parsanol 1.3.13-arm-linux

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (100) hide show
  1. checksums.yaml +7 -0
  2. data/HISTORY.txt +98 -0
  3. data/LICENSE +23 -0
  4. data/README.adoc +905 -0
  5. data/Rakefile +141 -0
  6. data/lib/parsanol/3.2/parsanol_native.so +0 -0
  7. data/lib/parsanol/3.3/parsanol_native.so +0 -0
  8. data/lib/parsanol/3.4/parsanol_native.so +0 -0
  9. data/lib/parsanol/4.0/parsanol_native.so +0 -0
  10. data/lib/parsanol/ast_visitor.rb +122 -0
  11. data/lib/parsanol/atoms/alternative.rb +123 -0
  12. data/lib/parsanol/atoms/base.rb +208 -0
  13. data/lib/parsanol/atoms/can_flatten.rb +194 -0
  14. data/lib/parsanol/atoms/capture.rb +38 -0
  15. data/lib/parsanol/atoms/context.rb +357 -0
  16. data/lib/parsanol/atoms/context_optimized.rb +38 -0
  17. data/lib/parsanol/atoms/custom.rb +110 -0
  18. data/lib/parsanol/atoms/cut.rb +66 -0
  19. data/lib/parsanol/atoms/dsl.rb +96 -0
  20. data/lib/parsanol/atoms/dynamic.rb +39 -0
  21. data/lib/parsanol/atoms/entity.rb +75 -0
  22. data/lib/parsanol/atoms/ignored.rb +37 -0
  23. data/lib/parsanol/atoms/infix.rb +167 -0
  24. data/lib/parsanol/atoms/lookahead.rb +85 -0
  25. data/lib/parsanol/atoms/named.rb +74 -0
  26. data/lib/parsanol/atoms/re.rb +83 -0
  27. data/lib/parsanol/atoms/repetition.rb +277 -0
  28. data/lib/parsanol/atoms/scope.rb +35 -0
  29. data/lib/parsanol/atoms/sequence.rb +195 -0
  30. data/lib/parsanol/atoms/str.rb +109 -0
  31. data/lib/parsanol/atoms/visitor.rb +91 -0
  32. data/lib/parsanol/atoms.rb +46 -0
  33. data/lib/parsanol/buffer.rb +133 -0
  34. data/lib/parsanol/builder_callbacks.rb +353 -0
  35. data/lib/parsanol/cause.rb +122 -0
  36. data/lib/parsanol/context.rb +39 -0
  37. data/lib/parsanol/convenience.rb +36 -0
  38. data/lib/parsanol/edit_tracker.rb +111 -0
  39. data/lib/parsanol/error_reporter/contextual.rb +99 -0
  40. data/lib/parsanol/error_reporter/deepest.rb +120 -0
  41. data/lib/parsanol/error_reporter/tree.rb +63 -0
  42. data/lib/parsanol/error_reporter.rb +100 -0
  43. data/lib/parsanol/expression/treetop.rb +154 -0
  44. data/lib/parsanol/expression.rb +106 -0
  45. data/lib/parsanol/fast_mode.rb +187 -0
  46. data/lib/parsanol/first_set.rb +79 -0
  47. data/lib/parsanol/grammar_builder.rb +179 -0
  48. data/lib/parsanol/incremental_parser.rb +182 -0
  49. data/lib/parsanol/interval_tree.rb +226 -0
  50. data/lib/parsanol/lazy_result.rb +179 -0
  51. data/lib/parsanol/mermaid.rb +142 -0
  52. data/lib/parsanol/native/batch_decoder.rb +255 -0
  53. data/lib/parsanol/native/dynamic.rb +238 -0
  54. data/lib/parsanol/native/parser.rb +102 -0
  55. data/lib/parsanol/native/serializer.rb +252 -0
  56. data/lib/parsanol/native/transformer.rb +604 -0
  57. data/lib/parsanol/native/types.rb +29 -0
  58. data/lib/parsanol/native.rb +223 -0
  59. data/lib/parsanol/optimizer.rb +85 -0
  60. data/lib/parsanol/optimizers/choice_optimizer.rb +78 -0
  61. data/lib/parsanol/optimizers/cut_inserter.rb +182 -0
  62. data/lib/parsanol/optimizers/lookahead_optimizer.rb +56 -0
  63. data/lib/parsanol/optimizers/quantifier_optimizer.rb +60 -0
  64. data/lib/parsanol/optimizers/sequence_optimizer.rb +97 -0
  65. data/lib/parsanol/options/zero_copy.rb +127 -0
  66. data/lib/parsanol/options.rb +21 -0
  67. data/lib/parsanol/parallel.rb +128 -0
  68. data/lib/parsanol/parser.rb +242 -0
  69. data/lib/parsanol/parslet.rb +151 -0
  70. data/lib/parsanol/pattern/binding.rb +91 -0
  71. data/lib/parsanol/pattern.rb +162 -0
  72. data/lib/parsanol/pool.rb +219 -0
  73. data/lib/parsanol/pools/array_pool.rb +75 -0
  74. data/lib/parsanol/pools/buffer_pool.rb +182 -0
  75. data/lib/parsanol/pools/position_pool.rb +92 -0
  76. data/lib/parsanol/pools/slice_pool.rb +64 -0
  77. data/lib/parsanol/position.rb +94 -0
  78. data/lib/parsanol/resettable.rb +29 -0
  79. data/lib/parsanol/result.rb +46 -0
  80. data/lib/parsanol/result_builder.rb +208 -0
  81. data/lib/parsanol/result_stream.rb +266 -0
  82. data/lib/parsanol/rig/rspec.rb +71 -0
  83. data/lib/parsanol/rope.rb +81 -0
  84. data/lib/parsanol/scope.rb +104 -0
  85. data/lib/parsanol/slice.rb +160 -0
  86. data/lib/parsanol/source/line_cache.rb +102 -0
  87. data/lib/parsanol/source.rb +185 -0
  88. data/lib/parsanol/source_location.rb +167 -0
  89. data/lib/parsanol/streaming_parser.rb +124 -0
  90. data/lib/parsanol/string_view.rb +198 -0
  91. data/lib/parsanol/transform.rb +226 -0
  92. data/lib/parsanol/version.rb +5 -0
  93. data/lib/parsanol/wasm/README.md +80 -0
  94. data/lib/parsanol/wasm/package.json +51 -0
  95. data/lib/parsanol/wasm/parsanol.js +252 -0
  96. data/lib/parsanol/wasm/parslet.d.ts +129 -0
  97. data/lib/parsanol/wasm_parser.rb +240 -0
  98. data/lib/parsanol.rb +278 -0
  99. data/parsanol.gemspec +67 -0
  100. metadata +279 -0
@@ -0,0 +1,223 @@
1
+ # frozen_string_literal: true
2
+
3
+ require "json"
4
+ require "digest"
5
+
6
+ require "parsanol/native/types"
7
+ require "parsanol/native/parser"
8
+ require "parsanol/native/serializer"
9
+ require "parsanol/native/batch_decoder"
10
+
11
+ module Parsanol
12
+ module Native
13
+ class << self
14
+ # Check if native extension is available
15
+ def available?
16
+ Parser.available?
17
+ end
18
+
19
+ # Parse input with a Ruby grammar, returning clean AST with lazy line/column.
20
+ #
21
+ # Uses batch FFI format for maximum performance (3-5x faster than object-by-object).
22
+ # The Rust-side transformation (to_parslet_compatible) produces Parslet-compatible
23
+ # output that can be consumed directly by Builder.build without additional
24
+ # Ruby-side transformation.
25
+ #
26
+ # @param grammar [Parsanol::Atoms::Base] Ruby grammar definition
27
+ # @param input [String] Input string to parse
28
+ # @return [Hash, Array, Parsanol::Slice] Transformed AST
29
+ #
30
+ # @example Simple parsing
31
+ # result = Parsanol::Native.parse(str('hello').as(:greeting), 'hello')
32
+ # # => {greeting: "hello"@0}
33
+ #
34
+ # @example With lazy line/column
35
+ # result = Parsanol::Native.parse(str('hello').as(:greeting), "hello\nworld")
36
+ # result[:greeting].line_and_column # => [1, 1]
37
+ #
38
+ def parse(grammar, input)
39
+ raise LoadError, "Native parser not available" unless available?
40
+
41
+ # Handle both grammar atoms and pre-serialized JSON strings
42
+ grammar_json = if grammar.is_a?(String)
43
+ grammar
44
+ else
45
+ Parser.serialize_grammar(grammar)
46
+ end
47
+
48
+ # Use _parse_raw which returns properly tagged Ruby arrays via transform_ast.
49
+ # The batch format doesn't preserve :repetition/:sequence tags, so we use
50
+ # the direct FFI path. Apply the Ruby transformer to handle tags correctly.
51
+ begin
52
+ raw_ast = _parse_raw(grammar_json, input)
53
+ rescue RuntimeError => e
54
+ raise_native_parse_error(e, grammar, input)
55
+ end
56
+ BatchDecoder.decode_and_flatten(raw_ast, input, Parsanol::Slice)
57
+ end
58
+
59
+ # Memory-bounded parsing without packrat cache.
60
+ #
61
+ # This creates a fresh arena and empty cache per call, bounding memory
62
+ # to AST size rather than input × atoms. Use for large files.
63
+ #
64
+ # @param grammar [Parsanol::Atoms::Base] Ruby grammar definition
65
+ # @param input [String] Input string to parse
66
+ # @return [Hash, Array, Parsanol::Slice] Transformed AST
67
+ def parse_fresh(grammar, input)
68
+ raise LoadError, "Native parser not available" unless available?
69
+
70
+ grammar_json = if grammar.is_a?(String)
71
+ grammar
72
+ else
73
+ Parser.serialize_grammar(grammar)
74
+ end
75
+
76
+ begin
77
+ raw_ast = _parse_fresh_raw(grammar_json, input)
78
+ rescue RuntimeError => e
79
+ raise_native_parse_error(e, grammar, input)
80
+ end
81
+ BatchDecoder.decode_and_flatten(raw_ast, input, Parsanol::Slice)
82
+ end
83
+
84
+ # Parse and return RAW AST without transformation.
85
+ #
86
+ # This returns the raw Parslet intermediate format before any transformation.
87
+ # Use this only if you need the raw AST for custom processing.
88
+ #
89
+ # For most use cases (including Expressir), use parse() instead which
90
+ # returns properly transformed AST.
91
+ #
92
+ # @param grammar [Parsanol::Atoms::Base] Ruby grammar definition
93
+ # @param input [String] Input string to parse
94
+ # @return [Hash, Array] Raw untransformed AST
95
+ #
96
+ # @example Raw parsing
97
+ # result = Parsanol::Native.parse_raw(str('hello').as(:greeting), 'hello')
98
+ # # => {:syntax => [{:spaces => ...}, {:greeting => "hello"@0}, {:spaces => ...}]}
99
+ #
100
+ def parse_raw(grammar, input)
101
+ raise LoadError, "Native parser not available" unless available?
102
+
103
+ # Handle both grammar atoms and pre-serialized JSON strings
104
+ grammar_json = if grammar.is_a?(String)
105
+ grammar
106
+ else
107
+ Parser.serialize_grammar(grammar)
108
+ end
109
+
110
+ # Use batch_raw format for raw AST (no transformation)
111
+ slice_class = Parsanol::Slice
112
+ batch_data = _parse_batch_raw(grammar_json, input)
113
+
114
+ # Decode without transformation - raw AST format
115
+ BatchDecoder.decode(batch_data, input, slice_class)
116
+ end
117
+
118
+ # Fast batch parsing - uses u64 array format to minimize FFI overhead.
119
+ #
120
+ # This is 3-5x faster than regular parse() for large grammars.
121
+ # The batch format passes a flat u64 array across FFI, then decodes
122
+ # in pure Ruby, avoiding expensive per-node FFI calls.
123
+ #
124
+ # Returns RAW AST without transformation. For Expressir use case,
125
+ # use parse() instead which returns properly transformed AST.
126
+ #
127
+ # @param grammar_json [String] Pre-serialized grammar JSON
128
+ # @param input [String] Input string to parse
129
+ # @param slice_class [Class] The Slice class to use for string refs
130
+ # @return [Hash, Array, Slice] Raw AST (not transformed)
131
+ def parse_batch(grammar_json, input, slice_class)
132
+ raise LoadError, "Native parser not available" unless available?
133
+
134
+ # Call native extension's _parse_batch_raw method (named with _raw suffix
135
+ # to avoid conflict with this Ruby wrapper method)
136
+ batch_data = _parse_batch_raw(grammar_json, input)
137
+ BatchDecoder.decode(batch_data, input, slice_class)
138
+ end
139
+
140
+ # Serialize a Ruby grammar to JSON (cached).
141
+ #
142
+ # @param root_atom [Parsanol::Atoms::Base] Root atom of the grammar
143
+ # @return [String] JSON string
144
+ def serialize_grammar(root_atom)
145
+ Parser.serialize_grammar(root_atom)
146
+ end
147
+
148
+ # Clear grammar caches (call if grammar changes)
149
+ def clear_cache
150
+ Parser.clear_cache
151
+ clear_grammar_cache if available?
152
+ end
153
+
154
+ # Clear the Rust grammar cache to free memory.
155
+ #
156
+ # This is useful for batch processing scenarios where you want to
157
+ # limit memory usage by clearing unused grammars.
158
+ #
159
+ # @return [nil]
160
+ def clear_grammar_cache
161
+ raise LoadError, "Native parser not available" unless available?
162
+
163
+ _clear_grammar_cache
164
+ end
165
+
166
+ # Get the current number of cached grammars in Rust.
167
+ #
168
+ # @return [Integer] Number of cached grammars
169
+ def grammar_cache_size
170
+ raise LoadError, "Native parser not available" unless available?
171
+
172
+ _grammar_cache_size
173
+ end
174
+
175
+ # Get the grammar cache capacity.
176
+ #
177
+ # @return [Integer] Maximum cache capacity
178
+ def grammar_cache_capacity
179
+ raise LoadError, "Native parser not available" unless available?
180
+
181
+ _grammar_cache_capacity
182
+ end
183
+
184
+ # Get cache statistics
185
+ def cache_stats
186
+ stats = Parser.cache_stats
187
+ if available?
188
+ stats[:rust_grammar_cache_size] = grammar_cache_size
189
+ stats[:rust_grammar_cache_capacity] = grammar_cache_capacity
190
+ end
191
+ stats
192
+ end
193
+
194
+ # Translates a native backend failure into the Parsanol error protocol.
195
+ #
196
+ # When the grammar atom is at hand, reparses through the pure Ruby
197
+ # backend: a Ruby failure raises Parsanol::ParseFailed with the full
198
+ # cause-tree diagnostics, and a Ruby success recovers grammars the
199
+ # native serializer cannot express (e.g. custom atoms). For
200
+ # pre-serialized JSON grammars the native message is wrapped in a
201
+ # Parsanol::ParseFailed directly.
202
+ def raise_native_parse_error(error, grammar, input)
203
+ return grammar.parse(input) if grammar.respond_to?(:parse)
204
+
205
+ source = Parsanol::Source.new(input)
206
+ cause = Parsanol::Cause.new(error.message, source, source.bytepos)
207
+ raise Parsanol::ParseFailed.new(cause.to_s, cause)
208
+ end
209
+ end
210
+ end
211
+ end
212
+
213
+ # Attempt to load native extension
214
+ begin
215
+ ruby_version = RUBY_VERSION.split(".").take(2).join(".")
216
+ require "parsanol/#{ruby_version}/parsanol_native"
217
+ rescue LoadError
218
+ begin
219
+ require "parsanol/parsanol_native"
220
+ rescue LoadError
221
+ # Native extension not built yet
222
+ end
223
+ end
@@ -0,0 +1,85 @@
1
+ # frozen_string_literal: true
2
+
3
+ require_relative "ast_visitor"
4
+ require_relative "optimizers/quantifier_optimizer"
5
+ require_relative "optimizers/sequence_optimizer"
6
+ require_relative "optimizers/choice_optimizer"
7
+ require_relative "optimizers/lookahead_optimizer"
8
+ require_relative "optimizers/cut_inserter"
9
+
10
+ # Grammar-level optimizations for Parslet parsers
11
+ # These optimizations transform the parser AST to reduce runtime overhead
12
+ # without changing semantics.
13
+ #
14
+ # Architecture:
15
+ # - Uses Visitor pattern for clean separation of traversal and transformation
16
+ # - Each optimizer is a separate class inheriting from ASTVisitor
17
+ # - Optimizer module provides facade methods for easy access
18
+ module Parsanol
19
+ module Optimizer
20
+ # Simplifies redundant quantifiers in a parslet tree
21
+ # Example: str('a').repeat(1, 1) => str('a')
22
+ # str('a').repeat(0, 1).repeat(0, 1) => str('a').repeat(0, 1)
23
+ #
24
+ # @param parslet [Parsanol::Atoms::Base] parslet to simplify
25
+ # @return [Parsanol::Atoms::Base] simplified parslet
26
+ def self.simplify_quantifiers(parslet)
27
+ Optimizers::QuantifierOptimizer.new.visit(parslet)
28
+ end
29
+
30
+ # Simplifies sequences by flattening and merging adjacent strings
31
+ # Example: str('a') >> str('b') => str('ab')
32
+ # (str('a') >> str('b')) >> str('c') => str('abc')
33
+ #
34
+ # @param parslet [Parsanol::Atoms::Base] parslet to simplify
35
+ # @return [Parsanol::Atoms::Base] simplified parslet
36
+ def self.simplify_sequences(parslet)
37
+ Optimizers::SequenceOptimizer.new.visit(parslet)
38
+ end
39
+
40
+ # Simplifies choice/alternative patterns
41
+ # Example: (A | B) | C => A | B | C
42
+ # A | B | A => A | B
43
+ #
44
+ # @param parslet [Parsanol::Atoms::Base] parslet to simplify
45
+ # @return [Parsanol::Atoms::Base] simplified parslet
46
+ def self.simplify_choices(parslet)
47
+ Optimizers::ChoiceOptimizer.new.visit(parslet)
48
+ end
49
+
50
+ # Simplifies lookahead patterns
51
+ # Example: !(!x) => &x (double negation elimination)
52
+ #
53
+ # @param parslet [Parsanol::Atoms::Base] parslet to simplify
54
+ # @return [Parsanol::Atoms::Base] simplified parslet
55
+ def self.simplify_lookaheads(parslet)
56
+ Optimizers::LookaheadOptimizer.new.visit(parslet)
57
+ end
58
+
59
+ # Automatically insert cut operators where safe (AC-FIRST algorithm)
60
+ # Inserts cuts after deterministic prefixes when alternatives have disjoint FIRST sets
61
+ # This enables O(1) space complexity by allowing aggressive cache eviction
62
+ #
63
+ # Example: str('if') >> x | str('while') >> y
64
+ # => str('if').cut >> x | str('while').cut >> y
65
+ #
66
+ # @param parslet [Parsanol::Atoms::Base] parslet to optimize
67
+ # @return [Parsanol::Atoms::Base] optimized parslet with cuts inserted
68
+ def self.insert_cuts(parslet)
69
+ Optimizers::CutInserter.new.optimize(parslet)
70
+ end
71
+
72
+ # Apply all optimizations in recommended order
73
+ # This is a convenience method that applies all optimizer passes
74
+ #
75
+ # @param parslet [Parsanol::Atoms::Base] parslet to optimize
76
+ # @return [Parsanol::Atoms::Base] fully optimized parslet
77
+ def self.optimize_all(parslet)
78
+ result = simplify_quantifiers(parslet)
79
+ result = simplify_sequences(result)
80
+ result = simplify_choices(result)
81
+ result = simplify_lookaheads(result)
82
+ insert_cuts(result)
83
+ end
84
+ end
85
+ end
@@ -0,0 +1,78 @@
1
+ # frozen_string_literal: true
2
+
3
+ require_relative "../ast_visitor"
4
+
5
+ module Parsanol
6
+ module Optimizers
7
+ # Optimizes alternative/choice patterns in the AST
8
+ # Follows visitor pattern for clean separation of concerns
9
+ #
10
+ # Transformations:
11
+ # - (A | B) | C => A | B | C (flatten nested alternatives)
12
+ # - A | B | A => A | B (remove duplicates)
13
+ # - Alternative(A) => A (unwrap single-element alternatives)
14
+ class ChoiceOptimizer < ASTVisitor
15
+ # Visit an alternative node and apply choice optimizations
16
+ # @param parslet [Parsanol::Atoms::Alternative] alternative to optimize
17
+ # @return [Parsanol::Atoms::Base] optimized parslet
18
+ def visit_alternative(parslet)
19
+ # First optimize children recursively
20
+ new_alternatives = parslet.alternatives.map { |p| visit(p) }
21
+
22
+ # Optimization 1: Flatten nested alternatives
23
+ flattened = flatten_alternatives(new_alternatives)
24
+
25
+ # Optimization 2: Remove duplicate alternatives
26
+ deduplicated = deduplicate_alternatives(flattened)
27
+
28
+ # Optimization 3: Unwrap single-element alternatives
29
+ return deduplicated[0] if deduplicated.size == 1
30
+
31
+ # Return optimized alternative if changed
32
+ if deduplicated == parslet.alternatives
33
+ parslet
34
+ else
35
+ Parsanol::Atoms::Alternative.new(*deduplicated)
36
+ end
37
+ end
38
+
39
+ private
40
+
41
+ # Flatten nested alternatives into a single level
42
+ # @param alternatives [Array<Parsanol::Atoms::Base>] array of alternatives
43
+ # @return [Array<Parsanol::Atoms::Base>] flattened array
44
+ def flatten_alternatives(alternatives)
45
+ result = []
46
+ alternatives.each do |alt|
47
+ if alt.is_a?(Parsanol::Atoms::Alternative)
48
+ result.concat(alt.alternatives)
49
+ else
50
+ result << alt
51
+ end
52
+ end
53
+ result
54
+ end
55
+
56
+ # Remove duplicate alternatives using structural equality
57
+ # @param alternatives [Array<Parsanol::Atoms::Base>] array of alternatives
58
+ # @return [Array<Parsanol::Atoms::Base>] deduplicated array
59
+ def deduplicate_alternatives(alternatives)
60
+ return alternatives if alternatives.size < 2
61
+
62
+ # Use to_s as proxy for structural equality
63
+ seen = {}
64
+ result = []
65
+
66
+ alternatives.each do |alt|
67
+ key = alt.to_s
68
+ unless seen[key]
69
+ seen[key] = true
70
+ result << alt
71
+ end
72
+ end
73
+
74
+ result
75
+ end
76
+ end
77
+ end
78
+ end
@@ -0,0 +1,182 @@
1
+ # frozen_string_literal: true
2
+
3
+ # Automatic Cut Insertion (AC-FIRST Algorithm)
4
+ #
5
+ # This optimizer implements the AC-FIRST algorithm from Mizushima et al. (2010)
6
+ # to automatically insert cut operators when alternatives have disjoint FIRST sets.
7
+ #
8
+ # When all alternatives in a choice have non-overlapping FIRST sets, we can safely
9
+ # insert a cut after the deterministic prefix, since backtracking will never be
10
+ # needed.
11
+ #
12
+ # Example:
13
+ # str('if') >> condition >> then_clause |
14
+ # str('while') >> condition >> body |
15
+ # str('print') >> expression
16
+ #
17
+ # Becomes:
18
+ # str('if').cut >> condition >> then_clause |
19
+ # str('while').cut >> condition >> body |
20
+ # str('print').cut >> expression
21
+ #
22
+ # Reference: Mizushima et al. (2010) "Packrat Parsers Can Handle Practical
23
+ # Grammars in Mostly Constant Space"
24
+ #
25
+ module Parsanol
26
+ module Optimizers
27
+ class CutInserter
28
+ # Optimize a parslet by inserting cuts where safe
29
+ # Recursively traverses the grammar AST
30
+ #
31
+ # @param parslet [Parsanol::Atoms::Base] The parslet to optimize
32
+ # @return [Parsanol::Atoms::Base] Optimized parslet with cuts inserted
33
+ def optimize(parslet)
34
+ case parslet
35
+ when Parsanol::Atoms::Alternative
36
+ optimize_alternative(parslet)
37
+ when Parsanol::Atoms::Sequence
38
+ optimize_sequence(parslet)
39
+ when Parsanol::Atoms::Repetition
40
+ optimize_repetition(parslet)
41
+ when Parsanol::Atoms::Named
42
+ optimize_named(parslet)
43
+ else
44
+ # Return atom unchanged (Str, Re, Lookahead, etc.)
45
+ parslet
46
+ end
47
+ end
48
+
49
+ private
50
+
51
+ # Optimize an Alternative atom by inserting cuts when all alternatives
52
+ # have disjoint FIRST sets
53
+ def optimize_alternative(alt)
54
+ alternatives = alt.alternatives
55
+ first_sets = alternatives.map(&:first_set)
56
+
57
+ # Only optimize if all FIRST sets are disjoint
58
+ unless Parsanol::FirstSet.all_disjoint?(first_sets)
59
+ # Not safe to insert cuts - return alternatives with recursive optimization
60
+ optimized = alternatives.map { |a| optimize(a) }
61
+ return Parsanol::Atoms::Alternative.new(*optimized)
62
+ end
63
+
64
+ # All FIRST sets are disjoint - safe to insert cuts!
65
+ # Insert cuts after deterministic prefixes
66
+ optimized = alternatives.map do |alternative|
67
+ insert_cut_if_safe(alternative)
68
+ end
69
+
70
+ Parsanol::Atoms::Alternative.new(*optimized)
71
+ end
72
+
73
+ # Optimize a Sequence atom by recursively optimizing its elements
74
+ def optimize_sequence(seq)
75
+ optimized_parslets = seq.parslets.map { |p| optimize(p) }
76
+ Parsanol::Atoms::Sequence.new(*optimized_parslets)
77
+ end
78
+
79
+ # Optimize a Repetition atom by recursively optimizing its parslet
80
+ def optimize_repetition(rep)
81
+ optimized_parslet = optimize(rep.parslet)
82
+ # Create new repetition with same min/max
83
+ # Note: We use default tag since it's not exposed as a reader
84
+ Parsanol::Atoms::Repetition.new(
85
+ optimized_parslet,
86
+ rep.min,
87
+ rep.max,
88
+ )
89
+ end
90
+
91
+ # Optimize a Named atom by recursively optimizing its parslet
92
+ def optimize_named(named)
93
+ optimized_parslet = optimize(named.parslet)
94
+ optimized_parslet.as(named.name)
95
+ end
96
+
97
+ # Insert a cut after the deterministic prefix if safe
98
+ # For sequences: find longest prefix without EPSILON
99
+ # For other atoms: cut the whole thing if it doesn't include EPSILON
100
+ def insert_cut_if_safe(parslet)
101
+ # For sequences, find the longest safe prefix
102
+ if parslet.is_a?(Parsanol::Atoms::Sequence)
103
+ prefix_parslets = find_deterministic_prefix(parslet)
104
+ if prefix_parslets && !prefix_parslets.empty?
105
+ return build_cut_sequence(parslet,
106
+ prefix_parslets)
107
+ end
108
+ end
109
+
110
+ # For other atoms, cut the whole thing if safe
111
+ return parslet.cut if safe_to_cut?(parslet)
112
+
113
+ # Not safe to cut - recursively optimize and return
114
+ optimize(parslet)
115
+ end
116
+
117
+ # Find the longest deterministic prefix of a sequence
118
+ # A deterministic prefix doesn't include EPSILON in its FIRST set
119
+ #
120
+ # @param sequence [Parsanol::Atoms::Sequence] The sequence to analyze
121
+ # @return [Array<Parsanol::Atoms::Base>] Prefix parslets, or nil if none
122
+ def find_deterministic_prefix(sequence)
123
+ parslets = sequence.parslets
124
+ prefix_length = 0
125
+
126
+ # Find longest prefix where no element can match empty
127
+ parslets.each do |p|
128
+ break if p.first_set.include?(Parsanol::FirstSet::EPSILON)
129
+
130
+ prefix_length += 1
131
+ end
132
+
133
+ prefix_length.positive? ? parslets[0...prefix_length] : nil
134
+ end
135
+
136
+ # Check if it's safe to cut after this parslet
137
+ # Safe if the parslet doesn't have EPSILON in its FIRST set
138
+ # (i.e., it always consumes input)
139
+ def safe_to_cut?(parslet)
140
+ first = parslet.first_set
141
+ # Don't cut if EPSILON is in FIRST set (might not consume)
142
+ # Also don't cut if FIRST set contains only nil (unknown)
143
+ return false if first.include?(Parsanol::FirstSet::EPSILON)
144
+ return false if first.all?(&:nil?)
145
+
146
+ true
147
+ end
148
+
149
+ # Build a new sequence with a cut after the prefix
150
+ #
151
+ # @param sequence [Parsanol::Atoms::Sequence] Original sequence
152
+ # @param prefix_parslets [Array] Parslets forming the deterministic prefix
153
+ # @return [Parsanol::Atoms::Base] New sequence with cut inserted
154
+ def build_cut_sequence(sequence, prefix_parslets)
155
+ # Recursively optimize prefix parslets
156
+ optimized_prefix = prefix_parslets.map { |p| optimize(p) }
157
+
158
+ # Build prefix (single parslet or sequence)
159
+ prefix = if optimized_prefix.length == 1
160
+ optimized_prefix.first
161
+ else
162
+ Parsanol::Atoms::Sequence.new(*optimized_prefix)
163
+ end
164
+
165
+ # Get remaining parslets after prefix
166
+ remaining = sequence.parslets[prefix_parslets.length..]
167
+
168
+ # Recursively optimize remaining parslets
169
+ optimized_remaining = remaining.map { |p| optimize(p) }
170
+
171
+ # Build final sequence with cut
172
+ if optimized_remaining.empty?
173
+ # Prefix is the entire sequence
174
+ prefix.cut
175
+ else
176
+ # Prefix + cut + remaining
177
+ Parsanol::Atoms::Sequence.new(prefix.cut, *optimized_remaining)
178
+ end
179
+ end
180
+ end
181
+ end
182
+ end
@@ -0,0 +1,56 @@
1
+ # frozen_string_literal: true
2
+
3
+ require_relative "../ast_visitor"
4
+
5
+ module Parsanol
6
+ module Optimizers
7
+ # Optimizes lookahead patterns in the AST
8
+ # Follows visitor pattern for clean separation of concerns
9
+ #
10
+ # Transformations:
11
+ # - !(!x) => &x (double negation elimination)
12
+ # - &(&x) => &x (positive lookahead is idempotent)
13
+ # - !(&x) => !x (negative of positive)
14
+ # - &(!x) => !x (positive of negative)
15
+ class LookaheadOptimizer < ASTVisitor
16
+ # Visit a lookahead node and apply lookahead optimizations
17
+ # @param parslet [Parsanol::Atoms::Lookahead] lookahead to optimize
18
+ # @return [Parsanol::Atoms::Base] optimized parslet
19
+ def visit_lookahead(parslet)
20
+ # First optimize the child
21
+ inner = visit(parslet.bound_parslet)
22
+
23
+ # If inner is also a lookahead, simplify nested lookaheads
24
+ if inner.is_a?(Parsanol::Atoms::Lookahead)
25
+ outer_positive = parslet.positive
26
+ inner_positive = inner.positive
27
+
28
+ # !(!x) => &x (double negation)
29
+ if !outer_positive && !inner_positive
30
+ return Parsanol::Atoms::Lookahead.new(inner.bound_parslet,
31
+ true)
32
+ end
33
+
34
+ # &(&x) => &x (idempotent)
35
+ return inner if outer_positive && inner_positive
36
+
37
+ # !(&x) => !x (negative of positive)
38
+ if !outer_positive && inner_positive
39
+ return Parsanol::Atoms::Lookahead.new(inner.bound_parslet,
40
+ false)
41
+ end
42
+
43
+ # &(!x) => !x (positive of negative)
44
+ return inner if outer_positive && !inner_positive
45
+ end
46
+
47
+ # Return lookahead with optimized child
48
+ if inner.equal?(parslet.bound_parslet)
49
+ parslet
50
+ else
51
+ Parsanol::Atoms::Lookahead.new(inner, parslet.positive)
52
+ end
53
+ end
54
+ end
55
+ end
56
+ end
@@ -0,0 +1,60 @@
1
+ # frozen_string_literal: true
2
+
3
+ require_relative "../ast_visitor"
4
+
5
+ module Parsanol
6
+ module Optimizers
7
+ # Optimizes repetition/quantifier patterns in the AST
8
+ # Follows visitor pattern for clean separation of concerns
9
+ #
10
+ # Transformations:
11
+ # - repeat(1,1) => unwrap (identity transformation)
12
+ # - repeat(0,1).repeat(0,1) => repeat(0,1) (idempotent)
13
+ # - repeat(n,n).repeat(m,m) => repeat(n*m,n*m) (multiply exact counts)
14
+ class QuantifierOptimizer < ASTVisitor
15
+ # Visit a repetition node and apply quantifier optimizations
16
+ # @param parslet [Parsanol::Atoms::Repetition] repetition to optimize
17
+ # @return [Parsanol::Atoms::Base] optimized parslet
18
+ def visit_repetition(parslet)
19
+ # First optimize the child
20
+ inner = visit(parslet.parslet)
21
+
22
+ # Optimization 1: repeat(1,1) is identity - unwrap it
23
+ return inner if parslet.min == 1 && parslet.max == 1
24
+
25
+ # Optimization 2: Nested repetitions
26
+ if inner.is_a?(Parsanol::Atoms::Repetition)
27
+ # repeat(0,1).repeat(0,1) => repeat(0,1) (idempotent)
28
+ if parslet.min.zero? && parslet.max == 1 &&
29
+ inner.min.zero? && inner.max == 1
30
+ return inner
31
+ end
32
+
33
+ # repeat(n,n).repeat(m,m) => repeat(n*m,n*m) for exact counts
34
+ if parslet.min == parslet.max && inner.min == inner.max &&
35
+ parslet.max && inner.max
36
+ new_count = parslet.min * inner.min
37
+ return Parsanol::Atoms::Repetition.new(
38
+ inner.parslet,
39
+ new_count,
40
+ new_count,
41
+ parslet.result_tag,
42
+ )
43
+ end
44
+ end
45
+
46
+ # Return optimized repetition with simplified child
47
+ if inner.equal?(parslet.parslet)
48
+ parslet
49
+ else
50
+ Parsanol::Atoms::Repetition.new(
51
+ inner,
52
+ parslet.min,
53
+ parslet.max,
54
+ parslet.result_tag,
55
+ )
56
+ end
57
+ end
58
+ end
59
+ end
60
+ end