parsanol 1.3.13-arm-linux
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +7 -0
- data/HISTORY.txt +98 -0
- data/LICENSE +23 -0
- data/README.adoc +905 -0
- data/Rakefile +141 -0
- data/lib/parsanol/3.2/parsanol_native.so +0 -0
- data/lib/parsanol/3.3/parsanol_native.so +0 -0
- data/lib/parsanol/3.4/parsanol_native.so +0 -0
- data/lib/parsanol/4.0/parsanol_native.so +0 -0
- data/lib/parsanol/ast_visitor.rb +122 -0
- data/lib/parsanol/atoms/alternative.rb +123 -0
- data/lib/parsanol/atoms/base.rb +208 -0
- data/lib/parsanol/atoms/can_flatten.rb +194 -0
- data/lib/parsanol/atoms/capture.rb +38 -0
- data/lib/parsanol/atoms/context.rb +357 -0
- data/lib/parsanol/atoms/context_optimized.rb +38 -0
- data/lib/parsanol/atoms/custom.rb +110 -0
- data/lib/parsanol/atoms/cut.rb +66 -0
- data/lib/parsanol/atoms/dsl.rb +96 -0
- data/lib/parsanol/atoms/dynamic.rb +39 -0
- data/lib/parsanol/atoms/entity.rb +75 -0
- data/lib/parsanol/atoms/ignored.rb +37 -0
- data/lib/parsanol/atoms/infix.rb +167 -0
- data/lib/parsanol/atoms/lookahead.rb +85 -0
- data/lib/parsanol/atoms/named.rb +74 -0
- data/lib/parsanol/atoms/re.rb +83 -0
- data/lib/parsanol/atoms/repetition.rb +277 -0
- data/lib/parsanol/atoms/scope.rb +35 -0
- data/lib/parsanol/atoms/sequence.rb +195 -0
- data/lib/parsanol/atoms/str.rb +109 -0
- data/lib/parsanol/atoms/visitor.rb +91 -0
- data/lib/parsanol/atoms.rb +46 -0
- data/lib/parsanol/buffer.rb +133 -0
- data/lib/parsanol/builder_callbacks.rb +353 -0
- data/lib/parsanol/cause.rb +122 -0
- data/lib/parsanol/context.rb +39 -0
- data/lib/parsanol/convenience.rb +36 -0
- data/lib/parsanol/edit_tracker.rb +111 -0
- data/lib/parsanol/error_reporter/contextual.rb +99 -0
- data/lib/parsanol/error_reporter/deepest.rb +120 -0
- data/lib/parsanol/error_reporter/tree.rb +63 -0
- data/lib/parsanol/error_reporter.rb +100 -0
- data/lib/parsanol/expression/treetop.rb +154 -0
- data/lib/parsanol/expression.rb +106 -0
- data/lib/parsanol/fast_mode.rb +187 -0
- data/lib/parsanol/first_set.rb +79 -0
- data/lib/parsanol/grammar_builder.rb +179 -0
- data/lib/parsanol/incremental_parser.rb +182 -0
- data/lib/parsanol/interval_tree.rb +226 -0
- data/lib/parsanol/lazy_result.rb +179 -0
- data/lib/parsanol/mermaid.rb +142 -0
- data/lib/parsanol/native/batch_decoder.rb +255 -0
- data/lib/parsanol/native/dynamic.rb +238 -0
- data/lib/parsanol/native/parser.rb +102 -0
- data/lib/parsanol/native/serializer.rb +252 -0
- data/lib/parsanol/native/transformer.rb +604 -0
- data/lib/parsanol/native/types.rb +29 -0
- data/lib/parsanol/native.rb +223 -0
- data/lib/parsanol/optimizer.rb +85 -0
- data/lib/parsanol/optimizers/choice_optimizer.rb +78 -0
- data/lib/parsanol/optimizers/cut_inserter.rb +182 -0
- data/lib/parsanol/optimizers/lookahead_optimizer.rb +56 -0
- data/lib/parsanol/optimizers/quantifier_optimizer.rb +60 -0
- data/lib/parsanol/optimizers/sequence_optimizer.rb +97 -0
- data/lib/parsanol/options/zero_copy.rb +127 -0
- data/lib/parsanol/options.rb +21 -0
- data/lib/parsanol/parallel.rb +128 -0
- data/lib/parsanol/parser.rb +242 -0
- data/lib/parsanol/parslet.rb +151 -0
- data/lib/parsanol/pattern/binding.rb +91 -0
- data/lib/parsanol/pattern.rb +162 -0
- data/lib/parsanol/pool.rb +219 -0
- data/lib/parsanol/pools/array_pool.rb +75 -0
- data/lib/parsanol/pools/buffer_pool.rb +182 -0
- data/lib/parsanol/pools/position_pool.rb +92 -0
- data/lib/parsanol/pools/slice_pool.rb +64 -0
- data/lib/parsanol/position.rb +94 -0
- data/lib/parsanol/resettable.rb +29 -0
- data/lib/parsanol/result.rb +46 -0
- data/lib/parsanol/result_builder.rb +208 -0
- data/lib/parsanol/result_stream.rb +266 -0
- data/lib/parsanol/rig/rspec.rb +71 -0
- data/lib/parsanol/rope.rb +81 -0
- data/lib/parsanol/scope.rb +104 -0
- data/lib/parsanol/slice.rb +160 -0
- data/lib/parsanol/source/line_cache.rb +102 -0
- data/lib/parsanol/source.rb +185 -0
- data/lib/parsanol/source_location.rb +167 -0
- data/lib/parsanol/streaming_parser.rb +124 -0
- data/lib/parsanol/string_view.rb +198 -0
- data/lib/parsanol/transform.rb +226 -0
- data/lib/parsanol/version.rb +5 -0
- data/lib/parsanol/wasm/README.md +80 -0
- data/lib/parsanol/wasm/package.json +51 -0
- data/lib/parsanol/wasm/parsanol.js +252 -0
- data/lib/parsanol/wasm/parslet.d.ts +129 -0
- data/lib/parsanol/wasm_parser.rb +240 -0
- data/lib/parsanol.rb +278 -0
- data/parsanol.gemspec +67 -0
- metadata +279 -0
|
@@ -0,0 +1,223 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require "json"
|
|
4
|
+
require "digest"
|
|
5
|
+
|
|
6
|
+
require "parsanol/native/types"
|
|
7
|
+
require "parsanol/native/parser"
|
|
8
|
+
require "parsanol/native/serializer"
|
|
9
|
+
require "parsanol/native/batch_decoder"
|
|
10
|
+
|
|
11
|
+
module Parsanol
|
|
12
|
+
module Native
|
|
13
|
+
class << self
|
|
14
|
+
# Check if native extension is available
|
|
15
|
+
def available?
|
|
16
|
+
Parser.available?
|
|
17
|
+
end
|
|
18
|
+
|
|
19
|
+
# Parse input with a Ruby grammar, returning clean AST with lazy line/column.
|
|
20
|
+
#
|
|
21
|
+
# Uses batch FFI format for maximum performance (3-5x faster than object-by-object).
|
|
22
|
+
# The Rust-side transformation (to_parslet_compatible) produces Parslet-compatible
|
|
23
|
+
# output that can be consumed directly by Builder.build without additional
|
|
24
|
+
# Ruby-side transformation.
|
|
25
|
+
#
|
|
26
|
+
# @param grammar [Parsanol::Atoms::Base] Ruby grammar definition
|
|
27
|
+
# @param input [String] Input string to parse
|
|
28
|
+
# @return [Hash, Array, Parsanol::Slice] Transformed AST
|
|
29
|
+
#
|
|
30
|
+
# @example Simple parsing
|
|
31
|
+
# result = Parsanol::Native.parse(str('hello').as(:greeting), 'hello')
|
|
32
|
+
# # => {greeting: "hello"@0}
|
|
33
|
+
#
|
|
34
|
+
# @example With lazy line/column
|
|
35
|
+
# result = Parsanol::Native.parse(str('hello').as(:greeting), "hello\nworld")
|
|
36
|
+
# result[:greeting].line_and_column # => [1, 1]
|
|
37
|
+
#
|
|
38
|
+
def parse(grammar, input)
|
|
39
|
+
raise LoadError, "Native parser not available" unless available?
|
|
40
|
+
|
|
41
|
+
# Handle both grammar atoms and pre-serialized JSON strings
|
|
42
|
+
grammar_json = if grammar.is_a?(String)
|
|
43
|
+
grammar
|
|
44
|
+
else
|
|
45
|
+
Parser.serialize_grammar(grammar)
|
|
46
|
+
end
|
|
47
|
+
|
|
48
|
+
# Use _parse_raw which returns properly tagged Ruby arrays via transform_ast.
|
|
49
|
+
# The batch format doesn't preserve :repetition/:sequence tags, so we use
|
|
50
|
+
# the direct FFI path. Apply the Ruby transformer to handle tags correctly.
|
|
51
|
+
begin
|
|
52
|
+
raw_ast = _parse_raw(grammar_json, input)
|
|
53
|
+
rescue RuntimeError => e
|
|
54
|
+
raise_native_parse_error(e, grammar, input)
|
|
55
|
+
end
|
|
56
|
+
BatchDecoder.decode_and_flatten(raw_ast, input, Parsanol::Slice)
|
|
57
|
+
end
|
|
58
|
+
|
|
59
|
+
# Memory-bounded parsing without packrat cache.
|
|
60
|
+
#
|
|
61
|
+
# This creates a fresh arena and empty cache per call, bounding memory
|
|
62
|
+
# to AST size rather than input × atoms. Use for large files.
|
|
63
|
+
#
|
|
64
|
+
# @param grammar [Parsanol::Atoms::Base] Ruby grammar definition
|
|
65
|
+
# @param input [String] Input string to parse
|
|
66
|
+
# @return [Hash, Array, Parsanol::Slice] Transformed AST
|
|
67
|
+
def parse_fresh(grammar, input)
|
|
68
|
+
raise LoadError, "Native parser not available" unless available?
|
|
69
|
+
|
|
70
|
+
grammar_json = if grammar.is_a?(String)
|
|
71
|
+
grammar
|
|
72
|
+
else
|
|
73
|
+
Parser.serialize_grammar(grammar)
|
|
74
|
+
end
|
|
75
|
+
|
|
76
|
+
begin
|
|
77
|
+
raw_ast = _parse_fresh_raw(grammar_json, input)
|
|
78
|
+
rescue RuntimeError => e
|
|
79
|
+
raise_native_parse_error(e, grammar, input)
|
|
80
|
+
end
|
|
81
|
+
BatchDecoder.decode_and_flatten(raw_ast, input, Parsanol::Slice)
|
|
82
|
+
end
|
|
83
|
+
|
|
84
|
+
# Parse and return RAW AST without transformation.
|
|
85
|
+
#
|
|
86
|
+
# This returns the raw Parslet intermediate format before any transformation.
|
|
87
|
+
# Use this only if you need the raw AST for custom processing.
|
|
88
|
+
#
|
|
89
|
+
# For most use cases (including Expressir), use parse() instead which
|
|
90
|
+
# returns properly transformed AST.
|
|
91
|
+
#
|
|
92
|
+
# @param grammar [Parsanol::Atoms::Base] Ruby grammar definition
|
|
93
|
+
# @param input [String] Input string to parse
|
|
94
|
+
# @return [Hash, Array] Raw untransformed AST
|
|
95
|
+
#
|
|
96
|
+
# @example Raw parsing
|
|
97
|
+
# result = Parsanol::Native.parse_raw(str('hello').as(:greeting), 'hello')
|
|
98
|
+
# # => {:syntax => [{:spaces => ...}, {:greeting => "hello"@0}, {:spaces => ...}]}
|
|
99
|
+
#
|
|
100
|
+
def parse_raw(grammar, input)
|
|
101
|
+
raise LoadError, "Native parser not available" unless available?
|
|
102
|
+
|
|
103
|
+
# Handle both grammar atoms and pre-serialized JSON strings
|
|
104
|
+
grammar_json = if grammar.is_a?(String)
|
|
105
|
+
grammar
|
|
106
|
+
else
|
|
107
|
+
Parser.serialize_grammar(grammar)
|
|
108
|
+
end
|
|
109
|
+
|
|
110
|
+
# Use batch_raw format for raw AST (no transformation)
|
|
111
|
+
slice_class = Parsanol::Slice
|
|
112
|
+
batch_data = _parse_batch_raw(grammar_json, input)
|
|
113
|
+
|
|
114
|
+
# Decode without transformation - raw AST format
|
|
115
|
+
BatchDecoder.decode(batch_data, input, slice_class)
|
|
116
|
+
end
|
|
117
|
+
|
|
118
|
+
# Fast batch parsing - uses u64 array format to minimize FFI overhead.
|
|
119
|
+
#
|
|
120
|
+
# This is 3-5x faster than regular parse() for large grammars.
|
|
121
|
+
# The batch format passes a flat u64 array across FFI, then decodes
|
|
122
|
+
# in pure Ruby, avoiding expensive per-node FFI calls.
|
|
123
|
+
#
|
|
124
|
+
# Returns RAW AST without transformation. For Expressir use case,
|
|
125
|
+
# use parse() instead which returns properly transformed AST.
|
|
126
|
+
#
|
|
127
|
+
# @param grammar_json [String] Pre-serialized grammar JSON
|
|
128
|
+
# @param input [String] Input string to parse
|
|
129
|
+
# @param slice_class [Class] The Slice class to use for string refs
|
|
130
|
+
# @return [Hash, Array, Slice] Raw AST (not transformed)
|
|
131
|
+
def parse_batch(grammar_json, input, slice_class)
|
|
132
|
+
raise LoadError, "Native parser not available" unless available?
|
|
133
|
+
|
|
134
|
+
# Call native extension's _parse_batch_raw method (named with _raw suffix
|
|
135
|
+
# to avoid conflict with this Ruby wrapper method)
|
|
136
|
+
batch_data = _parse_batch_raw(grammar_json, input)
|
|
137
|
+
BatchDecoder.decode(batch_data, input, slice_class)
|
|
138
|
+
end
|
|
139
|
+
|
|
140
|
+
# Serialize a Ruby grammar to JSON (cached).
|
|
141
|
+
#
|
|
142
|
+
# @param root_atom [Parsanol::Atoms::Base] Root atom of the grammar
|
|
143
|
+
# @return [String] JSON string
|
|
144
|
+
def serialize_grammar(root_atom)
|
|
145
|
+
Parser.serialize_grammar(root_atom)
|
|
146
|
+
end
|
|
147
|
+
|
|
148
|
+
# Clear grammar caches (call if grammar changes)
|
|
149
|
+
def clear_cache
|
|
150
|
+
Parser.clear_cache
|
|
151
|
+
clear_grammar_cache if available?
|
|
152
|
+
end
|
|
153
|
+
|
|
154
|
+
# Clear the Rust grammar cache to free memory.
|
|
155
|
+
#
|
|
156
|
+
# This is useful for batch processing scenarios where you want to
|
|
157
|
+
# limit memory usage by clearing unused grammars.
|
|
158
|
+
#
|
|
159
|
+
# @return [nil]
|
|
160
|
+
def clear_grammar_cache
|
|
161
|
+
raise LoadError, "Native parser not available" unless available?
|
|
162
|
+
|
|
163
|
+
_clear_grammar_cache
|
|
164
|
+
end
|
|
165
|
+
|
|
166
|
+
# Get the current number of cached grammars in Rust.
|
|
167
|
+
#
|
|
168
|
+
# @return [Integer] Number of cached grammars
|
|
169
|
+
def grammar_cache_size
|
|
170
|
+
raise LoadError, "Native parser not available" unless available?
|
|
171
|
+
|
|
172
|
+
_grammar_cache_size
|
|
173
|
+
end
|
|
174
|
+
|
|
175
|
+
# Get the grammar cache capacity.
|
|
176
|
+
#
|
|
177
|
+
# @return [Integer] Maximum cache capacity
|
|
178
|
+
def grammar_cache_capacity
|
|
179
|
+
raise LoadError, "Native parser not available" unless available?
|
|
180
|
+
|
|
181
|
+
_grammar_cache_capacity
|
|
182
|
+
end
|
|
183
|
+
|
|
184
|
+
# Get cache statistics
|
|
185
|
+
def cache_stats
|
|
186
|
+
stats = Parser.cache_stats
|
|
187
|
+
if available?
|
|
188
|
+
stats[:rust_grammar_cache_size] = grammar_cache_size
|
|
189
|
+
stats[:rust_grammar_cache_capacity] = grammar_cache_capacity
|
|
190
|
+
end
|
|
191
|
+
stats
|
|
192
|
+
end
|
|
193
|
+
|
|
194
|
+
# Translates a native backend failure into the Parsanol error protocol.
|
|
195
|
+
#
|
|
196
|
+
# When the grammar atom is at hand, reparses through the pure Ruby
|
|
197
|
+
# backend: a Ruby failure raises Parsanol::ParseFailed with the full
|
|
198
|
+
# cause-tree diagnostics, and a Ruby success recovers grammars the
|
|
199
|
+
# native serializer cannot express (e.g. custom atoms). For
|
|
200
|
+
# pre-serialized JSON grammars the native message is wrapped in a
|
|
201
|
+
# Parsanol::ParseFailed directly.
|
|
202
|
+
def raise_native_parse_error(error, grammar, input)
|
|
203
|
+
return grammar.parse(input) if grammar.respond_to?(:parse)
|
|
204
|
+
|
|
205
|
+
source = Parsanol::Source.new(input)
|
|
206
|
+
cause = Parsanol::Cause.new(error.message, source, source.bytepos)
|
|
207
|
+
raise Parsanol::ParseFailed.new(cause.to_s, cause)
|
|
208
|
+
end
|
|
209
|
+
end
|
|
210
|
+
end
|
|
211
|
+
end
|
|
212
|
+
|
|
213
|
+
# Attempt to load native extension
|
|
214
|
+
begin
|
|
215
|
+
ruby_version = RUBY_VERSION.split(".").take(2).join(".")
|
|
216
|
+
require "parsanol/#{ruby_version}/parsanol_native"
|
|
217
|
+
rescue LoadError
|
|
218
|
+
begin
|
|
219
|
+
require "parsanol/parsanol_native"
|
|
220
|
+
rescue LoadError
|
|
221
|
+
# Native extension not built yet
|
|
222
|
+
end
|
|
223
|
+
end
|
|
@@ -0,0 +1,85 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require_relative "ast_visitor"
|
|
4
|
+
require_relative "optimizers/quantifier_optimizer"
|
|
5
|
+
require_relative "optimizers/sequence_optimizer"
|
|
6
|
+
require_relative "optimizers/choice_optimizer"
|
|
7
|
+
require_relative "optimizers/lookahead_optimizer"
|
|
8
|
+
require_relative "optimizers/cut_inserter"
|
|
9
|
+
|
|
10
|
+
# Grammar-level optimizations for Parslet parsers
|
|
11
|
+
# These optimizations transform the parser AST to reduce runtime overhead
|
|
12
|
+
# without changing semantics.
|
|
13
|
+
#
|
|
14
|
+
# Architecture:
|
|
15
|
+
# - Uses Visitor pattern for clean separation of traversal and transformation
|
|
16
|
+
# - Each optimizer is a separate class inheriting from ASTVisitor
|
|
17
|
+
# - Optimizer module provides facade methods for easy access
|
|
18
|
+
module Parsanol
|
|
19
|
+
module Optimizer
|
|
20
|
+
# Simplifies redundant quantifiers in a parslet tree
|
|
21
|
+
# Example: str('a').repeat(1, 1) => str('a')
|
|
22
|
+
# str('a').repeat(0, 1).repeat(0, 1) => str('a').repeat(0, 1)
|
|
23
|
+
#
|
|
24
|
+
# @param parslet [Parsanol::Atoms::Base] parslet to simplify
|
|
25
|
+
# @return [Parsanol::Atoms::Base] simplified parslet
|
|
26
|
+
def self.simplify_quantifiers(parslet)
|
|
27
|
+
Optimizers::QuantifierOptimizer.new.visit(parslet)
|
|
28
|
+
end
|
|
29
|
+
|
|
30
|
+
# Simplifies sequences by flattening and merging adjacent strings
|
|
31
|
+
# Example: str('a') >> str('b') => str('ab')
|
|
32
|
+
# (str('a') >> str('b')) >> str('c') => str('abc')
|
|
33
|
+
#
|
|
34
|
+
# @param parslet [Parsanol::Atoms::Base] parslet to simplify
|
|
35
|
+
# @return [Parsanol::Atoms::Base] simplified parslet
|
|
36
|
+
def self.simplify_sequences(parslet)
|
|
37
|
+
Optimizers::SequenceOptimizer.new.visit(parslet)
|
|
38
|
+
end
|
|
39
|
+
|
|
40
|
+
# Simplifies choice/alternative patterns
|
|
41
|
+
# Example: (A | B) | C => A | B | C
|
|
42
|
+
# A | B | A => A | B
|
|
43
|
+
#
|
|
44
|
+
# @param parslet [Parsanol::Atoms::Base] parslet to simplify
|
|
45
|
+
# @return [Parsanol::Atoms::Base] simplified parslet
|
|
46
|
+
def self.simplify_choices(parslet)
|
|
47
|
+
Optimizers::ChoiceOptimizer.new.visit(parslet)
|
|
48
|
+
end
|
|
49
|
+
|
|
50
|
+
# Simplifies lookahead patterns
|
|
51
|
+
# Example: !(!x) => &x (double negation elimination)
|
|
52
|
+
#
|
|
53
|
+
# @param parslet [Parsanol::Atoms::Base] parslet to simplify
|
|
54
|
+
# @return [Parsanol::Atoms::Base] simplified parslet
|
|
55
|
+
def self.simplify_lookaheads(parslet)
|
|
56
|
+
Optimizers::LookaheadOptimizer.new.visit(parslet)
|
|
57
|
+
end
|
|
58
|
+
|
|
59
|
+
# Automatically insert cut operators where safe (AC-FIRST algorithm)
|
|
60
|
+
# Inserts cuts after deterministic prefixes when alternatives have disjoint FIRST sets
|
|
61
|
+
# This enables O(1) space complexity by allowing aggressive cache eviction
|
|
62
|
+
#
|
|
63
|
+
# Example: str('if') >> x | str('while') >> y
|
|
64
|
+
# => str('if').cut >> x | str('while').cut >> y
|
|
65
|
+
#
|
|
66
|
+
# @param parslet [Parsanol::Atoms::Base] parslet to optimize
|
|
67
|
+
# @return [Parsanol::Atoms::Base] optimized parslet with cuts inserted
|
|
68
|
+
def self.insert_cuts(parslet)
|
|
69
|
+
Optimizers::CutInserter.new.optimize(parslet)
|
|
70
|
+
end
|
|
71
|
+
|
|
72
|
+
# Apply all optimizations in recommended order
|
|
73
|
+
# This is a convenience method that applies all optimizer passes
|
|
74
|
+
#
|
|
75
|
+
# @param parslet [Parsanol::Atoms::Base] parslet to optimize
|
|
76
|
+
# @return [Parsanol::Atoms::Base] fully optimized parslet
|
|
77
|
+
def self.optimize_all(parslet)
|
|
78
|
+
result = simplify_quantifiers(parslet)
|
|
79
|
+
result = simplify_sequences(result)
|
|
80
|
+
result = simplify_choices(result)
|
|
81
|
+
result = simplify_lookaheads(result)
|
|
82
|
+
insert_cuts(result)
|
|
83
|
+
end
|
|
84
|
+
end
|
|
85
|
+
end
|
|
@@ -0,0 +1,78 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require_relative "../ast_visitor"
|
|
4
|
+
|
|
5
|
+
module Parsanol
|
|
6
|
+
module Optimizers
|
|
7
|
+
# Optimizes alternative/choice patterns in the AST
|
|
8
|
+
# Follows visitor pattern for clean separation of concerns
|
|
9
|
+
#
|
|
10
|
+
# Transformations:
|
|
11
|
+
# - (A | B) | C => A | B | C (flatten nested alternatives)
|
|
12
|
+
# - A | B | A => A | B (remove duplicates)
|
|
13
|
+
# - Alternative(A) => A (unwrap single-element alternatives)
|
|
14
|
+
class ChoiceOptimizer < ASTVisitor
|
|
15
|
+
# Visit an alternative node and apply choice optimizations
|
|
16
|
+
# @param parslet [Parsanol::Atoms::Alternative] alternative to optimize
|
|
17
|
+
# @return [Parsanol::Atoms::Base] optimized parslet
|
|
18
|
+
def visit_alternative(parslet)
|
|
19
|
+
# First optimize children recursively
|
|
20
|
+
new_alternatives = parslet.alternatives.map { |p| visit(p) }
|
|
21
|
+
|
|
22
|
+
# Optimization 1: Flatten nested alternatives
|
|
23
|
+
flattened = flatten_alternatives(new_alternatives)
|
|
24
|
+
|
|
25
|
+
# Optimization 2: Remove duplicate alternatives
|
|
26
|
+
deduplicated = deduplicate_alternatives(flattened)
|
|
27
|
+
|
|
28
|
+
# Optimization 3: Unwrap single-element alternatives
|
|
29
|
+
return deduplicated[0] if deduplicated.size == 1
|
|
30
|
+
|
|
31
|
+
# Return optimized alternative if changed
|
|
32
|
+
if deduplicated == parslet.alternatives
|
|
33
|
+
parslet
|
|
34
|
+
else
|
|
35
|
+
Parsanol::Atoms::Alternative.new(*deduplicated)
|
|
36
|
+
end
|
|
37
|
+
end
|
|
38
|
+
|
|
39
|
+
private
|
|
40
|
+
|
|
41
|
+
# Flatten nested alternatives into a single level
|
|
42
|
+
# @param alternatives [Array<Parsanol::Atoms::Base>] array of alternatives
|
|
43
|
+
# @return [Array<Parsanol::Atoms::Base>] flattened array
|
|
44
|
+
def flatten_alternatives(alternatives)
|
|
45
|
+
result = []
|
|
46
|
+
alternatives.each do |alt|
|
|
47
|
+
if alt.is_a?(Parsanol::Atoms::Alternative)
|
|
48
|
+
result.concat(alt.alternatives)
|
|
49
|
+
else
|
|
50
|
+
result << alt
|
|
51
|
+
end
|
|
52
|
+
end
|
|
53
|
+
result
|
|
54
|
+
end
|
|
55
|
+
|
|
56
|
+
# Remove duplicate alternatives using structural equality
|
|
57
|
+
# @param alternatives [Array<Parsanol::Atoms::Base>] array of alternatives
|
|
58
|
+
# @return [Array<Parsanol::Atoms::Base>] deduplicated array
|
|
59
|
+
def deduplicate_alternatives(alternatives)
|
|
60
|
+
return alternatives if alternatives.size < 2
|
|
61
|
+
|
|
62
|
+
# Use to_s as proxy for structural equality
|
|
63
|
+
seen = {}
|
|
64
|
+
result = []
|
|
65
|
+
|
|
66
|
+
alternatives.each do |alt|
|
|
67
|
+
key = alt.to_s
|
|
68
|
+
unless seen[key]
|
|
69
|
+
seen[key] = true
|
|
70
|
+
result << alt
|
|
71
|
+
end
|
|
72
|
+
end
|
|
73
|
+
|
|
74
|
+
result
|
|
75
|
+
end
|
|
76
|
+
end
|
|
77
|
+
end
|
|
78
|
+
end
|
|
@@ -0,0 +1,182 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
# Automatic Cut Insertion (AC-FIRST Algorithm)
|
|
4
|
+
#
|
|
5
|
+
# This optimizer implements the AC-FIRST algorithm from Mizushima et al. (2010)
|
|
6
|
+
# to automatically insert cut operators when alternatives have disjoint FIRST sets.
|
|
7
|
+
#
|
|
8
|
+
# When all alternatives in a choice have non-overlapping FIRST sets, we can safely
|
|
9
|
+
# insert a cut after the deterministic prefix, since backtracking will never be
|
|
10
|
+
# needed.
|
|
11
|
+
#
|
|
12
|
+
# Example:
|
|
13
|
+
# str('if') >> condition >> then_clause |
|
|
14
|
+
# str('while') >> condition >> body |
|
|
15
|
+
# str('print') >> expression
|
|
16
|
+
#
|
|
17
|
+
# Becomes:
|
|
18
|
+
# str('if').cut >> condition >> then_clause |
|
|
19
|
+
# str('while').cut >> condition >> body |
|
|
20
|
+
# str('print').cut >> expression
|
|
21
|
+
#
|
|
22
|
+
# Reference: Mizushima et al. (2010) "Packrat Parsers Can Handle Practical
|
|
23
|
+
# Grammars in Mostly Constant Space"
|
|
24
|
+
#
|
|
25
|
+
module Parsanol
|
|
26
|
+
module Optimizers
|
|
27
|
+
class CutInserter
|
|
28
|
+
# Optimize a parslet by inserting cuts where safe
|
|
29
|
+
# Recursively traverses the grammar AST
|
|
30
|
+
#
|
|
31
|
+
# @param parslet [Parsanol::Atoms::Base] The parslet to optimize
|
|
32
|
+
# @return [Parsanol::Atoms::Base] Optimized parslet with cuts inserted
|
|
33
|
+
def optimize(parslet)
|
|
34
|
+
case parslet
|
|
35
|
+
when Parsanol::Atoms::Alternative
|
|
36
|
+
optimize_alternative(parslet)
|
|
37
|
+
when Parsanol::Atoms::Sequence
|
|
38
|
+
optimize_sequence(parslet)
|
|
39
|
+
when Parsanol::Atoms::Repetition
|
|
40
|
+
optimize_repetition(parslet)
|
|
41
|
+
when Parsanol::Atoms::Named
|
|
42
|
+
optimize_named(parslet)
|
|
43
|
+
else
|
|
44
|
+
# Return atom unchanged (Str, Re, Lookahead, etc.)
|
|
45
|
+
parslet
|
|
46
|
+
end
|
|
47
|
+
end
|
|
48
|
+
|
|
49
|
+
private
|
|
50
|
+
|
|
51
|
+
# Optimize an Alternative atom by inserting cuts when all alternatives
|
|
52
|
+
# have disjoint FIRST sets
|
|
53
|
+
def optimize_alternative(alt)
|
|
54
|
+
alternatives = alt.alternatives
|
|
55
|
+
first_sets = alternatives.map(&:first_set)
|
|
56
|
+
|
|
57
|
+
# Only optimize if all FIRST sets are disjoint
|
|
58
|
+
unless Parsanol::FirstSet.all_disjoint?(first_sets)
|
|
59
|
+
# Not safe to insert cuts - return alternatives with recursive optimization
|
|
60
|
+
optimized = alternatives.map { |a| optimize(a) }
|
|
61
|
+
return Parsanol::Atoms::Alternative.new(*optimized)
|
|
62
|
+
end
|
|
63
|
+
|
|
64
|
+
# All FIRST sets are disjoint - safe to insert cuts!
|
|
65
|
+
# Insert cuts after deterministic prefixes
|
|
66
|
+
optimized = alternatives.map do |alternative|
|
|
67
|
+
insert_cut_if_safe(alternative)
|
|
68
|
+
end
|
|
69
|
+
|
|
70
|
+
Parsanol::Atoms::Alternative.new(*optimized)
|
|
71
|
+
end
|
|
72
|
+
|
|
73
|
+
# Optimize a Sequence atom by recursively optimizing its elements
|
|
74
|
+
def optimize_sequence(seq)
|
|
75
|
+
optimized_parslets = seq.parslets.map { |p| optimize(p) }
|
|
76
|
+
Parsanol::Atoms::Sequence.new(*optimized_parslets)
|
|
77
|
+
end
|
|
78
|
+
|
|
79
|
+
# Optimize a Repetition atom by recursively optimizing its parslet
|
|
80
|
+
def optimize_repetition(rep)
|
|
81
|
+
optimized_parslet = optimize(rep.parslet)
|
|
82
|
+
# Create new repetition with same min/max
|
|
83
|
+
# Note: We use default tag since it's not exposed as a reader
|
|
84
|
+
Parsanol::Atoms::Repetition.new(
|
|
85
|
+
optimized_parslet,
|
|
86
|
+
rep.min,
|
|
87
|
+
rep.max,
|
|
88
|
+
)
|
|
89
|
+
end
|
|
90
|
+
|
|
91
|
+
# Optimize a Named atom by recursively optimizing its parslet
|
|
92
|
+
def optimize_named(named)
|
|
93
|
+
optimized_parslet = optimize(named.parslet)
|
|
94
|
+
optimized_parslet.as(named.name)
|
|
95
|
+
end
|
|
96
|
+
|
|
97
|
+
# Insert a cut after the deterministic prefix if safe
|
|
98
|
+
# For sequences: find longest prefix without EPSILON
|
|
99
|
+
# For other atoms: cut the whole thing if it doesn't include EPSILON
|
|
100
|
+
def insert_cut_if_safe(parslet)
|
|
101
|
+
# For sequences, find the longest safe prefix
|
|
102
|
+
if parslet.is_a?(Parsanol::Atoms::Sequence)
|
|
103
|
+
prefix_parslets = find_deterministic_prefix(parslet)
|
|
104
|
+
if prefix_parslets && !prefix_parslets.empty?
|
|
105
|
+
return build_cut_sequence(parslet,
|
|
106
|
+
prefix_parslets)
|
|
107
|
+
end
|
|
108
|
+
end
|
|
109
|
+
|
|
110
|
+
# For other atoms, cut the whole thing if safe
|
|
111
|
+
return parslet.cut if safe_to_cut?(parslet)
|
|
112
|
+
|
|
113
|
+
# Not safe to cut - recursively optimize and return
|
|
114
|
+
optimize(parslet)
|
|
115
|
+
end
|
|
116
|
+
|
|
117
|
+
# Find the longest deterministic prefix of a sequence
|
|
118
|
+
# A deterministic prefix doesn't include EPSILON in its FIRST set
|
|
119
|
+
#
|
|
120
|
+
# @param sequence [Parsanol::Atoms::Sequence] The sequence to analyze
|
|
121
|
+
# @return [Array<Parsanol::Atoms::Base>] Prefix parslets, or nil if none
|
|
122
|
+
def find_deterministic_prefix(sequence)
|
|
123
|
+
parslets = sequence.parslets
|
|
124
|
+
prefix_length = 0
|
|
125
|
+
|
|
126
|
+
# Find longest prefix where no element can match empty
|
|
127
|
+
parslets.each do |p|
|
|
128
|
+
break if p.first_set.include?(Parsanol::FirstSet::EPSILON)
|
|
129
|
+
|
|
130
|
+
prefix_length += 1
|
|
131
|
+
end
|
|
132
|
+
|
|
133
|
+
prefix_length.positive? ? parslets[0...prefix_length] : nil
|
|
134
|
+
end
|
|
135
|
+
|
|
136
|
+
# Check if it's safe to cut after this parslet
|
|
137
|
+
# Safe if the parslet doesn't have EPSILON in its FIRST set
|
|
138
|
+
# (i.e., it always consumes input)
|
|
139
|
+
def safe_to_cut?(parslet)
|
|
140
|
+
first = parslet.first_set
|
|
141
|
+
# Don't cut if EPSILON is in FIRST set (might not consume)
|
|
142
|
+
# Also don't cut if FIRST set contains only nil (unknown)
|
|
143
|
+
return false if first.include?(Parsanol::FirstSet::EPSILON)
|
|
144
|
+
return false if first.all?(&:nil?)
|
|
145
|
+
|
|
146
|
+
true
|
|
147
|
+
end
|
|
148
|
+
|
|
149
|
+
# Build a new sequence with a cut after the prefix
|
|
150
|
+
#
|
|
151
|
+
# @param sequence [Parsanol::Atoms::Sequence] Original sequence
|
|
152
|
+
# @param prefix_parslets [Array] Parslets forming the deterministic prefix
|
|
153
|
+
# @return [Parsanol::Atoms::Base] New sequence with cut inserted
|
|
154
|
+
def build_cut_sequence(sequence, prefix_parslets)
|
|
155
|
+
# Recursively optimize prefix parslets
|
|
156
|
+
optimized_prefix = prefix_parslets.map { |p| optimize(p) }
|
|
157
|
+
|
|
158
|
+
# Build prefix (single parslet or sequence)
|
|
159
|
+
prefix = if optimized_prefix.length == 1
|
|
160
|
+
optimized_prefix.first
|
|
161
|
+
else
|
|
162
|
+
Parsanol::Atoms::Sequence.new(*optimized_prefix)
|
|
163
|
+
end
|
|
164
|
+
|
|
165
|
+
# Get remaining parslets after prefix
|
|
166
|
+
remaining = sequence.parslets[prefix_parslets.length..]
|
|
167
|
+
|
|
168
|
+
# Recursively optimize remaining parslets
|
|
169
|
+
optimized_remaining = remaining.map { |p| optimize(p) }
|
|
170
|
+
|
|
171
|
+
# Build final sequence with cut
|
|
172
|
+
if optimized_remaining.empty?
|
|
173
|
+
# Prefix is the entire sequence
|
|
174
|
+
prefix.cut
|
|
175
|
+
else
|
|
176
|
+
# Prefix + cut + remaining
|
|
177
|
+
Parsanol::Atoms::Sequence.new(prefix.cut, *optimized_remaining)
|
|
178
|
+
end
|
|
179
|
+
end
|
|
180
|
+
end
|
|
181
|
+
end
|
|
182
|
+
end
|
|
@@ -0,0 +1,56 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require_relative "../ast_visitor"
|
|
4
|
+
|
|
5
|
+
module Parsanol
|
|
6
|
+
module Optimizers
|
|
7
|
+
# Optimizes lookahead patterns in the AST
|
|
8
|
+
# Follows visitor pattern for clean separation of concerns
|
|
9
|
+
#
|
|
10
|
+
# Transformations:
|
|
11
|
+
# - !(!x) => &x (double negation elimination)
|
|
12
|
+
# - &(&x) => &x (positive lookahead is idempotent)
|
|
13
|
+
# - !(&x) => !x (negative of positive)
|
|
14
|
+
# - &(!x) => !x (positive of negative)
|
|
15
|
+
class LookaheadOptimizer < ASTVisitor
|
|
16
|
+
# Visit a lookahead node and apply lookahead optimizations
|
|
17
|
+
# @param parslet [Parsanol::Atoms::Lookahead] lookahead to optimize
|
|
18
|
+
# @return [Parsanol::Atoms::Base] optimized parslet
|
|
19
|
+
def visit_lookahead(parslet)
|
|
20
|
+
# First optimize the child
|
|
21
|
+
inner = visit(parslet.bound_parslet)
|
|
22
|
+
|
|
23
|
+
# If inner is also a lookahead, simplify nested lookaheads
|
|
24
|
+
if inner.is_a?(Parsanol::Atoms::Lookahead)
|
|
25
|
+
outer_positive = parslet.positive
|
|
26
|
+
inner_positive = inner.positive
|
|
27
|
+
|
|
28
|
+
# !(!x) => &x (double negation)
|
|
29
|
+
if !outer_positive && !inner_positive
|
|
30
|
+
return Parsanol::Atoms::Lookahead.new(inner.bound_parslet,
|
|
31
|
+
true)
|
|
32
|
+
end
|
|
33
|
+
|
|
34
|
+
# &(&x) => &x (idempotent)
|
|
35
|
+
return inner if outer_positive && inner_positive
|
|
36
|
+
|
|
37
|
+
# !(&x) => !x (negative of positive)
|
|
38
|
+
if !outer_positive && inner_positive
|
|
39
|
+
return Parsanol::Atoms::Lookahead.new(inner.bound_parslet,
|
|
40
|
+
false)
|
|
41
|
+
end
|
|
42
|
+
|
|
43
|
+
# &(!x) => !x (positive of negative)
|
|
44
|
+
return inner if outer_positive && !inner_positive
|
|
45
|
+
end
|
|
46
|
+
|
|
47
|
+
# Return lookahead with optimized child
|
|
48
|
+
if inner.equal?(parslet.bound_parslet)
|
|
49
|
+
parslet
|
|
50
|
+
else
|
|
51
|
+
Parsanol::Atoms::Lookahead.new(inner, parslet.positive)
|
|
52
|
+
end
|
|
53
|
+
end
|
|
54
|
+
end
|
|
55
|
+
end
|
|
56
|
+
end
|
|
@@ -0,0 +1,60 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require_relative "../ast_visitor"
|
|
4
|
+
|
|
5
|
+
module Parsanol
|
|
6
|
+
module Optimizers
|
|
7
|
+
# Optimizes repetition/quantifier patterns in the AST
|
|
8
|
+
# Follows visitor pattern for clean separation of concerns
|
|
9
|
+
#
|
|
10
|
+
# Transformations:
|
|
11
|
+
# - repeat(1,1) => unwrap (identity transformation)
|
|
12
|
+
# - repeat(0,1).repeat(0,1) => repeat(0,1) (idempotent)
|
|
13
|
+
# - repeat(n,n).repeat(m,m) => repeat(n*m,n*m) (multiply exact counts)
|
|
14
|
+
class QuantifierOptimizer < ASTVisitor
|
|
15
|
+
# Visit a repetition node and apply quantifier optimizations
|
|
16
|
+
# @param parslet [Parsanol::Atoms::Repetition] repetition to optimize
|
|
17
|
+
# @return [Parsanol::Atoms::Base] optimized parslet
|
|
18
|
+
def visit_repetition(parslet)
|
|
19
|
+
# First optimize the child
|
|
20
|
+
inner = visit(parslet.parslet)
|
|
21
|
+
|
|
22
|
+
# Optimization 1: repeat(1,1) is identity - unwrap it
|
|
23
|
+
return inner if parslet.min == 1 && parslet.max == 1
|
|
24
|
+
|
|
25
|
+
# Optimization 2: Nested repetitions
|
|
26
|
+
if inner.is_a?(Parsanol::Atoms::Repetition)
|
|
27
|
+
# repeat(0,1).repeat(0,1) => repeat(0,1) (idempotent)
|
|
28
|
+
if parslet.min.zero? && parslet.max == 1 &&
|
|
29
|
+
inner.min.zero? && inner.max == 1
|
|
30
|
+
return inner
|
|
31
|
+
end
|
|
32
|
+
|
|
33
|
+
# repeat(n,n).repeat(m,m) => repeat(n*m,n*m) for exact counts
|
|
34
|
+
if parslet.min == parslet.max && inner.min == inner.max &&
|
|
35
|
+
parslet.max && inner.max
|
|
36
|
+
new_count = parslet.min * inner.min
|
|
37
|
+
return Parsanol::Atoms::Repetition.new(
|
|
38
|
+
inner.parslet,
|
|
39
|
+
new_count,
|
|
40
|
+
new_count,
|
|
41
|
+
parslet.result_tag,
|
|
42
|
+
)
|
|
43
|
+
end
|
|
44
|
+
end
|
|
45
|
+
|
|
46
|
+
# Return optimized repetition with simplified child
|
|
47
|
+
if inner.equal?(parslet.parslet)
|
|
48
|
+
parslet
|
|
49
|
+
else
|
|
50
|
+
Parsanol::Atoms::Repetition.new(
|
|
51
|
+
inner,
|
|
52
|
+
parslet.min,
|
|
53
|
+
parslet.max,
|
|
54
|
+
parslet.result_tag,
|
|
55
|
+
)
|
|
56
|
+
end
|
|
57
|
+
end
|
|
58
|
+
end
|
|
59
|
+
end
|
|
60
|
+
end
|