parsanol 1.3.115 → 1.3.117
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/Cargo.lock +4 -4
- data/ext/parsanol_native/Cargo.toml +2 -6
- data/ext/parsanol_native/extconf.rb +9 -0
- data/lib/parsanol/atoms/alternative.rb +3 -1
- data/lib/parsanol/atoms/base.rb +21 -8
- data/lib/parsanol/atoms/context.rb +37 -35
- data/lib/parsanol/atoms/repetition.rb +33 -0
- data/lib/parsanol/native/parser.rb +11 -5
- data/lib/parsanol/native.rb +45 -1
- data/lib/parsanol/parg/artifact.rb +6 -0
- data/lib/parsanol/parg/bindings.rb +48 -2
- data/lib/parsanol/parg/compiler.rb +28 -5
- data/lib/parsanol/parg/parser.rb +8 -3
- data/lib/parsanol/parser.rb +27 -2
- data/lib/parsanol/version.rb +1 -1
- metadata +1 -1
checksums.yaml
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
---
|
|
2
2
|
SHA256:
|
|
3
|
-
metadata.gz:
|
|
4
|
-
data.tar.gz:
|
|
3
|
+
metadata.gz: '069ffc4a0b824942b8c6e21cbab7a9b0e450636049338c4f38fce28d3109e2d7'
|
|
4
|
+
data.tar.gz: 3b1341f63b428c23cfe66169eb0c9c5cf52b98c7af09ee785954f3256fa0db1f
|
|
5
5
|
SHA512:
|
|
6
|
-
metadata.gz:
|
|
7
|
-
data.tar.gz:
|
|
6
|
+
metadata.gz: 7c1d2039bab57923c91d24bf6a791ea86b23d71d576e51e51140f4bcd5044b962fb6fd333ba1c0df4ecdcd8828d0547f380c2d5c54520e3bb7a588c62b427ddb
|
|
7
|
+
data.tar.gz: 7d321301a11d0a437a964930c77d84a80bdd112be959476f8be087b585649efd36fd197f0b915bb51e5b865953ef3670ddbe77e66b910f450c7bb3976958d68d
|
data/Cargo.lock
CHANGED
|
@@ -307,9 +307,9 @@ checksum = "9f7c3e4beb33f85d45ae3e3a1792185706c8e16d043238c593331cc7cd313b50"
|
|
|
307
307
|
|
|
308
308
|
[[package]]
|
|
309
309
|
name = "parsanol"
|
|
310
|
-
version = "0.19.
|
|
310
|
+
version = "0.19.14"
|
|
311
311
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
312
|
-
checksum = "
|
|
312
|
+
checksum = "27faaa398cc92b331b0e1d08abfa475e697e8d23b0ea90e211a3dc8590e42bbd"
|
|
313
313
|
dependencies = [
|
|
314
314
|
"ahash",
|
|
315
315
|
"getrandom 0.3.4",
|
|
@@ -326,9 +326,9 @@ dependencies = [
|
|
|
326
326
|
|
|
327
327
|
[[package]]
|
|
328
328
|
name = "parsanol-derive"
|
|
329
|
-
version = "0.19.
|
|
329
|
+
version = "0.19.14"
|
|
330
330
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
331
|
-
checksum = "
|
|
331
|
+
checksum = "b6a65826a4a1f7ec796e29916963803a3deff86f9f9de9e8180f49f6173292a4"
|
|
332
332
|
dependencies = [
|
|
333
333
|
"proc-macro2",
|
|
334
334
|
"quote",
|
|
@@ -27,12 +27,8 @@ rb-sys = { version = "0.9.124", features = ["global-allocator"] }
|
|
|
27
27
|
# magnus for Ruby bindings (from crates.io, patched locally for Ruby 4.0 support)
|
|
28
28
|
magnus = { version = "0.9" }
|
|
29
29
|
|
|
30
|
-
#
|
|
31
|
-
|
|
32
|
-
# parsanol parser library (crates.io; 0.17.0 carries the LookSpec regex
|
|
33
|
-
# lookbehind (rs#163 gap 1) and the widened opaque writeback: the
|
|
34
|
-
# lazy-continuation fix, parsanol-ruby#160)
|
|
35
|
-
parsanol = { version = "0.19.1", features = ["ruby"] }
|
|
30
|
+
# Native legacy compatibility and compact recursive parser results.
|
|
31
|
+
parsanol = { version = "0.19.12", features = ["ruby"] }
|
|
36
32
|
|
|
37
33
|
# Logging
|
|
38
34
|
log = "0.4"
|
|
@@ -30,6 +30,15 @@ create_rust_makefile("parsanol/parsanol_native") do |r|
|
|
|
30
30
|
# Create debug builds in dev, release in production
|
|
31
31
|
r.profile = ENV.fetch("RB_SYS_CARGO_PROFILE", :dev).to_sym
|
|
32
32
|
|
|
33
|
+
# Persist RubyInstaller's headers into Make's later Cargo subprocess.
|
|
34
|
+
if RUBY_PLATFORM.include?("mingw") && ENV.key?("RI_DEVKIT") && ENV.key?("MSYSTEM_PREFIX")
|
|
35
|
+
sysroot = ENV.fetch("RI_DEVKIT").tr("\\", "/") + ENV.fetch("MSYSTEM_PREFIX")
|
|
36
|
+
r.env["BINDGEN_EXTRA_CLANG_ARGS"] = ENV.fetch("BINDGEN_EXTRA_CLANG_ARGS") do
|
|
37
|
+
"--target=x86_64-pc-windows-gnu --sysroot=#{sysroot} " \
|
|
38
|
+
"-D__AVX512VLFP16INTRIN_H -D__AVX512FP16INTRIN_H"
|
|
39
|
+
end
|
|
40
|
+
end
|
|
41
|
+
|
|
33
42
|
# cargo rustc builds dependency cdylibs verbatim, and parsanol ships
|
|
34
43
|
# a cdylib crate-type (the ffi tier). On macOS every dylib in the
|
|
35
44
|
# build resolves ruby symbols from the host process at load time, so
|
|
@@ -117,7 +117,9 @@ module Parsanol
|
|
|
117
117
|
# The reporting pass (the re-parse after a failure) must try every
|
|
118
118
|
# branch so the failure cause tree stays complete; the literal index
|
|
119
119
|
# only prunes the fast non-reporting pass.
|
|
120
|
-
|
|
120
|
+
# Legacy memo answers depend on attempted branches, including a
|
|
121
|
+
# prefix success in a branch that later fails its literal suffix.
|
|
122
|
+
unless context.reporting? || context.parslet_compatible?
|
|
121
123
|
indexed = candidate_indexes(source)
|
|
122
124
|
if indexed
|
|
123
125
|
return try_selected(indexed, options, source, context, consume_all)
|
data/lib/parsanol/atoms/base.rb
CHANGED
|
@@ -27,17 +27,27 @@ module Parsanol
|
|
|
27
27
|
# @param options [Hash] parsing options
|
|
28
28
|
# @option options [Parsanol::ErrorReporter] :reporter error collector
|
|
29
29
|
# @option options [Boolean] :prefix allow partial parse (default: false)
|
|
30
|
+
# @option options [Symbol] :compatibility :strict (default), or :parslet
|
|
31
|
+
# for legacy consume-all memo decisions on the Ruby interpreter
|
|
30
32
|
# @return [Object] the parsed result
|
|
31
33
|
# @raise [Parsanol::ParseFailed] on parse failure
|
|
32
34
|
def parse(source, options = {})
|
|
33
35
|
must_consume_all = !options[:prefix]
|
|
36
|
+
compatibility = options.fetch(:compatibility, :strict)
|
|
37
|
+
unless %i[strict parslet].include?(compatibility)
|
|
38
|
+
raise ArgumentError, "Unknown compatibility policy: #{compatibility}"
|
|
39
|
+
end
|
|
40
|
+
|
|
41
|
+
context_options = compatibility == :strict ? {} : { compatibility: compatibility }
|
|
34
42
|
|
|
35
43
|
# Compiled-VM fast path: String input, full-consumption semantics.
|
|
36
44
|
# On failure (or unsupported grammar / budget exhaustion) fall
|
|
37
45
|
# through to the interpreter, which also produces the exact
|
|
38
46
|
# cause-tree diagnostics. run_for memoizes heavy-backtracking
|
|
39
47
|
# grammars transparently.
|
|
40
|
-
|
|
48
|
+
# The VM implements strict memo semantics. The opt-in legacy policy
|
|
49
|
+
# is currently supported by the Ruby interpreter only.
|
|
50
|
+
if compatibility == :strict && must_consume_all && source.is_a?(String) &&
|
|
41
51
|
(program = VM.program_for(self))
|
|
42
52
|
begin
|
|
43
53
|
result = VM.run_for(self, program, source, true)
|
|
@@ -62,12 +72,12 @@ module Parsanol
|
|
|
62
72
|
input = normalize_input(source)
|
|
63
73
|
|
|
64
74
|
# Initial parse attempt (no error collection)
|
|
65
|
-
success, value = run_with_context(input, nil, must_consume_all)
|
|
75
|
+
success, value = run_with_context(input, nil, must_consume_all, **context_options)
|
|
66
76
|
return finalize_result(value) if success
|
|
67
77
|
|
|
68
78
|
# Reparse with error reporting for diagnostics
|
|
69
79
|
report_detailed_error(input, must_consume_all, options[:reporter],
|
|
70
|
-
value)
|
|
80
|
+
value, **context_options)
|
|
71
81
|
end
|
|
72
82
|
|
|
73
83
|
# Creates a new parsing context and executes the atom.
|
|
@@ -76,10 +86,11 @@ module Parsanol
|
|
|
76
86
|
# @param reporter [Object, nil] error reporter
|
|
77
87
|
# @param consume_all [Boolean] require complete consumption
|
|
78
88
|
# @return [Array(Boolean, Object)] outcome tuple
|
|
79
|
-
def run_with_context(input, reporter, consume_all)
|
|
89
|
+
def run_with_context(input, reporter, consume_all, compatibility: :strict)
|
|
80
90
|
parser_class = detect_parser_class
|
|
81
91
|
context = Parsanol::Atoms::Context.new(reporter,
|
|
82
|
-
parser_class: parser_class
|
|
92
|
+
parser_class: parser_class,
|
|
93
|
+
compatibility: compatibility)
|
|
83
94
|
apply(input, context, consume_all)
|
|
84
95
|
end
|
|
85
96
|
|
|
@@ -242,10 +253,12 @@ module Parsanol
|
|
|
242
253
|
# consumer's file:line:col degrades to "line 1 char 1" when the
|
|
243
254
|
# interpreter raises the outermost sequence instead. Explicit
|
|
244
255
|
# reporters (Tree for full cause trees) are honored unchanged.
|
|
245
|
-
def report_detailed_error(input, consume_all, reporter, _initial_error)
|
|
256
|
+
def report_detailed_error(input, consume_all, reporter, _initial_error, compatibility: :strict)
|
|
246
257
|
input.bytepos = 0
|
|
247
|
-
error_reporter = reporter ||
|
|
248
|
-
|
|
258
|
+
error_reporter = reporter ||
|
|
259
|
+
(compatibility == :parslet ? Parsanol::ErrorReporter::Tree.new : Parsanol::ErrorReporter::Deepest.new)
|
|
260
|
+
context_options = compatibility == :strict ? {} : { compatibility: compatibility }
|
|
261
|
+
success, cause = run_with_context(input, error_reporter, consume_all, **context_options)
|
|
249
262
|
|
|
250
263
|
# Second parse should also fail
|
|
251
264
|
raise "Invariant violation: parse succeeded during error reporting" if success
|
|
@@ -44,11 +44,24 @@ module Parsanol
|
|
|
44
44
|
# @param interval_cache: [Boolean] enable GPeg-style interval caching
|
|
45
45
|
# @param adaptive_cache_threshold: [Integer, nil] minimum input size for caching
|
|
46
46
|
# @param parser_class: [Class, nil] parser class for threshold selection
|
|
47
|
+
# @param compatibility: [Symbol] :strict or opt-in :parslet. Legacy
|
|
48
|
+
# decisions use eager, parse-lifetime memo entries; mutable capture
|
|
49
|
+
# and dynamic evaluations still retain their cache barriers.
|
|
47
50
|
#
|
|
48
51
|
def initialize(error_reporter = Parsanol::ErrorReporter::Tree.new,
|
|
49
52
|
interval_cache: false,
|
|
50
53
|
adaptive_cache_threshold: nil,
|
|
51
|
-
parser_class: nil
|
|
54
|
+
parser_class: nil,
|
|
55
|
+
compatibility: :strict)
|
|
56
|
+
unless %i[strict parslet].include?(compatibility)
|
|
57
|
+
raise ArgumentError, "Unknown compatibility policy: #{compatibility}"
|
|
58
|
+
end
|
|
59
|
+
|
|
60
|
+
@parslet_compatibility = compatibility == :parslet
|
|
61
|
+
if @parslet_compatibility && interval_cache
|
|
62
|
+
raise NotImplementedError, "Parslet compatibility does not yet support interval caching"
|
|
63
|
+
end
|
|
64
|
+
|
|
52
65
|
# Core memoization cache: position -> { atom_id -> [result, advance] }
|
|
53
66
|
# Auto-vivifying sub-hashes: a probe-phase position can reach the
|
|
54
67
|
# active-path lookup before any entry exists at that position
|
|
@@ -84,11 +97,6 @@ module Parsanol
|
|
|
84
97
|
@array_pool = Parsanol::Pools::ArrayPool.new(size: 10_000, preallocate: false)
|
|
85
98
|
@buffer_pool = Parsanol::Pools::BufferPool.new(pool_size: 100)
|
|
86
99
|
|
|
87
|
-
# Selective memoization tracking
|
|
88
|
-
@hit_stats = Hash.new(0)
|
|
89
|
-
@miss_stats = Hash.new(0)
|
|
90
|
-
@min_hits_for_cache = 2
|
|
91
|
-
|
|
92
100
|
# Optional GPeg-style interval caching
|
|
93
101
|
@use_intervals = interval_cache
|
|
94
102
|
if @use_intervals
|
|
@@ -117,7 +125,7 @@ module Parsanol
|
|
|
117
125
|
|
|
118
126
|
@adaptive_threshold = threshold
|
|
119
127
|
@input_len = nil
|
|
120
|
-
@caching_active =
|
|
128
|
+
@caching_active = @parslet_compatibility
|
|
121
129
|
@backtrack_events = 0
|
|
122
130
|
|
|
123
131
|
# Monotonic counter of cache-unsafe events (dynamic-atom evaluations
|
|
@@ -132,6 +140,10 @@ module Parsanol
|
|
|
132
140
|
# @return [Integer] monotonic count of cache-unsafe events so far
|
|
133
141
|
attr_reader :cache_unsafe_events
|
|
134
142
|
|
|
143
|
+
def parslet_compatible?
|
|
144
|
+
@parslet_compatibility
|
|
145
|
+
end
|
|
146
|
+
|
|
135
147
|
# Records that mutable parse state was read or written (dynamic atom
|
|
136
148
|
# evaluated, capture stored). Called by atoms; results computed across
|
|
137
149
|
# such events are excluded from memoization.
|
|
@@ -152,7 +164,11 @@ module Parsanol
|
|
|
152
164
|
#
|
|
153
165
|
def try_with_cache(atom, src, must_consume_all)
|
|
154
166
|
# Skip caching for atoms that don't benefit from it
|
|
155
|
-
|
|
167
|
+
legacy_pure = @parslet_compatibility &&
|
|
168
|
+
(atom.instance_of?(Parsanol::Atoms::Entity) ||
|
|
169
|
+
atom.instance_of?(Parsanol::Atoms::Str) || atom.instance_of?(Parsanol::Atoms::Re))
|
|
170
|
+
cacheable = atom.cached? || legacy_pure
|
|
171
|
+
return atom.try(src, self, must_consume_all) unless cacheable
|
|
156
172
|
|
|
157
173
|
# Use interval-based caching if enabled
|
|
158
174
|
return try_with_interval(atom, src, must_consume_all) if @use_intervals
|
|
@@ -171,7 +187,9 @@ module Parsanol
|
|
|
171
187
|
@furthest_pos = pos
|
|
172
188
|
@evict_counter += 1
|
|
173
189
|
|
|
174
|
-
|
|
190
|
+
# Legacy prefix decisions remain observable on later backtracking;
|
|
191
|
+
# retain them for this parse rather than silently re-evaluating.
|
|
192
|
+
if !@parslet_compatibility && @evict_counter >= @evict_interval
|
|
175
193
|
@evict_counter = 0
|
|
176
194
|
cutoff = pos - @evict_threshold
|
|
177
195
|
@memo.delete_if { |p, _| p < cutoff }
|
|
@@ -182,26 +200,21 @@ module Parsanol
|
|
|
182
200
|
# prefix-success fallback for ordered choice)
|
|
183
201
|
cached_key = cached_entry_key(@memo[pos], atom, must_consume_all)
|
|
184
202
|
if cached_key
|
|
185
|
-
@hit_stats[cached_key] += 1
|
|
186
203
|
outcome, delta = @memo[pos][cached_key]
|
|
187
204
|
src.bytepos = pos + delta
|
|
188
205
|
return outcome
|
|
189
206
|
end
|
|
190
207
|
|
|
191
208
|
# Cache miss - execute and store
|
|
192
|
-
@miss_stats[key] += 1
|
|
193
209
|
unsafe_before = @cache_unsafe_events
|
|
194
210
|
outcome = atom.try(src, self, must_consume_all)
|
|
195
211
|
delta = src.bytepos - pos
|
|
196
212
|
|
|
197
|
-
#
|
|
198
|
-
#
|
|
199
|
-
#
|
|
200
|
-
if
|
|
201
|
-
|
|
202
|
-
if attempts <= @min_hits_for_cache || @hit_stats[key].positive?
|
|
203
|
-
(@memo[pos] ||= {})[key] = [outcome, delta]
|
|
204
|
-
end
|
|
213
|
+
# Once backtracking activates memoization, retain every safe answer.
|
|
214
|
+
# An atom-wide attempts limit suppresses its entries at later positions
|
|
215
|
+
# and can leave recursive alternatives with exponential repeated work.
|
|
216
|
+
if @cache_unsafe_events == unsafe_before
|
|
217
|
+
(@memo[pos] ||= {})[key] = [outcome, delta]
|
|
205
218
|
end
|
|
206
219
|
|
|
207
220
|
outcome
|
|
@@ -217,23 +230,21 @@ module Parsanol
|
|
|
217
230
|
def try_with_interval(atom, src, must_consume_all)
|
|
218
231
|
pos = src.bytepos
|
|
219
232
|
key = scoped_cache_key(atom, must_consume_all)
|
|
220
|
-
|
|
221
|
-
|
|
233
|
+
_cached_key, cached = cached_interval_entry(atom, pos,
|
|
234
|
+
must_consume_all)
|
|
222
235
|
|
|
223
236
|
if cached
|
|
224
|
-
@hit_stats[cached_key] += 1
|
|
225
237
|
outcome, delta = cached
|
|
226
238
|
src.bytepos = pos + delta
|
|
227
239
|
return outcome
|
|
228
240
|
end
|
|
229
241
|
|
|
230
|
-
@miss_stats[key] += 1
|
|
231
242
|
unsafe_before = @cache_unsafe_events
|
|
232
243
|
outcome = atom.try(src, self, must_consume_all)
|
|
233
244
|
delta = src.bytepos - pos
|
|
234
245
|
end_pos = pos + delta
|
|
235
246
|
|
|
236
|
-
if
|
|
247
|
+
if @cache_unsafe_events == unsafe_before
|
|
237
248
|
@interval_trees[key].insert(pos, end_pos, [outcome, delta])
|
|
238
249
|
end
|
|
239
250
|
|
|
@@ -533,6 +544,8 @@ module Parsanol
|
|
|
533
544
|
end
|
|
534
545
|
|
|
535
546
|
def scoped_cache_key(atom, must_consume_all)
|
|
547
|
+
return shared_cache_key(atom) if @parslet_compatibility
|
|
548
|
+
|
|
536
549
|
must_consume_all ? strict_cache_key(atom) : shared_cache_key(atom)
|
|
537
550
|
end
|
|
538
551
|
|
|
@@ -554,17 +567,6 @@ module Parsanol
|
|
|
554
567
|
!must_consume_all && share_prefix_success_cache?(atom)
|
|
555
568
|
end
|
|
556
569
|
|
|
557
|
-
# An outcome may be memoized only when no cache-unsafe event (dynamic
|
|
558
|
-
# evaluation, capture write) happened while computing it, and the
|
|
559
|
-
# attempts heuristic deems the entry worthwhile.
|
|
560
|
-
def storable_outcome?(key, outcome, unsafe_before)
|
|
561
|
-
return false unless @cache_unsafe_events == unsafe_before
|
|
562
|
-
|
|
563
|
-
outcome.first ||
|
|
564
|
-
@hit_stats[key] + @miss_stats[key] <= @min_hits_for_cache ||
|
|
565
|
-
@hit_stats[key].positive?
|
|
566
|
-
end
|
|
567
|
-
|
|
568
570
|
def successful_prefix_entry?(entry)
|
|
569
571
|
entry && entry[0].is_a?(Array) && entry[0].first == true
|
|
570
572
|
end
|
|
@@ -74,6 +74,8 @@ module Parsanol
|
|
|
74
74
|
# @param consume_all [Boolean] require full consumption
|
|
75
75
|
# @return [Array(Boolean, Object)] result
|
|
76
76
|
def try(source, context, consume_all)
|
|
77
|
+
return try_compatible(source, context, consume_all) if context.parslet_compatible?
|
|
78
|
+
|
|
77
79
|
# Check for tree memoization support
|
|
78
80
|
if context.use_tree_memoization?
|
|
79
81
|
return with_tree_cache(source, context, consume_all)
|
|
@@ -119,6 +121,37 @@ module Parsanol
|
|
|
119
121
|
first
|
|
120
122
|
end
|
|
121
123
|
|
|
124
|
+
# Legacy repetition checks all children as prefixes. A bounded success
|
|
125
|
+
# returns before the outer consume-all guard; an exhausted optional
|
|
126
|
+
# failure checks consume-all inside #try. Their raw memo answers differ.
|
|
127
|
+
def try_compatible(source, context, consume_all)
|
|
128
|
+
start_pos = source.bytepos
|
|
129
|
+
result = [@result_tag]
|
|
130
|
+
count = 0
|
|
131
|
+
failure = nil
|
|
132
|
+
loop do
|
|
133
|
+
previous = source.bytepos
|
|
134
|
+
success, value = @parslet.apply(source, context, false)
|
|
135
|
+
unless success
|
|
136
|
+
failure = value
|
|
137
|
+
source.bytepos = previous
|
|
138
|
+
break
|
|
139
|
+
end
|
|
140
|
+
|
|
141
|
+
count += 1
|
|
142
|
+
result << value
|
|
143
|
+
return ok(result) if @max && count >= @max
|
|
144
|
+
# Retain the current engine's protection against empty loops.
|
|
145
|
+
break if source.bytepos == previous
|
|
146
|
+
end
|
|
147
|
+
return context.err_at(self, source, min_error, start_pos, [failure]) if count < @min
|
|
148
|
+
if consume_all && source.chars_left.positive?
|
|
149
|
+
return context.err(self, source, @extra_error, [failure])
|
|
150
|
+
end
|
|
151
|
+
|
|
152
|
+
ok(result)
|
|
153
|
+
end
|
|
154
|
+
|
|
122
155
|
# Optional match (0 or 1)
|
|
123
156
|
def try_maybe(source, context, _consume_all)
|
|
124
157
|
success, value = @parslet.apply(source, context, false)
|
|
@@ -57,9 +57,9 @@ module Parsanol
|
|
|
57
57
|
#
|
|
58
58
|
# @param grammar [Parsanol::Atoms::Base] Ruby grammar or JSON string
|
|
59
59
|
# @param input [String] Input string to parse
|
|
60
|
-
def parse(grammar, input)
|
|
60
|
+
def parse(grammar, input, **)
|
|
61
61
|
# Delegate to Parsanol::Native.parse for consistency
|
|
62
|
-
Parsanol::Native.parse(grammar, input)
|
|
62
|
+
Parsanol::Native.parse(grammar, input, **)
|
|
63
63
|
end
|
|
64
64
|
|
|
65
65
|
# Parse input with a Ruby grammar, returning the parslet-shaped
|
|
@@ -84,8 +84,13 @@ module Parsanol
|
|
|
84
84
|
# Resolve a Rust-side handle for the grammar, registering it once.
|
|
85
85
|
# Per-call cost is the memoized structure-hash lookup plus one Hash
|
|
86
86
|
# access — no JSON marshal, no re-hash inside Rust.
|
|
87
|
-
def grammar_handle(root_atom)
|
|
88
|
-
|
|
87
|
+
def grammar_handle(root_atom, compatibility: :strict)
|
|
88
|
+
root_atom = root_atom.root if root_atom.is_a?(::Parsanol::Parser)
|
|
89
|
+
legacy = compatibility == :parslet
|
|
90
|
+
# Legacy decisions depend on the exact shared-node graph. The
|
|
91
|
+
# strict structure hash deliberately omits repeated-node targets.
|
|
92
|
+
# Retaining the root in this bounded key also prevents id recycling.
|
|
93
|
+
cache_key = legacy ? [:parslet, root_atom] : grammar_cache_key(root_atom)
|
|
89
94
|
handle = HANDLE_CACHE[cache_key]
|
|
90
95
|
if handle
|
|
91
96
|
# Refresh recency: reinsertion moves the entry to the end,
|
|
@@ -94,7 +99,8 @@ module Parsanol
|
|
|
94
99
|
return handle
|
|
95
100
|
end
|
|
96
101
|
|
|
97
|
-
|
|
102
|
+
json = legacy ? GrammarSerializer.serialize(root_atom) : grammar_json(root_atom)
|
|
103
|
+
handle = Native._register_grammar(json)
|
|
98
104
|
HANDLE_CACHE[cache_key] = handle
|
|
99
105
|
trim_caches
|
|
100
106
|
handle
|
data/lib/parsanol/native.rb
CHANGED
|
@@ -58,9 +58,24 @@ module Parsanol
|
|
|
58
58
|
# result = Parsanol::Native.parse(str('hello').as(:greeting), "hello\nworld")
|
|
59
59
|
# result[:greeting].line_and_column # => [1, 1]
|
|
60
60
|
#
|
|
61
|
-
def parse(grammar, input)
|
|
61
|
+
def parse(grammar, input, compatibility: :strict)
|
|
62
|
+
unless %i[strict parslet].include?(compatibility)
|
|
63
|
+
raise ArgumentError, "Unknown compatibility policy: #{compatibility}"
|
|
64
|
+
end
|
|
62
65
|
raise LoadError, "Native parser not available" unless available?
|
|
63
66
|
|
|
67
|
+
if compatibility == :parslet
|
|
68
|
+
unless Parser.extension_loaded? && respond_to?(:_parse_handle_with_compatibility)
|
|
69
|
+
raise NotImplementedError,
|
|
70
|
+
"Native Parslet compatibility requires an extension with _parse_handle_with_compatibility"
|
|
71
|
+
end
|
|
72
|
+
if grammar.is_a?(String)
|
|
73
|
+
raise NotImplementedError, "Native Parslet compatibility requires grammar atoms, not JSON"
|
|
74
|
+
end
|
|
75
|
+
|
|
76
|
+
return parse_compatible_atom_grammar(grammar, input)
|
|
77
|
+
end
|
|
78
|
+
|
|
64
79
|
# ffi-gem tier (JRuby/TruffleRuby/no-binary MRI): same contract,
|
|
65
80
|
# batch-decoded results, single reporter-pass error fallback.
|
|
66
81
|
return Ffi.parse(grammar, input) unless Parser.extension_loaded?
|
|
@@ -366,6 +381,35 @@ module Parsanol
|
|
|
366
381
|
end
|
|
367
382
|
end
|
|
368
383
|
|
|
384
|
+
# Compatibility failures are authoritative native refusals. Unlike the
|
|
385
|
+
# legacy strict helper, this path never reparses or accepts in Ruby.
|
|
386
|
+
def parse_compatible_atom_grammar(grammar, input)
|
|
387
|
+
handle = Parser.grammar_handle(grammar, compatibility: :parslet)
|
|
388
|
+
raw = begin
|
|
389
|
+
_parse_handle_with_compatibility(handle, input, "parslet")
|
|
390
|
+
rescue ArgumentError
|
|
391
|
+
Parser.invalidate_handle(handle)
|
|
392
|
+
begin
|
|
393
|
+
_parse_handle_with_compatibility(
|
|
394
|
+
Parser.grammar_handle(grammar, compatibility: :parslet), input, "parslet"
|
|
395
|
+
)
|
|
396
|
+
rescue RuntimeError => e
|
|
397
|
+
raise_compatible_parse_error(e, input)
|
|
398
|
+
end
|
|
399
|
+
rescue RuntimeError => e
|
|
400
|
+
raise_compatible_parse_error(e, input)
|
|
401
|
+
end
|
|
402
|
+
BatchDecoder.decode_and_flatten(raw, input, Parsanol::Slice)
|
|
403
|
+
end
|
|
404
|
+
|
|
405
|
+
def raise_compatible_parse_error(error, input)
|
|
406
|
+
message = error.message
|
|
407
|
+
position = message[NATIVE_POS_MARKER, 1].to_i
|
|
408
|
+
cause = Parsanol::Cause.new(message.sub(NATIVE_POS_MARKER, ""),
|
|
409
|
+
Parsanol::Source.new(input), position)
|
|
410
|
+
raise Parsanol::ParseFailed.new(cause.to_s, cause), cause: nil
|
|
411
|
+
end
|
|
412
|
+
|
|
369
413
|
# Reference-walker path (TODO.sota/8 B1): the plain recursive
|
|
370
414
|
# relation — no VM, no memo tables, no fused scans, no terminal
|
|
371
415
|
# fast paths. Deliberately slow; the cross-engine differential
|
|
@@ -86,6 +86,12 @@ module Parsanol
|
|
|
86
86
|
|
|
87
87
|
def compiler_version = envelope["compiler_version"]
|
|
88
88
|
|
|
89
|
+
# The behavior fingerprint (ruby#377): stable across producer
|
|
90
|
+
# metadata, so freshness gates can tell "re-baked by a newer
|
|
91
|
+
# compiler" from "the grammar changed". nil on artifacts baked
|
|
92
|
+
# before the field existed.
|
|
93
|
+
def semantics_hash = envelope["semantics_hash"]
|
|
94
|
+
|
|
89
95
|
def fresh_against?(source_text)
|
|
90
96
|
return false if source_hash.nil?
|
|
91
97
|
|
|
@@ -20,7 +20,7 @@ module Parsanol
|
|
|
20
20
|
assign_scalar(out, artifact, binding, captures)
|
|
21
21
|
end
|
|
22
22
|
arrays.group_by { |binding| binding["path"].split("[]").first }.each do |prefix, group|
|
|
23
|
-
assign_array(out, artifact, prefix, group, captures)
|
|
23
|
+
assign_array(out, artifact, prefix, group, captures, shape)
|
|
24
24
|
end
|
|
25
25
|
out
|
|
26
26
|
end
|
|
@@ -63,7 +63,32 @@ module Parsanol
|
|
|
63
63
|
out[leaf_key(binding)] = finalize(artifact, binding, value)
|
|
64
64
|
end
|
|
65
65
|
|
|
66
|
-
def assign_array(out, artifact, prefix, group, captures)
|
|
66
|
+
def assign_array(out, artifact, prefix, group, captures, shape = nil)
|
|
67
|
+
# Position-aware pairing (parsanol-ruby#381): when the shape
|
|
68
|
+
# nests the group's instances under the prefix key, each
|
|
69
|
+
# instance's subtree owns its captures — pairing by OCCURRENCE
|
|
70
|
+
# index landed a sparse capture (one that fires in only some
|
|
71
|
+
# instances) on the wrong instance. Per-instance collection
|
|
72
|
+
# fixes the pairing and is identical to the old flat pairing
|
|
73
|
+
# when every instance fires every capture. Captures outside
|
|
74
|
+
# any instance keep the legacy occurrence pairing.
|
|
75
|
+
instances = shape && group_instances(shape, prefix)
|
|
76
|
+
if instances
|
|
77
|
+
list = (out[prefix.to_sym] ||= [])
|
|
78
|
+
instances.each do |instance|
|
|
79
|
+
found = collect_captures(instance)
|
|
80
|
+
element = {}
|
|
81
|
+
group.each do |binding|
|
|
82
|
+
value = found[binding["capture"].to_sym].first
|
|
83
|
+
next if value.nil?
|
|
84
|
+
|
|
85
|
+
element[leaf_key(binding)] = finalize(artifact, binding, value)
|
|
86
|
+
end
|
|
87
|
+
list << element
|
|
88
|
+
end
|
|
89
|
+
return
|
|
90
|
+
end
|
|
91
|
+
|
|
67
92
|
count = group.map { |binding| captures[binding["capture"].to_sym].length }.max || 0
|
|
68
93
|
return if count.zero?
|
|
69
94
|
|
|
@@ -82,6 +107,27 @@ module Parsanol
|
|
|
82
107
|
end
|
|
83
108
|
end
|
|
84
109
|
|
|
110
|
+
# The `prefix[]` group's instance subtrees: the first array under
|
|
111
|
+
# a key naming the prefix, walked in collect_captures' BFS order.
|
|
112
|
+
# A repetition-root entry's shape IS the instances array (the wire
|
|
113
|
+
# has no key for it) — that is the fallback when no key matches.
|
|
114
|
+
def group_instances(shape, prefix)
|
|
115
|
+
queue = [shape]
|
|
116
|
+
until queue.empty?
|
|
117
|
+
node = queue.shift
|
|
118
|
+
case node
|
|
119
|
+
when Hash
|
|
120
|
+
node.each do |key, value|
|
|
121
|
+
return value if key.to_s == prefix && value.is_a?(Array)
|
|
122
|
+
|
|
123
|
+
queue << value if value.is_a?(Hash) || value.is_a?(Array)
|
|
124
|
+
end
|
|
125
|
+
when Array then queue.concat(node)
|
|
126
|
+
end
|
|
127
|
+
end
|
|
128
|
+
shape if shape.is_a?(Array) && shape.any?(Hash)
|
|
129
|
+
end
|
|
130
|
+
|
|
85
131
|
def validate_path!(binding)
|
|
86
132
|
path = binding["path"].to_s
|
|
87
133
|
return unless path.include?(".") || path.include?("[")
|
|
@@ -32,6 +32,18 @@ module Parsanol
|
|
|
32
32
|
"sha256:#{Digest::SHA256.hexdigest(JSON.generate(canonicalize(payload)))}"
|
|
33
33
|
end
|
|
34
34
|
|
|
35
|
+
# Producer metadata: the checksum pins the exact producer, but the
|
|
36
|
+
# semantics hash (ruby#377) skips these fields so a re-bake by a
|
|
37
|
+
# newer compiler — or a different Ruby patch stamping
|
|
38
|
+
# unicode_tables — is byte-stable when the grammar's behavior is
|
|
39
|
+
# unchanged. Mirrors the cross-engine parity view.
|
|
40
|
+
SEMANTICS_EXCLUDED = %w[checksum compiler_version unicode_tables semantics_hash].freeze
|
|
41
|
+
|
|
42
|
+
def self.semantics_hash(envelope)
|
|
43
|
+
payload = envelope.except(*SEMANTICS_EXCLUDED)
|
|
44
|
+
"sha256:#{Digest::SHA256.hexdigest(JSON.generate(canonicalize(payload)))}"
|
|
45
|
+
end
|
|
46
|
+
|
|
35
47
|
def self.canonicalize(obj)
|
|
36
48
|
case obj
|
|
37
49
|
when Hash then obj.sort.to_h.transform_values { |value| canonicalize(value) }
|
|
@@ -137,14 +149,24 @@ module Parsanol
|
|
|
137
149
|
|
|
138
150
|
artifact = Artifact.from_json(JSON.generate(envelope), tables_dir: tables_dir)
|
|
139
151
|
default = envelope["default_entry"] || artifact.entries.first
|
|
152
|
+
# `Native.available?` is true under either engine tier, but the
|
|
153
|
+
# JSON-grammar fast path rides the MRI C-API extension's
|
|
154
|
+
# `_parse_raw` — an ffi-cdylib install is available without it.
|
|
155
|
+
# Probe rather than assume (same doctrine as
|
|
156
|
+
# Native.reference_available?): without the C-API symbol the
|
|
157
|
+
# self-test runs through the artifact's own parse lane, which
|
|
158
|
+
# picks native-or-pure-Ruby honestly instead of raising
|
|
159
|
+
# NoMethodError mid-run.
|
|
160
|
+
json_capable = Native.respond_to?(:_parse_raw)
|
|
140
161
|
grammars = {}
|
|
141
162
|
tests.filter_map do |test|
|
|
142
163
|
entry = test["entry"] || default
|
|
143
|
-
|
|
144
|
-
|
|
145
|
-
|
|
146
|
-
|
|
147
|
-
|
|
164
|
+
parse_with =
|
|
165
|
+
if json_capable
|
|
166
|
+
grammar_json = (grammars[entry] ||= JSON.generate(artifact.entry(entry)["grammar"]))
|
|
167
|
+
->(input) { Native.parse_json_grammar(grammar_json, input) }
|
|
168
|
+
end
|
|
169
|
+
artifact.run_test_list([test], parse_with: parse_with)
|
|
148
170
|
end.flatten
|
|
149
171
|
end
|
|
150
172
|
|
|
@@ -1352,6 +1374,7 @@ module Parsanol
|
|
|
1352
1374
|
envelope["customs"] = @document.customs unless @document.customs.empty?
|
|
1353
1375
|
envelope["states"] = @document.states unless @document.states.empty?
|
|
1354
1376
|
envelope["dynamic"] = true if @dynamic
|
|
1377
|
+
envelope["semantics_hash"] = Compiler.semantics_hash(envelope)
|
|
1355
1378
|
envelope["checksum"] = Compiler.checksum(envelope)
|
|
1356
1379
|
envelope
|
|
1357
1380
|
end
|
data/lib/parsanol/parg/parser.rb
CHANGED
|
@@ -571,11 +571,16 @@ module Parsanol
|
|
|
571
571
|
|
|
572
572
|
def parse_postfixed
|
|
573
573
|
sign, behind = parse_guard_sign
|
|
574
|
-
node =
|
|
574
|
+
node = parse_primary
|
|
575
575
|
if sign.nil?
|
|
576
|
-
node
|
|
576
|
+
capture_postfix(node)
|
|
577
577
|
else
|
|
578
|
-
|
|
578
|
+
# The capture binds the GUARD, not the guarded primary
|
|
579
|
+
# (parsanol-ruby#380): `!digit as year` is `( !digit ) as
|
|
580
|
+
# year` — a capture inside the lookahead would never bind
|
|
581
|
+
# (a negated guard consumes nothing), so `as` must wrap the
|
|
582
|
+
# pred, matching the rep path's postfix order (ruby#197).
|
|
583
|
+
capture_postfix(Node.new(:pred, sign, node, behind ? :behind : nil))
|
|
579
584
|
end
|
|
580
585
|
end
|
|
581
586
|
|
data/lib/parsanol/parser.rb
CHANGED
|
@@ -176,6 +176,9 @@ module Parsanol
|
|
|
176
176
|
# Options:
|
|
177
177
|
# - :reporter - Custom error reporter instance
|
|
178
178
|
# - :prefix - Allow partial matching (default: false)
|
|
179
|
+
# - :compatibility - :strict (default), or :parslet for legacy memo
|
|
180
|
+
# decisions. Auto mode selects native when available; an older native
|
|
181
|
+
# extension without the compatibility entrypoint raises unsupported.
|
|
179
182
|
#
|
|
180
183
|
# @return [Hash, Array, Parsanol::Slice] parsed result with position info
|
|
181
184
|
# @raise [Parsanol::ParseFailed] when parsing fails
|
|
@@ -200,8 +203,22 @@ module Parsanol
|
|
|
200
203
|
# parse(input, {mode: :ruby, ...}) is the same call shape a subclass
|
|
201
204
|
# forwarding options via super produces; :mode must be honored
|
|
202
205
|
# wherever it appears, or those callers silently get the native path.
|
|
203
|
-
|
|
204
|
-
|
|
206
|
+
compatibility = opts.fetch(:compatibility, :strict)
|
|
207
|
+
unless %i[strict parslet].include?(compatibility)
|
|
208
|
+
raise ArgumentError, "Unknown compatibility policy: #{compatibility}"
|
|
209
|
+
end
|
|
210
|
+
|
|
211
|
+
requested_mode = opts.delete(:mode)
|
|
212
|
+
mode = requested_mode ||
|
|
213
|
+
if compatibility == :parslet
|
|
214
|
+
Parsanol::Native.available? ? :native : :ruby
|
|
215
|
+
else
|
|
216
|
+
Parsanol::Native.available? && native_expressible? ? :native : :ruby
|
|
217
|
+
end
|
|
218
|
+
if compatibility == :parslet && mode == :json
|
|
219
|
+
raise NotImplementedError, "Parslet compatibility does not yet support JSON mode"
|
|
220
|
+
end
|
|
221
|
+
|
|
205
222
|
case mode
|
|
206
223
|
when :ruby
|
|
207
224
|
super(input, opts)
|
|
@@ -291,6 +308,14 @@ module Parsanol
|
|
|
291
308
|
# @return [Object] parse result with Slice objects for position info
|
|
292
309
|
#
|
|
293
310
|
def parse_native(input, opts)
|
|
311
|
+
if opts[:compatibility] == :parslet
|
|
312
|
+
if opts[:prefix]
|
|
313
|
+
raise NotImplementedError, "Native Parslet compatibility does not yet support prefix mode"
|
|
314
|
+
end
|
|
315
|
+
|
|
316
|
+
return Parsanol::Native.parse(root, input, compatibility: :parslet)
|
|
317
|
+
end
|
|
318
|
+
|
|
294
319
|
unless Parsanol::Native.available?
|
|
295
320
|
Parsanol::Atoms::Base.instance_method(:parse).bind_call(self, input,
|
|
296
321
|
opts)
|
data/lib/parsanol/version.rb
CHANGED