parsanol 1.3.108 → 1.3.112
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/Cargo.lock +4 -4
- data/ext/parsanol_cdylib/Cargo.toml +1 -1
- data/ext/parsanol_native/Cargo.toml +1 -1
- data/lib/parsanol/atoms/repetition.rb +22 -0
- data/lib/parsanol/parg/artifact.rb +37 -8
- data/lib/parsanol/parg/authoring.rb +196 -5
- data/lib/parsanol/parg/cli.rb +108 -0
- data/lib/parsanol/parg/compiler.rb +263 -5
- data/lib/parsanol/parg/diff.rb +264 -0
- data/lib/parsanol/parg/document.rb +15 -2
- data/lib/parsanol/parg/export.rb +191 -0
- data/lib/parsanol/parg/import.rb +3 -1
- data/lib/parsanol/parg/importers/abnf.rb +148 -28
- data/lib/parsanol/parg/importers/antlr.rb +407 -0
- data/lib/parsanol/parg/lexer.rb +1 -0
- data/lib/parsanol/parg/lints.rb +448 -1
- data/lib/parsanol/parg/parser.rb +76 -2
- data/lib/parsanol/parg.rb +2 -0
- data/lib/parsanol/slice.rb +13 -4
- data/lib/parsanol/version.rb +1 -1
- data/lib/parsanol/vm.rb +50 -24
- metadata +4 -1
|
@@ -66,6 +66,19 @@ module Parsanol
|
|
|
66
66
|
|
|
67
67
|
# QoL shorthands (ABNF core rules): materialized on reference when
|
|
68
68
|
# the grammar does not define its own rule of that name.
|
|
69
|
+
# Character-class core rules (TODO.sota/9 T1, ruby#303): the RFC
|
|
70
|
+
# 5234 prelude that is expressible as a class. A grammar's own
|
|
71
|
+
# rule of the same name shadows these — fetch_rule checks the
|
|
72
|
+
# document first. ALNUM/WORD/ANY predate the RFC set.
|
|
73
|
+
# Unicode general categories (U3): one-letter groups plus the
|
|
74
|
+
# two-letter codes grammars actually reach for. Both regex
|
|
75
|
+
# engines carry the tables; this gate rejects misspellings at
|
|
76
|
+
# compile time instead of silently matching nothing.
|
|
77
|
+
GENERAL_CATEGORIES = %w[
|
|
78
|
+
L Lu Ll Lt Lm Lo M Mn Mc Me N Nd Nl No P Pc Pd Ps Pe Pi Pf Po
|
|
79
|
+
S Sm Sc Sk So Z Zs Zl Zp C Cc Cf Co Cn
|
|
80
|
+
].freeze
|
|
81
|
+
|
|
69
82
|
BUILTIN_CLASSES = {
|
|
70
83
|
"DIGIT" => [[0x30, 0x39]],
|
|
71
84
|
"ALPHA" => [[0x41, 0x5A], [0x61, 0x7A]],
|
|
@@ -73,15 +86,30 @@ module Parsanol
|
|
|
73
86
|
"WORD" => [[0x30, 0x39], [0x41, 0x5A], [0x5F, 0x5F], [0x61, 0x7A]],
|
|
74
87
|
"HEXDIG" => [[0x30, 0x39], [0x41, 0x46], [0x61, 0x66]],
|
|
75
88
|
"ANY" => [[0x00, 0x10FFFF]],
|
|
89
|
+
"BIT" => [[0x30, 0x31]],
|
|
90
|
+
"CHAR" => [[0x01, 0x7F]],
|
|
91
|
+
"CR" => [[0x0D, 0x0D]],
|
|
92
|
+
"CTL" => [[0x00, 0x1F], [0x7F, 0x7F]],
|
|
93
|
+
"DQUOTE" => [[0x22, 0x22]],
|
|
94
|
+
"HTAB" => [[0x09, 0x09]],
|
|
95
|
+
"LF" => [[0x0A, 0x0A]],
|
|
96
|
+
"OCTET" => [[0x00, 0xFF]],
|
|
97
|
+
"SP" => [[0x20, 0x20]],
|
|
98
|
+
"VCHAR" => [[0x21, 0x7E]],
|
|
99
|
+
"WSP" => [[0x20, 0x20], [0x09, 0x09]],
|
|
76
100
|
}.freeze
|
|
77
101
|
|
|
78
102
|
def compile
|
|
103
|
+
inject_core_rules!
|
|
79
104
|
expand_left_recursion!
|
|
80
105
|
@document.rules.each_key { |name| atom_for(name) }
|
|
81
106
|
errors = Lints.errors(@document, self)
|
|
82
107
|
raise CompileError, errors.join("\n") unless errors.empty?
|
|
83
108
|
|
|
84
|
-
|
|
109
|
+
# The author sees every finding; the envelope carries only the
|
|
110
|
+
# semantic subset (see Lints::Advisory).
|
|
111
|
+
@warnings = Lints.warnings(@document, self).map(&:to_s)
|
|
112
|
+
@envelope_warnings = Lints.envelope_warnings(@document, self)
|
|
85
113
|
failures = run_tests
|
|
86
114
|
raise CompileError, failures.join("\n") unless failures.empty?
|
|
87
115
|
|
|
@@ -120,6 +148,65 @@ module Parsanol
|
|
|
120
148
|
end.flatten
|
|
121
149
|
end
|
|
122
150
|
|
|
151
|
+
# The structured core rules (TODO.sota/9 T1, ruby#303): CRLF and
|
|
152
|
+
# LWSP are not character classes, so BUILTIN_CLASSES cannot serve
|
|
153
|
+
# them — they inject as real rules when referenced and missing,
|
|
154
|
+
# transitively (LWSP needs CRLF). Class-expressible members of
|
|
155
|
+
# the RFC 5234 set ride BUILTIN_CLASSES above; a grammar's own
|
|
156
|
+
# rule of the same name shadows either mechanism. Injected at the
|
|
157
|
+
# front so definitions precede references for the order lint.
|
|
158
|
+
STRUCTURED_CORE_RULES_SOURCE = <<~PARG
|
|
159
|
+
grammar CoreRules version "1.0.0" {
|
|
160
|
+
CRLF = %x0D.0A
|
|
161
|
+
LWSP = *( WSP / CRLF WSP )
|
|
162
|
+
}
|
|
163
|
+
PARG
|
|
164
|
+
|
|
165
|
+
STRUCTURED_CORE_RULE_NAMES = %w[CRLF LWSP].freeze
|
|
166
|
+
|
|
167
|
+
def inject_core_rules!
|
|
168
|
+
return if @core_rules_injected
|
|
169
|
+
|
|
170
|
+
@core_rules_injected = true
|
|
171
|
+
missing = STRUCTURED_CORE_RULE_NAMES.select do |name|
|
|
172
|
+
!@document.rules.key?(name) &&
|
|
173
|
+
@document.rules.each_value.any? { |node| references_atom?(node, name) }
|
|
174
|
+
end
|
|
175
|
+
return if missing.empty?
|
|
176
|
+
|
|
177
|
+
# Snapshot before the prelude merges: lints distinguish an
|
|
178
|
+
# authored shadow from an injected prelude rule.
|
|
179
|
+
@document.authored_rules ||= @document.rules.keys
|
|
180
|
+
prelude = Parser.new(STRUCTURED_CORE_RULES_SOURCE).parse
|
|
181
|
+
injected = {}
|
|
182
|
+
until missing.empty?
|
|
183
|
+
name = missing.pop
|
|
184
|
+
next if injected.key?(name) || @document.rules.key?(name)
|
|
185
|
+
|
|
186
|
+
node = prelude.rules.fetch(name)
|
|
187
|
+
injected[name] = node
|
|
188
|
+
STRUCTURED_CORE_RULE_NAMES.each do |dep|
|
|
189
|
+
missing << dep if !injected.key?(dep) && !@document.rules.key?(dep) &&
|
|
190
|
+
references_atom?(node, dep)
|
|
191
|
+
end
|
|
192
|
+
end
|
|
193
|
+
# rules is deliberately read-only on the document: prepend in place.
|
|
194
|
+
@document.rules.replace(injected.merge(@document.rules))
|
|
195
|
+
end
|
|
196
|
+
|
|
197
|
+
def references_atom?(node, name)
|
|
198
|
+
walk = lambda do |n|
|
|
199
|
+
case n.kind
|
|
200
|
+
when :ref then return true if n.a == name
|
|
201
|
+
when :alt, :seq then n.a.each { |b| return true if references_atom?(b, name) }
|
|
202
|
+
when :cap, :pred then return true if references_atom?(n.b, name)
|
|
203
|
+
when :rep, :opt then return true if references_atom?(n.a, name)
|
|
204
|
+
end
|
|
205
|
+
false
|
|
206
|
+
end
|
|
207
|
+
walk.call(node) == true
|
|
208
|
+
end
|
|
209
|
+
|
|
123
210
|
def resolve_custom(name)
|
|
124
211
|
class_name = @document.customs.fetch(name)
|
|
125
212
|
klass = Object.const_get(class_name)
|
|
@@ -141,7 +228,146 @@ module Parsanol
|
|
|
141
228
|
@resync_cache[rule.to_sym] ||= build_atom(pattern)
|
|
142
229
|
end
|
|
143
230
|
|
|
231
|
+
# Modern mode (ruby#304, the #315 spec): every rule reference
|
|
232
|
+
# that produces no capture is wrapped in Named(rule_name) at
|
|
233
|
+
# compile time — the tree shape consumers read, opt-in via
|
|
234
|
+
# `mode` in the grammar header. Engines see ordinary
|
|
235
|
+
# Named atoms; identity rides the differential corpus.
|
|
236
|
+
def apply_modern_captures(atom)
|
|
237
|
+
return atom if @document.edition == "v1"
|
|
238
|
+
|
|
239
|
+
wrap_uncaptured(atom)
|
|
240
|
+
end
|
|
241
|
+
|
|
242
|
+
def wrap_uncaptured(atom)
|
|
243
|
+
case atom
|
|
244
|
+
when Atoms::Named
|
|
245
|
+
atom
|
|
246
|
+
when Atoms::Entity
|
|
247
|
+
# A rule that can only match whitespace is glue, not a
|
|
248
|
+
# semantic capture (ruby#352): wrapping it named every gap
|
|
249
|
+
# in the tree and broke consumers that fold captures into
|
|
250
|
+
# parameters. Conservatively: the rule's body is literals
|
|
251
|
+
# (through sequence/alternation/repetition composition) and
|
|
252
|
+
# every literal strips to empty.
|
|
253
|
+
# rubocop:disable Lint/DuplicateBranch -- trivia-glue and
|
|
254
|
+
# capture-producing refs both stay unwrapped; the bodies
|
|
255
|
+
# agree by design.
|
|
256
|
+
if trivia_only?(atom.rule_name.to_s, { atom.rule_name.to_s => true }) ||
|
|
257
|
+
produces_capture?(atom.parslet)
|
|
258
|
+
atom
|
|
259
|
+
else
|
|
260
|
+
Atoms::Named.new(atom, atom.rule_name.to_s)
|
|
261
|
+
end
|
|
262
|
+
# rubocop:enable Lint/DuplicateBranch
|
|
263
|
+
when Atoms::Sequence, Atoms::Alternative
|
|
264
|
+
# rubocop:disable Lint/DuplicateBranch -- the two composites
|
|
265
|
+
# rebuild with their own constructors; identical bodies,
|
|
266
|
+
# different decisions
|
|
267
|
+
kids = atom.respond_to?(:alternatives) ? atom.alternatives : atom.parslets
|
|
268
|
+
atom.class.new(*kids.map { |child| wrap_uncaptured(child) })
|
|
269
|
+
when Atoms::Repetition
|
|
270
|
+
Atoms::Repetition.new(wrap_uncaptured(atom.parslet), atom.min, atom.max, atom.result_tag)
|
|
271
|
+
else
|
|
272
|
+
atom
|
|
273
|
+
end
|
|
274
|
+
end
|
|
275
|
+
|
|
276
|
+
# Whether the named rule's body is provably whitespace-only:
|
|
277
|
+
# literals strip to empty; sequence/alternation/repetition
|
|
278
|
+
# compose the verdict; references resolve through the atom cache
|
|
279
|
+
# with a cycle set; anything else (classes, customs, authored
|
|
280
|
+
# captures) disqualifies.
|
|
281
|
+
def trivia_only?(name, seen)
|
|
282
|
+
return false unless @document.rules.key?(name)
|
|
283
|
+
|
|
284
|
+
trivia_walk(atom_for(name), seen)
|
|
285
|
+
end
|
|
286
|
+
|
|
287
|
+
def trivia_walk(atom, seen)
|
|
288
|
+
case atom
|
|
289
|
+
when Atoms::Str
|
|
290
|
+
atom.str.to_s.strip.empty?
|
|
291
|
+
when Atoms::Sequence, Atoms::Alternative
|
|
292
|
+
kids = atom.respond_to?(:alternatives) ? atom.alternatives : atom.parslets
|
|
293
|
+
kids.all? { |child| trivia_walk(child, seen) }
|
|
294
|
+
when Atoms::Repetition
|
|
295
|
+
trivia_walk(atom.parslet, seen)
|
|
296
|
+
when Atoms::Entity
|
|
297
|
+
name = atom.rule_name.to_s
|
|
298
|
+
return false if seen[name]
|
|
299
|
+
|
|
300
|
+
seen[name] = true
|
|
301
|
+
trivia_only?(name, seen)
|
|
302
|
+
else
|
|
303
|
+
false
|
|
304
|
+
end
|
|
305
|
+
end
|
|
306
|
+
|
|
307
|
+
# Whether the atom's flattening surfaces any Named capture:
|
|
308
|
+
# rule references resolve through (their own wrap pass already
|
|
309
|
+
# ran — a wrapped ref produces a capture), trivia never
|
|
310
|
+
# surfaces, everything else recurses structurally.
|
|
311
|
+
def produces_capture?(atom, seen = [])
|
|
312
|
+
return false if seen.include?(atom.object_id)
|
|
313
|
+
|
|
314
|
+
seen << atom.object_id
|
|
315
|
+
case atom
|
|
316
|
+
when Atoms::Named then true
|
|
317
|
+
when Atoms::Entity
|
|
318
|
+
wrapped = atom.parslet.is_a?(Atoms::Named)
|
|
319
|
+
wrapped || produces_capture?(atom.parslet, seen)
|
|
320
|
+
when Atoms::Sequence, Atoms::Alternative
|
|
321
|
+
kids = atom.respond_to?(:alternatives) ? atom.alternatives : atom.parslets
|
|
322
|
+
kids.any? { |child| produces_capture?(child, seen) }
|
|
323
|
+
when Atoms::Repetition
|
|
324
|
+
produces_capture?(atom.parslet, seen)
|
|
325
|
+
else
|
|
326
|
+
false
|
|
327
|
+
end
|
|
328
|
+
end
|
|
329
|
+
|
|
330
|
+
# The migration radar (ruby#304 `parg lint --captures`): for a
|
|
331
|
+
# LEGACY grammar, every rule reference modern mode would wrap —
|
|
332
|
+
# reference site, target rule. The compiler's state is warm
|
|
333
|
+
# (all rules built) when the lint calls this.
|
|
334
|
+
def modern_radar
|
|
335
|
+
# The radar is the classic dialect's migration tool: on a v2
|
|
336
|
+
# grammar every reference is already named.
|
|
337
|
+
return [] if @document.edition != "v1"
|
|
338
|
+
|
|
339
|
+
lines = []
|
|
340
|
+
@document.rules.each_key do |name|
|
|
341
|
+
atom = @atom_cache[name]
|
|
342
|
+
next unless atom
|
|
343
|
+
|
|
344
|
+
radar_walk(atom, name, lines, [])
|
|
345
|
+
end
|
|
346
|
+
lines
|
|
347
|
+
end
|
|
348
|
+
|
|
349
|
+
def radar_walk(atom, site, lines, seen)
|
|
350
|
+
return if seen.include?(atom.object_id)
|
|
351
|
+
|
|
352
|
+
seen << atom.object_id
|
|
353
|
+
case atom
|
|
354
|
+
when Atoms::Named then nil
|
|
355
|
+
when Atoms::Entity
|
|
356
|
+
unless produces_capture?(atom.parslet)
|
|
357
|
+
lines << "rule #{site}: reference to #{atom.rule_name} produces no capture -- " \
|
|
358
|
+
"modern mode would name it #{atom.rule_name}"
|
|
359
|
+
end
|
|
360
|
+
radar_walk(atom.parslet, site, lines, seen)
|
|
361
|
+
when Atoms::Sequence, Atoms::Alternative
|
|
362
|
+
kids = atom.respond_to?(:alternatives) ? atom.alternatives : atom.parslets
|
|
363
|
+
kids.each { |child| radar_walk(child, site, lines, seen) }
|
|
364
|
+
when Atoms::Repetition
|
|
365
|
+
radar_walk(atom.parslet, site, lines, seen)
|
|
366
|
+
end
|
|
367
|
+
end
|
|
368
|
+
|
|
144
369
|
def atom_for(name)
|
|
370
|
+
inject_core_rules!
|
|
145
371
|
expand_left_recursion! unless @expanded
|
|
146
372
|
@atom_cache[name] ||= begin
|
|
147
373
|
atom = build_atom(fetch_rule(name))
|
|
@@ -159,6 +385,15 @@ module Parsanol
|
|
|
159
385
|
atom = Atoms::Sequence.new(skip_maybe_atom, atom, skip_maybe_atom)
|
|
160
386
|
end
|
|
161
387
|
end
|
|
388
|
+
# The modern wrap runs LAST — the guide's spec table: trivia
|
|
389
|
+
# atoms are in place first, so the pass sees the final tree.
|
|
390
|
+
# The unwrapped tree lands in the cache BEFORE the wrap (the
|
|
391
|
+
# wrap's capture probes resolve rule references through
|
|
392
|
+
# atom_for; a mutual-reference grammar must hit the cache,
|
|
393
|
+
# not rebuild), then the cache takes the wrapped tree.
|
|
394
|
+
@atom_cache[name] = atom
|
|
395
|
+
atom = apply_modern_captures(atom)
|
|
396
|
+
@atom_cache[name] = atom
|
|
162
397
|
atom
|
|
163
398
|
end
|
|
164
399
|
end
|
|
@@ -600,6 +835,14 @@ module Parsanol
|
|
|
600
835
|
Atoms::Str.new(node.a)
|
|
601
836
|
end
|
|
602
837
|
when :class then Atoms::Re.new(class_pattern(node.a))
|
|
838
|
+
when :category
|
|
839
|
+
code = node.a
|
|
840
|
+
unless GENERAL_CATEGORIES.include?(code)
|
|
841
|
+
raise CompileError,
|
|
842
|
+
"unknown general category %p{#{code}} -- use one of " \
|
|
843
|
+
"#{GENERAL_CATEGORIES.join(' ')}"
|
|
844
|
+
end
|
|
845
|
+
Atoms::Re.new("[\\p{#{code}}]")
|
|
603
846
|
when :until
|
|
604
847
|
# QoL: (*( !terminal any_char )) — consume everything up to
|
|
605
848
|
# (and excluding) the delimiter.
|
|
@@ -858,9 +1101,10 @@ module Parsanol
|
|
|
858
1101
|
def compute_first(node)
|
|
859
1102
|
case node.kind
|
|
860
1103
|
when :lit
|
|
861
|
-
# The zero-width marker ("" as name) consumes nothing:
|
|
862
|
-
# first set
|
|
863
|
-
|
|
1104
|
+
# The zero-width marker ("" as name) consumes nothing: EPS in
|
|
1105
|
+
# its first set marks the node nullable (the convention every
|
|
1106
|
+
# nullable? check uses).
|
|
1107
|
+
return Set.new([EPS]) if node.a.empty?
|
|
864
1108
|
|
|
865
1109
|
set = Set.new(node.a[0].bytes)
|
|
866
1110
|
set += node.a[0].upcase.bytes + node.a[0].downcase.bytes if node.b
|
|
@@ -954,7 +1198,7 @@ module Parsanol
|
|
|
954
1198
|
"entries" => entries,
|
|
955
1199
|
"preprocess" => @document.preprocess,
|
|
956
1200
|
"tables" => table_manifest,
|
|
957
|
-
"lint" => { "order_warnings" => @
|
|
1201
|
+
"lint" => { "order_warnings" => @envelope_warnings.uniq },
|
|
958
1202
|
"default_entry" => begin
|
|
959
1203
|
resolve_default_entry
|
|
960
1204
|
rescue StandardError
|
|
@@ -980,8 +1224,22 @@ module Parsanol
|
|
|
980
1224
|
# back to a reference implementation instead of failing obscurely
|
|
981
1225
|
# at parse time. Checksummed like the rest of the envelope.
|
|
982
1226
|
"source_hash" => Digest::SHA256.hexdigest(@document.source.to_s),
|
|
1227
|
+
# The U3 provenance stamp (rs#297): which Unicode table source
|
|
1228
|
+
# the compiling engine's categories compiled against. An
|
|
1229
|
+
# engine-specific stamp (checksummed here, excluded from the
|
|
1230
|
+
# cross-engine parity set like compiler_version); the
|
|
1231
|
+
# observable behavior stays pinned by the unicode corpus
|
|
1232
|
+
# fixtures.
|
|
1233
|
+
"unicode_tables" => "onigmo (Ruby #{RUBY_VERSION})",
|
|
983
1234
|
"compiler_version" => Parsanol::VERSION,
|
|
984
1235
|
}
|
|
1236
|
+
# ruby#304: the envelope always names the capture dialect —
|
|
1237
|
+
# consumers read the tree-shape contract before parsing.
|
|
1238
|
+
envelope["edition"] = @document.edition
|
|
1239
|
+
# ruby#353: the language-semantics level the trees were frozen
|
|
1240
|
+
# against (1.3.108's implicit-capture shapes = 1; the
|
|
1241
|
+
# explicit end_guard/marker/hyphen nodes = 2). Consumers freeze
|
|
1242
|
+
# tree shapes and mappers; the loader asserts the level at
|
|
985
1243
|
if @document.skip_declaration
|
|
986
1244
|
# parsanol-ruby#134: declared trivia rule. Injection is baked
|
|
987
1245
|
# into the compiled trees (plain atom kinds), so consumers
|
|
@@ -0,0 +1,264 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module Parsanol
|
|
4
|
+
module PARG
|
|
5
|
+
# ruby#306: `parg diff OLD NEW` — semantic classification of grammar
|
|
6
|
+
# changes against the artifact contract. Atoms are compared by
|
|
7
|
+
# canonical rendering, never by JSON bytes, so compiler-version churn
|
|
8
|
+
# and key order never read as a grammar change.
|
|
9
|
+
#
|
|
10
|
+
# Levels: :major breaks consumers (capture/entry/branch removals,
|
|
11
|
+
# binding changes, mode change), :minor extends them (new entries,
|
|
12
|
+
# captures, branches), :patch is behavior-preserving detail (literal
|
|
13
|
+
# fixes, doc changes, added tests). Ambiguous cases land at the
|
|
14
|
+
# higher level with the evidence attached — never guessed down.
|
|
15
|
+
module Diff
|
|
16
|
+
Finding = Struct.new(:level, :kind, :detail)
|
|
17
|
+
|
|
18
|
+
SEVERITY = { none: 0, patch: 1, minor: 2, major: 3 }.freeze
|
|
19
|
+
|
|
20
|
+
module_function
|
|
21
|
+
|
|
22
|
+
def diff(old_envelope, new_envelope)
|
|
23
|
+
findings = []
|
|
24
|
+
findings.concat(diff_mode(old_envelope, new_envelope))
|
|
25
|
+
findings.concat(diff_entries(old_envelope, new_envelope))
|
|
26
|
+
findings.concat(diff_bindings(old_envelope, new_envelope))
|
|
27
|
+
findings.concat(diff_tests(old_envelope, new_envelope))
|
|
28
|
+
findings
|
|
29
|
+
end
|
|
30
|
+
|
|
31
|
+
def level(findings)
|
|
32
|
+
findings.reduce(:none) { |acc, finding| higher(acc, finding.level) }
|
|
33
|
+
end
|
|
34
|
+
|
|
35
|
+
def higher(left, right)
|
|
36
|
+
SEVERITY[left] >= SEVERITY[right] ? left : right
|
|
37
|
+
end
|
|
38
|
+
|
|
39
|
+
def suggest_bump(findings)
|
|
40
|
+
level = level(findings)
|
|
41
|
+
level == :none ? "no bump needed" : "bump #{level}"
|
|
42
|
+
end
|
|
43
|
+
|
|
44
|
+
# -- entries ------------------------------------------------------
|
|
45
|
+
|
|
46
|
+
def diff_entries(old_env, new_env)
|
|
47
|
+
old_entries = old_env["entries"] || {}
|
|
48
|
+
new_entries = new_env["entries"] || {}
|
|
49
|
+
findings = (old_entries.keys - new_entries.keys).map do |name|
|
|
50
|
+
Finding.new(:major, :entry_removed, "entry #{name.inspect} removed")
|
|
51
|
+
end
|
|
52
|
+
(new_entries.keys - old_entries.keys).each do |name|
|
|
53
|
+
findings << Finding.new(:minor, :entry_added, "entry #{name.inspect} added")
|
|
54
|
+
end
|
|
55
|
+
old_entries.each do |name, old_entry|
|
|
56
|
+
new_entry = new_entries[name]
|
|
57
|
+
next unless new_entry
|
|
58
|
+
|
|
59
|
+
findings.concat(diff_entry(name, old_entry, new_entry))
|
|
60
|
+
end
|
|
61
|
+
findings
|
|
62
|
+
end
|
|
63
|
+
|
|
64
|
+
def diff_entry(name, old_entry, new_entry)
|
|
65
|
+
old_atoms = old_entry.dig("grammar", "atoms") || []
|
|
66
|
+
new_atoms = new_entry.dig("grammar", "atoms") || []
|
|
67
|
+
old_root = old_entry.dig("grammar", "root")
|
|
68
|
+
new_root = new_entry.dig("grammar", "root")
|
|
69
|
+
return [Finding.new(:major, :entry_changed, "entry #{name.inspect} lost its root")] if old_root.nil? || new_root.nil?
|
|
70
|
+
|
|
71
|
+
old_render = render_atom(old_atoms, old_root)
|
|
72
|
+
new_render = render_atom(new_atoms, new_root)
|
|
73
|
+
return [] if old_render == new_render
|
|
74
|
+
|
|
75
|
+
findings = []
|
|
76
|
+
findings.concat(diff_captures(name, old_render, new_render))
|
|
77
|
+
findings.concat(diff_branches(name, mask(old_render), mask(new_render)))
|
|
78
|
+
if findings.empty?
|
|
79
|
+
findings << if mask(old_render) == mask(new_render)
|
|
80
|
+
# Same captures, same branches, same structure: the
|
|
81
|
+
# difference is inside literals, bounds, or trivia markers —
|
|
82
|
+
# behavior-preserving.
|
|
83
|
+
Finding.new(:patch, :literal_or_bound_change,
|
|
84
|
+
"entry #{name.inspect}: literal/bound/doc-level change")
|
|
85
|
+
else
|
|
86
|
+
# Structure reshuffled without a detectable capture or
|
|
87
|
+
# branch delta — evidence over a guess.
|
|
88
|
+
Finding.new(:minor, :structure_changed,
|
|
89
|
+
"entry #{name.inspect}: structure changed " \
|
|
90
|
+
"(no capture or branch removed): " \
|
|
91
|
+
"#{mask(old_render)[0, 60]} -> #{mask(new_render)[0, 60]}")
|
|
92
|
+
end
|
|
93
|
+
end
|
|
94
|
+
findings
|
|
95
|
+
end
|
|
96
|
+
|
|
97
|
+
# -- captures -----------------------------------------------------
|
|
98
|
+
|
|
99
|
+
def diff_captures(entry_name, old_render, new_render)
|
|
100
|
+
old_names = capture_names(old_render)
|
|
101
|
+
new_names = capture_names(new_render)
|
|
102
|
+
removed = old_names - new_names
|
|
103
|
+
added = new_names - old_names
|
|
104
|
+
removed.map { |n| Finding.new(:major, :capture_removed, "entry #{entry_name.inspect}: capture #{n.inspect} removed") } +
|
|
105
|
+
added.map { |n| Finding.new(:minor, :capture_added, "entry #{entry_name.inspect}: capture #{n.inspect} added") }
|
|
106
|
+
end
|
|
107
|
+
|
|
108
|
+
def capture_names(render)
|
|
109
|
+
render.scan(/\(name "([^"]+)"/).flatten.uniq
|
|
110
|
+
end
|
|
111
|
+
|
|
112
|
+
# -- alternative branches ----------------------------------------
|
|
113
|
+
|
|
114
|
+
# A branch that stops matching is the ambiguous-breaking case:
|
|
115
|
+
# inputs it used to accept now reject. Removed branches are major;
|
|
116
|
+
# added-only is the documented minor.
|
|
117
|
+
def diff_branches(entry_name, old_render, new_render)
|
|
118
|
+
old_branches = alt_branches(old_render)
|
|
119
|
+
new_branches = alt_branches(new_render)
|
|
120
|
+
removed = old_branches - new_branches
|
|
121
|
+
added = new_branches - old_branches
|
|
122
|
+
removed.map { |b| Finding.new(:major, :branch_removed, "entry #{entry_name.inspect}: alternative branch #{b} no longer matches") } +
|
|
123
|
+
added.map { |b| Finding.new(:minor, :branch_added, "entry #{entry_name.inspect}: alternative branch #{b} added") }
|
|
124
|
+
end
|
|
125
|
+
|
|
126
|
+
# Literals masked: branch identity is structural, not spelling —
|
|
127
|
+
# a literal fix inside a branch is not that branch's removal.
|
|
128
|
+
def mask(render)
|
|
129
|
+
render.gsub(/str:"[^"]*"/, "str:#").gsub(/const:[^\s)]+/, "const:#")
|
|
130
|
+
end
|
|
131
|
+
|
|
132
|
+
# Every alternative branch's canonical rendering, anywhere in the
|
|
133
|
+
# tree (alternatives inside repetitions, groups, captures…).
|
|
134
|
+
# Balanced-paren walk: branches nest.
|
|
135
|
+
def alt_branches(render)
|
|
136
|
+
branches = []
|
|
137
|
+
index = render.index("(alt ")
|
|
138
|
+
while index
|
|
139
|
+
start = index + 5
|
|
140
|
+
depth = 1
|
|
141
|
+
cursor = start
|
|
142
|
+
while depth.positive? && cursor < render.length
|
|
143
|
+
case render[cursor]
|
|
144
|
+
when "(" then depth += 1
|
|
145
|
+
when ")" then depth -= 1
|
|
146
|
+
end
|
|
147
|
+
cursor += 1
|
|
148
|
+
end
|
|
149
|
+
body = render[start...(cursor - 1)]
|
|
150
|
+
child_depth = 0
|
|
151
|
+
current = +""
|
|
152
|
+
body.each_char do |ch|
|
|
153
|
+
case ch
|
|
154
|
+
when "(" then child_depth += 1
|
|
155
|
+
when ")" then child_depth -= 1
|
|
156
|
+
when " "
|
|
157
|
+
if child_depth.zero?
|
|
158
|
+
branches << current unless current.empty?
|
|
159
|
+
current = +""
|
|
160
|
+
next
|
|
161
|
+
end
|
|
162
|
+
end
|
|
163
|
+
current << ch
|
|
164
|
+
end
|
|
165
|
+
branches << current unless current.empty?
|
|
166
|
+
index = render.index("(alt ", cursor)
|
|
167
|
+
end
|
|
168
|
+
branches.uniq.sort
|
|
169
|
+
end
|
|
170
|
+
|
|
171
|
+
# -- mode ---------------------------------------------------------
|
|
172
|
+
|
|
173
|
+
def diff_mode(old_env, new_env)
|
|
174
|
+
old_mode = old_env["mode"]
|
|
175
|
+
new_mode = new_env["mode"]
|
|
176
|
+
return [] if old_mode == new_mode
|
|
177
|
+
|
|
178
|
+
[Finding.new(:major, :mode_changed,
|
|
179
|
+
"mode changed #{old_mode.inspect} -> #{new_mode.inspect} " \
|
|
180
|
+
"(every uncaptured node's tree shape changes)")]
|
|
181
|
+
end
|
|
182
|
+
|
|
183
|
+
# -- bindings -----------------------------------------------------
|
|
184
|
+
|
|
185
|
+
def diff_bindings(old_env, new_env)
|
|
186
|
+
old_bindings = old_env["bindings"]
|
|
187
|
+
new_bindings = new_env["bindings"]
|
|
188
|
+
return [] if old_bindings == new_bindings
|
|
189
|
+
return [Finding.new(:minor, :bindings_added, "bindings section added")] if old_bindings.nil?
|
|
190
|
+
return [Finding.new(:major, :bindings_removed, "bindings section removed")] if new_bindings.nil?
|
|
191
|
+
|
|
192
|
+
[Finding.new(:major, :bindings_changed,
|
|
193
|
+
"bindings changed — binding paths/types re-validate " \
|
|
194
|
+
"against the tree")]
|
|
195
|
+
end
|
|
196
|
+
|
|
197
|
+
# -- tests --------------------------------------------------------
|
|
198
|
+
|
|
199
|
+
def diff_tests(old_env, new_env)
|
|
200
|
+
old_tests = old_env["tests"].to_a
|
|
201
|
+
new_tests = new_env["tests"].to_a
|
|
202
|
+
return [] if old_tests == new_tests
|
|
203
|
+
|
|
204
|
+
# Pins per kind: a pin whose INPUT changed alongside a literal
|
|
205
|
+
# fix is the pin following the grammar (patch), not a removal.
|
|
206
|
+
old_counts = old_tests.each_with_object(Hash.new(0)) { |t, counts| counts[t["kind"]] += 1 }
|
|
207
|
+
new_counts = new_tests.each_with_object(Hash.new(0)) { |t, counts| counts[t["kind"]] += 1 }
|
|
208
|
+
dropped = old_counts.keys.select { |kind| new_counts[kind] < old_counts[kind] }
|
|
209
|
+
unless dropped.empty?
|
|
210
|
+
return [Finding.new(:major, :tests_removed,
|
|
211
|
+
"#{dropped.join(', ')} pin(s) removed — the contract weakened")]
|
|
212
|
+
end
|
|
213
|
+
|
|
214
|
+
[Finding.new(:patch, :tests_added,
|
|
215
|
+
"test pins changed/added (#{old_tests.size} -> #{new_tests.size})")]
|
|
216
|
+
end
|
|
217
|
+
|
|
218
|
+
# -- canonical rendering ------------------------------------------
|
|
219
|
+
|
|
220
|
+
# Canonical s-expression of one atom index. Entities resolve; keyed
|
|
221
|
+
# hashes render in fixed field order; nothing escapes — this is a
|
|
222
|
+
# diff key, not output text.
|
|
223
|
+
def render_atom(atoms, idx, depth = 0)
|
|
224
|
+
return "…" if depth > 64
|
|
225
|
+
|
|
226
|
+
a = atoms[idx]
|
|
227
|
+
return "missing:#{idx}" if a.nil?
|
|
228
|
+
|
|
229
|
+
if (str = a["Str"])
|
|
230
|
+
"str:#{str['pattern'].inspect}"
|
|
231
|
+
elsif (re = a["Re"])
|
|
232
|
+
"re:#{re['pattern']}"
|
|
233
|
+
elsif (sequence = a["Sequence"])
|
|
234
|
+
"(seq #{sequence['atoms'].map { |i| render_atom(atoms, i, depth + 1) }.join(' ')})"
|
|
235
|
+
elsif (alt = a["Alternative"])
|
|
236
|
+
"(alt #{alt['atoms'].map { |i| render_atom(atoms, i, depth + 1) }.join(' ')})"
|
|
237
|
+
elsif (rep = a["Repetition"])
|
|
238
|
+
"(rep #{rep['min']} #{rep['max'].nil? ? '*' : rep['max']} #{rep['tag']} #{render_atom(atoms, rep['atom'], depth + 1)})"
|
|
239
|
+
elsif (named = a["Named"])
|
|
240
|
+
"(name \"#{named['name']}\" #{render_atom(atoms, named['atom'], depth + 1)})"
|
|
241
|
+
elsif (entity = a["Entity"])
|
|
242
|
+
render_atom(atoms, entity["atom"], depth + 1)
|
|
243
|
+
elsif (look = a["Lookahead"])
|
|
244
|
+
"(look #{look['positive'] ? '&' : '!'} #{render_atom(atoms, look['atom'], depth + 1)})"
|
|
245
|
+
elsif (behind = a["Lookbehind"])
|
|
246
|
+
"(behind #{behind['positive'] ? '&' : '!'} #{behind['look'].inspect})"
|
|
247
|
+
elsif a["Cut"]
|
|
248
|
+
"cut"
|
|
249
|
+
elsif (ignore = a["Ignore"])
|
|
250
|
+
"(ignore #{render_atom(atoms, ignore['atom'], depth + 1)})"
|
|
251
|
+
elsif (trivia = a["Trivia"])
|
|
252
|
+
"(trivia #{render_atom(atoms, trivia['atom'], depth + 1)})"
|
|
253
|
+
elsif (capture_trivia = a["TriviaCapture"])
|
|
254
|
+
"(trivia-cap #{capture_trivia['rules'].map { |m, k| "#{m}=#{k}" }.join(',')} " \
|
|
255
|
+
"#{render_atom(atoms, capture_trivia['atom'], depth + 1)})"
|
|
256
|
+
elsif (constant = a["Constant"])
|
|
257
|
+
"const:#{constant['value'].inspect}"
|
|
258
|
+
else
|
|
259
|
+
"unknown:#{a.keys.first}"
|
|
260
|
+
end
|
|
261
|
+
end
|
|
262
|
+
end
|
|
263
|
+
end
|
|
264
|
+
end
|
|
@@ -17,19 +17,29 @@ module Parsanol
|
|
|
17
17
|
|
|
18
18
|
Test = Struct.new(:entry, :kind, :input, :expect)
|
|
19
19
|
|
|
20
|
-
attr_accessor :grammar_name, :version, :source, :uses, :skip_declaration, :skip_source,
|
|
21
|
-
:atomic_rules, :skip_capture_key
|
|
20
|
+
attr_accessor :grammar_name, :version, :edition, :source, :uses, :skip_declaration, :skip_source,
|
|
21
|
+
:atomic_rules, :skip_capture_key, :brace_rules
|
|
22
22
|
# R2 declared resync: rule name -> skip pattern (the span matched
|
|
23
23
|
# at a recovery point is consumed as annotated error text).
|
|
24
24
|
attr_reader :resyncs
|
|
25
25
|
attr_reader :rules, :bindings, :preprocess, :entries, :docs, :tests, :render, :derive,
|
|
26
|
+
:declaration_order,
|
|
26
27
|
:customs, :states
|
|
27
28
|
# Entries authored in this file (pre-import); nil-entry tests resolve
|
|
28
29
|
# against these first.
|
|
29
30
|
attr_accessor :own_entries
|
|
31
|
+
# Rule names the grammar itself wrote, snapshotted before the
|
|
32
|
+
# compiler injects core rules — lints must not mistake an injected
|
|
33
|
+
# prelude rule for an authored shadow.
|
|
34
|
+
attr_accessor :authored_rules
|
|
30
35
|
|
|
31
36
|
def initialize
|
|
32
37
|
@grammar_name = nil
|
|
38
|
+
# The default capture dialect is v1 (the owner's reversal,
|
|
39
|
+
# 2026-10-28): runtime compiles of unpinned sources keep the
|
|
40
|
+
# legacy shapes — the pubid fleet froze against them. mode "v2"
|
|
41
|
+
# stays the named-trees opt-in via the grammar header.
|
|
42
|
+
@edition = "v1"
|
|
33
43
|
@version = "0.0.0"
|
|
34
44
|
@source = nil
|
|
35
45
|
@rules = {}
|
|
@@ -38,6 +48,7 @@ module Parsanol
|
|
|
38
48
|
@bindings = {}
|
|
39
49
|
@preprocess = {}
|
|
40
50
|
@entries = {}
|
|
51
|
+
@declaration_order = []
|
|
41
52
|
@docs = {}
|
|
42
53
|
@tests = []
|
|
43
54
|
@uses = []
|
|
@@ -48,6 +59,8 @@ module Parsanol
|
|
|
48
59
|
# suppressed inside their bodies (token-shaped rules match
|
|
49
60
|
# contiguously).
|
|
50
61
|
@atomic_rules = []
|
|
62
|
+
@brace_rules = []
|
|
63
|
+
@authored_rules = nil
|
|
51
64
|
# R2 declared resync: rule -> skip pattern. Recovery consumes
|
|
52
65
|
# each pattern match at a failure point as annotated error text
|
|
53
66
|
# and resumes the iteration after it.
|