parsanol 1.3.66 → 1.3.68

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: da894739a391336f9b9f65a65abc87da1fe51b3f3268c0c2334638e53f5adb8f
-  data.tar.gz: 6777a602a63a37e599fdc1ed9c38f82eb8b01a30a12a23c24bbed7518f37e5cf
+  metadata.gz: 5917e9971751a101c27984a0ccec6aa6254813b77d18a45eea7f6a5538ead517
+  data.tar.gz: 9513f097d858d7c394a10c8002e372523212bf11629813867ec59a91dacbc41d
 SHA512:
-  metadata.gz: 0fad1aa5febfc712a767578c4ca4d9c9c6cf3def48ed7f6fe97a66a95e45d0e628a4285660eb14b40e6d0b72bbd6422868b85ea901d84318382f23aa5744378b
-  data.tar.gz: 6ab9b3b4461592a3a5e91accd57d0341c59c7f878985b01fd89b73fe91c8abd71eacedd6246d0c79a2fd57547607415c0932e73cfb2c2e61b3adb2af5165a9a6
+  metadata.gz: 65cf06e07d16879100effb50a7ed8012e95774ed6b8c86f97c997d3b6ce7a41525a07e3943d4670dc3c6f65e91d587953e719668decba35f90610bdf29a7c011
+  data.tar.gz: 84cd53317231f028e6510d90c7c426379566e47570594e960b022766ae45fffb8305afdcc5f044339fbbdb3a30c74e44e25ce03a735c7e3672221f70f26b0ab1

data/lib/parsanol/atoms/context.rb

@@ -63,6 +63,10 @@ module Parsanol
 
         # Cache eviction state
         @furthest_pos = 0
+
+        # Injected-trivia depth: while positive, reported causes are
+        # marked trivia and never surface in diagnostics.
+        @trivia_depth = 0
         @evict_threshold = 200
         @evict_counter = 0
         @evict_interval = 100
@@ -230,15 +234,17 @@ module Parsanol
       # Pre-allocated result constants
       SUCCESS_RESULT = [true, nil].freeze
       ERROR_RESULT = [false, nil].freeze
+      TRIVIA_ERROR = [false, Cause::TriviaCause.instance].freeze
 
       # Reports an error at a specific position.
       #
       # @return [Array(Boolean, Object)] error result tuple
       #
       def err_at(*)
-        return [false, @reporter.err_at(*)] if @reporter
+        return TRIVIA_ERROR if @trivia_depth.positive?
+        return ERROR_RESULT unless @reporter
 
-        ERROR_RESULT
+        [false, @reporter.err_at(*)]
       end
 
       # Reports an error at the current position.
@@ -246,9 +252,22 @@ module Parsanol
       # @return [Array(Boolean, Object)] error result tuple
       #
       def err(*)
-        return [false, @reporter.err(*)] if @reporter
+        return TRIVIA_ERROR if @trivia_depth.positive?
+        return ERROR_RESULT unless @reporter
+
+        [false, @reporter.err(*)]
+      end
 
-        ERROR_RESULT
+      # Brackets a parse inside injected trivia: causes reported while
+      # the depth is positive carry the trivia flag and never surface
+      # in rendered trees or deepest-failure positions.
+      def with_trivia
+        @trivia_depth += 1
+        begin
+          yield
+        ensure
+          @trivia_depth -= 1
+        end
       end
 
       # Checks if this context is collecting diagnostic errors.

data/lib/parsanol/atoms/sequence.rb

@@ -77,7 +77,11 @@ module Parsanol
       # @param prec [Integer] precedence
       # @return [String]
       def to_s_inner(prec)
-        @parslets.map { |p| p.to_s(prec) }.join(" ")
+        # Injected trivia is invisible in diagnostics: a failed
+        # sequence describes what the grammar asks for, not the skip
+        # machinery interleaved around it.
+        @parslets.grep_v(Parsanol::Atoms::Trivia)
+          .map { |p| p.to_s(prec) }.join(" ")
       end
 
       # FIRST set is first element's FIRST set (with epsilon propagation).

data/lib/parsanol/atoms/trivia.rb

@@ -0,0 +1,21 @@
+# frozen_string_literal: true
+
+module Parsanol
+  module Atoms
+    # The injected skip wrapper: Ignored semantics (matched bytes
+    # contribute nothing to values or spans) plus diagnostics
+    # transparency — failures inside trivia are marked and never
+    # surface in rendered trees or deepest-failure positions.
+    class Trivia < Ignored
+      def apply(source, context, consume_all)
+        context.with_trivia do
+          super
+        end
+      end
+
+      def to_s_inner(prec)
+        "trivia(#{@wrapped_atom.to_s(prec)})"
+      end
+    end
+  end
+end

data/lib/parsanol/atoms.rb

@@ -27,6 +27,7 @@ module Parsanol
     require "parsanol/atoms/base"
     require "parsanol/atoms/custom"
     require "parsanol/atoms/ignored"
+    require "parsanol/atoms/trivia"
     require "parsanol/atoms/named"
     require "parsanol/atoms/lookahead"
     require "parsanol/atoms/cut"

data/lib/parsanol/cause.rb

@@ -16,6 +16,8 @@
 #   cause.children  # => []
 #   cause.to_s  # => "Expected at least one"
 #
+require "singleton"
+
 module Parsanol
   class Cause
     # @return [Array<String>] Error message parts
@@ -33,6 +35,15 @@ module Parsanol
     # @return [Array<Cause>] Child causes
     attr_reader :children
 
+    # True when this failure happened inside injected skip trivia:
+    # trivia positions never surface in diagnostics (rendered trees
+    # prune trivia causes; deepest-failure tracking ignores them).
+    attr_accessor :trivia
+
+    def trivia?
+      @trivia == true
+    end
+
     # Creates a new cause for parse failure
     #
     # @param message [String, Array<String>] Error description
@@ -45,6 +56,7 @@ module Parsanol
       @position = position
       @children = children.nil? ? [] : children
       @parsing_label = nil
+      @trivia = false
     end
 
     # Factory method for creating a cause
@@ -89,6 +101,24 @@ module Parsanol
       output.string
     end
 
+    # Stand-in for failures inside injected skip trivia: the reporting
+    # context returns it instead of calling the reporter (reporters and
+    # deepest-failure tracking never see trivia failures), and rendered
+    # trees prune it (visible).
+    class TriviaCause < Cause
+      include Singleton
+
+      def initialize
+        super("trivia", nil, 0)
+        @trivia = true
+      end
+
+      # Label propagation would mutate the shared instance.
+      def set_label(_label)
+        nil
+      end
+    end
+
     # Raises a ParseFailed exception with this cause's information
     #
     # @raise [Parsanol::ParseFailed] Always
@@ -99,12 +129,18 @@ module Parsanol
 
     private
 
+    # Trivia causes stay out of rendered trees entirely.
+    def visible(node)
+      node.children.reject(&:trivia?)
+    end
+
     def build_tree_recursive(node, stream, prefix_flags)
       render_prefix(stream, prefix_flags)
       stream.puts node.to_s
 
-      node.children.each do |child|
-        is_last_child = (node.children.last == child)
+      children = visible(node)
+      children.each do |child|
+        is_last_child = (children.last == child)
         build_tree_recursive(child, stream, prefix_flags + [is_last_child])
       end
     end

data/lib/parsanol/native/ffi.rb

@@ -43,8 +43,18 @@ module Parsanol
             json = ::Parsanol::Native::Parser.serialize_grammar(grammar)
             atom = grammar
           end
-          handle = ((@handles ||= {})[json] ||= @binding.c_register(json))
-          raise Error, "grammar registration failed" if handle.zero?
+          handle = @handles && @handles[json]
+          if handle
+            # Refresh recency: reinsertion moves the entry to the end,
+            # so churn cannot evict a hot grammar's handle.
+            @handles[json] = @handles.delete(json)
+          else
+            handle = @binding.c_register(json)
+            raise Error, "grammar registration failed" if handle.zero?
+
+            (@handles ||= {})[json] = handle
+            trim_handles
+          end
 
           written = parse_into(handle, input)
           if written.positive?
@@ -65,6 +75,19 @@ module Parsanol
 
         private
 
+        # FIFO cap with recency refresh, mirroring Parser::CACHE_LIMIT.
+        # Without it, every grammar ever parsed kept its Rust-side
+        # HandleEntry (parsed Grammar + engine state) forever, and the
+        # map keys are full grammar JSON strings. Evicted handles are
+        # released through c_release; a later miss re-registers.
+        def trim_handles
+          while @handles.size > ::Parsanol::Native::Parser::CACHE_LIMIT
+            json, handle = @handles.first
+            @handles.delete(json)
+            @binding.c_release(handle)
+          end
+        end
+
         # Search order: explicit override, gem-vendored cdylib next to
         # this file, then the system linker path.
         def locate_library
@@ -110,8 +133,10 @@ module Parsanol
                             %i[uint64], :void
           end
           # Probe the symbols now so a mismatched library fails loudly
-          # at availability time, not at first parse.
+          # at availability time, not at first parse. c_release(0) is a
+          # no-op (unknown handle), so the probe has no side effect.
           probe = binding_mod.c_last_error
+          binding_mod.c_release(0)
           @binding = binding_mod
           @buffer = nil
           !probe.nil? || true

data/lib/parsanol/native/parser.rb

@@ -6,7 +6,21 @@ module Parsanol
   module Native
     # Core parsing functionality using Rust native extension
     module Parser
-      GRAMMAR_HASH_CACHE = Hash.new
+      # Cap for the grammar-keyed caches. The handle cache pins a
+      # Rust-side HandleEntry (parsed Grammar + compiled program) per
+      # live entry — without the cap, every grammar ever registered
+      # accumulates on both sides of the FFI boundary. 32 mirrors
+      # VM::PROGRAM_CACHE_LIMIT; eviction re-registers on demand (the
+      # Rust artifact cache short-circuits the re-compile).
+      CACHE_LIMIT = 32
+
+      # Identity-keyed memo: root atom object => structure hash. Keyed
+      # by the OBJECT, not object_id — a recycled id must never return a
+      # dead grammar's hash for a new grammar (through GRAMMAR_CACHE /
+      # HANDLE_CACHE that would alias the new grammar to the old
+      # grammar's serialized JSON and Rust handle: a silent wrong
+      # native parse).
+      GRAMMAR_HASH_CACHE = Hash.new.compare_by_identity
       GRAMMAR_CACHE = Hash.new
       # structure-hash => Rust-side grammar handle. Keyed by content so a
       # recycled object_id can never alias a different grammar.
@@ -72,7 +86,18 @@ module Parsanol
         # access — no JSON marshal, no re-hash inside Rust.
         def grammar_handle(root_atom)
           cache_key = grammar_cache_key(root_atom)
-          HANDLE_CACHE[cache_key] ||= Native._register_grammar(grammar_json(root_atom))
+          handle = HANDLE_CACHE[cache_key]
+          if handle
+            # Refresh recency: reinsertion moves the entry to the end,
+            # so churn cannot evict a hot grammar's handle.
+            HANDLE_CACHE[cache_key] = HANDLE_CACHE.delete(cache_key)
+            return handle
+          end
+
+          handle = Native._register_grammar(grammar_json(root_atom))
+          HANDLE_CACHE[cache_key] = handle
+          trim_caches
+          handle
         end
 
         # Drop a handle whose Rust-side entry no longer exists; the next
@@ -82,6 +107,7 @@ module Parsanol
         end
 
         def clear_cache
+          HANDLE_CACHE.each_value { |handle| Native._release_grammar(handle) }
           GRAMMAR_HASH_CACHE.clear
           GRAMMAR_CACHE.clear
           HANDLE_CACHE.clear
@@ -99,14 +125,38 @@ module Parsanol
 
         def grammar_cache_key(root_atom)
           root_atom = root_atom.root if root_atom.is_a?(::Parsanol::Parser)
-          obj_id = root_atom.object_id
-          GRAMMAR_HASH_CACHE[obj_id] ||= grammar_structure_hash(root_atom)
+          cached = GRAMMAR_HASH_CACHE[root_atom]
+          return cached if cached
+
+          hash = grammar_structure_hash(root_atom)
+          GRAMMAR_HASH_CACHE[root_atom] = hash
+          GRAMMAR_HASH_CACHE.shift while GRAMMAR_HASH_CACHE.size > CACHE_LIMIT
+          hash
         end
 
         def grammar_json(root_atom)
           root_atom = root_atom.root if root_atom.is_a?(::Parsanol::Parser)
           cache_key = grammar_cache_key(root_atom)
-          GRAMMAR_CACHE[cache_key] ||= GrammarSerializer.serialize(root_atom)
+          json = GRAMMAR_CACHE[cache_key]
+          return json if json
+
+          json = GrammarSerializer.serialize(root_atom)
+          GRAMMAR_CACHE[cache_key] = json
+          json
+        end
+
+        # FIFO eviction past the cap; evicted handles are released
+        # Rust-side so HandleEntries (Grammar + program) do not
+        # accumulate. Re-registration on a later miss is cheap: the
+        # Rust artifact cache reloads the compiled program.
+        def trim_caches
+          while HANDLE_CACHE.size > CACHE_LIMIT
+            key, handle = HANDLE_CACHE.first
+            HANDLE_CACHE.delete(key)
+            GRAMMAR_CACHE.delete(key)
+            Native._release_grammar(handle)
+          end
+          GRAMMAR_CACHE.shift while GRAMMAR_CACHE.size > CACHE_LIMIT
         end
 
         def grammar_structure_hash(atom)

data/lib/parsanol/parg/compiler.rb

@@ -55,6 +55,17 @@ module Parsanol
         @warnings = []
       end
 
+      # QoL shorthands (ABNF core rules): materialized on reference when
+      # the grammar does not define its own rule of that name.
+      BUILTIN_CLASSES = {
+        "DIGIT" => [[0x30, 0x39]],
+        "ALPHA" => [[0x41, 0x5A], [0x61, 0x7A]],
+        "ALNUM" => [[0x30, 0x39], [0x41, 0x5A], [0x61, 0x7A]],
+        "WORD" => [[0x30, 0x39], [0x41, 0x5A], [0x5F, 0x5F], [0x61, 0x7A]],
+        "HEXDIG" => [[0x30, 0x39], [0x41, 0x46], [0x61, 0x66]],
+        "ANY" => [[0x00, 0x10FFFF]],
+      }.freeze
+
       def compile
         @document.rules.each_key { |name| atom_for(name) }
         errors = Lints.errors(@document, self)
@@ -79,7 +90,146 @@ module Parsanol
       end
 
       def atom_for(name)
-        @atom_cache[name] ||= build_atom(fetch_rule(name))
+        @atom_cache[name] ||= begin
+          atom = build_atom(fetch_rule(name))
+          # Atomic rules build without injection entirely: token-shaped
+          # rules match contiguously (no inter-iteration trivia, no
+          # entry-boundary skips).
+          if @document.skip_declaration && !skip_exempt_names.include?(name) &&
+              !@document.atomic_rules.include?(name)
+            atom = inject_skip_trivia(atom)
+            # Entry points take leading and trailing skips: tail trivia
+            # (the tail-comment case) has no following terminal to
+            # trigger an injection, and full-consumption would reject
+            # it; leading trivia has no preceding interleave either.
+            if entry_rule?(name)
+              atom = Atoms::Sequence.new(skip_maybe_atom, atom, skip_maybe_atom)
+            end
+          end
+          atom
+        end
+      end
+
+      def entry_rule?(name)
+        @document.entries.value?(name) || @document.own_entries&.include?(name)
+      end
+
+      # The skip declaration's rule references (and their transitive
+      # rule references) build without injection: the skip atom must
+      # not contain itself.
+      def skip_exempt_names
+        @skip_exempt_names ||= begin
+          names = Set.new
+          walk_refs = lambda do |node|
+            case node.kind
+            when :ref then names << node.a
+            when :alt, :seq then node.a.each { |b| walk_refs.call(b) }
+            when :cap, :pred then walk_refs.call(node.b)
+            when :rep, :opt then walk_refs.call(node.a)
+            end
+          end
+          walk_refs.call(@document.skip_declaration) if @document.skip_declaration
+          closed = Set.new
+          until names.subset?(closed)
+            (names - closed).each do |ref|
+              closed << ref
+              walk_refs.call(@document.rules[ref]) if @document.rules.key?(ref)
+            end
+            names += closed
+          end
+          names
+        end
+      end
+
+      def skip_wrapper?(atom)
+        @skip_wrapper_ids&.key?(atom)
+      end
+
+      def skip_maybe_atom
+        @skip_maybe_atom ||= begin
+          atom = build_atom(@document.skip_declaration)
+          while atom.is_a?(Atoms::Entity)
+            atom = atom.parslet
+          end
+          # the VM's internal compiler owns the ATOM-level nullability
+          # analysis (the node-level nullable? here works on rule nodes)
+          if Parsanol::VM::Compiler.new.nullable?(atom)
+            raise CompileError,
+                  "skip rule must be non-nullable: a skip that can match " \
+                  "empty loops forever at injection points"
+          end
+          # Trivia: consumed but contributes nothing to the enclosing
+          # sequence/repetition value — trivia never leaks into
+          # span-joined captures — and failures inside it never surface
+          # in diagnostics (rendered trees and deepest-failure
+          # positions skip trivia causes).
+          wrapper = Atoms::Trivia.new(Atoms::Repetition.new(atom, 0, 1))
+          (@skip_wrapper_ids ||= {}.compare_by_identity)[wrapper] = true
+          wrapper
+        end
+      end
+
+      # parsanol-ruby#134: inject an optional match of the declared skip
+      # rule before every terminal of every rule. Injected skips are
+      # capture-free (a bare Repetition contributes no Named results),
+      # optional (ordered-choice/backtracking semantics preserved), and
+      # leave first sets unchanged (an optional prefix is epsilon-able).
+      def inject_skip_trivia(atom)
+        return atom if @document.skip_declaration.nil?
+
+        case atom
+        # rubocop:disable Lint/DuplicateBranch -- pass-through is the
+        # deliberate default; identical bodies, different decisions
+        when Atoms::Sequence
+          kids = atom.parslets.map { |child| inject_skip_trivia(child) }
+          out = []
+          kids.each do |child|
+            out << skip_maybe_atom unless skip_wrapper?(child)
+            out << child
+          end
+          out.length == 1 ? out.first : Atoms::Sequence.new(*out)
+        when Atoms::Alternative
+          Atoms::Alternative.new(*atom.alternatives.map { |a| inject_skip_trivia(a) })
+        when Atoms::Repetition
+          # Inside repetition bodies, trivia is injected only before
+          # rule-reference children: structural iterations
+          # (`( sep item )*`) still span trivia between iterations,
+          # while char-scan spans (`( !"x" ANY )*` quoted-string bodies,
+          # WORD runs, comment bodies) stay verbatim — injecting there
+          # corrupted them (spaces inside strings eaten as whitespace,
+          # `//` inside a URL read as a comment). A single-ref body
+          # (`1*pair`) needs no interior injection: each iteration's own
+          # rule-level leading injection consumes the trivia.
+          body = atom.parslet
+          if body.is_a?(Atoms::Sequence) &&
+              body.parslets.any?(Atoms::Entity)
+            kids = body.parslets.flat_map do |child|
+              child.is_a?(Atoms::Entity) ? [skip_maybe_atom, child] : child
+            end
+            rebuilt = kids.length == 1 ? kids.first : Atoms::Sequence.new(*kids)
+            Atoms::Repetition.new(rebuilt, atom.min, atom.max, atom.result_tag)
+          else
+            atom
+          end
+        when Atoms::Named
+          # Descend, keeping the capture's span clean: injected skips
+          # are Ignored-wrapped, so the bytes they consume never join
+          # span-joined captures. Composites must interleave around
+          # their own children (refs included) — the old pass-through
+          # relied on every referenced leaf self-injecting, which
+          # `atomic` rules deliberately stop doing.
+          Atoms::Named.new(inject_skip_trivia(atom.parslet), atom.name)
+        when Atoms::Lookahead
+          Atoms::Lookahead.new(inject_skip_trivia(atom.bound_parslet), atom.positive)
+        when Atoms::Str, Atoms::Re
+          # bare terminal as a whole rule body: leading trivia position
+          Atoms::Sequence.new(skip_maybe_atom, atom)
+        # pass-through is the deliberate default; identical bodies,
+        # different decisions
+        else
+          atom
+        end
+        # rubocop:enable Lint/DuplicateBranch
       end
 
       def record_warning(message)
@@ -174,6 +324,10 @@ module Parsanol
         if !@document.rules.key?(name) && @document.customs.key?(name)
           return Node.new(:lit, "\u0001", false)
         end
+        if !@document.rules.key?(name) && BUILTIN_CLASSES.key?(name)
+          # QoL shorthand stand-in: the analysis passes see the class
+          return Node.new(:class, BUILTIN_CLASSES[name])
+        end
 
         @document.rules[name] || raise(CompileError, "unknown rule #{name.inspect}")
       end
@@ -193,6 +347,15 @@ module Parsanol
             Atoms::Str.new(node.a)
           end
         when :class then Atoms::Re.new(class_pattern(node.a))
+        when :until
+          # QoL: (*( !terminal any_char )) — consume everything up to
+          # (and excluding) the delimiter.
+          any_char = Atoms::Re.new(class_pattern([[0x00, 0x10FFFF]]))
+          Atoms::Repetition.new(
+            Atoms::Sequence.new(Atoms::Lookahead.new(Atoms::Str.new(node.a), false),
+                                any_char),
+            0, nil
+          )
         when :seq
           atoms = node.a.map { |child| build_atom(child) }
           atoms.length == 1 ? atoms.first : Atoms::Sequence.new(*atoms)
@@ -234,6 +397,10 @@ module Parsanol
           if @document.customs.key?(node.a)
             @dynamic = true
             resolve_custom(node.a)
+          elsif !@document.rules.key?(node.a) && BUILTIN_CLASSES.key?(node.a)
+            # QoL: character-class shorthands (ABNF core rules) — usable
+            # unless the grammar defines its own rule of the same name.
+            Atoms::Re.new(class_pattern(BUILTIN_CLASSES[node.a]))
           else
             Atoms::Entity.new(node.a) { atom_for(node.a) }
           end
@@ -472,6 +639,13 @@ module Parsanol
           "docs" => @document.docs,
           "source" => @document.source,
         }
+        if @document.skip_declaration
+          # parsanol-ruby#134: declared trivia rule. Injection is baked
+          # into the compiled trees (plain atom kinds), so consumers
+          # without skip support still parse these artifacts correctly;
+          # the field records the grammar-level declaration.
+          envelope["skip"] = @document.skip_source
+        end
         envelope["customs"] = @document.customs unless @document.customs.empty?
         envelope["states"] = @document.states unless @document.states.empty?
         envelope["dynamic"] = true if @dynamic

data/lib/parsanol/parg/document.rb

@@ -9,7 +9,8 @@ module Parsanol
 
       Test = Struct.new(:entry, :kind, :input, :expect)
 
-      attr_accessor :grammar_name, :version, :source, :uses
+      attr_accessor :grammar_name, :version, :source, :uses, :skip_declaration, :skip_source,
+                    :atomic_rules
       attr_reader :rules, :bindings, :preprocess, :entries, :docs, :tests, :render, :derive,
                   :customs, :states
       # Entries authored in this file (pre-import); nil-entry tests resolve
@@ -32,6 +33,10 @@ module Parsanol
         @own_entries = nil
         @customs = {}
         @states = {}
+        # Rules declared `atomic <name> = ...`: skip-trivia injection is
+        # suppressed inside their bodies (token-shaped rules match
+        # contiguously).
+        @atomic_rules = []
       end
 
       def validate!

data/lib/parsanol/parg/parser.rb

@@ -20,7 +20,8 @@ module Parsanol
 
       STATE_KEYWORDS = %w[custom state].freeze
       KEYWORDS = %w[grammar as alt from_table column bindings render derive
-                    preprocess entry table_lookup custom state set switch].freeze
+                    preprocess entry table_lookup custom state set switch
+                    skip until atomic].freeze
 
       def initialize(text)
         @tokens = Lexer.new(text).tokens
@@ -161,6 +162,19 @@ module Parsanol
             skip_newlines
             next
           end
+          if peek&.type == :ident && peek.value == "skip"
+            advance
+            punct("=")
+            if document.skip_declaration
+              raise ParseError, "duplicate skip declaration"
+            end
+
+            node = parse_choice
+            document.skip_declaration = validate_skip_node(node)
+            document.skip_source = skip_source_string(node)
+            skip_newlines
+            next
+          end
           if peek&.type == :ident && STATE_KEYWORDS.include?(peek.value)
             keyword = advance.value
             slot = ident.value
@@ -174,6 +188,11 @@ module Parsanol
             skip_newlines
             next
           end
+          # `atomic <rule> = ...` suppresses skip-trivia injection inside
+          # the rule body: token-shaped rules match their input
+          # contiguously instead of spanning trivia between iterations.
+          atomic = peek&.type == :ident && peek.value == "atomic"
+          advance if atomic
           name = rule_name
           document.docs[name] = docs.join("\n") unless docs.empty?
           punct("=")
@@ -183,11 +202,36 @@ module Parsanol
           end
 
           document.rules[name] = node
+          document.atomic_rules << name if atomic
           skip_newlines
         end
         punct("}")
       end
 
+      # The skip declaration may only name rules (a rule reference or an
+      # alternation of rule references) — arbitrary expressions would
+      # make the injected trivia unanalyzable.
+      def skip_source_string(node)
+        case node.kind
+        when :ref then node.a
+        when :alt then node.a.map { |b| skip_source_string(b) }.join(" / ")
+        end
+      end
+
+      def validate_skip_node(node)
+        case node.kind
+        when :ref
+          node
+        when :alt
+          node.a.each { |branch| validate_skip_node(branch) }
+          node
+        else
+          raise ParseError,
+                "skip must be a rule reference or an alternation of rule " \
+                "references, got #{node.kind.inspect}"
+        end
+      end
+
       def rule_name
         token = ident
         if KEYWORDS.include?(token.value)
@@ -325,6 +369,10 @@ module Parsanol
           end
           punct("}")
           Node.new(:switch, slot, arms)
+        when "until"
+          # QoL: any-until terminal — (*( !terminal any_char )), the
+          # idiomatic "consume everything up to the delimiter" run.
+          Node.new(:until, unquote(expect(:str).value))
         when "alt"
           ident("from_table")
           table = unquote(expect(:str).value)

data/lib/parsanol/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Parsanol
-  VERSION = "1.3.66"
+  VERSION = "1.3.68"
 end

data/lib/parsanol/vm.rb

@@ -85,11 +85,26 @@ module Parsanol
     MEMO_BUDGET_FACTOR = 4_096
     MEMO_BUDGET_CELLS_FACTOR = 16
 
+    # Compiled-program and heavy-flag caches hold at most this many
+    # grammars. A process rarely uses more than a handful concurrently;
+    # the cap keeps worst-case retention explicit and bounded instead
+    # of growing with every grammar ever parsed.
+    PROGRAM_CACHE_LIMIT = 32
+
     class << self
-      @programs = {}
+      # Compiled-program and heavy-flag caches, keyed by the root atom
+      # object itself (identity semantics — two structurally identical
+      # grammars are distinct entries). This replaces the original
+      # object_id-keyed plain Hash, where entries outlived their
+      # grammars forever and a recycled object_id could serve a fresh
+      # grammar another grammar's program. Holding the object as the
+      # key does pin the grammar while cached — bounded by
+      # PROGRAM_CACHE_LIMIT, FIFO eviction.
+      @programs = {}.compare_by_identity
+      @heavy = {}.compare_by_identity
 
       # Cached compile keyed by root-atom object identity. Atoms are
-      # effectively immutable once constructed; false caches grammars the
+      # effectively immutable once constructed; nil caches grammars the
       # VM cannot handle so we do not re-walk them on every parse.
       #
       # A grammar whose VM run failed once (unsupported backtracking shape
@@ -97,29 +112,32 @@ module Parsanol
       # attempt entirely — the interpreter result is identical.
       def program_for(atom)
         root = atom.is_a?(Parsanol::Parser) ? atom.root : atom
-        key = root.object_id
-        cache = (@programs ||= {})
-        cached = cache[key]
-        return nil if cached == :fallback
-        return cached if key?(cache, key)
+        cache = (@programs ||= {}.compare_by_identity)
+        cached = cache[root]
+        return nil if cached.equal?(:fallback)
+        return cached if key?(cache, root)
 
         # A grammar the compiler cannot compile — e.g. one containing a
         # lazily resolved Entity that a select-first literal index would
         # never select — falls back to the interpreter, the source of
         # truth. The interpreter resolves such entities lazily, so an
         # unselected branch simply never raises.
-        begin
-          cache[key] = compile(atom)
-        rescue NotImplementedError
-          cache[key] = :fallback
-          nil
-        end
+        program =
+          begin
+            compile(atom)
+          rescue NotImplementedError
+            :fallback
+          end
+        cache[root] = program
+        trim_cache(cache)
+        program.equal?(:fallback) ? nil : program
       end
 
       # Marks a grammar as VM-incompatible after a runtime failure.
       def disable_for!(atom)
         root = atom.is_a?(Parsanol::Parser) ? atom.root : atom
-        (@programs ||= {})[root.object_id] = :fallback # rubocop:disable Lint/HashCompareByIdentity -- object_id keys avoid holding strong references to grammar atoms
+        (@programs ||= {}.compare_by_identity)[root] = :fallback
+        trim_cache(@programs)
       end
 
       def key?(cache, key) # rubocop:disable Naming/PredicateMethod -- mirrors Hash#key?
@@ -131,6 +149,12 @@ module Parsanol
         @heavy&.clear
       end
 
+      # FIFO eviction past the cache limit.
+      def trim_cache(cache)
+        cache.shift while cache.size > PROGRAM_CACHE_LIMIT
+        nil
+      end
+
       # Compiles the grammar rooted at +atom+. Returns the flat program
       # Array, or nil when the grammar uses unsupported atoms.
       def compile(atom)
@@ -168,8 +192,8 @@ module Parsanol
         # included — the seed is recomputed at every compile), so the
         # cold-start never pays the doomed unmemoized exploration.
         if compiler.backtracking_prone
-          # rubocop:disable-next Lint/HashCompareByIdentity -- object_id keys mirror run_for's heavy flag
-          (@heavy ||= {})[root.object_id] = true
+          (@heavy ||= {}.compare_by_identity)[root] = true
+          trim_cache(@heavy)
         end
 
         compiler.to_program
@@ -194,6 +218,13 @@ module Parsanol
             break if walk_branch.nil?
           end
           next if walk_branch.nil?
+          # A bare zero-width assertion is a guard, not a content
+          # matcher: `&X / rest` commits to its empty match exactly
+          # where X succeeds, and `rest` is reachable exactly where X
+          # fails — conditional control flow, not accidental shadowing
+          # (#137). Both engines already stop repetitions on
+          # zero-width iterations, so guard branches cannot loop.
+          next if walk_branch.is_a?(Parsanol::Atoms::Lookahead)
           next unless Compiler.new.nullable?(walk_branch)
 
           "alternative branch #{idx} (#{branch.inspect}) can match " \
@@ -287,9 +318,8 @@ module Parsanol
       # interpreter.
       def run_for(atom, program, input, consume_all)
         root = atom.is_a?(Parsanol::Parser) ? atom.root : atom
-        id = root.object_id
-        heavy = (@heavy ||= {})
-        if heavy[id]
+        heavy = (@heavy ||= {}.compare_by_identity)
+        if heavy[root]
           result = Executor.new(program, input, consume_all,
                                 memoize: true).execute
           return BAIL if result.equal?(BUDGET)
@@ -300,7 +330,8 @@ module Parsanol
         result = Executor.new(program, input, consume_all).execute
         if result.equal?(BUDGET) ||
             (result.is_a?(Array) && result.first == :heavy)
-          heavy[id] = true
+          heavy[root] = true
+          trim_cache(heavy)
           memo_result = Executor.new(program, input, consume_all,
                                      memoize: true).execute
           # A heavy naive success already answered; the memo pass only

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: parsanol
 version: !ruby/object:Gem::Version
-  version: 1.3.66
+  version: 1.3.68
 platform: ruby
 authors:
 - Ribose Inc.
@@ -72,6 +72,7 @@ files:
 - lib/parsanol/atoms/sequence.rb
 - lib/parsanol/atoms/state.rb
 - lib/parsanol/atoms/str.rb
+- lib/parsanol/atoms/trivia.rb
 - lib/parsanol/atoms/visitor.rb
 - lib/parsanol/buffer.rb
 - lib/parsanol/builder_callbacks.rb