parsanol 1.3.80 → 1.3.82

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: f1cf70af68139adf8136c539ac11fc7e82c2e4c3975d8d47970b3c90d3842574
-  data.tar.gz: 30334852e279a556f196c1c0738b3348a9436cde57492638eddb43464567ec6e
+  metadata.gz: c56435897ed6f65bc101a9df1cdb0c0510917824295fa003893cdf7e447a32a4
+  data.tar.gz: 8308c7b4bc8179fbf7867ab1769c53b1f6759ce6ad8cd7ca2dc9362e8c5a3c58
 SHA512:
-  metadata.gz: d8ca94e8d24797ea68e041668e9e6d024b0e9a1c8c30d07c4a9a1c7063e1c23db9cdfbf67bd9ee66978df0ee43f6b53a390b6f360b89bd756169e4d48fdc8df4
-  data.tar.gz: 2137eb6520d33c6a0654940c791caf28f560b5975dbf83055965b095c9410420a229f6aaa9498bfbf856ca0618f8da6c8d18c34c8896d671859eabec5bd7cc87
+  metadata.gz: 7ad6958bee6a30a11ce020e95a1b084bd26155f7293c2c127c2ff264850da1df22bbde4eb1f321f8e442448492d5612adc25fc3ccd19fe8a65b3e9ea7a2f9731
+  data.tar.gz: 4e353b492c5113f197822c528bf37622b54bc6eae4d2a4bb73f171cbce7edadd00c96ed55928eda73a9d2746d68e14fb4fd212ec2c768bca30a733123a32c1c8

data/Cargo.lock

@@ -307,9 +307,9 @@ checksum = "9f7c3e4beb33f85d45ae3e3a1792185706c8e16d043238c593331cc7cd313b50"
 
 [[package]]
 name = "parsanol"
-version = "0.17.2"
+version = "0.18.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "2b6296dcfe8a2055273a1cb76ffb383c586dfbb6e647ff11b9f396f1b8f6fc36"
+checksum = "dc899494d6b94ea8df14279b7dcbac89d59ba4d1be52f7b8ffe840674c6c8e3c"
 dependencies = [
  "ahash",
  "getrandom 0.3.4",
@@ -326,9 +326,9 @@ dependencies = [
 
 [[package]]
 name = "parsanol-derive"
-version = "0.17.2"
+version = "0.18.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "60e96bd9d40164ef69ba879c97ba0d3efcf5e4183062ef6bf1eb66c4cdf91ae5"
+checksum = "b437046956f356eee0eb9b407ff1f1f6286af1f70f1af1d305c54a64c65b64e2"
 dependencies = [
  "proc-macro2",
  "quote",

data/ext/parsanol_cdylib/Cargo.toml

@@ -13,4 +13,4 @@ doc = false
 
 [dependencies]
 # 0.17.0 carries LookSpec + the widened opaque writeback
-parsanol = { version = "0.17.2" }
+parsanol = { version = "0.18.0" }

data/ext/parsanol_native/Cargo.toml

@@ -32,7 +32,7 @@ magnus = { version = "0.9" }
 # parsanol parser library (crates.io; 0.17.0 carries the LookSpec regex
 # lookbehind (rs#163 gap 1) and the widened opaque writeback: the
 # lazy-continuation fix, parsanol-ruby#160)
-parsanol = { version = "0.17.2", features = ["ruby"] }
+parsanol = { version = "0.18.0", features = ["ruby"] }
 
 # Logging
 log = "0.4"

data/lib/parsanol/atoms/sequence.rb

@@ -18,14 +18,25 @@ module Parsanol
       def initialize(*components)
         super()
         @parslets = components
-
-        # Pre-built error message
-        @fail_msg = "Failed to match sequence (#{inspect})"
+        # The failure message renders LAZILY (error_msgs): constructing
+        # sequences is hot-path work for the builder/injector, and
+        # stringifying the whole subtree eagerly is exponential on
+        # shared cross-referencing grammars (parsanol-ruby#180 side
+        # observation: a 25-rule Fibonacci-shaped grammar took 6s).
+        @fail_msg = nil
       end
 
       # Error messages hash (for compatibility)
       def error_msgs
-        { failed: @fail_msg }
+        { failed: fail_msg }
+      end
+
+      # Rendered lazily (see initialize); construction is hot-path
+      # builder/injector work and stringifying the whole subtree
+      # eagerly is exponential on shared cross-referencing grammars
+      # (parsanol-ruby#180 side observation).
+      def fail_msg
+        @fail_msg ||= "Failed to match sequence (#{inspect})"
       end
 
       # Appends a parser to this sequence with flattening.
@@ -104,7 +115,7 @@ module Parsanol
       # Single element sequence
       def match_single(parser, source, context, consume_all)
         success, value = parser.apply(source, context, consume_all)
-        return context.err(self, source, @fail_msg, [value]) unless success
+        return context.err(self, source, fail_msg, [value]) unless success
 
         ok([:sequence, value])
       end
@@ -112,10 +123,10 @@ module Parsanol
       # Two-element sequence
       def match_pair(p1, p2, source, context, consume_all)
         success, v1 = p1.apply(source, context, false)
-        return context.err(self, source, @fail_msg, [v1]) unless success
+        return context.err(self, source, fail_msg, [v1]) unless success
 
         success, v2 = p2.apply(source, context, consume_all)
-        return context.err(self, source, @fail_msg, [v2]) unless success
+        return context.err(self, source, fail_msg, [v2]) unless success
 
         ok([:sequence, v1, v2])
       end
@@ -123,13 +134,13 @@ module Parsanol
       # Three-element sequence
       def match_triple(p1, p2, p3, source, context, consume_all)
         success, v1 = p1.apply(source, context, false)
-        return context.err(self, source, @fail_msg, [v1]) unless success
+        return context.err(self, source, fail_msg, [v1]) unless success
 
         success, v2 = p2.apply(source, context, false)
-        return context.err(self, source, @fail_msg, [v2]) unless success
+        return context.err(self, source, fail_msg, [v2]) unless success
 
         success, v3 = p3.apply(source, context, consume_all)
-        return context.err(self, source, @fail_msg, [v3]) unless success
+        return context.err(self, source, fail_msg, [v3]) unless success
 
         ok([:sequence, v1, v2, v3])
       end
@@ -148,7 +159,7 @@ module Parsanol
           success, value = components[idx].apply(source, context, must_consume)
 
           unless success
-            return context.err(self, source, @fail_msg, [value])
+            return context.err(self, source, fail_msg, [value])
           end
 
           result[idx + 1] = value

data/lib/parsanol/atoms/trivia.rb

@@ -28,12 +28,16 @@ module Parsanol
       # +capturers+ maps leading-literal marker -> kind label (the
       # compiler derives markers from each capturer rule's first Str
       # atom, e.g. "//" => :line_comment); units not matching any
-      # marker are whitespace-shaped and stay unrecorded.
-      attr_reader :capturers
+      # marker are whitespace-shaped and stay unrecorded — unless
+      # +whitespace_kind+ is declared (parsanol-ruby#180): every
+      # unit then records under that label, giving source-preserving
+      # grammars the full ordered trivia stream.
+      attr_reader :capturers, :whitespace_kind
 
-      def initialize(atom, capturers)
+      def initialize(atom, capturers, whitespace_kind: nil)
         super(atom)
         @capturers = capturers
+        @whitespace_kind = whitespace_kind
       end
 
       def apply(source, context, consume_all)
@@ -55,16 +59,16 @@ module Parsanol
       private
 
       def record(source, context, start_pos, end_pos)
-        return unless @capturers
-
         text = source.input[start_pos...end_pos]
         # A trivia unit may lead with whitespace before the comment
         # shape; markers test the comment head.
         stripped = text.lstrip
-        marker, label = @capturers.find { |m, _label| stripped.start_with?(m) }
-        return unless marker
-
-        context.push_captured_trivia(label, stripped.strip)
+        marker, label = @capturers&.find { |m, _label| stripped.start_with?(m) }
+        if marker
+          context.push_captured_trivia(label, stripped.strip)
+        elsif @whitespace_kind
+          context.push_captured_trivia(@whitespace_kind, text)
+        end
       end
     end
   end

data/lib/parsanol/native/parser.rb

@@ -205,6 +205,19 @@ module Parsanol
             [:constant, atom.value.inspect]
           when ::Parsanol::Atoms::Lookbehind
             [:lookbehind, atom.count, atom.pattern, atom.positive]
+          # Subclass first: TriviaCapture < Trivia < Ignored (the same
+          # dispatch discipline as the serializer's).
+          when ::Parsanol::Atoms::TriviaCapture
+            # The capturer table differentiates capture-carrying
+            # grammars from plain-trivia ones — without it the grammar
+            # cache serves one variant's wire to every other and the
+            # wrong marker table reaches the engine (parsanol-ruby#180).
+            [:trivia_capture, atom.capturers.to_a.sort,
+             atom_structure(atom.wrapped_atom, visited)]
+          when ::Parsanol::Atoms::Trivia
+            [:trivia, atom_structure(atom.wrapped_atom, visited)]
+          when ::Parsanol::Atoms::Ignored
+            [:ignored, atom_structure(atom.wrapped_atom, visited)]
           else
             [:unknown, atom.class.name]
           end

data/lib/parsanol/native/serializer.rb

@@ -332,12 +332,16 @@ module Parsanol
 
     def serialize_trivia_capture(atom)
       inner_id = serialize_atom(atom.wrapped_atom)
-      {
-        "TriviaCapture" => {
-          "atom" => inner_id,
-          "rules" => atom.capturers.map { |marker, label| [marker, label.to_s] },
-        },
+      payload = {
+        "atom" => inner_id,
+        "rules" => atom.capturers.map { |marker, label| [marker, label.to_s] },
       }
+      # parsanol-ruby#180: the optional whitespace kind — units that
+      # match no marker record under it (source-preserving grammars).
+      # Optional on the wire: engines predating it deserialize and
+      # simply don't record whitespace units (documented caveat).
+      payload["whitespace"] = atom.whitespace_kind.to_s if atom.whitespace_kind
+      { "TriviaCapture" => payload }
     end
 
     def serialize_trivia(atom)

data/lib/parsanol/parg/artifact.rb

@@ -65,24 +65,26 @@ module Parsanol
       def parse(entry_name, input, mode: :native)
         case mode
         when :native
-          if Native.available? && !envelope["dynamic"]
-            # Fast lane (parsanol-ruby#162): the per-instance atom
-            # tree rides the DSL serialization/registration caches
+          if Native.available?
+            # Fast lane (parsanol-ruby#162): Native.parse with the
+            # per-instance root atom registers the grammar once
             # (GRAMMAR_CACHE / HANDLE_CACHE keyed on the stable root
-            # atom), so an artifact parses at compiled speed after
-            # the first call. The old path re-serialized the whole
-            # grammar JSON and re-registered it per parse (~8x).
-            # Dynamic artifacts carry ruby-tier atoms their wire
-            # cannot express — they keep the interpreter.
-            begin
-              return root_atom(entry_name).parse(input, mode: :native)
-            rescue Parsanol::ParseFailed
-              # The atom-tree native path reports the root position; the
-              # wire-registration path carries the deepest-failure
-              # diagnostics (parsanol-ruby#171). Failures pay the
-              # serialization once; successes keep the fast lane.
-              return Native.parse(JSON.generate(entry(entry_name).fetch("grammar")), input)
-            end
+            # atom) and every parse after the first runs the native
+            # engine by handle — the same path a DSL parser's
+            # mode: :native takes. Atoms::Base#parse must NOT carry
+            # this lane: it ignores :mode, so it would silently run
+            # the pure-Ruby VM and its interpreter fallback (the
+            # 5-7x real-document gap in #162, and the root-position
+            # diagnostics in #171 before the wire fallback).
+            #
+            # Dynamic-flagged artifacts parse natively too (rs#204):
+            # the state atoms (StateSet/StateMatch/StateSwitch)
+            # evaluate in the engine, riding the capture store's
+            # rollback discipline. Reaching a CustomRef aborts the
+            # native pass outright — Native.parse's error path then
+            # re-runs the interpreter, so customs stay Ruby-tier with
+            # identical trees.
+            return Native.parse(root_atom(entry_name), input)
           end
 
           root_atom(entry_name).parse(input)

data/lib/parsanol/parser.rb

@@ -52,7 +52,13 @@ module Parsanol
         define_method(:root) do
           atom = __send__(rule_name)
           skip_name = self.class.skip_rule_name
-          skip_name ? Parsanol::Skip.inject(self, atom, skip_name)[0] : atom
+          if skip_name
+            Parsanol::Skip.inject(self, atom, skip_name,
+                                  captures: self.class.skip_captures,
+                                  whitespace: self.class.skip_whitespace)[0]
+          else
+            atom
+          end
         end
       end
 
@@ -63,6 +69,13 @@ module Parsanol
       # everything it references) builds without injection; it must be
       # non-nullable. Order-independent with +root+.
       #
+      # +capture:+ (parsanol-ruby#180) declares trivia capturers — a
+      # map of rule name to kind label. Units of the skip rule
+      # matching a capturer's leading literal attach to the next
+      # Named capture under +comments:+ (the PARG
+      # `skip = trivia capture: comments` shape) instead of being
+      # discarded; whitespace-shaped units never record.
+      #
       # @example
       #   class MyParser < Parsanol::Parser
       #     rule(:spaces) { match(/[ \t]/).repeat(1) }
@@ -72,8 +85,15 @@ module Parsanol
       #     root :doc
       #   end
       #
-      def skip(rule_name)
+      # @example capturing comments
+      #   rule(:trivia) { spaces | line_comment }
+      #   rule(:line_comment) { str("//") >> match(/[^\n]/).repeat }
+      #   skip :trivia, capture: { line_comment: :line_comment }
+      #
+      def skip(rule_name, capture: nil, whitespace: nil)
         @skip_rule_name = rule_name
+        @skip_captures = capture
+        @skip_whitespace = whitespace
         nil
       end
 
@@ -82,6 +102,18 @@ module Parsanol
       def skip_rule_name
         @skip_rule_name
       end
+
+      # The declared capturer map ({rule => kind label}), or nil —
+      # see {.skip}.
+      def skip_captures
+        @skip_captures
+      end
+
+      # The declared whitespace kind (units matching no capturer
+      # record under it), or nil — see {.skip}.
+      def skip_whitespace
+        @skip_whitespace
+      end
     end
 
     # Delegates matching to the root rule.

data/lib/parsanol/skip.rb

@@ -13,7 +13,15 @@ module Parsanol
     # methods. Returns [injected_root, skip_atom] with the wrapper
     # built and the lint applied, or raises GrammarError for a
     # nullable skip.
-    def self.inject(parser_instance, root_atom, skip_rule_name)
+    #
+    # +captures+ (parsanol-ruby#180) maps capturer rule names to kind
+    # labels (`skip :trivia, capture: { line_comment: :line }`); the
+    # injected wrapper becomes a TriviaCapture whose units matching a
+    # capturer's leading literal attach to the next Named capture
+    # under `comments:` — the PARG `skip = trivia capture: comments`
+    # shape. Whitespace-shaped units never record.
+    def self.inject(parser_instance, root_atom, skip_rule_name, captures: nil,
+                    whitespace: nil)
       skip_atom = parser_instance.__send__(skip_rule_name)
       skip_atom = skip_atom.parslet while skip_atom.is_a?(Atoms::Entity)
 
@@ -23,24 +31,88 @@ module Parsanol
               "empty loops forever at injection points"
       end
 
-      injector = Injector.new(skip_atom)
+      markers = capturer_markers(parser_instance, captures)
+      whitespace_kind = whitespace&.to_sym
+      injector = Injector.new(skip_atom, markers, whitespace_kind)
       [injector.inject(root_atom, entry: true), skip_atom]
     end
 
+    # Derives each capturer's leading literal (the leftmost Str the
+    # rule can start with) and keys the marker table by it — the same
+    # derivation the PARG compiler applies to its skip declaration. A
+    # declared capturer without a derivable, non-blank literal is a
+    # declaration error: its units could never be told apart from
+    # whitespace-shaped trivia and it would silently record nothing.
+    def self.capturer_markers(parser_instance, captures)
+      return nil if captures.nil? || captures.empty?
+
+      captures.each_with_object({}) do |(rule_name, kind), markers|
+        atom = parser_instance.__send__(rule_name)
+        atom = atom.parslet while atom.is_a?(Atoms::Entity)
+        literal = leading_literal(atom)
+        if literal.nil? || literal.strip.empty?
+          raise GrammarError,
+                "skip capturer #{rule_name.inspect} has no leading " \
+                "literal: its units cannot be told apart from " \
+                "whitespace-shaped trivia"
+        end
+        markers[literal] = kind || rule_name
+      end
+    end
+
+    # The leftmost literal the atom can start with, or nil when the
+    # shape is not literal-led (alternatives with disagreeing
+    # branches, regexes, lookaheads).
+    def self.leading_literal(atom)
+      case atom
+      when Atoms::Str then atom.str
+      when Atoms::Sequence
+        atom.parslets.each do |child|
+          literal = leading_literal(child)
+          return literal if literal
+        end
+        nil
+      when Atoms::Alternative
+        first = nil
+        atom.alternatives.each do |branch|
+          literal = leading_literal(branch)
+          return nil if literal.nil? || (first && literal != first)
+
+          first ||= literal
+        end
+        first
+      when Atoms::Repetition
+        atom.min&.positive? ? leading_literal(atom.parslet) : nil
+      when Atoms::Named, Atoms::Capture, Atoms::Ignored, Atoms::Trivia
+        leading_literal(atom.parslet)
+      end
+    end
+
     # The injection walk. Mirrors the PARG compiler's semantics:
     # sequences interleave the wrapper before every child, alternatives
     # and repetitions descend, named captures descend (spans stay
     # clean — the wrapper is Trivia: Ignored + diagnostics
     # transparency), bare terminals gain a leading wrapper, and the
     # entry root takes leading and trailing wrappers.
+    # The walk is a pure function of the atom subtree, so results are
+    # memoized: dense cross-referencing grammars (expressir's ~250
+    # rules) revisit shared subtrees once per referencing path, which
+    # made the path-scoped visited-set exponential — a 25-rule
+    # Fibonacci-shaped grammar took 6.2s to build (parsanol-ruby#180
+    # side observation). In-progress atoms mark a cycle re-entry and
+    # return raw, exactly as the path-scoped set did.
     class Injector
-      def initialize(skip_atom)
+      def initialize(skip_atom, markers = nil, whitespace_kind = nil)
         @skip_atom = skip_atom
+        @markers = markers
+        @whitespace_kind = whitespace_kind
         @wrapper_ids = {}.compare_by_identity
+        @memo = {}.compare_by_identity
+        @in_progress = {}.compare_by_identity
       end
 
       def inject(atom, entry: false)
-        injected = walk(atom, {})
+        injected = walk(atom)
         return injected unless entry
 
         Atoms::Sequence.new(maybe, injected, maybe)
@@ -50,7 +122,14 @@ module Parsanol
 
       def maybe
         @maybe ||= begin
-          wrapper = Atoms::Trivia.new(Atoms::Repetition.new(@skip_atom, 0, 1))
+          inner = Atoms::Repetition.new(@skip_atom, 0, 1)
+          wrapper =
+            if @markers || @whitespace_kind
+              Atoms::TriviaCapture.new(inner, @markers,
+                                       whitespace_kind: @whitespace_kind)
+            else
+              Atoms::Trivia.new(inner)
+            end
           @wrapper_ids[wrapper] = true
           wrapper
         end
@@ -60,50 +139,56 @@ module Parsanol
         @wrapper_ids.key?(atom)
       end
 
-      def walk(atom, visited)
-        return atom if visited.key?(atom)
+      def walk(atom)
+        cached = @memo[atom]
+        return cached if cached
+        return atom if @in_progress.key?(atom)
         return atom if atom.equal?(@skip_atom)
 
-        visited = visited.merge(atom => true)
-        case atom
-        when Atoms::Sequence
-          out = []
-          atom.parslets.each do |child|
-            injected = walk(child, visited)
-            out << maybe unless wrapper?(injected) || injected.is_a?(Atoms::Trivia)
-            out << injected
-          end
-          out.length == 1 ? out.first : Atoms::Sequence.new(*out)
-        when Atoms::Alternative
-          Atoms::Alternative.new(*atom.alternatives.map { |a| walk(a, visited) })
-        when Atoms::Repetition
-          if wrapper?(atom)
-            atom
+        @in_progress[atom] = true
+        injected =
+          case atom
+          when Atoms::Sequence
+            out = []
+            atom.parslets.each do |child|
+              injected = walk(child)
+              out << maybe unless wrapper?(injected) || injected.is_a?(Atoms::Trivia)
+              out << injected
+            end
+            out.length == 1 ? out.first : Atoms::Sequence.new(*out)
+          when Atoms::Alternative
+            Atoms::Alternative.new(*atom.alternatives.map { |a| walk(a) })
+          when Atoms::Repetition
+            if wrapper?(atom)
+              atom
+            else
+              Atoms::Repetition.new(walk(atom.parslet), atom.min, atom.max,
+                                    atom.result_tag)
+            end
+          when Atoms::Named
+            Atoms::Named.new(walk(atom.parslet), atom.name)
+          when Atoms::Lookahead
+            Atoms::Lookahead.new(walk(atom.bound_parslet), atom.positive)
+          when Atoms::Ignored
+            Atoms::Ignored.new(walk(atom.wrapped_atom))
+          when Atoms::Entity
+            # Rule bodies own their injection when the grammar declares
+            # skip on every rule (PARG); in the DSL surface the root tree
+            # is walked once, so descend through resolved bodies with the
+            # identity visited-set guarding recursion.
+            begin
+              walk(atom.parslet)
+            rescue StandardError
+              atom
+            end
+          when Atoms::Str, Atoms::Re
+            Atoms::Sequence.new(maybe, atom)
           else
-            Atoms::Repetition.new(walk(atom.parslet, visited), atom.min, atom.max,
-                                  atom.result_tag)
-          end
-        when Atoms::Named
-          Atoms::Named.new(walk(atom.parslet, visited), atom.name)
-        when Atoms::Lookahead
-          Atoms::Lookahead.new(walk(atom.bound_parslet, visited), atom.positive)
-        when Atoms::Ignored
-          Atoms::Ignored.new(walk(atom.wrapped_atom, visited))
-        when Atoms::Entity
-          # Rule bodies own their injection when the grammar declares
-          # skip on every rule (PARG); in the DSL surface the root tree
-          # is walked once, so descend through resolved bodies with the
-          # identity visited-set guarding recursion.
-          begin
-            walk(atom.parslet, visited)
-          rescue StandardError
             atom
           end
-        when Atoms::Str, Atoms::Re
-          Atoms::Sequence.new(maybe, atom)
-        else
-          atom
-        end
+        @in_progress.delete(atom)
+        @memo[atom] = injected
+        injected
       end
     end
   end

data/lib/parsanol/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Parsanol
-  VERSION = "1.3.80"
+  VERSION = "1.3.82"
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: parsanol
 version: !ruby/object:Gem::Version
-  version: 1.3.80
+  version: 1.3.82
 platform: ruby
 authors:
 - Ribose Inc.