parsanol 1.3.81 → 1.3.82

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 2e47e2e84e0ec0b80a3091242ecab779b2944849f9311878ff9c08450eddbbd2
-  data.tar.gz: c3643004c06fdaa151824e59036e2212c73536a99a4cf33edfcb8691a705a3ea
+  metadata.gz: c56435897ed6f65bc101a9df1cdb0c0510917824295fa003893cdf7e447a32a4
+  data.tar.gz: 8308c7b4bc8179fbf7867ab1769c53b1f6759ce6ad8cd7ca2dc9362e8c5a3c58
 SHA512:
-  metadata.gz: 66c16945fad70ded8871867dd8ba6b7d6eb82b31d29279369dcaec3e1d9d88e3bfb2c2a4efd97d14245feb76c8b945beba6f9ac130f9033d39dc72a0ef697e80
-  data.tar.gz: 236eb7a3645fab92e8d59fe50f2027669c86009ebcca81e559a1e46c092da4bdb8184dcf3844e2370b5327646065818783fb27fbeb46f834daa63a51c96ee28b
+  metadata.gz: 7ad6958bee6a30a11ce020e95a1b084bd26155f7293c2c127c2ff264850da1df22bbde4eb1f321f8e442448492d5612adc25fc3ccd19fe8a65b3e9ea7a2f9731
+  data.tar.gz: 4e353b492c5113f197822c528bf37622b54bc6eae4d2a4bb73f171cbce7edadd00c96ed55928eda73a9d2746d68e14fb4fd212ec2c768bca30a733123a32c1c8

data/Cargo.lock

@@ -307,9 +307,9 @@ checksum = "9f7c3e4beb33f85d45ae3e3a1792185706c8e16d043238c593331cc7cd313b50"
 
 [[package]]
 name = "parsanol"
-version = "0.17.3"
+version = "0.18.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "43bb64bc897b111cd43a78a4319e8a5517f57f049778de9c6534afb726a2acf4"
+checksum = "dc899494d6b94ea8df14279b7dcbac89d59ba4d1be52f7b8ffe840674c6c8e3c"
 dependencies = [
  "ahash",
  "getrandom 0.3.4",
@@ -326,9 +326,9 @@ dependencies = [
 
 [[package]]
 name = "parsanol-derive"
-version = "0.17.3"
+version = "0.18.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "eb9a1b9a01344027ee6cc33bebc5105c6ff9b8a852378de00178ab7fbed9667b"
+checksum = "b437046956f356eee0eb9b407ff1f1f6286af1f70f1af1d305c54a64c65b64e2"
 dependencies = [
  "proc-macro2",
  "quote",

data/ext/parsanol_cdylib/Cargo.toml

@@ -13,4 +13,4 @@ doc = false
 
 [dependencies]
 # 0.17.0 carries LookSpec + the widened opaque writeback
-parsanol = { version = "0.17.3" }
+parsanol = { version = "0.18.0" }

data/ext/parsanol_native/Cargo.toml

@@ -32,7 +32,7 @@ magnus = { version = "0.9" }
 # parsanol parser library (crates.io; 0.17.0 carries the LookSpec regex
 # lookbehind (rs#163 gap 1) and the widened opaque writeback: the
 # lazy-continuation fix, parsanol-ruby#160)
-parsanol = { version = "0.17.3", features = ["ruby"] }
+parsanol = { version = "0.18.0", features = ["ruby"] }
 
 # Logging
 log = "0.4"

data/lib/parsanol/atoms/sequence.rb

@@ -18,14 +18,25 @@ module Parsanol
       def initialize(*components)
         super()
         @parslets = components
-
-        # Pre-built error message
-        @fail_msg = "Failed to match sequence (#{inspect})"
+        # The failure message renders LAZILY (error_msgs): constructing
+        # sequences is hot-path work for the builder/injector, and
+        # stringifying the whole subtree eagerly is exponential on
+        # shared cross-referencing grammars (parsanol-ruby#180 side
+        # observation: a 25-rule Fibonacci-shaped grammar took 6s).
+        @fail_msg = nil
       end
 
       # Error messages hash (for compatibility)
       def error_msgs
-        { failed: @fail_msg }
+        { failed: fail_msg }
+      end
+
+      # Rendered lazily (see initialize); construction is hot-path
+      # builder/injector work and stringifying the whole subtree
+      # eagerly is exponential on shared cross-referencing grammars
+      # (parsanol-ruby#180 side observation).
+      def fail_msg
+        @fail_msg ||= "Failed to match sequence (#{inspect})"
       end
 
       # Appends a parser to this sequence with flattening.
@@ -104,7 +115,7 @@ module Parsanol
       # Single element sequence
       def match_single(parser, source, context, consume_all)
         success, value = parser.apply(source, context, consume_all)
-        return context.err(self, source, @fail_msg, [value]) unless success
+        return context.err(self, source, fail_msg, [value]) unless success
 
         ok([:sequence, value])
       end
@@ -112,10 +123,10 @@ module Parsanol
       # Two-element sequence
       def match_pair(p1, p2, source, context, consume_all)
         success, v1 = p1.apply(source, context, false)
-        return context.err(self, source, @fail_msg, [v1]) unless success
+        return context.err(self, source, fail_msg, [v1]) unless success
 
         success, v2 = p2.apply(source, context, consume_all)
-        return context.err(self, source, @fail_msg, [v2]) unless success
+        return context.err(self, source, fail_msg, [v2]) unless success
 
         ok([:sequence, v1, v2])
       end
@@ -123,13 +134,13 @@ module Parsanol
       # Three-element sequence
       def match_triple(p1, p2, p3, source, context, consume_all)
         success, v1 = p1.apply(source, context, false)
-        return context.err(self, source, @fail_msg, [v1]) unless success
+        return context.err(self, source, fail_msg, [v1]) unless success
 
         success, v2 = p2.apply(source, context, false)
-        return context.err(self, source, @fail_msg, [v2]) unless success
+        return context.err(self, source, fail_msg, [v2]) unless success
 
         success, v3 = p3.apply(source, context, consume_all)
-        return context.err(self, source, @fail_msg, [v3]) unless success
+        return context.err(self, source, fail_msg, [v3]) unless success
 
         ok([:sequence, v1, v2, v3])
       end
@@ -148,7 +159,7 @@ module Parsanol
           success, value = components[idx].apply(source, context, must_consume)
 
           unless success
-            return context.err(self, source, @fail_msg, [value])
+            return context.err(self, source, fail_msg, [value])
           end
 
           result[idx + 1] = value

data/lib/parsanol/atoms/trivia.rb

@@ -28,12 +28,16 @@ module Parsanol
       # +capturers+ maps leading-literal marker -> kind label (the
       # compiler derives markers from each capturer rule's first Str
       # atom, e.g. "//" => :line_comment); units not matching any
-      # marker are whitespace-shaped and stay unrecorded.
-      attr_reader :capturers
+      # marker are whitespace-shaped and stay unrecorded — unless
+      # +whitespace_kind+ is declared (parsanol-ruby#180): every
+      # unit then records under that label, giving source-preserving
+      # grammars the full ordered trivia stream.
+      attr_reader :capturers, :whitespace_kind
 
-      def initialize(atom, capturers)
+      def initialize(atom, capturers, whitespace_kind: nil)
         super(atom)
         @capturers = capturers
+        @whitespace_kind = whitespace_kind
       end
 
       def apply(source, context, consume_all)
@@ -55,16 +59,16 @@ module Parsanol
       private
 
       def record(source, context, start_pos, end_pos)
-        return unless @capturers
-
         text = source.input[start_pos...end_pos]
         # A trivia unit may lead with whitespace before the comment
         # shape; markers test the comment head.
         stripped = text.lstrip
-        marker, label = @capturers.find { |m, _label| stripped.start_with?(m) }
-        return unless marker
-
-        context.push_captured_trivia(label, stripped.strip)
+        marker, label = @capturers&.find { |m, _label| stripped.start_with?(m) }
+        if marker
+          context.push_captured_trivia(label, stripped.strip)
+        elsif @whitespace_kind
+          context.push_captured_trivia(@whitespace_kind, text)
+        end
       end
     end
   end

data/lib/parsanol/native/serializer.rb

@@ -332,12 +332,16 @@ module Parsanol
 
     def serialize_trivia_capture(atom)
       inner_id = serialize_atom(atom.wrapped_atom)
-      {
-        "TriviaCapture" => {
-          "atom" => inner_id,
-          "rules" => atom.capturers.map { |marker, label| [marker, label.to_s] },
-        },
+      payload = {
+        "atom" => inner_id,
+        "rules" => atom.capturers.map { |marker, label| [marker, label.to_s] },
       }
+      # parsanol-ruby#180: the optional whitespace kind — units that
+      # match no marker record under it (source-preserving grammars).
+      # Optional on the wire: engines predating it deserialize and
+      # simply don't record whitespace units (documented caveat).
+      payload["whitespace"] = atom.whitespace_kind.to_s if atom.whitespace_kind
+      { "TriviaCapture" => payload }
     end
 
     def serialize_trivia(atom)

data/lib/parsanol/parser.rb

@@ -54,7 +54,8 @@ module Parsanol
           skip_name = self.class.skip_rule_name
           if skip_name
             Parsanol::Skip.inject(self, atom, skip_name,
-                                  captures: self.class.skip_captures)[0]
+                                  captures: self.class.skip_captures,
+                                  whitespace: self.class.skip_whitespace)[0]
           else
             atom
           end
@@ -89,9 +90,10 @@ module Parsanol
       #   rule(:line_comment) { str("//") >> match(/[^\n]/).repeat }
       #   skip :trivia, capture: { line_comment: :line_comment }
       #
-      def skip(rule_name, capture: nil)
+      def skip(rule_name, capture: nil, whitespace: nil)
         @skip_rule_name = rule_name
         @skip_captures = capture
+        @skip_whitespace = whitespace
         nil
       end
 
@@ -106,6 +108,12 @@ module Parsanol
       def skip_captures
         @skip_captures
       end
+
+      # The declared whitespace kind (units matching no capturer
+      # record under it), or nil — see {.skip}.
+      def skip_whitespace
+        @skip_whitespace
+      end
     end
 
     # Delegates matching to the root rule.

data/lib/parsanol/skip.rb

@@ -20,7 +20,8 @@ module Parsanol
     # capturer's leading literal attach to the next Named capture
     # under `comments:` — the PARG `skip = trivia capture: comments`
     # shape. Whitespace-shaped units never record.
-    def self.inject(parser_instance, root_atom, skip_rule_name, captures: nil)
+    def self.inject(parser_instance, root_atom, skip_rule_name, captures: nil,
+                    whitespace: nil)
       skip_atom = parser_instance.__send__(skip_rule_name)
       skip_atom = skip_atom.parslet while skip_atom.is_a?(Atoms::Entity)
 
@@ -31,7 +32,8 @@ module Parsanol
       end
 
       markers = capturer_markers(parser_instance, captures)
-      injector = Injector.new(skip_atom, markers)
+      whitespace_kind = whitespace&.to_sym
+      injector = Injector.new(skip_atom, markers, whitespace_kind)
       [injector.inject(root_atom, entry: true), skip_atom]
     end
 
@@ -92,15 +94,25 @@ module Parsanol
     # clean — the wrapper is Trivia: Ignored + diagnostics
     # transparency), bare terminals gain a leading wrapper, and the
     # entry root takes leading and trailing wrappers.
+    # The walk is a pure function of the atom subtree, so results are
+    # memoized: dense cross-referencing grammars (expressir's ~250
+    # rules) revisit shared subtrees once per referencing path, which
+    # made the path-scoped visited-set exponential — a 25-rule
+    # Fibonacci-shaped grammar took 6.2s to build (parsanol-ruby#180
+    # side observation). In-progress atoms mark a cycle re-entry and
+    # return raw, exactly as the path-scoped set did.
     class Injector
-      def initialize(skip_atom, markers = nil)
+      def initialize(skip_atom, markers = nil, whitespace_kind = nil)
         @skip_atom = skip_atom
         @markers = markers
+        @whitespace_kind = whitespace_kind
         @wrapper_ids = {}.compare_by_identity
+        @memo = {}.compare_by_identity
+        @in_progress = {}.compare_by_identity
       end
 
       def inject(atom, entry: false)
-        injected = walk(atom, {})
+        injected = walk(atom)
         return injected unless entry
 
         Atoms::Sequence.new(maybe, injected, maybe)
@@ -112,8 +124,9 @@ module Parsanol
         @maybe ||= begin
           inner = Atoms::Repetition.new(@skip_atom, 0, 1)
           wrapper =
-            if @markers
-              Atoms::TriviaCapture.new(inner, @markers)
+            if @markers || @whitespace_kind
+              Atoms::TriviaCapture.new(inner, @markers,
+                                       whitespace_kind: @whitespace_kind)
             else
               Atoms::Trivia.new(inner)
             end
@@ -126,50 +139,56 @@ module Parsanol
         @wrapper_ids.key?(atom)
       end
 
-      def walk(atom, visited)
-        return atom if visited.key?(atom)
+      def walk(atom)
+        cached = @memo[atom]
+        return cached if cached
+        return atom if @in_progress.key?(atom)
         return atom if atom.equal?(@skip_atom)
 
-        visited = visited.merge(atom => true)
-        case atom
-        when Atoms::Sequence
-          out = []
-          atom.parslets.each do |child|
-            injected = walk(child, visited)
-            out << maybe unless wrapper?(injected) || injected.is_a?(Atoms::Trivia)
-            out << injected
-          end
-          out.length == 1 ? out.first : Atoms::Sequence.new(*out)
-        when Atoms::Alternative
-          Atoms::Alternative.new(*atom.alternatives.map { |a| walk(a, visited) })
-        when Atoms::Repetition
-          if wrapper?(atom)
-            atom
+        @in_progress[atom] = true
+        injected =
+          case atom
+          when Atoms::Sequence
+            out = []
+            atom.parslets.each do |child|
+              injected = walk(child)
+              out << maybe unless wrapper?(injected) || injected.is_a?(Atoms::Trivia)
+              out << injected
+            end
+            out.length == 1 ? out.first : Atoms::Sequence.new(*out)
+          when Atoms::Alternative
+            Atoms::Alternative.new(*atom.alternatives.map { |a| walk(a) })
+          when Atoms::Repetition
+            if wrapper?(atom)
+              atom
+            else
+              Atoms::Repetition.new(walk(atom.parslet), atom.min, atom.max,
+                                    atom.result_tag)
+            end
+          when Atoms::Named
+            Atoms::Named.new(walk(atom.parslet), atom.name)
+          when Atoms::Lookahead
+            Atoms::Lookahead.new(walk(atom.bound_parslet), atom.positive)
+          when Atoms::Ignored
+            Atoms::Ignored.new(walk(atom.wrapped_atom))
+          when Atoms::Entity
+            # Rule bodies own their injection when the grammar declares
+            # skip on every rule (PARG); in the DSL surface the root tree
+            # is walked once, so descend through resolved bodies with the
+            # identity visited-set guarding recursion.
+            begin
+              walk(atom.parslet)
+            rescue StandardError
+              atom
+            end
+          when Atoms::Str, Atoms::Re
+            Atoms::Sequence.new(maybe, atom)
           else
-            Atoms::Repetition.new(walk(atom.parslet, visited), atom.min, atom.max,
-                                  atom.result_tag)
-          end
-        when Atoms::Named
-          Atoms::Named.new(walk(atom.parslet, visited), atom.name)
-        when Atoms::Lookahead
-          Atoms::Lookahead.new(walk(atom.bound_parslet, visited), atom.positive)
-        when Atoms::Ignored
-          Atoms::Ignored.new(walk(atom.wrapped_atom, visited))
-        when Atoms::Entity
-          # Rule bodies own their injection when the grammar declares
-          # skip on every rule (PARG); in the DSL surface the root tree
-          # is walked once, so descend through resolved bodies with the
-          # identity visited-set guarding recursion.
-          begin
-            walk(atom.parslet, visited)
-          rescue StandardError
             atom
           end
-        when Atoms::Str, Atoms::Re
-          Atoms::Sequence.new(maybe, atom)
-        else
-          atom
-        end
+        @in_progress.delete(atom)
+        @memo[atom] = injected
+        injected
       end
     end
   end

data/lib/parsanol/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Parsanol
-  VERSION = "1.3.81"
+  VERSION = "1.3.82"
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: parsanol
 version: !ruby/object:Gem::Version
-  version: 1.3.81
+  version: 1.3.82
 platform: ruby
 authors:
 - Ribose Inc.