parsanol 1.3.70 → 1.3.72

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 46a9a34dd51e63ba24ee155131eb13326947d08cf08a9814a6d600046aead88a
-  data.tar.gz: 679d459fa6c46a6b27816c042af13b0c6e9f15ccae847786140c473731b25ebd
+  metadata.gz: bd9773776674bd865171d5b4b378628a063b0dee92e2a4442fcf3fe0182262da
+  data.tar.gz: 6e8554e163d07070a54b9255d6bed7ca348fd72fd6a8968a17308814323d0d4c
 SHA512:
-  metadata.gz: 5bddc1e0b863c8f50be6573f29e09b17091c95dbfd0db57c660aea348d090b681bb7d99e3870276f3b05f32fcc1944098f71777bdaf4db2b34f348c1e847fbaa
-  data.tar.gz: bbb0ce39775917e628fcc19a04d15e2f252437d6fdcfe9b9b69f33711e096b2cb9a2b9a40908177068f66e54c3dc9ceb4d4ba25396b4cfc0ac465de34667b65d
+  metadata.gz: fa0c3526c5ab79f7d9b6ed08e7523738cf6b46e2b8a6c30e2a021845d6bdeeabd5bb3e7ddd4d31f36725c1682ad94e650975289889c1bc991364d48bd9e2c8d7
+  data.tar.gz: ea7fd569d5a432d56583d76ec896c5a0c03ab8b769d9af7863041f02e64343ce1280ee226f450efde730bcb98bb4a0391fb7fcd03d3f490c7e35b133f6f427e1

data/Cargo.lock

@@ -307,34 +307,16 @@ checksum = "9f7c3e4beb33f85d45ae3e3a1792185706c8e16d043238c593331cc7cd313b50"
 
 [[package]]
 name = "parsanol"
-version = "0.10.2"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "dd43c104276def3f79a54ce6e8d26642ea330c13f97d66380bd94fc057f0a849"
-dependencies = [
- "ahash",
- "getrandom 0.3.4",
- "getrandom 0.4.3",
- "hashbrown",
- "parsanol-derive 0.10.2",
- "regex",
- "regex-syntax",
- "serde",
- "serde_json",
- "sha2",
-]
-
-[[package]]
-name = "parsanol"
-version = "0.12.0"
+version = "0.13.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "b3754baf8f7f9761748bac497cd60efd1bab93ff54c4212e899e27cf3f09100a"
+checksum = "126b715ffe53b3bc75ed8ff753333e350d51aaea64a0a51a6eb8588f0786d925"
 dependencies = [
  "ahash",
  "getrandom 0.3.4",
  "getrandom 0.4.3",
  "hashbrown",
  "magnus",
- "parsanol-derive 0.12.0",
+ "parsanol-derive",
  "regex",
  "regex-syntax",
  "serde",
@@ -344,20 +326,9 @@ dependencies = [
 
 [[package]]
 name = "parsanol-derive"
-version = "0.10.2"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "eca430ba21d7cc2d674080d40dd37a5567e1a0acdc37faef9a9b3738b8651dd6"
-dependencies = [
- "proc-macro2",
- "quote",
- "syn 3.0.6",
-]
-
-[[package]]
-name = "parsanol-derive"
-version = "0.12.0"
+version = "0.13.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "7cdc8438988a014ec5a7cb6d7891c3c467154de134b353e8790bd42d4ae2ddd5"
+checksum = "6f1413076b79ef60c0af56973f8bbfddcc6034bd3cb47fbc5205fcd188bc8d79"
 dependencies = [
  "proc-macro2",
  "quote",
@@ -368,7 +339,7 @@ dependencies = [
 name = "parsanol_cdylib"
 version = "1.3.52"
 dependencies = [
- "parsanol 0.10.2",
+ "parsanol",
 ]
 
 [[package]]
@@ -377,7 +348,7 @@ version = "1.0.2"
 dependencies = [
  "log",
  "magnus",
- "parsanol 0.12.0",
+ "parsanol",
  "rb-sys",
 ]
 

data/ext/parsanol_cdylib/Cargo.toml

@@ -12,4 +12,5 @@ crate-type = ["cdylib"]
 doc = false
 
 [dependencies]
-parsanol = { version = "0.10" }
+# 0.13 carries rs#174 + rs#172 (Trivia diagnostics parity)
+parsanol = { version = "0.13" }

data/ext/parsanol_native/Cargo.toml

@@ -29,7 +29,9 @@ magnus = { version = "0.9" }
 
 # parsanol parser library (crates.io; 0.12 carries the rs#174 native
 # backtracking fix: dynamic-dependent memoization + outcome cache)
-parsanol = { version = "0.12.0", features = ["ruby"] }
+# parsanol parser library (crates.io; 0.13 carries the rs#172 Trivia
+# atom: native diagnostics skip trivia positions, matching ruby #145)
+parsanol = { version = "0.13.0", features = ["ruby"] }
 
 # Logging
 log = "0.4"

data/lib/parsanol/atoms/context.rb

@@ -67,6 +67,11 @@ module Parsanol
         # Injected-trivia depth: while positive, reported causes are
         # marked trivia and never surface in diagnostics.
         @trivia_depth = 0
+        # Captured trivia channel (parsanol-ruby#152): when the grammar
+        # declares `skip ... capture:`, matched trivia units land here
+        # and the NEXT successful Named capture attaches (and drains)
+        # them under `comments:`.
+        @pending_trivia = []
         @evict_threshold = 200
         @evict_counter = 0
         @evict_interval = 100
@@ -258,6 +263,34 @@ module Parsanol
         [false, @reporter.err(*)]
       end
 
+      # Records one captured trivia unit (kind, text) for attachment to
+      # the next successful Named capture.
+      def push_captured_trivia(kind, text)
+        @pending_trivia << { kind: kind, text: text }
+        nil
+      end
+
+      # Snapshot for backtracking: a failed branch must not leak its
+      # half-recorded trivia into the successful alternative.
+      def take_pending_trivia_snapshot
+        @pending_trivia.dup
+      end
+
+      def restore_pending_trivia(snapshot)
+        @pending_trivia = snapshot
+        nil
+      end
+
+      # Drains pending captured trivia (attached by the next
+      # successful Named capture). Backtracking discards whatever has
+      # not been attached yet — the caller snapshots position and
+      # restores this list alongside.
+      def take_pending_trivia
+        list = @pending_trivia
+        @pending_trivia = []
+        list
+      end
+
       # Brackets a parse inside injected trivia: causes reported while
       # the depth is positive carry the trivia flag and never surface
       # in rendered trees or deepest-failure positions.

data/lib/parsanol/atoms/named.rb

@@ -32,10 +32,32 @@ module Parsanol
       # @param consume_all [Boolean] require full consumption
       # @return [Array(Boolean, Object)] result
       def apply(source, context, consume_all)
+        trivia_before = context.take_pending_trivia_snapshot
         success, value = @parslet.apply(source, context, consume_all)
-        return [false, value] unless success
+        unless success
+          context.restore_pending_trivia(trivia_before)
+          return [false, value]
+        end
 
-        ok(wrap_result(value))
+        comments = context.take_pending_trivia
+        result = wrap_result(value)
+        result = attach_comments(result, comments) unless comments.empty?
+        ok(result)
+      end
+
+      private
+
+      # parsanol-ruby#152: trivia captured ahead of this capture rides
+      # with it under `comments:`. The attachment only fires when the
+      # grammar declared capture AND trivia actually preceded — key
+      # absence everywhere else keeps v1 trees byte-identical.
+      def attach_comments(result, comments)
+        list = comments.map { |unit| { unit[:kind] => unit[:text] } }
+        if result.is_a?(Hash)
+          result.merge(comments: list)
+        else
+          { @name => result, comments: list }
+        end
       end
 
       # Named wrappers skip caching (inner parser handles it).
@@ -60,8 +82,6 @@ module Parsanol
         @parslet.first_set
       end
 
-      private
-
       # Wraps matched value in labeled hash.
       #
       # @param matched [Object] matched value

data/lib/parsanol/atoms/trivia.rb

@@ -17,5 +17,53 @@ module Parsanol
         "trivia(#{@wrapped_atom.to_s(prec)})"
       end
     end
+
+    # parsanol-ruby#152: the skip-declared capture wrapper. Whitespace
+    # trivia stays Ignored; comment-shaped units (the grammar names the
+    # capture rules in the declaration) are recorded into the context
+    # channel for attachment to the next successful Named capture.
+    # Span discipline is unchanged: the matched bytes still contribute
+    # nothing to values or capture spans.
+    class TriviaCapture < Trivia
+      # +capturers+ maps leading-literal marker -> kind label (the
+      # compiler derives markers from each capturer rule's first Str
+      # atom, e.g. "//" => :line_comment); units not matching any
+      # marker are whitespace-shaped and stay unrecorded.
+      def initialize(atom, capturers)
+        super(atom)
+        @capturers = capturers
+      end
+
+      def apply(source, context, consume_all)
+        context.with_trivia do
+          start = source.bytepos
+          ok, _value = @wrapped_atom.apply(source, context, consume_all)
+          if ok && source.bytepos > start
+            record(source, context, start, source.bytepos)
+            return ok(nil)
+          end
+          [ok, _value]
+        end
+      end
+
+      def to_s_inner(_prec)
+        "trivia!(@wrapped_atom.to_s(prec))"
+      end
+
+      private
+
+      def record(source, context, start_pos, end_pos)
+        return unless @capturers
+
+        text = source.input[start_pos...end_pos]
+        # A trivia unit may lead with whitespace before the comment
+        # shape; markers test the comment head.
+        stripped = text.lstrip
+        marker, label = @capturers.find { |m, _label| stripped.start_with?(m) }
+        return unless marker
+
+        context.push_captured_trivia(label, stripped.strip)
+      end
+    end
   end
 end

data/lib/parsanol/native/serializer.rb

@@ -78,6 +78,13 @@ module Parsanol
                      serialize_custom_ref(atom)
                    when Parsanol::Atoms::Dynamic
                      serialize_dynamic(atom)
+                   when Parsanol::Atoms::Trivia
+                     # Wire-distinct from Ignore (rs#172): the Rust
+                     # engine suppresses trivia-position diagnostics by
+                     # atom kind. Engines predating the Trivia tag
+                     # reject skip-carrying artifacts loudly (unknown
+                     # variant), which is the declared gate.
+                     serialize_trivia(atom)
                    when Parsanol::Atoms::Ignored
                      serialize_ignored(atom)
                    else
@@ -276,6 +283,15 @@ module Parsanol
             "parse this grammar with mode: :ruby"
     end
 
+    def serialize_trivia(atom)
+      inner_id = serialize_atom(atom.wrapped_atom)
+      {
+        "Trivia" => {
+          "atom" => inner_id,
+        },
+      }
+    end
+
     def serialize_ignored(atom)
       inner_id = serialize_atom(atom.wrapped_atom)
       {

data/lib/parsanol/parg/artifact.rb

@@ -71,7 +71,10 @@ module Parsanol
 
           root_atom(entry_name).parse(input)
         when :ruby
-          root_atom(entry_name).parse(input)
+          # Deepest-failure reporting parity with the native engine: the
+          # default Tree reporter keeps the most recent (root) failure,
+          # which reports position 1:1 for grammars with entry wrappers.
+          root_atom(entry_name).parse(input, reporter: ErrorReporter::Deepest.new)
         else
           raise ArgumentError, "unknown mode #{mode.inspect} (use :native or :ruby)"
         end

data/lib/parsanol/parg/compiler.rb

@@ -141,6 +141,51 @@ module Parsanol
         end
       end
 
+      # parsanol-ruby#152: capturer rules are the rules transitively
+      # referenced by the skip declaration whose leading literal is a
+      # non-whitespace marker — the comment shapes ("//", "/*", ...).
+      # Whitespace rules stay unrecorded trivia.
+      def skip_capturer_markers
+        @skip_capturer_markers ||= begin
+          markers = {}
+          seen = {}
+          collect = lambda do |node|
+            case node.kind
+            when :ref
+              next if seen[node.a]
+
+              seen[node.a] = true
+              literal = leading_literal(node.a)
+              markers[literal] = node.a.to_sym if literal && !literal.strip.empty?
+              collect.call(@document.rules[node.a]) if @document.rules.key?(node.a)
+            when :alt, :seq then node.a.each { |b| collect.call(b) }
+            when :rep, :opt, :cap then collect.call(node.a)
+            when :pred then collect.call(node.b)
+            end
+          end
+          collect.call(@document.skip_declaration)
+          markers
+        end
+      end
+
+      def leading_literal(rule_name)
+        node = @document.rules[rule_name]
+        return nil unless node
+
+        literal = nil
+        walk = lambda do |n|
+          break if literal
+
+          case n.kind
+          when :lit then literal = n.a
+          when :seq, :alt then n.a.each { |c| walk.call(c) unless literal }
+          when :rep, :opt, :cap, :pred then walk.call(n.a || n.b) unless literal
+          end
+        end
+        walk.call(node)
+        literal
+      end
+
       def skip_wrapper?(atom)
         @skip_wrapper_ids&.key?(atom)
       end
@@ -163,7 +208,13 @@ module Parsanol
           # span-joined captures — and failures inside it never surface
           # in diagnostics (rendered trees and deepest-failure
           # positions skip trivia causes).
-          wrapper = Atoms::Trivia.new(Atoms::Repetition.new(atom, 0, 1))
+          inner = Atoms::Repetition.new(atom, 0, 1)
+          wrapper =
+            if @document.skip_capture_key
+              Atoms::TriviaCapture.new(inner, skip_capturer_markers)
+            else
+              Atoms::Trivia.new(inner)
+            end
           (@skip_wrapper_ids ||= {}.compare_by_identity)[wrapper] = true
           wrapper
         end

data/lib/parsanol/parg/document.rb

@@ -10,7 +10,7 @@ module Parsanol
       Test = Struct.new(:entry, :kind, :input, :expect)
 
       attr_accessor :grammar_name, :version, :source, :uses, :skip_declaration, :skip_source,
-                    :atomic_rules
+                    :atomic_rules, :skip_capture_key
       attr_reader :rules, :bindings, :preprocess, :entries, :docs, :tests, :render, :derive,
                   :customs, :states
       # Entries authored in this file (pre-import); nil-entry tests resolve

data/lib/parsanol/parg/parser.rb

@@ -21,7 +21,7 @@ module Parsanol
       STATE_KEYWORDS = %w[custom state].freeze
       KEYWORDS = %w[grammar as alt from_table column bindings render derive
                     preprocess entry table_lookup custom state set switch
-                    skip until atomic].freeze
+                    skip until atomic capture].freeze
 
       def initialize(text)
         @tokens = Lexer.new(text).tokens
@@ -170,6 +170,15 @@ module Parsanol
             end
 
             node = parse_choice
+            # parsanol-ruby#152 phase 2: `skip = trivia capture:
+            # comments` attaches comment-shaped trivia (the rules the
+            # declaration references, identified by leading literal)
+            # to the next successful capture under `comments:`.
+            if peek&.type == :ident && peek.value == "capture"
+              advance
+              punct(":")
+              document.skip_capture_key = ident.value.to_sym
+            end
             document.skip_declaration = validate_skip_node(node)
             document.skip_source = skip_source_string(node)
             skip_newlines
@@ -260,7 +269,12 @@ module Parsanol
 
       def stop_element?
         peek.nil? || peek.type == :newline ||
-          (peek.type == :punct && ["/", ")", "]", "}"].include?(peek.value))
+          (peek.type == :punct && ["/", ")", "]", "}"].include?(peek.value)) ||
+          # `capture:` suffix terminates the skip declaration's RHS
+          # (parsanol-ruby#152); everywhere else parse_element's guard
+          # rejects it.
+          (peek&.type == :ident && peek.value == "capture" &&
+           @tokens[@pos + 1]&.type == :punct && @tokens[@pos + 1]&.value == ":")
       end
 
       def parse_element
@@ -269,6 +283,13 @@ module Parsanol
           raise ParseError, "unexpected end of input inside sequence"
         end
 
+        # `capture:` is a skip-declaration suffix, never an expression
+        # element (reserved keyword; parsanol-ruby#152).
+        if token.type == :ident && token.value == "capture" &&
+            @tokens[@pos + 1]&.type == :punct && @tokens[@pos + 1]&.value == ":"
+          raise ParseError, "capture: only modifies the skip declaration"
+        end
+
         if token.type == :num
           advance
           min = token.value.to_i

data/lib/parsanol/parser.rb

@@ -49,7 +49,38 @@ module Parsanol
       def root(rule_name)
         # Remove any existing root method before redefining
         undef_method :root if method_defined?(:root)
-        define_method(:root) { __send__(rule_name) }
+        define_method(:root) do
+          atom = __send__(rule_name)
+          skip_name = self.class.skip_rule_name
+          skip_name ? Parsanol::Skip.inject(self, atom, skip_name)[0] : atom
+        end
+      end
+
+      # Declares the grammar's skip rule (parsanol-ruby#134 DSL
+      # surface): an optional match of the named rule is injected
+      # before every terminal of the root's rule tree, and the entry
+      # takes leading and trailing skips. The referenced rule (and
+      # everything it references) builds without injection; it must be
+      # non-nullable. Order-independent with +root+.
+      #
+      # @example
+      #   class MyParser < Parsanol::Parser
+      #     rule(:spaces) { match(/[ \t]/).repeat(1) }
+      #     rule(:word) { match(/[a-z]/).repeat(1).as(:w) }
+      #     rule(:doc) { word >> str(",") >> word }
+      #     skip :spaces
+      #     root :doc
+      #   end
+      #
+      def skip(rule_name)
+        @skip_rule_name = rule_name
+        nil
+      end
+
+      # The declared skip rule name, or nil (class-level; subclasses
+      # inherit unless they declare their own).
+      def skip_rule_name
+        @skip_rule_name
       end
     end
 

data/lib/parsanol/skip.rb

@@ -0,0 +1,110 @@
+# frozen_string_literal: true
+
+module Parsanol
+  # Skip-rule trivia injection for Ruby-DSL grammars (the PARG surface
+  # compiles the same semantics into artifacts; see
+  # Parsanol::PARG::Compiler#inject_skip_trivia). A parser declares
+  # `skip :trivia`; the resolved rule's atom becomes the injected
+  # optional trivia wrapper before every terminal of every rule body,
+  # with the declaration's transitive closure exempt (the skip atom
+  # must not contain itself).
+  module Skip
+    # +skip_rule_name+ resolves against the parser instance's rule
+    # methods. Returns [injected_root, skip_atom] with the wrapper
+    # built and the lint applied, or raises GrammarError for a
+    # nullable skip.
+    def self.inject(parser_instance, root_atom, skip_rule_name)
+      skip_atom = parser_instance.__send__(skip_rule_name)
+      skip_atom = skip_atom.parslet while skip_atom.is_a?(Atoms::Entity)
+
+      if VM::Compiler.new.nullable?(skip_atom)
+        raise GrammarError,
+              "skip rule must be non-nullable: a skip that can match " \
+              "empty loops forever at injection points"
+      end
+
+      injector = Injector.new(skip_atom)
+      [injector.inject(root_atom, entry: true), skip_atom]
+    end
+
+    # The injection walk. Mirrors the PARG compiler's semantics:
+    # sequences interleave the wrapper before every child, alternatives
+    # and repetitions descend, named captures descend (spans stay
+    # clean — the wrapper is Trivia: Ignored + diagnostics
+    # transparency), bare terminals gain a leading wrapper, and the
+    # entry root takes leading and trailing wrappers.
+    class Injector
+      def initialize(skip_atom)
+        @skip_atom = skip_atom
+        @wrapper_ids = {}.compare_by_identity
+      end
+
+      def inject(atom, entry: false)
+        injected = walk(atom, {})
+        return injected unless entry
+
+        Atoms::Sequence.new(maybe, injected, maybe)
+      end
+
+      private
+
+      def maybe
+        @maybe ||= begin
+          wrapper = Atoms::Trivia.new(Atoms::Repetition.new(@skip_atom, 0, 1))
+          @wrapper_ids[wrapper] = true
+          wrapper
+        end
+      end
+
+      def wrapper?(atom)
+        @wrapper_ids.key?(atom)
+      end
+
+      def walk(atom, visited)
+        return atom if visited.key?(atom)
+        return atom if atom.equal?(@skip_atom)
+
+        visited = visited.merge(atom => true)
+        case atom
+        when Atoms::Sequence
+          out = []
+          atom.parslets.each do |child|
+            injected = walk(child, visited)
+            out << maybe unless wrapper?(injected) || injected.is_a?(Atoms::Trivia)
+            out << injected
+          end
+          out.length == 1 ? out.first : Atoms::Sequence.new(*out)
+        when Atoms::Alternative
+          Atoms::Alternative.new(*atom.alternatives.map { |a| walk(a, visited) })
+        when Atoms::Repetition
+          if wrapper?(atom)
+            atom
+          else
+            Atoms::Repetition.new(walk(atom.parslet, visited), atom.min, atom.max,
+                                  atom.result_tag)
+          end
+        when Atoms::Named
+          Atoms::Named.new(walk(atom.parslet, visited), atom.name)
+        when Atoms::Lookahead
+          Atoms::Lookahead.new(walk(atom.bound_parslet, visited), atom.positive)
+        when Atoms::Ignored
+          Atoms::Ignored.new(walk(atom.wrapped_atom, visited))
+        when Atoms::Entity
+          # Rule bodies own their injection when the grammar declares
+          # skip on every rule (PARG); in the DSL surface the root tree
+          # is walked once, so descend through resolved bodies with the
+          # identity visited-set guarding recursion.
+          begin
+            walk(atom.parslet, visited)
+          rescue StandardError
+            atom
+          end
+        when Atoms::Str, Atoms::Re
+          Atoms::Sequence.new(maybe, atom)
+        else
+          atom
+        end
+      end
+    end
+  end
+end

data/lib/parsanol/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Parsanol
-  VERSION = "1.3.70"
+  VERSION = "1.3.72"
 end

data/lib/parsanol/vm.rb

@@ -649,6 +649,12 @@ module Parsanol
           compile_entity(atom, consume_all)
         when Parsanol::Atoms::Lookahead
           compile_lookahead(atom, consume_all)
+        when Parsanol::Atoms::TriviaCapture
+          # Captured trivia records into the context channel at match
+          # time (parsanol-ruby#152); the bytecode program has no
+          # channel, so the interpreter (which does) must own the
+          # grammar. Declining here routes program_for to :fallback.
+          nil
         when Parsanol::Atoms::Ignored
           inner = atom.wrapped_atom
           return nil if inner.nil?

data/lib/parsanol.rb

@@ -290,6 +290,7 @@ module Parsanol
   autoload :Pattern, "parsanol/pattern"
   autoload :Transform, "parsanol/transform"
   autoload :VM, "parsanol/vm"
+  autoload :Skip, "parsanol/skip"
   autoload :Parser, "parsanol/parser"
   autoload :ErrorReporter, "parsanol/error_reporter"
   autoload :Scope, "parsanol/scope"

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: parsanol
 version: !ruby/object:Gem::Version
-  version: 1.3.70
+  version: 1.3.72
 platform: ruby
 authors:
 - Ribose Inc.
@@ -155,6 +155,7 @@ files:
 - lib/parsanol/rig/rspec.rb
 - lib/parsanol/rope.rb
 - lib/parsanol/scope.rb
+- lib/parsanol/skip.rb
 - lib/parsanol/slice.rb
 - lib/parsanol/source.rb
 - lib/parsanol/source/line_cache.rb