parsanol 1.3.36 → 1.3.37

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: dbeca0f263a186fedfaebecb581dc25694d3c3d79e98898db861e1ec18a6296f
-  data.tar.gz: d82324e707dcddd112f535d20988f144981fd3df9a3ff86b9cb812c3cab2816e
+  metadata.gz: f3ee6550442261f9f42d66c0b51e9b5fe5f186c54c9b36a2932abc0410608a42
+  data.tar.gz: 9f0476cfd0d65c1642c81bf86bd36a2a0a6627d1c2978ba65f00989e84e6bd44
 SHA512:
-  metadata.gz: fbe5a790335f4f2fd4c9db8816ea9cf154c71c20bbcb233d8bc4f1665691bbd0fe5fbe00bf6f360afe815c3434ef0e8047f770244403dbc8655f0ab6d7d15a05
-  data.tar.gz: 441073e36388645c1a429aa6c6918051607bdab1b4f00514b2b81f68614d6e33033ae1155efbb7a5d6082809166c3602347ac834505e78fbba22ff0784957c7b
+  metadata.gz: b1dbb402205ebfb234148f7f87b5d9bcf6cd201b7f72f3c73bd096c7a0c3ba092582ad1a37a543f1665fd2542d8b2c6841b5ca9e4f21525a2ecfac68e343bad7
+  data.tar.gz: dc490cc9e39a4feb101139c1fd9a2d47e5509d495cb99920f6856cf65eac3303147e8cb95c13b25f8086d2fc31cb77be5193438527e28d4d261f08f28a7842cb

data/Cargo.lock

@@ -252,8 +252,9 @@ checksum = "9f7c3e4beb33f85d45ae3e3a1792185706c8e16d043238c593331cc7cd313b50"
 
 [[package]]
 name = "parsanol"
-version = "0.7.2"
-source = "git+https://github.com/parsanol/parsanol-rs?branch=main#5f04c5bda75c6f5f3f5d82bd18f30a36488ec1ad"
+version = "0.7.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "1f87f23ea48eb961366d013b90eac82e8f1d6e129d102ab33a7d56f5ec26a883"
 dependencies = [
  "ahash",
  "getrandom 0.3.4",
@@ -269,8 +270,9 @@ dependencies = [
 
 [[package]]
 name = "parsanol-derive"
-version = "0.7.2"
-source = "git+https://github.com/parsanol/parsanol-rs?branch=main#5f04c5bda75c6f5f3f5d82bd18f30a36488ec1ad"
+version = "0.7.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "a4043d26b37aa72b4a8efc02e6dc0a7225cb28e484b07bd290ede35c8a06a566"
 dependencies = [
  "proc-macro2",
  "quote",

data/ext/parsanol_native/Cargo.toml

@@ -28,7 +28,7 @@ rb-sys = { version = "0.9.124", features = ["global-allocator"] }
 magnus = { version = "0.9" }
 
 # parsanol parser library (from git for latest features)
-parsanol = { git = "https://github.com/parsanol/parsanol-rs", branch = "main", features = ["ruby"] }
+parsanol = { version = "0.7.3", features = ["ruby"] }
 
 # Logging
 log = "0.4"

data/lib/parsanol/atoms/base.rb

@@ -39,7 +39,15 @@ module Parsanol
         # grammars transparently.
         if must_consume_all && source.is_a?(String) &&
             (program = VM.program_for(self))
-          result = VM.run_for(self, program, source, true)
+          begin
+            result = VM.run_for(self, program, source, true)
+          rescue ArgumentError, TypeError
+            # Executor crash on a malformed program (GH-68): the
+            # interpreter is the source of truth — fall through and
+            # stop using the VM for this grammar.
+            VM.disable_for!(self)
+            result = VM::BAIL
+          end
           if result == VM::BAIL
             # Internal bail: fall back to the interpreter and skip the VM
             # for this grammar from now on.

data/lib/parsanol/atoms/can_flatten.rb

@@ -216,7 +216,10 @@ module Parsanol
             end
             piece = e.instance_of?(Parsanol::Slice) ? e.content : e.to_s
             if content.nil?
-              content = +piece
+              # Copy: unary + returns self for unfrozen strings, and
+              # appending into an input slice's buffer corrupts it when
+              # the same cached subtree is flattened again (GH-67).
+              content = String.new(piece)
             else
               content << piece
             end

data/lib/parsanol/atoms/re.rb

@@ -1,11 +1,16 @@
 # frozen_string_literal: true
 
-# Regular expression matcher for single characters.
-# Matches one character against a character class pattern.
+# Regular expression matcher.
+#
+# A plain character class matches one character. Like Parslet, a
+# pattern with an embedded quantifier (match('a+') etc.) is a regex
+# matched greedily at the current position: the whole regex match is
+# consumed in one step.
 #
 # @example Character classes
-#   match('[a-z]')  # matches a-z
-#   match('\d')     # matches digits
+#   match('[a-z]')  # matches one a-z character
+#   match('a+')     # matches a maximal run of a (Parslet parity)
+#   match('\d')     # matches one digit
 #   any             # matches any character
 #
 module Parsanol
@@ -20,10 +25,15 @@ module Parsanol
       # Creates a new regex matcher.
       #
       # @param pattern [String, Object] regex character class
+      QUANTIFIER_RE = /[+*?]|\{\d+(?:,\s*\d*)?\}/
+
       def initialize(pattern)
         super()
         @match = pattern.to_s
         @re = Regexp.new(@match, Regexp::MULTILINE)
+        # Embedded quantifier: one greedy regex match instead of a
+        # single character (Parslet parity, GH-69).
+        @quantified = QUANTIFIER_RE.match?(@match)
 
         # Extract pattern for display (strip delimiters)
         @display = @match.inspect[1..-2] || @match
@@ -40,7 +50,19 @@ module Parsanol
       # @param _consume_all [Boolean] ignored
       # @return [Array(Boolean, Object)] result
       def try(source, context, _consume_all)
-        # Fast path: check if next char matches
+        if @quantified
+          # One greedy regex match (Parslet parity): consume the whole
+          # match. A zero-length match is treated as no match so
+          # repetitions always advance.
+          matched = source.match_bytes(@re)
+          return ok(source.consume_bytes(matched)) if matched&.positive?
+
+          return context.err(self, source, @eof_error) if source.chars_left < 1
+
+          return context.err(self, source, @no_match_error)
+        end
+
+        # Fast path: single-character class
         return ok(source.consume(1)) if source.matches?(@re)
 
         # No input left

data/lib/parsanol/native/event_player.rb

@@ -0,0 +1,93 @@
+# frozen_string_literal: true
+
+module Parsanol
+  module Native
+    # Reference consumer for the flat event stream returned by
+    # {Parsanol::Native::Parser.parse_events}: replays the opcode
+    # stream into the same tree Parsanol::Native.parse produces.
+    #
+    # Domain builders (e.g. an EXPRESS model builder) should not
+    # rebuild this tree — they consume the events directly, which is
+    # the point of the format. This player exists to document the
+    # protocol and to conformance-test the stream against the shaped
+    # tree.
+    #
+    # Opcodes (see parsanol-rs portable::events):
+    #   0 NIL                  4 BEGIN_ARR   6 BEGIN_HASH
+    #   1 STR <pool>           5 END_ARR     7 END_HASH
+    #   2 SLICE <offset> <len> 3 KEY <pool>
+    class EventPlayer
+      OP_NIL = 0
+      OP_STR = 1
+      OP_SLICE = 2
+      OP_KEY = 3
+      OP_BEGIN_ARR = 4
+      OP_END_ARR = 5
+      OP_BEGIN_HASH = 6
+      OP_END_HASH = 7
+
+      # One-shot replay: returns the tree for the stream.
+      def self.play(events, strings, input)
+        player = new(events, strings, input)
+        tree = player.read_value
+        unless player.pos == events.length
+          raise ArgumentError, "event stream not fully consumed"
+        end
+
+        tree
+      end
+
+      def initialize(events, strings, input)
+        @events = events
+        @strings = strings
+        @input = input
+        @pos = 0
+      end
+
+      attr_reader :pos
+
+      def read_value
+        op = @events[@pos]
+        @pos += 1
+        case op
+        when OP_NIL then nil
+        when OP_STR
+          value = @strings[@events[@pos]]
+          @pos += 1
+          value
+        when OP_SLICE
+          offset = @events[@pos]
+          length = @events[@pos + 1]
+          @pos += 2
+          ::Parsanol::Slice.new(offset, @input.byteslice(offset, length), @input)
+        when OP_BEGIN_HASH
+          hash = {}
+          until @events[@pos] == OP_END_HASH
+            @pos += 1 # KEY opcode
+            key = symbol(@strings[@events[@pos]])
+            @pos += 1
+            hash[key] = read_value
+          end
+          @pos += 1
+          hash
+        when OP_BEGIN_ARR
+          array = []
+          until @events[@pos] == OP_END_ARR
+            array << read_value
+          end
+          @pos += 1
+          array
+        else
+          raise ArgumentError, "unknown event opcode #{op} at #{@pos - 1}"
+        end
+      end
+
+      private
+
+      def symbol(key)
+        @@symbol_cache ||= {}
+        @@symbol_cache[key] ||= key.to_sym
+      end
+    end
+  end
+end

data/lib/parsanol/native/parser.rb

@@ -48,6 +48,20 @@ module Parsanol
           Parsanol::Native.parse(grammar, input)
         end
 
+        # Parse input with a Ruby grammar, returning the parslet-shaped
+        # AST as a flat event stream [[events], [strings]] — one FFI
+        # return, no per-node Ruby objects. Replay with EventPlayer or
+        # consume the opcodes directly.
+        #
+        # @param grammar [Parsanol::Atoms::Base] Ruby grammar definition
+        # @param input [String] Input string to parse
+        # @return [Array<Array<Integer>, Array<String>>]
+        def parse_events(grammar, input)
+          handle = grammar_handle(grammar)
+          blob, strings = Native._parse_handle_events(handle, input)
+          [blob.unpack("q*"), strings]
+        end
+
         # Serialize a Ruby grammar to JSON (cached).
         def serialize_grammar(root_atom)
           grammar_json(root_atom)

data/lib/parsanol/native.rb

@@ -5,6 +5,7 @@ require "digest"
 
 require "parsanol/native/types"
 require "parsanol/native/parser"
+require "parsanol/native/event_player"
 require "parsanol/native/serializer"
 require "parsanol/native/batch_decoder"
 

data/lib/parsanol/parser.rb

@@ -230,22 +230,25 @@ module Parsanol
       raise
     end
 
-    # Engine selection for the default mode, decided at registration
-    # time (never mid-parse): grammars the Rust backend cannot express
-    # run on the Ruby engine, announced once per grammar — loud, not a
-    # silent parse-time fallback. An explicit mode: :native still raises
-    # UnsupportedGrammar from registration.
+    # Public expressibility check (GH-71): true when the Rust native
+    # backend can run this grammar, false when it must run on the Ruby
+    # engine (Dynamic/custom atoms). Use it to pick an engine
+    # explicitly via parse(input, mode: :native | :ruby).
+    #
+    # The degradation warning prints once per reason per process — not
+    # per parser instance and not per parse.
     def native_expressible?
       Parsanol::Native::Parser.grammar_handle(root)
       true
     rescue Parsanol::Native::UnsupportedGrammar => e
-      warned = (@@unsupported_warned ||= {}.compare_by_identity)
-      unless warned.key?(root)
-        warned[root] = true
+      warned = (@@unsupported_warned ||= {})
+      unless warned.key?(e.message)
+        warned[e.message] = true
         warn "parsanol: parsing with the Ruby engine (#{e.message})"
       end
       false
     end
+    public :native_expressible?
 
     # Feed the native deepest-failure diagnostics to a user-supplied
     # reporter: one err_at event carrying the cause the native engine

data/lib/parsanol/scope.rb

@@ -52,6 +52,11 @@ module Parsanol
         end
       end
 
+      # Chain-aware membership test without raising.
+      def key?(key)
+        @bindings.key?(key) || (@parent_frame&.key?(key) || false)
+      end
+
       # Stores a value in the current frame.
       #
       # @param key [Symbol] the variable name
@@ -79,6 +84,15 @@ module Parsanol
       @active_frame.fetch(key)
     end
 
+    # Chain-aware membership test, mirroring Parslet's captures.key?.
+    # Checks the current frame and all parents without raising.
+    #
+    # @param key [Symbol] the variable name to look up
+    # @return [Boolean] whether any frame binds the key
+    def key?(key)
+      @active_frame.key?(key)
+    end
+
     # Stores a value in the current frame.
     #
     # @param key [Symbol] the variable name

data/lib/parsanol/source.rb

@@ -69,6 +69,59 @@ module Parsanol
     end
     alias match matches?
 
+    # Byte length of the pattern's match at the current position
+    # (StringScanner#match? returns it), or nil when there is no match.
+    #
+    # @param pattern [Regexp] pattern to probe
+    # @return [Integer, nil] matched byte length
+    #
+    def match_bytes(pattern) # rubocop:disable Naming/PredicateMethod -- returns byte length or nil, not a boolean
+      @scanner.match?(pattern)
+    end
+
+    # Consumes n bytes from input and returns them as a pooled Slice.
+    # Companion to #match_bytes for atoms that measure their match in
+    # bytes (quantified regexes).
+    #
+    # @param count [Integer] number of bytes to consume
+    # @return [Parsanol::Slice] slice containing consumed bytes
+    #
+    # The underlying input string. Custom atoms doing position math
+    # (e.g. lookbehind rewinds) need direct access to it (GH-70).
+    #
+
+    # Rewinds n characters from the current position, walking back over
+    # UTF-8 continuation bytes so multibyte input stays aligned. Bounds:
+    # never rewinds past the start of the input.
+    #
+    # @param count [Integer] number of characters to move back
+    # @return [Integer] the new byte position
+    def rewind_chars(count)
+      pos = @scanner.pos
+      remaining = count
+      while remaining.positive? && pos.positive?
+        pos -= 1
+        remaining -= 1 unless (@raw_string.getbyte(pos) & 0xC0) == 0x80
+      end
+      @scanner.pos = pos
+      pos
+    end
+
+    # The underlying input string. Custom atoms doing position math
+    # (e.g. lookbehind rewinds) need direct access to it (GH-70).
+    #
+    # @return [String] the full input
+    def input
+      @raw_string
+    end
+
+    def consume_bytes(count)
+      current_pos = @scanner.pos
+      content = @raw_string.byteslice(current_pos, count)
+      @scanner.pos = current_pos + count
+      @slice_pool.acquire_with(current_pos, content, @line_data)
+    end
+
     # Consumes n characters from input and returns them as a pooled Slice.
     #
     # @param count [Integer] number of characters to consume

data/lib/parsanol/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Parsanol
-  VERSION = "1.3.36"
+  VERSION = "1.3.37"
 end

data/lib/parsanol.rb

@@ -129,31 +129,10 @@ module Parsanol
   def match(pattern = nil)
     return CharacterClassBuilder.new unless pattern
 
-    match_quantifier_warning(pattern)
     Atoms::Re.new(pattern)
   end
   module_function :match
 
-  # `match` consumes exactly one character (Parslet parity), so a regex
-  # quantifier is silently truncated: match(/[0-9]+/) matches a single
-  # digit. Warn once per offending pattern — the intended spelling is
-  # `match("[0-9]").repeat(1)`.
-  # Deliberately mutable: it accumulates warned patterns per process.
-  QUANTIFIER_WARNED = {} # rubocop:disable Style/MutableConstant
-  private_constant :QUANTIFIER_WARNED
-
-  def match_quantifier_warning(pattern)
-    return if QUANTIFIER_WARNED.key?(pattern)
-    return unless pattern.is_a?(String)
-    return unless quantified_pattern?(pattern)
-
-    QUANTIFIER_WARNED[pattern] = true
-    warn "Parsanol: match(#{pattern.inspect}) matches exactly one character " \
-         "(Parslet parity); the quantifier is ignored. " \
-         "Use match(...).repeat(1) to match repeatedly."
-  end
-  module_function :match_quantifier_warning
-
   # True when the pattern carries a top-level quantifier (+, *, ?, or
   # {n,m}) outside a character class.
   def quantified_pattern?(pattern)

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: parsanol
 version: !ruby/object:Gem::Version
-  version: 1.3.36
+  version: 1.3.37
 platform: ruby
 authors:
 - Ribose Inc.
@@ -89,6 +89,7 @@ files:
 - lib/parsanol/native.rb
 - lib/parsanol/native/batch_decoder.rb
 - lib/parsanol/native/dynamic.rb
+- lib/parsanol/native/event_player.rb
 - lib/parsanol/native/ffi.rb
 - lib/parsanol/native/parser.rb
 - lib/parsanol/native/serializer.rb