parsanol 1.3.63 → 1.3.65

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: f7ad59a2f852639eec49d8aaf0cfce4be63d8e13a20941b7db8f325244445e47
-  data.tar.gz: b4f7021d4c4dd9f30df4eddf64b2109da9005291b4d39edadd5075e54c6600c0
+  metadata.gz: a40f12d1d7a5fc67b1ef3255e9d9c5f4c38fd6a8fb266995974beec982728ca1
+  data.tar.gz: 297a013726a6e9ff9e81b385a8074e45afd5e9a22739e28852219f35cd1ad3a4
 SHA512:
-  metadata.gz: d5fe08eb23fa452e06432cb8b72cb101b257b854fb33ae71455fd8c2a0d81356704f66fd11af330392783aa6c6a5b4730df1884718459c5d4acfe3abc74b8b48
-  data.tar.gz: b2d8251fc66c8d47085bf6338457975b6f29e707a173c8e107d5755137bd850ead86c6d130b6410b29e4155e87ae4c67cac3739278f0ddd4a7826fc94ffba53e
+  metadata.gz: 6f171865bebc41146db262fcbdf5f05bdd3daf0e424cf29eaf26351408ca5d35db904057dfd64f0b57bd6a18190e48596bfd2af21ebfbf75d79a7fbc421c2bd1
+  data.tar.gz: d69f48dcc19e090e4b7c0fa880f142d9a76ff95e1aa6d8f73803558902c66621abbc7f28d24ca683ecadab7a5f6855fb88913dfecf05e5699a12780e98722ff7

data/lib/parsanol/atoms/state.rb

@@ -0,0 +1,151 @@
+# frozen_string_literal: true
+
+module Parsanol
+  module Atoms
+    # PARG runtime-state atoms (parsanol-ruby#129): per-parse mutable
+    # slots living in the parse context's capture store, so capture
+    # rollback, scope discipline and the dispatch semantics are the
+    # proven dynamic-callback ones. All four are pure data — they
+    # serialize as structured tags and rebuild identically in any
+    # runtime that implements the state machine.
+    class StateBase < Base
+      def cached?
+        false
+      end
+
+      private
+
+      # Slots and captures may live in a parent scope frame (an earlier
+      # element's frame); Scope#[] only reads the active one.
+      def lookup(context, key)
+        context.captures.fetch(key)
+      rescue Parsanol::Scope::UndefinedVariable
+        nil
+      end
+
+      def mark_unsafe(context)
+        # The atom reads mutable parse state (captures), so no enclosing
+        # composite result may be memoized — replaying it would skip the
+        # re-evaluation (same contract as Atoms::Dynamic).
+        context.mark_cache_unsafe!
+      end
+    end
+
+    # Writes a state slot. Two forms: a literal value (always succeeds,
+    # consumes nothing) or an inline expression — the expression matches
+    # at the position and the slot stores its consumed text (the
+    # block_open pattern: match the delimiter, remember it).
+    class StateSet < StateBase
+      def initialize(slot, value: nil, atom: nil)
+        super()
+        @slot = slot
+        @value = value
+        @atom = atom
+      end
+
+      attr_reader :slot, :value, :atom
+
+      def try(source, context, _consume_all)
+        mark_unsafe(context)
+        if @atom.nil?
+          context.captures[@slot] = @value
+          return [true, ""]
+        end
+
+        start = source.bytepos
+        outcome = @atom.apply(source, context, false)
+        return outcome unless outcome.first
+
+        span = source.input.byteslice(start, source.bytepos - start)
+        context.captures[@slot] = span
+        outcome
+      end
+
+      def to_s_inner(_prec)
+        "set #{@slot}=#{@value || '…'}"
+      end
+    end
+
+    # Matches the state slot's current value verbatim at the position —
+    # the block-delimiter comparison ("exactly what was captured above").
+    class StateMatch < StateBase
+      def initialize(slot)
+        super()
+        @slot = slot
+      end
+
+      attr_reader :slot
+
+      def try(source, context, _consume_all)
+        mark_unsafe(context)
+        expected = lookup(context, @slot)
+        return [false, nil] if expected.nil?
+
+        actual = source.peek(expected.length)
+        return [false, nil] if actual != expected
+
+        source.bytepos += expected.length
+        [true, actual]
+      end
+
+      def to_s_inner(_prec)
+        "state(#{@slot})"
+      end
+    end
+
+    # Dispatches on the slot's current value to a rule reference. The
+    # resolver maps a rule name to its atom lazily (rules may still be
+    # under construction when this atom is built). A nil arm fails the
+    # atom — the native dynamic-callback parity.
+    class StateSwitch < StateBase
+      def initialize(slot, arms, default, resolver)
+        super()
+        @slot = slot
+        @arms = arms
+        @default = default
+        @resolver = resolver
+      end
+
+      attr_reader :slot, :arms, :default
+
+      def try(source, context, consume_all)
+        mark_unsafe(context)
+        current = lookup(context, @slot)
+        rule_name = @arms[current] || @default
+        return [false, nil] if rule_name.nil?
+
+        @resolver.call(rule_name).apply(source, context, consume_all)
+      end
+
+      def to_s_inner(_prec)
+        "switch(#{@slot})"
+      end
+    end
+
+    # A foreign atom: a class resolved from the artifact's customs
+    # binding at compile time. The instance runs as-is on the Ruby
+    # engine; the serialized tag carries the binding so any runtime can
+    # decide support loudly.
+    class CustomRef < Base
+      def initialize(name, klass)
+        super()
+        @name = name
+        @atom = klass.new
+      end
+
+      attr_reader :name
+
+      def cached?
+        false
+      end
+
+      def try(source, context, consume_all)
+        @atom.try(source, context, consume_all)
+      end
+
+      def to_s_inner(_prec)
+        "custom(#{@name})"
+      end
+    end
+  end
+end

data/lib/parsanol/atoms.rb

@@ -38,6 +38,7 @@ module Parsanol
     require "parsanol/atoms/entity"
     require "parsanol/atoms/capture"
     require "parsanol/atoms/dynamic"
+    require "parsanol/atoms/state"
     require "parsanol/atoms/scope"
     require "parsanol/atoms/infix"
     # Load visitor pattern (must be after all atom classes)

data/lib/parsanol/native/serializer.rb

@@ -68,6 +68,14 @@ module Parsanol
                      serialize_capture(atom)
                    when Parsanol::Atoms::Scope
                      serialize_scope(atom)
+                   when Parsanol::Atoms::StateSet
+                     serialize_state_set(atom)
+                   when Parsanol::Atoms::StateMatch
+                     serialize_state_match(atom)
+                   when Parsanol::Atoms::StateSwitch
+                     serialize_state_switch(atom)
+                   when Parsanol::Atoms::CustomRef
+                     serialize_custom_ref(atom)
                    when Parsanol::Atoms::Dynamic
                      serialize_dynamic(atom)
                    when Parsanol::Atoms::Ignored
@@ -219,6 +227,34 @@ module Parsanol
       }
     end
 
+    def serialize_state_set(atom)
+      payload = { "slot" => atom.slot.to_s }
+      if atom.atom
+        payload["expr"] = serialize_atom(atom.atom)
+      else
+        payload["value"] = atom.value
+      end
+      { "StateSet" => payload }
+    end
+
+    def serialize_state_match(atom)
+      { "StateMatch" => { "slot" => atom.slot.to_s } }
+    end
+
+    def serialize_state_switch(atom)
+      {
+        "StateSwitch" => {
+          "slot" => atom.slot.to_s,
+          "arms" => atom.arms.transform_keys(&:to_s),
+          "default" => atom.default,
+        },
+      }
+    end
+
+    def serialize_custom_ref(atom)
+      { "CustomRef" => { "name" => atom.name } }
+    end
+
     def serialize_dynamic(atom)
       # Dynamic evaluates a Ruby block at parse time.
       # Register the block and get a callback ID for FFI.

data/lib/parsanol/parg/artifact.rb

@@ -57,10 +57,15 @@ module Parsanol
       # path is a straight register-and-run — no Ruby recompilation. The
       # Ruby atom runtime (mode: :ruby) remains available explicitly, and
       # serves as the fallback on platforms without the extension.
+      #
+      # Dynamic artifacts (runtime state, custom atoms — #129) route to
+      # the Ruby engine: the envelope carries the PARG source precisely
+      # so a Ruby runtime can rebuild the live atom tree; the closures
+      # are not portable JSON.
       def parse(entry_name, input, mode: :native)
         case mode
         when :native
-          if Native.available?
+          if !envelope["dynamic"] && Native.available?
             return Native.parse(JSON.generate(entry(entry_name).fetch("grammar")), input)
           end
 

data/lib/parsanol/parg/compiler.rb

@@ -50,6 +50,7 @@ module Parsanol
         @atom_cache = {}
         @first_cache = {}.compare_by_identity
         @ref_stack = []
+        @dynamic = false
         @tables = {}
         @warnings = []
       end
@@ -67,6 +68,16 @@ module Parsanol
         Result.new(@atom_cache, @warnings, envelope)
       end
 
+      def resolve_custom(name)
+        class_name = @document.customs.fetch(name)
+        klass = Object.const_get(class_name)
+        Atoms::CustomRef.new(name, klass)
+      rescue NameError
+        raise CompileError,
+              "custom atom #{name.inspect} cannot resolve #{class_name.inspect} " \
+              "(bind the class before compiling, or fix the custom declaration)"
+      end
+
       def atom_for(name)
         @atom_cache[name] ||= build_atom(fetch_rule(name))
       end
@@ -157,6 +168,13 @@ module Parsanol
       end
 
       def fetch_rule(name)
+        # Customs are opaque terminals for the analysis passes (first
+        # sets, left-recursion): a literal stand-in keeps them
+        # non-nullable with a singleton first set.
+        if !@document.rules.key?(name) && @document.customs.key?(name)
+          return Node.new(:lit, "\u0001", false)
+        end
+
         @document.rules[name] || raise(CompileError, "unknown rule #{name.inspect}")
       end
 
@@ -194,7 +212,31 @@ module Parsanol
           end
 
           Atoms::Named.new(build_atom(node.b), node.a.to_sym)
-        when :ref then Atoms::Entity.new(node.a) { atom_for(node.a) }
+        when :set
+          @dynamic = true
+          slot = node.a.to_sym
+          value = node.b
+          if value.is_a?(String)
+            Atoms::StateSet.new(slot, value: value)
+          else
+            Atoms::StateSet.new(slot, atom: build_atom(value))
+          end
+        when :state_match
+          @dynamic = true
+          Atoms::StateMatch.new(node.a.to_sym)
+        when :switch
+          @dynamic = true
+          slot = node.a.to_sym
+          arms = node.b.to_h { |match, rule| [match, rule] }
+          default = arms.delete(:default)
+          Atoms::StateSwitch.new(slot, arms, default, method(:atom_for))
+        when :ref
+          if @document.customs.key?(node.a)
+            @dynamic = true
+            resolve_custom(node.a)
+          else
+            Atoms::Entity.new(node.a) { atom_for(node.a) }
+          end
         when :table
           values = table_column(node.a, node.b)
           Atoms::Alternative.new(*values.map { |value| Atoms::Str.new(value) })
@@ -430,6 +472,9 @@ module Parsanol
           "docs" => @document.docs,
           "source" => @document.source,
         }
+        envelope["customs"] = @document.customs unless @document.customs.empty?
+        envelope["states"] = @document.states unless @document.states.empty?
+        envelope["dynamic"] = true if @dynamic
         envelope["checksum"] = Compiler.checksum(envelope)
         envelope
       end

data/lib/parsanol/parg/document.rb

@@ -10,7 +10,8 @@ module Parsanol
       Test = Struct.new(:entry, :kind, :input, :expect)
 
       attr_accessor :grammar_name, :version, :source, :uses
-      attr_reader :rules, :bindings, :preprocess, :entries, :docs, :tests, :render, :derive
+      attr_reader :rules, :bindings, :preprocess, :entries, :docs, :tests, :render, :derive,
+                  :customs, :states
       # Entries authored in this file (pre-import); nil-entry tests resolve
       # against these first.
       attr_accessor :own_entries
@@ -29,6 +30,8 @@ module Parsanol
         @tests = []
         @uses = []
         @own_entries = nil
+        @customs = {}
+        @states = {}
       end
 
       def validate!

data/lib/parsanol/parg/parser.rb

@@ -18,8 +18,9 @@ module Parsanol
 
       IN_BLOCK_SECTIONS = %w[entry bindings preprocess test].freeze
 
+      STATE_KEYWORDS = %w[custom state].freeze
       KEYWORDS = %w[grammar as alt from_table column bindings render derive
-                    preprocess entry table_lookup].freeze
+                    preprocess entry table_lookup custom state set switch].freeze
 
       def initialize(text)
         @tokens = Lexer.new(text).tokens
@@ -160,6 +161,19 @@ module Parsanol
             skip_newlines
             next
           end
+          if peek&.type == :ident && STATE_KEYWORDS.include?(peek.value)
+            keyword = advance.value
+            slot = ident.value
+            if keyword == "custom"
+              punct("=")
+              document.customs[slot] = unquote(expect(:str).value)
+            else
+              punct(":")
+              document.states[slot] = ident.value
+            end
+            skip_newlines
+            next
+          end
           name = rule_name
           document.docs[name] = docs.join("\n") unless docs.empty?
           punct("=")
@@ -274,6 +288,43 @@ module Parsanol
 
       def parse_ident(token)
         case token.value
+        when "state"
+          Node.new(:state_match, ident.value)
+        when "set"
+          slot = ident.value
+          punct("=")
+          # A string writes the literal; a bare name copies the matched
+          # text of a capture (the block-delimiter pattern: set the slot
+          # from what the opening delimiter matched).
+          value = if peek&.type == :str
+                    unquote(expect(:str).value)
+                  else
+                    punct("(")
+                    inner = parse_choice
+                    punct(")")
+                    inner
+                  end
+          Node.new(:set, slot, value)
+        when "switch"
+          slot = ident.value
+          punct("{")
+          skip_newlines
+          arms = []
+          until peek&.type == :punct && peek.value == "}"
+            match = if peek&.type == :ident && peek.value == "_"
+                      advance
+                      :default
+                    else
+                      unquote(expect(:str).value)
+                    end
+            arrow = advance
+            raise ParseError, "expected \"->\", got #{arrow&.type}:#{arrow&.value}" unless arrow&.value == "->"
+
+            arms << [match, ident.value]
+            skip_newlines
+          end
+          punct("}")
+          Node.new(:switch, slot, arms)
         when "alt"
           ident("from_table")
           table = unquote(expect(:str).value)

data/lib/parsanol/parser.rb

@@ -238,6 +238,11 @@ module Parsanol
     # The degradation warning prints once per reason per process — not
     # per parser instance and not per parse.
     def native_expressible?
+      # The ext attaches its methods lazily (available? triggers the load);
+      # without this guard a first-touch call NoMethodErrors inside the
+      # serializer (Dynamic.register -> Native.register_callback).
+      return false unless Parsanol::Native.available?
+
       Parsanol::Native::Parser.grammar_handle(root)
       true
     rescue Parsanol::Native::UnsupportedGrammar => e

data/lib/parsanol/scope.rb

@@ -84,6 +84,14 @@ module Parsanol
       @active_frame.fetch(key)
     end
 
+    # Chain-aware read: the active frame, then its parents ([] is
+    # deliberately active-frame-only for capture-scope discipline; state
+    # atoms read across frames — a slot may have been written inside an
+    # earlier sibling element's frame).
+    def fetch(key)
+      @active_frame.fetch(key)
+    end
+
     # Chain-aware membership test, mirroring Parslet's captures.key?.
     # Checks the current frame and all parents without raising.
     #

data/lib/parsanol/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Parsanol
-  VERSION = "1.3.63"
+  VERSION = "1.3.65"
 end

data/lib/parsanol/vm.rb

@@ -536,11 +536,19 @@ module Parsanol
           if tables.all? && pairwise_disjoint?(tables)
             dispatch_idx = emit(BYTE_DISPATCH, nil, nil, nil)
             starts = []
+            # A dispatched branch that matched must not fall through into
+            # the next branch's code: the tables are lead-byte disjoint,
+            # but the remaining input can still satisfy a later branch's
+            # terminals (pubid: " - " — the '-' branch ran the ' ' branch's
+            # STR on the byte after the dash). Jump every branch but the
+            # last past the dispatch.
+            jumps = []
             idx = 0
             while idx < count
               starts << flat(@ops.size)
               return nil unless compile_atom(alts[idx], consume_all)
 
+              jumps << emit(JMP, nil, nil, nil) unless idx == count - 1
               idx += 1
             end
             table = Array.new(256, -1)
@@ -555,6 +563,7 @@ module Parsanol
               bi2
             end
             patch(dispatch_idx, 1, table)
+            jumps.each { |j| patch(j, 1, flat(@ops.size)) }
             return self
           end
 

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: parsanol
 version: !ruby/object:Gem::Version
-  version: 1.3.63
+  version: 1.3.65
 platform: ruby
 authors:
 - Ribose Inc.
@@ -70,6 +70,7 @@ files:
 - lib/parsanol/atoms/repetition.rb
 - lib/parsanol/atoms/scope.rb
 - lib/parsanol/atoms/sequence.rb
+- lib/parsanol/atoms/state.rb
 - lib/parsanol/atoms/str.rb
 - lib/parsanol/atoms/visitor.rb
 - lib/parsanol/buffer.rb