parsanol 1.3.13-arm-linux

data/lib/parsanol/native/batch_decoder.rb

@@ -0,0 +1,255 @@
+# frozen_string_literal: true
+
+require "parsanol/native/transformer"
+
+module Parsanol
+  module Native
+    # Decodes flat u64 arrays from Rust batch parser into Ruby AST
+    #
+    # The batch format uses tagged u64 values:
+    # - 0x00 = nil
+    # - 0x01 + value = bool (0 or 1)
+    # - 0x02 + value = int
+    # - 0x03 + bits = float (IEEE 754 bits)
+    # - 0x04 + offset + length = input string reference
+    # - 0x05 ... 0x06 = array (start ... end)
+    # - 0x07 ... 0x08 = hash (start ... end)
+    # - 0x09 + len + data... = hash key
+    # - 0x0A + len + data... = inline string
+    module BatchDecoder
+      TAG_NIL = 0x00
+      TAG_BOOL = 0x01
+      TAG_INT = 0x02
+      TAG_FLOAT = 0x03
+      TAG_STRING = 0x04
+      TAG_ARRAY_START = 0x05
+      TAG_ARRAY_END = 0x06
+      TAG_HASH_START = 0x07
+      TAG_HASH_END = 0x08
+      TAG_HASH_KEY = 0x09
+      TAG_INLINE_STRING = 0x0A
+      TAG_SYMBOL = 0x0B
+      TAG_REPETITION = 0x0C
+      TAG_SEQUENCE = 0x0D
+
+      class << self
+        # Decode a flat u64 array into Ruby AST with Slice objects
+        #
+        # @param data [Array<Integer>] Flat u64 array from batch parser
+        # @param input [String] Original input string (for Slice references)
+        # @param slice_class [Class] The Slice class to use
+        # @return [Object] Ruby AST (Hash, Array, Slice, etc.)
+        def decode(data, input, slice_class)
+          @input = input
+          @input_bytes = input.b
+          @slice_class = slice_class
+          @pos = 0
+          @data = data
+          decode_value
+        end
+
+        # Decode batch format to Ruby AST and apply transformation.
+        #
+        # The Rust parser produces raw AST that needs transformation to match
+        # Ruby parser behavior (merging duplicate keys, etc.)
+        #
+        # @param data [Array<Integer>|Object] Either flat u64 array from batch parser OR
+        #   pre-decoded Ruby value from _parse_raw
+        # @param input [String] Original input string (for Slice references)
+        # @param slice_class [Class] The Slice class to use
+        # @return [Object] Transformed Ruby AST
+        def decode_and_flatten(data, input, slice_class)
+          # Check if data is batch data (flat u64 array) or already a Ruby value
+          if data.is_a?(Integer) || (data.is_a?(Array) && data.first.is_a?(Integer))
+            # Batch data (flat u64 array) - decode first, then transform
+            raw_ast = decode(data, input, slice_class)
+            AstTransformer.transform(raw_ast)
+          else
+            # Already decoded Ruby value from _parse_raw - apply transformer directly
+            AstTransformer.transform(data)
+          end
+        end
+
+        # Join consecutive Slice objects in arrays into single Slices
+        # This matches what transform_ast does in Rust (join_slices_from_array)
+        #
+        # @param value [Object] AST value
+        # @param slice_class [Class] The Slice class to check for
+        # @param input [String] Original input string
+        # @return [Object] AST with joined slices
+        def join_consecutive_slices(value, slice_class, input)
+          input_bytes = input.b
+
+          case value
+          when Array
+            # Recursively process array elements
+            processed = value.map do |v|
+              join_consecutive_slices(v, slice_class, input)
+            end
+
+            # Check if all non-nil elements are Slices
+            non_nil = processed.compact
+            if non_nil.all?(slice_class)
+              # Check if slices are consecutive
+              if slices_consecutive?(non_nil)
+                # Join into single slice
+                join_slices(non_nil, slice_class, input_bytes, input)
+              else
+                processed
+              end
+            else
+              processed
+            end
+          when Hash
+            # Process hash values recursively
+            result = {}
+            value.each do |k, v|
+              result[k] = join_consecutive_slices(v, slice_class, input)
+            end
+            result
+          else
+            value
+          end
+        end
+
+        private
+
+        def slices_consecutive?(slices)
+          return true if slices.empty?
+
+          slices.each_cons(2).all? do |a, b|
+            a.offset + a.content.bytesize == b.offset
+          end
+        end
+
+        def join_slices(slices, slice_class, input_bytes, input)
+          return nil if slices.empty?
+          return slices.first if slices.length == 1
+
+          first = slices.first
+          last = slices.last
+          total_length = last.offset + last.content.bytesize - first.offset
+          content = input_bytes[first.offset, total_length]
+          content = content.force_encoding("UTF-8") if content
+          slice_class.new(first.offset, content, input)
+        end
+
+        def decode_value
+          tag = @data[@pos]
+          @pos += 1
+
+          case tag
+          when TAG_NIL
+            nil
+          when TAG_BOOL
+            val = @data[@pos]
+            @pos += 1
+            val != 0
+          when TAG_INT
+            val = @data[@pos]
+            @pos += 1
+            # Handle negative numbers (signed i64 stored as u64)
+            if val >= 0x8000_0000_0000_0000
+              val = val - 0x1_0000_0000_0000_0000
+            end
+            val
+          when TAG_FLOAT
+            bits = @data[@pos]
+            @pos += 1
+            # Convert IEEE 754 bits to float
+            [bits].pack("Q").unpack1("D")
+          when TAG_STRING
+            offset = @data[@pos]
+            length = @data[@pos + 1]
+            @pos += 2
+            create_slice(offset, length)
+          when TAG_SYMBOL
+            # Symbol is encoded like inline string: len, then u64 chunks
+            len = @data[@pos]
+            @pos += 1
+            str = decode_inline_string_bytes(len)
+            str.to_sym
+          when TAG_REPETITION
+            inner = decode_value
+            [:repetition, inner].compact
+          when TAG_SEQUENCE
+            inner = decode_value
+            [:sequence, inner].compact
+          when TAG_ARRAY_START
+            decode_array
+          when TAG_HASH_START
+            decode_hash
+          else
+            raise "Unknown tag: #{tag} at position #{@pos - 1}"
+          end
+        end
+
+        def decode_array
+          result = []
+          loop do
+            tag = @data[@pos]
+            break if tag == TAG_ARRAY_END
+
+            result << decode_value
+          end
+          @pos += 1 # consume TAG_ARRAY_END
+          result
+        end
+
+        def decode_hash
+          result = {}
+          loop do
+            tag = @data[@pos]
+            break if tag == TAG_HASH_END
+
+            # Read key
+            raise "Expected TAG_HASH_KEY, got #{tag}" unless tag == TAG_HASH_KEY
+
+            @pos += 1
+            key = decode_inline_string
+
+            # Read value
+            value = decode_value
+
+            # Keep original key format (camelCase) for Ruby parser compatibility
+            result[key.to_sym] = value
+          end
+          @pos += 1 # consume TAG_HASH_END
+          result
+        end
+
+        def decode_inline_string
+          len = @data[@pos]
+          @pos += 1
+          decode_inline_string_bytes(len)
+        end
+
+        # Decode inline string bytes given the length
+        # @param len [Integer] Length of the string in bytes
+        # @return [String] Decoded string
+        def decode_inline_string_bytes(len)
+          # Read u64 chunks
+          chunks = (len + 7) / 8
+          bytes = String.new(encoding: "ASCII-8BIT", capacity: len)
+          chunks.times do
+            chunk = @data[@pos]
+            @pos += 1
+            8.times do |byte_idx|
+              break if bytes.bytesize >= len
+
+              bytes << ((chunk >> (byte_idx * 8)) & 0xFF)
+            end
+          end
+
+          bytes.force_encoding("UTF-8")
+        end
+
+        def create_slice(offset, length)
+          content = @input_bytes[offset, length]
+          content = content.force_encoding("UTF-8") if content
+          @slice_class.new(offset, content, @input)
+        end
+      end
+    end
+  end
+end

data/lib/parsanol/native/dynamic.rb

@@ -0,0 +1,238 @@
+# frozen_string_literal: true
+
+require "json"
+
+module Parsanol
+  module Native
+    # Manages Ruby callbacks for dynamic atoms
+    #
+    # Dynamic atoms allow runtime-determined parsing by invoking Ruby code
+    # during parsing. This module provides:
+    # - Registration of Ruby procs as callbacks
+    # - Thread-safe callback storage
+    # - GC-safe references (callbacks are kept alive while registered)
+    #
+    # @example Basic usage
+    #   # Register a callback
+    #   callback_id = Parsanol::Native::Dynamic.register(->(ctx) {
+    #     ctx[:mode] == 'A' ? str('alpha') : str('beta')
+    #   })
+    #
+    #   # Use in grammar
+    #   grammar = str('MODE:').capture(:mode) >> dynamic(callback_id)
+    #
+    #   # Unregister when done
+    #   Parsanol::Native::Dynamic.unregister(callback_id)
+    #
+    module Dynamic
+      # Callback storage (callback_id => block)
+      # This keeps strong references to prevent GC
+      @callbacks = {}
+      @mutex = Mutex.new
+      @next_id = 1_000_000 # Start high to avoid conflicts with Rust-side IDs
+
+      class << self
+        # Register a Ruby block as a dynamic callback
+        #
+        # @param block [Proc] The block to register (must accept a context hash)
+        # @param description [String, nil] Optional description for debugging
+        # @return [Integer] Unique callback ID for use in grammar
+        #
+        # @example
+        #   id = Parsanol::Native::Dynamic.register(->(ctx) {
+        #     case ctx[:type]
+        #     when 'int' then str('integer')
+        #     when 'str' then str('string')
+        #     else nil
+        #     end
+        #   })
+        #
+        def register(block, description: nil)
+          # Register with Rust FFI
+          ffi_id = Native.register_callback(@next_id,
+                                            description || "Ruby callback ##{@next_id}")
+
+          # Also keep a Ruby-side reference for GC safety
+          @mutex.synchronize do
+            @callbacks[ffi_id] = {
+              block: block,
+              description: description || "Ruby callback ##{ffi_id}",
+            }
+          end
+
+          ffi_id
+        end
+
+        # Unregister a callback (free memory)
+        #
+        # @param callback_id [Integer] The callback ID to remove
+        # @return [Boolean] True if the callback was found and removed
+        #
+        def unregister(callback_id)
+          # Remove from Rust FFI
+          Native.unregister_callback(callback_id)
+
+          # Remove from Ruby storage
+          @mutex.synchronize do
+            @callbacks.delete(callback_id)
+          end
+        end
+
+        # Get the description of a registered callback
+        #
+        # @param callback_id [Integer] The callback ID
+        # @return [String, nil] The description or nil if not found
+        #
+        def description(callback_id)
+          # Try Ruby-side first
+          ruby_desc = @mutex.synchronize do
+            @callbacks[callback_id]&.dig(:description)
+          end
+
+          return ruby_desc if ruby_desc
+
+          # Fall back to FFI
+          Native.get_callback_description(callback_id)
+        end
+
+        # Get the number of registered callbacks
+        #
+        # @return [Integer] Number of registered callbacks
+        #
+        def count
+          Native.callback_count
+        end
+
+        # Clear all callbacks (for testing)
+        #
+        # WARNING: This clears all callbacks globally, including those
+        # registered by other code. Use with caution.
+        #
+        def clear
+          Native.clear_callbacks
+          @mutex.synchronize { @callbacks.clear }
+        end
+
+        # Check if a callback is registered
+        #
+        # @param callback_id [Integer] The callback ID
+        # @return [Boolean] True if registered
+        #
+        def registered?(callback_id)
+          @mutex.synchronize { @callbacks.key?(callback_id) } ||
+            Native.has_callback(callback_id)
+        end
+
+        # Invoke a callback from Rust (called via FFI)
+        #
+        # @param callback_id [Integer] The callback ID
+        # @param context [Hash] The context hash from Rust
+        # @return [Object, nil] The returned atom (parslet) or nil
+        #
+        def invoke_from_rust(callback_id, context)
+          block = @mutex.synchronize { @callbacks[callback_id]&.dig(:block) }
+          return nil unless block
+
+          # Build DynamicContext from hash
+          ctx = DynamicContext.new(
+            context[:input],
+            context[:pos],
+            context[:captures].transform_keys(&:to_sym),
+          )
+
+          # Call the block
+          result = block.call(ctx)
+
+          return nil unless result
+
+          # Return the result (should be a parslet/atom)
+          result
+        rescue StandardError => e
+          warn "[Parsanol::Native::Dynamic] Invoke error: #{e.message}"
+          nil
+        end
+      end
+    end
+
+    # Context object passed to dynamic callbacks
+    #
+    # Provides read-only access to the parsing context including
+    # input string, current position, and captured values.
+    #
+    # @example
+    #   dynamic { |ctx|
+    #     if ctx[:mode] == 'strict'
+    #       str('strict_value')
+    #     else
+    #       str('relaxed_value')
+    #     end
+    #   }
+    #
+    class DynamicContext
+      # @return [String] The full input string being parsed
+      attr_reader :input
+
+      # @return [Integer] Current byte position in the input
+      attr_reader :pos
+
+      # @return [Hash<Symbol, String>] Captured values
+      attr_reader :captures
+
+      def initialize(input, pos, captures)
+        @input = input
+        @pos = pos
+        @captures = captures.transform_keys(&:to_sym)
+      end
+
+      # Get a captured value by name
+      #
+      # @param name [Symbol, String] The capture name
+      # @return [String, nil] The captured value or nil
+      #
+      def [](name)
+        @captures[name.to_sym]
+      end
+
+      # Check if a capture exists
+      #
+      # @param name [Symbol, String] The capture name
+      # @return [Boolean] True if the capture exists
+      #
+      def key?(name)
+        @captures.key?(name.to_sym)
+      end
+      alias has_key? key?
+
+      # Get the remaining input from the current position
+      #
+      # @return [String] The remaining input
+      #
+      def remaining
+        @input[@pos..] || ""
+      end
+
+      # Check if at end of input
+      #
+      # @return [Boolean] True if at end
+      #
+      def eos?
+        @pos >= @input.length
+      end
+      alias at_end? eos?
+
+      # Get a slice of the input
+      #
+      # @param start [Integer] Start position (relative to current pos if negative)
+      # @param length [Integer, nil] Length of slice (nil = to end)
+      # @return [String] The sliced input
+      #
+      def slice(start, length = nil)
+        if length
+          @input[@pos + start, length]
+        else
+          @input[(@pos + start)..]
+        end
+      end
+    end
+  end
+end

data/lib/parsanol/native/parser.rb

@@ -0,0 +1,102 @@
+# frozen_string_literal: true
+
+require "digest"
+
+module Parsanol
+  module Native
+    # Core parsing functionality using Rust native extension
+    module Parser
+      GRAMMAR_HASH_CACHE = Hash.new
+      GRAMMAR_CACHE = Hash.new
+
+      class << self
+        @cached_available = nil
+
+        def available?
+          return @cached_available unless @cached_available.nil?
+
+          @cached_available = begin
+            # Try versioned path first (released gem), then non-versioned (local dev)
+            ruby_version = RUBY_VERSION.split(".").take(2).join(".")
+            begin
+              require "parsanol/#{ruby_version}/parsanol_native"
+            rescue LoadError
+              require "parsanol/parsanol_native"
+            end
+            Parsanol::Native.is_available
+          rescue LoadError
+            false
+          end
+        end
+
+        # Parse input with a Ruby grammar, returning clean AST.
+        #
+        # @param grammar [Parsanol::Atoms::Base] Ruby grammar or JSON string
+        # @param input [String] Input string to parse
+        def parse(grammar, input)
+          # Delegate to Parsanol::Native.parse for consistency
+          Parsanol::Native.parse(grammar, input)
+        end
+
+        # Serialize a Ruby grammar to JSON (cached).
+        def serialize_grammar(root_atom)
+          root_atom = root_atom.root if root_atom.is_a?(::Parsanol::Parser)
+          obj_id = root_atom.object_id
+          cache_key = GRAMMAR_HASH_CACHE[obj_id] ||= grammar_structure_hash(root_atom)
+          GRAMMAR_CACHE[cache_key] ||= GrammarSerializer.serialize(root_atom)
+        end
+
+        def clear_cache
+          GRAMMAR_HASH_CACHE.clear
+          GRAMMAR_CACHE.clear
+        end
+
+        def cache_stats
+          {
+            hash_cache_size: GRAMMAR_HASH_CACHE.size,
+            grammar_cache_size: GRAMMAR_CACHE.size,
+          }
+        end
+
+        private
+
+        def grammar_structure_hash(atom)
+          Digest::MD5.hexdigest(atom_structure(atom).to_s)
+        end
+
+        def atom_structure(atom, visited = {})
+          # Cycle detection - return a placeholder if we've seen this atom before
+          obj_id = atom.object_id
+          if visited[obj_id]
+            return [:cycle, atom.class.name]
+          end
+
+          visited[obj_id] = true
+
+          case atom
+          when ::Parsanol::Atoms::Entity
+            # Recursively resolve entity to get actual structure for hash
+            atom_structure(atom.parslet, visited)
+          when ::Parsanol::Atoms::Str
+            [:str, atom.str]
+          when ::Parsanol::Atoms::Re
+            [:re, atom.match]
+          when ::Parsanol::Atoms::Sequence
+            [:seq, atom.parslets.map { |p| atom_structure(p, visited) }]
+          when ::Parsanol::Atoms::Alternative
+            [:alt, atom.alternatives.map { |p| atom_structure(p, visited) }]
+          when ::Parsanol::Atoms::Repetition
+            [:rep, atom.min, atom.max, atom_structure(atom.parslet, visited)]
+          when ::Parsanol::Atoms::Named
+            [:named, atom.name.to_s, atom_structure(atom.parslet, visited)]
+          when ::Parsanol::Atoms::Lookahead
+            [:lookahead, atom.positive,
+             atom_structure(atom.bound_parslet, visited)]
+          else
+            [:unknown, atom.class.name]
+          end
+        end
+      end
+    end
+  end
+end