parsanol 1.3.13-arm-linux

data/lib/parsanol/source/line_cache.rb

@@ -0,0 +1,102 @@
+# frozen_string_literal: true
+
+# Line position caching for efficient line/column lookups.
+# Stores line ending positions to enable O(log n) line number queries.
+#
+# Inspired by Parslet (MIT License).
+
+module Parsanol
+  class Source
+    # Caches line ending positions for quick line/column resolution.
+    # Uses binary search for efficient position lookup.
+    class LineCache
+      def initialize
+        # Array of byte offsets where each line ends
+        @breaks = []
+        @breaks.extend(IntervalLookup)
+        @max_scanned = nil
+      end
+
+      # Converts a byte offset to [line_number, column_number].
+      # Line and column numbers are 1-indexed.
+      #
+      # @param position [Integer, #bytepos] the byte offset to convert
+      # @return [Array<Integer, Integer>] [line, column] tuple
+      def line_and_column(position)
+        position = position.bytepos if position.respond_to?(:bytepos)
+
+        line_idx = @breaks.lower_bound_index(position)
+
+        if line_idx
+          # Found a line ending after this position
+          line_start = line_idx.positive? ? @breaks[line_idx - 1] : 0
+          [line_idx + 1, position - line_start + 1]
+        else
+          # Position is beyond all known line endings
+          line_start = @breaks.last || 0
+          [@breaks.size + 1, position - line_start + 1]
+        end
+      end
+
+      # Scans a string buffer for line endings and caches their positions.
+      # Avoids re-scanning already processed regions.
+      #
+      # @param start_offset [Integer] the byte offset where buffer starts
+      # @param buffer [String] the string content to scan
+      def scan_for_line_endings(start_offset, buffer)
+        return unless buffer
+
+        scanner = StringScanner.new(buffer)
+        return unless scanner.exist?(/\n/)
+
+        # Skip already-scanned content
+        scanner.pos = @max_scanned - start_offset if @max_scanned && start_offset < @max_scanned
+
+        # Record all newline positions
+        while scanner.skip_until(/\n/)
+          @max_scanned = start_offset + scanner.pos
+          @breaks << @max_scanned
+        end
+      end
+    end
+
+    # Mixin providing binary search for interval containment queries.
+    # Treats array values as interval endpoints where each interval [n-1, n]
+    # is represented by value at index n.
+    #
+    # @example
+    #   [10, 20, 30] represents intervals [0,10], (10,20], (20,30]
+    module IntervalLookup
+      # Calculates midpoint index for binary search.
+      # Uses floor to ensure integer result.
+      def midpoint_index(lo, hi)
+        lo + ((hi - lo) / 2).floor
+      end
+
+      # Finds the index of the first value greater than the bound.
+      # Returns nil if no such value exists.
+      #
+      # @param bound [Numeric] the value to search against
+      # @return [Integer, nil] index of first value > bound, or nil
+      def lower_bound_index(bound)
+        return nil if empty?
+        return nil unless last > bound
+
+        lo = 0
+        hi = size - 1
+
+        loop do
+          mid = midpoint_index(lo, hi)
+
+          if self[mid] > bound
+            hi = mid
+          else
+            lo = mid + 1
+          end
+
+          return hi if hi <= lo
+        end
+      end
+    end
+  end
+end

data/lib/parsanol/source.rb

@@ -0,0 +1,185 @@
+# frozen_string_literal: true
+
+require "stringio"
+require "strscan"
+
+require "parsanol/position"
+require "parsanol/source/line_cache"
+require "parsanol/pools/slice_pool"
+require "parsanol/pools/position_pool"
+
+module Parsanol
+  # Encapsulates input source for parsing operations. Provides position tracking,
+  # character consumption, line/column calculation, and object pooling for
+  # memory efficiency.
+  #
+  # @example Creating a source
+  #   src = Parsanol::Source.new("input string")
+  #   src.matches?(/a/)  # => true if 'a' is at current position
+  #   src.consume(1)     # => Slice containing one character
+  #
+  # Inspired by source/position tracking patterns in parser implementations.
+  #
+  class Source
+    # @return [Parsanol::Pools::SlicePool] pool for Slice objects
+    attr_reader :slice_pool
+
+    # @return [Parsanol::Pools::PositionPool] pool for Position objects
+    attr_reader :position_pool
+
+    # Creates a new source wrapper around a string.
+    #
+    # @param input [#to_str] string-like object to parse
+    # @raise [ArgumentError] if input doesn't respond to to_str
+    #
+    def initialize(input)
+      unless input.respond_to?(:to_str)
+        raise ArgumentError,
+              "Source requires a string-like object (responds to to_str)"
+      end
+
+      # Core scanner for input traversal
+      @scanner = StringScanner.new(input)
+      @raw_string = input.to_str
+
+      # Regex cache: maps count n to /(.|$){n}/m pattern
+      @regex_cache = Hash.new do |h, count|
+        h[count] = Regexp.new("(.|$){#{count}}", Regexp::MULTILINE)
+      end
+
+      # Line ending cache for position-to-line/column mapping
+      @line_data = LineCache.new
+      @line_data.scan_for_line_endings(0, input)
+
+      # Object pools for memory efficiency
+      # SlicePool: reduces Slice allocations during matching
+      @slice_pool = Parsanol::Pools::SlicePool.new(size: 5000, preallocate: false)
+
+      # PositionPool: reduces Position allocations for error reporting
+      @position_pool = Parsanol::Pools::PositionPool.new(size: 1000)
+    end
+
+    # Checks if a pattern matches at the current input position without consuming.
+    #
+    # @param pattern [Regexp] pattern to test
+    # @return [Boolean] true if pattern matches at current position
+    #
+    def matches?(pattern)
+      @scanner.match?(pattern)
+    end
+    alias match matches?
+
+    # Consumes n characters from input and returns them as a pooled Slice.
+    #
+    # @param count [Integer] number of characters to consume
+    # @return [Parsanol::Slice] slice containing consumed characters
+    #
+    def consume(count)
+      current_pos = @scanner.pos
+      content = @scanner.scan(@regex_cache[count])
+      @slice_pool.acquire_with(current_pos, content, @line_data)
+    end
+
+    # Creates a pooled slice at a specific position.
+    # Preferred method for atoms to construct slices.
+    #
+    # @param offset [Integer] byte position in source
+    # @param content [String] slice content
+    # @return [Parsanol::Slice] pooled slice instance
+    #
+    def slice(offset, content)
+      @slice_pool.acquire_with(offset, content, @line_data)
+    end
+
+    # Returns a slice to the pool for reuse.
+    #
+    # @param sl [Parsanol::Slice] slice to release
+    #
+    def release_slice(sl)
+      @slice_pool.release(sl)
+    end
+
+    # Returns count of remaining characters in input.
+    #
+    # @return [Integer] characters left to consume
+    #
+    def chars_left
+      @scanner.rest_size
+    end
+
+    # Counts characters from current position until a target string.
+    # Returns chars_left if target is not found.
+    #
+    # @param target [String] string to search for
+    # @return [Integer] count of chars until target or remaining chars
+    #
+    def chars_until(target)
+      found = @scanner.check_until(Regexp.new(Regexp.escape(target)))
+      return chars_left unless found
+
+      found.size - target.size
+    end
+
+    # Finds the byte position of the next occurrence of a character.
+    # Does not move the scanner position.
+    #
+    # @param ch [String] character to search for
+    # @return [Integer, nil] byte position or nil if not found
+    #
+    def index_of_char(ch)
+      rel_idx = @scanner.rest.index(ch)
+      return nil unless rel_idx
+
+      @scanner.pos + rel_idx
+    end
+
+    # Returns current byte position in input.
+    #
+    # @return [Integer] current byte offset
+    # @note Encoding-aware: position is in bytes, not characters
+    #
+    def pos
+      @scanner.pos
+    end
+    alias bytepos pos
+
+    # Sets the current byte position.
+    #
+    # @param new_pos [Integer] target byte position
+    #
+    def bytepos=(new_pos)
+      @scanner.pos = new_pos
+    rescue RangeError
+      # Silently ignore out-of-range positions
+    end
+
+    # Converts a byte position to line and column numbers.
+    #
+    # @param offset [Integer, nil] byte position (defaults to current)
+    # @return [Array<Integer, Integer>] [line, column] tuple (1-indexed)
+    #
+    def line_and_column(offset = nil)
+      effective = offset || @scanner.pos
+      @line_data.line_and_column(effective)
+    end
+
+    # Creates a pooled Position object for error reporting.
+    #
+    # @param offset [Integer, nil] byte position (defaults to current)
+    # @return [Parsanol::Position] pooled position instance
+    #
+    def position(offset = nil)
+      effective = offset || @scanner.pos
+      line_and_column(effective)
+
+      # Character position approximation
+      char_pos = @raw_string.byteslice(0, effective).size
+
+      @position_pool.acquire_with(
+        string: @raw_string,
+        bytepos: effective,
+        charpos: char_pos,
+      )
+    end
+  end
+end

data/lib/parsanol/source_location.rb

@@ -0,0 +1,167 @@
+# frozen_string_literal: true
+
+# Parsanol::SourceLocation - Source Location Tracking
+#
+# Track source positions (line, column, offset) through the parsing and
+# transformation pipeline. This is useful for error reporting, IDE integration,
+# and source mapping.
+#
+# Usage:
+#   # Parse with source tracking
+#   result = parser.parse_with_spans("hello world")
+#   tree = result.tree
+#   spans = result.spans
+#
+#   # Access span for a node
+#   span = spans[node_id]
+#   puts "Matched at line #{span.start.line}, column #{span.start.column}"
+#
+# Requires native extension for full functionality.
+
+module Parsanol
+  # Represents a position in source code
+  class SourcePosition
+    attr_reader :offset, :line, :column
+
+    def initialize(offset:, line:, column:)
+      @offset = offset
+      @line = line
+      @column = column
+    end
+
+    def to_s
+      "line #{@line}, column #{@column} (offset #{@offset})"
+    end
+
+    def to_h
+      { offset: @offset, line: @line, column: @column }
+    end
+
+    def ==(other)
+      return false unless other.is_a?(SourcePosition)
+
+      @offset == other.offset && @line == other.line && @column == other.column
+    end
+
+    def eql?(other)
+      self == other
+    end
+
+    def hash
+      [@offset, @line, @column].hash
+    end
+  end
+
+  # Represents a span in source code (from start to end position)
+  class SourceSpan
+    attr_reader :start, :end
+
+    def initialize(start_pos:, end_pos:)
+      @start = start_pos.is_a?(SourcePosition) ? start_pos : SourcePosition.new(**start_pos)
+      @end = end_pos.is_a?(SourcePosition) ? end_pos : SourcePosition.new(**end_pos)
+    end
+
+    # Create a span from offsets (computes line/column from input)
+    def self.from_offsets(input, start_offset, end_offset)
+      start_pos = compute_position(input, start_offset)
+      end_pos = compute_position(input, end_offset)
+      new(start_pos: start_pos, end_pos: end_pos)
+    end
+
+    # Merge two spans (returns a new span covering both)
+    def merge(other)
+      return self if other.nil?
+
+      SourceSpan.new(
+        start_pos: [@start, other.start].min_by(&:offset),
+        end_pos: [@end, other.end].max_by(&:offset),
+      )
+    end
+
+    # Check if this span overlaps with another
+    def overlaps?(other)
+      return false if other.nil?
+
+      @start.offset < other.end.offset && @end.offset > other.start.offset
+    end
+
+    # Check if this span is adjacent to another
+    def adjacent?(other)
+      return false if other.nil?
+
+      @end.offset == other.start.offset || other.end.offset == @start.offset
+    end
+
+    # Check if a position is within this span
+    def contains?(position)
+      offset = position.is_a?(SourcePosition) ? position.offset : position
+      offset.between?(@start.offset, @end.offset)
+    end
+
+    # Get the length of the span in bytes
+    def length
+      @end.offset - @start.offset
+    end
+
+    # Extract the source text from the input
+    def extract(input)
+      input.byteslice(@start.offset, length)
+    end
+
+    def to_s
+      "#{@start} - #{@end}"
+    end
+
+    def to_h
+      { start: @start.to_h, end: @end.to_h }
+    end
+
+    def ==(other)
+      return false unless other.is_a?(SourceSpan)
+
+      @start == other.start && @end == other.end
+    end
+
+    # Compute line and column from offset
+    def self.compute_position(input, offset)
+      line = 1
+      column = 1
+      current_offset = 0
+
+      input.each_char do |char|
+        break if current_offset >= offset
+
+        if char == "\n"
+          line += 1
+          column = 1
+        else
+          column += 1
+        end
+
+        current_offset += 1
+      end
+
+      SourcePosition.new(offset: offset, line: line, column: column)
+    end
+  end
+
+  # Result wrapper for parse_with_spans
+  class ParseResultWithSpans
+    attr_reader :tree, :spans
+
+    def initialize(tree:, spans:)
+      @tree = tree
+      @spans = spans
+    end
+
+    # Get span for a specific node
+    def span_for(node_id)
+      @spans[node_id]
+    end
+
+    # Get all spans that contain a position
+    def spans_at(offset)
+      @spans.values.select { |span| span.contains?(offset) }
+    end
+  end
+end

data/lib/parsanol/streaming_parser.rb

@@ -0,0 +1,124 @@
+# frozen_string_literal: true
+
+# Parsanol::StreamingParser - Streaming Parser for Large Inputs
+#
+# Parse large inputs in chunks without loading the entire input into memory.
+# Useful for file parsing, network streams, or very large documents.
+#
+# Usage:
+#   parser = Parsanol::StreamingParser.new(json_grammar)
+#
+#   File.open("large.json") do |f|
+#     parser.parse_stream(f) do |partial_result|
+#       # Process each complete element as it's parsed
+#       process_item(partial_result)
+#     end
+#   end
+#
+# Requires native extension for full functionality.
+
+module Parsanol
+  class StreamingParser
+    # Default chunk size (4KB)
+    DEFAULT_CHUNK_SIZE = 4096
+
+    # Create a new streaming parser
+    #
+    # @param grammar [Parsanol::Parser, Parsanol::Atoms::Base] Grammar to use
+    # @param chunk_size [Integer] Size of chunks to read (default: 4096)
+    def initialize(grammar, chunk_size: DEFAULT_CHUNK_SIZE)
+      @grammar = grammar
+      @chunk_size = chunk_size
+
+      if Parsanol::Native.available?
+        grammar_json = Parsanol::Native.serialize_grammar(grammar.root)
+        @native_parser = Parsanol::Native.streaming_parser_new(grammar_json)
+      else
+        @native_parser = nil
+      end
+
+      @buffer = +""
+      @position = 0
+    end
+
+    # Add a chunk of input
+    #
+    # @param chunk [String] Input chunk to add
+    # @return [Boolean] True if more chunks needed, false if ready for parsing
+    def add_chunk(chunk)
+      @buffer << chunk
+
+      if @native_parser
+        Parsanol::Native.streaming_parser_add_chunk(@native_parser, chunk)
+      else
+        # Pure Ruby fallback
+        false
+      end
+    end
+
+    # Parse what we have so far
+    #
+    # @return [Object, nil] Parsed result or nil if need more data
+    def parse_chunk
+      if @native_parser
+        Parsanol::Native.streaming_parser_parse_chunk(@native_parser)
+      else
+        # Pure Ruby fallback - not supported
+        raise NotImplementedError,
+              "Streaming parser requires native extension for full functionality."
+      end
+    end
+
+    # Check if we have enough data to make progress
+    #
+    # @return [Boolean] True if parser can make progress
+    def enough_data?
+      if @native_parser
+        !Parsanol::Native.streaming_parser_parse_chunk(@native_parser).nil?
+      else
+        false
+      end
+    end
+
+    # Parse entire stream (yields partial results)
+    #
+    # @param io [IO, StringIO] Input source to read from
+    # @param chunk_size [Integer] Size of chunks to read
+    # @yield [Object] Each complete element as it's parsed
+    # @return [Array] All parsed results
+    def parse_stream(io, chunk_size: @chunk_size)
+      results = []
+
+      loop do
+        chunk = io.read(chunk_size)
+        break if chunk.nil? || chunk.empty?
+
+        add_chunk(chunk)
+
+        while (result = parse_chunk)
+          results << result
+          yield result if block_given?
+        end
+      end
+
+      results
+    end
+
+    # Reset the parser for reuse
+    def reset
+      @buffer = +""
+      @position = 0
+
+      return unless @native_parser
+
+      grammar_json = Parsanol::Native.serialize_grammar(@grammar.root)
+      @native_parser = Parsanol::Native.streaming_parser_new(grammar_json)
+    end
+
+    # Get the current buffer
+    attr_reader :buffer
+
+    # Get the chunk size
+    attr_reader :chunk_size
+  end
+end