parsanol 3.0.0

data/lib/parsanol/slice.rb

@@ -0,0 +1,172 @@
+# frozen_string_literal: true
+# A slice is a small part from the parse input. A slice mainly behaves like
+# any other string, except that it remembers where it came from (offset in
+# original input).
+#
+# == Extracting line and column
+#
+# Using the #line_and_column method, you can extract the line and column in
+# the original input where this slice starts.
+#
+# Example:
+#   slice.line_and_column # => [1, 13]
+#   slice.offset          # => 12
+#
+# == Likeness to strings
+#
+# Parsanol::Slice behaves in many ways like a Ruby String. This likeness
+# however is not complete - many of the myriad of operations String supports
+# are not yet in Slice. You can always extract the internal string instance by
+# calling #to_s.
+#
+# These omissions are somewhat intentional. Rather than maintaining a full
+# delegation, we opt for a partial emulation that gets the job done.
+#
+class Parsanol::Slice
+  attr_reader :str, :line_cache
+
+  # Construct a slice using an integer byte position, a string, and an optional line cache.
+  # The line cache should be able to answer to the #line_and_column message.
+  #
+  # @param bytepos [Integer] Byte position in the original input
+  # @param string [String] The slice content
+  # @param line_cache [Object] Optional line cache for line/column info
+  #
+  def initialize(bytepos=0, string='', line_cache = nil)
+    @bytepos = bytepos
+    @str = string
+    @line_cache = line_cache
+  end
+
+  # Reset the slice for reuse in object pooling.
+  # This allows the slice to be reinitialized with new values for efficient reuse.
+  #
+  # @param bytepos [Integer] New byte position in the original input
+  # @param string [String] New slice content
+  # @param line_cache [Object] Optional line cache for line/column info
+  # @return [self] Returns self for method chaining
+  #
+  def reset!(bytepos=0, string='', line_cache=nil)
+    @bytepos = bytepos
+    @str = string
+    @line_cache = line_cache
+    self
+  end
+
+  # Create a Slice from a Rope.
+  # The rope is converted to a string and used to create the slice.
+  #
+  # @param rope [Parsanol::Rope] The rope to convert
+  # @param bytepos [Integer] Byte position in the input
+  # @param line_cache [Object] Optional line cache for line/column info
+  # @return [Parsanol::Slice] A new slice with the rope's content
+  #
+  def self.from_rope(rope, bytepos, line_cache = nil)
+    new(bytepos, rope.to_s, line_cache)
+  end
+
+  # Returns the byte position of this slice in the original input.
+  # This is the primary position tracking mechanism.
+  #
+  def offset
+    @bytepos
+  end
+
+  # Alias for offset - returns byte position.
+  # For backward compatibility and clarity.
+  #
+  alias bytepos offset
+
+  # Alias for offset - returns byte position.
+  # Note: For ASCII text, bytepos == charpos.
+  # For UTF-8, this is an approximation (byte position, not character position).
+  #
+  alias charpos offset
+
+  # Compares slices to other slices or strings.
+  # Fast path: Compare strings directly, most common case
+  #
+  def ==(other)
+    # Fast path: direct string comparison
+    return str == other if other.is_a?(String)
+    # Slice to Slice comparison
+    return str == other.str if other.is_a?(Parsanol::Slice)
+    str == other
+  end
+
+  # Type-strict equality comparison.
+  # This only returns true for Slice-to-Slice comparison with equal content.
+  #
+  def eql?(other)
+    other.is_a?(Parsanol::Slice) && str.eql?(other.str)
+  end
+
+  # Hash code for using Slices as hash keys.
+  # Incorporates both the string content and position to distinguish
+  # Slices from plain Strings and from Slices at different positions.
+  #
+  def hash
+    [str, offset].hash
+  end
+
+  # Match regular expressions.
+  #
+  def match(regexp)
+    str.match(regexp)
+  end
+
+  # Returns the slices size in characters.
+  #
+  def size
+    str.size
+  end
+
+  alias length size
+
+  # Concatenate two slices; it is assumed that the second slice begins
+  # where the first one ends. The offset of the resulting slice is the same
+  # as the one of this slice.
+  #
+  def +(other)
+    self.class.new(@bytepos, str + other.to_s, line_cache)
+  end
+
+  # Returns a <line, column> tuple referring to the original input.
+  # LineCache expects an integer byte position.
+  #
+  def line_and_column
+    raise ArgumentError, 'No line cache was given, cannot infer line and column.' \
+      unless line_cache
+
+    line_cache.line_and_column(@bytepos)
+  end
+
+  # Conversion operators -----------------------------------------------------
+  def to_str
+    str.is_a?(String) ? str : str.to_s
+  end
+  alias to_s to_str
+
+  def to_slice
+    self
+  end
+
+  def to_sym
+    str.to_sym
+  end
+
+  def to_i
+    self.str.to_i
+  end
+
+  def to_f
+    str.to_f
+  end
+
+  # Inspection & Debugging ---------------------------------------------------
+
+  # Prints the slice as <code>"string"@offset</code>.
+  def inspect
+    str.inspect + "@#{offset}"
+  end
+end

data/lib/parsanol/source/line_cache.rb

@@ -0,0 +1,99 @@
+
+
+class Parsanol::Source
+  # A cache for line start positions. 
+  #
+  class LineCache 
+    def initialize
+      # Stores line endings as a simple position number. The first line always
+      # starts at 0; numbers beyond the biggest entry are on any line > size, 
+      # but probably make a scan to that position neccessary.
+      @line_ends = []
+      @line_ends.extend RangeSearch
+      @last_line_end = nil
+    end
+
+    # Returns a <line, column> tuple for the given input position. Input
+    # position must be given as byte offset into original string. 
+    #
+    def line_and_column(pos)
+      pos = pos.bytepos if pos.respond_to? :bytepos
+      eol_idx = @line_ends.lbound(pos)
+
+      if eol_idx
+        # eol_idx points to the offset that ends the current line.
+        # Let's try to find the offset that starts it: 
+        offset = eol_idx>0 && @line_ends[eol_idx-1] || 0
+        return [eol_idx+1, pos-offset+1]
+      else
+        # eol_idx is nil, that means that we're beyond the last line end that
+        # we know about. Pretend for now that we're just on the last line.
+        offset = @line_ends.last || 0
+        return [@line_ends.size+1, pos-offset+1]
+      end
+    end
+
+    def scan_for_line_endings(start_pos, buf)
+      return unless buf
+
+      buf = StringScanner.new(buf)
+      return unless buf.exist?(/\n/)
+
+      ## If we have already read part or all of buf, we already know about
+      ## line ends in that portion. remove it and correct cur (search index)
+      if @last_line_end && start_pos < @last_line_end
+        # Let's not search the range from start_pos to last_line_end again.
+        buf.pos = @last_line_end - start_pos
+      end
+
+      ## Scan the string for line endings; store the positions of all endings
+      ## in @line_ends. 
+      while buf.skip_until(/\n/)
+        @last_line_end = start_pos + buf.pos
+        @line_ends << @last_line_end
+      end
+    end
+  end
+
+  # Mixin for arrays that implicitly give a number of ranges, where one range
+  # begins where the other one ends.
+  # 
+  #   Example: 
+  #
+  #     [10, 20, 30]
+  #     # would describe [0, 10], (10, 20], (20, 30]
+  #
+  module RangeSearch 
+    def find_mid(left, right)
+      # NOTE: Jonathan Hinkle reported that when mathn is required, just
+      # dividing and relying on the integer truncation is not enough.
+      left + ((right - left) / 2).floor
+    end  
+    
+    # Scans the array for the first number that is > than bound. Returns the 
+    # index of that number. 
+    #
+    def lbound(bound)
+      return nil if empty?
+      return nil unless last > bound
+
+      left = 0
+      right = size - 1 
+
+      loop do
+        mid = find_mid(left, right)
+
+        if self[mid] > bound
+          right = mid
+        else
+          # assert: self[mid] <= bound
+          left = mid+1
+        end
+
+        if right <= left
+          return right
+        end
+      end
+    end
+  end
+end

data/lib/parsanol/source.rb

@@ -0,0 +1,171 @@
+# frozen_string_literal: true
+
+require 'stringio'
+require 'strscan'
+
+require 'parsanol/position'
+require 'parsanol/source/line_cache'
+require 'parsanol/pools/slice_pool'
+require 'parsanol/pools/position_pool'
+
+module Parsanol
+  # Wraps the input string for parslet.
+  #
+  class Source
+    attr_reader :slice_pool, :position_pool
+    
+    def initialize(str)
+      raise(
+        ArgumentError,
+        "Must construct Source with a string like object."
+      ) unless str.respond_to?(:to_str)
+
+      @str = StringScanner.new(str)
+      @source_str = str.to_str
+
+      # maps 1 => /./m, 2 => /../m, etc...
+      @re_cache = Hash.new { |h,k|
+        h[k] = /(.|$){#{k}}/m }
+
+      @line_cache = LineCache.new
+      @line_cache.scan_for_line_endings(0, str)
+      
+      # Phase 1.2: SlicePool for reducing GC pressure during parsing
+      # Size of 5000 handles most typical parsing scenarios
+      @slice_pool = Parsanol::Pools::SlicePool.new(size: 5000)
+      
+      # Phase 1.4: PositionPool for error reporting positions
+      # Size of 1000 handles typical error reporting scenarios
+      @position_pool = Parsanol::Pools::PositionPool.new(size: 1000)
+    end
+
+    # Checks if the given pattern matches at the current input position.
+    #
+    # @param pattern [Regexp] pattern to check for
+    # @return [Boolean] true if the pattern matches at #pos
+    #
+    def matches?(pattern)
+      @str.match?(pattern)
+    end
+    alias match matches?
+
+    # Consumes n characters from the input, returning them as a slice of the
+    # input.
+    #
+    def consume(n)
+      bytepos = self.bytepos
+      slice_str = @str.scan(@re_cache[n])
+      slice = @slice_pool.acquire_with(bytepos, slice_str, @line_cache)
+
+      return slice
+    end
+    
+    # Helper method to acquire a pooled slice.
+    # This is the preferred way for atoms to create slices.
+    #
+    # @param bytepos [Integer] Byte position in the input
+    # @param str [String] The slice content
+    # @return [Parsanol::Slice] A pooled slice ready for use
+    #
+    def slice(bytepos, str)
+      @slice_pool.acquire_with(bytepos, str, @line_cache)
+    end
+    
+    # Release a slice back to the pool.
+    # Currently not actively used - slices are released when pool is recycled.
+    #
+    # @param slice [Parsanol::Slice] The slice to release
+    #
+    def release_slice(slice)
+      @slice_pool.release(slice)
+    end
+
+    # Returns how many chars remain in the input.
+    #
+    def chars_left
+      @str.rest_size
+    end
+
+    # Returns how many chars there are between current position and the
+    # string given. If the string given doesn't occur in the source, then
+    # the remaining chars (#chars_left) are returned.
+    #
+    # @return [Fixnum] count of chars until str or #chars_left
+    #
+    def chars_until str
+      slice_str = @str.check_until(Regexp.new(Regexp.escape(str)))
+      return chars_left unless slice_str
+      return slice_str.size - str.size
+    end
+
+    # Phase 31: Scan forward to find the next occurrence of a character.
+    # Returns the byte position of the next occurrence, or nil if not found.
+    # Does not move the scanner position.
+    #
+    # @param char [String] single character to search for
+    # @return [Integer, nil] byte position or nil if not found
+    #
+    def index_of_char(char)
+      # Use StringScanner's string directly for fast indexOf
+      idx = @str.rest.index(char)
+      return nil unless idx
+      @str.pos + idx
+    end
+
+    # Position of the parse as a byte offset into the original string.
+    # Returns an integer byte position instead of a Position object.
+    #
+    # @return [Integer] Current byte position in the input
+    # @note Please be aware of encodings at this point.
+    #
+    def pos
+      @str.pos
+    end
+
+    # Alias for pos - returns the current byte position.
+    # Provided for clarity and backward compatibility.
+    #
+    # @return [Integer] Current byte position in the input
+    #
+    alias bytepos pos
+
+    # @note Please be aware of encodings at this point.
+    #
+    def bytepos=(n)
+      @str.pos = n
+    rescue RangeError
+    end
+
+    # Returns a <line, column> tuple for the given position. If no position is
+    # given, line/column information is returned for the current position
+    # given by #pos.
+    #
+    def line_and_column(position=nil)
+      @line_cache.line_and_column(position || self.bytepos)
+    end
+    
+    # Creates a pooled Position object for the given byte position.
+    # This is used for error reporting where Position objects are needed.
+    #
+    # Phase 1.4: Uses PositionPool to reduce GC pressure when materializing
+    # position objects for error messages.
+    #
+    # @param bytepos [Integer] Byte position in source (defaults to current position)
+    # @return [Parsanol::Position] A pooled Position object
+    #
+    def position(bytepos = nil)
+      effective_pos = bytepos || self.bytepos
+      line, column = line_and_column(effective_pos)
+      
+      # Calculate character position (approximation for ASCII, exact for UTF-8)
+      charpos = @source_str.byteslice(0, effective_pos).size
+      
+      @position_pool.acquire_with(
+        string: @source_str,
+        bytepos: effective_pos,
+        charpos: charpos
+      )
+    end
+    
+  end
+end

data/lib/parsanol/source_location.rb

@@ -0,0 +1,164 @@
+# frozen_string_literal: true
+
+# Parsanol::SourceLocation - Source Location Tracking
+#
+# Track source positions (line, column, offset) through the parsing and
+# transformation pipeline. This is useful for error reporting, IDE integration,
+# and source mapping.
+#
+# Usage:
+#   # Parse with source tracking
+#   result = parser.parse_with_spans("hello world")
+#   tree = result.tree
+#   spans = result.spans
+#
+#   # Access span for a node
+#   span = spans[node_id]
+#   puts "Matched at line #{span.start.line}, column #{span.start.column}"
+#
+# Requires native extension for full functionality.
+
+module Parsanol
+  # Represents a position in source code
+  class SourcePosition
+    attr_reader :offset, :line, :column
+
+    def initialize(offset:, line:, column:)
+      @offset = offset
+      @line = line
+      @column = column
+    end
+
+    def to_s
+      "line #{@line}, column #{@column} (offset #{@offset})"
+    end
+
+    def to_h
+      { offset: @offset, line: @line, column: @column }
+    end
+
+    def ==(other)
+      return false unless other.is_a?(SourcePosition)
+      @offset == other.offset && @line == other.line && @column == other.column
+    end
+
+    def eql?(other)
+      self == other
+    end
+
+    def hash
+      [@offset, @line, @column].hash
+    end
+  end
+
+  # Represents a span in source code (from start to end position)
+  class SourceSpan
+    attr_reader :start, :end
+
+    def initialize(start_pos:, end_pos:)
+      @start = start_pos.is_a?(SourcePosition) ? start_pos : SourcePosition.new(**start_pos)
+      @end = end_pos.is_a?(SourcePosition) ? end_pos : SourcePosition.new(**end_pos)
+    end
+
+    # Create a span from offsets (computes line/column from input)
+    def self.from_offsets(input, start_offset, end_offset)
+      start_pos = compute_position(input, start_offset)
+      end_pos = compute_position(input, end_offset)
+      new(start_pos: start_pos, end_pos: end_pos)
+    end
+
+    # Merge two spans (returns a new span covering both)
+    def merge(other)
+      return self if other.nil?
+      SourceSpan.new(
+        start_pos: [@start, other.start].min_by(&:offset),
+        end_pos: [@end, other.end].max_by(&:offset)
+      )
+    end
+
+    # Check if this span overlaps with another
+    def overlaps?(other)
+      return false if other.nil?
+      @start.offset < other.end.offset && @end.offset > other.start.offset
+    end
+
+    # Check if this span is adjacent to another
+    def adjacent?(other)
+      return false if other.nil?
+      @end.offset == other.start.offset || other.end.offset == @start.offset
+    end
+
+    # Check if a position is within this span
+    def contains?(position)
+      offset = position.is_a?(SourcePosition) ? position.offset : position
+      offset >= @start.offset && offset <= @end.offset
+    end
+
+    # Get the length of the span in bytes
+    def length
+      @end.offset - @start.offset
+    end
+
+    # Extract the source text from the input
+    def extract(input)
+      input.byteslice(@start.offset, length)
+    end
+
+    def to_s
+      "#{@start} - #{@end}"
+    end
+
+    def to_h
+      { start: @start.to_h, end: @end.to_h }
+    end
+
+    def ==(other)
+      return false unless other.is_a?(SourceSpan)
+      @start == other.start && @end == other.end
+    end
+
+    private
+
+    # Compute line and column from offset
+    def self.compute_position(input, offset)
+      line = 1
+      column = 1
+      current_offset = 0
+
+      input.each_char do |char|
+        break if current_offset >= offset
+
+        if char == "\n"
+          line += 1
+          column = 1
+        else
+          column += 1
+        end
+
+        current_offset += 1
+      end
+
+      SourcePosition.new(offset: offset, line: line, column: column)
+    end
+  end
+
+  # Result wrapper for parse_with_spans
+  class ParseResultWithSpans
+    attr_reader :tree, :spans
+
+    def initialize(tree:, spans:)
+      @tree = tree
+      @spans = spans
+    end
+
+    # Get span for a specific node
+    def span_for(node_id)
+      @spans[node_id]
+    end
+
+    # Get all spans that contain a position
+    def spans_at(offset)
+      @spans.values.select { |span| span.contains?(offset) }
+    end
+  end
+end