parsanol 3.0.0

data/lib/parsanol/interval_tree.rb

@@ -0,0 +1,215 @@
+# Interval tree implementation for GPeg-style incremental parsing
+# Based on the GPeg paper: "Fast Incremental PEG Parsing" (Yedidia, SLE 2021)
+#
+# This data structure stores memoization results keyed by position intervals [start, end)
+# rather than single positions, enabling efficient invalidation of changed regions.
+#
+# Performance characteristics:
+# - Insert: O(log n)
+# - Query: O(log n + k) where k is number of overlapping intervals
+# - Delete overlapping: O(log n + k)
+#
+class Parsanol::IntervalTree
+  # A node in the interval tree
+  # Each node stores an interval [low, high) and associated data
+  class Node
+    attr_accessor :interval, :data, :max, :left, :right
+
+    def initialize(low, high, data)
+      @interval = [low, high]  # [start, end) half-open interval
+      @data = data
+      @max = high  # Maximum endpoint in subtree
+      @left = nil
+      @right = nil
+    end
+
+    def low
+      @interval[0]
+    end
+
+    def high
+      @interval[1]
+    end
+  end
+
+  def initialize
+    @root = nil
+    @size = 0
+  end
+
+  attr_reader :size
+
+  # Insert an interval with associated data
+  # @param low [Integer] Start position (inclusive)
+  # @param high [Integer] End position (exclusive)
+  # @param data [Object] Data to associate with this interval
+  def insert(low, high, data)
+    @root = insert_recursive(@root, low, high, data)
+    @size += 1
+  end
+
+  # Query for all intervals that overlap with [low, high)
+  # @param low [Integer] Start position (inclusive)
+  # @param high [Integer] End position (exclusive)
+  # @return [Array<Object>] Array of data from overlapping intervals
+  def query_overlapping(low, high)
+    # Empty intervals cannot overlap with anything
+    return [] if low >= high
+
+    results = []
+    query_recursive(@root, low, high, results)
+    results
+  end
+
+  # Query for exact interval match
+  # @param low [Integer] Start position (inclusive)
+  # @param high [Integer] End position (exclusive)
+  # @return [Object, nil] Data if exact match found, nil otherwise
+  def query_exact(low, high)
+    find_exact(@root, low, high)
+  end
+
+  # Delete all intervals that overlap with [low, high)
+  # Returns array of deleted data
+  # @param low [Integer] Start position (inclusive)
+  # @param high [Integer] End position (exclusive)
+  # @return [Array<Object>] Array of data from deleted intervals
+  def delete_overlapping(low, high)
+    deleted = []
+    @root = delete_overlapping_recursive(@root, low, high, deleted)
+    @size -= deleted.size
+    deleted
+  end
+
+  # Clear all intervals
+  def clear
+    @root = nil
+    @size = 0
+  end
+
+  # Check if tree is empty
+  def empty?
+    @root.nil?
+  end
+
+  private
+
+  # Insert node recursively maintaining BST property on interval start
+  def insert_recursive(node, low, high, data)
+    return Node.new(low, high, data) if node.nil?
+
+    # BST insertion based on interval start position
+    if low < node.low
+      node.left = insert_recursive(node.left, low, high, data)
+    else
+      node.right = insert_recursive(node.right, low, high, data)
+    end
+
+    # Update max endpoint in this subtree
+    node.max = [node.max, high].max
+    node.max = [node.max, node.left.max].max if node.left
+    node.max = [node.max, node.right.max].max if node.right
+
+    node
+  end
+
+  # Query recursively for overlapping intervals
+  def query_recursive(node, low, high, results)
+    return if node.nil?
+
+    # If no interval in this subtree can overlap, prune search
+    return if node.max <= low
+
+    # Check left subtree (may have overlapping intervals)
+    query_recursive(node.left, low, high, results) if node.left
+
+    # Check current node for overlap
+    # Two intervals [a,b) and [c,d) overlap if: a < d AND c < b
+    if node.low < high && low < node.high
+      results << node.data
+    end
+
+    # Check right subtree
+    # Only search right if intervals starting there could overlap
+    query_recursive(node.right, low, high, results) if node.right && node.low < high
+  end
+
+  # Find exact interval match
+  def find_exact(node, low, high)
+    return nil if node.nil?
+
+    if node.low == low && node.high == high
+      return node.data
+    end
+
+    # Search in appropriate subtree
+    if low < node.low
+      find_exact(node.left, low, high)
+    else
+      find_exact(node.right, low, high)
+    end
+  end
+
+  # Delete overlapping intervals recursively
+  def delete_overlapping_recursive(node, low, high, deleted)
+    return nil if node.nil?
+
+    # Recursively delete from left subtree
+    node.left = delete_overlapping_recursive(node.left, low, high, deleted) if node.left
+
+    # Recursively delete from right subtree
+    node.right = delete_overlapping_recursive(node.right, low, high, deleted) if node.right
+
+    # Check if current node overlaps
+    if node.low < high && low < node.high
+      # This node overlaps - delete it
+      deleted << node.data
+
+      # Remove this node and reinsert children
+      if node.left.nil?
+        return node.right
+      elsif node.right.nil?
+        return node.left
+      else
+        # Node has two children - replace with inorder successor
+        # Find minimum node in right subtree
+        min_node = find_min(node.right)
+
+        # Replace current node's interval and data with successor's
+        node.interval = min_node.interval
+        node.data = min_node.data
+
+        # Delete the successor from right subtree
+        node.right = delete_min(node.right)
+      end
+    end
+
+    # Update max for this node after potential deletions
+    if node
+      node.max = node.high
+      node.max = [node.max, node.left.max].max if node.left
+      node.max = [node.max, node.right.max].max if node.right
+    end
+
+    node
+  end
+
+  # Find minimum node in subtree (leftmost)
+  def find_min(node)
+    return node if node.left.nil?
+    find_min(node.left)
+  end
+
+  # Delete minimum node from subtree
+  def delete_min(node)
+    return node.right if node.left.nil?
+    node.left = delete_min(node.left)
+
+    # Update max
+    node.max = node.high
+    node.max = [node.max, node.left.max].max if node.left
+    node.max = [node.max, node.right.max].max if node.right
+
+    node
+  end
+end

data/lib/parsanol/lazy_result.rb

@@ -0,0 +1,178 @@
+# frozen_string_literal: true
+
+module Parsanol
+  # Lazy wrapper around Buffer that defers array materialization.
+  #
+  # LazyResult wraps a Buffer and only creates an Array when the result
+  # is actually accessed. This reduces allocations for results that are
+  # never used (cache hits, backtracking, etc.).
+  #
+  # == Usage
+  #
+  #   lazy = LazyResult.new(buffer, context)
+  #   # No array allocated yet
+  #   
+  #   lazy.to_a  # Now array is materialized and cached
+  #   lazy.to_a  # Returns cached array
+  #
+  # == Transparency
+  #
+  # LazyResult acts like an Array for most operations:
+  # - Enumerable methods work (each, map, select, etc.)
+  # - Array access works ([], size, empty?, etc.)
+  # - Can be used in transforms without changes
+  #
+  class LazyResult
+    # @return [Buffer] The underlying buffer
+    attr_reader :buffer
+    
+    # @return [Context] The context (for buffer release)
+    attr_reader :context
+    
+    # @return [Array, nil] Cached materialized array
+    attr_reader :materialized
+
+    # Initialize a new LazyResult.
+    #
+    # @param buffer [Buffer] Buffer containing elements
+    # @param context [Context] Context for buffer management
+    #
+    def initialize(buffer, context)
+      @buffer = buffer
+      @context = context
+      @materialized = nil
+    end
+
+    # Materialize to array (with caching).
+    #
+    # First call creates array from buffer, subsequent calls return cached.
+    #
+    # @return [Array] Materialized array
+    #
+    def to_a
+      @materialized ||= @buffer.to_a
+    end
+
+    # Get element at index (materializes if needed).
+    #
+    # @param index [Integer] Zero-based index
+    # @return [Object] Element at index
+    #
+    def [](index)
+      to_a[index]
+    end
+
+    # Get number of elements.
+    #
+    # @return [Integer] Number of elements
+    #
+    def size
+      @buffer.size
+    end
+    
+    alias length size
+
+    # Check if empty.
+    #
+    # @return [Boolean] true if no elements
+    #
+    def empty?
+      @buffer.empty?
+    end
+
+    # Iterate over elements (materializes if needed).
+    #
+    # @yield [element] Each element
+    # @return [Enumerator, self] Enumerator if no block, self otherwise
+    #
+    def each(&block)
+      return to_enum(:each) unless block_given?
+      to_a.each(&block)
+      self
+    end
+
+    # Check if acts like an array.
+    #
+    # @param other [Class] Class to check against
+    # @return [Boolean] true if Array
+    #
+    def is_a?(other)
+      other == Array || super
+    end
+    
+    alias kind_of? is_a?
+
+    # Respond to array methods.
+    #
+    # @param method [Symbol] Method name
+    # @param include_private [Boolean] Include private methods
+    # @return [Boolean] true if responds
+    #
+    def respond_to?(method, include_private = false)
+      super || to_a.respond_to?(method, include_private)
+    end
+
+    # Delegate unknown methods to materialized array.
+    #
+    # @param method [Symbol] Method name
+    # @param args [Array] Arguments
+    # @param block [Proc] Block if given
+    # @return [Object] Result of method call
+    #
+    def method_missing(method, *args, &block)
+      if to_a.respond_to?(method)
+        to_a.public_send(method, *args, &block)
+      else
+        super
+      end
+    end
+
+    # Support respond_to_missing? for proper method_missing implementation.
+    #
+    # @param method [Symbol] Method name
+    # @param include_private [Boolean] Include private methods
+    # @return [Boolean] true if method is supported
+    #
+    def respond_to_missing?(method, include_private = false)
+      to_a.respond_to?(method, include_private) || super
+    end
+
+    # Compare with another object.
+    # LazyResult compares equal to arrays with the same content.
+    #
+    # @param other [Object] Object to compare with
+    # @return [Boolean] true if equal
+    #
+    def ==(other)
+      if other.is_a?(Array)
+        to_a == other
+      elsif other.is_a?(LazyResult)
+        to_a == other.to_a
+      else
+        super
+      end
+    end
+    
+    alias eql? ==
+
+    # Hash code based on materialized array.
+    #
+    # @return [Integer] Hash code
+    #
+    def hash
+      to_a.hash
+    end
+
+    # Inspect for debugging.
+    #
+    # @return [String] Inspection string
+    #
+    def inspect
+      if @materialized
+        "#<LazyResult:#{object_id} materialized=#{@materialized.inspect}>"
+      else
+        "#<LazyResult:#{object_id} buffer.size=#{@buffer.size}>"
+      end
+    end
+  end
+end

data/lib/parsanol/lexer.rb

@@ -0,0 +1,146 @@
+# frozen_string_literal: true
+
+require "parsanol/native"
+
+module Parsanol
+  # Generic lexer for fast tokenization
+  #
+  # Create a lexer by subclassing and defining tokens:
+  #
+  #   class JsonLexer < Parsanol::Lexer
+  #     token :string, /"[^"]*"/
+  #     token :number, /-?[0-9]+(\.[0-9]+)?/
+  #     token :true, /true/
+  #     token :false, /false/
+  #     token :null, /null/
+  #     token :lbrace, /\{/
+  #     token :rbrace, /\}/
+  #     token :lbracket, /\[/
+  #     token :rbracket, /\]/
+  #     token :colon, /:/
+  #     token :comma, /,/
+  #
+  #     ignore /\s+/
+  #   end
+  #
+  #   lexer = JsonLexer.new
+  #   tokens = lexer.tokenize('{"name": "test"}')
+  #
+  class Lexer
+    class << self
+      # Define a token pattern
+      #
+      # @param name [Symbol] Token type name
+      # @param pattern [Regexp] Pattern to match
+      # @param priority [Integer] Priority for conflict resolution (higher = preferred)
+      # @param block [Proc] Optional block to transform the matched value
+      def token(name, pattern, priority: 0, &block)
+        token_definitions << Definition.new(
+          name: name.to_s,
+          pattern: pattern.source,
+          priority: priority,
+          ignore: false,
+          transform: block
+        )
+      end
+
+      # Define patterns to ignore (e.g., whitespace, comments)
+      #
+      # @param pattern [Regexp] Pattern to ignore
+      def ignore(pattern)
+        token_definitions << Definition.new(
+          name: "__ignore__",
+          pattern: pattern.source,
+          priority: 0,
+          ignore: true,
+          transform: nil
+        )
+      end
+
+      # Define keywords (identifiers with higher priority)
+      #
+      # @param keywords [Array<Symbol>] Keyword names
+      # @param priority [Integer] Priority (default: 100)
+      def keyword(*keywords, priority: 100)
+        keywords.each do |kw|
+          token_definitions << Definition.new(
+            name: kw.to_s.upcase,
+            pattern: Regexp.new(Regexp.escape(kw.to_s), Regexp::IGNORECASE).source,
+            priority: priority,
+            ignore: false,
+            transform: nil
+          )
+        end
+      end
+
+      # Get token definitions for this lexer class
+      #
+      # @return [Array<Definition>] Token definitions
+      def token_definitions
+        @token_definitions ||= []
+      end
+
+      # Inherit token definitions from parent class
+      def inherited(subclass)
+        super
+        subclass.instance_variable_set(:@token_definitions, token_definitions.dup)
+      end
+    end
+
+    # Token definition
+    Definition = Struct.new(:name, :pattern, :priority, :ignore, :transform)
+
+    # Initialize the lexer
+    def initialize
+      @lexer_id = nil
+      @transforms = build_transforms
+    end
+
+    # Tokenize input string
+    #
+    # @param input [String] Input to tokenize
+    # @return [Array<Hash>] Array of tokens with type, value, and location
+    def tokenize(input)
+      ensure_lexer_created
+
+      tokens = Native.tokenize_with_lexer(@lexer_id, input)
+
+      # Apply any transforms
+      tokens.map do |token|
+        transform = @transforms[token["type"]]
+        if transform
+          token = token.dup
+          token["value"] = transform.call(token["value"])
+        end
+        token
+      end
+    end
+
+    private
+
+    def ensure_lexer_created
+      return if @lexer_id
+
+      definitions = self.class.token_definitions.map do |d|
+        {
+          "name" => d.name,
+          "pattern" => d.pattern,
+          "priority" => d.priority,
+          "ignore" => d.ignore
+        }
+      end
+
+      @lexer_id = Native.create_lexer(definitions)
+    end
+
+    def build_transforms
+      transforms = {}
+      self.class.token_definitions.each do |d|
+        if d.transform && !d.ignore
+          transforms[d.name] = d.transform
+        end
+      end
+      transforms
+    end
+  end
+end