parsanol 1.0.1-aarch64-linux

data/lib/parsanol/interval_tree.rb

@@ -0,0 +1,217 @@
+# frozen_string_literal: true
+
+# Interval tree implementation for GPeg-style incremental parsing
+# Based on the GPeg paper: "Fast Incremental PEG Parsing" (Yedidia, SLE 2021)
+#
+# This data structure stores memoization results keyed by position intervals [start, end)
+# rather than single positions, enabling efficient invalidation of changed regions.
+#
+# Performance characteristics:
+# - Insert: O(log n)
+# - Query: O(log n + k) where k is number of overlapping intervals
+# - Delete overlapping: O(log n + k)
+#
+module Parsanol
+  class IntervalTree
+    # A node in the interval tree
+    # Each node stores an interval [low, high) and associated data
+    class Node
+      attr_accessor :interval, :data, :max, :left, :right
+
+      def initialize(low, high, data)
+        @interval = [low, high] # [start, end) half-open interval
+        @data = data
+        @max = high # Maximum endpoint in subtree
+        @left = nil
+        @right = nil
+      end
+
+      def low
+        @interval[0]
+      end
+
+      def high
+        @interval[1]
+      end
+    end
+
+    def initialize
+      @root = nil
+      @size = 0
+    end
+
+    attr_reader :size
+
+    # Insert an interval with associated data
+    # @param low [Integer] Start position (inclusive)
+    # @param high [Integer] End position (exclusive)
+    # @param data [Object] Data to associate with this interval
+    def insert(low, high, data)
+      @root = insert_recursive(@root, low, high, data)
+      @size += 1
+    end
+
+    # Query for all intervals that overlap with [low, high)
+    # @param low [Integer] Start position (inclusive)
+    # @param high [Integer] End position (exclusive)
+    # @return [Array<Object>] Array of data from overlapping intervals
+    def query_overlapping(low, high)
+      # Empty intervals cannot overlap with anything
+      return [] if low >= high
+
+      results = []
+      query_recursive(@root, low, high, results)
+      results
+    end
+
+    # Query for exact interval match
+    # @param low [Integer] Start position (inclusive)
+    # @param high [Integer] End position (exclusive)
+    # @return [Object, nil] Data if exact match found, nil otherwise
+    def query_exact(low, high)
+      find_exact(@root, low, high)
+    end
+
+    # Delete all intervals that overlap with [low, high)
+    # Returns array of deleted data
+    # @param low [Integer] Start position (inclusive)
+    # @param high [Integer] End position (exclusive)
+    # @return [Array<Object>] Array of data from deleted intervals
+    def delete_overlapping(low, high)
+      deleted = []
+      @root = delete_overlapping_recursive(@root, low, high, deleted)
+      @size -= deleted.size
+      deleted
+    end
+
+    # Clear all intervals
+    def clear
+      @root = nil
+      @size = 0
+    end
+
+    # Check if tree is empty
+    def empty?
+      @root.nil?
+    end
+
+    private
+
+    # Insert node recursively maintaining BST property on interval start
+    def insert_recursive(node, low, high, data)
+      return Node.new(low, high, data) if node.nil?
+
+      # BST insertion based on interval start position
+      if low < node.low
+        node.left = insert_recursive(node.left, low, high, data)
+      else
+        node.right = insert_recursive(node.right, low, high, data)
+      end
+
+      # Update max endpoint in this subtree
+      node.max = [node.max, high].max
+      node.max = [node.max, node.left.max].max if node.left
+      node.max = [node.max, node.right.max].max if node.right
+
+      node
+    end
+
+    # Query recursively for overlapping intervals
+    def query_recursive(node, low, high, results)
+      return if node.nil?
+
+      # If no interval in this subtree can overlap, prune search
+      return if node.max <= low
+
+      # Check left subtree (may have overlapping intervals)
+      query_recursive(node.left, low, high, results) if node.left
+
+      # Check current node for overlap
+      # Two intervals [a,b) and [c,d) overlap if: a < d AND c < b
+      results << node.data if node.low < high && low < node.high
+
+      # Check right subtree
+      # Only search right if intervals starting there could overlap
+      query_recursive(node.right, low, high, results) if node.right && node.low < high
+    end
+
+    # Find exact interval match
+    def find_exact(node, low, high)
+      return nil if node.nil?
+
+      return node.data if node.low == low && node.high == high
+
+      # Search in appropriate subtree
+      if low < node.low
+        find_exact(node.left, low, high)
+      else
+        find_exact(node.right, low, high)
+      end
+    end
+
+    # Delete overlapping intervals recursively
+    def delete_overlapping_recursive(node, low, high, deleted)
+      return nil if node.nil?
+
+      # Recursively delete from left subtree
+      node.left = delete_overlapping_recursive(node.left, low, high, deleted) if node.left
+
+      # Recursively delete from right subtree
+      node.right = delete_overlapping_recursive(node.right, low, high, deleted) if node.right
+
+      # Check if current node overlaps
+      if node.low < high && low < node.high
+        # This node overlaps - delete it
+        deleted << node.data
+
+        # Remove this node and reinsert children
+        if node.left.nil?
+          return node.right
+        elsif node.right.nil?
+          return node.left
+        else
+          # Node has two children - replace with inorder successor
+          # Find minimum node in right subtree
+          min_node = find_min(node.right)
+
+          # Replace current node's interval and data with successor's
+          node.interval = min_node.interval
+          node.data = min_node.data
+
+          # Delete the successor from right subtree
+          node.right = delete_min(node.right)
+        end
+      end
+
+      # Update max for this node after potential deletions
+      if node
+        node.max = node.high
+        node.max = [node.max, node.left.max].max if node.left
+        node.max = [node.max, node.right.max].max if node.right
+      end
+
+      node
+    end
+
+    # Find minimum node in subtree (leftmost)
+    def find_min(node)
+      return node if node.left.nil?
+
+      find_min(node.left)
+    end
+
+    # Delete minimum node from subtree
+    def delete_min(node)
+      return node.right if node.left.nil?
+
+      node.left = delete_min(node.left)
+
+      # Update max
+      node.max = node.high
+      node.max = [node.max, node.left.max].max if node.left
+      node.max = [node.max, node.right.max].max if node.right
+
+      node
+    end
+  end
+end

data/lib/parsanol/lazy_result.rb

@@ -0,0 +1,179 @@
+# frozen_string_literal: true
+
+module Parsanol
+  # Lazy wrapper around Buffer that defers array materialization.
+  #
+  # LazyResult wraps a Buffer and only creates an Array when the result
+  # is actually accessed. This reduces allocations for results that are
+  # never used (cache hits, backtracking, etc.).
+  #
+  # == Usage
+  #
+  #   lazy = LazyResult.new(buffer, context)
+  #   # No array allocated yet
+  #
+  #   lazy.to_a  # Now array is materialized and cached
+  #   lazy.to_a  # Returns cached array
+  #
+  # == Transparency
+  #
+  # LazyResult acts like an Array for most operations:
+  # - Enumerable methods work (each, map, select, etc.)
+  # - Array access works ([], size, empty?, etc.)
+  # - Can be used in transforms without changes
+  #
+  class LazyResult
+    # @return [Buffer] The underlying buffer
+    attr_reader :buffer
+
+    # @return [Context] The context (for buffer release)
+    attr_reader :context
+
+    # @return [Array, nil] Cached materialized array
+    attr_reader :materialized
+
+    # Initialize a new LazyResult.
+    #
+    # @param buffer [Buffer] Buffer containing elements
+    # @param context [Context] Context for buffer management
+    #
+    def initialize(buffer, context)
+      @buffer = buffer
+      @context = context
+      @materialized = nil
+    end
+
+    # Materialize to array (with caching).
+    #
+    # First call creates array from buffer, subsequent calls return cached.
+    #
+    # @return [Array] Materialized array
+    #
+    def to_a
+      @materialized ||= @buffer.to_a
+    end
+
+    # Get element at index (materializes if needed).
+    #
+    # @param index [Integer] Zero-based index
+    # @return [Object] Element at index
+    #
+    def [](index)
+      to_a[index]
+    end
+
+    # Get number of elements.
+    #
+    # @return [Integer] Number of elements
+    #
+    def size
+      @buffer.size
+    end
+
+    alias length size
+
+    # Check if empty.
+    #
+    # @return [Boolean] true if no elements
+    #
+    def empty?
+      @buffer.empty?
+    end
+
+    # Iterate over elements (materializes if needed).
+    #
+    # @yield [element] Each element
+    # @return [Enumerator, self] Enumerator if no block, self otherwise
+    #
+    def each(&block)
+      return to_enum(:each) unless block_given?
+
+      to_a.each(&block)
+      self
+    end
+
+    # Check if acts like an array.
+    #
+    # @param other [Class] Class to check against
+    # @return [Boolean] true if Array
+    #
+    def is_a?(other)
+      other == Array || super
+    end
+
+    alias kind_of? is_a?
+
+    # Respond to array methods.
+    #
+    # @param method [Symbol] Method name
+    # @param include_private [Boolean] Include private methods
+    # @return [Boolean] true if responds
+    #
+    def respond_to?(method, include_private = false)
+      super || to_a.respond_to?(method, include_private)
+    end
+
+    # Delegate unknown methods to materialized array.
+    #
+    # @param method [Symbol] Method name
+    # @param args [Array] Arguments
+    # @param block [Proc] Block if given
+    # @return [Object] Result of method call
+    #
+    def method_missing(method, ...)
+      if to_a.respond_to?(method)
+        to_a.public_send(method, ...)
+      else
+        super
+      end
+    end
+
+    # Support respond_to_missing? for proper method_missing implementation.
+    #
+    # @param method [Symbol] Method name
+    # @param include_private [Boolean] Include private methods
+    # @return [Boolean] true if method is supported
+    #
+    def respond_to_missing?(method, include_private = false)
+      to_a.respond_to?(method, include_private) || super
+    end
+
+    # Compare with another object.
+    # LazyResult compares equal to arrays with the same content.
+    #
+    # @param other [Object] Object to compare with
+    # @return [Boolean] true if equal
+    #
+    def ==(other)
+      if other.is_a?(Array)
+        to_a == other
+      elsif other.is_a?(LazyResult)
+        to_a == other.to_a
+      else
+        super
+      end
+    end
+
+    alias eql? ==
+
+    # Hash code based on materialized array.
+    #
+    # @return [Integer] Hash code
+    #
+    def hash
+      to_a.hash
+    end
+
+    # Inspect for debugging.
+    #
+    # @return [String] Inspection string
+    #
+    def inspect
+      if @materialized
+        "#<LazyResult:#{object_id} materialized=#{@materialized.inspect}>"
+      else
+        "#<LazyResult:#{object_id} buffer.size=#{@buffer.size}>"
+      end
+    end
+  end
+end

data/lib/parsanol/lexer.rb

@@ -0,0 +1,144 @@
+# frozen_string_literal: true
+
+require 'parsanol/native'
+
+module Parsanol
+  # Generic lexer for fast tokenization
+  #
+  # Create a lexer by subclassing and defining tokens:
+  #
+  #   class JsonLexer < Parsanol::Lexer
+  #     token :string, /"[^"]*"/
+  #     token :number, /-?[0-9]+(\.[0-9]+)?/
+  #     token :true, /true/
+  #     token :false, /false/
+  #     token :null, /null/
+  #     token :lbrace, /\{/
+  #     token :rbrace, /\}/
+  #     token :lbracket, /\[/
+  #     token :rbracket, /\]/
+  #     token :colon, /:/
+  #     token :comma, /,/
+  #
+  #     ignore /\s+/
+  #   end
+  #
+  #   lexer = JsonLexer.new
+  #   tokens = lexer.tokenize('{"name": "test"}')
+  #
+  class Lexer
+    class << self
+      # Define a token pattern
+      #
+      # @param name [Symbol] Token type name
+      # @param pattern [Regexp] Pattern to match
+      # @param priority [Integer] Priority for conflict resolution (higher = preferred)
+      # @param block [Proc] Optional block to transform the matched value
+      def token(name, pattern, priority: 0, &block)
+        token_definitions << Definition.new(
+          name: name.to_s,
+          pattern: pattern.source,
+          priority: priority,
+          ignore: false,
+          transform: block
+        )
+      end
+
+      # Define patterns to ignore (e.g., whitespace, comments)
+      #
+      # @param pattern [Regexp] Pattern to ignore
+      def ignore(pattern)
+        token_definitions << Definition.new(
+          name: '__ignore__',
+          pattern: pattern.source,
+          priority: 0,
+          ignore: true,
+          transform: nil
+        )
+      end
+
+      # Define keywords (identifiers with higher priority)
+      #
+      # @param keywords [Array<Symbol>] Keyword names
+      # @param priority [Integer] Priority (default: 100)
+      def keyword(*keywords, priority: 100)
+        keywords.each do |kw|
+          token_definitions << Definition.new(
+            name: kw.to_s.upcase,
+            pattern: Regexp.new(Regexp.escape(kw.to_s), Regexp::IGNORECASE).source,
+            priority: priority,
+            ignore: false,
+            transform: nil
+          )
+        end
+      end
+
+      # Get token definitions for this lexer class
+      #
+      # @return [Array<Definition>] Token definitions
+      def token_definitions
+        @token_definitions ||= []
+      end
+
+      # Inherit token definitions from parent class
+      def inherited(subclass)
+        super
+        subclass.instance_variable_set(:@token_definitions, token_definitions.dup)
+      end
+    end
+
+    # Token definition
+    Definition = Struct.new(:name, :pattern, :priority, :ignore, :transform)
+
+    # Initialize the lexer
+    def initialize
+      @lexer_id = nil
+      @transforms = build_transforms
+    end
+
+    # Tokenize input string
+    #
+    # @param input [String] Input to tokenize
+    # @return [Array<Hash>] Array of tokens with type, value, and location
+    def tokenize(input)
+      ensure_lexer_created
+
+      tokens = Native.tokenize_with_lexer(@lexer_id, input)
+
+      # Apply any transforms
+      tokens.map do |token|
+        transform = @transforms[token['type']]
+        if transform
+          token = token.dup
+          token['value'] = transform.call(token['value'])
+        end
+        token
+      end
+    end
+
+    private
+
+    def ensure_lexer_created
+      return if @lexer_id
+
+      definitions = self.class.token_definitions.map do |d|
+        {
+          'name' => d.name,
+          'pattern' => d.pattern,
+          'priority' => d.priority,
+          'ignore' => d.ignore
+        }
+      end
+
+      @lexer_id = Native.create_lexer(definitions)
+    end
+
+    def build_transforms
+      transforms = {}
+      self.class.token_definitions.each do |d|
+        transforms[d.name] = d.transform if d.transform && !d.ignore
+      end
+      transforms
+    end
+  end
+end

data/lib/parsanol/mermaid.rb

@@ -0,0 +1,139 @@
+# frozen_string_literal: true
+
+# Generates Mermaid diagram visualizations of parser grammars.
+# Mermaid is widely supported by GitHub, GitLab, Notion, and many other tools.
+#
+# @example Generate Mermaid diagram
+#   parser = MyParser.new
+#   puts parser.to_mermaid
+#
+# @example Generate diagram for specific rule
+#   puts parser.mermaid_for_rule(:expression)
+#
+# Inspired by Parslet (MIT License).
+
+module Parsanol
+  # Generates Mermaid diagram syntax from parser atoms.
+  class MermaidBuilder
+    def initialize
+      @lines = ['graph TD']
+      @node_counter = 0
+      @connections = []
+      @seen_rules = Set.new
+    end
+
+    # Entry point for parser visualization
+    def visit_parser(root_atom)
+      add_node('Parser', 'root')
+      traverse(root_atom, 'Parser')
+      finalize
+    end
+
+    # Handles named rules
+    def visit_entity(rule_name, rule_block)
+      return if @seen_rules.include?(rule_name)
+
+      @seen_rules << rule_name
+
+      node_id = add_node(rule_name.to_s.upcase, 'rule')
+      connect(current_parent, node_id)
+      traverse(rule_block.call, node_id)
+    end
+
+    # Pass through named captures
+    def visit_named(_label, atom)
+      traverse(atom, current_parent)
+    end
+
+    # Pass through repetition
+    def visit_repetition(_tag, _min, _max, atom)
+      traverse(atom, current_parent)
+    end
+
+    # Process alternatives
+    def visit_alternative(alternatives)
+      alternatives.each { |alt| traverse(alt, current_parent) }
+    end
+
+    # Process sequence
+    def visit_sequence(members)
+      members.each { |member| traverse(member, current_parent) }
+    end
+
+    # Pass through lookahead
+    def visit_lookahead(_positive, atom)
+      traverse(atom, current_parent)
+    end
+
+    # Leaf nodes
+    def visit_re(regexp)
+      add_node("match(#{regexp.inspect})", 'terminal', style: 'ellipse')
+    end
+
+    def visit_str(string)
+      add_node("'#{string}'", 'terminal', style: 'ellipse')
+    end
+
+    private
+
+    attr_reader :current_parent
+
+    def add_node(label, _shape_type = 'rect', _style = nil)
+      @node_counter += 1
+      node_id = "node_#{@node_counter}"
+      @lines << "    #{node_id}[\"#{escape_mermaid(label)}\"]"
+      node_id
+    end
+
+    def connect(from_id, to_id)
+      @connections << [from_id, to_id]
+    end
+
+    def escape_mermaid(text)
+      text.gsub('"', "'").gsub('\n', '\\n')
+    end
+
+    def finalize
+      @connections.each do |from, to|
+        @lines << "    #{from} --> #{to}"
+      end
+      @lines << ''
+      @lines.join("\n")
+    end
+
+    def traverse(atom, parent)
+      @current_parent = parent
+      atom.accept(self)
+    end
+  end
+
+  # Mixin module that adds Mermaid diagram generation to parsers
+  module MermaidDiagram
+    # Generates a Mermaid diagram of the parser.
+    #
+    # @return [String] Mermaid diagram source
+    def to_mermaid
+      builder = MermaidBuilder.new
+      new.accept(builder)
+      builder.output
+    end
+
+    # Generates Mermaid diagram for a specific rule.
+    #
+    # @param rule_name [Symbol] name of the rule
+    # @return [String] Mermaid diagram source
+    def mermaid_for_rule(rule_name)
+      builder = MermaidBuilder.new
+      rule_method = method(rule_name)
+      raise NotImplementedError, "Rule '#{rule_name}' not found" unless rule_method
+
+      rule_method.call.accept(builder)
+      builder.output
+    end
+  end
+
+  # Extend Parser with Mermaid diagram generation
+  class Parser
+    extend MermaidDiagram
+  end
+end