parsanol 1.0.0

data/lib/parsanol/source_location.rb

@@ -0,0 +1,167 @@
+# frozen_string_literal: true
+
+# Parsanol::SourceLocation - Source Location Tracking
+#
+# Track source positions (line, column, offset) through the parsing and
+# transformation pipeline. This is useful for error reporting, IDE integration,
+# and source mapping.
+#
+# Usage:
+#   # Parse with source tracking
+#   result = parser.parse_with_spans("hello world")
+#   tree = result.tree
+#   spans = result.spans
+#
+#   # Access span for a node
+#   span = spans[node_id]
+#   puts "Matched at line #{span.start.line}, column #{span.start.column}"
+#
+# Requires native extension for full functionality.
+
+module Parsanol
+  # Represents a position in source code
+  class SourcePosition
+    attr_reader :offset, :line, :column
+
+    def initialize(offset:, line:, column:)
+      @offset = offset
+      @line = line
+      @column = column
+    end
+
+    def to_s
+      "line #{@line}, column #{@column} (offset #{@offset})"
+    end
+
+    def to_h
+      { offset: @offset, line: @line, column: @column }
+    end
+
+    def ==(other)
+      return false unless other.is_a?(SourcePosition)
+
+      @offset == other.offset && @line == other.line && @column == other.column
+    end
+
+    def eql?(other)
+      self == other
+    end
+
+    def hash
+      [@offset, @line, @column].hash
+    end
+  end
+
+  # Represents a span in source code (from start to end position)
+  class SourceSpan
+    attr_reader :start, :end
+
+    def initialize(start_pos:, end_pos:)
+      @start = start_pos.is_a?(SourcePosition) ? start_pos : SourcePosition.new(**start_pos)
+      @end = end_pos.is_a?(SourcePosition) ? end_pos : SourcePosition.new(**end_pos)
+    end
+
+    # Create a span from offsets (computes line/column from input)
+    def self.from_offsets(input, start_offset, end_offset)
+      start_pos = compute_position(input, start_offset)
+      end_pos = compute_position(input, end_offset)
+      new(start_pos: start_pos, end_pos: end_pos)
+    end
+
+    # Merge two spans (returns a new span covering both)
+    def merge(other)
+      return self if other.nil?
+
+      SourceSpan.new(
+        start_pos: [@start, other.start].min_by(&:offset),
+        end_pos: [@end, other.end].max_by(&:offset)
+      )
+    end
+
+    # Check if this span overlaps with another
+    def overlaps?(other)
+      return false if other.nil?
+
+      @start.offset < other.end.offset && @end.offset > other.start.offset
+    end
+
+    # Check if this span is adjacent to another
+    def adjacent?(other)
+      return false if other.nil?
+
+      @end.offset == other.start.offset || other.end.offset == @start.offset
+    end
+
+    # Check if a position is within this span
+    def contains?(position)
+      offset = position.is_a?(SourcePosition) ? position.offset : position
+      offset.between?(@start.offset, @end.offset)
+    end
+
+    # Get the length of the span in bytes
+    def length
+      @end.offset - @start.offset
+    end
+
+    # Extract the source text from the input
+    def extract(input)
+      input.byteslice(@start.offset, length)
+    end
+
+    def to_s
+      "#{@start} - #{@end}"
+    end
+
+    def to_h
+      { start: @start.to_h, end: @end.to_h }
+    end
+
+    def ==(other)
+      return false unless other.is_a?(SourceSpan)
+
+      @start == other.start && @end == other.end
+    end
+
+    # Compute line and column from offset
+    def self.compute_position(input, offset)
+      line = 1
+      column = 1
+      current_offset = 0
+
+      input.each_char do |char|
+        break if current_offset >= offset
+
+        if char == "\n"
+          line += 1
+          column = 1
+        else
+          column += 1
+        end
+
+        current_offset += 1
+      end
+
+      SourcePosition.new(offset: offset, line: line, column: column)
+    end
+  end
+
+  # Result wrapper for parse_with_spans
+  class ParseResultWithSpans
+    attr_reader :tree, :spans
+
+    def initialize(tree:, spans:)
+      @tree = tree
+      @spans = spans
+    end
+
+    # Get span for a specific node
+    def span_for(node_id)
+      @spans[node_id]
+    end
+
+    # Get all spans that contain a position
+    def spans_at(offset)
+      @spans.values.select { |span| span.contains?(offset) }
+    end
+  end
+end

data/lib/parsanol/streaming_parser.rb

@@ -0,0 +1,124 @@
+# frozen_string_literal: true
+
+# Parsanol::StreamingParser - Streaming Parser for Large Inputs
+#
+# Parse large inputs in chunks without loading the entire input into memory.
+# Useful for file parsing, network streams, or very large documents.
+#
+# Usage:
+#   parser = Parsanol::StreamingParser.new(json_grammar)
+#
+#   File.open("large.json") do |f|
+#     parser.parse_stream(f) do |partial_result|
+#       # Process each complete element as it's parsed
+#       process_item(partial_result)
+#     end
+#   end
+#
+# Requires native extension for full functionality.
+
+module Parsanol
+  class StreamingParser
+    # Default chunk size (4KB)
+    DEFAULT_CHUNK_SIZE = 4096
+
+    # Create a new streaming parser
+    #
+    # @param grammar [Parsanol::Parser, Parsanol::Atoms::Base] Grammar to use
+    # @param chunk_size [Integer] Size of chunks to read (default: 4096)
+    def initialize(grammar, chunk_size: DEFAULT_CHUNK_SIZE)
+      @grammar = grammar
+      @chunk_size = chunk_size
+
+      if Parsanol::Native.available?
+        grammar_json = Parsanol::Native.serialize_grammar(grammar.root)
+        @native_parser = Parsanol::Native.streaming_parser_new(grammar_json)
+      else
+        @native_parser = nil
+      end
+
+      @buffer = String.new
+      @position = 0
+    end
+
+    # Add a chunk of input
+    #
+    # @param chunk [String] Input chunk to add
+    # @return [Boolean] True if more chunks needed, false if ready for parsing
+    def add_chunk(chunk)
+      @buffer << chunk
+
+      if @native_parser
+        Parsanol::Native.streaming_parser_add_chunk(@native_parser, chunk)
+      else
+        # Pure Ruby fallback
+        false
+      end
+    end
+
+    # Parse what we have so far
+    #
+    # @return [Object, nil] Parsed result or nil if need more data
+    def parse_chunk
+      if @native_parser
+        Parsanol::Native.streaming_parser_parse_chunk(@native_parser)
+      else
+        # Pure Ruby fallback - not supported
+        raise NotImplementedError,
+              'Streaming parser requires native extension for full functionality.'
+      end
+    end
+
+    # Check if we have enough data to make progress
+    #
+    # @return [Boolean] True if parser can make progress
+    def enough_data?
+      if @native_parser
+        !Parsanol::Native.streaming_parser_parse_chunk(@native_parser).nil?
+      else
+        false
+      end
+    end
+
+    # Parse entire stream (yields partial results)
+    #
+    # @param io [IO, StringIO] Input source to read from
+    # @param chunk_size [Integer] Size of chunks to read
+    # @yield [Object] Each complete element as it's parsed
+    # @return [Array] All parsed results
+    def parse_stream(io, chunk_size: @chunk_size)
+      results = []
+
+      loop do
+        chunk = io.read(chunk_size)
+        break if chunk.nil? || chunk.empty?
+
+        add_chunk(chunk)
+
+        while (result = parse_chunk)
+          results << result
+          yield result if block_given?
+        end
+      end
+
+      results
+    end
+
+    # Reset the parser for reuse
+    def reset
+      @buffer = String.new
+      @position = 0
+
+      return unless @native_parser
+
+      grammar_json = Parsanol::Native.serialize_grammar(@grammar.root)
+      @native_parser = Parsanol::Native.streaming_parser_new(grammar_json)
+    end
+
+    # Get the current buffer
+    attr_reader :buffer
+
+    # Get the chunk size
+    attr_reader :chunk_size
+  end
+end

data/lib/parsanol/string_view.rb

@@ -0,0 +1,195 @@
+# frozen_string_literal: true
+
+module Parsanol
+  # Zero-copy string view that references original input.
+  #
+  # StringView avoids string copies by maintaining a reference to the
+  # original string with offset and length. Strings are only materialized
+  # when explicitly requested via #to_s.
+  #
+  # == Usage
+  #
+  #   view = StringView.new(input_string, offset: 10, length: 5)
+  #   view.to_s  # Materializes string only when needed
+  #   view[0]    # Direct character access without copying
+  #
+  # == Performance
+  #
+  # - No string allocation until to_s called
+  # - Direct character access without copying
+  # - Reduced GC pressure from intermediate strings
+  # - Caches materialized strings for reuse
+  #
+  # == Design Principles
+  #
+  # 1. Zero-Copy: Reference original string, don't copy
+  # 2. Lazy Materialization: Create strings only when to_s called
+  # 3. Caching: Cache materialized strings for reuse
+  # 4. Compatibility: Acts like String where needed
+  # 5. Extensibility: Foundation for Rope (Phase 3.2)
+  #
+  class StringView
+    include Resettable
+
+    # @return [String] Original input string
+    attr_reader :string
+
+    # @return [Integer] Byte offset into string
+    attr_reader :offset
+
+    # @return [Integer] Length in bytes
+    attr_reader :length
+
+    # Initialize a new StringView.
+    #
+    # @param string [String] Original input string
+    # @param offset [Integer] Byte offset (default: 0)
+    # @param length [Integer] Length in bytes (default: string.bytesize)
+    #
+    def initialize(string, offset: 0, length: nil)
+      @string = string
+      @offset = offset
+      @length = length || (string.bytesize - offset)
+      @materialized = nil
+    end
+
+    # Materialize to string (with caching).
+    #
+    # First call creates string slice, subsequent calls return cached.
+    # This implements lazy evaluation - strings are only created when
+    # explicitly needed, not during parsing.
+    #
+    # @return [String] Materialized string
+    #
+    def to_s
+      @materialized ||= @string.byteslice(@offset, @length)
+    end
+
+    # Get character at index (zero-copy).
+    #
+    # Direct access to character in original string without creating
+    # intermediate string objects.
+    #
+    # @param index [Integer] Zero-based index
+    # @return [String, nil] Character at index or nil
+    #
+    def [](index)
+      return nil if index.negative? || index >= @length
+
+      @string.byteslice(@offset + index, 1)
+    end
+
+    # Get byte size.
+    #
+    # @return [Integer] Length in bytes
+    #
+    def bytesize
+      @length
+    end
+
+    alias size bytesize
+    alias length bytesize
+
+    # Check if empty.
+    #
+    # @return [Boolean] true if length is 0
+    #
+    def empty?
+      @length.zero?
+    end
+
+    # Compare with another object.
+    #
+    # StringViews are only equal if they reference the exact same string object
+    # (by object_id) and have the same offset/length. This is consistent with
+    # the view pattern - they're views of a specific string instance.
+    #
+    # When comparing with a String, content is compared.
+    #
+    # @param other [Object] Object to compare with
+    # @return [Boolean] true if equal
+    #
+    def ==(other)
+      case other
+      when String
+        to_s == other
+      when StringView
+        # Only equal if viewing the exact same string object with same range
+        @string.equal?(other.string) &&
+          @offset == other.offset &&
+          @length == other.length
+      else
+        super
+      end
+    end
+
+    alias eql? ==
+
+    # Hash code for hashing.
+    #
+    # Uses object_id of string to avoid materializing the view.
+    #
+    # @return [Integer] Hash code
+    #
+    def hash
+      [@string.object_id, @offset, @length].hash
+    end
+
+    # Create substring view (zero-copy).
+    #
+    # Returns a new StringView referencing a substring of this view.
+    # No string allocation occurs - just a new view with adjusted offset.
+    #
+    # @param start [Integer] Start offset (relative to view)
+    # @param len [Integer] Length
+    # @return [StringView] New view of substring
+    #
+    def slice(start, len)
+      # Handle edge cases
+      return self.class.new(@string, offset: @offset, length: 0) if len <= 0 || start >= @length
+
+      # Clamp start to valid range [0, @length)
+      clamped_start = [[start, 0].max, @length].min
+
+      # Calculate actual offset in original string
+      actual_offset = @offset + clamped_start
+
+      # Calculate actual length (min of requested and available)
+      available = @length - clamped_start
+      actual_length = [len, available].min
+
+      self.class.new(@string, offset: actual_offset, length: actual_length)
+    end
+
+    # Inspect for debugging.
+    #
+    # Shows whether string has been materialized.
+    #
+    # @return [String] Inspection string
+    #
+    def inspect
+      if @materialized
+        "#<StringView:#{object_id} @offset=#{@offset} @length=#{@length} cached=#{@materialized.inspect}>"
+      else
+        "#<StringView:#{object_id} @offset=#{@offset} @length=#{@length}>"
+      end
+    end
+
+    # Reset for pooling (if needed in future phases).
+    #
+    # Allows StringView objects to be reused from a pool.
+    #
+    # @param string [String] New string
+    # @param offset [Integer] New offset
+    # @param length [Integer] New length
+    # @return [self]
+    #
+    def reset!(string, offset, length)
+      @string = string
+      @offset = offset
+      @length = length
+      @materialized = nil
+      self
+    end
+  end
+end

data/lib/parsanol/transform.rb

@@ -0,0 +1,226 @@
+# frozen_string_literal: true
+
+require 'parsanol/pattern'
+
+# Tree transformation engine for converting parse trees into abstract syntax trees.
+#
+# Transforms expression trees through depth-first post-order traversal.
+# When a rule pattern matches a node, that node is replaced by the result
+# of the rule's transformation block. Unmatched nodes pass through unchanged.
+#
+# @example Basic transformation class
+#   class NumberTransform < Parsanol::Transform
+#     rule(int: simple(:value)) { Integer(value) }
+#     rule(float: simple(:value)) { Float(value) }
+#   end
+#
+#   transform = NumberTransform.new
+#   transform.apply({ int: '42' })  # => 42
+#
+# @example Inline transformation definition
+#   transform = Parsanol::Transform.new do
+#     rule(a: simple(:x)) { x.upcase }
+#   end
+#   transform.apply({ a: 'hello' })  # => 'HELLO'
+#
+# @example Using context for external dependencies
+#   builder = AstBuilder.new
+#   transform = Parsanol::Transform.new do
+#     rule(expr: simple(:e)) { builder.build_node(e) }
+#     rule(expr: simple(:e)) { |ctx| ctx[:builder].build_node(e) }
+#   end
+#   transform.apply(tree, builder: builder)
+#
+# Rule blocks can have two forms:
+# - Zero-arity: executed in a context where pattern bindings are local variables
+# - Arity-1: receives a hash of bindings as the argument
+#
+# Inspired by tree transformation patterns in parser combinators.
+#
+module Parsanol
+  class Transform
+    include Parsanol
+
+    # Class-level rule definition for subclass inheritance.
+    class << self
+      include Parsanol
+
+      # Defines a transformation rule at the class level.
+      # Rules are inherited by subclasses and evaluated in reverse order
+      # (most recently defined rules have highest precedence).
+      #
+      # @param expression [Object] pattern to match against tree nodes
+      # @yield block to execute when pattern matches, receives bindings
+      # @return [Array] the updated rules list
+      #
+      def rule(expression, &transformer)
+        class_rules.unshift([Parsanol::Pattern.new(expression), transformer])
+      end
+
+      # Returns all class-level rules defined for this transform.
+      #
+      # @return [Array<Array>] array of [pattern, block] pairs
+      #
+      def class_rules
+        @class_rules ||= []
+      end
+
+      # Ensures subclasses inherit parent rules.
+      def inherited(subclass)
+        super
+        subclass.instance_variable_set(:@class_rules, class_rules.dup)
+      end
+    end
+
+    # Creates a new transform instance.
+    #
+    # @param strict [Boolean] if true, raises on unmatched hash nodes
+    # @yield optional block for inline rule definition
+    #
+    def initialize(strict = false, &definition)
+      @strict_mode = strict
+      @instance_rules = []
+
+      instance_eval(&definition) if definition
+    end
+
+    # Defines an instance-level transformation rule.
+    # Instance rules are checked before class rules.
+    #
+    # @param expression [Object] pattern to match
+    # @yield transformation block
+    #
+    def rule(expression, &transformer)
+      @instance_rules.unshift([Parsanol::Pattern.new(expression), transformer])
+    end
+
+    # Applies transformation to a parse tree.
+    #
+    # Performs depth-first post-order traversal, transforming nodes
+    # from leaves to root. Context values are available in rule blocks.
+    #
+    # @param tree [Object] parse tree to transform
+    # @param context [Hash, nil] optional context bindings
+    # @return [Object] transformed tree
+    #
+    def apply(tree, context = nil)
+      # First, recursively transform children (depth-first)
+      transformed = transform_children(tree, context)
+
+      # Then, try to match and transform this node
+      attempt_transformation(transformed, context)
+    end
+
+    # Returns combined class and instance rules.
+    # Instance rules take precedence over class rules.
+    #
+    # @return [Array<Array>] all applicable rules
+    #
+    def all_rules
+      @instance_rules + self.class.class_rules
+    end
+
+    # Executes a transformation block with the given bindings.
+    # Public API for testing and advanced usage.
+    #
+    # @param bindings [Hash] pattern bindings
+    # @param block [Proc] transformation block
+    # @return [Object] block result
+    #
+    def call_on_match(bindings, block)
+      return nil unless block
+
+      if block.arity == 1
+        block.call(bindings)
+      else
+        Context.new(bindings).instance_eval(&block)
+      end
+    end
+
+    private
+
+    # Recursively transforms child nodes based on tree type.
+    #
+    # @param node [Object] current tree node
+    # @param ctx [Hash, nil] context bindings
+    # @return [Object] node with transformed children
+    #
+    def transform_children(node, ctx)
+      case node
+      when Hash
+        transform_hash_children(node, ctx)
+      when Array
+        transform_array_children(node, ctx)
+      else
+        node
+      end
+    end
+
+    # Transforms all values in a hash.
+    #
+    def transform_hash_children(hash, ctx)
+      result = {}
+      hash.each { |key, val| result[key] = apply(val, ctx) }
+      result
+    end
+
+    # Transforms all elements in an array.
+    #
+    def transform_array_children(array, ctx)
+      array.map { |element| apply(element, ctx) }
+    end
+
+    # Attempts to match a node against all rules and transform if matched.
+    #
+    # @param node [Object] node to potentially transform
+    # @param ctx [Hash, nil] context bindings
+    # @return [Object] transformed node or original if no match
+    #
+    def attempt_transformation(node, ctx)
+      all_rules.each do |pattern, block|
+        bindings = pattern.match(node, ctx)
+        next unless bindings
+
+        return execute_block(block, bindings)
+      end
+
+      # No rule matched
+      handle_unmatched(node)
+    end
+
+    # Executes a transformation block with proper binding context.
+    #
+    # @param block [Proc] transformation block
+    # @param bindings [Hash] matched pattern bindings
+    # @return [Object] block result
+    #
+    def execute_block(block, bindings)
+      return nil unless block
+
+      if block.arity == 1
+        # Block expects bindings as argument
+        block.call(bindings)
+      else
+        # Block executes in context with bindings as local variables
+        Context.new(bindings).instance_eval(&block)
+      end
+    end
+
+    # Handles nodes that didn't match any rule.
+    #
+    # @param node [Object] unmatched node
+    # @return [Object] the node (or raises in strict mode)
+    # @raise [NotImplementedError] if strict mode and node is a Hash
+    #
+    def handle_unmatched(node)
+      return node unless @strict_mode
+      return node unless node.is_a?(Hash)
+
+      # In strict mode, provide helpful error about what wasn't matched
+      signature = node.transform_values(&:class)
+      raise NotImplementedError, "Failed to match #{signature.inspect}"
+    end
+  end
+end
+
+require 'parsanol/context'

data/lib/parsanol/version.rb

@@ -0,0 +1,5 @@
+# frozen_string_literal: true
+
+module Parsanol
+  VERSION = '1.0.0'
+end