parsanol 1.0.0

data/lib/parsanol/expression.rb

@@ -0,0 +1,106 @@
+# frozen_string_literal: true
+
+# Parses treetop-style expression strings and converts them to Parsanol atoms.
+#
+# This allows specifying parser rules as strings using treetop syntax instead
+# of building atoms explicitly with the DSL.
+#
+# == Performance Note
+#
+# The expression parser is implemented in pure Ruby and is NOT accelerated by
+# the Rust native extension. This is intentional and acceptable because:
+#
+# 1. Expression parsing happens at grammar definition time (once)
+# 2. Expression strings are typically short (< 100 characters)
+# 3. The resulting atoms can still be used with Rust-accelerated parsing
+#
+# If you need maximum performance for dynamically generated parsers, consider
+# building atoms directly with the DSL (str, match, any, etc.) instead.
+#
+# == Syntax
+#
+# The treetop syntax supports:
+#
+# - Strings: 'hello' (single quotes)
+# - Character classes: [a-z], [0-9]
+# - Any character: .
+# - Sequence: 'a' 'b' (concatenation)
+# - Alternative: 'a' / 'b'
+# - Optional: 'a' ? (space before ? required)
+# - Zero or more: 'a' * (space before * required)
+# - One or more: 'a' + (space before + required)
+# - Repetition: 'a'{1,3}
+# - Grouping: ('a' / 'b')+
+#
+# == Example
+#
+#   # Using exp()
+#   rule(:word) { exp("'a' 'b' ?") }
+#
+#   # Equivalent DSL:
+#   rule(:word) { str('a') >> str('b').maybe }
+#
+# == Result Usage
+#
+# The atoms produced by exp() can be used with Rust-accelerated parsing:
+#
+#   atom = Parsanol.exp("'a' +")
+#
+#   # Ruby parsing
+#   atom.parse('aaa')
+#
+#   # Rust-accelerated parsing (if native extension available)
+#   Parsanol::Native.parse_with_grammar(atom, 'aaa')
+#
+module Parsanol
+  class Expression
+    include Parsanol
+
+    autoload :Treetop, 'parsanol/expression/treetop'
+
+    # Creates a parser atom from a treetop-style expression string.
+    #
+    # @param str [String] a treetop expression
+    # @param opts [Hash] options (:type => :treetop, default)
+    # @return [Parsanol::Expression] expression object (call #to_parslet for atom)
+    #
+    # @example
+    #   expr = Parsanol::Expression.new("'a' 'b' ?")
+    #   atom = expr.to_parslet
+    #   atom.parse('a')  # => "a"@0
+    #
+    def initialize(str, opts = {}, _context = self)
+      @type = opts[:type] || :treetop
+      @exp = str
+      @parslet = transform(parse(str))
+    end
+
+    # Transforms the parse tree into a parser atom.
+    #
+    # @param tree [Hash] parse tree from Treetop::Parser
+    # @return [Parsanol::Atoms::Base] parser atom
+    def transform(tree)
+      transform = Treetop::Transform.new
+      transform.apply(tree)
+    rescue StandardError
+      warn "Could not transform: #{tree.inspect}"
+      raise
+    end
+
+    # Parses the expression string and returns a parse tree.
+    #
+    # @param str [String] treetop expression
+    # @return [Hash] parse tree
+    def parse(str)
+      parser = Treetop::Parser.new
+      parser.parse(str)
+    end
+
+    # Returns the parser atom for this expression.
+    #
+    # @return [Parsanol::Atoms::Base] parser atom
+    def to_parslet
+      @parslet
+    end
+  end
+end

data/lib/parsanol/fast_mode.rb

@@ -0,0 +1,149 @@
+# frozen_string_literal: true
+
+# Fast mode patch for Parslet - matches vanilla parslet 2.0 behavior.
+#
+# For grammars with many small allocations (like EXPRESS), this is faster
+# because the overhead of pool management exceeds the benefit.
+#
+# Usage:
+#   require 'parslet'
+#   require 'parsanol/fast_mode'
+#   # Now all parsing uses fast mode methods
+#
+
+module Parsanol
+  FAST_MODE = true
+
+  module Atoms
+    # Fast mode Context - matches vanilla parslet 2.0 simplicity
+    class Context
+      # Override try_with_cache with vanilla-like version (no eviction, no pooling)
+      def try_with_cache(obj, source, consume_all)
+        beg = source.bytepos
+
+        # Not in cache yet? Return early.
+        unless (entry = @cache[beg]&.[](obj.object_id))
+          result = obj.try(source, self, consume_all)
+
+          (@cache[beg] ||= {})[obj.object_id] = [result, source.bytepos - beg] if obj.cached?
+
+          return result
+        end
+
+        # Cache hit
+        result, advance = entry
+        source.bytepos = beg + advance
+        result
+      end
+    end
+
+    # Fast mode Sequence - direct array creation, no lazy evaluation
+    class Sequence
+      def try(source, context, consume_all)
+        parslets = @parslets
+
+        case parslets.size
+        when 1
+          success, value = parslets[0].apply(source, context, consume_all)
+          success ? succ([:sequence, value]) : context.err(self, source, @error_msg, [value])
+        when 2
+          success, v1 = parslets[0].apply(source, context, false)
+          return context.err(self, source, @error_msg, [v1]) unless success
+
+          success, v2 = parslets[1].apply(source, context, consume_all)
+          success ? succ([:sequence, v1, v2]) : context.err(self, source, @error_msg, [v2])
+        when 3
+          success, v1 = parslets[0].apply(source, context, false)
+          return context.err(self, source, @error_msg, [v1]) unless success
+
+          success, v2 = parslets[1].apply(source, context, false)
+          return context.err(self, source, @error_msg, [v2]) unless success
+
+          success, v3 = parslets[2].apply(source, context, consume_all)
+          success ? succ([:sequence, v1, v2, v3]) : context.err(self, source, @error_msg, [v3])
+        else
+          result = [:sequence]
+          last_idx = parslets.size - 1
+          i = 0
+          while i <= last_idx
+            success, value = parslets[i].apply(source, context, consume_all && i == last_idx)
+            return context.err(self, source, @error_msg, [value]) unless success
+
+            result << value
+            i += 1
+          end
+          succ(result)
+        end
+      end
+    end
+
+    # Fast mode Repetition - direct array creation, no lazy evaluation
+    class Repetition
+      EMPTY_REPETITION_ARRAY = [:repetition].freeze
+
+      def try(source, context, consume_all)
+        parslet = @parslet
+        min = @min
+        max = @max
+        tag = @tag
+
+        # Fast path for .maybe
+        if min.zero? && max == 1
+          success, value = parslet.apply(source, context, false)
+          return succ([tag, value]) if success
+
+          return succ(tag == :repetition ? EMPTY_REPETITION_ARRAY : [tag])
+        end
+
+        # Fast path for exact count
+        if min == max && max && max <= 3
+          case max
+          when 1
+            success, value = parslet.apply(source, context, consume_all)
+            return success ? succ([tag, value]) : context.err_at(self, source, @error_msg, source.bytepos, [value])
+          when 2
+            success, v1 = parslet.apply(source, context, false)
+            return context.err_at(self, source, @error_msg, source.bytepos, [v1]) unless success
+
+            success, v2 = parslet.apply(source, context, consume_all)
+            return success ? succ([tag, v1, v2]) : context.err_at(self, source, @error_msg, source.bytepos, [v2])
+          when 3
+            success, v1 = parslet.apply(source, context, false)
+            return context.err_at(self, source, @error_msg, source.bytepos, [v1]) unless success
+
+            success, v2 = parslet.apply(source, context, false)
+            return context.err_at(self, source, @error_msg, source.bytepos, [v2]) unless success
+
+            success, v3 = parslet.apply(source, context, consume_all)
+            return success ? succ([tag, v1, v2, v3]) : context.err_at(self, source, @error_msg, source.bytepos, [v3])
+          end
+        end
+
+        # General case
+        start_pos = source.bytepos
+        occ = 0
+        result = [tag]
+        break_on = nil
+
+        loop do
+          success, value = parslet.apply(source, context, false)
+          break_on = value
+          break unless success
+
+          occ += 1
+          result << value
+          break if max && occ >= max
+        end
+
+        if occ < min
+          source.bytepos = start_pos
+          return context.err_at(self, source, @error_msg, start_pos, [break_on])
+        end
+
+        return context.err(self, source, @unconsumed_msg, [break_on]) if consume_all && source.chars_left.positive?
+
+        succ(result)
+      end
+    end
+  end
+end

data/lib/parsanol/first_set.rb

@@ -0,0 +1,79 @@
+# frozen_string_literal: true
+
+# FIRST Set Analysis for PEG Grammars
+#
+# FIRST sets help identify which terminals can appear at the beginning of
+# a parse. This is essential for:
+# 1. Automatic cut operator insertion (AC-FIRST algorithm)
+# 2. Grammar analysis and optimization
+# 3. Detecting ambiguous choices
+#
+# Reference: Mizushima et al. (2010) "Packrat Parsers Can Handle Practical
+# Grammars in Mostly Constant Space"
+#
+module Parsanol
+  module FirstSet
+    # Sentinel value representing the empty string (ε)
+    EPSILON = :epsilon
+
+    # Compute the FIRST set for this parslet atom
+    # Returns a Set containing:
+    # - Terminal atoms (Str, Re) that can match first
+    # - EPSILON if the atom can match empty string
+    # - nil elements represent unknown/variable terminals (e.g., any)
+    #
+    # @return [Set] FIRST set containing terminal atoms or EPSILON
+    def first_set
+      @first_set ||= compute_first_set
+    end
+
+    # Clear cached FIRST set (useful after grammar modifications)
+    def clear_first_set_cache
+      @first_set = nil
+    end
+
+    protected
+
+    # Override in subclasses to compute FIRST set
+    # Default: conservative approximation (unknown)
+    def compute_first_set
+      Set.new([nil]) # nil = unknown terminal
+    end
+
+    # Class methods for FIRST set analysis
+    class << self
+      # Check if two FIRST sets are disjoint
+      # Two sets are disjoint if they have no common elements
+      # EPSILON is ignored when checking disjointness
+      #
+      # @param set1 [Set] First FIRST set
+      # @param set2 [Set] Second FIRST set
+      # @return [Boolean] true if sets are disjoint
+      def disjoint?(set1, set2)
+        # Remove EPSILON and nil from both sets for comparison
+        real_set1 = set1.reject { |x| x == EPSILON || x.nil? }
+        real_set2 = set2.reject { |x| x == EPSILON || x.nil? }
+
+        # If either set is empty (only EPSILON/nil), consider disjoint
+        return true if real_set1.empty? || real_set2.empty?
+
+        # Check if intersection is empty (using to_a for Opal compatibility)
+        (real_set1.to_a & real_set2.to_a).empty?
+      end
+
+      # Check if all FIRST sets in a collection are mutually disjoint
+      # This is critical for AC-FIRST algorithm - we can only insert
+      # cuts when all alternatives have non-overlapping FIRST sets
+      #
+      # @param sets [Array<Set>] Collection of FIRST sets
+      # @return [Boolean] true if all pairs are disjoint
+      def all_disjoint?(sets)
+        # Need at least 2 sets to check disjointness
+        return true if sets.length < 2
+
+        # Check all pairs
+        sets.combination(2).all? { |s1, s2| disjoint?(s1, s2) }
+      end
+    end
+  end
+end

data/lib/parsanol/grammar_builder.rb

@@ -0,0 +1,177 @@
+# frozen_string_literal: true
+
+# Parsanol::GrammarBuilder - Grammar Composition
+#
+# Build complex grammars by importing and composing smaller grammars.
+# This enables reusable grammar modules.
+#
+# Usage:
+#   # Define reusable grammars
+#   expression_grammar = GrammarBuilder.new
+#     .rule("expr", str("a") | str("b"))
+#     .build
+#
+#   type_grammar = GrammarBuilder.new
+#     .rule("type", str("int") | str("str"))
+#     .build
+#
+#   # Compose into a new grammar
+#   combined = GrammarBuilder.new
+#     .import(expression_grammar, prefix: "expr")
+#     .import(type_grammar, prefix: "type")
+#     .rule("typed", seq([ref("expr:root"), str(":"), ref("type:root")]))
+#     .build
+#
+# Requires native extension for full functionality.
+
+module Parsanol
+  class GrammarBuilder
+    # Create a new grammar builder
+    def initialize
+      @rules = {}
+      @imports = []
+      @root = nil
+    end
+
+    # Define a rule
+    #
+    # @param name [String, Symbol] Rule name
+    # @param parslet [Parsanol::Atoms::Base] Parslet atom
+    # @return [self] For chaining
+    def rule(name, parslet)
+      @rules[name.to_s] = parslet
+      self
+    end
+
+    # Get a rule for modification
+    #
+    # @param name [String, Symbol] Rule name
+    # @return [Parsanol::Atoms::Base, nil] The rule atom
+    def [](name)
+      @rules[name.to_s]
+    end
+
+    # Set the root rule
+    #
+    # @param name [String, Symbol] Root rule name
+    # @return [self] For chaining
+    def root(name)
+      @root = name.to_s
+      self
+    end
+
+    # Import another grammar with optional prefix
+    #
+    # @param grammar [GrammarBuilder, Hash] Grammar to import
+    # @param prefix [String, nil] Optional prefix for imported rules
+    # @return [self] For chaining
+    def import(grammar, prefix: nil)
+      grammar_data = case grammar
+                     when GrammarBuilder
+                       grammar.to_h
+                     when Hash
+                       grammar
+                     else
+                       raise ArgumentError, "Expected GrammarBuilder or Hash, got #{grammar.class}"
+                     end
+
+      @imports << { grammar: grammar_data, prefix: prefix }
+      self
+    end
+
+    # Import with explicit rule mapping
+    #
+    # @param grammar [GrammarBuilder, Hash] Grammar to import
+    # @param prefix [String, nil] Optional prefix
+    # @param rules [Hash] Rule mapping {from_rule: to_rule}
+    # @return [self] For chaining
+    def import_with_rules(grammar, prefix: nil, rules: {})
+      grammar_data = case grammar
+                     when GrammarBuilder
+                       grammar.to_h
+                     when Hash
+                       grammar
+                     else
+                       raise ArgumentError, "Expected GrammarBuilder or Hash, got #{grammar.class}"
+                     end
+
+      @imports << { grammar: grammar_data, prefix: prefix, rules: rules }
+      self
+    end
+
+    # Build the grammar
+    #
+    # @return [Hash] Grammar representation
+    def build
+      {
+        rules: @rules,
+        root: @root,
+        imports: @imports
+      }
+    end
+
+    # Convert to JSON for native parser
+    #
+    # @return [String] JSON representation
+    def to_json(*_args)
+      build.to_json
+    end
+
+    # Get as a Hash
+    #
+    # @return [Hash] Grammar representation
+    def to_h
+      build
+    end
+
+    # Reference another rule in this grammar
+    #
+    # @param name [String, Symbol] Rule name
+    # @return [Parsanol::Atoms::Entity] Entity referencing the rule
+    def ref(name)
+      Parsanol::Atoms::Entity.new(name)
+    end
+
+    # Reference the root of another grammar
+    #
+    # @param grammar_name [String] Name of the grammar (for prefixed imports)
+    # @return [Parsanol::Atoms::Entity] Entity referencing the root
+    def ref_root(grammar_name = nil)
+      if grammar_name
+        ref("#{grammar_name}:root")
+      else
+        ref('root')
+      end
+    end
+
+    class << self
+      # Create a grammar from a block
+      #
+      # @yield [GrammarBuilder] Builder to configure
+      # @return [Hash] Built grammar
+      def build(&block)
+        builder = new
+        builder.instance_eval(&block)
+        builder.build
+      end
+
+      # Import a grammar from JSON string
+      #
+      # @param json [String] JSON representation
+      # @return [Hash] Grammar representation
+      def from_json(json)
+        JSON.parse(json)
+      end
+    end
+  end
+
+  # Module methods for DSL
+  module GrammarBuilderDSL
+    # Create a new grammar builder
+    #
+    # @return [GrammarBuilder] New builder
+    def grammar(&block)
+      GrammarBuilder.build(&block)
+    end
+  end
+end

data/lib/parsanol/incremental_parser.rb

@@ -0,0 +1,177 @@
+# frozen_string_literal: true
+
+# Parsanol::IncrementalParser - Incremental Parser for Editor Integration
+#
+# Parse with support for incremental edits. This is useful for editor integration
+# where the input changes frequently (e.g., as the user types).
+#
+# Usage:
+#   parser = Parsanol::IncrementalParser.new(grammar, initial_text)
+#
+#   # When text changes
+#   parser.apply_edit(start: 5, deleted: 3, inserted: "new")
+#   result = parser.reparse
+#
+# Requires native extension for full functionality.
+
+module Parsanol
+  # Represents an edit to apply to the input
+  class Edit
+    attr_reader :start, :deleted, :inserted
+
+    def initialize(start:, deleted:, inserted: '')
+      @start = start
+      @deleted = deleted
+      @inserted = inserted
+    end
+
+    # Get the old range that was replaced
+    def old_range
+      @start...(@start + @deleted)
+    end
+
+    # Check if this edit affects a specific position
+    def affects_position?(position)
+      position >= @start && position < @start + @deleted + @inserted.length
+    end
+
+    # Get the new position after this edit
+    def new_position
+      @start + @inserted.length
+    end
+
+    # Apply this edit to a string
+    def apply(input)
+      input[0...@start] + @inserted + input[(@start + @deleted)..]
+    end
+
+    def to_s
+      "Edit(#{@start}, +#{@inserted.length}, -#{@deleted})"
+    end
+
+    def ==(other)
+      return false unless other.is_a?(Edit)
+
+      @start == other.start && @deleted == other.deleted && @inserted == other.inserted
+    end
+  end
+
+  class IncrementalParser
+    # Create a new incremental parser
+    #
+    # @param grammar [Parsanol::Parser, Parsanol::Atoms::Base] Grammar to use
+    # @param initial_input [String] Initial input string
+    def initialize(grammar, initial_input = '')
+      @grammar = grammar
+      @input = initial_input
+
+      if Parsanol::Native.available?
+        grammar_json = Parsanol::Native.serialize_grammar(grammar.root)
+        @native_parser = Parsanol::Native.incremental_parser_new(grammar_json, initial_input)
+      else
+        @native_parser = nil
+      end
+
+      @edits = []
+      @cached_result = nil
+    end
+
+    # Apply an edit to the parser
+    #
+    # @param start [Integer] Start position of edit
+    # @param deleted [Integer] Number of characters deleted
+    # @param inserted [String] Text to insert
+    def apply_edit(start:, deleted:, inserted: '')
+      edit = Edit.new(start: start, deleted: deleted, inserted: inserted)
+      @edits << edit
+
+      # Update cached input
+      @input = edit.apply(@input)
+
+      # Invalidate cached result
+      @cached_result = nil
+
+      return unless @native_parser
+
+      Parsanol::Native.incremental_parser_apply_edit(@native_parser, start, deleted, inserted)
+    end
+
+    # Convenience method to apply multiple edits
+    #
+    # @param edits [Array<Hash>] Array of {start:, deleted:, inserted:} hashes
+    def apply_edits(edits)
+      edits.each do |edit_hash|
+        apply_edit(**edit_hash)
+      end
+    end
+
+    # Reparse with current input (or optional new input)
+    #
+    # @param new_input [String, nil] Optional new input (replaces current)
+    # @return [Object] Parse result
+    def reparse(new_input = nil)
+      if new_input
+        @input = new_input
+        @edits.clear
+        @cached_result = nil
+      end
+
+      return @cached_result if @cached_result
+
+      if @native_parser
+        @cached_result = Parsanol::Native.incremental_parser_reparse(@native_parser, @input)
+      else
+        # Pure Ruby fallback - reparse from scratch
+        root = @grammar.root
+        @cached_result = root.parse(@input)
+      end
+
+      @cached_result
+    end
+
+    # Invalidate a range (for external changes)
+    #
+    # @param start [Integer] Start position
+    # @param end_pos [Integer] End position
+    def invalidate_range(_start, _end_pos)
+      # Clear cached result if the invalidated range might affect it
+      @cached_result = nil
+
+      nil unless @native_parser
+      # Native implementation handles invalidation
+    end
+
+    # Get the current input
+    #
+    # @return [String] Current input
+    attr_reader :input
+
+    # Get all applied edits
+    #
+    # @return [Array<Edit>] Array of edits
+    def edits
+      @edits.dup
+    end
+
+    # Check if there are unapplied edits
+    #
+    # @return [Boolean] True if there are pending edits
+    def dirty?
+      @cached_result.nil? && !@edits.empty?
+    end
+
+    # Reset to initial state
+    #
+    # @param new_input [String, nil] Optional new initial input
+    def reset(new_input = nil)
+      @input = new_input || ''
+      @edits.clear
+      @cached_result = nil
+
+      return unless @native_parser && new_input
+
+      grammar_json = Parsanol::Native.serialize_grammar(@grammar.root)
+      @native_parser = Parsanol::Native.incremental_parser_new(grammar_json, @input)
+    end
+  end
+end