parsanol 3.0.0

data/lib/parsanol/fast_mode.rb

@@ -0,0 +1,145 @@
+# frozen_string_literal: true
+
+# Fast mode patch for Parslet - matches vanilla parslet 2.0 behavior.
+#
+# For grammars with many small allocations (like EXPRESS), this is faster
+# because the overhead of pool management exceeds the benefit.
+#
+# Usage:
+#   require 'parslet'
+#   require 'parsanol/fast_mode'
+#   # Now all parsing uses fast mode methods
+#
+
+module Parsanol
+  FAST_MODE = true
+
+  module Atoms
+    # Fast mode Context - matches vanilla parslet 2.0 simplicity
+    class Context
+      # Override try_with_cache with vanilla-like version (no eviction, no pooling)
+      def try_with_cache(obj, source, consume_all)
+        beg = source.bytepos
+
+        # Not in cache yet? Return early.
+        unless (entry = @cache[beg]&.[](obj.object_id))
+          result = obj.try(source, self, consume_all)
+
+          if obj.cached?
+            (@cache[beg] ||= {})[obj.object_id] = [result, source.bytepos - beg]
+          end
+
+          return result
+        end
+
+        # Cache hit
+        result, advance = entry
+        source.bytepos = beg + advance
+        result
+      end
+    end
+
+    # Fast mode Sequence - direct array creation, no lazy evaluation
+    class Sequence
+      def try(source, context, consume_all)
+        parslets = @parslets
+
+        case parslets.size
+        when 1
+          success, value = parslets[0].apply(source, context, consume_all)
+          return success ? succ([:sequence, value]) : context.err(self, source, @error_msg, [value])
+        when 2
+          success, v1 = parslets[0].apply(source, context, false)
+          return context.err(self, source, @error_msg, [v1]) unless success
+          success, v2 = parslets[1].apply(source, context, consume_all)
+          return success ? succ([:sequence, v1, v2]) : context.err(self, source, @error_msg, [v2])
+        when 3
+          success, v1 = parslets[0].apply(source, context, false)
+          return context.err(self, source, @error_msg, [v1]) unless success
+          success, v2 = parslets[1].apply(source, context, false)
+          return context.err(self, source, @error_msg, [v2]) unless success
+          success, v3 = parslets[2].apply(source, context, consume_all)
+          return success ? succ([:sequence, v1, v2, v3]) : context.err(self, source, @error_msg, [v3])
+        else
+          result = [:sequence]
+          last_idx = parslets.size - 1
+          i = 0
+          while i <= last_idx
+            success, value = parslets[i].apply(source, context, consume_all && i == last_idx)
+            return context.err(self, source, @error_msg, [value]) unless success
+            result << value
+            i += 1
+          end
+          succ(result)
+        end
+      end
+    end
+
+    # Fast mode Repetition - direct array creation, no lazy evaluation
+    class Repetition
+      EMPTY_REPETITION_ARRAY = [:repetition].freeze
+
+      def try(source, context, consume_all)
+        parslet = @parslet
+        min = @min
+        max = @max
+        tag = @tag
+
+        # Fast path for .maybe
+        if min == 0 && max == 1
+          success, value = parslet.apply(source, context, false)
+          return succ([tag, value]) if success
+          return succ(tag == :repetition ? EMPTY_REPETITION_ARRAY : [tag])
+        end
+
+        # Fast path for exact count
+        if min == max && max && max <= 3
+          case max
+          when 1
+            success, value = parslet.apply(source, context, consume_all)
+            return success ? succ([tag, value]) : context.err_at(self, source, @error_msg, source.bytepos, [value])
+          when 2
+            success, v1 = parslet.apply(source, context, false)
+            return context.err_at(self, source, @error_msg, source.bytepos, [v1]) unless success
+            success, v2 = parslet.apply(source, context, consume_all)
+            return success ? succ([tag, v1, v2]) : context.err_at(self, source, @error_msg, source.bytepos, [v2])
+          when 3
+            success, v1 = parslet.apply(source, context, false)
+            return context.err_at(self, source, @error_msg, source.bytepos, [v1]) unless success
+            success, v2 = parslet.apply(source, context, false)
+            return context.err_at(self, source, @error_msg, source.bytepos, [v2]) unless success
+            success, v3 = parslet.apply(source, context, consume_all)
+            return success ? succ([tag, v1, v2, v3]) : context.err_at(self, source, @error_msg, source.bytepos, [v3])
+          end
+        end
+
+        # General case
+        start_pos = source.bytepos
+        occ = 0
+        result = [tag]
+        break_on = nil
+
+        loop do
+          success, value = parslet.apply(source, context, false)
+          break_on = value
+          break unless success
+
+          occ += 1
+          result << value
+          break if max && occ >= max
+        end
+
+        if occ < min
+          source.bytepos = start_pos
+          return context.err_at(self, source, @error_msg, start_pos, [break_on])
+        end
+
+        if consume_all && source.chars_left > 0
+          return context.err(self, source, @unconsumed_msg, [break_on])
+        end
+
+        succ(result)
+      end
+    end
+  end
+end

data/lib/parsanol/first_set.rb

@@ -0,0 +1,75 @@
+# FIRST Set Analysis for PEG Grammars
+#
+# FIRST sets help identify which terminals can appear at the beginning of
+# a parse. This is essential for:
+# 1. Automatic cut operator insertion (AC-FIRST algorithm)
+# 2. Grammar analysis and optimization
+# 3. Detecting ambiguous choices
+#
+# Reference: Mizushima et al. (2010) "Packrat Parsers Can Handle Practical
+# Grammars in Mostly Constant Space"
+#
+module Parsanol::FirstSet
+  # Sentinel value representing the empty string (ε)
+  EPSILON = :epsilon
+
+  # Compute the FIRST set for this parslet atom
+  # Returns a Set containing:
+  # - Terminal atoms (Str, Re) that can match first
+  # - EPSILON if the atom can match empty string
+  # - nil elements represent unknown/variable terminals (e.g., any)
+  #
+  # @return [Set] FIRST set containing terminal atoms or EPSILON
+  def first_set
+    @first_set_cache ||= compute_first_set
+  end
+
+  # Clear cached FIRST set (useful after grammar modifications)
+  def clear_first_set_cache
+    @first_set_cache = nil
+  end
+
+  protected
+
+  # Override in subclasses to compute FIRST set
+  # Default: conservative approximation (unknown)
+  def compute_first_set
+    Set.new([nil])  # nil = unknown terminal
+  end
+
+  # Class methods for FIRST set analysis
+  class << self
+    # Check if two FIRST sets are disjoint
+    # Two sets are disjoint if they have no common elements
+    # EPSILON is ignored when checking disjointness
+    #
+    # @param set1 [Set] First FIRST set
+    # @param set2 [Set] Second FIRST set
+    # @return [Boolean] true if sets are disjoint
+    def disjoint?(set1, set2)
+      # Remove EPSILON and nil from both sets for comparison
+      real_set1 = set1.reject { |x| x == EPSILON || x.nil? }
+      real_set2 = set2.reject { |x| x == EPSILON || x.nil? }
+
+      # If either set is empty (only EPSILON/nil), consider disjoint
+      return true if real_set1.empty? || real_set2.empty?
+
+      # Check if intersection is empty (using to_a for Opal compatibility)
+      (real_set1.to_a & real_set2.to_a).empty?
+    end
+
+    # Check if all FIRST sets in a collection are mutually disjoint
+    # This is critical for AC-FIRST algorithm - we can only insert
+    # cuts when all alternatives have non-overlapping FIRST sets
+    #
+    # @param sets [Array<Set>] Collection of FIRST sets
+    # @return [Boolean] true if all pairs are disjoint
+    def all_disjoint?(sets)
+      # Need at least 2 sets to check disjointness
+      return true if sets.length < 2
+
+      # Check all pairs
+      sets.combination(2).all? { |s1, s2| disjoint?(s1, s2) }
+    end
+  end
+end

data/lib/parsanol/grammar_builder.rb

@@ -0,0 +1,177 @@
+# frozen_string_literal: true
+
+# Parsanol::GrammarBuilder - Grammar Composition
+#
+# Build complex grammars by importing and composing smaller grammars.
+# This enables reusable grammar modules.
+#
+# Usage:
+#   # Define reusable grammars
+#   expression_grammar = GrammarBuilder.new
+#     .rule("expr", str("a") | str("b"))
+#     .build
+#
+#   type_grammar = GrammarBuilder.new
+#     .rule("type", str("int") | str("str"))
+#     .build
+#
+#   # Compose into a new grammar
+#   combined = GrammarBuilder.new
+#     .import(expression_grammar, prefix: "expr")
+#     .import(type_grammar, prefix: "type")
+#     .rule("typed", seq([ref("expr:root"), str(":"), ref("type:root")]))
+#     .build
+#
+# Requires native extension for full functionality.
+
+module Parsanol
+  class GrammarBuilder
+    # Create a new grammar builder
+    def initialize
+      @rules = {}
+      @imports = []
+      @root = nil
+    end
+
+    # Define a rule
+    #
+    # @param name [String, Symbol] Rule name
+    # @param parslet [Parsanol::Atoms::Base] Parslet atom
+    # @return [self] For chaining
+    def rule(name, parslet)
+      @rules[name.to_s] = parslet
+      self
+    end
+
+    # Get a rule for modification
+    #
+    # @param name [String, Symbol] Rule name
+    # @return [Parsanol::Atoms::Base, nil] The rule atom
+    def [](name)
+      @rules[name.to_s]
+    end
+
+    # Set the root rule
+    #
+    # @param name [String, Symbol] Root rule name
+    # @return [self] For chaining
+    def root(name)
+      @root = name.to_s
+      self
+    end
+
+    # Import another grammar with optional prefix
+    #
+    # @param grammar [GrammarBuilder, Hash] Grammar to import
+    # @param prefix [String, nil] Optional prefix for imported rules
+    # @return [self] For chaining
+    def import(grammar, prefix: nil)
+      grammar_data = case grammar
+                     when GrammarBuilder
+                       grammar.to_h
+                     when Hash
+                       grammar
+                     else
+                       raise ArgumentError, "Expected GrammarBuilder or Hash, got #{grammar.class}"
+                     end
+
+      @imports << { grammar: grammar_data, prefix: prefix }
+      self
+    end
+
+    # Import with explicit rule mapping
+    #
+    # @param grammar [GrammarBuilder, Hash] Grammar to import
+    # @param prefix [String, nil] Optional prefix
+    # @param rules [Hash] Rule mapping {from_rule: to_rule}
+    # @return [self] For chaining
+    def import_with_rules(grammar, prefix: nil, rules: {})
+      grammar_data = case grammar
+                     when GrammarBuilder
+                       grammar.to_h
+                     when Hash
+                       grammar
+                     else
+                       raise ArgumentError, "Expected GrammarBuilder or Hash, got #{grammar.class}"
+                     end
+
+      @imports << { grammar: grammar_data, prefix: prefix, rules: rules }
+      self
+    end
+
+    # Build the grammar
+    #
+    # @return [Hash] Grammar representation
+    def build
+      {
+        rules: @rules,
+        root: @root,
+        imports: @imports
+      }
+    end
+
+    # Convert to JSON for native parser
+    #
+    # @return [String] JSON representation
+    def to_json
+      build.to_json
+    end
+
+    # Get as a Hash
+    #
+    # @return [Hash] Grammar representation
+    def to_h
+      build
+    end
+
+    # Reference another rule in this grammar
+    #
+    # @param name [String, Symbol] Rule name
+    # @return [Parsanol::Atoms::Entity] Entity referencing the rule
+    def ref(name)
+      Parsanol::Atoms::Entity.new(name)
+    end
+
+    # Reference the root of another grammar
+    #
+    # @param grammar_name [String] Name of the grammar (for prefixed imports)
+    # @return [Parsanol::Atoms::Entity] Entity referencing the root
+    def ref_root(grammar_name = nil)
+      if grammar_name
+        ref("#{grammar_name}:root")
+      else
+        ref('root')
+      end
+    end
+
+    class << self
+      # Create a grammar from a block
+      #
+      # @yield [GrammarBuilder] Builder to configure
+      # @return [Hash] Built grammar
+      def build(&block)
+        builder = new
+        builder.instance_eval(&block)
+        builder.build
+      end
+
+      # Import a grammar from JSON string
+      #
+      # @param json [String] JSON representation
+      # @return [Hash] Grammar representation
+      def from_json(json)
+        JSON.parse(json)
+      end
+    end
+  end
+
+  # Module methods for DSL
+  module GrammarBuilderDSL
+    # Create a new grammar builder
+    #
+    # @return [GrammarBuilder] New builder
+    def grammar(&block)
+      GrammarBuilder.build(&block)
+    end
+  end
+end

data/lib/parsanol/graphviz.rb

@@ -0,0 +1,97 @@
+
+# Paints a graphviz graph of your parser.
+
+begin
+  require 'ruby-graphviz'
+rescue LoadError
+  puts "Please install the 'ruby-graphviz' gem first."
+  fail
+end
+
+require 'set'
+require 'parsanol/atoms/visitor'
+
+module Parsanol
+  class GraphvizVisitor
+    def initialize g
+      @graph = g
+      @known_links = Set.new
+      @visited = Set.new
+    end
+
+    attr_reader :parent
+
+    def visit_parser(root)
+      recurse root, node('parser')
+    end
+    def visit_entity(name, block)
+      s = node(name)
+
+      downwards s
+
+      return if @visited.include?(name)
+      @visited << name
+
+      recurse block.call, s
+    end
+    def visit_named(name, atom)
+      recurse atom, parent
+    end
+    def visit_repetition(tag, min, max, atom)
+      recurse atom, parent
+    end
+    def visit_alternative(alternatives)
+      p = parent
+      alternatives.each do |atom|
+        recurse atom, p
+      end
+    end
+    def visit_sequence(sequence)
+      p = parent
+      sequence.each do |atom|
+        recurse atom, p
+      end
+    end
+    def visit_lookahead(positive, atom)
+      recurse atom, parent
+    end
+    def visit_re(regexp)
+      # downwards node(regexp.object_id, label: escape("re(#{regexp.inspect})"))
+    end
+    def visit_str(str)
+      # downwards node(str.object_id, label: escape("#{str.inspect}"))
+    end
+
+    def escape str
+      str.gsub('"', "'")
+    end
+    def node name, opts={}
+      @graph.add_nodes name.to_s, opts
+    end
+    def downwards child
+      if @parent && !@known_links.include?([@parent, child])
+        @graph.add_edges(@parent, child)
+        @known_links << [@parent, child]
+      end
+    end
+    def recurse node, current
+      @parent = current
+      node.accept(self)
+    end
+  end
+
+  module Graphable
+    def graph opts
+      g = GraphViz.new(:G, type: :digraph)
+      visitor = GraphvizVisitor.new(g)
+
+      new.accept(visitor)
+
+      g.output opts
+    end
+  end
+
+  class Parser # reopen for introducing the .graph method
+    extend Graphable
+  end
+end

data/lib/parsanol/incremental_parser.rb

@@ -0,0 +1,179 @@
+# frozen_string_literal: true
+
+# Parsanol::IncrementalParser - Incremental Parser for Editor Integration
+#
+# Parse with support for incremental edits. This is useful for editor integration
+# where the input changes frequently (e.g., as the user types).
+#
+# Usage:
+#   parser = Parsanol::IncrementalParser.new(grammar, initial_text)
+#
+#   # When text changes
+#   parser.apply_edit(start: 5, deleted: 3, inserted: "new")
+#   result = parser.reparse
+#
+# Requires native extension for full functionality.
+
+module Parsanol
+  # Represents an edit to apply to the input
+  class Edit
+    attr_reader :start, :deleted, :inserted
+
+    def initialize(start:, deleted:, inserted: '')
+      @start = start
+      @deleted = deleted
+      @inserted = inserted
+    end
+
+    # Get the old range that was replaced
+    def old_range
+      @start...(@start + @deleted)
+    end
+
+    # Check if this edit affects a specific position
+    def affects_position?(position)
+      position >= @start && position < @start + @deleted + @inserted.length
+    end
+
+    # Get the new position after this edit
+    def new_position
+      @start + @inserted.length
+    end
+
+    # Apply this edit to a string
+    def apply(input)
+      input[0...@start] + @inserted + input[@start + @deleted..]
+    end
+
+    def to_s
+      "Edit(#{@start}, +#{@inserted.length}, -#{@deleted})"
+    end
+
+    def ==(other)
+      return false unless other.is_a?(Edit)
+      @start == other.start && @deleted == other.deleted && @inserted == other.inserted
+    end
+  end
+
+  class IncrementalParser
+    # Create a new incremental parser
+    #
+    # @param grammar [Parsanol::Parser, Parsanol::Atoms::Base] Grammar to use
+    # @param initial_input [String] Initial input string
+    def initialize(grammar, initial_input = '')
+      @grammar = grammar
+      @input = initial_input
+
+      if Parsanol::Native.available?
+        grammar_json = Parsanol::Native.serialize_grammar(grammar.root)
+        @native_parser = Parsanol::Native.incremental_parser_new(grammar_json, initial_input)
+      else
+        @native_parser = nil
+      end
+
+      @edits = []
+      @cached_result = nil
+    end
+
+    # Apply an edit to the parser
+    #
+    # @param start [Integer] Start position of edit
+    # @param deleted [Integer] Number of characters deleted
+    # @param inserted [String] Text to insert
+    def apply_edit(start:, deleted:, inserted: '')
+      edit = Edit.new(start: start, deleted: deleted, inserted: inserted)
+      @edits << edit
+
+      # Update cached input
+      @input = edit.apply(@input)
+
+      # Invalidate cached result
+      @cached_result = nil
+
+      if @native_parser
+        Parsanol::Native.incremental_parser_apply_edit(@native_parser, start, deleted, inserted)
+      end
+    end
+
+    # Convenience method to apply multiple edits
+    #
+    # @param edits [Array<Hash>] Array of {start:, deleted:, inserted:} hashes
+    def apply_edits(edits)
+      edits.each do |edit_hash|
+        apply_edit(**edit_hash)
+      end
+    end
+
+    # Reparse with current input (or optional new input)
+    #
+    # @param new_input [String, nil] Optional new input (replaces current)
+    # @return [Object] Parse result
+    def reparse(new_input = nil)
+      if new_input
+        @input = new_input
+        @edits.clear
+        @cached_result = nil
+      end
+
+      return @cached_result if @cached_result
+
+      if @native_parser
+        @cached_result = Parsanol::Native.incremental_parser_reparse(@native_parser, @input)
+      else
+        # Pure Ruby fallback - reparse from scratch
+        root = @grammar.root
+        @cached_result = root.parse(@input)
+      end
+
+      @cached_result
+    end
+
+    # Invalidate a range (for external changes)
+    #
+    # @param start [Integer] Start position
+    # @param end_pos [Integer] End position
+    def invalidate_range(start, end_pos)
+      # Clear cached result if the invalidated range might affect it
+      @cached_result = nil
+
+      if @native_parser
+        # Native implementation handles invalidation
+      end
+    end
+
+    # Get the current input
+    #
+    # @return [String] Current input
+    def input
+      @input
+    end
+
+    # Get all applied edits
+    #
+    # @return [Array<Edit>] Array of edits
+    def edits
+      @edits.dup
+    end
+
+    # Check if there are unapplied edits
+    #
+    # @return [Boolean] True if there are pending edits
+    def dirty?
+      @cached_result.nil? && !@edits.empty?
+    end
+
+    # Reset to initial state
+    #
+    # @param new_input [String, nil] Optional new initial input
+    def reset(new_input = nil)
+      @input = new_input || ''
+      @edits.clear
+      @cached_result = nil
+
+      if @native_parser && new_input
+        grammar_json = Parsanol::Native.serialize_grammar(@grammar.root)
+        @native_parser = Parsanol::Native.incremental_parser_new(grammar_json, @input)
+      end
+    end
+  end
+end