parsanol 3.0.0

data/lib/parsanol/native/types.rb

@@ -0,0 +1,41 @@
+# frozen_string_literal: true
+
+module Parsanol
+  module Native
+    # Type tags used in AST serialization
+    # These must match the tags used by the Rust parser
+    module Types
+      # AST node type tags (must match Rust parser output)
+      TAG_NIL = 0x00
+      TAG_BOOL = 0x01
+      TAG_INT = 0x02
+      TAG_FLOAT = 0x03
+      TAG_STRING_REF = 0x04
+      TAG_ARRAY_START = 0x05
+      TAG_ARRAY_END = 0x06
+      TAG_HASH_START = 0x07
+      TAG_HASH_END = 0x08
+      TAG_HASH_KEY = 0x09
+      TAG_INLINE_STRING = 0x0A
+
+      # Frozen string constants for transformer (avoid allocations)
+      SEQUENCE_TAG = ':sequence'.freeze
+      REPETITION_TAG = ':repetition'.freeze
+      EMPTY_STRING = ''.freeze
+      EMPTY_ARRAY = [].freeze
+      EMPTY_HASH = {}.freeze
+    end
+
+    # Symbol cache to avoid repeated string-to-symbol conversions
+    # This is a class variable to share across all transformations
+    @@symbol_cache = {}
+
+    # Convert string key to symbol with caching
+    # @param key [String, Symbol] The key to convert
+    # @return [Symbol] The symbol version of the key
+    def self.cached_symbol(key)
+      return key if key.is_a?(Symbol)
+      @@symbol_cache[key] ||= key.to_sym
+    end
+  end
+end

data/lib/parsanol/native.rb

@@ -0,0 +1,217 @@
+# frozen_string_literal: true
+
+require 'json'
+require 'digest'
+
+# Entry point for native parsing functionality
+# Requires the individual components
+require 'parsanol/native/types'
+require 'parsanol/native/parser'
+require 'parsanol/native/transformer'
+require 'parsanol/native/serializer'
+
+module Parsanol
+  module Native
+    VERSION = '0.1.0'
+
+    class << self
+      # Delegate to Parser module
+      def available?
+        Parser.available?
+      end
+
+      def parse(grammar_json, input)
+        Parser.parse(grammar_json, input)
+      end
+
+      def parse_with_grammar(root_atom, input)
+        Parser.parse_with_grammar(root_atom, input)
+      end
+
+      def parse_parslet_compatible(root_atom, input)
+        Parser.parse_parslet_compatible(root_atom, input)
+      end
+
+      def parse_batch_inputs(root_atom, inputs)
+        Parser.parse_batch_inputs(root_atom, inputs)
+      end
+
+      def parse_batch_with_transform(root_atom, inputs)
+        Parser.parse_batch_with_transform(root_atom, inputs)
+      end
+
+      def parse_raw(root_atom, input)
+        Parser.parse_raw(root_atom, input)
+      end
+
+      def serialize_grammar(root_atom)
+        Parser.serialize_grammar(root_atom)
+      end
+
+      def clear_cache
+        Parser.clear_cache
+      end
+
+      def cache_stats
+        Parser.cache_stats
+      end
+
+      # Serialized Mode (JSON Output)
+      def parse_to_json(grammar_json, input)
+        Parser.parse_to_json(grammar_json, input)
+      end
+
+      # ZeroCopy Mode (Direct Ruby Objects)
+      def parse_to_objects(grammar_json, input, type_map = nil)
+        Parser.parse_to_objects(grammar_json, input, type_map)
+      end
+
+      def convert_slices(obj, input)
+        Parser.convert_slices(obj, input)
+      end
+
+      # Source Location Tracking
+      def parse_with_spans(grammar_json, input)
+        Parser.parse_with_spans(grammar_json, input)
+      end
+
+      def get_span(result, node_id)
+        Parser.get_span(result, node_id)
+      end
+
+      # Grammar Composition
+      def grammar_import(builder_json, grammar_json, prefix = nil)
+        Parser.grammar_import(builder_json, grammar_json, prefix)
+      end
+
+      def grammar_rule_mut(builder_json, rule_name)
+        Parser.grammar_rule_mut(builder_json, rule_name)
+      end
+
+      # Streaming Parser
+      def streaming_parser_new(grammar_json)
+        Parser.streaming_parser_new(grammar_json)
+      end
+
+      def streaming_parser_add_chunk(parser, chunk)
+        Parser.streaming_parser_add_chunk(parser, chunk)
+      end
+
+      def streaming_parser_parse_chunk(parser)
+        Parser.streaming_parser_parse_chunk(parser)
+      end
+
+      # Incremental Parser
+      def incremental_parser_new(grammar_json, initial_input)
+        Parser.incremental_parser_new(grammar_json, initial_input)
+      end
+
+      def incremental_parser_apply_edit(parser, start, deleted, inserted = '')
+        Parser.incremental_parser_apply_edit(parser, start, deleted, inserted)
+      end
+
+      def incremental_parser_reparse(parser, new_input = nil)
+        Parser.incremental_parser_reparse(parser, new_input)
+      end
+
+      # Streaming Builder - uses native parse_with_builder directly (exposed from Rust)
+      # The native function is exposed directly on Parsanol::Native module
+
+      # Alias for parse_with_builder (same functionality)
+      def parse_with_callback(grammar_json, input, callback)
+        parse_with_builder(grammar_json, input, callback)
+      end
+
+      # Parallel Parsing - uses native _parse_batch_parallel
+      def parse_batch_parallel(grammar_json, inputs, num_threads: nil)
+        _parse_batch_parallel(grammar_json, inputs, num_threads || 0)
+      end
+
+      # Security / Limits - uses native _parse_with_limits
+      def parse_with_limits(grammar_json, input, max_input_size: 100 * 1024 * 1024, max_recursion_depth: 1000)
+        _parse_with_limits(grammar_json, input, max_input_size, max_recursion_depth)
+      end
+
+      # Debug Tools
+      def parse_with_trace(grammar_json, input)
+        Parser.parse_with_trace(grammar_json, input)
+      end
+
+      def grammar_to_mermaid(grammar_json)
+        Parser.grammar_to_mermaid(grammar_json)
+      end
+
+      def grammar_to_dot(grammar_json)
+        Parser.grammar_to_dot(grammar_json)
+      end
+
+      # Legacy internal methods (for backward compatibility)
+      def _parse_with_spans(grammar_json, input)
+        Parser.send(:_parse_with_spans, grammar_json, input)
+      end
+
+      def _get_span(result, node_id)
+        Parser.send(:_get_span, result, node_id)
+      end
+
+      def _grammar_import(builder_json, grammar_json, prefix)
+        Parser.send(:_grammar_import, builder_json, grammar_json, prefix)
+      end
+
+      def _grammar_rule_mut(builder_json, rule_name)
+        Parser.send(:_grammar_rule_mut, builder_json, rule_name)
+      end
+
+      def _streaming_parser_new(grammar_json)
+        Parser.send(:_streaming_parser_new, grammar_json)
+      end
+
+      def _streaming_parser_add_chunk(parser, chunk)
+        Parser.send(:_streaming_parser_add_chunk, parser, chunk)
+      end
+
+      def _streaming_parser_parse_chunk(parser)
+        Parser.send(:_streaming_parser_parse_chunk, parser)
+      end
+
+      def _incremental_parser_new(grammar_json, initial_input)
+        Parser.send(:_incremental_parser_new, grammar_json, initial_input)
+      end
+
+      def _incremental_parser_apply_edit(parser, start, deleted, inserted)
+        Parser.send(:_incremental_parser_apply_edit, parser, start, deleted, inserted)
+      end
+
+      def _incremental_parser_reparse(parser, new_input)
+        Parser.send(:_incremental_parser_reparse, parser, new_input)
+      end
+
+      def _parse_batch_parallel(grammar_json, inputs, num_threads)
+        Parser.send(:_parse_batch_parallel, grammar_json, inputs, num_threads)
+      end
+
+      def _parse_with_limits(grammar_json, input, max_input_size, max_recursion_depth)
+        Parser.send(:_parse_with_limits, grammar_json, input, max_input_size, max_recursion_depth)
+      end
+
+      def _parse_with_trace(grammar_json, input)
+        Parser.send(:_parse_with_trace, grammar_json, input)
+      end
+
+      def _grammar_to_mermaid(grammar_json)
+        Parser.send(:_grammar_to_mermaid, grammar_json)
+      end
+
+      def _grammar_to_dot(grammar_json)
+        Parser.send(:_grammar_to_dot, grammar_json)
+      end
+    end
+  end
+end
+
+# Attempt to load native extension
+begin
+  require 'parsanol/parsanol_native'
+rescue LoadError
+  # Native extension not built yet
+end

data/lib/parsanol/optimizer.rb

@@ -0,0 +1,86 @@
+# frozen_string_literal: true
+
+require_relative 'ast_visitor'
+require_relative 'optimizers/quantifier_optimizer'
+require_relative 'optimizers/sequence_optimizer'
+require_relative 'optimizers/choice_optimizer'
+require_relative 'optimizers/lookahead_optimizer'
+require_relative 'optimizers/cut_inserter'
+
+# Grammar-level optimizations for Parslet parsers
+# These optimizations transform the parser AST to reduce runtime overhead
+# without changing semantics.
+#
+# Architecture:
+# - Uses Visitor pattern for clean separation of traversal and transformation
+# - Each optimizer is a separate class inheriting from ASTVisitor
+# - Optimizer module provides facade methods for easy access
+module Parsanol
+  module Optimizer
+    # Simplifies redundant quantifiers in a parslet tree
+    # Example: str('a').repeat(1, 1) => str('a')
+    #          str('a').repeat(0, 1).repeat(0, 1) => str('a').repeat(0, 1)
+    #
+    # @param parslet [Parsanol::Atoms::Base] parslet to simplify
+    # @return [Parsanol::Atoms::Base] simplified parslet
+    def self.simplify_quantifiers(parslet)
+      Optimizers::QuantifierOptimizer.new.visit(parslet)
+    end
+
+    # Simplifies sequences by flattening and merging adjacent strings
+    # Example: str('a') >> str('b') => str('ab')
+    #          (str('a') >> str('b')) >> str('c') => str('abc')
+    #
+    # @param parslet [Parsanol::Atoms::Base] parslet to simplify
+    # @return [Parsanol::Atoms::Base] simplified parslet
+    def self.simplify_sequences(parslet)
+      Optimizers::SequenceOptimizer.new.visit(parslet)
+    end
+
+    # Simplifies choice/alternative patterns
+    # Example: (A | B) | C => A | B | C
+    #          A | B | A => A | B
+    #
+    # @param parslet [Parsanol::Atoms::Base] parslet to simplify
+    # @return [Parsanol::Atoms::Base] simplified parslet
+    def self.simplify_choices(parslet)
+      Optimizers::ChoiceOptimizer.new.visit(parslet)
+    end
+
+    # Simplifies lookahead patterns
+    # Example: !(!x) => &x (double negation elimination)
+    #
+    # @param parslet [Parsanol::Atoms::Base] parslet to simplify
+    # @return [Parsanol::Atoms::Base] simplified parslet
+    def self.simplify_lookaheads(parslet)
+      Optimizers::LookaheadOptimizer.new.visit(parslet)
+    end
+
+    # Automatically insert cut operators where safe (AC-FIRST algorithm)
+    # Inserts cuts after deterministic prefixes when alternatives have disjoint FIRST sets
+    # This enables O(1) space complexity by allowing aggressive cache eviction
+    #
+    # Example: str('if') >> x | str('while') >> y
+    #       => str('if').cut >> x | str('while').cut >> y
+    #
+    # @param parslet [Parsanol::Atoms::Base] parslet to optimize
+    # @return [Parsanol::Atoms::Base] optimized parslet with cuts inserted
+    def self.insert_cuts(parslet)
+      Optimizers::CutInserter.new.optimize(parslet)
+    end
+
+    # Apply all optimizations in recommended order
+    # This is a convenience method that applies all optimizer passes
+    #
+    # @param parslet [Parsanol::Atoms::Base] parslet to optimize
+    # @return [Parsanol::Atoms::Base] fully optimized parslet
+    def self.optimize_all(parslet)
+      result = simplify_quantifiers(parslet)
+      result = simplify_sequences(result)
+      result = simplify_choices(result)
+      result = simplify_lookaheads(result)
+      result = insert_cuts(result)
+      result
+    end
+  end
+end

data/lib/parsanol/optimizers/choice_optimizer.rb

@@ -0,0 +1,78 @@
+# frozen_string_literal: true
+
+require_relative '../ast_visitor'
+
+module Parsanol
+  module Optimizers
+    # Optimizes alternative/choice patterns in the AST
+    # Follows visitor pattern for clean separation of concerns
+    #
+    # Transformations:
+    # - (A | B) | C => A | B | C (flatten nested alternatives)
+    # - A | B | A => A | B (remove duplicates)
+    # - Alternative(A) => A (unwrap single-element alternatives)
+    class ChoiceOptimizer < ASTVisitor
+      # Visit an alternative node and apply choice optimizations
+      # @param parslet [Parsanol::Atoms::Alternative] alternative to optimize
+      # @return [Parsanol::Atoms::Base] optimized parslet
+      def visit_alternative(parslet)
+        # First optimize children recursively
+        new_alternatives = parslet.alternatives.map { |p| visit(p) }
+
+        # Optimization 1: Flatten nested alternatives
+        flattened = flatten_alternatives(new_alternatives)
+
+        # Optimization 2: Remove duplicate alternatives
+        deduplicated = deduplicate_alternatives(flattened)
+
+        # Optimization 3: Unwrap single-element alternatives
+        return deduplicated[0] if deduplicated.size == 1
+
+        # Return optimized alternative if changed
+        if deduplicated != parslet.alternatives
+          Parsanol::Atoms::Alternative.new(*deduplicated)
+        else
+          parslet
+        end
+      end
+
+      private
+
+      # Flatten nested alternatives into a single level
+      # @param alternatives [Array<Parsanol::Atoms::Base>] array of alternatives
+      # @return [Array<Parsanol::Atoms::Base>] flattened array
+      def flatten_alternatives(alternatives)
+        result = []
+        alternatives.each do |alt|
+          if alt.is_a?(Parsanol::Atoms::Alternative)
+            result.concat(alt.alternatives)
+          else
+            result << alt
+          end
+        end
+        result
+      end
+
+      # Remove duplicate alternatives using structural equality
+      # @param alternatives [Array<Parsanol::Atoms::Base>] array of alternatives
+      # @return [Array<Parsanol::Atoms::Base>] deduplicated array
+      def deduplicate_alternatives(alternatives)
+        return alternatives if alternatives.size < 2
+
+        # Use to_s as proxy for structural equality
+        seen = {}
+        result = []
+
+        alternatives.each do |alt|
+          key = alt.to_s
+          unless seen[key]
+            seen[key] = true
+            result << alt
+          end
+        end
+
+        result
+      end
+    end
+  end
+end

data/lib/parsanol/optimizers/cut_inserter.rb

@@ -0,0 +1,175 @@
+# Automatic Cut Insertion (AC-FIRST Algorithm)
+#
+# This optimizer implements the AC-FIRST algorithm from Mizushima et al. (2010)
+# to automatically insert cut operators when alternatives have disjoint FIRST sets.
+#
+# When all alternatives in a choice have non-overlapping FIRST sets, we can safely
+# insert a cut after the deterministic prefix, since backtracking will never be
+# needed.
+#
+# Example:
+#   str('if') >> condition >> then_clause |
+#   str('while') >> condition >> body |
+#   str('print') >> expression
+#
+# Becomes:
+#   str('if').cut >> condition >> then_clause |
+#   str('while').cut >> condition >> body |
+#   str('print').cut >> expression
+#
+# Reference: Mizushima et al. (2010) "Packrat Parsers Can Handle Practical
+# Grammars in Mostly Constant Space"
+#
+class Parsanol::Optimizers::CutInserter
+  # Optimize a parslet by inserting cuts where safe
+  # Recursively traverses the grammar AST
+  #
+  # @param parslet [Parsanol::Atoms::Base] The parslet to optimize
+  # @return [Parsanol::Atoms::Base] Optimized parslet with cuts inserted
+  def optimize(parslet)
+    case parslet
+    when Parsanol::Atoms::Alternative
+      optimize_alternative(parslet)
+    when Parsanol::Atoms::Sequence
+      optimize_sequence(parslet)
+    when Parsanol::Atoms::Repetition
+      optimize_repetition(parslet)
+    when Parsanol::Atoms::Named
+      optimize_named(parslet)
+    else
+      # Return atom unchanged (Str, Re, Lookahead, etc.)
+      parslet
+    end
+  end
+
+  private
+
+  # Optimize an Alternative atom by inserting cuts when all alternatives
+  # have disjoint FIRST sets
+  def optimize_alternative(alt)
+    alternatives = alt.alternatives
+    first_sets = alternatives.map(&:first_set)
+
+    # Only optimize if all FIRST sets are disjoint
+    unless Parsanol::FirstSet.all_disjoint?(first_sets)
+      # Not safe to insert cuts - return alternatives with recursive optimization
+      optimized = alternatives.map { |a| optimize(a) }
+      return Parsanol::Atoms::Alternative.new(*optimized)
+    end
+
+    # All FIRST sets are disjoint - safe to insert cuts!
+    # Insert cuts after deterministic prefixes
+    optimized = alternatives.map do |alternative|
+      insert_cut_if_safe(alternative)
+    end
+
+    Parsanol::Atoms::Alternative.new(*optimized)
+  end
+
+  # Optimize a Sequence atom by recursively optimizing its elements
+  def optimize_sequence(seq)
+    optimized_parslets = seq.parslets.map { |p| optimize(p) }
+    Parsanol::Atoms::Sequence.new(*optimized_parslets)
+  end
+
+  # Optimize a Repetition atom by recursively optimizing its parslet
+  def optimize_repetition(rep)
+    optimized_parslet = optimize(rep.parslet)
+    # Create new repetition with same min/max
+    # Note: We use default tag since it's not exposed as a reader
+    Parsanol::Atoms::Repetition.new(
+      optimized_parslet,
+      rep.min,
+      rep.max
+    )
+  end
+
+  # Optimize a Named atom by recursively optimizing its parslet
+  def optimize_named(named)
+    optimized_parslet = optimize(named.parslet)
+    optimized_parslet.as(named.name)
+  end
+
+  # Insert a cut after the deterministic prefix if safe
+  # For sequences: find longest prefix without EPSILON
+  # For other atoms: cut the whole thing if it doesn't include EPSILON
+  def insert_cut_if_safe(parslet)
+    # For sequences, find the longest safe prefix
+    if parslet.is_a?(Parsanol::Atoms::Sequence)
+      prefix_parslets = find_deterministic_prefix(parslet)
+      if prefix_parslets && !prefix_parslets.empty?
+        return build_cut_sequence(parslet, prefix_parslets)
+      end
+    end
+
+    # For other atoms, cut the whole thing if safe
+    if safe_to_cut?(parslet)
+      return parslet.cut
+    end
+
+    # Not safe to cut - recursively optimize and return
+    optimize(parslet)
+  end
+
+  # Find the longest deterministic prefix of a sequence
+  # A deterministic prefix doesn't include EPSILON in its FIRST set
+  #
+  # @param sequence [Parsanol::Atoms::Sequence] The sequence to analyze
+  # @return [Array<Parsanol::Atoms::Base>] Prefix parslets, or nil if none
+  def find_deterministic_prefix(sequence)
+    parslets = sequence.parslets
+    prefix_length = 0
+
+    # Find longest prefix where no element can match empty
+    parslets.each do |p|
+      break if p.first_set.include?(Parsanol::FirstSet::EPSILON)
+      prefix_length += 1
+    end
+
+    prefix_length > 0 ? parslets[0...prefix_length] : nil
+  end
+
+  # Check if it's safe to cut after this parslet
+  # Safe if the parslet doesn't have EPSILON in its FIRST set
+  # (i.e., it always consumes input)
+  def safe_to_cut?(parslet)
+    first = parslet.first_set
+    # Don't cut if EPSILON is in FIRST set (might not consume)
+    # Also don't cut if FIRST set contains only nil (unknown)
+    return false if first.include?(Parsanol::FirstSet::EPSILON)
+    return false if first.all?(&:nil?)
+    true
+  end
+
+  # Build a new sequence with a cut after the prefix
+  #
+  # @param sequence [Parsanol::Atoms::Sequence] Original sequence
+  # @param prefix_parslets [Array] Parslets forming the deterministic prefix
+  # @return [Parsanol::Atoms::Base] New sequence with cut inserted
+  def build_cut_sequence(sequence, prefix_parslets)
+    # Recursively optimize prefix parslets
+    optimized_prefix = prefix_parslets.map { |p| optimize(p) }
+
+    # Build prefix (single parslet or sequence)
+    prefix = if optimized_prefix.length == 1
+      optimized_prefix.first
+    else
+      Parsanol::Atoms::Sequence.new(*optimized_prefix)
+    end
+
+    # Get remaining parslets after prefix
+    remaining = sequence.parslets[prefix_parslets.length..-1]
+
+    # Recursively optimize remaining parslets
+    optimized_remaining = remaining.map { |p| optimize(p) }
+
+    # Build final sequence with cut
+    if optimized_remaining.empty?
+      # Prefix is the entire sequence
+      prefix.cut
+    else
+      # Prefix + cut + remaining
+      Parsanol::Atoms::Sequence.new(prefix.cut, *optimized_remaining)
+    end
+  end
+end

data/lib/parsanol/optimizers/lookahead_optimizer.rb

@@ -0,0 +1,58 @@
+# frozen_string_literal: true
+
+require_relative '../ast_visitor'
+
+module Parsanol
+  module Optimizers
+    # Optimizes lookahead patterns in the AST
+    # Follows visitor pattern for clean separation of concerns
+    #
+    # Transformations:
+    # - !(!x) => &x (double negation elimination)
+    # - &(&x) => &x (positive lookahead is idempotent)
+    # - !(&x) => !x (negative of positive)
+    # - &(!x) => !x (positive of negative)
+    class LookaheadOptimizer < ASTVisitor
+      # Visit a lookahead node and apply lookahead optimizations
+      # @param parslet [Parsanol::Atoms::Lookahead] lookahead to optimize
+      # @return [Parsanol::Atoms::Base] optimized parslet
+      def visit_lookahead(parslet)
+        # First optimize the child
+        inner = visit(parslet.bound_parslet)
+
+        # If inner is also a lookahead, simplify nested lookaheads
+        if inner.is_a?(Parsanol::Atoms::Lookahead)
+          outer_positive = parslet.positive
+          inner_positive = inner.positive
+
+          # !(!x) => &x (double negation)
+          if !outer_positive && !inner_positive
+            return Parsanol::Atoms::Lookahead.new(inner.bound_parslet, true)
+          end
+
+          # &(&x) => &x (idempotent)
+          if outer_positive && inner_positive
+            return inner
+          end
+
+          # !(&x) => !x (negative of positive)
+          if !outer_positive && inner_positive
+            return Parsanol::Atoms::Lookahead.new(inner.bound_parslet, false)
+          end
+
+          # &(!x) => !x (positive of negative)
+          if outer_positive && !inner_positive
+            return inner
+          end
+        end
+
+        # Return lookahead with optimized child
+        if inner.equal?(parslet.bound_parslet)
+          parslet
+        else
+          Parsanol::Atoms::Lookahead.new(inner, parslet.positive)
+        end
+      end
+    end
+  end
+end