parsanol 1.0.0

data/lib/parsanol/native/types.rb

@@ -0,0 +1,42 @@
+# frozen_string_literal: true
+
+module Parsanol
+  module Native
+    # Type tags used in AST serialization
+    # These must match the tags used by the Rust parser
+    module Types
+      # AST node type tags (must match Rust parser output)
+      TAG_NIL = 0x00
+      TAG_BOOL = 0x01
+      TAG_INT = 0x02
+      TAG_FLOAT = 0x03
+      TAG_STRING_REF = 0x04
+      TAG_ARRAY_START = 0x05
+      TAG_ARRAY_END = 0x06
+      TAG_HASH_START = 0x07
+      TAG_HASH_END = 0x08
+      TAG_HASH_KEY = 0x09
+      TAG_INLINE_STRING = 0x0A
+
+      # Frozen string constants for transformer (avoid allocations)
+      SEQUENCE_TAG = ':sequence'
+      REPETITION_TAG = ':repetition'
+      EMPTY_STRING = ''
+      EMPTY_ARRAY = [].freeze
+      EMPTY_HASH = {}.freeze
+    end
+
+    # Symbol cache to avoid repeated string-to-symbol conversions
+    # This is a class variable to share across all transformations
+    @@symbol_cache = {}
+
+    # Convert string key to symbol with caching
+    # @param key [String, Symbol] The key to convert
+    # @return [Symbol] The symbol version of the key
+    def self.cached_symbol(key)
+      return key if key.is_a?(Symbol)
+
+      @@symbol_cache[key] ||= key.to_sym
+    end
+  end
+end

data/lib/parsanol/native.rb

@@ -0,0 +1,217 @@
+# frozen_string_literal: true
+
+require 'json'
+require 'digest'
+
+# Entry point for native parsing functionality
+# Requires the individual components
+require 'parsanol/native/types'
+require 'parsanol/native/parser'
+require 'parsanol/native/transformer'
+require 'parsanol/native/serializer'
+
+module Parsanol
+  module Native
+    VERSION = '0.1.0'
+
+    class << self
+      # Delegate to Parser module
+      def available?
+        Parser.available?
+      end
+
+      def parse(grammar_json, input)
+        Parser.parse(grammar_json, input)
+      end
+
+      def parse_with_grammar(root_atom, input)
+        Parser.parse_with_grammar(root_atom, input)
+      end
+
+      def parse_parslet_compatible(root_atom, input)
+        Parser.parse_parslet_compatible(root_atom, input)
+      end
+
+      def parse_batch_inputs(root_atom, inputs)
+        Parser.parse_batch_inputs(root_atom, inputs)
+      end
+
+      def parse_batch_with_transform(root_atom, inputs)
+        Parser.parse_batch_with_transform(root_atom, inputs)
+      end
+
+      def parse_raw(root_atom, input)
+        Parser.parse_raw(root_atom, input)
+      end
+
+      def serialize_grammar(root_atom)
+        Parser.serialize_grammar(root_atom)
+      end
+
+      def clear_cache
+        Parser.clear_cache
+      end
+
+      def cache_stats
+        Parser.cache_stats
+      end
+
+      # Serialized Mode (JSON Output)
+      def parse_to_json(grammar_json, input)
+        Parser.parse_to_json(grammar_json, input)
+      end
+
+      # ZeroCopy Mode (Direct Ruby Objects)
+      def parse_to_objects(grammar_json, input, type_map = nil)
+        Parser.parse_to_objects(grammar_json, input, type_map)
+      end
+
+      def convert_slices(obj, input)
+        Parser.convert_slices(obj, input)
+      end
+
+      # Source Location Tracking
+      def parse_with_spans(grammar_json, input)
+        Parser.parse_with_spans(grammar_json, input)
+      end
+
+      def get_span(result, node_id)
+        Parser.get_span(result, node_id)
+      end
+
+      # Grammar Composition
+      def grammar_import(builder_json, grammar_json, prefix = nil)
+        Parser.grammar_import(builder_json, grammar_json, prefix)
+      end
+
+      def grammar_rule_mut(builder_json, rule_name)
+        Parser.grammar_rule_mut(builder_json, rule_name)
+      end
+
+      # Streaming Parser
+      def streaming_parser_new(grammar_json)
+        Parser.streaming_parser_new(grammar_json)
+      end
+
+      def streaming_parser_add_chunk(parser, chunk)
+        Parser.streaming_parser_add_chunk(parser, chunk)
+      end
+
+      def streaming_parser_parse_chunk(parser)
+        Parser.streaming_parser_parse_chunk(parser)
+      end
+
+      # Incremental Parser
+      def incremental_parser_new(grammar_json, initial_input)
+        Parser.incremental_parser_new(grammar_json, initial_input)
+      end
+
+      def incremental_parser_apply_edit(parser, start, deleted, inserted = '')
+        Parser.incremental_parser_apply_edit(parser, start, deleted, inserted)
+      end
+
+      def incremental_parser_reparse(parser, new_input = nil)
+        Parser.incremental_parser_reparse(parser, new_input)
+      end
+
+      # Streaming Builder - uses native parse_with_builder directly (exposed from Rust)
+      # The native function is exposed directly on Parsanol::Native module
+
+      # Alias for parse_with_builder (same functionality)
+      def parse_with_callback(grammar_json, input, callback)
+        parse_with_builder(grammar_json, input, callback)
+      end
+
+      # Parallel Parsing - uses native _parse_batch_parallel
+      def parse_batch_parallel(grammar_json, inputs, num_threads: nil)
+        _parse_batch_parallel(grammar_json, inputs, num_threads || 0)
+      end
+
+      # Security / Limits - uses native _parse_with_limits
+      def parse_with_limits(grammar_json, input, max_input_size: 100 * 1024 * 1024, max_recursion_depth: 1000)
+        _parse_with_limits(grammar_json, input, max_input_size, max_recursion_depth)
+      end
+
+      # Debug Tools
+      def parse_with_trace(grammar_json, input)
+        Parser.parse_with_trace(grammar_json, input)
+      end
+
+      def grammar_to_mermaid(grammar_json)
+        Parser.grammar_to_mermaid(grammar_json)
+      end
+
+      def grammar_to_dot(grammar_json)
+        Parser.grammar_to_dot(grammar_json)
+      end
+
+      # Legacy internal methods (for backward compatibility)
+      def _parse_with_spans(grammar_json, input)
+        Parser.send(:_parse_with_spans, grammar_json, input)
+      end
+
+      def _get_span(result, node_id)
+        Parser.send(:_get_span, result, node_id)
+      end
+
+      def _grammar_import(builder_json, grammar_json, prefix)
+        Parser.send(:_grammar_import, builder_json, grammar_json, prefix)
+      end
+
+      def _grammar_rule_mut(builder_json, rule_name)
+        Parser.send(:_grammar_rule_mut, builder_json, rule_name)
+      end
+
+      def _streaming_parser_new(grammar_json)
+        Parser.send(:_streaming_parser_new, grammar_json)
+      end
+
+      def _streaming_parser_add_chunk(parser, chunk)
+        Parser.send(:_streaming_parser_add_chunk, parser, chunk)
+      end
+
+      def _streaming_parser_parse_chunk(parser)
+        Parser.send(:_streaming_parser_parse_chunk, parser)
+      end
+
+      def _incremental_parser_new(grammar_json, initial_input)
+        Parser.send(:_incremental_parser_new, grammar_json, initial_input)
+      end
+
+      def _incremental_parser_apply_edit(parser, start, deleted, inserted)
+        Parser.send(:_incremental_parser_apply_edit, parser, start, deleted, inserted)
+      end
+
+      def _incremental_parser_reparse(parser, new_input)
+        Parser.send(:_incremental_parser_reparse, parser, new_input)
+      end
+
+      def _parse_batch_parallel(grammar_json, inputs, num_threads)
+        Parser.send(:_parse_batch_parallel, grammar_json, inputs, num_threads)
+      end
+
+      def _parse_with_limits(grammar_json, input, max_input_size, max_recursion_depth)
+        Parser.send(:_parse_with_limits, grammar_json, input, max_input_size, max_recursion_depth)
+      end
+
+      def _parse_with_trace(grammar_json, input)
+        Parser.send(:_parse_with_trace, grammar_json, input)
+      end
+
+      def _grammar_to_mermaid(grammar_json)
+        Parser.send(:_grammar_to_mermaid, grammar_json)
+      end
+
+      def _grammar_to_dot(grammar_json)
+        Parser.send(:_grammar_to_dot, grammar_json)
+      end
+    end
+  end
+end
+
+# Attempt to load native extension
+begin
+  require 'parsanol/parsanol_native'
+rescue LoadError
+  # Native extension not built yet
+end

data/lib/parsanol/optimizer.rb

@@ -0,0 +1,85 @@
+# frozen_string_literal: true
+
+require_relative 'ast_visitor'
+require_relative 'optimizers/quantifier_optimizer'
+require_relative 'optimizers/sequence_optimizer'
+require_relative 'optimizers/choice_optimizer'
+require_relative 'optimizers/lookahead_optimizer'
+require_relative 'optimizers/cut_inserter'
+
+# Grammar-level optimizations for Parslet parsers
+# These optimizations transform the parser AST to reduce runtime overhead
+# without changing semantics.
+#
+# Architecture:
+# - Uses Visitor pattern for clean separation of traversal and transformation
+# - Each optimizer is a separate class inheriting from ASTVisitor
+# - Optimizer module provides facade methods for easy access
+module Parsanol
+  module Optimizer
+    # Simplifies redundant quantifiers in a parslet tree
+    # Example: str('a').repeat(1, 1) => str('a')
+    #          str('a').repeat(0, 1).repeat(0, 1) => str('a').repeat(0, 1)
+    #
+    # @param parslet [Parsanol::Atoms::Base] parslet to simplify
+    # @return [Parsanol::Atoms::Base] simplified parslet
+    def self.simplify_quantifiers(parslet)
+      Optimizers::QuantifierOptimizer.new.visit(parslet)
+    end
+
+    # Simplifies sequences by flattening and merging adjacent strings
+    # Example: str('a') >> str('b') => str('ab')
+    #          (str('a') >> str('b')) >> str('c') => str('abc')
+    #
+    # @param parslet [Parsanol::Atoms::Base] parslet to simplify
+    # @return [Parsanol::Atoms::Base] simplified parslet
+    def self.simplify_sequences(parslet)
+      Optimizers::SequenceOptimizer.new.visit(parslet)
+    end
+
+    # Simplifies choice/alternative patterns
+    # Example: (A | B) | C => A | B | C
+    #          A | B | A => A | B
+    #
+    # @param parslet [Parsanol::Atoms::Base] parslet to simplify
+    # @return [Parsanol::Atoms::Base] simplified parslet
+    def self.simplify_choices(parslet)
+      Optimizers::ChoiceOptimizer.new.visit(parslet)
+    end
+
+    # Simplifies lookahead patterns
+    # Example: !(!x) => &x (double negation elimination)
+    #
+    # @param parslet [Parsanol::Atoms::Base] parslet to simplify
+    # @return [Parsanol::Atoms::Base] simplified parslet
+    def self.simplify_lookaheads(parslet)
+      Optimizers::LookaheadOptimizer.new.visit(parslet)
+    end
+
+    # Automatically insert cut operators where safe (AC-FIRST algorithm)
+    # Inserts cuts after deterministic prefixes when alternatives have disjoint FIRST sets
+    # This enables O(1) space complexity by allowing aggressive cache eviction
+    #
+    # Example: str('if') >> x | str('while') >> y
+    #       => str('if').cut >> x | str('while').cut >> y
+    #
+    # @param parslet [Parsanol::Atoms::Base] parslet to optimize
+    # @return [Parsanol::Atoms::Base] optimized parslet with cuts inserted
+    def self.insert_cuts(parslet)
+      Optimizers::CutInserter.new.optimize(parslet)
+    end
+
+    # Apply all optimizations in recommended order
+    # This is a convenience method that applies all optimizer passes
+    #
+    # @param parslet [Parsanol::Atoms::Base] parslet to optimize
+    # @return [Parsanol::Atoms::Base] fully optimized parslet
+    def self.optimize_all(parslet)
+      result = simplify_quantifiers(parslet)
+      result = simplify_sequences(result)
+      result = simplify_choices(result)
+      result = simplify_lookaheads(result)
+      insert_cuts(result)
+    end
+  end
+end

data/lib/parsanol/optimizers/choice_optimizer.rb

@@ -0,0 +1,78 @@
+# frozen_string_literal: true
+
+require_relative '../ast_visitor'
+
+module Parsanol
+  module Optimizers
+    # Optimizes alternative/choice patterns in the AST
+    # Follows visitor pattern for clean separation of concerns
+    #
+    # Transformations:
+    # - (A | B) | C => A | B | C (flatten nested alternatives)
+    # - A | B | A => A | B (remove duplicates)
+    # - Alternative(A) => A (unwrap single-element alternatives)
+    class ChoiceOptimizer < ASTVisitor
+      # Visit an alternative node and apply choice optimizations
+      # @param parslet [Parsanol::Atoms::Alternative] alternative to optimize
+      # @return [Parsanol::Atoms::Base] optimized parslet
+      def visit_alternative(parslet)
+        # First optimize children recursively
+        new_alternatives = parslet.alternatives.map { |p| visit(p) }
+
+        # Optimization 1: Flatten nested alternatives
+        flattened = flatten_alternatives(new_alternatives)
+
+        # Optimization 2: Remove duplicate alternatives
+        deduplicated = deduplicate_alternatives(flattened)
+
+        # Optimization 3: Unwrap single-element alternatives
+        return deduplicated[0] if deduplicated.size == 1
+
+        # Return optimized alternative if changed
+        if deduplicated == parslet.alternatives
+          parslet
+        else
+          Parsanol::Atoms::Alternative.new(*deduplicated)
+        end
+      end
+
+      private
+
+      # Flatten nested alternatives into a single level
+      # @param alternatives [Array<Parsanol::Atoms::Base>] array of alternatives
+      # @return [Array<Parsanol::Atoms::Base>] flattened array
+      def flatten_alternatives(alternatives)
+        result = []
+        alternatives.each do |alt|
+          if alt.is_a?(Parsanol::Atoms::Alternative)
+            result.concat(alt.alternatives)
+          else
+            result << alt
+          end
+        end
+        result
+      end
+
+      # Remove duplicate alternatives using structural equality
+      # @param alternatives [Array<Parsanol::Atoms::Base>] array of alternatives
+      # @return [Array<Parsanol::Atoms::Base>] deduplicated array
+      def deduplicate_alternatives(alternatives)
+        return alternatives if alternatives.size < 2
+
+        # Use to_s as proxy for structural equality
+        seen = {}
+        result = []
+
+        alternatives.each do |alt|
+          key = alt.to_s
+          unless seen[key]
+            seen[key] = true
+            result << alt
+          end
+        end
+
+        result
+      end
+    end
+  end
+end

data/lib/parsanol/optimizers/cut_inserter.rb

@@ -0,0 +1,179 @@
+# frozen_string_literal: true
+
+# Automatic Cut Insertion (AC-FIRST Algorithm)
+#
+# This optimizer implements the AC-FIRST algorithm from Mizushima et al. (2010)
+# to automatically insert cut operators when alternatives have disjoint FIRST sets.
+#
+# When all alternatives in a choice have non-overlapping FIRST sets, we can safely
+# insert a cut after the deterministic prefix, since backtracking will never be
+# needed.
+#
+# Example:
+#   str('if') >> condition >> then_clause |
+#   str('while') >> condition >> body |
+#   str('print') >> expression
+#
+# Becomes:
+#   str('if').cut >> condition >> then_clause |
+#   str('while').cut >> condition >> body |
+#   str('print').cut >> expression
+#
+# Reference: Mizushima et al. (2010) "Packrat Parsers Can Handle Practical
+# Grammars in Mostly Constant Space"
+#
+module Parsanol
+  module Optimizers
+    class CutInserter
+      # Optimize a parslet by inserting cuts where safe
+      # Recursively traverses the grammar AST
+      #
+      # @param parslet [Parsanol::Atoms::Base] The parslet to optimize
+      # @return [Parsanol::Atoms::Base] Optimized parslet with cuts inserted
+      def optimize(parslet)
+        case parslet
+        when Parsanol::Atoms::Alternative
+          optimize_alternative(parslet)
+        when Parsanol::Atoms::Sequence
+          optimize_sequence(parslet)
+        when Parsanol::Atoms::Repetition
+          optimize_repetition(parslet)
+        when Parsanol::Atoms::Named
+          optimize_named(parslet)
+        else
+          # Return atom unchanged (Str, Re, Lookahead, etc.)
+          parslet
+        end
+      end
+
+      private
+
+      # Optimize an Alternative atom by inserting cuts when all alternatives
+      # have disjoint FIRST sets
+      def optimize_alternative(alt)
+        alternatives = alt.alternatives
+        first_sets = alternatives.map(&:first_set)
+
+        # Only optimize if all FIRST sets are disjoint
+        unless Parsanol::FirstSet.all_disjoint?(first_sets)
+          # Not safe to insert cuts - return alternatives with recursive optimization
+          optimized = alternatives.map { |a| optimize(a) }
+          return Parsanol::Atoms::Alternative.new(*optimized)
+        end
+
+        # All FIRST sets are disjoint - safe to insert cuts!
+        # Insert cuts after deterministic prefixes
+        optimized = alternatives.map do |alternative|
+          insert_cut_if_safe(alternative)
+        end
+
+        Parsanol::Atoms::Alternative.new(*optimized)
+      end
+
+      # Optimize a Sequence atom by recursively optimizing its elements
+      def optimize_sequence(seq)
+        optimized_parslets = seq.parslets.map { |p| optimize(p) }
+        Parsanol::Atoms::Sequence.new(*optimized_parslets)
+      end
+
+      # Optimize a Repetition atom by recursively optimizing its parslet
+      def optimize_repetition(rep)
+        optimized_parslet = optimize(rep.parslet)
+        # Create new repetition with same min/max
+        # Note: We use default tag since it's not exposed as a reader
+        Parsanol::Atoms::Repetition.new(
+          optimized_parslet,
+          rep.min,
+          rep.max
+        )
+      end
+
+      # Optimize a Named atom by recursively optimizing its parslet
+      def optimize_named(named)
+        optimized_parslet = optimize(named.parslet)
+        optimized_parslet.as(named.name)
+      end
+
+      # Insert a cut after the deterministic prefix if safe
+      # For sequences: find longest prefix without EPSILON
+      # For other atoms: cut the whole thing if it doesn't include EPSILON
+      def insert_cut_if_safe(parslet)
+        # For sequences, find the longest safe prefix
+        if parslet.is_a?(Parsanol::Atoms::Sequence)
+          prefix_parslets = find_deterministic_prefix(parslet)
+          return build_cut_sequence(parslet, prefix_parslets) if prefix_parslets && !prefix_parslets.empty?
+        end
+
+        # For other atoms, cut the whole thing if safe
+        return parslet.cut if safe_to_cut?(parslet)
+
+        # Not safe to cut - recursively optimize and return
+        optimize(parslet)
+      end
+
+      # Find the longest deterministic prefix of a sequence
+      # A deterministic prefix doesn't include EPSILON in its FIRST set
+      #
+      # @param sequence [Parsanol::Atoms::Sequence] The sequence to analyze
+      # @return [Array<Parsanol::Atoms::Base>] Prefix parslets, or nil if none
+      def find_deterministic_prefix(sequence)
+        parslets = sequence.parslets
+        prefix_length = 0
+
+        # Find longest prefix where no element can match empty
+        parslets.each do |p|
+          break if p.first_set.include?(Parsanol::FirstSet::EPSILON)
+
+          prefix_length += 1
+        end
+
+        prefix_length.positive? ? parslets[0...prefix_length] : nil
+      end
+
+      # Check if it's safe to cut after this parslet
+      # Safe if the parslet doesn't have EPSILON in its FIRST set
+      # (i.e., it always consumes input)
+      def safe_to_cut?(parslet)
+        first = parslet.first_set
+        # Don't cut if EPSILON is in FIRST set (might not consume)
+        # Also don't cut if FIRST set contains only nil (unknown)
+        return false if first.include?(Parsanol::FirstSet::EPSILON)
+        return false if first.all?(&:nil?)
+
+        true
+      end
+
+      # Build a new sequence with a cut after the prefix
+      #
+      # @param sequence [Parsanol::Atoms::Sequence] Original sequence
+      # @param prefix_parslets [Array] Parslets forming the deterministic prefix
+      # @return [Parsanol::Atoms::Base] New sequence with cut inserted
+      def build_cut_sequence(sequence, prefix_parslets)
+        # Recursively optimize prefix parslets
+        optimized_prefix = prefix_parslets.map { |p| optimize(p) }
+
+        # Build prefix (single parslet or sequence)
+        prefix = if optimized_prefix.length == 1
+                   optimized_prefix.first
+                 else
+                   Parsanol::Atoms::Sequence.new(*optimized_prefix)
+                 end
+
+        # Get remaining parslets after prefix
+        remaining = sequence.parslets[prefix_parslets.length..]
+
+        # Recursively optimize remaining parslets
+        optimized_remaining = remaining.map { |p| optimize(p) }
+
+        # Build final sequence with cut
+        if optimized_remaining.empty?
+          # Prefix is the entire sequence
+          prefix.cut
+        else
+          # Prefix + cut + remaining
+          Parsanol::Atoms::Sequence.new(prefix.cut, *optimized_remaining)
+        end
+      end
+    end
+  end
+end

data/lib/parsanol/optimizers/lookahead_optimizer.rb

@@ -0,0 +1,50 @@
+# frozen_string_literal: true
+
+require_relative '../ast_visitor'
+
+module Parsanol
+  module Optimizers
+    # Optimizes lookahead patterns in the AST
+    # Follows visitor pattern for clean separation of concerns
+    #
+    # Transformations:
+    # - !(!x) => &x (double negation elimination)
+    # - &(&x) => &x (positive lookahead is idempotent)
+    # - !(&x) => !x (negative of positive)
+    # - &(!x) => !x (positive of negative)
+    class LookaheadOptimizer < ASTVisitor
+      # Visit a lookahead node and apply lookahead optimizations
+      # @param parslet [Parsanol::Atoms::Lookahead] lookahead to optimize
+      # @return [Parsanol::Atoms::Base] optimized parslet
+      def visit_lookahead(parslet)
+        # First optimize the child
+        inner = visit(parslet.bound_parslet)
+
+        # If inner is also a lookahead, simplify nested lookaheads
+        if inner.is_a?(Parsanol::Atoms::Lookahead)
+          outer_positive = parslet.positive
+          inner_positive = inner.positive
+
+          # !(!x) => &x (double negation)
+          return Parsanol::Atoms::Lookahead.new(inner.bound_parslet, true) if !outer_positive && !inner_positive
+
+          # &(&x) => &x (idempotent)
+          return inner if outer_positive && inner_positive
+
+          # !(&x) => !x (negative of positive)
+          return Parsanol::Atoms::Lookahead.new(inner.bound_parslet, false) if !outer_positive && inner_positive
+
+          # &(!x) => !x (positive of negative)
+          return inner if outer_positive && !inner_positive
+        end
+
+        # Return lookahead with optimized child
+        if inner.equal?(parslet.bound_parslet)
+          parslet
+        else
+          Parsanol::Atoms::Lookahead.new(inner, parslet.positive)
+        end
+      end
+    end
+  end
+end