parsanol 3.0.0

data/lib/parsanol/optimizers/quantifier_optimizer.rb

@@ -0,0 +1,62 @@
+# frozen_string_literal: true
+
+require_relative '../ast_visitor'
+
+module Parsanol
+  module Optimizers
+    # Optimizes repetition/quantifier patterns in the AST
+    # Follows visitor pattern for clean separation of concerns
+    #
+    # Transformations:
+    # - repeat(1,1) => unwrap (identity transformation)
+    # - repeat(0,1).repeat(0,1) => repeat(0,1) (idempotent)
+    # - repeat(n,n).repeat(m,m) => repeat(n*m,n*m) (multiply exact counts)
+    class QuantifierOptimizer < ASTVisitor
+      # Visit a repetition node and apply quantifier optimizations
+      # @param parslet [Parsanol::Atoms::Repetition] repetition to optimize
+      # @return [Parsanol::Atoms::Base] optimized parslet
+      def visit_repetition(parslet)
+        # First optimize the child
+        inner = visit(parslet.parslet)
+
+        # Optimization 1: repeat(1,1) is identity - unwrap it
+        if parslet.min == 1 && parslet.max == 1
+          return inner
+        end
+
+        # Optimization 2: Nested repetitions
+        if inner.is_a?(Parsanol::Atoms::Repetition)
+          # repeat(0,1).repeat(0,1) => repeat(0,1) (idempotent)
+          if parslet.min == 0 && parslet.max == 1 &&
+             inner.min == 0 && inner.max == 1
+            return inner
+          end
+
+          # repeat(n,n).repeat(m,m) => repeat(n*m,n*m) for exact counts
+          if parslet.min == parslet.max && inner.min == inner.max &&
+             parslet.max && inner.max
+            new_count = parslet.min * inner.min
+            return Parsanol::Atoms::Repetition.new(
+              inner.parslet,
+              new_count,
+              new_count,
+              parslet.instance_variable_get(:@tag)
+            )
+          end
+        end
+
+        # Return optimized repetition with simplified child
+        if inner.equal?(parslet.parslet)
+          parslet
+        else
+          Parsanol::Atoms::Repetition.new(
+            inner,
+            parslet.min,
+            parslet.max,
+            parslet.instance_variable_get(:@tag)
+          )
+        end
+      end
+    end
+  end
+end

data/lib/parsanol/optimizers/sequence_optimizer.rb

@@ -0,0 +1,97 @@
+# frozen_string_literal: true
+
+require_relative '../ast_visitor'
+
+module Parsanol
+  module Optimizers
+    # Optimizes sequence patterns in the AST
+    # Follows visitor pattern for clean separation of concerns
+    #
+    # Transformations:
+    # - str('a') >> str('b') => str('ab') (merge adjacent strings)
+    # - (A >> B) >> C => A >> B >> C (flatten nested sequences)
+    # - Sequence(A) => A (unwrap single-element sequences)
+    class SequenceOptimizer < ASTVisitor
+      # Visit a sequence node and apply sequence optimizations
+      # @param parslet [Parsanol::Atoms::Sequence] sequence to optimize
+      # @return [Parsanol::Atoms::Base] optimized parslet
+      def visit_sequence(parslet)
+        # First optimize children recursively
+        new_parslets = parslet.parslets.map { |p| visit(p) }
+
+        # Optimization 1: Flatten nested sequences
+        flattened = flatten_sequences(new_parslets)
+
+        # Optimization 2: Merge adjacent string literals
+        merged = merge_adjacent_strings(flattened)
+
+        # Optimization 3: Unwrap single-element sequences
+        return merged[0] if merged.size == 1
+
+        # Return optimized sequence if changed
+        if merged != parslet.parslets
+          Parsanol::Atoms::Sequence.new(*merged)
+        else
+          parslet
+        end
+      end
+
+      private
+
+      # Flatten nested sequences into a single level
+      # @param parslets [Array<Parsanol::Atoms::Base>] array of parslets
+      # @return [Array<Parsanol::Atoms::Base>] flattened array
+      def flatten_sequences(parslets)
+        result = []
+        parslets.each do |p|
+          if p.is_a?(Parsanol::Atoms::Sequence)
+            result.concat(p.parslets)
+          else
+            result << p
+          end
+        end
+        result
+      end
+
+      # Merge adjacent Str atoms into single Str atoms
+      # @param parslets [Array<Parsanol::Atoms::Base>] array of parslets
+      # @return [Array<Parsanol::Atoms::Base>] array with merged strings
+      def merge_adjacent_strings(parslets)
+        return parslets if parslets.size < 2
+
+        result = []
+        i = 0
+
+        while i < parslets.size
+          current = parslets[i]
+
+          if current.is_a?(Parsanol::Atoms::Str)
+            # Look ahead for consecutive Str atoms using Rope for O(1) append
+            rope = Parsanol::Rope.new.append(current.str)
+            j = i + 1
+
+            while j < parslets.size && parslets[j].is_a?(Parsanol::Atoms::Str)
+              rope.append(parslets[j].str)
+              j += 1
+            end
+
+            # Create merged Str if we found consecutive strings
+            # O(n) join happens once at the end instead of O(n²) repeated concatenation
+            if j > i + 1
+              result << Parsanol::Atoms::Str.new(rope.to_s)
+              i = j
+            else
+              result << current
+              i += 1
+            end
+          else
+            result << current
+            i += 1
+          end
+        end
+
+        result
+      end
+    end
+  end
+end

data/lib/parsanol/options/ruby_transform.rb

@@ -0,0 +1,109 @@
+# frozen_string_literal: true
+
+# Parsanol::RubyTransform - Ruby Transform Mode (Parslet-Compatible)
+#
+# This is the default parsing mode that provides maximum flexibility.
+# - Parsing can use Rust (if available) or pure Ruby
+# - Transformation happens in Ruby using Parslet-style Transform class
+#
+# Usage:
+#   class MyParser < Parsanol::Parser
+#     include Parsanol::RubyTransform
+#     rule(:number) { match('[0-9]').repeat(1).as(:int) }
+#     root(:number)
+#   end
+#
+#   parser = MyParser.new
+#   tree = parser.parse("42")  # Returns generic tree
+#   ast = transform.apply(tree)  # Transform in Ruby
+#
+# To use Rust backend for parsing:
+#   class MyParser < Parsanol::Parser
+#     include Parsanol::RubyTransform
+#     parse_backend :rust  # Will raise if native extension not available
+#     ...
+#   end
+
+module Parsanol
+  module RubyTransform
+    def self.included(base)
+      base.extend(ClassMethods)
+    end
+
+    module ClassMethods
+      # Get or set the parsing backend
+      # @param backend [Symbol] :ruby (default) or :rust
+      # @return [Symbol] Current backend setting
+      def parse_backend(backend = nil)
+        if backend
+          @parse_backend = backend
+        end
+        @parse_backend ||= :ruby
+      end
+
+      # Setter for parsing backend
+      # @param backend [Symbol] :ruby or :rust
+      def parse_backend=(backend)
+        @parse_backend = backend
+      end
+
+      # Configure parsing to use Rust backend
+      # Raises LoadError if native extension not available
+      def use_rust_backend!
+        unless Parsanol::Native.available?
+          raise LoadError,
+            "Rust backend requested but native extension not available. " \
+            "Run `rake compile` to build the extension."
+        end
+        @parse_backend = :rust
+      end
+
+      # Configure parsing to use pure Ruby (default)
+      def use_ruby_backend!
+        @parse_backend = :ruby
+      end
+    end
+
+    # Parse input and return generic tree
+    #
+    # @param input [String] The input string to parse
+    # @param options [Hash] Parse options
+    # @option options [Boolean] :consume_all (true) Consume entire input
+    # @return [Hash, Array, String, Parsanol::Slice] Parse tree
+    # @raise [Parsanol::ParseFailed] If parsing fails
+    def parse(input, options = {})
+      if self.class.parse_backend == :rust && Parsanol::Native.available?
+        parse_with_rust(input, options)
+      else
+        parse_with_ruby(input, options)
+      end
+    end
+
+    # Parse and apply transform in one step
+    #
+    # @param input [String] The input string to parse
+    # @param transform [Parsanol::Transform] Transform to apply
+    # @param options [Hash] Parse options
+    # @return [Object] Transformed result
+    def parse_with_transform(input, transform, options = {})
+      tree = parse(input, options)
+      transform.apply(tree)
+    end
+
+    private
+
+    # Parse using Rust native extension
+    def parse_with_rust(input, options = {})
+      consume_all = options.fetch(:consume_all, true)
+
+      # Use native parser with Parslet-compatible output
+      Parsanol::Native.parse_parslet_compatible(root, input)
+    end
+
+    # Parse using pure Ruby
+    def parse_with_ruby(input, options = {})
+      # Call the root parslet's parse method directly
+      root.parse(input, options)
+    end
+  end
+end

data/lib/parsanol/options/serialized.rb

@@ -0,0 +1,94 @@
+# frozen_string_literal: true
+
+require 'json'
+
+# Parsanol::Serialized - Serialized Transform Mode (JSON Output)
+#
+# This mode provides cross-language compatibility through JSON serialization.
+# - Parsing AND transformation happen in Rust for maximum performance
+# - Output is a JSON string that can be deserialized to any format
+# - REQUIRES native extension (will raise LoadError if not available)
+#
+# Usage:
+#   class MyParser < Parsanol::Parser
+#     include Parsanol::Serialized
+#     rule(:number) { match('[0-9]').repeat(1).as(:int) }
+#     root(:number)
+#   end
+#
+#   parser = MyParser.new
+#   json = parser.parse_to_json("42")  # Returns JSON string
+#   # => '{"int": "42"}'
+#
+#   # With a deserializer class
+#   result = parser.parse_to_struct("42", MyDeserializer)
+#
+# Performance: Faster than RubyTransform because transform happens in Rust.
+# Memory: Higher than ZeroCopy due to JSON serialization overhead.
+
+module Parsanol
+  module Serialized
+    def self.included(base)
+      base.extend(ClassMethods)
+    end
+
+    module ClassMethods
+      # Define output schema for transformation
+      # This is optional but helps with type checking
+      #
+      # @param schema [Hash] Schema definition
+      # @example
+      #   output_schema(
+      #     number: { type: :integer },
+      #     binop: { type: :object, properties: [:left, :op, :right] }
+      #   )
+      def output_schema(schema = nil)
+        @output_schema = schema if schema
+        @output_schema ||= {}
+      end
+    end
+
+    # Parse input and return JSON string
+    #
+    # @param input [String] The input string to parse
+    # @return [String] JSON string representing the parse result
+    # @raise [LoadError] If native extension not available
+    # @raise [Parsanol::ParseFailed] If parsing fails
+    def parse_to_json(input)
+      unless Parsanol::Native.available?
+        raise LoadError,
+          "Serialized mode requires native extension for JSON serialization. " \
+          "Run `rake compile` or use Parsanol::RubyTransform for Ruby-only parsing."
+      end
+
+      grammar_json = Parsanol::Native.serialize_grammar(root)
+      Parsanol::Native.parse_to_json(grammar_json, input)
+    end
+
+    # Parse input and deserialize to a Ruby object
+    #
+    # @param input [String] The input string to parse
+    # @param deserializer_class [Class] Class with .from_json method
+    # @return [Object] Deserialized object
+    # @raise [LoadError] If native extension not available
+    # @raise [Parsanol::ParseFailed] If parsing fails
+    def parse_to_struct(input, deserializer_class)
+      json = parse_to_json(input)
+      deserializer_class.from_json(json)
+    end
+
+    # Parse input and return Ruby Hash (parsed JSON)
+    #
+    # @param input [String] The input string to parse
+    # @return [Hash, Array] Ruby object from JSON
+    # @raise [LoadError] If native extension not available
+    # @raise [Parsanol::ParseFailed] If parsing fails
+    def parse(input, options = {})
+      json = parse_to_json(input)
+      JSON.parse(json)
+    end
+
+    # Alias for consistency with other modes
+    alias parse_to_hash parse
+  end
+end

data/lib/parsanol/options/zero_copy.rb

@@ -0,0 +1,130 @@
+# frozen_string_literal: true
+
+# Parsanol::ZeroCopy - Zero-Copy Transform Mode (Direct FFI Object Construction)
+#
+# This mode provides MAXIMUM PERFORMANCE through zero-copy FFI.
+# - Rust directly constructs Ruby objects via rb_class_new, rb_ivar_set
+# - No serialization overhead whatsoever
+# - REQUIRES native extension AND type mapping definitions
+#
+# Usage:
+#   # Define Ruby classes that mirror Rust types
+#   module Calculator
+#     class Number < Expr
+#       attr_reader :value
+#       def initialize(value); @value = value; end
+#       def eval = @value
+#     end
+#
+#     class BinOp < Expr
+#       attr_reader :left, :op, :right
+#       def eval; ...; end
+#     end
+#   end
+#
+#   class CalculatorParser < Parsanol::Parser
+#     include Parsanol::ZeroCopy
+#
+#     rule(:number) { ... }
+#     root(:expression)
+#
+#     # Type mapping (tells Rust which Ruby classes to construct)
+#     output_types(
+#       number: Calculator::Number,
+#       binop: Calculator::BinOp
+#     )
+#   end
+#
+#   parser = CalculatorParser.new
+#   expr = parser.parse("42+8")  # Returns Calculator::Number or BinOp DIRECTLY
+#   puts expr.eval  # No transform needed!
+#
+# Performance: FASTEST mode (18-44x faster than pure Ruby)
+# Memory: Lowest overhead (zero-copy, no serialization)
+
+module Parsanol
+  module ZeroCopy
+    def self.included(base)
+      base.extend(ClassMethods)
+    end
+
+    module ClassMethods
+      # Define output type mapping for zero-copy construction
+      #
+      # This tells the Rust parser which Ruby classes to instantiate
+      # for each named capture in the grammar.
+      #
+      # @param types [Hash] Mapping of rule names to Ruby classes
+      # @example
+      #   output_types(
+      #     number: Calculator::Number,
+      #     binop: Calculator::BinOp,
+      #     expr: Calculator::Expr
+      #   )
+      def output_types(types = nil)
+        @output_types = types if types
+        @output_types ||= {}
+      end
+
+      # Define a single output type mapping
+      #
+      # @param rule_name [Symbol, String] Name of the rule
+      # @param ruby_class [Class] Ruby class to instantiate
+      # @example
+      #   output_type :number, Calculator::Number
+      def output_type(rule_name, ruby_class)
+        output_types[rule_name.to_sym] = ruby_class
+      end
+
+      # Get output types as a hash suitable for FFI
+      #
+      # @return [Hash] String keys, class names as values
+      def output_types_for_ffi
+        output_types.transform_keys(&:to_s).transform_values do |klass|
+          klass.is_a?(Class) ? klass.name : klass.to_s
+        end
+      end
+    end
+
+    # Parse input and return direct Ruby objects (no serialization)
+    #
+    # @param input [String] The input string to parse
+    # @param options [Hash] Parse options (ignored for zero-copy)
+    # @return [Object] Direct Ruby object (type depends on grammar)
+    # @raise [LoadError] If native extension not available
+    # @raise [Parsanol::ParseFailed] If parsing fails
+    def parse(input, options = {})
+      unless Parsanol::Native.available?
+        raise LoadError,
+          "ZeroCopy mode requires native extension for direct FFI object construction. " \
+          "Run `rake compile` to build the extension, or use " \
+          "Parsanol::RubyTransform for Ruby-only parsing."
+      end
+
+      grammar_json = Parsanol::Native.serialize_grammar(root)
+      type_map = self.class.output_types_for_ffi
+
+      if type_map.empty?
+        raise ArgumentError,
+          "ZeroCopy mode requires output_types to be defined. " \
+          "Add `output_types(number: MyNumberClass)` to your parser class."
+      end
+
+      Parsanol::Native.parse_to_objects(grammar_json, input, type_map)
+    end
+
+    # Parse with explicit type map override
+    #
+    # @param input [String] The input string to parse
+    # @param type_map [Hash] Override type mapping for this parse
+    # @return [Object] Direct Ruby object
+    def parse_with_types(input, type_map)
+      unless Parsanol::Native.available?
+        raise LoadError, "ZeroCopy mode requires native extension."
+      end
+
+      grammar_json = Parsanol::Native.serialize_grammar(root)
+      Parsanol::Native.parse_to_objects(grammar_json, input, type_map)
+    end
+  end
+end

data/lib/parsanol/options.rb

@@ -0,0 +1,20 @@
+# frozen_string_literal: true
+
+# Parsanol Transform Mode Options
+#
+# This module provides three transformation modes for parsing:
+#
+# 1. RubyTransform - Parse in Rust/Ruby, Transform in Ruby (default, most flexible)
+# 2. Serialized - Parse + Transform in Rust, JSON output (requires native extension)
+# 3. ZeroCopy - Direct FFI object construction (requires native extension, fastest)
+#
+# Usage:
+#   class MyParser < Parsanol::Parser
+#     include Parsanol::RubyTransform  # or Serialized, or ZeroCopy
+#     rule(:number) { match('[0-9]').repeat(1).as(:int) }
+#     root(:number)
+#   end
+
+require 'parsanol/options/ruby_transform'
+require 'parsanol/options/serialized'
+require 'parsanol/options/zero_copy'

data/lib/parsanol/parallel.rb

@@ -0,0 +1,133 @@
+# frozen_string_literal: true
+
+module Parsanol
+  # Parallel parsing support for batch processing multiple inputs.
+  # Uses rayon for linear speedup on multi-core systems.
+  #
+  # @example Parse multiple files in parallel
+  #   grammar = MyParser.new.serialize_grammar
+  #   inputs = Dir.glob("*.json").map { |f| File.read(f) }
+  #
+  #   results = Parsanol::Parallel.parse_batch(grammar, inputs)
+  #   results.each_with_index do |result, i|
+  #     case result
+  #     when Hash then puts "File #{i} parsed: #{result}"
+  #     when Parsanol::ParseFailed then puts "File #{i} failed: #{result.message}"
+  #     end
+  #   end
+  #
+  module Parallel
+    # Configuration for parallel parsing.
+    #
+    # @example Configure with 8 threads
+    #   config = Parsanol::Parallel::Config.new
+    #     .with_num_threads(8)
+    #     .with_min_chunk_size(50)
+    #
+    #   results = Parsanol::Parallel.parse_batch(grammar, inputs, config: config)
+    #
+    class Config
+      # @return [Integer, nil] Number of threads (nil = auto-detect based on CPU cores)
+      attr_accessor :num_threads
+
+      # @return [Integer] Minimum inputs per thread (default: 10)
+      attr_accessor :min_chunk_size
+
+      def initialize
+        @num_threads = nil  # Auto-detect
+        @min_chunk_size = 10
+      end
+
+      # Set the number of threads to use.
+      #
+      # @param n [Integer] Number of threads
+      # @return [Config] self for chaining
+      def with_num_threads(n)
+        @num_threads = n
+        self
+      end
+
+      # Set the minimum chunk size per thread.
+      #
+      # @param size [Integer] Minimum inputs per thread
+      # @return [Config] self for chaining
+      def with_min_chunk_size(size)
+        @min_chunk_size = size
+        self
+      end
+    end
+
+    class << self
+      # Parse multiple inputs in parallel.
+      #
+      # When the native extension with parallel feature is available,
+      # this uses rayon for parallel execution. Otherwise, falls back
+      # to sequential parsing.
+      #
+      # @param grammar_json [String] JSON-serialized grammar
+      # @param inputs [Array<String>] Array of input strings to parse
+      # @param config [Config] Parallel configuration (optional)
+      # @return [Array<Object>] Array of parse results in same order as inputs
+      #
+      # @example Basic usage
+      #   results = Parsanol::Parallel.parse_batch(grammar, inputs)
+      #
+      # @example With configuration
+      #   config = Parsanol::Parallel::Config.new.with_num_threads(4)
+      #   results = Parsanol::Parallel.parse_batch(grammar, inputs, config: config)
+      #
+      def parse_batch(grammar_json, inputs, config: Config.new)
+        unless Parsanol::Native.available?
+          raise LoadError,
+            "Parallel parsing requires native extension. " \
+            "Run `rake compile` to build the extension."
+        end
+
+        # Try to use native parallel parsing
+        if respond_to?(:_parse_batch_parallel, true)
+          Parsanol::Native.parse_batch_parallel(
+            grammar_json,
+            inputs,
+            num_threads: config.num_threads
+          )
+        else
+          # Fallback to sequential if parallel not available
+          inputs.map { |input| Parsanol::Native.parse(grammar_json, input) }
+        end
+      end
+
+      # Parse multiple inputs in parallel with transformation.
+      #
+      # @param grammar_json [String] JSON-serialized grammar
+      # @param inputs [Array<String>] Array of input strings to parse
+      # @param transform [Parsanol::Transform] Transform to apply to each result
+      # @param config [Config] Parallel configuration (optional)
+      # @return [Array<Object>] Array of transformed results
+      #
+      def parse_batch_with_transform(grammar_json, inputs, transform, config: Config.new)
+        results = parse_batch(grammar_json, inputs, config: config)
+        results.map { |result| transform.apply(result) }
+      end
+
+      # Get the number of available CPU cores for parallel processing.
+      #
+      # @return [Integer] Number of available cores
+      def available_cores
+        require 'etc'
+        Etc.nprocessors
+      rescue StandardError
+        1
+      end
+
+      # Estimate optimal number of threads for a given input size.
+      #
+      # @param input_count [Integer] Number of inputs to process
+      # @return [Integer] Recommended number of threads
+      def optimal_threads(input_count)
+        cores = available_cores
+        # Don't use more threads than inputs
+        [cores, input_count].min
+      end
+    end
+  end
+end