parsanol 3.0.0

data/spec/parsanol/builder_callbacks_spec.rb

@@ -0,0 +1,377 @@
+# frozen_string_literal: true
+
+require 'spec_helper'
+
+RSpec.describe Parsanol::BuilderCallbacks do
+  let(:builder_class) do
+    Class.new do
+      include Parsanol::BuilderCallbacks
+
+      attr_reader :events
+
+      def initialize
+        @events = []
+      end
+
+      def on_start(input)
+        @events << [:start, input]
+      end
+
+      def on_success
+        @events << [:success]
+      end
+
+      def on_error(message)
+        @events << [:error, message]
+      end
+
+      def on_string(value, offset, length)
+        @events << [:string, value, offset, length]
+      end
+
+      def on_int(value)
+        @events << [:int, value]
+      end
+
+      def on_float(value)
+        @events << [:float, value]
+      end
+
+      def on_bool(value)
+        @events << [:bool, value]
+      end
+
+      def on_nil
+        @events << [:nil]
+      end
+
+      def on_hash_start(size = nil)
+        @events << [:hash_start, size]
+      end
+
+      def on_hash_end(size)
+        @events << [:hash_end, size]
+      end
+
+      def on_hash_key(key)
+        @events << [:hash_key, key]
+      end
+
+      def on_hash_value(key)
+        @events << [:hash_value, key]
+      end
+
+      def on_array_start(size = nil)
+        @events << [:array_start, size]
+      end
+
+      def on_array_element(index)
+        @events << [:array_element, index]
+      end
+
+      def on_array_end(size)
+        @events << [:array_end, size]
+      end
+
+      def on_named_start(name)
+        @events << [:named_start, name]
+      end
+
+      def on_named_end(name)
+        @events << [:named_end, name]
+      end
+
+      def finish
+        @events
+      end
+    end
+  end
+
+  let(:builder) { builder_class.new }
+
+  describe 'included methods' do
+    it 'provides default no-op implementations' do
+      noop_builder = Class.new { include Parsanol::BuilderCallbacks }.new
+
+      expect { noop_builder.on_string('test', 0, 4) }.not_to raise_error
+      expect { noop_builder.on_int(42) }.not_to raise_error
+      expect { noop_builder.on_float(3.14) }.not_to raise_error
+      expect { noop_builder.on_bool(true) }.not_to raise_error
+      expect { noop_builder.on_nil }.not_to raise_error
+      expect { noop_builder.on_hash_start(nil) }.not_to raise_error
+      expect { noop_builder.on_hash_end(0) }.not_to raise_error
+      expect { noop_builder.on_hash_key('name') }.not_to raise_error
+      expect { noop_builder.on_array_start(nil) }.not_to raise_error
+      expect { noop_builder.on_array_element(0) }.not_to raise_error
+      expect { noop_builder.on_array_end(0) }.not_to raise_error
+      expect { noop_builder.on_named_start('test') }.not_to raise_error
+      expect { noop_builder.on_named_end('test') }.not_to raise_error
+      expect { noop_builder.on_start('input') }.not_to raise_error
+      expect { noop_builder.on_success }.not_to raise_error
+      expect { noop_builder.on_error('msg') }.not_to raise_error
+      expect { noop_builder.finish }.not_to raise_error
+    end
+  end
+
+  describe 'callback invocations' do
+    it 'records string callbacks' do
+      builder.on_string('hello', 0, 5)
+      expect(builder.events).to eq([[:string, 'hello', 0, 5]])
+    end
+
+    it 'records int callbacks' do
+      builder.on_int(42)
+      expect(builder.events).to eq([[:int, 42]])
+    end
+
+    it 'records float callbacks' do
+      builder.on_float(3.14)
+      expect(builder.events).to eq([[:float, 3.14]])
+    end
+
+    it 'records bool callbacks' do
+      builder.on_bool(true)
+      expect(builder.events).to eq([[:bool, true]])
+    end
+
+    it 'records nil callbacks' do
+      builder.on_nil
+      expect(builder.events).to eq([[:nil]])
+    end
+
+    it 'records hash callbacks' do
+      builder.on_hash_start(nil)
+      builder.on_hash_key('name')
+      builder.on_string('John', 0, 4)
+      builder.on_hash_end(1)
+
+      expect(builder.events).to eq([
+        [:hash_start, nil],
+        [:hash_key, 'name'],
+        [:string, 'John', 0, 4],
+        [:hash_end, 1]
+      ])
+    end
+
+    it 'records array callbacks' do
+      builder.on_array_start(nil)
+      builder.on_array_element(0)
+      builder.on_int(1)
+      builder.on_array_element(1)
+      builder.on_int(2)
+      builder.on_array_element(2)
+      builder.on_int(3)
+      builder.on_array_end(3)
+
+      expect(builder.events).to eq([
+        [:array_start, nil],
+        [:array_element, 0],
+        [:int, 1],
+        [:array_element, 1],
+        [:int, 2],
+        [:array_element, 2],
+        [:int, 3],
+        [:array_end, 3]
+      ])
+    end
+
+    it 'returns events from finish' do
+      builder.on_string('test', 0, 4)
+      expect(builder.finish).to eq([[:string, 'test', 0, 4]])
+    end
+  end
+end
+
+RSpec.describe Parsanol::Builders::DebugBuilder do
+  let(:builder) { Parsanol::Builders::DebugBuilder.new }
+
+  describe '#events' do
+    it 'collects all event types' do
+      builder.on_string('hello', 0, 5)
+      builder.on_int(42)
+      builder.on_float(3.14)
+      builder.on_bool(true)
+      builder.on_nil
+      builder.on_hash_start(nil)
+      builder.on_hash_key('name')
+      builder.on_hash_end(1)
+      builder.on_array_start(nil)
+      builder.on_array_end(0)
+
+      result = builder.finish
+
+      expect(result).to include('string: "hello" @ 0(5)')
+      expect(result).to include('int: 42')
+      expect(result).to include('float: 3.14')
+      expect(result).to include('bool: true')
+      expect(result).to include('nil')
+      expect(result).to include('hash_start(nil)')
+      expect(result).to include('hash_end(1)')
+      expect(result).to include('hash_key: "name"')
+      expect(result).to include('array_start(nil)')
+      expect(result).to include('array_end(0)')
+    end
+  end
+end
+
+RSpec.describe Parsanol::Builders::StringCollector do
+  let(:builder) { Parsanol::Builders::StringCollector.new }
+
+  describe '#strings' do
+    it 'collects only string values' do
+      builder.on_string('hello', 0, 5)
+      builder.on_int(42)
+      builder.on_string('world', 6, 5)
+
+      expect(builder.strings).to eq(['hello', 'world'])
+    end
+
+    it 'returns strings from finish' do
+      builder.on_string('test', 0, 4)
+      expect(builder.finish).to eq(['test'])
+    end
+  end
+end
+
+RSpec.describe Parsanol::Builders::NodeCounter do
+  let(:builder) { Parsanol::Builders::NodeCounter.new }
+
+  describe '#counts' do
+    it 'counts nodes by type' do
+      builder.on_string('a', 0, 1)
+      builder.on_string('b', 1, 1)
+      builder.on_int(1)
+      builder.on_int(2)
+      builder.on_int(3)
+      builder.on_float(1.0)
+      builder.on_bool(true)
+      builder.on_nil
+      builder.on_hash_start(nil)
+      builder.on_array_start(nil)
+      builder.on_array_start(nil)
+
+      expect(builder.counts[:string]).to eq(2)
+      expect(builder.counts[:int]).to eq(3)
+      expect(builder.counts[:float]).to eq(1)
+      expect(builder.counts[:bool]).to eq(1)
+      expect(builder.counts[:nil]).to eq(1)
+      expect(builder.counts[:hash]).to eq(1)
+      expect(builder.counts[:array]).to eq(2)
+    end
+
+    it 'returns counts from finish' do
+      builder.on_string('test', 0, 4)
+      builder.on_int(42)
+
+      result = builder.finish
+      expect(result[:string]).to eq(1)
+      expect(result[:int]).to eq(1)
+    end
+  end
+end
+
+RSpec.describe Parsanol::Parallel do
+  describe Parsanol::Parallel::Config do
+    let(:config) { Parsanol::Parallel::Config.new }
+
+    it 'has default values' do
+      expect(config.num_threads).to be_nil
+      expect(config.min_chunk_size).to eq(10)
+    end
+
+    it 'supports chaining' do
+      result = config.with_num_threads(8).with_min_chunk_size(50)
+
+      expect(result.num_threads).to eq(8)
+      expect(result.min_chunk_size).to eq(50)
+      expect(result).to eq(config)
+    end
+  end
+
+  describe '.available_cores' do
+    it 'returns an integer' do
+      result = Parsanol::Parallel.available_cores
+      expect(result).to be_a(Integer)
+      expect(result).to be >= 1
+    end
+  end
+
+  describe '.optimal_threads' do
+    it 'returns minimum of cores and input count' do
+      cores = Parsanol::Parallel.available_cores
+
+      # Small input count
+      expect(Parsanol::Parallel.optimal_threads(2)).to eq([2, cores].min)
+
+      # Large input count
+      expect(Parsanol::Parallel.optimal_threads(1000)).to eq(cores)
+    end
+  end
+
+  describe '.parse_batch' do
+    context 'when native extension is not available' do
+      before do
+        allow(Parsanol::Native).to receive(:available?).and_return(false)
+      end
+
+      it 'raises LoadError' do
+        expect {
+          Parsanol::Parallel.parse_batch('{}', ['input'])
+        }.to raise_error(LoadError, /requires native extension/)
+      end
+    end
+  end
+end
+
+RSpec.describe "Native parse_with_builder integration", :native do
+  # Define a simple parser for testing - captures only words, not spaces
+  let(:simple_parser) do
+    Class.new(Parsanol::Parser) do
+      rule(:word) { match('[a-z]+').as(:word) }
+      rule(:words) { word >> (match('\s') >> word).repeat }
+      root(:words)
+    end
+  end
+
+  describe "Parsanol::Native.parse_with_builder" do
+    context "when native extension is available" do
+      before do
+        skip "Native extension not available" unless Parsanol::Native.available?
+      end
+
+      it "parses input using a custom builder callback" do
+        grammar = Parsanol::Native.serialize_grammar(simple_parser.new.root)
+        input = "hello world"
+
+        builder = Parsanol::Builders::StringCollector.new
+        result = Parsanol::Native.parse_with_builder(grammar, input, builder)
+
+        # The parser captures all matches including spaces
+        expect(result).to include("hello", "world")
+      end
+
+      it "works with DebugBuilder" do
+        grammar = Parsanol::Native.serialize_grammar(simple_parser.new.root)
+        input = "a b"
+
+        builder = Parsanol::Builders::DebugBuilder.new
+        result = Parsanol::Native.parse_with_builder(grammar, input, builder)
+
+        expect(result).to be_a(String)
+        expect(result).to include("string:")
+      end
+
+      it "works with NodeCounter" do
+        grammar = Parsanol::Native.serialize_grammar(simple_parser.new.root)
+        input = "one two three"
+
+        builder = Parsanol::Builders::NodeCounter.new
+        result = Parsanol::Native.parse_with_builder(grammar, input, builder)
+
+        expect(result).to be_a(Hash)
+        # At minimum we have 3 words, plus spaces (5 total matches)
+        expect(result[:string]).to be >= 3
+      end
+    end
+  end
+end

data/spec/parsanol/choice_optimizer_spec.rb

@@ -0,0 +1,231 @@
+# frozen_string_literal: true
+
+require 'spec_helper'
+
+describe Parsanol::Optimizer, '.simplify_choices' do
+  let(:str_a) { Parsanol::Atoms::Str.new('a') }
+  let(:str_b) { Parsanol::Atoms::Str.new('b') }
+  let(:str_c) { Parsanol::Atoms::Str.new('c') }
+
+  describe 'unwrapping single alternatives' do
+    it 'unwraps Alternative with single element' do
+      alt = Parsanol::Atoms::Alternative.new(str_a)
+      result = Parsanol::Optimizer.simplify_choices(alt)
+
+      expect(result).to eq(str_a)
+    end
+
+    it 'leaves Alternative with multiple elements unchanged' do
+      alt = Parsanol::Atoms::Alternative.new(str_a, str_b)
+      result = Parsanol::Optimizer.simplify_choices(alt)
+
+      expect(result).to be_a(Parsanol::Atoms::Alternative)
+      expect(result.alternatives.size).to eq(2)
+    end
+  end
+
+  describe 'flattening nested alternatives' do
+    it 'flattens Alternative(Alternative(a, b), c)' do
+      inner = Parsanol::Atoms::Alternative.new(str_a, str_b)
+      outer = Parsanol::Atoms::Alternative.new(inner, str_c)
+
+      result = Parsanol::Optimizer.simplify_choices(outer)
+
+      expect(result).to be_a(Parsanol::Atoms::Alternative)
+      expect(result.alternatives.size).to eq(3)
+      expect(result.alternatives).to eq([str_a, str_b, str_c])
+    end
+
+    it 'flattens Alternative(a, Alternative(b, c))' do
+      inner = Parsanol::Atoms::Alternative.new(str_b, str_c)
+      outer = Parsanol::Atoms::Alternative.new(str_a, inner)
+
+      result = Parsanol::Optimizer.simplify_choices(outer)
+
+      expect(result).to be_a(Parsanol::Atoms::Alternative)
+      expect(result.alternatives.size).to eq(3)
+      expect(result.alternatives).to eq([str_a, str_b, str_c])
+    end
+
+    it 'flattens Alternative(Alternative(a, b), Alternative(c, d))' do
+      inner1 = Parsanol::Atoms::Alternative.new(str_a, str_b)
+      inner2 = Parsanol::Atoms::Alternative.new(str_c, Parsanol::Atoms::Str.new('d'))
+      outer = Parsanol::Atoms::Alternative.new(inner1, inner2)
+
+      result = Parsanol::Optimizer.simplify_choices(outer)
+
+      expect(result).to be_a(Parsanol::Atoms::Alternative)
+      expect(result.alternatives.size).to eq(4)
+    end
+
+    it 'flattens deeply nested alternatives' do
+      # Alternative(Alternative(Alternative(a, b), c), d)
+      inner1 = Parsanol::Atoms::Alternative.new(str_a, str_b)
+      inner2 = Parsanol::Atoms::Alternative.new(inner1, str_c)
+      outer = Parsanol::Atoms::Alternative.new(inner2, Parsanol::Atoms::Str.new('d'))
+
+      result = Parsanol::Optimizer.simplify_choices(outer)
+
+      expect(result).to be_a(Parsanol::Atoms::Alternative)
+      expect(result.alternatives.size).to eq(4)
+    end
+  end
+
+  describe 'deduplicating alternatives' do
+    it 'removes duplicate alternatives' do
+      alt = Parsanol::Atoms::Alternative.new(str_a, str_a, str_b)
+      result = Parsanol::Optimizer.simplify_choices(alt)
+
+      expect(result).to be_a(Parsanol::Atoms::Alternative)
+      expect(result.alternatives.size).to eq(2)
+      expect(result.alternatives).to eq([str_a, str_b])
+    end
+
+    it 'removes all duplicates keeping first occurrence' do
+      alt = Parsanol::Atoms::Alternative.new(str_a, str_b, str_a, str_c, str_b, str_a)
+      result = Parsanol::Optimizer.simplify_choices(alt)
+
+      expect(result).to be_a(Parsanol::Atoms::Alternative)
+      expect(result.alternatives.size).to eq(3)
+      expect(result.alternatives).to eq([str_a, str_b, str_c])
+    end
+
+    it 'unwraps when deduplication leaves single alternative' do
+      alt = Parsanol::Atoms::Alternative.new(str_a, str_a, str_a)
+      result = Parsanol::Optimizer.simplify_choices(alt)
+
+      expect(result).to eq(str_a)
+    end
+  end
+
+  describe 'combined optimizations' do
+    it 'flattens and deduplicates in one pass' do
+      # Alternative(Alternative(a, b), a, b)
+      inner = Parsanol::Atoms::Alternative.new(str_a, str_b)
+      outer = Parsanol::Atoms::Alternative.new(inner, str_a, str_b)
+
+      result = Parsanol::Optimizer.simplify_choices(outer)
+
+      expect(result).to be_a(Parsanol::Atoms::Alternative)
+      expect(result.alternatives.size).to eq(2)
+      expect(result.alternatives).to eq([str_a, str_b])
+    end
+
+    it 'flattens, deduplicates, and unwraps' do
+      # Alternative(Alternative(a, a), a) => a
+      inner = Parsanol::Atoms::Alternative.new(str_a, str_a)
+      outer = Parsanol::Atoms::Alternative.new(inner, str_a)
+
+      result = Parsanol::Optimizer.simplify_choices(outer)
+
+      expect(result).to eq(str_a)
+    end
+  end
+
+  describe 'recursive optimization' do
+    it 'optimizes alternatives nested in sequences' do
+      # Sequence(Alternative(a, a), b) => Sequence(a, b)
+      alt = Parsanol::Atoms::Alternative.new(str_a, str_a)
+      seq = Parsanol::Atoms::Sequence.new(alt, str_b)
+
+      result = Parsanol::Optimizer.simplify_choices(seq)
+
+      expect(result).to be_a(Parsanol::Atoms::Sequence)
+      expect(result.parslets[0]).to eq(str_a)
+      expect(result.parslets[1]).to eq(str_b)
+    end
+
+    it 'optimizes alternatives nested in repetitions' do
+      # Repetition(Alternative(a, a)) => Repetition(a)
+      alt = Parsanol::Atoms::Alternative.new(str_a, str_a)
+      rep = Parsanol::Atoms::Repetition.new(alt, 0, nil, nil)
+
+      result = Parsanol::Optimizer.simplify_choices(rep)
+
+      expect(result).to be_a(Parsanol::Atoms::Repetition)
+      expect(result.parslet).to eq(str_a)
+    end
+
+    it 'optimizes alternatives nested in lookaheads' do
+      # Lookahead(Alternative(a, a)) => Lookahead(a)
+      alt = Parsanol::Atoms::Alternative.new(str_a, str_a)
+      la = Parsanol::Atoms::Lookahead.new(alt, true)
+
+      result = Parsanol::Optimizer.simplify_choices(la)
+
+      expect(result).to be_a(Parsanol::Atoms::Lookahead)
+      expect(result.bound_parslet).to eq(str_a)
+    end
+
+    it 'optimizes alternatives nested in named atoms' do
+      # Named(Alternative(a, a)) => Named(a)
+      alt = Parsanol::Atoms::Alternative.new(str_a, str_a)
+      named = Parsanol::Atoms::Named.new(alt, :test)
+
+      result = Parsanol::Optimizer.simplify_choices(named)
+
+      expect(result).to be_a(Parsanol::Atoms::Named)
+      expect(result.parslet).to eq(str_a)
+    end
+  end
+
+  describe 'preserving semantics' do
+    it 'does not modify leaf atoms' do
+      result = Parsanol::Optimizer.simplify_choices(str_a)
+      expect(result).to eq(str_a)
+    end
+
+    it 'preserves alternative order' do
+      alt = Parsanol::Atoms::Alternative.new(str_a, str_b, str_c)
+      result = Parsanol::Optimizer.simplify_choices(alt)
+
+      expect(result.alternatives).to eq([str_a, str_b, str_c])
+    end
+
+    it 'works with empty alternatives array (edge case)' do
+      # This shouldn't happen in practice, but test robustness
+      alt = Parsanol::Atoms::Alternative.new
+      result = Parsanol::Optimizer.simplify_choices(alt)
+
+      # Should return something reasonable (empty alternative or nil)
+      # The exact behavior depends on how Alternative handles empty case
+    end
+  end
+
+  describe 'functional test with parser' do
+    it 'correctly parses after optimization' do
+      # Create an alternative with duplicate choices
+      alt = str_a | str_b | str_a | str_c | str_b
+
+      # Optimize it
+      optimized = Parsanol::Optimizer.simplify_choices(alt)
+
+      # Should have 3 unique alternatives instead of 5
+      expect(optimized).to be_a(Parsanol::Atoms::Alternative)
+      expect(optimized.alternatives.size).to eq(3)
+
+      # Should still parse correctly
+      expect(optimized.parse('a')).to eq('a')
+      expect(optimized.parse('b')).to eq('b')
+      expect(optimized.parse('c')).to eq('c')
+    end
+
+    it 'correctly parses nested alternatives after optimization' do
+      # Create nested alternatives
+      alt = (str_a | str_b) | (str_c | Parsanol::Atoms::Str.new('d'))
+
+      # Optimize it
+      optimized = Parsanol::Optimizer.simplify_choices(alt)
+
+      # Should be flattened to 4 alternatives
+      expect(optimized).to be_a(Parsanol::Atoms::Alternative)
+      expect(optimized.alternatives.size).to eq(4)
+
+      # Should still parse all options
+      expect(optimized.parse('a')).to eq('a')
+      expect(optimized.parse('b')).to eq('b')
+      expect(optimized.parse('c')).to eq('c')
+      expect(optimized.parse('d')).to eq('d')
+    end
+  end
+end

data/spec/parsanol/convenience_spec.rb

@@ -0,0 +1,54 @@
+require 'spec_helper'
+
+describe 'parslet/convenience' do
+  require 'parsanol/convenience'
+  include Parsanol
+
+  class FooParser < Parsanol::Parser
+    rule(:foo) { str('foo') }
+    root(:foo)
+  end
+
+  describe 'parse_with_debug' do
+    let(:parser) { FooParser.new }
+
+    context 'internal' do
+      before do
+        # Suppress output.
+        #
+        allow(parser).to receive(:puts)
+      end
+
+      it 'exists' do
+        -> { parser.parse_with_debug('anything') }.should_not raise_error
+      end
+
+      it 'catches ParseFailed exceptions' do
+        -> { parser.parse_with_debug('bar') }.should_not raise_error
+      end
+
+      it 'parses correct input like #parse' do
+        -> { parser.parse_with_debug('foo') }.should_not raise_error
+      end
+    end
+
+    context 'output' do
+      it 'putses once for tree output' do
+        expect(parser).to receive(:puts).once
+
+        parser.parse_with_debug('incorrect')
+      end
+
+      it 'putses once for the error on unconsumed input' do
+        expect(parser).to receive(:puts).once
+
+        parser.parse_with_debug('foobar')
+      end
+    end
+
+    it 'works for all parslets' do
+      str('foo').parse_with_debug('foo')
+      match['bar'].parse_with_debug('a')
+    end
+  end
+end