parsanol 3.0.0

data/spec/parsanol/options/serialized_spec.rb

@@ -0,0 +1,69 @@
+# frozen_string_literal: true
+
+require 'spec_helper'
+require 'json'
+
+describe Parsanol::Serialized do
+  let(:parser_class) do
+    Class.new(Parsanol::Parser) do
+      include Parsanol::Serialized
+
+      rule(:number) { match('[0-9]').repeat(1).as(:int) }
+      rule(:space) { match('\s').repeat }
+      rule(:add_op) { match('[+-]').as(:op) >> space }
+      rule(:expression) { (number.as(:left) >> add_op.as(:op) >> expression.as(:right)).as(:binop) | number }
+      root(:expression)
+    end
+  end
+
+  let(:parser) { parser_class.new }
+
+  describe '#parse_to_json' do
+    context 'when native extension is not available' do
+      before do
+        allow(Parsanol::Native).to receive(:available?).and_return(false)
+      end
+
+      it 'raises LoadError' do
+        expect { parser.parse_to_json('42') }.to raise_error(LoadError, /Serialized mode requires native extension/)
+      end
+    end
+  end
+
+  describe '#parse_to_struct' do
+    context 'when native extension is not available' do
+      before do
+        allow(Parsanol::Native).to receive(:available?).and_return(false)
+      end
+
+      it 'raises LoadError' do
+        deserializer = Class.new do
+          def self.from_json(json); JSON.parse(json); end
+        end
+        expect { parser.parse_to_struct('42', deserializer) }.to raise_error(LoadError)
+      end
+    end
+  end
+
+  describe '#parse' do
+    context 'when native extension is not available' do
+      before do
+        allow(Parsanol::Native).to receive(:available?).and_return(false)
+      end
+
+      it 'raises LoadError' do
+        expect { parser.parse('42') }.to raise_error(LoadError, /Serialized mode requires native extension/)
+      end
+    end
+  end
+
+  describe '.output_schema' do
+    it 'allows defining output schema' do
+      parser_class.output_schema(
+        number: { type: :integer },
+        binop: { type: :object, properties: [:left, :op, :right] }
+      )
+      expect(parser_class.output_schema).to be_a(Hash)
+    end
+  end
+end

data/spec/parsanol/options/zero_copy_spec.rb

@@ -0,0 +1,230 @@
+# frozen_string_literal: true
+
+require 'spec_helper'
+
+describe Parsanol::ZeroCopy do
+  # Define test AST classes
+  before(:all) do
+    module TestCalculator
+      class Expr
+        def eval
+          raise NotImplementedError
+        end
+      end
+
+      class Number < Expr
+        attr_reader :value
+
+        def initialize(value)
+          @value = value
+        end
+
+        def eval = @value
+      end
+
+      class BinOp < Expr
+        attr_reader :left, :op, :right
+
+        def initialize(left:, op:, right:)
+          @left = left
+          @op = op
+          @right = right
+        end
+
+        def eval
+          case @op
+          when '+' then @left.eval + @right.eval
+          when '-' then @left.eval - @right.eval
+          end
+        end
+      end
+    end
+  end
+
+  let(:parser_class) do
+    Class.new(Parsanol::Parser) do
+      include Parsanol::ZeroCopy
+
+      rule(:number) { match('[0-9]').repeat(1).as(:int) }
+      rule(:space) { match('\s').repeat }
+      rule(:add_op) { match('[+-]').as(:op) >> space }
+      rule(:expression) { (number.as(:left) >> add_op.as(:op) >> expression.as(:right)).as(:binop) | number }
+      root(:expression)
+    end
+  end
+
+  let(:parser) { parser_class.new }
+
+  describe '.output_types' do
+    it 'allows defining output types' do
+      parser_class.output_types(
+        number: TestCalculator::Number,
+        binop: TestCalculator::BinOp
+      )
+      expect(parser_class.output_types[:number]).to eq(TestCalculator::Number)
+      expect(parser_class.output_types[:binop]).to eq(TestCalculator::BinOp)
+    end
+
+    it 'returns empty hash by default' do
+      fresh_parser_class = Class.new(Parsanol::Parser) do
+        include Parsanol::ZeroCopy
+      end
+      expect(fresh_parser_class.output_types).to eq({})
+    end
+  end
+
+  describe '.output_type' do
+    it 'allows defining a single output type' do
+      parser_class.output_type(:number, TestCalculator::Number)
+      expect(parser_class.output_types[:number]).to eq(TestCalculator::Number)
+    end
+  end
+
+  describe '.output_types_for_ffi' do
+    it 'converts types to FFI-compatible format' do
+      parser_class.output_types(
+        number: TestCalculator::Number,
+        binop: TestCalculator::BinOp
+      )
+      ffi_types = parser_class.output_types_for_ffi
+      expect(ffi_types['number']).to eq('TestCalculator::Number')
+      expect(ffi_types['binop']).to eq('TestCalculator::BinOp')
+    end
+  end
+
+  describe '#parse' do
+    context 'when native extension is not available' do
+      before do
+        allow(Parsanol::Native).to receive(:available?).and_return(false)
+      end
+
+      it 'raises LoadError' do
+        expect { parser.parse('42') }.to raise_error(LoadError, /ZeroCopy mode requires native extension/)
+      end
+    end
+
+    context 'when output_types is not defined' do
+      before do
+        allow(Parsanol::Native).to receive(:available?).and_return(true)
+        allow(Parsanol::Native).to receive(:serialize_grammar).and_return('{}')
+      end
+
+      it 'raises ArgumentError' do
+        fresh_parser_class = Class.new(Parsanol::Parser) do
+          include Parsanol::ZeroCopy
+          rule(:test) { str('a') }
+          root(:test)
+        end
+        fresh_parser = fresh_parser_class.new
+        expect { fresh_parser.parse('a') }.to raise_error(ArgumentError, /ZeroCopy mode requires output_types/)
+      end
+    end
+  end
+
+  describe '#parse_with_types' do
+    context 'when native extension is not available' do
+      before do
+        allow(Parsanol::Native).to receive(:available?).and_return(false)
+      end
+
+      it 'raises LoadError' do
+        expect { parser.parse_with_types('42', {}) }.to raise_error(LoadError, /ZeroCopy mode requires native extension/)
+      end
+    end
+  end
+
+  # Tests for parse_to_ruby_objects FFI function
+  # These tests verify that the new FFI function directly constructs
+  # Parsanol::Slice objects without intermediate Hash markers.
+  describe 'parse_to_ruby_objects FFI' do
+    # Simple string match grammar
+    let(:string_grammar) do
+      {
+        atoms: [{ Str: { pattern: 'hello' } }],
+        root: 0
+      }.to_json
+    end
+
+    # Named capture grammar
+    let(:named_grammar) do
+      {
+        atoms: [
+          { Str: { pattern: 'hello' } },
+          { Named: { name: 'greeting', atom: 0 } }
+        ],
+        root: 1
+      }.to_json
+    end
+
+    # Sequence grammar
+    let(:sequence_grammar) do
+      {
+        atoms: [
+          { Str: { pattern: 'hello' } },
+          { Str: { pattern: ' ' } },
+          { Str: { pattern: 'world' } },
+          { Sequence: { atoms: [0, 1, 2] } }
+        ],
+        root: 3
+      }.to_json
+    end
+
+    context 'with native extension available' do
+      before do
+        skip 'Native extension not available' unless Parsanol::Native.available?
+      end
+
+      it 'returns Slice object for simple string match' do
+        result = Parsanol::Native.parse_to_ruby_objects(string_grammar, 'hello')
+        expect(result).to be_a(Parsanol::Slice)
+        expect(result.to_s).to eq('hello')
+        expect(result.offset).to eq(0)
+      end
+
+      it 'returns Hash with Slice values for named captures' do
+        result = Parsanol::Native.parse_to_ruby_objects(named_grammar, 'hello')
+        expect(result).to be_a(Hash)
+        expect(result.keys).to eq(['greeting'])
+
+        greeting = result['greeting']
+        expect(greeting).to be_a(Parsanol::Slice)
+        expect(greeting.to_s).to eq('hello')
+        expect(greeting.offset).to eq(0)
+      end
+
+      it 'returns Array of Slice objects for sequences' do
+        result = Parsanol::Native.parse_to_ruby_objects(sequence_grammar, 'hello world')
+        expect(result).to be_a(Array)
+        expect(result.length).to eq(3)
+
+        # First slice: "hello"
+        expect(result[0]).to be_a(Parsanol::Slice)
+        expect(result[0].to_s).to eq('hello')
+        expect(result[0].offset).to eq(0)
+
+        # Second slice: " "
+        expect(result[1]).to be_a(Parsanol::Slice)
+        expect(result[1].to_s).to eq(' ')
+        expect(result[1].offset).to eq(5)
+
+        # Third slice: "world"
+        expect(result[2]).to be_a(Parsanol::Slice)
+        expect(result[2].to_s).to eq('world')
+        expect(result[2].offset).to eq(6)
+      end
+
+      it 'preserves correct byte offsets for multi-byte characters' do
+        # Test with input containing UTF-8 characters
+        utf8_grammar = {
+          atoms: [{ Str: { pattern: '日本語' } }],
+          root: 0
+        }.to_json
+
+        result = Parsanol::Native.parse_to_ruby_objects(utf8_grammar, '日本語')
+        expect(result).to be_a(Parsanol::Slice)
+        expect(result.to_s).to eq('日本語')
+        expect(result.offset).to eq(0)
+      end
+    end
+  end
+end

data/spec/parsanol/parser_spec.rb

@@ -0,0 +1,36 @@
+require 'spec_helper'
+
+describe Parsanol::Parser do
+  include Parsanol
+  class FooParser < Parsanol::Parser
+    rule(:foo) { str('foo') }
+    root(:foo)
+  end
+
+  describe '<- .root' do
+    parser = Class.new(Parsanol::Parser) do
+      def root_parslet
+        :answer
+      end
+    end
+    parser.root :root_parslet
+
+    it "has defined a 'root' method, returning the root" do
+      parser_instance = parser.new
+      expect(parser_instance.root).to eq(:answer)
+    end
+  end
+
+  it "parses 'foo'" do
+    FooParser.new.parse('foo').should == 'foo'
+  end
+
+  context 'composition' do
+    let(:parser) { FooParser.new }
+
+    it 'allows concatenation' do
+      composite = parser >> str('bar')
+      composite.should parse('foobar')
+    end
+  end
+end

data/spec/parsanol/parslet_spec.rb

@@ -0,0 +1,38 @@
+require 'spec_helper'
+
+describe Parsanol do
+  include Parsanol
+  
+  describe Parsanol::ParseFailed do
+    it "should be caught by an empty rescue" do
+      begin
+        raise Parsanol::ParseFailed
+      rescue
+        # Success! Ignore this.
+      end
+    end 
+  end
+  describe "<- .rule" do
+    # Rules define methods. This can be easily tested by defining them right 
+    # here. 
+    context "empty rule" do
+      rule(:empty) { }
+      
+      it "should raise a NotImplementedError" do
+        lambda {
+          empty.parslet
+        }.should raise_error(NotImplementedError)
+      end 
+    end
+    
+    context "containing 'any'" do
+      rule(:any_rule) { any }
+      subject { any_rule }
+      
+      it { should be_a Parsanol::Atoms::Entity }
+      it "should memoize the returned instance" do
+        any_rule.object_id.should == any_rule.object_id
+      end 
+    end
+  end
+end

data/spec/parsanol/pattern_spec.rb

@@ -0,0 +1,272 @@
+require 'spec_helper'
+
+require 'parsanol/parslet'
+
+describe Parsanol::Pattern do
+  include Parsanol
+  
+  # These two factory methods help make the specs more robust to interface
+  # changes. They also help to label trees (t) and patterns (p).
+  def p(pattern)
+    Parsanol::Pattern.new(pattern)
+  end
+  def t(obj)
+    obj
+  end
+  
+  # Tries to match pattern to the tree, and verifies the bindings hash. Don't
+  # use this for new examples.
+  #
+  RSpec::Matchers.define :match_with_bind do |pattern, exp_bindings|
+    unless respond_to?(:failure_message)
+      alias_method :failure_message_for_should, :failure_message
+    end
+    
+    failure_message do |tree|
+      "expected #{pattern.inspect} to match #{tree.inspect}, but didn't. (block wasn't called or not correctly)"
+    end
+    match do |tree|
+      bindings = Parsanol::Pattern.new(pattern).match(tree)
+      bindings && bindings == exp_bindings
+    end
+  end
+
+  # This is the more modern version of verifying a match: (uses 'exp'
+  # implicitly). Checks for a match of pattern in +exp+ and yields the
+  # matched variables.
+  #
+  def with_match_locals(pattern, &block) 
+    bindings = p(pattern).match(exp)
+    bindings.should_not be_nil
+    
+    block.call(bindings) if block
+  end
+  
+  # Can't use #match here, so I went to the Thesaurus.
+  #
+  RSpec::Matchers.define :detect do |pattern|
+    match do |tree|
+      bindings = Parsanol::Pattern.new(pattern).match(tree)
+
+      bindings ? true : false
+    end
+  end
+  
+  describe "<- #match" do
+    context "injecting bindings" do
+      let(:pattern) { p(simple(:x)) }
+      
+      it "should not modify the original bindings hash" do
+        h = {}
+        b=pattern.match('a', h)
+        h.size.should == 0
+        b.size.should == 1
+      end
+      it "should return nil when no match succeeds" do
+        pattern.match([], :foo => :bar).should be_nil
+      end
+      context "when matching simple(:x) against 'a'" do
+        let(:bindings) { pattern.match(t('a'), :foo => :bar) }
+        
+        before(:each) { bindings.should_not be_nil }
+        it "should return the injected bindings" do
+          bindings[:foo].should == :bar
+        end
+        it "should return the new bindings" do  
+          bindings[:x].should == 'a'
+        end
+      end
+    end
+    context "simple strings" do
+      let(:exp) { 'aaaa' }
+
+      it "should match simple strings" do
+        exp.should match_with_bind(simple(:x), :x => 'aaaa')
+      end 
+    end
+    context "simple hash {:a => 'b'}" do
+      attr_reader :exp
+      before(:each) do
+        @exp = t(:a => 'b')
+      end
+
+      it "should not match {:a => simple(:x), :b => simple(:y)}" do
+        exp.should_not detect(:a => simple(:x), :b => simple(:y))
+      end
+      it "should match {:a => simple(:x)}, binding 'x' to the first argument" do
+        exp.should match_with_bind({:a => simple(:x)}, :x => 'b')
+      end 
+      it "should match {:a => 'b'} with no binds" do
+        exp.should match_with_bind({:a => 'b'}, {})
+      end 
+    end
+    context "a more complex hash {:a => {:b => 'c'}}" do
+      attr_reader :exp
+      before(:each) do
+        @exp = t(:a => {:b => 'c'})
+      end
+      
+      it "should match wholly with {:a => {:b => simple(:x)}}" do
+        exp.should match_with_bind({:a => {:b => simple(:x)}}, :x => 'c')
+      end
+      it "should match wholly with {:a => subtree(:t)}" do
+        with_match_locals(:a => subtree(:t)) do |dict|
+          dict[:t].should == {:b => 'c'}
+        end
+      end
+      it "should not bind subtrees to variables in {:a => simple(:x)}" do
+        p(:a => simple(:x)).should_not detect(exp)
+      end
+    end
+    context "a more complex hash {:a => 'a', :b => 'b'}" do
+      attr_reader :exp
+      before(:each) do
+        @exp = t({:a => 'a', :b => 'b'})
+      end
+
+      it "should not match partially" do
+        Parsanol::Pattern.new(:a => simple(:x)).match(exp).should be_nil
+      end 
+      it "should match completely" do
+        exp.should match_with_bind({:a => simple(:x), :b => simple(:y)}, 
+          :x => 'a', 
+          :y => 'b')
+      end 
+    end
+    context "an array of 'a', 'b', 'c'" do
+      let(:exp) { ['a', 'b', 'c'] }
+
+      it "should match all elements at once" do
+        exp.should match_with_bind(
+          [simple(:x), simple(:y), simple(:z)], 
+          :x => 'a', :y => 'b', :z => 'c')
+      end 
+    end
+    context "{:a => 'a', :b => 'b'}" do
+      attr_reader :exp
+      before(:each) do
+        @exp = t(:a => 'a', :b => 'b')
+      end
+
+      it "should match both elements simple(:x), simple(:y)" do
+        exp.should match_with_bind(
+          {:a => simple(:x), :b => simple(:y)}, 
+          :x => 'a', :y => 'b')
+      end
+      it "should not match a constrained match (simple(:x) != simple(:y))"  do
+        exp.should_not detect({:a => simple(:x), :b => simple(:x)})
+      end
+    end
+    context "{:a => 'a', :b => 'a'}" do
+      attr_reader :exp
+      before(:each) do
+        @exp = t(:a => 'a', :b => 'a')
+      end
+
+      it "should match constrained pattern" do
+        exp.should match_with_bind(
+          {:a => simple(:x), :b => simple(:x)}, 
+          :x => 'a')
+      end
+    end
+    context "{:sub1 => {:a => 'a'}, :sub2 => {:a => 'a'}}" do
+      attr_reader :exp
+      before(:each) do
+        @exp = t({
+          :sub1 => {:a => 'a'}, 
+          :sub2 => {:a => 'a'} 
+        })
+      end
+
+      it "should verify constraints over several subtrees" do
+        exp.should match_with_bind({
+          :sub1 => {:a => simple(:x)}, 
+          :sub2 => {:a => simple(:x)} 
+        }, :x => 'a')
+      end
+      it "should return both bind variables simple(:x), simple(:y)" do
+        exp.should match_with_bind({
+          :sub1 => {:a => simple(:x)}, 
+          :sub2 => {:a => simple(:y)} 
+        }, :x => 'a', :y => 'a')
+      end  
+    end
+    context "{:sub1 => {:a => 'a'}, :sub2 => {:a => 'b'}}" do
+      attr_reader :exp
+      before(:each) do
+        @exp = t({
+          :sub1 => {:a => 'a'}, 
+          :sub2 => {:a => 'b'} 
+        })
+      end
+
+      it "should verify constraints over several subtrees" do
+        exp.should_not match_with_bind({
+          :sub1 => {:a => simple(:x)}, 
+          :sub2 => {:a => simple(:x)} 
+        }, :x => 'a')
+      end
+      it "should return both bind variables simple(:x), simple(:y)" do
+        exp.should match_with_bind({
+          :sub1 => {:a => simple(:x)}, 
+          :sub2 => {:a => simple(:y)} 
+        }, :x => 'a', :y => 'b')
+      end  
+    end
+    context "[{:a => 'x'}, {:a => 'y'}]" do
+      attr_reader :exp  
+      before(:each) do
+        @exp = t([{:a => 'x'}, {:a => 'y'}])
+      end
+      
+      it "should not match sequence(:x) (as a whole)" do
+        exp.should_not detect(sequence(:x))
+      end
+    end
+    context "['x', 'y', 'z']" do
+      attr_reader :exp  
+      before(:each) do
+        @exp = t(['x', 'y', 'z'])
+      end
+
+      it "should match [simple(:x), simple(:y), simple(:z)]" do
+        with_match_locals([simple(:x), simple(:y), simple(:z)]) do |dict|
+          dict[:x].should == 'x'
+          dict[:y].should == 'y'
+          dict[:z].should == 'z'
+        end
+      end
+      it "should match %w(x y z)" do
+        exp.should match_with_bind(%w(x y z), { })
+      end 
+      it "should not match [simple(:x), simple(:y), simple(:x)]" do
+        exp.should_not detect([simple(:x), simple(:y), simple(:x)])
+      end
+      it "should not match [simple(:x), simple(:y)]" do
+        exp.should_not detect([simple(:x), simple(:y), simple(:x)])
+      end
+      it "should match sequence(:x) (as array)" do
+        exp.should match_with_bind(sequence(:x), :x => ['x', 'y', 'z'])
+      end
+    end
+    context "{:a => [1,2,3]}" do
+      attr_reader :exp  
+      before(:each) do
+        @exp = t(:a => [1,2,3])
+      end
+
+      it "should match :a => sequence(:x) (binding x to the whole array)" do
+        exp.should match_with_bind({:a => sequence(:x)}, {:x => [1,2,3]})
+      end
+    end
+    context "with differently ordered hashes" do
+      it "should still match" do
+        t(:a => 'a', :b => 'b').should detect(:a => 'a', :b => 'b')
+        t(:a => 'a', :b => 'b').should detect(:b => 'b', :a => 'a')
+
+        t(:b => 'b', :a => 'a').should detect(:b => 'b', :a => 'a')
+        t(:b => 'b', :a => 'a').should detect(:a => 'a', :b => 'b')
+      end 
+    end
+  end
+end