parsanol 3.0.0

data/spec/lexer_spec.rb

@@ -0,0 +1,231 @@
+# frozen_string_literal: true
+
+require "spec_helper"
+require "parsanol/lexer"
+
+RSpec.describe Parsanol::Lexer do
+  describe "basic tokenization" do
+    before do
+      class TestLexer < Parsanol::Lexer
+        token :word, /[a-z]+/
+        token :number, /[0-9]+/
+        ignore /\s+/
+      end
+    end
+
+    let(:lexer) { TestLexer.new }
+
+    it "tokenizes words" do
+      tokens = lexer.tokenize("hello world")
+      expect(tokens.size).to eq(3) # 2 words + eof
+      expect(tokens[0]["type"]).to eq("word")
+      expect(tokens[0]["value"]).to eq("hello")
+      expect(tokens[1]["type"]).to eq("word")
+      expect(tokens[1]["value"]).to eq("world")
+    end
+
+    it "tokenizes numbers" do
+      tokens = lexer.tokenize("123 456")
+      expect(tokens.size).to eq(3)
+      expect(tokens[0]["type"]).to eq("number")
+      expect(tokens[0]["value"]).to eq("123")
+      expect(tokens[1]["type"]).to eq("number")
+      expect(tokens[1]["value"]).to eq("456")
+    end
+
+    it "ignores whitespace" do
+      tokens = lexer.tokenize("hello   world")
+      expect(tokens.size).to eq(3)
+    end
+
+    it "includes location information" do
+      tokens = lexer.tokenize("hello")
+      expect(tokens[0]["location"]["line"]).to eq(1)
+      expect(tokens[0]["location"]["column"]).to eq(1)
+      expect(tokens[0]["location"]["offset"]).to eq(0)
+    end
+
+    it "tracks line and column correctly" do
+      tokens = lexer.tokenize("hello\nworld")
+      expect(tokens[0]["location"]["line"]).to eq(1)
+      expect(tokens[0]["location"]["column"]).to eq(1)
+      expect(tokens[1]["location"]["line"]).to eq(2)
+      expect(tokens[1]["location"]["column"]).to eq(1)
+    end
+
+    it "adds eof token at the end" do
+      tokens = lexer.tokenize("hello")
+      expect(tokens.last["type"]).to eq("eof")
+    end
+  end
+
+  describe "priority handling" do
+    before do
+      class PriorityLexer < Parsanol::Lexer
+        token :keyword, /if|else|while/, priority: 100
+        token :identifier, /[a-z]+/, priority: 1
+        ignore /\s+/
+      end
+    end
+
+    let(:lexer) { PriorityLexer.new }
+
+    it "matches higher priority patterns first" do
+      tokens = lexer.tokenize("if else while")
+      expect(tokens[0]["type"]).to eq("keyword")
+      expect(tokens[0]["value"]).to eq("if")
+      expect(tokens[1]["type"]).to eq("keyword")
+      expect(tokens[1]["value"]).to eq("else")
+      expect(tokens[2]["type"]).to eq("keyword")
+      expect(tokens[2]["value"]).to eq("while")
+    end
+
+    it "falls back to lower priority when no match" do
+      tokens = lexer.tokenize("if variable else")
+      expect(tokens[0]["type"]).to eq("keyword")
+      expect(tokens[1]["type"]).to eq("identifier")
+      expect(tokens[1]["value"]).to eq("variable")
+      expect(tokens[2]["type"]).to eq("keyword")
+    end
+  end
+
+  describe "keyword helper" do
+    before do
+      class KeywordLexer < Parsanol::Lexer
+        keyword :if, :then, :else, priority: 50
+        token :identifier, /[a-z]+/, priority: 1
+        ignore /\s+/
+      end
+    end
+
+    let(:lexer) { KeywordLexer.new }
+
+    it "creates keyword tokens with high priority" do
+      tokens = lexer.tokenize("if x then y")
+      expect(tokens[0]["type"]).to eq("IF")
+      expect(tokens[1]["type"]).to eq("identifier")
+      expect(tokens[2]["type"]).to eq("THEN")
+    end
+  end
+
+  describe "longest match rule" do
+    before do
+      class LongestMatchLexer < Parsanol::Lexer
+        token :string, /"[^"]*"/
+        token :quote, /"/
+        ignore /\s+/
+      end
+    end
+
+    let(:lexer) { LongestMatchLexer.new }
+
+    it "prefers longer matches" do
+      tokens = lexer.tokenize('"hello"')
+      expect(tokens.size).to eq(2) # string + eof
+      expect(tokens[0]["type"]).to eq("string")
+      expect(tokens[0]["value"]).to eq('"hello"')
+    end
+  end
+
+  describe "inheritance" do
+    before do
+      class BaseLexer < Parsanol::Lexer
+        token :word, /[a-z]+/
+        ignore /\s+/
+      end
+
+      class ExtendedLexer < BaseLexer
+        token :number, /[0-9]+/
+      end
+    end
+
+    let(:base_lexer) { BaseLexer.new }
+    let(:extended_lexer) { ExtendedLexer.new }
+
+    it "inherits tokens from parent class" do
+      tokens = extended_lexer.tokenize("hello 123")
+      # word + number + eof = 3 tokens
+      expect(tokens.size).to eq(3)
+      expect(tokens[0]["type"]).to eq("word")
+      expect(tokens[0]["value"]).to eq("hello")
+      expect(tokens[1]["type"]).to eq("number")
+      expect(tokens[1]["value"]).to eq("123")
+    end
+
+    it "does not modify parent class" do
+      tokens = base_lexer.tokenize("hello world")
+      # Base lexer should only match words
+      expect(tokens.size).to eq(3) # word + word + eof
+      expect(tokens[0]["type"]).to eq("word")
+      expect(tokens[0]["value"]).to eq("hello")
+      expect(tokens[1]["type"]).to eq("word")
+      expect(tokens[1]["value"]).to eq("world")
+    end
+  end
+
+  describe "error handling" do
+    before do
+      class ErrorLexer < Parsanol::Lexer
+        token :word, /[a-z]+/
+        ignore /\s+/
+      end
+    end
+
+    let(:lexer) { ErrorLexer.new }
+
+    it "produces error token for unrecognized input" do
+      tokens = lexer.tokenize("hello 123 world")
+      # 123 should produce error tokens (1, 2, 3)
+      error_tokens = tokens.select { |t| t["type"] == "error" }
+      expect(error_tokens.size).to be >= 1
+    end
+  end
+
+  describe "JSON example" do
+    before do
+      class JsonLexer < Parsanol::Lexer
+        token :string, /"[^"]*"/
+        token :number, /-?[0-9]+(\.[0-9]+)?([eE][+-]?[0-9]+)?/
+        token :true, /true/
+        token :false, /false/
+        token :null, /null/
+        token :lbrace, /\{/
+        token :rbrace, /\}/
+        token :lbracket, /\[/
+        token :rbracket, /\]/
+        token :colon, /:/
+        token :comma, /,/
+        ignore /\s+/
+      end
+    end
+
+    let(:lexer) { JsonLexer.new }
+
+    it "tokenizes simple JSON object" do
+      tokens = lexer.tokenize('{"name": "test"}')
+      types = tokens.map { |t| t["type"] }
+      expect(types).to eq(%w[lbrace string colon string rbrace eof])
+    end
+
+    it "tokenizes JSON with numbers" do
+      tokens = lexer.tokenize('{"count": 42}')
+      types = tokens.map { |t| t["type"] }
+      expect(types).to eq(%w[lbrace string colon number rbrace eof])
+    end
+
+    it "tokenizes JSON with boolean" do
+      tokens = lexer.tokenize('{"active": true}')
+      types = tokens.map { |t| t["type"] }
+      expect(types).to eq(%w[lbrace string colon true rbrace eof])
+    end
+
+    it "tokenizes complex JSON" do
+      tokens = lexer.tokenize('{"items": [1, 2, 3], "nested": {"x": true}}')
+      types = tokens.map { |t| t["type"] }
+      expect(types).to eq(%w[
+        lbrace string colon lbracket number comma number comma number rbracket
+        comma string colon lbrace string colon true rbrace rbrace eof
+      ])
+    end
+  end
+end

data/spec/parsanol/atom_results_spec.rb

@@ -0,0 +1,39 @@
+require 'spec_helper'
+
+describe 'Result of a Parsanol#parse' do
+  include Parsanol; extend Parsanol
+
+  describe 'regression' do
+    [
+      # Behaviour with maybe-nil
+      [str('foo').maybe >> str('bar'), 'bar', 'bar'],
+      [str('bar') >> str('foo').maybe, 'bar', 'bar'],
+
+      # These might be hard to understand; look at the result of
+      #   str.maybe >> str
+      # and
+      #   str.maybe >> str first.
+      [(str('f').maybe >> str('b')).repeat, 'bb', 'bb'],
+      [(str('b') >> str('f').maybe).repeat, 'bb', 'bb'],
+
+      [str('a').as(:a) >> (str('b') >> str('c').as(:a)).repeat, 'abc',
+       [{ a: 'a' }, { a: 'c' }]],
+
+      [str('a').as(:a).repeat >> str('b').as(:b).repeat, 'ab', [{ a: 'a' }, { b: 'b' }]],
+
+      # Repetition behaviour / named vs. unnamed
+      [str('f').repeat, '', ''],
+      [str('f').repeat.as(:f), '', { f: [] }],
+
+      # Maybe behaviour / named vs. unnamed
+      [str('f').maybe, '', ''],
+      [str('f').maybe.as(:f), '', { f: nil }],
+    ].each do |parslet, input, result|
+      context parslet.inspect.to_s do
+        it "parses \"#{input}\" into \"#{result}\"" do
+          expect(strip_positions(parslet.parse(input))).to eq(result)
+        end
+      end
+    end
+  end
+end

data/spec/parsanol/atoms/alternative_spec.rb

@@ -0,0 +1,26 @@
+require 'spec_helper'
+
+describe Parsanol::Atoms::Alternative do
+  include Parsanol
+  
+  describe '| shortcut' do
+    let(:alternative) { str('a') | str('b') }
+    
+    context "when chained with different atoms" do
+      before(:each) { 
+        # Chain something else to the alternative parslet. If it modifies the
+        # parslet atom in place, we'll notice: 
+        
+        alternative | str('d')
+      }
+      let!(:chained) { alternative | str('c') }
+      
+      
+      it "is side-effect free" do
+        chained.should parse('c')
+        chained.should parse('a')
+        chained.should_not parse('d')
+      end 
+    end
+  end
+end

data/spec/parsanol/atoms/base_spec.rb

@@ -0,0 +1,127 @@
+require 'spec_helper'
+
+describe Parsanol::Atoms::Base do
+  let(:parslet) { Parsanol::Atoms::Base.new }
+  let(:context) { Parsanol::Atoms::Context.new }
+
+  describe '<- #try(io)' do
+    it 'raises NotImplementedError' do
+      lambda {
+        parslet.try(double(:io), context, false)
+      }.should raise_error(NotImplementedError)
+    end
+  end
+
+  describe '<- #flatten_sequence' do
+    [
+      # 9 possibilities for making a word of 2 letters from the alphabeth of
+      # A(rray), H(ash) and S(tring). Make sure that all results are valid.
+      #
+      %w[a b], 'ab',                             # S S
+      [['a'], ['b']], %w[a b],                   # A A
+      [{ a: 'a' }, { b: 'b' }], { a: 'a', b: 'b' }, # H H
+
+      [{ a: 'a' }, ['a']], [{ a: 'a' }, 'a'],         # H A
+      [{ a: 'a' }, 's'],   { a: 'a' },                # H S
+
+      [['a'], { a: 'a' }], ['a', { a: 'a' }],         # A H (symmetric to H A)
+      [['a'], 'b'], ['a'], # A S
+
+      ['a', { b: 'b' }], { b: 'b' }, # S H (symmetric to H S)
+      ['a', ['b']], ['b'],                          # S A (symmetric to A S)
+
+      [nil, ['a']], ['a'],                          # handling of lhs nil
+      [nil, { a: 'a' }], { a: 'a' },
+      [['a'], nil], ['a'],                          # handling of rhs nil
+      [{ a: 'a' }, nil], { a: 'a' }
+    ].each_slice(2) do |sequence, result|
+      context 'for ' + sequence.inspect do
+        it "equals #{result.inspect}" do
+          parslet.flatten_sequence(sequence).should == result
+        end
+      end
+    end
+  end
+
+  describe '<- #flatten_repetition' do
+    def unnamed(obj)
+      parslet.flatten_repetition(obj, false)
+    end
+
+    it 'gives subtrees precedence' do
+      unnamed([[{ a: 'a' }, { m: 'm' }], { a: 'a' }]).should == [{ a: 'a' }]
+    end
+  end
+
+  describe '#parse(source)' do
+    context 'when given something that looks like a source' do
+      let(:source) do
+        double('source lookalike',
+               line_and_column: [1, 2],
+               bytepos: 1,
+               chars_left: 0)
+      end
+
+      it 'does not rewrap in a source' do
+        expect(Parsanol::Source).not_to receive(:new)
+
+        begin
+          parslet.parse(source)
+        rescue NotImplementedError
+        end
+      end
+    end
+  end
+
+  context 'when the parse fails, the exception' do
+    it 'contains a string' do
+      Parsanol.str('foo').parse('bar')
+    rescue Parsanol::ParseFailed => e
+      e.message.should be_kind_of(String)
+    end
+  end
+
+  context 'when not all input is consumed' do
+    let(:parslet) { Parsanol.str('foo') }
+
+    it 'raises with a proper error message' do
+      error = catch_failed_parse do
+        parslet.parse('foobar')
+      end
+
+      error.to_s.should == "Don't know what to do with \"bar\" at line 1 char 4."
+    end
+  end
+
+  context 'when only parsing string prefix' do
+    let(:parslet) { Parsanol.str('foo') >> Parsanol.str('bar') }
+
+    it 'returns the first half on a prefix parse' do
+      parslet.parse('foobarbaz', prefix: true).should == 'foobar'
+    end
+  end
+
+  describe ':reporter option' do
+    let(:parslet) { Parsanol.str('test') >> Parsanol.str('ing') }
+    let(:reporter) { double(:reporter) }
+
+    it 'replaces the default reporter' do
+      cause = double(:cause)
+
+      # Two levels of the parse, calling two different error reporting
+      # methods.
+      expect(reporter).to receive(:err_at).once
+      expect(reporter).to receive(:err).and_return(cause).once
+      expect(reporter).to receive(:succ).once
+
+      # The final cause will be sent the #raise method.
+      expect(cause).to receive(:raise).once.and_throw(:raise)
+
+      catch(:raise) do
+        parslet.parse('testung', reporter: reporter)
+
+        raise 'NEVER REACHED'
+      end
+    end
+  end
+end

data/spec/parsanol/atoms/capture_spec.rb

@@ -0,0 +1,21 @@
+require 'spec_helper'
+
+describe Parsanol::Atoms::Capture do
+  include Parsanol
+
+  let(:context) { Parsanol::Atoms::Context.new(nil) }
+
+  def inject string, parser
+    source = Parsanol::Source.new(string)
+    parser.apply(source, context, true)
+  end
+
+  it "should capture simple results" do
+    inject 'a', str('a').capture(:a)
+    strip_positions(context.captures[:a]).should == 'a'
+  end
+  it "should capture complex results" do
+    inject 'a', str('a').as(:b).capture(:a)
+    strip_positions(context.captures[:a]).should == {:b => 'a'}
+  end
+end

data/spec/parsanol/atoms/combinations_spec.rb

@@ -0,0 +1,5 @@
+require 'spec_helper'
+
+describe "Parsanol combinations" do
+  include Parsanol
+end

data/spec/parsanol/atoms/custom_spec.rb

@@ -0,0 +1,79 @@
+# frozen_string_literal: true
+
+require 'spec_helper'
+
+class TestCustomAtom < Parsanol::Atoms::Custom
+  def initialize(match_string)
+    @match_string = match_string
+    super()
+  end
+
+  def try_match(source, context, consume_all)
+    pos = source.bytepos
+
+    # Try to match the string using consume (returns a Slice)
+    result = source.consume(@match_string.length)
+    if result == @match_string
+      # Success - source.consume already returned a Slice
+      [true, result]
+    else
+      # Restore position on failure
+      source.bytepos = pos
+      [false, nil]
+    end
+  end
+
+  def to_s_inner(prec = nil)
+    "test_custom(#{@match_string.inspect})"
+  end
+end
+
+RSpec.describe Parsanol::Atoms::Custom do
+  let(:atom) { TestCustomAtom.new('hello') }
+
+  it 'can be created and used for parsing' do
+    result = atom.parse('hello')
+    expect(result.to_s).to eq('hello')
+  end
+
+  it 'fails on non-matching input' do
+    expect { atom.parse('world') }.to raise_error(Parsanol::ParseFailed)
+  end
+
+  it 'raises NotImplementedError when try_match is not implemented' do
+    custom = Class.new(Parsanol::Atoms::Custom).new
+    expect { custom.parse('test') }.to raise_error(NotImplementedError)
+  end
+
+  it 'can be combined with other atoms using sequence' do
+    # Use as() to label results for structured output
+    combined = atom.as(:first) >> Parsanol.str(' world').as(:second)
+    result = combined.parse('hello world')
+    expect(result).to eq({ first: 'hello', second: ' world' })
+  end
+
+  it 'supports repetition' do
+    # Basic repetition - returns concatenated result by default
+    repeated = atom.repeat(2, 2)
+    result = repeated.parse('hellohello')
+    expect(result.to_s).to eq('hellohello')
+  end
+
+  it 'can be used in alternative' do
+    alt = atom | Parsanol.str('world')
+    expect(alt.parse('hello').to_s).to eq('hello')
+    expect(alt.parse('world').to_s).to eq('world')
+  end
+
+  it 'supports maybe' do
+    maybe = atom.maybe
+    expect(maybe.parse('hello').to_s).to eq('hello')
+    # Maybe returns empty string when it doesn't match (standard Parslet behavior)
+    expect(maybe.parse('')).to eq('')
+  end
+
+  it 'provides custom to_s_inner for debugging' do
+    expect(atom.to_s).to include('test_custom')
+    expect(atom.to_s).to include('hello')
+  end
+end

data/spec/parsanol/atoms/dsl_spec.rb

@@ -0,0 +1,7 @@
+require 'spec_helper'
+
+describe Parsanol::Atoms::DSL do
+  describe "deprecated methods" do
+    let(:parslet) { Parsanol.str('foo') }
+  end
+end

data/spec/parsanol/atoms/entity_spec.rb

@@ -0,0 +1,77 @@
+require 'spec_helper'
+
+describe Parsanol::Atoms::Entity do
+  context "when constructed with str('bar') inside" do
+    let(:named) { Parsanol::Atoms::Entity.new('name', &proc { Parsanol.str('bar') }) }
+
+    it "should parse 'bar' without raising exceptions" do
+      named.parse('bar')
+    end 
+    it "should raise when applied to 'foo'" do
+      lambda {
+        named.parse('foo')
+      }.should raise_error(Parsanol::ParseFailed)
+    end 
+
+    describe "#inspect" do
+      it "should return the name of the entity" do
+        named.inspect.should == 'NAME'
+      end 
+    end
+  end
+  context "when constructed with empty block" do
+    let(:entity) { Parsanol::Atoms::Entity.new('name', &proc { }) }
+    
+    it "should raise NotImplementedError" do
+      lambda {
+        entity.parse('some_string')
+      }.should raise_error(NotImplementedError)
+    end 
+  end
+  
+  context "recursive definition parser" do
+    class RecDefParser
+      include Parsanol
+      rule :recdef do
+        str('(') >> atom >> str(')')
+      end
+      rule :atom do
+        str('a') | str('b') | recdef
+      end
+    end
+    let(:parser) { RecDefParser.new }
+    
+    it "should parse balanced parens" do
+      parser.recdef.parse("(((a)))")
+    end
+    it "should not throw 'stack level too deep' when printing errors" do
+      cause = catch_failed_parse { parser.recdef.parse('(((a))') }
+      cause.ascii_tree
+    end
+  end
+
+  context "when constructed with a label" do
+    let(:named) { Parsanol::Atoms::Entity.new('name', 'label', &proc { Parsanol.str('bar') }) }
+
+    it "should parse 'bar' without raising exceptions" do
+      named.parse('bar')
+    end 
+    it "should raise when applied to 'foo'" do
+      lambda {
+        named.parse('foo')
+      }.should raise_error(Parsanol::ParseFailed)
+    end 
+
+    describe "#inspect" do
+      it "should return the label of the entity" do
+        named.inspect.should == 'label'
+      end 
+    end
+
+    describe "#parslet" do
+      it "should set the label on the cached parslet" do
+        named.parslet.label.should == 'label'
+      end
+    end
+  end
+end

data/spec/parsanol/atoms/ignored_spec.rb

@@ -0,0 +1,15 @@
+require 'spec_helper'
+
+describe Parsanol::Atoms::Ignored do
+  include Parsanol
+
+  describe "ignore" do
+    it "ignores parts of the input" do
+      str('a').ignore.parse('a').should == nil
+      (str('a') >> str('b').ignore >> str('c')).parse('abc').should == 'ac'
+      (str('a') >> str('b').as(:name).ignore >> str('c')).parse('abc').should == 'ac'
+      (str('a') >> str('b').maybe.ignore >> str('c')).parse('abc').should == 'ac'
+      (str('a') >> str('b').maybe.ignore >> str('c')).parse('ac').should == 'ac'
+    end
+  end
+end

data/spec/parsanol/atoms/infix_spec.rb

@@ -0,0 +1,5 @@
+require 'spec_helper'
+
+describe Parsanol::Atoms::Infix do
+
+end

data/spec/parsanol/atoms/lookahead_spec.rb

@@ -0,0 +1,22 @@
+require 'spec_helper'
+
+describe Parsanol::Atoms::Lookahead do
+  include Parsanol
+  
+  describe 'negative lookahead' do
+    it "influences the error tree" do
+      parser = str('f').absent? >> str('b')
+      cause = catch_failed_parse { parser.parse('f') }
+      
+      cause.ascii_tree.should == "Failed to match sequence (!'f' 'b') at line 1 char 1.\n`- Input should not start with 'f' at line 1 char 1.\n"
+    end 
+  end
+  describe 'positive lookahead' do
+    it "influences the error tree" do
+      parser = str('f').present? >> str('b')
+      cause = catch_failed_parse { parser.parse('b') }
+      
+      cause.ascii_tree.should == "Failed to match sequence (&'f' 'b') at line 1 char 1.\n`- Input should start with 'f' at line 1 char 1.\n"
+    end 
+  end
+end

data/spec/parsanol/atoms/named_spec.rb

@@ -0,0 +1,4 @@
+require 'spec_helper'
+
+describe Parsanol::Atoms::Named do
+end

data/spec/parsanol/atoms/re_spec.rb

@@ -0,0 +1,14 @@
+require 'spec_helper'
+
+describe Parsanol::Atoms::Re do
+  describe "construction" do
+    include Parsanol
+    
+    it "should allow match(str) form" do
+      match('[a]').should be_a(Parsanol::Atoms::Re)
+    end 
+    it "should allow match[str] form" do
+      match['a'].should be_a(Parsanol::Atoms::Re)
+    end 
+  end
+end