citrus 1.8.0 → 2.0.0

data/lib/citrus/debug.rb

@@ -3,6 +3,16 @@ require 'builder'
 
 module Citrus
   class Match
+    # The offset at which this match was found in the input.
+    attr_accessor :offset
+
+    def debug_attrs
+      { "names"   => names.join(','),
+        "text"    => to_s,
+        "offset"  => offset
+      }
+    end
+
     # Creates a Builder::XmlMarkup object from this match. Useful when
     # inspecting a nested match. The +xml+ argument may be a Hash of
     # Builder::XmlMarkup options.
@@ -13,12 +23,10 @@ module Citrus
         xml.instruct!
       end
 
-      attrs = { "names" => names.join(','), "text" => to_s, "offset" => offset }
-
       if matches.empty?
-        xml.match(attrs)
+        xml.match(debug_attrs)
       else
-        xml.match(attrs) do
+        xml.match(debug_attrs) do
           matches.each {|m| m.to_markup(xml) }
         end
       end
@@ -36,4 +44,26 @@ module Citrus
       to_xml
     end
   end
+
+  # Hijack all classes that use Rule#create_match to create matches. Now, when
+  # matches are created they will also record their offset to help debugging.
+  # This functionality is included in this file because calculating the offset
+  # of every match as it is created can slow things down quite a bit.
+  [ Terminal,
+    AndPredicate,
+    NotPredicate,
+    ButPredicate,
+    Repeat,
+    Sequence
+  ].each do |rule_class|
+    rule_class.class_eval do
+      alias original_match match
+  
+      def match(input)
+        m = original_match(input)
+        m.offset = input.pos - m.length if m
+        m
+      end
+    end
+  end
 end

data/test/file_test.rb

@@ -49,12 +49,12 @@ class CitrusFileTest < Test::Unit::TestCase
     match = grammar.parse('""')
     assert(match)
     assert_kind_of(Rule, match.value)
-    assert_instance_of(FixedWidth, match.value)
+    assert_instance_of(Terminal, match.value)
 
     match = grammar.parse('"a"')
     assert(match)
     assert_kind_of(Rule, match.value)
-    assert_instance_of(FixedWidth, match.value)
+    assert_instance_of(Terminal, match.value)
 
     match = grammar.parse('"a" "b"')
     assert(match)
@@ -69,17 +69,17 @@ class CitrusFileTest < Test::Unit::TestCase
     match = grammar.parse('.')
     assert(match)
     assert_kind_of(Rule, match.value)
-    assert_instance_of(Expression, match.value)
+    assert_instance_of(Terminal, match.value)
 
     match = grammar.parse('[a-z]')
     assert(match)
     assert_kind_of(Rule, match.value)
-    assert_instance_of(Expression, match.value)
+    assert_instance_of(Terminal, match.value)
 
     match = grammar.parse('/./')
     assert(match)
     assert_kind_of(Rule, match.value)
-    assert_instance_of(Expression, match.value)
+    assert_instance_of(Terminal, match.value)
 
     match = grammar.parse('/./ /./')
     assert(match)
@@ -98,7 +98,7 @@ class CitrusFileTest < Test::Unit::TestCase
     match = grammar.parse('"" {}')
     assert(match)
     assert_kind_of(Rule, match.value)
-    assert_instance_of(FixedWidth, match.value)
+    assert_instance_of(Terminal, match.value)
 
     match = grammar.parse('""* {}')
     assert(match)
@@ -180,7 +180,7 @@ class CitrusFileTest < Test::Unit::TestCase
     match = grammar.parse("[0-9] {\n  def value\n    text.to_i\n  end\n}\n")
     assert(match)
     assert_kind_of(Rule, match.value)
-    assert_instance_of(Expression, match.value)
+    assert_instance_of(Terminal, match.value)
 
     match = grammar.parse("[0-9]+ {\n  def value\n    text.to_i\n  end\n}\n")
     assert(match)
@@ -297,7 +297,7 @@ class CitrusFileTest < Test::Unit::TestCase
     match = grammar.parse('"a"')
     assert(match)
     assert_kind_of(Rule, match.value)
-    assert_instance_of(FixedWidth, match.value)
+    assert_instance_of(Terminal, match.value)
   end
 
 
@@ -362,25 +362,25 @@ class CitrusFileTest < Test::Unit::TestCase
     match = grammar.parse('"a"')
     assert(match)
     assert_kind_of(Rule, match.value)
-    assert_instance_of(FixedWidth, match.value)
+    assert_instance_of(Terminal, match.value)
     assert(match.value.terminal?)
 
     match = grammar.parse('[a-z]')
     assert(match)
     assert_kind_of(Rule, match.value)
-    assert_instance_of(Expression, match.value)
+    assert_instance_of(Terminal, match.value)
     assert(match.value.terminal?)
 
     match = grammar.parse('.')
     assert(match)
     assert_kind_of(Rule, match.value)
-    assert_instance_of(Expression, match.value)
+    assert_instance_of(Terminal, match.value)
     assert(match.value.terminal?)
 
     match = grammar.parse('/./')
     assert(match)
     assert_kind_of(Rule, match.value)
-    assert_instance_of(Expression, match.value)
+    assert_instance_of(Terminal, match.value)
     assert(match.value.terminal?)
   end
 

data/test/helper.rb

@@ -11,9 +11,7 @@ class Test::Unit::TestCase
     Input.new(str)
   end
 
-  module TestGrammar
-    include Citrus::Grammar
-
+  TestGrammar = Grammar.new do
     rule :alpha do
       /[a-zA-Z]/
     end
@@ -27,6 +25,30 @@ class Test::Unit::TestCase
     end
   end
 
+  Double = Grammar.new do
+    include TestGrammar
+
+    root :double
+
+    rule :double do
+      one_or_more(:num)
+    end
+  end
+
+  Words = Grammar.new do
+    include TestGrammar
+
+    root :words
+
+    rule :word do
+      one_or_more(:alpha)
+    end
+
+    rule :words do
+      [ :word, zero_or_more([ ' ', :word ]) ]
+    end
+  end
+
   class EqualRule
     include Citrus::Rule
 
@@ -34,8 +56,8 @@ class Test::Unit::TestCase
       @value = value
     end
 
-    def match(input, offset=0)
-      create_match(@value.to_s.dup, offset) if @value.to_s == input.string
+    def match(input)
+      create_match(@value.to_s.dup) if @value.to_s == input.string
     end
   end
 

data/test/match_test.rb

@@ -2,25 +2,6 @@ require File.expand_path('../helper', __FILE__)
 
 class MatchTest < Test::Unit::TestCase
 
-  Double = Grammar.new {
-    include MatchTest::TestGrammar
-    root :double
-    rule :double do
-      one_or_more(:num)
-    end
-  }
-
-  Sentence = Grammar.new {
-    include MatchTest::TestGrammar
-    root :sentence
-    rule :word do
-      one_or_more(:alpha)
-    end
-    rule :sentence do
-      [ :word, zero_or_more([ ' ', :word ]) ]
-    end
-  }
-
   def test_string
     match = Match.new('hello')
     assert_equal('hello', match)
@@ -33,20 +14,7 @@ class MatchTest < Test::Unit::TestCase
     match = Match.new([match1, match2])
     assert_equal('ab', match)
     assert_equal(2, match.length)
-  end
-
-  def test_match_data
-    match = Match.new('hello world'.match(/^(\w+) /))
-    assert_equal('hello ', match)
-    assert_equal(6, match.length)
-  end
-
-  def test_array_match_data
-    match1 = Match.new('hello')
-    match2 = Match.new(' world'.match(/.+/))
-    match = Match.new([match1, match2])
-    assert_equal('hello world', match)
-    assert_equal(11, match.length)
+    assert_equal(2, match.matches.length)
   end
 
   def test_equality
@@ -80,9 +48,25 @@ class MatchTest < Test::Unit::TestCase
   end
 
   def test_matches_deep
-    match = Sentence.parse('one two three four')
+    match = Words.parse('one two three four')
     assert(match)
     assert_equal(15, match.find(:alpha).length)
   end
 
+  def test_offset
+    match = Words.parse('one two')
+    assert(match)
+    assert_equal(0, match.offset)
+
+    words = match.find(:word)
+    assert(match)
+    assert_equal(2, words.length)
+
+    assert_equal('one', words[0])
+    assert_equal(0, words[0].offset)
+
+    assert_equal('two', words[1])
+    assert_equal(4, words[1].offset)
+  end
+
 end

data/test/parse_error_test.rb

@@ -0,0 +1,56 @@
+require File.expand_path('../helper', __FILE__)
+
+class ParseErrorTest < Test::Unit::TestCase
+
+  Sentence = Grammar.new do
+    include Words
+
+    rule :sentence do
+      all(:capital_word, one_or_more([ :space, :word ]), :period)
+    end
+
+    rule :capital_word do
+      all(/[A-Z]/, zero_or_more(:alpha))
+    end
+
+    rule :space do
+      one_or_more(any(" ", "\n", "\r\n"))
+    end
+
+    rule :period, '.'
+  end
+
+  def test_basic
+    begin
+      TestGrammar.parse('#')
+    rescue ParseError => e
+      assert_equal(0, e.offset)
+      assert_equal('#', e.line)
+      assert_equal(1, e.line_number)
+      assert_equal(0, e.line_offset)
+    end
+  end
+
+  def test_single_line
+    begin
+      Sentence.parse('Once upon 4 time.')
+    rescue ParseError => e
+      assert_equal(10, e.offset)
+      assert_equal('Once upon 4 time.', e.line)
+      assert_equal(1, e.line_number)
+      assert_equal(10, e.line_offset)
+    end
+  end
+
+  def test_multi_line
+    begin
+      Sentence.parse("Once\nupon a\r\ntim3.")
+    rescue ParseError => e
+      assert_equal(16, e.offset)
+      assert_equal('tim3.', e.line)
+      assert_equal(3, e.line_number)
+      assert_equal(3, e.line_offset)
+    end
+  end
+
+end

data/test/terminal_test.rb

@@ -0,0 +1,56 @@
+require File.expand_path('../helper', __FILE__)
+
+class TerminalTest < Test::Unit::TestCase
+
+  def test_terminal?
+    rule = Terminal.new
+    assert(rule.terminal?)
+  end
+
+  def test_regexp_match
+    rule = Terminal.new(/\d+/)
+    match = rule.match(input('123 456'))
+    assert(match)
+    assert_equal('123', match)
+    assert_equal(3, match.length)
+  end
+
+  def test_regexp_match_failure
+    rule = Terminal.new(/\d+/)
+    match = rule.match(input(' 456'))
+    assert_equal(nil, match)
+  end
+
+  def test_regexp_to_s
+    rule = Terminal.new(/\d+/)
+    assert_equal('/\\d+/', rule.to_s)
+  end
+
+  def test_string_match
+    rule = Terminal.new('abc')
+    match = rule.match(input('abc'))
+    assert(match)
+    assert_equal('abc', match)
+    assert_equal(3, match.length)
+  end
+
+  def test_string_match_short
+    rule = Terminal.new('abc')
+    match = rule.match(input('ab'))
+    assert_equal(nil, match)
+  end
+
+  def test_string_match_long
+    rule = Terminal.new('abc')
+    match = rule.match(input('abcd'))
+    assert(match)
+    assert_equal('abc', match)
+    assert_equal(3, match.length)
+  end
+
+  def test_string_to_s
+    rule = Terminal.new('abc')
+    assert_equal('"abc"', rule.to_s)
+  end
+
+end

metadata

@@ -3,10 +3,10 @@ name: citrus
 version: !ruby/object:Gem::Version 
   prerelease: false
   segments: 
-  - 1
-  - 8
+  - 2
   - 0
-  version: 1.8.0
+  - 0
+  version: 2.0.0
 platform: ruby
 authors: 
 - Michael Jackson
@@ -52,6 +52,9 @@ extensions: []
 extra_rdoc_files: 
 - README
 files: 
+- benchmark/after.dat
+- benchmark/before.dat
+- benchmark/master.dat
 - benchmark/seqpar.citrus
 - benchmark/seqpar.gnuplot
 - benchmark/seqpar.rb
@@ -83,19 +86,19 @@ files:
 - test/calc_file_test.rb
 - test/calc_test.rb
 - test/choice_test.rb
-- test/expression_test.rb
 - test/file_test.rb
-- test/fixed_width_test.rb
 - test/grammar_test.rb
 - test/helper.rb
 - test/label_test.rb
 - test/match_test.rb
 - test/multibyte_test.rb
 - test/not_predicate_test.rb
+- test/parse_error_test.rb
 - test/repeat_test.rb
 - test/rule_test.rb
 - test/sequence_test.rb
 - test/super_test.rb
+- test/terminal_test.rb
 - citrus.gemspec
 - Rakefile
 - README
@@ -143,15 +146,15 @@ test_files:
 - test/calc_file_test.rb
 - test/calc_test.rb
 - test/choice_test.rb
-- test/expression_test.rb
 - test/file_test.rb
-- test/fixed_width_test.rb
 - test/grammar_test.rb
 - test/label_test.rb
 - test/match_test.rb
 - test/multibyte_test.rb
 - test/not_predicate_test.rb
+- test/parse_error_test.rb
 - test/repeat_test.rb
 - test/rule_test.rb
 - test/sequence_test.rb
 - test/super_test.rb
+- test/terminal_test.rb