kpeg 0.10.0 → 1.3.0

data/lib/kpeg/string_escape.rb

@@ -13,12 +13,12 @@ class KPeg::StringEscape
     # Prepares for parsing +str+.  If you define a custom initialize you must
     # call this method before #parse
     def setup_parser(str, debug=false)
-      @string = str
-      @pos = 0
+      set_string str, 0
       @memoizations = Hash.new { |h,k| h[k] = {} }
       @result = nil
       @failed_rule = nil
       @failing_rule_offset = -1
+      @line_offsets = nil
 
       setup_foreign_grammar
     end
@@ -27,7 +27,6 @@ class KPeg::StringEscape
     attr_reader :failing_rule_offset
     attr_accessor :result, :pos
 
-    
     def current_column(target=pos)
       if c = string.rindex("\n", target-1)
         return target - c - 1
@@ -36,17 +35,32 @@ class KPeg::StringEscape
       target + 1
     end
 
-    def current_line(target=pos)
-      cur_offset = 0
-      cur_line = 0
+    if [].respond_to? :bsearch_index
+      def current_line(target=pos)
+        unless @line_offsets
+          @line_offsets = []
+          total = 0
+          string.each_line do |line|
+            total += line.size
+            @line_offsets << total
+          end
+        end
 
-      string.each_line do |line|
-        cur_line += 1
-        cur_offset += line.size
-        return cur_line if cur_offset >= target
+        @line_offsets.bsearch_index {|x| x >= target } + 1 || -1
       end
+    else
+      def current_line(target=pos)
+        cur_offset = 0
+        cur_line = 0
+
+        string.each_line do |line|
+          cur_line += 1
+          cur_offset += line.size
+          return cur_line if cur_offset >= target
+        end
 
-      -1
+        -1
+      end
     end
 
     def lines
@@ -61,6 +75,13 @@ class KPeg::StringEscape
       @string[start..@pos-1]
     end
 
+    # Sets the string and current parsing position for the parser.
+    def set_string string, pos
+      @string = string
+      @string_size = string ? string.size : 0
+      @pos = pos
+    end
+
     def show_pos
       width = 10
       if @pos < width
@@ -158,28 +179,27 @@ class KPeg::StringEscape
     end
 
     def scan(reg)
-      if m = reg.match(@string[@pos..-1])
-        width = m.end(0)
-        @pos += width
+      if m = reg.match(@string, @pos)
+        @pos = m.end(0)
         return true
       end
 
       return nil
     end
 
-    if "".respond_to? :getbyte
+    if "".respond_to? :ord
       def get_byte
-        if @pos >= @string.size
+        if @pos >= @string_size
           return nil
         end
 
-        s = @string.getbyte @pos
+        s = @string[@pos].ord
         @pos += 1
         s
       end
     else
       def get_byte
-        if @pos >= @string.size
+        if @pos >= @string_size
           return nil
         end
 
@@ -228,8 +248,7 @@ class KPeg::StringEscape
       old_pos = @pos
       old_string = @string
 
-      @pos = other.pos
-      @string = other.string
+      set_string other.string, other.pos
 
       begin
         if val = __send__(rule, *args)
@@ -240,8 +259,7 @@ class KPeg::StringEscape
         end
         val
       ensure
-        @pos = old_pos
-        @string = old_string
+        set_string old_string, old_pos
       end
     end
 
@@ -275,8 +293,6 @@ class KPeg::StringEscape
         else
           return ans
         end
-
-        return ans
       end
     end
 
@@ -309,8 +325,6 @@ class KPeg::StringEscape
         else
           return ans
         end
-
-        return ans
       end
     end
 
@@ -359,7 +373,7 @@ class KPeg::StringEscape
   # :stopdoc:
   def setup_foreign_grammar; end
 
-  # segment = (< /[\w ]+/ > { text } | "\\" { "\\\\" } | "\n" { "\\n" } | "\t" { "\\t" } | "\b" { "\\b" } | "\"" { "\\\"" } | < . > { text })
+  # segment = (< /[\w ]+/ > { text } | "\\" { "\\\\" } | "\n" { "\\n" } | "\r" { "\\r" } | "\t" { "\\t" } | "\b" { "\\b" } | "\"" { "\\\"" } | < . > { text })
   def _segment
 
     _save = self.pos
@@ -368,7 +382,7 @@ class KPeg::StringEscape
       _save1 = self.pos
       while true # sequence
         _text_start = self.pos
-        _tmp = scan(/\A(?-mix:[\w ]+)/)
+        _tmp = scan(/\G(?-mix:[\w ]+)/)
         if _tmp
           text = get_text(_text_start)
         end
@@ -425,12 +439,12 @@ class KPeg::StringEscape
 
       _save4 = self.pos
       while true # sequence
-        _tmp = match_string("\t")
+        _tmp = match_string("\r")
         unless _tmp
           self.pos = _save4
           break
         end
-        @result = begin;  "\\t" ; end
+        @result = begin;  "\\r" ; end
         _tmp = true
         unless _tmp
           self.pos = _save4
@@ -443,12 +457,12 @@ class KPeg::StringEscape
 
       _save5 = self.pos
       while true # sequence
-        _tmp = match_string("\b")
+        _tmp = match_string("\t")
         unless _tmp
           self.pos = _save5
           break
         end
-        @result = begin;  "\\b" ; end
+        @result = begin;  "\\t" ; end
         _tmp = true
         unless _tmp
           self.pos = _save5
@@ -461,12 +475,12 @@ class KPeg::StringEscape
 
       _save6 = self.pos
       while true # sequence
-        _tmp = match_string("\"")
+        _tmp = match_string("\b")
         unless _tmp
           self.pos = _save6
           break
         end
-        @result = begin;  "\\\"" ; end
+        @result = begin;  "\\b" ; end
         _tmp = true
         unless _tmp
           self.pos = _save6
@@ -478,6 +492,24 @@ class KPeg::StringEscape
       self.pos = _save
 
       _save7 = self.pos
+      while true # sequence
+        _tmp = match_string("\"")
+        unless _tmp
+          self.pos = _save7
+          break
+        end
+        @result = begin;  "\\\"" ; end
+        _tmp = true
+        unless _tmp
+          self.pos = _save7
+        end
+        break
+      end # end sequence
+
+      break if _tmp
+      self.pos = _save
+
+      _save8 = self.pos
       while true # sequence
         _text_start = self.pos
         _tmp = get_byte
@@ -485,13 +517,13 @@ class KPeg::StringEscape
           text = get_text(_text_start)
         end
         unless _tmp
-          self.pos = _save7
+          self.pos = _save8
           break
         end
         @result = begin;  text ; end
         _tmp = true
         unless _tmp
-          self.pos = _save7
+          self.pos = _save8
         end
         break
       end # end sequence
@@ -599,7 +631,7 @@ class KPeg::StringEscape
   end
 
   Rules = {}
-  Rules[:_segment] = rule_info("segment", "(< /[\\w ]+/ > { text } | \"\\\\\" { \"\\\\\\\\\" } | \"\\n\" { \"\\\\n\" } | \"\\t\" { \"\\\\t\" } | \"\\b\" { \"\\\\b\" } | \"\\\"\" { \"\\\\\\\"\" } | < . > { text })")
+  Rules[:_segment] = rule_info("segment", "(< /[\\w ]+/ > { text } | \"\\\\\" { \"\\\\\\\\\" } | \"\\n\" { \"\\\\n\" } | \"\\r\" { \"\\\\r\" } | \"\\t\" { \"\\\\t\" } | \"\\b\" { \"\\\\b\" } | \"\\\"\" { \"\\\\\\\"\" } | < . > { text })")
   Rules[:_root] = rule_info("root", "segment*:s { @text = s.join }")
   Rules[:_embed_seg] = rule_info("embed_seg", "(\"\#\" { \"\\\\\#\" } | segment)")
   Rules[:_embed] = rule_info("embed", "embed_seg*:s { @text = s.join }")

data/lib/kpeg.rb

@@ -1,6 +1,6 @@
 module KPeg
 
-  VERSION = "0.10.0"
+  VERSION = "1.3.0"
 
   def self.grammar
     g = Grammar.new

data/test/test_kpeg.rb

@@ -2,7 +2,7 @@ require 'minitest/autorun'
 require 'kpeg'
 require 'stringio'
 
-class TestKPeg < MiniTest::Unit::TestCase
+class TestKPeg < Minitest::Test
   def assert_match(m, str)
     assert_kind_of KPeg::MatchString, m
     assert_equal str, m.string

data/test/test_kpeg_code_generator.rb

@@ -4,7 +4,7 @@ require 'kpeg'
 require 'kpeg/code_generator'
 require 'stringio'
 
-class TestKPegCodeGenerator < MiniTest::Unit::TestCase
+class TestKPegCodeGenerator < Minitest::Test
   def test_dot
     gram = KPeg.grammar do |g|
       g.root = g.dot
@@ -80,7 +80,7 @@ class Test < KPeg::CompiledParser
 
   # root = /[0-9]/
   def _root
-    _tmp = scan(/\\A(?-mix:[0-9])/)
+    _tmp = scan(/\\G(?-mix:[0-9])/)
     set_failed_rule :_root unless _tmp
     return _tmp
   end
@@ -114,7 +114,7 @@ class Test < KPeg::CompiledParser
 
   # root = /./
   def _root
-    _tmp = scan(/\\A(?-mix:.)/)
+    _tmp = scan(/\\G(?-mix:.)/)
     set_failed_rule :_root unless _tmp
     return _tmp
   end
@@ -133,7 +133,7 @@ class Test < KPeg::CompiledParser
 
   # root = /./u
   def _root
-    _tmp = scan(/\\A(?-mix:.)/u)
+    _tmp = scan(/\\G(?-mix:.)/u)
     set_failed_rule :_root unless _tmp
     return _tmp
   end
@@ -1661,15 +1661,18 @@ class Test < KPeg::CompiledParser
       end
     end
   end
-  def bracket(receiver, argument)
-    AST::BracketOperator.new(receiver, argument)
-  end
-  def simple()
-    AST::Simple.new()
-  end
-  def simple2()
-    AST::Simple2.new()
+  module ASTConstruction
+    def bracket(receiver, argument)
+      AST::BracketOperator.new(receiver, argument)
+    end
+    def simple()
+      AST::Simple.new()
+    end
+    def simple2()
+      AST::Simple2.new()
+    end
   end
+  include ASTConstruction
 
   # root = .
   def _root
@@ -1715,9 +1718,12 @@ class Test < KPeg::CompiledParser
       attr_reader :argument
     end
   end
-  def bracket(receiver, argument)
-    MegaAST::BracketOperator.new(receiver, argument)
+  module MegaASTConstruction
+    def bracket(receiver, argument)
+      MegaAST::BracketOperator.new(receiver, argument)
+    end
   end
+  include MegaASTConstruction
 
   # root = .
   def _root

data/test/test_kpeg_compiled_parser.rb

@@ -3,10 +3,11 @@ require 'kpeg'
 require 'kpeg/compiled_parser'
 require 'stringio'
 
-class TestKPegCompiledParser < MiniTest::Unit::TestCase
+class TestKPegCompiledParser < Minitest::Test
 
   gram = <<-GRAM
   letter = [a-z]
+  number = [0-9]
   root = letter
   GRAM
 
@@ -14,7 +15,7 @@ class TestKPegCompiledParser < MiniTest::Unit::TestCase
 
   gram = <<-GRAM
   %test = TestKPegCompiledParser::TestParser
-  root = %test.letter "!"
+  root = %test.letter %test.number? "!"
   GRAM
 
   KPeg.compile gram, "CompTestParser", self
@@ -78,4 +79,12 @@ class TestKPegCompiledParser < MiniTest::Unit::TestCase
     assert_equal expected, r.failure_oneline
   end
 
+  def test_composite_two_char_error
+    r = CompTestParser.new "aa"
+    assert_nil r.parse, "should not parse"
+
+    expected = "@1:2 failed rule 'TestKPegCompiledParser::TestParser#_number', got 'a'"
+    assert_equal expected, r.failure_oneline
+  end
+
 end

data/test/test_kpeg_format.rb

@@ -5,7 +5,7 @@ require 'kpeg/grammar_renderer'
 require 'stringio'
 require 'rubygems'
 
-class TestKPegFormat < MiniTest::Unit::TestCase
+class TestKPegFormat < Minitest::Test
   G = KPeg::Grammar.new
 
   gram = File.read File.expand_path("../../lib/kpeg/format_parser.kpeg", __FILE__)
@@ -157,9 +157,9 @@ b(p) = x
   end
 
   def test_regexp
-    assert_rule G.reg(/foo/), match('a=/foo/')
-    assert_rule G.reg(/foo\/bar/), match('a=/foo\/bar/')
-    assert_rule G.reg(/[^"]/), match('a=/[^"]/')
+    assert_rule G.reg('foo'), match('a=/foo/')
+    assert_rule G.reg('foo\\/bar'), match('a=/foo\/bar/')
+    assert_rule G.reg('[^"]'), match('a=/[^"]/')
   end
 
   def test_regexp_options
@@ -431,6 +431,25 @@ a=b
     assert_equal expected, m.directives
   end
 
+  def test_parser_directive_single_quote
+    m = match <<-GRAMMAR
+%% header {
+# It's a bug I found
+}
+
+a=b
+    GRAMMAR
+
+    assert_rule G.ref("b"), m
+
+    expected = {
+      "header" => KPeg::Action.new("\n# It's a bug I found\n")
+    }
+
+    assert_equal expected, m.directives
+  end
+  
+
   def test_parser_setup
     m = match "%% { def initialize; end }\na=b"
     assert_rule G.ref("b"), m

data/test/test_kpeg_format_parser_round_trip.rb

@@ -5,7 +5,7 @@ require 'kpeg/grammar_renderer'
 require 'kpeg/code_generator'
 require 'stringio'
 
-class TestKPegFormatParserRoundtrip < MiniTest::Unit::TestCase
+class TestKPegFormatParserRoundtrip < Minitest::Test
   PATH = File.expand_path("../../lib/kpeg/format_parser.kpeg", __FILE__)
   def test_roundtrip
     data = File.read(PATH)

data/test/test_kpeg_grammar.rb

@@ -4,7 +4,7 @@ require 'kpeg/format_parser'
 require 'kpeg/code_generator'
 require 'stringio'
 
-class TestKpegGrammar < MiniTest::Unit::TestCase
+class TestKpegGrammar < Minitest::Test
   LEFT_RECURSION = <<-'STR'
 
   name = name:n "[]" { [:array, n] }

data/test/test_kpeg_grammar_renderer.rb

@@ -3,7 +3,7 @@ require 'kpeg'
 require 'kpeg/grammar_renderer'
 require 'stringio'
 
-class TestKPegGrammarRenderer < MiniTest::Unit::TestCase
+class TestKPegGrammarRenderer < Minitest::Test
   def test_escape
     str = "hello\nbob"
     assert_equal 'hello\nbob', KPeg::GrammarRenderer.escape(str)

data/test/test_kpeg_string_escape.rb

@@ -0,0 +1,42 @@
+require 'minitest/autorun'
+require 'kpeg'
+require 'kpeg/string_escape'
+
+class TestKPegStringEscape < Minitest::Test
+
+  def test_bell
+    assert_equal '\b', parse("\b")
+  end
+
+  def test_carriage_return
+    assert_equal '\r', parse("\r")
+  end
+
+  def test_newline
+    assert_equal '\n', parse("\n")
+  end
+
+  def test_quote
+    assert_equal '\\\\\"', parse('\\"')
+  end
+
+  def test_slash
+    assert_equal '\\\\', parse('\\')
+  end
+
+  def test_tab
+    assert_equal '\t', parse("\t")
+  end
+
+  def parse(str, embed = false)
+    se = KPeg::StringEscape.new(str)
+
+    rule = (embed ? 'embed' : nil)
+
+    se.raise_error unless se.parse(rule)
+
+    se.text
+  end
+
+end
+