kpeg 0.9.0 → 1.2.0

data/lib/kpeg/string_escape.rb

@@ -1,13 +1,380 @@
-require 'kpeg/compiled_parser'
+class KPeg::StringEscape
+  # :stopdoc:
+
+    # This is distinct from setup_parser so that a standalone parser
+    # can redefine #initialize and still have access to the proper
+    # parser setup code.
+    def initialize(str, debug=false)
+      setup_parser(str, debug)
+    end
+
+
+
+    # Prepares for parsing +str+.  If you define a custom initialize you must
+    # call this method before #parse
+    def setup_parser(str, debug=false)
+      set_string str, 0
+      @memoizations = Hash.new { |h,k| h[k] = {} }
+      @result = nil
+      @failed_rule = nil
+      @failing_rule_offset = -1
+      @line_offsets = nil
+
+      setup_foreign_grammar
+    end
+
+    attr_reader :string
+    attr_reader :failing_rule_offset
+    attr_accessor :result, :pos
+
+    def current_column(target=pos)
+      if c = string.rindex("\n", target-1)
+        return target - c - 1
+      end
+
+      target + 1
+    end
+
+    if [].respond_to? :bsearch_index
+      def current_line(target=pos)
+        unless @line_offsets
+          @line_offsets = [-1]
+          total = 0
+          string.each_line do |line|
+            @line_offsets << total
+            total += line.size
+          end
+          @line_offsets << total
+        end
+
+        @line_offsets.bsearch_index {|x| x >= target } || -1
+      end
+    else
+      def current_line(target=pos)
+        cur_offset = 0
+        cur_line = 0
+
+        string.each_line do |line|
+          cur_line += 1
+          cur_offset += line.size
+          return cur_line if cur_offset >= target
+        end
+
+        -1
+      end
+    end
+
+    def lines
+      lines = []
+      string.each_line { |l| lines << l }
+      lines
+    end
+
+
+
+    def get_text(start)
+      @string[start..@pos-1]
+    end
+
+    # Sets the string and current parsing position for the parser.
+    def set_string string, pos
+      @string = string
+      @string_size = string ? string.size : 0
+      @pos = pos
+    end
+
+    def show_pos
+      width = 10
+      if @pos < width
+        "#{@pos} (\"#{@string[0,@pos]}\" @ \"#{@string[@pos,width]}\")"
+      else
+        "#{@pos} (\"... #{@string[@pos - width, width]}\" @ \"#{@string[@pos,width]}\")"
+      end
+    end
+
+    def failure_info
+      l = current_line @failing_rule_offset
+      c = current_column @failing_rule_offset
+
+      if @failed_rule.kind_of? Symbol
+        info = self.class::Rules[@failed_rule]
+        "line #{l}, column #{c}: failed rule '#{info.name}' = '#{info.rendered}'"
+      else
+        "line #{l}, column #{c}: failed rule '#{@failed_rule}'"
+      end
+    end
+
+    def failure_caret
+      l = current_line @failing_rule_offset
+      c = current_column @failing_rule_offset
+
+      line = lines[l-1]
+      "#{line}\n#{' ' * (c - 1)}^"
+    end
+
+    def failure_character
+      l = current_line @failing_rule_offset
+      c = current_column @failing_rule_offset
+      lines[l-1][c-1, 1]
+    end
+
+    def failure_oneline
+      l = current_line @failing_rule_offset
+      c = current_column @failing_rule_offset
+
+      char = lines[l-1][c-1, 1]
+
+      if @failed_rule.kind_of? Symbol
+        info = self.class::Rules[@failed_rule]
+        "@#{l}:#{c} failed rule '#{info.name}', got '#{char}'"
+      else
+        "@#{l}:#{c} failed rule '#{@failed_rule}', got '#{char}'"
+      end
+    end
+
+    class ParseError < RuntimeError
+    end
+
+    def raise_error
+      raise ParseError, failure_oneline
+    end
+
+    def show_error(io=STDOUT)
+      error_pos = @failing_rule_offset
+      line_no = current_line(error_pos)
+      col_no = current_column(error_pos)
+
+      io.puts "On line #{line_no}, column #{col_no}:"
+
+      if @failed_rule.kind_of? Symbol
+        info = self.class::Rules[@failed_rule]
+        io.puts "Failed to match '#{info.rendered}' (rule '#{info.name}')"
+      else
+        io.puts "Failed to match rule '#{@failed_rule}'"
+      end
+
+      io.puts "Got: #{string[error_pos,1].inspect}"
+      line = lines[line_no-1]
+      io.puts "=> #{line}"
+      io.print(" " * (col_no + 3))
+      io.puts "^"
+    end
+
+    def set_failed_rule(name)
+      if @pos > @failing_rule_offset
+        @failed_rule = name
+        @failing_rule_offset = @pos
+      end
+    end
+
+    attr_reader :failed_rule
+
+    def match_string(str)
+      len = str.size
+      if @string[pos,len] == str
+        @pos += len
+        return str
+      end
+
+      return nil
+    end
+
+    def scan(reg)
+      if m = reg.match(@string, @pos)
+        @pos = m.end(0)
+        return true
+      end
+
+      return nil
+    end
+
+    if "".respond_to? :ord
+      def get_byte
+        if @pos >= @string_size
+          return nil
+        end
+
+        s = @string[@pos].ord
+        @pos += 1
+        s
+      end
+    else
+      def get_byte
+        if @pos >= @string_size
+          return nil
+        end
+
+        s = @string[@pos]
+        @pos += 1
+        s
+      end
+    end
+
+    def parse(rule=nil)
+      # We invoke the rules indirectly via apply
+      # instead of by just calling them as methods because
+      # if the rules use left recursion, apply needs to
+      # manage that.
+
+      if !rule
+        apply(:_root)
+      else
+        method = rule.gsub("-","_hyphen_")
+        apply :"_#{method}"
+      end
+    end
+
+    class MemoEntry
+      def initialize(ans, pos)
+        @ans = ans
+        @pos = pos
+        @result = nil
+        @set = false
+        @left_rec = false
+      end
+
+      attr_reader :ans, :pos, :result, :set
+      attr_accessor :left_rec
+
+      def move!(ans, pos, result)
+        @ans = ans
+        @pos = pos
+        @result = result
+        @set = true
+        @left_rec = false
+      end
+    end
+
+    def external_invoke(other, rule, *args)
+      old_pos = @pos
+      old_string = @string
+
+      set_string other.string, other.pos
+
+      begin
+        if val = __send__(rule, *args)
+          other.pos = @pos
+          other.result = @result
+        else
+          other.set_failed_rule "#{self.class}##{rule}"
+        end
+        val
+      ensure
+        set_string old_string, old_pos
+      end
+    end
+
+    def apply_with_args(rule, *args)
+      memo_key = [rule, args]
+      if m = @memoizations[memo_key][@pos]
+        @pos = m.pos
+        if !m.set
+          m.left_rec = true
+          return nil
+        end
+
+        @result = m.result
 
-class KPeg::StringEscape < KPeg::CompiledParser
+        return m.ans
+      else
+        m = MemoEntry.new(nil, @pos)
+        @memoizations[memo_key][@pos] = m
+        start_pos = @pos
+
+        ans = __send__ rule, *args
+
+        lr = m.left_rec
+
+        m.move! ans, @pos, @result
+
+        # Don't bother trying to grow the left recursion
+        # if it's failing straight away (thus there is no seed)
+        if ans and lr
+          return grow_lr(rule, args, start_pos, m)
+        else
+          return ans
+        end
+      end
+    end
+
+    def apply(rule)
+      if m = @memoizations[rule][@pos]
+        @pos = m.pos
+        if !m.set
+          m.left_rec = true
+          return nil
+        end
+
+        @result = m.result
+
+        return m.ans
+      else
+        m = MemoEntry.new(nil, @pos)
+        @memoizations[rule][@pos] = m
+        start_pos = @pos
+
+        ans = __send__ rule
+
+        lr = m.left_rec
+
+        m.move! ans, @pos, @result
+
+        # Don't bother trying to grow the left recursion
+        # if it's failing straight away (thus there is no seed)
+        if ans and lr
+          return grow_lr(rule, nil, start_pos, m)
+        else
+          return ans
+        end
+      end
+    end
+
+    def grow_lr(rule, args, start_pos, m)
+      while true
+        @pos = start_pos
+        @result = m.result
+
+        if args
+          ans = __send__ rule, *args
+        else
+          ans = __send__ rule
+        end
+        return nil unless ans
+
+        break if @pos <= m.pos
+
+        m.move! ans, @pos, @result
+      end
+
+      @result = m.result
+      @pos = m.pos
+      return m.ans
+    end
+
+    class RuleInfo
+      def initialize(name, rendered)
+        @name = name
+        @rendered = rendered
+      end
+
+      attr_reader :name, :rendered
+    end
+
+    def self.rule_info(name, rendered)
+      RuleInfo.new(name, rendered)
+    end
+
+
+  # :startdoc:
 
 
   attr_reader :text
 
 
+  # :stopdoc:
+  def setup_foreign_grammar; end
 
-  # segment = (< /[\w ]+/ > { text } | "\\" { "\\\\" } | "\n" { "\\n" } | "\t" { "\\t" } | "\b" { "\\b" } | "\"" { "\\\"" } | < . > { text })
+  # segment = (< /[\w ]+/ > { text } | "\\" { "\\\\" } | "\n" { "\\n" } | "\r" { "\\r" } | "\t" { "\\t" } | "\b" { "\\b" } | "\"" { "\\\"" } | < . > { text })
   def _segment
 
     _save = self.pos
@@ -16,7 +383,7 @@ class KPeg::StringEscape < KPeg::CompiledParser
       _save1 = self.pos
       while true # sequence
         _text_start = self.pos
-        _tmp = scan(/\A(?-mix:[\w ]+)/)
+        _tmp = scan(/\G(?-mix:[\w ]+)/)
         if _tmp
           text = get_text(_text_start)
         end
@@ -73,12 +440,12 @@ class KPeg::StringEscape < KPeg::CompiledParser
 
       _save4 = self.pos
       while true # sequence
-        _tmp = match_string("\t")
+        _tmp = match_string("\r")
         unless _tmp
           self.pos = _save4
           break
         end
-        @result = begin;  "\\t" ; end
+        @result = begin;  "\\r" ; end
         _tmp = true
         unless _tmp
           self.pos = _save4
@@ -91,12 +458,12 @@ class KPeg::StringEscape < KPeg::CompiledParser
 
       _save5 = self.pos
       while true # sequence
-        _tmp = match_string("\b")
+        _tmp = match_string("\t")
         unless _tmp
           self.pos = _save5
           break
         end
-        @result = begin;  "\\b" ; end
+        @result = begin;  "\\t" ; end
         _tmp = true
         unless _tmp
           self.pos = _save5
@@ -109,12 +476,12 @@ class KPeg::StringEscape < KPeg::CompiledParser
 
       _save6 = self.pos
       while true # sequence
-        _tmp = match_string("\"")
+        _tmp = match_string("\b")
         unless _tmp
           self.pos = _save6
           break
         end
-        @result = begin;  "\\\"" ; end
+        @result = begin;  "\\b" ; end
         _tmp = true
         unless _tmp
           self.pos = _save6
@@ -126,6 +493,24 @@ class KPeg::StringEscape < KPeg::CompiledParser
       self.pos = _save
 
       _save7 = self.pos
+      while true # sequence
+        _tmp = match_string("\"")
+        unless _tmp
+          self.pos = _save7
+          break
+        end
+        @result = begin;  "\\\"" ; end
+        _tmp = true
+        unless _tmp
+          self.pos = _save7
+        end
+        break
+      end # end sequence
+
+      break if _tmp
+      self.pos = _save
+
+      _save8 = self.pos
       while true # sequence
         _text_start = self.pos
         _tmp = get_byte
@@ -133,13 +518,13 @@ class KPeg::StringEscape < KPeg::CompiledParser
           text = get_text(_text_start)
         end
         unless _tmp
-          self.pos = _save7
+          self.pos = _save8
           break
         end
         @result = begin;  text ; end
         _tmp = true
         unless _tmp
-          self.pos = _save7
+          self.pos = _save8
         end
         break
       end # end sequence
@@ -247,8 +632,9 @@ class KPeg::StringEscape < KPeg::CompiledParser
   end
 
   Rules = {}
-  Rules[:_segment] = rule_info("segment", "(< /[\\w ]+/ > { text } | \"\\\\\" { \"\\\\\\\\\" } | \"\\n\" { \"\\\\n\" } | \"\\t\" { \"\\\\t\" } | \"\\b\" { \"\\\\b\" } | \"\\\"\" { \"\\\\\\\"\" } | < . > { text })")
+  Rules[:_segment] = rule_info("segment", "(< /[\\w ]+/ > { text } | \"\\\\\" { \"\\\\\\\\\" } | \"\\n\" { \"\\\\n\" } | \"\\r\" { \"\\\\r\" } | \"\\t\" { \"\\\\t\" } | \"\\b\" { \"\\\\b\" } | \"\\\"\" { \"\\\\\\\"\" } | < . > { text })")
   Rules[:_root] = rule_info("root", "segment*:s { @text = s.join }")
   Rules[:_embed_seg] = rule_info("embed_seg", "(\"\#\" { \"\\\\\#\" } | segment)")
   Rules[:_embed] = rule_info("embed", "embed_seg*:s { @text = s.join }")
+  # :startdoc:
 end

data/lib/kpeg.rb

@@ -1,6 +1,6 @@
 module KPeg
 
-  VERSION = "0.9.0"
+  VERSION = "1.2.0"
 
   def self.grammar
     g = Grammar.new

data/test/test_kpeg.rb

@@ -2,7 +2,7 @@ require 'minitest/autorun'
 require 'kpeg'
 require 'stringio'
 
-class TestKPeg < MiniTest::Unit::TestCase
+class TestKPeg < Minitest::Test
   def assert_match(m, str)
     assert_kind_of KPeg::MatchString, m
     assert_equal str, m.string