depager 0.1.9 → 0.2.0

data/lib/depager/lr.rb

@@ -1,31 +1,32 @@
-module Depager
-  class G
-    attr_accessor :rulelist, :gh, :syms, :nonterms, :terms, :closure, :ismap, :nssize, :tla
-    attr_accessor :precs, :epr, :epr2, :f0e, :first1
-    Eps = 0x3fffffff #epsilon
-    #Eps = []
-    def initialize rulelist, ts, ns, precs = nil
-      @nssize = ns.size
-      @precs = precs
-      @syms = ns.invert.merge(ts.invert)
-      @tla = @syms.size
-
-      @rulelist = rulelist
-      @gh = []; @nssize.times{|i| @gh[i] = []}
-      @epr = {}
-      @f0e = {}
-      @rulelist.each_with_index do |rule, rx|
-        rule.g = self
-        rule.n = rx
-        @gh[rule.l].push rule
-        if rule.r.empty?
-          @epr[rule.l] = rule.n 
-          @f0e[rule.l] = { rule.n => [Eps] }
-        end
-      end
-      make_sym_first
+require 'pp'
+require 'depager/grammar.rb'
+module Depager::LALR
+  class SingleParserGenerator < Simple::SingleGenerator
+    Tmplfile = 'single_lalr_parser.erb'
+    def initialize d_parser
+      super
+      @parsing_method = LALR
+      @basis_name = @d_parser.target_name + '.new()'
+    end
+  end
+
+  class ExtensionParserGenerator < Simple::ExtensionGenerator
+    TmplfileMaster = "#{Tmpldir}/extension_lalr_master.erb"
+    TmplfileSlave  = "#{Tmpldir}/extension_lalr_slave.erb"
+    def initialize d_parser
+      super
+      @parsing_method = LALR
+      @basis_name = 'self'
+    end
+  end
+
+  Eps = Depager::ParsingMethodCommon::G::Eps
+  class Grammar < Depager::ParsingMethodCommon::G
+    attr_accessor :closure, :memo_closure1
+    def initialize_depend
       make_sym_f0e
       @closure = {}
+      @memo_closure1 = {}
     end
 
     def make_sym_f0e
@@ -60,127 +61,46 @@ module Depager
         end
       end while cnt
     end
-
-    def make_sym_first
-      @first1 = {@tla => [@tla]}
-      @syms.each_key do |s|
-        if terms? s
-          @first1[s] = [s]
-        else
-          @first1[s] = []        
-          @first1[s].push Eps if @epr[s]
-        end
-      end
-      begin
-        cnt = false
-        @syms.each_key do |s|
-          @gh[s].each do |rule|
-            rule.r.each do |es|
-              ss = @first1[es] - @first1[s]
-              unless ss.empty?
-                @first1[s] |= ss
-                cnt = true
-              end
-              break unless @first1[es].include? Eps
-            end
-            unless @first1[s].include? Eps
-              if rule.r.all?{|i| @first1[i].include? Eps }
-                @first1.push Eps
-                cnt = true
-              end
-            end
-          end if nonterms? s
-        end
-      end while cnt
-    end
-
-    def first p
-      r = []
-      x = 0
-      p.each do |i|
-        if i == Eps
-        elsif i.class == Array
-          i.each do |j|
-            r |= @first1[j]
-          end
-          break unless i.any?{|j| @epr[j] || j==Eps }
-        else
-          r |= @first1[i]
-          break unless @epr[i]
-        end
-        x += 1
-      end
-      r -= [Eps]
-      r |= [Eps] if x == p.size
-      return r
-    end
-
-    def terms? i
-      i && i >= @nssize
-    end
-
-    def nonterms? i
-      i && i < @nssize
-    end
-
-    def [](n)
-      @rulelist[n]
-    end
   end
-
-  class Rule
-    attr_accessor :l, :r, :g, :n, :act, :epr, :prec
-    def initialize l, r, g = nil, n = nil, act = nil, prec = nil
-      @l = l
-      @r = r
-      @g = g
-      @n = n
-      @act = act
-      @epr = []
-      @prec = prec
-    end
-    def self.[] l, r, prec = nil
-      Rule.new l, r, nil, nil, nil, prec
-    end
-
-    def to_s
-      r = @r.map{|i| @g.syms[i]}.join(' ')    
-      "(#{'%03s'%@n}) #{@g.syms[@l]} : #{r}"
+  class Rule <  Depager::ParsingMethodCommon::Rule
+    def initialize_depend
     end
-
-    def hash
-      (@l.hash % 37) * (@r.hash % 37)
-    end
-    def eql? i
-      @l == i.l && @r == i.r
-    end
-    alias == eql?
-
     def closure1 n, la
-      i = 0
-      j = [ LRItem[self, n, la] ]
-      h = {j[0] => j.size-1}
+      key = [self.n, n, la]
+      return @g.memo_closure1[key] if @g.memo_closure1[key]
+      _G = @g
+      _I = [ LRItem[self, n, la] ]
+      memo = { _I[0].rule.n => _I[0] }
+      _I.each do |_IA|
+        _B, beta = _IA.dotsym, _IA.dotrest
+        next if !_B or !_G.gh[_B]
+
+        b = _G.first(beta)
+        if b.include? Eps
+          b.delete(Eps)
+          b |= _IA.la 
+        end
 
-      while i < j.size
-        ji = j[i]
-        b = ji.rule.r[(ji.n+1) .. ji.rule.r.size]
-        b = [] unless b
-
-        if (x = ji.rule.r[ji.n]) && (a = @g.gh[x])
-          a.each do |rule|
-            li = LRItem[rule, 0]
-            if x = h[li]
-              j[x].la.concat @g.first(b + [ji.la])
-            else
-              li.la = @g.first(b + [ji.la])
-              j << li
-              h[li] = j.size-1
+        _G.gh[_B].each do |_Brule|
+          if li = memo[_Brule.n]
+            #warn "= #{li} ;"+
+            #     "   [#{(beta+[a]).map{|i| @g.syms[i] || '$' }}]"+
+            #     " :: #{b.map{|i| @g.syms[i] }}"
+            s0 = li.la.size
+            li.la |= b
+            if li.la.size > s0 # (b - li.la).empty?
+              _I << li
+              # warn "! #{b.map{|i| @g.syms[i]}}; #{li}"
             end
+          else
+            li = memo[_Brule.n] = LRItem[_Brule, 0, b]
+            _I << li
+            # warn "+ #{memo[_Brule.n]}"
           end
         end
-        i += 1
       end
-      j
+      @g.memo_closure1[key] = _I
+      _I
     end
 
     def closure n, base=nil
@@ -249,6 +169,12 @@ module Depager
         @goto[x] = I[]
       end
     end
+    def dotsym
+      @rule.r[@n]
+    end
+    def dotrest
+      @rule.r[@n+1 .. -1]
+    end
   end
 
   class I
@@ -309,13 +235,14 @@ module Depager
     end
   end
 
-  class LALRTable
+  class Table
     attr_accessor :g, :action_table, :goto_table
     attr_accessor :defred_table, :defred_after_shift_table
     def initialize g
       @g = g
       @c = nil
-      
+      @warnings = []
+
       items
       mkset
       mktable
@@ -378,7 +305,7 @@ module Depager
     def checkconf newv, oldv, key, g = nil
       return newv unless oldv
       if oldv == 'ACC'
-        warn "'ACC' conflict #{g}."
+        @warnings << "'ACC' conflict #{g}."
         return oldv
       end
       # warn "\n-- N:#{newv} O:#{oldv} K:#{@g.syms[key]} "
@@ -392,7 +319,7 @@ module Depager
           # warn "+:reduce #{@g[-newv]}"
           newv
         else
-          warn "shift/reduce conflict #{@g.syms[key]} #{g}."
+          @warnings << "shift/reduce conflict '#{@g.syms[key]}'\n#{g}."
           oldv
         end
       elsif newv > 0 and oldv < 0
@@ -404,7 +331,7 @@ module Depager
           # warn "reduce"
           oldv
         else
-          warn "shift/reduce conflict #{g}."
+          @warnings << "shift/reduce conflict\n#{g}."
           newv
         end
       elsif newv < 0 and oldv < 0
@@ -412,7 +339,7 @@ module Depager
           warn "?? newv == oldv == #{newv}."
           newv
         else
-          warn "reduce/reduce conflict #{g}."
+          @warnings << "reduce/reduce conflict\n#{g}."
           newv
         end
       else
@@ -494,6 +421,10 @@ module Depager
       put_table if $DEBUG
     end
 
+    def check_table dp
+      @warnings.each{|i| dp.warning i }
+    end
+    
     def put_table
       syms, first1, f0e, rulelist = @g.syms, @g.first1, @g.f0e, @g.rulelist
 
@@ -501,14 +432,14 @@ module Depager
 
       f_ = first1.map {|k, v|
         "#{syms[k]} => " << v.map{|i|
-          i == G::Eps ? ':eps' : syms[i]
+          i == Eps ? ':eps' : syms[i]
         }.join(' ')
       }.join("\n")
 
       e = f0e.map {|k, v| 
         "#{syms[k].inspect} => " << v.map{|n, f|
           "  #{rulelist[n]} ? " << f.map{|i|
-            i == G::Eps ? ':eps' : syms[i].inspect
+            i == Eps ? ':eps' : syms[i].inspect
           }.join(' ')
         }.join("\n")
       }.join("\n")
@@ -546,10 +477,16 @@ module Depager
         g << "\n"
       }
 
+      gh = ""
+      @g.gh.each_with_index{|i, x|
+        gh << "#{syms[x]}(#{x}) => #{i}\n"
+      }
+
       warn "** SYMS **\n\n#{s}\n\n" <<
         "** FIRST1 **\n\n#{f_}\n\n" <<
         "** Eps Reduce Rule **\n\n#{e}\n\n" <<
         "*** Grammar ***\n\n#{r}\n\n" <<
+        "** GH **\n\n#{gh}\n\n" <<
         "*** States ***\n\n#{c}\n\n" <<
         "*** Action Table ***\n\n#{a}\n" <<
         "*** Goto Table ***\n\n#{g}\n"
@@ -558,39 +495,36 @@ module Depager
     def mkset
       warn '** cal LA **' if $MP_DEBUG
       trn = []
-      @c.each do |ci|
-        ci.items.each do |k|
-          k.closure1([@g.tla]).each do |j|          
-            j.la.each do |ai|
-              if ai != @g.tla
-                ci.goto(j.rule.r[j.n]).items.each do |i|
-                  if i.rule == j.rule && i.n == j.n+1
-                    i.la.push ai unless i.la.include?(ai)
-                    break
-                  end
-                end
-              elsif ai == @g.tla
-                ci.goto(j.rule.r[j.n]).items.each do |i|
-                  if i.rule == j.rule && i.n == j.n+1
-                    trn.push [k, i]
-                    break
+      @c.each do |_K|
+        _K.items.each do |_B|
+          _B.closure1([@g.tla]).each do |_A|
+            _K.goto(_A.dotsym).items.each do |i|
+              if i.rule == _A.rule and i.n == _A.n+1
+                _A.la.each do |a|
+                  if a != @g.tla
+                    i.la << a unless i.la.include?(a)
+                  else
+                    trn.push [_B, i, _K, _K.goto(_A.dotsym)]
                   end
                 end
+                break
               end
             end
           end
         end
       end
 
+      #puts @g.closure.values.join("\n")
+      #puts "-----"
+      #puts @g.memo_closure1.sort_by{|k,v| k}.map{|k,v| "#{k.inspect}:\n#{v.join("\n")}"}
       #puts 'mk'; mk.each{|k, v| p "#{k}=>#{v}"}
-      #puts 'trn',trn.sort{|a,b| a[0].rule.l.to_s <=> b[0].rule.l.to_s}\
-      #  .map{|a| "#{a[0]} => #{a[1]}"}
-
+      #trn = trn.uniq
+      #puts 'trn',trn.map{|a| "<#{a[2].n}>#{a[0]} => <#{a[3].n}>#{a[1]}"}
       warn '** mk LALR(1) **' if $MP_DEBUG
       @c[0].items[0].la = [@g.nssize] # '$'
       begin
         change = false
-        trn.each do |k, v|
+        trn.each do |k, v, _|
           k.la.each do |i|
             unless v.la.include? i
               change = true

data/lib/depager/parse_action.rb

@@ -18,7 +18,7 @@ module Depager::ActionParser
       end
     end
     @line = $'
-    #warn val
+    # warn "--action--\n#{val}"
     return val
   end
 end

data/lib/depager/parser.rb

@@ -1,11 +1,38 @@
 require "depager/utils.rb"
-module Depager
-  #
-  # Token
-  # 
-  Token = Struct.new(:value, :lineno, :filename)
 
-  # = Parser Runtime
+#
+# Token
+# 
+class Depager::Token
+  def self.[] *args
+    self.new(*args)
+  end
+  attr_accessor :value, :lineno, :filename
+  def initialize value, lineno=nil, filename=nil
+    @value = value
+    @lineno = lineno
+    @filename = filename
+  end
+  def [](n)
+    case n
+    when 0 ; @value
+    when 1 ; @lineno
+    when 2 ; @filename
+    else   ; nil
+    end
+  end
+  def to_s
+    "#{@value.inspect}:#{@lineno}@#{@filename}"
+  end
+  def inspect
+    "#<#{self.class.name}:0x#{object_id.to_s(16)}" <<
+      " #{@value.inspect}:#{@lineno}@#{@filename}>"
+  end
+end
+
+# LALR runtime
+module Depager::LALR
+  # = Parser runtime
   # == Stack
   # s::  state
   # la:: look ahead
@@ -107,7 +134,7 @@ module Depager
 
     def _Token sym, val=nil, lineno=nil
       lineno ||= file.lineno
-      return sym, Token[val, lineno, file.path]
+      return sym, Depager::Token[val, lineno, file.path]
     end
 
     public

data/lib/depager/slex.dr

@@ -25,30 +25,45 @@
   def postRhs
     j = 0
     rhs = g_parser.rhs
-    @ins.each do |i, m|
+    @ins.each do |i, option_list, debug|
       state_name = "#{lhs_name}_#{nrhs}_#{i}"
+      m_name     = "_lex_#{state_name}"
       if i != rhs.size
-        isym     = g_parser.add_nonterm("__#{state_name}__".intern)
-        new_rule = Rule[isym, []]
-        val      = ( ":_lex_#{state_name}" )
-        @addition << [ new_rule, rhs, val, i+j, isym ]
+        nt_name  = "__#{state_name}__"
+        isym     = g_parser.add_nonterm(nt_name)
+        g_parser.rhs_insert_sym(i+j, isym, nt_name)
+        @addition << [ isym, m_name ]
         j += 1
       else
-        @on_reduce[nrules] = ( ":_lex_#{state_name}" )
+        @on_reduce[nrules] = ':'+m_name
+      end
+
+      codes = option_list.map do |k, m|
+        case k
+        when :GOTO
+          "@basis.lex_state = #{m}" <<
+            (debug ? ";warn 'MODE:->#{m}'" : '')
+        when :ADD 
+          "@basis.lex_context[#{m}] = true" <<
+            (debug ? ";warn 'CONTEXT:+#{m}/\#{@context.inspect}'" : '')
+        when :SUB
+          "@basis.lex_context.delete #{m}" <<
+            (debug ? ";warn 'CONTEXT:-#{m}'/\#{@context.inspect}" : '')
+        else
+        end
       end
       @optouter.push << %{
-        def _lex_#{lhs_name}_#{nrhs}_#{i} val
-          @basis.lex_state = #{m}
+        def #{m_name} val
+          #{codes.join("\n          ")}
         end
       }
     end
     @ins.clear
   end
   def postRhsList
-    @addition.each do |new_rule, rhs, val, pos, isym|      
-      g_parser.g << new_rule
-      rhs.insert(pos, isym)
-      @on_reduce[nrules] = val
+    @addition.each do |isym, m_name|  
+      g_parser.add_rule isym, []
+      @on_reduce[nrules] = ':'+m_name
     end
     @addition.clear
   end
@@ -63,10 +78,18 @@
 %LEX{
   /\s+/, /#.*\Z/           { }
   /[A-Z]+/                 { yield _Token(:ID, $&) }
-  /\%\}\s*\Z/              { @line = $'; yield nil,nil }
-  /\/(([^\/\\]+|\\.)*)\//,
-  /'([^'\\]+|\\.)*\'/      { yield _Token(:LEX, "/\\A#{$1}/") }
-  /\{/                     { ln = lineno; yield :ACTION, Token[parse_action, ln]; //=~'' }
+  /\%\}\s*\Z/              { yield nil,nil }
+  /\/(([^\/\\]+|\\.)*)\//  { yield _Token(:LEX, "/\\A#{$1}/") }
+  /'([^'\\]+|\\.)*'/,
+  /"([^"\\]+|\\.)*"/
+  { 
+    yield _Token(:LEX, "/\\A#{Regexp.escape($1).gsub('/', '\/')}/")
+  }
+  '{' !
+  { 
+    _lineno = lineno
+    yield _Token(:ACTION, parse_action, _lineno)
+  }
   /./                      { yield _Token($&, $&) }
 %}
 #begin-rule
@@ -75,32 +98,31 @@
       {
         ll = ''
         _mode_list.each do |m, s, l|
-          elsec = if s
-                    "_rest = lex_#{s}(&block)"
-                  else
-                    %!raise RuntimeError, "must not happen \#{@line}"!
-                  end
+          else_code = if s
+                        "lex_#{s}(&block)"
+                      else
+                        'raise RuntimeError, "must not happen #{@line}"'
+                      end
           ll << %{
             def lex_#{m}(&block)
-              _rest = ''
               case @line
                 #{l.map{|i| ' '*4+i}}
               else
-                #{elsec}
+                #{else_code}
               end
-              return _rest
             end
           }; #code
         end
         g_parser.optinner << ll << %{
-          attr_accessor :lex_state
+          attr_accessor :lex_state, :lex_context
           def lex(&block)
-            @lex_state ||= :START
+            @lex_state   ||= :START
+            @lex_context ||= {}
             begin
               until @line.empty?
-                @line = self.send "lex_\#{@lex_state}", &block
+                self.send "lex_\#{@lex_state}", &block
               end
-            end while @line = getline
+            end while @line0 = @line = getline
             yield nil, nil
           end
         }; #code
@@ -122,15 +144,19 @@
     | lexactlist lexact { _lexactlist << _lexact }
   ;
   lexact:
-      lexlist ACTION
+      lexlist opt_noskip ACTION
       {
         %{
           when #{_lexlist.join(', ')}
-            _rest = $'
+            #{ _opt_noskip ? '#' : "@oldline = @line; @line = $'" }
             #{ _ACTION.value }
         }; #code
       }
   ;
+  opt_noskip:          
+                        { false }
+    | '!'               { true  }
+  ;
   lexlist:
       LEX               { [ _LEX.value ] }
     | lexlist ',' LEX   { _lexlist.push _LEX }
@@ -144,18 +170,32 @@
 %banner '[> ...]'
 %%
 %LEX{
-  /\s+/         { }
-  /:[a-zA-Z_]+/ { yield _Token(:SYMBOL, $&) }
-  /\]/          { yield _Token($&, $&); @line = $'; yield nil, nil }
-  /./           { yield _Token($&, $&) }
+  /\s+/            { }
+  /:[a-zA-Z_]+\??/ { yield _Token(:SYMBOL, $&) }
+  /\]/             { yield _Token($&, $&); @line = $'; yield nil, nil }
+  /./              { yield _Token($&, $&) }
 %}
 
 #begin-rule
   start:
-      '[' '>' SYMBOL ']'
+      '[' option_list ']'
       {
-        master.ins.push [g_parser.rhs.size, _SYMBOL.value]
+        master.ins.push [g_parser.rhs.size, _option_list, false]
       }
+    | '[' option_list '?' ']'
+      {
+        master.ins.push [g_parser.rhs.size, _option_list, true]
+      }
+
+  ;
+  option_list:
+      option             { [_option] }
+    | option_list option { _option_list << _option }
+  ;
+  option:
+      '>' SYMBOL { [:GOTO, _SYMBOL.value] }
+    | '+' SYMBOL { [:ADD, _SYMBOL.value]  }
+    | '-' SYMBOL { [:SUB, _SYMBOL.value]  }
   ;
 #end-rule
 %%