rubylexer 0.7.6 → 0.7.7

data/lib/rubylexer/.document

@@ -0,0 +1,9 @@
+charhandler.rb
+charset.rb
+context.rb
+lextable.rb
+rubycode.rb
+rulexer.rb
+symboltable.rb
+tokenprinter.rb
+token.rb

data/lib/rubylexer/charhandler.rb

@@ -1,4 +1,4 @@
-=begin legal crap
+=begin
     rubylexer - a ruby lexer written in ruby
     Copyright (C) 2004,2005,2008  Caleb Clausen
 
@@ -22,23 +22,29 @@ class RubyLexer
 #------------------------------------
 class CharHandler
   #-----------------------------------
-  CHARSETSPECIALS=CharSet[?[ ,?] ,?\\ ,?-]
+  if ?A.is_a? String #ruby >= 1.9
+    CHARSETSPECIALS=/[\[\]\\\-]/
+  else
+    CHARSETSPECIALS=CharSet[?[ ,?] ,?\\ ,?-]
+  end
   def initialize(receiver,default,hash) 
     @default=default 
     @receiver=receiver
- #breakpoint
-    @table=Array.new(0)
+    if ?A.is_a? String #ruby >= 1.9
+      @table={}
+    else
+      @table=Array.new(0)
+    end
     @matcher='^[^'
 
     hash.each_pair {|pattern,action|
       case pattern
       when Range
         pattern.each { |c|
-          c.kind_of? String and c=c[0] #cvt to integer  #still needed?
           self[c]=action
         }
       when String
-        pattern.each_byte {|b| self[b]=action }
+        CharHandler.each_char(pattern) {|b| self[b]=action }
       when Fixnum
         self[pattern]=action
       else
@@ -47,15 +53,26 @@ class CharHandler
     }
 
     @matcher += ']$'
-    @matcher=Regexp.new(@matcher)
+    @matcher=Regexp.new(@matcher,0,'n')
 
     freeze
   end
 
+  #-----------------------------------
+  if String===?a 
+    def self.each_char(str,&block)
+      str.each_char(&block)
+    end
+  else
+    def self.each_char(str,&block)
+      str.each_byte(&block)
+    end
+  end
+
   #-----------------------------------
   def []=(b,action)  #for use in initialize only
     assert b >= ?\x00
-    assert b <= ?\xFF
+    assert b <= ?\x7F
     assert !frozen?
 
     @table[b]=action
@@ -69,17 +86,6 @@ class CharHandler
     @receiver.send((@table[b] or @default), b.chr, *args)
   end
 
-  #-----------------------------------
-  def eat_file(file,blocksize,*args)
-    begin
-      chars=file.read(blocksize)
-      md=@matcher.match(chars)
-      mychar=md[0][0]
-      #get file back in the right pos
-      file.pos+=md.offset(0)[0] - chars.length
-      @receiver.send(@default,md[0])
-    end until go(mychar,*args)
-  end
 end
 end
 

data/lib/rubylexer/context.rb

@@ -1,4 +1,4 @@
-=begin legal crap
+=begin 
     rubylexer - a ruby lexer written in ruby
     Copyright (C) 2008  Caleb Clausen
 
@@ -74,6 +74,7 @@ module NestedContexts
       super('{','}',linenum)
     end
     attr_accessor :wanting_stabby_block_body
+    #attr_writer :starter,:ender
   end
 
   class BeginEndContext  < NestedContext
@@ -118,6 +119,9 @@ module NestedContexts
   class UnparenedParamListLhsContext < ImplicitLhsContext
     def starter; huh end  #" " ???
     def ender; huh end    #; or \n when from method def, { or do when from stabby block
+    def endtoken offset
+      KwParamListEndToken.new offset
+    end
   end
 
   class ImplicitContext < ListContext
@@ -129,6 +133,9 @@ module NestedContexts
     end
     def lhs; false end
     def wantarrow; true end
+    def endtoken offset
+      ImplicitParamListEndToken.new offset
+    end
   end
 
   class KWParamListContextNoParen < ParamListContextNoParen
@@ -159,6 +166,9 @@ module NestedContexts
     def multi_assign?
       @multi if defined? @multi 
     end
+    def endtoken offset
+      AssignmentRhsEndToken.new offset
+    end
   end
 
   class WantsEndContext < NestedContext
@@ -201,6 +211,9 @@ module NestedContexts
     end
 
     attr :in_body
+
+    attr_writer :has_parens
+    def has_parens?; @has_parens end
   end
 
   class StringContext < NestedContext #not used yet
@@ -283,13 +296,13 @@ module NestedContexts
       stack=lxr.parsestack
       assert msg!=:for
       case msg
-      when :for: WantsEndContext===stack.last or raise 'syntax error: for not expected at this time'
+      when :for; WantsEndContext===stack.last or raise 'syntax error: for not expected at this time'
                  #local var defined in this state
                  #never actually used?
-      when :in:  self.equal? stack.pop or raise 'syntax error: in not expected at this time'
+      when :in;  self.equal? stack.pop or raise 'syntax error: in not expected at this time'
                  stack.push ExpectDoOrNlContext.new("for",/(do|;|:|\n)/,@linenum) 
                  #pop self off owning context stack and push ExpectDoOrNlContext
-      when :comma, :splat: return
+      when :comma, :splat; return
       else super
       end
       LEGAL_SUCCESSORS[@state].include? msg or raise "for syntax error: #{msg} unexpected in #@state"

data/lib/rubylexer/rubycode.rb

@@ -41,7 +41,7 @@ class RubyCode < Token
       result=[]
       keepwsprinter=KeepWsTokenPrinter.new('',@linenum)
       ident.each{|tok| result << keepwsprinter.sprint(tok) }
-      return result.to_s
+      return result.join
    end
 end
 end

data/lib/rubylexer/rulexer.rb

@@ -38,17 +38,10 @@ require 'sequence/file'
 require 'sequence/list'
 #-----------------------------------
 assert !defined? ::RubyLexer
-$RuLexer=Class.new{}
-class RubyLexer < $RuLexer
-  RuLexer=$RuLexer
-end
-$RuLexer=nil
-#------------------------------------
 class RubyLexer
   FASTER_STRING_ESCAPES=true
   warn "FASTER_STRING_ESCAPES is off" unless FASTER_STRING_ESCAPES
   AUTO_UNESCAPE_STRINGS=false
-  class RuLexer
    WHSP=" \t\r\v\f"
    WHSPLF=WHSP+"\n"
    #maybe \r should be in WHSPLF instead
@@ -75,11 +68,13 @@ class RubyLexer
       @moretokens=[ RubyLexer::FileAndLineToken.new(@filename, @linenum, input_position) ]
       @endsets={}
    end
+   alias rulexer_initialize initialize
 
    #-----------------------------------
    def endoffile_detected s=''
      EoiToken.new(s,@original_file, input_position-s.size)
    end
+   alias rulexer_endoffile_detected endoffile_detected
 
    #-----------------------------------
    def get1token
@@ -92,6 +87,7 @@ class RubyLexer
 
       @toptable.go( nextchar )
    end
+   alias rulexer_get1token get1token
 
    #-----------------------------------
    def no_more?
@@ -143,40 +139,41 @@ private
       if @rubyversion>=1.9
         named_brs=[]
         if result.elems.size==1 and String===result.elems.first
+            elem=result.elems.first
             index=0
-            huh
-            while index=elem.index(/#{EVEN_BS_S}( \(\?[<'] | \(\?\# | \[ )/xo,index)
-              huh
-              case alt
-              when "(?<"; huh
+            while index=elem.index(/(#{EVEN_BS_S})( \(\?[<'] | \(\?\# | \[ )/xo,index)
+              index+=$1.size
+              case $2
+              when "(?<"
                 index=elem.index(/\G...(#{LCLETTER}#{LETTER_DIGIT}+)>/o,index)
-                index or huh
-                index+=$1.size+4
+                break lexerror(result, "malformed named backreference") unless index
+                index+=$&.size
                 named_brs<<$1
-              when "(?'"; huh
+              when "(?'"
                 index=elem.index(/\G...(#{LCLETTER}#{LETTER_DIGIT}+)'/o,index)
-                index or huh
-                index+=$1.size+4
+                break lexerror(result, "malformed named backreference") unless index
+                index+=$&.size
                 named_brs<<$1
-              when "(?#"; huh
+              when "(?#"
                 index+=3
-                index=elem.index(/#{EVEN_BS_S}\)/,index)
-                index or huh
-                index+=1
-              when "["; huh
+                index=elem.index(/#{EVEN_BS_S}\)/o,index)
+                break lexerror(result, "unterminated regexp comment") unless index
+                index+=$&.size
+              when "["
                 index+=1
                 paren_ctr=1
                 loop do
                   index=elem.index(/#{EVEN_BS_S}(&&\[\^|\])/o,index)
-                  index or huh
+                  break lexerror(result, "unterminated character class") unless index
                   index+=$&.size
-                  unless $1[-1]==?]
-                    paren_ctr+=1
-                  else 
+                  if $1==']'
                     paren_ctr-=1
                     break if paren_ctr==0 
+                  else 
+                    paren_ctr+=1
                   end
                 end
+                break unless index
                 
               end
             end
@@ -190,6 +187,7 @@ private
 
    #-----------------------------------
    def single_char_token(str)  getchar   end
+   alias rulexer_single_char_token single_char_token
 
    #-----------------------------------
    def illegal_char(ch)
@@ -239,12 +237,12 @@ else
 end
       result=send(strlex, beg, type, close=(PAIRS[beg] or beg))
       case ch
-      when /^[Wwr]$/:
+      when /^[Wwr]$/
         str=result
         result=RenderExactlyStringToken.new(type).append_token(result)
         result.open=str.open; result.close=str.close
         result.line=@linenum
-      when 's': 
+      when 's'
         result.open=open+beg
         result.close=close
         result=SymbolToken.new result,nil,"%s"
@@ -274,12 +272,12 @@ end
    #-----------------------------------
    INTERIOR_REX_CACHE={}
    EVEN_BS_S=/
-     ($|
+     (?:\G|
       [^\\c-]|
-      ($|[^\\])(c|[CM]-)|
-      ($|[^CM])-
+      (?:\G|[^\\])(?:c|[CM]-)|
+      (?:\G|[^CM])-
      )
-     (\\(?:c|[CM]-)?){2}*
+     (?:\\(?:c|[CM]-)?){2}*
    /x
    ILLEGAL_ESCAPED=/#{EVEN_BS_S}(\\([CM][^-]|x[^a-fA-F0-9]))/o #whaddaya do with this?
    def all_quote(nester, type, delimiter, bs_handler=nil)
@@ -298,12 +296,12 @@ if FASTER_STRING_ESCAPES
         single_quotish=true
         special=/\\./m
       else
-        crunch=/#(?=[^{$@])/
+        crunch=/\#(?=[^{$@])/
         escaped=/\\(?>[^xcCM0-7]|(?>c|[CM].)(?>[^\\]|(?=\\))|(?>x.[0-9a-fA-F]?)|(?>[0-7]{1,3}))/m
         special=
           case delimiter
-          when '\\': crunch
-          when '#': escaped
+          when '\\'; crunch
+          when '#'; escaped
           else /#{escaped}|#{crunch}/o
           end
         special_char<< maybe_crunch="#"
@@ -318,8 +316,8 @@ if FASTER_STRING_ESCAPES
       str=StringToken.new type
       str.bs_handler ||= case type
         when '/' then :regex_esc_seq
-        when '{' then :Wquote_esc_seq
-        when '"','`',':' then :dquote_esc_seq
+        when '{' then @rubyversion>=1.9 ? :Wquote19_esc_seq : :Wquote_esc_seq
+        when '"','`',':' then @rubyversion>=1.9 ? :dquote19_esc_seq : :dquote_esc_seq
         when "'"     then :squote_esc_seq
         when "["     then :wquote_esc_seq
         else raise "unknown quote type: #{type}"
@@ -431,8 +429,8 @@ else
 
       bs_handler ||= case type
         when '/' then :regex_esc_seq
-        when '{' then :Wquote_esc_seq
-        when '"','`',':' then :dquote_esc_seq
+        when '{' then @rubyversion>=1.9 ? :Wquote19_esc_seq : :Wquote_esc_seq
+        when '"','`',':' then @rubyversion>=1.9 ? :dquote19_esc_seq : :dquote_esc_seq
         when "'"     then :squote_esc_seq
         when "["     then :wquote_esc_seq
         else raise "unknown quote type: #{type}"
@@ -525,16 +523,25 @@ end
             k.tr(ESCAPECHRS,ESCAPESEQS)
          when "M"
             eat_next_if(?-) or raise 'bad \\M sequence'
-            (getchar_maybe_escape | 0x80).chr
+            ch=getchar_maybe_escape[0]
+            ch=ch.ord if ch.respond_to? :ord
+            ch>=0xFF and raise 'bad \\M sequence'
+            (ch | 0x80).chr
 
          when "C"
             eat_next_if(?-) or raise 'bad \\C sequence'
             nextchar==?? and getchar and return "\177" #wtf?
-            (getchar_maybe_escape & 0x9F).chr
+            ch=getchar_maybe_escape[0]
+            ch=ch.ord if ch.respond_to? :ord
+            ch>=0xFF and raise 'bad \\M sequence'
+            (ch & 0x9F).chr
 
          when "c"
             nextchar==?? and getchar and return "\177" #wtf?
-            (getchar_maybe_escape & 0x9F).chr
+            ch=getchar_maybe_escape[0]
+            ch=ch.ord if ch.respond_to? :ord
+            ch>=0xFF and raise 'bad \\M sequence'
+            (ch & 0x9F).chr
 
          when /^[0-7]$/
             str=k
@@ -556,6 +563,33 @@ end
       end
    end
 
+   #-----------------------------------
+   def dquote19_esc_seq(ch,nester,delimiter)
+      assert ch == '\\'
+      case ch=getchar
+      when 'u'
+        case ch=getchar
+        when /[a-f0-9]/i
+          u=read(4)
+          raise "bad unicode escape" unless /[0-9a-f]{4}/i===u
+          [u.hex].pack "U"
+        when '{'
+          result=[]
+          until eat_next_if '}'
+            u=@file.scan(/\A[0-9a-f]{1,6}[ \t]?/i,7)
+            result<<u.hex
+          end
+          result=result.pack "U*"
+        else raise "bad unicode escape"
+        end 
+      else 
+        back1char
+        result=dquote_esc_seq('\\',nester,delimiter)
+        #/\s|\v/===result and result="\\"+result
+        result
+      end
+   end
+
    #-----------------------------------
    def regex_esc_seq(ch,nester,delimiter)
       assert ch == '\\'
@@ -571,9 +605,9 @@ end
    def Wquote_esc_seq(ch,nester,delimiter)
       assert ch == '\\'
       case ch=getchar
-      when "\n": @linenum+=1; ch
-      when nester,delimiter: ch
-      when /[\s\v\\]/: ch
+      when "\n"; @linenum+=1; ch
+      when nester,delimiter; ch
+      when /[\s\v\\]/; ch
       else 
         back1char
         result=dquote_esc_seq('\\',nester,delimiter)
@@ -582,6 +616,21 @@ end
       end
    end
 
+   #-----------------------------------
+   def Wquote19_esc_seq(ch,nester,delimiter)
+      assert ch == '\\'
+      case ch=getchar
+      when "\n"; @linenum+=1; ch
+      when nester,delimiter; ch
+      when /[\s\v\\]/; ch
+      else 
+        back1char
+        result=dquote19_esc_seq('\\',nester,delimiter)
+        #/\s|\v/===result and result="\\"+result
+        result
+      end
+   end
+
    #-----------------------------------
    def wquote_esc_seq(ch,nester,delimiter)
       assert(ch=='\\')
@@ -592,10 +641,10 @@ end
          #all \ sequences 
          #are unescaped; actual
          #newlines are counted but not changed
-         when delimiter,nester,'\\': escchar
-#         when delimiter,nester: escchar
-         when "\n": @linenum+=1; escchar
-         when /[\s\v]/: escchar
+         when delimiter,nester,'\\'; escchar
+#         when delimiter,nester; escchar
+         when "\n"; @linenum+=1; escchar
+         when /[\s\v]/; escchar
          else       "\\"+escchar
       end
    end
@@ -610,9 +659,9 @@ end
          #all \ sequences 
          #are unescaped; actual
          #newlines are counted but not changed
-         when delimiter,nester,'\\': escchar
-#         when delimiter,nester: escchar
-         when "\n": @linenum+=1; "\\"+escchar
+         when delimiter,nester,'\\'; escchar
+#         when delimiter,nester; escchar
+         when "\n"; @linenum+=1; "\\"+escchar
          else       "\\"+escchar
       end
    end
@@ -627,9 +676,9 @@ end
          #all \ sequences 
          #are unescaped; actual
          #newlines are counted but not changed
-         when delimiter,nester: escchar
-#         when delimiter,nester: escchar
-         when "\n": @linenum+=1; "\\"+escchar
+         when delimiter,nester; escchar
+#         when delimiter,nester; escchar
+         when "\n"; @linenum+=1; "\\"+escchar
          else       "\\"+escchar
       end
    end
@@ -649,9 +698,11 @@ end
 #   alias squote_esc_seq	wquote_esc_seq
 
   module RecursiveRubyLexer
+=begin
     def initial_nonblock_levels
       @localvars_stack.size==1 ? 2 : 1
     end
+=end
   end
 
   def initial_nonblock_levels; 1 end
@@ -663,8 +714,8 @@ end
 
   def merge_levels levels, nil_empty_class
     case (levels.size rescue 0)
-    when 0: {} unless nil_empty_class
-    when 1: levels.first.dup
+    when 0; {} unless nil_empty_class
+    when 1; levels.first.dup
     else levels.inject{|a,b| a.merge b} 
     end
   end
@@ -713,30 +764,7 @@ end
       #pass current local vars into new parser
       #must pass the lists of nonblock, parentblock and currentblock vars separately
       #then a table increment after each
-      nonblocky,blocky,current=decompose_lvars(true)
-      nonblocky.keys.each{|varname|
-        rl.localvars[varname]=true
-      }
-      rl.localvars.start_block  
-      #incremental table, tells us what :local vars are defined in the str inclusion
-
-      if blocky
-        rl.localvars.start_block  
-        blocky.keys.each{|varname|
-          rl.localvars[varname]=true
-        }
-        rl.localvars.start_block
-        #incremental table, tells us what :block vars are defined in the str inclusion
-      end
-
-      if current
-        rl.localvars.start_block  
-        current.keys.each{|varname|
-          rl.localvars[varname]=true
-        }
-        rl.localvars.start_block
-        #incremental table, tells us what :current vars are defined in the str inclusion
-      end
+      rl.localvars_stack=@localvars_stack.map{|lvs| lvs.deep_copy}
       
       rl.pending_here_bodies=@pending_here_bodies
 
@@ -790,19 +818,13 @@ end
 #      @pending_here_bodies=rl.pending_here_bodies      
 
       #local vars defined in inclusion get propagated to outer parser
-      newvars=rl.localvars.__locals_lists[1..-1].map{|bag| bag.keys }.flatten
-      newvars.each{|newvar| localvars[newvar]=true }
+      @localvars_stack=rl.localvars_stack
 
       result=RubyCode.new(tokens,@filename,@linenum)
       @linenum=rl.linenum
       return result
    end
    
-   #-----------------------------------
-   def here_spread_over_ruby_code(rl,tok)
-     lexerror tok, 'here body outside string inclusion'
-   end
-   
 
    #-----------------------------------
 #   BINCHARS=?0..?1
@@ -913,6 +935,7 @@ else
       IgnoreToken.new(til_charset(/[\r\n]/))
    end
 end
+  alias rulexer_comment comment
 
    #-----------------------------------
    def whitespace(ch)
@@ -944,16 +967,17 @@ end
       @moretokens << FileAndLineToken.new( @filename, @linenum, offset+1 )
       return NewlineToken.new("\n",offset)
    end
-
+   alias rulexer_newline newline
 
    #-----------------------------------
    def getchar_maybe_escape
       eof? and raise "unterminated dq string"
-      c=getc
-
-      c == ?\\ and
-         (c = (dquote_esc_seq('\\')[-1] or ?\n))
+      c=getc.chr
 
+      if c == "\\"
+         c = @rubyversion >= 1.9 ? dquote19_esc_seq('\\') : dquote_esc_seq('\\')
+         c = "\n" if c.empty?
+      end
       return c
    end
 
@@ -962,6 +986,7 @@ protected
   require 'forwardable'
   extend Forwardable
   def_delegators :@file, :readahead, :readback, :read, :eof?
+  alias rulexer_eof? eof?
 
   def til_charset cs,len=16; @file.read_til_charset cs,len end
   def getc; @file.read1 end
@@ -995,13 +1020,14 @@ protected
 
   #-----------------------------------
   def input_position; @file.pos end
+  alias rulexer_input_position input_position
 
   #-----------------------------------
   def input_position_set x; @file.pos=x end
 
   #-----------------------------------
   def self.save_offsets_in(*funcnames)
-    eval funcnames.collect{|fn| <<-endeval }.to_s
+    eval funcnames.collect{|fn| <<-endeval }.join
       class ::#{self}
         alias #{fn}__no_offset #{fn}   #rename old ver of fn
         def #{fn}(*args)               #create new version
@@ -1015,7 +1041,6 @@ protected
     endeval
   end
 
-  end
 
 end