rubylexer 0.7.7 → 0.8.0

data/lib/rubylexer/charhandler.rb

@@ -1,6 +1,6 @@
 =begin
     rubylexer - a ruby lexer written in ruby
-    Copyright (C) 2004,2005,2008  Caleb Clausen
+    Copyright (C) 2004,2005,2008, 2011  Caleb Clausen
 
     This library is free software; you can redistribute it and/or
     modify it under the terms of the GNU Lesser General Public

data/lib/rubylexer/charset.rb

@@ -1,6 +1,6 @@
 =begin copyright
     rubylexer - a ruby lexer written in ruby
-    Copyright (C) 2004,2005  Caleb Clausen
+    Copyright (C) 2004,2005, 2011  Caleb Clausen
 
     This library is free software; you can redistribute it and/or
     modify it under the terms of the GNU Lesser General Public
@@ -34,7 +34,7 @@ class CharSet
       case chars
       when ::String
          chars.each_byte {|c| @bitset |= (1<<c) }
-      when ::Fixnum then        @bitset |= (1<<chars)
+      when ::Fixnum then      @bitset |= (1<<chars)
       else chars.each    {|c| @bitset |= (1<<c) }
       end
    end
@@ -49,17 +49,25 @@ class CharSet
       #this math works right with bignums... (i'm pretty sure)
    end
 
-   def ===(c) #c is String|Fixnum|nil
-      c.nil? and return false
-      c.kind_of? String and c=c[0]
-      return ( @bitset[c] != 0 )
+   if String==="a"[0]
+     def ===(c) #c is String|Fixnum|nil
+       c.nil? and return false
+       c.kind_of? String and c=c.getbyte(0)
+       return ( @bitset[c] != 0 )
+     end
+   else
+     def ===(c) #c is String|Fixnum|nil
+       c.nil? and return false
+       c.kind_of? String and c=c[0]
+       return ( @bitset[c] != 0 )
+     end
    end
 
    #enumerate the chars in n AS INTEGERS
    def each_byte(&block)
       #should use ffs... not available in ruby
       (0..255).each { |n|
-         @bitset[n] and block[n]
+         @bitset[n].nonzero? and block[n]
       }
    end
 

data/lib/rubylexer/context.rb

@@ -1,6 +1,6 @@
 =begin 
     rubylexer - a ruby lexer written in ruby
-    Copyright (C) 2008  Caleb Clausen
+    Copyright (C) 2008, 2011  Caleb Clausen
 
     This library is free software; you can redistribute it and/or
     modify it under the terms of the GNU Lesser General Public
@@ -36,6 +36,7 @@ module NestedContexts
     
     def see lxr,msg; end
     def lhs=*x; end #do nothing
+    def lhs; false end
     def wantarrow; false end
   end
 
@@ -51,6 +52,12 @@ module NestedContexts
     end
     def wantarrow; true end
   end
+  
+  class StringInclusionContext < NestedContext
+    def initialize(linenum)
+      super("{","}" ,linenum)
+    end
+  end
 
   class ParenContext < NestedContext
     def initialize(linenum)
@@ -120,7 +127,7 @@ module NestedContexts
     def starter; huh end  #" " ???
     def ender; huh end    #; or \n when from method def, { or do when from stabby block
     def endtoken offset
-      KwParamListEndToken.new offset
+      ImplicitParamListEndToken.new offset
     end
   end
 
@@ -324,5 +331,6 @@ module NestedContexts
       dflt_initialize('?',':',linenum)
     end
   end
+
 end
 end

data/lib/rubylexer/lextable.rb

@@ -1,3 +1,4 @@
+#Copyright (c) 2011 Caleb Clausen
 class RubyLexer
   class Rule
     def initialize(lead,matcher,*actions)

data/lib/rubylexer/rubycode.rb

@@ -1,6 +1,6 @@
 =begin copyright
     rubylexer - a ruby lexer written in ruby
-    Copyright (C) 2004,2005  Caleb Clausen
+    Copyright (C) 2004,2005, 2011  Caleb Clausen
 
     This library is free software; you can redistribute it and/or
     modify it under the terms of the GNU Lesser General Public

data/lib/rubylexer/rulexer.rb

@@ -1,6 +1,7 @@
+#encoding: binary
 =begin
     rubylexer - a ruby lexer written in ruby
-    Copyright (C) 2004,2005,2008  Caleb Clausen
+    Copyright (C) 2004,2005,2008, 2011  Caleb Clausen
 
     This library is free software; you can redistribute it and/or
     modify it under the terms of the GNU Lesser General Public
@@ -17,9 +18,9 @@
     Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
 =end
 
-#warn "hacking $LOAD_PATH to find latest sequence"
-#$:<<"../sequence/lib"
-
+if defined? RubyLexer #sigh
+  Object.send :remove_const, :RubyLexer
+end
 
 require "assert"
 #require "charhandler"
@@ -32,10 +33,18 @@ rescue LoadError=>e
   raise unless /rubygems/===e.message
   #hope we don't need it
 end
-#require 'sequence'
-require 'sequence/indexed'
-require 'sequence/file'
-require 'sequence/list'
+begin
+  #require 'sequence'
+  require 'sequence/indexed'
+  require 'sequence/file'
+  require 'sequence/list'
+rescue LoadError
+  trydir=File.expand_path File.dirname(__FILE__)+"/../../../sequence/lib"
+  raise if $:.include? trydir
+  warn "hacking $LOAD_PATH to find latest sequence"
+  $:<<trydir
+  retry
+end
 #-----------------------------------
 assert !defined? ::RubyLexer
 class RubyLexer
@@ -46,7 +55,7 @@ class RubyLexer
    WHSPLF=WHSP+"\n"
    #maybe \r should be in WHSPLF instead
 
-   LEGALCHARS=/[ -~#{WHSPLF}\x80-\xFF]/
+   LEGALCHARS=/[!-~#{WHSPLF}\x80-\xFF]/
 
    PAIRS={ '{'=>'}', '['=>']', '('=>')', '<'=>'>'}
 
@@ -72,20 +81,23 @@ class RubyLexer
 
    #-----------------------------------
    def endoffile_detected s=''
-     EoiToken.new(s,@original_file, input_position-s.size)
+     EoiToken.new(s,@original_file, input_position-s.size,@linenum)
    end
    alias rulexer_endoffile_detected endoffile_detected
 
    #-----------------------------------
    def get1token
-      @moretokens.empty? or return @moretokens.shift
+      @moretokens.empty? or return result=@moretokens.shift
 
       if eof?
          #@moretokens<<nil
-         return endoffile_detected()
+         return result=endoffile_detected()
       end
 
-      @toptable.go( nextchar )
+      return result=@toptable.go( nextchar )
+   ensure
+      #hacky: result.endline should already be set
+      result.endline||=@linenum if result
    end
    alias rulexer_get1token get1token
 
@@ -135,8 +147,8 @@ private
 
    #-----------------------------------
    def regex(ch=nil)
-      result=RenderExactlyStringToken.new('/').append_token double_quote("/")
-      if @rubyversion>=1.9
+      result= double_quote("/")
+      if false and @rubyversion>=1.9
         named_brs=[]
         if result.elems.size==1 and String===result.elems.first
             elem=result.elems.first
@@ -219,7 +231,7 @@ private
          when /^#{LCLETTER().gsub('_','')}$/o
             error= "unrecognized %string type: "+ch; '"'
          when ''
-            result= lexerror( StringToken.new('', oldpos), "unexpected eof in %string")
+            result= lexerror( assign_encoding!(StringToken.new('', oldpos)), "unexpected eof in %string")
             result.line=@linenum
             return result
 
@@ -239,7 +251,6 @@ end
       case ch
       when /^[Wwr]$/
         str=result
-        result=RenderExactlyStringToken.new(type).append_token(result)
         result.open=str.open; result.close=str.close
         result.line=@linenum
       when 's'
@@ -269,6 +280,11 @@ end
      return result
    end
 
+   #-----------------------------------
+   def assign_encoding! str
+     str
+   end
+
    #-----------------------------------
    INTERIOR_REX_CACHE={}
    EVEN_BS_S=/
@@ -316,14 +332,14 @@ if FASTER_STRING_ESCAPES
       str=StringToken.new type
       str.bs_handler ||= case type
         when '/' then :regex_esc_seq
-        when '{' then @rubyversion>=1.9 ? :Wquote19_esc_seq : :Wquote_esc_seq
-        when '"','`',':' then @rubyversion>=1.9 ? :dquote19_esc_seq : :dquote_esc_seq
+        when '{' then Wquote_handler_name() #@rubyversion>=1.9 ? :Wquote19_esc_seq : :Wquote_esc_seq
+        when '"','`',':' then dquote_handler_name #@rubyversion>=1.9 ? :dquote19_esc_seq : :dquote_esc_seq
         when "'"     then :squote_esc_seq
         when "["     then :wquote_esc_seq
         else raise "unknown quote type: #{type}"
       end
 
-      old_linenum=@linenum
+      str.startline=old_linenum=@linenum
       nestlevel=1
       loop{ 
          str.append(@file.scan( interior ))
@@ -404,7 +420,7 @@ if FASTER_STRING_ESCAPES
            /ox #and this?
 
            #shouldn't tolerate ILLEGAL_ESCAPED in str (unless single quotish)....
-           lexerror str, "illegal escape sequence" if /#{@@ILLEGAL_CRUNCH}|#{ILLEGAL_ESCAPED}/===b 
+           lexerror str, "illegal escape sequence" if /#{@@ILLEGAL_CRUNCH}|#{ILLEGAL_ESCAPED}/o===b 
          end
 
          str.append b
@@ -429,8 +445,8 @@ else
 
       bs_handler ||= case type
         when '/' then :regex_esc_seq
-        when '{' then @rubyversion>=1.9 ? :Wquote19_esc_seq : :Wquote_esc_seq
-        when '"','`',':' then @rubyversion>=1.9 ? :dquote19_esc_seq : :dquote_esc_seq
+        when '{' then Wquote_handler_name #@rubyversion>=1.9 ? :Wquote19_esc_seq : :Wquote_esc_seq
+        when '"','`',':' then dquote_handler_name #@rubyversion>=1.9 ? :dquote19_esc_seq : :dquote_esc_seq
         when "'"     then :squote_esc_seq
         when "["     then :wquote_esc_seq
         else raise "unknown quote type: #{type}"
@@ -506,6 +522,24 @@ else
       str.line=@linenum
       str
 end
+   ensure
+     assign_encoding!(str) if str
+   end
+
+   #-----------------------------------
+   def dquote_handle(ch)
+     @rubyversion >= 1.9 ? dquote19_esc_seq(ch) : dquote_esc_seq(ch) 
+     #factored
+   end
+   #-----------------------------------
+   def dquote_handler_name
+     @rubyversion>=1.9 ? :dquote19_esc_seq : :dquote_esc_seq
+     #factored
+   end
+   #-----------------------------------
+   def Wquote_handler_name
+     @rubyversion>=1.9 ? :Wquote19_esc_seq : :Wquote_esc_seq
+     #factored
    end
 
    #-----------------------------------
@@ -570,13 +604,14 @@ end
       when 'u'
         case ch=getchar
         when /[a-f0-9]/i
-          u=read(4)
+          u=ch+read(3)
           raise "bad unicode escape" unless /[0-9a-f]{4}/i===u
           [u.hex].pack "U"
         when '{'
           result=[]
           until eat_next_if '}'
-            u=@file.scan(/\A[0-9a-f]{1,6}[ \t]?/i,7)
+            u=@file.scan( /\A[0-9a-f]{1,6}[ \t]?/i )
+            raise "bad unicode escape" unless u
             result<<u.hex
           end
           result=result.pack "U*"
@@ -607,7 +642,7 @@ end
       case ch=getchar
       when "\n"; @linenum+=1; ch
       when nester,delimiter; ch
-      when /[\s\v\\]/; ch
+      when /[#@@WSCHARS\\]/o; ch
       else 
         back1char
         result=dquote_esc_seq('\\',nester,delimiter)
@@ -622,7 +657,7 @@ end
       case ch=getchar
       when "\n"; @linenum+=1; ch
       when nester,delimiter; ch
-      when /[\s\v\\]/; ch
+      when /[#@@WSCHARS\\]/o; ch
       else 
         back1char
         result=dquote19_esc_seq('\\',nester,delimiter)
@@ -644,7 +679,7 @@ end
          when delimiter,nester,'\\'; escchar
 #         when delimiter,nester; escchar
          when "\n"; @linenum+=1; escchar
-         when /[\s\v]/; escchar
+         when /[#@@WSCHARS]/o; escchar
          else       "\\"+escchar
       end
    end
@@ -755,6 +790,12 @@ end
       assert ch[/^[{(@$]$/]
       klass= RubyLexer===self ? self.class : RubyLexer
       rl=klass.new(@filename,@file,@linenum,offset_adjust(),:rubyversion=>@rubyversion)
+      modules=[]
+      class<<self;ancestors;end.each{|anc| 
+        break if Class===anc
+        modules<<anc 
+      }
+      modules.reverse.each{|m| rl.extend m }
       rl.extend RecursiveRubyLexer
       rl.enable_macros! if @enable_macro
       rl.in_def=true if inside_method_def?
@@ -927,12 +968,12 @@ if (defined? DEBUGGER__ or defined? Debugger)
       #assert str == '#'
       Process.kill("INT",0) if readahead(11)==%/#breakpoint/ 
      
-      IgnoreToken.new(til_charset(/[\r\n]/))
+      IgnoreToken.new(til_charset(/\n/))
    end
 else
    #-----------------------------------
    def comment(str=nil)
-      IgnoreToken.new(til_charset(/[\r\n]/))
+      IgnoreToken.new(til_charset(/\n/))
    end
 end
   alias rulexer_comment comment
@@ -956,6 +997,7 @@ end
       nl or return nil
       assert((1..2)===nl.length)
       @linenum+=1
+      @offset_adjust2=0
       read nl.length
    end
 
@@ -975,7 +1017,7 @@ end
       c=getc.chr
 
       if c == "\\"
-         c = @rubyversion >= 1.9 ? dquote19_esc_seq('\\') : dquote_esc_seq('\\')
+         c = dquote_handle('\\') #@rubyversion >= 1.9 ? dquote19_esc_seq('\\') : dquote_esc_seq('\\')
          c = "\n" if c.empty?
       end
       return c
@@ -1025,6 +1067,18 @@ protected
   #-----------------------------------
   def input_position_set x; @file.pos=x end
 
+  #-----------------------------------
+  def adjust_linenums_in_moretokens!(tok2)
+    line=tok2.endline
+    @moretokens.each{|tok|
+      if tok.linecount.zero?
+        tok.endline||=line
+      else
+        line+=tok.linecount
+      end
+    }
+  end
+
   #-----------------------------------
   def self.save_offsets_in(*funcnames)
     eval funcnames.collect{|fn| <<-endeval }.join
@@ -1032,15 +1086,35 @@ protected
         alias #{fn}__no_offset #{fn}   #rename old ver of fn
         def #{fn}(*args)               #create new version
           pos= input_position
+          ln=@linenum
           result=#{fn}__no_offset(*args)
-          assert Token===result
+          assert Token===result, "lexer output was not a Token"
           result.offset||=pos
+          result.endline||=ln
+          adjust_linenums_in_moretokens!(result)
           return result
         end
       end
     endeval
   end
 
+  #-----------------------------------
+  def self.save_linenums_in(*funcnames)
+    eval funcnames.collect{|fn| <<-endeval }.join
+      class ::#{self}
+        alias #{fn}__no_linenum #{fn}   #rename old ver of fn
+        def #{fn}(*args)               #create new version
+          ln=@linenum
+          result=#{fn}__no_linenum(*args)
+          assert Token===result
+          result.endline||=ln
+          adjust_linenums_in_moretokens!(result)
+          return result
+         end
+      end
+    endeval
+  end
+
 
 end
 

data/lib/rubylexer/symboltable.rb

@@ -1,6 +1,6 @@
 =begin copyright
     rubylexer - a ruby lexer written in ruby
-    Copyright (C) 2004,2005  Caleb Clausen
+    Copyright (C) 2004,2005, 2011  Caleb Clausen
 
     This library is free software; you can redistribute it and/or
     modify it under the terms of the GNU Lesser General Public

data/lib/rubylexer/test/oneliners.rb

@@ -65,10 +65,8 @@ wwww,eeee=1,2
 x{a.b,c.d=1,2}
 x{proc{|a.b,c.d|}}
 
-p % foo
 p % foo
 
-p(% foo )
 p(% foo )
 
 p eval "%\sfoo\s"
@@ -250,8 +248,6 @@ p 0x123456789abcdefABCDEF01
 p "Hi, my name is #{"Slim #{(4)>2?"Whitman":"Shady"} "}."
 p "Hi, my name is #{"Slim #{(4)<2?"Whitman":"Shady"} "}."
 
-p(String *Class)
-
 def String.*(right) [self,right] end
 def String.<<(right) [self,:<<,right] end
 def String./(right) [self,:/,right] end
@@ -463,7 +459,6 @@ p 0x80
 p ?p
 p 0.1
 p 0.8
-p 0.9
 p(-1)
 p %/p/
 p %Q[<LI>]
@@ -576,3 +571,18 @@ def a.b; end rescue b0
 def maybe(chance = 0.5)end
 return rval / precision
 0e0
+
+while false do end
+while false do; end
+until false do; end
+for i in [] do; end
+while (((((((((((((((((((((((((((((((false))))))))))))))))))))))))))))))) do; end
+while (((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((false))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))) do; end
+
+* = z
+tuf while buf=sread 4096
+{:n?=>1}
+{:n!=>1}
+{:n==>1}
+{:n=>1}
+case;when I; JIS;else case; when sjis__length; EJP ;else 55; end;end