rubylexer 0.7.6 → 0.7.7

data/lib/rubylexer/symboltable.rb

@@ -37,11 +37,32 @@ class SymbolTable
       list or raise "unbalanced end block"
       list.each_key {|sym|
          @symbols[sym].pop
-         @symbols[sym].empty? and @symbols[sym]=nil
+         @symbols[sym].empty? and @symbols.delete sym
       }
       assert @locals_lists.last
    end
 
+   def deep_copy
+     result=SymbolTable.allocate
+     new_symbols={}
+     @symbols.each_pair{|str,stack|
+       new_symbols[str.clone]=stack.map{|elem| elem.clone rescue elem }
+     }
+     new_locals_lists=[]
+     @locals_lists.each{|hash|
+       new_locals_lists.push({})
+       hash.each_pair{|str,bool|
+         new_locals_lists.last[str.dup]=bool
+       }
+     }
+     new_locals_lists.push({}) if new_locals_lists.empty?
+     result.instance_eval{
+       @symbols=new_symbols
+       @locals_lists=new_locals_lists
+     }
+     return result
+   end
+
    def names
      @symbols.keys
    end

data/lib/rubylexer/test/oneliners.rb

@@ -3,6 +3,23 @@ def x; yield end #this must be first!!!!
 #modifying the list of known local variables. it may be omitted
 #in cases where it is known that no local vars are defined.
 
+module A 55 end
+module Array([Array]).first::E; end
+module Array ([Array]).first::E; end
+module Array([Array]).first::E include M; end
+module Array ([Array]).first::E include M; end
+module Array([Array]).first::E include M end
+module Array ([Array]).first::E include M end
+x{a=Array; module a::Foo; 55 end}
+x{a=Array; module a::Foo include a; 55 end}
+x{a=Array; module a::Foo include a; a end}
+module a.b.c.d.e.f::Quux; YYY=534 end
+module[Array][0]::Foo; Bazz=556 end
+
+module(@a)::Foo; Bar=111 end
+
+module@x::Foo; Bar=1 end
+
 module A::B; end
 
 p a rescue b
@@ -499,6 +516,9 @@ def ~@; :foo end
 undef ~@
 alias ~@ non
 alias non ~@
+undef :~@
+alias :~@ non
+alias non :~@
 p :~@
 a.~@
 a::~@

data/lib/rubylexer/test/oneliners_1.9.rb

@@ -0,0 +1,146 @@
+module Ruby1_9OneLiners
+  EXPECT_NO_METHODS=[ #no errors either
+    '->a; h do 123 end',
+    '->{}',
+    '-> {}',
+    '->;{}',
+    '->(;){}',
+    '->a{}',
+    '->a {}',
+    '->a;{}',
+    '->(a;){}',
+    '->a,b{}',
+    '->a,b;{}',
+    '->a,b;c{}',
+    '->(a,b;){}',
+    '->(a,b;c){}',
+    '$f.($x,$y)',
+    '$f::($x,$y)',
+    '__ENCODING__',
+    '__ENCODING__ +"foo"',
+    'module __ENCODING__::A; end',
+    '/(?<foo>bar)/ =~ "baz"; foo +1',
+    "/(?'foo'bar)/ =~ 'baz'; foo +1",
+    '/\\\\(?<foo>bar)/ =~ "baz"; foo +1',
+    "/\\\\(?'foo'bar)/ =~ 'baz'; foo +1",
+    '/\\c\\(?<foo>bar)/ =~ "baz"; foo +1',
+    "/\\c\\(?'foo'bar)/ =~ 'baz'; foo +1",
+    '/\\\\c(?<foo>bar)/ =~ "baz"; foo +1',
+    "/\\\\c(?'foo'bar)/ =~ 'baz'; foo +1",
+    '/\\C-\\(?<foo>bar)/ =~ "baz"; foo +1',
+    "/\\C-\\(?'foo'bar)/ =~ 'baz'; foo +1",
+    '/\\\\C-(?<foo>bar)/ =~ "baz"; foo +1',
+    "/\\\\C-(?'foo'bar)/ =~ 'baz'; foo +1",
+    '/\\(?#(?<foo>bar))/ =~ "baz"; foo +1',
+    "/\\(?#(?'foo'bar))/ =~ 'baz'; foo +1",
+    '/\\[(?<foo>bar)]/ =~ "baz"; foo +1',
+    "/\\[(?'foo'bar)]/ =~ 'baz'; foo +1",
+
+    '/z(?<foo>bar)/ =~ "baz"; foo +1',
+    "/z(?'foo'bar)/ =~ 'baz'; foo +1",
+    '/z\\\\(?<foo>bar)/ =~ "baz"; foo +1',
+    "/z\\\\(?'foo'bar)/ =~ 'baz'; foo +1",
+    '/z\\c\\(?<foo>bar)/ =~ "baz"; foo +1',
+    "/z\\c\\(?'foo'bar)/ =~ 'baz'; foo +1",
+    '/z\\\\c(?<foo>bar)/ =~ "baz"; foo +1',
+    "/z\\\\c(?'foo'bar)/ =~ 'baz'; foo +1",
+    '/z\\C-\\(?<foo>bar)/ =~ "baz"; foo +1',
+    "/z\\C-\\(?'foo'bar)/ =~ 'baz'; foo +1",
+    '/z\\\\C-(?<foo>bar)/ =~ "baz"; foo +1',
+    "/z\\\\C-(?'foo'bar)/ =~ 'baz'; foo +1",
+    '/z\\(?#(?<foo>bar))/ =~ "baz"; foo +1',
+    "/z\\(?#(?'foo'bar))/ =~ 'baz'; foo +1",
+    '/z\\[(?<foo>bar)]/ =~ "baz"; foo +1',
+    "/z\\[(?'foo'bar)]/ =~ 'baz'; foo +1",
+  ]
+
+  EXPECT_1_METHOD=[
+    'def self.foo; 1 end',
+    '->{ foo=1 }; foo',
+    '->do foo=1 end; foo',
+    'def __FILE__.foo; 1 end',
+    'def __LINE__.foo; 1 end',
+    'def a(b,*c,d) 1 end',
+    'def a(*c,d) 1 end',
+    'def Z::a(b,*c,d) 1 end',
+    'def Z::a(*c,d) 1 end',
+    'a{|b,*c,d| 1 }',
+    'a{|*c,d| 1 }',
+    'def a(b,(x,y),d) 1 end',
+    'def a((x,y),d) 1 end',
+    'def Z::a(b,(x,y),d) 1 end',
+    'def Z::a((x,y),d) 1 end',
+    'a{|b,(x,y),d| 1 }',
+    'a{|(x,y),d| 1 }',
+    'def a(b,(x,*y),d) 1 end',
+    'def a((x,*y),d) 1 end',
+    'def Z::a(b,(x,*y),d) 1 end',
+    'def Z::a((x,*y),d) 1 end',
+    'a{|b,(x,*y),d| 1 }',
+    'a{|(x,*y),d| 1 }',
+    'def a(b,(x,(y,z)),d) 1 end',
+    'def a((x,(y,z)),d) 1 end',
+    'def Z::a(b,(x,(y,z)),d) 1 end',
+    'def Z::a((x,(y,z)),d) 1 end',
+    'a{|b,(x,(y,z)),d| 1 }',
+    'a{|(x,(y,z)),d| 1 }',
+    'module __ENCODING__::A include B; end',
+    'def __ENCODING__; 342 end',
+    'def __ENCODING__.foo; 1 end',
+    'def Z::__ENCODING__; 342 end',
+    #'def Z::__ENCODING__.foo; 1 end', #oops, 2 methods here
+    '/\\(?<foo>bar)/ =~ "baz"; foo +1',
+    "/\\(?'foo'bar)/ =~ 'baz'; foo +1",
+    '/\\c(?<foo>bar)/ =~ "baz"; foo +1',
+    "/\\c(?'foo'bar)/ =~ 'baz'; foo +1",
+    '/\\C-(?<foo>bar)/ =~ "baz"; foo +1',
+    "/\\C-(?'foo'bar)/ =~ 'baz'; foo +1",
+    '/\\M-(?<foo>bar)/ =~ "baz"; foo +1',
+    "/\\M-(?'foo'bar)/ =~ 'baz'; foo +1",
+    '/\\\\\\(?<foo>bar)/ =~ "baz"; foo +1',
+    "/\\\\\\(?'foo'bar)/ =~ 'baz'; foo +1",
+    '/\\\\c\\(?<foo>bar)/ =~ "baz"; foo +1',
+    "/\\\\c\\(?'foo'bar)/ =~ 'baz'; foo +1",
+    '/(?#(?<foo>bar))/ =~ "baz"; foo +1',
+    "/(?#(?'foo'bar))/ =~ 'baz'; foo +1",
+    '/[(?<foo>bar)]/ =~ "baz"; foo +1',
+    "/[(?'foo'bar)]/ =~ 'baz'; foo +1",
+    '/[qwe&&[^ty](?<foo>bar)]/ =~ "baz"; foo +1',
+    "/[qwe&&[^ty](?'foo'bar)]/ =~ 'baz'; foo +1",
+    '/[qwe&&[^(?<foo>bar)]]/ =~ "baz"; foo +1',
+    "/[qwe&&[^(?'foo'bar)]]/ =~ 'baz'; foo +1",
+    '/[qwe&&[^ty&&[^uip]](?<foo>bar)]/ =~ "baz"; foo +1',
+    "/[qwe&&[^ty&&[^uip]](?'foo'bar)]/ =~ 'baz'; foo +1",
+    '/[qwe&&[^ty&&[^uip](?<foo>bar)]]/ =~ "baz"; foo +1',
+    "/[qwe&&[^ty&&[^uip](?'foo'bar)]]/ =~ 'baz'; foo +1",
+    '/[qwe&&[^ty&&[^uip(?<foo>bar)]]]/ =~ "baz"; foo +1',
+    "/[qwe&&[^ty&&[^uip(?'foo'bar)]]]/ =~ 'baz'; foo +1",
+
+    '/z\\(?<foo>bar)/ =~ "baz"; foo +1',
+    "/z\\(?'foo'bar)/ =~ 'baz'; foo +1",
+    '/z\\c(?<foo>bar)/ =~ "baz"; foo +1',
+    "/z\\c(?'foo'bar)/ =~ 'baz'; foo +1",
+    '/z\\C-(?<foo>bar)/ =~ "baz"; foo +1',
+    "/z\\C-(?'foo'bar)/ =~ 'baz'; foo +1",
+    '/z\\M-(?<foo>bar)/ =~ "baz"; foo +1',
+    "/z\\M-(?'foo'bar)/ =~ 'baz'; foo +1",
+    '/z\\\\\\(?<foo>bar)/ =~ "baz"; foo +1',
+    "/z\\\\\\(?'foo'bar)/ =~ 'baz'; foo +1",
+    '/z\\\\c\\(?<foo>bar)/ =~ "baz"; foo +1',
+    "/z\\\\c\\(?'foo'bar)/ =~ 'baz'; foo +1",
+    '/z(?#(?<foo>bar))/ =~ "baz"; foo +1',
+    "/z(?#(?'foo'bar))/ =~ 'baz'; foo +1",
+    '/z[(?<foo>bar)]/ =~ "baz"; foo +1',
+    "/z[(?'foo'bar)]/ =~ 'baz'; foo +1",
+    '/z[qwe&&[^ty](?<foo>bar)]/ =~ "baz"; foo +1',
+    "/z[qwe&&[^ty](?'foo'bar)]/ =~ 'baz'; foo +1",
+    '/z[qwe&&[^(?<foo>bar)]]/ =~ "baz"; foo +1',
+    "/z[qwe&&[^(?'foo'bar)]]/ =~ 'baz'; foo +1",
+    '/z[qwe&&[^ty&&[^uip]](?<foo>bar)]/ =~ "baz"; foo +1',
+    "/z[qwe&&[^ty&&[^uip]](?'foo'bar)]/ =~ 'baz'; foo +1",
+    '/z[qwe&&[^ty&&[^uip](?<foo>bar)]]/ =~ "baz"; foo +1',
+    "/z[qwe&&[^ty&&[^uip](?'foo'bar)]]/ =~ 'baz'; foo +1",
+    '/z[qwe&&[^ty&&[^uip(?<foo>bar)]]]/ =~ "baz"; foo +1',
+    "/z[qwe&&[^ty&&[^uip(?'foo'bar)]]]/ =~ 'baz'; foo +1",
+ ]
+end

data/lib/rubylexer/test/testcases.rb

@@ -8,9 +8,13 @@ module TestCases
   ILLEGAL_STANZAS=IO.read(rldir+'/rubylexer/test/illegal_stanzas.rb').split("\n\n").grep(/./).reverse
 
   datadir=$:.find{|dir| File.exist? dir+'/../test/data/p.rb' }
-  FILENAMES=Dir[datadir+'/../test/data/*'].reject{|fn| File.directory? fn}
+  FILENAMES=Dir[datadir+'/../test/data/*.rb'].reject{|fn| File.directory? fn}
   FILES=FILENAMES.map{|fn| File.read fn }
 
+  ILLEGAL_FILENAMES=(Dir[datadir+'/../test/data/*']-Dir[datadir+'/../test/data/*.rb'])
+  ILLEGAL_FILENAMES.reject!{|fn| File.directory? fn}
+  ILLEGAL_FILES=ILLEGAL_FILENAMES.map{|fn| File.read fn }
+
   TESTCASES=ONELINERS+STANZAS+FILES
-  ILLEGAL_TESTCASES=ILLEGAL_ONELINERS+ILLEGAL_STANZAS
+  ILLEGAL_TESTCASES=ILLEGAL_ONELINERS+ILLEGAL_STANZAS+ILLEGAL_FILES
 end

data/lib/rubylexer/token.rb

@@ -63,17 +63,24 @@ class KeywordToken < WToken   #also some operators
     @callsite if defined? @callsite
   end
 
+  attr_accessor :value
 
   #-----------------------------------
   def set_infix! 
     @infix=true
   end
 
+  #-----------------------------------
+  def unary= flag
+    @infix=!flag
+  end
+
   #-----------------------------------
   def infix? 
     @infix ||= nil
   end
   def prefix?; !infix? end
+  alias unary prefix?
 
   #-----------------------------------
   def has_end!
@@ -103,9 +110,16 @@ end
 #-------------------------
 class OperatorToken < WToken
   attr_accessor :unary
+  attr_writer :as
   alias prefix? unary
   def infix?; !prefix? end
 
+  def as
+    return @as if defined? @as
+    if tag and ident[/^[,*&]$/]
+      tag.to_s+ident
+    end
+  end       
 end
 
 
@@ -212,6 +226,8 @@ class StringToken < Token
 
    attr_accessor :open #exact sequence of chars used to start the str
    attr_accessor :close #exact seq of (1) char to stop the str
+
+   attr_accessor :lvars #names used in named backrefs if this is a regex
    
    def with_line(line)
      @line=line
@@ -255,16 +271,16 @@ class StringToken < Token
       ender=close
       elems.each{|e|
         case e
-        when String: result<<e
+        when String; result<<e
 #        strfrag=translate_escapes strfrag if RubyLexer::FASTER_STRING_ESCAPES
 #        result << send(transname,strfrag,starter,ender)
-        when VarNameToken: 
+        when VarNameToken;
           if /^[$@]/===e.to_s
             result << '#' + e.to_s
           else 
             result << "\#{#{e}}"
           end
-        when RubyCode: result << '#' + e.to_s
+        when RubyCode; result << '#' + e.to_s
         else fail
         end
       }
@@ -429,7 +445,7 @@ class HerePlaceholderToken < WToken
 
       #@termex=/^#{'[\s\v]*' if dash}#{Regexp.escape ender}$/
       @termex=Regexp.new \
-         ["^", ('[\s\v]*' if dash), Regexp.escape(ender), "$"].to_s
+         ["^", ('[\s\v]*' if dash), Regexp.escape(ender), "$"].join
       @bodyclass=HereBodyToken
    end
 
@@ -442,7 +458,7 @@ class HerePlaceholderToken < WToken
         else
           @ender
         end
-        return ["<<",@dash,@quote_real&&@quote,result,@quote_real&&@quote].to_s
+        return ["<<",@dash,@quote_real&&@quote,result,@quote_real&&@quote].join
 #      else
 #        assert !unsafe_to_use
 #        return @string.to_s
@@ -557,7 +573,7 @@ EndDefHeaderToken=EndHeaderToken
 
 #-------------------------
 class EscNlToken < IgnoreToken
-   def initialize(filename,linenum,ident="\\\n",offset=nil)
+   def initialize(ident,offset,filename=nil,linenum=nil)
       super(ident,offset)
       #@char='\\'
       @filename=filename

data/lib/rubylexer/tokenprinter.rb

@@ -87,9 +87,9 @@ end
    end end
    class ZwToken; def ws_munge(tp)
       case tp.showzw 
-      when 2: explicit_form_all
-      when 1: explicit_form
-      when 0: nil
+      when 2; explicit_form_all
+      when 1; explicit_form
+      when 0; nil
       else raise 'unknown showzw'
       end
    end end
@@ -117,7 +117,7 @@ class KeepWsTokenPrinter
    def pprint(tok,output=$stdout)
       @accum<<aprint(tok).to_s
       if (@accum.size>ACCUMSIZE and NewlineToken===tok) or EoiToken===tok
-         output.print(@accum)
+         output.print(@accum.join)
          @accum=[]
       end
    end
@@ -139,10 +139,10 @@ class KeepWsTokenPrinter
       result=
       case tok
       when ZwToken,EoiToken,NoWsToken, HereBodyToken, NewlineToken,
-           ImplicitParamListStartToken,ImplicitParamListEndToken:
+           ImplicitParamListStartToken,ImplicitParamListEndToken;
         tok
       else
-        [@sep.dup,tok]
+        @sep.dup<<tok.to_s
       end unless NoWsToken===lasttok
       
       if str_needs_escnls

data/lib/rubylexer/version.rb

@@ -1,3 +1,3 @@
 class RubyLexer
-  VERSION='0.7.6'
+  VERSION='0.7.7'
 end

data/rubylexer.gemspec

@@ -0,0 +1,40 @@
+# -*- encoding: utf-8 -*-
+
+require "#{File.dirname(__FILE__)}/lib/rubylexer/version"
+RubyLexer::Description=open("README.txt"){|f| f.read[/^==+ ?description[^\n]*?\n *\n?(.*?\n *\n.*?)\n *\n/im,1] }
+RubyLexer::Latest_changes="###"+open("History.txt"){|f| f.read[/\A===(.*?)(?====)/m,1] }
+
+Gem::Specification.new do |s|
+  s.name = "rubylexer"
+  s.version = RubyLexer::VERSION
+  s.date = Time.now.strftime("%Y-%m-%d")
+  s.authors = ["Caleb Clausen"]
+  s.email = %q{caleb (at) inforadical (dot) net}
+  s.summary = "RubyLexer is a lexer library for Ruby, written in Ruby."
+  s.description = RubyLexer::Description
+  s.homepage = %{http://github.com/coatl/rubylexer}
+  s.rubyforge_project = %q{rubylexer}
+
+  s.files = `git ls-files`.split
+  s.test_files = %w[test/test_all.rb]
+  s.require_paths = ["lib"]
+  s.extra_rdoc_files = ["README.txt", "COPYING"]
+  s.has_rdoc = true
+  s.rdoc_options = %w[--main README.txt]
+
+  s.rubygems_version = %q{1.3.0}
+  s.required_rubygems_version = Gem::Requirement.new(">= 0") if s.respond_to? :required_rubygems_version=
+
+  if s.respond_to? :specification_version then
+    current_version = Gem::Specification::CURRENT_SPECIFICATION_VERSION
+    s.specification_version = 2
+
+    if Gem::Version.new(Gem::RubyGemsVersion) >= Gem::Version.new('1.2.0') then
+      s.add_runtime_dependency("sequence", [">= 0.2.3"])
+    else
+      s.add_dependency("sequence", [">= 0.2.3"])
+    end
+  else
+    s.add_dependency("sequence", [">= 0.2.3"])
+  end
+end

data/test/code/coloruby.rb

@@ -0,0 +1,154 @@
+require 'rubygems'
+require 'rubylexer'
+require 'term/ansicolor'
+
+
+def coloruby file,fd=open(file)
+  lexer=RubyLexer.new(file,fd)
+  begin
+    token=lexer.get1token
+    print token.colorize
+  end until RubyLexer::EoiToken===token
+ensure
+  print Term::ANSIColor.reset
+end
+
+class RubyLexer
+  class Token
+    include Term::ANSIColor
+
+    def colorize
+      color+ident.to_s
+    end
+  end
+
+  class VarNameToken
+    alias color blue
+  end
+
+  class MethNameToken
+    alias color green
+  end
+
+  class OperatorToken
+    alias color cyan
+  end
+
+  class KeywordToken
+    def colorize
+      if /[^a-z]/i===ident
+        yellow+ident
+      else
+        red+ident
+      end
+    end
+  end
+
+  class WsToken
+    def colorize
+      black+ident
+    end
+  end
+
+  class NewlineToken
+    alias color black
+  end
+
+  class ZwToken
+    def colorize; '' end
+  end
+
+  class FileAndLineToken
+    def colorize; '' end
+  end
+
+  class ImplicitParamListStartToken
+    def colorize; '' end    
+  end
+
+  class ImplicitParamListEndToken
+    def colorize; '' end    
+  end
+
+  class IgnoreToken
+    def colorize
+      dark+green+ident+reset
+    end
+  end
+ 
+  class EscNlToken
+    def colorize
+      yellow+ident
+    end
+  end
+
+  class NumberToken
+    def colorize
+      dark+blue+ident+reset
+    end
+  end
+
+  class SymbolToken
+    def colorize
+      if StringToken===ident
+        dark+blue+':'+ident.colorize+reset
+      else
+        dark+blue+ident+reset
+      end
+    end
+  end
+
+  class StringToken
+    def colorize
+      magenta+open+
+        elems.map{|elem|
+          if String===elem
+            magenta+elem
+          else
+            yellow+'#'+elem.colorize
+          end
+        }.join+
+      magenta+close
+    end
+  end
+
+  class RubyCode
+    def colorize
+      ident.map{|tok| tok.colorize }.join
+    end
+  end
+
+  class HerePlaceholderToken
+    def colorize
+      "#{blue}<<#{'-' if @dash}#{ender}"
+    end
+  end
+
+  class HereBodyToken
+    def colorize
+      headtok.string.colorize
+    end
+  end
+
+  class EoiToken
+    def colorize
+      return '' #gyaah, @offset has off by one errors
+      data=begin
+             @file.pos=@offset-1
+             @file.read
+           rescue
+             @file[@offset-1..-1]
+           end
+      dark+green+data+reset
+    end
+  end
+end
+
+if __FILE__==$0
+  if ARGV.first=='-e'
+    coloruby ARGV[1],ARGV[1]
+  else
+    coloruby ARGV.first
+  end
+end
+