rubylexer 0.6.2 → 0.7.0

data/{symboltable.rb → lib/rubylexer/symboltable.rb}

@@ -17,7 +17,7 @@
     Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
 =end
 
-
+class RubyLexer
 class SymbolTable
    def initialize
       #note: below Stack means Array (used as a stack)
@@ -42,6 +42,14 @@ class SymbolTable
       assert @locals_lists.last
    end
 
+   def names
+     @symbols.keys
+   end
+
+   def __locals_lists
+     @locals_lists
+   end
+
    def [](name)
       assert @locals_lists.last
       (stack=@symbols[name]) and stack.last
@@ -63,3 +71,4 @@ class SymbolTable
       return val
    end
 end
+end

data/{token.rb → lib/rubylexer/token.rb}

@@ -17,8 +17,9 @@
     Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
 =end
 
-require "rubycode"
 
+
+class RubyLexer
 #-------------------------
 class Token
    attr_accessor :ident
@@ -31,6 +32,8 @@ class Token
    end
    
    def error; end
+
+   def has_no_block?; false end
 end
 
 #-------------------------
@@ -45,22 +48,33 @@ end
 class KeywordToken < WToken   #also some operators
 
   #-----------------------------------
-  def set_callsite!
+  def set_callsite!  #not needed
     @callsite=true
   end
 
   #-----------------------------------
-  def callsite?
+  def callsite?  #not used
     @callsite ||= nil
   end
 
+
+  #-----------------------------------
+  def set_infix! 
+    @infix=true
+  end
+
+  #-----------------------------------
+  def infix? 
+    @infix ||= nil
+  end
+  def prefix?; !infix? end
+
   #-----------------------------------
   def has_end!
     assert self===RubyLexer::BEGINWORDS
     @has_end=true
   end
 
-
   #-----------------------------------
   def has_end?
     self===RubyLexer::BEGINWORDS and @has_end||=nil
@@ -69,6 +83,9 @@ end
 
 #-------------------------
 class OperatorToken < WToken
+  attr_accessor :unary
+  alias prefix? unary
+  def infix?; !prefix? end
 end
 
 
@@ -87,8 +104,8 @@ module TokenPat
    end
 end
 
-class String; include TokenPat; end
-class Regexp; include TokenPat; end
+class ::String; include TokenPat; end
+class ::Regexp; include TokenPat; end
 
 #-------------------------
 class VarNameToken < WToken
@@ -112,6 +129,7 @@ class MethNameToken  < Token # < SymbolToken
    def initialize(ident,offset=nil)
       @ident= (VarNameToken===ident)? ident.ident : ident
       @offset=offset
+      @has_no_block=false
    #   @char=''
    end
 
@@ -121,6 +139,14 @@ class MethNameToken  < Token # < SymbolToken
    def ===(pattern)
       pattern===@ident
    end
+
+   def has_no_block!
+     @has_no_block=true
+   end
+
+   def has_no_block?
+     @has_no_block
+   end
 end
 
 #-------------------------
@@ -137,14 +163,21 @@ class StringToken < Token
 
    attr_accessor :modifiers    #for regex only
    attr_accessor :elems
+   attr_accessor :line  #line on which the string ENDS
+   
+   def with_line(line)
+     @line=line
+     self
+   end
 
    def initialize(type='"',ident='')
       super(ident)
       type=="'" and type='"'
       @char=type
-      assert(@char[/^[\[{"`\/]$/])
+      assert @char[/^[\[{"`\/]$/]  #"
       @elems=[ident.dup]     #why .dup?
       @modifiers=nil
+      @line=nil
    end
 
    DQUOTE_ESCAPE_TABLE = [
@@ -161,7 +194,7 @@ class StringToken < Token
    SUFFIXERS={ '['=>"]",   '{'=>'}' }
 
    def to_s(transname=:transform)
-      assert(@char[/[\[{"`\/]/])
+      assert @char[/[\[{"`\/]/] #"
       #on output, all single-quoted strings become double-quoted
       assert(@elems.length==1)  if @char=='['
 
@@ -274,7 +307,7 @@ end
 
 #-------------------------
 class HerePlaceholderToken < WToken
-   attr_reader :termex, :quote, :ender
+   attr_reader :termex, :quote, :ender, :dash
    attr_accessor :unsafe_to_use, :string
    attr_accessor :bodyclass
 
@@ -292,14 +325,15 @@ class HerePlaceholderToken < WToken
    def ===(bogus); false end
 
    def to_s
-      if unsafe_to_use
-        result="<<"
-        result << if/[^a-z_0-9]/i===@ender
+      if @bodyclass==OutlinedHereBodyToken
+        result=if/[^a-z_0-9]/i===@ender
           %["#{@ender.gsub(/[\\"]/, '\\\\'+'\\&')}"]
         else
           @ender
         end
+        ["<<",@quote,@ender,@quote].to_s
       else
+        assert !unsafe_to_use
         @string.to_s
       end
    end
@@ -307,11 +341,22 @@ class HerePlaceholderToken < WToken
    def append s; @string.append s end
 
    def append_token tok; @string.append_token tok  end
+   
+   #def with_line(line) @string.line=line; self end
+   
+   def line; @string.line end
+   def line=line; @string.line=line end
+
+end
+
+#-------------------------
+module StillIgnoreToken 
 
 end
 
 #-------------------------
 class IgnoreToken < Token
+  include StillIgnoreToken
 end
 
 #-------------------------
@@ -338,15 +383,20 @@ class NoWsToken < ZwToken
   end
 end
 
-class ImplicitParamListStartToken < ZwToken
-  def explicit_form
-    '('
+class ImplicitParamListStartToken < KeywordToken
+  include StillIgnoreToken
+  def initialize(offset)
+    super("(",offset)
   end
+  def to_s; '' end
 end
-class ImplicitParamListEndToken < ZwToken
-  def explicit_form
-    ')'
+
+class ImplicitParamListEndToken < KeywordToken
+  include StillIgnoreToken
+  def initialize(offset)
+    super(")",offset)
   end
+  def to_s; '' end
 end
 
 class AssignmentRhsListStartToken < ZwToken
@@ -409,6 +459,7 @@ class HereBodyToken < IgnoreToken
     @headtok=headtok
   end
 
+  attr :headtok
 end
 
 #-------------------------
@@ -438,8 +489,7 @@ class OutlinedHereBodyToken < HereBodyToken
     assert HerePlaceholderToken===@headtok
     result=@headtok.string
     result=result.to_s(:simple_transform).match(/^"(.*)"$/m)[1]
-    return "\n" +
-           result +
+    return result +
            @headtok.ender +
            "\n"
   end
@@ -482,5 +532,7 @@ class DecoratorToken < SubitemToken
    def value()     @subitem    end
 end
 
+end
 
+require "rubylexer/rubycode"
 

data/{tokenprinter.rb → lib/rubylexer/tokenprinter.rb}

@@ -22,7 +22,7 @@
 require "assert"
 
 
-
+class RubyLexer
 
 #-------------------------------
 class SimpleTokenPrinter
@@ -34,7 +34,7 @@ class SimpleTokenPrinter
    TOKENSPERLINE=8
    TOKENSMAGICMAP="\n"+' '*(TOKENSPERLINE-1)
 
-   def pprint(tok) print(sprint(tok)) end
+   def pprint(tok,output=$stdout) output.print(sprint(tok)) end
 
    def sprint(tok)
       case tok
@@ -46,14 +46,19 @@ class SimpleTokenPrinter
    end
 end
 
-   class EscNl; def ws_munge(tp)
+   class EscNlToken; def ws_munge(tp)
       tp.lasttok=self
-      return to_s
+      return " \\\n"
    end end
    class FileAndLineToken; def ws_munge(tp)
+      result=''
+      
+      #faugh, doesn't fix it
+      #result= "\\\n"*(line-tp.lastfal.line) if StringToken===tp.lasttok 
+      
       tp.lasttok=self
       tp.lastfal=self
-      return ''
+      return result
    end end
    class Newline; def ws_munge(tp)
       tp.lasttok=self
@@ -62,14 +67,20 @@ end
    class IgnoreToken; def ws_munge(tp)
          #tp.latestline+= to_s.scan("\n").size
          tp.lasttok=self
-         unless tp.inws
+         result=unless tp.inws
            tp.inws=true
-           return ' '
+           ' '
+         else
+           ''
+         end
+         if ?= == @ident.to_s[0]
+           result+="\\\n"*@ident.to_s.scan(/\r\n?|\n\r?/).size
          end
-         return ''
+         
+         return result
    end end
    class OutlinedHereBodyToken; def ws_munge(tp)
-      nil
+     nil
    end end
    class ZwToken; def ws_munge(tp)
       case tp.showzw 
@@ -100,24 +111,35 @@ class KeepWsTokenPrinter
       @showzw=showzw
    end
 
-   def pprint(tok)
-
+   def pprint(tok,output=$stdout)
       @accum<<aprint(tok).to_s
-      if @accum.size>ACCUMSIZE or EoiToken===tok
-         print(@accum)
+      if (@accum.size>ACCUMSIZE and NewlineToken===tok) or EoiToken===tok
+         output.print(@accum)
          @accum=[]
       end
    end
 
    def aprint(tok)
+      if StringToken===tok or 
+         (HerePlaceholderToken===tok and 
+          tok.bodyclass!=OutlinedHereBodyToken
+         )
+            str_needs_escnls=(tok.line-@lastfal.line).nonzero?
+      end
       result=tok.ws_munge(self) and return result
 
+
       #insert extra ws unless an ambiguous op immediately follows
       #id or num, in which case ws would change the meaning
-      result=if (ZwToken===tok or NoWsToken===@lasttok)
-        tok.to_s
+      result=if (ZwToken===tok or NoWsToken===@lasttok or ImplicitParamListStartToken===tok or ImplicitParamListEndToken===tok)
+        tok
       else
-        [@sep.dup,tok.to_s]
+        [@sep.dup,tok]
+      end
+      
+      if str_needs_escnls
+        result=result.to_s
+        result.gsub!(/(["`\/])$/){ "\\\n"*str_needs_escnls+$1 }
       end
 
       @lasttok=tok
@@ -147,6 +169,7 @@ class KeepWsTokenPrinter
             /^[$@a-zA-Z_]/===@lasttok)) #lasttok is id or num?
    end
 end
+end
 
 #-------------------------------
 

data/lib/rubylexer/version.rb

@@ -0,0 +1,3 @@
+class RubyLexer
+  VERSION='0.7.0'
+end

data/test/code/dl_all_gems.rb

@@ -0,0 +1,43 @@
+require 'rubygems'
+Gem.manage_gems
+
+class Gem::SourceInfoCache
+  public :read_cache
+end
+
+if ARGV.empty?
+  limit=1.0/0
+else
+  limit=ARGV.first.to_i
+end
+
+gemdir="gems/"
+
+Dir.mkdir gemdir rescue nil
+total=0
+db4=[]
+db=Gem::SourceInfoCache.new.read_cache
+db.each_pair{|site,db2|
+  newest={}
+  db3=db2.source_index.instance_variable_get(:@gems)
+  db3.each_pair{|filename,gemdata|
+    version=gemdata.version
+    newest[gemdata.name]=[version,filename,site] unless
+      newest[gemdata.name] and newest[gemdata.name].first>=version
+  }
+  newest.each_pair{|name,triad| triad.shift }
+  db4.push newest
+}
+
+
+  db4.each{|hash| hash.each_pair{|filename,(fn,site)|
+#    fn=filename+".gem"
+    next if File.exist? gemdir+fn
+    fn+=".gem"
+    next if File.exist? gemdir+fn
+    url=site+"/gems/"+fn
+    puts url
+    system "wget #{url} -O "+gemdir+fn
+    total+=(4096.0+1.01*File.size(gemdir+fn)) rescue 0
+    exit if total>limit
+  }}

data/{testcode → test/code}/dumptokens.rb

@@ -1,17 +1,20 @@
-#!/usr/bin/env ruby -dw
+#!/usr/bin/env ruby
+$Debug=true
 require 'rubylexer'
 require 'getoptlong'
 
-
+#def puts(x) end
 
 #a Token#inspect that omits the object id
+class RubyLexer
 class Token
-  def inspect
-    ["#<",self.class,": ",instance_variables.sort.collect{|v| 
+  def strify
+    [self.class.name[/[^:]+$/],": ",instance_variables.sort.collect{|v| 
       [v,"=",instance_variable_get(v).inspect," "]
-    }].to_s.sub(/ $/,'>')
+    }].to_s
   end
 end
+end
 
 file=nil
 
@@ -28,11 +31,11 @@ file||=if name=ARGV.first
     File.open(name)
   else 
     name='-'
-    $stdout
+    $stdin
   end
 
 lexer=RubyLexer.new(name, file) 
-until EoiToken===(tok=lexer.get1token)
-  p tok
+until RubyLexer::EoiToken===(tok=lexer.get1token)
+  puts tok.strify
 end
-p tok #print eoi token
+puts tok.strify #print eoi token