rubylexer 0.7.0 → 0.7.1

data/lib/rubylexer/token.rb

@@ -1,6 +1,6 @@
-=begin copyright
+=begin legal crap
     rubylexer - a ruby lexer written in ruby
-    Copyright (C) 2004,2005  Caleb Clausen
+    Copyright (C) 2004,2005,2008  Caleb Clausen
 
     This library is free software; you can redistribute it and/or
     modify it under the terms of the GNU Lesser General Public
@@ -25,6 +25,8 @@ class Token
    attr_accessor :ident
    alias to_s ident
    attr_accessor :offset #file offset of start of this token
+   attr_accessor :as #if defined, a KeywordToken which this token stands in for.
+   attr_accessor :allow_ooo_offset #hack
 
    def initialize(ident,offset=nil)
       @ident=ident
@@ -48,8 +50,8 @@ end
 class KeywordToken < WToken   #also some operators
 
   #-----------------------------------
-  def set_callsite!  #not needed
-    @callsite=true
+  def set_callsite!(x=true)  #not needed
+    @callsite=x
   end
 
   #-----------------------------------
@@ -79,6 +81,17 @@ class KeywordToken < WToken   #also some operators
   def has_end?
     self===RubyLexer::BEGINWORDS and @has_end||=nil
   end
+
+  attr_accessor :comma_type
+
+  def has_no_block!
+     @has_no_block=true
+  end
+
+  def has_no_block?
+     @has_no_block
+  end
+
 end
 
 #-------------------------
@@ -109,6 +122,8 @@ class ::Regexp; include TokenPat; end
 
 #-------------------------
 class VarNameToken < WToken
+  attr_accessor :lvar_type
+  attr_accessor :in_def
 end
 
 #-------------------------
@@ -118,15 +133,30 @@ end
 
 #-------------------------
 class SymbolToken < Token
-   def initialize(ident,offset=nil)
-      super ":#{ident}", offset
-   #   @char=':'
-   end
+  attr_accessor :open,:close
+  attr :raw
+  def initialize(ident,offset=nil,starter=':')
+    @raw=ident
+    str=ident.to_s
+    str[0,2]='' if /\A%s/===str
+    super starter+str, offset
+    @open=":"
+    @close=""
+#   @char=':'
+
+  end
+
+  def to_s
+    return @ident
+    raw=@raw.to_s
+    raw=raw[1...-1] if StringToken===@raw
+    @open+raw+@close 
+  end
 end
 
 #-------------------------
 class MethNameToken  < Token # < SymbolToken
-   def initialize(ident,offset=nil)
+   def initialize(ident,offset=nil,bogus=nil)
       @ident= (VarNameToken===ident)? ident.ident : ident
       @offset=offset
       @has_no_block=false
@@ -155,6 +185,7 @@ class NewlineToken < Token
       super(nlstr,offset)
       #@char=''
    end
+   def as; ';' end
 end
 
 #-------------------------
@@ -164,6 +195,10 @@ class StringToken < Token
    attr_accessor :modifiers    #for regex only
    attr_accessor :elems
    attr_accessor :line  #line on which the string ENDS
+   attr_accessor :bs_handler
+
+   attr_accessor :open #exact sequence of chars used to start the str
+   attr_accessor :close #exact seq of (1) char to stop the str
    
    def with_line(line)
      @line=line
@@ -193,26 +228,39 @@ class StringToken < Token
    PREFIXERS={ '['=>"%w[", '{'=>'%W{' }
    SUFFIXERS={ '['=>"]",   '{'=>'}' }
 
-   def to_s(transname=:transform)
+   def has_str_inc?
+     elems.size>1 or RubyCode===elems.first
+   end
+
+   def to_s transname=:transform
       assert @char[/[\[{"`\/]/] #"
       #on output, all single-quoted strings become double-quoted
       assert(@elems.length==1)  if @char=='['
 
-      result=(PREFIXERS[@char] or @char).dup
+      result=open.dup
       starter=result[-1,1]
-      ender=(SUFFIXERS[@char] or @char).dup
-      0.step(@elems.length-1,2) { |i|
-         strfrag=@elems[i].dup
-         result << send(transname,strfrag,starter,ender)
-
-         if e=@elems[i+1]
-            assert(e.kind_of?(RubyCode))
+      ender=close
+      elems.each{|e|
+        case e
+        when String: result<<e
+#        strfrag=translate_escapes strfrag if RubyLexer::FASTER_STRING_ESCAPES
+#        result << send(transname,strfrag,starter,ender)
+        when VarNameToken: 
+          if /^[$@]/===e.to_s
             result << '#' + e.to_s
-         end
+          else 
+            result << "\#{#{e}}"
+          end
+        when RubyCode: result << '#' + e.to_s
+        else fail
+        end
       }
       result << ender
 
-      modifiers and result << modifiers #regex only
+      if @char=='/'
+        result << modifiers if modifiers #regex only
+        result="%r"+result if RubyLexer::WHSPLF[result[1,1]]
+      end
 
       return result
    end
@@ -231,18 +279,18 @@ class StringToken < Token
    end
 
    def append(glob)
-      assert @elems.last.kind_of?(String)
+      #assert @elems.last.kind_of?(String)
       case glob
       when String,Integer then append_str! glob
       when RubyCode then append_code! glob
       else raise "bad string contents: #{glob}, a #{glob.class}"
       end
-      assert @elems.last.kind_of?(String)
+      #assert @elems.last.kind_of?(String)
    end
 
    def append_token(strtok)
       assert @elems.last.kind_of?(String)
-      assert strtok.elems.last.kind_of?(String)
+      #assert strtok.elems.last.kind_of?(String)
       assert strtok.elems.first.kind_of?(String)
 
       @elems.last << strtok.elems.shift
@@ -256,17 +304,60 @@ class StringToken < Token
       assert((!@modifiers or !strtok.modifiers))
       @modifiers||=strtok.modifiers
 
-      assert @elems.last.kind_of?(String)
+      #assert @elems.last.kind_of?(String)
+
+      @bs_handler ||=strtok.bs_handler
 
       return self
    end
 
+   def translate_escapes(str)
+     rl=RubyLexer.new("(string escape translation hack...)",'')
+     result=str.dup
+     seq=result.to_sequence
+     rl.instance_eval{@file=seq}
+     repls=[]
+     i=0
+     #ugly ugly ugly
+     while i<result.size and bs_at=result.index(/\\./m,i) 
+         seq.pos=$~.end(0)-1
+         ch=rl.send(bs_handler,"\\",@open[-1,1],@close)
+         result[bs_at...seq.pos]=ch
+         i=bs_at+ch.size
+     end
+
+     return  result
+   end
+
 private
+   UNESC_DELIMS={}
+
    #simpler transform, preserves original exactly
    def simple_transform(strfrag,starter,ender)
-      #assert('[{/'[@char])
-      #strfrag.gsub!(/#([{$@])/,'\\#\\1') unless @char=='['
-      strfrag.gsub!(Regexp.new("[\\"+starter+"\\"+ender+"]"), '\\\\\&')
+      assert('[{/'[@char])
+      #strfrag.gsub!(/(\A|[^\\])(?:\\\\)*\#([{$@])/){$1+'\\#'+$2} unless @char=='['
+      delimchars=Regexp.quote starter+ender
+      delimchars+=Regexp.quote("#") unless @char=='['  #escape beginning of string iterpolations
+
+      #i think most or all of this method is useless now...
+
+      #escape curly brace in string interpolations (%W only)
+      strfrag.gsub!('#{', '#\\{') if @char=='{'
+
+      ckey=starter+ender
+      unesc_delim=
+        UNESC_DELIMS[ckey]||=
+          /(\A|[^\\](?:\\\\)*)([#{delimchars}]+)/
+#          /(\\)([^#{delimchars}#{RubyLexer::WHSPLF}]|\Z)/
+      
+      #an even number (esp 0) of backslashes before delim becomes escaped delim
+      strfrag.gsub!(unesc_delim){ 
+        pre=$1; toesc=$2
+        pre+toesc.gsub(/(.)/){ "\\"+$1 }
+      }
+
+      #no need to double backslashes anymore... they should come pre-doubled
+
       return strfrag
    end
 
@@ -286,15 +377,21 @@ private
    end
 
    def append_str!(str)
-      assert @elems.last.kind_of?(String)
-      @elems.last << str
+      if @elems.last.kind_of?(String)
+        @elems.last << str
+      else
+        @elems << str
+      end
       @ident << str
       assert @elems.last.kind_of?(String)
    end
 
    def append_code!(code)
-      assert @elems.last.kind_of?(String)
-      @elems.concat [code, '']
+      if @elems.last.kind_of?(String)
+      else
+        @elems.push ''
+      end
+      @elems.push code,''
       @ident <<  "\#{#{code}}"
       assert @elems.last.kind_of?(String)
    end
@@ -310,9 +407,10 @@ class HerePlaceholderToken < WToken
    attr_reader :termex, :quote, :ender, :dash
    attr_accessor :unsafe_to_use, :string
    attr_accessor :bodyclass
+   attr_accessor :open, :close
 
-   def initialize(dash,quote,ender)
-      @dash,@quote,@ender=dash,quote,ender
+   def initialize(dash,quote,ender,quote_real=true)
+      @dash,@quote,@ender,@quote_real=dash,quote,ender,quote_real
       @unsafe_to_use=true
       @string=StringToken.new
 
@@ -325,17 +423,17 @@ class HerePlaceholderToken < WToken
    def ===(bogus); false end
 
    def to_s
-      if @bodyclass==OutlinedHereBodyToken
+#      if @bodyclass==OutlinedHereBodyToken
         result=if/[^a-z_0-9]/i===@ender
-          %["#{@ender.gsub(/[\\"]/, '\\\\'+'\\&')}"]
+          @ender.gsub(/[\\"]/, '\\\\'+'\\&')
         else
           @ender
         end
-        ["<<",@quote,@ender,@quote].to_s
-      else
-        assert !unsafe_to_use
-        @string.to_s
-      end
+        return ["<<",@dash,@quote_real&&@quote,result,@quote_real&&@quote].to_s
+#      else
+#        assert !unsafe_to_use
+#        return @string.to_s
+#      end
    end
 
    def append s; @string.append s end
@@ -374,6 +472,7 @@ class ZwToken < IgnoreToken
   def explicit_form_all; explicit_form end
 end
 
+#-------------------------
 class NoWsToken < ZwToken
   def explicit_form_all
     "#nows#"
@@ -383,34 +482,41 @@ class NoWsToken < ZwToken
   end
 end
 
+#-------------------------
 class ImplicitParamListStartToken < KeywordToken
   include StillIgnoreToken
   def initialize(offset)
     super("(",offset)
   end
   def to_s; '' end
+  def as; "(" end
 end
 
+#-------------------------
 class ImplicitParamListEndToken < KeywordToken
   include StillIgnoreToken
   def initialize(offset)
     super(")",offset)
   end
   def to_s; '' end
+  def as; ")" end
 end
 
+#-------------------------
 class AssignmentRhsListStartToken < ZwToken
   def explicit_form
     '*['
   end
 end
 
+#-------------------------
 class AssignmentRhsListEndToken < ZwToken
   def explicit_form
     ']'
   end
 end
 
+#-------------------------
 class KwParamListStartToken  < ZwToken
   def explicit_form_all
     "#((#"
@@ -420,6 +526,7 @@ class KwParamListStartToken  < ZwToken
   end
 end
 
+#-------------------------
 class KwParamListEndToken  < ZwToken
   def explicit_form_all
     "#))#"
@@ -429,6 +536,11 @@ class KwParamListEndToken  < ZwToken
   end
 end
 
+#-------------------------
+class EndDefHeaderToken < ZwToken
+  def as; ";" end
+end
+
 #-------------------------
 class EscNlToken < IgnoreToken
    def initialize(filename,linenum,ident="\\\n",offset=nil)
@@ -440,7 +552,7 @@ class EscNlToken < IgnoreToken
 end
 
 #-------------------------
-class EoiToken < IgnoreToken
+class EoiToken < Token
    attr :file
    alias :pos :offset
 
@@ -453,18 +565,29 @@ end
 #-------------------------
 class HereBodyToken < IgnoreToken
   #attr_accessor :ender
-  def initialize(headtok)
+  attr_accessor :open,:close
+  def initialize(headtok,linecount)
     assert HerePlaceholderToken===headtok
     super(headtok.string,headtok.string.offset)
     @headtok=headtok
+    @linecount=linecount
+  end
+
+  def line
+    @ident.line
+  end
+
+  def to_s
+    @ident.to_s
   end
 
   attr :headtok
+  attr :linecount #num lines here body spans (including terminator)
 end
 
 #-------------------------
 class FileAndLineToken < IgnoreToken
-   attr :line
+   attr_accessor :line
 
    def initialize(ident,line,offset=nil)
 

data/lib/rubylexer/tokenprinter.rb

@@ -1,6 +1,6 @@
-=begin copyright
+=begin legal crap
     rubylexer - a ruby lexer written in ruby
-    Copyright (C) 2004,2005  Caleb Clausen
+    Copyright (C) 2004,2005,2008  Caleb Clausen
 
     This library is free software; you can redistribute it and/or
     modify it under the terms of the GNU Lesser General Public
@@ -79,6 +79,9 @@ end
          
          return result
    end end
+   class HereBodyToken; def ws_munge(tp) #experimental
+     nil
+   end end
    class OutlinedHereBodyToken; def ws_munge(tp)
      nil
    end end
@@ -121,21 +124,26 @@ class KeepWsTokenPrinter
 
    def aprint(tok)
       if StringToken===tok or 
-         (HerePlaceholderToken===tok and 
-          tok.bodyclass!=OutlinedHereBodyToken
-         )
+          HereBodyToken===tok
+#         (HerePlaceholderToken===tok and 
+#          tok.bodyclass!=OutlinedHereBodyToken
+#         )
             str_needs_escnls=(tok.line-@lastfal.line).nonzero?
-      end
+      end if false
       result=tok.ws_munge(self) and return result
 
 
       #insert extra ws unless an ambiguous op immediately follows
       #id or num, in which case ws would change the meaning
-      result=if (ZwToken===tok or NoWsToken===@lasttok or ImplicitParamListStartToken===tok or ImplicitParamListEndToken===tok)
+      result=tok
+      result=
+      case tok
+      when ZwToken,EoiToken,NoWsToken, HereBodyToken, NewlineToken,
+           ImplicitParamListStartToken,ImplicitParamListEndToken:
         tok
       else
         [@sep.dup,tok]
-      end
+      end unless NoWsToken===lasttok
       
       if str_needs_escnls
         result=result.to_s