rubylexer 0.7.1 → 0.7.2

data/History.txt

@@ -1,4 +1,19 @@
-=== 0.7.1/10-29-2008
+=== 0.7.2/10-12-2008
+* 12 Minor Enhancements:
+  * a new context for then kw expected
+  * disable all backtracking when scanning string interiors
+  * ternary flag distinguishes ternary : from other uses
+  * EndDefHeaderToken renamed to EndHeaderToken
+  * ^ now gets its own scanning method
+  * correct # of parens emitted after of kw used as (or like) method
+  * more special casing for break return and next
+  * abort_noparens! now better if When context on the stack
+  * semicolon may now be omitted after module header
+  * { and } in BEGIN/END expr masquerade as do and end
+  * trying to make 'rake test' work right
+  * certain other changes of no importance whatsoever
+
+=== 0.7.1/8-29-2008
 * 6 Major Enhancements:
   * handling of empty string fragments now more closely mirrors ruby; this resolves many warnings
   * yet more hacks in aid of string inclusions

data/Rakefile

@@ -15,7 +15,7 @@ require 'lib/rubylexer/version.rb'
      _.email = "rubylexer-owner @at@ inforadical .dot. net"
      _.url = ["http://rubylexer.rubyforge.org/", "http://rubyforge.org/projects/rubylexer/"]
      _.extra_deps << ['sequence', '>= 0.2.0']
-     _.test_globs=["test/{code/*,data/*rb*,results/}"]
+     _.test_globs=["test/code/regression.rb"]
      _.description=desc
      _.summary=desc[/\A[^.]+\./]
      _.spec_extras={:bindir=>''}

data/lib/rubylexer.rb

@@ -83,7 +83,7 @@ class RubyLexer
          ?, => :comma,
          ?; => :semicolon,
 
-         ?^ => :biop,
+         ?^ => :caret,
          ?~ => :tilde,
          ?= => :equals,
          ?! => :exclam,
@@ -130,6 +130,9 @@ class RubyLexer
       @in_def_name=false
       @last_operative_token=nil
       @last_token_maybe_implicit=nil
+      @enable_macro=nil
+      @base_file=nil
+      @progress_thread=nil
 
       @toptable=CharHandler.new(self, :illegal_char, CHARMAPPINGS)
 
@@ -442,12 +445,12 @@ private
    end
    
    #-----------------------------------
-   def in_lvar_define_state
+   def in_lvar_define_state lasttok=@last_operative_token
      #@defining_lvar is a hack
      @defining_lvar or case ctx=@parsestack.last
        #when ForSMContext; ctx.state==:for
        when RescueSMContext
-         @last_operative_token.ident=="=>" and @file.match? /\A[\s\v]*([:;#\n]|then[^a-zA-Z0-9_])/m
+         lasttok.ident=="=>" and @file.match? /\A[\s\v]*([:;#\n]|then[^a-zA-Z0-9_])/m
        #when BlockParamListLhsContext; true
      end 
    end
@@ -472,7 +475,7 @@ private
 
      assert String===name
      
-     was_in_lvar_define_state=in_lvar_define_state
+     was_in_lvar_define_state=in_lvar_define_state(lasttok)
      #maybe_local really means 'maybe local or constant'
      maybe_local=case name
        when /[^a-z_0-9]$/i #do nothing
@@ -500,8 +503,6 @@ private
            if /^[a-z_][a-zA-Z_0-9]*$/===name 
              assert !(lasttok===/^(\.|::)$/)
              localvars[name]=true
-           else
-             lexerror tok,"not a valid variable name: #{name}"
            end
            return result.unshift(tok)
          elsif maybe_local
@@ -534,7 +535,6 @@ private
      if (assignment_coming && !(lasttok===/^(\.|::)$/) or was_in_lvar_define_state)
         tok=assign_lvar_type! VarNameToken.new(name,pos)
         if /[^a-z_0-9]$/i===name 
-          lexerror tok,"not a valid variable name: #{name}"
         elsif /^[a-z_]/===name and !(lasttok===/^(\.|::)$/)
           localvars[name]=true
         end
@@ -573,7 +573,7 @@ private
                  !(KeywordToken===oldlast and oldlast===/\A(\.|::)\Z/)
          x
 =end
-       when ?(; 
+       when ?(
          maybe_local=false
          lastid=lasttok&&lasttok.ident
          case lastid
@@ -595,10 +595,12 @@ private
 #         ignored_tokens(true)
 #         want_parens=false if nextchar==?)
 #         @file.pos=afterparen
-
+         want_parens=true if /^(return|break|next)$/===@last_operative_token.ident and not(
+              KeywordToken===lasttok and /^(.|::)$/===lasttok.ident
+            )
          want_parens ? 1 : 0
-       when ?},?],?),?;,?^, ?|, ?>, ?,, ?., ?=; 2
-       when ?+, ?-, ?%, ?/
+       when ?},?],?),?;,(?^ unless @enable_macro), ?|, ?>, ?,, ?., ?=; 2
+       when ?+, ?-, ?%, ?/, (?^ if @enable_macro)
          if /^(return|break|next)$/===@last_operative_token.ident and not(
               KeywordToken===lasttok and /^(.|::)$/===lasttok.ident
             )
@@ -607,7 +609,7 @@ private
            (ws_toks.empty? || readahead(2)[/^.[#{WHSPLF}]/o]) ? 2 : 3
          end
        when ?*, ?&
-         lasttok=@last_operative_token
+ #        lasttok=@last_operative_token
          if /^(return|break|next)$/===@last_operative_token.ident and not(
               KeywordToken===lasttok and /^(.|::)$/===lasttok.ident
             )
@@ -626,7 +628,12 @@ private
                    /^\?([#{WHSPLF}]|[a-z_][a-z_0-9])/io===next3 ? 2 : 3
 #       when ?:,??; (readahead(2)[/^.[#{WHSPLF}]/o]) ? 2 : 3
        when ?<; (!ws_toks.empty? && readahead(4)[/^<<-?["'`a-zA-Z_0-9]/]) ? 3 : 2 
-       when ?[; ws_toks.empty?&&!(KeywordToken===oldlast and /^(return|break|next)$/===oldlast.ident) ? 2 : 3
+       when ?[; 
+           if ws_toks.empty? 
+             (KeywordToken===oldlast and /^(return|break|next)$/===oldlast.ident) ? 3 : 2
+           else
+             3
+           end
        when ?\\, ?\s, ?\t, ?\n, ?\r, ?\v, ?#; raise 'failure'
        else raise "unknown char after ident: #{nc=nextchar ? nc.chr : "<<EOF>>"}"
      end
@@ -704,8 +711,8 @@ private
   CONTEXT2ENDTOK={
     AssignmentRhsContext=>AssignmentRhsListEndToken, 
     ParamListContextNoParen=>ImplicitParamListEndToken,
-    KWParamListContextNoParen=>ImplicitParamListEndToken,
-    WhenParamListContext=>KwParamListEndToken,
+    KWParamListContextNoParen=>ImplicitParamListEndToken, #break,next,return
+    WhenParamListContext=>KwParamListEndToken, 
     RescueSMContext=>KwParamListEndToken
   }
   def abort_noparens!(str='')
@@ -713,7 +720,8 @@ private
     result=[]
     while klass=CONTEXT2ENDTOK[@parsestack.last.class]
       result << klass.new(input_position-str.length)
-      break if RescueSMContext===@parsestack.last
+      break if RescueSMContext===@parsestack.last #and str==':'
+      break if WhenParamListContext===@parsestack.last and str==':'
       @parsestack.pop 
     end
     return result
@@ -724,7 +732,7 @@ private
     AssignmentRhsContext=>AssignmentRhsListEndToken, 
     ParamListContextNoParen=>ImplicitParamListEndToken,
     KWParamListContextNoParen=>ImplicitParamListEndToken,
-    WhenParamListContext=>KwParamListEndToken,
+    WhenParamListContext=>KwParamListEndToken,      #I think this isn't needed...
     RescueSMContext=>KwParamListEndToken
   }
   def abort_noparens_for_rescue!(str='')
@@ -791,6 +799,39 @@ private
      return result
    end
 
+   #-----------------------------------
+   def enable_macros!
+     @enable_macro="macro"
+   end
+   public :enable_macros!
+
+
+   #-----------------------------------
+   @@SPACES=/[\ \t\v\f\v]/
+   @@WSTOK=/\r?\n|\r*#@@SPACES+(?:#@@SPACES|\r(?!\n))*|\#[^\n]*\n|\\\r?\n|
+            ^=begin[\s\n](?:(?!=end).*\n)*=end[\s\n].*\n/x
+   @@WSTOKS=/(?!=begin)#@@WSTOK+/o
+   def divide_ws(ws,offset)
+     result=[]
+     ws.scan(/\G#@@WSTOK/o){|ws|
+       incr= $~.begin(0)
+       klass=case ws
+       when /\A[\#=]/: CommentToken
+       when /\n\Z/: EscNlToken
+       else WsToken
+       end
+       result << klass.new(ws,offset+incr)
+     }
+     result.each_with_index{|ws,i|
+       if WsToken===ws
+         ws.ident << result.delete(i+1).ident while WsToken===result[i+1]
+       end
+     }
+     return result
+   end
+   
+
+
    #-----------------------------------
    #parse keywords now, to prevent confusion over bare symbols
    #and match end with corresponding preceding def or class or whatever.
@@ -823,7 +864,33 @@ private
          result.first.has_end!
          @parsestack.push WantsEndContext.new(str,@linenum)
          @localvars_stack.push SymbolTable.new 
-         
+         offset=input_position
+         @file.scan(/\A(#@@WSTOKS)?(::)?/o) 
+         md=@file.last_match
+         all,ws,dc=*md
+         fail if all.empty?
+         @moretokens.concat divide_ws(ws,offset) if ws
+         @moretokens.push KeywordToken.new('::',offset+md.end(0)-2) if dc
+         loop do
+           offset=input_position
+           @file.scan(/\A(#@@WSTOKS)?([A-Z][a-zA-Z_0-9]*)(::)?/o)
+           #this regexp---^ will need to change in order to support utf8 properly.
+           md=@file.last_match
+           all,ws,name,dc=*md
+           if ws
+             @moretokens.concat divide_ws(ws,offset)
+             incr=ws.size
+           else
+             incr=0
+           end
+           @moretokens.push VarNameToken.new(name,offset+incr)
+           break unless dc
+           @moretokens.push KeywordToken.new('::',offset+md.end(0)-2)
+         end
+         @moretokens.push EndHeaderToken.new(input_position)
+           
+        
+
       when "class"
          result.first.has_end!
          @parsestack.push ClassContext.new(str,@linenum)
@@ -832,11 +899,12 @@ private
          if after_nonid_op?{false} #prefix form
             result.first.has_end!
             @parsestack.push WantsEndContext.new(str,@linenum)
-            
-
+            @parsestack.push ExpectThenOrNlContext.new(str,@linenum)
          else #infix form
            result.unshift(*abort_noparens!(str))
          end
+      when "elsif"
+         @parsestack.push ExpectThenOrNlContext.new(str,@linenum)
       when "begin","case"
          result.first.has_end!
          @parsestack.push WantsEndContext.new(str,@linenum)
@@ -868,12 +936,12 @@ private
             localvars.start_block
             block_param_list_lookahead
          end
-      when "def"
+      when "def",@enable_macro
          result.first.has_end!
          @parsestack.push ctx=DefContext.new(@linenum)
          ctx.state=:saw_def
          safe_recurse { |aa|
-            set_last_token KeywordToken.new "def" #hack
+            set_last_token KeywordToken.new str #hack
             result.concat ignored_tokens
 
             #read an expr like a.b.c or a::b::c
@@ -957,7 +1025,7 @@ private
                   else 
                     ofs+=listend.to_s.size
                   end
-                  result.insert end_index+1,EndDefHeaderToken.new(ofs)
+                  result.insert end_index+1,EndHeaderToken.new(ofs)
                   break
                end
 
@@ -983,7 +1051,7 @@ private
                   if endofs
                     result.insert -2,ImplicitParamListEndToken.new(tok.offset)
                   end
-                  result.insert -2, EndDefHeaderToken.new(tok.offset)
+                  result.insert -2, EndHeaderToken.new(tok.offset)
                   break
                else
                   lexerror(tok, "bizarre token in def name: " +
@@ -1060,6 +1128,11 @@ private
       when "then"
          result.unshift(*abort_noparens!(str))
          @parsestack.last.see self,:then
+
+         if ExpectThenOrNlContext===@parsestack.last
+           @parsestack.pop
+         else #error... does anyone care?
+         end
          
       when "in"
          result.unshift KwParamListEndToken.new( offset)
@@ -1083,7 +1156,7 @@ private
          #END blocks are visible to subsequent code. (Why??) 
          #That difference forces a custom parsing.
          if @last_operative_token===/^(\.|::)$/
-           result=yield nil #should pass a keyword token here
+           result=yield MethNameToken.new(str) #should pass a methname token here
          else
            safe_recurse{
              old=result.first
@@ -1096,17 +1169,18 @@ private
              getchar=='{' or lexerror(result.first,"expected { after #{str}")
              result.push KeywordToken.new('{',input_position-1)
              result.last.set_infix!
+             result.last.as="do"
              @parsestack.push BeginEndContext.new(str,offset)
            }
          end
 
       when FUNCLIKE_KEYWORDS
-         result=yield nil #should be a keyword token
+         result=yield MethNameToken.new(str) #should be a keyword token?
 
       when RUBYKEYWORDS
         #do nothing
         
-      else result=yield nil
+      else result=yield  MethNameToken.new(str)
 
       end
 
@@ -1307,7 +1381,7 @@ end
 #         if  !eof? and nextchar.chr[/[iuw\/<|>+\-*&%?:({]/] and
 #             !(NewlineToken===@last_operative_token) and
 #             !(/^(end|;)$/===@last_operative_token)
-           #result<<EndDefHeaderToken.new(result.last.offset+result.last.to_s.size)
+           #result<<EndHeaderToken.new(result.last.offset+result.last.to_s.size)
            set_last_token KeywordToken.new ';'
            result<< get1token
 #         end
@@ -1426,10 +1500,18 @@ end
    #-----------------------------------
    def symbol_or_op(ch)
       startpos= input_position
+
+
       qe= colon_quote_expected?(ch)
       lastchar=prevchar
       eat_next_if(ch[0]) or raise "needed: "+ch
 
+      if nextchar==?( and @enable_macro
+        result= OperatorToken.new(':', startpos)
+        result.unary=true
+        return result
+      end
+
       #handle quoted symbols like  :"foobar",  :"[]"
       qe and return symbol(':')
 
@@ -1437,17 +1519,23 @@ end
       unless eat_next_if(?:) 
         #cancel implicit contexts...
         @moretokens.push(*abort_noparens!(':'))
-        @moretokens.push KeywordToken.new(':',startpos)
+        @moretokens.push tok=KeywordToken.new(':',startpos)
         
         case @parsestack.last
-        when TernaryContext: @parsestack.pop #should be in the context's see handler
+        when TernaryContext: 
+          tok.ternary=true
+          @parsestack.pop #should be in the context's see handler
         when ExpectDoOrNlContext: #should be in the context's see handler
           @parsestack.pop
           assert @parsestack.last.starter[/^(while|until|for)$/]
-          @moretokens.last.as=";"
+          tok.as=";"
+        when ExpectThenOrNlContext,WhenParamListContext: 
+          #should be in the context's see handler
+          @parsestack.pop
+          tok.as="then"
         when RescueSMContext:
-          @moretokens.last.as=";"
-        else @moretokens.last.as="then"
+          tok.as=";"
+        else fail ": not expected in #{@parsestack.last.class}->#{@parsestack.last.starter}"
         end
         
         #end ternary context, if any
@@ -1503,7 +1591,9 @@ end
              /[A-Z_0-9]$/i===result and klass=VarNameToken
            end
            result
-         else error= "unexpected char starting symbol: #{nc.chr}"
+         else 
+           error= "unexpected char starting symbol: #{nc.chr}"
+           '_'
        end
      result= lexerror(klass.new(result,start,notbare ?  ':' : ''),error)
      if open
@@ -1843,7 +1933,10 @@ end
       then   #hack-o-rama: probly cases left out above
         @offset_adjust=@min_offset_adjust
         a= abort_noparens!
-        ExpectDoOrNlContext===@parsestack.last and @parsestack.pop
+        case @parsestack.last  #these should be in the see:semi handler
+          when ExpectDoOrNlContext: @parsestack.pop
+          when ExpectThenOrNlContext: @parsestack.pop        
+        end
         assert !@parsestack.empty?
         @parsestack.last.see self,:semi
 
@@ -2011,7 +2104,7 @@ end
          when /^(#{RUBYOPERATORREX}|#{INNERBOUNDINGWORDS}|do)$/o.token_pat
             #regexs above must match whole string
             #assert(@last_operative_token==$&) #disabled 'cause $& is now always nil :(
-            return true
+            return true if OperatorToken===@last_operative_token || KeywordToken===@last_operative_token
          when NewlineToken, nil,   #nil means we're still at beginning of file
               /^([({\[]|or|not|and|if|unless|then|elsif|else|class|module|def|
                  while|until|begin|for|in|case|when|ensure|defined\?)$
@@ -2065,15 +2158,31 @@ end
    end
 
 
-   #-----------------------------------
-   def biop(ch) #match /%=?/ (% or %=)
+    #-----------------------------------
+    def caret(ch) #match /^=?/ (^ or ^=) (maybe unary ^ too)
+      if @enable_macro and (@last_token_maybe_implicit and
+         @last_token_maybe_implicit.ident=='(') || unary_op_expected?(ch)
+           result=OperatorToken.new(read(1),input_position)
+           result.unary=true
+           result
+      else
+           biop ch
+      end
+    end
+
+    #-----------------------------------
+    def biop(ch) #match /%=?/ (% or %=)
       assert(ch[/^[%^]$/])
+      oldpos=input_position
       result=getchar
       if eat_next_if(?=)
          result << ?=
       end
-      return operator_or_methname_token( result)
+      result= operator_or_methname_token( result)
+      result.offset=oldpos
+      return result
    end
+
    #-----------------------------------
    def tilde(ch) #match ~
       assert(ch=='~')
@@ -2322,10 +2431,8 @@ end
         lexerror kw,"mismatched braces: #{origch}#{ch}\n" +
                  "matching brace location", @filename, line
       end
-      if BlockContext===ctx
-        localvars.end_block
-        @moretokens.last.as="end"
-      end
+      localvars.end_block if BlockContext===ctx
+      @moretokens.last.as="end" if BlockContext===ctx or BeginEndContext===ctx
       if ParamListContext==ctx.class
         assert ch==')'
         kw.set_callsite! #not needed?
@@ -2372,6 +2479,7 @@ end
        ParamListContext===@parsestack[-2] || 
        ParamListContextNoParen===@parsestack[-2] ||
        WhenParamListContext===@parsestack[-2] ||
+       ListImmedContext===@parsestack[-2] ||
        (RescueSMContext===@parsestack[-2] && @parsestack[-2].state==:rescue) ||
        (DefContext===@parsestack[-2] && !@parsestack[-2].in_body)
          @parsestack.pop
@@ -2394,7 +2502,10 @@ end
     assert @moretokens.empty?
     @moretokens.push(*abort_noparens!)
     @parsestack.last.see self,:semi
-    if ExpectDoOrNlContext===@parsestack.last #should be in context's see:semi handler
+    case @parsestack.last #should be in context's see:semi handler
+    when ExpectThenOrNlContext
+      @parsestack.pop
+    when ExpectDoOrNlContext
       @parsestack.pop
       assert @parsestack.last.starter[/^(while|until|for)$/]
     end
@@ -2417,7 +2528,7 @@ end
   #-----------------------------------
   #tokenify_results_of  :identifier
   save_offsets_in(*CHARMAPPINGS.values.uniq-[
-    :symbol_or_op,:open_brace,:whitespace,:exclam,:backquote  
+    :symbol_or_op,:open_brace,:whitespace,:exclam,:backquote,:caret
   ])
   #save_offsets_in :symbol