rubylexer 0.7.5 → 0.7.6

data/History.txt

@@ -1,3 +1,16 @@
+=== 0.7.6/7-01-2009
+* 5 Bugfixes:
+  * don't treat <, <=, <=> as starting variables (only << for here header)
+  * space between break/return/next and following open paren is ignored
+  * fixed unusual whitespace is unlikely places (module header)
+  * some parentheses weren't being decorated right
+  * rescue should not end implicit parameter lists... unless its an op
+
+* 3 new ruby 1.9 features:
+  * stabby blocks
+  * dot at beginning of line
+  * !, !=, !~ are now valid method/symbol names
+
 === 0.7.5/5-23-2009
 * 1 Bugfix:
   * fixed problem with parsing shebang lines

data/Rakefile

@@ -30,7 +30,7 @@ require 'lib/rubylexer/version.rb'
      _.test_globs=["test/code/regression.rb"]
      _.description=desc
      _.summary=desc[/\A[^.]+\./]
-     _.spec_extras={:bindir=>'',:rdoc_options=>'-x lib/rubylexer/test'}
+     _.spec_extras={:bindir=>'',:rdoc_options=>'-x lib/rubylexer/test/oneliners.rb'}
      #_.rdoc_pattern=/\A(howtouse\.txt|testing\.txt|README\.txt|lib\/[^\/]*\.rb|lib\/rubylexer\/[^\d][^\/]*\.rb)\Z/
    end
 

data/lib/rubylexer.rb

@@ -170,6 +170,11 @@ class RubyLexer
       @progress_thread=nil
       @rubyversion=options[:rubyversion]
       @encoding=options[:encoding]||:detect
+      @method_operators=if @rubyversion>=1.9 
+                          /#{RUBYSYMOPERATORREX}|\A![=~]?\Z/o
+                        else
+                          RUBYSYMOPERATORREX
+                        end
 
       @toptable=CharHandler.new(self, :illegal_char, CHARMAPPINGS)
 
@@ -670,9 +675,9 @@ private
          maybe_local=false
          lastid=lasttok&&lasttok.ident
          case lastid
-           when /\A[;(]|do\Z/: was_after_nonid_op=false
-           when '|':  was_after_nonid_op=false unless BlockParamListLhsContext===@parsestack.last
-           when '{': was_after_nonid_op=false if  BlockContext===@parsestack.last or BeginEndContext===@parsestack.last
+           when /\A[;(]|do\Z/; was_after_nonid_op=false
+           when '|';  was_after_nonid_op=false unless BlockParamListLhsContext===@parsestack.last
+           when '{'; was_after_nonid_op=false if  BlockContext===@parsestack.last or BeginEndContext===@parsestack.last
          end if KeywordToken===lasttok
          was_after_nonid_op=false if NewlineToken===lasttok or lasttok.nil?
          want_parens=!(ws_toks.empty? or was_after_nonid_op) #or
@@ -752,20 +757,21 @@ private
        result.unshift ImplicitParamListStartToken.new(oldpos),
                  ImplicitParamListEndToken.new(oldpos)
      when 1,3;
-       arr,pass=*param_list_coming_with_2_or_more_params?
-       result.push( *arr )
-       unless pass
+       if /^(break|next|return)$/===name and
+                !(KeywordToken===lasttok and /^(\.|::)$/===lasttok.ident)
          #only 1 param in list
          result.unshift ImplicitParamListStartToken.new(oldpos)
-         last=result.last
-         last.set_callsite! false if last.respond_to? :callsite? and last.callsite? #KeywordToken===last and last.ident==')'
-         if /^(break|next|return)$/===name and 
-            !(KeywordToken===lasttok and /^(\.|::)$/===lasttok.ident)
-           ty=KWParamListContextNoParen 
-         else
-           ty=ParamListContextNoParen
+         @parsestack.push ParamListContextNoParen.new(@linenum)
+       else
+         arr,pass=*param_list_coming_with_2_or_more_params?
+         result.push( *arr )
+         unless pass
+           #only 1 param in list
+           result.unshift ImplicitParamListStartToken.new(oldpos)
+           last=result.last
+           last.set_callsite! false if last.respond_to? :callsite? and last.callsite?
+           @parsestack.push ParamListContextNoParen.new(@linenum)
          end
-         @parsestack.push ty.new(@linenum)
        end
      when 0; #do nothing
      else raise 'invalid value of implicit_parens_to_emit'
@@ -782,8 +788,11 @@ private
    end
 
    #-----------------------------------
+   #read ahead to see if there's method param list (with real parentheses)
+   #and 2 or more parameters (and hence a comma to separate them)
+   #ugly, lexer recursion
    def param_list_coming_with_2_or_more_params?
-     WHSPCHARS[prevchar] && (?(==nextchar) or return [[],false]
+       return [[],false] unless WHSPCHARS[prevchar] && (?(==nextchar)
        basesize=@parsestack.size
        result=[get1token]
        pass=loop{
@@ -799,6 +808,8 @@ private
            lexerror tok, "unexpected eof in parameter list"
          end
        }
+       result.concat @moretokens
+       @moretokens.replace []
        return [result,pass]
    end
 
@@ -907,22 +918,22 @@ private
    #-----------------------------------
    @@SPACES=/[\ \t\v\f\v]/
    @@WSTOK=/\r?\n|\r*#@@SPACES+(?:#@@SPACES|\r(?!\n))*|\#[^\n]*\n|\\\r?\n|
-            ^=begin[\s\n](?:(?!=end).*\n)*=end[\s\n].*\n/x
+            ^=begin(?:[\s].*)?\n(?:(?!=end).*\n)*=end[\s\n].*\n/x
    @@WSTOKS=/(?!=begin)#@@WSTOK+/o
    def divide_ws(ws,offset)
      result=[]
      ws.scan(/\G#@@WSTOK/o){|ws|
        incr= $~.begin(0)
        klass=case ws
-       when /\A[\#=]/: CommentToken
-       when /\n\Z/: EscNlToken
+       when /\A[\#=]/; CommentToken
+       when /\n\Z/; EscNlToken
        else WsToken
        end
        result << klass.new(ws,offset+incr)
      }
      result.each_with_index{|ws,i|
        if WsToken===ws
-         ws.ident << result.delete(i+1).ident while WsToken===result[i+1]
+         ws.ident << result.delete_at(i+1).ident while WsToken===result[i+1]
        end
      }
      return result
@@ -1052,15 +1063,21 @@ private
 
    def keyword_do(str,offset,result)
          result.unshift(*abort_noparens_for_do!(str))
-         if ExpectDoOrNlContext===@parsestack.last
+         ctx=@parsestack.last
+         if ExpectDoOrNlContext===ctx
             @parsestack.pop
             assert WantsEndContext===@parsestack.last
             result.last.as=";"
          else
             result.last.has_end!
-            @parsestack.push WantsEndContext.new(str,@linenum)
-            localvars.start_block
-            block_param_list_lookahead
+            if BlockContext===ctx and ctx.wanting_stabby_block_body
+              ctx.wanting_stabby_block_body=false
+              ctx.starter,ctx.ender="do","end"
+            else
+              @parsestack.push WantsEndContext.new(str,@linenum)            
+              localvars.start_block
+              block_param_list_lookahead
+            end
          end
          return result
    end
@@ -1262,7 +1279,7 @@ private
            result.push KwParamListStartToken.new(offset+str.length)
            #corresponding EndToken emitted by abort_noparens! on leaving rescue context
            @parsestack.push RescueSMContext.new(@linenum)
-           result.unshift(*abort_noparens!(str))  
+#           result.unshift(*abort_noparens!(str))  
          end
          return result
    end
@@ -1386,16 +1403,17 @@ private
 
    
    #-----------------------------------
-   def block_param_list_lookahead
+   def block_param_list_lookahead starter=?|, ctx_type=BlockParamListLhsContext
       safe_recurse{ |la|
          set_last_token KeywordToken.new(  ';' )
          a=ignored_tokens
 
-         if eat_next_if(?|)
-           a<< KeywordToken.new("|", input_position-1)
+         if eat_next_if(starter)
+           mycontext=ctx_type.new(@linenum)
+           a<< KeywordToken.new(mycontext.starter, input_position-1)
 if true
-           @parsestack.push mycontext=BlockParamListLhsContext.new(@linenum)
-           nextchar==?| and a.push NoWsToken.new(input_position)
+           @parsestack.push mycontext
+           nextchar==mycontext.ender[0] and a.push NoWsToken.new(input_position)
 else
            if eat_next_if(?|)
              a.concat [NoWsToken.new(input_position-1),
@@ -1430,8 +1448,11 @@ else
              a<< KeywordToken.new('|',tok.offset)
              @moretokens.empty? or
                fixme %#moretokens might be set from get1token call above...might be bad#
-end
            end
+end
+         elsif starter==?(
+           ctx_type=UnparenedParamListLhsContext #hacky... should be a param?
+           @parsestack.push ctx_type.new(@linenum)
          end
 
          set_last_token KeywordToken.new( ';' )
@@ -1594,7 +1615,8 @@ end
          when AssignmentRhsContext; result.tag= :rhs
          when ParamListContext,ParamListContextNoParen; #:call
          when ListImmedContext; #:array
-         when BlockParamListLhsContext; #:block
+         when BlockParamListLhsContext,UnparenedParamListLhsContext; #:block or stabby proc
+         when ParenedParamListLhsContext; #:stabby proc or method def'n?
          when KnownNestedLhsParenContext; #:nested
          else          result.tag=     :lhs if cill
          end
@@ -1647,7 +1669,8 @@ end
 
      s=tok.to_s
      case s
-     when /^[@$<]/; true
+     when /^[@$]/; true
+     when /^<</; HerePlaceholderToken===tok
      when /(?!#@@LETTER_DIGIT).$/o; false
 #     when /^#@@LCLETTER/o; localvars===s or VARLIKE_KEYWORDS===s
      when /^#@@LETTER/o; VarNameToken===tok
@@ -1714,8 +1737,9 @@ end
           end
         when RescueSMContext
           tok.as=";"
-        end or 
+        end or
           fail ": not expected in #{@parsestack.last.class}->#{@parsestack.last.starter}"
+
         
         #end ternary context, if any
         @parsestack.last.see self,:colon
@@ -1748,7 +1772,7 @@ end
      klass=(notbare ? SymbolToken : MethNameToken)
      
      #look for operators
-     opmatches=readahead(3)[RUBYSYMOPERATORREX]
+     opmatches=readahead(3)[@method_operators]
      result= opmatches ? read(opmatches.size) :
        case nc=nextchar
          when ?" #"
@@ -1782,27 +1806,31 @@ end
      return result
    end
 
+   #-----------------------------------
    def merge_assignment_op_in_setter_callsites?
      false
    end
+
    #-----------------------------------
    def callsite_symbol(tok_to_errify)
      start= input_position
      
      #look for operators
-     opmatches=readahead(3)[RUBYSYMOPERATORREX]
-     return [opmatches ? read(opmatches.size) :
-       case nc=nextchar
-         when ?` then read(1) #`
-         when ?_,?a..?z,?A..?Z,NONASCII then 
+     opmatches=readahead(3)[@method_operators]
+     return [read(opmatches.size), start] if opmatches
+     case nc=nextchar
+         when ?` #`
+           return [read(1),start] 
+         when ?_,?a..?z,?A..?Z,NONASCII
            context=merge_assignment_op_in_setter_callsites? ? ?: : nc
-           identifier_as_string(context)
-         else 
-           set_last_token KeywordToken.new(';')
-           lexerror(tok_to_errify,"unexpected char starting callsite symbol: #{nc.chr}, tok=#{tok_to_errify.inspect}")
-           nil
-       end, start
-      ]
+           return [identifier_as_string(context), start]
+         when ?(
+           return [nil,start] if @enable_macro
+     end
+
+     set_last_token KeywordToken.new(';')
+     lexerror(tok_to_errify,"unexpected char starting callsite symbol: #{nc.chr}, tok=#{tok_to_errify.inspect}")
+     return [nil, start]
    end
 
    #-----------------------------------
@@ -2112,13 +2140,15 @@ end
       pre=FileAndLineToken.new(@filename,@linenum+1,input_position)
       pre.allow_ooo_offset=true
 
-      if NewlineToken===@last_operative_token or #hack
-         (KeywordToken===@last_operative_token and 
-          @last_operative_token.ident=="rescue" and
-          !@last_operative_token.infix?)  or 
-         #/^(;|begin|do|#{INNERBOUNDINGWORDS})$/ or #hack
-         !after_nonid_op?{false}
-      then   #hack-o-rama: probly cases left out above
+      hard=NewlineToken===@last_operative_token || #hack
+           (KeywordToken===@last_operative_token and
+            @last_operative_token.ident=="rescue" and
+            !@last_operative_token.infix?)  ||
+           !after_nonid_op?{false}
+
+      hard=false if @rubyversion>=1.9 and @file.check /\A\n(?:#@@WSTOKS)?\.[^.]/o
+
+      if hard
         @offset_adjust=@min_offset_adjust
         a= abort_noparens!
         case @parsestack.last  #these should be in the see:semi handler
@@ -2402,6 +2432,15 @@ end
          /^(return|break|next)$/===@last_operative_token.ident
         if (?0..?9)===readahead(2)[1]
           return number(ch)
+        elsif @rubyversion>=1.9 and '->' == readahead(2) #stabby proc
+          #push down block context
+          localvars.start_block
+          @parsestack.push ctx=BlockContext.new(@linenum)
+          ctx.wanting_stabby_block_body=true
+          #read optional proc params
+          block_param_list_lookahead ?(, ParenedParamListLhsContext
+
+          
         else #unary operator
           result=getchar
           WHSPLF[nextchar.chr] or
@@ -2575,8 +2614,13 @@ end
       when '('
         lasttok=last_token_maybe_implicit #last_operative_token
         #could be: lasttok===/^#@@LETTER/o
-        if (VarNameToken===lasttok or MethNameToken===lasttok or
-            lasttok===FUNCLIKE_KEYWORDS)
+        method_params= (
+          VarNameToken===lasttok or 
+          MethNameToken===lasttok or
+          lasttok===FUNCLIKE_KEYWORDS or
+          (@enable_macro and lasttok and lasttok.ident==')')
+        )
+        if method_params
           unless WHSPCHARS[lastchar]
                @moretokens << tokch
                tokch= NoWsToken.new(input_position-1)
@@ -2589,6 +2633,8 @@ end
             !(KeywordToken===lasttok && lasttok.ident=="def")
           if maybe_def or 
              BlockParamListLhsContext===ctx or 
+             ParenedParamListLhsContext===ctx or 
+             UnparenedParamListLhsContext===ctx or 
              ParenContext===ctx && ctx.lhs
             @parsestack.push KnownNestedLhsParenContext.new(@linenum)
           else
@@ -2614,9 +2660,13 @@ end
         end
 #=end
 
-        localvars.start_block
-        @parsestack.push BlockContext.new(@linenum)
-        block_param_list_lookahead
+        if BlockContext===@parsestack.last and @parsestack.last.wanting_stabby_block_body
+          @parsestack.last.wanting_stabby_block_body=false
+        else
+          localvars.start_block
+          @parsestack.push BlockContext.new(@linenum)
+          block_param_list_lookahead
+        end
       end
       end
       return (tokch)
@@ -2645,6 +2695,10 @@ end
         assert ch==')'
         kw.set_callsite! #not needed?
       end
+      if ParenedParamListLhsContext===ctx
+        assert @parsestack.last.wanting_stabby_block_body
+        assert ch==')'
+      end
       return @moretokens.shift
    end
 
@@ -2705,7 +2759,8 @@ end
     when AssignmentRhsContext; token.tag=:rhs
     when ParamListContext,ParamListContextNoParen; #:call
     when ListImmedContext; #:array
-    when BlockParamListLhsContext; #:block
+    when BlockParamListLhsContext,UnparenedParamListLhsContext; #:block or stabby proc
+    when ParenedParamListLhsContext; #stabby proc or method def'n?
     when KnownNestedLhsParenContext; #:nested
     else
       token.tag=:lhs if comma_in_lvalue_list? 

data/lib/rubylexer/context.rb

@@ -73,6 +73,7 @@ module NestedContexts
     def initialize(linenum)
       super('{','}',linenum)
     end
+    attr_accessor :wanting_stabby_block_body
   end
 
   class BeginEndContext  < NestedContext
@@ -109,6 +110,16 @@ module NestedContexts
     def ender; '|' end
   end
 
+  class ParenedParamListLhsContext < ImplicitLhsContext
+    def starter; '(' end
+    def ender; ')' end
+  end
+
+  class UnparenedParamListLhsContext < ImplicitLhsContext
+    def starter; huh end  #" " ???
+    def ender; huh end    #; or \n when from method def, { or do when from stabby block
+  end
+
   class ImplicitContext < ListContext
   end
 

data/lib/rubylexer/version.rb

@@ -1,3 +1,3 @@
 class RubyLexer
-  VERSION='0.7.5'
+  VERSION='0.7.6'
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification 
 name: rubylexer
 version: !ruby/object:Gem::Version 
-  version: 0.7.5
+  version: 0.7.6
 platform: ruby
 authors: 
 - Caleb Clausen
@@ -9,7 +9,7 @@ autorequire:
 bindir: ""
 cert_chain: []
 
-date: 2009-05-23 00:00:00 -07:00
+date: 2009-07-06 00:00:00 -07:00
 default_executable: 
 dependencies: 
 - !ruby/object:Gem::Dependency 
@@ -166,7 +166,7 @@ has_rdoc: true
 homepage: http://github.com/coatl/rubylexer/
 post_install_message: 
 rdoc_options: 
-- -x lib/rubylexer/test
+- -x lib/rubylexer/test/oneliners.rb
 require_paths: 
 - lib
 required_ruby_version: !ruby/object:Gem::Requirement