rubylexer 0.6.2

data/rlold.rb

@@ -0,0 +1,12 @@
+class RubyLexer06 < RubyLexer
+  alias lexerror lexerror_exception
+end
+
+class<<RubyLexer
+  def version(num)
+    case num
+      when 0.0..0.6: RubyLexer06 
+      else RubyLexer
+    end
+  end 
+end

data/rubycode.rb

@@ -0,0 +1,44 @@
+=begin copyright
+    rubylexer - a ruby lexer written in ruby
+    Copyright (C) 2004,2005  Caleb Clausen
+
+    This library is free software; you can redistribute it and/or
+    modify it under the terms of the GNU Lesser General Public
+    License as published by the Free Software Foundation; either
+    version 2.1 of the License, or (at your option) any later version.
+
+    This library is distributed in the hope that it will be useful,
+    but WITHOUT ANY WARRANTY; without even the implied warranty of
+    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+    Lesser General Public License for more details.
+
+    You should have received a copy of the GNU Lesser General Public
+    License along with this library; if not, write to the Free Software
+    Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
+=end
+
+
+
+require "token.rb"
+require "tokenprinter.rb"
+
+class RubyCode < Token
+   def initialize(tokens,filename,linenum)
+      super(tokens)
+      @filename=filename
+      @linenum=linenum
+   end
+
+   def [](*args)
+      exec? ident.huh
+   end
+
+   def to_s()
+      result=[]
+      keepwsprinter=KeepWsTokenPrinter.new('',@linenum)
+      ident.each{|tok| result << keepwsprinter.sprint(tok) }
+      return result.to_s
+   end
+end
+
+

data/rubylexer.rb

@@ -0,0 +1,1589 @@
+=begin copyright
+    rubylexer - a ruby lexer written in ruby
+    Copyright (C) 2004,2005  Caleb Clausen
+
+    This library is free software; you can redistribute it and/or
+    modify it under the terms of the GNU Lesser General Public
+    License as published by the Free Software Foundation; either
+    version 2.1 of the License, or (at your option) any later version.
+
+    This library is distributed in the hope that it will be useful,
+    but WITHOUT ANY WARRANTY; without even the implied warranty of
+    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+    Lesser General Public License for more details.
+
+    You should have received a copy of the GNU Lesser General Public
+    License along with this library; if not, write to the Free Software
+    Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
+=end
+
+
+
+require "rulexer"
+require "symboltable"
+require "io.each_til_charset"
+require "context.rb"
+
+
+
+#-----------------------------------
+class RubyLexer < RuLexer
+  include NestedContexts
+ 
+   RUBYSYMOPERATORREX=
+      %r{^([&|^/%~]|=(==?|~)|>[=>]?|<(<|=>?)?|[+\-]@?|\*\*?|\[\]=?)}
+      # (nasty beastie, eh?)
+      #these are the overridable operators
+      #does not match flow-control operators like: || && ! or and if not
+      #or op= ops like: += -= ||=
+      #or .. ... ?:
+      #for that use:
+   RUBYNONSYMOPERATORREX=
+      %r{^([%^~/\-+]=|(\|\|?|&&?)=?|(<<|>>|\*\*?)=|\.{1,3}|[?:,;]|=>?|![=~]?)$}
+   RUBYOPERATORREX=/#{RUBYSYMOPERATORREX}|#{RUBYNONSYMOPERATORREX}/o
+   UNSYMOPS=/^[~!]$/ #always unary
+   UBSYMOPS=/^([*&+-]|::)$/  #ops that could be unary or binary
+   WHSPCHARS=WHSPLF+"\\#"
+   OPORBEGINWORDS="(if|unless|while|until)"
+   BEGINWORDS=/^(def|class|module|begin|for|case|do|#{OPORBEGINWORDS})$/o
+   FUNCLIKE_KEYWORDS=/^(break|next|redo|return|raise|yield|defined\?|retry|super|BEGIN|END)$/
+   VARLIKE_KEYWORDS=/^(__FILE__|__LINE__|false|nil|self|true)$/
+   INNERBOUNDINGWORDS="(else|elsif|ensure|in|then|rescue|when)"
+   BINOPWORDS="(and|or)"
+   NEVERSTARTPARAMLISTWORDS=/^(#{OPORBEGINWORDS}|#{INNERBOUNDINGWORDS}|#{BINOPWORDS}|end)[^a-zA-Z0-9_!?=]?/o
+   NEVERSTARTPARAMLISTFIRST=CharSet[%[aoeitrwu]]  #char set that begins NEVERSTARTPARAMLIST
+   NEVERSTARTPARAMLISTMAXLEN=7            #max len of a NEVERSTARTPARAMLIST
+   
+   RUBYKEYWORDS=%r{
+     ^(alias|#{BINOPWORDS}|not|undef|__END__|end|
+       #{VARLIKE_KEYWORDS}|#{FUNCLIKE_KEYWORDS}|
+       #{INNERBOUNDINGWORDS}|#{BEGINWORDS}
+     )$
+   }xo
+   CHARMAPPINGS = {
+         ?$ => :dollar_identifier,
+         ?@ => :at_identifier,
+         ?a..?z => :identifier,
+         ?A..?Z => :identifier,
+         ?_     => :identifier,
+         ?0..?9 => :number,
+         ?" => :double_quote,
+         ?' => :single_quote,
+         ?` => :back_quote,
+
+         WHSP => :whitespace, #includes \r
+         ?, => :comma,
+         ?; => :semicolon,
+
+         ?^ => :biop,
+         ?~ => :tilde,
+         ?= => :equals,
+         ?! => :exclam,
+         ?. => :dot,
+
+         #these ones could signal either an op or a term
+         ?/ => :regex_or_div,
+         "|>" => :quadriop,
+         "*&" => :star_or_amp,        #could be unary
+         "+-" => :plusminus, #could be unary
+         ?< => :lessthan,
+         ?% => :percent,
+         ?? => :char_literal_or_op,  #single-char int literal
+         ?: => :symbol_or_op,
+         ?\n => :newline, #implicitly escaped after op
+         #?\r => :newline, #implicitly escaped after op
+
+         ?\\ => :escnewline,
+         ?\0 => :eof,
+
+         "[({" => :open_brace,
+         "])}" => :close_brace,
+
+
+         ?# => :comment
+   }
+
+   attr :incomplete_here_tokens
+
+
+   #-----------------------------------
+   def initialize(filename,file,linenum=1)
+      super(filename,file, linenum)
+      @start_linenum=linenum
+      @bracestack=[TopLevelContext.new]
+      @incomplete_here_tokens=[]
+      @localvars=SymbolTable.new
+      @defining_lvar=nil
+
+      @toptable=CharHandler.new(self, :illegal_char, CHARMAPPINGS)
+
+      start_of_line_directives
+   end
+
+   #-----------------------------------
+   def get1token
+      result=super  #most of the action's here
+
+      #now cleanup and housekeeping
+
+
+      #check for bizarre token types
+      case result
+      when IgnoreToken#,nil
+         return result
+      when Token#,String
+      else
+         raise "#{@filename}:#{linenum}:token is a #{result.class}, last is #{@last_operative_token}"
+      end
+
+      @last_operative_token=result
+
+      return result
+   end
+   
+
+
+   #-----------------------------------
+   def balanced_braces?
+
+       #@bracestack.empty?
+       @bracestack.size==1 and TopLevelContext===@bracestack.first
+   end
+
+   #-----------------------------------
+   def dollar_identifier(ch=nil)
+      s=eat_next_if(?$) or return nil
+
+      if t=((identifier_as_string(?$) or special_global))
+        s<<t
+      else error= "missing $id name"
+      end
+
+      return lexerror(VarNameToken.new(s),error)
+   end
+
+   #-----------------------------------
+   def at_identifier(ch=nil)
+      result =  (eat_next_if(?@) or return nil)
+      result << (eat_next_if(?@)or'')
+      if t=identifier_as_string(?@)
+        result<<t
+      else error= "missing @id name"
+      end
+      return lexerror(VarNameToken.new(result),error)
+   end
+
+private
+   #-----------------------------------
+   def here_spread_over_ruby_code(rl,tok)
+     assert(!rl.incomplete_here_tokens.empty?)
+     @incomplete_here_tokens += rl.incomplete_here_tokens
+   end
+
+   #-----------------------------------
+   def expect_do_or_end_or_nl!(st)
+     @bracestack.push ExpectDoOrNlContext.new(st,/(do|;|:|\n)/,@linenum)
+   end
+
+  #-----------------------------------
+  #match NoWstoken, ws, comment, or (escaped?) newline repeatedly
+  def maybe_no_ws_token
+    result=[]
+    while IgnoreToken===(tok=get1token)
+      EoiToken===tok and lexerror tok,"end of file not expected here"
+      result << tok
+    end
+    assert((not IgnoreToken===tok))
+    @moretokens.unshift tok
+    return result
+  end
+
+  #-----------------------------------
+  WSCHARSET=CharSet["#\\\n\s\t\v\r\f"]
+  def ignored_tokens(allow_eof=false)
+    result=[]
+    result<<@moretokens.shift while IgnoreToken===@moretokens.first
+    @moretokens.empty? or return result
+if true
+    loop do
+      unless @moretokens.empty?
+        IgnoreToken===@moretokens.first or NewlineToken===@moretokens.first or
+        break
+      else
+        WSCHARSET===nextchar or break
+      end
+
+      tok=get1token
+      result<<tok
+      case tok
+        when NewlineToken : block_given? and yield tok
+        when EoiToken : allow_eof or lexerror tok,"end of file not expected here(2)"
+        when IgnoreToken
+        else raise "impossible"
+      end
+    end
+
+else
+      @whsphandler||=CharHandler.new(self, :==,
+         "#" => :comment,
+         "\n" => :newline,
+         "\\" => :escnewline,
+         "\s\t\v\r\f" => :whitespace
+      )
+      #tok=nil
+      while tok=@whsphandler.go((nextchar or return result))
+         block_given? and NewlineToken===tok and yield tok
+         result << tok
+      end
+end
+    return result
+  end
+
+   #-----------------------------------
+   def safe_recurse
+      old_moretokens=@moretokens
+      #old_bracestack=@bracestack.dup
+      @moretokens=[]
+      result=   yield @moretokens
+      #assert @incomplete_here_tokens.empty?
+      #assert @bracestack==old_bracestack
+      @moretokens= old_moretokens.concat @moretokens
+      return result
+      #need to do something with @last_operative_token?
+   end
+
+   #-----------------------------------
+   def special_global   #handle $-a and friends
+      assert prevchar=='$'
+      result = ((
+      #order matters here, but it shouldn't
+      #(but til_charset must be last)
+         eat_next_if(/^[!@&+`'=~\/\\,.;<>*"$?:]$/) or
+         (eat_next_if('-') and ("-"+getchar)) or
+         (?0..?9)===nextchar ? til_charset(/[^\d]/) : nil
+      ))
+   end
+
+   #-----------------------------------
+   def identifier(context=nil)
+      oldpos=@file.pos
+      str=identifier_as_string(context)
+
+      #skip keyword processing if 'escaped' as it were, by def, . or ::
+      #or if in a non-bare context
+      #just asserts because those contexts are never encountered.
+      #control goes through symbol(<...>,nil) 
+      assert( /^[a-z_]$/i===context)
+      assert !(@last_operative_token===/^(\.|::|(un)?def|alias)$/)
+
+      @moretokens.unshift(*parse_keywords(str,oldpos) do
+        #if not a keyword,
+        case str
+          when FUNCLIKE_KEYWORDS: #do nothing
+          when VARLIKE_KEYWORDS,RUBYKEYWORDS: raise "shouldnt see keywords here, now"
+        end
+        safe_recurse { |a| var_or_meth_name(str,@last_operative_token,oldpos) }
+      end)
+      return @moretokens.shift
+   end
+
+   #-----------------------------------
+   def identifier_as_string(context)
+      #must begin w/ letter or underscore
+      str=eat_next_if(/^[_a-z]$/i) or return nil
+
+      #equals, question mark, and exclamation mark
+      #might be allowed at the end in some contexts.
+      #(in def headers and symbols)
+      #otherwise, =,?, and ! are to be considered
+      #separate tokens. confusing, eh?
+      #i hope i've captured all right conditions....
+      #context should always be ?: right after def, ., and :: now
+
+      maybe_eq,maybe_qm,maybe_ex = case context
+         when ?@,?$ then [nil,nil,nil]
+         when ?:    then [?=, ??, ?!]
+         else            [nil,??, ?!]
+      end
+
+      str<<til_charset(/[^a-z0-9_]/i)
+
+      #look for ?, !, or =, if allowed
+      case b=@file.getc
+      when nil #means we're at eof
+         #handling nil here prevents b from ever matching
+         #a nil value of maybe_qm, maybe_ex or maybe_eq
+      when maybe_qm
+         str << b
+      when maybe_ex
+         nc=(nextchar unless @file.eof?)
+         #does ex appear to be part of a larger operator?
+         if nc==?= #or nc==?~
+           back1char
+         else
+           str << b
+         end
+      when maybe_eq
+         nc=(nextchar unless @file.eof?)
+         #does eq appear to be part of a larger operator?
+         if nc==?= or nc==?~ or nc==?>
+           back1char
+         else
+           str << b
+         end
+      else
+         back1char
+      end
+
+
+      return str
+   end
+
+  #-----------------------------------
+  #contexts in which comma may appear in ruby:
+    #multiple lhs (terminated by assign op)
+    #multiple rhs (in implicit context) (tbd)
+    #method actual param list (in ( or implicit context)
+    #method formal param list (in ( or implicit context)
+    #block  formal param list (in | context) (tbd)
+    #hash immediate (in imm{ context)
+    #array immediate (in imm[ context)
+    #element reference/assignment (in [] or []= method actual parameter context)
+    #list after for
+    #list after rescue
+    #list after when
+    #list after undef
+
+    #note: comma in parens not around a param list is illegal
+
+   #-----------------------------------
+   #a comma has been seen. are we in an
+   #lvalue list or some other construct that uses commas?
+   def comma_in_lvalue_list?
+     not ListContext===@bracestack.last
+   end
+   
+   #-----------------------------------
+   def in_lvar_define_state
+     #@defining_lvar is a hack
+     @defining_lvar or case ctx=@bracestack.last
+       when ForSMContext: ctx.state==:for
+       when RescueSMContext: ctx.state==:arrow
+       when BlockParamListContext: true
+     end 
+   end
+   
+   #-----------------------------------
+   #determine if an alphabetic identifier refers to a variable
+   #or method name. generates implicit parenthes(es) if it is a
+   #call site and no explicit parens are present. starts an implicit param list
+   #if appropriate. adds tok to the
+   #local var table if its a local var being defined for the first time.
+   
+   #note: what we here call variables (rather, constants) following ::
+   #might actually be methods at runtime, but that's immaterial to tokenization.
+   
+   #note: this routine should determine the correct token type for name and
+   #create the appropriate token. currently this is not done because callers
+   #typically have done it (perhaps incorrectly) already.
+   def var_or_meth_name(name,lasttok,pos)
+     #look for call site if not a keyword or keyword is function-like
+     #look for and ignore local variable names
+
+     assert String===name
+
+     #fixme: keywords shouldn't be treated specially after :: and .
+     
+     #maybe_local really means 'maybe local or constant'
+     maybe_local=case name
+       when /[^a-z_0-9]$/i: #do nothing
+       when /^[a-z_]/:  (@localvars===name or VARLIKE_KEYWORDS===name or in_lvar_define_state) and not lasttok===/^(\.|::)$/
+       when /^[A-Z]/: is_const=true;not lasttok==='.'  #this is the right algorithm for constants... 
+     end 
+          
+     assert(@moretokens.empty?)
+
+     tok=@last_operative_token=VarNameToken.new(name,pos)
+
+     oldpos=@file.pos
+     sawnl=false
+     result=ws_toks=ignored_tokens(true) {|nl| sawnl=true }
+     sawnl || @file.eof?  and  return result.unshift(
+         *if  maybe_local : [tok] 
+         else [MethNameToken.new(name,pos),  #insert implicit parens right after tok
+               ImplicitParamListStartToken.new( oldpos),
+               ImplicitParamListEndToken.new( oldpos) ]
+         end
+     )
+     
+     #if next op is assignment (or comma in lvalue list)
+     #then omit implicit parens
+     assignment_coming=case nc=nextchar
+       when ?=:  not /^=[=~]$/===readahead(2)
+       when ?,: comma_in_lvalue_list?
+       when ?>,?<: /^([<>])\1=$/===readahead(3)
+       when ?*,?|,?&: /^([*|&])\1?=/===readahead(3)
+       when ?%,?/,?-,?+,?^: readahead(2)[1..1]=='='
+     end 
+     if (assignment_coming or in_lvar_define_state)
+        tok=VarNameToken.new(name,pos)
+        if /[^a-z_0-9]$/i===name 
+          lexerror tok,"not a valid variable name: #{name}"
+        elsif /^[a-z_]/===name and !(lasttok===/^(\.|::)$/)
+          @localvars[name]=true
+        end
+        return result.unshift(tok)
+     end
+
+     implicit_parens_to_emit=case nc
+       when ?!: readahead(2)=='!=' ? 2 : 1
+       when NEVERSTARTPARAMLISTFIRST
+         (NEVERSTARTPARAMLISTWORDS===readahead(NEVERSTARTPARAMLISTMAXLEN)) ? 2 : 1
+       when ?",?',?`,?a..?z,?A..?Z,?0..?9,?_,?@,?$,?~: 1
+       when ?{: maybe_local=false; 2
+       when ?(: maybe_local=false; 0
+       when ?},?],?),?;,?^, ?|, ?>, ?,, ?., ?=: 2
+       when ?+, ?-, ?*, ?&, ?%, ?/, ?:, ??: (ws_toks.empty? || readahead(2)[/^.[#{WHSPLF}]/]) ? 2 : 3
+       when ?<: (ws_toks.empty? || readahead(3)[/^<<[^"'`a-zA-Z_0-9-]/]) ? 2 : 3 
+       when ?[: ws_toks.empty? ? 2 : 3
+       when ?\\, ?\s, ?\t, ?\n, ?\r, ?\v, ?#: raise 'failure'
+       else raise "unknown char after ident: #{nextchar.chr}"
+     end
+     
+     implicit_parens_to_emit==3 and is_const and implicit_parens_to_emit=1
+     
+     tok=if maybe_local and implicit_parens_to_emit>=2 
+       implicit_parens_to_emit=0
+       VarNameToken
+     else
+       MethNameToken
+     end.new(name,pos)
+     
+     
+     case implicit_parens_to_emit
+     when 2:
+       result.unshift ImplicitParamListStartToken.new(oldpos),
+                 ImplicitParamListEndToken.new(oldpos)
+     when 1,3:
+       result.unshift ImplicitParamListStartToken.new(oldpos)
+       @bracestack.push ParamListContextNoParen.new(@linenum)
+     when 0: #do nothing
+     else raise 'invalid value of implicit_parens_to_emit'
+     end
+     return result.unshift(tok)
+     # 'ok:'
+     # 'if unless while until {'
+     # '\n (unescaped) and or'
+     # 'then else elsif rescue ensure (illegal in value context)'
+
+     # 'need to pop noparen from bracestack on these tokens: (in operator context)'
+     # 'not ok:'
+     # 'not (but should it be?)'
+   end
+
+  #-----------------------------------
+  CONTEXT2ENDTOK={AssignmentRhsContext=>AssignmentRhsListEndToken, 
+                  ParamListContextNoParen=>ImplicitParamListEndToken,
+                  KwParamListContext=>KwParamListEndToken
+                 }
+  def abort_noparens!(str='')
+    #assert @moretokens.empty?
+    result=[]
+    while klass=CONTEXT2ENDTOK[@bracestack.last.class]
+      result << klass.new(@file.pos-str.length)
+      @bracestack.pop
+    end
+    return result
+  end
+
+if false #no longer used
+   #-----------------------------------
+   def abort_1_noparen!(offs=0)
+     assert @moretokens.empty?
+     result=[]
+     while AssignmentRhsContext===@bracestack.last
+       @bracestack.pop
+       result << AssignmentRhsListEndToken.new(@file.pos-offs)
+     end
+     ParamListContextNoParen===@bracestack.last or lexerror huh,'{} with no matching callsite'
+       @bracestack.pop
+       result << ImplicitParamListEndToken.new(@file.pos-offs)
+     return result
+   end
+end
+
+   #-----------------------------------
+   #parse keywords now, to prevent confusion over bare symbols
+   #and match end with corresponding preceding def or class or whatever.
+   #if arg is not a keyword, the block is called
+   def parse_keywords(str,offset)
+      assert @moretokens.empty?
+      result=[KeywordToken.new(str,offset)]
+
+      case str
+      when "end"
+         result.unshift(*abort_noparens!(str))
+         @bracestack.last.see @bracestack,:semi #sorta hacky... should make an :end event instead?
+
+=begin not needed?
+         if ExpectDoOrNlContext===@bracestack.last
+            @bracestack.pop
+            assert @bracestack.last.starter[/^(while|until|for)$/]
+         end
+=end
+
+         WantsEndContext===@bracestack.last or lexerror result.last, 'unbalanced end'
+         ctx=@bracestack.pop
+         start,line=ctx.starter,ctx.linenum
+         BEGINWORDS===start or lexerror result.last, "end does not match #{start or "nil"}"
+         /^(class|module|def|do)$/===start and @localvars.end_block
+
+      when "class","module"
+         result.first.has_end!
+         @bracestack.push WantsEndContext.new(str,@linenum)
+         @localvars.start_block
+
+      when "if","unless" #could be infix form without end
+         if after_nonid_op?{false} #prefix form
+            result.first.has_end!
+            @bracestack.push WantsEndContext.new(str,@linenum)
+            
+
+         else #infix form
+           result.unshift(*abort_noparens!(str))
+         end
+      when "begin","case"
+         result.first.has_end!
+         @bracestack.push WantsEndContext.new(str,@linenum)
+      when "while","until" #could be infix form without end
+         if after_nonid_op?{false} #prefix form
+           result.first.has_end!
+           @bracestack.push WantsEndContext.new(str,@linenum)
+           expect_do_or_end_or_nl! str
+
+         else #infix form
+           result.unshift(*abort_noparens!(str))
+         end
+      when "for"
+         result.first.has_end!
+         @bracestack.push WantsEndContext.new(str,@linenum)
+         #expect_do_or_end_or_nl! str #handled by ForSMContext now
+         @bracestack.push ForSMContext.new(@linenum)
+      when "do"
+         result.unshift(*abort_noparens!(str))
+         if ExpectDoOrNlContext===@bracestack.last
+            @bracestack.pop
+            assert WantsEndContext===@bracestack.last
+         else
+            result.last.has_end!
+            @bracestack.push WantsEndContext.new(str,@linenum)
+            @localvars.start_block
+            block_param_list_lookahead
+         end
+      when "def"
+         result.first.has_end!
+         @bracestack.push WantsEndContext.new("def",@linenum)
+         @localvars.start_block
+         safe_recurse { |aa|
+            @last_operative_token=KeywordToken.new "def" #hack
+            result.concat ignored_tokens
+
+            #read an expr like a.b.c or a::b::c
+            #or (expr).b.c
+            if nextchar==?( #look for optional parenthesised head
+              old_size=@bracestack.size
+              parencount=0
+              begin
+                tok=get1token
+                case tok
+                when/^\($/.token_pat then parencount+=1
+                when/^\)$/.token_pat then parencount-=1
+                end
+                EoiToken===tok and lexerror tok, "eof in def header"
+                result<<tok
+              end until  parencount==0 #@bracestack.size==old_size
+            else #no parentheses, all tail
+              @last_operative_token=KeywordToken.new "." #hack hack
+              result << symbol(false,false)
+              #this isn't quite right.... if a.b.c.d is seen, a, b, and c
+              #should be considered maybe varname instead of methnames.
+              #the last (d in the example) is always considered a methname;
+              #it's what's being defined.
+            end
+            #read tail: .b.c.d etc
+            @last_operative_token=result.last
+            state=:expect_op
+            loop do
+
+               #look for start of parameter list
+               nc=(@moretokens.first or nextchar.chr)
+               if state==:expect_op and /^[a-z_(&*]/i===nc
+                  result.concat def_param_list
+                  break
+               end
+
+               tok=get1token
+               result<<tok
+               case tok
+               when EoiToken
+                  lexerror tok,'unexpected eof in def header'
+               when IgnoreToken
+               when MethNameToken #,VarNameToken # /^[a-z_]/i.token_pat
+                  lexerror tok,'expected . or ::' unless state==:expect_name
+                  state=:expect_op
+               when /^(\.|::)$/.token_pat
+                  lexerror tok,'expected ident' unless state==:expect_op
+                  state=:expect_name
+               when /^(;|end)$/.token_pat, NewlineToken #are we done with def name?
+                  state==:expect_op or lexerror tok,'expected identifier'
+                  break
+               else
+                  lexerror(tok, "bizarre token in def name: " +
+                           "#{tok}:#{tok.class}")
+               end
+            end
+         }
+      when "alias"
+         safe_recurse { |a|
+            @last_operative_token=KeywordToken.new "alias" #hack
+            result.concat ignored_tokens
+            res=symbol(eat_next_if(?:),false) 
+            res ? result<<res : lexerror(result.first,"bad symbol in alias")
+            @last_operative_token=KeywordToken.new "alias" #hack
+            result.concat ignored_tokens
+            res=symbol(eat_next_if(?:),false) 
+            res ? result<<res : lexerror(result.first,"bad symbol in alias")
+         }
+      when "undef"
+         safe_recurse { |a|
+            loop do
+               @last_operative_token=KeywordToken.new "," #hack
+               result.concat ignored_tokens
+               tok=symbol(eat_next_if(?:),false)
+               tok or lexerror(result.first,"bad symbol in undef")
+               result<< tok
+               @last_operative_token=tok
+
+               sawnl=false
+               result.concat ignored_tokens(true){|nl| sawnl=true}
+
+               break if sawnl or nextchar != ?,
+               tok= single_char_token(?,)
+               result<< tok
+            end
+         }
+         
+#      when "defined?"
+#         huh
+         #defined? might have a baresymbol following it
+         #does it need to be handled specially?
+
+      when "when"
+         result.unshift(*abort_noparens!(str))
+         @bracestack.push KwParamListContext.new(str,@linenum)
+
+      when "rescue"
+         result.unshift(*abort_noparens!(str))
+         @bracestack.push RescueSMContext.new(@linenum)
+
+      when "then","in"
+         result.unshift(*abort_noparens!(str))
+         @bracestack.last.see @bracestack,str.to_sym
+         
+      when /^(#{BINOPWORDS}|#{INNERBOUNDINGWORDS})$/o
+         result.unshift(*abort_noparens!(str))
+
+      when FUNCLIKE_KEYWORDS: result=yield
+
+      when RUBYKEYWORDS
+        #do nothing
+        
+      else result=yield
+
+      end
+
+      return result
+   end
+
+
+   #-----------------------------------
+   def block_param_list_lookahead
+      safe_recurse{ |la|
+         @last_operative_token=KeywordToken.new  ';'
+         a=ignored_tokens
+
+         if eat_next_if(?|)
+           a<<KeywordToken.new("|",@file.pos-1)
+           if eat_next_if(?|)
+             a.concat [NoWsToken.new(@file.pos-1),
+                       KeywordToken.new('|',@file.pos-1)]
+           else
+             assert !@defining_lvar
+             @defining_lvar=true
+             assert((@last_operative_token===';' or NewlineToken===@last_operative_token))
+             @bracestack.push BlockParamListContext.new(@linenum)
+             #block param initializers are not supported here, because ruby doesn't allow them!
+             begin
+               tok=get1token
+               EoiToken===tok and lexerror tok,"eof in block parameter list"
+               a<<tok
+             end until tok==='|'
+             assert@defining_lvar
+             @defining_lvar=false
+             BlockParamListContext===@bracestack.last or raise 'expected BlockParamListContext atop @bracestack'
+             @bracestack.pop
+             @moretokens.empty? or
+               fixme %#moretokens might be set from get1token call above...might be bad#
+           end
+         end
+
+         @last_operative_token=KeywordToken.new ';'
+         #a.concat ignored_tokens
+
+         #assert @last_operative_token===';'
+         #a<<get1token
+
+         la[0,0]=a
+      }
+   end
+
+   #-----------------------------------
+   #handle parameter list of a method declaration.
+   #parentheses are optional... if missing param list
+   #is ended by (unescaped) newline or semicolon (at the same bracing level)
+   #expect a brace as the next token,
+   #then match the following tokens until
+   #the matching endbrace is found
+   def def_param_list
+      result=[]
+      normal_comma_level=old_bracestack_size=@bracestack.size
+      safe_recurse { |a|
+         assert(@moretokens.empty?)
+         assert((not IgnoreToken===@moretokens[0]))
+         assert((@moretokens[0] or not nextchar.chr[WHSPCHARS]))
+
+         #have parentheses?
+         if '('==@moretokens[0] or nextchar==?(
+            #get open paren token
+            result.concat maybe_no_ws_token
+            result << tok=get1token
+            assert(tok==='(')
+
+
+            #bracestack was changed by get1token above...
+            normal_comma_level+=1
+            assert(normal_comma_level==@bracestack.size)
+            endingblock=proc{|tok| tok===')' }
+         else
+            endingblock=proc{|tok| tok===';' or NewlineToken===tok}
+         end
+         class << endingblock
+            alias === call
+         end
+
+         @last_operative_token=KeywordToken.new ',' #hack
+         #read local parameter names
+         loop do
+            expect_name=(@last_operative_token===',' and
+                         normal_comma_level==@bracestack.size)
+            expect_name and @defining_lvar||=true
+            result << tok=get1token
+            lexerror tok, "unexpected eof in def header" if EoiToken===tok
+
+            #break if at end of param list
+               endingblock===tok and 
+                 old_bracestack_size>=@bracestack.size and break
+
+            #next token is a local var name
+            #(or the one after that if unary ops present)
+            #result.concat ignored_tokens
+            expect_name and case tok
+               when IgnoreToken#, /^[A-Z]/ #do nothing
+               when VarNameToken
+                  assert@defining_lvar
+                  @defining_lvar=false
+                  assert((not @last_operative_token===','))
+               when /^[&*]$/.token_pat #unary form...
+                  #a NoWsToken is also expected... read it now
+                  result.concat maybe_no_ws_token #not needed?
+                  @last_operative_token=KeywordToken.new ','
+               else lexerror tok,"unfamiliar var name '#{tok}'"
+            end
+         end
+         
+         @defining_lvar=false
+                   
+
+         assert(@bracestack.size <= old_bracestack_size)
+         assert(endingblock[tok])
+
+         #hack: force next token to look like start of a
+         #new stmt, if the last ignored_tokens
+         #call above did not find a newline
+         #(just in case the next token parsed
+         #happens to call quote_expected? or after_nonid_op)
+         result.concat ignored_tokens
+         if  nextchar.chr[/[iuw\/<|>+\-*&%?:]/] and
+             !(NewlineToken===@last_operative_token) and
+             !(/^(end|;)$/===@last_operative_token)
+           @last_operative_token=KeywordToken.new ';'
+           result<< get1token
+         end
+      }
+
+      return result
+   end
+
+
+   #-----------------------------------
+   #handle % in ruby code. is it part of fancy quote or a modulo operator?
+   def percent(ch)
+      if quote_expected? ch
+         fancy_quote ch
+      else
+         biop ch
+      end
+   end
+
+   #-----------------------------------
+   #handle * in ruby code. is unary or binary operator?
+   def star_or_amp(ch)
+      assert('*&'[ch])
+      if unary_op_expected? ch
+         #readahead(2)[1..1][/[\s\v#\\]/] or #not needed?
+         result=operator_or_methname_token getchar
+         WHSPLF[nextchar.chr] or
+           @moretokens << NoWsToken.new(@file.pos)
+         return result
+      else
+         return(quadriop ch)
+      end
+         #result should distinguish unary+binary *&
+   end
+
+   #-----------------------------------
+   #handle ? in ruby code. is it part of ?..: or a character literal?
+   def char_literal_or_op(ch)
+      if colon_quote_expected? ch
+         getchar
+         NumberToken.new getchar_maybe_escape
+      else
+         @bracestack.push TernaryContext.new(@linenum)
+         KeywordToken.new getchar   #operator
+      end
+   end
+
+   #-----------------------------------
+   def regex_or_div(ch) 
+   #space after slash always means / operator, rather than regex start
+      if after_nonid_op?{ !is_var_name? and WHSPLF[prevchar] and !readahead(2)[%r{^/\s}] }
+        return regex(ch)
+      else #/ is operator
+        result=getchar
+        if eat_next_if(?=)
+          result << '='
+        end
+        return(operator_or_methname_token result)
+      end
+   end
+
+   #-----------------------------------
+   #return true if tok corresponds to a variable or constant, false if its for a method, nil for something else
+   #we assume tok is a valid token with a correctly formed name.
+   #...should really be called was_var_name
+   def is_var_name?
+     (tok=@last_operative_token)
+
+     s=tok.to_s
+     case s
+     when /[^a-z_0-9]$/i: false
+     when /^[a-z_]/: @localvars===s or VARLIKE_KEYWORDS===s
+     when /^[A-Z]/: VarNameToken===tok
+     when /^[@$<]/: true
+     else raise "not var or method name: #{s}"
+     end   
+   end
+   
+   #-----------------------------------
+   def colon_quote_expected?(ch) #yukko hack
+     assert ':?'[ch]
+     readahead(2)[/^(\?[^#{WHSPLF}]|:[$@a-zA-Z_'"`\[*~+\-\/%<=>&|^])$/o]   or return false
+
+     after_nonid_op? {
+       #possible func-call as operator
+
+       !is_var_name?
+     }
+   end
+
+   #-----------------------------------
+   def symbol_or_op(ch)
+      startpos=@file.pos
+      qe= colon_quote_expected?(ch)
+      lastchar=prevchar
+      eat_next_if(ch) or raise "needed: "+ch
+
+      #handle quoted symbols like  :"foobar",  :"[]"
+      qe and return symbol(':')
+
+      #look for another colon; return single : if not found
+      unless eat_next_if(?:) 
+        #cancel implicit contexts...
+        @moretokens.push(*abort_noparens!(':'))
+        
+        #end ternary context, if any
+        @bracestack.last.see @bracestack,:colon
+        
+        TernaryContext===@bracestack.last and @bracestack.pop #should be in the context's see handler
+        
+        if ExpectDoOrNlContext===@bracestack.last #should be in the context's see handler
+          @bracestack.pop
+          assert @bracestack.last.starter[/^(while|until|for)$/]
+        end
+        
+        @moretokens.push KeywordToken.new(':',startpos)
+        return @moretokens.shift
+      end
+      
+      #we definately found a ::
+
+      colon2=KeywordToken.new( '::',startpos)
+      lasttok=@last_operative_token
+      assert !(String===lasttok)
+      if (VarNameToken===lasttok or MethNameToken===lasttok) and
+          lasttok===/^[$@a-zA-Z_]/ and !WHSPCHARS[lastchar]
+      then
+         @moretokens << colon2
+         result= NoWsToken.new(startpos)
+      else
+         result=colon2
+      end
+      dot_rhs(colon2)
+      return result
+   end
+
+   #-----------------------------------
+   def symbol(notbare,couldbecallsite=!notbare)
+     assert !couldbecallsite
+     start=@file.pos
+     notbare and start-=1
+     klass=(notbare ? SymbolToken : MethNameToken)
+     
+     #look for operators
+     opmatches=readahead(3)[RUBYSYMOPERATORREX]
+     result= opmatches ? @file.read(opmatches.size) :
+       case nc=nextchar
+         when ?" then assert notbare;double_quote('"')
+         when ?' then assert notbare;double_quote("'")
+         when ?` then @file.read(1)
+         when ?@ then at_identifier.to_s
+         when ?$ then dollar_identifier.to_s
+         when ?_,?a..?z then identifier_as_string(?:)
+         when ?A..?Z then 
+           result=identifier_as_string(?:)
+           if @last_operative_token==='::' 
+             assert klass==MethNameToken
+             /[A-Z_0-9]$/i===result and klass=VarNameToken
+           end
+           result
+         else error= "unexpected char starting symbol: #{nc.chr}"
+       end
+     return lexerror(klass.new(result,start),error)
+   end
+
+   #-----------------------------------
+   def callsite_symbol(tok_to_errify)
+     start=@file.pos
+     
+     #look for operators
+     opmatches=readahead(3)[RUBYSYMOPERATORREX]
+     return [opmatches ? @file.read(opmatches.size) :
+       case nc=nextchar
+         when ?` then @file.read(1)
+         when ?_,?a..?z,?A..?Z then identifier_as_string(?:)
+         else 
+           @last_operative_token=KeywordToken.new(';')
+           lexerror(tok_to_errify,"unexpected char starting symbol: #{nc.chr}")
+           nil
+       end, start
+      ]
+   end
+
+   #-----------------------------------
+   def here_header
+      @file.read(2)=='<<' or raise "parser insanity"
+
+      dash=eat_next_if(?-)
+      quote=eat_next_if( /^['"`]$/)
+      if quote
+        ender=til_charset(/[#{quote}]/)
+        (quote==getchar) or 
+          return lexerror(HerePlaceholderToken.new( dash, quote, ender ), "mismatched quotes in here doc")
+      else
+        quote='"'
+        ender=til_charset(/[^a-zA-Z0-9_]/)
+        ender.length >= 1  or 
+          return lexerror(HerePlaceholderToken.new( dash, quote, ender ), "invalid here header")
+      end
+
+      res= HerePlaceholderToken.new( dash, quote, ender )
+      @incomplete_here_tokens.push res
+
+      #hack: normally this should just be in get1token
+      #this fixup is necessary because the call the get1token below
+      #makes a recursion.
+      @last_operative_token=res
+
+      safe_recurse { |a|
+         assert(a.object_id==@moretokens.object_id)
+         toks=[]
+         begin
+           #yech. 
+           #handle case of here header in a string inclusion, but
+           #here body outside it.
+           cnt=0
+           1.upto @bracestack.size do |i|
+             case @bracestack[-i]
+               when AssignmentRhsContext,ParamListContextNoParen,TopLevelContext
+               else cnt+=1
+             end
+           end
+           if nextchar==?} and cnt==1
+             res.bodyclass=OutlinedHereBodyToken
+             break
+           end
+           
+           tok=get1token
+           assert(a.object_id==@moretokens.object_id)
+           toks<<tok
+           EoiToken===tok and lexerror tok, "here body expected before eof"
+         end while res.unsafe_to_use
+         assert(a.object_id==@moretokens.object_id)
+         a[0,0]= toks   #same as a=toks+a, but keeps a's id
+      }
+
+      return res
+
+      #the action continues in newline, where
+      #the rest of the here token is read after a
+      #newline has been seen and res.affix is eventually called
+   end
+
+   #-----------------------------------
+   def lessthan(ch) #match quadriop('<') or here doc or spaceship op
+      case readahead(3)
+        when /^<<['"`\-a-z0-9_]$/i
+           if quote_expected?(ch) #and @last_operative_token!='class' #not needed?
+              here_header
+           else
+              operator_or_methname_token @file.read(2)
+           end
+        when "<=>" then operator_or_methname_token @file.read(3)
+        else quadriop(ch)
+      end
+   end
+
+   #-----------------------------------
+   def escnewline(ch)
+      assert ch == '\\'
+      
+      pos=@file.pos
+      result=getchar
+      if nl=readnl
+        result+=nl
+      else
+        error='illegal escape sequence'
+      end
+      lexerror EscNlToken.new(@filename,@linenum,result,pos), error
+   end
+
+   #-----------------------------------
+   def newline(ch)
+      assert("\r\n"[nextchar.chr])
+
+      #handle here bodies queued up by previous line
+      #(we should be more compatible with dos/mac style newlines...)
+      if tofill=@incomplete_here_tokens.shift
+         tofill.string.offset=@file.pos
+         loop {
+            assert("\r\n"[nextchar.chr])
+
+            #retr evrything til next nl
+            line=all_quote(/^[\r\n]$/, tofill.quote, /^[\r\n]$/, :regex_esc_seq)
+            #(you didn't know all_quote could take a regex, did you?)
+
+            #get rid of fals that otherwise appear to be in the middle of
+            #a string (and are emitted out of order)
+            fal=@moretokens.pop
+            assert FileAndLineToken===fal || fal.nil?
+
+            back1char
+            assert("\r\n"[nextchar.chr])
+
+            #matches terminating reg expr?
+            break if line.elems.size==1 and
+                     line.elems[0][tofill.termex]
+
+            tofill.append_token line
+            tofill.append readnl
+            back1char
+         }
+
+         assert("\r\n"[nextchar.chr])
+         tofill.unsafe_to_use=false
+
+         return tofill.bodyclass.new(tofill)
+      end
+
+      #ordinary newline handling (possibly implicitly escaped)
+      assert("\r\n"[nextchar.chr])
+      assert @moretokens.empty?
+      result=if NewlineToken===@last_operative_token or #hack
+                @last_operative_token===/^(;|begin|do|#{INNERBOUNDINGWORDS})$/ or #hack
+                !after_nonid_op?{false}
+             then   #hack-o-rama: probly cases left out above
+                   a= abort_noparens!
+                   ExpectDoOrNlContext===@bracestack.last and @bracestack.pop
+                   @bracestack.last.see @bracestack,:semi
+
+                   a << super(ch)
+                   @moretokens.replace a+@moretokens
+                   @moretokens.shift
+             else
+                   offset=@file.pos
+                   #@moretokens <<
+                      EscNlToken.new(@filename,@linenum,readnl,offset)
+                   #WsToken.new ' ' #why?  #should be "\\\n" ?
+             end
+
+      start_of_line_directives
+
+      return result
+   end
+
+   #-----------------------------------
+   EQBEGIN=%r/^=begin[^a-zA-Z_0-9]$/
+   EQBEGINLENGTH=7
+   EQEND='=end'
+   ENDMARKER=/^__END__[\r\n]$/
+   ENDMARKERLENGTH=8
+   def start_of_line_directives
+      #handle =begin...=end (at start of a line)
+      while EQBEGIN===readahead(EQBEGINLENGTH)
+         startpos=@file.pos
+         more=@file.read(EQBEGINLENGTH-1)   #get =begin
+
+         #keep reading til /\n=end.*\n/
+         @file.each(EQEND) {|cblock|
+            more << cblock
+            #must be at start of line
+            break if /^[\r\n]#{EQEND}/o===readback(EQEND.length+1)
+         }
+         #read rest of line after =end
+         more << @file.til_charset(/[\r\n]/)  
+         assert((?\r===nextchar or ?\n===nextchar))
+         assert !(/[\r\n]/===more[-1,1])
+
+         newls= more.scan(/\r\n?|\n\r?/)
+         @linenum+= newls.size
+
+         #inject the fresh comment into future token results
+         @moretokens.push IgnoreToken.new(more,startpos)
+      end
+
+      #handle __END__
+      if ENDMARKER===readahead(ENDMARKERLENGTH)
+         assert !(ImplicitContext===@bracestack.last)
+         @moretokens.unshift endoffile_detected(@file.read(6))
+         @file.pos=@file.stat.size
+      end
+   end
+
+
+
+  #-----------------------------------
+  #used to resolve the ambiguity of
+  # unary ops (+, -, *, &, ~ !) in ruby
+  #returns whether current token is to be the start of a literal
+  IDBEGINCHAR=/^[a-zA-Z_$@]/
+  def unary_op_expected?(ch) #yukko hack
+    '*&='[readahead(2)[1..1]] and return false
+
+    after_nonid_op? {
+      #possible func-call as operator
+
+      not is_var_name? and
+        WHSPLF[prevchar]
+    }
+  end
+
+   #-----------------------------------
+   #used to resolve the ambiguity of
+   # <<, %, ? in ruby
+   #returns whether current token is to be the start of a literal
+   #/ is not handled right here if whitespace immediately follows the /
+   def quote_expected?(ch) #yukko hack
+     case ch[0]
+          when ?? then readahead(2)[/^\?[#{WHSPLF}]$/o] #not needed?
+          when ?% then readahead(3)[/^%([a-ps-vyzA-PR-VX-Z]|[QqrwWx][a-zA-Z0-9])/]
+          when ?< then !readahead(4)[/^<<-?['"`a-z0-9_]/i]
+          else raise 'unexpected ch (#{ch}) in quote_expected?'
+     #     when ?+,?-,?&,?*,?~,?! then '*&='[readahead(2)[1..1]]
+     end and return false
+
+     after_nonid_op? {
+       #possible func-call as operator
+
+       not is_var_name? and
+         WHSPLF[prevchar] and not WHSPLF[readahead(2)[1..1]]
+     }
+   end
+
+   #-----------------------------------
+   #used to resolve the ambiguity of
+   # <<, %, /, ?, :, and newline in ruby
+   def after_nonid_op?
+      case @last_operative_token
+         when MethNameToken,VarNameToken, FUNCLIKE_KEYWORDS.token_pat
+            return yield
+         when StringToken, SymbolToken, NumberToken, HerePlaceholderToken,
+              %r{^(class|module|do|end|self|true|false|nil|
+                   __FILE__|__LINE__|[\})\]]|alias|(un)?def|for
+              )$}x.token_pat
+            #do shouldn't be in above list... dunno about def/undef
+            #maybe class/module shouldn't either?  
+            #for is also in NewlineToken branch, below.
+            #what about rescue?
+            return false
+         when /^(#{RUBYOPERATORREX}|#{INNERBOUNDINGWORDS})$/o.token_pat
+            #regexs above must match whole string
+            #assert(@last_operative_token==$&) #disabled 'cause $& is now always nil :(
+            return true
+         when NewlineToken, nil,   #nil means we're still at beginning of file
+              /^([({\[]|or|not|and|if|unless|then|elsif|else|
+                 while|until|begin|for|in|case|when|ensure)$
+              /x.token_pat
+            return true
+         #when KeywordToken
+         #   return true
+         when IgnoreToken
+            raise "last_operative_token shouldn't be ignoreable"
+         else
+            raise "after_nonid_op? after #{@last_operative_token}:#{@last_operative_token.class} -- now what"
+      end
+   end
+
+   #-----------------------------------
+   def quadriop(ch) #match /&&?=?/ (&, &&, &=, or &&=)
+      assert(%w[& * | < >].include?(ch))
+#      '&*'[ch] and qe=quote_expected?(ch) #not needed?
+      result=getchar + (eat_next_if(ch)or'')
+      if eat_next_if(?=)
+         result << ?=
+#      elsif qe and result[/^[&*]$/] #not needed?
+#         @moretokens<<NoWsToken.new(@file.pos) #not needed?
+      end
+      return operator_or_methname_token(result)
+   end
+
+   #-----------------------------------
+   def biop(ch) #match /%=?/ (% or %=)
+      assert(ch[/^[%^~]$/])
+      result=getchar
+      if eat_next_if(?=)
+         result <<?=
+      end
+      return operator_or_methname_token( result)
+   end
+
+   #-----------------------------------
+   def tilde(ch) #match /~=?/ (~ or ~=)
+      assert(ch=='~')
+      result=getchar
+#      eat_next_if(?=) ?
+#         result <<?= :
+         WHSPLF[nextchar.chr] ||
+           @moretokens << NoWsToken.new(@file.pos)
+      #why is the NoWsToken necessary at this point?    
+      return operator_or_methname_token( result)
+         #result should distinguish unary ~
+   end
+
+   #-----------------------------------
+   def want_op_name
+      KeywordToken===@last_operative_token and
+         @last_operative_token===/^(alias|(un)?def|\.|::)$/
+   end
+
+   #-----------------------------------
+   #match /[+\-]=?/ (+ or +=)
+   #could be beginning of number, too
+   #fixme: handle +@ and -@ here as well... (currently, this is done in symbol()?)
+   def plusminus(ch)
+      assert(/^[+\-]$/===ch)
+      if unary_op_expected?(ch)
+        if (?0..?9)===readahead(2)[1]
+          return number(ch)
+        else #unary operator
+          result=getchar
+          WHSPLF[nextchar.chr] or
+            @moretokens << NoWsToken.new(@file.pos)
+          return(operator_or_methname_token result)
+          #todo: result should distinguish unary+binary +-
+        end
+      else #binary operator
+         assert(! want_op_name)
+         result=getchar
+         if eat_next_if(?=)
+            result << ?=
+         end
+         return(operator_or_methname_token result)
+         #todo: result should distinguish unary+binary +-
+      end
+   end
+
+   #-----------------------------------
+   def equals(ch) #match /=(>|~|==?)?/ (= or == or =~ or === or =>)
+      offset=@file.pos
+      str=getchar
+      assert str=='='
+      c=(eat_next_if(/^[~=>]$/)or'')
+      str << c
+      case c
+      when '=': str<< (eat_next_if(?=)or'')
+      
+      when '>': @bracestack.last.see @bracestack,:arrow
+      when '': #record local variable definitions
+
+        @bracestack.push AssignmentRhsContext.new(@linenum)
+        @moretokens.unshift AssignmentRhsListStartToken.new( offset+1)
+      end
+      return operator_or_methname_token( str,offset)
+   end
+
+   #-----------------------------------
+   def exclam(ch) #match /![~=]?/ (! or != or !~)
+      assert nextchar==?!
+      result=getchar
+      k=eat_next_if(/^[~=]$/)
+      if k
+        result+=k
+      else
+        WHSPLF[nextchar.chr] or
+          @moretokens << NoWsToken.new(@file.pos)
+      end
+      return KeywordToken.new(result)
+      #result should distinguish unary !
+   end
+
+   #-----------------------------------
+   def dot(ch)
+      str=''
+      eat_next_if(?.) or raise "lexer confusion"
+
+      #three lumps of sugar or two?
+      eat_next_if(?.) and
+         return KeywordToken.new(eat_next_if(?.)? "..." : "..")
+
+      #else saw just single .
+      #match a valid ruby id after the dot
+      result= KeywordToken.new( ".")
+      dot_rhs(result)
+      return result
+   end
+   
+   #-----------------------------------
+   def dot_rhs(prevtok)
+      safe_recurse { |a|
+         @last_operative_token=prevtok
+         aa= ignored_tokens
+         tok,pos=callsite_symbol(prevtok)
+         tok and aa.push(*var_or_meth_name(tok,prevtok,pos)) 
+         a.unshift(*aa)
+      }     
+   end
+
+   #-----------------------------------
+   def single_quote(ch=nil)
+      double_quote(ch)
+   end
+
+  #-----------------------------------
+  def back_quote(ch=nil)
+    oldpos=@file.pos
+    @last_operative_token===/^(def|::|\.)$/ and return MethNameToken.new(
+       (eat_next_if(?`) or raise "insanity"), oldpos
+    )
+    double_quote(ch)
+  end
+
+   #-----------------------------------
+   def comment(str)
+     result=""
+     #loop{
+       result<<super(nil).to_s
+
+       if /^\#.*\#$/===result #if comment was ended by a crunch
+
+         #that's not a legal comment end in ruby, so just keep reading
+         assert(result.to_s[-1]==?#)
+         result.chomp! '#'
+
+         #back up one char in input so that the
+         #super will see that # on the next go round.
+         #this hack makes the ruma comment lexer work with ruby too.
+         back1char
+
+         assert nextchar==?#
+       #else break #not a crunch... just exit
+       end
+     #}
+
+     return IgnoreToken.new(result)
+   end
+
+   #-----------------------------------
+   def open_brace(ch)
+      assert((ch!='[' or !want_op_name))
+      assert(@moretokens.empty?)
+      lastchar=prevchar
+      ch=eat_next_if(/^[({\[]$/)or raise "lexer confusion"
+      tokch=KeywordToken.new(ch,@file.pos-1)
+
+      #maybe emitting of NoWsToken can be moved into var_or_meth_name ??
+      case tokch.ident
+      when '['
+        #fixme: in contexts expecting an (operator) method name, we 
+        #       should match [] or []= at this point
+        @bracestack.push ListImmedContext.new(ch,@linenum)
+        lasttok=last_operative_token
+        #could be: lasttok===/^[a-z_]/i
+        if (VarNameToken===lasttok or MethNameToken===lasttok or
+            lasttok===FUNCLIKE_KEYWORDS) and !WHSPCHARS[lastchar]
+               @moretokens << (tokch)
+               tokch= NoWsToken.new(@file.pos-1)
+        end
+      when '('
+        lasttok=last_operative_token
+        #could be: lasttok===/^[a-z_]/i
+        if (VarNameToken===lasttok or MethNameToken===lasttok or
+            lasttok===FUNCLIKE_KEYWORDS)
+          unless WHSPCHARS[lastchar]
+               @moretokens << tokch
+               tokch= NoWsToken.new(@file.pos-1)
+          end
+          @bracestack.push ParamListContext.new(@linenum)
+        else
+          @bracestack.push ParenContext.new(@linenum)
+        end
+
+      when '{'
+      #check if we are in a hash literal or string inclusion (#{}),
+      #in which case below would be bad.
+      if after_nonid_op?{false}
+        @bracestack.push ListImmedContext.new(ch,@linenum) #that is, a hash
+      else
+=begin not needed now, i think
+        # 'need to find matching callsite context and end it if implicit'
+        lasttok=last_operative_token
+        unless lasttok===')' and lasttok.callsite?
+          @moretokens.push *(abort_1_noparen!(1).push tokch)
+          tokch=@moretokens.shift
+        end
+=end
+
+        @localvars.start_block
+        @bracestack.push BlockContext.new(@linenum)
+        block_param_list_lookahead
+      end
+      end
+      return (tokch)
+   end
+
+   #-----------------------------------
+   def close_brace(ch)
+      ch==eat_next_if(/[)}\]]/) or raise "lexer confusion"
+      @moretokens.concat abort_noparens!(ch)
+      @moretokens<< kw=KeywordToken.new( ch,@file.pos-1)
+      @bracestack.last.see @bracestack,:semi #hack
+      if @bracestack.empty? 
+        lexerror kw,"unmatched brace: #{ch}"
+        return @moretokens.shift
+      end
+      ctx=@bracestack.pop
+      origch,line=ctx.starter,ctx.linenum
+      ch==PAIRS[origch] or
+        lexerror kw,"mismatched braces: #{origch}#{ch}\n" +
+                 "matching brace location", @filename, line
+      BlockContext===ctx and @localvars.end_block
+      if ParamListContext==ctx.class
+        assert ch==')'
+        #kw.set_callsite! #not needed?
+      end
+      return @moretokens.shift
+   end
+
+   #-----------------------------------
+   def eof(ch=nil)
+       #this must be the very last character...
+       oldpos=@file.pos
+       assert(?\0==@file.getc)
+
+       result= "\0#{ignored_tokens(true).delete_if{|t|FileAndLineToken===t}}"
+
+       @file.pos==@file.stat.size or
+          lexerror result,'nul character is not at the end of file'
+       @file.pos=@file.stat.size
+       return(endoffile_detected result)
+   end
+
+   #-----------------------------------
+   def endoffile_detected(s='')
+     @moretokens.push( *(abort_noparens!.push super(s)))
+     result= @moretokens.shift
+     balanced_braces? or (lexerror result,"unbalanced braces at eof. bracestack=#{@bracestack.inspect}")
+     result
+   end
+
+  #-----------------------------------
+  def single_char_token(ch)
+    KeywordToken.new super(ch), @file.pos-1
+  end
+
+  #-----------------------------------
+  def comma(ch)
+    single_char_token(ch)
+  end
+  
+  #-----------------------------------
+  def semicolon(ch)
+      assert @moretokens.empty?
+    @moretokens.push(*abort_noparens!)
+    @bracestack.last.see @bracestack,:semi
+         if ExpectDoOrNlContext===@bracestack.last #should be in context's see:semi handler
+            @bracestack.pop
+            assert @bracestack.last.starter[/^(while|until|for)$/]
+         end
+    @moretokens.push single_char_token(ch)
+    return @moretokens.shift
+  end
+
+  #-----------------------------------
+  def operator_or_methname_token(s,offset=nil)
+    assert RUBYOPERATORREX===s
+    if RUBYNONSYMOPERATORREX===s
+      KeywordToken
+    elsif @last_operative_token===/^(\.|::|def|undef|alias|defined\?)$/
+      MethNameToken
+    else 
+      OperatorToken
+    end.new(s,offset)
+  end
+
+  #-----------------------------------
+  #tokenify_results_of  :identifier
+  save_offsets_in(*CHARMAPPINGS.values.uniq-[:symbol_or_op,:open_brace,:whitespace])
+  #save_offsets_in :symbol
+
+end
+