rubylexer 0.6.2 → 0.7.0

data/lib/rubylexer/0.6.2.rb

@@ -0,0 +1,39 @@
+require 'rubylexer/0.7.0'
+
+#make ImplicitParamList Start and End tokens descend from IgnoreToken again
+class RubyLexer
+  remove_const :ImplicitParamListStartToken
+  remove_const :ImplicitParamListEndToken
+
+  class ImplicitParamListStartToken < IgnoreToken
+#    include StillIgnoreToken
+    def initialize(offset)
+      super("(",offset)
+    end
+    def to_s; '' end
+  end
+
+  class ImplicitParamListEndToken < IgnoreToken
+#    include StillIgnoreToken
+    def initialize(offset)
+      super(")",offset)
+    end
+    def to_s; '' end
+  end
+end
+
+RubyLexer.constants.map{|k| 
+  k.name[/[^:]+$/] if Token>=k or Context>=k
+}.compact + %w[
+  RuLexer CharHandler CharSet SymbolTable 
+  SimpleTokenPrinter KeepWsTokenPrinter
+].each{|name|
+  Object.const_set name, RubyLexer.const_get name
+}
+
+
+class RubyLexer
+  def merge_assignment_op_in_setter_callsites?
+    true
+  end
+end

data/lib/rubylexer/0.6.rb

@@ -0,0 +1,5 @@
+require 'rubylexer/0.6.2'
+
+class RubyLexer
+  alias lexerror lexerror_exception
+end

data/lib/rubylexer/0.7.0.rb

@@ -0,0 +1,2 @@
+require 'rubylexer'
+#nothing else (yet)

data/{charhandler.rb → lib/rubylexer/charhandler.rb}

@@ -17,7 +17,8 @@
     Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
 =end
 
-require "charset"
+require "rubylexer/charset"
+class RubyLexer
 #------------------------------------
 class CharHandler
   #-----------------------------------
@@ -41,7 +42,7 @@ class CharHandler
       when Fixnum
         self[pattern]=action
       else
-        raise "invalid pattern class #{pattern.class}"
+        raise "invalid pattern class #{pattern.class}: #{pattern}"
       end
     }
 
@@ -80,5 +81,6 @@ class CharHandler
     end until go(mychar,*args)
   end
 end
+end
 
 

data/{charset.rb → lib/rubylexer/charset.rb}

@@ -17,6 +17,7 @@
     Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
 =end
 
+class RubyLexer
 class CharSet
    def initialize(*charss)
       clear
@@ -31,9 +32,9 @@ class CharSet
 
    def add(chars)
       case chars
-      when String
+      when ::String
          chars.each_byte {|c| @bitset |= (1<<c) }
-      when Fixnum then        @bitset |= (1<<chars)
+      when ::Fixnum then        @bitset |= (1<<chars)
       else chars.each    {|c| @bitset |= (1<<c) }
       end
    end
@@ -72,5 +73,5 @@ class CharSet
       return result
    end
 end
-
+end
 

data/{context.rb → lib/rubylexer/context.rb}

@@ -1,3 +1,4 @@
+class RubyLexer
 module NestedContexts
   class NestedContext
     attr :starter
@@ -13,7 +14,8 @@ module NestedContexts
       @ender==tok
     end
     
-    def see stack,msg; end
+    def see lxr,msg; end
+    def lhs=*x; end #do nothing
   end
 
   class ListContext < NestedContext 
@@ -30,6 +32,7 @@ module NestedContexts
     def initialize(linenum)
       super('(', ')' ,linenum)
     end
+    attr_accessor :lhs
   end
 
   class BlockContext  < NestedContext
@@ -38,16 +41,31 @@ module NestedContexts
     end
   end
 
-  class BlockParamListContext  < ListContext
-    def initialize(linenum)
-      super('|','|',linenum)
-    end
-  end
+#  class BlockParamListContext  < ListContext
+#    def initialize(linenum)
+#      super('|','|',linenum)
+#    end
+#  end
 
   class ParamListContext < ListContext
     def initialize(linenum)
       super('(', ')',linenum)
     end
+    def lhs; false end
+  end
+  
+  class ImplicitLhsContext < NestedContext
+    def initialize(linenum)
+      @linenum=linenum
+    end
+    def lhs; true end
+    def starter; nil end
+    def ender; '=' end
+  end
+  
+  class BlockParamListLhsContext < ImplicitLhsContext    
+    def starter; '|' end
+    def ender; '|' end
   end
   
   class ImplicitContext < ListContext
@@ -55,19 +73,26 @@ module NestedContexts
 
   class ParamListContextNoParen < ImplicitContext
     def initialize(linenum)
-      dflt_initialize(nil,nil,linenum)
+      super(nil,nil,linenum)
     end
+    def lhs; false end
   end
 
-  class KwParamListContext < ImplicitContext
+  class WhenParamListContext < ImplicitContext
     def initialize(starter,linenum)
-      dflt_initialize(starter,nil,linenum)
+      super(starter,nil,linenum)
+    end
+  end
+
+  class AssignmentContext < NestedContext
+    def initialize(linenum)
+      super("assignment context", "=",linenum)
     end
   end
 
   class AssignmentRhsContext < ImplicitContext
     def initialize(linenum)
-      dflt_initialize(nil,nil,linenum)
+      super(nil,nil,linenum)
     end
   end
 
@@ -76,8 +101,8 @@ module NestedContexts
       super(starter,'end',linenum)
     end
     
-    def see stack,msg
-      msg==:rescue ? stack.push_rescue_sm : super
+    def see lxr,msg
+      msg==:rescue ? lxr.parsestack.push_rescue_sm : super
     end
   end
 
@@ -115,7 +140,8 @@ module NestedContexts
       @state=:rescue 
     end
     
-    def see(stack,msg)
+    def see(lxr,msg)
+      stack=lxr.parsestack
       case msg
       when :rescue: 
         WantsEndContext===stack.last or 
@@ -125,7 +151,7 @@ module NestedContexts
       when :arrow: #local var defined in this state
       when :then,:semi,:colon:
         msg=:then
-        RescueSMContext===stack.pop or raise 'syntax error: then not expected at this time'
+        self.equal? stack.pop or raise 'syntax error: then not expected at this time'
                   #pop self off owning context stack
       else super
       end
@@ -149,16 +175,19 @@ module NestedContexts
       @state=:for
     end
     
-    def see(stack,msg)
+    def see(lxr,msg)
+      stack=lxr.parsestack
+      assert msg!=:for
       case msg
       when :for: WantsEndContext===stack.last or raise 'syntax error: for not expected at this time'
                  #local var defined in this state
-      when :in:  ForSMContext===stack.pop or raise 'syntax error: in not expected at this time'
+                 #never actually used?
+      when :in:  self.equal? stack.pop or raise 'syntax error: in not expected at this time'
                  stack.push ExpectDoOrNlContext.new("for",/(do|;|:|\n)/,@linenum) 
                  #pop self off owning context stack and push ExpectDoOrNlContext
       else super
       end
-      LEGAL_SUCCESSORS[@state] == msg or raise 'for syntax error: #{msg} unexpected in #@state'
+      LEGAL_SUCCESSORS[@state] == msg or raise "for syntax error: #{msg} unexpected in #@state"
       @state=msg
     end    
   end
@@ -171,4 +200,5 @@ module NestedContexts
       dflt_initialize('?',':',linenum)
     end
   end
-end
+end
+end

data/{rubycode.rb → lib/rubylexer/rubycode.rb}

@@ -19,8 +19,10 @@
 
 
 
-require "token.rb"
-require "tokenprinter.rb"
+#require "token.rb"
+#require "tokenprinter.rb"
+
+class RubyLexer
 
 class RubyCode < Token
    def initialize(tokens,filename,linenum)
@@ -40,5 +42,5 @@ class RubyCode < Token
       return result.to_s
    end
 end
-
+end
 

data/{rulexer.rb → lib/rubylexer/rulexer.rb}

@@ -20,14 +20,27 @@
 
 
 require "assert"
-require "charhandler"
+#require "charhandler"
 #require "term"
-require "rubycode"
-require "io.each_til_charset"
-
+#require "rubycode"
+#require "io.each_til_charset"
+#begin
+require 'rubygems'
+#rescue Exception:
+#end
+#require 'sequence'
+require 'sequence/indexed'
+require 'sequence/file'
+#-----------------------------------
+assert !defined? ::RubyLexer
+$RuLexer=Class.new{}
+class RubyLexer < $RuLexer
+  RuLexer=$RuLexer
+end
+$RuLexer=nil
 #------------------------------------
-class RuLexer
-
+class RubyLexer
+  class RuLexer
    WHSP=" \t\r\v\f"
    WHSPLF=WHSP+"\n"
    #maybe \r should be in WHSPLF instead
@@ -41,25 +54,28 @@ class RuLexer
    #-----------------------------------
    def initialize(filename, file, line)
       @filename=filename
-      String===file && file=IOext::FakeFile.new(file)
-      file.binmode
-      @file=file
+
+#      String===file && file=IOext::FakeFile.new(file)
+      file.binmode if File===file
+      @original_file=file
+      @file=file.to_sequence
       @linenum=line
       @toptable=nil   #descendants must fill this out
-      @moretokens=[ FileAndLineToken.new(@filename, @linenum, @file.pos) ]
+      @moretokens=[ RubyLexer::FileAndLineToken.new(@filename, @linenum, input_position) ]
       @last_operative_token=nil
+      @endsets={}
    end
 
    #-----------------------------------
    def endoffile_detected s=''
-     EoiToken.new(s,@file,@file.pos-s.size)
+     EoiToken.new(s,@original_file, input_position-s.size)
    end
 
    #-----------------------------------
    def get1token
       @moretokens.empty? or return @moretokens.shift
 
-      if @file.eof?
+      if eof?
          #@moretokens<<nil
          return endoffile_detected()
       end
@@ -73,6 +89,14 @@ class RuLexer
      return true
    end
 
+   #-----------------------------------
+   def each
+     begin yield tok = get1token
+     end until tok.is_a? EoiToken
+   end
+
+   include Enumerable
+
 private
    #-----------------------------------
    def lexerror_errortoken(tok,str,file=@filename,line=@linenum)
@@ -99,16 +123,16 @@ private
    def regex(ch=nil)
       result=RenderExactlyStringToken.new('/').
          append_token double_quote("/")
-
+      result.line=@linenum
       return result
    end
 
    #-----------------------------------
-   def single_char_token(str)   return @file.getc.chr   end
+   def single_char_token(str)  getchar   end
 
    #-----------------------------------
    def illegal_char(ch)
-     pos=@file.pos
+     pos= input_position
      LEGALCHARS===ch and return( lexerror WsToken.new(getchar,pos), "legal (?!) bad char (code: #{ch[0]})" )
      lexerror WsToken.new(til_charset(LEGALCHARS),pos), "bad char (code: #{ch[0]})"   
    end
@@ -116,7 +140,7 @@ private
    #-----------------------------------
    def fancy_quote (ch)
       assert ch=='%'
-      oldpos=@file.pos
+      oldpos= input_position
       eat_next_if(ch) or raise "fancy_quote, no "+ch
 
       ch=getchar
@@ -133,7 +157,9 @@ private
          when /^[a-z0-9]$/oi
             error= "unrecognized %string type: "+ch; '"'
          when ''
-            return lexerror( StringToken.new('', oldpos), "unexpected eof in %string")
+            result= lexerror( StringToken.new('', oldpos), "unexpected eof in %string")
+            result.line=@linenum
+            return result
          else back1char; '"' #no letter means string too
       end
 
@@ -144,8 +170,10 @@ private
 
       result=double_quote(beg, type, (PAIRS[beg] or beg))
       case ch
-      when /^[Wwr]$/: result=RenderExactlyStringToken.new(type).append_token(result)
-      when 's': result=SymbolToken.new(result.to_s)
+      when /^[Wwr]$/; 
+        result=RenderExactlyStringToken.new(type).append_token(result)
+        result.line=@linenum
+      when 's'; result=SymbolToken.new(result.to_s)
       end
       result.offset=oldpos
       return lexerror(result,error)
@@ -159,11 +187,15 @@ private
 
    #-----------------------------------
    def all_quote(nester, type, delimiter, bs_handler=nil)
+      endset="\r\n\\\\"
+
       #string must start with nester
       if nester==INET_NL_REX
         readnl
       else
-        eat_next_if(nester)
+        endset<< "\\"+nester
+        endset<< "\\"+delimiter if nester!=delimiter
+        eat_next_if(nester[0])
       end or return nil
 
       bs_handler ||= case type
@@ -177,9 +209,12 @@ private
       str=StringToken.new type
       old_linenum=@linenum
       nestlevel=1
-      maybe_crunch= "'["[type] ? nil : "#"
-      @file.each_byte {|b|
-         b=b.chr
+      endset<<maybe_crunch="#" unless "'["[type]
+      endset=
+        @endsets[endset] ||= /[#{endset}]/
+      loop{ 
+         str.append(til_charset( endset ))
+         b=getchar
          if /^[\r\n]$/===b
            back1char
            b=readnl
@@ -189,8 +224,9 @@ private
                if (nestlevel-=1)==0
                   str.modifiers=til_charset(/[^eioumnsx]/) if '/'==type
                   #emit eol marker later if line has changed
+                  str.line=@linenum
                   @linenum != old_linenum and @moretokens <<
-                     FileAndLineToken.new(@filename,@linenum,@file.pos)
+                     FileAndLineToken.new(@filename,@linenum, input_position)
                   return str
                end
             when nester
@@ -198,17 +234,25 @@ private
                assert(nester!=delimiter)
                nestlevel+=1
             when "\\"
+            begin
                b= send(bs_handler,'\\',nester,delimiter)
+            rescue e
+               lexerror str, e.message
+            end
             when nil then raise "nil char from each_byte?" #never happens
             when maybe_crunch
                nc=nextchar.chr
                nc[/^[{@$]$/] and b=ruby_code(nc)
+            when ""  #eof
+               lexerror str, "unterminated #{delimiter}-string at eof"
+               break               
          end
          str.append b
       }
 
-      assert @file.eof?
-      lexerror str,"unterminated #{delimiter}-string"
+      assert eof?
+      str.line=@linenum
+      str
    end
 
    #-----------------------------------
@@ -216,7 +260,7 @@ private
    ESCAPESEQS="\a\b\e\f\n\r\s\t\v"
    def dquote_esc_seq(ch,nester=nil,delimiter=nil)
       assert ch == '\\'
-      #see ruby book, p 205 for documentation of escape sequences
+      #see pickaxe (1st ed), p 205 for documentation of escape sequences
       return case k=getchar
          when "\n" then @linenum+=1; ""
          when "\\" then "\\"
@@ -236,11 +280,11 @@ private
          when "v"  then "\v"
 =end
          when "M"
-            eat_next_if(?-) or lexerror 'bad \\M sequence'
+            eat_next_if(?-) or raise 'bad \\M sequence'
             (getchar_maybe_escape | 0x80).chr
 
          when "C"
-            eat_next_if(?-) or lexerror 'bad \\C sequence'
+            eat_next_if(?-) or raise 'bad \\C sequence'
             (getchar_maybe_escape & 0x9F).chr
 
          when "c"
@@ -249,16 +293,16 @@ private
          when /^[0-7]$/
             str=k
             while str.length < 3
-               str << (eat_next_if(/^[0-7]$/) or break)
+               str << (eat_next_if(/[0-7]/) or break)
             end
             (str.oct&0xFF).chr
 
          when "x"
             str=''
             while str.length < 2
-               str << (eat_next_if(/^[0-9A-F]$/i) or break)
+               str << (eat_next_if(/[0-9A-F]/i) or break)
             end
-            str=='' and lexerror "bad \\x sequence"
+            str=='' and raise "bad \\x sequence"
             str.hex.chr
 
          else
@@ -281,7 +325,7 @@ private
          #when "M","C"
          #   eat_next_if(?-) or
          #         lexerror "illegal \\#{c}- esc sequence"
-         #   ch + c + '-' + (eat_next_if(/^[^\\]$/)or'')
+         #   ch + c + '-' + (eat_next_if(/[^\\]/)or'')
          #   #if this \M- or \C- sequence is continued by
          #   #another backslash, we'll just leave the
          #   #backslash on the input, to be read by the next pass
@@ -332,7 +376,11 @@ private
       klass= RubyLexer===self ? self.class : RubyLexer
       rl=klass.new(@filename,@file,@linenum)
 
-
+      #pass current local vars into new parser
+      localvars.names.each{|varname|
+        rl.localvars[varname]=true
+      }
+      rl.localvars.start_block
 
       case ch
       when '@'
@@ -363,6 +411,10 @@ private
       rl.no_more? or
         raise 'uh-oh, ruby tokens were lexed past end of ruby code'
 
+      #local vars defined in inclusion get propagated to outer parser
+      newvars=rl.localvars.__locals_lists[1..-1].map{|bag| bag.keys }.flatten
+      newvars.each{|newvar| localvars[newvar]=true }
+
       result=RubyCode.new(tokens,@filename,@linenum)
       @linenum=rl.linenum
       return result
@@ -375,10 +427,14 @@ private
    
 
    #-----------------------------------
-   BINCHARS=?0..?1
-   OCTCHARS=?0..?7
-   DECCHARS=?0..?9
-   HEXCHARS=CharSet[?0..?9, ?A..?F, ?a..?f]
+#   BINCHARS=?0..?1
+#   OCTCHARS=?0..?7
+#   DECCHARS=?0..?9
+#   HEXCHARS=CharSet[?0..?9, ?A..?F, ?a..?f]
+   BINCHARS=/[^01_]/
+   OCTCHARS=/[^0-7_]/
+   DECCHARS=/[^0-9_]/
+   HEXCHARS=/[^0-9a-f_]/i
    #0-9
    #-----------------------------------
    def number(str)
@@ -386,83 +442,85 @@ private
       return nil unless /^[0-9+\-]$/===str
 
       interp=:to_i
-      str=  (eat_next_if(/^[+\-]$/)or'')
-      str<< (eat_next_if(/^[0-9]$/)or'')
+      str=  (eat_next_if(/[+\-]/)or'')
+      str<< (eat_next_if(?0)or'')
 
-      if str[-1] == ?0 and nextchar !=?.
-         typechar=eat_next_if(/^[BOX]$/i)||'o'
+      if str[-1] == ?0 and !eof? and !nextchar.chr[/[.eE]/]
+         typechar=eat_next_if(/[BOXD]/i)||'o'
          str << typechar
          interp=:oct
-         allowed=case typechar
-           when 'b','B': BINCHARS
-           when 'x','X': HEXCHARS
-           when 'o','O': OCTCHARS
+         unallowed=case typechar
+           when 'b','B'; BINCHARS
+           when 'x','X'; HEXCHARS
+           when 'o','O'; OCTCHARS
+           when 'd','D'; interp=:to_i; DECCHARS
            else raise  :impossible
          end
       else
          interp=:to_i
-         allowed =DECCHARS
+         unallowed =DECCHARS
       end
 
-      addl_dig_seqs= ((!typechar)? 2 : 0)      #den 210
+      addl_dig_seqs= (typechar)? 0 : 2      #den 210
       error=nil
-      @file.each_byte { |b|
-         if allowed === b or ?_ == b
-            str << b
-         else
+      
+#      @file.each_byte { |b|
+#         if unallowed === b or ?_ == b
+#            str << b
+#         else
+       str<<til_charset(unallowed)
+       b=getc
             #digits must follow and precede . and e
-            if ?.==b and addl_dig_seqs==2 and allowed===nextchar
-               addl_dig_seqs=1
+            if ?.==b and addl_dig_seqs==2 and !(unallowed===nextchar.chr)
+               #addl_dig_seqs=1
                str << b
+               str<<til_charset(unallowed)
+               b=getc
+               interp=:to_s
+            end
             #digits must follow and precede . and e
-            elsif (?e==b or ?E==b) and addl_dig_seqs>=1 and
+            if (?e==b or ?E==b) and addl_dig_seqs>=1 and
                   readahead(2)[/^[-+]?[0-9]/]
-               addl_dig_seqs=0
+               #addl_dig_seqs=0
                str << b
                str << (eat_next_if(/[+\-]/)or'')
-            else
-               back1char
-               #return(str.send(interp))
-               break
+               str<<til_charset(unallowed)
+               b=getc
+               interp=:to_s
             end
-            #OCTCHARS allowed here to permit constants like this: 01.2
-            allowed == DECCHARS or allowed == OCTCHARS or error= "floats are always decimal (currently)"
-            allowed = DECCHARS
-            interp=:to_s
-         end
-      }
+               back1char if b
+               #return(str.send(interp))
+#               break
+#            #OCTCHARS allowed here to permit constants like this: 01.2
+#            unallowed == DECCHARS or unallowed == OCTCHARS or error= "floats are always decimal (currently)"
+#            unallowed = DECCHARS
+#            interp=:to_s
+#         end
+#      }
 
       assert(str[/[0-9]/])
       lexerror NumberToken.new(str.send(interp)), error
    end
 
+if (defined? DEBUGGER__ or defined? Debugger) 
    #-----------------------------------
    def comment(str=nil)
       #assert str == '#'
-      str=eat_next_if(?#) or return nil
-      Process.kill("INT",0) if
-         readahead(10)==%/breakpoint/ and defined? DEBUGGER__
-if false
-     @file.each_byte {|b|
-         if b==?\n #leave \n's on input for newline to eat
-            back1char
-         else
-            str << b
-         end
-         return IgnoreToken.new(str) if b==?\n or b==?#
-      }
-      #eof...
+      Process.kill("INT",0) if readahead(11)==%/#breakpoint/ 
+     
+      IgnoreToken.new(til_charset(/[\r\n]/))
+   end
 else
-      str<<til_charset(/[\r\n#]/)
-      eat_next_if ?# and str<<?#
-end
-      return IgnoreToken.new(str)
+   #-----------------------------------
+   def comment(str=nil)
+      IgnoreToken.new(til_charset(/[\r\n]/))
    end
+end
 
    #-----------------------------------
    def whitespace(ch)
       assert ch[/^[#{WHSP}]$/o]
-      oldpos=@file.pos
+      oldpos= input_position
       str=til_charset(/[^#{WHSP}]/o)
       return WsToken.new(str,oldpos)
    end
@@ -471,26 +529,29 @@ end
    INET_NL_REX=/^(\r\n?|\n\r?)/
    def readnl
       #compatible with dos/mac style newlines...
+
+      eof? and return ''
+
       nl=readahead(2)[INET_NL_REX]
       nl or return nil
       assert((1..2)===nl.length)
       @linenum+=1
-      @file.read nl.length
+      read nl.length
    end
 
    #-----------------------------------
    def newline(ch)
-      offset=@file.pos
+      offset= input_position
       nl=readnl
-      @moretokens << FileAndLineToken.new( @filename, @linenum, @file.pos )
+      @moretokens << FileAndLineToken.new( @filename, @linenum, input_position )
       return NewlineToken.new( nl,offset)
    end
 
 
    #-----------------------------------
    def getchar_maybe_escape
-      @file.eof? and lexerror huh,"unterminated dq string"
-      c=@file.getc
+      eof? and raise "unterminated dq string"
+      c=getc
 
       c == ?\\ and
          (c = (dquote_esc_seq('\\')[-1] or ?\n))
@@ -498,26 +559,41 @@ end
       return c
    end
 
-  #-----------------------------------
-  def RuLexer.delegate_to(obj,*names)
-    eval names.collect {|name|
-      "define_method(:#{name}) do|*args|
-        #{obj}.#{name}(*args)
-      end
-      "
-    }.to_s
+protected
+#  delegate_to :@file, :eat_next_if,:prevchar,:nextchar,:getchar,:getc,:back1char
+  require 'forwardable'
+  extend Forwardable
+  def_delegators :@file, :readahead,:readback, :read, :eof?
+
+  def til_charset cs,len=16; @file.read_til_charset cs,len end
+  def getc; @file.read1 end
+  def getchar; @file.read 1 end
+  def back1char; @file.move( -1 )end
+  def prevchar; @file.readbehind 1 end
+  def nextchar; @file.readahead1 end
+  def eat_next_if(ch); 
+    saw=getchar
+    if Integer===ch
+      ch==saw[0]
+    else
+      ch===saw
+    end or (back1char; return)
+    return saw
   end
 
-protected
-  delegate_to :@file, :eat_next_if,:prevchar,:nextchar,:getchar,:back1char,:readahead,:readback,:til_charset
+  #-----------------------------------
+  def input_position; @file.pos end
+
+  #-----------------------------------
+  def input_position_set x; @file.pos=x end
 
   #-----------------------------------
-  def RuLexer.save_offsets_in(*funcnames)
+  def self.save_offsets_in(*funcnames)
     eval funcnames.collect{|fn| <<-endeval }.to_s
       class ::#{self}
         alias #{fn}__no_offset #{fn}   #rename old ver of fn
         def #{fn}(*args)               #create new version
-          pos=@file.pos
+          pos= input_position
           result=#{fn}__no_offset(*args)
           assert Token===result
           result.offset||=pos
@@ -527,6 +603,8 @@ protected
     endeval
   end
 
-
+  end
 
 end
+
+