rubylexer 0.6.2

data/rulexer.rb

@@ -0,0 +1,532 @@
+=begin copyright
+    rubylexer - a ruby lexer written in ruby
+    Copyright (C) 2004,2005  Caleb Clausen
+
+    This library is free software; you can redistribute it and/or
+    modify it under the terms of the GNU Lesser General Public
+    License as published by the Free Software Foundation; either
+    version 2.1 of the License, or (at your option) any later version.
+
+    This library is distributed in the hope that it will be useful,
+    but WITHOUT ANY WARRANTY; without even the implied warranty of
+    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+    Lesser General Public License for more details.
+
+    You should have received a copy of the GNU Lesser General Public
+    License along with this library; if not, write to the Free Software
+    Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
+=end
+
+
+
+require "assert"
+require "charhandler"
+#require "term"
+require "rubycode"
+require "io.each_til_charset"
+
+#------------------------------------
+class RuLexer
+
+   WHSP=" \t\r\v\f"
+   WHSPLF=WHSP+"\n"
+   #maybe \r should be in WHSPLF instead
+
+   LEGALCHARS=/[ -~#{WHSPLF}]/
+
+   PAIRS={ '{'=>'}', '['=>']', '('=>')', '<'=>'>'}
+
+   attr_reader :linenum,:last_operative_token
+
+   #-----------------------------------
+   def initialize(filename, file, line)
+      @filename=filename
+      String===file && file=IOext::FakeFile.new(file)
+      file.binmode
+      @file=file
+      @linenum=line
+      @toptable=nil   #descendants must fill this out
+      @moretokens=[ FileAndLineToken.new(@filename, @linenum, @file.pos) ]
+      @last_operative_token=nil
+   end
+
+   #-----------------------------------
+   def endoffile_detected s=''
+     EoiToken.new(s,@file,@file.pos-s.size)
+   end
+
+   #-----------------------------------
+   def get1token
+      @moretokens.empty? or return @moretokens.shift
+
+      if @file.eof?
+         #@moretokens<<nil
+         return endoffile_detected()
+      end
+
+      @toptable.go( nextchar )
+   end
+
+   #-----------------------------------
+   def no_more?
+     @moretokens.each{|t| FileAndLineToken===t or return false }
+     return true
+   end
+
+private
+   #-----------------------------------
+   def lexerror_errortoken(tok,str,file=@filename,line=@linenum)
+      str or return tok
+      tok.extend(ErrorToken).error=str
+      tok
+   end
+
+   #-----------------------------------
+   def lexerror_exception(tok,str,file=@filename,line=@linenum)
+      str or return tok
+      raise [file,line,'  '+str].join(':')
+   end
+
+   #-----------------------------------
+   alias lexerror lexerror_errortoken
+
+   #-----------------------------------
+   def handler_loop(handler)
+      @file.each_byte {|b|   handler.go(b) or break   }
+   end
+
+   #-----------------------------------
+   def regex(ch=nil)
+      result=RenderExactlyStringToken.new('/').
+         append_token double_quote("/")
+
+      return result
+   end
+
+   #-----------------------------------
+   def single_char_token(str)   return @file.getc.chr   end
+
+   #-----------------------------------
+   def illegal_char(ch)
+     pos=@file.pos
+     LEGALCHARS===ch and return( lexerror WsToken.new(getchar,pos), "legal (?!) bad char (code: #{ch[0]})" )
+     lexerror WsToken.new(til_charset(LEGALCHARS),pos), "bad char (code: #{ch[0]})"   
+   end
+
+   #-----------------------------------
+   def fancy_quote (ch)
+      assert ch=='%'
+      oldpos=@file.pos
+      eat_next_if(ch) or raise "fancy_quote, no "+ch
+
+      ch=getchar
+      #ch.tr!('qwQWrx','"["{/`')
+      type=case ch
+         when 'q' then "'"
+         when 'w' then "[" #word array
+         when 'Q' then '"' #regular string
+         when 'W' then '{' #dquotish word array
+         when 'r' then '/' #regex
+         when 'x' then '`' #exec it
+         when 's' then '"' #symbol
+         #other letters, nums are illegal here
+         when /^[a-z0-9]$/oi
+            error= "unrecognized %string type: "+ch; '"'
+         when ''
+            return lexerror( StringToken.new('', oldpos), "unexpected eof in %string")
+         else back1char; '"' #no letter means string too
+      end
+
+      beg=nextchar.chr
+      if /^[\r\n]$/===beg  then 
+           beg=INET_NL_REX
+      end
+
+      result=double_quote(beg, type, (PAIRS[beg] or beg))
+      case ch
+      when /^[Wwr]$/: result=RenderExactlyStringToken.new(type).append_token(result)
+      when 's': result=SymbolToken.new(result.to_s)
+      end
+      result.offset=oldpos
+      return lexerror(result,error)
+   end
+
+   #-----------------------------------
+   #this method is now misnamed, since it handles single quotes as well
+   def double_quote(nester, type=nester, delimiter=nester)
+      all_quote(nester,type,delimiter)
+   end
+
+   #-----------------------------------
+   def all_quote(nester, type, delimiter, bs_handler=nil)
+      #string must start with nester
+      if nester==INET_NL_REX
+        readnl
+      else
+        eat_next_if(nester)
+      end or return nil
+
+      bs_handler ||= case type
+        when '/','{' then :regex_esc_seq
+        when '"','`',':' then :dquote_esc_seq
+        when "'"     then :squote_esc_seq
+        when "["     then :wquote_esc_seq
+        else raise "unknown quote type: #{type}"
+      end
+
+      str=StringToken.new type
+      old_linenum=@linenum
+      nestlevel=1
+      maybe_crunch= "'["[type] ? nil : "#"
+      @file.each_byte {|b|
+         b=b.chr
+         if /^[\r\n]$/===b
+           back1char
+           b=readnl
+         end
+         case b
+            when delimiter
+               if (nestlevel-=1)==0
+                  str.modifiers=til_charset(/[^eioumnsx]/) if '/'==type
+                  #emit eol marker later if line has changed
+                  @linenum != old_linenum and @moretokens <<
+                     FileAndLineToken.new(@filename,@linenum,@file.pos)
+                  return str
+               end
+            when nester
+               #this branch ignored if nester==delimiter
+               assert(nester!=delimiter)
+               nestlevel+=1
+            when "\\"
+               b= send(bs_handler,'\\',nester,delimiter)
+            when nil then raise "nil char from each_byte?" #never happens
+            when maybe_crunch
+               nc=nextchar.chr
+               nc[/^[{@$]$/] and b=ruby_code(nc)
+         end
+         str.append b
+      }
+
+      assert @file.eof?
+      lexerror str,"unterminated #{delimiter}-string"
+   end
+
+   #-----------------------------------
+   ESCAPECHRS="abefnrstv"
+   ESCAPESEQS="\a\b\e\f\n\r\s\t\v"
+   def dquote_esc_seq(ch,nester=nil,delimiter=nil)
+      assert ch == '\\'
+      #see ruby book, p 205 for documentation of escape sequences
+      return case k=getchar
+         when "\n" then @linenum+=1; ""
+         when "\\" then "\\"
+         when '"' then '"'
+         when '#' then '#'
+         when /^[#{ESCAPECHRS}]$/o
+            k.tr(ESCAPECHRS,ESCAPESEQS)
+=begin not needed anymore
+         when "a"  then "\a"
+         when "b"  then "\b"
+         when "e"  then "\e"
+         when "f"  then "\f"
+         when "n"  then "\n"
+         when "r"  then "\r"
+         when "s"  then "\ "
+         when "t"  then "\t"
+         when "v"  then "\v"
+=end
+         when "M"
+            eat_next_if(?-) or lexerror 'bad \\M sequence'
+            (getchar_maybe_escape | 0x80).chr
+
+         when "C"
+            eat_next_if(?-) or lexerror 'bad \\C sequence'
+            (getchar_maybe_escape & 0x9F).chr
+
+         when "c"
+            (getchar_maybe_escape & 0x9F).chr
+
+         when /^[0-7]$/
+            str=k
+            while str.length < 3
+               str << (eat_next_if(/^[0-7]$/) or break)
+            end
+            (str.oct&0xFF).chr
+
+         when "x"
+            str=''
+            while str.length < 2
+               str << (eat_next_if(/^[0-9A-F]$/i) or break)
+            end
+            str=='' and lexerror "bad \\x sequence"
+            str.hex.chr
+
+         else
+            '\\'+k
+      end
+   end
+
+   #-----------------------------------
+   def regex_esc_seq(ch,nester,delimiter)
+      assert ch == '\\'
+      c=getchar
+      return case c
+         when "\n"
+            @linenum+=1
+            ''#ch+c
+         when nester,delimiter  ,"/"
+            c
+         #when "c"
+         #   ch + c + getchar
+         #when "M","C"
+         #   eat_next_if(?-) or
+         #         lexerror "illegal \\#{c}- esc sequence"
+         #   ch + c + '-' + (eat_next_if(/^[^\\]$/)or'')
+         #   #if this \M- or \C- sequence is continued by
+         #   #another backslash, we'll just leave the
+         #   #backslash on the input, to be read by the next pass
+         else
+            ch+c
+      end
+   end
+
+   #-----------------------------------
+   def wquote_esc_seq(ch,nester,delimiter)
+      assert(ch=='\\')
+
+      #get the escaped character
+      escchar=getchar
+      return (case escchar
+         #all \ sequences but \delimiter, \nester
+         #are passed thru unchanged; actual
+         #newlines are counted but not changed
+         when delimiter,nester
+              ''
+         when "\n"
+              @linenum+=1; "\\"
+         else '\\'
+      end+escchar)
+   end
+
+   #-----------------------------------
+   def squote_esc_seq(ch,nester,delimiter)
+      assert(ch=='\\')
+
+      #get the escaped character
+      escchar=getchar
+      return (case escchar
+         #all \ sequences but \delimiter, \nester and \\
+         #are passed thru unchanged; actual
+         #newlines are counted but not changed
+         when delimiter,nester,'\\'
+              ''
+         when "\n"
+              @linenum+=1; "\\"
+         else '\\'
+      end+escchar)
+   end
+
+   #-----------------------------------
+   def ruby_code(ch='{')
+      assert ch[/^[{(@$]$/]
+      klass= RubyLexer===self ? self.class : RubyLexer
+      rl=klass.new(@filename,@file,@linenum)
+
+
+
+      case ch
+      when '@'
+         tokens=[rl.at_identifier]
+      when '$'
+         tokens=[rl.dollar_identifier]
+      when '{','('
+         tokens=[]
+         loop {
+            tok=rl.get1token
+            EoiToken===tok and lexerror tok,"unterminated string inclusion"
+            tokens << tok
+            break if tok===PAIRS[ch] and rl.no_more? and rl.balanced_braces?
+         }
+      else
+         raise 'hell'
+      end
+
+      if @linenum != rl.linenum
+        last=tokens.pop
+        fal=FileAndLineToken.new(@filename,@linenum, last.offset)
+        tokens.push fal,last
+      end
+
+      #need to verify that rl's @moretokens, @incomplete_here_tokens are empty
+      rl.incomplete_here_tokens.empty? or 
+        here_spread_over_ruby_code rl,tokens.last
+      rl.no_more? or
+        raise 'uh-oh, ruby tokens were lexed past end of ruby code'
+
+      result=RubyCode.new(tokens,@filename,@linenum)
+      @linenum=rl.linenum
+      return result
+   end
+   
+   #-----------------------------------
+   def here_spread_over_ruby_code(rl,tok)
+     lexerror tok, 'here body outside string inclusion'
+   end
+   
+
+   #-----------------------------------
+   BINCHARS=?0..?1
+   OCTCHARS=?0..?7
+   DECCHARS=?0..?9
+   HEXCHARS=CharSet[?0..?9, ?A..?F, ?a..?f]
+   #0-9
+   #-----------------------------------
+   def number(str)
+
+      return nil unless /^[0-9+\-]$/===str
+
+      interp=:to_i
+      str=  (eat_next_if(/^[+\-]$/)or'')
+      str<< (eat_next_if(/^[0-9]$/)or'')
+
+      if str[-1] == ?0 and nextchar !=?.
+         typechar=eat_next_if(/^[BOX]$/i)||'o'
+         str << typechar
+         interp=:oct
+         allowed=case typechar
+           when 'b','B': BINCHARS
+           when 'x','X': HEXCHARS
+           when 'o','O': OCTCHARS
+           else raise  :impossible
+         end
+      else
+         interp=:to_i
+         allowed =DECCHARS
+      end
+
+      addl_dig_seqs= ((!typechar)? 2 : 0)      #den 210
+      error=nil
+      @file.each_byte { |b|
+         if allowed === b or ?_ == b
+            str << b
+         else
+            #digits must follow and precede . and e
+            if ?.==b and addl_dig_seqs==2 and allowed===nextchar
+               addl_dig_seqs=1
+               str << b
+            #digits must follow and precede . and e
+            elsif (?e==b or ?E==b) and addl_dig_seqs>=1 and
+                  readahead(2)[/^[-+]?[0-9]/]
+               addl_dig_seqs=0
+               str << b
+               str << (eat_next_if(/[+\-]/)or'')
+            else
+               back1char
+               #return(str.send(interp))
+               break
+            end
+            #OCTCHARS allowed here to permit constants like this: 01.2
+            allowed == DECCHARS or allowed == OCTCHARS or error= "floats are always decimal (currently)"
+            allowed = DECCHARS
+            interp=:to_s
+         end
+      }
+
+      assert(str[/[0-9]/])
+      lexerror NumberToken.new(str.send(interp)), error
+   end
+
+   #-----------------------------------
+   def comment(str=nil)
+      #assert str == '#'
+      str=eat_next_if(?#) or return nil
+      Process.kill("INT",0) if
+         readahead(10)==%/breakpoint/ and defined? DEBUGGER__
+if false
+     @file.each_byte {|b|
+         if b==?\n #leave \n's on input for newline to eat
+            back1char
+         else
+            str << b
+         end
+         return IgnoreToken.new(str) if b==?\n or b==?#
+      }
+      #eof...
+else
+      str<<til_charset(/[\r\n#]/)
+      eat_next_if ?# and str<<?#
+end
+      return IgnoreToken.new(str)
+   end
+
+   #-----------------------------------
+   def whitespace(ch)
+      assert ch[/^[#{WHSP}]$/o]
+      oldpos=@file.pos
+      str=til_charset(/[^#{WHSP}]/o)
+      return WsToken.new(str,oldpos)
+   end
+
+   #-----------------------------------
+   INET_NL_REX=/^(\r\n?|\n\r?)/
+   def readnl
+      #compatible with dos/mac style newlines...
+      nl=readahead(2)[INET_NL_REX]
+      nl or return nil
+      assert((1..2)===nl.length)
+      @linenum+=1
+      @file.read nl.length
+   end
+
+   #-----------------------------------
+   def newline(ch)
+      offset=@file.pos
+      nl=readnl
+      @moretokens << FileAndLineToken.new( @filename, @linenum, @file.pos )
+      return NewlineToken.new( nl,offset)
+   end
+
+
+   #-----------------------------------
+   def getchar_maybe_escape
+      @file.eof? and lexerror huh,"unterminated dq string"
+      c=@file.getc
+
+      c == ?\\ and
+         (c = (dquote_esc_seq('\\')[-1] or ?\n))
+
+      return c
+   end
+
+  #-----------------------------------
+  def RuLexer.delegate_to(obj,*names)
+    eval names.collect {|name|
+      "define_method(:#{name}) do|*args|
+        #{obj}.#{name}(*args)
+      end
+      "
+    }.to_s
+  end
+
+protected
+  delegate_to :@file, :eat_next_if,:prevchar,:nextchar,:getchar,:back1char,:readahead,:readback,:til_charset
+
+  #-----------------------------------
+  def RuLexer.save_offsets_in(*funcnames)
+    eval funcnames.collect{|fn| <<-endeval }.to_s
+      class ::#{self}
+        alias #{fn}__no_offset #{fn}   #rename old ver of fn
+        def #{fn}(*args)               #create new version
+          pos=@file.pos
+          result=#{fn}__no_offset(*args)
+          assert Token===result
+          result.offset||=pos
+          return result
+        end
+      end
+    endeval
+  end
+
+
+
+end