rubysl-irb 1.0.0

data/lib/irb/ruby-token.rb

@@ -0,0 +1,273 @@
+#
+#   irb/ruby-token.rb - ruby tokens 
+#   	$Release Version: 0.9.5$
+#   	$Revision: 11708 $
+#   	$Date: 2007-02-12 15:01:19 -0800 (Mon, 12 Feb 2007) $
+#   	by Keiju ISHITSUKA(keiju@ruby-lang.org)
+#
+# --
+#
+#   
+#
+module RubyToken
+  EXPR_BEG = :EXPR_BEG
+  EXPR_MID = :EXPR_MID
+  EXPR_END = :EXPR_END
+  EXPR_ARG = :EXPR_ARG
+  EXPR_FNAME = :EXPR_FNAME
+  EXPR_DOT = :EXPR_DOT
+  EXPR_CLASS = :EXPR_CLASS
+
+  # for ruby 1.4X
+  if !defined?(Symbol)
+    Symbol = Integer
+  end
+  
+  class Token
+    def initialize(seek, line_no, char_no)
+      @seek = seek
+      @line_no = line_no
+      @char_no = char_no
+    end
+    attr :seek
+    attr :line_no
+    attr :char_no
+  end
+
+  class TkNode < Token
+    def initialize(seek, line_no, char_no)
+      super
+    end
+    attr :node
+  end
+
+  class TkId < Token
+    def initialize(seek, line_no, char_no, name)
+      super(seek, line_no, char_no)
+      @name = name
+    end
+    attr :name
+  end
+
+  class TkVal < Token
+    def initialize(seek, line_no, char_no, value = nil)
+      super(seek, line_no, char_no)
+      @value = value
+    end
+    attr :value
+  end
+
+  class TkOp < Token
+    attr :name, true
+  end
+
+  class TkOPASGN < TkOp
+    def initialize(seek, line_no, char_no, op)
+      super(seek, line_no, char_no)
+      op = TkReading2Token[op][0] unless op.kind_of?(Symbol)
+      @op = op
+    end
+    attr :op
+  end
+
+  class TkUnknownChar < Token
+    def initialize(seek, line_no, char_no, id)
+      super(seek, line_no, char_no)
+      @name = name
+    end
+    attr :name
+  end
+
+  class TkError < Token
+  end
+
+  def Token(token, value = nil)
+    case token
+    when String
+      if (tk = TkReading2Token[token]).nil?
+	IRB.fail TkReading2TokenNoKey, token
+      end
+      tk = Token(tk[0], value) 
+      if tk.kind_of?(TkOp)
+	tk.name = token
+      end
+      return tk
+    when Symbol
+      if (tk = TkSymbol2Token[token]).nil?
+	IRB.fail TkSymbol2TokenNoKey, token
+      end
+      return Token(tk[0], value) 
+    else 
+      if (token.ancestors & [TkId, TkVal, TkOPASGN, TkUnknownChar]).empty?
+	token.new(@prev_seek, @prev_line_no, @prev_char_no)
+      else
+	token.new(@prev_seek, @prev_line_no, @prev_char_no, value)
+      end
+    end
+  end
+
+  TokenDefinitions = [
+    [:TkCLASS,      TkId,  "class",  EXPR_CLASS],
+    [:TkMODULE,     TkId,  "module", EXPR_BEG],
+    [:TkDEF,	    TkId,  "def",    EXPR_FNAME],
+    [:TkUNDEF,      TkId,  "undef",  EXPR_FNAME],
+    [:TkBEGIN,      TkId,  "begin",  EXPR_BEG],
+    [:TkRESCUE,     TkId,  "rescue", EXPR_MID],
+    [:TkENSURE,     TkId,  "ensure", EXPR_BEG],
+    [:TkEND,	    TkId,  "end",    EXPR_END],
+    [:TkIF,         TkId,  "if",     EXPR_BEG, :TkIF_MOD],
+    [:TkUNLESS,     TkId,  "unless", EXPR_BEG, :TkUNLESS_MOD],
+    [:TkTHEN,	    TkId,  "then",   EXPR_BEG],
+    [:TkELSIF,      TkId,  "elsif",  EXPR_BEG],
+    [:TkELSE,	    TkId,  "else",   EXPR_BEG],
+    [:TkCASE,	    TkId,  "case",   EXPR_BEG],
+    [:TkWHEN,	    TkId,  "when",   EXPR_BEG],
+    [:TkWHILE,      TkId,  "while",  EXPR_BEG, :TkWHILE_MOD],
+    [:TkUNTIL,      TkId,  "until",  EXPR_BEG, :TkUNTIL_MOD],
+    [:TkFOR,	    TkId,  "for",    EXPR_BEG],
+    [:TkBREAK,      TkId,  "break",  EXPR_END],
+    [:TkNEXT,	    TkId,  "next",   EXPR_END],
+    [:TkREDO,	    TkId,  "redo",   EXPR_END],
+    [:TkRETRY,      TkId,  "retry",  EXPR_END],
+    [:TkIN,	    TkId,  "in",     EXPR_BEG],
+    [:TkDO,	    TkId,  "do",     EXPR_BEG],
+    [:TkRETURN,     TkId,  "return", EXPR_MID],
+    [:TkYIELD,      TkId,  "yield",  EXPR_END],
+    [:TkSUPER,      TkId,  "super",  EXPR_END],
+    [:TkSELF,	    TkId,  "self",   EXPR_END],
+    [:TkNIL, 	    TkId,  "nil",    EXPR_END],
+    [:TkTRUE,	    TkId,  "true",   EXPR_END],
+    [:TkFALSE,      TkId,  "false",  EXPR_END],
+    [:TkAND,	    TkId,  "and",    EXPR_BEG],
+    [:TkOR, 	    TkId,  "or",     EXPR_BEG],
+    [:TkNOT,	    TkId,  "not",    EXPR_BEG],
+    [:TkIF_MOD,     TkId],
+    [:TkUNLESS_MOD, TkId],
+    [:TkWHILE_MOD,  TkId],
+    [:TkUNTIL_MOD,  TkId],
+    [:TkALIAS,      TkId,  "alias",    EXPR_FNAME],
+    [:TkDEFINED,    TkId,  "defined?", EXPR_END],
+    [:TklBEGIN,     TkId,  "BEGIN",    EXPR_END],
+    [:TklEND,	    TkId,  "END",      EXPR_END],
+    [:Tk__LINE__,   TkId,  "__LINE__", EXPR_END],
+    [:Tk__FILE__,   TkId,  "__FILE__", EXPR_END],
+
+    [:TkIDENTIFIER, TkId],
+    [:TkFID,	    TkId],
+    [:TkGVAR,	    TkId],
+    [:TkCVAR,	    TkId],
+    [:TkIVAR,	    TkId],
+    [:TkCONSTANT,   TkId],
+
+    [:TkINTEGER,    TkVal],
+    [:TkFLOAT,      TkVal],
+    [:TkSTRING,     TkVal],
+    [:TkXSTRING,    TkVal],
+    [:TkREGEXP,     TkVal],
+    [:TkSYMBOL,     TkVal],
+
+    [:TkDSTRING,    TkNode],
+    [:TkDXSTRING,   TkNode],
+    [:TkDREGEXP,    TkNode],
+    [:TkNTH_REF,    TkNode],
+    [:TkBACK_REF,   TkNode],
+
+    [:TkUPLUS,      TkOp,   "+@"],
+    [:TkUMINUS,     TkOp,   "-@"],
+    [:TkPOW,	    TkOp,   "**"],
+    [:TkCMP,	    TkOp,   "<=>"],
+    [:TkEQ,	    TkOp,   "=="],
+    [:TkEQQ,	    TkOp,   "==="],
+    [:TkNEQ,	    TkOp,   "!="],
+    [:TkGEQ,	    TkOp,   ">="],
+    [:TkLEQ,	    TkOp,   "<="],
+    [:TkANDOP,      TkOp,   "&&"],
+    [:TkOROP,	    TkOp,   "||"],
+    [:TkMATCH,      TkOp,   "=~"],
+    [:TkNMATCH,     TkOp,   "!~"],
+    [:TkDOT2,	    TkOp,   ".."],
+    [:TkDOT3,	    TkOp,   "..."],
+    [:TkAREF,	    TkOp,   "[]"],
+    [:TkASET,	    TkOp,   "[]="],
+    [:TkLSHFT,      TkOp,   "<<"],
+    [:TkRSHFT,      TkOp,   ">>"],
+    [:TkCOLON2,     TkOp],
+    [:TkCOLON3,     TkOp],
+#   [:OPASGN,	    TkOp],               # +=, -=  etc. #
+    [:TkASSOC,      TkOp,   "=>"],
+    [:TkQUESTION,   TkOp,   "?"],	 #?
+    [:TkCOLON,      TkOp,   ":"],        #:
+    
+    [:TkfLPAREN],         # func( #
+    [:TkfLBRACK],         # func[ #
+    [:TkfLBRACE],         # func{ #
+    [:TkSTAR],            # *arg
+    [:TkAMPER],           # &arg #
+    [:TkSYMBEG],          # :SYMBOL
+
+    [:TkGT,	    TkOp,   ">"],
+    [:TkLT,	    TkOp,   "<"],
+    [:TkPLUS,	    TkOp,   "+"],
+    [:TkMINUS,      TkOp,   "-"],
+    [:TkMULT,	    TkOp,   "*"],
+    [:TkDIV,	    TkOp,   "/"],
+    [:TkMOD,	    TkOp,   "%"],
+    [:TkBITOR,      TkOp,   "|"],
+    [:TkBITXOR,     TkOp,   "^"],
+    [:TkBITAND,     TkOp,   "&"],
+    [:TkBITNOT,     TkOp,   "~"],
+    [:TkNOTOP,      TkOp,   "!"],
+
+    [:TkBACKQUOTE,  TkOp,   "`"],
+
+    [:TkASSIGN,     Token,  "="],
+    [:TkDOT,	    Token,  "."],
+    [:TkLPAREN,     Token,  "("],  #(exp)
+    [:TkLBRACK,     Token,  "["],  #[arry]
+    [:TkLBRACE,     Token,  "{"],  #{hash}
+    [:TkRPAREN,     Token,  ")"],
+    [:TkRBRACK,     Token,  "]"],
+    [:TkRBRACE,     Token,  "}"],
+    [:TkCOMMA,      Token,  ","],
+    [:TkSEMICOLON,  Token,  ";"],
+
+    [:TkCOMMENT],
+    [:TkRD_COMMENT],
+    [:TkSPACE],
+    [:TkNL],
+    [:TkEND_OF_SCRIPT],
+
+    [:TkBACKSLASH,  TkUnknownChar,  "\\"],
+    [:TkAT,	    TkUnknownChar,  "@"],
+    [:TkDOLLAR,     TkUnknownChar,  "$"],
+  ]
+
+  # {reading => token_class}
+  # {reading => [token_class, *opt]}
+  TkReading2Token = {}
+  TkSymbol2Token = {}
+
+  def RubyToken.def_token(token_n, super_token = Token, reading = nil, *opts)
+    token_n = token_n.id2name if token_n.kind_of?(Symbol)
+    if RubyToken.const_defined?(token_n)
+      IRB.fail AlreadyDefinedToken, token_n
+    end
+    token_c = eval("class #{token_n} < #{super_token}; end; #{token_n}")
+    
+    if reading
+      if TkReading2Token[reading]
+	IRB.fail TkReading2TokenDuplicateError, token_n, reading
+      end
+      if opts.empty?
+	TkReading2Token[reading] = [token_c]
+      else
+	TkReading2Token[reading] = [token_c].concat(opts)
+      end
+    end
+    TkSymbol2Token[token_n.intern] = token_c
+  end
+
+  for defs in TokenDefinitions
+    def_token(*defs)
+  end
+end

data/lib/irb/slex.rb

@@ -0,0 +1,285 @@
+#
+#   irb/slex.rb - symple lex analizer
+#   	$Release Version: 0.9.5$
+#   	$Revision: 11708 $
+#   	$Date: 2007-02-12 15:01:19 -0800 (Mon, 12 Feb 2007) $
+#   	by Keiju ISHITSUKA(keiju@ruby-lang.org)
+#
+# --
+#
+#   
+#
+
+require "e2mmap"
+require "irb/notifier"
+
+module IRB
+  class SLex
+    @RCS_ID='-$Id: slex.rb 11708 2007-02-12 23:01:19Z shyouhei $-'
+
+    extend Exception2MessageMapper
+    def_exception :ErrNodeNothing, "node nothing"
+    def_exception :ErrNodeAlreadyExists, "node already exists"
+
+    DOUT = Notifier::def_notifier("SLex::")
+    D_WARN = DOUT::def_notifier(1, "Warn: ")
+    D_DEBUG = DOUT::def_notifier(2, "Debug: ")
+    D_DETAIL = DOUT::def_notifier(4, "Detail: ")
+    
+    DOUT.level = Notifier::D_NOMSG
+
+    def initialize
+      @head = Node.new("")
+    end
+    
+    def def_rule(token, preproc = nil, postproc = nil, &block)
+      D_DETAIL.pp token
+
+      postproc = block if block_given?
+      node = create(token, preproc, postproc)
+    end
+    
+    def def_rules(*tokens, &block)
+      if block_given?
+	p = block
+      end
+      for token in tokens
+	def_rule(token, nil, p)
+      end
+    end
+    
+    def preproc(token, proc)
+      node = search(token)
+      node.preproc=proc
+    end
+    
+    #$BMW%A%'%C%/(B? 
+    def postproc(token)
+      node = search(token, proc)
+      node.postproc=proc
+    end
+    
+    def search(token)
+      @head.search(token.split(//))
+    end
+
+    def create(token, preproc = nil, postproc = nil)
+      @head.create_subnode(token.split(//), preproc, postproc)
+    end
+    
+    def match(token)
+      case token
+      when Array
+      when String
+	return match(token.split(//))
+      else
+	return @head.match_io(token)
+      end
+      ret = @head.match(token)
+      D_DETAIL.exec_if{D_DEATIL.printf "match end: %s:%s\n", ret, token.inspect}
+      ret
+    end
+    
+    def inspect
+      format("<SLex: @head = %s>", @head.inspect)
+    end
+
+    #----------------------------------------------------------------------
+    #
+    #   class Node - 
+    #
+    #----------------------------------------------------------------------
+    class Node
+      # if postproc is nil, this node is an abstract node.
+      # if postproc is non-nil, this node is a real node.
+      def initialize(preproc = nil, postproc = nil)
+	@Tree = {}
+	@preproc = preproc
+	@postproc = postproc
+      end
+
+      attr_accessor :preproc
+      attr_accessor :postproc
+      
+      def search(chrs, opt = nil)
+	return self if chrs.empty?
+	ch = chrs.shift
+	if node = @Tree[ch]
+	  node.search(chrs, opt)
+	else
+	  if opt
+	    chrs.unshift ch
+	    self.create_subnode(chrs)
+	  else
+	    SLex.fail ErrNodeNothing
+	  end
+	end
+      end
+      
+      def create_subnode(chrs, preproc = nil, postproc = nil)
+	if chrs.empty?
+	  if @postproc
+	    D_DETAIL.pp node
+	    SLex.fail ErrNodeAlreadyExists
+	  else
+	    D_DEBUG.puts "change abstract node to real node."
+	    @preproc = preproc
+	    @postproc = postproc
+	  end
+	  return self
+	end
+	
+	ch = chrs.shift
+	if node = @Tree[ch]
+	  if chrs.empty?
+	    if node.postproc
+	      DebugLogger.pp node
+	      DebugLogger.pp self
+	      DebugLogger.pp ch
+	      DebugLogger.pp chrs
+	      SLex.fail ErrNodeAlreadyExists
+	    else
+	      D_WARN.puts "change abstract node to real node"
+	      node.preproc = preproc
+	      node.postproc = postproc
+	    end
+	  else
+	    node.create_subnode(chrs, preproc, postproc)
+	  end
+	else
+	  if chrs.empty?
+	    node = Node.new(preproc, postproc)
+	  else
+	    node = Node.new
+	    node.create_subnode(chrs, preproc, postproc)
+	  end
+	  @Tree[ch] = node
+	end
+	node
+      end
+
+      #
+      # chrs: String
+      #       character array
+      #       io must have getc()/ungetc(); and ungetc() must be
+      #       able to be called arbitrary number of times. 
+      #
+      def match(chrs, op = "")
+	D_DETAIL.print "match>: ", chrs, "op:", op, "\n"
+	if chrs.empty?
+	  if @preproc.nil? || @preproc.call(op, chrs)
+	    DOUT.printf(D_DETAIL, "op1: %s\n", op)
+	    @postproc.call(op, chrs)
+	  else
+	    nil
+	  end
+	else
+	  ch = chrs.shift
+	  if node = @Tree[ch]
+	    if ret = node.match(chrs, op+ch)
+	      return ret
+	    else
+	      chrs.unshift ch
+	      if @postproc and @preproc.nil? || @preproc.call(op, chrs)
+		DOUT.printf(D_DETAIL, "op2: %s\n", op.inspect)
+		ret = @postproc.call(op, chrs)
+		return ret
+	      else
+		return nil
+	      end
+	    end
+	  else
+	    chrs.unshift ch
+	    if @postproc and @preproc.nil? || @preproc.call(op, chrs)
+	      DOUT.printf(D_DETAIL, "op3: %s\n", op)
+	      @postproc.call(op, chrs)
+	      return ""
+	    else
+	      return nil
+	    end
+	  end
+	end
+      end
+
+      def match_io(io, op = "")
+	if op == ""
+	  ch = io.getc
+	  if ch == nil
+	    return nil
+	  end
+	else
+	  ch = io.getc_of_rests
+	end
+	if ch.nil?
+	  if @preproc.nil? || @preproc.call(op, io)
+	    D_DETAIL.printf("op1: %s\n", op)
+	    @postproc.call(op, io)
+	  else
+	    nil
+	  end
+	else
+	  if node = @Tree[ch]
+	    if ret = node.match_io(io, op+ch)
+	      ret
+	    else
+	      io.ungetc ch
+	      if @postproc and @preproc.nil? || @preproc.call(op, io)
+		DOUT.exec_if{D_DETAIL.printf "op2: %s\n", op.inspect}
+		@postproc.call(op, io)
+	      else
+		nil
+	      end
+	    end
+	  else
+	    io.ungetc ch
+	    if @postproc and @preproc.nil? || @preproc.call(op, io)
+	      D_DETAIL.printf("op3: %s\n", op)
+	      @postproc.call(op, io)
+	    else
+	      nil
+	    end
+	  end
+	end
+      end
+    end
+  end
+end
+
+SLex=IRB::SLex
+
+if $0 == __FILE__
+  #    Tracer.on
+  case $1
+  when "1"
+    tr = SLex.new
+    print "0: ", tr.inspect, "\n"
+    tr.def_rule("=") {print "=\n"}
+    print "1: ", tr.inspect, "\n"
+    tr.def_rule("==") {print "==\n"}
+    print "2: ", tr.inspect, "\n"
+    
+    print "case 1:\n"
+    print tr.match("="), "\n"
+    print "case 2:\n"
+    print tr.match("=="), "\n"
+    print "case 3:\n"
+    print tr.match("=>"), "\n"
+    
+  when "2"
+    tr = SLex.new
+    print "0: ", tr.inspect, "\n"
+    tr.def_rule("=") {print "=\n"}
+    print "1: ", tr.inspect, "\n"
+    tr.def_rule("==", proc{false}) {print "==\n"}
+    print "2: ", tr.inspect, "\n"
+    
+    print "case 1:\n"
+    print tr.match("="), "\n"
+    print "case 2:\n"
+    print tr.match("=="), "\n"
+    print "case 3:\n"
+    print tr.match("=>"), "\n"
+  end
+  exit
+end
+