rubysl-irb 1.0.2 → 2.0.3

data/lib/irb/magic-file.rb

@@ -0,0 +1,37 @@
+module IRB
+  class << (MagicFile = Object.new)
+    # see parser_magic_comment in parse.y
+    ENCODING_SPEC_RE = %r"coding\s*[=:]\s*([[:alnum:]\-_]+)"
+
+    def open(path)
+      io = File.open(path, 'rb')
+      line = io.gets
+      line = io.gets if line[0,2] == "#!"
+      encoding = detect_encoding(line)
+      internal_encoding = encoding
+      encoding ||= default_src_encoding
+      io.rewind
+      io.set_encoding(encoding, internal_encoding)
+
+      if block_given?
+        begin
+          return (yield io)
+        ensure
+          io.close
+        end
+      else
+        return io
+      end
+    end
+
+    private
+    def detect_encoding(line)
+      return unless line[0] == ?#
+      line = line[1..-1]
+      line = $1 if line[/-\*-\s*(.*?)\s*-*-$/]
+      return nil unless ENCODING_SPEC_RE =~ line
+      encoding = $1
+      return encoding.sub(/-(?:mac|dos|unix)/i, '')
+    end
+  end
+end

data/lib/irb/notifier.rb

@@ -1,79 +1,120 @@
 #
-#   notifier.rb - optput methods used by irb 
-#   	$Release Version: 0.9.5$
-#   	$Revision: 11708 $
-#   	$Date: 2007-02-12 15:01:19 -0800 (Mon, 12 Feb 2007) $
+#   notifier.rb - output methods used by irb
+#   	$Release Version: 0.9.6$
+#   	$Revision$
 #   	by Keiju ISHITSUKA(keiju@ruby-lang.org)
 #
 # --
 #
-#   
+#
 #
 
 require "e2mmap"
 require "irb/output-method"
 
 module IRB
+  # An output formatter used internally by the lexer.
   module Notifier
     extend Exception2MessageMapper
-    def_exception :ErrUndefinedNotifier, 
+    def_exception :ErrUndefinedNotifier,
       "undefined notifier level: %d is specified"
-    def_exception :ErrUnrecognizedLevel, 
+    def_exception :ErrUnrecognizedLevel,
       "unrecognized notifier level: %s is specified"
 
+    # Define a new Notifier output source, returning a new CompositeNotifier
+    # with the given +prefix+ and +output_method+.
+    #
+    # The optional +prefix+ will be appended to all objects being inspected
+    # during output, using the given +output_method+ as the output source. If
+    # no +output_method+ is given, StdioOuputMethod will be used, and all
+    # expressions will be sent directly to STDOUT without any additional
+    # formatting.
     def def_notifier(prefix = "", output_method = StdioOutputMethod.new)
       CompositeNotifier.new(prefix, output_method)
     end
     module_function :def_notifier
-  
-    class AbstructNotifier
+
+    # An abstract class, or superclass, for CompositeNotifier and
+    # LeveledNotifier to inherit. It provides several wrapper methods for the
+    # OutputMethod object used by the Notifier.
+    class AbstractNotifier
+      # Creates a new Notifier object
       def initialize(prefix, base_notifier)
 	@prefix = prefix
 	@base_notifier = base_notifier
       end
 
+      # The +prefix+ for this Notifier, which is appended to all objects being
+      # inspected during output.
       attr_reader :prefix
 
+      # A wrapper method used to determine whether notifications are enabled.
+      #
+      # Defaults to +true+.
       def notify?
 	true
       end
 
+      # See OutputMethod#print for more detail.
       def print(*opts)
 	@base_notifier.print prefix, *opts if notify?
       end
 
+      # See OutputMethod#printn for more detail.
       def printn(*opts)
 	@base_notifier.printn prefix, *opts if notify?
       end
 
+      # See OutputMethod#printf for more detail.
       def printf(format, *opts)
 	@base_notifier.printf(prefix + format, *opts) if notify?
       end
 
+      # See OutputMethod#puts for more detail.
       def puts(*objs)
 	if notify?
 	  @base_notifier.puts(*objs.collect{|obj| prefix + obj.to_s})
 	end
       end
 
+      # Same as #ppx, except it uses the #prefix given during object
+      # initialization.
+      # See OutputMethod#ppx for more detail.
       def pp(*objs)
 	if notify?
 	  @base_notifier.ppx @prefix, *objs
 	end
       end
 
+      # Same as #pp, except it concatenates the given +prefix+ with the #prefix
+      # given during object initialization.
+      #
+      # See OutputMethod#ppx for more detail.
       def ppx(prefix, *objs)
 	if notify?
 	  @base_notifier.ppx @prefix+prefix, *objs
 	end
       end
 
+      # Execute the given block if notifications are enabled.
       def exec_if
 	yield(@base_notifier) if notify?
       end
     end
 
-    class CompositeNotifier<AbstructNotifier
+    # A class that can be used to create a group of notifier objects with the
+    # intent of representing a leveled notification system for irb.
+    #
+    # This class will allow you to generate other notifiers, and assign them
+    # the appropriate level for output.
+    #
+    # The Notifier class provides a class-method Notifier.def_notifier to
+    # create a new composite notifier. Using the first composite notifier
+    # object you create, sibling notifiers can be initialized with
+    # #def_notifier.
+    class CompositeNotifier<AbstractNotifier
+      # Create a new composite notifier object with the given +prefix+, and
+      # +base_notifier+ to use for output.
       def initialize(prefix, base_notifier)
 	super
 
@@ -81,20 +122,42 @@ module IRB
 	@level_notifier = D_NOMSG
       end
 
+      # List of notifiers in the group
       attr_reader :notifiers
 
+      # Creates a new LeveledNotifier in the composite #notifiers group.
+      #
+      # The given +prefix+ will be assigned to the notifier, and +level+ will
+      # be used as the index of the #notifiers Array.
+      #
+      # This method returns the newly created instance.
       def def_notifier(level, prefix = "")
 	notifier = LeveledNotifier.new(self, level, prefix)
 	@notifiers[level] = notifier
 	notifier
       end
 
+      # Returns the leveled notifier for this object
       attr_reader :level_notifier
       alias level level_notifier
 
+      # Sets the leveled notifier for this object.
+      #
+      # When the given +value+ is an instance of AbstractNotifier,
+      # #level_notifier is set to the given object.
+      #
+      # When an Integer is given, #level_notifier is set to the notifier at the
+      # index +value+ in the #notifiers Array.
+      #
+      # If no notifier exists at the index +value+ in the #notifiers Array, an
+      # ErrUndefinedNotifier exception is raised.
+      #
+      # An ErrUnrecognizedLevel exception is raised if the given +value+ is not
+      # found in the existing #notifiers Array, or an instance of
+      # AbstractNotifier
       def level_notifier=(value)
 	case value
-	when AbstructNotifier
+	when AbstractNotifier
 	  @level_notifier = value
 	when Integer
 	  l = @notifiers[value]
@@ -108,38 +171,61 @@ module IRB
       alias level= level_notifier=
     end
 
-    class LeveledNotifier<AbstructNotifier
+    # A leveled notifier is comparable to the composite group from
+    # CompositeNotifier#notifiers.
+    class LeveledNotifier<AbstractNotifier
       include Comparable
 
+      # Create a new leveled notifier with the given +base+, and +prefix+ to
+      # send to AbstractNotifier.new
+      #
+      # The given +level+ is used to compare other leveled notifiers in the
+      # CompositeNotifier group to determine whether or not to output
+      # notifications.
       def initialize(base, level, prefix)
 	super(prefix, base)
-	
+
 	@level = level
       end
 
+      # The current level of this notifier object
       attr_reader :level
 
+      # Compares the level of this notifier object with the given +other+
+      # notifier.
+      #
+      # See the Comparable module for more information.
       def <=>(other)
 	@level <=> other.level
       end
-      
+
+      # Whether to output messages to the output method, depending on the level
+      # of this notifier object.
       def notify?
 	@base_notifier.level >= self
       end
     end
 
+    # NoMsgNotifier is a LeveledNotifier that's used as the default notifier
+    # when creating a new CompositeNotifier.
+    #
+    # This notifier is used as the +zero+ index, or level +0+, for
+    # CompositeNotifier#notifiers, and will not output messages of any sort.
     class NoMsgNotifier<LeveledNotifier
+      # Creates a new notifier that should not be used to output messages.
       def initialize
 	@base_notifier = nil
 	@level = 0
 	@prefix = ""
       end
 
+      # Ensures notifications are ignored, see AbstractNotifier#notify? for
+      # more information.
       def notify?
 	false
       end
     end
 
-    D_NOMSG = NoMsgNotifier.new
+    D_NOMSG = NoMsgNotifier.new # :nodoc:
   end
 end

data/lib/irb/output-method.rb

@@ -1,33 +1,38 @@
 #
-#   output-method.rb - optput methods used by irb 
-#   	$Release Version: 0.9.5$
-#   	$Revision: 11708 $
-#   	$Date: 2007-02-12 15:01:19 -0800 (Mon, 12 Feb 2007) $
+#   output-method.rb - output methods used by irb
+#   	$Release Version: 0.9.6$
+#   	$Revision$
 #   	by Keiju ISHITSUKA(keiju@ruby-lang.org)
 #
 # --
 #
-#   
+#
 #
 
 require "e2mmap"
 
 module IRB
-  # OutputMethod
-  #   StdioOutputMethod
-
+  # An abstract output class for IO in irb. This is mainly used internally by
+  # IRB::Notifier. You can define your own output method to use with Irb.new,
+  # or Context.new
   class OutputMethod
-    @RCS_ID='-$Id: output-method.rb 11708 2007-02-12 23:01:19Z shyouhei $-'
+    extend Exception2MessageMapper
+    def_exception :NotImplementedError, "Need to define `%s'"
+
 
+    # Open this method to implement your own output method, raises a
+    # NotImplementedError if you don't define #print in your own class.
     def print(*opts)
-      IRB.fail NotImplementError, "print"
+      OutputMethod.Raise NotImplementedError, "print"
     end
 
+    # Prints the given +opts+, with a newline delimiter.
     def printn(*opts)
       print opts.join(" "), "\n"
     end
 
-    # extend printf
+    # Extends IO#printf to format the given +opts+ for Kernel#sprintf using
+    # #parse_printf_format
     def printf(format, *opts)
       if /(%*)%I/ =~ format
 	format, opts = parse_printf_format(format, opts)
@@ -35,31 +40,22 @@ module IRB
       print sprintf(format, *opts)
     end
 
-    # %
-    # <�ե饰>  [#0- +]
-    # <�Ǿ��ե��������> (\*|\*[1-9][0-9]*\$|[1-9][0-9]*)
-    # <����>.(\*|\*[1-9][0-9]*\$|[1-9][0-9]*|)?
-    # #<Ĺ������ʸ��>(hh|h|l|ll|L|q|j|z|t)
-    # <�Ѵ�����ʸ��>[diouxXeEfgGcsb%] 
+    # Returns an array of the given +format+ and +opts+ to be used by
+    # Kernel#sprintf, if there was a successful Regexp match in the given
+    # +format+ from #printf
+    #
+    #     %
+    #     <flag>  [#0- +]
+    #     <minimum field width> (\*|\*[1-9][0-9]*\$|[1-9][0-9]*)
+    #     <precision>.(\*|\*[1-9][0-9]*\$|[1-9][0-9]*|)?
+    #     #<length modifier>(hh|h|l|ll|L|q|j|z|t)
+    #     <conversion specifier>[diouxXeEfgGcsb%]
     def parse_printf_format(format, opts)
       return format, opts if $1.size % 2 == 1
     end
 
-    def foo(format)
-      pos = 0
-      inspects = []
-      format.scan(/%[#0\-+ ]?(\*(?=[^0-9])|\*[1-9][0-9]*\$|[1-9][0-9]*(?=[^0-9]))?(\.(\*(?=[^0-9])|\*[1-9][0-9]*\$|[1-9][0-9]*(?=[^0-9])))?(([1-9][0-9]*\$)*)([diouxXeEfgGcsb%])/) {|f, p, pp, pos, new_pos, c|
-	puts [f, p, pp, pos, new_pos, c].join("!")
-	pos = new_pos if new_pos
-	if c == "I"
-	  inspects.push pos.to_i 
-	  (f||"")+(p||"")+(pp||"")+(pos||"")+"s"
-	else
-	  $&
-	end
-      }
-    end
-
+    # Calls #print on each element in the given +objs+, followed by a newline
+    # character.
     def puts(*objs)
       for obj in objs
 	print(*obj)
@@ -67,17 +63,27 @@ module IRB
       end
     end
 
+    # Prints the given +objs+ calling Object#inspect on each.
+    #
+    # See #puts for more detail.
     def pp(*objs)
       puts(*objs.collect{|obj| obj.inspect})
     end
 
+    # Prints the given +objs+ calling Object#inspect on each and appending the
+    # given +prefix+.
+    #
+    # See #puts for more detail.
     def ppx(prefix, *objs)
       puts(*objs.collect{|obj| prefix+obj.inspect})
     end
 
   end
 
+  # A standard output printer
   class StdioOutputMethod<OutputMethod
+    # Prints the given +opts+ to standard output, see IO#print for more
+    # information.
     def print(*opts)
       STDOUT.print(*opts)
     end

data/lib/irb/ruby-lex.rb

@@ -1,32 +1,32 @@
 #
-#   irb/ruby-lex.rb - ruby lexcal analizer
-#   	$Release Version: 0.9.5$
-#   	$Revision: 11708 $
-#   	$Date: 2007-02-12 15:01:19 -0800 (Mon, 12 Feb 2007) $
+#   irb/ruby-lex.rb - ruby lexcal analyzer
+#   	$Release Version: 0.9.6$
+#   	$Revision$
 #   	by Keiju ISHITSUKA(keiju@ruby-lang.org)
 #
 # --
 #
-#   
+#
 #
 
 require "e2mmap"
 require "irb/slex"
 require "irb/ruby-token"
 
+# :stopdoc:
 class RubyLex
-  @RCS_ID='-$Id: ruby-lex.rb 11708 2007-02-12 23:01:19Z shyouhei $-'
+  @RCS_ID='-$Id$-'
 
   extend Exception2MessageMapper
   def_exception(:AlreadyDefinedToken, "Already defined token(%s)")
   def_exception(:TkReading2TokenNoKey, "key nothing(key='%s')")
   def_exception(:TkSymbol2TokenNoKey, "key nothing(key='%s')")
-  def_exception(:TkReading2TokenDuplicateError, 
+  def_exception(:TkReading2TokenDuplicateError,
 		"key duplicate(token_n='%s', key='%s')")
   def_exception(:SyntaxError, "%s")
 
   def_exception(:TerminateLineInput, "Terminate Line Input")
-  
+
   include RubyToken
 
   class << self
@@ -54,7 +54,8 @@ class RubyLex
     @lex_state = EXPR_BEG
     @space_seen = false
     @here_header = false
-    
+    @post_symbeg = false
+
     @continue = false
     @line = ""
 
@@ -77,7 +78,7 @@ class RubyLex
   # io functions
   def set_input(io, p = nil, &block)
     @io = io
-    if p and p.respond_to?(:call)
+    if p.respond_to?(:call)
       @input = p
     elsif block_given?
       @input = block
@@ -87,12 +88,12 @@ class RubyLex
   end
 
   def get_readed
-    if idx = @readed.reverse.index("\n")
-      @base_char_no = idx
+    if idx = @readed.rindex("\n")
+      @base_char_no = @readed.size - (idx + 1)
     else
       @base_char_no += @readed.size
     end
-    
+
     readed = @readed.join("")
     @readed = []
     readed
@@ -111,7 +112,7 @@ class RubyLex
     end
     @seek += 1
     if c == "\n"
-      @line_no += 1 
+      @line_no += 1
       @char_no = 0
     else
       @char_no += 1
@@ -148,12 +149,12 @@ class RubyLex
       c2 = @here_readed.pop
     end
     c = c2 unless c
-    @rests.unshift c #c = 
-      @seek -= 1
+    @rests.unshift c #c =
+    @seek -= 1
     if c == "\n"
-      @line_no -= 1 
-      if idx = @readed.reverse.index("\n")
-	@char_no = @readed.size - idx
+      @line_no -= 1
+      if idx = @readed.rindex("\n")
+	@char_no = idx + 1
       else
 	@char_no = @base_char_no + @readed.size
       end
@@ -188,7 +189,7 @@ class RubyLex
     prompt
     line = @input.call
     return nil unless line
-    @rests.concat line.split(//)
+    @rests.concat line.chars.to_a
     true
   end
   private :buf_input
@@ -216,14 +217,16 @@ class RubyLex
     @lex_state = EXPR_BEG
     @space_seen = false
     @here_header = false
-    
+
     @continue = false
+    @post_symbeg = false
+
     prompt
 
     @line = ""
     @exp_line_no = @line_no
   end
-  
+
   def each_top_level_statement
     initialize_input
     catch(:TERM_INPUT) do
@@ -234,13 +237,13 @@ class RubyLex
 	  unless l = lex
 	    throw :TERM_INPUT if @line == ''
 	  else
-	    #p l
 	    @line.concat l
 	    if @ltype or @continue or @indent > 0
 	      next
 	    end
 	  end
 	  if @line != "\n"
+            @line.force_encoding(@io.encoding)
 	    yield @line, @exp_line_no
 	  end
 	  break unless l
@@ -286,6 +289,8 @@ class RubyLex
       begin
 	tk = @OP.match(self)
 	@space_seen = tk.kind_of?(TkSPACE)
+	@lex_state = EXPR_END if @post_symbeg && tk.kind_of?(TkOp)
+	@post_symbeg = tk.kind_of?(TkSYMBEG)
       rescue SyntaxError
 	raise if @exception_on_syntax_error
 	tk = TkError.new(@seek, @line_no, @char_no)
@@ -297,7 +302,7 @@ class RubyLex
     #      Tracer.off
     tk
   end
-  
+
   ENINDENT_CLAUSE = [
     "case", "class", "def", "do", "for", "if",
     "module", "unless", "until", "while", "begin" #, "when"
@@ -312,9 +317,11 @@ class RubyLex
     "r" => "/",
     "w" => "]",
     "W" => "]",
+    "i" => "]",
+    "I" => "]",
     "s" => ":"
   }
-  
+
   PERCENT_PAREN = {
     "{" => "}",
     "[" => "]",
@@ -354,7 +361,7 @@ class RubyLex
     end
 
     @OP.def_rule("=begin",
-		 proc{|op, io| @prev_char_no == 0 && peek(0) =~ /\s/}) do 
+		 proc{|op, io| @prev_char_no == 0 && peek(0) =~ /\s/}) do
       |op, io|
       @ltype = "="
       until getc == "\n"; end
@@ -374,8 +381,8 @@ class RubyLex
       else
 	@continue = false
 	@lex_state = EXPR_BEG
-	until (@indent_stack.empty? || 
-	       [TkLPAREN, TkLBRACK, TkLBRACE, 
+	until (@indent_stack.empty? ||
+	       [TkLPAREN, TkLBRACK, TkLBRACE,
 		 TkfLPAREN, TkfLBRACK, TkfLBRACE].include?(@indent_stack.last))
 	  @indent_stack.pop
 	end
@@ -385,11 +392,12 @@ class RubyLex
       Token(TkNL)
     end
 
-    @OP.def_rules("*", "**",	
-		  "=", "==", "===", 
-		  "=~", "<=>",	
+    @OP.def_rules("*", "**",
+		  "=", "==", "===",
+		  "=~", "<=>",
 		  "<", "<=",
-		  ">", ">=", ">>") do
+		  ">", ">=", ">>",
+		  "!", "!=", "!~") do
       |op, io|
       case @lex_state
       when EXPR_FNAME, EXPR_DOT
@@ -400,19 +408,13 @@ class RubyLex
       Token(op)
     end
 
-    @OP.def_rules("!", "!=", "!~") do
-      |op, io|
-      @lex_state = EXPR_BEG
-      Token(op)
-    end
-
     @OP.def_rules("<<") do
       |op, io|
       tk = nil
       if @lex_state != EXPR_END && @lex_state != EXPR_CLASS &&
 	  (@lex_state != EXPR_ARG || @space_seen)
 	c = peek(0)
-	if /\S/ =~ c && (/["'`]/ =~ c || /[\w_]/ =~ c || c == "-")
+	if /\S/ =~ c && (/["'`]/ =~ c || /\w/ =~ c || c == "-")
 	  tk = identify_here_document
 	end
       end
@@ -455,7 +457,7 @@ class RubyLex
 	  @lex_state = EXPR_BEG;
 	  Token(TkQUESTION)
 	else
-	  if (ch == '\\') 
+	  if (ch == '\\')
 	    read_escape
 	  end
 	  @lex_state = EXPR_END
@@ -469,8 +471,8 @@ class RubyLex
       @lex_state = EXPR_BEG
       Token(op)
     end
-    
-    @OP.def_rules("+=", "-=", "*=", "**=", 
+
+    @OP.def_rules("+=", "-=", "*=", "**=",
 		  "&=", "|=", "^=", "<<=", ">>=", "||=", "&&=") do
       |op, io|
       @lex_state = EXPR_BEG
@@ -529,7 +531,7 @@ class RubyLex
 
     lex_int2
   end
-  
+
   def lex_int2
     @OP.def_rules("]", "}", ")") do
       |op, io|
@@ -545,7 +547,7 @@ class RubyLex
 	@lex_state = EXPR_BEG
 	Token(TkCOLON)
       else
-	@lex_state = EXPR_FNAME;
+	@lex_state = EXPR_FNAME
 	Token(TkSYMBEG)
       end
     end
@@ -572,7 +574,7 @@ class RubyLex
 	Token(TkOPASGN, "/") #/)
       elsif @lex_state == EXPR_ARG and @space_seen and peek(0) !~ /\s/
 	identify_string(op)
-      else 
+      else
 	@lex_state = EXPR_BEG
 	Token("/") #/)
       end
@@ -588,7 +590,7 @@ class RubyLex
     # 	@lex_state = EXPR_BEG
     # 	Token(OP_ASGN, :^)
     #       end
-    
+
     @OP.def_rules(",") do
       |op, io|
       @lex_state = EXPR_BEG
@@ -598,8 +600,8 @@ class RubyLex
     @OP.def_rules(";") do
       |op, io|
       @lex_state = EXPR_BEG
-      until (@indent_stack.empty? || 
-	     [TkLPAREN, TkLBRACK, TkLBRACE, 
+      until (@indent_stack.empty? ||
+	     [TkLPAREN, TkLBRACK, TkLBRACE,
 	       TkfLPAREN, TkfLBRACK, TkfLBRACE].include?(@indent_stack.last))
 	@indent_stack.pop
       end
@@ -617,7 +619,7 @@ class RubyLex
       @lex_state = EXPR_BEG
       Token("~")
     end
-    
+
     @OP.def_rule("(") do
       |op, io|
       @indent += 1
@@ -629,7 +631,7 @@ class RubyLex
 	tk_c = TkLPAREN
       end
       @indent_stack.push tk_c
-      tk = Token(tk_c)
+      Token(tk_c)
     end
 
     @OP.def_rule("[]", proc{|op, io| @lex_state == EXPR_FNAME}) do
@@ -683,7 +685,7 @@ class RubyLex
 	@continue = true
 	Token(TkSPACE)
       else
-	ungetc
+	read_escape
 	Token("\\")
       end
     end
@@ -710,7 +712,7 @@ class RubyLex
 
     @OP.def_rule('@') do
       |op, io|
-      if peek(0) =~ /[\w_@]/
+      if peek(0) =~ /[\w@]/
 	ungetc
 	identify_identifier
       else
@@ -718,7 +720,7 @@ class RubyLex
       end
     end
 
-    #       @OP.def_rule("def", proc{|op, io| /\s/ =~ io.peek(0)}) do 
+    #       @OP.def_rule("def", proc{|op, io| /\s/ =~ io.peek(0)}) do
     # 	|op, io|
     # 	@indent += 1
     # 	@lex_state = EXPR_FNAME
@@ -733,19 +735,19 @@ class RubyLex
       printf "MATCH: start %s: %s\n", op, io.inspect if RubyLex.debug?
       if peek(0) =~ /[0-9]/
 	t = identify_number
-      elsif peek(0) =~ /[\w_]/
+      elsif peek(0) =~ /[^\x00-\/:-@\[-^`{-\x7F]/
 	t = identify_identifier
       end
       printf "MATCH: end %s: %s\n", op, io.inspect if RubyLex.debug?
       t
     end
-    
+
     p @OP if RubyLex.debug?
   end
-  
+
   def identify_gvar
     @lex_state = EXPR_END
-    
+
     case ch = getc
     when /[~_*$?!@\/\\;,=:<>".]/   #"
       Token(TkGVAR, "$" + ch)
@@ -761,12 +763,12 @@ class RubyLex
       ungetc
       ungetc
       identify_identifier
-    else 
+    else
       ungetc
       Token("$")
     end
   end
-  
+
   def identify_identifier
     token = ""
     if peek(0) =~ /[$@]/
@@ -776,12 +778,12 @@ class RubyLex
       end
     end
 
-    while (ch = getc) =~ /\w|_/
+    while (ch = getc) =~ /[^\x00-\/:-@\[-^`{-\x7F]/
       print ":", ch, ":" if RubyLex.debug?
       token.concat ch
     end
     ungetc
-    
+
     if (ch == "!" || ch == "?") && token[0,1] =~ /\w/ && peek(0) != "="
       token.concat getc
     end
@@ -799,7 +801,7 @@ class RubyLex
       @lex_state = EXPR_END
       return Token(TkIVAR, token)
     end
-    
+
     if @lex_state != EXPR_DOT
       print token, "\n" if RubyLex.debug?
 
@@ -822,11 +824,11 @@ class RubyLex
 	      when "class"
 		valid = false unless peek_match?(/^\s*(<<|\w|::)/)
 	      when "def"
-		valid = false if peek_match?(/^\s*(([+-\/*&\|^]|<<|>>|\|\||\&\&)=|\&\&|\|\|)/)
+		valid = false if peek_match?(/^\s*(([+\-\/*&\|^]|<<|>>|\|\||\&\&)=|\&\&|\|\|)/)
 	      when "do"
-		valid = false if peek_match?(/^\s*([+-\/*]?=|\*|<|>|\&)/)
+		valid = false if peek_match?(/^\s*([+\-\/*]?=|\*|<|>|\&)/)
 	      when *ENINDENT_CLAUSE
-		valid = false if peek_match?(/^\s*([+-\/*]?=|\*|<|>|\&|\|)/)
+		valid = false if peek_match?(/^\s*([+\-\/*]?=|\*|<|>|\&|\|)/)
 	      else
 		# no nothing
 	      end
@@ -910,10 +912,25 @@ class RubyLex
     end
 
     @here_header = false
-    while l = gets
-      l = l.sub(/(:?\r)?\n\z/, '')
-      if (indent ? l.strip : l) == quoted
- 	break
+#     while l = gets
+#       l = l.sub(/(:?\r)?\n\z/, '')
+#       if (indent ? l.strip : l) == quoted
+#  	break
+#       end
+#     end
+
+    line = ""
+    while ch = getc
+      if ch == "\n"
+	if line == quoted
+	  break
+	end
+	line = ""
+      else
+	line.concat ch unless indent && line == "" && /\s/ =~ ch
+	if @ltype != "'" && ch == "#" && peek(0) == "{"
+	  identify_string_dvar
+	end
       end
     end
 
@@ -927,7 +944,7 @@ class RubyLex
     @lex_state = EXPR_END
     Token(Ltype2Token[lt])
   end
-  
+
   def identify_quotation
     ch = getc
     if lt = PERCENT_LTYPE[ch]
@@ -967,11 +984,11 @@ class RubyLex
       when /[0-7]/
 	match = /[0-7_]/
       when /[89]/
-	RubyLex.fail SyntaxError, "Illegal octal digit"
-      else 
+	RubyLex.fail SyntaxError, "Invalid octal digit"
+      else
 	return Token(TkINTEGER)
       end
-      
+
       len0 = true
       non_digit = false
       while ch = getc
@@ -999,7 +1016,7 @@ class RubyLex
       end
       return Token(TkINTEGER)
     end
-    
+
     type = TkINTEGER
     allow_point = true
     allow_e = true
@@ -1042,7 +1059,7 @@ class RubyLex
     end
     Token(type)
   end
-  
+
   def identify_string(ltype, quoted = ltype)
     @ltype = ltype
     @quoted = quoted
@@ -1052,12 +1069,20 @@ class RubyLex
       while ch = getc
 	if @quoted == ch and nest == 0
 	  break
+	elsif @ltype != "'" && ch == "#" && peek(0) == "{"
+	  identify_string_dvar
 	elsif @ltype != "'" && @ltype != "]" && @ltype != ":" and ch == "#"
 	  subtype = true
+	elsif ch == '\\' and @ltype == "'" #'
+	  case ch = getc
+	  when "\\", "\n", "'"
+	  else
+	    ungetc
+	  end
 	elsif ch == '\\' #'
 	  read_escape
 	end
-	if PERCENT_PAREN.values.include?(@quoted) 
+	if PERCENT_PAREN.values.include?(@quoted)
 	  if PERCENT_PAREN[ch] == @quoted
 	    nest += 1
 	  elsif ch == @quoted
@@ -1066,7 +1091,7 @@ class RubyLex
 	end
       end
       if @ltype == "/"
-	if peek(0) =~ /i|m|x|o|e|s|u|n/
+        while /[imxoesun]/ =~ peek(0)
 	  getc
 	end
       end
@@ -1081,7 +1106,43 @@ class RubyLex
       @lex_state = EXPR_END
     end
   end
-  
+
+  def identify_string_dvar
+    begin
+      getc
+
+      reserve_continue = @continue
+      reserve_ltype = @ltype
+      reserve_indent = @indent
+      reserve_indent_stack = @indent_stack
+      reserve_state = @lex_state
+      reserve_quoted = @quoted
+
+      @ltype = nil
+      @quoted = nil
+      @indent = 0
+      @indent_stack = []
+      @lex_state = EXPR_BEG
+
+      loop do
+	@continue = false
+	prompt
+	tk = token
+	if @ltype or @continue or @indent > 0
+	  next
+	end
+	break if tk.kind_of?(TkRBRACE)
+      end
+    ensure
+      @continue = reserve_continue
+      @ltype = reserve_ltype
+      @indent = reserve_indent
+      @indent_stack = reserve_indent_stack
+      @lex_state = reserve_state
+      @quoted = reserve_quoted
+    end
+  end
+
   def identify_comment
     @ltype = "#"
 
@@ -1097,7 +1158,7 @@ class RubyLex
     end
     return Token(TkCOMMENT)
   end
-  
+
   def read_escape
     case ch = getc
     when "\n", "\r", "\f"
@@ -1114,7 +1175,7 @@ class RubyLex
 	  break
 	end
       end
-      
+
     when "x"
       2.times do
 	case ch = getc
@@ -1143,7 +1204,8 @@ class RubyLex
 	read_escape
       end
     else
-      # other characters 
+      # other characters
     end
   end
 end
+# :startdoc: