rdoc 6.0.0.beta1 → 6.0.0.beta2

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 3806537c526d973e02f6e7c7b73ba88d56bd29f0
-  data.tar.gz: 12c3bcb2879e2d188cf2067202999cb6e7417ba7
+  metadata.gz: 79f073ce205804f7cb64ae64f2918b47aa208b80
+  data.tar.gz: 3c5c07e3985797e49ec9d43e3f68ac2b0246061a
 SHA512:
-  metadata.gz: '009cbfdd5ddf8238ca4219bcf1a44974ea3fc16c794929e03595e8e3713eb130c7c2ed43667d702363586d89b9b52323fab680d4a9102ba79bfc33d1514c5efc'
-  data.tar.gz: 02cb698212d59e25b3799becb7a743503eaa762e2dfc863cdc99facaf500a03969550ac6198286bee076611677eaa3b9e0e241d6db3f4b3cb4611c14943b607b
+  metadata.gz: cdecae483f5d749fed6339751c6ba2f28cd5c608024ead2d24ca20635548ecb9df5dc44c8a04f2052b3c09e9ee9534381960281ed8275042bd4ff9a14fc6aa88
+  data.tar.gz: 2bbe546d5745c41bce196a253e708e6dca4ed21cd9a0695ee7c61ef1163b6750b6a8648149b3afcfdadde2873af9ba326d1b3c62d9ad29495c92d72100dd1665

data/.travis.yml

@@ -2,23 +2,13 @@
 before_install:
   - gem install bundler --no-document
 language: ruby
-notifications:
-  email:
-  - mail@zzak.io
 rvm:
   - 2.2.7
   - 2.3.4
   - 2.4.1
   - ruby-head
-  - rbx-2
-  - jruby-1.7.26
-  - jruby-9.1.9.0
+  - jruby-9.1.13.0
 env:
   global:
     NOBENCHMARK=1
-script: rake
-matrix:
-  allow_failures:
-    - rvm: jruby-1.7.26
-    - rvm: jruby-9.1.9.0
-    - rvm: rbx-2
+script: bundle exec rake

data/appveyor.yml

@@ -0,0 +1,18 @@
+---
+install:
+  - SET PATH=C:\Ruby%ruby_version%\bin;%PATH%
+  - gem install bundler --no-document
+  - bundle install
+build: off
+test_script:
+  - rake
+deploy: off
+environment:
+  NOBENCHMARK: 1
+  matrix:
+    - ruby_version: "22"
+    - ruby_version: "22-x64"
+    - ruby_version: "23"
+    - ruby_version: "23-x64"
+    - ruby_version: "24"
+    - ruby_version: "24-x64"

data/lib/rdoc.rb

@@ -65,7 +65,7 @@ module RDoc
   ##
   # RDoc version you are using
 
-  VERSION = '6.0.0.beta1'
+  VERSION = '6.0.0.beta2'
 
   ##
   # Method visibilities
@@ -148,7 +148,7 @@ module RDoc
 
   autoload :KNOWN_CLASSES,  'rdoc/known_classes'
 
-  autoload :RubyLex,        'rdoc/ruby_lex'
+  autoload :RipperStateLex, 'rdoc/parser/ripper_state_lex'
   autoload :RubyToken,      'rdoc/ruby_token'
   autoload :TokenStream,    'rdoc/token_stream'
 

data/lib/rdoc/constant.rb

@@ -36,7 +36,7 @@ class RDoc::Constant < RDoc::CodeObject
     @value = value
 
     @is_alias_for = nil
-    @visibility   = nil
+    @visibility   = :public
 
     self.comment = comment
   end
@@ -136,7 +136,7 @@ class RDoc::Constant < RDoc::CodeObject
     initialize array[1], nil, array[5]
 
     @full_name     = array[2]
-    @visibility    = array[3]
+    @visibility    = array[3] || :public
     @is_alias_for  = array[4]
     #                      5 handled above
     #                      6 handled below

data/lib/rdoc/context.rb

@@ -1079,6 +1079,7 @@ class RDoc::Context < RDoc::CodeObject
     return if [:private, :nodoc].include? min_visibility
     remove_invisible_in @method_list, min_visibility
     remove_invisible_in @attributes, min_visibility
+    remove_invisible_in @constants, min_visibility
   end
 
   ##
@@ -1165,6 +1166,17 @@ class RDoc::Context < RDoc::CodeObject
     end
   end
 
+  ##
+  # Given an array +names+ of constants, set the visibility of each constant to
+  # +visibility+
+
+  def set_constant_visibility_for(names, visibility)
+    names.each do |name|
+      constant = @constants_hash[name] or next
+      constant.visibility = visibility
+    end
+  end
+
   ##
   # Sorts sections alphabetically (default) or in TomDoc fashion (none,
   # Public, Internal, Deprecated)

data/lib/rdoc/markup/to_html.rb

@@ -200,10 +200,12 @@ class RDoc::Markup::ToHtml < RDoc::Markup::Formatter
 
     content = if verbatim.ruby? or parseable? text then
                 begin
-                  tokens = RDoc::RubyLex.tokenize text, @options
+                  tokens = RDoc::RipperStateLex.parse text
                   klass  = ' class="ruby"'
 
-                  RDoc::TokenStream.to_html tokens
+                  result = RDoc::TokenStream.to_html tokens
+                  result = result + "\n" unless "\n" == result[-1]
+                  result
                 rescue RDoc::RubyLex::Error
                   CGI.escapeHTML text
                 end
@@ -212,7 +214,7 @@ class RDoc::Markup::ToHtml < RDoc::Markup::Formatter
               end
 
     if @options.pipe then
-      @res << "\n<pre><code>#{CGI.escapeHTML text}</code></pre>\n"
+      @res << "\n<pre><code>#{CGI.escapeHTML text}\n</code></pre>\n"
     else
       @res << "\n<pre#{klass}>#{content}</pre>\n"
     end

data/lib/rdoc/parser/c.rb

@@ -666,8 +666,7 @@ class RDoc::Parser::C < RDoc::Parser
 
       #meth_obj.params = params
       meth_obj.start_collecting_tokens
-      tk = RDoc::RubyToken::Token.new nil, 1, 1
-      tk.set_text body
+      tk = { :line_no => 1, :char_no => 1, :text => body }
       meth_obj.add_token tk
       meth_obj.comment = comment
       meth_obj.line    = file_content[0, offset].count("\n") + 1
@@ -684,8 +683,7 @@ class RDoc::Parser::C < RDoc::Parser
       find_modifiers comment, meth_obj
 
       meth_obj.start_collecting_tokens
-      tk = RDoc::RubyToken::Token.new nil, 1, 1
-      tk.set_text body
+      tk = { :line_no => 1, :char_no => 1, :text => body }
       meth_obj.add_token tk
       meth_obj.comment = comment
       meth_obj.line    = file_content[0, offset].count("\n") + 1

data/lib/rdoc/parser/ripper_state_lex.rb

@@ -0,0 +1,587 @@
+require 'ripper'
+
+class RDoc::RipperStateLex
+  EXPR_NONE = 0
+  EXPR_BEG = 1
+  EXPR_END = 2
+  EXPR_ENDARG = 4
+  EXPR_ENDFN = 8
+  EXPR_ARG = 16
+  EXPR_CMDARG = 32
+  EXPR_MID = 64
+  EXPR_FNAME = 128
+  EXPR_DOT = 256
+  EXPR_CLASS = 512
+  EXPR_LABEL = 1024
+  EXPR_LABELED = 2048
+  EXPR_FITEM = 4096
+  EXPR_VALUE = EXPR_BEG
+  EXPR_BEG_ANY  =  (EXPR_BEG | EXPR_MID | EXPR_CLASS)
+  EXPR_ARG_ANY  =  (EXPR_ARG | EXPR_CMDARG)
+  EXPR_END_ANY  =  (EXPR_END | EXPR_ENDARG | EXPR_ENDFN)
+
+  class InnerStateLex < Ripper::Filter
+    attr_accessor :lex_state
+
+    def initialize(code)
+      @lex_state = EXPR_BEG
+      @in_fname = false
+      @continue = false
+      reset
+      super(code)
+    end
+
+    def reset
+      @command_start = false
+      @cmd_state = @command_start
+    end
+
+    def on_nl(tok, data)
+      case @lex_state
+      when EXPR_FNAME, EXPR_DOT
+        @continue = true
+      else
+        @continue = false
+        @lex_state = EXPR_BEG unless (EXPR_LABEL & @lex_state) != 0
+      end
+      @callback.call({ :line_no => lineno, :char_no => column, :kind => __method__, :text => tok, :state => @lex_state})
+    end
+
+    def on_ignored_nl(tok, data)
+      case @lex_state
+      when EXPR_FNAME, EXPR_DOT
+        @continue = true
+      else
+        @continue = false
+        @lex_state = EXPR_BEG unless (EXPR_LABEL & @lex_state) != 0
+      end
+      @callback.call({ :line_no => lineno, :char_no => column, :kind => __method__, :text => tok, :state => @lex_state})
+    end
+
+    def on_op(tok, data)
+      case tok
+      when '&', '|', '!', '!=', '!~'
+        case @lex_state
+        when EXPR_FNAME, EXPR_DOT
+          @lex_state = EXPR_ARG
+        else
+          @lex_state = EXPR_BEG
+        end
+      when '<<'
+        # TODO next token?
+        case @lex_state
+        when EXPR_FNAME, EXPR_DOT
+          @lex_state = EXPR_ARG
+        else
+          @lex_state = EXPR_BEG
+        end
+      when '?'
+        @lex_state = EXPR_BEG
+      when '&&', '||', '+=', '-=', '*=', '**=',
+           '&=', '|=', '^=', '<<=', '>>=', '||=', '&&='
+        @lex_state = EXPR_BEG
+      else
+        case @lex_state
+        when EXPR_FNAME, EXPR_DOT
+          @lex_state = EXPR_ARG
+        else
+          @lex_state = EXPR_BEG
+        end
+      end
+      @callback.call({ :line_no => lineno, :char_no => column, :kind => __method__, :text => tok, :state => @lex_state})
+    end
+
+    def on_kw(tok, data)
+      case tok
+      when 'class'
+        @lex_state = EXPR_CLASS
+        @in_fname = true
+      when 'def'
+        @lex_state = EXPR_FNAME
+        @continue = true
+        @in_fname = true
+      when 'if', 'unless', 'while', 'until'
+        if ((EXPR_END | EXPR_ENDARG | EXPR_ENDFN | EXPR_ARG | EXPR_CMDARG) & @lex_state) != 0 # postfix if
+          @lex_state = EXPR_BEG | EXPR_LABEL
+        else
+          @lex_state = EXPR_BEG
+        end
+      when 'begin'
+        @lex_state = EXPR_BEG
+      else
+        if @lex_state == EXPR_FNAME
+          @lex_state = EXPR_END
+        else
+          @lex_state = EXPR_END
+        end
+      end
+      @callback.call({ :line_no => lineno, :char_no => column, :kind => __method__, :text => tok, :state => @lex_state})
+    end
+
+    def on_tstring_beg(tok, data)
+      @lex_state = EXPR_BEG
+      @callback.call({ :line_no => lineno, :char_no => column, :kind => __method__, :text => tok, :state => @lex_state})
+    end
+
+    def on_tstring_end(tok, data)
+      @lex_state = EXPR_END | EXPR_ENDARG
+      @callback.call({ :line_no => lineno, :char_no => column, :kind => __method__, :text => tok, :state => @lex_state})
+    end
+
+    def on_CHAR(tok, data)
+      @lex_state = EXPR_END
+      @callback.call({ :line_no => lineno, :char_no => column, :kind => __method__, :text => tok, :state => @lex_state})
+    end
+
+    def on_period(tok, data)
+      @lex_state = EXPR_DOT
+      @callback.call({ :line_no => lineno, :char_no => column, :kind => __method__, :text => tok, :state => @lex_state})
+    end
+
+    def on_int(tok, data)
+      @lex_state = EXPR_END | EXPR_ENDARG
+      @callback.call({ :line_no => lineno, :char_no => column, :kind => __method__, :text => tok, :state => @lex_state})
+    end
+
+    def on_float(tok, data)
+      @lex_state = EXPR_END | EXPR_ENDARG
+      @callback.call({ :line_no => lineno, :char_no => column, :kind => __method__, :text => tok, :state => @lex_state})
+    end
+
+    def on_rational(tok, data)
+      @lex_state = EXPR_END | EXPR_ENDARG
+      @callback.call({ :line_no => lineno, :char_no => column, :kind => __method__, :text => tok, :state => @lex_state})
+    end
+
+    def on_imaginary(tok, data)
+      @lex_state = EXPR_END | EXPR_ENDARG
+      @callback.call({ :line_no => lineno, :char_no => column, :kind => __method__, :text => tok, :state => @lex_state})
+    end
+
+    def on_symbeg(tok, data)
+      @lex_state = EXPR_FNAME
+      @continue = true
+      @in_fname = true
+      @callback.call({ :line_no => lineno, :char_no => column, :kind => __method__, :text => tok, :state => @lex_state})
+    end
+
+    private def on_variables(event, tok, data)
+      if @in_fname
+        @lex_state = EXPR_ENDFN
+        @in_fname = false
+        @continue = false
+      elsif @continue
+        case @lex_state
+        when EXPR_DOT
+          @lex_state = EXPR_ARG
+        else
+          @lex_state = EXPR_ENDFN
+          @continue = false
+        end
+      else
+        @lex_state = EXPR_CMDARG
+      end
+      @callback.call({ :line_no => lineno, :char_no => column, :kind => event, :text => tok, :state => @lex_state})
+    end
+
+    def on_ident(tok, data)
+      on_variables(__method__, tok, data)
+    end
+
+    def on_ivar(tok, data)
+      @lex_state = EXPR_END
+      on_variables(__method__, tok, data)
+    end
+
+    def on_cvar(tok, data)
+      @lex_state = EXPR_END
+      on_variables(__method__, tok, data)
+    end
+
+    def on_gvar(tok, data)
+      @lex_state = EXPR_END
+      on_variables(__method__, tok, data)
+    end
+
+    def on_backref(tok, data)
+      @lex_state = EXPR_END
+      on_variables(__method__, tok, data)
+    end
+
+    def on_lparen(tok, data)
+      @lex_state = EXPR_LABEL | EXPR_BEG
+      @callback.call({ :line_no => lineno, :char_no => column, :kind => __method__, :text => tok, :state => @lex_state})
+    end
+
+    def on_rparen(tok, data)
+      @lex_state = EXPR_ENDFN
+      @callback.call({ :line_no => lineno, :char_no => column, :kind => __method__, :text => tok, :state => @lex_state})
+    end
+
+    def on_lbrace(tok, data)
+      @lex_state = EXPR_LABEL | EXPR_BEG
+      @callback.call({ :line_no => lineno, :char_no => column, :kind => __method__, :text => tok, :state => @lex_state})
+    end
+
+    def on_rbrace(tok, data)
+      @lex_state = EXPR_ENDARG
+      @callback.call({ :line_no => lineno, :char_no => column, :kind => __method__, :text => tok, :state => @lex_state})
+    end
+
+    def on_lbracket(tok, data)
+      @lex_state = EXPR_LABEL | EXPR_BEG
+      @callback.call({ :line_no => lineno, :char_no => column, :kind => __method__, :text => tok, :state => @lex_state})
+    end
+
+    def on_rbracket(tok, data)
+      @lex_state = EXPR_ENDARG
+      @callback.call({ :line_no => lineno, :char_no => column, :kind => __method__, :text => tok, :state => @lex_state})
+    end
+
+    def on_const(tok, data)
+      case @lex_state
+      when EXPR_FNAME
+        @lex_state = EXPR_ENDFN
+      when EXPR_CLASS
+        @lex_state = EXPR_ARG
+      else
+        @lex_state = EXPR_CMDARG
+      end
+      @callback.call({ :line_no => lineno, :char_no => column, :kind => __method__, :text => tok, :state => @lex_state})
+    end
+
+    def on_sp(tok, data)
+      @callback.call({ :line_no => lineno, :char_no => column, :kind => __method__, :text => tok, :state => @lex_state})
+    end
+
+    def on_comma(tok, data)
+      @lex_state = EXPR_BEG | EXPR_LABEL if (EXPR_ARG_ANY & @lex_state) != 0
+      @callback.call({ :line_no => lineno, :char_no => column, :kind => __method__, :text => tok, :state => @lex_state})
+    end
+
+    def on_comment(tok, data)
+      @lex_state = EXPR_BEG unless (EXPR_LABEL & @lex_state) != 0
+      @callback.call({ :line_no => lineno, :char_no => column, :kind => __method__, :text => tok, :state => @lex_state})
+    end
+
+    def on_ignored_sp(tok, data)
+      @lex_state = EXPR_BEG unless (EXPR_LABEL & @lex_state) != 0
+      @callback.call({ :line_no => lineno, :char_no => column, :kind => __method__, :text => tok, :state => @lex_state})
+    end
+
+    def on_heredoc_end(tok, data)
+      @callback.call({ :line_no => lineno, :char_no => column, :kind => __method__, :text => tok, :state => @lex_state})
+      @lex_state = EXPR_BEG
+    end
+
+    def on_default(event, tok, data)
+      reset
+      @callback.call({ :line_no => lineno, :char_no => column, :kind => event, :text => tok, :state => @lex_state})
+    end
+
+    def each(&block)
+      @callback = block
+      parse
+    end
+  end
+
+  def get_squashed_tk
+    if @buf.empty?
+      tk = @inner_lex_enumerator.next
+    else
+      tk = @buf.shift
+    end
+    case tk[:kind]
+    when :on_symbeg then
+      tk = get_symbol_tk(tk)
+    when :on_tstring_beg then
+      tk = get_string_tk(tk)
+    when :on_backtick then
+      if (EXPR_FNAME & tk[:state]) != 0
+        @inner_lex.lex_state = EXPR_ARG
+        tk[:kind] = :on_ident
+        tk[:state] = @inner_lex.lex_state
+      else
+        tk = get_string_tk(tk)
+      end
+    when :on_regexp_beg then
+      tk = get_regexp_tk(tk)
+    when :on_embdoc_beg then
+      tk = get_embdoc_tk(tk)
+    when :on_heredoc_beg then
+      @heredoc_queue << retrieve_heredoc_info(tk)
+      @inner_lex.lex_state = EXPR_END
+    when :on_nl, :on_ignored_nl, :on_comment, :on_heredoc_end then
+      unless @heredoc_queue.empty?
+        get_heredoc_tk(*@heredoc_queue.shift)
+      end
+    when :on_words_beg then
+      tk = get_words_tk(tk)
+    when :on_qwords_beg then
+      tk = get_words_tk(tk)
+    when :on_symbols_beg then
+      tk = get_words_tk(tk)
+    when :on_qsymbols_beg then
+      tk = get_words_tk(tk)
+    when :on_op then
+      if '&.' == tk[:text]
+        tk[:kind] = :on_period
+      else
+        tk = get_op_tk(tk)
+      end
+    end
+    tk
+  end
+
+  private def get_symbol_tk(tk)
+    is_symbol = true
+    symbol_tk = { :line_no => tk[:line_no], :char_no => tk[:char_no], :kind => :on_symbol }
+    if ":'" == tk[:text] or ':"' == tk[:text]
+      tk1 = get_string_tk(tk)
+      symbol_tk[:text] = tk1[:text]
+      symbol_tk[:state] = tk1[:state]
+    else
+      case (tk1 = get_squashed_tk)[:kind]
+      when :on_ident
+        symbol_tk[:text] = ":#{tk1[:text]}"
+        symbol_tk[:state] = tk1[:state]
+      when :on_tstring_content
+        symbol_tk[:text] = ":#{tk1[:text]}"
+        symbol_tk[:state] = get_squashed_tk[:state] # skip :on_tstring_end
+      when :on_tstring_end
+        symbol_tk[:text] = ":#{tk1[:text]}"
+        symbol_tk[:state] = tk1[:state]
+      when :on_op
+        symbol_tk[:text] = ":#{tk1[:text]}"
+        symbol_tk[:state] = tk1[:state]
+      when :on_ivar
+        symbol_tk[:text] = ":#{tk1[:text]}"
+        symbol_tk[:state] = tk1[:state]
+      when :on_cvar
+        symbol_tk[:text] = ":#{tk1[:text]}"
+        symbol_tk[:state] = tk1[:state]
+      when :on_gvar
+        symbol_tk[:text] = ":#{tk1[:text]}"
+        symbol_tk[:state] = tk1[:state]
+      when :on_const
+        symbol_tk[:text] = ":#{tk1[:text]}"
+        symbol_tk[:state] = tk1[:state]
+      when :on_kw
+        symbol_tk[:text] = ":#{tk1[:text]}"
+        symbol_tk[:state] = tk1[:state]
+      else
+        is_symbol = false
+        tk = tk1
+      end
+    end
+    if is_symbol
+      tk = symbol_tk
+    end
+    tk
+  end
+
+  private def get_string_tk(tk)
+    string = tk[:text]
+    state = nil
+    kind = :on_tstring
+    loop do
+      inner_str_tk = get_squashed_tk
+      if inner_str_tk.nil?
+        break
+      elsif :on_tstring_end == inner_str_tk[:kind]
+        string = string + inner_str_tk[:text]
+        state = inner_str_tk[:state]
+        break
+      elsif :on_label_end == inner_str_tk[:kind]
+        string = string + inner_str_tk[:text]
+        state = inner_str_tk[:state]
+        kind = :on_symbol
+        break
+      else
+        string = string + inner_str_tk[:text]
+        if :on_embexpr_beg == inner_str_tk[:kind] then
+          kind = :on_dstring if :on_tstring == kind
+        end
+      end
+    end
+    {
+      :line_no => tk[:line_no],
+      :char_no => tk[:char_no],
+      :kind => kind,
+      :text => string,
+      :state => state
+    }
+  end
+
+  private def get_regexp_tk(tk)
+    string = tk[:text]
+    state = nil
+    loop do
+      inner_str_tk = get_squashed_tk
+      if inner_str_tk.nil?
+        break
+      elsif :on_regexp_end == inner_str_tk[:kind]
+        string = string + inner_str_tk[:text]
+        state = inner_str_tk[:state]
+        break
+      else
+        string = string + inner_str_tk[:text]
+      end
+    end
+    {
+      :line_no => tk[:line_no],
+      :char_no => tk[:char_no],
+      :kind => :on_regexp,
+      :text => string,
+      :state => state
+    }
+  end
+
+  private def get_embdoc_tk(tk)
+    string = tk[:text]
+    until :on_embdoc_end == (embdoc_tk = get_squashed_tk)[:kind] do
+      string = string + embdoc_tk[:text]
+    end
+    string = string + embdoc_tk[:text]
+    {
+      :line_no => tk[:line_no],
+      :char_no => tk[:char_no],
+      :kind => :on_embdoc,
+      :text => string,
+      :state => embdoc_tk[:state]
+    }
+  end
+
+  private def get_heredoc_tk(heredoc_name, indent)
+    string = ''
+    start_tk = nil
+    prev_tk = nil
+    until heredoc_end?(heredoc_name, indent, tk = @inner_lex_enumerator.next) do
+      start_tk = tk unless start_tk
+      if (prev_tk.nil? or "\n" == prev_tk[:text][-1]) and 0 != tk[:char_no]
+        string = string + (' ' * tk[:char_no])
+      end
+      string = string + tk[:text]
+      prev_tk = tk
+    end
+    start_tk = tk unless start_tk
+    prev_tk = tk unless prev_tk
+    @buf.unshift tk # closing heredoc
+    heredoc_tk = {
+      :line_no => start_tk[:line_no],
+      :char_no => start_tk[:char_no],
+      :kind => :on_heredoc,
+      :text => string,
+      :state => prev_tk[:state]
+    }
+    @buf.unshift heredoc_tk
+  end
+
+  private def retrieve_heredoc_info(tk)
+    name = tk[:text].gsub(/\A<<[-~]?(['"`]?)(.+)\1\z/, '\2')
+    indent = tk[:text] =~ /\A<<[-~]/
+    [name, indent]
+  end
+
+  private def heredoc_end?(name, indent, tk)
+    result = false
+    if :on_heredoc_end == tk[:kind] then
+      tk_name = (indent ? tk[:text].gsub(/^ *(.+)\n?$/, '\1') : tk[:text].gsub(/\n\z/, ''))
+      if name == tk_name
+        result = true
+      end
+    end
+    result
+  end
+
+  private def get_words_tk(tk)
+    string = ''
+    start_token = tk[:text]
+    start_quote = tk[:text].rstrip[-1]
+    line_no = tk[:line_no]
+    char_no = tk[:char_no]
+    state = tk[:state]
+    end_quote =
+      case start_quote
+      when ?( then ?)
+      when ?[ then ?]
+      when ?{ then ?}
+      when ?< then ?>
+      else start_quote
+      end
+    end_token = nil
+    loop do
+      tk = get_squashed_tk
+      if tk.nil?
+        end_token = end_quote
+        break
+      elsif :on_tstring_content == tk[:kind] then
+        string += tk[:text]
+      elsif :on_words_sep == tk[:kind] or :on_tstring_end == tk[:kind] then
+        if end_quote == tk[:text].strip then
+          end_token = tk[:text]
+          break
+        else
+          string += tk[:text]
+        end
+      else
+        string += tk[:text]
+      end
+    end
+    text = "#{start_token}#{string}#{end_token}"
+    {
+      :line_no => line_no,
+      :char_no => char_no,
+      :kind => :on_dstring,
+      :text => text,
+      :state => state
+    }
+  end
+
+  private def get_op_tk(tk)
+    redefinable_operators = %w[! != !~ % & * ** + +@ - -@ / < << <= <=> == === =~ > >= >> [] []= ^ ` | ~]
+    if redefinable_operators.include?(tk[:text]) and EXPR_ARG == tk[:state] then
+      @inner_lex.lex_state = EXPR_ARG
+      tk[:kind] = :on_ident
+      tk[:state] = @inner_lex.lex_state
+    elsif tk[:text] =~ /^[-+]$/ then
+      tk_ahead = get_squashed_tk
+      case tk_ahead[:kind]
+      when :on_int, :on_float, :on_rational, :on_imaginary then
+        tk[:text] += tk_ahead[:text]
+        tk[:kind] = tk_ahead[:kind]
+        tk[:state] = tk_ahead[:state]
+      else
+        @buf.unshift tk_ahead
+      end
+    end
+    tk
+  end
+
+  def initialize(code)
+    @buf = []
+    @heredoc_queue = []
+    @inner_lex = InnerStateLex.new(code)
+    @inner_lex_enumerator = Enumerator.new do |y|
+      @inner_lex.each do |tk|
+        y << tk
+      end
+    end
+  end
+
+  def self.parse(code)
+    lex = self.new(code)
+    tokens = []
+    begin
+      while tk = lex.get_squashed_tk
+        tokens.push tk
+      end
+    rescue StopIteration
+    end
+    tokens
+  end
+
+  def self.end?(token)
+    (token[:state] & EXPR_END)
+  end
+end