rdoc 6.0.0.beta1 → 6.0.0.beta2

data/lib/rdoc/parser/ruby_tools.rb

@@ -5,8 +5,6 @@
 
 module RDoc::Parser::RubyTools
 
-  include RDoc::RubyToken
-
   ##
   # Adds a token listener +obj+, but you should probably use token_listener
 
@@ -22,16 +20,25 @@ module RDoc::Parser::RubyTools
     tk = nil
 
     if @tokens.empty? then
-      tk = @scanner.token
-      @read.push @scanner.get_readed
-      puts "get_tk1 => #{tk.inspect}" if $TOKEN_DEBUG
+      if @scanner_point >= @scanner.size
+        return nil
+      else
+        tk = @scanner[@scanner_point]
+        @scanner_point += 1
+        @read.push tk[:text]
+        puts "get_tk1 => #{tk.inspect}" if $TOKEN_DEBUG
+      end
     else
       @read.push @unget_read.shift
       tk = @tokens.shift
       puts "get_tk2 => #{tk.inspect}" if $TOKEN_DEBUG
     end
 
-    tk = nil if TkEND_OF_SCRIPT === tk
+    if tk == nil || :on___end__ == tk[:kind]
+      tk = nil
+    end
+
+    return nil unless tk
 
     # inform any listeners of our shiny new token
     @token_listeners.each do |obj|
@@ -102,19 +109,24 @@ module RDoc::Parser::RubyTools
     @tokens     = []
     @unget_read = []
     @nest = 0
+    @scanner_point = 0
+  end
+
+  def tk_nl?(tk)
+    :on_nl == tk[:kind] or :on_ignored_nl == tk[:kind]
   end
 
   ##
   # Skips whitespace tokens including newlines if +skip_nl+ is true
 
-  def skip_tkspace(skip_nl = true) # HACK dup
+  def skip_tkspace(skip_nl = true)
     tokens = []
 
-    while TkSPACE === (tk = get_tk) or (skip_nl and TkNL === tk) do
-      tokens.push tk
+    while (tk = get_tk) and (:on_sp == tk[:kind] or (skip_nl and tk_nl?(tk))) do
+      tokens.push(tk)
     end
 
-    unget_tk tk
+    unget_tk(tk)
     tokens
   end
 

data/lib/rdoc/stats/normal.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: false
-require 'io/console/size'
+require 'io/console'
 
 ##
 # Stats printer that prints just the files being documented with a progress
@@ -23,7 +23,7 @@ class RDoc::Stats::Normal < RDoc::Stats::Quiet
 
     # Print a progress bar, but make sure it fits on a single line. Filename
     # will be truncated if necessary.
-    terminal_width = IO.console_size[1].to_i.nonzero? || 80
+    terminal_width = IO.console.winsize[1].to_i.nonzero? || 80
     max_filename_size = terminal_width - progress_bar.size
 
     if filename.size > max_filename_size then

data/lib/rdoc/token_stream.rb

@@ -30,28 +30,41 @@ module RDoc::TokenStream
     token_stream.map do |t|
       next unless t
 
-      style = case t
-              when RDoc::RubyToken::TkCONSTANT then 'ruby-constant'
-              when RDoc::RubyToken::TkKW       then 'ruby-keyword'
-              when RDoc::RubyToken::TkIVAR     then 'ruby-ivar'
-              when RDoc::RubyToken::TkOp       then 'ruby-operator'
-              when RDoc::RubyToken::TkId       then 'ruby-identifier'
-              when RDoc::RubyToken::TkREGEXP   then 'ruby-regexp'
-              when RDoc::RubyToken::TkDREGEXP  then 'ruby-regexp'
-              when RDoc::RubyToken::TkNode     then 'ruby-node'
-              when RDoc::RubyToken::TkCOMMENT  then 'ruby-comment'
-              when RDoc::RubyToken::TkXSTRING  then 'ruby-string'
-              when RDoc::RubyToken::TkSTRING   then 'ruby-string'
-              when RDoc::RubyToken::TkVal      then 'ruby-value'
+      style = case t[:kind]
+              when :on_const   then 'ruby-constant'
+              when :on_kw      then 'ruby-keyword'
+              when :on_ivar    then 'ruby-ivar'
+              when :on_cvar    then 'ruby-identifier'
+              when :on_gvar    then 'ruby-identifier'
+              when '=' != t[:text] && :on_op then
+                if RDoc::RipperStateLex::EXPR_ARG == t[:state] then
+                                    'ruby-identifier'
+                else
+                                    'ruby-operator'
+                end
+              when :on_tlambda then 'ruby-operator'
+              when :on_ident   then 'ruby-identifier'
+              when :on_label   then 'ruby-value'
+              when :on_backref, :on_dstring
+                               then 'ruby-node'
+              when :on_comment then 'ruby-comment'
+              when :on_embdoc  then 'ruby-comment'
+              when :on_regexp  then 'ruby-regexp'
+              when :on_tstring then 'ruby-string'
+              when :on_int, :on_float,
+                   :on_rational, :on_imaginary,
+                   :on_heredoc,
+                   :on_symbol, :on_CHAR then 'ruby-value'
+              when :on_heredoc_beg, :on_heredoc_end
+                               then 'ruby-identifier'
               end
 
       comment_with_nl = false
-      case t
-      when RDoc::RubyToken::TkRD_COMMENT, RDoc::RubyToken::TkHEREDOCEND
-        comment_with_nl = true if t.text =~ /\n$/
-        text = t.text.rstrip
+      if :on_comment == t[:kind] or :on_embdoc == t[:kind] or :on_heredoc_end == t[:kind]
+        comment_with_nl = true if "\n" == t[:text][-1]
+        text = t[:text].rstrip
       else
-        text = t.text
+        text = t[:text]
       end
       text = CGI.escapeHTML text
 

data/rdoc.gemspec

@@ -1,11 +1,10 @@
-# -*- encoding: utf-8 -*-
-$:.unshift File.expand_path("../lib", __FILE__)
-require 'rdoc'
+require_relative "lib/rdoc"
 
 Gem::Specification.new do |s|
   s.name = "rdoc"
   s.version = RDoc::VERSION
-
+  s.date = "2017-09-12"
+  
   s.authors = [
     "Eric Hodel",
     "Dave Thomas",
@@ -28,7 +27,7 @@ RDoc includes the +rdoc+ and +ri+ tools for generating and displaying documentat
   s.executables = ["rdoc", "ri"]
   s.require_paths = ["lib"]
   # for ruby core repository. It was generated by `git ls-files -z`.split("\x0").reject { |f| f.match(%r{^(test|spec|features)/}) }
-  s.files = [".document", ".gitignore", ".travis.yml", "CONTRIBUTING.rdoc", "CVE-2013-0256.rdoc", "ExampleMarkdown.md", "ExampleRDoc.rdoc", "Gemfile", "History.rdoc", "LEGAL.rdoc", "LICENSE.rdoc", "README.rdoc", "RI.rdoc", "Rakefile", "TODO.rdoc", "bin/console", "bin/setup", "exe/rdoc", "exe/ri", "lib/gauntlet_rdoc.rb", "lib/rdoc.rb", "lib/rdoc/alias.rb", "lib/rdoc/anon_class.rb", "lib/rdoc/any_method.rb", "lib/rdoc/attr.rb", "lib/rdoc/class_module.rb", "lib/rdoc/code_object.rb", "lib/rdoc/code_objects.rb", "lib/rdoc/comment.rb", "lib/rdoc/constant.rb", "lib/rdoc/context.rb", "lib/rdoc/context/section.rb", "lib/rdoc/cross_reference.rb", "lib/rdoc/encoding.rb", "lib/rdoc/erb_partial.rb", "lib/rdoc/erbio.rb", "lib/rdoc/extend.rb", "lib/rdoc/generator.rb", "lib/rdoc/generator/darkfish.rb", "lib/rdoc/generator/json_index.rb", "lib/rdoc/generator/markup.rb", "lib/rdoc/generator/pot.rb", "lib/rdoc/generator/pot/message_extractor.rb", "lib/rdoc/generator/pot/po.rb", "lib/rdoc/generator/pot/po_entry.rb", "lib/rdoc/generator/ri.rb", "lib/rdoc/generator/template/darkfish/.document", "lib/rdoc/generator/template/darkfish/_footer.rhtml", "lib/rdoc/generator/template/darkfish/_head.rhtml","lib/rdoc/generator/template/darkfish/_sidebar_VCS_info.rhtml", "lib/rdoc/generator/template/darkfish/_sidebar_classes.rhtml", "lib/rdoc/generator/template/darkfish/_sidebar_extends.rhtml", "lib/rdoc/generator/template/darkfish/_sidebar_in_files.rhtml", "lib/rdoc/generator/template/darkfish/_sidebar_includes.rhtml", "lib/rdoc/generator/template/darkfish/_sidebar_installed.rhtml", "lib/rdoc/generator/template/darkfish/_sidebar_methods.rhtml", "lib/rdoc/generator/template/darkfish/_sidebar_navigation.rhtml", "lib/rdoc/generator/template/darkfish/_sidebar_pages.rhtml", "lib/rdoc/generator/template/darkfish/_sidebar_parent.rhtml", "lib/rdoc/generator/template/darkfish/_sidebar_search.rhtml", "lib/rdoc/generator/template/darkfish/_sidebar_sections.rhtml", "lib/rdoc/generator/template/darkfish/_sidebar_table_of_contents.rhtml", "lib/rdoc/generator/template/darkfish/class.rhtml", "lib/rdoc/generator/template/darkfish/css/fonts.css", "lib/rdoc/generator/template/darkfish/css/rdoc.css", "lib/rdoc/generator/template/darkfish/fonts/Lato-Light.ttf", "lib/rdoc/generator/template/darkfish/fonts/Lato-LightItalic.ttf", "lib/rdoc/generator/template/darkfish/fonts/Lato-Regular.ttf", "lib/rdoc/generator/template/darkfish/fonts/Lato-RegularItalic.ttf", "lib/rdoc/generator/template/darkfish/fonts/SourceCodePro-Bold.ttf", "lib/rdoc/generator/template/darkfish/fonts/SourceCodePro-Regular.ttf", "lib/rdoc/generator/template/darkfish/images/add.png", "lib/rdoc/generator/template/darkfish/images/arrow_up.png", "lib/rdoc/generator/template/darkfish/images/brick.png", "lib/rdoc/generator/template/darkfish/images/brick_link.png", "lib/rdoc/generator/template/darkfish/images/bug.png", "lib/rdoc/generator/template/darkfish/images/bullet_black.png", "lib/rdoc/generator/template/darkfish/images/bullet_toggle_minus.png", "lib/rdoc/generator/template/darkfish/images/bullet_toggle_plus.png", "lib/rdoc/generator/template/darkfish/images/date.png", "lib/rdoc/generator/template/darkfish/images/delete.png", "lib/rdoc/generator/template/darkfish/images/find.png", "lib/rdoc/generator/template/darkfish/images/loadingAnimation.gif", "lib/rdoc/generator/template/darkfish/images/macFFBgHack.png", "lib/rdoc/generator/template/darkfish/images/package.png", "lib/rdoc/generator/template/darkfish/images/page_green.png", "lib/rdoc/generator/template/darkfish/images/page_white_text.png", "lib/rdoc/generator/template/darkfish/images/page_white_width.png", "lib/rdoc/generator/template/darkfish/images/plugin.png", "lib/rdoc/generator/template/darkfish/images/ruby.png", "lib/rdoc/generator/template/darkfish/images/tag_blue.png", "lib/rdoc/generator/template/darkfish/images/tag_green.png", "lib/rdoc/generator/template/darkfish/images/transparent.png", "lib/rdoc/generator/template/darkfish/images/wrench.png", "lib/rdoc/generator/template/darkfish/images/wrench_orange.png", "lib/rdoc/generator/template/darkfish/images/zoom.png", "lib/rdoc/generator/template/darkfish/index.rhtml", "lib/rdoc/generator/template/darkfish/js/darkfish.js", "lib/rdoc/generator/template/darkfish/js/jquery.js", "lib/rdoc/generator/template/darkfish/js/search.js", "lib/rdoc/generator/template/darkfish/page.rhtml", "lib/rdoc/generator/template/darkfish/servlet_not_found.rhtml", "lib/rdoc/generator/template/darkfish/servlet_root.rhtml", "lib/rdoc/generator/template/darkfish/table_of_contents.rhtml", "lib/rdoc/generator/template/json_index/.document", "lib/rdoc/generator/template/json_index/js/navigation.js", "lib/rdoc/generator/template/json_index/js/searcher.js", "lib/rdoc/ghost_method.rb", "lib/rdoc/i18n.rb", "lib/rdoc/i18n/locale.rb", "lib/rdoc/i18n/text.rb", "lib/rdoc/include.rb", "lib/rdoc/known_classes.rb", "lib/rdoc/markdown.kpeg", "lib/rdoc/markdown/entities.rb", "lib/rdoc/markdown/literals.kpeg", "lib/rdoc/markdown/literals.rb", "lib/rdoc/markup.rb", "lib/rdoc/markup/attr_changer.rb", "lib/rdoc/markup/attr_span.rb", "lib/rdoc/markup/attribute_manager.rb", "lib/rdoc/markup/attributes.rb", "lib/rdoc/markup/blank_line.rb", "lib/rdoc/markup/block_quote.rb", "lib/rdoc/markup/document.rb", "lib/rdoc/markup/formatter.rb", "lib/rdoc/markup/formatter_test_case.rb", "lib/rdoc/markup/hard_break.rb", "lib/rdoc/markup/heading.rb", "lib/rdoc/markup/include.rb", "lib/rdoc/markup/indented_paragraph.rb", "lib/rdoc/markup/inline.rb", "lib/rdoc/markup/list.rb", "lib/rdoc/markup/list_item.rb", "lib/rdoc/markup/paragraph.rb", "lib/rdoc/markup/parser.rb", "lib/rdoc/markup/pre_process.rb", "lib/rdoc/markup/raw.rb", "lib/rdoc/markup/rule.rb", "lib/rdoc/markup/special.rb", "lib/rdoc/markup/text_formatter_test_case.rb", "lib/rdoc/markup/to_ansi.rb", "lib/rdoc/markup/to_bs.rb", "lib/rdoc/markup/to_html.rb", "lib/rdoc/markup/to_html_crossref.rb", "lib/rdoc/markup/to_html_snippet.rb", "lib/rdoc/markup/to_joined_paragraph.rb", "lib/rdoc/markup/to_label.rb", "lib/rdoc/markup/to_markdown.rb", "lib/rdoc/markup/to_rdoc.rb", "lib/rdoc/markup/to_table_of_contents.rb", "lib/rdoc/markup/to_test.rb", "lib/rdoc/markup/to_tt_only.rb", "lib/rdoc/markup/verbatim.rb", "lib/rdoc/meta_method.rb", "lib/rdoc/method_attr.rb", "lib/rdoc/mixin.rb", "lib/rdoc/normal_class.rb", "lib/rdoc/normal_module.rb", "lib/rdoc/options.rb", "lib/rdoc/parser.rb", "lib/rdoc/parser/c.rb", "lib/rdoc/parser/changelog.rb", "lib/rdoc/parser/markdown.rb", "lib/rdoc/parser/rd.rb", "lib/rdoc/parser/ruby.rb", "lib/rdoc/parser/ruby_tools.rb", "lib/rdoc/parser/simple.rb", "lib/rdoc/parser/text.rb", "lib/rdoc/rd.rb", "lib/rdoc/rd/block_parser.ry", "lib/rdoc/rd/inline.rb", "lib/rdoc/rd/inline_parser.ry", "lib/rdoc/rdoc.rb", "lib/rdoc/require.rb", "lib/rdoc/ri.rb", "lib/rdoc/ri/driver.rb", "lib/rdoc/ri/formatter.rb", "lib/rdoc/ri/paths.rb", "lib/rdoc/ri/store.rb", "lib/rdoc/ri/task.rb", "lib/rdoc/ruby_lex.rb", "lib/rdoc/ruby_token.rb", "lib/rdoc/rubygems_hook.rb", "lib/rdoc/servlet.rb", "lib/rdoc/single_class.rb", "lib/rdoc/stats.rb", "lib/rdoc/stats/normal.rb", "lib/rdoc/stats/quiet.rb", "lib/rdoc/stats/verbose.rb", "lib/rdoc/store.rb", "lib/rdoc/task.rb", "lib/rdoc/test_case.rb", "lib/rdoc/text.rb", "lib/rdoc/token_stream.rb", "lib/rdoc/tom_doc.rb", "lib/rdoc/top_level.rb", "rdoc.gemspec"]
+  s.files = [".document", ".gitignore", ".travis.yml", "CONTRIBUTING.rdoc", "CVE-2013-0256.rdoc", "ExampleMarkdown.md", "ExampleRDoc.rdoc", "Gemfile", "History.rdoc", "LEGAL.rdoc", "LICENSE.rdoc", "README.rdoc", "RI.rdoc", "Rakefile", "TODO.rdoc", "appveyor.yml", "bin/console", "bin/setup", "exe/rdoc", "exe/ri", "lib/rdoc.rb", "lib/rdoc/alias.rb", "lib/rdoc/anon_class.rb", "lib/rdoc/any_method.rb", "lib/rdoc/attr.rb", "lib/rdoc/class_module.rb", "lib/rdoc/code_object.rb", "lib/rdoc/code_objects.rb", "lib/rdoc/comment.rb", "lib/rdoc/constant.rb", "lib/rdoc/context.rb", "lib/rdoc/context/section.rb", "lib/rdoc/cross_reference.rb", "lib/rdoc/encoding.rb", "lib/rdoc/erb_partial.rb", "lib/rdoc/erbio.rb", "lib/rdoc/extend.rb", "lib/rdoc/generator.rb", "lib/rdoc/generator/darkfish.rb", "lib/rdoc/generator/json_index.rb", "lib/rdoc/generator/markup.rb", "lib/rdoc/generator/pot.rb", "lib/rdoc/generator/pot/message_extractor.rb", "lib/rdoc/generator/pot/po.rb", "lib/rdoc/generator/pot/po_entry.rb", "lib/rdoc/generator/ri.rb", "lib/rdoc/generator/template/darkfish/.document", "lib/rdoc/generator/template/darkfish/_footer.rhtml", "lib/rdoc/generator/template/darkfish/_head.rhtml", "lib/rdoc/generator/template/darkfish/_sidebar_VCS_info.rhtml", "lib/rdoc/generator/template/darkfish/_sidebar_classes.rhtml", "lib/rdoc/generator/template/darkfish/_sidebar_extends.rhtml", "lib/rdoc/generator/template/darkfish/_sidebar_in_files.rhtml", "lib/rdoc/generator/template/darkfish/_sidebar_includes.rhtml", "lib/rdoc/generator/template/darkfish/_sidebar_installed.rhtml", "lib/rdoc/generator/template/darkfish/_sidebar_methods.rhtml", "lib/rdoc/generator/template/darkfish/_sidebar_navigation.rhtml", "lib/rdoc/generator/template/darkfish/_sidebar_pages.rhtml", "lib/rdoc/generator/template/darkfish/_sidebar_parent.rhtml", "lib/rdoc/generator/template/darkfish/_sidebar_search.rhtml", "lib/rdoc/generator/template/darkfish/_sidebar_sections.rhtml", "lib/rdoc/generator/template/darkfish/_sidebar_table_of_contents.rhtml", "lib/rdoc/generator/template/darkfish/class.rhtml", "lib/rdoc/generator/template/darkfish/css/fonts.css", "lib/rdoc/generator/template/darkfish/css/rdoc.css", "lib/rdoc/generator/template/darkfish/fonts/Lato-Light.ttf", "lib/rdoc/generator/template/darkfish/fonts/Lato-LightItalic.ttf", "lib/rdoc/generator/template/darkfish/fonts/Lato-Regular.ttf", "lib/rdoc/generator/template/darkfish/fonts/Lato-RegularItalic.ttf", "lib/rdoc/generator/template/darkfish/fonts/SourceCodePro-Bold.ttf", "lib/rdoc/generator/template/darkfish/fonts/SourceCodePro-Regular.ttf", "lib/rdoc/generator/template/darkfish/images/add.png", "lib/rdoc/generator/template/darkfish/images/arrow_up.png", "lib/rdoc/generator/template/darkfish/images/brick.png", "lib/rdoc/generator/template/darkfish/images/brick_link.png", "lib/rdoc/generator/template/darkfish/images/bug.png", "lib/rdoc/generator/template/darkfish/images/bullet_black.png", "lib/rdoc/generator/template/darkfish/images/bullet_toggle_minus.png", "lib/rdoc/generator/template/darkfish/images/bullet_toggle_plus.png", "lib/rdoc/generator/template/darkfish/images/date.png", "lib/rdoc/generator/template/darkfish/images/delete.png", "lib/rdoc/generator/template/darkfish/images/find.png", "lib/rdoc/generator/template/darkfish/images/loadingAnimation.gif", "lib/rdoc/generator/template/darkfish/images/macFFBgHack.png", "lib/rdoc/generator/template/darkfish/images/package.png", "lib/rdoc/generator/template/darkfish/images/page_green.png", "lib/rdoc/generator/template/darkfish/images/page_white_text.png", "lib/rdoc/generator/template/darkfish/images/page_white_width.png", "lib/rdoc/generator/template/darkfish/images/plugin.png", "lib/rdoc/generator/template/darkfish/images/ruby.png", "lib/rdoc/generator/template/darkfish/images/tag_blue.png", "lib/rdoc/generator/template/darkfish/images/tag_green.png", "lib/rdoc/generator/template/darkfish/images/transparent.png", "lib/rdoc/generator/template/darkfish/images/wrench.png", "lib/rdoc/generator/template/darkfish/images/wrench_orange.png", "lib/rdoc/generator/template/darkfish/images/zoom.png", "lib/rdoc/generator/template/darkfish/index.rhtml", "lib/rdoc/generator/template/darkfish/js/darkfish.js", "lib/rdoc/generator/template/darkfish/js/jquery.js", "lib/rdoc/generator/template/darkfish/js/search.js", "lib/rdoc/generator/template/darkfish/page.rhtml", "lib/rdoc/generator/template/darkfish/servlet_not_found.rhtml", "lib/rdoc/generator/template/darkfish/servlet_root.rhtml", "lib/rdoc/generator/template/darkfish/table_of_contents.rhtml", "lib/rdoc/generator/template/json_index/.document", "lib/rdoc/generator/template/json_index/js/navigation.js", "lib/rdoc/generator/template/json_index/js/searcher.js", "lib/rdoc/ghost_method.rb", "lib/rdoc/i18n.rb", "lib/rdoc/i18n/locale.rb", "lib/rdoc/i18n/text.rb", "lib/rdoc/include.rb", "lib/rdoc/known_classes.rb", "lib/rdoc/markdown.kpeg", "lib/rdoc/markdown/entities.rb", "lib/rdoc/markdown/literals.kpeg", "lib/rdoc/markdown/literals.rb", "lib/rdoc/markup.rb", "lib/rdoc/markup/attr_changer.rb", "lib/rdoc/markup/attr_span.rb", "lib/rdoc/markup/attribute_manager.rb", "lib/rdoc/markup/attributes.rb", "lib/rdoc/markup/blank_line.rb", "lib/rdoc/markup/block_quote.rb", "lib/rdoc/markup/document.rb", "lib/rdoc/markup/formatter.rb", "lib/rdoc/markup/formatter_test_case.rb", "lib/rdoc/markup/hard_break.rb", "lib/rdoc/markup/heading.rb", "lib/rdoc/markup/include.rb", "lib/rdoc/markup/indented_paragraph.rb", "lib/rdoc/markup/inline.rb", "lib/rdoc/markup/list.rb", "lib/rdoc/markup/list_item.rb", "lib/rdoc/markup/paragraph.rb", "lib/rdoc/markup/parser.rb", "lib/rdoc/markup/pre_process.rb", "lib/rdoc/markup/raw.rb", "lib/rdoc/markup/rule.rb", "lib/rdoc/markup/special.rb", "lib/rdoc/markup/text_formatter_test_case.rb", "lib/rdoc/markup/to_ansi.rb", "lib/rdoc/markup/to_bs.rb", "lib/rdoc/markup/to_html.rb", "lib/rdoc/markup/to_html_crossref.rb", "lib/rdoc/markup/to_html_snippet.rb", "lib/rdoc/markup/to_joined_paragraph.rb", "lib/rdoc/markup/to_label.rb", "lib/rdoc/markup/to_markdown.rb", "lib/rdoc/markup/to_rdoc.rb", "lib/rdoc/markup/to_table_of_contents.rb", "lib/rdoc/markup/to_test.rb", "lib/rdoc/markup/to_tt_only.rb", "lib/rdoc/markup/verbatim.rb", "lib/rdoc/meta_method.rb", "lib/rdoc/method_attr.rb", "lib/rdoc/mixin.rb", "lib/rdoc/normal_class.rb", "lib/rdoc/normal_module.rb", "lib/rdoc/options.rb", "lib/rdoc/parser.rb", "lib/rdoc/parser/c.rb", "lib/rdoc/parser/changelog.rb", "lib/rdoc/parser/markdown.rb", "lib/rdoc/parser/rd.rb", "lib/rdoc/parser/ripper_state_lex.rb", "lib/rdoc/parser/ruby.rb", "lib/rdoc/parser/ruby_tools.rb", "lib/rdoc/parser/simple.rb", "lib/rdoc/parser/text.rb", "lib/rdoc/rd.rb", "lib/rdoc/rd/block_parser.ry", "lib/rdoc/rd/inline.rb", "lib/rdoc/rd/inline_parser.ry", "lib/rdoc/rdoc.rb", "lib/rdoc/require.rb", "lib/rdoc/ri.rb", "lib/rdoc/ri/driver.rb", "lib/rdoc/ri/formatter.rb", "lib/rdoc/ri/paths.rb", "lib/rdoc/ri/store.rb", "lib/rdoc/ri/task.rb", "lib/rdoc/ruby_token.rb", "lib/rdoc/rubygems_hook.rb", "lib/rdoc/servlet.rb", "lib/rdoc/single_class.rb", "lib/rdoc/stats.rb", "lib/rdoc/stats/normal.rb", "lib/rdoc/stats/quiet.rb", "lib/rdoc/stats/verbose.rb", "lib/rdoc/store.rb", "lib/rdoc/task.rb", "lib/rdoc/test_case.rb", "lib/rdoc/text.rb", "lib/rdoc/token_stream.rb", "lib/rdoc/tom_doc.rb", "lib/rdoc/top_level.rb", "rdoc.gemspec"]
   # files from .gitignore
   s.files << "lib/rdoc/rd/block_parser.rb" << "lib/rdoc/rd/inline_parser.rb" << "lib/rdoc/markdown.rb"
 
@@ -54,4 +53,5 @@ RDoc includes the +rdoc+ and +ri+ tools for generating and displaying documentat
   s.add_development_dependency("racc", "> 1.4.10")
   s.add_development_dependency("kpeg")
   s.add_development_dependency("minitest", "~> 4")
+  s.add_development_dependency("json")
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: rdoc
 version: !ruby/object:Gem::Version
-  version: 6.0.0.beta1
+  version: 6.0.0.beta2
 platform: ruby
 authors:
 - Eric Hodel
@@ -13,7 +13,7 @@ authors:
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2017-08-29 00:00:00.000000000 Z
+date: 2017-09-12 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rake
@@ -71,6 +71,20 @@ dependencies:
     - - "~>"
       - !ruby/object:Gem::Version
         version: '4'
+- !ruby/object:Gem::Dependency
+  name: json
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
 description: |
   RDoc produces HTML and command-line documentation for Ruby projects.
   RDoc includes the +rdoc+ and +ri+ tools for generating and displaying documentation from the command-line.
@@ -112,11 +126,11 @@ files:
 - RI.rdoc
 - Rakefile
 - TODO.rdoc
+- appveyor.yml
 - bin/console
 - bin/setup
 - exe/rdoc
 - exe/ri
-- lib/gauntlet_rdoc.rb
 - lib/rdoc.rb
 - lib/rdoc/alias.rb
 - lib/rdoc/anon_class.rb
@@ -263,6 +277,7 @@ files:
 - lib/rdoc/parser/changelog.rb
 - lib/rdoc/parser/markdown.rb
 - lib/rdoc/parser/rd.rb
+- lib/rdoc/parser/ripper_state_lex.rb
 - lib/rdoc/parser/ruby.rb
 - lib/rdoc/parser/ruby_tools.rb
 - lib/rdoc/parser/simple.rb
@@ -281,7 +296,6 @@ files:
 - lib/rdoc/ri/paths.rb
 - lib/rdoc/ri/store.rb
 - lib/rdoc/ri/task.rb
-- lib/rdoc/ruby_lex.rb
 - lib/rdoc/ruby_token.rb
 - lib/rdoc/rubygems_hook.rb
 - lib/rdoc/servlet.rb

data/lib/gauntlet_rdoc.rb

@@ -1,82 +0,0 @@
-Gem.load_yaml
-require 'rdoc'
-require 'gauntlet'
-require 'fileutils'
-
-##
-# Allows for testing of RDoc against every gem
-
-class RDoc::Gauntlet < Gauntlet
-
-  def initialize # :nodoc:
-    super
-
-    @args = nil
-    @type = nil
-  end
-
-  ##
-  # Runs an RDoc generator for gem +name+
-
-  def run name
-    return if self.data.key? name
-
-    dir = File.expand_path "~/.gauntlet/data/#{@type}/#{name}"
-    FileUtils.rm_rf dir if File.exist? dir
-
-    yaml = File.read 'gemspec'
-    begin
-      spec = Gem::Specification.from_yaml yaml
-    rescue Psych::SyntaxError
-      puts "bad spec #{name}"
-      self.data[name] = false
-      return
-    end
-
-    args = @args.dup
-    args << '--op' << dir
-    args.concat spec.rdoc_options
-    args << spec.require_paths
-    args << spec.extra_rdoc_files
-    args = args.flatten.map { |a| a.to_s }
-    args.delete '--quiet'
-
-    puts "#{name} - rdoc #{args.join ' '}"
-
-    self.dirty = true
-    r = RDoc::RDoc.new
-
-    begin
-      r.document args
-      self.data[name] = true
-      puts 'passed'
-      FileUtils.rm_rf dir
-    rescue Interrupt, StandardError, RDoc::Error, SystemStackError => e
-      puts "failed - (#{e.class}) #{e.message}"
-      self.data[name] = false
-    end
-  rescue Gem::Exception
-    puts "bad gem #{name}"
-  ensure
-    puts
-  end
-
-  ##
-  # Runs the gauntlet with the given +type+ (rdoc or ri) and +filter+ for
-  # which gems to run
-
-  def run_the_gauntlet type = 'rdoc', filter = nil
-    @type = type || 'rdoc'
-    @args = type == 'rdoc' ? [] : %w[--ri]
-    @data_file = "#{DATADIR}/#{@type}-data.yml"
-
-    super filter
-  end
-
-end
-
-type = ARGV.shift
-filter = ARGV.shift
-filter = /#{filter}/ if filter
-
-RDoc::Gauntlet.new.run_the_gauntlet type, filter

data/lib/rdoc/ruby_lex.rb

@@ -1,1521 +0,0 @@
-# coding: US-ASCII
-# frozen_string_literal: false
-
-#--
-#   irb/ruby-lex.rb - ruby lexcal analyzer
-#   	$Release Version: 0.9.5$
-#   	$Revision: 17979 $
-#   	$Date: 2008-07-09 10:17:05 -0700 (Wed, 09 Jul 2008) $
-#   	by Keiju ISHITSUKA(keiju@ruby-lang.org)
-#
-#++
-
-require "e2mmap"
-require "irb/slex"
-require "stringio"
-
-##
-# Ruby lexer adapted from irb.
-#
-# The internals are not documented because they are scary.
-
-class RDoc::RubyLex
-
-  ##
-  # Raised upon invalid input
-
-  class Error < RDoc::Error
-  end
-
-  # :stopdoc:
-
-  extend Exception2MessageMapper
-
-  def_exception(:AlreadyDefinedToken, "Already defined token(%s)")
-  def_exception(:TkReading2TokenNoKey, "key nothing(key='%s')")
-  def_exception(:TkSymbol2TokenNoKey, "key nothing(key='%s')")
-  def_exception(:TkReading2TokenDuplicateError,
-                "key duplicate(token_n='%s', key='%s')")
-  def_exception(:SyntaxError, "%s")
-
-  def_exception(:TerminateLineInput, "Terminate Line Input")
-
-  include RDoc::RubyToken
-  include IRB
-
-  attr_accessor :continue
-  attr_accessor :lex_state
-  attr_accessor :first_in_method_statement
-  attr_reader :reader
-
-  class << self
-    attr_accessor :debug_level
-  end
-
-  def self.debug?
-    @debug_level > 0
-  end
-
-  self.debug_level = 0
-
-  # :startdoc:
-
-  ##
-  # Returns an Array of +ruby+ tokens.  See ::new for a description of
-  # +options+.
-
-  def self.tokenize ruby, options
-    tokens = []
-
-    scanner = RDoc::RubyLex.new ruby, options
-    scanner.exception_on_syntax_error = true
-
-    while token = scanner.token do
-      tokens << token
-    end
-
-    tokens
-  end
-
-  ##
-  # Creates a new lexer for +content+.  +options+ is an RDoc::Options, only
-  # +tab_width is used.
-
-  def initialize(content, options)
-    lex_init
-
-    if /\t/ =~ content then
-      tab_width = options.tab_width
-      content = content.split(/\n/).map do |line|
-        1 while line.gsub!(/\t+/) {
-          ' ' * (tab_width*$&.length - $`.length % tab_width)
-        }  && $~
-        line
-      end.join("\n")
-    end
-
-    content << "\n" unless content[-1, 1] == "\n"
-
-    set_input StringIO.new content
-
-    @base_char_no = 0
-    @char_no = 0
-    @exp_line_no = @line_no = 1
-    @here_readed = []
-    @readed = []
-    @current_readed = @readed
-    @rests = []
-    @seek = 0
-
-    @heredoc_queue = []
-
-    @indent = 0
-    @indent_stack = []
-    @lex_state = :EXPR_BEG
-    @space_seen = false
-    @escaped_nl = false
-    @first_in_method_statement = false
-    @after_question = false
-
-    @continue = false
-    @line = ""
-
-    @skip_space = false
-    @readed_auto_clean_up = false
-    @exception_on_syntax_error = true
-
-    @prompt = nil
-    @prev_seek = nil
-    @ltype = nil
-  end
-
-  # :stopdoc:
-
-  def inspect # :nodoc:
-    "#<%s:0x%x pos %d lex_state %p space_seen %p>" % [
-      self.class, object_id,
-      @io.pos, @lex_state, @space_seen,
-    ]
-  end
-
-  attr_accessor :skip_space
-  attr_accessor :readed_auto_clean_up
-  attr_accessor :exception_on_syntax_error
-
-  attr_reader :seek
-  attr_reader :char_no
-  attr_reader :line_no
-  attr_reader :indent
-
-  # io functions
-  def set_input(io, p = nil, &block)
-    @io = io
-    if p.respond_to?(:call)
-      @input = p
-    elsif block_given?
-      @input = block
-    else
-      @input = Proc.new{@io.gets}
-    end
-  end
-
-  def get_readed
-    if idx = @readed.rindex("\n")
-      @base_char_no = @readed.size - (idx + 1)
-    else
-      @base_char_no += @readed.size
-    end
-
-    readed = @readed.join("")
-    @readed.clear
-    readed
-  end
-
-  def getc
-    while @rests.empty?
-      #      return nil unless buf_input
-      @rests.push nil unless buf_input
-    end
-    c = @rests.shift
-    @current_readed.push c
-    @seek += 1
-    if c == "\n".freeze
-      @line_no += 1
-      @char_no = 0
-    else
-      @char_no += 1
-    end
-
-    c
-  end
-
-  def gets
-    l = ""
-    while c = getc
-      l.concat(c)
-      break if c == "\n"
-    end
-    return nil if l == "" and c.nil?
-    l
-  end
-
-  def eof?
-    @io.eof?
-  end
-
-  def getc_of_rests
-    if @rests.empty?
-      nil
-    else
-      getc
-    end
-  end
-
-  def ungetc(c = nil)
-    if @here_readed.empty?
-      c2 = @readed.pop
-    else
-      c2 = @here_readed.pop
-    end
-    c = c2 unless c
-    @rests.unshift c #c =
-    @seek -= 1
-    if c == "\n"
-      @line_no -= 1
-      if idx = @readed.rindex("\n")
-        @char_no = idx + 1
-      else
-        @char_no = @base_char_no + @readed.size
-      end
-    else
-      @char_no -= 1
-    end
-  end
-
-  def peek_equal?(str)
-    chrs = str.split(//)
-    until @rests.size >= chrs.size
-      return false unless buf_input
-    end
-    @rests[0, chrs.size] == chrs
-  end
-
-  def peek_match?(regexp)
-    while @rests.empty?
-      return false unless buf_input
-    end
-    regexp =~ @rests.join("")
-  end
-
-  def peek(i = 0)
-    while @rests.size <= i
-      return nil unless buf_input
-    end
-    @rests[i]
-  end
-
-  def buf_input
-    prompt
-    line = @input.call
-    return nil unless line
-    @rests.concat line.split(//)
-    true
-  end
-  private :buf_input
-
-  def set_prompt(p = nil, &block)
-    p = block if block_given?
-    if p.respond_to?(:call)
-      @prompt = p
-    else
-      @prompt = Proc.new{print p}
-    end
-  end
-
-  def prompt
-    if @prompt
-      @prompt.call(@ltype, @indent, @continue, @line_no)
-    end
-  end
-
-  def initialize_input
-    @ltype = nil
-    @quoted = nil
-    @indent = 0
-    @indent_stack = []
-    @lex_state = :EXPR_BEG
-    @space_seen = false
-    @current_readed = @readed
-
-    @continue = false
-    prompt
-
-    @line = ""
-    @exp_line_no = @line_no
-  end
-
-  def each_top_level_statement
-    initialize_input
-    catch(:TERM_INPUT) do
-      loop do
-        begin
-          @continue = false
-          prompt
-          unless l = lex
-            throw :TERM_INPUT if @line == ''
-          else
-            #p l
-            @line.concat l
-            if @ltype or @continue or @indent > 0
-              next
-            end
-          end
-          if @line != "\n"
-            yield @line, @exp_line_no
-          end
-          break unless l
-          @line = ''
-          @exp_line_no = @line_no
-
-          @indent = 0
-          @indent_stack = []
-          prompt
-        rescue TerminateLineInput
-          initialize_input
-          prompt
-          get_readed
-        end
-      end
-    end
-  end
-
-  def lex
-    until (((tk = token).kind_of?(TkNL) || tk.kind_of?(TkEND_OF_SCRIPT)) &&
-           !@continue or
-      tk.nil?)
-      #p tk
-      #p @lex_state
-      #p self
-    end
-    line = get_readed
-    #      print self.inspect
-    if line == "" and tk.kind_of?(TkEND_OF_SCRIPT) || tk.nil?
-      nil
-    else
-      line
-    end
-  end
-
-  def token
-    #      require "tracer"
-    #      Tracer.on
-    @prev_seek = @seek
-    @prev_line_no = @line_no
-    @prev_char_no = @char_no
-    begin
-      begin
-        tk = @OP.match(self)
-        @space_seen = tk.kind_of?(TkSPACE)
-        @first_in_method_statement = false if !@space_seen && @first_in_method_statement
-      rescue SyntaxError => e
-        raise Error, "syntax error: #{e.message}" if
-          @exception_on_syntax_error
-
-        tk = TkError.new(@seek, @line_no, @char_no)
-      end
-    end while @skip_space and tk.kind_of?(TkSPACE)
-
-    if @readed_auto_clean_up
-      get_readed
-    end
-
-    if TkSYMBEG === tk then
-      tk1 = token
-      set_token_position tk.seek, tk.line_no, tk.char_no
-
-      case tk1
-      when TkId, TkOp, TkSTRING, TkDSTRING, TkSTAR, TkAMPER then
-        if tk1.respond_to?(:name) then
-          tk = Token(TkSYMBOL, ":" + tk1.name)
-        else
-          tk = Token(TkSYMBOL, ":" + tk1.text)
-        end
-      else
-        tk = tk1
-      end
-    elsif (TkPLUS === tk or TkMINUS === tk) and peek(0) =~ /\d/ then
-      tk1 = token
-      set_token_position tk.seek, tk.line_no, tk.char_no
-      tk = Token(tk1.class, tk.text + tk1.text)
-    end
-    @after_question = false if @after_question and !(TkQUESTION === tk)
-
-    #      Tracer.off
-    tk
-  end
-
-  ENINDENT_CLAUSE = [
-    "case", "class", "def", "do", "for", "if",
-    "module", "unless", "until", "while", "begin" #, "when"
-  ]
-
-  DEINDENT_CLAUSE = ["end" #, "when"
-  ]
-
-  PERCENT_LTYPE = {
-    "q" => "\'",
-    "Q" => "\"",
-    "x" => "\`",
-    "r" => "/",
-    "w" => "]",
-    "W" => "]",
-    "s" => ":",
-    "i" => "]",
-    "I" => "]"
-  }
-
-  PERCENT_PAREN = {
-    "{" => "}",
-    "[" => "]",
-    "<" => ">",
-    "(" => ")"
-  }
-
-  PERCENT_PAREN_REV = PERCENT_PAREN.invert
-
-  Ltype2Token = {
-    "\'" => TkSTRING,
-    "\"" => TkSTRING,
-    "\`" => TkXSTRING,
-    "/" => TkREGEXP,
-    "]" => TkDSTRING,
-    ":" => TkSYMBOL
-  }
-  DLtype2Token = {
-    "\"" => TkDSTRING,
-    "\`" => TkDXSTRING,
-    "/" => TkDREGEXP,
-  }
-
-  def lex_init()
-    @OP = IRB::SLex.new
-    @OP.def_rules("\0", "\004", "\032") do |op, io|
-      Token(TkEND_OF_SCRIPT, '')
-    end
-
-    @OP.def_rules(" ", "\t", "\f", "\r", "\13") do |op, io|
-      @space_seen = true
-      str = op
-      while (ch = getc) =~ /[ \t\f\r\13]/ do
-        str << ch
-      end
-      ungetc
-      Token TkSPACE, str
-    end
-
-    @OP.def_rule("#") do |op, io|
-      identify_comment
-    end
-
-    @OP.def_rule("=begin",
-                 proc{|op, io| @prev_char_no == 0 && peek(0) =~ /\s/}) do
-      |op, io|
-      @ltype = "="
-      res = op
-      until (ch = getc) == "\n" do
-        res << ch
-      end
-      res << ch
-
-      until ( peek_equal?("=end") && peek(4) =~ /\s/ ) do
-        (ch = getc)
-        res << ch
-      end
-
-      res << gets # consume =end
-
-      @ltype = nil
-      Token(TkRD_COMMENT, res)
-    end
-
-    @OP.def_rule("\n") do |op, io|
-      print "\\n\n" if RDoc::RubyLex.debug?
-      unless @heredoc_queue.empty?
-        info = @heredoc_queue[0]
-        if !info[:started] # "\n"
-          info[:started] = true
-          ungetc "\n"
-        elsif info[:heredoc_end].nil? # heredoc body
-          tk, heredoc_end = identify_here_document_body(info[:quoted], info[:lt], info[:indent])
-          info[:heredoc_end] = heredoc_end
-          ungetc "\n"
-        else # heredoc end
-          @heredoc_queue.shift
-          @lex_state = :EXPR_BEG
-          tk = Token(TkHEREDOCEND, info[:heredoc_end])
-          if !@heredoc_queue.empty?
-            @heredoc_queue[0][:started] = true
-            ungetc "\n"
-          end
-        end
-      end
-      unless tk
-        case @lex_state
-        when :EXPR_BEG, :EXPR_FNAME, :EXPR_DOT
-          @continue = true
-        else
-          @continue = false
-          @lex_state = :EXPR_BEG unless @escaped_nl
-          until (@indent_stack.empty? ||
-                 [TkLPAREN, TkLBRACK, TkLBRACE,
-                   TkfLPAREN, TkfLBRACK, TkfLBRACE].include?(@indent_stack.last))
-            @indent_stack.pop
-          end
-        end
-        @current_readed = @readed
-        @here_readed.clear
-        tk = Token(TkNL)
-      end
-      @escaped_nl = false
-      tk
-    end
-
-    @OP.def_rules("=") do
-      |op, io|
-      case @lex_state
-      when :EXPR_FNAME, :EXPR_DOT
-        @lex_state = :EXPR_ARG
-      else
-        @lex_state = :EXPR_BEG
-      end
-      Token(op)
-    end
-
-    @OP.def_rules("*", "**",
-                  "==", "===",
-                  "=~", "<=>",
-                  "<", "<=",
-                  ">", ">=", ">>", "=>") do
-      |op, io|
-      case @lex_state
-      when :EXPR_FNAME, :EXPR_DOT
-        tk = Token(TkId, op)
-        @lex_state = :EXPR_ARG
-      else
-        tk = Token(op)
-        @lex_state = :EXPR_BEG
-      end
-      tk
-    end
-
-    @OP.def_rules("->") do
-      |op, io|
-      @lex_state = :EXPR_ENDFN
-      Token(op)
-    end
-
-    @OP.def_rules("!", "!=", "!~") do
-      |op, io|
-      case @lex_state
-      when :EXPR_FNAME, :EXPR_DOT
-        @lex_state = :EXPR_ARG
-        Token(TkId, op)
-      else
-        @lex_state = :EXPR_BEG
-        Token(op)
-      end
-    end
-
-    @OP.def_rules("<<") do
-      |op, io|
-      tk = nil
-      if @lex_state != :EXPR_END && @lex_state != :EXPR_CLASS &&
-         (@lex_state != :EXPR_ARG || @space_seen)
-        c = peek(0)
-        if /\S/ =~ c && (/["'`]/ =~ c || /\w/ =~ c || c == "-" || c == "~")
-          tk = identify_here_document(op)
-        end
-      end
-      unless tk
-        case @lex_state
-        when :EXPR_FNAME, :EXPR_DOT
-          tk = Token(TkId, op)
-          @lex_state = :EXPR_ARG
-        else
-          tk = Token(op)
-          @lex_state = :EXPR_BEG
-        end
-      end
-      tk
-    end
-
-    @OP.def_rules("'", '"') do
-      |op, io|
-      identify_string(op)
-    end
-
-    @OP.def_rules("`") do
-      |op, io|
-      if :EXPR_FNAME == @lex_state or :EXPR_DOT == @lex_state
-        @lex_state = :EXPR_ARG
-        Token(TkId, op)
-      else
-        identify_string(op)
-      end
-    end
-
-    @OP.def_rules('?') do
-      |op, io|
-      if @lex_state == :EXPR_END
-        @lex_state = :EXPR_BEG
-        @after_question = true
-        Token(TkQUESTION)
-      else
-        ch = getc
-        if @lex_state == :EXPR_ARG && ch =~ /\s/
-          ungetc
-          @lex_state = :EXPR_BEG;
-          Token(TkQUESTION)
-        else
-          @lex_state = :EXPR_END
-          ch << getc if "\\" == ch
-          Token(TkCHAR, "?#{ch}")
-        end
-      end
-    end
-
-    @OP.def_rules("&&", "||") do
-      |op, io|
-      @lex_state = :EXPR_BEG
-      Token(op)
-    end
-
-    @OP.def_rules("&", "|") do
-      |op, io|
-      case @lex_state
-      when :EXPR_FNAME, :EXPR_DOT
-        tk = Token(TkId, op)
-        @lex_state = :EXPR_ARG
-      else
-        tk = Token(op)
-        @lex_state = :EXPR_BEG
-      end
-      tk
-    end
-
-    @OP.def_rules("+=", "-=", "*=", "**=",
-                  "&=", "|=", "^=", "<<=", ">>=", "||=", "&&=") do
-      |op, io|
-      @lex_state = :EXPR_BEG
-      op =~ /^(.*)=$/
-      Token(TkOPASGN, $1)
-    end
-
-    @OP.def_rule("+@", proc{|op, io| @lex_state == :EXPR_FNAME}) do
-      |op, io|
-      @lex_state = :EXPR_ARG
-      Token(TkId, op)
-    end
-
-    @OP.def_rule("-@", proc{|op, io| @lex_state == :EXPR_FNAME}) do
-      |op, io|
-      @lex_state = :EXPR_ARG
-      Token(TkId, op)
-    end
-
-    @OP.def_rules("+", "-") do
-      |op, io|
-      catch(:RET) do
-        if :EXPR_FNAME == @lex_state or :EXPR_DOT == @lex_state
-          tk = Token(TkId, op)
-          @lex_state = :EXPR_ARG
-        elsif @lex_state == :EXPR_ARG
-          if @space_seen and peek(0) =~ /[0-9]/
-            throw :RET, identify_number(op)
-          else
-            @lex_state = :EXPR_BEG
-          end
-        elsif @lex_state != :EXPR_END and peek(0) =~ /[0-9]/
-          throw :RET, identify_number(op)
-        else
-          @lex_state = :EXPR_BEG
-        end
-        tk = Token(op) unless tk
-        tk
-      end
-    end
-
-    @OP.def_rules(".", "&.") do
-      |op, io|
-      @lex_state = :EXPR_BEG
-      if peek(0) =~ /[0-9]/
-        ungetc
-        identify_number
-      else
-        # for "obj.if" or "obj&.if" etc.
-        @lex_state = :EXPR_DOT
-        Token(op)
-      end
-    end
-
-    @OP.def_rules("..", "...") do
-      |op, io|
-      @lex_state = :EXPR_BEG
-      Token(op)
-    end
-
-    lex_int2
-  end
-
-  def lex_int2
-    @OP.def_rules("]", "}", ")") do
-      |op, io|
-      @lex_state = :EXPR_END
-      @indent -= 1
-      @indent_stack.pop
-      Token(op)
-    end
-
-    @OP.def_rule(":") do
-      |op, io|
-      if @lex_state == :EXPR_END || peek(0) =~ /\s/
-        @lex_state = :EXPR_BEG
-        Token(TkCOLON)
-      else
-        @lex_state = :EXPR_FNAME;
-        Token(TkSYMBEG)
-      end
-    end
-
-    @OP.def_rule("::") do
-      |op, io|
-      #      p @lex_state.id2name, @space_seen
-      if @lex_state == :EXPR_BEG or @lex_state == :EXPR_ARG && @space_seen
-        @lex_state = :EXPR_BEG
-        Token(TkCOLON3)
-      else
-        @lex_state = :EXPR_DOT
-        Token(TkCOLON2)
-      end
-    end
-
-    @OP.def_rule("/") do
-      |op, io|
-      if :EXPR_FNAME == @lex_state or :EXPR_DOT == @lex_state
-        @lex_state = :EXPR_ARG
-        Token(TkId, op)
-      elsif @lex_state == :EXPR_BEG || @lex_state == :EXPR_MID || @first_in_method_statement
-        identify_string(op)
-      elsif peek(0) == '='
-        getc
-        @lex_state = :EXPR_BEG
-        Token(TkOPASGN, "/") #/)
-      elsif @lex_state == :EXPR_ARG and @space_seen and peek(0) !~ /\s/
-        identify_string(op)
-      else
-        @lex_state = :EXPR_BEG
-        Token("/") #/)
-      end
-    end
-
-    @OP.def_rules("^") do
-      |op, io|
-      case @lex_state
-      when :EXPR_FNAME, :EXPR_DOT
-        tk = Token(TkId, op)
-        @lex_state = :EXPR_ARG
-      else
-        tk = Token(op)
-        @lex_state = :EXPR_BEG
-      end
-      tk
-    end
-
-    #       @OP.def_rules("^=") do
-    # 	@lex_state = :EXPR_BEG
-    # 	Token(OP_ASGN, :^)
-    #       end
-
-    @OP.def_rules(",") do
-      |op, io|
-      @lex_state = :EXPR_BEG
-      Token(op)
-    end
-
-    @OP.def_rules(";") do
-      |op, io|
-      @lex_state = :EXPR_BEG
-      until (@indent_stack.empty? ||
-             [TkLPAREN, TkLBRACK, TkLBRACE,
-               TkfLPAREN, TkfLBRACK, TkfLBRACE].include?(@indent_stack.last))
-        @indent_stack.pop
-      end
-      Token(op)
-    end
-
-    @OP.def_rule("~") do
-      |op, io|
-      case @lex_state
-      when :EXPR_FNAME, :EXPR_DOT
-        @lex_state = :EXPR_ARG
-        Token(TkId, op)
-      else
-        @lex_state = :EXPR_BEG
-        Token(op)
-      end
-    end
-
-    @OP.def_rule("~@", proc{|op, io| @lex_state == :EXPR_FNAME}) do
-      |op, io|
-      @lex_state = :EXPR_BEG
-      Token("~")
-    end
-
-    @OP.def_rule("(") do
-      |op, io|
-      @indent += 1
-      if @lex_state == :EXPR_BEG || @lex_state == :EXPR_MID
-        @lex_state = :EXPR_BEG
-        tk_c = TkfLPAREN
-      else
-        @lex_state = :EXPR_BEG
-        tk_c = TkLPAREN
-      end
-      @indent_stack.push tk_c
-      Token tk_c
-    end
-
-    @OP.def_rule("[]", proc{|op, io| @lex_state == :EXPR_FNAME}) do
-      |op, io|
-      @lex_state = :EXPR_ARG
-      Token(TkId, op)
-    end
-
-    @OP.def_rule("[]=", proc{|op, io| @lex_state == :EXPR_FNAME}) do
-      |op, io|
-      @lex_state = :EXPR_ARG
-      Token(TkId, op)
-    end
-
-    @OP.def_rule("[") do
-      |op, io|
-      text = nil
-      @indent += 1
-      if @lex_state == :EXPR_FNAME
-        tk_c = TkfLBRACK
-      else
-        if @lex_state == :EXPR_BEG || @lex_state == :EXPR_MID
-          tk_c = TkLBRACK
-        elsif @lex_state == :EXPR_ARG && @space_seen
-          tk_c = TkLBRACK
-        elsif @lex_state == :EXPR_DOT
-          if peek(0) == "]"
-            tk_c = TkIDENTIFIER
-            getc
-            if peek(0) == "="
-              text = "[]="
-            else
-              text = "[]"
-            end
-          else
-            tk_c = TkOp
-          end
-        else
-          tk_c = TkfLBRACK
-        end
-        @lex_state = :EXPR_BEG
-      end
-      @indent_stack.push tk_c
-      Token(tk_c, text)
-    end
-
-    @OP.def_rule("{") do
-      |op, io|
-      @indent += 1
-      if @lex_state != :EXPR_END && @lex_state != :EXPR_ARG
-        tk_c = TkLBRACE
-      else
-        tk_c = TkfLBRACE
-      end
-      @lex_state = :EXPR_BEG
-      @indent_stack.push tk_c
-      Token(tk_c)
-    end
-
-    @OP.def_rule('\\') do
-      |op, io|
-      if peek(0) == "\n"
-        @space_seen = true
-        @continue = true
-        @escaped_nl = true
-      end
-      Token("\\")
-    end
-
-    @OP.def_rule('%') do
-      |op, io|
-      if :EXPR_FNAME == @lex_state or :EXPR_DOT == @lex_state
-        @lex_state = :EXPR_ARG
-        Token(TkId, op)
-      elsif @lex_state == :EXPR_BEG || @lex_state == :EXPR_MID
-        identify_quotation
-      elsif peek(0) == '='
-        getc
-        @lex_state = :EXPR_BEG
-        Token(TkOPASGN, '%')
-      elsif @lex_state == :EXPR_ARG and @space_seen and peek(0) !~ /\s/
-        identify_quotation
-      else
-        @lex_state = :EXPR_BEG
-        Token("%") #))
-      end
-    end
-
-    @OP.def_rule('$') do
-      |op, io|
-      identify_gvar
-    end
-
-    @OP.def_rule('@') do
-      |op, io|
-      if peek(0) =~ /[\w@]/
-        ungetc
-        identify_identifier
-      else
-        Token("@")
-      end
-    end
-
-    #       @OP.def_rule("def", proc{|op, io| /\s/ =~ io.peek(0)}) do
-    # 	|op, io|
-    # 	@indent += 1
-    # 	@lex_state = :EXPR_FNAME
-    # #	@lex_state = :EXPR_END
-    # #	until @rests[0] == "\n" or @rests[0] == ";"
-    # #	  rests.shift
-    # #	end
-    #       end
-
-    @OP.def_rule("_") do
-      if peek_match?(/_END__/) and @lex_state == :EXPR_BEG then
-        6.times { getc }
-        Token(TkEND_OF_SCRIPT, '__END__')
-      else
-        ungetc
-        identify_identifier
-      end
-    end
-
-    @OP.def_rule("") do
-      |op, io|
-      printf "MATCH: start %s: %s\n", op, io.inspect if RDoc::RubyLex.debug?
-      if peek(0) =~ /[0-9]/
-        t = identify_number
-      else
-        t = identify_identifier
-      end
-      printf "MATCH: end %s: %s\n", op, io.inspect if RDoc::RubyLex.debug?
-      t
-    end
-
-    p @OP if RDoc::RubyLex.debug?
-  end
-
-  def identify_gvar
-    @lex_state = :EXPR_END
-
-    case ch = getc
-    when /[~_*$?!@\/\\;,=:<>".]/   #"
-      Token(TkGVAR, "$" + ch)
-    when "-"
-      Token(TkGVAR, "$-" + getc)
-    when "&", "`", "'", "+"
-      Token(TkBACK_REF, "$"+ch)
-    when /[1-9]/
-      ref = ch
-      while (ch = getc) =~ /[0-9]/ do ref << ch end
-      ungetc
-      Token(TkNTH_REF, "$#{ref}")
-    when /\w/
-      ungetc
-      ungetc
-      identify_identifier
-    else
-      ungetc
-      Token("$")
-    end
-  end
-
-  IDENT_RE = eval '/[\w\u{0080}-\u{FFFFF}]/u'
-
-  def identify_identifier
-    token = ""
-    if peek(0) =~ /[$@]/
-      token.concat(c = getc)
-      if c == "@" and peek(0) == "@"
-        token.concat getc
-      end
-    end
-
-    while (ch = getc) =~ IDENT_RE do
-      print " :#{ch}: " if RDoc::RubyLex.debug?
-      token.concat ch
-    end
-
-    ungetc
-
-    if ((ch == "!" && peek(1) != "=") || ch == "?") && token[0,1] =~ /\w/
-      token.concat getc
-    end
-
-    # almost fix token
-
-    case token
-    when /^\$/
-      return Token(TkGVAR, token)
-    when /^\@\@/
-      @lex_state = :EXPR_END
-      # p Token(TkCVAR, token)
-      return Token(TkCVAR, token)
-    when /^\@/
-      @lex_state = :EXPR_END
-      return Token(TkIVAR, token)
-    end
-
-    if @lex_state != :EXPR_DOT
-      print token, "\n" if RDoc::RubyLex.debug?
-
-      token_c, *trans = TkReading2Token[token]
-      if token_c
-        # reserved word?
-
-        if (@lex_state != :EXPR_BEG &&
-            @lex_state != :EXPR_FNAME &&
-            trans[1])
-          # modifiers
-          token_c = TkSymbol2Token[trans[1]]
-          @lex_state = trans[0]
-        else
-          if @lex_state != :EXPR_FNAME
-            if ENINDENT_CLAUSE.include?(token)
-              valid = peek(0) != ':'
-
-              # check for ``class = val'' etc.
-              case token
-              when "class"
-                valid = false unless peek_match?(/^\s*(<<|\w|::)/)
-              when "def"
-                valid = false if peek_match?(/^\s*(([+-\/*&\|^]|<<|>>|\|\||\&\&)=|\&\&|\|\|)/)
-              when "do"
-                valid = false if peek_match?(/^\s*([+-\/*]?=|\*|<|>|\&)/)
-              when *ENINDENT_CLAUSE
-                valid = false if peek_match?(/^\s*([+-\/*]?=|\*|<|>|\&|\|)/)
-              else
-                # no nothing
-              end if valid
-
-              if valid
-                if token == "do"
-                  if ![TkFOR, TkWHILE, TkUNTIL].include?(@indent_stack.last)
-                    @indent += 1
-                    @indent_stack.push token_c
-                  end
-                else
-                  @indent += 1
-                  @indent_stack.push token_c
-                end
-              else
-                token_c = TkIDENTIFIER
-              end
-
-            elsif DEINDENT_CLAUSE.include?(token)
-              @indent -= 1
-              @indent_stack.pop
-            end
-            @lex_state = trans[0]
-          else
-            @lex_state = :EXPR_END
-          end
-        end
-        if token_c.ancestors.include?(TkId) and peek(0) == ':' and !peek_match?(/^::/)
-          token.concat getc
-          token_c = TkSYMBOL
-        end
-        return Token(token_c, token)
-      end
-    end
-
-    if @lex_state == :EXPR_FNAME
-      @lex_state = :EXPR_END
-      if peek(0) == '=' and peek(1) != '>'
-        token.concat getc
-      end
-    elsif @lex_state == :EXPR_BEG || @lex_state == :EXPR_DOT ||
-          @lex_state == :EXPR_ARG || @lex_state == :EXPR_MID
-      @lex_state = :EXPR_ARG
-    else
-      @lex_state = :EXPR_END
-    end
-
-    if token[0, 1] =~ /[A-Z]/
-      if token[-1] =~ /[!?]/
-        token_c = TkIDENTIFIER
-      else
-        token_c = TkCONSTANT
-      end
-    elsif token[token.size - 1, 1] =~ /[!?]/
-      token_c = TkFID
-    else
-      token_c = TkIDENTIFIER
-    end
-    if peek(0) == ':' and !peek_match?(/^::/)
-      token.concat getc
-      return Token(TkSYMBOL, token)
-    else
-      return Token(token_c, token)
-    end
-  end
-
-  def identify_here_document(op)
-    ch = getc
-    start_token = op
-    #    if lt = PERCENT_LTYPE[ch]
-    if ch == "-" or ch == "~"
-      start_token.concat ch
-      ch = getc
-      indent = true
-    end
-    if /['"`]/ =~ ch
-      start_token.concat ch
-      user_quote = lt = ch
-      quoted = ""
-      while (c = getc) && c != lt
-        quoted.concat c
-      end
-      start_token.concat quoted
-      start_token.concat lt
-    else
-      user_quote = nil
-      lt = '"'
-      quoted = ch.dup
-      while (c = getc) && c =~ /\w/
-        quoted.concat c
-      end
-      start_token.concat quoted
-      ungetc
-    end
-
-    @heredoc_queue << {
-      quoted: quoted,
-      lt: lt,
-      indent: indent,
-      started: false
-    }
-    @lex_state = :EXPR_END
-    Token(RDoc::RubyLex::TkHEREDOCBEG, start_token)
-  end
-
-  def identify_here_document_body(quoted, lt, indent)
-    ltback, @ltype = @ltype, lt
-
-    doc = ""
-    heredoc_end = nil
-    while l = gets
-      l = l.sub(/(:?\r)?\n\z/, "\n")
-      if (indent ? l.strip : l.chomp) == quoted
-        heredoc_end = l
-        break
-      end
-      doc << l
-    end
-    raise Error, "Missing terminating #{quoted} for string" unless heredoc_end
-
-    @ltype = ltback
-    @lex_state = :EXPR_BEG
-    [Token(RDoc::RubyLex::TkHEREDOC, doc), heredoc_end]
-  end
-
-  def identify_quotation
-    type = ch = getc
-    if lt = PERCENT_LTYPE[type]
-      ch = getc
-    elsif type =~ /\W/
-      type = nil
-      lt = "\""
-    else
-      return Token(TkMOD, '%')
-    end
-    #     if ch !~ /\W/
-    #       ungetc
-    #       next
-    #     end
-    #@ltype = lt
-    @quoted = ch unless @quoted = PERCENT_PAREN[ch]
-    identify_string(lt, @quoted, type)
-  end
-
-  def identify_number(op = "")
-    @lex_state = :EXPR_END
-
-    num = op
-
-    if peek(0) == "0" && peek(1) !~ /[.eEri]/
-      num << getc
-
-      case peek(0)
-      when /[xX]/
-        ch = getc
-        match = /[0-9a-fA-F_]/
-      when /[bB]/
-        ch = getc
-        match = /[01_]/
-      when /[oO]/
-        ch = getc
-        match = /[0-7_]/
-      when /[dD]/
-        ch = getc
-        match = /[0-9_]/
-      when /[0-7]/
-        match = /[0-7_]/
-      when /[89]/
-        raise Error, "Illegal octal digit"
-      else
-        return Token(TkINTEGER, num)
-      end
-
-      num << ch if ch
-
-      len0 = true
-      non_digit = false
-      while ch = getc
-        num << ch
-        if match =~ ch
-          if ch == "_"
-            if non_digit
-              raise Error, "trailing `#{ch}' in number"
-            else
-              non_digit = ch
-            end
-          else
-            non_digit = false
-            len0 = false
-          end
-        else
-          ungetc
-          num[-1, 1] = ''
-          if len0
-            raise Error, "numeric literal without digits"
-          end
-          if non_digit
-            raise Error, "trailing `#{non_digit}' in number"
-          end
-          break
-        end
-      end
-      return Token(TkINTEGER, num)
-    end
-
-    type = TkINTEGER
-    allow_point = true
-    allow_e = true
-    allow_ri = true
-    non_digit = false
-    while ch = getc
-      num << ch
-      case ch
-      when /[0-9]/
-        non_digit = false
-      when "_"
-        non_digit = ch
-      when allow_point && "."
-        if non_digit
-          raise Error, "trailing `#{non_digit}' in number"
-        end
-        type = TkFLOAT
-        if peek(0) !~ /[0-9]/
-          type = TkINTEGER
-          ungetc
-          num[-1, 1] = ''
-          break
-        end
-        allow_point = false
-      when allow_e && "e", allow_e && "E"
-        if non_digit
-          raise Error, "trailing `#{non_digit}' in number"
-        end
-        type = TkFLOAT
-        if peek(0) =~ /[+-]/
-          num << getc
-        end
-        allow_e = false
-        allow_ri = false
-        allow_point = false
-        non_digit = ch
-      when allow_ri && "r"
-        if non_digit
-          raise Error, "trailing `#{non_digit}' in number"
-        end
-        type = TkRATIONAL
-        if peek(0) == 'i'
-          type = TkIMAGINARY
-          num << getc
-        end
-        break
-      when allow_ri && "i"
-        if non_digit && non_digit != "r"
-          raise Error, "trailing `#{non_digit}' in number"
-        end
-        type = TkIMAGINARY
-        break
-      else
-        if non_digit
-          raise Error, "trailing `#{non_digit}' in number"
-        end
-        ungetc
-        num[-1, 1] = ''
-        break
-      end
-    end
-
-    Token(type, num)
-  end
-
-  def identify_string(ltype, quoted = ltype, type = nil)
-    close = PERCENT_PAREN.values.include?(quoted)
-    @ltype = ltype
-    @quoted = quoted
-
-    str = if ltype == quoted and %w[" ' / `].include? ltype and type.nil? then
-            ltype.dup
-          else
-            "%#{type}#{PERCENT_PAREN_REV[quoted]||quoted}"
-          end
-
-    subtype = nil
-    begin
-      nest = 0
-
-      while ch = getc
-        str << ch
-
-        if @quoted == ch and nest <= 0
-          break
-        elsif @ltype != "'" && @ltype != "]" && @ltype != ":" and ch == "#"
-          ch = getc
-          if ch == "{" then
-            subtype = true
-            str << ch << skip_inner_expression
-            next
-          else
-            ungetc
-          end
-        elsif ch == '\\'
-          case @ltype
-          when "'" then
-            case ch = getc
-            when "'", '\\' then
-              str << ch
-            else
-              str << ch
-            end
-          else
-            str << read_escape
-          end
-        end
-
-        if close then
-          if PERCENT_PAREN[ch] == @quoted
-            nest += 1
-          elsif ch == @quoted
-            nest -= 1
-          end
-        end
-      end
-
-      if @ltype == "/"
-        while peek(0) =~ /i|m|x|o|e|s|u|n/
-          str << getc
-        end
-      end
-
-      if peek(0) == ':' and !peek_match?(/^::/) and :EXPR_BEG == @lex_state and !@after_question
-        str.concat getc
-        return Token(TkSYMBOL, str)
-      elsif subtype
-        Token(DLtype2Token[ltype], str)
-      else
-        Token(Ltype2Token[ltype], str)
-      end
-    ensure
-      @ltype = nil
-      @quoted = nil
-      @lex_state = :EXPR_END
-    end
-  end
-
-  def skip_inner_expression
-    res = ""
-    nest = 0
-    while ch = getc
-      res << ch
-      if ch == '}'
-        break if nest.zero?
-        nest -= 1
-      elsif ch == '{'
-        nest += 1
-      end
-    end
-    res
-  end
-
-  def identify_comment
-    @ltype = "#"
-
-    comment = '#'
-
-    while ch = getc
-      # if ch == "\\" #"
-      #   read_escape
-      # end
-      if ch == "\n"
-        @ltype = nil
-        ungetc
-        break
-      end
-
-      comment << ch
-    end
-
-    return Token(TkCOMMENT, comment)
-  end
-
-  def read_escape
-    escape = ''
-    ch = getc
-
-    case ch
-    when "\n", "\r", "\f"
-      escape << ch
-    when "\\", "n", "t", "r", "f", "v", "a", "e", "b", "s" #"
-      escape << ch
-    when /[0-7]/
-      ungetc ch
-      3.times do
-        ch = getc
-        case ch
-        when /[0-7]/
-          escape << ch
-        when nil
-          break
-        else
-          ungetc
-          break
-        end
-      end
-
-    when "x"
-      escape << ch
-
-      2.times do
-        ch = getc
-        case ch
-        when /[0-9a-fA-F]/
-          escape << ch
-        when nil
-          break
-        else
-          ungetc
-          break
-        end
-      end
-
-    when "M"
-      escape << ch
-
-      ch = getc
-      if ch != '-'
-        ungetc
-      else
-        escape << ch
-
-        ch = getc
-        if ch == "\\" #"
-          ungetc
-          escape << read_escape
-        else
-          escape << ch
-        end
-      end
-
-    when "C", "c" #, "^"
-      escape << ch
-
-      if ch == "C"
-        ch = getc
-
-        if ch == "-"
-          escape << ch
-          ch = getc
-          escape << ch
-
-          escape << read_escape if ch == "\\"
-        else
-          ungetc
-        end
-      elsif (ch = getc) == "\\" #"
-        escape << ch << read_escape
-      end
-    else
-      escape << ch
-
-      # other characters
-    end
-
-    escape
-  end
-
-  # :startdoc:
-
-end
-
-#RDoc::RubyLex.debug_level = 1