yard 0.2.2 → 0.2.3

data/lib/yard/handlers/ruby/method_condition_handler.rb

@@ -0,0 +1,7 @@
+class YARD::Handlers::Ruby::MethodConditionHandler < YARD::Handlers::Ruby::Base
+  handles :if_mod, :unless_mod
+  
+  def process
+    parse_block(statement.then_block, owner: owner)
+  end
+end

data/lib/yard/handlers/ruby/method_handler.rb

@@ -0,0 +1,48 @@
+class YARD::Handlers::Ruby::MethodHandler < YARD::Handlers::Ruby::Base
+  handles :def, :defs
+  
+  def process
+    nobj = namespace
+    mscope = scope
+    if statement.type == :defs
+      meth = statement[2][0]
+      nobj = P(namespace, statement[0].source) unless statement[0].source == "self"
+      args = format_args(statement[3])
+      blk = statement[4]
+      mscope = :class
+    else
+      meth = statement[0][0]
+      args = format_args(statement[1])
+      blk = statement[2]
+    end
+    
+    obj = register MethodObject.new(nobj, meth, mscope) do |o| 
+      o.visibility = visibility 
+      o.source = statement.source
+      o.signature = method_signature(meth)
+      o.explicit = true
+      o.parameters = args
+    end
+    
+    parse_block(blk, :owner => obj) # mainly for yield/exceptions
+  end
+  
+  def format_args(args)
+    args = args.jump(:params)
+    params = []
+    params += args.required_params.map {|a| [a.source, nil] } if args.required_params
+    params += args.optional_params.map {|a| [a[0].source, a[1].source] } if args.optional_params
+    params << ["*" + args.splat_param.source, nil] if args.splat_param
+    params += args.required_end_params.map {|a| [a.source, nil] } if args.required_end_params
+    params << ["&" + args.block_param.source, nil] if args.block_param
+    params
+  end
+  
+  def method_signature(method_name)
+    if statement[1]
+      "def #{method_name}(#{statement[1].jump(:params).source})"
+    else
+      "def #{method_name}"
+    end
+  end
+end

data/lib/yard/handlers/ruby/mixin_handler.rb

@@ -0,0 +1,25 @@
+class YARD::Handlers::Ruby::MixinHandler < YARD::Handlers::Ruby::Base
+  namespace_only
+  handles method_call(:include)
+  
+  def process
+    statement.parameters(false).each {|mixin| process_mixin(mixin) }
+  end
+
+  protected
+
+  def process_mixin(mixin)
+    unless mixin.ref?
+      raise YARD::Parser::UndocumentableError, "mixin #{mixin.source} for class #{namespace.path}"
+    end
+    
+    case obj = Proxy.new(namespace, mixin.source)
+    when Proxy
+      obj.type = :module
+    when ConstantObject # If a constant is included, use its value as the real object
+      obj = Proxy.new(namespace, obj.value)
+    end
+    
+    namespace.mixins(scope) << obj
+  end
+end

data/lib/yard/handlers/ruby/module_handler.rb

@@ -0,0 +1,9 @@
+class YARD::Handlers::Ruby::ModuleHandler < YARD::Handlers::Ruby::Base
+  handles :module
+  
+  def process
+    modname = statement[0].source
+    mod = register ModuleObject.new(namespace, modname)
+    parse_block(statement[1], :namespace => mod)
+  end
+end

data/lib/yard/handlers/ruby/visibility_handler.rb

@@ -0,0 +1,18 @@
+class YARD::Handlers::Ruby::VisibilityHandler < YARD::Handlers::Ruby::Base
+  handles method_call(:private)
+  handles method_call(:protected)
+  handles method_call(:public)
+  
+  def process
+    return if (ident = statement.jump(:ident)) == statement
+    case statement.type
+    when :var_ref
+      self.visibility = ident.first
+    when :fcall, :command
+      statement[1].traverse do |node|
+        next unless node.type == :ident || node.type == :string_content
+        MethodObject.new(namespace, node.source, scope) {|o| o.visibility = ident.first }
+      end
+    end
+  end
+end

data/lib/yard/handlers/ruby/yield_handler.rb

@@ -0,0 +1,28 @@
+class YARD::Handlers::Ruby::YieldHandler < YARD::Handlers::Ruby::Base
+  handles :yield, :yield0
+  
+  def process
+    return unless owner.is_a?(MethodObject) # Only methods yield
+    return if owner.has_tag? :yield         # Don't override yield tags
+    return if owner.has_tag? :yieldparam    # Same thing.
+
+    yieldtag = YARD::Tags::Tag.new(:yield, "", [])
+    owner.docstring.add_tag(yieldtag)
+    
+    if statement.type == :yield
+      statement.jump(:list).children.each do |item|
+        if item == s(:var_ref, s(:kw, "self"))
+          yieldtag.types << '_self'
+          owner.docstring.add_tag YARD::Tags::Tag.new(:yieldparam, 
+            "the object that the method was called on", owner.namespace.path, '_self')
+        elsif item == s(:zsuper)
+          yieldtag.types << '_super'
+          owner.docstring.add_tag YARD::Tags::Tag.new(:yieldparam, 
+            "the result of the method from the superclass", nil, '_super')
+        else
+          yieldtag.types << item.source
+        end
+      end
+    end
+  end
+end

data/lib/yard/parser/ruby/ast_node.rb

@@ -0,0 +1,263 @@
+module YARD
+  module Parser
+    module Ruby
+      def s(*args)
+        type = Symbol === args.first ? args.shift : :list
+        opts = Hash === args.last ? args.pop : {}
+        AstNode.node_class_for(type).new(type, args, opts)
+      end
+      
+      class AstNode < Array
+        attr_accessor :type, :parent, :docstring, :file, :full_source, :source
+        attr_accessor :source_range, :line_range
+        alias comments docstring
+        alias to_s source
+        
+        KEYWORDS = { class: true, alias: true, lambda: true, do_block: true,
+          def: true, defs: true, begin: true, rescue: true, rescue_mod: true,
+          if: true, if_mod: true, else: true, elsif: true, case: true,
+          when: true, next: true, break: true, retry: true, redo: true,
+          return: true, throw: true, catch: true, until: true, until_mod: true,
+          while: true, while_mod: true, yield: true, yield0: true, zsuper: true,
+          unless: true, unless_mod: true, for: true, super: true, return0: true }
+        
+        def self.node_class_for(type)
+          case type
+          when :params
+            ParameterNode
+          when :call, :fcall, :command, :command_call
+            MethodCallNode
+          when :if, :elsif, :if_mod, :unless, :unless_mod
+            ConditionalNode
+          when /_ref\Z/
+            ReferenceNode
+          else
+            AstNode
+          end
+        end
+
+        def initialize(type, arr, opts = {})
+          super(arr)
+          self.type = type
+          self.line_range = opts[:line]
+          self.source_range = opts[:char]
+          @fallback_line = opts[:listline]
+          @fallback_source = opts[:listchar]
+          @token = true if opts[:token]
+        end
+        
+        def ==(ast)
+          super && type == ast.type
+        end
+        
+        def show
+          "\t#{line}: #{first_line}"
+        end
+        
+        def source_range
+          reset_line_info unless @source_range
+          @source_range
+        end
+        
+        def line_range
+          reset_line_info unless @line_range
+          @line_range
+        end
+        
+        def has_line?
+          @line_range ? true : false
+        end
+        
+        def line
+          line_range && line_range.first
+        end
+        
+        def first_line
+          full_source.split(/\r?\n/)[line - 1].strip
+        end
+        
+        def jump(*node_types)
+          traverse {|child| return(child) if node_types.include?(child.type) }
+          self
+        end
+
+        def children
+          @children ||= select {|e| AstNode === e }
+        end
+
+        def token?
+          @token
+        end
+        
+        def ref?
+          false
+        end
+        
+        def literal?
+          @literal ||= type =~ /_literal$/ ? true : false
+        end
+        
+        def kw?
+          @kw ||= KEYWORDS.has_key?(type)
+        end
+        
+        def call?
+          false
+        end
+        
+        def condition?
+          false
+        end
+
+        def file
+          return parent.file if parent
+          @file
+        end
+
+        def full_source
+          return parent.full_source if parent
+          return @full_source if @full_source
+          return IO.read(@file) if file && File.exist?(file)
+        end
+
+        def source
+          return parent.full_source[source_range] if parent
+          full_source
+        end
+
+        def pretty_print(q)
+          objs = [*self.dup, :__last__]
+          objs.unshift(type) if type && type != :list
+
+          options = {}
+          if @docstring
+            options[:docstring] = docstring
+          end
+          if @source_range || @line_range
+            options[:line] = line_range
+            options[:source] = source_range
+          end
+          objs.pop if options.size == 0
+
+          q.group(3, 's(', ')') do
+            q.seplist(objs, nil, :each) do |v| 
+              if v == :__last__
+                q.seplist(options, nil, :each) do |k, v| 
+                  q.group(3) do 
+                    q.text k
+                    q.group(3) do 
+                      q.text ': '
+                      q.pp v 
+                    end
+                  end
+                end
+              else
+                q.pp v 
+              end
+            end
+          end
+        end
+
+        def inspect 
+          typeinfo = type && type != :list ? ':' + type.to_s + ', ' : ''
+          's(' + typeinfo + map(&:inspect).join(", ") + ')'
+        end
+
+        def traverse
+          nodes = [self]
+          nodes.each.with_index do |node, index|
+            yield node
+            nodes.insert index+1, *node.children
+          end
+        end
+        
+        private
+
+        def reset_line_info
+          if size == 0
+            self.line_range = @fallback_line
+            self.source_range = @fallback_source
+          elsif children.size > 0
+            f, l = children.first, children.last
+            self.line_range = Range.new(f.line_range.first, l.line_range.last)
+            self.source_range = Range.new(f.source_range.first, l.source_range.last)
+          elsif @fallback_line || @fallback_source
+            self.line_range = @fallback_line
+            self.source_range = @fallback_source
+          else
+            self.line_range = 0...0
+            self.source_range = 0...0
+          end
+        end
+      end
+      
+      class ReferenceNode < AstNode
+        def ref?; true end
+        
+        def path
+          Array.new flatten
+        end
+        
+        def namespace
+          Array.new flatten[0...-1]
+        end
+        
+        def source
+          super.split(/\s+/).first
+        end
+      end
+      
+      class ParameterNode < AstNode
+        def required_params; self[0] end
+        def required_end_params; self[3] end
+        def optional_params; self[1] end
+        def splat_param; self[2] ? self[2][0] : nil end
+        def block_param; self[4] ? self[4][0] : nil end
+      end
+      
+      class MethodCallNode < AstNode
+        def call?; true end
+        def namespace; first if index_adjust > 0 end
+
+        def method_name(name_only = false)
+          name = self[index_adjust]
+          name_only ? name.jump(:ident).first.to_sym : name
+        end
+
+        def parameters(include_block_param = true)
+          params = self[1 + index_adjust]
+          params = call_has_paren? ? params.first : params
+          include_block_param ? params : params[0...-1]
+        end
+        
+        def block_param; parameters.last end
+        
+        private
+        
+        def index_adjust
+          [:call, :command_call].include?(type) ? 2 : 0
+        end
+        
+        def call_has_paren? 
+          [:fcall, :call].include?(type)
+        end
+      end
+      
+      class ConditionalNode < AstNode
+        def condition?; true end
+        def condition; first end
+        def then_block; self[1] end
+        
+        def else_block
+          if self[2] && !cmod?
+            self[2].type == :elsif ? self[2] : self[2][0]
+          end
+        end
+        
+        private
+        
+        def cmod?; type =~ /_mod$/ end
+      end
+    end
+  end
+end

data/lib/yard/parser/{ruby_lex.rb → ruby/legacy/ruby_lex.rb}

@@ -2,7 +2,7 @@ require "e2mmap"
 require "irb/slex"
 
 module YARD
-  module Parser
+  module Parser::Ruby::Legacy
     module RubyToken
       EXPR_BEG   = :EXPR_BEG
       EXPR_MID   = :EXPR_MID
@@ -95,7 +95,7 @@ module YARD
         when String, Symbol
           source = token.kind_of?(String) ? TkReading2Token : TkSymbol2Token
           if (tk = source[token]).nil?
-          	IRB.fail TkReading2TokenNoKey, token
+            IRB.fail TkReading2TokenNoKey, token
           end
           tk = Token(tk[0], value) 
         else 
@@ -105,7 +105,7 @@ module YARD
                  else
                    token.new(@prev_line_no, @prev_char_no, value)
                  end
-               end
+          end
         end
         tk
       end
@@ -113,38 +113,38 @@ module YARD
       TokenDefinitions = [
         [:TkCLASS,      TkKW,  "class",  EXPR_CLASS],
         [:TkMODULE,     TkKW,  "module", EXPR_BEG],
-        [:TkDEF,	      TkKW,  "def",    EXPR_FNAME],
+        [:TkDEF,        TkKW,  "def",    EXPR_FNAME],
         [:TkUNDEF,      TkKW,  "undef",  EXPR_FNAME],
         [:TkBEGIN,      TkKW,  "begin",  EXPR_BEG],
         [:TkRESCUE,     TkKW,  "rescue", EXPR_MID],
         [:TkENSURE,     TkKW,  "ensure", EXPR_BEG],
-        [:TkEND,	      TkKW,  "end",    EXPR_END],
+        [:TkEND,        TkKW,  "end",    EXPR_END],
         [:TkIF,         TkKW,  "if",     EXPR_BEG, :TkIF_MOD],
         [:TkUNLESS,     TkKW,  "unless", EXPR_BEG, :TkUNLESS_MOD],
-        [:TkTHEN,	      TkKW,  "then",   EXPR_BEG],
+        [:TkTHEN,       TkKW,  "then",   EXPR_BEG],
         [:TkELSIF,      TkKW,  "elsif",  EXPR_BEG],
-        [:TkELSE,	      TkKW,  "else",   EXPR_BEG],
-        [:TkCASE,	      TkKW,  "case",   EXPR_BEG],
-        [:TkWHEN,	      TkKW,  "when",   EXPR_BEG],
+        [:TkELSE,       TkKW,  "else",   EXPR_BEG],
+        [:TkCASE,       TkKW,  "case",   EXPR_BEG],
+        [:TkWHEN,       TkKW,  "when",   EXPR_BEG],
         [:TkWHILE,      TkKW,  "while",  EXPR_BEG, :TkWHILE_MOD],
         [:TkUNTIL,      TkKW,  "until",  EXPR_BEG, :TkUNTIL_MOD],
-        [:TkFOR,	      TkKW,  "for",    EXPR_BEG],
+        [:TkFOR,        TkKW,  "for",    EXPR_BEG],
         [:TkBREAK,      TkKW,  "break",  EXPR_END],
-        [:TkNEXT,	      TkKW,  "next",   EXPR_END],
-        [:TkREDO,	      TkKW,  "redo",   EXPR_END],
+        [:TkNEXT,       TkKW,  "next",   EXPR_END],
+        [:TkREDO,       TkKW,  "redo",   EXPR_END],
         [:TkRETRY,      TkKW,  "retry",  EXPR_END],
-        [:TkIN,	        TkKW,  "in",     EXPR_BEG],
-        [:TkDO,	        TkKW,  "do",     EXPR_BEG],
+        [:TkIN,         TkKW,  "in",     EXPR_BEG],
+        [:TkDO,         TkKW,  "do",     EXPR_BEG],
         [:TkRETURN,     TkKW,  "return", EXPR_MID],
         [:TkYIELD,      TkKW,  "yield",  EXPR_END],
         [:TkSUPER,      TkKW,  "super",  EXPR_END],
-        [:TkSELF,	      TkKW,  "self",   EXPR_END],
-        [:TkNIL, 	      TkKW,  "nil",    EXPR_END],
-        [:TkTRUE,	      TkKW,  "true",   EXPR_END],
+        [:TkSELF,       TkKW,  "self",   EXPR_END],
+        [:TkNIL,        TkKW,  "nil",    EXPR_END],
+        [:TkTRUE,       TkKW,  "true",   EXPR_END],
         [:TkFALSE,      TkKW,  "false",  EXPR_END],
-        [:TkAND,	      TkKW,  "and",    EXPR_BEG],
-        [:TkOR, 	      TkKW,  "or",     EXPR_BEG],
-        [:TkNOT,	      TkKW,  "not",    EXPR_BEG],
+        [:TkAND,        TkKW,  "and",    EXPR_BEG],
+        [:TkOR,         TkKW,  "or",     EXPR_BEG],
+        [:TkNOT,        TkKW,  "not",    EXPR_BEG],
         [:TkIF_MOD,     TkKW],
         [:TkUNLESS_MOD, TkKW],
         [:TkWHILE_MOD,  TkKW],
@@ -152,14 +152,14 @@ module YARD
         [:TkALIAS,      TkKW,  "alias",    EXPR_FNAME],
         [:TkDEFINED,    TkKW,  "defined?", EXPR_END],
         [:TklBEGIN,     TkKW,  "BEGIN",    EXPR_END],
-        [:TklEND,	      TkKW,  "END",      EXPR_END],
+        [:TklEND,       TkKW,  "END",      EXPR_END],
         [:Tk__LINE__,   TkKW,  "__LINE__", EXPR_END],
         [:Tk__FILE__,   TkKW,  "__FILE__", EXPR_END],
 
         [:TkIDENTIFIER, TkId],
-        [:TkFID,	      TkId],
-        [:TkGVAR,	      TkId],
-        [:TkIVAR,	      TkId],
+        [:TkFID,        TkId],
+        [:TkGVAR,       TkId],
+        [:TkIVAR,       TkId],
         [:TkCONSTANT,   TkId],
 
         [:TkINTEGER,    TkVal],
@@ -178,28 +178,28 @@ module YARD
 
         [:TkUPLUS,      TkOp,   "+@"],
         [:TkUMINUS,     TkOp,   "-@"],
-        [:TkPOW,	      TkOp,   "**"],
-        [:TkCMP,	      TkOp,   "<=>"],
-        [:TkEQ,	        TkOp,   "=="],
-        [:TkEQQ,	      TkOp,   "==="],
-        [:TkNEQ,	      TkOp,   "!="],
-        [:TkGEQ,	      TkOp,   ">="],
-        [:TkLEQ,	      TkOp,   "<="],
+        [:TkPOW,        TkOp,   "**"],
+        [:TkCMP,        TkOp,   "<=>"],
+        [:TkEQ,         TkOp,   "=="],
+        [:TkEQQ,        TkOp,   "==="],
+        [:TkNEQ,        TkOp,   "!="],
+        [:TkGEQ,        TkOp,   ">="],
+        [:TkLEQ,        TkOp,   "<="],
         [:TkANDOP,      TkOp,   "&&"],
-        [:TkOROP,	      TkOp,   "||"],
+        [:TkOROP,       TkOp,   "||"],
         [:TkMATCH,      TkOp,   "=~"],
         [:TkNMATCH,     TkOp,   "!~"],
-        [:TkDOT2,	      TkOp,   ".."],
-        [:TkDOT3,	      TkOp,   "..."],
-        [:TkAREF,	      TkOp,   "[]"],
-        [:TkASET,	      TkOp,   "[]="],
+        [:TkDOT2,       TkOp,   ".."],
+        [:TkDOT3,       TkOp,   "..."],
+        [:TkAREF,       TkOp,   "[]"],
+        [:TkASET,       TkOp,   "[]="],
         [:TkLSHFT,      TkOp,   "<<"],
         [:TkRSHFT,      TkOp,   ">>"],
         [:TkCOLON2,     TkOp],
         [:TkCOLON3,     TkOp],
-        [:OPASGN,	      TkOp],               # +=, -=  etc. #
+        [:OPASGN,       TkOp],               # +=, -=  etc. #
         [:TkASSOC,      TkOp,   "=>"],
-        [:TkQUESTION,   TkOp,   "?"],	 #?
+        [:TkQUESTION,   TkOp,   "?"],  #?
         [:TkCOLON,      TkOp,   ":"],        #:
     
 #        [:TkfLPAREN],         # func( #
@@ -209,13 +209,13 @@ module YARD
         [:TkAMPER],           # &arg #
 #        [:TkSYMBOL,     TkId],          # :SYMBOL
         [:TkSYMBEG,     TkId], 
-        [:TkGT,	        TkOp,   ">"],
-        [:TkLT,	        TkOp,   "<"],
-        [:TkPLUS,	      TkOp,   "+"],
+        [:TkGT,         TkOp,   ">"],
+        [:TkLT,         TkOp,   "<"],
+        [:TkPLUS,       TkOp,   "+"],
         [:TkMINUS,      TkOp,   "-"],
-        [:TkMULT,	      TkOp,   "*"],
-        [:TkDIV,	      TkOp,   "/"],
-        [:TkMOD,	      TkOp,   "%"],
+        [:TkMULT,       TkOp,   "*"],
+        [:TkDIV,        TkOp,   "/"],
+        [:TkMOD,        TkOp,   "%"],
         [:TkBITOR,      TkOp,   "|"],
         [:TkBITXOR,     TkOp,   "^"],
         [:TkBITAND,     TkOp,   "&"],
@@ -225,7 +225,7 @@ module YARD
         [:TkBACKQUOTE,  TkOp,   "`"],
 
         [:TkASSIGN,     Token,  "="],
-        [:TkDOT,	      Token,  "."],
+        [:TkDOT,        Token,  "."],
         [:TkLPAREN,     Token,  "("],  #(exp)
         [:TkLBRACK,     Token,  "["],  #[arry]
         [:TkLBRACE,     Token,  "{"],  #{hash}
@@ -241,7 +241,7 @@ module YARD
         [:TkEND_OF_SCRIPT,  TkWhitespace],
 
         [:TkBACKSLASH,  TkUnknownChar,  "\\"],
-        [:TkAT,	        TkUnknownChar,  "@"],
+        [:TkAT,         TkUnknownChar,  "@"],
         [:TkDOLLAR,     TkUnknownChar,  "\$"], #"
       ]
 
@@ -253,21 +253,21 @@ module YARD
       def RubyToken.def_token(token_n, super_token = Token, reading = nil, *opts)
         token_n = token_n.id2name unless token_n.kind_of?(String)
         if RubyToken.const_defined?(token_n)
-          #IRB.fail AlreadyDefinedToken, token_n
+          # IRB.fail AlreadyDefinedToken, token_n
         end
 
         token_c =  Class.new super_token
         RubyToken.const_set token_n, token_c
-    #    token_c.inspect
+        # token_c.inspect
  
         if reading
           if TkReading2Token[reading]
-    	      IRB.fail TkReading2TokenDuplicateError, token_n, reading
+            IRB.fail TkReading2TokenDuplicateError, token_n, reading
           end
           if opts.empty?
-    	      TkReading2Token[reading] = [token_c]
+            TkReading2Token[reading] = [token_c]
           else
-    	      TkReading2Token[reading] = [token_c].concat(opts)
+            TkReading2Token[reading] = [token_c].concat(opts)
           end
         end
         TkSymbol2Token[token_n.intern] = token_c
@@ -414,7 +414,7 @@ module YARD
       def_exception(:TkReading2TokenNoKey, "key nothing(key='%s')")
       def_exception(:TkSymbol2TokenNoKey, "key nothing(key='%s')")
       def_exception(:TkReading2TokenDuplicateError, 
-    		"key duplicate(token_n='%s', key='%s')")
+        "key duplicate(token_n='%s', key='%s')")
       def_exception(:SyntaxError, "%s")
   
       include RubyToken
@@ -502,8 +502,8 @@ module YARD
       def lex
         catch(:eof) do
           until (((tk = token).kind_of?(TkNL) || tk.kind_of?(TkEND_OF_SCRIPT)) &&
-      	     !@continue or
-      	     tk.nil?)
+             !@continue or
+             tk.nil?)
           end
           line = get_read
 
@@ -520,17 +520,17 @@ module YARD
         catch(:eof) do
           begin
             begin
-      	      tk = @OP.match(self)
-      	      @space_seen = tk.kind_of?(TkSPACE)
+              tk = @OP.match(self)
+              @space_seen = tk.kind_of?(TkSPACE)
             rescue SyntaxError
-      	      abort if @exception_on_syntax_error
-      	      tk = TkError.new(line_no, char_no)
+              abort if @exception_on_syntax_error
+              tk = TkError.new(line_no, char_no)
             end
           end while @skip_space and tk.kind_of?(TkSPACE)
           if @read_auto_clean_up
             get_read
           end
-      #   throw :eof unless tk
+          # throw :eof unless tk
           p tk if $DEBUG
           tk.lex_state = lex_state if tk
           tk
@@ -629,20 +629,20 @@ module YARD
           @colonblock_seen = false
           case @lex_state
           when EXPR_BEG, EXPR_FNAME, EXPR_DOT
-    	      @continue = TRUE
+            @continue = TRUE
           else
-    	      @continue = FALSE
-    	      @lex_state = EXPR_BEG
+            @continue = FALSE
+            @lex_state = EXPR_BEG
           end
           Token(TkNL).set_text("\n")
         end
 
-        @OP.def_rules("*", "**",	
-    		  "!", "!=", "!~",
-    		  "=", "==", "===", 
-    		  "=~", "<=>",	
-    		  "<", "<=",
-    		  ">", ">=", ">>") do |op, io|
+        @OP.def_rules("*", "**",  
+          "!", "!=", "!~",
+          "=", "==", "===", 
+          "=~", "<=>",  
+          "<", "<=",
+          ">", ">=", ">>") do |op, io|
           @lex_state = EXPR_BEG
           Token(op).set_text(op)
         end
@@ -651,8 +651,8 @@ module YARD
           tk = nil
           if @lex_state != EXPR_END && @lex_state != EXPR_CLASS && 
              (@lex_state != EXPR_ARG || @space_seen)
-          	c = peek(0)
-          	tk = identify_here_document if /[-\w_\"\'\`]/ =~ c
+            c = peek(0)
+            tk = identify_here_document if /[-\w_\"\'\`]/ =~ c
           end
           if !tk
             @lex_state = EXPR_BEG
@@ -667,31 +667,31 @@ module YARD
 
         @OP.def_rules("`") do |op, io|
           if @lex_state == EXPR_FNAME
-    	      Token(op).set_text(op)
+            Token(op).set_text(op)
           else
-    	      identify_string(op)
+            identify_string(op)
           end
         end
 
         @OP.def_rules('?') do |op, io|
           if @lex_state == EXPR_END
-    	      @lex_state = EXPR_BEG
-    	      Token(TkQUESTION).set_text(op)
+            @lex_state = EXPR_BEG
+            Token(TkQUESTION).set_text(op)
           else
-    	      ch = getc
-    	      if @lex_state == EXPR_ARG && ch !~ /\s/
-    	        ungetc
-    	        @lex_state = EXPR_BEG
-    	        Token(TkQUESTION).set_text(op)
-    	      else
+            ch = getc
+            if @lex_state == EXPR_ARG && ch !~ /\s/
+              ungetc
+              @lex_state = EXPR_BEG
+              Token(TkQUESTION).set_text(op)
+            else
               str = op
               str << ch
-          	  if (ch == '\\') #'
-          	    str << read_escape
-          	  end
-          	  @lex_state = EXPR_END
-          	  Token(TkINTEGER).set_text(str)
-    	      end
+              if (ch == '\\') #'
+                str << read_escape
+              end
+              @lex_state = EXPR_END
+              Token(TkINTEGER).set_text(str)
+            end
           end
         end
 
@@ -701,7 +701,7 @@ module YARD
         end
     
         @OP.def_rules("+=", "-=", "*=", "**=", 
-    		  "&=", "|=", "^=", "<<=", ">>=", "||=", "&&=") do |op, io|
+          "&=", "|=", "^=", "<<=", ">>=", "||=", "&&=") do |op, io|
           @lex_state = EXPR_BEG
           op =~ /^(.*)=$/
           Token(TkOPASGN, $1).set_text(op)
@@ -717,30 +717,30 @@ module YARD
 
         @OP.def_rules("+", "-") do |op, io|
           catch(:RET) do
-          	if @lex_state == EXPR_ARG
-          	  if @space_seen and peek(0) =~ /[0-9]/
-          	    throw :RET, identify_number(op)
-          	  else
-          	    @lex_state = EXPR_BEG
-          	  end
-          	elsif @lex_state != EXPR_END and peek(0) =~ /[0-9]/
-          	  throw :RET, identify_number(op)
-          	else
-          	  @lex_state = EXPR_BEG
-          	end
-    	      Token(op).set_text(op)
+            if @lex_state == EXPR_ARG
+              if @space_seen and peek(0) =~ /[0-9]/
+                throw :RET, identify_number(op)
+              else
+                @lex_state = EXPR_BEG
+              end
+            elsif @lex_state != EXPR_END and peek(0) =~ /[0-9]/
+              throw :RET, identify_number(op)
+            else
+              @lex_state = EXPR_BEG
+            end
+            Token(op).set_text(op)
           end
         end
 
         @OP.def_rule(".") do
           @lex_state = EXPR_BEG
           if peek(0) =~ /[0-9]/
-          	ungetc
-          	identify_number("")
+            ungetc
+            identify_number("")
           else
-    	      # for obj.if
-    	      @lex_state = EXPR_DOT
-    	      Token(TkDOT).set_text(".")
+            # for obj.if
+            @lex_state = EXPR_DOT
+            Token(TkDOT).set_text(".")
           end
         end
 
@@ -761,39 +761,39 @@ module YARD
         end
 
         @OP.def_rule(":") do 
-          if @colonblock_seen || peek(0) =~ /\s/
-    	      @lex_state = EXPR_BEG
-    	      tk = Token(TkCOLON)
+          if (@colonblock_seen && @lex_state != EXPR_BEG) || peek(0) =~ /\s/
+            @lex_state = EXPR_BEG
+            tk = Token(TkCOLON)
           else
-    	      @lex_state = EXPR_FNAME
-    	      tk = Token(TkSYMBEG)
+            @lex_state = EXPR_FNAME
+            tk = Token(TkSYMBEG)
           end
           tk.set_text(":")
         end
 
         @OP.def_rule("::") do
-    #      p @lex_state.id2name, @space_seen
+          # p @lex_state.id2name, @space_seen
           if @lex_state == EXPR_BEG or @lex_state == EXPR_ARG && @space_seen
-    	      @lex_state = EXPR_BEG
-    	      tk = Token(TkCOLON3)
+            @lex_state = EXPR_BEG
+            tk = Token(TkCOLON3)
           else
-    	      @lex_state = EXPR_DOT
-    	      tk = Token(TkCOLON2)
+            @lex_state = EXPR_DOT
+            tk = Token(TkCOLON2)
           end
           tk.set_text("::")
         end
 
         @OP.def_rule("/") do |op, io|
           if @lex_state == EXPR_BEG || @lex_state == EXPR_MID
-    	      identify_string(op)
+            identify_string(op)
           elsif peek(0) == '='
-    	      getc
-    	      @lex_state = EXPR_BEG
-    	      Token(TkOPASGN, :/).set_text("/=") #")
+            getc
+            @lex_state = EXPR_BEG
+            Token(TkOPASGN, :/).set_text("/=") #")
           elsif @lex_state == EXPR_ARG and @space_seen and peek(0) !~ /\s/
-    	      identify_string(op)
+            identify_string(op)
           else 
-    	      @lex_state = EXPR_BEG
+            @lex_state = EXPR_BEG
             Token("/").set_text(op)
           end
         end
@@ -803,10 +803,10 @@ module YARD
           Token("^").set_text("^")
         end
 
-        #       @OP.def_rules("^=") do
-        # 	@lex_state = EXPR_BEG
-        # 	Token(TkOPASGN, :^)
-        #       end
+        # @OP.def_rules("^=") do
+        #   @lex_state = EXPR_BEG
+        #   Token(TkOPASGN, :^)
+        # end
     
         @OP.def_rules(",", ";") do |op, io|
           @colonblock_seen = false
@@ -826,13 +826,13 @@ module YARD
     
         @OP.def_rule("(") do
           @indent += 1
-          #if @lex_state == EXPR_BEG || @lex_state == EXPR_MID
-    	    #  @lex_state = EXPR_BEG
-    	    #  tk = Token(TkfLPAREN)
-          #else
-    	      @lex_state = EXPR_BEG
-    	      tk = Token(TkLPAREN)
-          #end
+          # if @lex_state == EXPR_BEG || @lex_state == EXPR_MID
+          #  @lex_state = EXPR_BEG
+          #  tk = Token(TkfLPAREN)
+          # else
+            @lex_state = EXPR_BEG
+            tk = Token(TkLPAREN)
+          # end
           tk.set_text("(")
         end
 
@@ -846,16 +846,16 @@ module YARD
 
         @OP.def_rule("[") do
           @indent += 1
-#          if @lex_state == EXPR_FNAME
-#    	      t = Token(TkfLBRACK)
-#          else
-#    	      if @lex_state == EXPR_BEG || @lex_state == EXPR_MID
-#    	        t = Token(TkLBRACK)
-#    	      elsif @lex_state == EXPR_ARG && @space_seen
-#    	      else
-#    	        t = Token(TkfLBRACK)
-#    	      end
-#          end
+          # if @lex_state == EXPR_FNAME
+          #   t = Token(TkfLBRACK)
+          # else
+          #   if @lex_state == EXPR_BEG || @lex_state == EXPR_MID
+          #     t = Token(TkLBRACK)
+          #   elsif @lex_state == EXPR_ARG && @space_seen
+          #   else
+          #     t = Token(TkfLBRACK)
+          #   end
+          # end
           t = Token(TkLBRACK)
           @lex_state = EXPR_BEG
           t.set_text("[")
@@ -863,11 +863,11 @@ module YARD
 
         @OP.def_rule("{") do
           @indent += 1
-          #if @lex_state != EXPR_END && @lex_state != EXPR_ARG
-    	    #  t = Token(TkLBRACE)
-          #else
-    	    #  t = Token(TkfLBRACE)
-          #end
+          # if @lex_state != EXPR_END && @lex_state != EXPR_ARG
+          #   t = Token(TkLBRACE)
+          # else
+          #   t = Token(TkfLBRACE)
+          # end
           t = Token(TkLBRACE)
           @lex_state = EXPR_BEG
           t.set_text("{")
@@ -875,27 +875,27 @@ module YARD
 
         @OP.def_rule('\\') do   #'
           if getc == "\n" 
-    	      @space_seen = true
-    	      @continue = true
-    	      Token(TkSPACE).set_text("\\\n")
+            @space_seen = true
+            @continue = true
+            Token(TkSPACE).set_text("\\\n")
           else 
-    	      ungetc
-    	      Token("\\").set_text("\\")  #"
+            ungetc
+            Token("\\").set_text("\\")  #"
           end 
         end 
 
         @OP.def_rule('%') do
           |op, io|
           if @lex_state == EXPR_BEG || @lex_state == EXPR_MID
-    	      identify_quotation('%')
+            identify_quotation('%')
           elsif peek(0) == '='
-    	      getc
-    	      Token(TkOPASGN, "%").set_text("%=")
+            getc
+            Token(TkOPASGN, "%").set_text("%=")
           elsif @lex_state == EXPR_ARG and @space_seen and peek(0) !~ /\s/
-    	      identify_quotation('%')
+            identify_quotation('%')
           else
-    	      @lex_state = EXPR_BEG
-    	      Token("%").set_text("%")
+            @lex_state = EXPR_BEG
+            Token("%").set_text("%")
           end
         end
 
@@ -905,34 +905,33 @@ module YARD
 
         @OP.def_rule('@') do
           if peek(0) =~ /[@\w_]/
-    	      ungetc
-    	      identify_identifier
+            ungetc
+            identify_identifier
           else
-    	      Token("@").set_text("@")
+            Token("@").set_text("@")
           end
         end
 
-        #       @OP.def_rule("def", proc{|op, io| /\s/ =~ io.peek(0)}) do 
-        # 	|op, io|
-        # 	@indent += 1
-        # 	@lex_state = EXPR_FNAME
-        # #	@lex_state = EXPR_END
-        # #	until @rests[0] == "\n" or @rests[0] == ";"
-        # #	  rests.shift
-        # #	end
-        #       end
+        # @OP.def_rule("def", proc{|op, io| /\s/ =~ io.peek(0)}) do 
+        #   |op, io|
+        #   @indent += 1
+        #   @lex_state = EXPR_FNAME
+        # # @lex_state = EXPR_END
+        # # until @rests[0] == "\n" or @rests[0] == ";"
+        # #   rests.shift
+        # # end
+        # end
 
         @OP.def_rule("__END__", proc{@prev_char_no == 0 && peek(0) =~ /[\r\n]/}) do
           throw :eof
         end
 
-        @OP.def_rule("") do
-          |op, io|
+        @OP.def_rule("") do |op, io|
           printf "MATCH: start %s: %s\n", op, io.inspect if RubyLex.debug?
           if peek(0) =~ /[0-9]/
-    	t = identify_number("")
+            t = identify_number("")
           elsif peek(0) =~ /[\w_]/
-    	t = identify_identifier
+            t = identify_identifier
           end
           printf "MATCH: end %s: %s\n", op, io.inspect if RubyLex.debug?
           t
@@ -1007,41 +1006,41 @@ module YARD
 
           token_c, *trans = TkReading2Token[token]
           if token_c
-    	# reserved word?
-
-    	if (@lex_state != EXPR_BEG &&
-    	    @lex_state != EXPR_FNAME &&
-    	    trans[1])
-    	  # modifiers
-    	  token_c = TkSymbol2Token[trans[1]]
-    	  @lex_state = trans[0]
-    	else
-    	  if @lex_state != EXPR_FNAME
-    	    if ENINDENT_CLAUSE.include?(token)
-    	      @indent += 1
-    	      
-    	      if ACCEPTS_COLON.include?(token)  
-    	        @colonblock_seen = true
-  	        else
-  	          @colonblock_seen = false
-	          end
-    	    elsif DEINDENT_CLAUSE.include?(token)
-    	      @indent -= 1
-    	      @colonblock_seen = false
-    	    end
-    	    @lex_state = trans[0]
-    	  else
-    	    @lex_state = EXPR_END
-    	  end
-    	end
-    	return Token(token_c, token).set_text(token)
+            # reserved word?
+
+            if (@lex_state != EXPR_BEG &&
+                @lex_state != EXPR_FNAME &&
+                trans[1])
+              # modifiers
+              token_c = TkSymbol2Token[trans[1]]
+              @lex_state = trans[0]
+            else
+              if @lex_state != EXPR_FNAME
+                if ENINDENT_CLAUSE.include?(token)
+                  @indent += 1
+            
+                  if ACCEPTS_COLON.include?(token)  
+                    @colonblock_seen = true
+                  else
+                    @colonblock_seen = false
+                  end
+                elsif DEINDENT_CLAUSE.include?(token)
+                  @indent -= 1
+                  @colonblock_seen = false
+                end
+                @lex_state = trans[0]
+              else
+                @lex_state = EXPR_END
+              end
+            end
+            return Token(token_c, token).set_text(token)
           end
         end
 
         if @lex_state == EXPR_FNAME
           @lex_state = EXPR_END
           if peek(0) == '='
-    	      token.concat getc
+            token.concat getc
           end
         elsif @lex_state == EXPR_BEG || @lex_state == EXPR_DOT
           @lex_state = EXPR_ARG
@@ -1068,13 +1067,13 @@ module YARD
           lt = ch
           quoted = ""
           while (c = getc) && c != lt
-    	quoted.concat c
+            quoted.concat c
           end
         else
           lt = '"'
           quoted = ch.dup
           while (c = getc) && c =~ /\w/
-    	quoted.concat c
+            quoted.concat c
           end
           ungetc
         end
@@ -1086,9 +1085,9 @@ module YARD
           reserve << ch
           if ch == "\\"    #"
             ch = getc
-    	reserve << ch
+            reserve << ch
           elsif ch == "\n"
-    	break
+            break
           end
         end
 
@@ -1115,13 +1114,13 @@ module YARD
         elsif ch =~ /\W/
           lt = "\""
         else
-#          RubyLex.fail SyntaxError, "unknown type of %string ('#{ch}')"
+          # RubyLex.fail SyntaxError, "unknown type of %string ('#{ch}')"
         end
-    #     if ch !~ /\W/
-    #       ungetc
-    #       next
-    #     end
-        #@ltype = lt
+        # if ch !~ /\W/
+        #   ungetc
+        #   next
+        # end
+        # @ltype = lt
         @quoted = ch unless @quoted = PERCENT_PAREN[ch]
         identify_string(lt, @quoted, ch, initial_char) if lt
       end
@@ -1164,25 +1163,25 @@ module YARD
             str << ch
 
           when allow_point && "."
-    	type = TkFLOAT
-    	if peek(0) !~ /[0-9]/
-    	  ungetc
-    	  break
-    	end
+            type = TkFLOAT
+            if peek(0) !~ /[0-9]/
+              ungetc
+              break
+            end
             str << ch
-    	allow_point = false
+            allow_point = false
 
           when allow_e && "e", allow_e && "E"
             str << ch
-    	type = TkFLOAT
-    	if peek(0) =~ /[+-]/
-    	  str << getc
-    	end
-    	allow_e = false
-    	allow_point = false
+            type = TkFLOAT
+            if peek(0) =~ /[+-]/
+              str << getc
+            end
+            allow_e = false
+            allow_point = false
           else
-    	ungetc
-    	break
+            ungetc
+            break
           end
         end
         Token(type).set_text(str)
@@ -1200,8 +1199,8 @@ module YARD
         nest = 0
         begin
           while ch = getc 
-    	str << ch
-    	if @quoted == ch 
+            str << ch
+            if @quoted == ch 
               if nest == 0
                 break
               else
@@ -1209,7 +1208,7 @@ module YARD
               end
             elsif opener == ch
               nest += 1
-    	elsif @ltype != "'" && @ltype != "]" and ch == "#"
+            elsif @ltype != "'" && @ltype != "]" and ch == "#"
               ch = getc
               if ch == "{"
                 subtype = true
@@ -1217,19 +1216,19 @@ module YARD
               else
                 ungetc(ch)
               end
-    	elsif ch == '\\' #'
-    	  str << read_escape
-    	end
+            elsif ch == '\\' #'
+              str << read_escape
+            end
           end
           if @ltype == "/"
-    	if peek(0) =~ /i|o|n|e|s/
-    	  str << getc
-    	end
+            if peek(0) =~ /i|o|n|e|s/
+              str << getc
+            end
           end
           if subtype
-    	      Token(DLtype2Token[ltype], str)
+            Token(DLtype2Token[ltype], str)
           else
-    	      Token(Ltype2Token[ltype], str)
+            Token(Ltype2Token[ltype], str)
           end.set_text(str)
         ensure
           @ltype = nil
@@ -1282,49 +1281,49 @@ module YARD
         when /[0-7]/
           ungetc ch
           3.times do
-        	case ch = getc
-        	when /[0-7]/
-        	when nil
-        	  break
-        	else
-        	  ungetc
-        	  break
-        	end
+          case ch = getc
+          when /[0-7]/
+          when nil
+            break
+          else
+            ungetc
+            break
+          end
           res << ch
         end
       
         when "x"
           res << ch
           2.times do
-        	case ch = getc
-        	when /[0-9a-fA-F]/
-        	when nil
-      	    break
-      	  else
-      	    ungetc
-      	    break
-      	  end
+          case ch = getc
+          when /[0-9a-fA-F]/
+          when nil
+            break
+          else
+            ungetc
+            break
+          end
             res << ch
           end
 
         when "M"
           res << ch
           if (ch = getc) != '-'
-    	      ungetc
+            ungetc
           else
             res << ch
-    	      if (ch = getc) == "\\" #"
+            if (ch = getc) == "\\" #"
               res << ch
-    	        res << read_escape
+              res << read_escape
             else
               res << ch
-    	      end
+            end
           end
 
         when "C", "c" #, "^"
           res << ch
           if ch == "C" and (ch = getc) != "-"
-    	      ungetc
+            ungetc
           else
             res << ch
             if (ch = getc) == "\\" #"
@@ -1341,4 +1340,4 @@ module YARD
       end
     end
   end
-end
+end