yard 0.2.2 → 0.2.3

data/lib/yard/parser/{statement.rb → ruby/legacy/statement.rb}

@@ -1,5 +1,5 @@
 module YARD
-  module Parser
+  module Parser::Ruby::Legacy
     class Statement 
       attr_reader :tokens, :comments, :block
 
@@ -9,8 +9,7 @@ module YARD
         @comments = comments
       end
       
-      def inspect(lines = nil)
-        flineno = tokens.first.line_no
+      def inspect
         buf = [""]
         tokens.each do |tk|
           if tk.is_a?(RubyToken::TkNL)
@@ -19,7 +18,12 @@ module YARD
             buf.last << tk.text
           end
         end
-        buf.map {|line| "\t#{flineno}: #{line}" }.join("\n")
+        buf.map {|text| "\t#{line}: #{text}" }.join("\n")
+      end
+      alias show inspect
+      
+      def line
+        tokens.first.line_no
       end
 
       private

data/lib/yard/parser/ruby/legacy/statement_list.rb

@@ -0,0 +1,262 @@
+module YARD
+  module Parser::Ruby::Legacy
+    class StatementList < Array
+      include RubyToken
+
+      # The following list of tokens will require a block to be opened 
+      # if used at the beginning of a statement.
+      OPEN_BLOCK_TOKENS = [TkCLASS, TkDEF, TkMODULE, TkUNTIL,
+                           TkIF, TkUNLESS, TkWHILE, TkFOR, TkCASE]
+
+      ##
+      # Creates a new statement list
+      #
+      # @param [TokenList, String] content the tokens to create the list from
+      def initialize(content)
+        if content.is_a? TokenList
+          @tokens = content.dup
+        elsif content.is_a? String
+          @tokens = TokenList.new(content)
+        else 
+          raise ArgumentError, "Invalid content for StatementList: #{content.inspect}:#{content.class}"
+        end
+
+        parse_statements
+      end
+      
+      def enumerator
+        self
+      end
+
+      private
+
+      def parse_statements
+        while stmt = next_statement do self << stmt end
+      end
+
+      ##
+      # Returns the next statement in the token stream
+      #
+      # @return [Statement] the next statement
+      def next_statement
+        @state = :first_statement
+        @statement_stack = []
+        @level = 0
+        @done = false
+        @current_block = nil
+        @statement, @block, @comments = TokenList.new, nil, nil
+        @last_tk, @last_ns_tk, @before_last_tk = nil, nil, nil
+
+        while !@done && tk = @tokens.shift
+          process_token(tk)
+
+          @before_last_tk = @last_tk
+          @last_tk = tk # Save last token
+          @last_ns_tk = tk unless [TkSPACE, TkNL, TkEND_OF_SCRIPT].include? tk.class
+        end
+
+        # Return the code block with starting token and initial comments
+        # If there is no code in the block, return nil
+        @comments = @comments.compact if @comments
+        if @block || !@statement.empty?
+          Statement.new(@statement, @block, @comments)
+        else
+          nil
+        end
+      end
+
+      ##
+      # Processes a single token
+      #
+      # @param [RubyToken::Token] tk the token to process
+      def process_token(tk)
+        case @state
+        when :first_statement
+          return if process_initial_comment(tk)
+          return if @statement.empty? && [TkSPACE, TkNL, TkCOMMENT].include?(tk.class)
+          return if process_simple_block_opener(tk)
+          push_token(tk)
+          return if process_complex_block_opener(tk)
+
+          if balances?(tk)
+            process_statement_end(tk)
+          else
+            @state = :balance
+          end
+        when :balance
+          @statement << tk
+          return unless balances?(tk)
+          @state = :first_statement
+          process_statement_end(tk)
+        when :block_statement
+          push_token(tk)
+          return unless balances?(tk)
+          process_statement_end(tk)
+        when :pre_block
+          @current_block = nil
+          process_block_token(tk) unless tk.class == TkSEMICOLON
+          @state = :block
+        when :block; process_block_token(tk)
+        when :post_block
+          if tk.class == TkSPACE
+            @statement << tk
+            return
+          end
+
+          process_statement_end(tk)
+          @state = :block
+        end
+      end
+
+      ##
+      # Processes a token in a block
+      #
+      # @param [RubyToken::Token] tk the token to process
+      def process_block_token(tk)
+        @block << tk
+        return unless balances?(tk)
+        process_statement_end(tk)
+      end
+
+      ##
+      # Processes a comment token that comes before a statement
+      #
+      # @param [RubyToken::Token] tk the token to process
+      # @return [Boolean] whether or not +tk+ was processed as an initial comment
+      def process_initial_comment(tk)
+        return unless tk.class == TkCOMMENT
+
+        # Two new-lines in a row will destroy any comment blocks
+        if @last_tk.class == TkNL && @before_last_tk &&
+            (@before_last_tk.class == TkNL || @before_last_tk.class == TkSPACE)
+          @comments = nil
+          return
+        end
+
+        # Remove the "#" and up to 1 space before the text
+        # Since, of course, the convention is to have "# text"
+        # and not "#text", which I deem ugly (you heard it here first)
+        @comments ||= []
+        @comments << tk.text.gsub(/^#+\s{0,1}/, '')
+        @comments.pop if @comments.size == 1 && @comments.first =~ /^\s*$/
+        true
+      end
+
+      ##
+      # Processes a simple block-opening token;
+      # that is, a block opener such as +begin+ or +do+
+      # that isn't followed by an expression
+      #
+      # @param [RubyToken::Token] tk the token to process
+      def process_simple_block_opener(tk)
+        return unless [TkLBRACE, TkDO, TkBEGIN].include?(tk.class) &&
+          # Make sure hashes are parsed as hashes, not as blocks
+          (@last_ns_tk.nil? || @last_ns_tk.lex_state != EXPR_BEG)
+
+        @block = TokenList.new
+        @block << tk
+        @level += 1
+        @state = :block
+
+        true
+      end
+
+      ##
+      # Processes a complex block-opening token;
+      # that is, a block opener such as +while+ or +for+
+      # that is followed by an expression
+      #
+      # @param [RubyToken::Token] tk the token to process
+      def process_complex_block_opener(tk)
+        return unless OPEN_BLOCK_TOKENS.include?(tk.class)
+        return if @last_ns_tk.class == TkALIAS
+
+        @current_block = tk.class
+        @state = :block_statement
+
+        true
+      end
+
+      ##
+      # Processes a token that closes a statement
+      #
+      # @param [RubyToken::Token] tk the token to process
+      def process_statement_end(tk)
+        # Whitespace means that we keep the same value of @new_statement as last token
+        return if tk.class == TkSPACE
+
+        return unless 
+          # We might be coming after a statement-ending token...
+          ((@last_tk && [TkSEMICOLON, TkNL, TkEND_OF_SCRIPT].include?(tk.class)) ||
+           # Or we might be at the beginning of an argument list
+           (@current_block == TkDEF && tk.class == TkRPAREN))
+
+        # Continue a possible existing new statement unless we just finished an expression...
+        return unless (@last_tk && [EXPR_END, EXPR_ARG].include?(@last_tk.lex_state)) ||
+          # Or we've opened a block and are ready to move into the body
+          (@current_block && [TkNL, TkSEMICOLON].include?(tk.class) &&
+           # Handle the case where the block statement's expression is on the next line
+           #
+           # while
+           #     foo
+           # end
+           @last_ns_tk.class != @current_block &&
+           # And the case where part of the expression is on the next line
+           #
+           # while foo ||
+           #     bar
+           # end
+           @last_tk.lex_state != EXPR_BEG)
+
+        # Continue with the statement if we've hit a comma in a def
+        return if @current_block == TkDEF && peek_no_space.class == TkCOMMA
+
+        unless @current_block
+          @done = true
+          return
+        end
+
+        @level += 1
+        @state = :pre_block unless @stat == :block_statement
+        @block = TokenList.new
+        @block << tk if @current_block && tk.class == TkNL
+      end
+
+      ##
+      # Handles the balancing of parentheses and blocks
+      #
+      # @param [RubyToken::Token] tk the token to process
+      # @return [Boolean] whether or not the current statement's parentheses and blocks
+      #   are balanced after +tk+
+      def balances?(tk)
+        if [TkLPAREN, TkLBRACK, TkLBRACE, TkDO, TkBEGIN].include?(tk.class)
+          @level += 1
+        elsif OPEN_BLOCK_TOKENS.include?(tk.class)
+          @level += 1 unless @last_ns_tk.class == TkALIAS
+        elsif [TkRPAREN, TkRBRACK, TkRBRACE, TkEND].include?(tk.class) && @level > 0
+          @level -= 1
+        end
+
+        @level == 0
+      end
+
+      ##
+      # Adds a token to the current statement,
+      # unless it's a newline, semicolon, or comment
+      #
+      # @param [RubyToken::Token] tk the token to process
+      def push_token(tk)
+        @statement << tk unless @level == 0 && [TkNL, TkSEMICOLON, TkCOMMENT].include?(tk.class)
+      end
+
+      ##
+      # Returns the next token in the stream that's not a space
+      #
+      # @return [RubyToken::Token] the next non-space token
+      def peek_no_space
+        return @tokens.first unless @tokens.first.class == TkSPACE
+        return @tokens[1]
+      end
+    end
+  end
+end

data/lib/yard/parser/{token_list.rb → ruby/legacy/token_list.rb}

@@ -1,5 +1,5 @@
 module YARD
-  module Parser
+  module Parser::Ruby::Legacy
     class TokenList < Array
       include RubyToken
       

data/lib/yard/parser/ruby/ruby_parser.rb

@@ -0,0 +1,307 @@
+require 'ripper'
+
+module YARD
+  module Parser
+    module Ruby
+      class ParserSyntaxError < UndocumentableError; end
+      
+      class RubyParser < Ripper
+        attr_reader :ast, :charno, :comments, :file, :tokens
+        alias root ast
+
+        def initialize(source, filename, *args)
+          super
+          @file = filename
+          @source = source
+          @tokens = []
+          @comments = {}
+          @map = {}
+          @ns_charno = 0
+          @list = []
+          @charno = 0
+        end
+
+        def parse
+          @ast = super
+          @ast.full_source = @source
+          @ast.file = @file
+          freeze_tree
+          insert_comments
+          self
+        end
+        
+        def enumerator
+          ast.children
+        end
+        
+        private
+        
+        MAPPINGS = {
+          :BEGIN => "BEGIN",
+          :END => "END",
+          :alias => "alias",
+          :array => :lbracket,
+          :aref => :lbracket,
+          :arg_paren => :lparen,
+          :begin => "begin",
+          :blockarg => "&",
+          :brace_block => :lbrace,
+          :break => "break",
+          :case => "case",
+          :class => "class",
+          :def => "def",
+          :defined => "defined?",
+          :defs => "def",
+          :do_block => "do",
+          :else => "else",
+          :elsif => "elsif",
+          :ensure => "ensure",
+          :for => "for",
+          :hash => :lbrace,
+          :if => "if",
+          :lambda => "lambda",
+          :module => "module",
+          :next => "next",
+          :paren => :lparen,
+          :qwords_literal => :qwords_beg,
+          :redo => "redo",
+          :regexp_literal => :regexp_beg,
+          :rescue => "rescue",
+          :rest_param => "*",
+          :retry => "retry",
+          :return => "return",
+          :return0 => "return",
+          :sclass => "class",
+          :string_embexpr => :embexpr_beg,
+          :string_literal => [:tstring_beg, :heredoc_beg],
+          :super => "super",
+          :symbol => :symbeg,
+          :undef => "undef",
+          :unless => "unless",
+          :until => "until",
+          :when => "when",
+          :while => "while",
+          :xstring_literal => :backtick,
+          :yield => "yield",
+          :yield0 => "yield",
+          :zsuper => "super"
+        }
+        REV_MAPPINGS = {}
+        
+        AST_TOKENS = [:CHAR, :backref, :const, :cvar, :gvar, :heredoc_end, :ident,
+          :int, :float, :ivar, :period, :regexp_end, :tstring_content, :backtick]
+
+        MAPPINGS.each do |k, v|
+          if Array === v
+            v.each {|_v| (REV_MAPPINGS[_v] ||= []) << k }
+          else
+            (REV_MAPPINGS[v] ||= []) << k
+          end
+        end
+
+        PARSER_EVENT_TABLE.each do |event, arity|
+          node_class = AstNode.node_class_for(event)
+          
+          if /_new\z/ =~ event and arity == 0
+            module_eval(<<-eof, __FILE__, __LINE__ + 1)
+              def on_#{event}(*args)
+                #{node_class}.new(:list, args, listchar: charno...charno, listline: lineno..lineno)
+              end
+            eof
+          elsif /_add(_.+)?\z/ =~ event
+            module_eval(<<-eof, __FILE__, __LINE__ + 1)
+              def on_#{event}(list, item)
+                list.push(item)
+                list
+              end
+            eof
+          elsif MAPPINGS.has_key?(event)
+            module_eval(<<-eof, __FILE__, __LINE__ + 1)
+              def on_#{event}(*args)
+                visit_event #{node_class}.new(:#{event}, args)
+              end
+            eof
+          else
+            module_eval(<<-eof, __FILE__, __LINE__ + 1)
+              def on_#{event}(*args)
+                #{node_class}.new(:#{event}, args, listline: lineno..lineno, listchar: charno...charno)
+              end
+            eof
+          end
+        end
+
+        SCANNER_EVENTS.each do |event|
+          ast_token = AST_TOKENS.include?(event)
+          module_eval(<<-eof, __FILE__, __LINE__ + 1)
+            def on_#{event}(tok)
+              visit_ns_token(:#{event}, tok, #{ast_token.inspect})
+            end
+          eof
+        end
+        
+        REV_MAPPINGS.select {|k| k.is_a?(Symbol) }.each do |event, value|
+          ast_token = AST_TOKENS.include?(event)
+          module_eval(<<-eof, __FILE__, __LINE__ + 1)
+            def on_#{event}(tok)
+              (@map[:#{event}] ||= []) << [lineno, charno]
+              visit_ns_token(:#{event}, tok, #{ast_token.inspect})
+            end
+          eof
+        end
+        
+        [:kw, :op].each do |event|
+          module_eval(<<-eof, __FILE__, __LINE__ + 1)
+            def on_#{event}(tok)
+              unless @tokens.last && @tokens.last[0] == :symbeg
+                (@map[tok] ||= []) << [lineno, charno]
+              end
+              visit_ns_token(:#{event}, tok, true)
+            end
+          eof
+        end
+
+        [:sp, :nl, :ignored_nl].each do |event|
+          module_eval(<<-eof, __FILE__, __LINE__ + 1)
+            def on_#{event}(tok)
+              add_token(:#{event}, tok)
+              @charno += tok.length
+            end
+          eof
+        end
+        
+        def visit_event(node)
+          lstart, sstart = *@map[MAPPINGS[node.type]].pop
+          node.source_range = Range.new(sstart, @ns_charno - 1)
+          node.line_range = Range.new(lstart, lineno)
+          node
+        end
+        
+        def visit_event_arr(node)
+          mapping = MAPPINGS[node.type].find {|k| @map[k] && !@map[k].empty? }
+          lstart, sstart = *@map[mapping].pop
+          node.source_range = Range.new(sstart, @ns_charno - 1)
+          node.line_range = Range.new(lstart, lineno)
+          node
+        end
+
+        def visit_ns_token(token, data, ast_token = false)
+          add_token(token, data)
+          ch = charno
+          @charno += data.length
+          @ns_charno = charno
+          if ast_token
+            AstNode.new(token, [data], line: lineno..lineno, char: ch..charno-1, token: true)
+          end
+        end
+        
+        def add_token(token, data)
+          if @tokens.last && @tokens.last[0] == :symbeg
+            @tokens[-1] = [:symbol, ":" + data]
+          else
+            @tokens << [token, data]
+          end
+        end
+
+        def on_program(*args)
+          args.first
+        end
+
+        def on_body_stmt(*args)
+          args.first
+        end
+        
+        def on_hash(*args)
+          visit_event AstNode.new(:hash, [args.first], listline: lineno..lineno, listchar: charno...charno)
+        end
+        
+        def on_assoc_new(*args)
+          AstNode.new(:assoc, args)
+        end
+        
+        def on_bare_assoc_hash(*args)
+          args.first
+        end
+        
+        def on_assoclist_from_args(*args)
+         args.first
+        end
+        
+        def on_qwords_new
+          visit_event AstNode.new(:qwords_literal, [])
+        end
+        
+        def on_string_literal(*args)
+          visit_event_arr AstNode.new(:string_literal, args)
+        end
+        
+        def on_string_content(*args)
+          AstNode.new(:string_content, args, listline: lineno..lineno, listchar: charno..charno)
+        end
+
+        def on_void_stmt
+          AstNode.new(:void_stmt, [], line: lineno..lineno, char: charno...charno)
+        end
+
+        def on_params(*args)
+          args.map! do |arg|
+            if arg.class == Array
+              if arg.first.class == Array
+                arg.map! do |sub_arg|
+                  if sub_arg.class == Array
+                    AstNode.new(:default_arg, sub_arg, listline: lineno..lineno, listchar: charno..charno)
+                  else
+                    sub_arg
+                  end
+                end
+              end
+              AstNode.new(:list, arg, listline: lineno..lineno, listchar: charno..charno)
+            else
+              arg
+            end
+          end
+          ParameterNode.new(:params, args, listline: lineno..lineno, listchar: charno..charno)
+        end
+
+        def on_comment(comment)
+          visit_ns_token(:comment, comment)
+          
+          comment = comment.gsub(/^\#{1,2}\s{0,1}/, '').chomp
+          append_comment = @comments[lineno - 1]
+          
+          if append_comment
+            @comments.delete(lineno - 1)
+            comment = append_comment + "\n" + comment
+          end
+          
+          @comments[lineno] = comment
+        end
+        
+        def on_parse_error(msg)
+          raise ParserSyntaxError, "syntax error in `#{file}`:(#{lineno},#{column}): #{msg}"
+        end
+        
+        def insert_comments
+          root.traverse do |node|
+            next if node.type == :list || node.parent.type != :list
+            node.line.downto(node.line - 2) do |line|
+              comment = @comments[line]
+              if comment && !comment.empty?
+                node.docstring = comment
+                comments.delete(line)
+                break
+              end
+            end
+          end
+        end
+        
+        def freeze_tree(node = nil)
+          node ||= root
+          node.children.each do |child|
+            child.parent = node
+            freeze_tree(child)
+          end
+        end
+      end
+    end
+  end
+end