jsduck 3.11.2 → 4.0.beta

data/lib/jsduck/js_parser.rb

@@ -1,430 +1,219 @@
-require 'jsduck/lexer'
-require 'jsduck/doc_parser'
-require 'jsduck/js_literal_parser'
-require 'jsduck/js_literal_builder'
+require 'jsduck/esprima'
 
 module JsDuck
 
-  class JsParser < JsLiteralParser
+  # JavaScript parser that internally uses Esprima.js
+  class JsParser
+
+    # Initializes the parser with JavaScript source code to be parsed.
     def initialize(input, options = {})
-      super(input)
-      @doc_parser = DocParser.new
-      @docs = []
-      @ext_namespaces = (options[:ext_namespaces] || ["Ext"]).map {|ns| tokenize_ns(ns) }
-    end
+      @input = input
 
-    # Splits namespace string into array like so:
-    #
-    # "Foo.Bar.Baz" --> ["Foo", ".", "Bar", ".", "Baz"]
-    #
-    def tokenize_ns(ns)
-      ns.split(".").reduce([]) do |res, x|
-        res << "." unless res.length == 0
-        res << x
-      end
+      # Initialize line number counting
+      @start_index = 0
+      @start_linenr = 1
     end
 
-    # Parses the whole JavaScript block and returns array where for
-    # each doc-comment there is a hash of three values: the comment
-    # structure created by DocParser, number of the line where the
-    # comment starts, and parsed structure of the code that
-    # immediately follows the comment.
+    # Parses JavaScript source code and returns array of hashes like this:
     #
-    # For example with the following JavaScript input:
-    #
-    # /**
-    #  * @param {String} foo
-    #  */
-    # MyClass.doIt = function(foo, bar) {
-    # }
-    #
-    # The return value of this function will be:
-    #
-    # [
-    #   {
-    #     :comment => [
-    #       {:tagname => :default, :doc => "Method description"},
-    #       {:tagname => :return, :type => "Number", :doc => ""},
-    #     ],
-    #     :linenr => 1,
-    #     :code => {
-    #       :type => :assignment,
-    #       :left => ["MyClass", "doIt"],
-    #       :right => {
-    #         :type => :function,
-    #         :name => nil,
-    #         :params => [
-    #           {:name => "foo"},
-    #           {:name => "bar"}
-    #         ]
-    #       }
+    #     {
+    #         :comment => "The contents of the comment",
+    #         :code => {...AST data structure for code following the comment...},
+    #         :linenr => 12,  // Beginning with 1
+    #         :type => :doc_comment, // or :plain_comment
     #     }
-    #   }
-    # ]
     #
     def parse
-      while !@lex.empty? do
-        if look(:doc_comment)
-          comment = @lex.next(true)
-          @docs << {
-            :comment => @doc_parser.parse(comment[:value]),
-            :linenr => comment[:linenr],
-            :code => code_block
-          }
-        else
-          @lex.next
-        end
-      end
-      @docs
-    end
+      @ast = Esprima.instance.parse(@input)
 
-    # The following is a recursive-descent parser for JavaScript that
-    # can possibly follow a doc-comment
-
-    # <code-block> := <function> | <var-declaration> | <ext-define> |
-    #                 <assignment> | <property-literal>
-    def code_block
-      if look(:function)
-        function
-      elsif look(:var)
-        var_declaration
-      elsif ext_look(:ns, ".", "define", "(", :string)
-        ext_define(:ns, ".", "define", "(", :string)
-      elsif ext_look(:ns, ".", "ClassManager", ".", "create", "(", :string)
-        ext_define(:ns, ".", "ClassManager", ".", "create", "(", :string)
-      elsif look(:ident, ":") || look(:string, ":")
-        property_literal
-      elsif look(",", :ident, ":") || look(",", :string, ":")
-        match(",")
-        property_literal
-      elsif look(:ident) || look(:this)
-        maybe_assignment
-      elsif look(:string)
-        {:type => :assignment, :left => [match(:string)[:value]]}
-      else
-        {:type => :nop}
-      end
+      @ast["comments"] = merge_comments(@ast["comments"])
+      locate_comments
     end
 
-    # <function> := "function" [ <ident> ] <function-parameters> <function-body>
-    def function
-      match(:function)
-      return {
-        :type => :function,
-        :name => look(:ident) ? match(:ident)[:value] : "",
-        :params => function_parameters,
-        :body => function_body,
-      }
-    end
-
-    # <ext-emptyfn> := "Ext" "." "emptyFn"
-    def ext_emptyfn
-      match(:ident, ".", "emptyFn")
-      return {
-        :type => :function,
-        :name => "",
-        :params => [],
-      }
-    end
+    private
 
-    # <function-parameters> := "(" [ <ident> [ "," <ident> ]* ] ")"
-    def function_parameters
-      match("(")
-      params = look(:ident) ? [{:name => match(:ident)[:value]}] : []
-      while look(",", :ident) do
-        params << {:name => match(",", :ident)[:value]}
-      end
-      match(")")
-      return params
-    end
+    # Merges consecutive line-comments and Establishes links between
+    # comments, so we can easily use comment["next"] to get to the
+    # next comment.
+    def merge_comments(original_comments)
+      result = []
 
-    # <function-body> := "{" ...
-    def function_body
-      match("{")
-    end
+      comment = original_comments[0]
+      i = 0
 
-    # <var-declaration> := "var" <assignment>
-    def var_declaration
-      match(:var)
-      maybe_assignment
-    end
+      while comment
+        i += 1
+        next_comment = original_comments[i]
 
-    # <maybe-assignment> := <ident-chain> ( "=" <expression> | ";" | "," )
-    def maybe_assignment
-      left = ident_chain
-      if look("=")
-        match("=")
-        right = expression
-      elsif look(";")
-        match(";")
-        right = nil
-      elsif look(",")
-        match(",")
-        right = nil
-      else
-        return {:type => :nop}
+        if next_comment && mergeable?(comment, next_comment)
+          # Merge next comment to current one
+          comment["value"] += "\n" + next_comment["value"]
+          comment["range"][1] = next_comment["range"][1]
+        else
+          # Create a link and continue with next comment
+          comment["next"] = next_comment
+          result << comment
+          comment = next_comment
+        end
       end
 
-      return {
-        :type => :assignment,
-        :left => left,
-        :right => right,
-      }
+      result
     end
 
-    # <ident-chain> := [ "this" | <ident> ]  [ "." <ident> ]*
-    def ident_chain
-      if look(:this)
-        match(:this)
-        chain = ["this"]
+    # Two comments can be merged if they are both line-comments and
+    # they are separated only by whitespace (but no newlines)
+    def mergeable?(c1, c2)
+      if c1["type"] == "Line" && c2["type"] == "Line"
+        /\A[ \t]*\Z/ =~ @input.slice((c1["range"][1])..(c2["range"][0]-1))
       else
-        chain = [match(:ident)[:value]]
-      end
-
-      while look(".", :ident) do
-        chain << match(".", :ident)[:value]
+        false
       end
-      return chain
     end
 
-    # <expression> := <function> | <ext-extend> | <ext-emptyfn> | <ext-base-css-prefix> | <literal>
-    def expression
-      if look(:function)
-        function
-      elsif ext_look(:ns, ".", "extend")
-        ext_extend
-      elsif ext_look(:ns, ".", "emptyFn")
-        ext_emptyfn
-      elsif ext_look(:ns, ".", "baseCSSPrefix", "+", :string)
-        ext_base_css_prefix
-      else
-        my_literal
-      end
-    end
+    def locate_comments
+      @ast["comments"].map do |comment|
+        # Detect comment type and strip * at the beginning of doc-comment
+        value = comment["value"]
+        if comment["type"] == "Block" && value =~ /\A\*/
+          type = :doc_comment
+          value = value.slice(1, value.length-1)
+        else
+          type = :plain_comment
+        end
 
-    # <literal> := ...see JsLiteralParser...
-    def my_literal
-      lit = literal
-      return unless lit && literal_expression_end?
-
-      cls_map = {
-        :string => "String",
-        :number => "Number",
-        :regex => "RegExp",
-        :array => "Array",
-        :object => "Object",
-      }
-
-      if cls_map[lit[:type]]
-        cls = cls_map[lit[:type]]
-      elsif lit[:type] == :ident && (lit[:value] == "true" || lit[:value] == "false")
-        cls = "Boolean"
-      else
-        cls = nil
+        {
+          :comment => value,
+          :code => stuff_after(comment),
+          :linenr => line_number(comment["range"][0]),
+          :type => type,
+        }
       end
-
-      value = JsLiteralBuilder.new.to_s(lit)
-
-      {:type => :literal, :class => cls, :value => value}
-    end
-
-    # True when we're at the end of literal expression.
-    # ",", ";" and "}" are the normal closing symbols, but for
-    # our docs purposes doc-comment and file end work too.
-    def literal_expression_end?
-      look(",") || look(";") || look("}") || look(:doc_comment) || @lex.empty?
     end
 
-    # <ext-extend> := "Ext" "." "extend" "(" <ident-chain> "," ...
-    def ext_extend
-      match(:ident, ".", "extend", "(")
-      return {
-        :type => :ext_extend,
-        :extend => ident_chain,
-      }
+    # Given index inside input string, returns the corresponding line number
+    def line_number(index)
+      # To speed things up, remember the index until which we counted,
+      # then next time just begin counting from there.  This way we
+      # only count each line once.
+      @start_linenr = @input[@start_index...index].count("\n") + @start_linenr
+      @start_index = index
+      return @start_linenr
     end
 
-    # <ext-base-css-prefix> := "Ext" "." "baseCSSPrefix" "+" <string>
-    def ext_base_css_prefix
-      match(:ident, ".", "baseCSSPrefix", "+")
-      return {
-        :type => :literal,
-        :class => "String",
-        :value => '"x-' + match(:string)[:value] + '"',
-      }
-    end
-
-    # <ext-define> := "Ext" "." ["define" | "ClassManager" "." "create" ] "(" <string> "," <ext-define-cfg>
-    def ext_define(*pattern)
-      name = ext_match(*pattern)[:value]
-
-      if look(",", "{")
-        match(",")
-        cfg = ext_define_cfg
+    # Sees if there is some code following the comment.
+    # Returns the code found.  But if the comment is instead
+    # followed by another comment, returns nil.
+    def stuff_after(comment)
+      code = code_after(comment["range"], @ast)
+      if code && comment["next"]
+        return code["range"][0] < comment["next"]["range"][0] ? code : nil
       else
-        cfg = {}
+        code
       end
-
-      cfg[:type] = :ext_define
-      cfg[:name] = name
-
-      cfg
     end
 
-    # <ext-define-cfg> := "{" ( <extend> | <mixins> | <alternate-class-name> | <alias> |
-    #                           <xtype> | <requires> | <uses> | <singleton> | <?> )*
-    def ext_define_cfg
-      match("{")
-      cfg = {}
-      found = true
-      while found
-        found = false
-        if found = ext_define_extend
-          cfg[:extend] = found
-        elsif found = ext_define_mixins
-          cfg[:mixins] = found
-        elsif found = ext_define_alternate_class_name
-          cfg[:alternateClassNames] = found
-        elsif found = ext_define_alias
-          cfg[:alias] = found
-        elsif found = ext_define_xtype
-          cfg[:xtype] = found
-        elsif found = ext_define_requires
-          cfg[:requires] = found
-        elsif found = ext_define_uses
-          cfg[:uses] = found
-        elsif found = ext_define_singleton
-          cfg[:singleton] = found
-        elsif found = ext_define_whatever
-          # ignore this
+    # Looks for code following the given range.
+    #
+    # The second argument is the parent node within which we perform
+    # our search.
+    def code_after(range, parent)
+      # Look through all child nodes of parent...
+      child_nodes(parent).each do |node|
+        if less(range, node["range"])
+          # If node is after our range, then that's it.  There could
+          # be comments in our way, but that's taken care of in
+          # #stuff_after method.
+          return node
+        elsif within(range, node["range"])
+          # Our range is within the node --> recurse
+          return code_after(range, node)
         end
-        match(",") if look(",")
       end
-      cfg
-    end
 
-    # <extend> := "extend" ":" <string>
-    def ext_define_extend
-      if look("extend", ":", :string)
-        match("extend", ":", :string)[:value]
-      end
+      return nil
     end
 
-    # <mixins> := "mixins" ":" [ <object-literal> | <array-literal> ]
-    def ext_define_mixins
-      if look("mixins", ":")
-        match("mixins", ":")
-        lit = literal
-        if lit && lit[:type] == :object
-          lit[:value].map {|x| x[:value][:value] }
-        elsif lit && lit[:type] == :array
-          lit[:value].map {|x| x[:value] }
-        else
-          nil
-        end
-      end
-    end
 
-    # <alternate-class-name> := "alternateClassName" ":" <string-or-list>
-    def ext_define_alternate_class_name
-      if look("alternateClassName", ":")
-        match("alternateClassName", ":")
-        string_or_list
-      end
+    # True if range A is less than range B
+    def less(a, b)
+      return a[1] < b[0]
     end
 
-    # <alias> := "alias" ":" <string-or-list>
-    def ext_define_alias
-      if look("alias", ":")
-        match("alias", ":")
-        string_or_list
-      end
+    # True if range A is greater than range B
+    def greater(a, b)
+      return a[0] > b[1]
     end
 
-    # <xtype> := "xtype" ":" <string-or-list>
-    def ext_define_xtype
-      if look("xtype", ":")
-        match("xtype", ":")
-        string_or_list
-      end
+    # True if range A is within range B
+    def within(a, b)
+      return b[0] < a[0] && a[1] < b[1]
     end
 
-    # <requires> := "requires" ":" <string-or-list>
-    def ext_define_requires
-      if look("requires", ":")
-        match("requires", ":")
-        string_or_list
-      end
-    end
 
-    # <uses> := "uses" ":" <string-or-list>
-    def ext_define_uses
-      if look("uses", ":")
-        match("uses", ":")
-        string_or_list
-      end
-    end
+    # Returns array of child nodes of given node
+    def child_nodes(node)
+      properties = NODE_TYPES[node["type"]]
 
-    # <singleton> := "singleton" ":" "true"
-    def ext_define_singleton
-      if look("singleton", ":", "true")
-        match("singleton", ":", "true")
-        true
+      unless properties
+        puts "Unknown node type: "+node["type"]
+        exit(1)
       end
-    end
-
-    # <?> := <ident> ":" <literal>
-    def ext_define_whatever
-      if look(:ident, ":")
-        match(:ident, ":")
-        literal
-      end
-    end
-
-    # <string-or-list> := ( <string> | <array-literal> )
-    def string_or_list
-      lit = literal
-      if lit && lit[:type] == :string
-        [ lit[:value] ]
-      elsif lit && lit[:type] == :array
-        lit[:value].map {|x| x[:value] }
-      else
-        []
-      end
-    end
 
-    # <property-literal> := ( <ident> | <string> ) ":" <expression>
-    def property_literal
-      left = look(:ident) ? match(:ident)[:value] : match(:string)[:value]
-      match(":")
-      right = expression
-      return {
-        :type => :assignment,
-        :left => [left],
-        :right => right,
-      }
+      properties.map {|p| node[p] }.compact.flatten
     end
 
-    # Like look() but tries to match as the first argument all the
-    # names listed in @ext_namespaces
-    def ext_look(placeholder, *args)
-      @ext_namespaces.each do |ns|
-        return true if look(*(ns + args))
-      end
-      return false
-    end
-
-    # Like match() but tries as the first argument all the names
-    # listed in @ext_namespaces
-    def ext_match(placeholder, *args)
-      @ext_namespaces.each do |ns|
-        pattern = ns + args
-        if look(*pattern)
-          return match(*pattern)
-        end
-      end
-    end
+    # All possible node types in Esprima-created abstract syntax tree
+    #
+    # Each node type maps to list of properties of that node into
+    # which we can recurse for further parsing.
+    NODE_TYPES = {
+      "Program" => ["body"],
+
+      "BlockStatement" => ["body"],
+      "BreakStatement" => [],
+      "ContinueStatement" => [],
+      "DoWhileStatement" => ["body", "test"],
+      "DebuggerStatement" => [],
+      "EmptyStatement" => [],
+      "ExpressionStatement" => ["expression"],
+      "ForStatement" => ["init", "test", "update", "body"],
+      "ForInStatement" => ["left", "right", "body"],
+      "IfStatement" => ["test", "consequent", "alternate"],
+      "LabeledStatement" => ["body"],
+      "ReturnStatement" => ["argument"],
+      "SwitchStatement" => ["discriminant", "cases"],
+      "SwitchCase" => ["test", "consequent"],
+      "ThrowStatement" => ["argument"],
+      "TryStatement" => ["block", "handlers", "finalizer"],
+      "CatchClause" => ["param", "body"],
+      "WhileStatement" => ["test", "body"],
+      "WithStatement" => ["object", "body"],
+
+      "FunctionDeclaration" => ["id", "params", "body"],
+      "VariableDeclaration" => ["declarations"],
+      "VariableDeclarator" => ["id", "init"],
+
+      "AssignmentExpression" => ["left", "right"],
+      "ArrayExpression" => ["elements"],
+      "BinaryExpression" => ["left", "right"],
+      "CallExpression" => ["callee", "arguments"],
+      "ConditionalExpression" => ["test", "consequent", "alternate"],
+      "FunctionExpression" => ["body"],
+
+      "LogicalExpression" => ["left", "right"],
+      "MemberExpression" => ["object", "property"],
+      "NewExpression" => ["callee", "arguments"],
+      "ObjectExpression" => ["properties"],
+      "Property" => ["key", "value"],
+
+      "SequenceExpression" => ["expressions"],
+      "ThisExpression" => [],
+      "UnaryExpression" => ["argument"],
+      "UpdateExpression" => ["argument"],
+
+      "Identifier" => [],
+      "Literal" => [],
+    }
 
   end
-
 end