jsduck 2.0.pre4 → 3.0.pre

data/lib/jsduck/js_parser.rb

@@ -1,11 +1,13 @@
 require 'jsduck/lexer'
 require 'jsduck/doc_parser'
+require 'jsduck/js_literal_parser'
+require 'jsduck/js_literal_builder'
 
 module JsDuck
 
-  class JsParser
+  class JsParser < JsLiteralParser
     def initialize(input)
-      @lex = Lexer.new(input)
+      super(input)
       @doc_parser = DocParser.new
       @docs = []
     end
@@ -86,7 +88,7 @@ module JsDuck
       elsif look(:ident) || look("this")
         maybe_assignment
       elsif look(:string)
-        {:type => :assignment, :left => [match(:string)]}
+        {:type => :assignment, :left => [match(:string)[:value]]}
       else
         {:type => :nop}
       end
@@ -97,7 +99,7 @@ module JsDuck
       match("function")
       return {
         :type => :function,
-        :name => look(:ident) ? match(:ident) : nil,
+        :name => look(:ident) ? match(:ident)[:value] : nil,
         :params => function_parameters,
         :body => function_body,
       }
@@ -106,9 +108,9 @@ module JsDuck
     # <function-parameters> := "(" [ <ident> [ "," <ident> ]* ] ")"
     def function_parameters
       match("(")
-      params = look(:ident) ? [{:name => match(:ident)}] : []
+      params = look(:ident) ? [{:name => match(:ident)[:value]}] : []
       while look(",", :ident) do
-        params << {:name => match(",", :ident)}
+        params << {:name => match(",", :ident)[:value]}
       end
       match(")")
       return params
@@ -142,33 +144,52 @@ module JsDuck
     # <ident-chain> := [ "this" | <ident> ]  [ "." <ident> ]*
     def ident_chain
       if look("this")
-        chain = [match("this")]
+        chain = [match("this")[:value]]
       else
-        chain = [match(:ident)]
+        chain = [match(:ident)[:value]]
       end
 
       while look(".", :ident) do
-        chain << match(".", :ident)
+        chain << match(".", :ident)[:value]
       end
       return chain
     end
 
     # <expression> := <function> | <ext-extend> | <literal>
-    # <literal> := <string> | <boolean> | <number> | <regex>
     def expression
       if look("function")
         function
       elsif look("Ext", ".", "extend")
         ext_extend
-      elsif look(:string)
-        {:type => :literal, :class => "String"}
-      elsif look("true") || look("false")
-        {:type => :literal, :class => "Boolean"}
-      elsif look(:number)
-        {:type => :literal, :class => "Number"}
-      elsif look(:regex)
-        {:type => :literal, :class => "RegExp"}
+      else
+        my_literal
+      end
+    end
+
+    # <literal> := ...see JsLiteralParser...
+    def my_literal
+      lit = literal
+      return unless lit
+
+      cls_map = {
+        :string => "String",
+        :number => "Number",
+        :regex => "RegExp",
+        :array => "Array",
+        :object => "Object",
+      }
+
+      if cls_map[lit[:type]]
+        cls = cls_map[lit[:type]]
+      elsif lit[:type] == :ident && (lit[:value] == "true" || lit[:value] == "false")
+        cls = "Boolean"
+      else
+        cls = nil
       end
+
+      value = JsLiteralBuilder.new.to_s(lit)
+
+      {:type => :literal, :class => cls, :value => value}
     end
 
     # <ext-extend> := "Ext" "." "extend" "(" <ident-chain> "," ...
@@ -184,7 +205,7 @@ module JsDuck
     def ext_define
       match("Ext", ".");
       look("define") ? match("define") : match("ClassManager", ".", "create");
-      name = match("(", :string)
+      name = match("(", :string)[:value]
 
       if look(",", "{")
         match(",")
@@ -199,103 +220,115 @@ module JsDuck
       cfg
     end
 
-    # <ext-define-cfg> := "{" ( <extend> | <mixins> | <alternate-class-name> | <alias> | <?> )*
+    # <ext-define-cfg> := "{" ( <extend> | <mixins> | <alternate-class-name> | <alias> |
+    #                           <requires> | <uses> | <singleton> | <?> )*
     def ext_define_cfg
       match("{")
       cfg = {}
       found = true
       while found
         found = false
-        if look("extend", ":", :string)
-          cfg[:extend] = ext_define_extend
-          found = true
-        elsif look("mixins", ":", "{")
-          cfg[:mixins] = ext_define_mixins
-          found = true
-        elsif look("alternateClassName", ":")
-          cfg[:alternateClassNames] = ext_define_alternate_class_names
-          found = true
-        elsif look("alias", ":")
-          cfg[:alias] = ext_define_alias
-          found = true
-        elsif look(:ident, ":")
-          match(:ident, ":")
-          if look(:string) || look(:number) || look(:regex) ||
-              look("true") || look("false") ||
-              look("null") || look("undefined")
-            # Some key with literal value -- ignore
-            @lex.next
-            found = true
-          elsif look("[")
-            # Some key with array of strings -- ignore
-            found = array_of_strings
-          end
+        if found = ext_define_extend
+          cfg[:extend] = found
+        elsif found = ext_define_mixins
+          cfg[:mixins] = found
+        elsif found = ext_define_alternate_class_name
+          cfg[:alternateClassNames] = found
+        elsif found = ext_define_alias
+          cfg[:alias] = found
+        elsif found = ext_define_requires
+          cfg[:requires] = found
+        elsif found = ext_define_uses
+          cfg[:uses] = found
+        elsif found = ext_define_singleton
+          cfg[:singleton] = found
+        elsif found = ext_define_whatever
+          # ignore this
         end
         match(",") if look(",")
       end
       cfg
     end
 
-    # <ext-define-extend> := "extend" ":" <string>
+    # <extend> := "extend" ":" <string>
     def ext_define_extend
-      match("extend", ":", :string)
+      if look("extend", ":", :string)
+        match("extend", ":", :string)[:value]
+      end
     end
 
-    # <ext-define-alternate-class-names> := "alternateClassName" ":" <string-or-list>
-    def ext_define_alternate_class_names
-      match("alternateClassName", ":")
-      string_or_list
+    # <mixins> := "mixins" ":" <object-literal>
+    def ext_define_mixins
+      if look("mixins", ":", "{")
+        match("mixins", ":")
+        lit = literal
+        lit && lit[:value].map {|x| x[:value][:value] }
+      end
     end
 
-    # <ext-define-alias> := "alias" ":" <string-or-list>
+    # <alternate-class-name> := "alternateClassName" ":" <string-or-list>
+    def ext_define_alternate_class_name
+      if look("alternateClassName", ":")
+        match("alternateClassName", ":")
+        string_or_list
+      end
+    end
+
+    # <alias> := "alias" ":" <string-or-list>
     def ext_define_alias
-      match("alias", ":")
-      string_or_list
+      if look("alias", ":")
+        match("alias", ":")
+        string_or_list
+      end
     end
 
-    # <string-or-list> := ( <string> | <array-of-strings> )
-    def string_or_list
-      if look(:string)
-        [ match(:string) ]
-      elsif look("[")
-        array_of_strings
-      else
-        []
+    # <requires> := "requires" ":" <string-or-list>
+    def ext_define_requires
+      if look("requires", ":")
+        match("requires", ":")
+        string_or_list
       end
     end
 
-    # <ext-define-mixins> := "mixins" ":" "{" [ <ident> ":" <string> ","? ]* "}"
-    def ext_define_mixins
-      match("mixins", ":", "{")
-      mixins = []
-      while look(:ident, ":", :string)
-        mixins << match(:ident, ":", :string)
-        match(",") if look(",")
+    # <uses> := "uses" ":" <string-or-list>
+    def ext_define_uses
+      if look("uses", ":")
+        match("uses", ":")
+        string_or_list
       end
-      match("}") if look("}")
-      mixins
     end
 
-    # <array-of-strings> := "[" [ <string> ","? ]* "]"
-    def array_of_strings
-      match("[")
-      strs = []
-      while look(:string)
-        strs << match(:string)
-        match(",") if look(",")
+    # <singleton> := "singleton" ":" "true"
+    def ext_define_singleton
+      if look("singleton", ":", "true")
+        match("singleton", ":", "true")
+        true
       end
+    end
 
-      if look("]")
-        match("]")
-        strs
+    # <?> := <ident> ":" <literal>
+    def ext_define_whatever
+      if look(:ident, ":")
+        match(:ident, ":")
+        literal
+      end
+    end
+
+    # <string-or-list> := ( <string> | <array-literal> )
+    def string_or_list
+      lit = literal
+      if lit && lit[:type] == :string
+        [ lit[:value] ]
+      elsif lit && lit[:type] == :array
+        lit[:value].map {|x| x[:value] }
       else
-        false
+        []
       end
     end
 
     # <property-literal> := ( <ident> | <string> ) ":" <expression>
     def property_literal
-      left = look(:ident) ? match(:ident) : match(:string)
+      left = look(:ident) ? match(:ident)[:value] : match(:string)[:value]
       match(":")
       right = expression
       return {
@@ -305,21 +338,6 @@ module JsDuck
       }
     end
 
-    # Matches all arguments, returns the value of last match
-    # When the whole sequence doesn't match, throws exception
-    def match(*args)
-      if look(*args)
-        last = nil
-        args.length.times { last = @lex.next }
-        last
-      else
-        throw "Expected: " + args.join(", ")
-      end
-    end
-
-    def look(*args)
-      @lex.look(*args)
-    end
   end
 
 end

data/lib/jsduck/json_duck.rb

@@ -0,0 +1,45 @@
+require 'json'
+
+module JsDuck
+
+  # Wrapper around the json gem for use in JsDuck.
+  #
+  # The main benefit of it is that we have a central place for
+  # controlling how the JSON is created (pretty-formatted or not).
+  class JsonDuck
+    @@pretty = false
+
+    # Set to true to turn on pretty-formatting of JSON
+    def self.pretty=(pretty)
+      @@pretty = pretty
+    end
+
+    # Turns object into JSON, places it inside JavaScript that calls the
+    # given callback name, and writes the result to file.
+    def self.write_jsonp(filename, callback_name, data)
+      jsonp = "Ext.data.JsonP." + callback_name + "(" + self.generate(data) + ");"
+      File.open(filename, 'w') {|f| f.write(jsonp) }
+    end
+
+    # Turns object into JSON and writes inside a file
+    def self.write_json(filename, data)
+      File.open(filename, 'w') {|f| f.write(self.generate(data)) }
+    end
+
+    # Generates JSON from object
+    def self.generate(data)
+      @@pretty ? JSON.pretty_generate(data) : JSON.generate(data)
+    end
+
+    # Reads and parses JSON from file
+    def self.read(filename)
+      self.parse(IO.read(filename))
+    end
+
+    # Parses JSON string
+    def self.parse(string)
+      JSON.parse(string)
+    end
+  end
+
+end

data/lib/jsduck/lexer.rb

@@ -19,9 +19,15 @@ module JsDuck
   # comments are ignored just as whitespace.
   #
   class Lexer
+    # Input can be either a String or StringScanner.
+    #
+    # In the latter case we ensure that only #next will advance the
+    # scanpointer of StringScanner - this allows context-switching
+    # while parsing some string.  Specifically we need this feature to
+    # parse some JavaScript inside doc-comments.
     def initialize(input)
-      @input = StringScanner.new(input)
-      tokenize
+      @input = input.is_a?(StringScanner) ? input : StringScanner.new(input)
+      @buffer = []
     end
 
     # Tests if given pattern matches the tokens that follow at current
@@ -33,12 +39,14 @@ module JsDuck
     #     look(:ident, "=", :regex)
     #
     def look(*tokens)
+      buffer_tokens(tokens.length)
       i = 0
       tokens.all? do |t|
-        tok = @tokens[i]
+        tok = @buffer[i]
         i += 1
-        return false if tok == nil
-        if t.instance_of?(Symbol)
+        if !tok
+          false
+        elsif t.instance_of?(Symbol)
           tok[:type] == t
         else
           tok[:value] == t
@@ -57,16 +65,39 @@ module JsDuck
     # pointing to the line where the doc-comment began.
     #
     def next(full=false)
-      tok = @tokens.shift
+      buffer_tokens(1)
+      tok = @buffer.shift
+      # advance the scanpointer to the position after this token
+      @input.pos = tok[:pos]
       full ? tok : tok[:value]
     end
 
     # True when no more tokens.
     def empty?
-      @tokens.empty?
+      buffer_tokens(1)
+      return !@buffer.first
+    end
+
+    # Ensures next n tokens are read in buffer
+    #
+    # At the end of buffering the initial position scanpointer is
+    # restored.  Only the #next method will advance the scanpointer in
+    # a way that's visible outside this class.
+    def buffer_tokens(n)
+      prev_pos = @input.pos
+      @input.pos = @buffer.last[:pos] if @buffer.last
+      (n - @buffer.length).times do
+        @previous_token = tok = next_token
+        if tok
+          # remember scanpointer position after each token
+          tok[:pos] = @input.pos
+          @buffer << tok
+        end
+      end
+      @input.pos = prev_pos
     end
 
-    # Goes through the whole input and tokenizes it
+    # Parses out next token from input stream.
     #
     # For efficency we look for tokens in order of frequency in
     # JavaScript source code:
@@ -79,36 +110,35 @@ module JsDuck
     # The remaining token types are less frequent, so these are left
     # to the end.
     #
-    def tokenize
-      @tokens = []
+    def next_token
       while !@input.eos? do
         skip_white
         if @input.check(/[.(),;={}:]/)
-          @tokens << {
+          return {
             :type => :operator,
             :value => @input.scan(/./)
           }
         elsif @input.check(/[a-zA-Z_]/)
           value = @input.scan(/\w+/)
-          @tokens << {
+          return {
             :type => KEYWORDS[value] ? :keyword : :ident,
             :value => value
           }
         elsif @input.check(/'/)
-          @tokens << {
+          return {
             :type => :string,
-            :value => @input.scan(/'([^'\\]|\\.)*'/).sub(/^'(.*)'$/, "\\1")
+            :value => @input.scan(/'([^'\\]|\\.)*'/m).sub(/^'(.*)'$/m, "\\1")
           }
         elsif @input.check(/"/)
-          @tokens << {
+          return {
             :type => :string,
-            :value => @input.scan(/"([^"\\]|\\.)*"/).sub(/^"(.*)"$/, "\\1")
+            :value => @input.scan(/"([^"\\]|\\.)*"/m).sub(/^"(.*)"$/m, "\\1")
           }
         elsif @input.check(/\//)
           # Several things begin with dash:
           # - comments, regexes, division-operators
           if @input.check(/\/\*\*/)
-            @tokens << {
+            return {
               :type => :doc_comment,
               # Calculate current line number, starting with 1
               :linenr => @input.string[0...@input.pos].count("\n") + 1,
@@ -121,30 +151,30 @@ module JsDuck
             # skip line comment
             @input.scan_until(/\n|\Z/)
           elsif regex?
-            @tokens << {
+            return {
               :type => :regex,
               :value => @input.scan(/\/([^\/\\]|\\.)*\/[gim]*/)
             }
           else
-            @tokens << {
+            return {
               :type => :operator,
               :value => @input.scan(/\//)
             }
           end
         elsif @input.check(/[0-9]+/)
           nr = @input.scan(/[0-9]+(\.[0-9]*)?/)
-          @tokens << {
+          return {
             :type => :number,
             :value => nr
           }
         elsif @input.check(/\$/)
           value = @input.scan(/\$\w*/)
-          @tokens << {
+          return {
             :type => :ident,
             :value => value
           }
         elsif  @input.check(/./)
-          @tokens << {
+          return {
             :type => :operator,
             :value => @input.scan(/./)
           }
@@ -160,9 +190,9 @@ module JsDuck
     # - closing square-bracket ]
     # Otherwise it's a beginning of regex
     def regex?
-      if @tokens.last
-        type = @tokens.last[:type]
-        value = @tokens.last[:value]
+      if @previous_token
+        type = @previous_token[:type]
+        value = @previous_token[:value]
         if type == :ident || type == :number
           return false
         elsif type == :keyword && value == "this"

data/lib/jsduck/lint.rb

@@ -0,0 +1,84 @@
+require 'jsduck/logger'
+
+module JsDuck
+
+  # Reports bugs and problems in documentation
+  class Lint
+    attr_accessor :relations
+
+    def initialize(relations)
+      @relations = relations
+    end
+
+    # Runs the linter
+    def run
+      warn_globals
+      warn_unnamed
+      warn_optional_params
+      warn_duplicate_params
+    end
+
+    # print warning for each global member
+    def warn_globals
+      global = @relations["global"]
+      return unless global
+      global.each_member do |member|
+        warn("Global #{member[:tagname]}: #{member[:name]}", member)
+      end
+    end
+
+    # print warning for each member or parameter with no name
+    def warn_unnamed
+      each_member do |member|
+        if !member[:name] || member[:name] == ""
+          warn("Unnamed #{member[:tagname]}", member)
+        end
+        (member[:params] || []).each do |p|
+          if !p[:name] || p[:name] == ""
+            warn("Unnamed parameter", member)
+          end
+        end
+      end
+    end
+
+    # print warning for each non-optional parameter that follows an optional parameter
+    def warn_optional_params
+      each_member do |member|
+        if member[:tagname] == :method
+          optional_found = false
+          member[:params].each do |p|
+            if optional_found && !p[:optional]
+              warn("Optional param can't be followed by regular param #{p[:name]}", member)
+            end
+            optional_found = optional_found || p[:optional]
+          end
+        end
+      end
+    end
+
+    # print warnings for duplicate parameter names
+    def warn_duplicate_params
+      each_member do |member|
+        params = {}
+        (member[:params] || []).each do |p|
+          if params[p[:name]]
+            warn("Duplicate parameter name #{p[:name]}", member)
+          end
+          params[p[:name]] = true
+        end
+      end
+    end
+
+    # Loops through all members of all classes
+    def each_member(&block)
+      @relations.each {|cls| cls.each_member(&block) }
+    end
+
+    # Prints warning + filename and linenumber from doc-context
+    def warn(msg, context)
+      Logger.instance.warn(msg + " in #{context[:filename]} line #{context[:linenr]}")
+    end
+
+  end
+
+end