jsduck 5.2.0 → 5.3.0

checksums.yaml

@@ -1,15 +1,15 @@
 ---
 !binary "U0hBMQ==":
   metadata.gz: !binary |-
-    YTQ4ZGE0ZGEzMTQxNDY1ZjhiY2FhNTliMmJjNTQ0OGY2NTZkY2VhMA==
+    Yjc3MDZjMmZlYWNkOGMzNzQ4MTdlZTA2MGU4M2Y3OWE1MmE5OWQ4MA==
   data.tar.gz: !binary |-
-    ZWEzMjZmMGEzOTBmOGM4OGQ4NzZiY2NhMjA5OTZkZGVlMzdhMTc4NQ==
+    ZDkwMmFjOWQ1MTRlYzYxYWIzZjRhNThjMDlhNDc2YzBhZDU1ZjA0OQ==
 !binary "U0hBNTEy":
   metadata.gz: !binary |-
-    ZjA3N2ZjODkwZjNhNjBiNWExNTNlZjI0YzA4Y2EyZmZjZDk5MWUyMzY1Y2Iy
-    NTlmYWY0MjJlODhjZjgwZGE3ZmY2YzZjZWI3MWE5OWVhM2ZjN2Q3YWJlODQy
-    Zjg0OGM1NDZmNTdkNDExNmJlN2ZmZTU4ODkxYWI1YTM3MWFhMjk=
+    MjBlMjA4YzMwNTQzOTFmODE4MDcxNmJiZTU1OGMxY2FjOWVjZTNkZjViZDA3
+    ZDk3MTZlNGFmODU3Y2NlN2QwMTc5YjI0ODc1Yzc0MGJkMzUyMDI1M2Y3N2U4
+    ODViMjRlNDg4NDVhYWU2NDk2YWM5ZjRjMzk2ZDlkMjQ4NDNmY2U=
   data.tar.gz: !binary |-
-    NTc0ZDQ1NDMxYTU5MDhiMTYyMTFlN2RiODNlYTczYThkZjUzMDFkNzk4NTU3
-    NzY5YzVkNDMyYzA0NWI1MjM5NWQ4M2VkYzI2ZDcxNzQ0MGI0Y2Y5OWVkNDYz
-    MjE0MjFiZjRjNWIxNWYzMjcxMWU2YjNkMjY3M2VmNTZmMzg2ODY=
+    M2U1ZTRiZDk0NjNiODUwZjNmMTRmYjIwNTA3NDg4ZTlkMGQ0MzZhYzY4YzA0
+    NmNiOWIwYmUwNTI0YjhkMDMzMzA3NTRjZWU0ZWE5YjlkNmM4Mzk1YzFmNTAy
+    NTU4ZDg2ZjBiMzU5ZGFjNzI4OTZjZTM3NTFlYzE2MDFiNjVlZGI=

data/README.md

@@ -98,6 +98,8 @@ Who's using JSDuck?
 - [CKEditor](http://docs.ckeditor.com)
 - [GeoExt 2](https://github.com/geoext/geoext2)
 - Rally Software [Rally App SDK](https://prod.help.rallydev.com/apps/2.0rc1/doc/)
+- Wikimedia Foundation [Mediawiki](https://doc.wikimedia.org/mediawiki-core/master/js/)
+  and [VisualEditor](https://doc.wikimedia.org/VisualEditor/master/)
 - [Sencha](http://docs.sencha.com) - obviously :)
 
 These are some that we know of. Want your project listed here? Drop us a line.

data/lib/jsduck/batch_parser.rb

@@ -3,6 +3,7 @@ require 'jsduck/util/io'
 require 'jsduck/parser'
 require 'jsduck/source/file'
 require 'jsduck/logger'
+require 'jsduck/cache'
 
 module JsDuck
 
@@ -11,17 +12,33 @@ module JsDuck
   class BatchParser
 
     def self.parse(opts)
-      Util::Parallel.map(opts.input_files) do |fname|
+      cache = Cache.create(opts)
+
+      results = Util::Parallel.map(opts.input_files) do |fname|
         Logger.log("Parsing", fname)
+
         begin
           source = Util::IO.read(fname)
-          docs = Parser.new.parse(source, fname, opts)
-          Source::File.new(source, docs, fname)
+          docs = nil
+
+          unless docs = cache.read(fname, source)
+            docs = Parser.new.parse(source, fname, opts)
+            cache.write(fname, source, docs)
+          end
+
+          {
+            :file => Source::File.new(source, docs, fname),
+            :cache => cache.previous_entry,
+          }
         rescue
           Logger.fatal_backtrace("Error while parsing #{fname}", $!)
           exit(1)
         end
       end
+
+      cache.cleanup( results.map {|r| r[:cache] }.compact )
+
+      return results.map {|r| r[:file] }
     end
 
   end

data/lib/jsduck/cache.rb

@@ -0,0 +1,137 @@
+require 'digest/md5'
+require 'fileutils'
+require 'jsduck/util/null_object'
+require 'set'
+
+module JsDuck
+
+  # Reads/writes parsed files in cache.
+  #
+  # When writing to cache:
+  #
+  # - makes MD5 hash of <file name> + <file contents>
+  # - Dumps the the parsed data structure using Marshal into <md5>.dat
+  #
+  # When reading from cache:
+  #
+  # - makes MD5 hash of <file name> + <file contents>
+  # - Reads the parsed data structure using Marshal from <md5>.dat
+  #
+  # Additionally a manifest.txt file is saved into the cache
+  # directory, the contents of which is a string like the following:
+  #
+  #     Ruby: 1.9.3, JSDuck: 5.2.0
+  #
+  # This file is consulted before all other cache operations.  When
+  # the version numbers in there don't match with current Ruby and
+  # JSDuck versions, the whole cache gets invalidated - all cached
+  # files get deleted.  This is to avoid problems with the Marshal
+  # file format changes between Ruby versions and parsed data
+  # structure changes between JSDuck versions.
+  #
+  # After all files have been checked into cache, the files that
+  # weren't touched get deleted (using the #cleanup method).  This
+  # ensures that the number of files in cache only grows when more
+  # files are added to the documentation.
+  #
+  class Cache
+
+    # Factory method to produce a cache object.  When caching is
+    # disabled, returns a NullObject which emulates a cache that's
+    # always empty.
+    def self.create(opts)
+      # Check also for cache_dir, which will be nil when output_dir is :stdout
+      if opts.cache && opts.cache_dir
+        Cache.new(opts)
+      else
+        Util::NullObject.new(
+          :read => nil,
+          :write => nil,
+          :previous_entry => nil,
+          :cleanup => nil
+          )
+      end
+    end
+
+    # The name of the cache file that was previously read or written.
+    # When the #read call failed to find the file, it will be nil.
+    # But it will always be available after the #write call.
+    attr_reader :previous_entry
+
+    def initialize(opts)
+      @jsduck_version = opts.version
+      @cache_dir = opts.cache_dir
+      @manifest_file = @cache_dir + "/manifest.txt"
+      @previous_entry = nil
+
+      FileUtils.mkdir_p(@cache_dir) unless File.exists?(@cache_dir)
+
+      # Invalidate the whole cache when it was generated with a
+      # different Ruby and/or JSDuck version.
+      invalidate_all! unless valid_manifest?
+    end
+
+    # Given the name and contents of a source file, reads the already
+    # parsed data structure from cache.  Returns nil when not found.
+    def read(file_name, file_contents)
+      fname = cache_file_name(file_name, file_contents)
+      if File.exists?(fname)
+        @previous_entry = fname
+        File.open(fname, "rb") {|file| Marshal::load(file) }
+      else
+        @previous_entry = nil
+        nil
+      end
+    end
+
+    # Writes parse data into cache under a name generated from the
+    # name and contents of a source file.
+    def write(file_name, file_contents, data)
+      fname = cache_file_name(file_name, file_contents)
+      @previous_entry = fname
+      File.open(fname, "wb") {|file| Marshal::dump(data, file) }
+    end
+
+    # Given listing of used cache files (those that were either read
+    # or written during this jsduck run) removes rest of the files
+    # from cache directory that were unused.
+    def cleanup(used_cache_entries)
+      used = Set.new(used_cache_entries)
+
+      Dir[@cache_dir + "/*.dat"].each do |file|
+        FileUtils.rm_rf(file) unless used.include?(file)
+      end
+    end
+
+    private
+
+    def cache_file_name(file_name, file_contents)
+      @cache_dir + "/" + md5(file_name + file_contents) + ".dat"
+    end
+
+    def md5(string)
+      Digest::MD5.hexdigest(string)
+    end
+
+    def valid_manifest?
+      manifest = File.exists?(@manifest_file) ? Util::IO.read(@manifest_file) : ""
+      return manifest == current_manifest
+    end
+
+    def invalidate_all!
+      FileUtils.rm_rf(@cache_dir)
+      FileUtils.mkdir(@cache_dir)
+      save_manifest
+    end
+
+    def save_manifest
+      File.open(@manifest_file, "w") {|f| f.write(current_manifest) }
+    end
+
+    def current_manifest
+      "Ruby: #{RUBY_VERSION}, JSDuck: #{@jsduck_version}\n"
+    end
+
+  end
+
+end

data/lib/jsduck/class_writer.rb

@@ -29,7 +29,8 @@ module JsDuck
     end
 
     def write_dir(dir, extension)
-      FileUtils.mkdir(dir)
+      FileUtils.mkdir(dir) unless File.exists?(dir)
+
       Util::Parallel.each(@relations.classes) do |cls|
         filename = dir + "/" + cls[:name] + extension
         Logger.log("Writing docs", filename)

data/lib/jsduck/export_writer.rb

@@ -4,6 +4,7 @@ require 'jsduck/exporter/examples'
 require 'jsduck/format/batch'
 require 'jsduck/class_writer'
 require 'jsduck/guide_writer'
+require 'jsduck/output_dir'
 require 'fileutils'
 
 module JsDuck
@@ -50,7 +51,7 @@ module JsDuck
     # -- util routines --
 
     def clean_output_dir
-      FileUtils.rm_rf(@opts.output_dir)
+      OutputDir.clean(@opts)
     end
 
     def format_classes

data/lib/jsduck/js/class.rb

@@ -122,7 +122,7 @@ module JsDuck
             when "inheritableStatics"
               cls[:members] += make_statics(value, {:inheritable => true})
             else
-              detect_method_or_property(cls, key, value, pair)
+              detect_method_or_property(cls, key, value, pair) if pair.raw["kind"] == "init"
             end
           end
         end
@@ -132,7 +132,7 @@ module JsDuck
       def detect_class_members_from_object(cls, ast)
         cls[:members] = []
         ast.each_property do |key, value, pair|
-          detect_method_or_property(cls, key, value, pair)
+          detect_method_or_property(cls, key, value, pair) if pair.raw["kind"] == "init"
         end
       end
 

data/lib/jsduck/js/method.rb

@@ -34,7 +34,7 @@ module JsDuck
           make(exp["id"].to_s || "", exp)
 
           # foo: function() {}
-        elsif ast.property? && ast["value"].function?
+        elsif ast.property? && ast.raw["kind"] == "init" && ast["value"].function?
           make(ast["key"].key_value, ast["value"])
 
           # Object.defineProperty(obj, "prop", {value: function() {} })

data/lib/jsduck/js/property.rb

@@ -41,9 +41,18 @@ module JsDuck
           # Object.defineProperty(obj, "prop", {value: x})
         elsif exp && exp.define_property?
           name = exp["arguments"][1].to_value
-          writable = exp.object_descriptor("writable")
-          readonly = writable ? !writable.to_value : true
-          make(name, exp.object_descriptor("value"), readonly)
+
+          if exp.object_descriptor("set")
+            # Object with a setter is not readonly
+            make(name)
+          elsif exp.object_descriptor("get")
+            # Object with a getter and no setter is readonly
+            make(name, nil, true)
+          else
+            writable = exp.object_descriptor("writable")
+            readonly = writable ? !writable.to_value : true
+            make(name, exp.object_descriptor("value"), readonly)
+          end
 
         else
           nil

data/lib/jsduck/js/rkelly_adapter.rb

@@ -216,32 +216,24 @@ module JsDuck
         when RKelly::Nodes::PropertyNode == node.class
           make(node, {
             "type" => "Property",
-            "key" =>
-              if node.name.is_a?(Numeric)
-                {
-                  "type" => "Literal",
-                  "value" => node.name,
-                  "raw" => node.name.to_s,
-                  "range" => offset_range(node, :name),
-                }
-              elsif node.name =~ /['"]/
-                {
-                  "type" => "Literal",
-                  "value" => string_value(node.name),
-                  "raw" => node.name,
-                  "range" => offset_range(node, :name),
-                }
-              else
-                {
-                  "type" => "Identifier",
-                  "name" => node.name,
-                  "range" => offset_range(node, :name),
-                }
-              end,
+            "key" => property_key(node),
             "value" => adapt_node(node.value),
             "kind" => "init",
           })
-
+        when RKelly::Nodes::GetterPropertyNode == node.class
+          make(node, {
+            "type" => "Property",
+            "key" => property_key(node),
+            "value" => adapt_node(node.value),
+            "kind" => "get",
+          })
+        when RKelly::Nodes::SetterPropertyNode == node.class
+          make(node, {
+            "type" => "Property",
+            "key" => property_key(node),
+            "value" => adapt_node(node.value),
+            "kind" => "set",
+          })
         # Statements
         when RKelly::Nodes::ExpressionStatementNode == node.class
           make(node, {
@@ -458,6 +450,30 @@ module JsDuck
         }
       end
 
+      def property_key(node)
+        if node.name.is_a?(Numeric)
+          {
+            "type" => "Literal",
+            "value" => node.name,
+            "raw" => node.name.to_s,
+            "range" => offset_range(node, :name),
+          }
+        elsif node.name =~ /['"]/
+          {
+            "type" => "Literal",
+            "value" => string_value(node.name),
+            "raw" => node.name,
+            "range" => offset_range(node, :name),
+          }
+        else
+          {
+            "type" => "Identifier",
+            "name" => node.name,
+            "range" => offset_range(node, :name),
+          }
+        end
+      end
+
       # Evaluates the actual value of a JavaScript string.
       # Importantly we avoid using Ruby's eval().
       def string_value(string)

data/lib/jsduck/options.rb

@@ -18,6 +18,7 @@ module JsDuck
     attr_accessor :ignore_global
     attr_accessor :external_classes
     attr_accessor :ext4_events
+    attr_accessor :version
 
     # Customizing output
     attr_accessor :title
@@ -48,6 +49,8 @@ module JsDuck
 
     # Debugging
     attr_accessor :warnings_exit_nonzero
+    attr_accessor :cache
+    attr_accessor :cache_dir
     attr_accessor :template_dir
     attr_accessor :template_links
     attr_accessor :extjs_path
@@ -99,7 +102,7 @@ module JsDuck
       ]
       @ext4_events = nil
 
-      @version = "5.2.0"
+      @version = "5.3.0"
       # Customizing output
       @title = "Documentation - JSDuck"
       @header = "<strong>Documentation</strong> JSDuck"
@@ -132,6 +135,8 @@ module JsDuck
 
       # Debugging
       @warnings_exit_nonzero = false
+      @cache = false
+      @cache_dir = nil
       @root_dir = File.dirname(File.dirname(File.dirname(__FILE__)))
       @template_dir = @root_dir + "/template-min"
       @template_links = false
@@ -203,16 +208,22 @@ module JsDuck
           "This option is REQUIRED.  When the directory exists,",
           "it will be overwritten.  Give dash '-' as argument",
           "to write docs to STDOUT (works only with --export).") do |path|
-          @output_dir = path == "-" ? :stdout : canonical(path)
+          if path == "-"
+            @output_dir = :stdout
+          else
+            @output_dir = canonical(path)
+            @cache_dir = @output_dir + "/.cache" unless @cache_dir
+          end
         end
 
-        opts.on('--export=TYPE',
+        opts.on('--export=full/examples',
           "Exports docs in JSON.",
           "",
-          "TYPE is one of:",
+          "For each JavaScript class a JSON file gets written,",
+          "the contents of which are as follows:",
           "",
-          "- full     - full class docs.",
-          "- examples - extracts inline examples from classes.") do |format|
+          "- full     - docs and metadata for class and its members.",
+          "- examples - inline examples from classes and guides.") do |format|
           export_type = format.to_sym
           if [:full, :examples].include?(export_type)
             @export = export_type
@@ -662,6 +673,63 @@ module JsDuck
           end
         end
 
+        opts.separator ""
+        opts.separator "Performance:"
+        opts.separator ""
+
+        opts.on('-p', '--processes=COUNT',
+          "The number of parallel processes to use.",
+          "",
+          "Defaults to the number of processors/cores.",
+          "",
+          "Set to 0 to disable parallel processing completely.",
+          "This is often useful in debugging to get deterministic",
+          "results.",
+          "",
+          "In Windows this option is disabled.") do |count|
+          Util::Parallel.in_processes = count.to_i
+        end
+
+        opts.on('--[no-]cache',
+          "Turns parser cache on/off (EXPERIMENTAL).",
+          "",
+          "Defaults to off.",
+          "",
+          "When enabled, the results of parsing source files is saved",
+          "inside the JSDuck output directory. Next time JSDuck runs,",
+          "only the files that have changed are parsed again, others",
+          "are read from the cache.",
+          "",
+          "Note that switching between Ruby and/or JSDuck versions",
+          "invalidates the whole cache.  But changes in custom tags",
+          "don't invalidate the cache, so avoid caching when developing",
+          "your custom tags.",
+          "",
+          "To change the cache directory location, use --cache-dir.") do |enabled|
+          @cache = enabled
+        end
+
+        opts.on('--cache-dir=PATH',
+          "Directory where to cache the parsed source.",
+          "",
+          "Defaults to: <output-dir>/.cache",
+          "",
+          "Each project needs to have a separate cache directory.",
+          "Instead of writing the cache into the output directory,",
+          "one might consider keeping it together with the source",
+          "files.",
+          "",
+          "Note that JSDuck ensures that the <output-dir>/.cache",
+          "dir is preserved when the rest of the <output-dir> gets",
+          "wiped clean during the docs generation.  If you specify",
+          "cache dir like <output-dir>/.mycache, then this will also",
+          "be cleaned up during docs generation, and the caching",
+          "won't work.",
+          "",
+          "This option only has an effect when --cache is also used.",) do |path|
+          @cache_dir = path
+        end
+
         opts.separator ""
         opts.separator "Debugging:"
         opts.separator ""
@@ -726,19 +794,6 @@ module JsDuck
           Logger.colors = on
         end
 
-        opts.on('-p', '--processes=COUNT',
-          "The number of parallel processes to use.",
-          "",
-          "Defaults to the number of processors/cores.",
-          "",
-          "Set to 0 to disable parallel processing completely.",
-          "This is often useful in debugging to get deterministic",
-          "results.",
-          "",
-          "In Windows this option is disabled.") do |count|
-          Util::Parallel.in_processes = count.to_i
-        end
-
         opts.on('--pretty-json',
           "Turns on pretty-printing of JSON.",
           "",