licensed 2.6.0 → 2.6.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 94e3cbc840104e6f0d4733c82ea38805aa2db680070708b88f107420405a1956
-  data.tar.gz: d04aea6f7ac531c7ab5826b90ece5f3bd0936845a7d8a0e0e44d54c7119e9f08
+  metadata.gz: 399ad5d09b5d158f3c1a51aa719dd0f353bf9a4714d495d3998dd0e38c6b7009
+  data.tar.gz: 72a61d16fa72990cb12421bb29db09c80c15436eb5a85f962ffab089e78f7c99
 SHA512:
-  metadata.gz: aab7fc74e7905ab4acad59d1e997c3ea95c8121299ba5cd52ae96bc2a7451314de1155754d5c41fe7834037217469e49b8b2fc4c88b68b1b08036bedc06a0ee7
-  data.tar.gz: 8572722b1cfca8af7527278f269b410a7c4edc699dc0ffba6e991e2ececb78d575addf0fe548ae95ac6347b7c59431ec553d1c0003b8a4abbd3f2dbfd534abc5
+  metadata.gz: 4c267eaf2523970f7e890e8e08eca972fa07cf3b616f66dc68ba010677014cd3ab8f08f7d7496c22345b4cd9801e6386015ff13e0064eed5ee850c12b068f8a5
+  data.tar.gz: ea1f68a031f398a43036648850325f34b44f28e5cfb9da630cc6d6bbdbe8e3b82b5d467b030256d4ed78bb4dda7e7c5820f5dd7fb97af08afe10105db8fc2946

data/.github/workflows/release.yml

@@ -18,7 +18,7 @@ jobs:
     - name: Set up Ruby 2.6
       uses: actions/setup-ruby@v1
       with:
-        version: 2.6.x
+        ruby-version: 2.6.x
 
     - name: Build package
       run: script/packages/linux
@@ -39,7 +39,7 @@ jobs:
     - name: Set up Ruby 2.6
       uses: actions/setup-ruby@v1
       with:
-        version: 2.6.x
+        ruby-version: 2.6.x
 
     - name: Build package
       run: script/packages/mac
@@ -60,7 +60,7 @@ jobs:
     - name: Set up Ruby 2.6
       uses: actions/setup-ruby@v1
       with:
-        version: 2.6.x
+        ruby-version: 2.6.x
 
     - name: Build gem
       run: gem build *.gemspec
@@ -88,7 +88,7 @@ jobs:
     - name: Set up Ruby 2.6
       uses: actions/setup-ruby@v1
       with:
-        version: 2.6.x
+        ruby-version: 2.6.x
 
     - name: Download linux package
       uses: actions/download-artifact@master

data/.github/workflows/test.yml

@@ -15,11 +15,11 @@ jobs:
     - name: Setup node
       uses: actions/setup-node@v1
       with:
-        version: 8
+        node-version: 8
     - name: Set up Ruby
       uses: actions/setup-ruby@v1
       with:
-        version: 2.6.x
+        ruby-version: 2.6.x
     - name: Bootstrap
       run: script/bootstrap
     - name: Set up fixtures
@@ -37,7 +37,7 @@ jobs:
     - name: Set up Ruby
       uses: actions/setup-ruby@v1
       with:
-        version: 2.6.x
+        ruby-version: 2.6.x
     - name: Set up Bundler
       run: |
         yes | gem uninstall bundler --all
@@ -60,7 +60,7 @@ jobs:
     - name: Set up Ruby
       uses: actions/setup-ruby@v1
       with:
-        version: 2.6.x
+        ruby-version: 2.6.x
     - name: Setup Haskell
       uses: actions/setup-haskell@v1
       with:
@@ -83,7 +83,7 @@ jobs:
     - name: Set up Ruby
       uses: actions/setup-ruby@v1
       with:
-        version: ${{matrix.ruby}}
+        ruby-version: ${{matrix.ruby}}
     - name: Set up Bundler
       run: gem install bundler
     - name: Bootstrap
@@ -104,7 +104,7 @@ jobs:
     - name: Set up Ruby
       uses: actions/setup-ruby@v1
       with:
-        version: 2.6.x
+        ruby-version: 2.6.x
     - name: Bootstrap
       run: script/bootstrap
     - name: Set up fixtures
@@ -126,7 +126,7 @@ jobs:
     - name: Set up Ruby
       uses: actions/setup-ruby@v1
       with:
-        version: 2.6.x
+        ruby-version: 2.6.x
     - name: Bootstrap
       run: script/bootstrap
     - name: Set up fixtures
@@ -144,11 +144,11 @@ jobs:
     - name: Setup node
       uses: actions/setup-node@v1
       with:
-        version: ${{ matrix.node_version }}
+        node-version: ${{ matrix.node_version }}
     - name: Set up Ruby
       uses: actions/setup-ruby@v1
       with:
-        version: 2.6.x
+        ruby-version: 2.6.x
     - name: Bootstrap
       run: script/bootstrap
     - name: Set up fixtures
@@ -163,7 +163,7 @@ jobs:
     - name: Set up Ruby
       uses: actions/setup-ruby@v1
       with:
-        version: 2.6.x
+        ruby-version: 2.6.x
     - name: Bootstrap
       run: script/bootstrap
     - name: Gradle version
@@ -186,7 +186,7 @@ jobs:
     - name: Set up Ruby
       uses: actions/setup-ruby@v1
       with:
-        version: 2.6.x
+        ruby-version: 2.6.x
     - name: Bootstrap
       run: script/bootstrap
     - name: Install virtualenv
@@ -208,7 +208,7 @@ jobs:
     - name: Set up Ruby
       uses: actions/setup-ruby@v1
       with:
-        version: 2.6.x
+        ruby-version: 2.6.x
     - name: Bootstrap
       run: script/bootstrap
     - name: Install pipenv
@@ -232,7 +232,7 @@ jobs:
     - name: Set up Ruby
       uses: actions/setup-ruby@v1
       with:
-        version: 2.6.x
+        ruby-version: 2.6.x
     - name: Bootstrap
       run: script/bootstrap
     - name: Set up fixtures
@@ -255,10 +255,23 @@ jobs:
     - name: Set up Ruby
       uses: actions/setup-ruby@v1
       with:
-        version: 2.6.x
+        ruby-version: 2.6.x
     - name: Bootstrap
       run: script/bootstrap
     - name: Set up fixtures
       run: script/source-setup/mix
     - name: Run tests
       run: script/test mix
+
+  manifest:
+    runs-on: ubuntu-latest
+    steps:
+    - uses: actions/checkout@master
+    - name: Set up Ruby
+      uses: actions/setup-ruby@v1
+      with:
+        ruby-version: 2.6.x
+    - name: Bootstrap
+      run: script/bootstrap
+    - name: Run tests
+      run: script/test manifest

data/CHANGELOG.md

@@ -6,6 +6,12 @@ and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0.
 
 ## [Unreleased]
 
+## 2.6.1
+2019-10-26
+
+### Changed
+- Performance improvements during dependency enumeration (:tada: @krzysztof-pawlik-gat https://github.com/github/licensed/pull/204, https://github.com/github/licensed/pull/207) (https://github.com/github/licensed/pull/210)
+
 ## 2.6.0
 2019-10-22
 
@@ -229,4 +235,4 @@ and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0.
 
 Initial release :tada:
 
-[Unreleased]: https://github.com/github/licensed/compare/2.6.0...HEAD
+[Unreleased]: https://github.com/github/licensed/compare/2.6.1...HEAD

data/lib/licensed/sources/bower.rb

@@ -5,7 +5,7 @@ module Licensed
   module Sources
     class Bower < Source
       def enabled?
-        [@config.pwd.join(".bowerrc"), @config.pwd.join("bower.json")].any? do |path|
+        [config.pwd.join(".bowerrc"), config.pwd.join("bower.json")].any? do |path|
           File.exist?(path)
         end
       end
@@ -30,7 +30,7 @@ module Licensed
       # Returns a parsed ".bowerrc" configuration, or an empty hash if not found
       def bower_config
         @bower_config ||= begin
-          path = @config.pwd.join(".bowerrc")
+          path = config.pwd.join(".bowerrc")
           path.exist? ? JSON.parse(path.read) : {}
         end
       end
@@ -38,7 +38,7 @@ module Licensed
       # Returns the expected path to bower components.
       # Note this does not validate that the returned path is valid
       def bower_path
-        pwd = bower_config["cwd"] ? Pathname.new(bower_config["cwd"]).expand_path : @config.pwd
+        pwd = bower_config["cwd"] ? Pathname.new(bower_config["cwd"]).expand_path : config.pwd
         pwd.join bower_config["directory"] || "bower_components"
       end
     end

data/lib/licensed/sources/bundler.rb

@@ -264,7 +264,7 @@ module Licensed
       # Defaults to [:development, :test] + ::Bundler.settings[:without]
       def exclude_groups
         @exclude_groups ||= begin
-          exclude = Array(@config.dig("bundler", "without"))
+          exclude = Array(config.dig("bundler", "without"))
           exclude = DEFAULT_WITHOUT_GROUPS if exclude.empty?
           exclude.uniq.map(&:to_sym)
         end
@@ -272,7 +272,7 @@ module Licensed
 
       # Returns the path to the Bundler Gemfile
       def gemfile_path
-        @gemfile_path ||= GEMFILES.map { |g| @config.pwd.join g }
+        @gemfile_path ||= GEMFILES.map { |g| config.pwd.join g }
                                   .find { |f| f.exist? }
       end
 
@@ -285,10 +285,10 @@ module Licensed
       # Returns the configured bundler executable to use, or "bundle" by default.
       def bundler_exe
         @bundler_exe ||= begin
-          exe = @config.dig("bundler", "bundler_exe")
+          exe = config.dig("bundler", "bundler_exe")
           return "bundle" unless exe
           return exe if Licensed::Shell.tool_available?(exe)
-          @config.root.join(exe)
+          config.root.join(exe)
         end
       end
 

data/lib/licensed/sources/cabal.rb

@@ -43,7 +43,7 @@ module Licensed
           end
         end
 
-        package_ids.map { |id| package_info(id) }.concat(missing)
+        Parallel.map(package_ids) { |id| package_info(id) }.concat(missing)
       end
 
       # Returns the packages document directory and search root directory
@@ -52,7 +52,7 @@ module Licensed
         unless package["haddock-html"]
           # default to a local vendor directory if haddock-html property
           # isn't available
-          return [File.join(@config.pwd, "vendor", package["name"]), nil]
+          return [File.join(config.pwd, "vendor", package["name"]), nil]
         end
 
         html_dir = package["haddock-html"]
@@ -85,7 +85,7 @@ module Licensed
 
         results.merge new_packages
 
-        dependencies = new_packages.flat_map { |n| package_dependencies(n) }
+        dependencies = Parallel.map(new_packages, &method(:package_dependencies)).flatten
 
         return results if dependencies.empty?
 
@@ -131,10 +131,10 @@ module Licensed
       # Returns an array of ghc package DB locations as specified in the app
       # configuration
       def package_db_args
-        @package_db_args ||= Array(@config.dig("cabal", "ghc_package_db")).map do |path|
+        @package_db_args ||= Array(config.dig("cabal", "ghc_package_db")).map do |path|
           next "--#{path}" if %w(global user).include?(path)
           path = realized_ghc_package_path(path)
-          path = File.expand_path(path, @config.root)
+          path = File.expand_path(path, config.root)
 
           next unless File.exist?(path)
           "--package-db=#{path}"
@@ -196,14 +196,14 @@ module Licensed
 
       # Returns the targets to search for `build-depends` in a cabal file
       def cabal_file_targets
-        targets = Array(@config.dig("cabal", "cabal_file_targets"))
+        targets = Array(config.dig("cabal", "cabal_file_targets"))
         targets.push(*DEFAULT_TARGETS) if targets.empty?
         targets
       end
 
       # Returns an array of the local directory cabal package files
       def cabal_files
-        @cabal_files ||= Dir.glob(File.join(@config.pwd, "*.cabal"))
+        @cabal_files ||= Dir.glob(File.join(config.pwd, "*.cabal"))
       end
 
       # Returns the ghc cli tool version

data/lib/licensed/sources/composer.rb

@@ -49,12 +49,12 @@ module Licensed
       end
 
       def composer_application_path
-        setting = @config.dig("composer", "application_path") || DEFAULT_COMPOSER_APPLICATON_PATH
-        File.expand_path(setting, @config.pwd)
+        setting = config.dig("composer", "application_path") || DEFAULT_COMPOSER_APPLICATON_PATH
+        File.expand_path(setting, config.pwd)
       end
 
       def composer_lock
-        @config.pwd.join("composer.lock")
+        config.pwd.join("composer.lock")
       end
     end
   end

data/lib/licensed/sources/dep.rb

@@ -10,8 +10,8 @@ module Licensed
 
       def enumerate_dependencies
         packages.map do |package|
-          package_dir = @config.pwd.join("vendor", package[:name])
-          search_root = @config.pwd.join("vendor", package[:project])
+          package_dir = config.pwd.join("vendor", package[:name])
+          search_root = config.pwd.join("vendor", package[:project])
 
           Dependency.new(
             name: package[:name],
@@ -52,7 +52,7 @@ module Licensed
       end
 
       def gopkg_lock_path
-        @config.pwd.join("Gopkg.lock")
+        config.pwd.join("Gopkg.lock")
       end
 
       # Returns a list of go standard packages

data/lib/licensed/sources/git_submodule.rb

@@ -20,7 +20,7 @@ module Licensed
         git_submodules_command.lines.map do |line|
           displaypath, toplevel, version, homepage = line.strip.split
           name = File.basename(displaypath)
-          submodule_path = if toplevel == @config.pwd.to_s
+          submodule_path = if toplevel == config.pwd.to_s
             name
           else
             parent = File.basename(toplevel)
@@ -31,7 +31,7 @@ module Licensed
           Licensed::Dependency.new(
             name: submodule_path,
             version: version,
-            path: @config.pwd.join(displaypath),
+            path: config.pwd.join(displaypath),
             metadata: {
               "type" => self.class.type,
               "name" => name,
@@ -50,7 +50,7 @@ module Licensed
       end
 
       def gitmodules_path
-        @config.pwd.join(".gitmodules")
+        config.pwd.join(".gitmodules")
       end
     end
   end

data/lib/licensed/sources/go.rb

@@ -54,7 +54,7 @@ module Licensed
       def root_package_deps
         # check for ignored packages to avoid raising errors calling `go list`
         # when ignored package is not found
-        Array(root_package["Deps"]).map { |name| package_info(name) }
+        Parallel.map(Array(root_package["Deps"])) { |name| package_info(name) }
       end
 
       # Returns the list of dependencies as returned by "go list -json -deps"
@@ -196,12 +196,12 @@ module Licensed
       def gopath
         return @gopath if defined?(@gopath)
 
-        path = @config.dig("go", "GOPATH")
+        path = config.dig("go", "GOPATH")
         @gopath = if path.nil? || path.empty?
                     ENV["GOPATH"]
                   else
                     root = begin
-                             @config.root
+                             config.root
                            rescue Licensed::Shell::Error
                              Pathname.pwd
                            end

data/lib/licensed/sources/manifest.rb

@@ -12,7 +12,7 @@ module Licensed
       end
 
       def enumerate_dependencies
-        packages.map do |package_name, sources|
+        Parallel.map(packages) do |package_name, sources|
           Licensed::Sources::Manifest::Dependency.new(
             name: package_name,
             version: contents_version(*sources),
@@ -28,10 +28,10 @@ module Licensed
 
       # Returns the license path for a package specified in the configuration.
       def configured_license_path(package_name)
-        license_path = @config.dig("manifest", "licenses", package_name)
+        license_path = config.dig("manifest", "licenses", package_name)
         return unless license_path
 
-        license_path = @config.root.join(license_path)
+        license_path = config.root.join(license_path)
         return unless license_path.exist?
         license_path
       end
@@ -46,7 +46,7 @@ module Licensed
           # don't allow the workspace root to be used as common prefix
           # the project this is run for should be excluded from the manifest,
           # or ignored in the config.  any license in the root should be ignored.
-          return common_prefix if common_prefix != @config.root
+          return common_prefix if common_prefix != config.root
         end
 
         # use the first (or only) sources directory to find license information
@@ -61,7 +61,7 @@ module Licensed
         manifest.each_with_object({}) do |(src, package_name), hsh|
           next if src.nil? || src.empty?
           hsh[package_name] ||= []
-          hsh[package_name] << File.join(@config.root, src)
+          hsh[package_name] << File.join(config.root, src)
         end
       end
 
@@ -79,15 +79,15 @@ module Licensed
 
       # Returns the manifest location for the app
       def manifest_path
-        path = @config.dig("manifest", "path")
-        return @config.root.join(path) if path
+        path = config.dig("manifest", "path")
+        return config.root.join(path) if path
 
-        @config.cache_path.join("manifest.json")
+        config.cache_path.join("manifest.json")
       end
 
       # Returns whether a manifest should be generated automatically
       def generate_manifest?
-        !File.exist?(manifest_path) && !@config.dig("manifest", "dependencies").nil?
+        !File.exist?(manifest_path) && !config.dig("manifest", "dependencies").nil?
       end
 
       # Returns a manifest of files generated automatically based on patterns
@@ -128,7 +128,7 @@ module Licensed
       # Returns the project dependencies specified from the licensed configuration
       def configured_dependencies
         @configured_dependencies ||= begin
-          dependencies = @config.dig("manifest", "dependencies")&.dup || {}
+          dependencies = config.dig("manifest", "dependencies")&.dup || {}
 
           dependencies.each do |name, patterns|
             # map glob pattern(s) listed for the dependency to a listing
@@ -142,7 +142,7 @@ module Licensed
 
       # Returns the set of project files that are included in dependency evaluation
       def included_files
-        @sources ||= all_files - files_from_pattern_list(@config.dig("manifest", "exclude"))
+        @sources ||= all_files - files_from_pattern_list(config.dig("manifest", "exclude"))
       end
 
       # Finds and returns all files in the project that match
@@ -151,7 +151,7 @@ module Licensed
         return Set.new if patterns.nil? || patterns.empty?
 
         # evaluate all patterns from the project root
-        Dir.chdir @config.root do
+        Dir.chdir config.root do
           Array(patterns).reduce(Set.new) do |files, pattern|
             if pattern.start_with?("!")
               # if the pattern is an exclusion, remove all matching files

data/lib/licensed/sources/npm.rb

@@ -9,11 +9,10 @@ module Licensed
       end
 
       def enabled?
-        Licensed::Shell.tool_available?("npm") && File.exist?(@config.pwd.join("package.json"))
+        Licensed::Shell.tool_available?("npm") && File.exist?(config.pwd.join("package.json"))
       end
 
       def enumerate_dependencies
-        @yarn_lock_present = File.exist?(@config.pwd.join("yarn.lock"))
         packages.map do |name, package|
           path = package["path"]
           Dependency.new(
@@ -49,7 +48,7 @@ module Licensed
       # package name to it's metadata
       def recursive_dependencies(dependencies, result = {})
         dependencies.each do |name, dependency|
-          next if @yarn_lock_present && dependency["missing"]
+          next if yarn_lock_present && dependency["missing"]
           (result[name] ||= []) << dependency
           recursive_dependencies(dependency["dependencies"] || {}, result)
         end
@@ -60,6 +59,11 @@ module Licensed
       def package_metadata_command
         Licensed::Shell.execute("npm", "list", "--json", "--production", "--long", allow_failure: true)
       end
+
+      # Returns true if a yarn.lock file exists in the current directory
+      def yarn_lock_present
+        @yarn_lock_present ||= File.exist?(config.pwd.join("yarn.lock"))
+      end
     end
   end
 end

data/lib/licensed/sources/pip.rb

@@ -1,6 +1,8 @@
 # frozen_string_literal: true
-require "json"
+
 require "English"
+require "json"
+require "parallel"
 
 module Licensed
   module Sources
@@ -10,11 +12,11 @@ module Licensed
 
       def enabled?
         return unless virtual_env_pip && Licensed::Shell.tool_available?(virtual_env_pip)
-        File.exist?(@config.pwd.join("requirements.txt"))
+        File.exist?(config.pwd.join("requirements.txt"))
       end
 
       def enumerate_dependencies
-        packages_from_requirements_txt.map do |package_name|
+        Parallel.map(packages_from_requirements_txt, in_threads: Parallel.processor_count) do |package_name|
           package = package_info(package_name)
           location = File.join(package["Location"], package["Name"].gsub("-", "_") +  "-" + package["Version"] + ".dist-info")
           Dependency.new(
@@ -33,7 +35,7 @@ module Licensed
       private
 
       def packages_from_requirements_txt
-        File.read(@config.pwd.join("requirements.txt"))
+        File.read(config.pwd.join("requirements.txt"))
             .lines
             .reject { |line| line.include?("://") }
             .map { |line| line.strip.match(PACKAGE_REGEX) { |match| match.captures.first } }
@@ -61,8 +63,8 @@ module Licensed
       def virtual_env_dir
         return @virtual_env_dir if defined?(@virtual_env_dir)
         @virtual_env_dir = begin
-          venv_dir = @config.dig("python", "virtual_env_dir")
-          File.expand_path(venv_dir, @config.root) if venv_dir
+          venv_dir = config.dig("python", "virtual_env_dir")
+          File.expand_path(venv_dir, config.root) if venv_dir
         end
       end
     end

data/lib/licensed/sources/pipenv.rb

@@ -1,14 +1,16 @@
 # frozen_string_literal: true
 
+require "parallel"
+
 module Licensed
   module Sources
     class Pipenv < Source
       def enabled?
-        Licensed::Shell.tool_available?("pipenv") && File.exist?(@config.pwd.join("Pipfile.lock"))
+        Licensed::Shell.tool_available?("pipenv") && File.exist?(config.pwd.join("Pipfile.lock"))
       end
 
       def enumerate_dependencies
-        pakages_from_pipfile_lock.map do |package_name|
+        Parallel.map(pakages_from_pipfile_lock, in_threads: Parallel.processor_count) do |package_name|
           package = package_info(package_name)
           location = File.join(package["Location"], package["Name"].gsub("-", "_") +  "-" + package["Version"] + ".dist-info")
           Dependency.new(

data/lib/licensed/version.rb

@@ -1,6 +1,6 @@
 # frozen_string_literal: true
 module Licensed
-  VERSION = "2.6.0".freeze
+  VERSION = "2.6.1".freeze
 
   def self.previous_major_versions
     major_version = Gem::Version.new(Licensed::VERSION).segments.first

data/licensed.gemspec

@@ -29,6 +29,7 @@ Gem::Specification.new do |spec|
   spec.add_dependency "tomlrb", "~> 1.2"
   spec.add_dependency "bundler", ">= 1.10"
   spec.add_dependency "ruby-xxHash", "~> 0.4"
+  spec.add_dependency "parallel", ">= 0.18.0"
 
   spec.add_development_dependency "rake", "~> 10.0"
   spec.add_development_dependency "minitest", "~> 5.8"

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: licensed
 version: !ruby/object:Gem::Version
-  version: 2.6.0
+  version: 2.6.1
 platform: ruby
 authors:
 - GitHub
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2019-10-22 00:00:00.000000000 Z
+date: 2019-10-26 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: licensee
@@ -94,6 +94,20 @@ dependencies:
     - - "~>"
       - !ruby/object:Gem::Version
         version: '0.4'
+- !ruby/object:Gem::Dependency
+  name: parallel
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 0.18.0
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 0.18.0
 - !ruby/object:Gem::Dependency
   name: rake
   requirement: !ruby/object:Gem::Requirement