licensed 2.7.0 → 2.8.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 4b2b964bfef1d9dd5d12c96a4586a4e1815d530a744c09484b548e1505a01a22
-  data.tar.gz: a14d76ad21ab8fb742f698eb0ca90702fb154184b5e887313f8f7a7f894a6437
+  metadata.gz: 2613c80ad97c8d19cea10560588a08406388e1860b21c18bfbd322843db428cc
+  data.tar.gz: b34fc3f921feccd667898554166d69cb8d7b327e3756c30a3e418364d57cc4a6
 SHA512:
-  metadata.gz: 46689f9c144234c03de03c3adc9ee8cf11042d51047bcd83321eac9e1fd9d03a974f0f1757777da831e105e784153d06a4f48ea57b4cbda8079c0221074212f5
-  data.tar.gz: 5e1fe805637db57b6d16b0cc0f6ff8dfe3d56fd5aa1d70ffc19c06cc41764afe98b20917697de45ca338a22fe3aa1fcfa2491e9c9f90db393b4b35d38de34dc4
+  metadata.gz: 720754f1245f1043a2ff4721e0c2b7a403d020cb886277992d1d56a4c2145c588514fdcb18c3a1cd5aaf0d9409a5f839d73691ef0b167fbce69304c9aaf061ed
+  data.tar.gz: f33e96346dbb63b48f78de856094992fa4bf58acd5ff811e9193e8e65e71308d91f4a4dc54bfa2b5b80bba1c3fc5348d94454bc00ddb7d012c20ecbed26ba441

data/.github/workflows/test.yml

@@ -16,6 +16,8 @@ jobs:
       uses: actions/setup-node@v1
       with:
         node-version: 8
+    - name: Install Bower
+      run: npm install -g bower
     - name: Set up Ruby
       uses: actions/setup-ruby@v1
       with:
@@ -340,3 +342,35 @@ jobs:
       run: script/source-setup/pipenv
     - name: Run tests
       run: script/test pipenv
+      
+  yarn:
+    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        # not using 1.0.0 because it doesn't support `yarn list --production`
+        yarn_version: [ 1.4.0, latest ]
+    steps:
+    - uses: actions/checkout@master
+    - name: Setup node
+      uses: actions/setup-node@v1
+      with:
+        node-version: 12
+    - name: Install Yarn
+      run: npm install -g yarn@${YARN_VERSION}
+      env:
+        YARN_VERSION: ${{ matrix.yarn_version }}
+    - name: Set up Ruby
+      uses: actions/setup-ruby@v1
+      with:
+        ruby-version: 2.6.x
+    - run: bundle lock
+    - uses: actions/cache@preview
+      with:
+        path: vendor/gems
+        key: ${{ runner.os }}-gem-2.6.x-${{ hashFiles(format('{0}{1}', github.workspace, '/Gemfile.lock')) }}
+    - name: Bootstrap
+      run: script/bootstrap
+    - name: Set up fixtures
+      run: script/source-setup/yarn
+    - name: Run tests
+      run: script/test yarn

data/.gitignore

@@ -12,26 +12,39 @@
 test/fixtures/bundler/.bundle/
 test/fixtures/bundler/vendor/
 test/fixtures/bundler/Gemfile.lock
+
 test/fixtures/bower/bower_components
+
 test/fixtures/npm/node_modules
 test/fixtures/npm/package-lock.json
+
 test/fixtures/go/src/*
 test/fixtures/go/pkg
 !test/fixtures/go/src/test
 !test/fixtures/go/src/modules_test
+
 test/fixtures/cabal/*
 !test/fixtures/cabal/app*
+
 test/fixtures/git_submodule/*
 !test/fixtures/git_submodule/README
+
 test/fixtures/pip/venv
+
 test/fixtures/pipenv/Pipfile.lock
+
 !test/fixtures/migrations/**/*
+
 test/fixtures/composer/**/*
 !test/fixtures/composer/composer.json
+
 test/fixtures/mix/_build
 test/fixtures/mix/deps
 test/fixtures/mix/mix.lock
 
+test/fixtures/yarn/*
+!test/fixtures/yarn/package.json
+
 vendor/licenses
 .licenses
 *.gem

data/CHANGELOG.md

@@ -6,6 +6,16 @@ and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0.
 
 ## [Unreleased]
 
+## 2.8.0
+2020-01-03
+
+### Added
+- Yarn source (https://github.com/github/licensed/pull/232, https://github.com/github/licensed/pull/233, https://github.com/github/licensed/pull/236)
+- NPM source has a new option to include non-production dependencies (https://github.com/github/licensed/pull/231)
+
+### Fixed
+- Cabal source will no longer crash if packages aren't found (https://github.com/github/licensed/pull/230)
+
 ## 2.7.0
 2019-11-10
 
@@ -255,4 +265,4 @@ and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0.
 
 Initial release :tada:
 
-[Unreleased]: https://github.com/github/licensed/compare/2.7.0...HEAD
+[Unreleased]: https://github.com/github/licensed/compare/2.8.0...HEAD

data/README.md

@@ -25,11 +25,11 @@ See the [migration documentation](./docs/migrating_to_newer_versions.md) for mor
 
 Licensed uses the `libgit2` bindings for Ruby provided by `rugged`. `rugged` requires `cmake` and `pkg-config` which you may need to install before you can install Licensed.
    
-   >  Ubuntu 
+   >  Ubuntu
     
     sudo apt-get install cmake pkg-config
     
-   >  OS X 
+   >  OS X
        
     brew install cmake pkg-config
 
@@ -110,6 +110,7 @@ Dependencies will be automatically detected for all of the following sources by
 1. [Pipenv](./docs/sources/pipenv.md)
 1. [Git Submodules (git_submodule)](./docs/sources/git_submodule.md)
 1. [Mix](./docs/sources/mix.md)
+1. [Yarn](./docs/sources/yarn.md)
 
 You can disable any of them in the configuration file:
 

data/docs/configuration.md

@@ -64,7 +64,7 @@ name: 'My application'
 # If not set, defaults to a git repository root
 root: 'relative/path/from/configuration/file/directory'
 
-# Path is relative to configuration root
+# Path is relative to configuration root and specifies where cached metadata will be stored.
 # If not set, defaults to '.licenses'
 cache_path: 'relative/path/to/cache'
 
@@ -78,16 +78,23 @@ sources:
   bower: true
   bundler: false
 
-# Dependencies with these licenses are allowed by default.
+# Dependencies with these licenses are allowed and will not raise errors or warnings.
+# This list does not have a default value and is required for `licensed status`
+# to succeed.
 allowed:
   - mit
   - apache-2.0
   - bsd-2-clause
   - bsd-3-clause
   - cc0-1.0
-
-# These dependencies are explicitly ignored.
-# They shouldn't be cached at all, and will not have cached metadata written to the repo.
+  - isc
+
+# These dependencies are ignored during enumeration.
+# They will not be cached, and will not raise errors or warnings.
+# This configuration is intended to be used for dependencies that don't need to
+# be included for compliance purposes, such as other projects owned by the current
+# project's owner, internal dependencies, and dependencies that aren't shipped with
+# the project like test frameworks.
 ignored:
   bundler:
     - some-internal-gem
@@ -101,8 +108,10 @@ ignored:
     # comparisons use the FNM_CASEFOLD and FNM_PATHNAME flags
     - github.com/internal-package/**/*
 
-# These dependencies have been reviewed.
-# They need to be cached and checked, but do not have a license found that matches the allowed configured licenses.
+# These dependencies have licenses not on the `allowed` list and have been reviewed.
+# They will be cached and checked, but will not raise errors or warnings for a
+# non-allowed license.  Dependencies on this list will still raise errors if
+# license text cannot be found for the dependency.
 reviewed:
   bundler:
     - bcrypt-ruby

data/docs/sources/npm.md

@@ -1,3 +1,12 @@
 # NPM
 
 The npm source will detect dependencies `package.json` is found at an apps `source_path`.  It uses `npm list` to enumerate dependencies and metadata.
+
+### Including development dependencies
+
+By default, the npm source will exclude all non-development dependencies.  To include development or test dependencies, set `production_only: false` in the licensed configuration.
+
+```yml
+npm:
+  production_only: false
+```

data/docs/sources/yarn.md

@@ -0,0 +1,16 @@
+# Yarn
+
+The yarn source will detect dependencies when `package.json` and `yarn.lock` are found at an app's `source_path`.
+
+It uses `yarn list` to enumerate dependencies and `yarn info` to get metadata on each package.
+
+### Including development dependencies
+
+Yarn versions < 1.3.0 will always include non-production dependencies due to a bug in those yarn versions.
+
+Starting with yarn version >= 1.3.0, the yarn source excludes non-production dependencies by default.  To include development and test dependencies, set `production_only: false` in `.licensed.yml`.
+
+```yml
+yarn:
+  production_only: false
+```

data/lib/licensed/sources.rb

@@ -15,5 +15,6 @@ module Licensed
     require "licensed/sources/pipenv"
     require "licensed/sources/gradle"
     require "licensed/sources/mix"
+    require "licensed/sources/yarn"
   end
 end

data/lib/licensed/sources/cabal.rb

@@ -31,24 +31,25 @@ module Licensed
 
       # Returns a list of all detected packages
       def packages
-        missing = []
         package_ids = Set.new
         cabal_file_dependencies.each do |target|
-          name, version = target.split(/\s/, 2)
+          name = target.split(/\s/)[0]
           package_id = cabal_package_id(name)
           if package_id.nil?
-            missing << { "name" => name, "version" => version, "error" => "package not found" }
+            package_ids << target
           else
             recursive_dependencies([package_id], package_ids)
           end
         end
 
-        Parallel.map(package_ids) { |id| package_info(id) }.concat(missing)
+        Parallel.map(package_ids) { |id| package_info(id) }
       end
 
       # Returns the packages document directory and search root directory
       # as an array
       def package_docs_dirs(package)
+        return [nil, nil] if package.nil? || package.empty?
+
         unless package["haddock-html"]
           # default to a local vendor directory if haddock-html property
           # isn't available
@@ -78,18 +79,17 @@ module Licensed
       # Recursively finds the dependencies for each cabal package.
       # Returns a `Set` containing the package names for all dependencies
       def recursive_dependencies(package_names, results = Set.new)
-        return [] if package_names.nil? || package_names.empty?
+        return results if package_names.nil? || package_names.empty?
 
         new_packages = Set.new(package_names) - results
-        return [] if new_packages.empty?
+        return results if new_packages.empty?
 
         results.merge new_packages
 
         dependencies = Parallel.map(new_packages, &method(:package_dependencies)).flatten
 
-        return results if dependencies.empty?
-
-        results.merge recursive_dependencies(dependencies, results)
+        recursive_dependencies(dependencies, results)
+        results
       end
 
       # Returns an array of dependency package names for the cabal package
@@ -108,7 +108,10 @@ module Licensed
 
       # Returns package information as a hash for the given id
       def package_info(id)
-        package_info_command(id).lines.each_with_object({}) do |line, info|
+        info = package_info_command(id).strip
+        return missing_package(id) if info.empty?
+
+        info.lines.each_with_object({}) do |line, info|
           key, value = line.split(":", 2).map(&:strip)
           next unless key && value
 
@@ -216,6 +219,17 @@ module Licensed
       def ghc?
         @ghc ||= Licensed::Shell.tool_available?("ghc")
       end
+
+      # Returns a package info structure with an error set
+      def missing_package(id)
+        name, _, version = if id.index(/\s/).nil?
+          id.rpartition("-") # e.g. to match the right-most dash from ipid fused-effects-1.0.0.0
+        else
+          id.partition(/\s/) # e.g. to match the left-most space from constraint fused-effects > 1.0.0.0
+        end
+
+        { "name" => name, "version" => version, "error" => "package not found" }
+      end
     end
   end
 end

data/lib/licensed/sources/npm.rb

@@ -57,13 +57,20 @@ module Licensed
 
       # Returns the output from running `npm list` to get package metadata
       def package_metadata_command
-        Licensed::Shell.execute("npm", "list", "--json", "--production", "--long", allow_failure: true)
+        args = %w(--json --long)
+        args << "--production" unless include_non_production?
+        Licensed::Shell.execute("npm", "list", *args, allow_failure: true)
       end
 
       # Returns true if a yarn.lock file exists in the current directory
       def yarn_lock_present
         @yarn_lock_present ||= File.exist?(config.pwd.join("yarn.lock"))
       end
+
+      # Returns whether to include non production dependencies based on the licensed configuration settings
+      def include_non_production?
+        config.dig("npm", "production_only") == false
+      end
     end
   end
 end

data/lib/licensed/sources/yarn.rb

@@ -0,0 +1,138 @@
+# frozen_string_literal: true
+require "json"
+
+module Licensed
+  module Sources
+    class Yarn < Source
+      # `yarn licenses list --json` returns data in a table format with header
+      # ordering specified in the output.  Look for these specific headers and use
+      # their indices to get data from the table body
+      YARN_NAME_HEAD = "Name".freeze
+      YARN_VERSION_HEAD = "Version".freeze
+      YARN_URL_HEAD = "URL".freeze
+
+      def enabled?
+        return unless Licensed::Shell.tool_available?("yarn")
+
+        config.pwd.join("package.json").exist? && config.pwd.join("yarn.lock").exist?
+      end
+
+      def enumerate_dependencies
+        packages.map do |name, package|
+          Dependency.new(
+            name: name,
+            version: package["version"],
+            path: package["path"],
+            metadata: {
+              "type"     => Yarn.type,
+              "name"     => package["name"],
+              "homepage" => dependency_urls[package["id"]]
+            }
+          )
+        end
+      end
+
+      # Finds packages that the current project relies on
+      def packages
+        return [] if yarn_package_tree.nil?
+        all_dependencies = {}
+        recursive_dependencies(config.pwd, yarn_package_tree).each do |name, results|
+          results.uniq! { |package| package["version"] }
+          if results.size == 1
+            # if there is only one package for a name, reference it by name
+            all_dependencies[name] = results[0]
+          else
+            # if there is more than one package for a name, reference each by
+            # "<name>-<version>"
+            results.each do |package|
+              all_dependencies["#{name}-#{package["version"]}"] = package
+            end
+          end
+        end
+
+        all_dependencies
+      end
+
+      # Recursively parse dependency JSON data.  Returns a hash mapping the
+      # package name to it's metadata
+      def recursive_dependencies(path, dependencies, result = {})
+        dependencies.each do |dependency|
+          # "shadow" indicate a dependency requirement only, not a
+          # resolved package identifier
+          next if dependency["shadow"]
+          name, _, version = dependency["name"].rpartition("@")
+
+          # the dependency should be found under the parent's "node_modules" path
+          dependency_path = path.join("node_modules", name)
+          (result[name] ||= []) << {
+            "id" => dependency["name"],
+            "name" => name,
+            "version" => version,
+            "path" => dependency_path
+          }
+          recursive_dependencies(dependency_path, dependency["children"], result)
+        end
+        result
+      end
+
+      # Finds and returns the yarn package tree listing from `yarn list` output
+      def yarn_package_tree
+        return @yarn_package_tree if defined?(@yarn_package_tree)
+        @yarn_package_tree = begin
+          # parse all lines of output to json and find one that is "type": "tree"
+          tree = yarn_list_command.lines
+                                  .map(&:strip)
+                                  .map(&JSON.method(:parse))
+                                  .find { |json| json["type"] == "tree" }
+          tree&.dig("data", "trees")
+        end
+      end
+
+      # Returns a mapping of unique dependency identifiers to urls
+      def dependency_urls
+        @dependency_urls ||= begin
+          table = yarn_licenses_command.lines
+                                       .map(&:strip)
+                                       .map(&JSON.method(:parse))
+                                       .find { |json| json["type"] == "table" }
+          return [] if table.nil?
+
+          head = table.dig("data", "head")
+          return [] if head.nil?
+
+          name_index = head.index YARN_NAME_HEAD
+          version_index = head.index YARN_VERSION_HEAD
+          url_index = head.index YARN_URL_HEAD
+          return [] if name_index.nil? || version_index.nil? || url_index.nil?
+
+          body = table.dig("data", "body")
+          return [] if body.nil?
+
+          body.each_with_object({}) do |row, hsh|
+            id = "#{row[name_index]}@#{row[version_index]}"
+            hsh[id] = row[url_index]
+          end
+        end
+      end
+
+      # Returns the output from running `yarn list` to get project dependencies
+      def yarn_list_command
+        args = %w(--json -s --no-progress)
+        args << "--production" unless include_non_production?
+        Licensed::Shell.execute("yarn", "list", *args, allow_failure: true)
+      end
+
+      # Returns the output from running `yarn licenses list` to get project urls
+      def yarn_licenses_command
+        args = %w(--json -s --no-progress)
+        args << "--production" unless include_non_production?
+        Licensed::Shell.execute("yarn", "licenses", "list", *args, allow_failure: true)
+      end
+
+      # Returns whether to include non production dependencies based on the licensed configuration settings
+      def include_non_production?
+        config.dig("yarn", "production_only") == false
+      end
+    end
+  end
+end

data/lib/licensed/version.rb

@@ -1,6 +1,6 @@
 # frozen_string_literal: true
 module Licensed
-  VERSION = "2.7.0".freeze
+  VERSION = "2.8.0".freeze
 
   def self.previous_major_versions
     major_version = Gem::Version.new(Licensed::VERSION).segments.first

data/script/source-setup/yarn

@@ -0,0 +1,17 @@
+#!/bin/bash
+set -e
+
+if [ -z "$(which yarn)" ]; then
+  echo "A local yarn installation is required for yarn development." >&2
+  exit 127
+fi
+
+# setup test fixtures
+BASE_PATH="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)"
+cd $BASE_PATH/test/fixtures/yarn
+
+if [ "$1" == "-f" ]; then
+  find . -not -regex "\.*" -and -not -name "package\.json" -print0 | xargs -0 rm -rf
+fi
+
+yarn install

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: licensed
 version: !ruby/object:Gem::Version
-  version: 2.7.0
+  version: 2.8.0
 platform: ruby
 authors:
 - GitHub
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2019-11-10 00:00:00.000000000 Z
+date: 2020-01-03 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: licensee
@@ -254,6 +254,7 @@ files:
 - docs/sources/pip.md
 - docs/sources/pipenv.md
 - docs/sources/stack.md
+- docs/sources/yarn.md
 - exe/licensed
 - lib/licensed.rb
 - lib/licensed/cli.rb
@@ -293,6 +294,7 @@ files:
 - lib/licensed/sources/pip.rb
 - lib/licensed/sources/pipenv.rb
 - lib/licensed/sources/source.rb
+- lib/licensed/sources/yarn.rb
 - lib/licensed/ui/shell.rb
 - lib/licensed/version.rb
 - licensed.gemspec
@@ -314,6 +316,7 @@ files:
 - script/source-setup/npm
 - script/source-setup/pip
 - script/source-setup/pipenv
+- script/source-setup/yarn
 - script/test
 homepage: https://github.com/github/licensed
 licenses: