licensed 1.0.1 → 1.1.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 9e7af8fc6ca23f14500bc88d7766153dbc21ef30
-  data.tar.gz: 50c1b4ffdf8ef72d4b7bfd20e3b0aff031e7e843
+  metadata.gz: 3baec95cfc92e05345d639e02f55185da93a50a2
+  data.tar.gz: 7b970c9ea7bfe7c0724fde3d75ea058011655554
 SHA512:
-  metadata.gz: 587c4b056a4741e0d25b41551612f287140636bef7a9ddb76dd1231a8e2316a1e50ea2438b599216f6f63c6fc2268b32761df9e098d04587c8848b8e68cbc0cb
-  data.tar.gz: ede02755d459ecf6a42f4c7b9c9a0339b0e9294d9db94c271d3b00984de3825c504a43fa8c8792ac3d355356f90119d8757077237078bb3ee2d973654a4c27af
+  metadata.gz: bab9dbb1f74d2658cf93f689b4f828a40666836c444b35ca4999a8265cd5f471a60fc0b93a53008130148f8d7ee6f69bc8701a31961eba99d967b28566aae793
+  data.tar.gz: 74920ce7597d046570c5f45e9bd4c341ddebdf0958a8059eb1f43015209a1d69d54a498e2d0ec4ee838fb074e27bad2f1c9ba181b18dda2c8f9992265a92dc67

data/.gitignore

@@ -7,7 +7,6 @@
 /pkg/
 /spec/reports/
 /tmp/
-.byebug_history
 
 # test fixtures
 test/fixtures/bundler/.bundle/
@@ -19,11 +18,13 @@ test/fixtures/npm/package-lock.json
 test/fixtures/go/src/*
 test/fixtures/go/pkg
 !test/fixtures/go/src/test
-test/fixtures/haskell/dist*
+test/fixtures/cabal/*
+!test/fixtures/cabal/app*
 
 vendor/licenses
 .licenses
 *.gem
 vendor/gems
+.byebug_history
 
 bin/

data/.travis.yml

@@ -8,13 +8,27 @@ matrix:
       script: bundle exec rake rubocop build
       env: NAME="Lint and Build"
 
-    # go tests
+    # go 1.7 tests
     - language: go
-      go: 1.7.x
+      go: "1.7.x"
       before_script: ./script/source-setup/go
       script: ./script/test go
       env: NAME="go"
 
+    # go 1.10 tests
+    - language: go
+      go: "1.10.x"
+      before_script: ./script/source-setup/go
+      script: ./script/test go
+      env: NAME="go"
+
+    # dep tests
+    - language: go
+      go: "1.10.x"
+      before_script: ./script/source-setup/go
+      script: ./script/test dep
+      env: NAME="go dep"
+
     # cabal tests
     - language: haskell
       ghc: "8.2"
@@ -51,5 +65,21 @@ matrix:
       script: ./script/test manifest
       env: NAME="manifest"
 
+    # python 2.7 tests
+    - language: python
+      python:
+        - "2.7"
+      before_script: ./script/source-setup/pip
+      script: ./script/test pip
+      env: NAME="pip"
+
+    # python 3.6 tests
+    - language: python
+      python:
+        - "3.6"
+      before_script: ./script/source-setup/pip
+      script: ./script/test pip
+      env: NAME="pip"
+
 notifications:
   disable: true

data/CHANGELOG.md

@@ -6,6 +6,21 @@ and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0.
 
 ## [Unreleased]
 
+## 1.1.0 - 2018-06-04
+### Added
+- Pip dependency source :tada:
+- Go Dep dependency source :tada:
+
+### Changed
+- Changed how `sources` configuration property affects which sources are enabled
+- Raise informative error messages when shell commands fail
+
+### Fixed
+- Don't reuse cached license when cached version metadata is missing
+- Disable dependency sources when dependent tools are not available
+- Vendored packages from the go std library are properly excluded
+- Cabal dependency enumeration properly includes executable targets
+
 ## 1.0.1 - 2018-04-26
 ### Added
 - GOPATH settable in configuration file

data/README.md

@@ -77,8 +77,10 @@ Dependencies will be automatically detected for
 2. [Bundler (rubygem)](./docs/sources/bundler.md)
 3. [Cabal](./docs/sources/cabal.md)
 4. [Go](./docs/sources/go.md)
-5. [Manifest lists](./docs/sources/manifests.md)
-6. [NPM](./docs/sources/npm.md)
+5. [Go Dep](./docs/sources/dep.md)
+6. [Manifest lists](./docs/sources/manifests.md)
+7. [NPM](./docs/sources/npm.md)
+8. [Pip](./docs/source/pip.md)
 
 You can disable any of them in the configuration file:
 
@@ -92,7 +94,13 @@ sources:
 
 ## Development
 
-After checking out the repo, run `script/bootstrap` to install dependencies. Then, run `script/cibuild` to run the tests. You can also run `script/console` for an interactive prompt that will allow you to experiment.
+To get started after checking out the repo, run
+1. `script/bootstrap` to install dependencies
+2. `script/setup` to setup test fixtures.
+  - `script/setup -f` will force a clean test fixture environment
+3. `script/cibuild` to run the tests.
+
+You can also run `script/console` for an interactive prompt that will allow you to experiment.
 
 To install this gem onto your local machine, run `bundle exec rake install`. To release a new version, update the version number in `version.rb`, and then run `bundle exec rake release`, which will create a git tag for the version, push git commits and tags, and push the `.gem` file to [rubygems.org](https://rubygems.org).
 

data/Rakefile

@@ -4,12 +4,15 @@ require "rake/testtask"
 require "rubocop/rake_task"
 
 desc "Run source setup scripts"
-task :setup do
+task :setup, [:arguments] do |task, args|
+  arguments = args[:arguments].to_s.split
+  force = arguments.include?("-f") ? "-f" : ""
+
   Dir["script/source-setup/*"].each do |script|
     # green
     puts "\033[32mRunning #{script}.\e[0m"
 
-    if system(script)
+    if Bundler.with_clean_env { system(script, force) }
       # green
       puts "\033[32mCompleted #{script}.\e[0m"
     elsif $?.exitstatus == 127

data/docs/configuration.md

@@ -4,6 +4,37 @@ A configuration file specifies the details of enumerating and operating on licen
 
 Configuration can be specified in either YML or JSON formats.  Examples below are given in YML.
 
+## Restricting sources
+
+The `sources` configuration property specifies which sources `licensed` will use to enumerate dependencies.
+By default, `licensed` will generally try to enumerate dependencies from all sources.  As a result,
+the configuration property should be used to explicitly disable sources rather than to enable a particular source.
+
+Be aware that this configuration is separate from an individual sources `#enabled?` method, which determines
+whether the source is valid for the current project.  Even if a source is enabled in the configuration
+it may still determine that it can't enumerate dependencies for a project.
+
+```yml
+sources:
+  bower: true
+  rubygem: false
+```
+
+`licensed` determines which sources will try to enumerate dependencies based on the following rules:
+1. If no sources are configured, all sources are enabled
+2. If no sources are set to true, any unconfigured sources are enabled
+```yml
+sources:
+  bower: false
+  # all other sources are enabled by default since there are no sources set to true
+```
+3. If any sources are set to true, any unconfigured sources are disabled
+```yml
+sources:
+  bower: true
+  # all other sources are disabled by default because a source was set to true
+```
+
 ## Applications
 
 What is an "app"?  In the context of `licensed`, an app is a combination of a source path and a cache path.
@@ -23,7 +54,6 @@ cache_path: 'relative/path/to/cache'
 source_path: 'relative/path/to/source'
 
 # Sources of metadata
-# All sources will attempt to run unless explicitly disabled
 sources:
   bower: true
   rubygem: false

data/docs/sources/bower.md

@@ -1,5 +1,5 @@
 # Bower
 
-The bower source will detect dependencies when the source is enabled and either `.bowerrc` or `bower.json` files are found at an apps `source_path`.
+The bower source will detect dependencies when either `.bowerrc` or `bower.json` files are found at an apps `source_path`.
 
 It enumerates dependencies and metadata from parsing `.bower.json` files for bower components.

data/docs/sources/bundler.md

@@ -1,6 +1,6 @@
 # Bundler (rubygem)
 
-The bundler source will detect dependencies when the source is enabled, `Gemfile` and `Gemfile.lock` files are found at an apps `source_path`.  The source uses the `Bundler` API to enumerate dependencies from `Gemfile` and `Gemfile.lock`.
+The bundler source will detect dependencies `Gemfile` and `Gemfile.lock` files are found at an apps `source_path`.  The source uses the `Bundler` API to enumerate dependencies from `Gemfile` and `Gemfile.lock`.
 
 The bundler source will exclude gems in the `:development` and `:test` groups.  Be aware that if you have a local
 bundler configuration (e.g. `.bundle`), that configuration will be respected as well.  For example, if you have a local

data/docs/sources/cabal.md

@@ -2,7 +2,25 @@
 
 The cabal source uses the `ghc-pkg` command to enumerate dependencies and provide metadata.  It is un-opinionated on GHC packagedb locations and requires some configuration to ensure that all packages are properly found.
 
-The rubygem source will detect dependencies when the source is enabled and a `.cabal` file is found at an apps `source_path`.
+The cabal source will detect dependencies when a `.cabal` file is found at an apps `source_path`.  By default, the cabal source will enumerate dependencies for all executable and library targets in a cabal file.
+
+### Specifying which cabal file targets should enumerate dependencies
+The cabal source can be configured to override which cabal file targets contain dependencies that need to be documented.
+
+The default configuration is equivalent to:
+```yml
+cabal:
+  cabal_file_targets:
+    - executable
+    - library
+```
+
+However if you only wanted to enumerate dependencies for a `my_cabal_exe` executable target, you could specify:
+```yml
+cabal:
+  cabal_file_targets:
+    - executable my_cabal_exe
+```
 
 ### Specifying GHC packagedb locations through environment
 You can configure the `cabal` source to use specific packagedb locations by setting the `GHC_PACKAGE_PATH` environment variable before running `licensed`.

data/docs/sources/dep.md

@@ -0,0 +1,33 @@
+# Warning!
+This source is intended to be used when all of a projects dependencies have been vendored and does not detect non-vendored packages installed at `$GOPATH/pkg`.  If your project uses dependencies that are not listed in `Gopkg.lock`, then you must use the go source to enumerate all project dependencies.
+
+# Go Dep
+
+The dep source will detect dependencies when the source is enabled and both `Gopkg.toml` and `Gopkg.lock` are found at an apps `source_path`.  It
+parses the `Gopkg.lock` file to find packages that have been vendored into the project directory.
+
+This source will self-disable if the `ignored` property in `Gopkg.toml` has any values.  While strongly discouraged, the source can be forced to run
+via configuration.
+
+```yml
+dep:
+  allow_ignored: true # force source to run even if `Gopkg.toml` is non-empty
+```
+
+#### Limitations
+
+The dep dependency source has some limitations compared to the general-purpose go source.
+1. Go std libraries are not filtered from enumerated dependencies if `go list std` is not available
+2. Summary information is not available for packages
+
+#### Go or Dep
+
+Reasons to choose the dep source over the go source
+1. The dep source does not have a hard dependency on go being installed
+  - some functionality is only available if go is available
+    1. filtering go std libs from the found dependencies
+2. The dep source should generally run much more quickly then the go source
+
+Reasons to choose the go source over the dep source
+1. Your project has dependencies not specified by `Gopkpg.lock`
+2. You require dependency summary information

data/docs/sources/npm.md

@@ -1,3 +1,3 @@
 # NPM
 
-The npm source will detect dependencies when the source is enabled and `package.json` is found at an apps `source_path`.  It uses `npm list` to enumerate dependencies and metadata.
+The npm source will detect dependencies `package.json` is found at an apps `source_path`.  It uses `npm list` to enumerate dependencies and metadata.

data/docs/sources/pip.md

@@ -0,0 +1,23 @@
+# Pip
+
+The pip source uses `pip` CLI commands to enumerate dependencies and properties. It is expected that `pip` is available in the `virtual_env_dir` specific directory before running `licensed`.
+
+Your repository root should also contain a `requirements.txt` file which contains all the packages and dependences that are needed. You can generate one with `pip` using the command:
+```
+pip freeze > requirements.txt
+```
+
+A `virtualenv` directory is required before running `licensed`. You can setup a `virtualenv` by running the command:
+```
+virtualenv <your_venv_dir>
+```
+_note_: `<your_venv_dir>` path should be relative to the repository root or can be specified as an absolute path.
+
+#### virtual_env_dir (Required)
+
+The `pip` command will be sourced from this directory.
+An example usage of this might look like:
+```yaml
+python:
+    virtual_env_dir:"/path/to/your/venv_dir"
+```

data/docs/sources/stack.md

@@ -1,3 +1,3 @@
 # HaskellStack
 
-It is not recommended to use this source.  Please see [cabal documentation](./cabal.md) for using the cabal source with stack.
+Please see [cabal documentation](./cabal.md) for using the cabal source with stack.

data/lib/licensed.rb

@@ -1,7 +1,6 @@
 # frozen_string_literal: true
 require "licensed/version"
 require "licensed/shell"
-require "licensed/configuration"
 require "licensed/license"
 require "licensed/dependency"
 require "licensed/git"
@@ -10,7 +9,10 @@ require "licensed/source/bower"
 require "licensed/source/manifest"
 require "licensed/source/npm"
 require "licensed/source/go"
+require "licensed/source/dep"
 require "licensed/source/cabal"
+require "licensed/source/pip"
+require "licensed/configuration"
 require "licensed/command/cache"
 require "licensed/command/status"
 require "licensed/command/list"

data/lib/licensed/command/cache.rb

@@ -11,17 +11,19 @@ module Licensed
       def run(force: false)
         summary = @config.apps.flat_map do |app|
           app_name = app["name"]
-          @config.ui.info "Caching licenes for #{app_name}:"
+          @config.ui.info "Caching licenses for #{app_name}:"
 
           # load the app environment
           Dir.chdir app.source_path do
 
             # map each available app source to it's dependencies
             app.sources.map do |source|
-              @config.ui.info "  #{source.type} dependencies:"
+              type = source.class.type
+
+              @config.ui.info "  #{type} dependencies:"
 
               names = []
-              cache_path = app.cache_path.join(source.type)
+              cache_path = app.cache_path.join(type)
 
               # exclude ignored dependencies
               dependencies = source.dependencies.select { |d| !app.ignored?(d) }
@@ -38,8 +40,9 @@ module Licensed
                 # or default to a blank license
                 license = Licensed::License.read(filename) || Licensed::License.new
 
-                # Version did not change, no need to re-cache
-                if !force && version == license["version"]
+                # cached version string exists and did not change, no need to re-cache
+                has_version = !license["version"].nil? && !license["version"].empty?
+                if !force && has_version && version == license["version"]
                   @config.ui.info "    Using #{name} (#{version})"
                   next
                 end
@@ -60,7 +63,7 @@ module Licensed
                 FileUtils.rm(file) unless names.include?(relative_path.chomp(".txt"))
               end
 
-              "* #{app_name} #{source.type} dependencies: #{dependencies.size}"
+              "* #{app_name} #{type} dependencies: #{dependencies.size}"
             end
           end
         end

data/lib/licensed/command/list.rb

@@ -13,14 +13,16 @@ module Licensed
           @config.ui.info "Displaying dependencies for #{app['name']}"
           Dir.chdir app.source_path do
             app.sources.each do |source|
-              @config.ui.info "  #{source.type} dependencies:"
+              type = source.class.type
+
+              @config.ui.info "  #{type} dependencies:"
 
               source_dependencies = dependencies(app, source)
               source_dependencies.each do |dependency|
                 @config.ui.info "    Found #{dependency['name']} (#{dependency['version']})"
               end
 
-              @config.ui.confirm "  * #{source.type} dependencies: #{source_dependencies.size}"
+              @config.ui.confirm "  * #{type} dependencies: #{source_dependencies.size}"
             end
           end
         end

data/lib/licensed/configuration.rb

@@ -9,6 +9,7 @@ module Licensed
       ".licensed.yaml".freeze,
       ".licensed.json".freeze
     ].freeze
+    SOURCE_TYPES = Source.constants.map { |c| Source.const_get(c) }.freeze
 
     def initialize(options = {}, inherited_options = {})
       super()
@@ -46,19 +47,16 @@ module Licensed
 
     # Returns an array of enabled app sources
     def sources
-      @sources ||= [
-        Source::Bundler.new(self),
-        Source::Bower.new(self),
-        Source::Cabal.new(self),
-        Source::Go.new(self),
-        Source::Manifest.new(self),
-        Source::NPM.new(self)
-      ].select(&:enabled?)
+      @sources ||= SOURCE_TYPES.select { |source_class| enabled?(source_class.type) }
+                               .map { |source_class| source_class.new(self) }
+                               .select(&:enabled?)
     end
 
     # Returns whether a source type is enabled
     def enabled?(source_type)
-      self["sources"].fetch(source_type, true)
+      # the default is false if any sources are set to true, true otherwise
+      default = !self["sources"].any? { |_, enabled| enabled }
+      self["sources"].fetch(source_type, default)
     end
 
     # Is the given dependency reviewed?

data/lib/licensed/git.rb

@@ -23,7 +23,7 @@ module Licensed
         file = File.directory?(descriptor) ? "." : File.basename(descriptor)
 
         Dir.chdir dir do
-          Licensed::Shell.execute("git", "rev-list", "-1", "HEAD", "--", file)
+          Licensed::Shell.execute("git", "rev-list", "-1", "HEAD", "--", file, allow_failure: true)
         end
       end
 

data/lib/licensed/shell.rb

@@ -3,12 +3,19 @@ require "open3"
 
 module Licensed
   module Shell
-    # Executes a command, returning it's STDOUT on success.  Returns an empty
-    # string on failure
-    def self.execute(cmd, *args)
-      output, _, status = Open3.capture3(cmd, *args)
-      return "" unless status.success?
-      output.strip
+    # Executes a command, returning its standard output on success. On failure,
+    # it raises an exception that contains the error output, unless
+    # `allow_failure` is true, in which case it returns an empty string.
+    def self.execute(cmd, *args, allow_failure: false)
+      stdout, stderr, status = Open3.capture3(cmd, *args)
+
+      if status.success?
+        stdout.strip
+      elsif allow_failure
+        ""
+      else
+        raise Error.new([cmd, *args], status.exitstatus, stderr)
+      end
     end
 
     # Executes a command and returns a boolean value indicating if the command
@@ -24,5 +31,31 @@ module Licensed
       output, err, status = Open3.capture3("which", tool)
       status.success? && !output.strip.empty? && err.strip.empty?
     end
+
+    class Error < RuntimeError
+      def initialize(cmd, status, stderr)
+        super()
+        @cmd = cmd
+        @exitstatus = status
+        @output = stderr
+      end
+
+      def message
+        output = @output.to_s.strip
+        extra = output.empty?? "" : "\n#{output.gsub(/^/, "    ")}"
+        "command exited with status #{@exitstatus}\n  #{escape_cmd}#{extra}"
+      end
+
+      def escape_cmd
+        @cmd.map do |arg|
+          if arg =~ /[\s'"]/
+            escaped = arg.gsub(/([\\"])/, '\\\\\1')
+            %("#{escaped}")
+          else
+            arg
+          end
+        end.join(" ")
+      end
+    end
   end
 end

data/lib/licensed/source/bower.rb

@@ -4,17 +4,15 @@ require "json"
 module Licensed
   module Source
     class Bower
-      def initialize(config)
-        @config = config
+      def self.type
+        "bower"
       end
 
-      def type
-        "bower"
+      def initialize(config)
+        @config = config
       end
 
       def enabled?
-        return false unless @config.enabled?(type)
-
         [@config.pwd.join(".bowerrc"), @config.pwd.join("bower.json")].any? do |path|
           File.exist?(path)
         end
@@ -25,7 +23,7 @@ module Licensed
           package = JSON.parse(File.read(file))
           path = bower_path.join(file).dirname.to_path
           Dependency.new(path, {
-            "type"     => type,
+            "type"     => Bower.type,
             "name"     => package["name"],
             "version"  => package["version"] || package["_release"],
             "summary"  => package["description"],

data/lib/licensed/source/bundler.rb

@@ -1,28 +1,31 @@
 # frozen_string_literal: true
-require "bundler"
+begin
+  require "bundler"
+rescue LoadError
+end
 
 module Licensed
   module Source
     class Bundler
       GEMFILES = %w{Gemfile gems.rb}.freeze
 
+      def self.type
+        "rubygem"
+      end
+
       def initialize(config)
         @config = config
       end
 
       def enabled?
-        @config.enabled?(type) && lockfile_path && lockfile_path.exist?
-      end
-
-      def type
-        "rubygem"
+        defined?(::Bundler) && lockfile_path && lockfile_path.exist?
       end
 
       def dependencies
         @dependencies ||= with_local_configuration do
           definition.specs_for(groups).map do |spec|
             Dependency.new(spec.gem_dir, {
-              "type"     => type,
+              "type"     => Bundler.type,
               "name"     => spec.name,
               "version"  => spec.version.to_s,
               "summary"  => spec.summary,

data/lib/licensed/source/cabal.rb

@@ -4,16 +4,19 @@ require "English"
 module Licensed
   module Source
     class Cabal
-      def initialize(config)
-        @config = config
-      end
+      DEPENDENCY_REGEX = /\s*.+?\s*/.freeze
+      DEFAULT_TARGETS = %w{executable library}.freeze
 
-      def type
+      def self.type
         "cabal"
       end
 
+      def initialize(config)
+        @config = config
+      end
+
       def enabled?
-        @config.enabled?(type) && cabal_packages.any? && ghc?
+        cabal_file_dependencies.any? && ghc?
       end
 
       def dependencies
@@ -22,7 +25,7 @@ module Licensed
 
           path, search_root = package_docs_dirs(package)
           Dependency.new(path, {
-            "type"     => type,
+            "type"     => Cabal.type,
             "name"     => package["name"],
             "version"  => package["version"],
             "summary"  => package["synopsis"],
@@ -63,8 +66,7 @@ module Licensed
 
       # Returns a `Set` of the package ids for all cabal dependencies
       def package_ids
-        deps = cabal_packages.flat_map { |n| package_dependencies(n, false) }
-        recursive_dependencies(deps)
+        recursive_dependencies(cabal_file_dependencies)
       end
 
       # Recursively finds the dependencies for each cabal package.
@@ -125,7 +127,7 @@ module Licensed
       # Runs a `ghc-pkg field` command for a given set of fields and arguments
       # Automatically includes ghc package DB locations in the command
       def ghc_pkg_field_command(id, fields, *args)
-        Licensed::Shell.execute("ghc-pkg", "field", id, fields.join(","), *args, *package_db_args)
+        Licensed::Shell.execute("ghc-pkg", "field", id, fields.join(","), *args, *package_db_args, allow_failure: true)
       end
 
       # Returns an array of ghc package DB locations as specified in the app
@@ -148,12 +150,56 @@ module Licensed
         path.gsub("<ghc_version>", ghc_version)
       end
 
-      # Return an array of the top-level cabal packages for the current app
-      def cabal_packages
-        cabal_files.map do |f|
-          name_match = File.read(f).match(/^name:\s*(.*)$/)
-          name_match[1] if name_match
-        end.compact
+      # Returns a set containing the top-level dependencies found in cabal files
+      def cabal_file_dependencies
+        cabal_files.each_with_object(Set.new) do |cabal_file, packages|
+          content = File.read(cabal_file)
+          next if content.nil? || content.empty?
+
+          # add any dependencies for matched targets from the cabal file.
+          # by default this will find executable and library dependencies
+          content.scan(cabal_file_regex).each do |match|
+            # match[1] is a string of "," separated dependencies
+            dependencies = match[1].split(",").map(&:strip)
+            dependencies.each do |dep|
+              # the dependency might have a version specifier.
+              # remove it so we can get the full id specifier for each package
+              id = cabal_package_id(dep.split(/\s/)[0])
+              packages.add(id) if id
+            end
+          end
+        end
+      end
+
+      # Returns an installed package id for the package.
+      def cabal_package_id(package_name)
+        field = ghc_pkg_field_command(package_name, ["id"])
+        id = field.split(":", 2)[1]
+        id.strip if id
+      end
+
+      # Find `build-depends` lists from specified targets in a cabal file
+      def cabal_file_regex
+        # this will match 0 or more occurences of
+        # match[0] - specifier, e.g. executable, library, etc
+        # match[1] - full list of matched dependencies
+        # match[2] - first matched dependency (required)
+        # match[3] - remainder of matched dependencies (not required)
+        @cabal_file_regex ||= /
+          # match a specifier, e.g. library or executable
+          ^(#{cabal_file_targets.join("|")})
+            .*? # stuff
+
+            # match a list of 1 or more dependencies
+            build-depends:(#{DEPENDENCY_REGEX}(,#{DEPENDENCY_REGEX})*)\n
+        /xmi
+      end
+
+      # Returns the targets to search for `build-depends` in a cabal file
+      def cabal_file_targets
+        targets = Array(@config.dig("cabal", "cabal_file_targets"))
+        targets.push(*DEFAULT_TARGETS) if targets.empty?
+        targets
       end
 
       # Returns an array of the local directory cabal package files

data/lib/licensed/source/dep.rb

@@ -0,0 +1,87 @@
+# frozen_string_literal: true
+require "tomlrb"
+
+module Licensed
+  module Source
+    class Dep
+      def self.type
+        "dep"
+      end
+
+      def initialize(config)
+        @config = config
+      end
+
+      def enabled?
+        go_dep_available?
+      end
+
+      def dependencies
+        @dependencies ||= begin
+          packages.map do |package|
+            package_dir = @config.pwd.join("vendor", package[:name])
+            search_root = @config.pwd.join("vendor", package[:project])
+
+            unless package_dir.exist?
+              next if @config.ignored?("type" => Dep.type, "name" => package[:name])
+              raise "couldn't find package for #{package[:name]}"
+            end
+
+            Dependency.new(package_dir.to_s, {
+              "type"        => Dep.type,
+              "name"        => package[:name],
+              "homepage"    => "https://#{package[:name]}",
+              "search_root" => search_root.to_s,
+              "version"     => package[:version]
+            })
+          end
+        end
+      end
+
+      # Returns an array of dependency packages specified from Gopkg.lock
+      def packages
+        gopkg_lock = Tomlrb.load_file(gopkg_lock_path, symbolize_keys: true)
+        return [] unless gopkg_lock && gopkg_lock[:projects]
+
+        gopkg_lock[:projects].flat_map do |project|
+          # map each package to a full import path
+          # then return a hash for each import path containing the path and the version
+          project[:packages].map { |package| package == "." ? project[:name] : "#{project[:name]}/#{package}" }
+                            .reject { |import_path| go_std_package?(import_path) }
+                            .map { |import_path| { name: import_path, version: project[:revision], project: project[:name] } }
+        end
+      end
+
+      # Returns whether the package is part of the go std list.  Replaces
+      # "golang.org" with "golang_org" to match packages listed in `go list std`
+      # as "vendor/golang_org/*" but are vendored as "vendor/golang.org/*"
+      def go_std_package?(import_path)
+        go_std_packages.include? "vendor/#{import_path.sub(/^golang.org/, "golang_org")}"
+      end
+
+      def go_dep_available?
+        return false unless gopkg_lock_path.exist? && gopkg_toml_path.exist?
+        return true if @config.dig("dep", "allow_ignored") == true
+
+        gopkg_toml = Tomlrb.load_file(gopkg_toml_path, symbolize_keys: true)
+        gopkg_toml[:ignored].nil? || gopkg_toml[:ignored].empty?
+      end
+
+      def gopkg_lock_path
+        @config.pwd.join("Gopkg.lock")
+      end
+
+      def gopkg_toml_path
+        @config.pwd.join("Gopkg.toml")
+      end
+
+      # Returns a list of go standard packages
+      def go_std_packages
+        @std_packages ||= begin
+          return [] unless Licensed::Shell.tool_available?("go")
+          Licensed::Shell.execute("go", "list", "std").lines.map(&:strip)
+        end
+      end
+    end
+  end
+end

data/lib/licensed/source/go.rb

@@ -1,20 +1,21 @@
 # frozen_string_literal: true
 require "json"
 require "English"
+require "pathname"
 
 module Licensed
   module Source
     class Go
-      def initialize(config)
-        @config = config
+      def self.type
+        "go"
       end
 
-      def type
-        "go"
+      def initialize(config)
+        @config = config
       end
 
       def enabled?
-        @config.enabled?(type) && go_source?
+        Licensed::Shell.tool_available?("go") && go_source?
       end
 
       def dependencies
@@ -24,13 +25,13 @@ module Licensed
             import_path = non_vendored_import_path(package_name)
 
             if package.empty?
-              next if @config.ignored?("type" => type, "name" => package_name)
+              next if @config.ignored?("type" => Go.type, "name" => package_name)
               raise "couldn't find package for #{import_path}"
             end
 
             package_dir = package["Dir"]
             Dependency.new(package_dir, {
-              "type"        => type,
+              "type"        => Go.type,
               "name"        => import_path,
               "summary"     => package["Doc"],
               "homepage"    => homepage(import_path),
@@ -61,6 +62,13 @@ module Licensed
           .uniq
           .select { |d| !go_std_packages.include?(d) }
           .select { |d| !d.start_with?(root_package["ImportPath"]) || vendored_path?(d) }
+          .select do |d|
+          # this removes the packages listed in `go list std` as "vendor/golang_org/*" but are vendored
+          # as "vendor/golang.org/*"
+            go_std_packages.none? do |std_pkg|
+              std_pkg.sub(%r{^vendor/golang_org/}, "#{root_package["ImportPath"]}/vendor/golang.org/") == d
+            end
+          end
       end
 
       # Returns the root directory to search for a package license
@@ -106,7 +114,7 @@ module Licensed
       # package - Go package import path
       def package_info_command(package)
         package ||= ""
-        Licensed::Shell.execute("go", "list", "-json", package)
+        Licensed::Shell.execute("go", "list", "-json", package, allow_failure: true)
       end
 
       # Returns the info for the package under test
@@ -133,7 +141,12 @@ module Licensed
         @gopath = if path.nil? || path.empty?
                     ENV["GOPATH"]
                   else
-                    File.expand_path(path, Licensed::Git.repository_root)
+                    root = begin
+                             Licensed::Git.repository_root
+                           rescue Licensed::Shell::Error
+                             Pathname.pwd
+                           end
+                    File.expand_path(path, root)
                   end
       end
 

data/lib/licensed/source/manifest.rb

@@ -4,22 +4,22 @@ require "pathname/common_prefix"
 module Licensed
   module Source
     class Manifest
+      def self.type
+        "manifest"
+      end
+
       def initialize(config)
         @config = config
       end
 
       def enabled?
-        @config.enabled?(type) && File.exist?(manifest_path)
-      end
-
-      def type
-        "manifest"
+        File.exist?(manifest_path)
       end
 
       def dependencies
         @dependencies ||= packages.map do |package_name, sources|
           Dependency.new(sources_license_path(sources), {
-            "type"     => type,
+            "type"     => Manifest.type,
             "name"     => package_name,
             "version"  => package_version(sources)
           })

data/lib/licensed/source/npm.rb

@@ -4,16 +4,16 @@ require "json"
 module Licensed
   module Source
     class NPM
-      def initialize(config)
-        @config = config
+      def self.type
+        "npm"
       end
 
-      def type
-        "npm"
+      def initialize(config)
+        @config = config
       end
 
       def enabled?
-        @config.enabled?(type) && File.exist?(@config.pwd.join("package.json"))
+        Licensed::Shell.tool_available?("npm") && File.exist?(@config.pwd.join("package.json"))
       end
 
       def dependencies
@@ -31,7 +31,7 @@ module Licensed
           path = package["realPath"] || locations["#{package["name"]}@#{package["version"]}"]
           fail "couldn't locate #{name} under node_modules/" unless path
           Dependency.new(path, {
-            "type"     => type,
+            "type"     => NPM.type,
             "name"     => package["name"],
             "version"  => package["version"],
             "summary"  => package["description"],

data/lib/licensed/source/pip.rb

@@ -0,0 +1,63 @@
+# frozen_string_literal: true
+require "json"
+require "English"
+
+module Licensed
+  module Source
+    class Pip
+      def self.type
+        "pip"
+      end
+
+      def initialize(config)
+        @config = config
+      end
+
+      def enabled?
+        File.exist?(@config.pwd.join("requirements.txt"))
+      end
+
+      def dependencies
+        @dependencies ||= parse_requirements_txt.map do |package_name|
+          package = package_info(package_name)
+          location = File.join(package["Location"], package["Name"] +  "-" + package["Version"] + ".dist-info")
+          Dependency.new(location, {
+                           "type"        => Pip.type,
+                           "name"        => package["Name"],
+                           "summary"     => package["Summary"],
+                           "homepage"    => package["Home-page"],
+                           "version"     => package["Version"]
+                         })
+        end
+      end
+
+      # Build the list of packages from a 'requirements.txt'
+      # Assumes that the requirements.txt follow the format pkg=1.0.0 or pkg==1.0.0
+      def parse_requirements_txt
+        File.open(@config.pwd.join("requirements.txt")).map do |line|
+          p_split = line.split("=")
+          p_split[0]
+        end
+      end
+
+      def package_info(package_name)
+        p_info = pip_command(package_name).lines
+        p_info.each_with_object(Hash.new(0)) { |pkg, a|
+          k, v = pkg.split(":", 2)
+          next if k.nil? || k.empty?
+          a[k.strip] = v&.strip
+        }
+      end
+
+      def pip_command(*args)
+        venv_dir = @config.dig("python", "virtual_env_dir")
+        if venv_dir.nil?
+          raise "Virtual env directory not set."
+        end
+        venv_dir = File.expand_path(venv_dir, Licensed::Git.repository_root)
+        pip = File.join(venv_dir, "bin", "pip")
+        Licensed::Shell.execute(pip, "--disable-pip-version-check", "show", *args)
+      end
+    end
+  end
+end

data/lib/licensed/version.rb

@@ -1,4 +1,4 @@
 # frozen_string_literal: true
 module Licensed
-  VERSION = "1.0.1".freeze
+  VERSION = "1.1.0".freeze
 end

data/licensed.gemspec

@@ -25,6 +25,7 @@ Gem::Specification.new do |spec|
   spec.add_dependency "thor", "~>0.19"
   spec.add_dependency "octokit", "~>4.0"
   spec.add_dependency "pathname-common_prefix", "~>0.0.1"
+  spec.add_dependency "tomlrb", "~>1.2"
 
   spec.add_development_dependency "bundler", "~> 1.10"
   spec.add_development_dependency "rake", "~> 10.0"

data/script/setup

@@ -2,4 +2,4 @@
 
 set -e
 
-bundle exec rake setup
+bundle exec rake setup["$@"]

data/script/source-setup/bower

@@ -9,4 +9,9 @@ fi
 # setup test fixtures
 BASE_PATH="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)"
 cd $BASE_PATH/test/fixtures/bower
+
+if [ "$1" == "-f" ]; then
+  find . -not -regex "\.*" -and -not -name "bower\.json" -print0 | xargs -0 rm -rf
+fi
+
 bower install

data/script/source-setup/bundler

@@ -13,4 +13,8 @@ cd $BASE_PATH/test/fixtures/bundler
 # unset any pre-existing gemfile when installing test fixtures
 unset BUNDLE_GEMFILE
 
-bundle install --path $BASE_PATH/test/fixtures/bundler/vendor/gems
+if [ "$1" == "-f" ]; then
+  find . -not -regex "\.*" -and -not -name "Gemfile" -print0 | xargs -0 rm -rf
+fi
+
+bundle install --path vendor/gems

data/script/source-setup/cabal

@@ -8,5 +8,10 @@ fi
 
 # setup test fixtures
 BASE_PATH="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)"
-cd $BASE_PATH/test/fixtures/haskell
+cd $BASE_PATH/test/fixtures/cabal
+
+if [ "$1" == "-f" ]; then
+  find . -not -regex "\.*" -and -not -path "*app*" -print0 | xargs -0 rm -rf
+fi
+
 cabal new-build

data/script/source-setup/go

@@ -9,5 +9,11 @@ fi
 # setup test fixtures
 BASE_PATH="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)"
 export GOPATH="$BASE_PATH/test/fixtures/go"
-cd $BASE_PATH/test/fixtures/go/src/test
+cd $BASE_PATH/test/fixtures/go
+
+if [ "$1" == "-f" ]; then
+  find . -not -regex "\.*" -and -not -path "*/src/test*" -and -not -path "*/src" -print0 | xargs -0 rm -rf
+fi
+
+cd src/test
 go get

data/script/source-setup/npm

@@ -9,4 +9,9 @@ fi
 # setup test fixtures
 BASE_PATH="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)"
 cd $BASE_PATH/test/fixtures/npm
+
+if [ "$1" == "-f" ]; then
+  find . -not -regex "\.*" -and -not -name "package\.json" -print0 | xargs -0 rm -rf
+fi
+
 npm install

data/script/source-setup/pip

@@ -0,0 +1,29 @@
+#!/bin/bash
+set -e
+
+if [ -z "$(which pip)" ]; then
+  echo "A local pip installation is required for python development." >&2
+  exit 127
+fi
+
+if [ -z "$(which virtualenv)" ]; then
+    echo "A local virtualenv installation is required for python development." >&2
+    exit 127
+fi
+
+
+# setup test fixtures
+BASE_PATH="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)"
+
+
+# clean up any previous fixture venv that might have been created.
+if [ "$1" == "-f" ]; then
+    echo "removing old fixture setup..."
+    rm -rf  $BASE_PATH/test/fixtures/pip/venv
+fi
+
+# set up a virtualenv and install the packages in the test requirements
+virtualenv $BASE_PATH/test/fixtures/pip/venv
+. $BASE_PATH/test/fixtures/pip/venv/bin/activate
+pip install -r $BASE_PATH/test/fixtures/pip/requirements.txt
+deactivate

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: licensed
 version: !ruby/object:Gem::Version
-  version: 1.0.1
+  version: 1.1.0
 platform: ruby
 authors:
 - GitHub
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2018-04-26 00:00:00.000000000 Z
+date: 2018-06-04 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: licensee
@@ -66,6 +66,20 @@ dependencies:
     - - "~>"
       - !ruby/object:Gem::Version
         version: 0.0.1
+- !ruby/object:Gem::Dependency
+  name: tomlrb
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.2'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.2'
 - !ruby/object:Gem::Dependency
   name: bundler
   requirement: !ruby/object:Gem::Requirement
@@ -201,9 +215,11 @@ files:
 - docs/sources/bower.md
 - docs/sources/bundler.md
 - docs/sources/cabal.md
+- docs/sources/dep.md
 - docs/sources/go.md
 - docs/sources/manifests.md
 - docs/sources/npm.md
+- docs/sources/pip.md
 - docs/sources/stack.md
 - exe/licensed
 - lib/licensed.rb
@@ -219,9 +235,11 @@ files:
 - lib/licensed/source/bower.rb
 - lib/licensed/source/bundler.rb
 - lib/licensed/source/cabal.rb
+- lib/licensed/source/dep.rb
 - lib/licensed/source/go.rb
 - lib/licensed/source/manifest.rb
 - lib/licensed/source/npm.rb
+- lib/licensed/source/pip.rb
 - lib/licensed/ui/shell.rb
 - lib/licensed/version.rb
 - licensed.gemspec
@@ -234,6 +252,7 @@ files:
 - script/source-setup/cabal
 - script/source-setup/go
 - script/source-setup/npm
+- script/source-setup/pip
 - script/test
 homepage: https://github.com/github/licensed
 licenses: