licensed 2.1.0 → 2.2.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: c7f39c048283d853075674325f0d371edaf124a8
-  data.tar.gz: 60530129f47ed34f51cac3152be83aab27e32a01
+  metadata.gz: ee8b9896bf3a7728b539c308822b7613b70ddc62
+  data.tar.gz: e3c4f581c2a1982b426a82dfd7c5217ca98d2838
 SHA512:
-  metadata.gz: 2b2bc431aea2a64f3f12139d533914ad2de740cc5eb00310166dbfa6160fc5a6821d73faf214ceb49fcb6207d9c695561fc44a9ff21e146d88cc4f4eb7de0d38
-  data.tar.gz: 53db8e3e0272e976428471bbe0f5a2c6e54cdb7743b2e309ac8b2d839a352073fb8a81026e8bc19838f155a09d0b2a74097736ee6e68bc4152eb2124177ceb7e
+  metadata.gz: 05bc23396c71a8d445412965cc9decf5abb6d5a6ebc450060e318266a3504368abb257649092ffe1eae615a93b6e69e24f9192a7ed475c39ff9e943332db2cb4
+  data.tar.gz: 2fa140721c4e3fbfe22fe91048c8cac718ae6e918efc675cb800ea606d260de6383d977fa13ce1cd309b81c36275245183bbbbeca0b1f2ccfb81c812fb0f06a8

data/CHANGELOG.md

@@ -6,6 +6,14 @@ and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0.
 
 ## [Unreleased]
 
+## 2.2.0 - 2019-05-11
+
+### Added
+- Content hash versioning strategy for go and manifest sources (https://github.com/github/licensed/pull/164)
+
+### Fixed
+- Python source handles urls and package names with "-" in requirements.txt (:tada: @krzysztof-pawlik-gat https://github.com/github/licensed/pull/165)
+
 ## 2.1.0 - 2019-04-16
 
 ### Added
@@ -154,4 +162,4 @@ and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0.
 
 Initial release :tada:
 
-[Unreleased]: https://github.com/github/licensed/compare/2.0.1...HEAD
+[Unreleased]: https://github.com/github/licensed/compare/2.2.0...HEAD

data/docs/sources/go.md

@@ -23,3 +23,20 @@ go:
 The setting supports absolute, relative and expandable (e.g. "~") paths.  Relative paths are considered relative to the repository root.
 
 Non-empty `GOPATH` configuration settings will override the `GOPATH` environment variable while enumerating `go` dependencies.  The `GOPATH` environment variable is restored once dependencies have been enumerated.
+
+#### Versioning
+
+The go source supports multiple versioning strategies to determine if cached dependency metadata is stale.  A version strategy is chosen based on the availability of go module information along with the current app configuration.
+
+1. Go Module version - This strategy uses the version of the go module.
+   - :exclamation: This strategy will always be used if go module information is available because the version comes from an externally provided identifier.  Locating the version of the source package used via this identifier will be easier than other strategies.
+2. Git commit SHA - This strategy uses the latest Git commit SHA available for the package's import path directory as the version.  This is the default strategy used if a go module version isn't available and the setting is not configured.
+   - :warning: The latest Git commit won't capture any changes that are committed alongside a cached file update.  Make sure to update cached files after all other changes are committed.
+
+   ```yaml
+   version_strategy: git # or leave this key unset
+   ```
+3. Contents hash - This strategy uses a hash of the files in the package's import path directory as the version.
+   ```yaml
+   version_strategy: contents
+   ```

data/docs/sources/manifests.md

@@ -145,3 +145,18 @@ manifest:
   licenses:
     package: path/to/LICENSE
 ```
+
+### License content versioning
+
+The manifest source supports multiple versioning strategies to determine if cached dependency metadata is stale.  A version strategy is chosen based on the current app configuration.
+
+1. Git commit SHA - This strategy uses the latest Git commit SHA available for the package's import path directory as the version.  This is the default strategy used if not otherwise configured.
+   - :warning: The latest Git commit won't capture any changes that are committed alongside a cached file update.  Make sure to update cached files after all other changes are committed.
+
+   ```yaml
+   version_strategy: git # or leave this key unset
+   ```
+2. Contents hash - This strategy uses a hash of the files in the package's import path directory as the version.
+   ```yaml
+   version_strategy: contents
+   ```

data/lib/licensed/sources/go.rb

@@ -1,10 +1,13 @@
 # frozen_string_literal: true
 require "json"
 require "pathname"
+require "licensed/sources/helpers/content_versioning"
 
 module Licensed
   module Sources
     class Go < Source
+      include Licensed::Sources::ContentVersioning
+
       def enabled?
         Licensed::Shell.tool_available?("go") && go_source?
       end
@@ -102,7 +105,19 @@ module Licensed
         # find most recent git SHA for a package, or nil if SHA is
         # not available
         Dir.chdir package_directory do
-          Licensed::Git.version(".")
+          contents_version *contents_version_arguments
+        end
+      end
+
+      # Determines the arguments to pass to contents_version based on which
+      # version strategy is selected
+      #
+      # Returns an array of arguments to pass to contents version
+      def contents_version_arguments
+        if version_strategy == Licensed::Sources::ContentVersioning::GIT
+          ["."]
+        else
+          Dir["*"]
         end
       end
 

data/lib/licensed/sources/helpers/content_versioning.rb

@@ -0,0 +1,72 @@
+# frozen_string_literal: true
+
+require "ruby-xxhash"
+
+module Licensed
+  module Sources
+    module ContentVersioning
+      GIT = "git".freeze
+      CONTENTS = "contents".freeze
+
+      # Find the version for a list of paths using the version strategy
+      # specified for the source from the configuration
+      #
+      # paths - list of paths to find version
+      #
+      # Returns a version identifier for the given files
+      def contents_version(*paths)
+        case version_strategy
+        when CONTENTS
+          contents_hash(paths)
+        when GIT
+          git_version(paths)
+        end
+      end
+
+      # Returns the version strategy configured for the source
+      def version_strategy
+        # default to git for backwards compatible behavior
+        @version_strategy ||= begin
+           case config.fetch("version_strategy", nil)
+           when CONTENTS
+             CONTENTS
+           when GIT
+             GIT
+           else
+             Licensed::Git.available? ? GIT : CONTENTS
+           end
+        end
+      end
+
+      # Find the version for a list of paths using Git commit information
+      #
+      # paths - list of paths to find version
+      #
+      # Returns the most recent git SHA from the given paths
+      def git_version(paths)
+        return if paths.nil?
+
+        paths.map { |path| Licensed::Git.version(path) }
+             .reject { |sha| sha.to_s.empty? }
+             .max_by { |sha| Licensed::Git.commit_date(sha) }
+      end
+
+      # Find the version for a list of paths using their file contents
+      #
+      # paths - list of paths to find version
+      #
+      # Returns a hash of the path contents as an identifier for the group
+      def contents_hash(paths)
+        return if paths.nil?
+
+        paths = paths.compact.select { |path| File.file?(path) }
+        return if paths.empty?
+
+        paths.sort
+             .reduce(Digest::XXHash64.new, :file)
+             .digest
+             .to_s(16) # convert to hex
+      end
+    end
+  end
+end

data/lib/licensed/sources/manifest.rb

@@ -1,9 +1,12 @@
 # frozen_string_literal: true
 require "pathname/common_prefix"
+require "licensed/sources/helpers/content_versioning"
 
 module Licensed
   module Sources
     class Manifest < Source
+      include Licensed::Sources::ContentVersioning
+
       def enabled?
         File.exist?(manifest_path) || generate_manifest?
       end
@@ -12,7 +15,7 @@ module Licensed
         packages.map do |package_name, sources|
           Licensed::Sources::Manifest::Dependency.new(
             name: package_name,
-            version: package_version(sources),
+            version: contents_version(*sources),
             path: configured_license_path(package_name) || sources_license_path(sources),
             sources: sources,
             metadata: {
@@ -23,15 +26,6 @@ module Licensed
         end
       end
 
-      # Returns the latest git SHA available from `sources`
-      def package_version(sources)
-        return if sources.nil? || sources.empty?
-
-        sources.map { |s| Licensed::Git.version(s) }
-               .compact
-               .max_by { |sha| Licensed::Git.commit_date(sha) }
-      end
-
       # Returns the license path for a package specified in the configuration.
       def configured_license_path(package_name)
         license_path = @config.dig("manifest", "licenses", package_name)

data/lib/licensed/sources/pip.rb

@@ -6,7 +6,7 @@ module Licensed
   module Sources
     class Pip < Source
       VERSION_OPERATORS = %w(< > <= >= == !=).freeze
-      PACKAGE_REGEX = /^(\w+)(#{VERSION_OPERATORS.join("|")})?/
+      PACKAGE_REGEX = /^([\w-]+)(#{VERSION_OPERATORS.join("|")})?/
 
       def enabled?
         return unless virtual_env_pip && Licensed::Shell.tool_available?(virtual_env_pip)
@@ -16,7 +16,7 @@ module Licensed
       def enumerate_dependencies
         packages_from_requirements_txt.map do |package_name|
           package = package_info(package_name)
-          location = File.join(package["Location"], package["Name"] +  "-" + package["Version"] + ".dist-info")
+          location = File.join(package["Location"], package["Name"].gsub("-", "_") +  "-" + package["Version"] + ".dist-info")
           Dependency.new(
             name: package["Name"],
             version: package["Version"],
@@ -35,6 +35,7 @@ module Licensed
       def packages_from_requirements_txt
         File.read(@config.pwd.join("requirements.txt"))
             .lines
+            .reject { |line| line.include?("://") }
             .map { |line| line.strip.match(PACKAGE_REGEX) { |match| match.captures.first } }
             .compact
       end

data/lib/licensed/version.rb

@@ -1,6 +1,6 @@
 # frozen_string_literal: true
 module Licensed
-  VERSION = "2.1.0".freeze
+  VERSION = "2.2.0".freeze
 
   def self.previous_major_versions
     major_version = Gem::Version.new(Licensed::VERSION).segments.first

data/licensed.gemspec

@@ -28,6 +28,7 @@ Gem::Specification.new do |spec|
   spec.add_dependency "pathname-common_prefix", "~> 0.0.1"
   spec.add_dependency "tomlrb", "~> 1.2"
   spec.add_dependency "bundler", ">= 1.10"
+  spec.add_dependency "ruby-xxHash", "~> 0.4"
 
   spec.add_development_dependency "rake", "~> 10.0"
   spec.add_development_dependency "minitest", "~> 5.8"

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: licensed
 version: !ruby/object:Gem::Version
-  version: 2.1.0
+  version: 2.2.0
 platform: ruby
 authors:
 - GitHub
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2019-04-17 00:00:00.000000000 Z
+date: 2019-05-11 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: licensee
@@ -80,6 +80,20 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: '1.10'
+- !ruby/object:Gem::Dependency
+  name: ruby-xxHash
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.4'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.4'
 - !ruby/object:Gem::Dependency
   name: rake
   requirement: !ruby/object:Gem::Requirement
@@ -236,6 +250,7 @@ files:
 - lib/licensed/sources/git_submodule.rb
 - lib/licensed/sources/go.rb
 - lib/licensed/sources/gradle.rb
+- lib/licensed/sources/helpers/content_versioning.rb
 - lib/licensed/sources/manifest.rb
 - lib/licensed/sources/npm.rb
 - lib/licensed/sources/pip.rb