license_scout 1.3.2 → 2.0.2

data/lib/license_scout/dependency_manager/mix.rb

@@ -0,0 +1,105 @@
+#
+# Copyright:: Copyright 2016, Chef Software Inc.
+# License:: Apache License, Version 2.0
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+require "license_scout/dependency_manager/base"
+require "license_scout/exceptions"
+
+require "mixlib/shellout"
+require "ffi_yajl"
+
+module LicenseScout
+  module DependencyManager
+    class Mix < Base
+
+      attr_reader :packaged_dependencies
+
+      def initialize(directory)
+        super(directory)
+
+        @packaged_dependencies = {}
+      end
+
+      def name
+        "elixir_mix"
+      end
+
+      def type
+        "elixir"
+      end
+
+      def signature
+        "mix.lock file"
+      end
+
+      def install_command
+        "mix deps.get"
+      end
+
+      def detected?
+        File.exist?(mix_lock_path)
+      end
+
+      def dependencies
+        parse_packaged_dependencies
+
+        # Some dependencies are obtained via 'pkg' identifier of rebar. These
+        # dependencies include their version in the rebar.lock file. Here we
+        # parse the rebar.lock and remember all the versions we find.
+        packaged_dependencies.map do |dep_name, dep_version|
+          dep_path = Dir.glob(File.join(directory, "**", "deps", dep_name)).first
+
+          dependency = new_dependency(dep_name, dep_version, dep_path)
+
+          Array(hex_info(dep_name).dig("meta", "licenses")).each do |license|
+            dependency.add_license(license, "https://hex.pm/api/packages/#{dep_name}")
+          end
+
+          dependency
+        end.compact
+      end
+
+      private
+
+      def parse_packaged_dependencies
+        mix_lock_to_json_path = File.expand_path("../../../bin/mix_lock_json", File.dirname(__FILE__))
+        s = Mixlib::ShellOut.new("#{LicenseScout::Config.escript_bin} #{mix_lock_to_json_path} #{mix_lock_path}", environment: LicenseScout::Config.environment)
+        s.run_command
+        s.error!
+
+        mix_lock_content = FFI_Yajl::Parser.parse(s.stdout)
+
+        mix_lock_content.each do |dep|
+          name = dep.keys.first
+          version = dep.values.first
+
+          @packaged_dependencies[name] = version
+        end
+      end
+
+      def mix_lock_path
+        File.join(directory, "mix.lock")
+      end
+
+      def hex_info(package_name)
+        FFI_Yajl::Parser.parse(open("https://hex.pm/api/packages/#{package_name}").read)
+      rescue OpenURI::HTTPError
+        LicenseScout::Log.debug("[elixir] Unable to download hex.pm info for #{package_name}")
+        {}
+      end
+    end
+  end
+end

data/lib/license_scout/dependency_manager/npm.rb

@@ -15,16 +15,28 @@
 # limitations under the License.
 #
 
-require "set" unless defined?(Set)
-require "ffi_yajl" unless defined?(FFI_Yajl)
+require "set"
+require "ffi_yajl"
 require "license_scout/dependency_manager/base"
 
 module LicenseScout
   module DependencyManager
-    class NPM < Base
+    class Npm < Base
 
       def name
-        "js_npm"
+        "nodejs_npm"
+      end
+
+      def type
+        "nodejs"
+      end
+
+      def signature
+        "node_modules directory"
+      end
+
+      def install_command
+        "npm install"
       end
 
       def detected?
@@ -32,33 +44,26 @@ module LicenseScout
       end
 
       def dependencies
-        packages = all_package_json_files.inject(Set.new) do |package_set, package_json_file|
+        all_package_json_files.inject(Set.new) do |uniq_deps, package_json_file|
           pkg_info = File.open(package_json_file) do |f|
             FFI_Yajl::Parser.parse(f)
           end
 
-          pkg_name = pkg_info["name"]
-          pkg_version = pkg_info["version"]
-          package_path = File.dirname(package_json_file)
+          dep_name = pkg_info["name"]
+          dep_version = pkg_info["version"]
+          dep_path = File.dirname(package_json_file)
 
-          license = options.overrides.license_for(name, pkg_name, pkg_version) ||
-            normalize_license_data(pkg_info["license"] || pkg_info["licence"] || pkg_info["licenses"])
+          dependency = new_dependency(dep_name, dep_version, dep_path)
 
-          override_license_files = options.overrides.license_files_for(name, pkg_name, pkg_version)
-          if override_license_files.empty?
-            license_files = find_license_files_in(package_path)
-          else
-            license_files = override_license_files.resolve_locations(package_path)
+          case pkg_info["license"]
+          when String
+            dependency.add_license(pkg_info["license"], "package.json")
+          when Hash
+            dependency.add_license(pkg_info["license"]["type"], "package.json", pkg_info["license"]["url"])
           end
 
-          package_set << create_dependency(
-            pkg_name,
-            pkg_version,
-            license,
-            license_files
-          )
-        end
-        packages.to_a
+          uniq_deps << dependency
+        end.to_a
       end
 
       private
@@ -79,7 +84,7 @@ module LicenseScout
       # package metadata is removed).
       def all_package_json_files
         all_files = []
-        package_dirs = [project_dir]
+        package_dirs = [directory]
         loop do
           break if package_dirs.empty?
 
@@ -98,70 +103,9 @@ module LicenseScout
         all_files
       end
 
-      def find_license_files_in(dir)
-        root_files = Dir["#{dir}/*"]
-        root_files.select { |f| POSSIBLE_LICENSE_FILES.include?(File.basename(f)) }
-      end
-
-      def normalize_license_data(license_metadata)
-        license_string =
-          case license_metadata
-          when nil
-            nil
-          when String
-            license_metadata
-          when Hash
-            license_metadata["type"]
-          when Array
-            if (map = license_metadata.first) && map.is_a?(Hash) && (type = map["type"])
-              type
-            else
-              nil
-            end
-          end
-        select_best_license(license_string)
-      end
-
-      # npm packages use SPDX "expressions" for their licenses; Thus far we've
-      # only seen a single license, optional multiple licenses like "(MIT OR Apache-2.0)"
-      # or mandatory multiple licenses like "(MIT AND CC-BY-3.0)"
-      #
-      # If there are multiple options, we want to pick just one to keep it simple.
-      def select_best_license(license_string)
-        return nil if license_string.nil?
-
-        options = license_string.tr("(", "").tr(")", "").split(" OR ")
-        options.inject do |selected_license, license|
-          if license_rank(selected_license) < license_rank(license)
-            selected_license
-          else
-            license
-          end
-        end
-      end
-
-      # Rank licenses when selecting one of multiple options. Licenses are
-      # converted to integer scores, the lower the better.
-      #
-      # We prefer Apache-2.0 since it matches our own projects, then MIT, then
-      # BSDs. Everything else is considered equal.
-      def license_rank(license)
-        case license
-        when "Apache-2.0"
-          0
-        when "MIT"
-          1
-        when /bsd/i
-          2
-        else
-          3
-        end
-      end
-
       def root_node_modules_path
-        File.join(project_dir, "node_modules")
+        File.join(directory, "node_modules")
       end
-
     end
   end
 end

data/lib/license_scout/dependency_manager/rebar.rb

@@ -16,12 +16,10 @@
 #
 
 require "license_scout/dependency_manager/base"
-require "license_scout/net_fetcher"
 require "license_scout/exceptions"
-require "license_scout/license_file_analyzer"
 
-require "mixlib/shellout" unless defined?(Mixlib::ShellOut)
-require "ffi_yajl" unless defined?(FFI_Yajl)
+require "mixlib/shellout"
+require "ffi_yajl"
 
 module LicenseScout
   module DependencyManager
@@ -29,8 +27,8 @@ module LicenseScout
 
       attr_reader :packaged_dependencies
 
-      def initialize(project_dir, options)
-        super(project_dir, options)
+      def initialize(directory)
+        super(directory)
 
         @packaged_dependencies = {}
       end
@@ -39,19 +37,29 @@ module LicenseScout
         "erlang_rebar"
       end
 
+      def type
+        "erlang"
+      end
+
+      def signature
+        "rebar.config file"
+      end
+
+      def install_command
+        "rebar get-deps"
+      end
+
       def detected?
         File.exist?(rebar_config_path)
       end
 
       def dependencies
-        dependencies = []
-
         # Some dependencies are obtained via 'pkg' identifier of rebar. These
         # dependencies include their version in the rebar.lock file. Here we
         # parse the rebar.lock and remember all the versions we find.
         parse_packaged_dependencies
 
-        Dir.glob("#{project_deps_dir}/*").each do |dep_dir|
+        Dir.glob("#{project_deps_dir}/*").map do |dep_dir|
           next unless File.directory?(dep_dir)
 
           dep_name = File.basename(dep_dir)
@@ -59,10 +67,10 @@ module LicenseScout
           # First check if this dependency is coming from the parent software.
           # If so we do not need to worry about its version or licenses because
           # it will be covered under the parent software's license.
-          next if File.directory?(File.join(project_dir, "apps", dep_name))
+          next if File.directory?(File.join(directory, "apps", dep_name))
 
           # Or skip if the dep name is the project name
-          next if File.exist?(File.join(project_dir, "_build/default/rel", dep_name))
+          next if File.exist?(File.join(directory, "_build/default/rel", dep_name))
 
           # While determining the dependency version we first check the cache we
           # built from rebar.lock for the dependencies that come via 'pkg'
@@ -74,22 +82,8 @@ module LicenseScout
                           git_rev_parse(dep_dir)
                         end
 
-          override_license_files = options.overrides.license_files_for(name, dep_name, dep_version)
-          license_files =
-            if override_license_files.empty?
-              Dir.glob("#{dep_dir}/*").select { |f| POSSIBLE_LICENSE_FILES.include?(File.basename(f)) }
-            else
-              override_license_files.resolve_locations(dep_dir)
-            end
-
-          license_name = options.overrides.license_for(name, dep_name, dep_version) || scan_licenses(license_files)
-
-          dep = create_dependency(dep_name, dep_version, license_name, license_files)
-
-          dependencies << dep
-        end
-
-        dependencies
+          new_dependency(dep_name, dep_version, dep_dir)
+        end.compact
       end
 
       private
@@ -99,12 +93,12 @@ module LicenseScout
       # determine the version of them via git, we try to parse the rebar.lock
       # file and remember their versions to use it later.
       def parse_packaged_dependencies
-        rebar_lock_path = File.join(project_dir, "rebar.lock")
+        rebar_lock_path = File.join(directory, "rebar.lock")
 
         return unless File.exist?(rebar_lock_path)
 
         rebar_lock_to_json_path = File.expand_path("../../../bin/rebar_lock_json", File.dirname(__FILE__))
-        s = Mixlib::ShellOut.new("#{rebar_lock_to_json_path} #{rebar_lock_path}", environment: options.environment)
+        s = Mixlib::ShellOut.new("#{LicenseScout::Config.escript_bin} #{rebar_lock_to_json_path} #{rebar_lock_path}", environment: LicenseScout::Config.environment)
         s.run_command
         s.error!
 
@@ -115,7 +109,7 @@ module LicenseScout
             source_name = source_info["pkg_name"]
             source_version = source_info["pkg_version"]
 
-            packaged_dependencies[source_name] = source_version
+            @packaged_dependencies[source_name] = source_version
           end
         end
       rescue Mixlib::ShellOut::ShellCommandFailed
@@ -139,27 +133,14 @@ module LicenseScout
       def project_deps_dir
         # rebar dependencies can be found in one of these two directories.
         ["deps", "_build/default/lib"].each do |dir|
-          dep_dir = File.join(project_dir, dir)
+          dep_dir = File.join(directory, dir)
           return dep_dir if File.exist?(dep_dir)
         end
       end
 
       def rebar_config_path
-        File.join(project_dir, "rebar.config")
-      end
-
-      def scan_licenses(license_files)
-        if license_files.empty?
-          nil
-        else
-          license_names = license_files.map do |license_file|
-            found_license = LicenseScout::LicenseFileAnalyzer.find_by_text(IO.read(license_file))
-            found_license ? found_license.short_name : nil
-          end
-          license_names.find { |x| x }
-        end
+        File.join(directory, "rebar.config")
       end
-
     end
   end
 end

data/lib/license_scout/dependency_manager.rb

@@ -15,20 +15,34 @@
 # limitations under the License.
 #
 
+require "license_scout/dependency_manager/base"
+
+require "license_scout/dependency_manager/berkshelf"
 require "license_scout/dependency_manager/bundler"
-require "license_scout/dependency_manager/rebar"
 require "license_scout/dependency_manager/cpanm"
-require "license_scout/dependency_manager/godep"
 require "license_scout/dependency_manager/dep"
 require "license_scout/dependency_manager/glide"
-require "license_scout/dependency_manager/berkshelf"
+require "license_scout/dependency_manager/godep"
+require "license_scout/dependency_manager/habitat"
+require "license_scout/dependency_manager/mix"
+require "license_scout/dependency_manager/rebar"
 require "license_scout/dependency_manager/npm"
-require "license_scout/dependency_manager/manual"
 
 module LicenseScout
   module DependencyManager
     def self.implementations
-      [Bundler, Rebar, Cpanm, Berkshelf, NPM, Godep, Dep, Glide, Manual]
+      [
+        Berkshelf,
+        Bundler,
+        Cpanm,
+        Dep,
+        Glide,
+        Godep,
+        Habitat,
+        Mix,
+        Rebar,
+        Npm,
+      ]
     end
   end
 end

data/lib/license_scout/exceptions.rb

@@ -18,48 +18,7 @@
 module LicenseScout
   module Exceptions
     class Error < RuntimeError; end
-
-    class ProjectDirectoryMissing < Error
-      def initialize(project_dir)
-        @project_dir = project_dir
-      end
-
-      def to_s
-        "Could not locate or access the provided project directory '#{@project_dir}'."
-      end
-    end
-
-    class UnsupportedProjectType < Error
-      def initialize(project_dir)
-        @project_dir = project_dir
-      end
-
-      def to_s
-        "Could not find a supported dependency manager for the project in the provided directory '#{@project_dir}'."
-      end
-    end
-
-    class InaccessibleDependency < Error; end
-    class InvalidOverride < Error; end
-    class InvalidOutputReport < Error; end
-    class InvalidManualDependency < Error; end
-
-    class NetworkError < Error
-
-      attr_reader :from_url
-      attr_reader :network_error
-
-      def initialize(from_url, network_error)
-        @from_url = from_url
-        @network_error = network_error
-      end
-
-      def to_s
-        [
-          "Network error while fetching '#{from_url}'",
-          network_error.to_s,
-        ].join("\n")
-      end
-    end
+    class ConfigError < Error; end
+    class MissingSourceDirectory < Error; end
   end
 end

data/lib/license_scout/license.rb

@@ -0,0 +1,126 @@
+#
+# Copyright:: Copyright 2018, Chef Software Inc.
+# License:: Apache License, Version 2.0
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+require "license_scout/spdx"
+
+module LicenseScout
+  class License
+    # A class that represents the components that make up a license.
+    class Record
+      attr_reader :id
+      attr_reader :parsed_expression
+      attr_reader :source
+      attr_reader :content
+      attr_reader :spdx_license_data
+
+      def initialize(license_id = nil, source = nil, content = nil, options = {})
+        @id = LicenseScout::SPDX.find(license_id, options[:force])
+        @parsed_expression = LicenseScout::SPDX.parse(id)
+        @source = source
+        @content = content
+      end
+
+      def to_h
+        {
+          id: id,
+          source: source,
+          content: content,
+        }
+      end
+    end
+
+    attr_reader :project
+    attr_reader :records
+
+    # @param path [String, nil] A path to give to Licensee to search for the license. Could be local path or GitHub URL.
+    def initialize(path = nil)
+      if path.nil?
+        @project = nil
+        @records = []
+      else
+        @project = Licensee.project(path, detect_readme: true)
+        @records = []
+
+        project.licenses.each_index do |i|
+          record = Record.new(
+            project.licenses[i].spdx_id,
+            project.matched_files[i].filename,
+            project.matched_files[i].content
+          )
+
+          # Favor records that have identified a license
+          record.id.nil? ? @records.push(record) : @records.unshift(record)
+        end
+      end
+    end
+
+    # Capture a license that was specified in metadata
+    #
+    # @param license_id [String] The license as specified in the metadata file
+    # @param source [String] Where we found the license info
+    # @param contents_url [String] Where we can find the contents of the license
+    # @param options [Hash] Options to control various behavior
+    #
+    # @return [void]
+    def add_license(license_id, source, contents_url, options)
+      content = license_content(license_id, contents_url)
+      @records.push(Record.new(license_id, source, content, options))
+    end
+
+    # @return [Boolean] Whether or not the license(s) are allowed
+    def is_allowed?
+      (records.map(&:parsed_expression).flatten.compact & LicenseScout::Config.allowed_licenses).any?
+    end
+
+    # @return [Boolean] Whether or not the license(s) are flagged
+    def is_flagged?
+      (records.map(&:parsed_expression).flatten.compact & LicenseScout::Config.flagged_licenses).any?
+    end
+
+    # @return [Boolean] Whether we were unable to determine a license
+    def undetermined?
+      (records.map(&:parsed_expression).flatten.compact).empty?
+    end
+
+    private
+
+    def license_content(license_id, contents_url)
+      if contents_url.nil?
+        nil
+      else
+        new_url = raw_github_url(contents_url)
+
+        begin
+          LicenseScout::Log.debug("[license] Pulling license content for #{license_id} from #{new_url}")
+          open(new_url).read
+        rescue OpenURI::HTTPError
+          LicenseScout::Log.warn("[license] Unable to download license for #{license_id} from #{new_url}")
+          nil
+        end
+      end
+    end
+
+    def raw_github_url(url)
+      case url
+      when /github.com\/(.+)\/blob\/(.+)/
+        "https://raw.githubusercontent.com/#{$1}/#{$2}"
+      else
+        url
+      end
+    end
+  end
+end

data/lib/license_scout/{license_file_analyzer.rb → log.rb}

@@ -1,5 +1,5 @@
 #
-# Copyright:: Copyright 2016, Chef Software Inc.
+# Copyright:: Copyright 2018, Chef Software Inc.
 # License:: Apache License, Version 2.0
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
@@ -15,12 +15,10 @@
 # limitations under the License.
 #
 
-require "license_scout/license_file_analyzer/definitions"
+require "mixlib/log"
 
 module LicenseScout
-  module LicenseFileAnalyzer
-    def self.find_by_text(text)
-      Definitions.all.detect { |l| l.matches_text? text }
-    end
+  class Log
+    extend Mixlib::Log
   end
 end