license_scout 1.3.2 → 2.0.2

data/lib/license_scout/dependency_manager/cpanm.rb

@@ -15,146 +15,96 @@
 # limitations under the License.
 #
 
-require "ffi_yajl" unless defined?(FFI_Yajl)
-require "psych"
-require "mixlib/shellout" unless defined?(Mixlib::ShellOut)
-
 require "license_scout/dependency_manager/base"
-require "license_scout/exceptions"
-require "license_scout/dependency"
+
+require "ffi_yajl"
+require "psych"
+require "mixlib/shellout"
 
 module LicenseScout
   module DependencyManager
     class Cpanm < Base
 
-      class CpanmDependency
-
-        LICENSE_TYPE_MAP = {
-          "perl_5" => "Perl-5",
-          "perl" => "Perl-5",
-          "apache_2_0" => "Apache-2.0",
-          "artistic_2" => "Artistic-2.0",
-          "gpl_3" => "GPL-3.0",
-        }.freeze
-
-        attr_reader :unpack_path
-        attr_reader :overrides
-        attr_reader :metadata
-
-        def initialize(unpack_path, overrides)
-          @unpack_path = unpack_path
-          @overrides = overrides
-        end
-
-        def to_dep
-          parse_metadata!
+      def name
+        "perl_cpanm"
+      end
 
-          Dependency.new(
-            name,
-            version.to_s,
-            license,
-            license_files,
-            "perl_cpanm"
-          )
-        end
+      def type
+        "perl"
+      end
 
-        def parse_metadata!
-          # Packages can contain metadata files named META.yml, META.json,
-          # MYMETA.json, MYMETA.yml. META.* files are created by the authors of
-          # the plugins whereas MYMETA.* files are created by the build system
-          # after dynamic dependencies are resolved. For our purposes META.*
-          # files are enough. And for no good reason we prioritize json files
-          # over yml files.
-          @metadata ||= begin
-            json_path = File.join(unpack_path, "META.json")
-            yml_path = File.join(unpack_path, "META.yml")
-
-            if File.exist?(json_path)
-              FFI_Yajl::Parser.parse(File.read(json_path))
-            elsif File.exist?(yml_path)
-              Psych.safe_load(File.read(yml_path))
-            else
-              raise LicenseScout::Exceptions::Error.new("Can not find a metadata file for the perl package at '#{unpack_path}'.")
-            end
-          end
-        end
+      def signature
+        File.exist?(meta_json_path) ? "META.json file" : "META.yml file"
+      end
 
-        def name
-          metadata["name"]
-        end
+      def install_command
+        "cpanm --installdeps ."
+      end
 
-        def version
-          metadata["version"]
-        end
+      # NOTE: it's possible that projects won't have a META.yml, but the two
+      # that we care about for Chef Server do have one. As of 2015, 84% of perl
+      # distribution packages have one: http://neilb.org/2015/10/18/spotters-guide.html
+      def detected?
+        File.exist?(meta_json_path) || File.exist?(meta_yml_path)
+      end
 
-        def license
-          @license ||= begin
-            override_license = overrides.license_for("perl_cpanm", name, version)
+      def dependencies
+        Dir.glob("#{cpanm_root}/latest-build/*").map do |dep_path|
+          next unless File.directory?(dep_path)
 
-            if override_license
-              override_license
-            elsif metadata && metadata.key?("license")
-              given_type = Array(metadata["license"]).reject { |l| l == "unknown" }.first
+          dep_data = manifest(dep_path)
+          metafile = dep_data["metafile"]
+          dep_name = dep_data["name"]
+          dep_version = dep_data["version"]
 
-              # Normalize the common perl license strings to the strings we commonly use
-              LICENSE_TYPE_MAP[given_type] || given_type
-            end
-          end
-        end
+          dependency = new_dependency(dep_name, dep_version, dep_path)
 
-        def license_files
-          @license_files ||= begin
-            override_license_files = overrides.license_files_for("perl_cpanm", name, version)
+          # CPANM projects contain license metadata - include it!
+          unless dep_data["license"].nil?
+            Array(dep_data["license"]).each do |license|
+              next if license == "unknown"
 
-            if override_license_files.empty?
-              find_license_files
-            else
-              override_license_files.resolve_locations(unpack_path)
+              dependency.add_license(license, metafile)
             end
           end
-        end
-
-        def find_license_files
-          Dir["#{unpack_path}/*"].select do |f|
-            Cpanm::POSSIBLE_LICENSE_FILES.include?(File.basename(f))
-          end
-        end
 
+          dependency
+        end.compact
       end
 
-      def name
-        "perl_cpanm"
-      end
+      private
 
-      def cpanm_root
-        # By default cpanm downloads all the dependencies into ~/.cpanm directory
-        File.expand_path("~/.cpanm")
+      def meta_yml_path
+        File.join(directory, "META.yml")
       end
 
-      def dependencies
-        @dependencies ||= begin
-          deps = []
-
-          Dir.glob("#{cpanm_root}/latest-build/*").each do |dep_path|
-            next unless File.directory?(dep_path)
-
-            deps << CpanmDependency.new(dep_path, options.overrides).to_dep
-          end
+      def meta_json_path
+        File.join(directory, "META.json")
+      end
 
-          deps
+      # Packages can contain metadata files named META.yml, META.json,
+      # MYMETA.json, MYMETA.yml. META.* files are created by the authors of
+      # the plugins whereas MYMETA.* files are created by the build system
+      # after dynamic dependencies are resolved. For our purposes META.*
+      # files are enough. And for no good reason we prioritize json files
+      # over yml files.
+      def manifest(unpack_path)
+        json_path = File.join(unpack_path, "META.json")
+        yml_path = File.join(unpack_path, "META.yml")
+
+        if File.exist?(json_path)
+          FFI_Yajl::Parser.parse(File.read(json_path)).merge({ "metafile" => "META.json" })
+        elsif File.exist?(yml_path)
+          Psych.safe_load(File.read(yml_path)).merge({ "metafile" => "META.yml" })
+        else
+          raise LicenseScout::Exceptions::Error.new("Can not find a metadata file for the perl package at '#{unpack_path}'.")
         end
       end
 
-      # NOTE: it's possible that projects won't have a META.yml, but the two
-      # that we care about for Chef Server do have one. As of 2015, 84% of perl
-      # distribution packages have one: http://neilb.org/2015/10/18/spotters-guide.html
-      def detected?
-        meta_yml_path = File.join(project_dir, "META.yml")
-        meta_json_path = File.join(project_dir, "META.json")
-
-        File.exist?(meta_yml_path) || File.exist?(meta_json_path)
+      def cpanm_root
+        # By default cpanm downloads all the dependencies into ~/.cpanm directory
+        File.expand_path(LicenseScout::Config.cpanm_root)
       end
-
     end
   end
 end

data/lib/license_scout/dependency_manager/dep.rb

@@ -15,8 +15,8 @@
 # limitations under the License.
 #
 
-require "ffi_yajl" unless defined?(FFI_Yajl)
-require "yaml" unless defined?(YAML)
+require "ffi_yajl"
+require "yaml"
 require "toml-rb"
 require "license_scout/dependency_manager/base"
 
@@ -26,62 +26,55 @@ module LicenseScout
     class Dep < Base
 
       def name
-        "go_dep"
+        "golang_dep"
       end
 
-      def detected?
-        File.exist?(root_dep_file)
+      def type
+        "golang"
       end
 
-      def dependencies
-        deps = File.open(root_dep_file) do |f|
-          TomlRB.parse(f)
-        end
-        return [] unless deps.key?("projects")
+      def signature
+        "Gopkg.lock file"
+      end
 
-        deps["projects"].map do |pkg_info|
-          pkg_import_name = pkg_info["name"]
-          pkg_file_name = pkg_import_name.tr("/", "_")
-          pkg_version = pkg_info["version"] || pkg_info["revision"]
-          license = options.overrides.license_for("go", pkg_import_name, pkg_version)
+      def install_command
+        "dep ensure"
+      end
 
-          override_license_files = options.overrides.license_files_for("go", pkg_import_name, pkg_version)
-          if override_license_files.empty?
-            license_files = find_license_files_for_package_in_gopath_or_vendor_dir(pkg_import_name)
-          else
-            license_files = override_license_files.resolve_locations(gopath(pkg_import_name))
-          end
+      def detected?
+        File.exist?(gopkg_lock_path)
+      end
 
-          if license.nil? && !license_files.empty?
-            license = scan_licenses(license_files)
-          end
+      def dependencies
+        Array(gopkg.dig("projects")).map do |pkg_info|
+          dep_name = pkg_info["name"]
+          dep_version = pkg_info["version"] || pkg_info["revision"]
+          dep_path = package_path(dep_name)
 
-          create_dependency(pkg_file_name, pkg_version, license, license_files)
-        end
+          new_dependency(dep_name, dep_version, dep_path)
+        end.compact
       end
 
       private
 
-      def scan_licenses(license_files)
-        found_license = LicenseScout::LicenseFileAnalyzer.find_by_text(IO.read(license_files.first))
-        found_license && found_license.short_name
+      def gopkg
+        File.open(gopkg_lock_path) { |f| TomlRB.parse(f) }
       end
 
-      def root_dep_file
-        File.join(project_dir, "Gopkg.lock")
+      def gopkg_lock_path
+        File.join(directory, "Gopkg.lock")
       end
 
       def gopath(pkg)
-        "#{ENV["GOPATH"]}/src/#{pkg}"
+        "#{ENV['GOPATH']}/src/#{pkg}"
       end
 
       def vendor_dir(pkg = nil)
-        File.join(project_dir, "vendor/#{pkg}")
+        File.join(directory, "vendor/#{pkg}")
       end
 
-      def find_license_files_for_package_in_gopath_or_vendor_dir(pkg)
-        root_files = Dir["#{gopath(pkg)}/*"] + Dir["#{vendor_dir(pkg)}/*"]
-        root_files.select { |f| POSSIBLE_LICENSE_FILES.include?(File.basename(f)) }
+      def package_path(pkg)
+        (Dir[vendor_dir(pkg)] + Dir[gopath(pkg)]).first
       end
     end
   end

data/lib/license_scout/dependency_manager/glide.rb

@@ -15,7 +15,7 @@
 # limitations under the License.
 #
 
-require "yaml" unless defined?(YAML)
+require "psych"
 require "license_scout/dependency_manager/base"
 
 module LicenseScout
@@ -23,56 +23,45 @@ module LicenseScout
     class Glide < Base
 
       def name
-        "go_glide"
+        "golang_glide"
       end
 
-      def detected?
-        File.exist?(glide_yaml)
+      def type
+        "golang"
       end
 
-      def dependencies
-        unless File.file?(glide_yaml_locked)
-          raise "Detected Go/Glide project that is missing its \"glide.lock\" "\
-                "file in #{project_dir}"
-        end
-
-        deps = YAML.safe_load(File.read(glide_yaml_locked), permitted_classes: [Date, Symbol, Time])
-        deps["imports"].map { |i| add_glide_dep(i) }
+      def signature
+        "glide.lock file"
       end
 
-      private
+      def install_command
+        "glide install"
+      end
 
-      def add_glide_dep(import_field)
-        pkg_import_name = import_field["name"]
-        pkg_file_name = pkg_import_name.tr("/", "_")
-        pkg_version = import_field["version"]
-        license = options.overrides.license_for("go", pkg_import_name, pkg_version)
+      def detected?
+        File.exist?(glide_lock_path)
+      end
 
-        override_license_files = options.overrides.license_files_for("go", pkg_import_name, pkg_version)
-        if override_license_files.empty?
-          license_files = find_license_files_for_package_in_gopath(pkg_import_name)
-        else
-          license_files = override_license_files.resolve_locations(gopath(pkg_import_name))
-        end
+      def dependencies
+        # We cannot use YAML.safe_load because Psych throws a fit about the
+        # updated field. We should circle back and see what we can do to fix that.
+        YAML.load(File.read(glide_lock_path))["imports"].map do |import|
+          dep_name = import["name"]
+          dep_version = import["version"]
+          dep_path = gopath(dep_name)
 
-        create_dependency(pkg_file_name, pkg_version, license, license_files)
+          new_dependency(dep_name, dep_version, dep_path)
+        end.compact
       end
 
-      def glide_yaml
-        File.join(project_dir, "glide.yaml")
-      end
+      private
 
-      def glide_yaml_locked
-        File.join(project_dir, "glide.lock")
+      def glide_lock_path
+        File.join(directory, "glide.lock")
       end
 
       def gopath(pkg)
-        "#{ENV["GOPATH"]}/src/#{pkg}"
-      end
-
-      def find_license_files_for_package_in_gopath(pkg)
-        root_files = Dir["#{gopath(pkg)}/*"]
-        root_files.select { |f| POSSIBLE_LICENSE_FILES.include?(File.basename(f)) }
+        "#{ENV['GOPATH']}/src/#{pkg}"
       end
     end
   end

data/lib/license_scout/dependency_manager/godep.rb

@@ -15,7 +15,7 @@
 # limitations under the License.
 #
 
-require "ffi_yajl" unless defined?(FFI_Yajl)
+require "ffi_yajl"
 require "license_scout/dependency_manager/base"
 
 module LicenseScout
@@ -23,7 +23,19 @@ module LicenseScout
     class Godep < Base
 
       def name
-        "go_godep"
+        "golang_godep"
+      end
+
+      def type
+        "golang"
+      end
+
+      def signature
+        "Godeps/Godeps.json file"
+      end
+
+      def install_command
+        "godep restore"
       end
 
       def detected?
@@ -31,40 +43,29 @@ module LicenseScout
       end
 
       def dependencies
-        godeps = File.open(root_godeps_file) do |f|
-          FFI_Yajl::Parser.parse(f)
-        end
-
         godeps["Deps"].map do |pkg_info|
-          pkg_import_name = pkg_info["ImportPath"]
-          pkg_file_name = pkg_import_name.tr("/", "_")
-          pkg_version = pkg_info["Comment"] || pkg_info["Rev"]
-          license = options.overrides.license_for("go", pkg_import_name, pkg_version)
+          dep_name = pkg_info["ImportPath"]
+          dep_version = pkg_info["Comment"] || pkg_info["Rev"]
+          dep_path = gopath(dep_name)
 
-          override_license_files = options.overrides.license_files_for("go", pkg_import_name, pkg_version)
-          if override_license_files.empty?
-            license_files = find_license_files_for_package_in_gopath(pkg_import_name)
-          else
-            license_files = override_license_files.resolve_locations(gopath(pkg_import_name))
-          end
-
-          create_dependency(pkg_file_name, pkg_version, license, license_files)
-        end
+          new_dependency(dep_name, dep_version, dep_path)
+        end.compact
       end
 
       private
 
-      def root_godeps_file
-        File.join(project_dir, "Godeps/Godeps.json")
+      def godeps
+        File.open(root_godeps_file) do |f|
+          FFI_Yajl::Parser.parse(f)
+        end
       end
 
-      def gopath(pkg)
-        "#{ENV["GOPATH"]}/src/#{pkg}"
+      def root_godeps_file
+        File.join(directory, "Godeps/Godeps.json")
       end
 
-      def find_license_files_for_package_in_gopath(pkg)
-        root_files = Dir["#{gopath(pkg)}/*"]
-        root_files.select { |f| POSSIBLE_LICENSE_FILES.include?(File.basename(f)) }
+      def gopath(pkg)
+        "#{ENV['GOPATH']}/src/#{pkg}"
       end
     end
   end

data/lib/license_scout/dependency_manager/habitat.rb

@@ -0,0 +1,126 @@
+#
+# Copyright:: Copyright 2018, Chef Software Inc.
+# License:: Apache License, Version 2.0
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+require "license_scout/dependency_manager/base"
+
+require "open-uri"
+require "mixlib/shellout"
+
+module LicenseScout
+  module DependencyManager
+    class Habitat < Base
+
+      def name
+        "habitat"
+      end
+
+      def type
+        "habitat"
+      end
+
+      def signature
+        File.exist?(habitat_plan_sh_path) ? "habitat/plan.sh file" : "plan.sh file"
+      end
+
+      def install_command
+        ""
+      end
+
+      def detected?
+        File.exist?(plan_sh_path) || File.exist?(habitat_plan_sh_path)
+      end
+
+      def dependencies
+        tdeps = Set.new(pkg_deps)
+
+        pkg_deps.each do |pkg_dep|
+          pkg_info(pkg_dep)["tdeps"].each { |dep| tdeps << to_ident(dep) }
+        end
+
+        tdeps.sort.map do |tdep|
+          o, n, v, r = tdep.split("/")
+          dep_name = "#{o}/#{n}"
+          dep_version = "#{v}-#{r}"
+
+          dependency = new_dependency(dep_name, dep_version, nil)
+
+          license_from_manifest(pkg_info(tdep)["manifest"]).each do |spdx|
+            dependency.add_license(spdx, "https://bldr.habitat.sh/v1/depot/channels/#{o}/stable/pkgs/#{n}/#{v}/#{r}")
+          end
+
+          dependency
+        end.compact
+      end
+
+      private
+
+      def license_from_manifest(manifest_content)
+        /^*\s+__License__:\s+(.+)$/.match(manifest_content)[1].strip.split("\s")
+      end
+
+      def pkg_deps
+        @pkg_deps ||= begin
+          plan_path = File.exist?(plan_sh_path) ? plan_sh_path : habitat_plan_sh_path
+
+          c = Mixlib::ShellOut.new("bash -ec 'export PLAN_CONTEXT=\"#{File.dirname(plan_path)}\"; source #{plan_path}; echo ${pkg_deps[*]}'", LicenseScout::Config.environment)
+          c.run_command
+          c.error!
+          pkg_deps = c.stdout.split("\s")
+
+          # Fetch the fully-qualified pkg_ident for each pkg
+          pkg_deps.map { |dep| to_ident(pkg_info(dep)["ident"]) }
+        end
+      end
+
+      def to_ident(ident_hash)
+        "#{ident_hash["origin"]}/#{ident_hash["name"]}/#{ident_hash["version"]}/#{ident_hash["release"]}"
+      end
+
+      def pkg_info(pkg_ident)
+        $habitat_pkg_info ||= {}
+        $habitat_pkg_info[pkg_ident] ||= begin
+          pkg_origin, pkg_name, pkg_version, pkg_release = pkg_ident.split("/")
+
+          base_api_uri = "https://bldr.habitat.sh/v1/depot/channels/#{pkg_origin}/stable/pkgs/#{pkg_name}"
+          if pkg_version.nil? && pkg_release.nil?
+            base_api_uri += "/latest"
+          elsif pkg_release.nil?
+            base_api_uri += "/#{pkg_version}/latest"
+          else
+            base_api_uri += "/#{pkg_version}/#{pkg_release}"
+          end
+
+          LicenseScout::Log.debug("[habitat] Fetching pkg_info from #{base_api_uri}")
+          FFI_Yajl::Parser.parse(open(base_api_uri).read)
+        rescue OpenURI::HTTPError
+          pkg_origin, pkg_name, = pkg_ident.split("/")
+
+          LicenseScout::Log.warn("[habitat] Could not find pkg_info for #{pkg_ident} - trying for the latest version of #{pkg_origin}/#{pkg_name}")
+          FFI_Yajl::Parser.parse(open("https://bldr.habitat.sh/v1/depot/channels/#{pkg_origin}/stable/pkgs/#{pkg_name}/latest").read)
+        end
+      end
+
+      def plan_sh_path
+        File.join(directory, "plan.sh")
+      end
+
+      def habitat_plan_sh_path
+        File.join(directory, "habitat", "plan.sh")
+      end
+    end
+  end
+end