license_scout 0.1.0 → 0.1.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: d48be3c19574905aef576b453b5750796b64e7ed
-  data.tar.gz: a9424c524ca4e52c478cf45610b229f9f4909268
+  metadata.gz: 20de647a9029e60eee95ccfdd6af26ca23f68943
+  data.tar.gz: 4d39bfea2c9974e8af4cd400eccfca1bbec4f4bd
 SHA512:
-  metadata.gz: b4c68b97c0370efab31ba8a43090ca05762a03275af7f00cd9af0dc4f384a352c701c4c045a4de537af132bb0e7cb1ebe7b5b83122afd3ad6b0d1326fbe6e69d
-  data.tar.gz: 083fa2b33c02f30de9a39976aa2b9f87f366eb90d3a2f90901c71446eb7c0e25892547555ef04a95bc1ed337517f81a58505e17916484aefee6cb58484695ad8
+  metadata.gz: f554b77a3215ce07597d48a64daa48844dc23463372ca5ec5bfda0726418836c25b7b53eb4e123942b5ffdf858a503e14e59d7aa196b6fa27fce0513ed985e8d
+  data.tar.gz: 990fd0c28d0f003e72f0cc1840198b0453c8a9f8c52c45f3e7aff9bae145e645b2a46a83901159a97de45d9e8f056bef92fe41ae3da42014350cdeffbdd6f30f

data/README.md

@@ -7,9 +7,15 @@ Currently supported project types are:
 
 * Ruby - bundler
 * Erlang - rebar
+* CPAN - perl
+* Berkshelf - chef
 
 ## Usage
 
+## Thanks
+
+Thanks to https://github.com/basho for `config_to_json` binary which helps with parsing Erlang config files. From: https://github.com/basho/erlang_template_helper
+
 ## Contributing
 
 This project is maintained by the contribution guidelines identified for

data/bin/license_scout

@@ -22,10 +22,12 @@ require "license_scout/collector"
 require "license_scout/overrides"
 require "license_scout/options"
 
-project_dir = File.expand_path(Dir.pwd)
+project_dir = ARGV[0] || File.expand_path(Dir.pwd)
 project_name = File.basename(project_dir)
 
-output_dir = project_dir
+# Create the output files under a specific directory in order not to pollute the
+# project_dir too much.
+output_dir = File.join(project_dir, "license-cache")
 
 overrides = LicenseScout::Overrides.new
 

data/lib/license_scout/collector.rb

@@ -17,6 +17,7 @@
 
 require "license_scout/exceptions"
 require "license_scout/dependency_manager"
+require "license_scout/reporter"
 
 require "ffi_yajl"
 
@@ -59,30 +60,7 @@ module LicenseScout
     end
 
     def issue_report
-      report = []
-      license_report = FFI_Yajl::Parser.parse(File.read(license_manifest_path))
-
-      license_report["dependency_managers"].each do |dependency_manager, dependencies|
-        dependencies.each do |dependency|
-          if dependency["name"].nil? || dependency["name"].empty?
-            report << "There is a dependency with a missing name in '#{dependency_manager}'."
-          end
-
-          if dependency["version"].nil? || dependency["version"].empty?
-            report << "Dependency '#{dependency["name"]}' under '#{dependency_manager}' is missing version information."
-          end
-
-          if dependency["license"].nil? || dependency["license"].empty?
-            report << "Dependency '#{dependency["name"]}' version '#{dependency["version"]}' under '#{dependency_manager}' is missing license information."
-          end
-
-          if dependency["license_files"].empty?
-            report << "Dependency '#{dependency["name"]}' version '#{dependency["version"]}' under '#{dependency_manager}' is missing license files information."
-          end
-        end
-      end
-
-      report
+      Reporter.new(output_dir).report
     end
 
     private

data/lib/license_scout/dependency_manager/berkshelf.rb

@@ -0,0 +1,102 @@
+#
+# Copyright:: Copyright 2016, Chef Software Inc.
+# License:: Apache License, Version 2.0
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+require "berkshelf"
+
+require "license_scout/dependency_manager/base"
+
+module LicenseScout
+  module DependencyManager
+    class Berkshelf < Base
+
+      def name
+        "chef_berkshelf"
+      end
+
+      def detected?
+        File.exists?(berksfile_path) && File.exists?(lockfile_path)
+      end
+
+      def dependencies
+        dependencies = []
+        cookbook_dependencies = nil
+
+        Dir.chdir(project_dir) do
+          berksfile = ::Berkshelf::Berksfile.from_file("./Berksfile")
+
+          # Berkshelf should not give an error when there are cookbooks in the
+          # lockfile that are no longer in the berksfile. It handles this case in
+          # the Installer class which we are not using here. So we handle this
+          # case in the same way Installer does.
+          berksfile.lockfile.reduce!
+
+          cookbook_dependencies = berksfile.list
+        end
+
+        cookbook_dependencies.each do |dep|
+          dependency_name = dep.name
+          dependency_version = dep.cached_cookbook.version
+
+          dependency_license_files = auto_detect_license_files(dep.cached_cookbook.path.to_s)
+
+          # Check license override and license_files override separately since
+          # only one might be set in the overrides.
+          dependency_license = options.overrides.license_for(name, dependency_name, dependency_version) || dep.cached_cookbook.license
+
+          override_license_files = options.overrides.license_files_for(name, dependency_name, dependency_version)
+          cookbook_path = dep.cached_cookbook.path.to_s
+
+          if override_license_files.empty?
+            dependency_license_files = auto_detect_license_files(cookbook_path)
+          else
+            dependency_license_files = override_license_files.resolve_locations(cookbook_path)
+          end
+
+          dependencies << Dependency.new(
+            dependency_name,
+            dependency_version,
+            dependency_license,
+            dependency_license_files
+          )
+        end
+
+        dependencies
+      end
+
+      private
+
+      def berksfile_path
+        File.join(project_dir, "Berksfile")
+      end
+
+      def lockfile_path
+        File.join(project_dir, "Berksfile.lock")
+      end
+
+      def auto_detect_license_files(cookbook_path)
+        unless File.exist?(cookbook_path)
+          raise LicenseScout::Exceptions::InaccessibleDependency.new "Autodetected cookbook path '#{cookbook_path}' does not exist"
+        end
+
+        Dir.glob("#{cookbook_path}/*").select do |f|
+          POSSIBLE_LICENSE_FILES.include?(File.basename(f))
+        end
+      end
+
+    end
+  end
+end

data/lib/license_scout/dependency_manager/bundler/_bundler_script.rb

@@ -32,13 +32,12 @@ require "bundler/setup"
 # We're only using things that are in the stdlib.
 require "json"
 
-definition = ::Bundler::Definition.build("./Gemfile", "./Gemfile.lock", nil)
 dependencies = []
 
-definition.specs_for(definition.groups).each do |gem_spec|
+Bundler.load.specs.each do |gem_spec|
   dependencies << {
     name: gem_spec.name,
-    version: gem_spec.version,
+    version: gem_spec.version.to_s,
     license: gem_spec.license,
     path: gem_spec.full_gem_path,
   }

data/lib/license_scout/dependency_manager/bundler.rb

@@ -33,11 +33,13 @@ module LicenseScout
       end
 
       def detected?
-        # We only check for the existence of Gemfile in order to declare a
-        # project a Bundler project. If the Gemfile.lock does not exist
-        # we will raise a specific error to indicate that "bundle install"
-        # needs to be run before proceeding.
-        File.exists?(gemfile_path)
+        # We check the existence of both Gemfile and Gemfile.lock. We need both
+        # of them to be able to get a concrete set of dependencies which we can
+        # search. We used to raise an error when Gemfile.lock did not exist but
+        # that created issues with projects like oc_bifrost which is a rebar
+        # project but have a Gemfile at its root to be able to run some rake
+        # commands.
+        File.exists?(gemfile_path) && File.exists?(lockfile_path)
       end
 
       def dependency_data
@@ -56,10 +58,6 @@ module LicenseScout
       end
 
       def dependencies
-        if !File.exists?(lockfile_path)
-          raise LicenseScout::Exceptions::DependencyManagerNotRun.new(project_dir, name)
-        end
-
         dependencies = []
         dependency_data.each do |gem_data|
           dependency_name = gem_data["name"]
@@ -74,6 +72,14 @@ module LicenseScout
             # bundler's lib/ dir, so we have to munge it.
             dependency_license = "MIT"
             dependency_license_files = [File.join(File.dirname(__FILE__), "bundler/LICENSE.md")]
+          elsif dependency_name == "json"
+            # json is different weird. When project is using the json that is prepackaged with
+            # Ruby, its included not as a full fledged gem but an *.rb file at:
+            # /opt/opscode/embedded/lib/ruby/2.2.0/json.rb
+            # Because of this its license is reported as nil and its license files can not be
+            # found. That is why we need to provide them manually here.
+            dependency_license = "Ruby"
+            dependency_license_files = [File.join(File.dirname(__FILE__), "json/README.md")]
           else
             # Check license override and license_files override separately since
             # only one might be set in the overrides.

data/lib/license_scout/dependency_manager/cpan.rb

@@ -0,0 +1,321 @@
+#
+# Copyright:: Copyright 2016, Chef Software Inc.
+# License:: Apache License, Version 2.0
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+require "rexml/document"
+
+require "ffi_yajl"
+require "psych"
+require "mixlib/shellout"
+
+require "license_scout/dependency_manager/base"
+require "license_scout/net_fetcher"
+require "license_scout/exceptions"
+require "license_scout/dependency"
+
+module LicenseScout
+  module DependencyManager
+    class CPAN < Base
+
+      class CPANDependency
+
+        LICENSE_TYPE_MAP = {
+          "perl_5"      => "Perl-5",
+          "perl"        => "Perl-5",
+          "apache_2_0"  => "Apache-2.0",
+          "artistic_2"  => "Artistic-2.0",
+          "gpl_3"       => "GPL-3.0",
+        }.freeze
+
+        attr_reader :module_name
+        attr_reader :dist
+        attr_reader :version
+        attr_reader :cpanfile
+
+        attr_reader :license_files
+        attr_reader :license
+
+        attr_reader :cache_root
+
+        attr_reader :overrides
+
+        def initialize(module_name:, dist:, version:, cpanfile:, cache_root:, overrides:)
+          @module_name = module_name
+          @dist = dist
+          @version = version
+          @cpanfile = cpanfile
+          @cache_root = cache_root
+          @overrides = overrides
+
+          @deps_list = nil
+
+          @license = nil
+          @license_files = []
+        end
+
+        def desc
+          "#{module_name} in #{dist} (#{version}) [#{license}]"
+        end
+
+        def to_dep
+          Dependency.new(
+            # we use dist for the name because there can be multiple modules in
+            # a dist, but the dist is the unit of packaging and licensing
+            dist,
+            version,
+            license,
+            license_files
+          )
+        end
+
+        def collect_licenses
+          ensure_cached
+          Dir.mktmpdir do |tmpdir|
+            FileUtils.cp(distribution_fullpath, tmpdir)
+            Dir.chdir(tmpdir) do
+              untar!
+              distribution_unpack_fullpath = File.join(tmpdir, distribution_unpack_relpath)
+              collect_licenses_in(distribution_unpack_fullpath)
+            end
+          end
+        end
+
+        def ensure_cached
+          cache_path = File.join(dist_cache_root, cpanfile)
+
+          # CPAN download URL is like:
+          # http://www.cpan.org/authors/id/R/RJ/RJBS/Sub-Install-0.928.tar.gz
+          # cpanfile is like:
+          # R/RJ/RJBS/Sub-Install-0.928.tar.gz
+          unless File.exist?(cache_path)
+
+            url = "http://www.cpan.org/authors/id/#{cpanfile}"
+            tmp_path = NetFetcher.cache(url)
+
+            FileUtils.mkdir_p(File.dirname(cache_path))
+            FileUtils.cp(tmp_path, cache_path)
+
+          end
+        end
+
+        def distribution_filename
+          File.basename(cpanfile)
+        end
+
+        def distribution_unpack_relpath
+          # Most packages have tar.gz extension but some have .tgz like
+          # IO-Pager-0.36.tgz
+          [".tar.gz", ".tgz"].each do |ext|
+            if distribution_filename.end_with?(ext)
+              return File.basename(distribution_filename, ext)
+            end
+          end
+        end
+
+        def distribution_fullpath
+          File.join(dist_cache_root, cpanfile)
+        end
+
+        # Untar the distribution.
+        #
+        # NOTE: On some platforms, you only get a usable version of tar as
+        # `gtar`, and on windows, symlinks break a lot of stuff. We (Chef
+        # Software) currently only use perl in server products, which we only
+        # build for a handful of Linux distros, so this is sufficient.
+        def untar!
+          s = Mixlib::ShellOut.new("tar zxf #{distribution_filename}")
+          s.run_command
+          s.error!
+          s.stdout
+        end
+
+        def collect_licenses_in(unpack_path)
+          collect_license_info_in(unpack_path)
+          collect_license_files_info_in(unpack_path)
+        end
+
+        def collect_license_info_in(unpack_path)
+          # Notice that we use "dist" as the dependency name
+          # See #to_dep for details.
+          @license = overrides.license_for("perl_cpan", dist, version) || begin
+            metadata = if File.exist?(meta_json_in(unpack_path))
+                         slurp_meta_json_in(unpack_path)
+                       elsif File.exist?(meta_yaml_in(unpack_path))
+                         slurp_meta_yaml_in(unpack_path)
+                       end
+
+            if metadata && metadata.key?("license")
+              given_type = Array(metadata["license"]).reject { |l| l == "unknown" }.first
+              normalize_license_type(given_type)
+            end
+          end
+        end
+
+        def collect_license_files_info_in(unpack_path)
+          override_license_files = overrides.license_files_for("perl_cpan", dist, version)
+
+          license_files = if override_license_files.empty?
+                            find_license_files_in(unpack_path)
+                          else
+                            override_license_files.resolve_locations(unpack_path)
+                          end
+
+          license_files.each do |f|
+            @license_files << cache_license_file(f)
+          end
+        end
+
+        # Copy license file to the cache. We unpack the CPAN dists in a tempdir
+        # and throw it away after we've inspected the contents, so we need to
+        # put the license file somewhere it can be copied from later.
+        def cache_license_file(unpacked_file)
+          basename = File.basename(unpacked_file)
+          license_cache_path = File.join(license_cache_root, "#{dist}-#{basename}")
+          FileUtils.mkdir_p(license_cache_root)
+          FileUtils.cp(unpacked_file, license_cache_path)
+          # In some cases, the license files get unpacked with 0444
+          # permissions which could make a re-run fail on the `cp` step.
+          FileUtils.chmod(0644, license_cache_path)
+          license_cache_path
+        end
+
+        def slurp_meta_yaml_in(unpack_path)
+          Psych.safe_load(File.read(meta_yaml_in(unpack_path)))
+        end
+
+        def slurp_meta_json_in(unpack_path)
+          FFI_Yajl::Parser.parse(File.read(meta_json_in(unpack_path)))
+        end
+
+        def license_cache_root
+          File.join(cache_root, "cpan-licenses")
+        end
+
+        def dist_cache_root
+          File.join(cache_root, "cpan-dists")
+        end
+
+        def normalize_license_type(given_type)
+          LICENSE_TYPE_MAP[given_type] || given_type
+        end
+
+        def meta_json_in(unpack_path)
+          File.join(unpack_path, "META.json")
+        end
+
+        def mymeta_json_in(unpack_path)
+          File.join(unpack_path, "MYMETA.json")
+        end
+
+        def meta_yaml_in(unpack_path)
+          File.join(unpack_path, "META.yml")
+        end
+
+        def find_license_files_in(unpack_path)
+          Dir["#{unpack_path}/*"].select do |f|
+            CPAN::POSSIBLE_LICENSE_FILES.include?(File.basename(f))
+          end
+        end
+
+      end
+
+      def initialize(*args, &block)
+        super
+        @dependencies = nil
+      end
+
+      def name
+        "perl_cpan"
+      end
+
+      def dependencies
+        return @dependencies if @dependencies
+        @dependencies = deps_list.map do |d|
+          d.collect_licenses
+          d.to_dep
+        end
+      end
+
+      def deps_list
+        return @deps_list if @deps_list
+
+        xml_doc = REXML::Document.new(dependency_graph_xml)
+
+        root = xml_doc.root
+
+        deps = root.get_elements("//dependency")
+
+        @deps_list = []
+
+        deps.each do |dep|
+          dep_module_name = dep.get_text("module").to_s
+          next if dep_module_name == module_name
+          @deps_list << CPANDependency.new(
+            module_name: dep_module_name,
+            dist: dep.get_text("dist").to_s,
+            version: dep.get_text("distversion").to_s,
+            cpanfile: dep.get_text("cpanfile").to_s,
+            cache_root: options.cpan_cache,
+            overrides: options.overrides
+          )
+        end
+
+        @deps_list
+      end
+
+      def dependency_graph_xml
+        @dependency_graph_xml ||=
+          begin
+            dependency_graph_xml_file = NetFetcher.cache(dependency_graph_url)
+            raw_xml = File.read(dependency_graph_xml_file)
+            FileUtils.rm_f(dependency_graph_xml_file)
+            raw_xml
+          end
+      end
+
+      # NOTE: there's no SSL version available. Take care handling any
+      # data/code referenced in responses from this site.
+      def dependency_graph_url
+        "http://deps.cpantesters.org/?xml=1;module=#{module_name};perl=5.24.0;os=any%20OS;pureperl=0"
+      end
+
+      # Infers the module name from the directory name. For Chef Server, the
+      # two perl packages we use are:
+      # * "App-Sqitch-VERSION" => "App::Sqitch"
+      # * "DBD-Pg-VERSION" => "DBD::Pg"
+      #
+      # NOTE: Distributions may contain multiple modules that would each have
+      # their own dependency graphs and it's possible to get a perl project
+      # that doesn't obey this convention (e.g., if you git clone it). But this
+      # meets our immediate needs.
+      def module_name
+        File.basename(project_dir).split("-")[0...-1].join("::")
+      end
+
+      # NOTE: it's possible that projects won't have a META.yml, but the two
+      # that we care about for Chef Server do have one. As of 2015, 84% of perl
+      # distribution packages have one: http://neilb.org/2015/10/18/spotters-guide.html
+      def detected?
+        File.exist?(meta_yml_path)
+      end
+
+      def meta_yml_path
+        File.join(project_dir, "META.yml")
+      end
+
+    end
+  end
+end