inspec 0.31.0 → 0.32.0

data/lib/inspec/dependencies/dependency_set.rb

@@ -1,5 +1,6 @@
 # encoding: utf-8
 require 'inspec/dependencies/vendor_index'
+require 'inspec/dependencies/requirement'
 require 'inspec/dependencies/resolver'
 
 module Inspec
@@ -10,17 +11,60 @@ module Inspec
   # VendorIndex and the Resolver.
   #
   class DependencySet
-    attr_reader :list, :vendor_path
+    #
+    # Return a dependency set given a lockfile.
+    #
+    # @param lockfile [Inspec::Lockfile] A lockfile to generate the dependency set from
+    # @param cwd [String] Current working directory for relative path includes
+    # @param vendor_path [String] Path to the vendor directory
+    #
+    def self.from_lockfile(lockfile, cwd, vendor_path)
+      vendor_index = VendorIndex.new(vendor_path)
+      dep_tree = lockfile.deps.map do |dep|
+        Inspec::Requirement.from_lock_entry(dep, cwd, vendor_index)
+      end
 
+      dep_list = flatten_dep_tree(dep_tree)
+      new(cwd, vendor_path, dep_list)
+    end
+
+    # This is experimental code to test the working of the
+    # dependency loader - perform a proper dependency related search
+    # in the future.
+    #
+    # Flatten tree because that is all we know how to deal with for
+    # right now. Last dep seen for a given name wins right now.
+    def self.flatten_dep_tree(dep_tree)
+      dep_list = {}
+      dep_tree.each do |d|
+        dep_list[d.name] = d
+        dep_list.merge!(flatten_dep_tree(d.dependencies))
+      end
+      dep_list
+    end
+
+    attr_reader :vendor_path
+    attr_writer :dep_list
     # initialize
     #
     # @param cwd [String] current working directory for relative path includes
     # @param vendor_path [String] path which contains vendored dependencies
     # @return [dependencies] this
-    def initialize(cwd, vendor_path)
+    def initialize(cwd, vendor_path, dep_list = nil)
       @cwd = cwd
-      @vendor_path = vendor_path || File.join(Dir.home, '.inspec', 'cache')
-      @list = nil
+      @vendor_path = vendor_path
+      @dep_list = dep_list
+    end
+
+    def list
+      @dep_list
+    end
+
+    def to_array
+      return [] if @dep_list.nil?
+      @dep_list.map do |_k, v|
+        v.to_hash
+      end.compact
     end
 
     #
@@ -29,10 +73,11 @@ module Inspec
     #
     # @param dependencies [Gem::Dependency] list of dependencies
     # @return [nil]
+    #
     def vendor(dependencies)
       return nil if dependencies.nil? || dependencies.empty?
       @vendor_index ||= VendorIndex.new(@vendor_path)
-      @list = Resolver.resolve(dependencies, @vendor_index, @cwd)
+      @dep_list = Resolver.resolve(dependencies, @vendor_index, @cwd)
     end
   end
 end

data/lib/inspec/dependencies/lockfile.rb

@@ -0,0 +1,94 @@
+# encoding: utf-8
+require 'yaml'
+
+module Inspec
+  class Lockfile
+    # When we finalize this feature, we should set these to 1
+    MINIMUM_SUPPORTED_VERSION = 0
+    CURRENT_LOCKFILE_VERSION = 0
+
+    def self.from_dependency_set(dep_set)
+      lockfile_content = {
+        'lockfile_version' => CURRENT_LOCKFILE_VERSION,
+        'depends' => dep_set.to_array,
+      }
+      new(lockfile_content)
+    end
+
+    def self.from_file(path)
+      parsed_content = YAML.load(File.read(path))
+      version = parsed_content['lockfile_version']
+      fail "No lockfile_version set in #{path}!" if version.nil?
+      validate_lockfile_version!(version.to_i)
+      new(parsed_content)
+    end
+
+    def self.validate_lockfile_version!(version)
+      if version < MINIMUM_SUPPORTED_VERSION
+        fail <<EOF
+This lockfile specifies a lockfile_version of #{version} which is
+lower than the minimum supported version #{MINIMUM_SUPPORTED_VERSION}.
+
+Please create a new lockfile for this project by running:
+
+    inspec vendor
+EOF
+      elsif version == 0
+        # Remove this case once this feature stablizes
+        $stderr.puts <<EOF
+WARNING: This is a version 0 lockfile. Thank you for trying the
+experimental dependency management feature. Please be aware you may
+need to regenerate this lockfile in future versions as the feature is
+currently in development.
+EOF
+      elsif version > CURRENT_LOCKFILE_VERSION
+        fail <<EOF
+This lockfile claims to be version #{version} which is greater than
+the most recent lockfile version(#{CURRENT_LOCKFILE_VERSION}).
+
+This may happen if you are using an older version of inspec than was
+used to create the lockfile.
+EOF
+      end
+    end
+
+    attr_reader :version, :deps
+    def initialize(lockfile_content_hash)
+      version = lockfile_content_hash['lockfile_version']
+      @version = version.to_i
+      parse_content_hash(lockfile_content_hash)
+    end
+
+    def to_yaml
+      {
+        'lockfile_version' => CURRENT_LOCKFILE_VERSION,
+        'depends' => @deps,
+      }.to_yaml
+    end
+
+    private
+
+    # Refactor this to be "version-wise" - i.e. make one dispatch
+    # function for each version so that even if it duplicates code,
+    # it can describe the part of the code that it expects to be
+    # different. Then that dispatch routine can call more well
+    # defined methods like "parse_v0_dependencies" or
+    # "parse_flat_dependencies" or what not as things generally
+    # develop. It does help people easily set breakpoints/track
+    # different entry points of the API.
+    def parse_content_hash(lockfile_content_hash)
+      case version
+      when 0
+        parse_content_hash_0(lockfile_content_hash)
+      else
+        # If we've gotten here, there is likely a mistake in the
+        # lockfile version validation in the constructor.
+        fail "No lockfile parser for version #{version}"
+      end
+    end
+
+    def parse_content_hash_0(lockfile_content_hash)
+      @deps = lockfile_content_hash['depends']
+    end
+  end
+end

data/lib/inspec/dependencies/requirement.rb

@@ -1,87 +1,134 @@
 # encoding: utf-8
 require 'inspec/fetcher'
+require 'digest'
 
 module Inspec
   #
   # Inspec::Requirement represents a given profile dependency, where
   # appropriate we delegate to Inspec::Profile directly.
   #
-  class Requirement
+  class Requirement # rubocop:disable Metrics/ClassLength
     attr_reader :name, :dep, :cwd, :opts
+    attr_writer :dependencies
+
+    def self.from_metadata(dep, vendor_index, opts)
+      fail 'Cannot load empty dependency.' if dep.nil? || dep.empty?
+      name = dep[:name] || fail('You must provide a name for all dependencies')
+      version = dep[:version]
+      new(name, version, vendor_index, opts[:cwd], opts.merge(dep))
+    end
+
+    def self.from_lock_entry(entry, cwd, vendor_index)
+      req = new(entry['name'],
+                entry['version_constraints'],
+                vendor_index,
+                cwd, { url: entry['resolved_source'] })
+
+      locked_deps = []
+      Array(entry['dependencies']).each do |dep_entry|
+        locked_deps << Inspec::Requirement.from_lock_entry(dep_entry, cwd, vendor_index)
+      end
+
+      req.lock_deps(locked_deps)
+      req
+    end
 
     def initialize(name, version_constraints, vendor_index, cwd, opts)
       @name = name
-      @dep = Gem::Dependency.new(name,
-                                 Gem::Requirement.new(Array(version_constraints)),
-                                 :runtime)
+      @version_requirement = Gem::Requirement.new(Array(version_constraints))
+      @dep = Gem::Dependency.new(name, @version_requirement, :runtime)
       @vendor_index = vendor_index
       @opts = opts
       @cwd = cwd
     end
 
-    def matches_spec?(spec)
-      params = spec.profile.metadata.params
-      @dep.match?(params[:name], params[:version])
+    def required_version
+      @version_requirement
+    end
+
+    def source_version
+      profile.metadata.params[:version]
+    end
+
+    def source_satisfies_spec?
+      name = profile.metadata.params[:name]
+      version = profile.metadata.params[:version]
+      @dep.match?(name, version)
+    end
+
+    def to_hash
+      h = {
+        'name' => name,
+        'resolved_source' => source_url,
+        'version_constraints' => @version_requirement.to_s,
+      }
+
+      if !dependencies.empty?
+        h['dependencies'] = dependencies.map(&:to_hash)
+      end
+
+      if is_vendored?
+        h['content_hash'] = content_hash
+      end
+      h
+    end
+
+    def lock_deps(dep_array)
+      @dependencies = dep_array
+    end
+
+    def is_vendored?
+      @vendor_index.exists?(@name, source_url)
+    end
+
+    def content_hash
+      @content_hash ||= begin
+                          archive_path = @vendor_index.archive_entry_for(@name, source_url)
+                          fail "No vendored archive path for #{self}, cannot take content hash" if archive_path.nil?
+                          Digest::SHA256.hexdigest File.read(archive_path)
+                        end
     end
 
     def source_url
-      case source_type
-      when :local_path
-        "file://#{File.expand_path(opts[:path])}"
-      when :url
-        @opts[:url]
+      if opts[:path]
+        "file://#{File.expand_path(opts[:path], @cwd)}"
+      elsif opts[:url]
+        opts[:url]
       end
     end
 
     def local_path
-      @local_path ||= case source_type
-                      when :local_path
-                        File.expand_path(opts[:path], @cwd)
+      @local_path ||= if fetcher.class == Fetchers::Local
+                        File.expand_path(fetcher.target, @cwd)
                       else
                         @vendor_index.prefered_entry_for(@name, source_url)
                       end
     end
 
-    def source_type
-      @source_type ||= if @opts[:path]
-                         :local_path
-                       elsif opts[:url]
-                         :url
-                       else
-                         fail "Cannot determine source type from #{opts}"
-                       end
-    end
-
-    def fetcher_class
-      @fetcher_class ||= case source_type
-                         when :local_path
-                           Fetchers::Local
-                         when :url
-                           Fetchers::Url
-                         else
-                           fail "No known fetcher for dependency #{name} with source_type #{source_type}"
-                         end
-
-      fail "No fetcher for #{name} (options: #{opts})" if @fetcher_class.nil?
-      @fetcher_class
+    def fetcher
+      @fetcher ||= Inspec::Fetcher.resolve(source_url)
+      fail "No fetcher for #{name} (options: #{opts})" if @fetcher.nil?
+      @fetcher
     end
 
     def pull
-      case source_type
-      when :local_path
+      # TODO(ssd): Dispatch on the class here is gross.  Seems like
+      # Fetcher is missing an API we want.
+      if fetcher.class == Fetchers::Local || @vendor_index.exists?(@name, source_url)
         local_path
       else
-        if @vendor_index.exists?(@name, source_url)
-          local_path
-        else
-          archive = fetcher_class.download_archive(source_url)
-          @vendor_index.add(@name, source_url, archive.path)
-        end
+        @vendor_index.add(@name, source_url, fetcher.archive_path)
+      end
+    end
+
+    def dependencies
+      @dependencies ||= profile.metadata.dependencies.map do |r|
+        Inspec::Requirement.from_metadata(r, @vendor_index, cwd: @cwd)
       end
     end
 
     def to_s
-      dep.to_s
+      "#{dep} (#{source_url})"
     end
 
     def path
@@ -92,12 +139,5 @@ module Inspec
       return nil if path.nil?
       @profile ||= Inspec::Profile.for_target(path, {})
     end
-
-    def self.from_metadata(dep, vendor_index, opts)
-      fail 'Cannot load empty dependency.' if dep.nil? || dep.empty?
-      name = dep[:name] || fail('You must provide a name for all dependencies')
-      version = dep[:version]
-      new(name, version, vendor_index, opts[:cwd], opts.merge(dep))
-    end
   end
 end

data/lib/inspec/dependencies/resolver.rb

@@ -1,188 +1,71 @@
 # encoding: utf-8
-# author: Dominik Richter
-# author: Christoph Hartmann
-require 'logger'
-require 'molinillo'
+# author: Steven Danna <steve@chef.io>
+require 'inspec/log'
 require 'inspec/errors'
-require 'inspec/dependencies/requirement'
 
 module Inspec
   #
-  # Inspec::Resolver is responsible for recursively resolving all the
-  # depenendencies for a given top-level dependency set.
+  # Inspec::Resolver is a simple dependency resolver. Unlike Bundler
+  # or Berkshelf, it does not attempt to resolve each named dependency
+  # to a single version. Rather, it traverses down the dependency tree
+  # and:
+  #
+  # - Fetches the dependency from the source
+  # - Checks the presence of cycles, and
+  # - Checks that the specified dependency source satisfies the
+  #   specified version constraint
+  #
+  # The full dependency tree is then available for the loader, which
+  # will provide the isolation necessary to support multiple versions
+  # of the same profile being used at runtime.
+  #
+  # Currently the fetching happens somewhat lazily depending on the
+  # implementation of the fetcher being used.
   #
   class Resolver
-    def self.resolve(requirements, vendor_index, cwd, opts = {})
-      reqs = requirements.map do |req|
-        req = Inspec::Requirement.from_metadata(req, vendor_index, cwd: cwd)
+    def self.resolve(dependencies, vendor_index, working_dir)
+      reqs = dependencies.map do |dep|
+        req = Inspec::Requirement.from_metadata(dep, vendor_index, cwd: working_dir)
         req || fail("Cannot initialize dependency: #{req}")
       end
-
-      new(vendor_index, opts.merge(cwd: cwd)).resolve(reqs)
-    end
-
-    def initialize(vendor_index, opts = {})
-      @logger = opts[:logger] || Logger.new(nil)
-      @debug_mode = false
-
-      @vendor_index = vendor_index
-      @cwd = opts[:cwd] || './'
-      @resolver = Molinillo::Resolver.new(self, self)
-      @search_cache = {}
-    end
-
-    # Resolve requirements.
-    #
-    # @param requirements [Array(Inspec::requirement)] Array of requirements
-    # @return [Array(String)] list of resolved dependency paths
-    def resolve(requirements)
-      requirements.each(&:pull)
-      @base_dep_graph = Molinillo::DependencyGraph.new
-      @dep_graph = @resolver.resolve(requirements, @base_dep_graph)
-      arr = @dep_graph.map(&:payload)
-      Hash[arr.map { |e| [e.name, e] }]
-    rescue Molinillo::VersionConflict => e
-      raise VersionConflict.new(e.conflicts.keys.uniq, e.message)
-    rescue Molinillo::CircularDependencyError => e
-      names = e.dependencies.sort_by(&:name).map { |d| "profile '#{d.name}'" }
-      raise CyclicDependencyError,
-            'Your profile has requirements that depend on each other, creating '\
-            "an infinite loop. Please remove #{names.count > 1 ? 'either ' : ''} "\
-            "#{names.join(' or ')} and try again."
-    end
-
-    # --------------------------------------------------------------------------
-    # SpecificationProvider
-
-    # Search for the specifications that match the given dependency.
-    # The specifications in the returned array will be considered in reverse
-    # order, so the latest version ought to be last.
-    # @note This method should be 'pure', i.e. the return value should depend
-    #   only on the `dependency` parameter.
-    #
-    # @param [Object] dependency
-    # @return [Array<Object>] the specifications that satisfy the given
-    #   `dependency`.
-    def search_for(dep)
-      unless dep.is_a?(Inspec::Requirement)
-        fail 'Internal error: Dependency resolver requires an Inspec::Requirement object for #search_for(dependency)'
-      end
-      @search_cache[dep] ||= uncached_search_for(dep)
-    end
-
-    def uncached_search_for(dep)
-      # pre-cached and specified dependencies
-      return [dep] unless dep.profile.nil?
-
-      results = @vendor_index.find(dep)
-      return [] unless results.any?
-
-      # TODO: load dep from vendor index
-      # vertex = @dep_graph.vertex_named(dep.name)
-      # locked_requirement = vertex.payload.requirement if vertex
-      fail NotImplementedError, "load dependency #{dep} from vendor index"
-    end
-
-    # Returns the dependencies of `specification`.
-    # @note This method should be 'pure', i.e. the return value should depend
-    #   only on the `specification` parameter.
-    #
-    # @param [Object] specification
-    # @return [Array<Object>] the dependencies that are required by the given
-    #   `specification`.
-    def dependencies_for(specification)
-      specification.profile.metadata.dependencies.map do |r|
-        Inspec::Requirement.from_metadata(r, @vendor_index, cwd: @cwd)
-      end
-    end
-
-    # Determines whether the given `requirement` is satisfied by the given
-    # `spec`, in the context of the current `activated` dependency graph.
-    #
-    # @param [Object] requirement
-    # @param [DependencyGraph] activated the current dependency graph in the
-    #   resolution process.
-    # @param [Object] spec
-    # @return [Boolean] whether `requirement` is satisfied by `spec` in the
-    #   context of the current `activated` dependency graph.
-    def requirement_satisfied_by?(requirement, _activated, spec)
-      requirement.matches_spec?(spec) || spec.is_a?(Inspec::Profile)
+      new.resolve(reqs)
     end
 
-    # Returns the name for the given `dependency`.
-    # @note This method should be 'pure', i.e. the return value should depend
-    #   only on the `dependency` parameter.
-    #
-    # @param [Object] dependency
-    # @return [String] the name for the given `dependency`.
-    def name_for(dependency)
-      unless dependency.is_a?(Inspec::Requirement)
-        fail 'Internal error: Dependency resolver requires an Inspec::Requirement object for #name_for(dependency)'
+    def resolve(deps, top_level = true, seen_items = {}, path_string = '')
+      graph = {}
+      if top_level
+        Inspec::Log.debug("Starting traversal of dependencies #{deps.map(&:name)}")
+      else
+        Inspec::Log.debug("Traversing dependency tree of transitive dependency #{deps.map(&:name)}")
       end
-      dependency.name
-    end
-
-    # @return [String] the name of the source of explicit dependencies, i.e.
-    #   those passed to {Resolver#resolve} directly.
-    def name_for_explicit_dependency_source
-      'inspec.yml'
-    end
 
-    # @return [String] the name of the source of 'locked' dependencies, i.e.
-    #   those passed to {Resolver#resolve} directly as the `base`
-    def name_for_locking_dependency_source
-      'inspec.lock'
-    end
-
-    # Sort dependencies so that the ones that are easiest to resolve are first.
-    # Easiest to resolve is (usually) defined by:
-    #   1) Is this dependency already activated?
-    #   2) How relaxed are the requirements?
-    #   3) Are there any conflicts for this dependency?
-    #   4) How many possibilities are there to satisfy this dependency?
-    #
-    # @param [Array<Object>] dependencies
-    # @param [DependencyGraph] activated the current dependency graph in the
-    #   resolution process.
-    # @param [{String => Array<Conflict>}] conflicts
-    # @return [Array<Object>] a sorted copy of `dependencies`.
-    def sort_dependencies(dependencies, activated, conflicts)
-      dependencies.sort_by do |dependency|
-        name = name_for(dependency)
-        [
-          activated.vertex_named(name).payload ? 0 : 1,
-          # amount_constrained(dependency), # TODO
-          conflicts[name] ? 0 : 1,
-          # activated.vertex_named(name).payload ? 0 : search_for(dependency).count, # TODO
-        ]
+      deps.each do |dep|
+        path_string = if path_string.empty?
+                        dep.name
+                      else
+                        path_string + " -> #{dep.name}"
+                      end
+
+        if seen_items.key?(dep.source_url)
+          fail Inspec::CyclicDependencyError, "Dependency #{dep} would cause a dependency cycle (#{path_string})"
+        else
+          seen_items[dep.source_url] = true
+        end
+
+        if !dep.source_satisfies_spec?
+          fail Inspec::UnsatisfiedVersionSpecification, "The profile #{dep.name} from #{dep.source_url} has a version #{dep.source_version} which doesn't match #{dep.required_version}"
+        end
+
+        Inspec::Log.debug("Adding #{dep.source_url}")
+        graph[dep.name] = dep
+        if !dep.dependencies.empty?
+          # Recursively resolve any transitive dependencies.
+          resolve(dep.dependencies, false, seen_items.dup, path_string)
+        end
       end
-    end
-
-    # Returns whether this dependency, which has no possible matching
-    # specifications, can safely be ignored.
-    #
-    # @param [Object] dependency
-    # @return [Boolean] whether this dependency can safely be skipped.
-    def allow_missing?(_dependency)
-      # TODO
-      false
-    end
-
-    # --------------------------------------------------------------------------
-    # UI
-
-    include Molinillo::UI
-
-    # The {IO} object that should be used to print output. `STDOUT`, by default.
-    #
-    # @return [IO]
-    def output
-      self
-    end
 
-    def print(what = '')
-      @logger.info(what)
+      Inspec::Log.debug('Dependency traversal complete.') if top_level
+      graph
     end
-    alias puts print
   end
 end