inspec 0.33.2 → 0.34.0

data/lib/fetchers/url.rb

@@ -8,21 +8,31 @@ require 'open-uri'
 
 module Fetchers
   class Url < Inspec.fetcher(1)
+    MIME_TYPES = {
+      'application/x-zip-compressed' => '.zip',
+      'application/zip' => '.zip',
+      'application/x-gzip' => '.tar.gz',
+      'application/gzip' => '.tar.gz',
+    }.freeze
+
     name 'url'
     priority 200
 
-    attr_reader :files
-
     def self.resolve(target, opts = {})
-      return nil unless target.is_a?(String)
+      if target.is_a?(Hash) && target.key?(:url)
+        resolve_from_string(target[:url], opts)
+      elsif target.is_a?(String)
+        resolve_from_string(target, opts)
+      end
+    end
+
+    def self.resolve_from_string(target, opts)
       uri = URI.parse(target)
       return nil if uri.nil? or uri.scheme.nil?
       return nil unless %{ http https }.include? uri.scheme
       target = transform(target)
-      # fetch this url and hand it off
-      res = new(target, opts)
-      resolve_next(res.archive.path, res)
-    rescue URI::Error => _e
+      new(target, opts)
+    rescue URI::Error
       nil
     end
 
@@ -44,38 +54,51 @@ module Fetchers
     # https://github.com/hardening-io/tests-os-hardening/tree/48bd4388ddffde68badd83aefa654e7af3231876
     # is transformed to
     # https://github.com/hardening-io/tests-os-hardening/archive/48bd4388ddffde68badd83aefa654e7af3231876.tar.gz
+    GITHUB_URL_REGEX = %r{^https?://(www\.)?github\.com/(?<user>[\w-]+)/(?<repo>[\w-]+)(\.git)?(/)?$}
+    GITHUB_URL_WITH_TREE_REGEX = %r{^https?://(www\.)?github\.com/(?<user>[\w-]+)/(?<repo>[\w-]+)/tree/(?<commit>[\w\.]+)(/)?$}
     def self.transform(target)
-      # support for default github url
-      m = %r{^https?://(www\.)?github\.com/(?<user>[\w-]+)/(?<repo>[\w-]+)(\.git)?(/)?$}.match(target)
-      return "https://github.com/#{m[:user]}/#{m[:repo]}/archive/master.tar.gz" if m
+      transformed_target = if m = GITHUB_URL_REGEX.match(target) # rubocop:disable Lint/AssignmentInCondition
+                             "https://github.com/#{m[:user]}/#{m[:repo]}/archive/master.tar.gz"
+                           elsif m = GITHUB_URL_WITH_TREE_REGEX.match(target) # rubocop:disable Lint/AssignmentInCondition
+                             "https://github.com/#{m[:user]}/#{m[:repo]}/archive/#{m[:commit]}.tar.gz"
+                           end
+
+      if transformed_target
+        Inspec::Log.warn("URL target #{target} transformed to #{transformed_target}. Consider using the git fetcher")
+        transformed_target
+      else
+        target
+      end
+    end
 
-      # support for branch and commit urls
-      m = %r{^https?://(www\.)?github\.com/(?<user>[\w-]+)/(?<repo>[\w-]+)/tree/(?<commit>[\w\.]+)(/)?$}.match(target)
-      return "https://github.com/#{m[:user]}/#{m[:repo]}/archive/#{m[:commit]}.tar.gz" if m
+    attr_reader :files, :archive_path
 
-      # if we could not find a match, return the original value
-      target
+    def initialize(url, opts)
+      @target = url
+      @insecure = opts['insecure']
+      @token = opts['token']
+      @config = opts
     end
 
-    MIME_TYPES = {
-      'application/x-zip-compressed' => '.zip',
-      'application/zip' => '.zip',
-      'application/x-gzip' => '.tar.gz',
-      'application/gzip' => '.tar.gz',
-    }.freeze
+    def fetch(path)
+      Inspec::Log.debug("Fetching URL: #{@target}")
+      @archive_path = download_archive(path)
+    end
+
+    def resolved_source
+      { url: @target }
+    end
+
+    private
 
     # download url into archive using opts,
     # returns File object and content-type from HTTP headers
-    def self.download_archive(url, opts = {})
+    def download_archive(path)
       http_opts = {}
-      # http_opts['http_basic_authentication'] = [opts['user'] || '', opts['password'] || ''] if opts['user']
-      http_opts['ssl_verify_mode'.to_sym] = OpenSSL::SSL::VERIFY_NONE if opts['insecure']
-      http_opts['Authorization'] = "Bearer #{opts['token']}" if opts['token']
+      http_opts['ssl_verify_mode'.to_sym] = OpenSSL::SSL::VERIFY_NONE if @insecure
+      http_opts['Authorization'] = "Bearer #{@token}" if @token
 
-      remote = open(
-        url,
-        http_opts,
-      )
+      remote = open(@target, http_opts)
 
       content_type = remote.meta['content-type']
       file_type = MIME_TYPES[content_type] ||
@@ -86,25 +109,16 @@ module Fetchers
       if file_type.nil?
         fail "Could not determine file type for content type #{content_type}."
       end
-
+      final_path = "#{path}#{file_type}"
       # download content
       archive = Tempfile.new(['inspec-dl-', file_type])
       archive.binmode
       archive.write(remote.read)
       archive.rewind
       archive.close
-      archive
-    end
-
-    attr_reader :archive
-
-    def initialize(url, opts)
-      @target = url
-      @archive = self.class.download_archive(url, opts)
-    end
-
-    def archive_path
-      @archive.path
+      FileUtils.mv(archive.path, final_path)
+      Inspec::Log.debug("Fetched archive moved to: #{final_path}")
+      final_path
     end
   end
 end

data/lib/inspec/dependencies/lockfile.rb

@@ -62,7 +62,7 @@ EOF
     def to_yaml
       {
         'lockfile_version' => CURRENT_LOCKFILE_VERSION,
-        'depends' => @deps,
+        'depends' => @deps.map { |i| stringify_keys(i) },
       }.to_yaml
     end
 
@@ -88,7 +88,33 @@ EOF
     end
 
     def parse_content_hash_0(lockfile_content_hash)
-      @deps = lockfile_content_hash['depends']
+      @deps = if lockfile_content_hash['depends']
+                lockfile_content_hash['depends'].map { |i| symbolize_keys(i) }
+              end
+    end
+
+    def mutate_hash_keys_with(hash, fun)
+      hash.each_with_object({}) do |v, memo|
+        key = fun.call(v[0])
+        value = if v[1].is_a?(Hash)
+                  mutate_hash_keys_with(v[1], fun)
+                elsif v[1].is_a?(Array)
+                  v[1].map do |i|
+                    i.is_a?(Hash) ? mutate_hash_keys_with(i, fun) : i
+                  end
+                else
+                  v[1]
+                end
+        memo[key] = value
+      end
+    end
+
+    def stringify_keys(hash)
+      mutate_hash_keys_with(hash, proc { |i| i.to_s })
+    end
+
+    def symbolize_keys(hash)
+      mutate_hash_keys_with(hash, proc { |i| i.to_sym })
     end
   end
 end

data/lib/inspec/dependencies/requirement.rb

@@ -8,7 +8,7 @@ module Inspec
   # Inspec::Requirement represents a given profile dependency, where
   # appropriate we delegate to Inspec::Profile directly.
   #
-  class Requirement # rubocop:disable Metrics/ClassLength
+  class Requirement
     attr_reader :name, :dep, :cwd, :opts
     attr_writer :dependencies
 
@@ -20,12 +20,11 @@ module Inspec
     end
 
     def self.from_lock_entry(entry, cwd, vendor_index, backend)
-      req = new(entry['name'],
-                entry['version_constraints'],
+      req = new(entry[:name],
+                entry[:version_constraints],
                 vendor_index,
                 cwd,
-                { url: entry['resolved_source'],
-                  backend: backend })
+                entry[:resolved_source].merge(backend: backend))
 
       locked_deps = []
       Array(entry['dependencies']).each do |dep_entry|
@@ -59,10 +58,14 @@ module Inspec
       @dep.match?(name, version)
     end
 
+    def resolved_source
+      @resolved_source ||= fetcher.resolved_source
+    end
+
     def to_hash
       h = {
         'name' => name,
-        'resolved_source' => source_url,
+        'resolved_source' => resolved_source,
         'version_constraints' => @version_requirement.to_s,
       }
 
@@ -70,9 +73,7 @@ module Inspec
         h['dependencies'] = dependencies.map(&:to_hash)
       end
 
-      if is_vendored?
-        h['content_hash'] = content_hash
-      end
+      h['content_hash'] = content_hash if content_hash
       h
     end
 
@@ -80,50 +81,21 @@ module Inspec
       @dependencies = dep_array
     end
 
-    def is_vendored?
-      @vendor_index.exists?(@name, source_url)
-    end
-
     def content_hash
       @content_hash ||= begin
-                          archive_path = @vendor_index.archive_entry_for(@name, source_url)
-                          fail "No vendored archive path for #{self}, cannot take content hash" if archive_path.nil?
-                          Digest::SHA256.hexdigest File.read(archive_path)
+                          archive_path = @vendor_index.archive_entry_for(fetcher.cache_key) || fetcher.archive_path
+                          if archive_path && File.file?(archive_path)
+                            Digest::SHA256.hexdigest File.read(archive_path)
+                          end
                         end
     end
 
-    def source_url
-      if opts[:path]
-        "file://#{File.expand_path(opts[:path], @cwd)}"
-      elsif opts[:url]
-        opts[:url]
-      end
-    end
-
-    def local_path
-      @local_path ||= if fetcher.class == Fetchers::Local
-                        File.expand_path(fetcher.target, @cwd)
-                      else
-                        @vendor_index.prefered_entry_for(@name, source_url)
-                      end
-    end
-
     def fetcher
-      @fetcher ||= Inspec::Fetcher.resolve(source_url)
+      @fetcher ||= Inspec::Fetcher.resolve(opts)
       fail "No fetcher for #{name} (options: #{opts})" if @fetcher.nil?
       @fetcher
     end
 
-    def pull
-      # TODO(ssd): Dispatch on the class here is gross.  Seems like
-      # Fetcher is missing an API we want.
-      if fetcher.class == Fetchers::Local || @vendor_index.exists?(@name, source_url)
-        local_path
-      else
-        @vendor_index.add(@name, source_url, fetcher.archive_path)
-      end
-    end
-
     def dependencies
       @dependencies ||= profile.metadata.dependencies.map do |r|
         Inspec::Requirement.from_metadata(r, @vendor_index, cwd: @cwd, backend: @backend)
@@ -131,20 +103,17 @@ module Inspec
     end
 
     def to_s
-      "#{dep} (#{source_url})"
-    end
-
-    def path
-      @path ||= pull
+      "#{dep} (#{resolved_source})"
     end
 
     def profile
-      return nil if path.nil?
-      opts = { backend: @backend }
+      opts = @opts.dup
+      opts[:cache] = @vendor_index
+      opts[:backend] = @backend
       if !@dependencies.nil?
         opts[:dependencies] = Inspec::DependencySet.from_array(@dependencies, @cwd, @vendor_index, @backend)
       end
-      @profile ||= Inspec::Profile.for_target(path, opts)
+      @profile ||= Inspec::Profile.for_target(opts, opts)
     end
   end
 end

data/lib/inspec/dependencies/resolver.rb

@@ -31,7 +31,23 @@ module Inspec
       new.resolve(reqs)
     end
 
-    def resolve(deps, top_level = true, seen_items = {}, path_string = '')
+    def detect_duplicates(deps, top_level, path_string)
+      seen_items_local = []
+      deps.each do |dep|
+        if seen_items_local.include?(dep.name)
+          problem_cookbook = if top_level
+                               'the inspec.yml for this profile.'
+                             else
+                               "the dependency information for #{path_string.split(' ').last}"
+                             end
+          fail Inspec::DuplicateDep, "The dependency #{dep.name} is listed twice in #{problem_cookbook}"
+        else
+          seen_items_local << dep.name
+        end
+      end
+    end
+
+    def resolve(deps, top_level = true, seen_items = {}, path_string = '') # rubocop:disable Metrics/AbcSize
       graph = {}
       if top_level
         Inspec::Log.debug("Starting traversal of dependencies #{deps.map(&:name)}")
@@ -39,28 +55,29 @@ module Inspec
         Inspec::Log.debug("Traversing dependency tree of transitive dependency #{deps.map(&:name)}")
       end
 
+      detect_duplicates(deps, top_level, path_string)
       deps.each do |dep|
-        path_string = if path_string.empty?
-                        dep.name
-                      else
-                        path_string + " -> #{dep.name}"
-                      end
+        new_seen_items = seen_items.dup
+        new_path_string = if path_string.empty?
+                            dep.name
+                          else
+                            path_string + " -> #{dep.name}"
+                          end
 
-        if seen_items.key?(dep.source_url)
-          fail Inspec::CyclicDependencyError, "Dependency #{dep} would cause a dependency cycle (#{path_string})"
+        if new_seen_items.key?(dep.resolved_source)
+          fail Inspec::CyclicDependencyError, "Dependency #{dep} would cause a dependency cycle (#{new_path_string})"
         else
-          seen_items[dep.source_url] = true
+          new_seen_items[dep.resolved_source] = true
         end
 
         if !dep.source_satisfies_spec?
-          fail Inspec::UnsatisfiedVersionSpecification, "The profile #{dep.name} from #{dep.source_url} has a version #{dep.source_version} which doesn't match #{dep.required_version}"
+          fail Inspec::UnsatisfiedVersionSpecification, "The profile #{dep.name} from #{dep.resolved_source} has a version #{dep.source_version} which doesn't match #{dep.required_version}"
         end
 
-        Inspec::Log.debug("Adding #{dep.source_url}")
+        Inspec::Log.debug("Adding dependency #{dep.name} (#{dep.resolved_source})")
         graph[dep.name] = dep
         if !dep.dependencies.empty?
-          # Recursively resolve any transitive dependencies.
-          resolve(dep.dependencies, false, seen_items.dup, path_string)
+          resolve(dep.dependencies, false, new_seen_items.dup, new_path_string)
         end
       end
 

data/lib/inspec/dependencies/vendor_index.rb

@@ -23,32 +23,17 @@ module Inspec
       FileUtils.mkdir_p(@path) unless File.directory?(@path)
     end
 
-    def add(name, source, path_from)
-      path_to = base_path_for(name, source)
-      path_to = if File.directory?(path_to)
-                  path_to
-                elsif path_from.end_with?('.zip')
-                  "#{path_to}.tar.gz"
-                elsif path_from.end_with?('.tar.gz')
-                  "#{path_to}.tar.gz"
-                else
-                  fail "Cannot add unknown archive #{path} to vendor index"
-                end
-      FileUtils.cp_r(path_from, path_to)
-      path_to
-    end
-
-    def prefered_entry_for(name, source_url)
-      path = base_path_for(name, source_url)
+    def prefered_entry_for(key)
+      path = base_path_for(key)
       if File.directory?(path)
         path
       else
-        archive_entry_for(name, source_url)
+        archive_entry_for(key)
       end
     end
 
-    def archive_entry_for(name, source_url)
-      path = base_path_for(name, source_url)
+    def archive_entry_for(key)
+      path = base_path_for(key)
       if File.exist?("#{path}.tar.gz")
         "#{path}.tar.gz"
       elsif File.exist?("#{path}.zip")
@@ -64,8 +49,8 @@ module Inspec
     # @param [String] source_url
     # @return [Boolean]
     #
-    def exists?(name, source_url)
-      path = base_path_for(name, source_url)
+    def exists?(key)
+      path = base_path_for(key)
       File.directory?(path) || File.exist?("#{path}.tar.gz") || File.exist?("#{path}.zip")
     end
 
@@ -80,26 +65,8 @@ module Inspec
     # @param [String] source_url
     # @return [String]
     #
-    def base_path_for(name, source_url)
-      File.join(@path, key_for(name, source_url))
-    end
-
-    private
-
-    #
-    # Return the key for a given profile in the vendor index.
-    #
-    # The `source_url` parameter should be a URI-like string that
-    # fully specifies the source of the exact version we want to pull
-    # down.
-    #
-    # @param [String] name
-    # @param [String] source_url
-    # @return [String]
-    #
-    def key_for(name, source_url)
-      source_hash = Digest::SHA256.hexdigest source_url
-      "#{name}-#{source_hash}"
+    def base_path_for(cache_key)
+      File.join(@path, cache_key)
     end
   end
 end

data/lib/inspec/errors.rb

@@ -8,4 +8,5 @@ module Inspec
   # dependency resolution
   class CyclicDependencyError < Error; end
   class UnsatisfiedVersionSpecification < Error; end
+  class DuplicateDep < Error; end
 end

data/lib/inspec/fetcher.rb

@@ -17,6 +17,5 @@ module Inspec
 end
 
 require 'fetchers/local'
-require 'fetchers/zip'
-require 'fetchers/tar'
 require 'fetchers/url'
+require 'fetchers/git'