berkshelf 6.0.1 → 6.1.0

data/features/step_definitions/environment_steps.rb

@@ -7,5 +7,5 @@ Given /^the environment variable (.+) is "(.+)"$/ do |variable, value|
 end
 
 Given /^the environment variable (.+) is \$TEST_BERKSHELF_ARTIFACTORY_API_KEY$/ do |variable|
-  set_environment_variable(variable, ENV['TEST_BERKSHELF_ARTIFACTORY_API_KEY'])
+  set_environment_variable(variable, ENV["TEST_BERKSHELF_ARTIFACTORY_API_KEY"])
 end

data/features/step_definitions/json_steps.rb

@@ -16,8 +16,8 @@ end
 
 # Pending Ridley allowing newer Faraday and Celluloid.
 def clean_json_output(output)
-  output.gsub(/^.+warning: constant ::Fixnum is deprecated$/, '') \
-        .gsub(/^.*forwarding to private method Celluloid::PoolManager#url_prefix$/, '')
+  output.gsub(/^.+warning: constant ::Fixnum is deprecated$/, "") \
+        .gsub(/^.*forwarding to private method Celluloid::PoolManager#url_prefix$/, "")
 end
 
 Then /^the output should contain JSON:$/ do |data|

data/features/support/env.rb

@@ -12,7 +12,6 @@ require "berkshelf/api/cucumber" unless windows?
 Dir["spec/support/**/*.rb"].each { |f| require File.expand_path(f) }
 
 World(Berkshelf::RSpec::PathHelpers)
-World(Berkshelf::RSpec::Kitchen)
 
 CHEF_SERVER_PORT = 26310
 BERKS_API_PORT   = 26210
@@ -34,7 +33,6 @@ Before do
   aruba.config.main_class = Berkshelf::Cli::Runner
   @dirs = ["spec/tmp/aruba"] # set aruba's temporary directory
 
-  stub_kitchen!
   clean_tmp_path
   Berkshelf.initialize_filesystem
   Berkshelf::CookbookStore.instance.initialize_filesystem

data/lib/berkshelf/berksfile.rb

@@ -198,7 +198,7 @@ module Berkshelf
     #
     # @return [Array<Source>]
     def source(api_url, **options)
-      source = Source.new(api_url, **options)
+      source = Source.new(self, api_url, **options)
       @sources[source.uri.to_s] = source
     end
     expose :source

data/lib/berkshelf/chef_repo_universe.rb

@@ -0,0 +1,45 @@
+require "berkshelf/api_client/remote_cookbook"
+require "ridley/chef/cookbook"
+
+module Berkshelf
+  # Shim to look like a Berkshelf::APIClient but for a chef repo folder.
+  #
+  # @since 6.1
+  class ChefRepoUniverse
+    def initialize(uri, **options)
+      @uri = uri
+      @path = options[:path]
+      @options = options
+    end
+
+    def universe
+      Dir.entries(cookbooks_path).sort.each_with_object([]) do |entry, cookbooks|
+        next if entry[0] == "." # Skip hidden folders.
+        entry_path = "#{cookbooks_path}/#{entry}"
+        next unless File.directory?(entry_path) # Skip non-dirs.
+        cookbook = begin
+          Ridley::Chef::Cookbook.from_path(entry_path)
+        rescue IOError
+          next # It wasn't a cookbook.
+        end
+        cookbooks << Berkshelf::APIClient::RemoteCookbook.new(
+          cookbook.cookbook_name,
+          cookbook.version,
+          location_type: "file_store",
+          location_path: entry_path,
+          dependencies: cookbook.metadata.dependencies
+        )
+      end
+    end
+
+    private
+
+    def cookbooks_path
+      if File.exist?("#{@path}/cookbooks")
+        "#{@path}/cookbooks"
+      else
+        @path
+      end
+    end
+  end
+end

data/lib/berkshelf/cli.rb

@@ -3,7 +3,6 @@ require_relative "config"
 require_relative "init_generator"
 require_relative "cookbook_generator"
 require_relative "commands/shelf"
-require_relative "commands/test_command"
 
 module Berkshelf
   class Cli < Thor
@@ -262,7 +261,7 @@ module Berkshelf
       banner: "URL"
     desc "search NAME", "Search the remote source for cookbooks matching the partial name"
     def search(name)
-      source = Source.new(options[:source])
+      source = Source.new(nil, options[:source])
       cookbooks = source.search(name)
       Berkshelf.formatter.search(cookbooks)
     end

data/lib/berkshelf/community_rest.rb

@@ -1,4 +1,4 @@
-require "open-uri"
+require "berkshelf/streaming_file_adapter"
 require "retryable"
 require "mixlib/archive"
 
@@ -80,7 +80,7 @@ module Berkshelf
           interval: @retry_interval,
           exceptions: [Faraday::Error::TimeoutError]
 
-        b.adapter :httpclient
+        b.use Berkshelf::StreamingFileAdapter
       end
 
       super(api_uri, options)
@@ -116,7 +116,7 @@ module Berkshelf
       response = get("cookbooks/#{name}/versions/#{self.class.uri_escape_version(version)}")
 
       # Artifactory responds with a 200 and blank body for unknown cookbooks.
-      if response.status == 200 && response.body.to_s == ''
+      if response.status == 200 && response.body.to_s == ""
         response.env.status = 404
       end
 
@@ -137,7 +137,7 @@ module Berkshelf
       response = get("cookbooks/#{name}")
 
       # Artifactory responds with a 200 and blank body for unknown cookbooks.
-      if response.status == 200 && response.body.to_s == ''
+      if response.status == 200 && response.body.to_s == ""
         response.env.status = 404
       end
 
@@ -189,44 +189,13 @@ module Berkshelf
       local = Tempfile.new("community-rest-stream")
       local.binmode
 
-      Retryable.retryable(tries: retries, on: OpenURI::HTTPError, sleep: retry_interval) do
-        open(target, "rb", open_uri_options(target)) do |remote|
-          local.write(remote.read)
-        end
+      Retryable.retryable(tries: retries, on: Faraday::Error::ConnectionFailed, sleep: retry_interval) do
+        get(target, nil, streaming_file: local)
       end
 
       local
     ensure
       local.close(false) unless local.nil?
     end
-
-    private
-
-    def open_uri_options(target)
-      options = {}
-      options.merge!(headers)
-      options.merge!(open_uri_proxy_options(target))
-      options.merge!(ssl_verify_mode: ssl_verify_mode)
-    end
-
-    def open_uri_proxy_options(target)
-      proxy_uri = URI.parse(target).find_proxy()
-      return {} if proxy_uri.nil?
-
-      proxy = Faraday::ProxyOptions.from(proxy_uri)
-      if proxy && proxy[:user] && proxy[:password]
-        { proxy_http_basic_authentication: [ proxy[:uri], proxy[:user], proxy[:password] ] }
-      else
-        { proxy: proxy[:uri] }
-      end
-    end
-
-    def ssl_verify_mode
-      if Berkshelf::Config.instance.ssl.verify.nil? || Berkshelf::Config.instance.ssl.verify
-        OpenSSL::SSL::VERIFY_PEER
-      else
-        OpenSSL::SSL::VERIFY_NONE
-      end
-    end
   end
 end

data/lib/berkshelf/config.rb

@@ -110,6 +110,9 @@ module Berkshelf
     attribute "chef.trusted_certs_dir",
       type: String,
       default: Berkshelf.chef_config.trusted_certs_dir
+    attribute "chef.artifactory_api_key",
+      type: String,
+      default: Berkshelf.chef_config.artifactory_api_key
     attribute "cookbook.copyright",
       type: String,
       default: Berkshelf.chef_config.cookbook_copyright

data/lib/berkshelf/downloader.rb

@@ -63,9 +63,9 @@ module Berkshelf
 
       case remote_cookbook.location_type
       when :opscode, :supermarket
-        options = {}
+        options = { ssl: source.options[:ssl] }
         if source.type == :artifactory
-          options[:headers] = {'X-Jfrog-Art-Api' => source.options[:api_key]}
+          options[:headers] = { "X-Jfrog-Art-Api" => source.options[:api_key] }
         end
         CommunityREST.new(remote_cookbook.location_path, options).download(name, version)
       when :chef_server

data/lib/berkshelf/formatters/human.rb

@@ -19,7 +19,9 @@ module Berkshelf
     def install(source, cookbook)
       message = "Installing #{cookbook.name} (#{cookbook.version})"
 
-      unless source.default?
+      if source.type == :chef_repo
+        message << " from #{cookbook.location_path}"
+      elsif !source.default?
         message << " from #{source}"
         message << " ([#{cookbook.location_type}] #{cookbook.location_path})"
       end

data/lib/berkshelf/installer.rb

@@ -1,4 +1,6 @@
 require "berkshelf/api-client"
+require "concurrent/executors"
+require "concurrent/future"
 
 module Berkshelf
   class Installer
@@ -10,14 +12,15 @@ module Berkshelf
     def initialize(berksfile)
       @berksfile  = berksfile
       @lockfile   = berksfile.lockfile
-      @worker     = Worker.pool(size: [(Celluloid.cores.to_i - 1), 2].max, args: [berksfile])
+      @pool       = Concurrent::FixedThreadPool.new([Concurrent.processor_count - 1, 2].max)
+      @worker     = Worker.new(berksfile)
     end
 
     def build_universe
       berksfile.sources.collect do |source|
         Thread.new do
           begin
-            Berkshelf.formatter.msg("Fetching cookbook index from #{source.uri}...")
+            Berkshelf.formatter.msg("Fetching cookbook index from #{source}...")
             source.build_universe
           rescue Berkshelf::APIClientError => ex
             Berkshelf.formatter.warn "Error retrieving universe from source: #{source}"
@@ -61,10 +64,9 @@ module Berkshelf
     private
 
     attr_reader :worker
+    attr_reader :pool
 
     class Worker
-      include Celluloid
-
       attr_reader :berksfile
       attr_reader :downloader
 
@@ -132,7 +134,10 @@ module Berkshelf
         build_universe
       end
 
-      cookbooks = dependencies.sort.map { |dependency| worker.future.install(dependency) }.map(&:value)
+      futures = dependencies.sort.map { |dependency| Concurrent::Future.execute(executor: pool) { worker.install(dependency) } }
+      cookbooks = futures.map(&:value)
+      rejects = futures.select(&:rejected?)
+      raise rejects.first.reason unless rejects.empty?
 
       [dependencies, cookbooks]
     end
@@ -173,7 +178,10 @@ module Berkshelf
 
       Berkshelf.log.debug "  Starting resolution..."
 
-      cookbooks = resolver.resolve.sort.map { |dependency| worker.future.install(dependency) }.map(&:value)
+      futures = resolver.resolve.sort.map { |dependency| Concurrent::Future.execute(executor: pool) { worker.install(dependency) } }
+      cookbooks = futures.map(&:value)
+      rejects = futures.select(&:rejected?)
+      raise rejects.first.reason unless rejects.empty?
 
       [dependencies, cookbooks]
     end

data/lib/berkshelf/locations/git.rb

@@ -1,5 +1,3 @@
-require "buff/shell_out"
-
 module Berkshelf
   class GitLocation < BaseLocation
     include Mixin::Git

data/lib/berkshelf/mixin/git.rb

@@ -1,8 +1,9 @@
-require "buff/shell_out"
+require "berkshelf/shell_out"
 
 module Berkshelf
   module Mixin
     module Git
+      include Berkshelf::ShellOut
       # Perform a git command.
       #
       # @param [String] command
@@ -17,9 +18,9 @@ module Berkshelf
           raise GitNotInstalled.new
         end
 
-        response = Buff::ShellOut.shell_out(%{git #{command}})
+        response = shell_out(%{git #{command}})
 
-        if error && !response.success?
+        if response.error?
           raise GitCommandError.new(command, cache_path, response.stderr)
         end
 

data/lib/berkshelf/shell_out.rb

@@ -0,0 +1,17 @@
+require "mixlib/shellout"
+
+module Berkshelf
+  module ShellOut
+    def shell_out(*args, **options)
+      cmd = Mixlib::ShellOut.new(*args, **options)
+      cmd.run_command
+      cmd
+    end
+
+    def shell_out!(*args)
+      cmd = shell_out(*args)
+      cmd.error!
+      cmd
+    end
+  end
+end

data/lib/berkshelf/source.rb

@@ -1,4 +1,5 @@
 require "berkshelf/api-client"
+require "berkshelf/chef_repo_universe"
 require "berkshelf/ssl_policies"
 require "openssl"
 
@@ -10,9 +11,10 @@ module Berkshelf
     attr_accessor :uri_string
     attr_accessor :options
 
+    # @param [Berkshelf::Berksfile] berksfile
     # @param [String, Berkshelf::SourceURI] source
-    def initialize(source, **options)
-      @options = {timeout: api_timeout, open_timeout: [(api_timeout / 10), 3].max, ssl: {}}
+    def initialize(berksfile, source, **options)
+      @options = { timeout: api_timeout, open_timeout: [(api_timeout / 10), 3].max, ssl: {} }
       @options.update(options)
       case source
       when String
@@ -35,7 +37,13 @@ module Berkshelf
         @options[:client_name] ||= Berkshelf::Config.instance.chef.node_name
         @options[:client_key] ||= Berkshelf::Config.instance.chef.client_key
       when :artifactory
-        @options[:api_key] ||= Berkshelf::Config.instance.chef.artifactory_api_key || ENV['ARTIFACTORY_API_KEY']
+        @options[:api_key] ||= Berkshelf::Config.instance.chef.artifactory_api_key || ENV["ARTIFACTORY_API_KEY"]
+      when :chef_repo
+        @options[:path] = uri_string
+        # If given a relative path, expand it against the Berksfile's folder.
+        @options[:path] = File.expand_path(@options[:path], File.dirname(berksfile ? berksfile.filepath : Dir.pwd))
+        # Lie because this won't actually parse as a URI.
+        @uri_string = "file://#{@options[:path]}"
       end
       # Set some default SSL options.
       Berkshelf::Config.instance.ssl.each do |key, value|
@@ -50,17 +58,19 @@ module Berkshelf
     end
 
     def api_client
-       @api_client ||= case type
-                       when :chef_server
-                         APIClient.chef_server(server_url: uri.to_s, **options)
-                       when :artifactory
-                         # Don't accidentally mutate the options.
-                         client_options = options.dup
-                         api_key = client_options.delete(:api_key)
-                         APIClient.new(uri, headers: {'X-Jfrog-Art-Api' => api_key}, **client_options)
-                       else
-                         APIClient.new(uri, **options)
-                       end
+      @api_client ||= case type
+                      when :chef_server
+                        APIClient.chef_server(server_url: uri.to_s, **options)
+                      when :artifactory
+                        # Don't accidentally mutate the options.
+                        client_options = options.dup
+                        api_key = client_options.delete(:api_key)
+                        APIClient.new(uri, headers: { "X-Jfrog-Art-Api" => api_key }, **client_options)
+                      when :chef_repo
+                        ChefRepoUniverse.new(uri_string, **options)
+                      else
+                        APIClient.new(uri, **options)
+                      end
     end
 
     def uri
@@ -133,15 +143,18 @@ module Berkshelf
     end
 
     def to_s
-      if type == :supermarket
-        "#{uri}"
+      case type
+      when :supermarket
+        uri.to_s
+      when :chef_repo
+        options[:path]
       else
         "#{type}: #{uri}"
       end
     end
 
     def inspect
-      "#<#{self.class.name} #{type}: #{uri.to_s.inspect}, #{options.map {|k, v| "#{k}: #{v.inspect}" }.join(', ')}>"
+      "#<#{self.class.name} #{type}: #{uri.to_s.inspect}, #{options.map { |k, v| "#{k}: #{v.inspect}" }.join(', ')}>"
     end
 
     def hash

data/lib/berkshelf/source_uri.rb

@@ -22,7 +22,7 @@ module Berkshelf
       end
     end
 
-    VALID_SCHEMES = %w{http https}.freeze
+    VALID_SCHEMES = %w{http https file}.freeze
 
     # @raise [Berkshelf::InvalidSourceURI]
     def validate

data/lib/berkshelf/ssl_policies.rb

@@ -22,7 +22,7 @@ module Berkshelf
     end
 
     def trusted_certs_dir
-      config_dir = Berkshelf.config.chef.trusted_certs_dir.to_s.tr('\\', '/')
+      config_dir = Berkshelf.config.chef.trusted_certs_dir.to_s.tr('\\', "/")
       if config_dir.empty? || !::File.exist?(config_dir)
         File.join(ENV["HOME"], ".chef", "trusted_certs")
       else

data/lib/berkshelf/streaming_file_adapter.rb

@@ -0,0 +1,22 @@
+require "faraday/adapter/net_http"
+
+module Berkshelf
+  class StreamingFileAdapter < Faraday::Adapter::NetHttp
+    def call(env)
+      env[:streaming_file] = env[:request_headers].delete(:streaming_file) if env[:request_headers] && env[:request_headers][:streaming_file]
+      super
+    end
+
+    def perform_request(http, env)
+      if env[:streaming_file]
+        http.request(create_request(env)) do |response|
+          response.read_body do |chunk|
+            env[:streaming_file].write(chunk) if response.code == "200"
+          end
+        end
+      else
+        super
+      end
+    end
+  end
+end