chef 12.13.37 → 12.14.60

data/lib/chef/chef_fs/parallelizer/parallel_enumerable.rb

@@ -184,9 +184,7 @@ class Chef
                 sleep(0.01)
               end
 
-              until @unconsumed_output.empty?
-                yield @unconsumed_output.pop
-              end
+              yield @unconsumed_output.pop until @unconsumed_output.empty?
 
               # If no one is working on our tasks and we're allowed to
               # work on them in the main thread, process an input to
@@ -227,9 +225,7 @@ class Chef
 
         def stop
           @unconsumed_input.clear
-          while @in_process.size > 0
-            sleep(0.05)
-          end
+          sleep(0.05) while @in_process.size > 0
           @unconsumed_output.clear
         end
 

data/lib/chef/cookbook/cookbook_version_loader.rb

@@ -3,6 +3,7 @@ require "chef/cookbook_version"
 require "chef/cookbook/chefignore"
 require "chef/cookbook/metadata"
 require "chef/util/path_helper"
+require "find"
 
 class Chef
   class Cookbook
@@ -168,7 +169,7 @@ class Chef
           when /\.json$/
             apply_json_metadata(metadata_file)
           else
-            raise RuntimeError, "Invalid metadata file: #{metadata_file} for cookbook: #{cookbook_version}"
+            raise "Invalid metadata file: #{metadata_file} for cookbook: #{cookbook_version}"
           end
         end
 
@@ -223,27 +224,31 @@ class Chef
       # however if the file is named ".uploaded-cookbook-version.json" it is
       # assumed to be managed by chef-zero and not part of the cookbook.
       def load_all_files
-        Dir.glob(File.join(Chef::Util::PathHelper.escape_glob_dir(cookbook_path), "*"), File::FNM_DOTMATCH).each do |fs_entry|
-          if File.directory?(fs_entry)
-            dir_relpath = Chef::Util::PathHelper.relative_path_from(@cookbook_path, fs_entry)
-
-            next if dir_relpath.to_s.start_with?(".")
-
-            Dir.glob(File.join(fs_entry, "**/*"), File::FNM_DOTMATCH).each do |file|
-              next if File.directory?(file)
-              file = Pathname.new(file).cleanpath.to_s
-              name = Chef::Util::PathHelper.relative_path_from(@cookbook_path, file)
-              cookbook_settings[:all_files][name] = file
-            end
-          elsif File.file?(fs_entry)
-            file = Pathname.new(fs_entry).cleanpath.to_s
-
-            next if File.basename(file) == UPLOADED_COOKBOOK_VERSION_FILE
-
-            name = Chef::Util::PathHelper.relative_path_from(@cookbook_path, file)
-            cookbook_settings[:all_files][name] = file
-          else # pipes, devices, other weirdness
-            next
+        return unless File.exist?(cookbook_path)
+
+        # If cookbook_path is a symlink, Find on Windows Ruby 2.3 will not traverse it.
+        # Dir.entries will do so on all platforms, so we iterate the top level using
+        # Dir.entries. Since we have different behavior at the top anyway (hidden
+        # directories at the top level are not included for backcompat), this
+        # actually keeps things a bit cleaner.
+        Dir.entries(cookbook_path).each do |top_filename|
+          # Skip top-level directories starting with "."
+          top_path = File.join(cookbook_path, top_filename)
+          next if File.directory?(top_path) && top_filename.start_with?(".")
+
+          # Use Find.find because it:
+          # (a) returns any children, recursively
+          # (b) includes top_path as well
+          # (c) skips symlinks, which is backcompat (no judgement on whether it was *right*)
+          Find.find(top_path) do |path|
+            # Only add files, not directories
+            next unless File.file?(path)
+            # Don't add .uploaded-cookbook-version.json
+            next if File.basename(path) == UPLOADED_COOKBOOK_VERSION_FILE
+
+            relative_path = Chef::Util::PathHelper.relative_path_from(cookbook_path, path)
+            path = Pathname.new(path).cleanpath.to_s
+            cookbook_settings[:all_files][relative_path] = path
           end
         end
       end

data/lib/chef/cookbook/metadata.rb

@@ -722,7 +722,7 @@ class Chef
         end
       end
 
-    private
+      private
 
       # Helper to match a gem style version (ohai_version/chef_version) against a set of
       # Gem::Dependency version constraints.  If none are present, it always matches.  if

data/lib/chef/cookbook/synchronizer.rb

@@ -152,11 +152,11 @@ class Chef
         queue << lambda do |lock|
           full_file_path = sync_file(file)
 
-          lock.synchronize {
+          lock.synchronize do
             # Save the full_path of the downloaded file to be restored in the manifest later
             save_full_file_path(file, full_file_path)
             mark_file_synced(file)
-          }
+          end
         end
       end
 
@@ -291,7 +291,7 @@ class Chef
     end
 
     def server_api
-      Chef::ServerAPI.new(Chef::Config[:chef_server_url])
+      Thread.current[:server_api] ||= Chef::ServerAPI.new(Chef::Config[:chef_server_url], keepalives: true)
     end
 
   end

data/lib/chef/cookbook_site_streaming_uploader.rb

@@ -31,7 +31,7 @@ class Chef
   # inspired by http://stanislavvitvitskiy.blogspot.com/2008/12/multipart-post-in-ruby.html
   class CookbookSiteStreamingUploader
 
-    DefaultHeaders = { "accept" => "application/json", "x-chef-version" => ::Chef::VERSION }
+    DefaultHeaders = { "accept" => "application/json", "x-chef-version" => ::Chef::VERSION } # rubocop:disable Style/ConstantName
 
     class << self
 
@@ -149,11 +149,11 @@ class Chef
           alias :to_s :body
 
           # BUGBUG this makes the response compatible with what respsonse_steps expects to test headers (response.headers[] -> response[])
-          def headers
+          def headers # rubocop:disable Lint/NestedMethodDefinition
             self
           end
 
-          def status
+          def status  # rubocop:disable Lint/NestedMethodDefinition
             code.to_i
           end
         end

data/lib/chef/cookbook_uploader.rb

@@ -55,7 +55,7 @@ class Chef
         checksum_files.merge!(cb.checksums)
       end
 
-      checksums = checksum_files.inject({}) { |memo, elt| memo[elt.first] = nil ; memo }
+      checksums = checksum_files.inject({}) { |memo, elt| memo[elt.first] = nil; memo }
       new_sandbox = rest.post("sandboxes", { :checksums => checksums })
 
       Chef::Log.info("Uploading files")

data/lib/chef/cookbook_version.rb

@@ -316,13 +316,13 @@ class Chef
           error_message << error_locations.join("\n")
           existing_files = segment_filenames(segment)
           # Strip the root_dir prefix off all files for readability
-          pretty_existing_files = existing_files.map { |path|
+          pretty_existing_files = existing_files.map do |path|
             if root_dir
               path[root_dir.length + 1..-1]
             else
               path
             end
-          }
+          end
           # Show the files that the cookbook does have. If the user made a typo,
           # hopefully they'll see it here.
           unless pretty_existing_files.empty?
@@ -599,12 +599,12 @@ class Chef
       end
     end
 
-    def <=>(o)
-      raise Chef::Exceptions::CookbookVersionNameMismatch if self.name != o.name
+    def <=>(other)
+      raise Chef::Exceptions::CookbookVersionNameMismatch if self.name != other.name
       # FIXME: can we change the interface to the Metadata class such
       # that metadata.version returns a Chef::Version instance instead
       # of a string?
-      Chef::Version.new(self.version) <=> Chef::Version.new(o.version)
+      Chef::Version.new(self.version) <=> Chef::Version.new(other.version)
     end
 
     private
@@ -623,7 +623,7 @@ class Chef
     # For each filename, produce a mapping of base filename (i.e. recipe name
     # or attribute file) to on disk location
     def filenames_by_name(filenames)
-      filenames.select { |filename| filename =~ /\.rb$/ }.inject({}) { |memo, filename| memo[File.basename(filename, ".rb")] = filename ; memo }
+      filenames.select { |filename| filename =~ /\.rb$/ }.inject({}) { |memo, filename| memo[File.basename(filename, ".rb")] = filename; memo }
     end
 
     def file_vendor

data/lib/chef/dsl/powershell.rb

@@ -21,7 +21,7 @@ require "chef/util/powershell/ps_credential"
 class Chef
   module DSL
     module Powershell
-      def ps_credential(username = "placeholder", password)
+      def ps_credential(username = "placeholder", password) # rubocop:disable Style/OptionalArguments
         Chef::Util::Powershell::PSCredential.new(username, password)
       end
     end

data/lib/chef/event_dispatch/base.rb

@@ -232,11 +232,11 @@ class Chef
       end
 
       # Called after the recipe has been loaded
-      def recipe_file_loaded(path)
+      def recipe_file_loaded(path, recipe)
       end
 
       # Called after a recipe file fails to load
-      def recipe_file_load_failed(path, exception)
+      def recipe_file_load_failed(path, exception, recipe)
       end
 
       # Called when a recipe cannot be resolved

data/lib/chef/exceptions.rb

@@ -184,7 +184,7 @@ class Chef
 
     # A different version of a cookbook was added to a
     # VersionedRecipeList than the one already there.
-    class CookbookVersionConflict < ArgumentError ; end
+    class CookbookVersionConflict < ArgumentError; end
 
     # does not follow X.Y.Z format. ArgumentError?
     class InvalidPlatformVersion < ArgumentError; end

data/lib/chef/file_cache.rb

@@ -85,7 +85,7 @@ class Chef
             File.join(create_cache_path(File.join(file_path_array), true), file_name)
           )
         else
-          raise RuntimeError, "Cannot move #{file} to #{path}!"
+          raise "Cannot move #{file} to #{path}!"
         end
       end
 

data/lib/chef/formatters/base.rb

@@ -203,12 +203,12 @@ class Chef
       end
 
       # Delegates to #file_loaded
-      def recipe_file_loaded(path)
+      def recipe_file_loaded(path, recipe)
         file_loaded(path)
       end
 
       # Delegates to #file_load_failed
-      def recipe_file_load_failed(path, exception)
+      def recipe_file_load_failed(path, exception, recipe)
         file_load_failed(path, exception)
       end
 

data/lib/chef/formatters/doc.rb

@@ -32,9 +32,9 @@ class Chef
 
       def pretty_elapsed_time
         time = elapsed_time
-        if time < 60 then
+        if time < 60
           message = Time.at(time).utc.strftime("%S seconds")
-        elsif time < 3600 then
+        elsif time < 3600
           message = Time.at(time).utc.strftime("%M minutes %S seconds")
         else
           message = Time.at(time).utc.strftime("%H hours %M minutes %S seconds")

data/lib/chef/http.rb

@@ -77,6 +77,9 @@ class Chef
 
     attr_reader :middlewares
 
+    # [Boolean] if we're doing keepalives or not
+    attr_reader :keepalives
+
     # Create a HTTP client object. The supplied +url+ is used as the base for
     # all subsequent requests. For example, when initialized with a base url
     # http://localhost:4000, a call to +get+ with 'nodes' will make an
@@ -87,6 +90,7 @@ class Chef
       @sign_on_redirect = true
       @redirects_followed = 0
       @redirect_limit = 10
+      @keepalives = options[:keepalives] || false
       @options = options
 
       @middlewares = []
@@ -228,6 +232,33 @@ class Chef
 
     def http_client(base_url = nil)
       base_url ||= url
+      if keepalives && !base_url.nil?
+        # only reuse the http_client if we want keepalives and have a base_url
+        @http_client ||= {}
+        # the per-host per-port cache here gets peristent connections correct when
+        # redirecting to different servers
+        if base_url.is_a?(String) # sigh, this kind of abuse can't happen with strongly typed languages
+          @http_client[base_url] ||= build_http_client(base_url)
+        else
+          @http_client[base_url.host] ||= {}
+          @http_client[base_url.host][base_url.port] ||= build_http_client(base_url)
+        end
+      else
+        build_http_client(base_url)
+      end
+    end
+
+    # DEPRECATED: This is only kept around to provide access to cache control data in
+    # lib/chef/provider/remote_file/http.rb
+    # FIXME: Find a better API.
+    def last_response
+      @last_response
+    end
+
+    private
+
+    # @api private
+    def build_http_client(base_url)
       if chef_zero_uri?(base_url)
         # PERFORMANCE CRITICAL: *MUST* lazy require here otherwise we load up webrick
         # via chef-zero and that hits DNS (at *require* time) which may timeout,
@@ -239,12 +270,11 @@ class Chef
 
         SocketlessChefZeroClient.new(base_url)
       else
-        BasicClient.new(base_url, :ssl_policy => Chef::HTTP::APISSLPolicy)
+        BasicClient.new(base_url, ssl_policy: Chef::HTTP::APISSLPolicy, keepalives: keepalives)
       end
     end
 
-    protected
-
+    # @api private
     def create_url(path)
       return path if path.is_a?(URI)
       if path =~ /^(http|https|chefzero):\/\//i
@@ -259,6 +289,7 @@ class Chef
       end
     end
 
+    # @api private
     def apply_request_middleware(method, url, headers, data)
       middlewares.inject([method, url, headers, data]) do |req_data, middleware|
         Chef::Log.debug("Chef::HTTP calling #{middleware.class}#handle_request")
@@ -266,6 +297,7 @@ class Chef
       end
     end
 
+    # @api private
     def apply_response_middleware(response, rest_request, return_value)
       middlewares.reverse.inject([response, rest_request, return_value]) do |res_data, middleware|
         Chef::Log.debug("Chef::HTTP calling #{middleware.class}#handle_response")
@@ -273,6 +305,7 @@ class Chef
       end
     end
 
+    # @api private
     def apply_stream_complete_middleware(response, rest_request, return_value)
       middlewares.reverse.inject([response, rest_request, return_value]) do |res_data, middleware|
         Chef::Log.debug("Chef::HTTP calling #{middleware.class}#handle_stream_complete")
@@ -280,6 +313,7 @@ class Chef
       end
     end
 
+    # @api private
     def log_failed_request(response, return_value)
       return_value ||= {}
       error_message = "HTTP Request Returned #{response.code} #{response.message}: "
@@ -287,12 +321,14 @@ class Chef
       Chef::Log.info(error_message)
     end
 
+    # @api private
     def success_response?(response)
       response.kind_of?(Net::HTTPSuccess) || response.kind_of?(Net::HTTPRedirection)
     end
 
     # Runs a synchronous HTTP request, with no middleware applied (use #request
     # to have the middleware applied). The entire response will be loaded into memory.
+    # @api private
     def send_http_request(method, url, headers, body, &response_handler)
       headers = build_headers(method, url, headers, body)
 
@@ -328,6 +364,7 @@ class Chef
     # Wraps an HTTP request with retry logic.
     # === Arguments
     # url:: URL of the request, used for error messages
+    # @api private
     def retrying_http_errors(url)
       http_attempts = 0
       begin
@@ -377,18 +414,22 @@ class Chef
       end
     end
 
+    # @api private
     def http_retry_delay
       config[:http_retry_delay]
     end
 
+    # @api private
     def http_retry_count
       config[:http_retry_count]
     end
 
+    # @api private
     def config
       Chef::Config
     end
 
+    # @api private
     def follow_redirect
       raise Chef::Exceptions::RedirectLimitExceeded if @redirects_followed >= redirect_limit
       @redirects_followed += 1
@@ -399,13 +440,13 @@ class Chef
       @redirects_followed = 0
     end
 
-    private
-
+    # @api private
     def chef_zero_uri?(uri)
       uri = URI.parse(uri) unless uri.respond_to?(:scheme)
       uri.scheme == "chefzero"
     end
 
+    # @api private
     def redirected_to(response)
       return nil  unless response.kind_of?(Net::HTTPRedirection)
       # Net::HTTPNotModified is undesired subclass of Net::HTTPRedirection so test for this
@@ -413,6 +454,7 @@ class Chef
       response["location"]
     end
 
+    # @api private
     def build_headers(method, url, headers = {}, json_body = false)
       headers = @default_headers.merge(headers)
       headers["Content-Length"] = json_body.bytesize.to_s if json_body
@@ -420,6 +462,7 @@ class Chef
       headers
     end
 
+    # @api private
     def stream_to_tempfile(url, response, &progress_block)
       content_length = response["Content-Length"]
       tf = Tempfile.open("chef-rest")
@@ -443,18 +486,5 @@ class Chef
       raise
     end
 
-    public
-
-    ############################################################################
-    # DEPRECATED
-    ############################################################################
-
-    # This is only kept around to provide access to cache control data in
-    # lib/chef/provider/remote_file/http.rb
-    # Find a better API.
-    def last_response
-      @last_response
-    end
-
   end
 end