chef 11.14.0.alpha.2-x86-mingw32 → 11.14.0.alpha.3-x86-mingw32

data/lib/chef/config.rb

@@ -142,7 +142,7 @@ class Chef
           end
         end
       else
-        platform_specific_path("/var/chef")
+        cache_path
       end
     end
 
@@ -240,10 +240,28 @@ class Chef
       if local_mode
         "#{config_dir}local-mode-cache"
       else
-        platform_specific_path("/var/chef")
+        primary_cache_root = platform_specific_path("/var")
+        primary_cache_path = platform_specific_path("/var/chef")
+        # Use /var/chef as the cache path only if that folder exists and we can read and write
+        # into it, or /var exists and we can read and write into it (we'll create /var/chef later).
+        # Otherwise, we'll create .chef under the user's home directory and use that as
+        # the cache path.
+        unless path_accessible?(primary_cache_path) || path_accessible?(primary_cache_root)
+          secondary_cache_path = File.join(user_home, '.chef')
+          secondary_cache_path.gsub!(File::SEPARATOR, platform_path_separator) # Safety, mainly for Windows...
+          Chef::Log.info("Unable to access cache at #{primary_cache_path}. Switching cache to #{secondary_cache_path}")
+          secondary_cache_path
+        else
+          primary_cache_path
+        end
       end
     end
 
+    # Returns true only if the path exists and is readable and writeable for the user.
+    def self.path_accessible?(path)
+      File.exists?(path) && File.readable?(path) && File.writable?(path)
+    end
+
     # Where cookbook files are stored on the server (by content checksum)
     default(:checksum_path) { path_join(cache_path, "checksums") }
 
@@ -307,6 +325,7 @@ class Chef
       config_strict_mode true
       default(:enabled) { Chef::Config.local_mode }
       default :port, 8889
+      default :host, 'localhost'
     end
     default :chef_server_url,   "https://localhost:443"
 
@@ -552,6 +571,21 @@ class Chef
     # immediately if 0.)
     default :run_lock_timeout, nil
 
+    # Number of worker threads for syncing cookbooks in parallel. Increasing
+    # this number can result in gateway errors from the server (namely 503 and 504).
+    # If you are seeing this behavior while using the default setting, reducing
+    # the number of threads will help.
+    default :cookbook_sync_threads, 10
+
+    # A whitelisted array of attributes you want sent over the wire when node
+    # data is saved.
+    # The default setting is nil, which collects all data. Setting to [] will not
+    # collect any data for save.
+    default :automatic_attribute_whitelist, nil
+    default :default_attribute_whitelist, nil
+    default :normal_attribute_whitelist, nil
+    default :override_attribute_whitelist, nil
+
     # If installed via an omnibus installer, this gives the path to the
     # "embedded" directory which contains all of the software packaged with
     # omnibus. This is used to locate the cacert.pem file on windows.

data/lib/chef/cookbook/cookbook_version_loader.rb

@@ -1,5 +1,4 @@
 
-require 'chef/config'
 require 'chef/cookbook_version'
 require 'chef/cookbook/chefignore'
 require 'chef/cookbook/metadata'

data/lib/chef/cookbook/synchronizer.rb

@@ -1,4 +1,5 @@
 require 'chef/client'
+require 'chef/util/threaded_job_queue'
 require 'singleton'
 
 class Chef
@@ -56,6 +57,8 @@ class Chef
   # Synchronizes the locally cached copies of cookbooks with the files on the
   # server.
   class CookbookSynchronizer
+    CookbookFile = Struct.new(:cookbook, :segment, :manifest_record)
+
     def initialize(cookbooks_by_name, events)
       @eager_segments = Chef::CookbookVersion::COOKBOOK_SEGMENTS.dup
       unless Chef::Config[:no_lazy_load]
@@ -87,6 +90,38 @@ class Chef
       @cookbooks_by_name.key?(cookbook_name)
     end
 
+    def files
+      @files ||= cookbooks.inject([]) do |memo, cookbook|
+        @eager_segments.each do |segment|
+          cookbook.manifest[segment].each do |manifest_record|
+            memo << CookbookFile.new(cookbook, segment, manifest_record)
+          end
+        end
+        memo
+      end
+    end
+
+    def files_by_cookbook
+      files.group_by { |file| file.cookbook }
+    end
+
+    def files_remaining_by_cookbook
+      @files_remaining_by_cookbook ||= begin
+        files_by_cookbook.inject({}) do |memo, (cookbook, files)|
+          memo[cookbook] = files.size
+          memo
+        end
+      end
+    end
+
+    def mark_file_synced(file)
+      files_remaining_by_cookbook[file.cookbook] -= 1
+
+      if files_remaining_by_cookbook[file.cookbook] == 0
+        @events.synchronized_cookbook(file.cookbook.name)
+      end
+    end
+
     # Synchronizes all the cookbooks from the chef-server.
     #)
     # === Returns
@@ -97,14 +132,19 @@ class Chef
 
       clear_obsoleted_cookbooks
 
-      @events.cookbook_sync_start(cookbook_count)
+      queue = Chef::Util::ThreadedJobQueue.new
 
-      # Synchronize each of the node's cookbooks, and add to the
-      # valid_cache_entries hash.
-      cookbooks.each do |cookbook|
-        sync_cookbook(cookbook)
+      files.each do |file|
+        queue << lambda do |lock|
+          sync_file(file)
+          lock.synchronize { mark_file_synced(file) }
+        end
       end
 
+      @events.cookbook_sync_start(cookbook_count)
+      queue.process(Chef::Config[:cookbook_sync_threads])
+      update_cookbook_filenames
+
     rescue Exception => e
       @events.cookbook_sync_failed(cookbooks, e)
       raise
@@ -129,61 +169,43 @@ class Chef
       @events.cookbook_clean_complete
     end
 
-    # Sync the eagerly loaded files contained by +cookbook+
-    #
-    # === Arguments
-    # cookbook<Chef::Cookbook>:: The cookbook to update
-    # valid_cache_entries<Hash>:: Out-param; Added to this hash are the files that
-    # were referred to by this cookbook
-    def sync_cookbook(cookbook)
-      Chef::Log.debug("Synchronizing cookbook #{cookbook.name} #{cookbook.version}")
-
-      # files and templates are lazily loaded, and will be done later.
-
-      @eager_segments.each do |segment|
-        segment_filenames = Array.new
-        cookbook.manifest[segment].each do |manifest_record|
-
-          cache_filename = sync_file_in_cookbook(cookbook, manifest_record)
-          # make the segment filenames a full path.
-          full_path_cache_filename = cache.load(cache_filename, false)
-          segment_filenames << full_path_cache_filename
-        end
+    def update_cookbook_filenames
+      files_by_cookbook.each do |cookbook, cookbook_files|
+        files_by_segment = cookbook_files.group_by { |file| file.segment }
+        @eager_segments.each do |segment|
+          segment_files = files_by_segment[segment]
+          next unless segment_files
 
-        # replace segment filenames with a full-path one.
-        if segment.to_sym == :recipes
-          cookbook.recipe_filenames = segment_filenames
-        elsif segment.to_sym == :attributes
-          cookbook.attribute_filenames = segment_filenames
-        else
-          cookbook.segment_filenames(segment).replace(segment_filenames)
+          filenames = segment_files.map { |file| file.manifest_record['path'] }
+          cookbook.replace_segment_filenames(segment, filenames)
         end
       end
-      @events.synchronized_cookbook(cookbook.name)
     end
 
     # Sync an individual file if needed. If there is an up to date copy
-    # locally, nothing is done.
+    # locally, nothing is done. Updates +file+'s manifest with the full path to
+    # the cached file.
     #
     # === Arguments
-    # file_manifest::: A Hash of the form {"path" => 'relative/path', "url" => "location to fetch the file"}
+    # file<CookbookFile>
     # === Returns
-    # Path to the cached file as a String
-    def sync_file_in_cookbook(cookbook, file_manifest)
-      cache_filename = File.join("cookbooks", cookbook.name, file_manifest['path'])
+    # Full path to the cached file as a String
+    def sync_file(file)
+      cache_filename = File.join("cookbooks", file.cookbook.name, file.manifest_record['path'])
       mark_cached_file_valid(cache_filename)
 
       # If the checksums are different between on-disk (current) and on-server
       # (remote, per manifest), do the update. This will also execute if there
       # is no current checksum.
-      if !cached_copy_up_to_date?(cache_filename, file_manifest['checksum'])
-        download_file(file_manifest['url'], cache_filename)
-        @events.updated_cookbook_file(cookbook.name, cache_filename)
+      if !cached_copy_up_to_date?(cache_filename, file.manifest_record['checksum'])
+        download_file(file.manifest_record['url'], cache_filename)
+        @events.updated_cookbook_file(file.cookbook.name, cache_filename)
       else
         Chef::Log.debug("Not storing #{cache_filename}, as the cache is up to date.")
       end
 
-      cache_filename
+      # Update the manifest with the full path to the cached file
+      file.manifest_record['path'] = cache.load(cache_filename, false)
     end
 
     def cached_copy_up_to_date?(local_path, expected_checksum)

data/lib/chef/cookbook_uploader.rb

@@ -7,29 +7,12 @@ require 'chef/digester'
 require 'chef/cookbook_version'
 require 'chef/cookbook/syntax_check'
 require 'chef/cookbook/file_system_file_vendor'
+require 'chef/util/threaded_job_queue'
 require 'chef/sandbox'
-require 'thread'
 
 class Chef
   class CookbookUploader
 
-    def self.work_queue
-      @work_queue ||= Queue.new
-    end
-
-    def self.setup_worker_threads(concurrency=10)
-      @worker_threads ||= begin
-        work_queue
-        (1..concurrency).map do
-          Thread.new do
-            loop do
-              work_queue.pop.call
-            end
-          end
-        end
-      end
-    end
-
     attr_reader :cookbooks
     attr_reader :path
     attr_reader :opts
@@ -61,8 +44,6 @@ class Chef
     end
 
     def upload_cookbooks
-      Thread.abort_on_exception = true
-
       # Syntax Check
       validate_cookbooks
       # generate checksums of cookbook files and create a sandbox
@@ -77,7 +58,7 @@ class Chef
 
       Chef::Log.info("Uploading files")
 
-      self.class.setup_worker_threads(concurrency)
+      queue = Chef::Util::ThreadedJobQueue.new
 
       checksums_to_upload = Set.new
 
@@ -86,15 +67,13 @@ class Chef
         if info['needs_upload'] == true
           checksums_to_upload << checksum
           Chef::Log.info("Uploading #{checksum_files[checksum]} (checksum hex = #{checksum}) to #{info['url']}")
-          self.class.work_queue << uploader_function_for(checksum_files[checksum], checksum, info['url'], checksums_to_upload)
+          queue << uploader_function_for(checksum_files[checksum], checksum, info['url'], checksums_to_upload)
         else
           Chef::Log.debug("#{checksum_files[checksum]} has not changed")
         end
       end
 
-      until checksums_to_upload.empty?
-        sleep 0.1
-      end
+      queue.process(@concurrency)
 
       sandbox_url = new_sandbox['uri']
       Chef::Log.debug("Committing sandbox")

data/lib/chef/cookbook_version.rb

@@ -20,9 +20,6 @@
 # limitations under the License.
 
 require 'chef/log'
-require 'chef/node'
-require 'chef/resource_definition_list'
-require 'chef/recipe'
 require 'chef/cookbook/file_vendor'
 require 'chef/cookbook/metadata'
 require 'chef/version_class'
@@ -169,14 +166,7 @@ class Chef
         next unless @manifest.has_key?(segment)
         filenames = @manifest[segment].map{|manifest_record| manifest_record['name']}
 
-        if segment == :recipes
-          self.recipe_filenames = filenames
-        elsif segment == :attributes
-          self.attribute_filenames = filenames
-        else
-          segment_filenames(segment).clear
-          filenames.each { |filename| segment_filenames(segment) << filename }
-        end
+        replace_segment_filenames(segment, filenames)
       end
     end
 
@@ -272,6 +262,17 @@ class Chef
       end
     end
 
+    def replace_segment_filenames(segment, filenames)
+      case segment.to_sym
+      when :recipes
+        self.recipe_filenames = filenames
+      when :attributes
+        self.attribute_filenames = filenames
+      else
+        segment_filenames(segment).replace(filenames)
+      end
+    end
+
     # Query whether a template file +template_filename+ is available. File
     # specificity for the given +node+ is obeyed in the lookup.
     def has_template_for_node?(node, template_filename)

data/lib/chef/formatters/error_inspectors/api_error_formatting.rb

@@ -88,7 +88,7 @@ E
       def format_rest_error
         Array(Chef::JSONCompat.from_json(exception.response.body)["error"]).join('; ')
       rescue Exception
-        exception.response.body
+        safe_format_rest_error
       end
 
       def username
@@ -107,6 +107,23 @@ E
         exception.response.body =~ /synchronize the clock/i
       end
 
+      def safe_format_rest_error
+        # When we get 504 from the server, sometimes the response body is non-readable.
+        #
+        # Stack trace:
+        #
+        # NoMethodError: undefined method `closed?' for nil:NilClass
+        # .../lib/ruby/1.9.1/net/http.rb:2789:in `stream_check'
+        # .../lib/ruby/1.9.1/net/http.rb:2709:in `read_body'
+        # .../lib/ruby/1.9.1/net/http.rb:2736:in `body'
+        # .../lib/chef/formatters/error_inspectors/api_error_formatting.rb:91:in `rescue in format_rest_error'
+        begin
+          exception.response.body
+        rescue Exception
+          "Cannot fetch the contents of the response."
+        end
+      end
+
     end
   end
 end

data/lib/chef/formatters/error_inspectors/cookbook_sync_error_inspector.rb

@@ -65,7 +65,7 @@ class Chef
           when Net::HTTPNotFound
           when Net::HTTPInternalServerError
             describe_500_error(error_description)
-          when Net::HTTPBadGateway, Net::HTTPServiceUnavailable
+          when Net::HTTPBadGateway, Net::HTTPServiceUnavailable, Net::HTTPGatewayTimeOut
             describe_503_error(error_description)
           else
             describe_http_error(error_description)
@@ -76,5 +76,3 @@ class Chef
     end
   end
 end
-
-

data/lib/chef/knife/bootstrap.rb

@@ -154,6 +154,26 @@ class Chef
         :description => "A file containing the secret key to use to encrypt data bag item values",
         :proc => Proc.new { |sf| Chef::Config[:knife][:secret_file] = sf }
 
+      option :bootstrap_url,
+        :long        => "--bootstrap-url URL",
+        :description => "URL to a custom installation script",
+        :proc        => Proc.new { |u| Chef::Config[:knife][:bootstrap_url] = u }
+
+      option :bootstrap_install_command,
+        :long        => "--bootstrap-install-command COMMANDS",
+        :description => "Custom command to install chef-client",
+        :proc        => Proc.new { |ic| Chef::Config[:knife][:bootstrap_install_command] = ic }
+
+      option :bootstrap_wget_options,
+        :long        => "--bootstrap-wget-options OPTIONS",
+        :description => "Add options to wget when installing chef-client",
+        :proc        => Proc.new { |wo| Chef::Config[:knife][:bootstrap_wget_options] = wo }
+
+      option :bootstrap_curl_options,
+        :long        => "--bootstrap-curl-options OPTIONS",
+        :description => "Add options to curl when install chef-client",
+        :proc        => Proc.new { |co| Chef::Config[:knife][:bootstrap_curl_options] = co }
+
       def find_template(template=nil)
         # Are we bootstrapping using an already shipped template?
         if config[:template_file]
@@ -206,7 +226,9 @@ class Chef
         begin
           knife_ssh.run
         rescue Net::SSH::AuthenticationFailed
-          unless config[:ssh_password]
+          if config[:ssh_password]
+            raise
+          else
             ui.info("Failed to authenticate #{config[:ssh_user]} - trying password auth")
             knife_ssh_with_password_auth.run
           end

data/lib/chef/knife/bootstrap/chef-aix.erb

@@ -0,0 +1,58 @@
+ksh -c '
+
+function exists {
+  if type $1 >/dev/null 2>&1
+  then
+    return 0
+  else
+    return 1
+  fi
+}
+
+if ! exists /usr/bin/chef-client; then
+  <% if @chef_config[:aix_package] -%>
+    # Read the download URL/location from knife.rb with option aix_package
+    rm -rf /tmp/chef_installer # ensure there no older pkg
+    echo "<%= @chef_config[:aix_package] %>"
+    perl -e '\''use LWP::Simple; getprint($ARGV[0]);'\'' <%= @chef_config[:aix_package] %> > /tmp/chef_installer
+    installp -aYF -d  /tmp/chef_installer chef
+  <% else -%>
+     echo ":aix_package location is not set in knife.rb"
+     exit
+  <% end -%>
+fi
+
+mkdir -p /etc/chef
+
+cat > /etc/chef/validation.pem <<'EOP'
+<%= validation_key %>
+EOP
+chmod 0600 /etc/chef/validation.pem
+
+<% if encrypted_data_bag_secret -%>
+cat > /etc/chef/encrypted_data_bag_secret <<'EOP'
+<%= encrypted_data_bag_secret %>
+EOP
+chmod 0600 /etc/chef/encrypted_data_bag_secret
+<% end -%>
+
+<%# Generate Ohai Hints -%>
+<% unless @chef_config[:knife][:hints].nil? || @chef_config[:knife][:hints].empty? -%>
+mkdir -p /etc/chef/ohai/hints
+
+<% @chef_config[:knife][:hints].each do |name, hash| -%>
+cat > /etc/chef/ohai/hints/<%= name %>.json <<'EOP'
+<%= hash.to_json %>
+EOP
+<% end -%>
+<% end -%>
+
+cat > /etc/chef/client.rb <<'EOP'
+<%= config_content %>
+EOP
+
+cat > /etc/chef/first-boot.json <<'EOP'
+<%= first_boot.to_json %>
+EOP
+
+<%= start_chef %>'