chef 11.18.12 → 12.0.0.alpha.0

data/lib/chef/chef_fs/file_system/chef_repository_file_system_entry.rb

@@ -19,7 +19,6 @@
 
 require 'chef/chef_fs/file_system/file_system_entry'
 require 'chef/chef_fs/file_system/not_found_error'
-require 'chef/json_compat'
 
 class Chef
   module ChefFS
@@ -42,7 +41,7 @@ class Chef
 
         def chef_object
           begin
-            return data_handler.chef_object(Chef::JSONCompat.parse(read, :create_additions => false))
+            return data_handler.chef_object(JSON.parse(read, :create_additions => false))
           rescue
             Chef::Log.error("Could not read #{path_for_printing} into a Chef object: #{$!}")
           end

data/lib/chef/chef_fs/file_system/cookbooks_dir.rb

@@ -106,7 +106,7 @@ class Chef
             cookbook_to_upload.freeze_version if options[:freeze]
 
             # Instantiate a new uploader based on the proxy loader
-            uploader = Chef::CookbookUploader.new(cookbook_to_upload, proxy_cookbook_path, :force => options[:force], :rest => root.chef_rest)
+            uploader = Chef::CookbookUploader.new(cookbook_to_upload, :force => options[:force], :rest => root.chef_rest)
 
             with_actual_cookbooks_dir(temp_cookbooks_path) do
               upload_cookbook!(uploader)
@@ -128,7 +128,7 @@ class Chef
         def upload_unversioned_cookbook(other, options)
           cookbook_to_upload = other.chef_object
           cookbook_to_upload.freeze_version if options[:freeze]
-          uploader = Chef::CookbookUploader.new(cookbook_to_upload, other.parent.file_path, :force => options[:force], :rest => root.chef_rest)
+          uploader = Chef::CookbookUploader.new(cookbook_to_upload, :force => options[:force], :rest => root.chef_rest)
 
           with_actual_cookbooks_dir(other.parent.file_path) do
             upload_cookbook!(uploader)

data/lib/chef/chef_fs/file_system/rest_list_dir.rb

@@ -19,7 +19,6 @@
 require 'chef/chef_fs/file_system/base_fs_dir'
 require 'chef/chef_fs/file_system/rest_list_entry'
 require 'chef/chef_fs/file_system/not_found_error'
-require 'chef/json_compat'
 
 class Chef
   module ChefFS
@@ -62,8 +61,8 @@ class Chef
 
         def create_child(name, file_contents)
           begin
-            object = Chef::JSONCompat.parse(file_contents, :create_additions => false)
-          rescue Chef::Exceptions::JSON::ParseError => e
+            object = JSON.parse(file_contents, :create_additions => false)
+          rescue JSON::ParserError => e
             raise Chef::ChefFS::FileSystem::OperationFailedError.new(:create_child, self, e), "Parse error reading JSON creating child '#{name}': #{e}"
           end
 

data/lib/chef/chef_fs/file_system/rest_list_entry.rb

@@ -19,7 +19,6 @@
 require 'chef/chef_fs/file_system/base_fs_object'
 require 'chef/chef_fs/file_system/not_found_error'
 require 'chef/chef_fs/file_system/operation_failed_error'
-require 'chef/json_compat'
 require 'chef/role'
 require 'chef/node'
 
@@ -129,8 +128,8 @@ class Chef
           value = minimize_value(value)
           value_json = Chef::JSONCompat.to_json_pretty(value)
           begin
-            other_value = Chef::JSONCompat.parse(other_value_json, :create_additions => false)
-          rescue Chef::Exceptions::JSON::ParseError => e
+            other_value = JSON.parse(other_value_json, :create_additions => false)
+          rescue JSON::ParserError => e
             Chef::Log.warn("Parse error reading #{other.path_for_printing} as JSON: #{e}")
             return [ nil, value_json, other_value_json ]
           end
@@ -146,8 +145,8 @@ class Chef
 
         def write(file_contents)
           begin
-            object = Chef::JSONCompat.parse(file_contents, :create_additions => false)
-          rescue Chef::Exceptions::JSON::ParseError => e
+            object = JSON.parse(file_contents, :create_additions => false)
+          rescue JSON::ParserError => e
             raise Chef::ChefFS::FileSystem::OperationFailedError.new(:write, self, e), "Parse error reading JSON: #{e}"
           end
 

data/lib/chef/config.rb

@@ -325,7 +325,7 @@ class Chef
       config_strict_mode true
       default(:enabled) { Chef::Config.local_mode }
       default :host, 'localhost'
-      default :port, 8889
+      default :port, 8889.upto(9999) # Will try ports from 8889-9999 until one works
     end
     default :chef_server_url,   "https://localhost:443"
 
@@ -567,7 +567,7 @@ class Chef
     # If false file staging is will be done via tempfiles that are
     # created under ENV['TMP'] otherwise tempfiles will be created in
     # the directory that files are going to reside.
-    default :file_staging_uses_destdir, false
+    default :file_staging_uses_destdir, true
 
     # Exit if another run is in progress and the chef-client is unable to
     # get the lock before time expires. If nil, no timeout is enforced. (Exits
@@ -580,6 +580,30 @@ class Chef
     # the number of threads will help.
     default :cookbook_sync_threads, 10
 
+    # At the beginning of the Chef Client run, the cookbook manifests are downloaded which
+    # contain URLs for every file in every relevant cookbook.  Most of the files
+    # (recipes, resources, providers, libraries, etc) are immediately synchronized
+    # at the start of the run.  The handling of "files" and "templates" directories,
+    # however, have two modes of operation.  They can either all be downloaded immediately
+    # at the start of the run (no_lazy_load==true) or else they can be lazily loaded as
+    # cookbook_file or template resources are converged which require them (no_lazy_load==false).
+    #
+    # The advantage of lazily loading these files is that unnecessary files are not
+    # synchronized.  This may be useful to users with large files checked into cookbooks which
+    # are only selectively downloaded to a subset of clients which use the cookbook.  However,
+    # better solutions are to either isolate large files into individual cookbooks and only
+    # include those cookbooks in the run lists of the servers that need them -- or move to
+    # using remote_file and a more appropriate backing store like S3 for large file
+    # distribution.
+    #
+    # The disadvantages of lazily loading files are that users some time find it
+    # confusing that their cookbooks are not fully synchronzied to the cache initially,
+    # and more importantly the time-sensitive URLs which are in the manifest may time
+    # out on long Chef runs before the resource that uses the file is converged
+    # (leading to many confusing 403 errors on template/cookbook_file resources).
+    #
+    default :no_lazy_load, true
+
     # A whitelisted array of attributes you want sent over the wire when node
     # data is saved.
     # The default setting is nil, which collects all data. Setting to [] will not

data/lib/chef/config_fetcher.rb

@@ -18,7 +18,7 @@ class Chef
       config_data = read_config
       begin
         Chef::JSONCompat.from_json(config_data)
-      rescue Chef::Exceptions::JSON::ParseError => error
+      rescue FFI_Yajl::ParseError => error
         Chef::Application.fatal!("Could not parse the provided JSON file (#{config_location}): " + error.message, 2)
       end
     end

data/lib/chef/cookbook/cookbook_version_loader.rb

@@ -170,7 +170,7 @@ class Chef
       def apply_ruby_metadata(file)
         begin
           @metadata.from_file(file)
-        rescue Chef::Exceptions::JSON::ParseError
+        rescue JSON::ParserError
           Chef::Log.error("Error evaluating metadata.rb for #@cookbook_name in " + file)
           raise
         end
@@ -179,7 +179,7 @@ class Chef
       def apply_json_metadata(file)
         begin
           @metadata.from_json(IO.read(file))
-        rescue Chef::Exceptions::JSON::ParseError
+        rescue JSON::ParserError
           Chef::Log.error("Couldn't parse cookbook metadata JSON for #@cookbook_name in " + file)
           raise
         end
@@ -189,7 +189,7 @@ class Chef
         begin
           data = Chef::JSONCompat.from_json(IO.read(file), :create_additions => false)
           @metadata.from_hash(data['metadata'])
-        rescue Chef::Exceptions::JSON::ParseError
+        rescue JSON::ParserError
           Chef::Log.error("Couldn't parse cookbook metadata JSON for #@cookbook_name in " + file)
           raise
         end
@@ -200,7 +200,7 @@ class Chef
           begin
             data = Chef::JSONCompat.from_json(IO.read(uploaded_cookbook_version_file), :create_additions => false)
             @frozen = data['frozen?']
-          rescue Chef::Exceptions::JSON::ParseError
+          rescue JSON::ParserError
             Chef::Log.error("Couldn't parse cookbook metadata JSON for #@cookbook_name in #{uploaded_cookbook_version_file}")
             raise
           end

data/lib/chef/cookbook/metadata.rb

@@ -441,7 +441,7 @@ class Chef
       end
 
       def to_json(*a)
-        Chef::JSONCompat.to_json(self.to_hash, *a)
+        self.to_hash.to_json(*a)
       end
 
       def self.from_hash(o)

data/lib/chef/cookbook/synchronizer.rb

@@ -10,6 +10,8 @@ class Chef
   # cache.
   class CookbookCacheCleaner
 
+    attr_accessor :skip_removal
+
     # Setup a notification to clear the valid_cache_entries when a Chef client
     # run starts
     Chef::Client.when_run_starts do |run_status|
@@ -40,15 +42,17 @@ class Chef
     end
 
     def cleanup_file_cache
-      unless Chef::Config[:solo]
+      unless Chef::Config[:solo] || skip_removal
         # Delete each file in the cache that we didn't encounter in the
         # manifest.
-        cache.find(File.join(%w{cookbooks ** *})).each do |cache_filename|
+        cache.find(File.join(%w{cookbooks ** {*,.*}})).each do |cache_filename|
           unless @valid_cache_entries[cache_filename]
             Chef::Log.info("Removing #{cache_filename} from the cache; it is no longer needed by chef-client.")
             cache.delete(cache_filename)
           end
         end
+      else
+        Chef::Log.info("Skipping removal of unused files from the cache")
       end
     end
 
@@ -59,6 +63,8 @@ class Chef
   class CookbookSynchronizer
     CookbookFile = Struct.new(:cookbook, :segment, :manifest_record)
 
+    attr_accessor :remove_obsoleted_files
+
     def initialize(cookbooks_by_name, events)
       @eager_segments = Chef::CookbookVersion::COOKBOOK_SEGMENTS.dup
       unless Chef::Config[:no_lazy_load]
@@ -70,6 +76,7 @@ class Chef
       @cookbooks_by_name, @events = cookbooks_by_name, events
 
       @cookbook_full_file_paths = {}
+      @remove_obsoleted_files = true
     end
 
     def cache
@@ -92,6 +99,10 @@ class Chef
       @cookbooks_by_name.key?(cookbook_name)
     end
 
+    def cookbook_segment(cookbook_name, segment)
+      @cookbooks_by_name[cookbook_name].manifest[segment]
+    end
+
     def files
       @files ||= cookbooks.inject([]) do |memo, cookbook|
         @eager_segments.each do |segment|
@@ -169,12 +180,10 @@ class Chef
       @cookbook_full_file_paths[file.cookbook][file.segment] << full_path
     end
 
-    # Iterates over cached cookbooks' files, removing files belonging to
-    # cookbooks that don't appear in +cookbook_hash+
-    def clear_obsoleted_cookbooks
-      @events.cookbook_clean_start
-      # Remove all cookbooks no longer relevant to this node
-      cache.find(File.join(%w{cookbooks ** *})).each do |cache_file|
+    # remove cookbooks that are not referenced in the expanded run_list at all
+    # (if we have an override run_list we may not want to do this)
+    def remove_old_cookbooks
+      cache.find(File.join(%w{cookbooks ** {*,.*}})).each do |cache_file|
         cache_file =~ /^cookbooks\/([^\/]+)\//
         unless have_cookbook?($1)
           Chef::Log.info("Removing #{cache_file} from the cache; its cookbook is no longer needed on this client.")
@@ -182,6 +191,39 @@ class Chef
           @events.removed_cookbook_file(cache_file)
         end
       end
+    end
+
+    # remove deleted files in cookbooks that are being used on the node
+    def remove_deleted_files
+      cache.find(File.join(%w{cookbooks ** {*,.*}})).each do |cache_file|
+        md = cache_file.match(/^cookbooks\/([^\/]+)\/([^\/]+)\/(.*)/)
+        next unless md
+        ( cookbook_name, segment, file ) = md[1..3]
+        if have_cookbook?(cookbook_name)
+          manifest_segment = cookbook_segment(cookbook_name, segment)
+          if manifest_segment.select { |manifest_record| manifest_record["path"] == "#{segment}/#{file}" }.empty?
+            Chef::Log.info("Removing #{cache_file} from the cache; its is no longer in the cookbook manifest.")
+            cache.delete(cache_file)
+            @events.removed_cookbook_file(cache_file)
+          end
+        end
+      end
+    end
+
+    # Iterates over cached cookbooks' files, removing files belonging to
+    # cookbooks that don't appear in +cookbook_hash+
+    def clear_obsoleted_cookbooks
+      @events.cookbook_clean_start
+
+      if remove_obsoleted_files
+        remove_old_cookbooks
+      else
+        Chef::Log.info("Skipping removal of obsoleted cookbooks from the cache")
+        CookbookCacheCleaner.instance.skip_removal = true
+      end
+
+      remove_deleted_files
+
       @events.cookbook_clean_complete
     end
 

data/lib/chef/cookbook_uploader.rb

@@ -1,6 +1,5 @@
 
 require 'set'
-require 'rest_client'
 require 'chef/exceptions'
 require 'chef/knife/cookbook_metadata'
 require 'chef/digester'
@@ -36,8 +35,8 @@ class Chef
     #           in Chef::Config.
     # * :concurrency   An integer that decided how many threads will be used to
     #           perform concurrent uploads
-    def initialize(cookbooks, path, opts={})
-      @path, @opts = path, opts
+    def initialize(cookbooks, opts={})
+      @opts = opts
       @cookbooks = Array(cookbooks)
       @rest = opts[:rest] || Chef::REST.new(Chef::Config[:chef_server_url])
       @concurrency = opts[:concurrency] || 10
@@ -54,7 +53,7 @@ class Chef
       end
 
       checksums = checksum_files.inject({}){|memo,elt| memo[elt.first]=nil ; memo}
-      new_sandbox = rest.post_rest("sandboxes", { :checksums => checksums })
+      new_sandbox = rest.post("sandboxes", { :checksums => checksums })
 
       Chef::Log.info("Uploading files")
 
@@ -81,7 +80,7 @@ class Chef
       # in eventual consistency)
       retries = 0
       begin
-        rest.put_rest(sandbox_url, {:is_completed => true})
+        rest.put(sandbox_url, {:is_completed => true})
       rescue Net::HTTPServerException => e
         if e.message =~ /^400/ && (retries += 1) <= 5
           sleep 2
@@ -95,7 +94,7 @@ class Chef
       cookbooks.each do |cb|
         save_url = opts[:force] ? cb.force_save_url : cb.save_url
         begin
-          rest.put_rest(save_url, cb)
+          rest.put(save_url, cb)
         rescue Net::HTTPServerException => e
           case e.response.code
           when "409"
@@ -109,32 +108,19 @@ class Chef
       Chef::Log.info("Upload complete!")
     end
 
-    def worker_thread(work_queue)
-    end
-
     def uploader_function_for(file, checksum, url, checksums_to_upload)
       lambda do
         # Checksum is the hexadecimal representation of the md5,
         # but we need the base64 encoding for the content-md5
         # header
         checksum64 = Base64.encode64([checksum].pack("H*")).strip
-        timestamp = Time.now.utc.iso8601
         file_contents = File.open(file, "rb") {|f| f.read}
-        # TODO - 5/28/2010, cw: make signing and sending the request streaming
+
+        # Custom headers. 'content-type' disables JSON serialization of the request body.
         headers = { 'content-type' => 'application/x-binary', 'content-md5' => checksum64, "accept" => 'application/json' }
-        if rest.signing_key
-          sign_obj = Mixlib::Authentication::SignedHeaderAuth.signing_object(
-                                                                             :http_method => :put,
-                                                                             :path        => URI.parse(url).path,
-                                                                             :body        => file_contents,
-                                                                             :timestamp   => timestamp,
-                                                                             :user_id     => rest.client_name
-                                                                             )
-          headers.merge!(sign_obj.sign(OpenSSL::PKey::RSA.new(rest.signing_key)))
-        end
 
         begin
-          Chef::HTTP::Simple.new(url, :headers=>headers).put(url, file_contents)
+          rest.put(url, file_contents, headers)
           checksums_to_upload.delete(checksum)
         rescue Net::HTTPServerException, Net::HTTPFatalError, Errno::ECONNREFUSED, Timeout::Error, Errno::ETIMEDOUT, SocketError => e
           error_message = "Failed to upload #{file} (#{checksum}) to #{url} : #{e.message}"
@@ -147,7 +133,7 @@ class Chef
 
     def validate_cookbooks
       cookbooks.each do |cb|
-        syntax_checker = Chef::Cookbook::SyntaxCheck.for_cookbook(cb.name, @user_cookbook_path)
+        syntax_checker = Chef::Cookbook::SyntaxCheck.for_cookbook(cb.name)
         Chef::Log.info("Validating ruby files")
         exit(1) unless syntax_checker.validate_ruby_files
         Chef::Log.info("Validating templates")

data/lib/chef/cookbook_version.rb

@@ -25,6 +25,7 @@ require 'chef/cookbook/metadata'
 require 'chef/version_class'
 require 'pathname'
 require 'chef/monkey_patches/pathname'
+require 'chef/digester'
 
 class Chef
 
@@ -459,7 +460,7 @@ class Chef
     def to_json(*a)
       result = self.to_hash
       result['json_class'] = self.class.name
-      Chef::JSONCompat.to_json(result, *a)
+      result.to_json(*a)
     end
 
     def self.json_create(o)
@@ -469,7 +470,7 @@ class Chef
       cookbook_version.manifest = o
 
       # We don't need the following step when we decide to stop supporting deprecated operators in the metadata (e.g. <<, >>)
-      cookbook_version.manifest["metadata"] = Chef::JSONCompat.from_json(Chef::JSONCompat.to_json(cookbook_version.metadata))
+      cookbook_version.manifest["metadata"] = Chef::JSONCompat.from_json(cookbook_version.metadata.to_json)
 
       cookbook_version.freeze_version if o["frozen?"]
       cookbook_version

data/lib/chef/data_bag.rb

@@ -63,7 +63,7 @@ class Chef
 
     # Serialize this object as a hash
     def to_json(*a)
-      Chef::JSONCompat.to_json(to_hash, *a)
+      to_hash.to_json(*a)
     end
 
     def chef_server_rest

data/lib/chef/data_bag_item.rb

@@ -118,7 +118,7 @@ class Chef
         "data_bag" => self.data_bag,
         "raw_data" => self.raw_data
       }
-      Chef::JSONCompat.to_json(result, *a)
+      result.to_json(*a)
     end
 
     def self.from_hash(h)

data/lib/chef/dsl/recipe.rb

@@ -28,6 +28,7 @@ class Chef
     # objects via method calls.
     module Recipe
 
+      include Chef::Mixin::ShellOut
       include Chef::Mixin::ConvertToClassName
 
       def method_missing(method_symbol, *args, &block)
@@ -84,20 +85,6 @@ class Chef
 
         resource = build_resource(type, name, created_at, &resource_attrs_block)
 
-        # Some resources (freebsd_package) can be invoked with multiple names
-        # (package || freebsd_package).
-        # https://github.com/opscode/chef/issues/1773
-        # For these resources we want to make sure
-        # their key in resource collection is same as the name they are declared
-        # as. Since this might be a breaking change for resources that define
-        # customer to_s methods, we are working around the issue by letting
-        # resources know of their created_as_type until this issue is fixed in
-        # Chef 12:
-        # https://github.com/opscode/chef/issues/1817
-        if resource.respond_to?(:created_as_type=)
-          resource.created_as_type = type
-        end
-
         run_context.resource_collection.insert(resource)
         resource
       end