chef 12.0.0.alpha.1 → 12.0.0.alpha.2

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 52f66afbe3a2085850ce9ab0d1e32644753864d9
-  data.tar.gz: bcc5aff93e2dcacec5086cf79a21fe2fd9652561
+  metadata.gz: b36dd2de3b06d6acb17dfd3c2d1f7beb0ba6d837
+  data.tar.gz: 1037ea4cbd93f2f6e1a4939537da08ab24f216fa
 SHA512:
-  metadata.gz: a40b8ba07c5997375d554bd31cbede4c177c624b116f688b4e063b60093707d8245173aa7ecefcd092892f5cb351a651e29fa00d22ffd528c9e073494e3cc91a
-  data.tar.gz: cd4e29fbccd881017e98e09cc75bc075a88c8a1c057e3cdbb481642387e77cb771e71fb9fc92d521bedaac326de9c26d34997f2e113371bc634d88122bdbf16f
+  metadata.gz: eb14cff2fbd9f333009de5a2cbe3e6b63b9f5ee537f04f2cbab77ab247ad12764708ba935948aae9e796f4dd0c5d295338f1eb4d44bd857f0d49510acbea5053
+  data.tar.gz: c19b61ef0c8c931fab94fcf8e94c0f7a392ef57badcb43c2a347d5bd9517489a3cb8a41c7896bf520d0af2d4c1b92efdbb59482651f227ac785fcc8e47ea4364

data/lib/chef/application.rb

@@ -46,6 +46,7 @@ class Chef::Application
     configure_chef
     configure_logging
     configure_proxy_environment_variables
+    configure_encoding
   end
 
   # Get this party started
@@ -81,10 +82,11 @@ class Chef::Application
 
   # Parse the config file
   def load_config_file
-    config_fetcher = Chef::ConfigFetcher.new(config[:config_file], Chef::Config.config_file_jail)
+    config_fetcher = Chef::ConfigFetcher.new(config[:config_file])
     if config[:config_file].nil?
       Chef::Log.warn("No config file found or specified on command line, using command line options.")
     elsif config_fetcher.config_missing?
+      pp config_missing: true
       Chef::Log.warn("*****************************************")
       Chef::Log.warn("Did not find config file: #{config[:config_file]}, using command line options.")
       Chef::Log.warn("*****************************************")
@@ -174,6 +176,11 @@ class Chef::Application
     configure_no_proxy
   end
 
+  # Sets the default external encoding to UTF-8 (users can change this, but they shouldn't)
+  def configure_encoding
+    Encoding.default_external = Chef::Config[:ruby_encoding]
+  end
+
   # Called prior to starting the application, by the run method
   def setup_application
     raise Chef::Exceptions::Application, "#{self.to_s}: you must override setup_application"

data/lib/chef/application/apply.rb

@@ -134,6 +134,10 @@ class Chef::Application::Apply < Chef::Application
       @recipe_text = STDIN.read
       temp_recipe_file
     else
+      if !ARGV[0]
+        puts opt_parser
+        Chef::Application.exit! "No recipe file provided", 1
+      end
       @recipe_filename = ARGV[0]
       @recipe_text,@recipe_fh = read_recipe_file @recipe_filename
     end

data/lib/chef/application/client.rb

@@ -24,6 +24,7 @@ require 'chef/daemon'
 require 'chef/log'
 require 'chef/config_fetcher'
 require 'chef/handler/error_report'
+require 'chef/workstation_config_loader'
 
 class Chef::Application::Client < Chef::Application
 
@@ -219,9 +220,10 @@ class Chef::Application::Client < Chef::Application
     :long         => "--chef-zero-port PORT",
     :description  => "Port (or port range) to start chef-zero on.  Port ranges like 1000,1010 or 8889-9999 will try all given ports until one works."
 
-  option :config_file_jail,
-    :long         => "--config-file-jail PATH",
-    :description  => "Directory under which config files are allowed to be loaded (no client.rb or knife.rb outside this path will be loaded)."
+  option :disable_config,
+    :long         => "--disable-config",
+    :description  => "Refuse to load a config file and use defaults. This is for development and not a stable API",
+    :boolean      => true
 
   option :run_lock_timeout,
     :long         => "--run-lock-timeout SECONDS",
@@ -273,11 +275,9 @@ class Chef::Application::Client < Chef::Application
   end
 
   def load_config_file
-    Chef::Config.config_file_jail = config[:config_file_jail] if config[:config_file_jail]
-    if !config.has_key?(:config_file)
+    if !config.has_key?(:config_file) && !config[:disable_config]
       if config[:local_mode]
-        require 'chef/knife'
-        config[:config_file] = Chef::Knife.locate_config_file
+        config[:config_file] = Chef::WorkstationConfigLoader.new(nil, Chef::Log).config_location
       else
         config[:config_file] = Chef::Config.platform_specific_path("/etc/chef/client.rb")
       end

data/lib/chef/application/solo.rb

@@ -185,24 +185,22 @@ class Chef::Application::Solo < Chef::Application
       Chef::Config[:interval] ||= 1800
     end
 
-    if Chef::Config[:json_attribs]
-      config_fetcher = Chef::ConfigFetcher.new(Chef::Config[:json_attribs])
-      @chef_client_json = config_fetcher.fetch_json
-    end
-
     if Chef::Config[:recipe_url]
       cookbooks_path = Array(Chef::Config[:cookbook_path]).detect{|e| e =~ /\/cookbooks\/*$/ }
       recipes_path = File.expand_path(File.join(cookbooks_path, '..'))
 
       Chef::Log.debug "Creating path #{recipes_path} to extract recipes into"
-      FileUtils.mkdir_p recipes_path
-      path = File.join(recipes_path, 'recipes.tgz')
-      File.open(path, 'wb') do |f|
-        open(Chef::Config[:recipe_url]) do |r|
-          f.write(r.read)
-        end
-      end
-      Chef::Mixin::Command.run_command(:command => "tar zxvf #{path} -C #{recipes_path}")
+      FileUtils.mkdir_p(recipes_path)
+      tarball_path = File.join(recipes_path, 'recipes.tgz')
+      fetch_recipe_tarball(Chef::Config[:recipe_url], tarball_path)
+      Chef::Mixin::Command.run_command(:command => "tar zxvf #{tarball_path} -C #{recipes_path}")
+    end
+
+    # json_attribs shuld be fetched after recipe_url tarball is unpacked.
+    # Otherwise it may fail if points to local file from tarball.
+    if Chef::Config[:json_attribs]
+      config_fetcher = Chef::ConfigFetcher.new(Chef::Config[:json_attribs])
+      @chef_client_json = config_fetcher.fetch_json
     end
   end
 
@@ -246,4 +244,14 @@ class Chef::Application::Solo < Chef::Application
     end
   end
 
+  private
+
+  def fetch_recipe_tarball(url, path)
+    Chef::Log.debug("Download recipes tarball from #{url} to #{path}")
+    File.open(path, 'wb') do |f|
+      open(url) do |r|
+        f.write(r.read)
+      end
+    end
+  end
 end

data/lib/chef/chef_fs/chef_fs_data_store.rb

@@ -27,7 +27,61 @@ require 'fileutils'
 
 class Chef
   module ChefFS
+    #
+    # Translation layer between chef-zero's DataStore (a place where it expects
+    # files to be stored) and ChefFS (the user's repository directory layout).
+    #
+    # chef-zero expects the data store to store files *its* way--for example, it
+    # expects get("nodes/blah") to return the JSON text for the blah node, and
+    # it expects get("cookbooks/blah/1.0.0") to return the JSON definition of
+    # the blah cookbook version 1.0.0.
+    #
+    # The repository is defined the way the *user* wants their layout.  These
+    # two things are very similar in layout (for example, nodes are stored under
+    # the nodes/ directory and their filename is the name of the node).
+    #
+    # However, there are a few differences that make this more than just a raw
+    # file store:
+    #
+    # 1. Cookbooks are stored much differently.
+    #   - chef-zero places JSON text with the checksums for the cookbook at
+    #     /cookbooks/NAME/VERSION, and expects the JSON to contain URLs to the
+    #     actual files, which are stored elsewhere.
+    #   - The repository contains an actual directory with just the cookbook
+    #     files and a metadata.rb containing a version #.  There is no JSON to
+    #     be found.
+    #   - Further, if versioned_cookbooks is false, that directory is named
+    #     /cookbooks/NAME and only one version exists.  If versioned_cookbooks
+    #     is true, the directory is named /cookbooks/NAME-VERSION.
+    #   - Therefore, ChefFSDataStore calculates the cookbook JSON by looking at
+    #     the files in the cookbook and checksumming them, and reading metadata.rb
+    #     for the version and dependency information.
+    #   - ChefFSDataStore also modifies the cookbook file URLs so that they point
+    #     to /file_store/repo/<filename> (the path to the actual file under the
+    #     repository root).  For example, /file_store/repo/apache2/metadata.rb or
+    #     /file_store/repo/cookbooks/apache2/recipes/default.rb).
+    #
+    # 2. Sandboxes don't exist in the repository.
+    #   - ChefFSDataStore lets cookbooks be uploaded into a temporary memory
+    #     storage, and when the cookbook is committed, copies the files onto the
+    #     disk in the correct place (/cookbooks/apache2/recipes/default.rb).
+    # 3. Data bags:
+    #   - The Chef server expects data bags in /data/BAG/ITEM
+    #   - The repository stores data bags in /data_bags/BAG/ITEM
+    #
+    # 4. JSON filenames are generally NAME.json in the repository (e.g. /nodes/foo.json).
+    #
     class ChefFSDataStore
+      #
+      # Create a new ChefFSDataStore
+      #
+      # ==== Arguments
+      #
+      # [chef_fs]
+      #   A +ChefFS::FileSystem+ object representing the repository root.
+      #   Generally will be a +ChefFS::FileSystem::ChefRepositoryFileSystemRoot+
+      #   object, created from +ChefFS::Config.local_fs+.
+      #
       def initialize(chef_fs)
         @chef_fs = chef_fs
         @memory_store = ChefZero::DataStore::MemoryStore.new
@@ -103,7 +157,7 @@ class Chef
                   value.each do |file|
                     if file.is_a?(Hash) && file.has_key?('checksum')
                       relative = ['file_store', 'repo', 'cookbooks']
-                      if Chef::Config.versioned_cookbooks
+                      if chef_fs.versioned_cookbooks
                         relative << "#{path[1]}-#{path[2]}"
                       else
                         relative << path[1]
@@ -190,7 +244,7 @@ class Chef
         elsif path[0] == 'cookbooks' && path.length == 1
           with_entry(path) do |entry|
             begin
-              if Chef::Config.versioned_cookbooks
+              if chef_fs.versioned_cookbooks
                 # /cookbooks/name-version -> /cookbooks/name
                 entry.children.map { |child| split_name_version(child.name)[0] }.uniq
               else
@@ -203,7 +257,7 @@ class Chef
           end
 
         elsif path[0] == 'cookbooks' && path.length == 2
-          if Chef::Config.versioned_cookbooks
+          if chef_fs.versioned_cookbooks
             result = with_entry([ 'cookbooks' ]) do |entry|
               # list /cookbooks/name = filter /cookbooks/name-version down to name
               entry.children.map { |child| split_name_version(child.name) }.
@@ -261,7 +315,7 @@ class Chef
       end
 
       def write_cookbook(path, data, *options)
-        if Chef::Config.versioned_cookbooks
+        if chef_fs.versioned_cookbooks
           cookbook_path = File.join('cookbooks', "#{path[1]}-#{path[2]}")
         else
           cookbook_path = File.join('cookbooks', path[1])
@@ -318,7 +372,7 @@ class Chef
         elsif path[0] == 'cookbooks'
           if path.length == 2
             raise ChefZero::DataStore::DataNotFoundError.new(path)
-          elsif Chef::Config.versioned_cookbooks
+          elsif chef_fs.versioned_cookbooks
             if path.length >= 3
               # cookbooks/name/version -> cookbooks/name-version
               path = [ path[0], "#{path[1]}-#{path[2]}" ] + path[3..-1]
@@ -351,7 +405,7 @@ class Chef
           end
 
         elsif path[0] == 'cookbooks'
-          if Chef::Config.versioned_cookbooks
+          if chef_fs.versioned_cookbooks
             # cookbooks/name-version/... -> cookbooks/name/version/...
             if path.length >= 2
               name, version = split_name_version(path[1])

data/lib/chef/chef_fs/config.rb

@@ -22,17 +22,87 @@ require 'chef/chef_fs/path_utils'
 class Chef
   module ChefFS
     #
-    # Helpers to take Chef::Config and create chef_fs and local_fs from it
+    # Helpers to take Chef::Config and create chef_fs and local_fs (ChefFS
+    # objects representing the server and local repository, respectively).
     #
     class Config
-      def initialize(chef_config = Chef::Config, cwd = Dir.pwd, options = {})
+      #
+      # Create a new Config object which can produce a chef_fs and local_fs.
+      #
+      # ==== Arguments
+      #
+      # [chef_config]
+      #   A hash that looks suspiciously like +Chef::Config+.  These hash keys
+      #   include:
+      #
+      #   :chef_repo_path::
+      #     The root where all local chef object data is stored.  Mirrors
+      #     +Chef::Config.chef_repo_path+
+      #   :cookbook_path, node_path, ...::
+      #     Paths to cookbooks/, nodes/, data_bags/, etc.  Mirrors
+      #     +Chef::Config.cookbook_path+, etc.  Defaults to
+      #     +<chef_repo_path>/cookbooks+, etc.
+      #   :repo_mode::
+      #     The directory format on disk.  'everything', 'hosted_everything' and
+      #     'static'.  Default: autodetected based on whether the URL has
+      #     "/organizations/NAME."
+      #   :versioned_cookbooks::
+      #     If true, the repository contains cookbooks with versions in their
+      #     name (apache2-1.0.0).  If false, the repository just has one version
+      #     of each cookbook and the directory has the cookbook name (apache2).
+      #     Default: +false+
+      #   :chef_server_url::
+      #     The URL to the Chef server, e.g. https://api.opscode.com/organizations/foo.
+      #     Used as the server for the remote chef_fs, and to "guess" repo_mode
+      #     if not specified.
+      #   :node_name:: The username to authenticate to the Chef server with.
+      #   :client_key:: The private key for the user for authentication
+      #   :environment:: The environment in which you are presently working
+      #   :repo_mode::
+      #     The repository mode, :hosted_everything, :everything or :static.
+      #     This determines the set of subdirectories the Chef server will offer
+      #     up.
+      #   :versioned_cookbooks:: Whether or not to include versions in cookbook names
+      #
+      # [cwd]
+      #   The current working directory to base relative Chef paths from.
+      #   Defaults to +Dir.pwd+.
+      #
+      # [options]
+      #   A hash of other, not-suspiciously-like-chef-config options:
+      #   :cookbook_version::
+      #     When downloading cookbooks, download this cookbook version instead
+      #     of the latest.
+      #
+      # [ui]
+      #   The object to print output to, with "output", "warn" and "error"
+      #   (looks a little like a Chef::Knife::UI object, obtainable from
+      #   Chef::Knife.ui).
+      #
+      # ==== Example
+      #
+      #   require 'chef/chef_fs/config'
+      #   config = Chef::ChefFS::Config.new
+      #   config.chef_fs.child('cookbooks').children.each do |cookbook|
+      #     puts "Cookbook on server: #{cookbook.name}"
+      #   end
+      #   config.local_fs.child('cookbooks').children.each do |cookbook|
+      #     puts "Local cookbook: #{cookbook.name}"
+      #   end
+      #
+      def initialize(chef_config = Chef::Config, cwd = Dir.pwd, options = {}, ui = nil)
         @chef_config = chef_config
         @cwd = cwd
         @cookbook_version = options[:cookbook_version]
 
+        if @chef_config[:repo_mode] == 'everything' && is_hosted? && !ui.nil?
+          ui.warn %Q{You have repo_mode set to 'everything', but your chef_server_url
+              looks like it might be a hosted setup.  If this is the case please use
+              hosted_everything or allow repo_mode to default}
+        end
         # Default to getting *everything* from the server.
         if !@chef_config[:repo_mode]
-          if @chef_config[:chef_server_url] =~ /\/+organizations\/.+/
+          if is_hosted?
             @chef_config[:repo_mode] = 'hosted_everything'
           else
             @chef_config[:repo_mode] = 'everything'
@@ -44,6 +114,10 @@ class Chef
       attr_reader :cwd
       attr_reader :cookbook_version
 
+      def is_hosted?
+        @chef_config[:chef_server_url] =~ /\/+organizations\/.+/
+      end
+
       def chef_fs
         @chef_fs ||= create_chef_fs
       end
@@ -59,7 +133,7 @@ class Chef
 
       def create_local_fs
         require 'chef/chef_fs/file_system/chef_repository_file_system_root_dir'
-        Chef::ChefFS::FileSystem::ChefRepositoryFileSystemRootDir.new(object_paths)
+        Chef::ChefFS::FileSystem::ChefRepositoryFileSystemRootDir.new(object_paths, Array(chef_config[:chef_repo_path]).flatten, @chef_config)
       end
 
       # Returns the given real path's location relative to the server root.

data/lib/chef/chef_fs/data_handler/acl_data_handler.rb

@@ -4,9 +4,9 @@ class Chef
   module ChefFS
     module DataHandler
       class AclDataHandler < DataHandlerBase
-        def normalize(node, entry)
+        def normalize(acl, entry)
           # Normalize the order of the keys for easier reading
-          result = normalize_hash(node, {
+          result = normalize_hash(acl, {
             'create' => {},
             'read' => {},
             'update' => {},

data/lib/chef/chef_fs/data_handler/client_data_handler.rb

@@ -22,7 +22,7 @@ class Chef
           result
         end
 
-        def preserve_key(key)
+        def preserve_key?(key)
           return key == 'name'
         end
 

data/lib/chef/chef_fs/data_handler/container_data_handler.rb

@@ -11,7 +11,7 @@ class Chef
           })
         end
 
-        def preserve_key(key)
+        def preserve_key?(key)
           return key == 'containername'
         end
 

data/lib/chef/chef_fs/data_handler/cookbook_data_handler.rb

@@ -23,7 +23,7 @@ class Chef
           })
         end
 
-        def preserve_key(key)
+        def preserve_key?(key)
           return key == 'cookbook_name' || key == 'version'
         end
 

data/lib/chef/chef_fs/data_handler/data_bag_item_data_handler.rb

@@ -34,7 +34,7 @@ class Chef
           normalize_for_post(data_bag_item, entry)
         end
 
-        def preserve_key(key)
+        def preserve_key?(key)
           return key == 'id'
         end
 

data/lib/chef/chef_fs/data_handler/data_handler_base.rb

@@ -1,17 +1,32 @@
 class Chef
   module ChefFS
     module DataHandler
+      #
+      # The base class for all *DataHandlers.
+      #
+      # DataHandlers' job is to know the innards of Chef objects and manipulate
+      # JSON for them, adding defaults and formatting them.
+      #
       class DataHandlerBase
+        #
+        # Remove all default values from a Chef object's JSON so that the only
+        # thing you see are the values that have been explicitly set.
+        # Achieves this by calling normalize({}, entry) to get the list of
+        # defaults, and subtracting anything that is the same.
+        #
         def minimize(object, entry)
           default_object = default(entry)
           object.each_pair do |key, value|
-            if default_object[key] == value && !preserve_key(key)
+            if default_object[key] == value && !preserve_key?(key)
               object.delete(key)
             end
           end
           object
         end
 
+        #
+        # Takes a name like blah.json and removes the .json from it.
+        #
         def remove_dot_json(name)
           if name.length < 5 || name[-5,5] != ".json"
             raise "Invalid name #{path}: must end in .json"
@@ -19,14 +34,34 @@ class Chef
           name[0,name.length-5]
         end
 
-        def preserve_key(key)
+        #
+        # Return true if minimize() should preserve a key even if it is the same
+        # as the default.  Often used for ids and names.
+        #
+        def preserve_key?(key)
           false
         end
 
+        #
+        # Get the default value for an entry.  Calls normalize({}, entry).
+        #
         def default(entry)
           normalize({}, entry)
         end
 
+        #
+        # Utility function to help subclasses do normalize().  Pass in a hash
+        # and a list of keys with defaults, and normalize will:
+        #
+        # 1. Fill in the defaults
+        # 2. Put the actual values in the order of the defaults
+        # 3. Move any other values to the end
+        #
+        # == Example
+        #
+        #   normalize_hash({x: 100, c: 2, a: 1}, { a: 10, b: 20, c: 30})
+        #   -> { a: 1, b: 20, c: 2, x: 100}
+        #
         def normalize_hash(object, defaults)
           # Make a normalized result in the specified order for diffing
           result = {}
@@ -39,14 +74,25 @@ class Chef
           result
         end
 
+        # Specialized function to normalize an object before POSTing it, since
+        # some object types want slightly different values on POST.
+        # If not overridden, this just calls normalize()
         def normalize_for_post(object, entry)
           normalize(object, entry)
         end
 
+        # Specialized function to normalize an object before PUTing it, since
+        # some object types want slightly different values on PUT.
+        # If not overridden, this just calls normalize().
         def normalize_for_put(object, entry)
           normalize(object, entry)
         end
 
+        #
+        # normalize a run list (an array of run list items).
+        # Leaves recipe[name] and role[name] alone, and translates
+        # name to recipe[name].  Then calls uniq on the result.
+        #
         def normalize_run_list(run_list)
           run_list.map{|item|
             case item.to_s
@@ -60,22 +106,46 @@ class Chef
           }.uniq
         end
 
+        #
+        # Bring in an instance of this object from Ruby.  (Like roles/x.rb)
+        #
         def from_ruby(ruby)
           chef_class.from_file(ruby).to_hash
         end
 
+        #
+        # Turn a JSON hash into a bona fide Chef object (like Chef::Node).
+        #
         def chef_object(object)
           chef_class.json_create(object)
         end
 
+        #
+        # Write out the Ruby file for this instance.  (Like roles/x.rb)
+        #
         def to_ruby(object)
           raise NotImplementedError
         end
 
+        #
+        # Get the class for instances of this type.  Must be overridden.
+        #
         def chef_class
           raise NotImplementedError
         end
 
+        #
+        # Helper to write out a Ruby file for a JSON hash.  Writes out only
+        # the keys specified in "keys"; anything else must be emitted by the
+        # caller.
+        #
+        # == Example
+        #
+        #   to_ruby_keys({"name" => "foo", "environment" => "desert", "foo": "bar"}, [ "name", "environment" ])
+        #   ->
+        #   'name "foo"
+        #   environment "desert"'
+        #
         def to_ruby_keys(object, keys)
           result = ''
           keys.each do |key|
@@ -115,6 +185,10 @@ class Chef
           result
         end
 
+        #
+        # Verify that the JSON hash for this type has a key that matches its name.
+        # Calls the on_error block with the error, if there is one.
+        #
         def verify_integrity(object, entry, &on_error)
           base_name = remove_dot_json(entry.name)
           if object['name'] != base_name