chef 0.9.18 → 0.10.0.beta.0

data/lib/chef/client.rb

@@ -3,15 +3,15 @@
 # Author:: Christopher Walters (<cw@opscode.com>)
 # Author:: Christopher Brown (<cb@opscode.com>)
 # Author:: Tim Hinderliter (<tim@opscode.com>)
-# Copyright:: Copyright (c) 2008-2010 Opscode, Inc.
+# Copyright:: Copyright (c) 2008-2011 Opscode, Inc.
 # License:: Apache License, Version 2.0
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
 # You may obtain a copy of the License at
-# 
+#
 #     http://www.apache.org/licenses/LICENSE-2.0
-# 
+#
 # Unless required by applicable law or agreed to in writing, software
 # distributed under the License is distributed on an "AS IS" BASIS,
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
@@ -28,6 +28,7 @@ require 'chef/role'
 require 'chef/file_cache'
 require 'chef/run_context'
 require 'chef/runner'
+require 'chef/run_status'
 require 'chef/cookbook/cookbook_collection'
 require 'chef/cookbook/file_vendor'
 require 'chef/cookbook/file_system_file_vendor'
@@ -41,6 +42,8 @@ class Chef
   # syncs cookbooks if necessary, and triggers convergence.
   class Client
 
+    SANE_PATHS = %w[/usr/local/sbin /usr/local/bin /usr/sbin /usr/bin /sbin /bin]
+
     # Clears all notifications for client run status events.
     # Primarily for testing purposes.
     def self.clear_notifications
@@ -126,7 +129,7 @@ class Chef
       @runner = nil
       @ohai = Ohai::System.new
     end
-    
+
     # Do a full run for this Chef::Client.  Calls:
     #
     #  * run_ohai - Collect information about the system
@@ -140,39 +143,21 @@ class Chef
     def run
       run_context = nil
 
+      enforce_path_sanity
       run_ohai
       register unless Chef::Config[:solo]
       build_node
-      
+
       begin
 
         run_status.start_clock
         Chef::Log.info("Starting Chef Run (Version #{Chef::VERSION})")
         run_started
-        
-        if Chef::Config[:solo]
-          Chef::Cookbook::FileVendor.on_create { |manifest| Chef::Cookbook::FileSystemFileVendor.new(manifest) }
-          run_context = Chef::RunContext.new(node, Chef::CookbookCollection.new(Chef::CookbookLoader.new))
-          run_status.run_context = run_context
-          assert_cookbook_path_not_empty(run_context)
-          converge(run_context)
-        else
-          # Sync_cookbooks eagerly loads all files except files and templates.
-          # It returns the cookbook_hash -- the return result from
-          # /nodes/#{nodename}/cookbooks -- which we will use for our
-          # run_context.
-          Chef::Cookbook::FileVendor.on_create { |manifest| Chef::Cookbook::RemoteFileVendor.new(manifest, rest) }
-          cookbook_hash = sync_cookbooks
-          run_context = Chef::RunContext.new(node, Chef::CookbookCollection.new(cookbook_hash))
-          run_status.run_context = run_context
-
-          assert_cookbook_path_not_empty(run_context)
-          
-          converge(run_context)
-          Chef::Log.debug("Saving the current state of node #{node_name}")
-          @node.save
-        end
-        
+
+        run_context = setup_run_context
+        converge(run_context)
+        save_updated_node
+
         run_status.stop_clock
         Chef::Log.info("Chef Run complete in #{run_status.elapsed_time} seconds")
         run_completed_successfully
@@ -186,6 +171,35 @@ class Chef
       ensure
         run_status = nil
       end
+      true
+    end
+
+
+    # Configures the Chef::Cookbook::FileVendor class to fetch file from the
+    # server or disk as appropriate, creates the run context for this run, and
+    # sanity checks the cookbook collection.
+    #===Returns
+    # Chef::RunContext:: the run context for this run.
+    def setup_run_context
+      if Chef::Config[:solo]
+        Chef::Cookbook::FileVendor.on_create { |manifest| Chef::Cookbook::FileSystemFileVendor.new(manifest, Chef::Config[:cookbook_path]) }
+        run_context = Chef::RunContext.new(node, Chef::CookbookCollection.new(Chef::CookbookLoader.new(Chef::Config[:cookbook_path])))
+      else
+        Chef::Cookbook::FileVendor.on_create { |manifest| Chef::Cookbook::RemoteFileVendor.new(manifest, rest) }
+        cookbook_hash = sync_cookbooks
+        run_context = Chef::RunContext.new(node, Chef::CookbookCollection.new(cookbook_hash))
+      end
+      run_status.run_context = run_context
+      run_context.load(@run_list_expansion)
+      assert_cookbook_path_not_empty(run_context)
+      run_context
+    end
+
+    def save_updated_node
+      unless Chef::Config[:solo]
+        Chef::Log.debug("Saving the current state of node #{node_name}")
+        @node.save
+      end
     end
 
     def run_ohai
@@ -203,14 +217,14 @@ class Chef
 
       name
     end
-    
+
     # Builds a new node object for this client.  Starts with querying for the FQDN of the current
     # host (unless it is supplied), then merges in the facts from Ohai.
     #
     # === Returns
     # node<Chef::Node>:: Returns the created node object, also stored in @node
     def build_node
-      Chef::Log.debug("Building node object for #{@node_name}")
+      Chef::Log.debug("Building node object for #{node_name}")
 
       if Chef::Config[:solo]
         @node = Chef::Node.build(node_name)
@@ -218,46 +232,74 @@ class Chef
         @node = Chef::Node.find_or_create(node_name)
       end
 
+      # Allow user to override the environment of a node by specifying
+      # a config parameter.
+      if Chef::Config[:environment] && !Chef::Config[:environment].chop.empty?
+        @node.chef_environment(Chef::Config[:environment])
+      end
 
-      @node.consume_external_attrs(ohai.data, @json_attribs)
-      @node.expand!
-      @node.save unless Chef::Config[:solo]
+      # consume_external_attrs may add items to the run_list. Save the
+      # expanded run_list, which we will pass to the server later to
+      # determine which versions of cookbooks to use.
       @node.reset_defaults_and_overrides
+      @node.consume_external_attrs(ohai.data, @json_attribs)
+      if Chef::Config[:solo]
+        @run_list_expansion = @node.expand!('disk')
+      else
+        @run_list_expansion = @node.expand!('server')
+      end
+
+      # @run_list_expansion is a RunListExpansion.
+      #
+      # Convert @expanded_run_list, which is an
+      # Array of Hashes of the form
+      #   {:name => NAME, :version_constraint => Chef::VersionConstraint },
+      # into @expanded_run_list_with_versions, an
+      # Array of Strings of the form
+      #   "#{NAME}@#{VERSION}"
+      @expanded_run_list_with_versions = @run_list_expansion.recipes.with_version_constraints_strings
+
+      Chef::Log.info("Run List is [#{@node.run_list}]")
+      Chef::Log.info("Run List expands (with versions) to [#{@expanded_run_list_with_versions.join(', ')}]")
 
       @run_status = Chef::RunStatus.new(@node)
 
       @node
     end
 
-    # 
+    #
     # === Returns
     # rest<Chef::REST>:: returns Chef::REST connection object
-    def register
-      if File.exists?(Chef::Config[:client_key])
-        Chef::Log.debug("Client key #{Chef::Config[:client_key]} is present - skipping registration")
+    def register(client_name=node_name, config=Chef::Config)
+      if File.exists?(config[:client_key])
+        Chef::Log.debug("Client key #{config[:client_key]} is present - skipping registration")
       else
-        Chef::Log.info("Client key #{Chef::Config[:client_key]} is not present - registering")
-        Chef::REST.new(Chef::Config[:client_url], Chef::Config[:validation_client_name], Chef::Config[:validation_key]).register(node_name, Chef::Config[:client_key])
+        Chef::Log.info("Client key #{config[:client_key]} is not present - registering")
+        Chef::REST.new(config[:client_url], config[:validation_client_name], config[:validation_key]).register(client_name, config[:client_key])
       end
       # We now have the client key, and should use it from now on.
-      self.rest = Chef::REST.new(Chef::Config[:chef_server_url], node_name, Chef::Config[:client_key])
+      self.rest = Chef::REST.new(config[:chef_server_url], client_name, config[:client_key])
     end
-    
-    # Synchronizes all the cookbooks from the chef-server.
+
+    # Sync_cookbooks eagerly loads all files except files and
+    # templates.  It returns the cookbook_hash -- the return result
+    # from /environments/#{node.chef_environment}/cookbook_versions,
+    # which we will use for our run_context.
     #
     # === Returns
-    # true:: Always returns true
+    # Hash:: The hash of cookbooks with download URLs as given by the server
     def sync_cookbooks
       Chef::Log.debug("Synchronizing cookbooks")
-      cookbook_hash = rest.get_rest("nodes/#{node_name}/cookbooks")
+      cookbook_hash = rest.post_rest("environments/#{@node.chef_environment}/cookbook_versions",
+                                     {:run_list => @expanded_run_list_with_versions})
       Chef::CookbookVersion.sync_cookbooks(cookbook_hash)
 
       # register the file cache path in the cookbook path so that CookbookLoader actually picks up the synced cookbooks
       Chef::Config[:cookbook_path] = File.join(Chef::Config[:file_cache_path], "cookbooks")
-      
+
       cookbook_hash
     end
-    
+
     # Converges the node.
     #
     # === Returns
@@ -268,23 +310,35 @@ class Chef
       runner.converge
       true
     end
-    
+
+    def enforce_path_sanity(env=ENV)
+      if Chef::Config[:enforce_path_sanity] && RUBY_PLATFORM !~ /mswin|mingw32|windows/
+        existing_paths = env["PATH"].split(':')
+        SANE_PATHS.each do |sane_path|
+          unless existing_paths.include?(sane_path)
+            env["PATH"] << ':' unless env["PATH"].empty?
+            env["PATH"] << sane_path
+          end
+        end
+      end
+    end
+
     private
-    
+
     def directory_not_empty?(path)
       File.exists?(path) && (Dir.entries(path).size > 2)
     end
-    
+
     def is_last_element?(index, object)
-      object.kind_of?(Array) ? index == object.size - 1 : true 
-    end  
-    
+      object.kind_of?(Array) ? index == object.size - 1 : true
+    end
+
     def assert_cookbook_path_not_empty(run_context)
       if Chef::Config[:solo]
         # Check for cookbooks in the path given
         # Chef::Config[:cookbook_path] can be a string or an array
         # if it's an array, go through it and check each one, raise error at the last one if no files are found
-        Chef::Log.debug "loading from cookbook_path: #{Array(Chef::Config[:cookbook_path]).map { |path| File.expand_path(path) }.join(', ')}" 
+        Chef::Log.debug "loading from cookbook_path: #{Array(Chef::Config[:cookbook_path]).map { |path| File.expand_path(path) }.join(', ')}"
         Array(Chef::Config[:cookbook_path]).each_with_index do |cookbook_path, index|
           if directory_not_empty?(cookbook_path)
             break
@@ -302,3 +356,7 @@ class Chef
   end
 end
 
+# HACK cannot load this first, but it must be loaded.
+require 'chef/cookbook_loader'
+require 'chef/cookbook_version'
+

data/lib/chef/config.rb

@@ -82,6 +82,7 @@ class Chef
     #
     config_attr_writer :log_location do |location|
       if location.respond_to? :sync=
+        location.sync = true
         location
       elsif location.respond_to? :to_str
         f = File.new(location.to_str, "a")
@@ -100,6 +101,9 @@ class Chef
       providers
     end
 
+    # Turn on "path sanity" by default. See also: http://wiki.opscode.com/display/chef/User+Environment+PATH+Sanity
+    enforce_path_sanity(true)
+
     # Used when OpenID authentication is enabled in the Web UI
     authorized_openid_identifiers nil
     authorized_openid_providers nil

data/lib/chef/cookbook/chefignore.rb

@@ -0,0 +1,66 @@
+#--
+# Author:: Daniel DeLeo (<dan@opscode.com>)
+# Copyright:: Copyright (c) 2011 Opscode, Inc.
+# License:: Apache License, Version 2.0
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+class Chef
+  class Cookbook
+    class Chefignore
+
+      COMMENTS_AND_WHITESPACE = /^\w*(?:#.*)?$/
+
+      attr_reader :ignores
+
+      def initialize(ignore_file_or_repo)
+        @ignore_file = find_ignore_file(ignore_file_or_repo)
+        @ignores = parse_ignore_file
+      end
+
+      def remove_ignores_from(file_list)
+        Array(file_list).inject([]) do |unignored, file|
+          ignored?(file) ? unignored : unignored << file
+        end
+      end
+
+      def ignored?(file_name)
+        @ignores.any? {|glob| File.fnmatch?(glob, file_name)}
+      end
+
+      private
+
+      def parse_ignore_file
+        ignore_globs = []
+        if File.exist?(@ignore_file) && File.readable?(@ignore_file)
+          File.foreach(@ignore_file) do |line|
+            ignore_globs << line.strip unless line =~ COMMENTS_AND_WHITESPACE
+          end
+        else
+          Chef::Log.debug("No chefignore file found at #@ignore_file no files will be ignored")
+        end
+        ignore_globs
+      end
+
+      def find_ignore_file(path)
+        if File.basename(path) =~ /chefignore/
+          path
+        else
+          File.join(path, 'chefignore')
+        end
+      end
+    end
+  end
+end
+

data/lib/chef/cookbook/cookbook_collection.rb

@@ -7,9 +7,9 @@
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
 # You may obtain a copy of the License at
-# 
+#
 #     http://www.apache.org/licenses/LICENSE-2.0
-# 
+#
 # Unless required by applicable law or agreed to in writing, software
 # distributed under the License is distributed on an "AS IS" BASIS,
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
@@ -17,7 +17,7 @@
 # limitations under the License.
 #
 
-require 'extlib'
+require 'chef/mash'
 
 class Chef
   # == Chef::CookbookCollection
@@ -35,10 +35,11 @@ class Chef
     # simply extract them
     def initialize(cookbook_versions={})
       super() do |hash, key|
-        raise Chef::Exceptions::CookbookNotFound, "Cookbook #{key} not found"
+        raise Chef::Exceptions::CookbookNotFound, "Cookbook #{key} not found. " <<
+          "If you're loading #{key} from another cookbook, make sure you configure the dependency in your metadata"
       end
       cookbook_versions.each{ |cookbook_name, cookbook_version| self[cookbook_name] = cookbook_version }
     end
-    
+
   end
 end

data/lib/chef/cookbook/cookbook_version_loader.rb

@@ -0,0 +1,151 @@
+
+require 'chef/config'
+require 'chef/cookbook_version'
+require 'chef/cookbook/chefignore'
+require 'chef/cookbook/metadata'
+
+class Chef
+  class Cookbook
+    class CookbookVersionLoader
+
+      FILETYPES_SUBJECT_TO_IGNORE = [ :attribute_filenames,
+                                      :definition_filenames,
+                                      :recipe_filenames,
+                                      :template_filenames,
+                                      :file_filenames,
+                                      :library_filenames,
+                                      :resource_filenames,
+                                      :provider_filenames]
+
+
+      attr_reader :cookbook_name
+      attr_reader :cookbook_settings
+      attr_reader :metadata_filenames
+
+      def initialize(path, chefignore=nil)
+        @cookbook_path = File.expand_path( path )
+        @cookbook_name = File.basename( path )
+        @chefignore = chefignore
+        @metadata = Hash.new
+        @relative_path = /#{Regexp.escape(@cookbook_path)}\/(.+)$/
+        @cookbook_settings = {
+          :attribute_filenames  => {},
+          :definition_filenames => {},
+          :recipe_filenames     => {},
+          :template_filenames   => {},
+          :file_filenames       => {},
+          :library_filenames    => {},
+          :resource_filenames   => {},
+          :provider_filenames   => {},
+          :root_filenames       => {}
+        }
+
+        @metadata_filenames = []
+      end
+
+      def load_cookbooks
+        load_as(:attribute_filenames, 'attributes', '*.rb')
+        load_as(:definition_filenames, 'definitions', '*.rb')
+        load_as(:recipe_filenames, 'recipes', '*.rb')
+        load_as(:library_filenames, 'libraries', '*.rb')
+        load_recursively_as(:template_filenames, "templates", "*")
+        load_recursively_as(:file_filenames, "files", "*")
+        load_recursively_as(:resource_filenames, "resources", "*.rb")
+        load_recursively_as(:provider_filenames, "providers", "*.rb")
+        load_root_files
+
+        remove_ignored_files
+
+        if File.exists?(File.join(@cookbook_path, "metadata.json"))
+          @metadata_filenames << File.join(@cookbook_path, "metadata.json")
+        end
+
+        if empty?
+          Chef::Log.warn "found a directory #{cookbook_name} in the cookbook path, but it contains no cookbook files. skipping."
+        end
+        @cookbook_settings
+      end
+
+      def cookbook_version
+        return nil if empty?
+
+        Chef::CookbookVersion.new(@cookbook_name.to_sym).tap do |c|
+          c.root_dir             = @cookbook_path
+          c.attribute_filenames  = cookbook_settings[:attribute_filenames].values
+          c.definition_filenames = cookbook_settings[:definition_filenames].values
+          c.recipe_filenames     = cookbook_settings[:recipe_filenames].values
+          c.template_filenames   = cookbook_settings[:template_filenames].values
+          c.file_filenames       = cookbook_settings[:file_filenames].values
+          c.library_filenames    = cookbook_settings[:library_filenames].values
+          c.resource_filenames   = cookbook_settings[:resource_filenames].values
+          c.provider_filenames   = cookbook_settings[:provider_filenames].values
+          c.root_filenames       = cookbook_settings[:root_filenames].values
+          c.metadata_filenames   = @metadata_filenames
+          c.metadata             = metadata(c)
+        end
+      end
+
+      # Generates the Cookbook::Metadata object
+      def metadata(cookbook_version)
+        @metadata = Chef::Cookbook::Metadata.new(cookbook_version)
+        @metadata_filenames.each do |meta_json|
+          begin
+            @metadata.from_json(IO.read(meta_json))
+          rescue JSON::ParserError
+            Chef::Log.error("Couldn't parse cookbook metadata JSON for #@cookbook_name in " + meta_json)
+            raise
+          end
+        end
+        @metadata
+      end
+
+      def empty?
+        cookbook_settings.inject(true) do |all_empty, files|
+          all_empty && files.last.empty?
+        end
+      end
+
+      def merge!(other_cookbook_loader)
+        other_cookbook_settings = other_cookbook_loader.cookbook_settings
+        @cookbook_settings.each do |file_type, file_list|
+          file_list.merge!(other_cookbook_settings[file_type])
+        end
+        @metadata_filenames.concat(other_cookbook_loader.metadata_filenames)
+      end
+
+      def chefignore
+        @chefignore ||= Chefignore.new(File.basename(@cookbook_path))
+      end
+
+      def load_root_files
+        Dir.glob(File.join(@cookbook_path, '*'), File::FNM_DOTMATCH).each do |file|
+          next if File.directory?(file)
+          @cookbook_settings[:root_filenames][file[@relative_path, 1]] = file
+        end
+      end
+
+      def load_recursively_as(category, category_dir, glob)
+        file_spec = File.join(@cookbook_path, category_dir, '**', glob)
+        Dir.glob(file_spec, File::FNM_DOTMATCH).each do |file|
+          next if File.directory?(file)
+          @cookbook_settings[category][file[@relative_path, 1]] = file
+        end
+      end
+
+      def load_as(category, *path_glob)
+        Dir[File.join(@cookbook_path, *path_glob)].each do |file|
+          @cookbook_settings[category][file[@relative_path, 1]] = file
+        end
+      end
+
+      def remove_ignored_files
+        @cookbook_settings.each_value do |file_list|
+          file_list.reject! do |relative_path, full_path|
+            chefignore.ignored?(relative_path)
+          end
+        end
+      end
+
+    end
+  end
+end