chef 10.28.2-x86-mingw32

data/lib/chef/cookbook/remote_file_vendor.rb

@@ -0,0 +1,84 @@
+#
+# Author:: Tim Hinderliter (<tim@opscode.com>)
+# Copyright:: Copyright (c) 2010 Opscode, Inc.
+# License:: Apache License, Version 2.0
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+require 'chef/cookbook/file_vendor'
+
+class Chef
+  class Cookbook
+    # == Chef::Cookbook::RemoteFileVendor
+    # This FileVendor loads files by either fetching them from the local cache, or
+    # if not available, loading them from the remote server.
+    class RemoteFileVendor < FileVendor
+
+      def initialize(manifest, rest)
+        @manifest = manifest
+        @cookbook_name = @manifest[:cookbook_name]
+        @rest = rest
+      end
+
+      # Implements abstract base's requirement. It looks in the
+      # Chef::Config.cookbook_path file hierarchy for the requested
+      # file.
+      def get_filename(filename)
+        if filename =~ /([^\/]+)\/(.+)$/
+          segment = $1
+        else
+          raise "get_filename: Cannot determine segment/filename for incoming filename #{filename}"
+        end
+
+        raise "No such segment #{segment} in cookbook #{@cookbook_name}" unless @manifest[segment]
+        found_manifest_record = @manifest[segment].find {|manifest_record| manifest_record[:path] == filename }
+        raise "No such file #{filename} in #{@cookbook_name}" unless found_manifest_record
+
+        cache_filename = File.join("cookbooks", @cookbook_name, found_manifest_record['path'])
+
+        # update valid_cache_entries so the upstream cache cleaner knows what
+        # we've used.
+        validate_cached_copy(cache_filename)
+
+        current_checksum = nil
+        if Chef::FileCache.has_key?(cache_filename)
+          current_checksum = Chef::CookbookVersion.checksum_cookbook_file(Chef::FileCache.load(cache_filename, false))
+        end
+
+        # If the checksums are different between on-disk (current) and on-server
+        # (remote, per manifest), do the update. This will also execute if there
+        # is no current checksum.
+        if current_checksum != found_manifest_record['checksum']
+          raw_file = @rest.get_rest(found_manifest_record[:url], true)
+
+          Chef::Log.debug("Storing updated #{cache_filename} in the cache.")
+          Chef::FileCache.move_to(raw_file.path, cache_filename)
+        else
+          Chef::Log.debug("Not fetching #{cache_filename}, as the cache is up to date.")
+          Chef::Log.debug("current checksum: #{current_checksum}; manifest checksum: #{found_manifest_record['checksum']})")
+        end
+
+        full_path_cache_filename = Chef::FileCache.load(cache_filename, false)
+
+        # return the filename, not the contents (second argument= false)
+        full_path_cache_filename
+      end
+
+      def validate_cached_copy(cache_filename)
+        CookbookCacheCleaner.instance.mark_file_as_valid(cache_filename)
+      end
+
+    end
+  end
+end

data/lib/chef/cookbook/synchronizer.rb

@@ -0,0 +1,218 @@
+require 'chef/client'
+require 'singleton'
+
+class Chef
+
+  # Keep track of the filenames that we use in both eager cookbook
+  # downloading (during sync_cookbooks) and lazy (during the run
+  # itself, through FileVendor). After the run is over, clean up the
+  # cache.
+  class CookbookCacheCleaner
+
+    # Setup a notification to clear the valid_cache_entries when a Chef client
+    # run starts
+    Chef::Client.when_run_starts do |run_status|
+      instance.reset!
+    end
+
+    # Register a notification to cleanup unused files from cookbooks
+    Chef::Client.when_run_completes_successfully do |run_status|
+      instance.cleanup_file_cache
+    end
+
+    include Singleton
+
+    def initialize
+      reset!
+    end
+
+    def reset!
+      @valid_cache_entries = {}
+    end
+
+    def mark_file_as_valid(cache_path)
+      @valid_cache_entries[cache_path] = true
+    end
+
+    def cache
+      Chef::FileCache
+    end
+
+    def cleanup_file_cache
+      unless Chef::Config[:solo]
+        # Delete each file in the cache that we didn't encounter in the
+        # manifest.
+        cache.find(File.join(%w{cookbooks ** *})).each do |cache_filename|
+          unless @valid_cache_entries[cache_filename]
+            Chef::Log.info("Removing #{cache_filename} from the cache; it is no longer needed by chef-client.")
+            cache.delete(cache_filename)
+          end
+        end
+      end
+    end
+
+  end
+
+  # Synchronizes the locally cached copies of cookbooks with the files on the
+  # server.
+  class CookbookSynchronizer
+    def initialize(cookbooks_by_name, events)
+      @eager_segments = Chef::CookbookVersion::COOKBOOK_SEGMENTS.dup
+      unless Chef::Config[:no_lazy_load]
+        @eager_segments.delete(:files)
+        @eager_segments.delete(:templates)
+      end
+      @eager_segments.freeze
+
+      @cookbooks_by_name, @events = cookbooks_by_name, events
+    end
+
+    def cache
+      Chef::FileCache
+    end
+
+    def cookbook_names
+      @cookbooks_by_name.keys
+    end
+
+    def cookbooks
+      @cookbooks_by_name.values
+    end
+
+    def cookbook_count
+      @cookbooks_by_name.size
+    end
+
+    def have_cookbook?(cookbook_name)
+      @cookbooks_by_name.key?(cookbook_name)
+    end
+
+    # Synchronizes all the cookbooks from the chef-server.
+    #)
+    # === Returns
+    # true:: Always returns true
+    def sync_cookbooks
+      Chef::Log.info("Loading cookbooks [#{cookbook_names.sort.join(', ')}]")
+      Chef::Log.debug("Cookbooks detail: #{cookbooks.inspect}")
+
+      clear_obsoleted_cookbooks
+
+      @events.cookbook_sync_start(cookbook_count)
+
+      # Synchronize each of the node's cookbooks, and add to the
+      # valid_cache_entries hash.
+      cookbooks.each do |cookbook|
+        sync_cookbook(cookbook)
+      end
+
+    rescue Exception => e
+      @events.cookbook_sync_failed(cookbooks, e)
+      raise
+    else
+      @events.cookbook_sync_complete
+      true
+    end
+
+    # Iterates over cached cookbooks' files, removing files belonging to
+    # cookbooks that don't appear in +cookbook_hash+
+    def clear_obsoleted_cookbooks
+      @events.cookbook_clean_start
+      # Remove all cookbooks no longer relevant to this node
+      cache.find(File.join(%w{cookbooks ** *})).each do |cache_file|
+        cache_file =~ /^cookbooks\/([^\/]+)\//
+        unless have_cookbook?($1)
+          Chef::Log.info("Removing #{cache_file} from the cache; its cookbook is no longer needed on this client.")
+          cache.delete(cache_file)
+          @events.removed_cookbook_file(cache_file)
+        end
+      end
+      @events.cookbook_clean_complete
+    end
+
+    # Sync the eagerly loaded files contained by +cookbook+
+    #
+    # === Arguments
+    # cookbook<Chef::Cookbook>:: The cookbook to update
+    # valid_cache_entries<Hash>:: Out-param; Added to this hash are the files that
+    # were referred to by this cookbook
+    def sync_cookbook(cookbook)
+      Chef::Log.debug("Synchronizing cookbook #{cookbook.name}")
+
+      # files and templates are lazily loaded, and will be done later.
+
+      @eager_segments.each do |segment|
+        segment_filenames = Array.new
+        cookbook.manifest[segment].each do |manifest_record|
+
+          cache_filename = sync_file_in_cookbook(cookbook, manifest_record)
+          # make the segment filenames a full path.
+          full_path_cache_filename = cache.load(cache_filename, false)
+          segment_filenames << full_path_cache_filename
+        end
+
+        # replace segment filenames with a full-path one.
+        if segment.to_sym == :recipes
+          cookbook.recipe_filenames = segment_filenames
+        elsif segment.to_sym == :attributes
+          cookbook.attribute_filenames = segment_filenames
+        else
+          cookbook.segment_filenames(segment).replace(segment_filenames)
+        end
+      end
+      @events.synchronized_cookbook(cookbook.name)
+    end
+
+    # Sync an individual file if needed. If there is an up to date copy
+    # locally, nothing is done.
+    #
+    # === Arguments
+    # file_manifest::: A Hash of the form {"path" => 'relative/path', "url" => "location to fetch the file"}
+    # === Returns
+    # Path to the cached file as a String
+    def sync_file_in_cookbook(cookbook, file_manifest)
+      cache_filename = File.join("cookbooks", cookbook.name, file_manifest['path'])
+      mark_cached_file_valid(cache_filename)
+
+      # If the checksums are different between on-disk (current) and on-server
+      # (remote, per manifest), do the update. This will also execute if there
+      # is no current checksum.
+      if !cached_copy_up_to_date?(cache_filename, file_manifest['checksum'])
+        download_file(file_manifest['url'], cache_filename)
+        @events.updated_cookbook_file(cookbook.name, cache_filename)
+      else
+        Chef::Log.debug("Not storing #{cache_filename}, as the cache is up to date.")
+      end
+
+      cache_filename
+    end
+
+    def cached_copy_up_to_date?(local_path, expected_checksum)
+      if cache.has_key?(local_path)
+        current_checksum = CookbookVersion.checksum_cookbook_file(cache.load(local_path, false))
+        expected_checksum == current_checksum
+      else
+        false
+      end
+    end
+
+    # Unconditionally download the file from the given URL. File will be
+    # downloaded to the path +destination+ which is relative to the Chef file
+    # cache root.
+    def download_file(url, destination)
+      raw_file = server_api.get_rest(url, true)
+
+      Chef::Log.info("Storing updated #{destination} in the cache.")
+      cache.move_to(raw_file.path, destination)
+    end
+
+    # Marks the given file as valid (non-stale).
+    def mark_cached_file_valid(cache_filename)
+      CookbookCacheCleaner.instance.mark_file_as_valid(cache_filename)
+    end
+
+    def server_api
+      Chef::REST.new(Chef::Config[:chef_server_url])
+    end
+
+  end
+end

data/lib/chef/cookbook/syntax_check.rb

@@ -0,0 +1,136 @@
+#
+# Author:: Daniel DeLeo (<dan@opscode.com>)
+# Copyright:: Copyright (c) 2010 Opscode, Inc.
+# License:: Apache License, Version 2.0
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+# 
+#     http://www.apache.org/licenses/LICENSE-2.0
+# 
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+require 'chef/checksum_cache'
+require 'chef/mixin/shell_out'
+
+class Chef
+  class Cookbook
+    # == Chef::Cookbook::SyntaxCheck
+    # Encapsulates the process of validating the ruby syntax of files in Chef
+    # cookbooks.
+    class SyntaxCheck
+      include Chef::Mixin::ShellOut
+
+      attr_reader :cookbook_path
+
+      # Creates a new SyntaxCheck given the +cookbook_name+ and a +cookbook_path+.
+      # If no +cookbook_path+ is given, +Chef::Config.cookbook_path+ is used.
+      def self.for_cookbook(cookbook_name, cookbook_path=nil)
+        cookbook_path ||= Chef::Config.cookbook_path
+        unless cookbook_path
+          raise ArgumentError, "Cannot find cookbook #{cookbook_name} unless Chef::Config.cookbook_path is set or an explicit cookbook path is given"
+        end
+        new(File.join(cookbook_path, cookbook_name.to_s))
+      end
+
+      # Create a new SyntaxCheck object
+      # === Arguments
+      # cookbook_path::: the (on disk) path to the cookbook
+      def initialize(cookbook_path)
+        @cookbook_path = cookbook_path
+      end
+
+      def cache
+        Chef::ChecksumCache.instance
+      end
+
+      def ruby_files
+        Dir[File.join(cookbook_path, '**', '*.rb')]
+      end
+
+      def untested_ruby_files
+        ruby_files.reject do |file|
+          if validated?(file)
+            Chef::Log.debug("Ruby file #{file} is unchanged, skipping syntax check")
+            true
+          else
+            false
+          end
+        end
+      end
+
+      def template_files
+        Dir[File.join(cookbook_path, '**', '*.erb')]
+      end
+
+      def untested_template_files
+        template_files.reject do |file| 
+          if validated?(file)
+            Chef::Log.debug("Template #{file} is unchanged, skipping syntax check")
+            true
+          else
+            false
+          end
+        end
+      end
+
+      def validated?(file)
+        !!cache.lookup_checksum(cache_key(file), File.stat(file))
+      end
+
+      def validated(file)
+        cache.generate_checksum(cache_key(file), file, File.stat(file))
+      end
+
+      def cache_key(file)
+        @cache_keys ||= {}
+        @cache_keys[file] ||= cache.generate_key(file, "chef-test")
+      end
+
+      def validate_ruby_files
+        untested_ruby_files.each do |ruby_file|
+          return false unless validate_ruby_file(ruby_file)
+          validated(ruby_file)
+        end
+      end
+
+      def validate_templates
+        untested_template_files.each do |template|
+          return false unless validate_template(template)
+          validated(template)
+        end
+      end
+
+      def validate_template(erb_file)
+        Chef::Log.debug("Testing template #{erb_file} for syntax errors...")
+        result = shell_out("erubis -x #{erb_file} | ruby -c")
+        result.error!
+        true
+      rescue Mixlib::ShellOut::ShellCommandFailed
+        file_relative_path = erb_file[/^#{Regexp.escape(cookbook_path+File::Separator)}(.*)/, 1]
+        Chef::Log.fatal("Erb template #{file_relative_path} has a syntax error:")
+        result.stderr.each_line { |l| Chef::Log.fatal(l.chomp) }
+        false
+      end
+      
+      def validate_ruby_file(ruby_file)
+        Chef::Log.debug("Testing #{ruby_file} for syntax errors...")
+        result = shell_out("ruby -c #{ruby_file}")
+        result.error!
+        true
+      rescue Mixlib::ShellOut::ShellCommandFailed
+        file_relative_path = ruby_file[/^#{Regexp.escape(cookbook_path+File::Separator)}(.*)/, 1]
+        Chef::Log.fatal("Cookbook file #{file_relative_path} has a ruby syntax error:")
+        result.stderr.each_line { |l| Chef::Log.fatal(l.chomp) }
+        false
+      end
+      
+    end
+  end
+end

data/lib/chef/cookbook_loader.rb

@@ -0,0 +1,121 @@
+#
+# Author:: Adam Jacob (<adam@opscode.com>)
+# Author:: Christopher Walters (<cw@opscode.com>)
+# Author:: Daniel DeLeo (<dan@kallistec.com>)
+# Copyright:: Copyright (c) 2008 Opscode, Inc.
+# Copyright:: Copyright (c) 2009 Daniel DeLeo
+# License:: Apache License, Version 2.0
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+require 'chef/config'
+require 'chef/exceptions'
+require 'chef/cookbook/cookbook_version_loader'
+require 'chef/cookbook_version'
+require 'chef/cookbook/chefignore'
+require 'chef/cookbook/metadata'
+
+class Chef
+  class CookbookLoader
+
+    attr_accessor :metadata
+    attr_reader :cookbooks_by_name
+    attr_reader :merged_cookbooks
+    attr_reader :cookbook_paths
+
+    include Enumerable
+
+    def initialize(*repo_paths)
+      @repo_paths = repo_paths.flatten
+      raise ArgumentError, "You must specify at least one cookbook repo path" if @repo_paths.empty?
+      @cookbooks_by_name = Mash.new
+      @loaded_cookbooks = {}
+      @metadata = Mash.new
+      @cookbooks_paths = Hash.new {|h,k| h[k] = []} # for deprecation warnings
+
+      # Used to track which cookbooks appear in multiple places in the cookbook repos
+      # and are merged in to a single cookbook by file shadowing. This behavior is
+      # deprecated, so users of this class may issue warnings to the user by checking
+      # this variable
+      @merged_cookbooks = []
+
+      load_cookbooks
+    end
+
+    def merged_cookbook_paths # for deprecation warnings
+      merged_cookbook_paths = {}
+      @merged_cookbooks.each {|c| merged_cookbook_paths[c] = @cookbooks_paths[c]}
+      merged_cookbook_paths
+    end
+
+    def load_cookbooks
+      cookbook_settings = Hash.new
+      @repo_paths.each do |repo_path|
+        repo_path = File.expand_path(repo_path)
+        chefignore = Cookbook::Chefignore.new(repo_path)
+        Dir[File.join(repo_path, "*")].each do |cookbook_path|
+          next unless File.directory?(cookbook_path)
+          loader = Cookbook::CookbookVersionLoader.new(cookbook_path, chefignore)
+          loader.load_cookbooks
+          next if loader.empty?
+          @cookbooks_paths[loader.cookbook_name] << cookbook_path # for deprecation warnings
+          if @loaded_cookbooks.key?(loader.cookbook_name)
+            @merged_cookbooks << loader.cookbook_name # for deprecation warnings
+            @loaded_cookbooks[loader.cookbook_name].merge!(loader)
+          else
+            @loaded_cookbooks[loader.cookbook_name] = loader
+          end
+        end
+      end
+
+      @loaded_cookbooks.each do |cookbook, loader|
+        cookbook_version = loader.cookbook_version
+        @cookbooks_by_name[cookbook] = cookbook_version
+        @metadata[cookbook] = cookbook_version.metadata
+      end
+      @cookbooks_by_name
+    end
+
+    def [](cookbook)
+      if @cookbooks_by_name.has_key?(cookbook.to_sym)
+        @cookbooks_by_name[cookbook.to_sym]
+      else
+        raise Exceptions::CookbookNotFoundInRepo, "Cannot find a cookbook named #{cookbook.to_s}; did you forget to add metadata to a cookbook? (http://wiki.opscode.com/display/chef/Metadata)"
+      end
+    end
+
+    alias :fetch :[]
+
+    def has_key?(cookbook_name)
+      @cookbooks_by_name.has_key?(cookbook_name)
+    end
+    alias :cookbook_exists? :has_key?
+    alias :key? :has_key?
+
+    def each
+      @cookbooks_by_name.keys.sort { |a,b| a.to_s <=> b.to_s }.each do |cname|
+        yield(cname, @cookbooks_by_name[cname])
+      end
+    end
+
+    def cookbook_names
+      @cookbooks_by_name.keys.sort
+    end
+
+    def values
+      @cookbooks_by_name.values
+    end
+    alias :cookbooks :values
+
+  end
+end