chef 10.34.6 → 11.0.0.beta.0

data/lib/chef/chef_fs/path_utils.rb

@@ -0,0 +1,64 @@
+#
+# Author:: John Keiser (<jkeiser@opscode.com>)
+# Copyright:: Copyright (c) 2012 Opscode, Inc.
+# License:: Apache License, Version 2.0
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+require 'chef/chef_fs'
+
+class Chef
+  module ChefFS
+    class PathUtils
+
+      # If you are in 'source', this is what you would have to type to reach 'dest'
+      # relative_to('/a/b/c/d/e', '/a/b/x/y') == '../../c/d/e'
+      # relative_to('/a/b', '/a/b') == ''
+      def self.relative_to(dest, source)
+        # Skip past the common parts
+        source_parts = Chef::ChefFS::PathUtils.split(source)
+        dest_parts = Chef::ChefFS::PathUtils.split(dest)
+        i = 0
+        until i >= source_parts.length || i >= dest_parts.length || source_parts[i] != source_parts[i]
+          i+=1
+        end
+        # dot-dot up from 'source' to the common ancestor, then
+        # descend to 'dest' from the common ancestor
+        result = Chef::ChefFS::PathUtils.join(*(['..']*(source_parts.length-i) + dest_parts[i,dest.length-i]))
+        result == '' ? '.' : result
+      end
+
+      def self.join(*parts)
+        return "" if parts.length == 0
+        # Determine if it started with a slash
+        absolute = parts[0].length == 0 || parts[0].length > 0 && parts[0] =~ /^#{regexp_path_separator}/
+        # Remove leading and trailing slashes from each part so that the join will work (and the slash at the end will go away)
+        parts = parts.map { |part| part.gsub(/^\/|\/$/, "") }
+        # Don't join empty bits
+        result = parts.select { |part| part != "" }.join("/")
+        # Put the / back on
+        absolute ? "/#{result}" : result
+      end
+
+      def self.split(path)
+        path.split(Regexp.new(regexp_path_separator))
+      end
+
+      def self.regexp_path_separator
+        Chef::ChefFS::windows? ? '[/\\]' : '/'
+      end
+
+    end
+  end
+end

data/lib/chef/client.rb

@@ -24,6 +24,7 @@ require 'chef/mixin/path_sanity'
 require 'chef/log'
 require 'chef/rest'
 require 'chef/api_client'
+require 'chef/api_client/registration'
 require 'chef/platform'
 require 'chef/node'
 require 'chef/role'
@@ -41,6 +42,7 @@ require 'chef/formatters/doc'
 require 'chef/formatters/minimal'
 require 'chef/version'
 require 'chef/resource_reporter'
+require 'chef/run_lock'
 require 'ohai'
 require 'rbconfig'
 
@@ -127,9 +129,9 @@ class Chef
     #--
     # TODO: timh/cw: 5-19-2010: json_attribs should be moved to RunContext?
     attr_reader :json_attribs
-
     attr_reader :run_status
-
+    attr_reader :events
+    
     # Creates a new Chef::Client.
     def initialize(json_attribs=nil, args={})
       @json_attribs = json_attribs
@@ -138,15 +140,6 @@ class Chef
       @runner = nil
       @ohai = Ohai::System.new
 
-      # If we want why-run output and user hasn't explicitly specified a format
-      # we need to use a formatter that will render whyrun output.
-      if Chef::Config.why_run
-        if Chef::Config.formatter == "null"
-          Chef::Log.warn("Forcing formatter of 'doc' to capture whyrun output.")
-          Chef::Config[:formatter] = 'doc'
-        end
-      end
-
       event_handlers = configure_formatters
       event_handlers += Array(Chef::Config[:event_handlers])
 
@@ -156,18 +149,42 @@ class Chef
     end
 
     def configure_formatters
-      [Chef::Formatters.new(Chef::Config.formatter, STDOUT, STDERR)]
+      formatters_for_run.map do |formatter_name, output_path|
+        if output_path.nil?
+          Chef::Formatters.new(formatter_name, STDOUT, STDERR)
+        else
+          io = File.open(output_path, "a+")
+          io.sync = true
+          Chef::Formatters.new(formatter_name, io, io)
+        end
+      end
+    end
+
+    def formatters_for_run
+      if Chef::Config.formatters.empty?
+        [default_formatter]
+      else
+        Chef::Config.formatters
+      end
+    end
+
+    def default_formatter
+      if (STDOUT.tty? && !Chef::Config[:force_logger]) || Chef::Config[:force_formatter]
+        [:doc]
+      else
+        [:null]
+      end
     end
 
     # Do a full run for this Chef::Client.  Calls:
     # * do_run
     #
-    # This provides a wrapper around #do_run allowing the
+    # This provides a wrapper around #do_run allowing the 
     # run to be optionally forked.
     # === Returns
     # boolean:: Return value from #do_run. Should always returns true.
     def run
-      if(Chef::Config[:client_fork] && Process.respond_to?(:fork) && !Chef::Platform.windows?)
+      if(Chef::Config[:client_fork] && Process.respond_to?(:fork))
         Chef::Log.info "Forking chef instance to converge..."
         pid = fork do
           Chef::Log.info "Forked instance now converging"
@@ -192,7 +209,9 @@ class Chef
     def setup_run_context
       if Chef::Config[:solo]
         Chef::Cookbook::FileVendor.on_create { |manifest| Chef::Cookbook::FileSystemFileVendor.new(manifest, Chef::Config[:cookbook_path]) }
-        cookbook_collection = Chef::CookbookCollection.new(Chef::CookbookLoader.new(Chef::Config[:cookbook_path]))
+        cl = Chef::CookbookLoader.new(Chef::Config[:cookbook_path])
+        cl.load_cookbooks
+        cookbook_collection = Chef::CookbookCollection.new(cl)
         run_context = Chef::RunContext.new(node, cookbook_collection, @events)
       else
         Chef::Cookbook::FileVendor.on_create { |manifest| Chef::Cookbook::RemoteFileVendor.new(manifest, rest) }
@@ -212,7 +231,7 @@ class Chef
         Chef::Log.debug("Saving the current state of node #{node_name}")
         if(@original_runlist)
           @node.run_list(*@original_runlist)
-          @node.set[:runlist_override_history] = {Time.now.to_i => @override_runlist.inspect}
+          @node.automatic_attrs[:runlist_override_history] = {Time.now.to_i => @override_runlist.inspect}
         end
         @node.save
       end
@@ -248,7 +267,7 @@ class Chef
     def build_node
       # Allow user to override the environment of a node by specifying
       # a config parameter.
-      if Chef::Config[:environment] && !Chef::Config[:environment].chomp.empty?
+      if Chef::Config[:environment] && !Chef::Config[:environment].chop.empty?
         @node.chef_environment(Chef::Config[:environment])
       end
 
@@ -329,7 +348,7 @@ class Chef
       else
         @events.registration_start(node_name, config)
         Chef::Log.info("Client key #{config[:client_key]} is not present - registering")
-        Chef::REST.new(config[:client_url], config[:validation_client_name], config[:validation_key]).register(client_name, config[:client_key])
+        Chef::ApiClient::Registration.new(node_name, config[:client_key]).run
         @events.registration_completed
       end
       # We now have the client key, and should use it from now on.
@@ -385,7 +404,7 @@ class Chef
       runner.converge
       @events.converge_complete
       true
-    rescue Exception => e
+    rescue Exception
       # TODO: should this be a separate #converge_failed(exception) method?
       @events.converge_complete
       raise
@@ -404,8 +423,10 @@ class Chef
     # === Returns
     # true:: Always returns true.
     def do_run
-      run_context = nil
+      runlock = RunLock.new(Chef::Config)
+      runlock.acquire
 
+      run_context = nil
       @events.run_start(Chef::VERSION)
       Chef::Log.info("*** Chef #{Chef::VERSION} ***")
       enforce_path_sanity
@@ -445,8 +466,9 @@ class Chef
         @events.run_failed(e)
         raise
       ensure
-        run_status = nil
+        @run_status = nil
         run_context = nil
+        runlock.release
         GC.start
       end
       true
@@ -503,3 +525,4 @@ end
 require 'chef/cookbook_loader'
 require 'chef/cookbook_version'
 require 'chef/cookbook/synchronizer'
+

data/lib/chef/config.rb

@@ -48,9 +48,6 @@ class Chef
     end
 
     def self.platform_specific_path(path)
-      #10.times { puts "* " * 40}
-      #pp caller
-
       if RUBY_PLATFORM =~ /mswin|mingw|windows/
         # turns /etc/chef/client.rb into C:/chef/client.rb
         system_drive = ENV['SYSTEMDRIVE'] ? ENV['SYSTEMDRIVE'] : ""
@@ -61,6 +58,14 @@ class Chef
       path
     end
 
+    def self.add_formatter(name, file_path=nil)
+      formatters << [name, file_path]
+    end
+
+    def self.formatters
+      @formatters ||= []
+    end
+
     # Override the config dispatch to set the value of multiple server options simultaneously
     #
     # === Parameters
@@ -104,7 +109,7 @@ class Chef
         begin
           f = File.new(location.to_str, "a")
           f.sync = true
-        rescue Errno::ENOENT => error
+        rescue Errno::ENOENT
           raise Chef::Exceptions::ConfigurationError("Failed to open or create log file at #{location.to_str}")
         end
           f
@@ -130,8 +135,6 @@ class Chef
     # Used when OpenID authentication is enabled in the Web UI
     authorized_openid_identifiers nil
     authorized_openid_providers nil
-    openid_cstore_couchdb false
-    openid_cstore_path "/var/chef/openid/cstore"
 
     # The number of times the client should retry when registering with the server
     client_registration_retries 5
@@ -150,29 +153,51 @@ class Chef
     # Where cookbook files are stored on the server (by content checksum)
     checksum_path "/var/chef/checksums"
 
-    # CouchDB database name to use
-    couchdb_database "chef"
-
-    couchdb_url "http://localhost:5984"
-
     # Where chef's cache files should be stored
     file_cache_path platform_specific_path("/var/chef/cache")
 
+    # By default, chef-client (or solo) creates a lockfile in
+    # `file_cache_path`/chef-client-running.pid
+    # If `lockfile` is explicitly set, this path will be used instead.
+    #
+    # If your `file_cache_path` resides on a NFS (or non-flock()-supporting
+    # fs), it's recommended to set this to something like
+    # '/tmp/chef-client-running.pid'
+    lockfile nil
+
     # Where backups of chef-managed files should go
     file_backup_path platform_specific_path("/var/chef/backup")
 
     ## Daemonization Settings ##
     # What user should Chef run as?
     user nil
-    # What group should the chef-server, -solr, -solr-indexer run as
     group nil
     umask 0022
 
+
+    # Valid log_levels are:
+    # * :debug
+    # * :info
+    # * :warn
+    # * :fatal
+    # These work as you'd expect. There is also a special `:auto` setting.
+    # When set to :auto, Chef will auto adjust the log verbosity based on
+    # context. When a tty is available (usually becase the user is running chef
+    # in a console), the log level is set to :warn, and output formatters are
+    # used as the primary mode of output. When a tty is not available, the
+    # logger is the primary mode of output, and the log level is set to :info
+    log_level :auto
+
+    # Using `force_formatter` causes chef to default to formatter output when STDOUT is not a tty
+    force_formatter false
+
+    # Using `force_logger` causes chef to default to logger output when STDOUT is a tty
+    force_logger false
+
     http_retry_count 5
     http_retry_delay 5
     interval nil
     json_attribs nil
-    log_level :info
     log_location STDOUT
     # toggle info level log items that can create a lot of output
     verbose_logging true
@@ -194,7 +219,6 @@ class Chef
     client_url "http://localhost:4042"
 
     rest_timeout 300
-    yum_timeout 120
     run_command_stderr_timeout 120
     run_command_stdout_timeout 120
     solo  false
@@ -222,23 +246,6 @@ class Chef
     # Where should chef-solo download recipes from?
     recipe_url nil
 
-    solr_url "http://localhost:8983/solr"
-    solr_jetty_path "/var/chef/solr-jetty"
-    solr_data_path "/var/chef/solr/data"
-    solr_home_path "/var/chef/solr"
-    solr_heap_size "256M"
-    solr_java_opts nil
-
-    # Parameters for connecting to RabbitMQ
-    amqp_host '0.0.0.0'
-    amqp_port '5672'
-    amqp_user 'chef'
-    amqp_pass 'testing'
-    amqp_vhost '/chef'
-    # Setting this to a UUID string also makes the queue durable
-    # (persist across rabbitmq restarts)
-    amqp_consumer_id "default"
-
     # Sets the version of the signed header authentication protocol to use (see
     # the 'mixlib-authorization' project for more detail). Currently, versions
     # 1.0 and 1.1 are available; however, the chef-server must first be
@@ -293,19 +300,25 @@ class Chef
     # Report Handlers
     report_handlers []
 
-    # Event Handlers
-    event_handlers []
-
     # Exception Handlers
     exception_handlers []
 
     # Start handlers
     start_handlers []
 
-    # Checksum Cache
-    # Uses Moneta on the back-end
-    cache_type "BasicFile"
-    cache_options({ :path => platform_specific_path("/var/chef/cache/checksums"), :skip_expires => true })
+    # Syntax Check Cache. Knife keeps track of files that is has already syntax
+    # checked by storing files in this directory. `syntax_check_cache_path` is
+    # the new (and preferred) configuration setting. If not set, knife will
+    # fall back to using cache_options[:path].
+    #
+    # Because many users will have knife configs with cache_options (generated
+    # by `knife configure`), the default for now is to *not* set
+    # syntax_check_cache_path, and thus fallback to cache_options[:path]. We
+    # leave that value to the same default as was previously set.
+    syntax_check_cache_path nil
+
+    # Deprecated:
+    cache_options({ :path => platform_specific_path("/var/chef/cache/checksums") })
 
     # Set to false to silence Chef 11 deprecation warnings:
     chef11_deprecation_warnings true
@@ -328,10 +341,6 @@ class Chef
 
     # returns a platform specific path to the user home dir
     windows_home_path = ENV['SYSTEMDRIVE'] + ENV['HOMEPATH'] if ENV['SYSTEMDRIVE'] && ENV['HOMEPATH']
-    user_home (ENV['HOME'] || windows_home_path || ENV['USERPROFILE'])
-
-    # CHEF-4631
-    # Enables the concatanation behavior instead of merging
-    deep_merge_array_concat true
+    user_home(ENV['HOME'] || windows_home_path || ENV['USERPROFILE'])
   end
 end

data/lib/chef/cookbook/synchronizer.rb

@@ -56,14 +56,12 @@ class Chef
   # Synchronizes the locally cached copies of cookbooks with the files on the
   # server.
   class CookbookSynchronizer
-    def initialize(cookbooks_by_name, events)
-      @eager_segments = Chef::CookbookVersion::COOKBOOK_SEGMENTS.dup
-      unless Chef::Config[:no_lazy_load]
-        @eager_segments.delete(:files)
-        @eager_segments.delete(:templates)
-      end
-      @eager_segments.freeze
+    EAGER_SEGMENTS = Chef::CookbookVersion::COOKBOOK_SEGMENTS.dup
+    EAGER_SEGMENTS.delete(:files)
+    EAGER_SEGMENTS.delete(:templates)
+    EAGER_SEGMENTS.freeze
 
+    def initialize(cookbooks_by_name, events)
       @cookbooks_by_name, @events = cookbooks_by_name, events
     end
 
@@ -140,7 +138,7 @@ class Chef
 
       # files and templates are lazily loaded, and will be done later.
 
-      @eager_segments.each do |segment|
+      EAGER_SEGMENTS.each do |segment|
         segment_filenames = Array.new
         cookbook.manifest[segment].each do |manifest_record|
 

data/lib/chef/cookbook/syntax_check.rb

@@ -16,8 +16,8 @@
 # limitations under the License.
 #
 
-require 'chef/checksum_cache'
 require 'chef/mixin/shell_out'
+require 'chef/mixin/checksum'
 
 class Chef
   class Cookbook
@@ -25,10 +25,65 @@ class Chef
     # Encapsulates the process of validating the ruby syntax of files in Chef
     # cookbooks.
     class SyntaxCheck
+
+      # == Chef::Cookbook::SyntaxCheck::PersistentSet
+      # Implements set behavior with disk-based persistence. Objects in the set
+      # are expected to be strings containing only characters that are valid in
+      # filenames.
+      #
+      # This class is used to track which files have been syntax checked so
+      # that known good files are not rechecked.
+      class PersistentSet
+
+        attr_reader :cache_path
+
+        # Create a new PersistentSet. Values in the set are persisted by
+        # creating a file in the +cache_path+ directory. If not given, the
+        # value of Chef::Config[:syntax_check_cache_path] is used; if that
+        # value is not configured, the value of
+        # Chef::Config[:cache_options][:path] is used.
+        #--
+        # history: prior to Chef 11, the cache implementation was based on
+        # moneta and configured via cache_options[:path]. Knife configs
+        # generated with Chef 11 will have `syntax_check_cache_path`, but older
+        # configs will have `cache_options[:path]`. `cache_options` is marked
+        # deprecated in chef/config.rb but doesn't currently trigger a warning.
+        # See also: CHEF-3715
+        def initialize(cache_path=nil)
+          @cache_path = cache_path || Chef::Config[:syntax_check_cache_path] || Chef::Config[:cache_options][:path]
+          @cache_path_created = false
+        end
+
+        # Adds +value+ to the set's collection.
+        def add(value)
+          ensure_cache_path_created
+          FileUtils.touch(File.join(cache_path, value))
+        end
+
+        # Returns true if the set includes +value+
+        def include?(value)
+          File.exist?(File.join(cache_path, value))
+        end
+
+        private
+
+        def ensure_cache_path_created
+          return true if @cache_path_created
+          FileUtils.mkdir_p(cache_path)
+          @cache_path_created = true
+        end
+
+      end
+
       include Chef::Mixin::ShellOut
+      include Chef::Mixin::Checksum
 
       attr_reader :cookbook_path
 
+      # A PersistentSet object that tracks which files have already been
+      # validated.
+      attr_reader :validated_files
+
       # Creates a new SyntaxCheck given the +cookbook_name+ and a +cookbook_path+.
       # If no +cookbook_path+ is given, +Chef::Config.cookbook_path+ is used.
       def self.for_cookbook(cookbook_name, cookbook_path=nil)
@@ -44,10 +99,7 @@ class Chef
       # cookbook_path::: the (on disk) path to the cookbook
       def initialize(cookbook_path)
         @cookbook_path = cookbook_path
-      end
-
-      def cache
-        Chef::ChecksumCache.instance
+        @validated_files = PersistentSet.new
       end
 
       def ruby_files
@@ -81,16 +133,11 @@ class Chef
       end
 
       def validated?(file)
-        !!cache.lookup_checksum(cache_key(file), File.stat(file))
+        validated_files.include?(checksum(file))
       end
 
       def validated(file)
-        cache.generate_checksum(cache_key(file), file, File.stat(file))
-      end
-
-      def cache_key(file)
-        @cache_keys ||= {}
-        @cache_keys[file] ||= cache.generate_key(file, "chef-test")
+        validated_files.add(checksum(file))
       end
 
       def validate_ruby_files
@@ -118,7 +165,7 @@ class Chef
         result.stderr.each_line { |l| Chef::Log.fatal(l.chomp) }
         false
       end
-      
+
       def validate_ruby_file(ruby_file)
         Chef::Log.debug("Testing #{ruby_file} for syntax errors...")
         result = shell_out("ruby -c #{ruby_file}")
@@ -130,7 +177,7 @@ class Chef
         result.stderr.each_line { |l| Chef::Log.fatal(l.chomp) }
         false
       end
-      
+
     end
   end
 end