chef 11.10.0.alpha.1 → 11.10.0.rc.0

data/lib/chef/config.rb

@@ -127,26 +127,6 @@ class Chef
     # properly.
     configurable(:daemonize).writes_value { |v| v }
 
-    # Override the config dispatch to set the value of log_location configuration option
-    #
-    # === Parameters
-    # location<IO||String>:: Logging location as either an IO stream or string representing log file path
-    #
-    config_attr_writer :log_location do |location|
-      if location.respond_to? :sync=
-        location.sync = true
-        location
-      elsif location.respond_to? :to_str
-        begin
-          f = File.new(location.to_str, "a")
-          f.sync = true
-        rescue Errno::ENOENT
-          raise Chef::Exceptions::ConfigurationError, "Failed to open or create log file at #{location.to_str}"
-        end
-        f
-      end
-    end
-
     # The root where all local chef object data is stored.  cookbooks, data bags,
     # environments are all assumed to be in separate directories under this.
     # chef-solo uses these directories for input data.  knife commands
@@ -299,6 +279,9 @@ class Chef
     # logger is the primary mode of output, and the log level is set to :info
     default :log_level, :auto
 
+    # Logging location as either an IO stream or string representing log file path
+    default :log_location, STDOUT
+
     # Using `force_formatter` causes chef to default to formatter output when STDOUT is not a tty
     default :force_formatter, false
 
@@ -310,7 +293,6 @@ class Chef
     default :interval, nil
     default :once, nil
     default :json_attribs, nil
-    default :log_location, STDOUT
     # toggle info level log items that can create a lot of output
     default :verbose_logging, true
     default :node_name, nil
@@ -338,6 +320,16 @@ class Chef
     default :enable_reporting, true
     default :enable_reporting_url_fatals, false
 
+    # Policyfile is an experimental feature where a node gets its run list and
+    # cookbook version set from a single document on the server instead of
+    # expanding the run list and having the server compute the cookbook version
+    # set based on environment constraints.
+    #
+    # Because this feature is experimental, it is not recommended for
+    # production use. Developent/release of this feature may not adhere to
+    # semver guidelines.
+    default :use_policyfile, false
+
     # Set these to enable SSL authentication / mutual-authentication
     # with the server
 
@@ -497,8 +489,12 @@ class Chef
 
       default :fatal_windows_admin_check, false
     else
-      default :user_valid_regex, [ /^([-a-zA-Z0-9_.]+[\\@]?[-a-zA-Z0-9_.]*)$/, /^\d+$/ ]
-      default :group_valid_regex, [ /^([-a-zA-Z0-9_.\\@^ ]+)$/, /^\d+$/ ]
+      # user/group cannot start with '-', '+' or '~'
+      # user/group cannot contain ':', ',' or non-space-whitespace or null byte
+      # everything else is allowed (UTF-8, spaces, etc) and we delegate to your O/S useradd program to barf or not
+      # copies: http://anonscm.debian.org/viewvc/pkg-shadow/debian/trunk/debian/patches/506_relaxed_usernames?view=markup
+      default :user_valid_regex, [ /^[^-+~:,\t\r\n\f\0]+[^:,\t\r\n\f\0]*$/ ]
+      default :group_valid_regex, [ /^[^-+~:,\t\r\n\f\0]+[^:,\t\r\n\f\0]*$/ ]
     end
 
     # returns a platform specific path to the user home dir

data/lib/chef/data_bag.rb

@@ -31,7 +31,7 @@ class Chef
     include Chef::Mixin::FromFile
     include Chef::Mixin::ParamsValidate
 
-    VALID_NAME = /^[\-[:alnum:]_]+$/
+    VALID_NAME = /^[\.\-[:alnum:]_]+$/
 
     def self.validate_name!(name)
       unless name =~ VALID_NAME

data/lib/chef/data_bag_item.rb

@@ -35,7 +35,7 @@ class Chef
     include Chef::Mixin::FromFile
     include Chef::Mixin::ParamsValidate
 
-    VALID_ID = /^[\-[:alnum:]_]+$/
+    VALID_ID = /^[\.\-[:alnum:]_]+$/
 
     def self.validate_id!(id_str)
       if id_str.nil? || ( id_str !~ VALID_ID )

data/lib/chef/exceptions.rb

@@ -297,5 +297,13 @@ class Chef
     # non-GET and non-HEAD request will thus raise an InvalidRedirect.
     class InvalidRedirect < StandardError; end
 
+    # Raised when the content length of a download does not match the content
+    # length declared in the http response.
+    class ContentLengthMismatch < RuntimeError
+      def initialize(response_length, content_length)
+        super "Response body length #{response_length} does not match HTTP Content-Length header #{content_length}."
+      end
+    end
+
   end
 end

data/lib/chef/formatters/doc.rb

@@ -230,6 +230,21 @@ class Chef
         @output.color("\n    * Whyrun not supported for #{resource}, bypassing load.", :yellow)
       end
 
+      # Called before handlers run
+      def handlers_start(handler_count)
+        puts "\nRunning handlers:"
+      end
+
+      # Called after an individual handler has run
+      def handler_executed(handler)
+        puts "  - #{handler.class.name}"
+      end
+
+      # Called after all handlers have executed
+      def handlers_completed
+        puts "Running handlers complete\n"
+      end
+
       # Called when a provider makes an assumption after a failed assertion
       # in whyrun mode, in order to allow execution to continue
       def whyrun_assumption(action, resource, message)

data/lib/chef/formatters/error_inspectors/api_error_formatting.rb

@@ -52,7 +52,8 @@ chef_server_url   "#{server_url}"
 node_name         "#{username}"
 client_key        "#{api_key}"
 
-If these settings are correct, your client_key may be invalid.
+If these settings are correct, your client_key may be invalid, or
+you may have a chef user with the same client name as this node.
 E
         end
       end

data/lib/chef/http.rb

@@ -165,18 +165,22 @@ class Chef
       response, rest_request, return_value = send_http_request(method, url, headers, data) do |http_response|
         if http_response.kind_of?(Net::HTTPSuccess)
           tempfile = stream_to_tempfile(url, http_response)
-          if block_given?
-            begin
-              yield tempfile
-            ensure
-              tempfile && tempfile.close!
-            end
-          end
         end
+        apply_stream_complete_middleware(http_response, rest_request, return_value)
       end
-      unless response.kind_of?(Net::HTTPSuccess) or response.kind_of?(Net::HTTPRedirection)
+
+      return nil if response.kind_of?(Net::HTTPRedirection)
+      unless response.kind_of?(Net::HTTPSuccess)
         response.error!
       end
+
+      if block_given?
+        begin
+          yield tempfile
+        ensure
+          tempfile && tempfile.close!
+        end
+      end
       tempfile
     rescue Exception => e
       log_failed_request(response, return_value) unless response.nil?
@@ -216,6 +220,12 @@ class Chef
       end
     end
 
+    def apply_stream_complete_middleware(response, rest_request, return_value)
+      middlewares.reverse.inject([response, rest_request, return_value]) do |res_data, middleware|
+        middleware.handle_stream_complete(*res_data)
+      end
+    end
+
     def log_failed_request(response, return_value)
       return_value ||= {}
       error_message = "HTTP Request Returned #{response.code} #{response.message}: "

data/lib/chef/http/authenticator.rb

@@ -52,6 +52,10 @@ class Chef
         nil
       end
 
+      def handle_stream_complete(http_response, rest_request, return_value)
+        [http_response, rest_request, return_value]
+      end
+
       def sign_requests?
         auth_credentials.sign_requests? && @sign_request
       end

data/lib/chef/http/cookie_manager.rb

@@ -50,6 +50,9 @@ class Chef
         nil
       end
 
+      def handle_stream_complete(http_response, rest_request, return_value)
+        [http_response, rest_request, return_value]
+      end
 
     end
   end

data/lib/chef/http/decompressor.rb

@@ -69,6 +69,10 @@ class Chef
         [http_response, rest_request, return_value]
       end
 
+      def handle_stream_complete(http_response, rest_request, return_value)
+        [http_response, rest_request, return_value]
+      end
+
       def decompress_body(response)
         if gzip_disabled? || response.body.nil?
           response.body

data/lib/chef/http/json_input.rb

@@ -48,6 +48,10 @@ class Chef
         nil
       end
 
+      def handle_stream_complete(http_response, rest_request, return_value)
+        [http_response, rest_request, return_value]
+      end
+
     end
   end
 end

data/lib/chef/http/json_output.rb

@@ -60,6 +60,10 @@ class Chef
         end
       end
 
+      def handle_stream_complete(http_response, rest_request, return_value)
+        [http_response, rest_request, return_value]
+      end
+
       def stream_response_handler(response)
         nil
       end

data/lib/chef/http/validate_content_length.rb

@@ -0,0 +1,94 @@
+#--
+# Author:: Lamont Granquist (<lamont@getchef.com>)
+# Copyright:: Copyright (c) 2013 Chef Software, Inc.
+# License:: Apache License, Version 2.0
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+require 'pp'
+require 'chef/log'
+
+class Chef
+  class HTTP
+
+    # Middleware that validates the Content-Length header against the downloaded number of bytes.
+    #
+    # This must run before the decompressor middleware, since otherwise we will count the uncompressed
+    # streamed bytes, rather than the on-the-wire compressed bytes.
+    class ValidateContentLength
+
+      class ContentLengthCounter
+        attr_accessor :content_length
+
+        def initialize
+          @content_length = 0
+        end
+
+        def handle_chunk(chunk)
+          @content_length += chunk.bytesize
+          chunk
+        end
+      end
+
+      def initialize(opts={})
+      end
+
+      def handle_request(method, url, headers={}, data=false)
+        [method, url, headers, data]
+      end
+
+      def handle_response(http_response, rest_request, return_value)
+        unless http_response['content-length']
+          Chef::Log.debug("HTTP server did not include a Content-Length header in response, cannot identify truncated downloads.")
+          return [http_response, rest_request, return_value]
+        end
+        validate(response_content_length(http_response), http_response.body.bytesize)
+        return [http_response, rest_request, return_value]
+      end
+
+      def handle_stream_complete(http_response, rest_request, return_value)
+        if http_response['content-length'].nil?
+          Chef::Log.debug("HTTP server did not include a Content-Length header in response, cannot idenfity streamed download.")
+        elsif @content_length_counter.nil?
+          Chef::Log.debug("No content-length information collected for the streamed download, cannot identify streamed download.")
+        else
+          validate(response_content_length(http_response), @content_length_counter.content_length)
+        end
+        return [http_response, rest_request, return_value]
+      end
+
+      def stream_response_handler(response)
+        @content_length_counter = ContentLengthCounter.new
+      end
+
+      private
+      def response_content_length(response)
+        if response['content-length'].is_a?(Array)
+          response['content-length'].first.to_i
+        else
+          response['content-length'].to_i
+        end
+      end
+
+      def validate(content_length, response_length)
+        Chef::Log.debug "Content-Length header = #{content_length}"
+        Chef::Log.debug "Response body length = #{response_length}"
+        if response_length != content_length
+          raise Chef::Exceptions::ContentLengthMismatch.new(response_length, content_length)
+        end
+        true
+      end
+    end
+  end
+end

data/lib/chef/knife.rb

@@ -245,7 +245,6 @@ class Chef
             ENV['PWD']
           end || Dir.pwd
 
-      puts "Working directory: #{a}"
       a
     end
 

data/lib/chef/knife/configure.rb

@@ -35,29 +35,29 @@ class Chef
       option :repository,
         :short => "-r REPO",
         :long => "--repository REPO",
-        :description => "The path to your chef-repo"
+        :description => "The path to the chef-repo"
 
       option :initial,
         :short => "-i",
         :long => "--initial",
         :boolean => true,
-        :description => "Create an initial API User"
+        :description => "Use to create a API client, typically an administrator client on a freshly-installed server"
 
       option :admin_client_name,
         :long => "--admin-client-name NAME",
-        :description => "The existing admin clientname (usually admin)"
+        :description => "The name of the client, typically the name of the admin client"
 
       option :admin_client_key,
         :long => "--admin-client-key PATH",
-        :description => "The path to the admin client's private key (usually a file named admin.pem)"
+        :description => "The path to the private key used by the client, typically a file named admin.pem"
 
       option :validation_client_name,
         :long => "--validation-client-name NAME",
-        :description => "The validation clientname (usually chef-validator)"
+        :description => "The name of the validation client, typically a client named chef-validator"
 
       option :validation_key,
         :long => "--validation-key PATH",
-        :description => "The location of the validation key (usually a file named validation.pem)"
+        :description => "The path to the validation key used by the client, typically a file named validation.pem"
 
       def configure_chef
         # We are just faking out the system so that you can do this without a key specified

data/lib/chef/knife/cookbook_create.rb

@@ -290,7 +290,7 @@ e.g.
 
 Attributes
 ----------
-TODO: List you cookbook attributes here.
+TODO: List your cookbook attributes here.
 
 e.g.
 #### #{cookbook_name}::default
@@ -358,7 +358,7 @@ Requirements
   toaster         #{cookbook_name} needs toaster to brown your bagel.
 
 Attributes
-  TODO: List you cookbook attributes here.
+  TODO: List your cookbook attributes here.
 
   #{cookbook_name}
   Key                                   Type        Description                           Default

data/lib/chef/knife/core/subcommand_loader.rb

@@ -60,9 +60,13 @@ class Chef
       # subcommand loader has been modified to load the plugins by using Kernel.load
       # with the absolute path.
       def gem_and_builtin_subcommands
-        # search all gems for chef/knife/*.rb
-        require 'rubygems'
-        find_subcommands_via_rubygems
+        if have_plugin_manifest?
+          find_subcommands_via_manifest
+        else
+          # search all gems for chef/knife/*.rb
+          require 'rubygems'
+          find_subcommands_via_rubygems
+        end
       rescue LoadError
         find_subcommands_via_dirglob
       end
@@ -71,6 +75,36 @@ class Chef
         @subcommand_files ||= (gem_and_builtin_subcommands.values + site_subcommands).flatten.uniq
       end
 
+      # If the user has created a ~/.chef/plugin_manifest.json file, we'll use
+      # that instead of inspecting the on-system gems to find the plugins. The
+      # file format is expected to look like:
+      #
+      #   { "plugins": {
+      #       "knife-ec2": {
+      #         "paths": [
+      #           "/home/alice/.rubymanagerthing/gems/knife-ec2-x.y.z/lib/chef/knife/ec2_server_create.rb",
+      #           "/home/alice/.rubymanagerthing/gems/knife-ec2-x.y.z/lib/chef/knife/ec2_server_delete.rb"
+      #         ]
+      #       }
+      #     }
+      #   }
+      #
+      # Extraneous content in this file is ignored. This intentional so that we
+      # can adapt the file format for potential behavior changes to knife in
+      # the future.
+      def find_subcommands_via_manifest
+        # Format of subcommand_files is "relative_path" (something you can
+        # Kernel.require()) => full_path. The relative path isn't used
+        # currently, so we just map full_path => full_path.
+        subcommand_files = {}
+        plugin_manifest["plugins"].each do |plugin_name, plugin_manifest|
+          plugin_manifest["paths"].each do |cmd_path|
+            subcommand_files[cmd_path] = cmd_path
+          end
+        end
+        subcommand_files.merge(find_subcommands_via_dirglob)
+      end
+
       def find_subcommands_via_dirglob
         # The "require paths" of the core knife subcommands bundled with chef
         files = Dir[File.expand_path('../../../knife/*.rb', __FILE__)]
@@ -93,6 +127,18 @@ class Chef
         subcommand_files.merge(find_subcommands_via_dirglob)
       end
 
+      def have_plugin_manifest?
+        ENV["HOME"] && File.exist?(plugin_manifest_path)
+      end
+
+      def plugin_manifest
+        Chef::JSONCompat.from_json(File.read(plugin_manifest_path))
+      end
+
+      def plugin_manifest_path
+        File.join(ENV['HOME'], '.chef', 'plugin_manifest.json')
+      end
+
       private
 
       def find_files_latest_gems(glob, check_load_path=true)