chef 13.12.14-universal-mingw32 → 14.0.190-universal-mingw32

data/lib/chef/cookbook/syntax_check.rb

@@ -124,7 +124,7 @@ class Chef
       def untested_ruby_files
         ruby_files.reject do |file|
           if validated?(file)
-            Chef::Log.debug("Ruby file #{file} is unchanged, skipping syntax check")
+            Chef::Log.trace("Ruby file #{file} is unchanged, skipping syntax check")
             true
           else
             false
@@ -139,7 +139,7 @@ class Chef
       def untested_template_files
         template_files.reject do |file|
           if validated?(file)
-            Chef::Log.debug("Template #{file} is unchanged, skipping syntax check")
+            Chef::Log.trace("Template #{file} is unchanged, skipping syntax check")
             true
           else
             false
@@ -170,12 +170,12 @@ class Chef
       end
 
       def validate_template(erb_file)
-        Chef::Log.debug("Testing template #{erb_file} for syntax errors...")
+        Chef::Log.trace("Testing template #{erb_file} for syntax errors...")
         validate_erb_file_inline(erb_file)
       end
 
       def validate_ruby_file(ruby_file)
-        Chef::Log.debug("Testing #{ruby_file} for syntax errors...")
+        Chef::Log.trace("Testing #{ruby_file} for syntax errors...")
         validate_ruby_file_inline(ruby_file)
       end
 

data/lib/chef/cookbook_loader.rb

@@ -2,7 +2,7 @@
 # Author:: Adam Jacob (<adam@chef.io>)
 # Author:: Christopher Walters (<cw@chef.io>)
 # Author:: Daniel DeLeo (<dan@kallistec.com>)
-# Copyright:: Copyright 2008-2016, Chef Software Inc.
+# Copyright:: Copyright 2008-2018, Chef Software Inc.
 # Copyright:: Copyright 2009-2016, Daniel DeLeo
 # License:: Apache License, Version 2.0
 #
@@ -78,7 +78,7 @@ class Chef
     # @api private
     def load_cookbooks_without_shadow_warning
       preload_cookbooks
-      @loaders_by_name.each do |cookbook_name, _loaders|
+      @loaders_by_name.each_key do |cookbook_name|
         load_cookbook(cookbook_name)
       end
       @cookbooks_by_name
@@ -142,6 +142,14 @@ class Chef
       end
     end
 
+    def each_key(&block)
+      cookbook_names.each(&block)
+    end
+
+    def each_value(&block)
+      values.each(&block)
+    end
+
     def cookbook_names
       @cookbooks_by_name.keys.sort
     end

data/lib/chef/cookbook_site_streaming_uploader.rb

@@ -31,7 +31,7 @@ class Chef
   # inspired by http://stanislavvitvitskiy.blogspot.com/2008/12/multipart-post-in-ruby.html
   class CookbookSiteStreamingUploader
 
-    DefaultHeaders = { "accept" => "application/json", "x-chef-version" => ::Chef::VERSION } # rubocop:disable Style/ConstantName
+    DefaultHeaders = { "accept" => "application/json", "x-chef-version" => ::Chef::VERSION } # rubocop:disable Naming/ConstantName
 
     class << self
 
@@ -41,19 +41,19 @@ class Chef
         tmp_cookbook_dir = tmp_cookbook_path.path
         File.unlink(tmp_cookbook_dir)
         FileUtils.mkdir_p(tmp_cookbook_dir)
-        Chef::Log.debug("Staging at #{tmp_cookbook_dir}")
+        Chef::Log.trace("Staging at #{tmp_cookbook_dir}")
         checksums_to_on_disk_paths = cookbook.checksums
         cookbook.each_file do |manifest_record|
           path_in_cookbook = manifest_record[:path]
           on_disk_path = checksums_to_on_disk_paths[manifest_record[:checksum]]
           dest = File.join(tmp_cookbook_dir, cookbook.name.to_s, path_in_cookbook)
           FileUtils.mkdir_p(File.dirname(dest))
-          Chef::Log.debug("Staging #{on_disk_path} to #{dest}")
+          Chef::Log.trace("Staging #{on_disk_path} to #{dest}")
           FileUtils.cp(on_disk_path, dest)
         end
 
         # First, generate metadata
-        Chef::Log.debug("Generating metadata")
+        Chef::Log.trace("Generating metadata")
         kcm = Chef::Knife::CookbookMetadata.new
         kcm.config[:cookbook_path] = [ tmp_cookbook_dir ]
         kcm.name_args = [ cookbook.name.to_s ]

data/lib/chef/cookbook_uploader.rb

@@ -71,14 +71,14 @@ class Chef
           Chef::Log.info("Uploading #{checksum_files[checksum]} (checksum hex = #{checksum}) to #{info['url']}")
           queue << uploader_function_for(checksum_files[checksum], checksum, info["url"], checksums_to_upload)
         else
-          Chef::Log.debug("#{checksum_files[checksum]} has not changed")
+          Chef::Log.trace("#{checksum_files[checksum]} has not changed")
         end
       end
 
       queue.process(@concurrency)
 
       sandbox_url = new_sandbox["uri"]
-      Chef::Log.debug("Committing sandbox")
+      Chef::Log.trace("Committing sandbox")
       # Retry if S3 is claims a checksum doesn't exist (the eventual
       # in eventual consistency)
       retries = 0
@@ -145,7 +145,6 @@ class Chef
         Chef::Log.info("Validating templates")
         exit(1) unless syntax_checker.validate_templates
         Chef::Log.info("Syntax OK")
-        true
       end
     end
 

data/lib/chef/cookbook_version.rb

@@ -4,7 +4,7 @@
 # Author:: Tim Hinderliter (<tim@chef.io>)
 # Author:: Seth Falcon (<seth@chef.io>)
 # Author:: Daniel DeLeo (<dan@chef.io>)
-# Copyright:: Copyright 2008-2016, Chef Software, Inc.
+# Copyright:: Copyright 2008-2018, Chef Software Inc.
 # License:: Apache License, Version 2.0
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
@@ -54,11 +54,6 @@ class Chef
     # CookbookVersion.
     attr_reader :metadata
 
-    attr_reader :recipe_filenames_by_name
-    attr_reader :attribute_filenames_by_short_filename
-
-    attr_accessor :chef_server_rest
-
     # The `identifier` field is used for cookbook_artifacts, which are
     # organized on the chef server according to their content. If the
     # policy_mode option to CookbookManifest is set to true it will include
@@ -84,7 +79,7 @@ class Chef
     def self.checksum_cookbook_file(filepath)
       Chef::Digester.generate_md5_checksum_for_file(filepath)
     rescue Errno::ENOENT
-      Chef::Log.debug("File #{filepath} does not exist, so there is no checksum to generate")
+      Chef::Log.trace("File #{filepath} does not exist, so there is no checksum to generate")
       nil
     end
 
@@ -157,7 +152,6 @@ class Chef
     def metadata=(metadata)
       @metadata = metadata
       @metadata.recipes_from_cookbook_version(self)
-      @metadata
     end
 
     def manifest
@@ -194,7 +188,7 @@ class Chef
         raise Chef::Exceptions::RecipeNotFound, "could not find recipe #{recipe_name} for cookbook #{name}"
       end
 
-      Chef::Log.debug("Found recipe #{recipe_name} in cookbook #{name}")
+      Chef::Log.trace("Found recipe #{recipe_name} in cookbook #{name}")
       recipe = Chef::Recipe.new(name, recipe_name, run_context)
       recipe_filename = recipe_filenames_by_name[recipe_name]
 
@@ -301,9 +295,9 @@ class Chef
         # we're just going to make cookbook_files out of these and make the
         # cookbook find them by filespecificity again. but it's the shortest
         # path to "success" for now.
-        if manifest_record_path =~ /(#{Regexp.escape(segment.to_s)}\/[^\/]*\/?#{Regexp.escape(dirname)})\/.+$/
+        if manifest_record_path =~ /(#{Regexp.escape(segment.to_s)}\/[^\/]+\/#{Regexp.escape(dirname)})\/.+$/
           specificity_dirname = $1
-          non_specific_path = manifest_record_path[/#{Regexp.escape(segment.to_s)}\/[^\/]*\/?#{Regexp.escape(dirname)}\/(.+)$/, 1]
+          non_specific_path = manifest_record_path[/#{Regexp.escape(segment.to_s)}\/[^\/]+\/#{Regexp.escape(dirname)}\/(.+)$/, 1]
           # Record the specificity_dirname only if it's in the list of
           # valid preferences
           if filenames_by_pref[specificity_dirname]

data/lib/chef/data_bag.rb

@@ -2,7 +2,7 @@
 # Author:: Adam Jacob (<adam@chef.io>)
 # Author:: Nuo Yan (<nuo@chef.io>)
 # Author:: Christopher Brown (<cb@chef.io>)
-# Copyright:: Copyright 2009-2016, Chef Software Inc.
+# Copyright:: Copyright 2009-2018, Chef Software Inc.
 # License:: Apache License, Version 2.0
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
@@ -35,8 +35,6 @@ class Chef
     VALID_NAME = /^[\.\-[:alnum:]_]+$/
     RESERVED_NAMES = /^(node|role|environment|client)$/
 
-    attr_accessor :chef_server_rest
-
     def self.validate_name!(name)
       unless name =~ VALID_NAME
         raise Exceptions::InvalidDataBagName, "DataBags must have a name matching #{VALID_NAME.inspect}, you gave #{name.inspect}"

data/lib/chef/data_bag_item.rb

@@ -2,7 +2,7 @@
 # Author:: Adam Jacob (<adam@chef.io>)
 # Author:: Nuo Yan (<nuo@chef.io>)
 # Author:: Christopher Brown (<cb@chef.io>)
-# Copyright:: Copyright 2009-2016, Chef Software, Inc.
+# Copyright:: Copyright 2009-2018, Chef Software Inc.
 # License:: Apache License, Version 2.0
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
@@ -38,8 +38,6 @@ class Chef
 
     VALID_ID = /^[\.\-[:alnum:]_]+$/
 
-    attr_accessor :chef_server_rest
-
     def self.validate_id!(id_str)
       if id_str.nil? || ( id_str !~ VALID_ID )
         raise Exceptions::InvalidDataBagItemID, "Data Bag items must have an id matching #{VALID_ID.inspect}, you gave: #{id_str.inspect}"
@@ -66,10 +64,6 @@ class Chef
       Chef::ServerAPI.new(Chef::Config[:chef_server_url])
     end
 
-    def raw_data
-      @raw_data
-    end
-
     def validate_id!(id_str)
       self.class.validate_id!(id_str)
     end

data/lib/chef/data_collector.rb

@@ -42,20 +42,20 @@ class Chef
     #   falsey value
     def self.register_reporter?
       if why_run?
-        Chef::Log.debug("data collector is disabled for why run mode")
+        Chef::Log.trace("data collector is disabled for why run mode")
         return false
       end
       unless reporter_enabled_for_current_mode?
-        Chef::Log.debug("data collector is configured to only run in " \
+        Chef::Log.trace("data collector is configured to only run in " \
                         "#{Chef::Config[:data_collector][:mode].inspect} modes, disabling it")
         return false
       end
-      unless data_collector_url_configured?
-        Chef::Log.debug("data collector URL is not configured, disabling data collector")
+      unless data_collector_url_configured? || data_collector_output_locations_configured?
+        Chef::Log.trace("Neither data collector URL or output locations have been configured, disabling data collector")
         return false
       end
       if solo? && !token_auth_configured?
-        Chef::Log.debug("Data collector token must be configured to use Chef Automate data collector with Chef Solo")
+        Chef::Log.trace("Data collector token must be configured to use Chef Automate data collector with Chef Solo")
       end
       if !solo? && token_auth_configured?
         Chef::Log.warn("Data collector token authentication is not recommended for client-server mode" \
@@ -69,6 +69,10 @@ class Chef
       !!Chef::Config[:data_collector][:server_url]
     end
 
+    def self.data_collector_output_locations_configured?
+      !!Chef::Config[:data_collector][:output_locations]
+    end
+
     def self.why_run?
       !!Chef::Config[:why_run]
     end
@@ -104,7 +108,7 @@ class Chef
 
       def initialize
         validate_data_collector_server_url!
-
+        validate_data_collector_output_locations! if data_collector_output_locations
         @all_resource_reports    = []
         @current_resource_loaded = nil
         @error_descriptions      = {}
@@ -112,7 +116,10 @@ class Chef
         @deprecations            = Set.new
         @enabled                 = true
 
-        @http = setup_http_client
+        @http = setup_http_client(data_collector_server_url)
+        if data_collector_output_locations
+          @http_output_locations = setup_http_output_locations if data_collector_output_locations[:urls]
+        end
       end
 
       # see EventDispatch::Base#run_started
@@ -125,11 +132,11 @@ class Chef
       def run_started(current_run_status)
         update_run_status(current_run_status)
 
+        message = Chef::DataCollector::Messages.run_start_message(current_run_status)
         disable_reporter_on_error do
-          send_to_data_collector(
-            Chef::DataCollector::Messages.run_start_message(current_run_status)
-          )
+          send_to_data_collector(message)
         end
+        send_to_output_locations(message) if data_collector_output_locations
       end
 
       # see EventDispatch::Base#run_completed
@@ -286,11 +293,17 @@ class Chef
       # intended to be used primarily for Chef Solo in which case no signing
       # key will be available (in which case `Chef::ServerAPI.new()` would
       # raise an exception.
-      def setup_http_client
+      def setup_http_client(url)
         if data_collector_token.nil?
-          Chef::ServerAPI.new(data_collector_server_url, validate_utf8: false)
+          Chef::ServerAPI.new(url, validate_utf8: false)
         else
-          Chef::HTTP::SimpleJSON.new(data_collector_server_url, validate_utf8: false)
+          Chef::HTTP::SimpleJSON.new(url, validate_utf8: false)
+        end
+      end
+
+      def setup_http_output_locations
+        Chef::Config[:data_collector][:output_locations][:urls].each_with_object({}) do |location_url, http_output_locations|
+          http_output_locations[location_url] = setup_http_client(location_url)
         end
       end
 
@@ -309,7 +322,8 @@ class Chef
              Errno::ECONNREFUSED, EOFError, Net::HTTPBadResponse,
              Net::HTTPHeaderSyntaxError, Net::ProtocolError, OpenSSL::SSL::SSLError,
              Errno::EHOSTDOWN => e
-        disable_data_collector_reporter
+        # Do not disable data collector reporter if additional output_locations have been specified
+        disable_data_collector_reporter unless data_collector_output_locations
         code = if e.respond_to?(:response) && e.response.code
                  e.response.code.to_s
                else
@@ -332,8 +346,29 @@ class Chef
 
       def send_to_data_collector(message)
         return unless data_collector_accessible?
+        http.post(nil, message, headers) if data_collector_server_url
+      end
+
+      def send_to_output_locations(message)
+        data_collector_output_locations.each do |type, location_list|
+          location_list.each do |l|
+            handle_output_location(type, l, message)
+          end
+        end
+      end
+
+      def handle_output_location(type, loc, message)
+        type == :urls ? send_to_http_location(loc, message) : send_to_file_location(loc, message)
+      end
 
-        http.post(nil, message, headers)
+      def send_to_file_location(file_name, message)
+        open(file_name, "a") { |f| f.puts message }
+      end
+
+      def send_to_http_location(http_url, message)
+        @http_output_locations[http_url].post(nil, message, headers) if @http_output_locations[http_url]
+      rescue
+        Chef::Log.trace("Data collector failed to send to URL location #{http_url}. Please check your configured data_collector.output_locations")
       end
 
       #
@@ -352,16 +387,18 @@ class Chef
         # we have nothing to report.
         return unless run_status
 
-        send_to_data_collector(
-          Chef::DataCollector::Messages.run_end_message(
-            run_status: run_status,
-            expanded_run_list: expanded_run_list,
-            resources: all_resource_reports,
-            status: opts[:status],
-            error_descriptions: error_descriptions,
-            deprecations: deprecations.to_a
-          )
-        )
+        message = Chef::DataCollector::Messages.run_end_message(
+                   run_status: run_status,
+                   expanded_run_list: expanded_run_list,
+                   resources: all_resource_reports,
+                   status: opts[:status],
+                   error_descriptions: error_descriptions,
+                   deprecations: deprecations.to_a
+                  )
+        disable_reporter_on_error do
+          send_to_data_collector(message)
+        end
+        send_to_output_locations(message) if data_collector_output_locations
       end
 
       def headers
@@ -379,6 +416,10 @@ class Chef
         Chef::Config[:data_collector][:server_url]
       end
 
+      def data_collector_output_locations
+        Chef::Config[:data_collector][:output_locations]
+      end
+
       def data_collector_token
         Chef::Config[:data_collector][:token]
       end
@@ -467,21 +508,56 @@ class Chef
         @current_resource_report && @current_resource_report.new_resource != new_resource
       end
 
+      def validate_and_return_uri(uri)
+        URI(uri)
+      rescue URI::InvalidURIError
+        nil
+      end
+
+      def validate_and_create_file(file)
+        send_to_file_location(file, "")
+        true
+      # Rescue exceptions raised by the file path being non-existent or not writeable and re-raise them to the user
+      # with clearer explanatory text.
+      rescue Errno::ENOENT
+        raise Chef::Exceptions::ConfigurationError,
+              "Chef::Config[:data_collector][:output_locations][:files] contains the location #{file}, which is a non existent file path."
+      rescue Errno::EACCES
+        raise Chef::Exceptions::ConfigurationError,
+              "Chef::Config[:data_collector][:output_locations][:files] contains the location #{file}, which cannnot be written to by Chef."
+      end
+
       def validate_data_collector_server_url!
-        if data_collector_server_url.empty?
-          raise Chef::Exceptions::ConfigurationError,
-            "Chef::Config[:data_collector][:server_url] is empty. Please supply a valid URL."
-        end
+        unless !data_collector_server_url && data_collector_output_locations
+          uri = validate_and_return_uri(data_collector_server_url)
+          unless uri
+            raise Chef::Exceptions::ConfigurationError, "Chef::Config[:data_collector][:server_url] (#{data_collector_server_url}) is not a valid URI."
+          end
 
-        begin
-          uri = URI(data_collector_server_url)
-        rescue URI::InvalidURIError
-          raise Chef::Exceptions::ConfigurationError, "Chef::Config[:data_collector][:server_url] (#{data_collector_server_url}) is not a valid URI."
+          if uri.host.nil?
+            raise Chef::Exceptions::ConfigurationError,
+              "Chef::Config[:data_collector][:server_url] (#{data_collector_server_url}) is a URI with no host. Please supply a valid URL."
+          end
         end
+      end
+
+      def handle_type(type, loc)
+        type == :urls ? validate_and_return_uri(loc) : validate_and_create_file(loc)
+      end
 
-        if uri.host.nil?
+      def validate_data_collector_output_locations!
+        if data_collector_output_locations.empty?
           raise Chef::Exceptions::ConfigurationError,
-            "Chef::Config[:data_collector][:server_url] (#{data_collector_server_url}) is a URI with no host. Please supply a valid URL."
+                "Chef::Config[:data_collector][:output_locations] is empty. Please supply an hash of valid URLs and / or local file paths."
+        end
+
+        data_collector_output_locations.each do |type, locations|
+          locations.each do |l|
+            unless handle_type(type, l)
+              raise Chef::Exceptions::ConfigurationError,
+                      "Chef::Config[:data_collector][:output_locations] contains the location #{l} which is not valid."
+            end
+          end
         end
       end
     end