chef 12.11.18 → 12.12.13

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 97b8d731ca0f6c028fb862f811e120dc265d5228
-  data.tar.gz: 3f3192a78b9d23a2536c9e911140fbeb19ed9a79
+  metadata.gz: 06fd3e81e49974a9b3c27291e92c045abf414b17
+  data.tar.gz: 6e798ffc8280bf201dd883f03b4d046dfa95be9c
 SHA512:
-  metadata.gz: 5154122e67402dc45487bda181cc291e1599b3fc8ccc0fba240400386c8ccdc38be2bca97a4b6734dfe4000606401382e5ac3aa851cc4ea0ede9f8e54e9eb1b6
-  data.tar.gz: c1c519e59a74cae11a8de2b316d14ee1a9725f5b4608a4e31aeafbcfaa2d14f67103684b3ba068b811ea412595d8916d77cf2a3aedeb91e46ac89251455bf5f3
+  metadata.gz: cbb907b8556c4dff3c85e589722e048af0ae070057f5338845f2105ec3b14efe55cdd24042342bf358aff2c43e2390189ab0b8a2f4e3a174daf2191347d009e2
+  data.tar.gz: 5353be5018040d24332f63f96e0bea3ca85f7cc49bf5468ccbfd154b9d6c3335610c38e0bc8163f8ab721ffddaa1f33dd7d918780f1ae5dbde4d56122b7d000f

data/Gemfile

@@ -13,15 +13,15 @@ gem "chef", path: "."
 
 gem "chef-config", path: File.expand_path("../chef-config", __FILE__) if File.exist?(File.expand_path("../chef-config", __FILE__))
 # Ensure that we can always install rake, regardless of gem groups
-gem "rake"
+gem "rake", group: [ :default, :omnibus_package, :development ]
 gem "bundler"
 gem "cheffish"
 
 group(:omnibus_package) do
   gem "appbundler"
   gem "rb-readline"
-  gem "nokogiri"
 end
+
 group(:omnibus_package, :pry) do
   gem "pry"
   gem "pry-byebug"
@@ -65,7 +65,7 @@ end
 
 group(:development, :test) do
   gem "simplecov"
-  gem "rack"
+  gem "rack", "< 2.0" # 2.0 requires Ruby 2.2+
 
   # for testing new chefstyle rules
   # gem 'chefstyle', github: 'chef/chefstyle'
@@ -78,7 +78,7 @@ end
 
 group(:travis) do
   # See `bundler-audit` in .travis.yml
-  gem "bundler-audit", git: "https://github.com/rubysec/bundler-audit.git", ref: "4e32fca"
+  gem "bundler-audit", git: "https://github.com/rubysec/bundler-audit.git"
 end
 
 instance_eval(ENV["GEMFILE_MOD"]) if ENV["GEMFILE_MOD"]

data/Rakefile

@@ -29,12 +29,13 @@ require_relative "tasks/cbgb"
 require_relative "tasks/dependencies"
 require_relative "tasks/changelog"
 
-ChefConfig::PackageTask.new(File.expand_path("..", __FILE__), "Chef") do |package|
+ChefConfig::PackageTask.new(File.expand_path("..", __FILE__), "Chef", "chef") do |package|
   package.component_paths = ["chef-config"]
   package.generate_version_class = true
 end
 # Add a conservative dependency update to version:bump (which was created by PackageTask)
-task "version:bump" => %w{version:bump_patch version:update bundle:install}
+task "version:bump" => %w{version:bump_patch version:update}
+task "version:bump" => %w{version:bump_patch version:update}
 
 task :pedant, :chef_zero_spec
 

data/VERSION

@@ -1 +1 @@
-12.11.18
+12.12.13

data/acceptance/Gemfile.lock

@@ -11,13 +11,13 @@ GEM
   specs:
     addressable (2.4.0)
     artifactory (2.3.2)
-    aws-sdk (2.3.9)
-      aws-sdk-resources (= 2.3.9)
-    aws-sdk-core (2.3.9)
+    aws-sdk (2.3.14)
+      aws-sdk-resources (= 2.3.14)
+    aws-sdk-core (2.3.14)
       jmespath (~> 1.0)
-    aws-sdk-resources (2.3.9)
-      aws-sdk-core (= 2.3.9)
-    berkshelf (4.3.3)
+    aws-sdk-resources (2.3.14)
+      aws-sdk-core (= 2.3.14)
+    berkshelf (4.3.5)
       addressable (~> 2.3, >= 2.3.4)
       berkshelf-api-client (~> 2.0, >= 2.0.2)
       buff-config (~> 1.0)
@@ -29,6 +29,7 @@ GEM
       faraday (~> 0.9)
       httpclient (~> 2.7)
       minitar (~> 0.5, >= 0.5.4)
+      mixlib-archive (~> 0.1)
       octokit (~> 4.0)
       retryable (~> 2.0)
       ridley (~> 4.5)
@@ -52,7 +53,7 @@ GEM
     celluloid-io (0.16.2)
       celluloid (>= 0.16.0)
       nio4r (>= 1.1.0)
-    chef-config (12.10.24)
+    chef-config (12.11.18)
       fuzzyurl (~> 0.8.0)
       mixlib-config (~> 2.0)
       mixlib-shellout (~> 2.0)
@@ -75,17 +76,17 @@ GEM
     hashie (3.4.4)
     hitimes (1.2.4)
     httpclient (2.7.2)
-    inspec (0.22.1)
+    inspec (0.26.0)
       hashie (~> 3.4)
       json (~> 1.8)
       method_source (~> 0.8)
       pry (~> 0)
-      r-train (~> 0.12)
       rainbow (~> 2)
       rspec (~> 3)
       rspec-its (~> 1.2)
       rubyzip (~> 1.1)
       thor (~> 0.19)
+      train (~> 0.13)
     jmespath (1.2.4)
       json_pure (>= 1.8.1)
     json (1.8.3)
@@ -107,13 +108,11 @@ GEM
       multi_json (~> 1.10)
     method_source (0.8.2)
     minitar (0.5.4)
-    mixlib-authentication (1.4.0)
+    mixlib-archive (0.1.0)
+    mixlib-authentication (1.4.1)
       mixlib-log
-      rspec-core (~> 3.2)
-      rspec-expectations (~> 3.2)
-      rspec-mocks (~> 3.2)
     mixlib-config (2.2.1)
-    mixlib-install (1.0.12)
+    mixlib-install (1.0.13)
       artifactory
       mixlib-shellout
       mixlib-versioning
@@ -134,14 +133,6 @@ GEM
       coderay (~> 1.1.0)
       method_source (~> 0.8.1)
       slop (~> 3.4)
-    r-train (0.12.1)
-      docker-api (~> 1.26)
-      json (~> 1.8)
-      mixlib-shellout (~> 2.0)
-      net-scp (~> 1.2)
-      net-ssh (>= 2.9, < 4.0)
-      winrm (~> 1.6)
-      winrm-fs (~> 0.3)
     rainbow (2.1.0)
     retryable (2.0.3)
     ridley (4.5.1)
@@ -189,7 +180,7 @@ GEM
     solve (2.0.3)
       molinillo (~> 0.4.2)
       semverse (~> 1.1)
-    test-kitchen (1.9.0)
+    test-kitchen (1.9.2)
       mixlib-install (~> 1.0, >= 1.0.4)
       mixlib-shellout (>= 1.2, < 3.0)
       net-scp (~> 1.1)
@@ -199,6 +190,14 @@ GEM
     thor (0.19.1)
     timers (4.0.4)
       hitimes
+    train (0.13.1)
+      docker-api (~> 1.26)
+      json (~> 1.8)
+      mixlib-shellout (~> 2.0)
+      net-scp (~> 1.2)
+      net-ssh (>= 2.9, < 4.0)
+      winrm (~> 1.6)
+      winrm-fs (~> 0.3)
     varia_model (0.4.1)
       buff-extensions (~> 1.0)
       hashie (>= 2.0.2, < 4.0.0)

data/acceptance/data-collector/test/integration/default/serverspec/default_spec.rb

@@ -110,6 +110,7 @@ shared_examples_for "run_converge.success payload check" do
         expanded_run_list
         message_type
         message_version
+        node
         node_name
         organization_name
         resources
@@ -150,6 +151,7 @@ shared_examples_for "run_converge.failure payload check" do
         expanded_run_list
         message_type
         message_version
+        node
         node_name
         organization_name
         resources
@@ -178,44 +180,6 @@ shared_examples_for "run_converge.failure payload check" do
   end
 end
 
-shared_examples_for "node-update payload check" do
-  describe "node update message" do
-    let(:required_fields) do
-      %w{
-        entity_name
-        entity_type
-        entity_uuid
-        id
-        message_type
-        message_version
-        organization_name
-        recorded_at
-        remote_hostname
-        requestor_name
-        requestor_type
-        run_id
-        service_hostname
-        source
-        task
-        user_agent
-      }
-    end
-    let(:optional_fields) { %{data} }
-
-    it "is not missing any required fields" do
-      payload = JSON.load(command("curl http://localhost:9292/cache/action").stdout)
-      missing_fields = required_fields.select { |key| !payload.key?(key) }
-      expect(missing_fields).to eq([])
-    end
-
-    it "does not have any extra fields" do
-      payload = JSON.load(command("curl http://localhost:9292/cache/action").stdout)
-      extra_fields = payload.keys.select { |key| !required_fields.include?(key) && !optional_fields.include?(key) }
-      expect(extra_fields).to eq([])
-    end
-  end
-end
-
 describe "CCR with no data collector URL configured" do
   include_examples "successful chef run", "chef-client -z -c /etc/chef/no-endpoint.rb"
   include_examples "counter checks", { "run_start" => nil, "run_converge.success" => nil, "run_converge.failure" => nil }
@@ -226,7 +190,6 @@ describe "CCR, local mode, config in solo mode" do
   include_examples "counter checks", { "run_start" => 1, "run_converge.success" => 1, "run_converge.failure" => nil }
   include_examples "run_start payload check"
   include_examples "run_converge.success payload check"
-  include_examples "node-update payload check"
 end
 
 describe "CCR, local mode, config in client mode" do
@@ -239,7 +202,6 @@ describe "CCR, local mode, config in both mode" do
   include_examples "counter checks", { "run_start" => 1, "run_converge.success" => 1, "run_converge.failure" => nil }
   include_examples "run_start payload check"
   include_examples "run_converge.success payload check"
-  include_examples "node-update payload check"
 end
 
 describe "CCR, local mode, config in solo mode, failed run" do
@@ -247,5 +209,4 @@ describe "CCR, local mode, config in solo mode, failed run" do
   include_examples "counter checks", { "run_start" => 1, "run_converge.success" => nil, "run_converge.failure" => 1 }
   include_examples "run_start payload check"
   include_examples "run_converge.failure payload check"
-  include_examples "node-update payload check"
 end

data/lib/chef/application/solo.rb

@@ -241,6 +241,13 @@ class Chef::Application::Solo < Chef::Application
         ARGV[dash_r] = "--recipe-url"
       end
 
+      # For back compat reasons, we need to ensure that we try and use the cache_path as a repo first
+      Chef::Log.debug "Current chef_repo_path is #{Chef::Config.chef_repo_path}"
+
+      if !Chef::Config.has_key?(:cookbook_path) && !Chef::Config.has_key?(:chef_repo_path)
+        Chef::Config.chef_repo_path = Chef::Config.find_chef_repo_path(Chef::Config[:cache_path])
+      end
+
       Chef::Config[:local_mode] = true
     else
       configure_legacy_mode!

data/lib/chef/chef_fs/file_system/multiplexed_dir.rb

@@ -41,7 +41,7 @@ class Chef
             child_entry = dir.child(name)
             if child_entry.exists?
               if result
-                Chef::Log.debug("Child with name '#{child_entry.name}' found in multiple directories: #{result.parent.path_for_printing} and #{child_entry.parent.path_for_printing}") unless seen[child.name].path_for_printing == child.path_for_printing
+                Chef::Log.debug("Child with name '#{child_entry.name}' found in multiple directories: #{result.parent.path_for_printing} and #{child_entry.parent.path_for_printing}")
               else
                 result = child_entry
               end

data/lib/chef/data_collector.rb

@@ -22,6 +22,7 @@ require "uri"
 require "chef/event_dispatch/base"
 require "chef/data_collector/messages"
 require "chef/data_collector/resource_report"
+require "ostruct"
 
 class Chef
 
@@ -51,13 +52,12 @@ class Chef
     # and exports its data through a webhook-like mechanism to a configured
     # endpoint.
     class Reporter < EventDispatch::Base
-      attr_reader :updated_resources, :status, :exception, :error_descriptions,
-                  :expanded_run_list, :run_status, :http, :resource_count,
+      attr_reader :all_resource_reports, :status, :exception, :error_descriptions,
+                  :expanded_run_list, :run_context, :run_status, :http,
                   :current_resource_report, :enabled
 
       def initialize
-        @updated_resources       = []
-        @resource_count          = 0
+        @all_resource_reports    = []
         @current_resource_loaded = nil
         @error_descriptions      = {}
         @expanded_run_list       = {}
@@ -94,6 +94,29 @@ class Chef
         send_run_completion(status: "failure")
       end
 
+      # see EventDispatch::Base#converge_start
+      # Upon receipt, we stash the run_context for use at the
+      # end of the run in order to determine what resource+action
+      # combinations have not yet fired so we can report on
+      # unprocessed resources.
+      def converge_start(run_context)
+        @run_context = run_context
+      end
+
+      # see EventDispatch::Base#converge_complete
+      # At the end of the converge, we add any unprocessed resources
+      # to our report list.
+      def converge_complete
+        detect_unprocessed_resources
+      end
+
+      # see EventDispatch::Base#converge_failed
+      # At the end of the converge, we add any unprocessed resources
+      # to our report list
+      def converge_failed(exception)
+        detect_unprocessed_resources
+      end
+
       # see EventDispatch::Base#resource_current_state_loaded
       # Create a new ResourceReport instance that we'll use to track
       # the state of this resource during the run. Nested resources are
@@ -101,56 +124,40 @@ class Chef
       # resource, and we only care about tracking top-level resources.
       def resource_current_state_loaded(new_resource, action, current_resource)
         return if nested_resource?(new_resource)
-        update_current_resource_report(
-          Chef::DataCollector::ResourceReport.new(
-            new_resource,
-            action,
-            current_resource
-          )
-        )
+        update_current_resource_report(create_resource_report(new_resource, action, current_resource))
       end
 
       # see EventDispatch::Base#resource_up_to_date
-      # Mark our ResourceReport status accordingly, and increment the total
-      # resource count.
+      # Mark our ResourceReport status accordingly
       def resource_up_to_date(new_resource, action)
         current_resource_report.up_to_date unless nested_resource?(new_resource)
-        increment_resource_count
       end
 
       # see EventDispatch::Base#resource_skipped
-      # Increment the total resource count. If this is a top-level resource,
-      # we also create a ResourceReport instance (because a skipped resource
-      # does not trigger the resource_current_state_loaded event), and flag
-      # it as skipped.
+      # If this is a top-level resource, we create a ResourceReport
+      # instance (because a skipped resource does not trigger the
+      # resource_current_state_loaded event), and flag it as skipped.
       def resource_skipped(new_resource, action, conditional)
-        increment_resource_count
         return if nested_resource?(new_resource)
 
-        update_current_resource_report(
-          Chef::DataCollector::ResourceReport.new(
-            new_resource,
-            action
-          )
-        )
-        current_resource_report.skipped(conditional)
+        resource_report = create_resource_report(new_resource, action)
+        resource_report.skipped(conditional)
+        update_current_resource_report(resource_report)
       end
 
       # see EventDispatch::Base#resource_updated
       # Flag the current ResourceReport instance as updated (as long as it's
-      # a top-level resource) and increment the total resource count.
+      # a top-level resource).
       def resource_updated(new_resource, action)
         current_resource_report.updated unless nested_resource?(new_resource)
-        increment_resource_count
       end
 
       # see EventDispatch::Base#resource_failed
       # Flag the current ResourceReport as failed and supply the exception as
-      # long as it's a top-level resource, increment the total resource count,
-      # and update the run error text with the proper Formatter.
+      # long as it's a top-level resource, and update the run error text
+      # with the proper Formatter.
       def resource_failed(new_resource, action, exception)
         current_resource_report.failed(exception) unless nested_resource?(new_resource)
-        increment_resource_count
         update_error_description(
           Formatters::ErrorMapper.resource_failed(
             new_resource,
@@ -161,13 +168,12 @@ class Chef
       end
 
       # see EventDispatch::Base#resource_completed
-      # Mark the ResourceReport instance as finished (for timing details)
-      # and add it to the list of resources encountered during this run.
+      # Mark the ResourceReport instance as finished (for timing details).
       # This marks the end of this resource during this run.
       def resource_completed(new_resource)
         if current_resource_report && !nested_resource?(new_resource)
           current_resource_report.finish
-          add_updated_resource(current_resource_report)
+          add_resource_report(current_resource_report)
           update_current_resource_report(nil)
         end
       end
@@ -273,13 +279,11 @@ class Chef
         # we have nothing to report.
         return unless run_status
 
-        send_to_data_collector(Chef::DataCollector::Messages.node_update_message(run_status).to_json)
         send_to_data_collector(
           Chef::DataCollector::Messages.run_end_message(
             run_status: run_status,
             expanded_run_list: expanded_run_list,
-            total_resource_count: resource_count,
-            updated_resources: updated_resources,
+            resources: all_resource_reports,
             status: opts[:status],
             error_descriptions: error_descriptions
           ).to_json
@@ -305,12 +309,12 @@ class Chef
         Chef::Config[:data_collector][:token]
       end
 
-      def increment_resource_count
-        @resource_count += 1
-      end
-
-      def add_updated_resource(resource_report)
-        @updated_resources << resource_report
+      def add_resource_report(resource_report)
+        @all_resource_reports << OpenStruct.new(
+          resource: resource_report.new_resource,
+          action: resource_report.action,
+          report_data: resource_report.to_hash
+        )
       end
 
       def disable_data_collector_reporter
@@ -333,6 +337,38 @@ class Chef
         @error_descriptions = discription_hash
       end
 
+      def create_resource_report(new_resource, action, current_resource = nil)
+        Chef::DataCollector::ResourceReport.new(
+          new_resource,
+          action,
+          current_resource
+        )
+      end
+
+      def detect_unprocessed_resources
+        # create a Set containing all resource+action combinations from
+        # the Resource Collection
+        collection_resources = Set.new
+        run_context.resource_collection.all_resources.each do |resource|
+          Array(resource.action).each do |action|
+            collection_resources.add([resource, action])
+          end
+        end
+
+        # Delete from the Set any resource+action combination we have
+        # already processed.
+        all_resource_reports.each do |report|
+          collection_resources.delete([report.resource, report.action])
+        end
+
+        # The items remaining in the Set are unprocessed resource+actions,
+        # so we'll create new resource reports for them which default to
+        # a state of "unprocessed".
+        collection_resources.each do |resource, action|
+          add_resource_report(create_resource_report(resource, action))
+        end
+      end
+
       # If we are getting messages about a resource while we are in the middle of
       # another resource's update, we assume that the nested resource is just the
       # implementation of a provider, and we want to hide it from the reporting