chef 12.10.24 → 12.11.18

data/README.md

@@ -157,9 +157,19 @@ Additionally, periodically Chef will update the desired versions of chef compone
 Whenever a change is checked in to `master`, the patch version of `chef` is bumped. To do this, the `lita-versioner` bot listens to github for merged PRs, and when it finds one, takes these actions:
 
 1. Bumps the patch version in `lib/chef/version.rb` (e.g. 0.9.14 -> 0.9.15).
-2. Runs `rake dependencies:update_conservative` to update the `Gemfile.lock` to include the new version.
+2. Runs `rake bundle:install` to update the `Gemfile.lock` to include the new version.
 3. Pushes to `master` and submits a new build to Chef's Jenkins cluster.
 
+## Bumping the minor version of Chef
+
+After each "official" stable release we need to bump the minor version. To do this:
+
+1. Manually increment the minor version in the VERSION file that is in the root of this repo. and reset the patch version to 0. Assuming the current version is `12.10.57` you would edit `VERSION` to be `12.11.0`.
+2. Run `bundle exec rake version` which will copy the version to the respective `version.rb` files in chef and chef-config.
+3. Run `bundle exec rake bundle:install` to update the base Gemfile.lock
+
+Submit a PR with the changes made by the above.
+
 ## Component Versions
 
 Chef has two sorts of component: ruby components like `berkshelf` and `test-kitchen`, and binary components like `openssl` and even `ruby` itself.

data/VERSION

@@ -1 +1 @@
-12.10.24
+12.11.18

data/acceptance/.bundle/config

@@ -0,0 +1,2 @@
+---
+BUNDLE_FROZEN: '1'

data/acceptance/Gemfile.lock

@@ -1,6 +1,6 @@
 GIT
   remote: git://github.com/chef/chef-acceptance.git
-  revision: 49458ec493dbd12588680eea9f2f9beb76463d09
+  revision: e92ddae46d2126864698b9c8e4fc4ec2dcc46c55
   specs:
     chef-acceptance (0.2.0)
       mixlib-shellout (~> 2.0)
@@ -11,13 +11,13 @@ GEM
   specs:
     addressable (2.4.0)
     artifactory (2.3.2)
-    aws-sdk (2.2.34)
-      aws-sdk-resources (= 2.2.34)
-    aws-sdk-core (2.2.34)
+    aws-sdk (2.3.9)
+      aws-sdk-resources (= 2.3.9)
+    aws-sdk-core (2.3.9)
       jmespath (~> 1.0)
-    aws-sdk-resources (2.2.34)
-      aws-sdk-core (= 2.2.34)
-    berkshelf (4.3.2)
+    aws-sdk-resources (2.3.9)
+      aws-sdk-core (= 2.3.9)
+    berkshelf (4.3.3)
       addressable (~> 2.3, >= 2.3.4)
       berkshelf-api-client (~> 2.0, >= 2.0.2)
       buff-config (~> 1.0)
@@ -52,14 +52,14 @@ GEM
     celluloid-io (0.16.2)
       celluloid (>= 0.16.0)
       nio4r (>= 1.1.0)
-    chef-config (12.9.38)
+    chef-config (12.10.24)
       fuzzyurl (~> 0.8.0)
       mixlib-config (~> 2.0)
       mixlib-shellout (~> 2.0)
     cleanroom (1.0.0)
     coderay (1.1.1)
     diff-lcs (1.2.5)
-    docker-api (1.26.2)
+    docker-api (1.28.0)
       excon (>= 0.38.0)
       json
     erubis (2.7.0)
@@ -72,14 +72,15 @@ GEM
       ffi (>= 1.0.1)
     gyoku (1.3.1)
       builder (>= 2.1.2)
-    hashie (3.4.3)
-    hitimes (1.2.3)
-    httpclient (2.7.1)
-    inspec (0.19.1)
+    hashie (3.4.4)
+    hitimes (1.2.4)
+    httpclient (2.7.2)
+    inspec (0.22.1)
+      hashie (~> 3.4)
       json (~> 1.8)
       method_source (~> 0.8)
       pry (~> 0)
-      r-train (~> 0.10.5)
+      r-train (~> 0.12)
       rainbow (~> 2)
       rspec (~> 3)
       rspec-its (~> 1.2)
@@ -95,8 +96,8 @@ GEM
       multi_json
       retryable (~> 2.0)
       test-kitchen (~> 1.4, >= 1.4.1)
-    kitchen-inspec (0.12.5)
-      inspec (>= 0.14.1, < 1.0.0)
+    kitchen-inspec (0.14.0)
+      inspec (>= 0.22.0, < 1.0.0)
       test-kitchen (~> 1.6)
     kitchen-vagrant (0.20.0)
       test-kitchen (~> 1.4)
@@ -112,15 +113,15 @@ GEM
       rspec-expectations (~> 3.2)
       rspec-mocks (~> 3.2)
     mixlib-config (2.2.1)
-    mixlib-install (1.0.7)
+    mixlib-install (1.0.12)
       artifactory
       mixlib-shellout
       mixlib-versioning
     mixlib-log (1.6.0)
     mixlib-shellout (2.2.6)
     mixlib-versioning (1.1.0)
-    molinillo (0.4.4)
-    multi_json (1.11.2)
+    molinillo (0.4.5)
+    multi_json (1.12.1)
     multipart-post (2.0.0)
     net-scp (1.2.1)
       net-ssh (>= 2.6.5)
@@ -133,17 +134,17 @@ GEM
       coderay (~> 1.1.0)
       method_source (~> 0.8.1)
       slop (~> 3.4)
-    r-train (0.10.5)
-      docker-api (~> 1.26.2)
+    r-train (0.12.1)
+      docker-api (~> 1.26)
       json (~> 1.8)
-      mixlib-shellout (~> 2.1)
+      mixlib-shellout (~> 2.0)
       net-scp (~> 1.2)
       net-ssh (>= 2.9, < 4.0)
       winrm (~> 1.6)
       winrm-fs (~> 0.3)
     rainbow (2.1.0)
     retryable (2.0.3)
-    ridley (4.5.0)
+    ridley (4.5.1)
       addressable
       buff-config (~> 1.0)
       buff-extensions (~> 1.0)
@@ -188,7 +189,7 @@ GEM
     solve (2.0.3)
       molinillo (~> 0.4.2)
       semverse (~> 1.1)
-    test-kitchen (1.7.3)
+    test-kitchen (1.9.0)
       mixlib-install (~> 1.0, >= 1.0.4)
       mixlib-shellout (>= 1.2, < 3.0)
       net-scp (~> 1.1)
@@ -203,7 +204,7 @@ GEM
       hashie (>= 2.0.2, < 4.0.0)
     windows_chef_zero (2.0.0)
       test-kitchen (>= 1.2.1)
-    winrm (1.7.3)
+    winrm (1.8.1)
       builder (>= 2.1.2)
       gssapi (~> 1.2)
       gyoku (~> 1.0)
@@ -232,4 +233,4 @@ DEPENDENCIES
   winrm-fs
 
 BUNDLED WITH
-   1.11.2
+   1.12.5

data/acceptance/basics/.kitchen/logs/chef-current-install-ubuntu-1404.log

@@ -0,0 +1,2 @@
+I, [2016-05-17T16:57:08.264892 #99884]  INFO -- chef-current-install-ubuntu-1404: -----> Destroying <chef-current-install-ubuntu-1404>...
+I, [2016-05-17T16:57:08.268893 #99884]  INFO -- chef-current-install-ubuntu-1404: Finished destroying <chef-current-install-ubuntu-1404> (0m0.00s).

data/acceptance/basics/.kitchen/logs/kitchen.log

@@ -0,0 +1,3 @@
+I, [2016-05-17T16:57:07.751600 #99884]  INFO -- Kitchen: -----> Starting Kitchen (v1.8.0)
+I, [2016-05-17T16:57:08.264892 #99884]  INFO -- Kitchen: -----> Destroying <chef-current-install-ubuntu-1404>...
+I, [2016-05-17T16:57:08.269892 #99884]  INFO -- Kitchen: -----> Kitchen is finished. (0m0.52s)

data/acceptance/data-collector/.acceptance/acceptance-cookbook/.gitignore

@@ -0,0 +1,2 @@
+nodes/
+tmp/

data/acceptance/data-collector/.acceptance/acceptance-cookbook/metadata.rb

@@ -0,0 +1,3 @@
+name 'acceptance-cookbook'
+depends "kitchen_acceptance"
+depends "data-collector-test"

data/acceptance/data-collector/.acceptance/acceptance-cookbook/recipes/destroy.rb

@@ -0,0 +1,2 @@
+log "Running 'destroy' recipe from the acceptance-cookbook in directory '#{node['chef-acceptance']['suite-dir']}'"
+kitchen "destroy"

data/acceptance/data-collector/.acceptance/acceptance-cookbook/recipes/provision.rb

@@ -0,0 +1,2 @@
+log "Running 'provision' recipe from the acceptance-cookbook in directory '#{node['chef-acceptance']['suite-dir']}'"
+kitchen "converge"

data/acceptance/data-collector/.acceptance/acceptance-cookbook/recipes/verify.rb

@@ -0,0 +1,2 @@
+log "Running 'verify' recipe from the acceptance-cookbook in directory '#{node['chef-acceptance']['suite-dir']}'"
+kitchen "verify"

data/acceptance/data-collector/.acceptance/data-collector-test/.gitignore

@@ -0,0 +1,16 @@
+.vagrant
+Berksfile.lock
+*~
+*#
+.#*
+\#*#
+.*.sw[a-z]
+*.un~
+
+# Bundler
+Gemfile.lock
+bin/*
+.bundle/*
+
+.kitchen/
+.kitchen.local.yml

data/acceptance/data-collector/.acceptance/data-collector-test/Berksfile

@@ -0,0 +1,3 @@
+source 'https://supermarket.chef.io'
+
+metadata

data/acceptance/data-collector/.acceptance/data-collector-test/files/default/api.rb

@@ -0,0 +1,85 @@
+require "json"
+require "sinatra"
+
+class Chef
+  class Node
+    # dummy class for JSON parsing
+  end
+end
+
+module ApiHelpers
+  def self.payload_type(payload)
+    message_type = payload["message_type"]
+    status       = payload["status"]
+
+    message_type == "run_converge" ? "#{message_type}.#{status}" : message_type
+  end
+end
+
+class Counter
+  def self.reset
+    @@counters = Hash.new { |h, k| h[k] = 0 }
+  end
+
+  def self.increment(payload)
+    counter_name = ApiHelpers.payload_type(payload)
+    @@counters[counter_name] += 1
+  end
+
+  def self.to_json
+    @@counters.to_json
+  end
+end
+
+class MessageCache
+  include ApiHelpers
+
+  def self.reset
+    @@message_cache = {}
+  end
+
+  def self.store(payload)
+    cache_key = ApiHelpers.payload_type(payload)
+
+    @@message_cache[cache_key] = payload
+  end
+
+  def self.fetch(cache_key)
+    @@message_cache[cache_key].to_json
+  end
+end
+
+Counter.reset
+
+get "/" do
+  "Data Collector API server"
+end
+
+get "/reset-counters" do
+  Counter.reset
+  "counters reset"
+end
+
+get "/counters" do
+  Counter.to_json
+end
+
+get "/cache/:key" do |cache_key|
+  MessageCache.fetch(cache_key)
+end
+
+get "/reset-cache" do
+  MessageCache.reset
+  "cache reset"
+end
+
+post "/data-collector/v0" do
+  body = request.body.read
+  payload = JSON.load(body)
+
+  Counter.increment(payload)
+  MessageCache.store(payload)
+
+  status 201
+  "message received"
+end

data/acceptance/data-collector/.acceptance/data-collector-test/files/default/apigemfile

@@ -0,0 +1,3 @@
+source "https://rubygems.org"
+
+gem "sinatra"

data/acceptance/data-collector/.acceptance/data-collector-test/files/default/client-rb-both-mode.rb

@@ -0,0 +1,4 @@
+chef_server_url           "http://localhost:8889"
+node_name                 "data-collector-test"
+data_collector.server_url "http://localhost:9292/data-collector/v0"
+data_collector.mode       :both

data/acceptance/data-collector/.acceptance/data-collector-test/files/default/client-rb-client-mode.rb

@@ -0,0 +1,4 @@
+chef_server_url           "http://localhost:8889"
+node_name                 "data-collector-test"
+data_collector.server_url "http://localhost:9292/data-collector/v0"
+data_collector.mode       :client

data/acceptance/data-collector/.acceptance/data-collector-test/files/default/client-rb-no-endpoint.rb

@@ -0,0 +1,2 @@
+chef_server_url "http://localhost:8889"
+node_name       "data-collector-test"

data/acceptance/data-collector/.acceptance/data-collector-test/files/default/client-rb-solo-mode.rb

@@ -0,0 +1,4 @@
+chef_server_url           "http://localhost:8889"
+node_name                 "data-collector-test"
+data_collector.server_url "http://localhost:9292/data-collector/v0"
+data_collector.mode       :solo

data/acceptance/data-collector/.acceptance/data-collector-test/files/default/config.ru

@@ -0,0 +1,2 @@
+require_relative "./api"
+run Sinatra::Application

data/acceptance/data-collector/.acceptance/data-collector-test/metadata.rb

@@ -0,0 +1,7 @@
+name 'data-collector-test'
+maintainer 'Adam Leff'
+maintainer_email 'adamleff@chef.io'
+license 'Apache 2.0'
+description 'Installs/Configures data-collector-test'
+long_description 'Installs/Configures data-collector-test'
+version '0.1.0'

data/acceptance/data-collector/.acceptance/data-collector-test/recipes/default.rb

@@ -0,0 +1,38 @@
+api_root_dir = "/var/opt/data_collector_api"
+
+directory api_root_dir do
+  recursive true
+end
+
+cookbook_file ::File.join(api_root_dir, "Gemfile") do
+  source "apigemfile"
+end
+
+cookbook_file ::File.join(api_root_dir, "config.ru")
+
+cookbook_file ::File.join(api_root_dir, "api.rb")
+
+execute "bundle install --binstubs" do
+  cwd api_root_dir
+end
+
+pid_file    = "/var/run/api.pid"
+running_pid = ::File.exist?(pid_file) ? ::File.read(pid_file).strip : nil
+
+execute "kill existing API process" do
+  command "kill #{running_pid}"
+  not_if { running_pid.nil? }
+end
+
+execute "start API" do
+  command "bin/rackup -D -P #{pid_file}"
+  cwd api_root_dir
+end
+
+directory "/etc/chef"
+
+["both-mode", "client-mode", "no-endpoint", "solo-mode"].each do |config_file|
+  cookbook_file "/etc/chef/#{config_file}.rb" do
+    source "client-rb-#{config_file}.rb"
+  end
+end

data/acceptance/data-collector/.kitchen.yml

@@ -0,0 +1,9 @@
+verifier:
+  name: busser
+
+suites:
+  - name: default
+    includes: 
+      - ubuntu-14.04
+    run_list:
+      - recipe[data-collector-test::default]

data/acceptance/data-collector/Berksfile

@@ -0,0 +1,3 @@
+source "https://supermarket.chef.io"
+
+cookbook "data-collector-test", path: File.join(File.expand_path(File.dirname(__FILE__)), ".acceptance", "data-collector-test")

data/acceptance/data-collector/Berksfile.lock

@@ -0,0 +1,6 @@
+DEPENDENCIES
+  data-collector-test
+    path: .acceptance/data-collector-test
+
+GRAPH
+  data-collector-test (0.1.0)

data/acceptance/data-collector/test/integration/default/serverspec/default_spec.rb

@@ -0,0 +1,251 @@
+#
+# Author:: Adam Leff (<adamleff@chef.io)
+#
+# Copyright:: Copyright 2012-2016, Chef Software Inc.
+# License:: Apache License, Version 2.0
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+require "json"
+require "serverspec"
+
+set :backend, :exec
+
+class Chef
+  class Node
+    # dummy class for parsing JSON action messages
+  end
+end
+
+shared_examples_for "reset counters" do
+  it "resets the counters" do
+    expect(command("curl http://localhost:9292/reset-counters").exit_status).to eq(0)
+  end
+end
+
+shared_examples_for "reset cache" do
+  it "resets the message cache" do
+    expect(command("curl http://localhost:9292/reset-cache").exit_status).to eq(0)
+  end
+end
+
+shared_examples_for "successful chef run" do |cmd|
+  include_examples "reset counters"
+  include_examples "reset cache"
+
+  it "runs chef and expects a zero exit status" do
+    expect(command(cmd).exit_status).to eq(0)
+  end
+end
+
+shared_examples_for "unsuccessful chef run" do |cmd|
+  include_examples "reset counters"
+  include_examples "reset cache"
+
+  it "runs chef and expects a non-zero exit status" do
+    expect(command(cmd).exit_status).not_to eq(0)
+  end
+end
+
+shared_examples_for "counter checks" do |counters_to_check|
+  counters_to_check.each do |counter, value|
+    it "counter #{counter} should equal #{value}" do
+      counter_values = JSON.load(command("curl http://localhost:9292/counters").stdout)
+      expect(counter_values[counter]).to eq(value)
+    end
+  end
+end
+
+shared_examples_for "run_start payload check" do
+  describe "run_start message" do
+    let(:required_fields) do
+      %w{
+        chef_server_fqdn
+        entity_uuid
+        id
+        message_version
+        message_type
+        node_name
+        organization_name
+        run_id
+        source
+        start_time
+      }
+    end
+    let(:optional_fields) { [] }
+
+    it "is not missing any required fields" do
+      payload = JSON.load(command("curl http://localhost:9292/cache/run_start").stdout)
+      missing_fields = required_fields.select { |key| !payload.key?(key) }
+      expect(missing_fields).to eq([])
+    end
+
+    it "does not have any extra fields" do
+      payload = JSON.load(command("curl http://localhost:9292/cache/run_start").stdout)
+      extra_fields = payload.keys.select { |key| !required_fields.include?(key) && !optional_fields.include?(key) }
+      expect(extra_fields).to eq([])
+    end
+  end
+end
+
+shared_examples_for "run_converge.success payload check" do
+  describe "run_converge success message" do
+    let(:required_fields) do
+      %w{
+        chef_server_fqdn
+        entity_uuid
+        id
+        end_time
+        expanded_run_list
+        message_type
+        message_version
+        node_name
+        organization_name
+        resources
+        run_id
+        run_list
+        source
+        start_time
+        status
+        total_resource_count
+        updated_resource_count
+      }
+    end
+    let(:optional_fields) { [] }
+
+    it "is not missing any required fields" do
+      payload = JSON.load(command("curl http://localhost:9292/cache/run_converge.success").stdout)
+      missing_fields = required_fields.select { |key| !payload.key?(key) }
+      expect(missing_fields).to eq([])
+    end
+
+    it "does not have any extra fields" do
+      payload = JSON.load(command("curl http://localhost:9292/cache/run_converge.success").stdout)
+      extra_fields = payload.keys.select { |key| !required_fields.include?(key) && !optional_fields.include?(key) }
+      expect(extra_fields).to eq([])
+    end
+  end
+end
+
+shared_examples_for "run_converge.failure payload check" do
+  describe "run_converge failure message" do
+    let(:required_fields) do
+      %w{
+        chef_server_fqdn
+        entity_uuid
+        error
+        id
+        end_time
+        expanded_run_list
+        message_type
+        message_version
+        node_name
+        organization_name
+        resources
+        run_id
+        run_list
+        source
+        start_time
+        status
+        total_resource_count
+        updated_resource_count
+      }
+    end
+    let(:optional_fields) { [] }
+
+    it "is not missing any required fields" do
+      payload = JSON.load(command("curl http://localhost:9292/cache/run_converge.failure").stdout)
+      missing_fields = required_fields.select { |key| !payload.key?(key) }
+      expect(missing_fields).to eq([])
+    end
+
+    it "does not have any extra fields" do
+      payload = JSON.load(command("curl http://localhost:9292/cache/run_converge.failure").stdout)
+      extra_fields = payload.keys.select { |key| !required_fields.include?(key) && !optional_fields.include?(key) }
+      expect(extra_fields).to eq([])
+    end
+  end
+end
+
+shared_examples_for "node-update payload check" do
+  describe "node update message" do
+    let(:required_fields) do
+      %w{
+        entity_name
+        entity_type
+        entity_uuid
+        id
+        message_type
+        message_version
+        organization_name
+        recorded_at
+        remote_hostname
+        requestor_name
+        requestor_type
+        run_id
+        service_hostname
+        source
+        task
+        user_agent
+      }
+    end
+    let(:optional_fields) { %{data} }
+
+    it "is not missing any required fields" do
+      payload = JSON.load(command("curl http://localhost:9292/cache/action").stdout)
+      missing_fields = required_fields.select { |key| !payload.key?(key) }
+      expect(missing_fields).to eq([])
+    end
+
+    it "does not have any extra fields" do
+      payload = JSON.load(command("curl http://localhost:9292/cache/action").stdout)
+      extra_fields = payload.keys.select { |key| !required_fields.include?(key) && !optional_fields.include?(key) }
+      expect(extra_fields).to eq([])
+    end
+  end
+end
+
+describe "CCR with no data collector URL configured" do
+  include_examples "successful chef run", "chef-client -z -c /etc/chef/no-endpoint.rb"
+  include_examples "counter checks", { "run_start" => nil, "run_converge.success" => nil, "run_converge.failure" => nil }
+end
+
+describe "CCR, local mode, config in solo mode" do
+  include_examples "successful chef run", "chef-client -z -c /etc/chef/solo-mode.rb"
+  include_examples "counter checks", { "run_start" => 1, "run_converge.success" => 1, "run_converge.failure" => nil }
+  include_examples "run_start payload check"
+  include_examples "run_converge.success payload check"
+  include_examples "node-update payload check"
+end
+
+describe "CCR, local mode, config in client mode" do
+  include_examples "successful chef run", "chef-client -z -c /etc/chef/client-mode.rb"
+  include_examples "counter checks", { "run_start" => nil, "run_converge.success" => nil, "run_converge.failure" => nil }
+end
+
+describe "CCR, local mode, config in both mode" do
+  include_examples "successful chef run", "chef-client -z -c /etc/chef/both-mode.rb"
+  include_examples "counter checks", { "run_start" => 1, "run_converge.success" => 1, "run_converge.failure" => nil }
+  include_examples "run_start payload check"
+  include_examples "run_converge.success payload check"
+  include_examples "node-update payload check"
+end
+
+describe "CCR, local mode, config in solo mode, failed run" do
+  include_examples "unsuccessful chef run", "chef-client -z -c /etc/chef/solo-mode.rb -r 'recipe[cookbook-that-does-not-exist::default]'"
+  include_examples "counter checks", { "run_start" => 1, "run_converge.success" => nil, "run_converge.failure" => 1 }
+  include_examples "run_start payload check"
+  include_examples "run_converge.failure payload check"
+  include_examples "node-update payload check"
+end