chef 16.10.8 → 17.0.242

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: c34d608da9c455be75a96a34306d2f48c696791569efad3a74d949ffa057a1c6
-  data.tar.gz: 53948ca32fb71521ba9b9502f7bd3a3aa10754a6f1971b1c3756ea8caa682de9
+  metadata.gz: 76e8bbb87457ae55ce29900a8a7686dbefcb4816c50fa818979bb83f6fab76c8
+  data.tar.gz: c6a402cf50ba0ec056631b412d5887c503d3ce177fc7f869692a64c487b26ef1
 SHA512:
-  metadata.gz: 07d4cc0280c0619c9f9caf772d5f8f5c1edbe5e05cea905a5033c218dd6c0fdfd2c3159dfc18eafdc66c1dc0ed7da28839bade97514f910967b8adf1930b6707
-  data.tar.gz: 00a4ee34617c879ab9de9dba5ea9b5842c3fd1da604c7ac8c9e14ae7ce2f1575b5b0227d2ac947937519df90802192f6d49340890f57ec08c7df0adb7b6f0734
+  metadata.gz: 278aff6f45d8a41f323677c768efa94d9e337a60add1445d598de6b15fa9d3cce3a308ce8e27ec9465e557a97bae37d6d29a7c254f35a52d47bba82d928142b5
+  data.tar.gz: caa051215e2ac1116cc818a6ba0bc0d6dd15683aaef9a26e9d08e7063e4a1263d5177baf30e56c08e13f60f8d5e515cfd632632397cd6f348902d38b01a59856

data/Gemfile

@@ -1,8 +1,5 @@
 source "https://rubygems.org"
 
-# pin until issues with Windows builds in 1.14.2 are resolved
-gem "ffi", "=1.13.1"
-
 # Note we do not use the gemspec DSL which restricts to the
 # gemspec for the current platform and filters out other platforms
 # during a bundle lock operation. We actually want dependencies from
@@ -10,7 +7,7 @@ gem "ffi", "=1.13.1"
 # of bundler versions prior to 1.12.0 (https://github.com/bundler/bundler/commit/193a14fe5e0d56294c7b370a0e59f93b2c216eed)
 gem "chef", path: "."
 
-gem "ohai", git: "https://github.com/chef/ohai.git", branch: "16-stable"
+gem "ohai", git: "https://github.com/chef/ohai.git", branch: "master"
 
 gem "chef-utils", path: File.expand_path("chef-utils", __dir__) if File.exist?(File.expand_path("chef-utils", __dir__))
 gem "chef-config", path: File.expand_path("chef-config", __dir__) if File.exist?(File.expand_path("chef-config", __dir__))
@@ -23,7 +20,7 @@ else
   gem "chef-bin" # rubocop:disable Bundler/DuplicatedGem
 end
 
-gem "cheffish", ">= 14"
+gem "cheffish", ">= 17"
 
 gem "chef-telemetry", ">=1.0.8" # 1.0.8 removes the http dep
 
@@ -36,19 +33,20 @@ end
 
 group(:omnibus_package, :pry) do
   gem "pry"
-  gem "pry-byebug"
+  # byebug does not install on freebsd on ruby 3.0
+  # gem "pry-byebug"
   gem "pry-stack_explorer"
 end
 
 # Everything except AIX
 group(:ruby_prof) do
-  # ruby-prof 1.3.0 does not compile on our centos6 builders/kitchen testers
-  gem "ruby-prof", "< 1.3.0"
+  gem "ruby-prof"
 end
 
 # Everything except AIX and Windows
 group(:ruby_shadow) do
-  gem "ruby-shadow", platforms: :ruby
+  # if ruby-shadow does a release that supports ruby-3.0 this can be removed
+  gem "ruby-shadow", git: "https://github.com/chef/ruby-shadow", branch: "lcg/ruby-3.0", platforms: :ruby
 end
 
 group(:development, :test) do
@@ -59,7 +57,8 @@ group(:development, :test) do
 end
 
 group(:chefstyle) do
-  gem "chefstyle", "= 1.5.9"
+  # for testing new chefstyle rules
+  gem "chefstyle", git: "https://github.com/chef/chefstyle.git", branch: "master"
 end
 
 instance_eval(ENV["GEMFILE_MOD"]) if ENV["GEMFILE_MOD"]

data/README.md

@@ -1,6 +1,6 @@
 # Chef Infra
 [![Code Climate](https://codeclimate.com/github/chef/chef.svg)](https://codeclimate.com/github/chef/chef)
-[![Build Status](https://badge.buildkite.com/c82093430ceec7d27af05febb9dcafe3aa331fff9d74c0ab9d.svg?branch=chef-16)](https://buildkite.com/chef-oss/chef-chef-chef-16-verify)
+[![Build Status](https://badge.buildkite.com/c82093430ceec7d27af05febb9dcafe3aa331fff9d74c0ab9d.svg?branch=master)](https://buildkite.com/chef-oss/chef-chef-master-verify)
 [![Gem Version](https://badge.fury.io/rb/chef.svg)](https://badge.fury.io/rb/chef)
 [![](https://img.shields.io/badge/Release%20Policy-Cadence%20Release-brightgreen.svg)](https://github.com/chef/chef/blob/master/docs/dev/design_documents/client_release_cadence.md)
 

data/Rakefile

@@ -74,6 +74,17 @@ task :install do
   end
 end
 
+namespace :install do
+  task local: "pre_install:all"
+
+  task :local do
+    chef_bin_path = ::File.join(::File.dirname(__FILE__), "chef-bin")
+    Dir.chdir(chef_bin_path) do
+      sh("rake install:local")
+    end
+  end
+end
+
 task :pedant, :chef_zero_spec
 
 task :build_eventlog do
@@ -97,8 +108,8 @@ task :update_chef_exec_dll do
 
   sh("hab pkg install chef/chef-powershell-shim")
   sh("hab pkg install chef/chef-powershell-shim-x86")
-  x64 = `hab pkg path chef/chef-powershell-shim`.chomp.gsub(/\\/, "/")
-  x86 = `hab pkg path chef/chef-powershell-shim-x86`.chomp.gsub(/\\/, "/")
+  x64 = `hab pkg path chef/chef-powershell-shim`.chomp.tr('\\', "/")
+  x86 = `hab pkg path chef/chef-powershell-shim-x86`.chomp.tr('\\', "/")
   FileUtils.rm_rf(Dir["distro/ruby_bin_folder/AMD64/*"])
   FileUtils.rm_rf(Dir["distro/ruby_bin_folder/x86/*"])
   puts "Copying #{x64}/bin/* to distro/ruby_bin_folder/AMD64"

data/chef-universal-mingw32.gemspec

@@ -14,7 +14,7 @@ gemspec.add_dependency "win32-service", ">= 2.1.5", "< 3.0"
 gemspec.add_dependency "wmi-lite", "~> 1.0"
 gemspec.add_dependency "win32-taskscheduler", "~> 2.0"
 gemspec.add_dependency "iso8601", ">= 0.12.1", "< 0.14" # validate 0.14 when it comes out
-gemspec.add_dependency "win32-certstore", "~> 0.5" # 0.5+ required for specifying user vs. system store
+gemspec.add_dependency "win32-certstore", "~> 0.6.2" # 0.5+ required for specifying user vs. system store
 gemspec.extensions << "ext/win32-eventlog/Rakefile"
 gemspec.files += Dir.glob("{distro,ext}/**/*")
 

data/chef.gemspec

@@ -35,16 +35,16 @@ Gem::Specification.new do |s|
   s.add_dependency "mixlib-authentication", ">= 2.1", "< 4"
   s.add_dependency "mixlib-shellout", ">= 3.1.1", "< 4.0"
   s.add_dependency "mixlib-archive", ">= 0.4", "< 2.0"
-  s.add_dependency "ohai", "~> 16.0"
+  s.add_dependency "ohai", "~> 17.0"
   s.add_dependency "inspec-core", "~> 4.23"
 
-  s.add_dependency "ffi", ">= 1.9.25"
+  s.add_dependency "ffi", ">= 1.5.0"
   s.add_dependency "ffi-yajl", "~> 2.2"
   s.add_dependency "net-ssh", ">= 5.1", "< 7"
   s.add_dependency "net-ssh-multi", "~> 1.2", ">= 1.2.1"
   s.add_dependency "net-sftp", ">= 2.1.2", "< 4.0"
   s.add_dependency "ed25519", "~> 1.2" # ed25519 ssh key support
-  s.add_dependency "bcrypt_pbkdf", "= 1.1.0.rc2" # ed25519 ssh key support
+  s.add_dependency "bcrypt_pbkdf", "~> 1.1" # ed25519 ssh key support
   s.add_dependency "highline", ">= 1.6.9", "< 3"
   s.add_dependency "tty-prompt", "~> 0.21" # knife ui.ask prompt
   s.add_dependency "tty-screen", "~> 0.6" # knife list
@@ -64,16 +64,14 @@ Gem::Specification.new do |s|
 
   s.add_dependency "proxifier", "~> 1.0"
 
-  # v1.10 is needed as a runtime dep now for 'bundler/inline'
-  # very deliberately avoiding putting a ceiling on this to avoid depsolver conflicts.
-  s.add_dependency "bundler", ">= 1.10"
-
   s.bindir       = "bin"
-  s.executables  = %w{ knife }
+  s.executables  = %w{ }
 
   s.require_paths = %w{ lib }
   s.files = %w{Gemfile Rakefile LICENSE README.md} +
-    Dir.glob("{lib,spec}/**/*", File::FNM_DOTMATCH).reject { |f| File.directory?(f) } +
+    Dir.glob("{lib,spec}/**/*", File::FNM_DOTMATCH).reject do |f|
+      File.directory?(f) || File.path(f).match(/knife*/)
+    end +
     Dir.glob("*.gemspec") +
     Dir.glob("tasks/rspec.rb")
 

data/lib/chef/application/base.rb

@@ -368,7 +368,7 @@ class Chef::Application::Base < Chef::Application
       FileUtils.cp(url, path)
     elsif URI::DEFAULT_PARSER.make_regexp.match?(url)
       File.open(path, "wb") do |f|
-        open(url) do |r|
+        URI.open(url) do |r|
           f.write(r.read)
         end
       end

data/lib/chef/applications.rb

@@ -1,4 +1,3 @@
 require_relative "application/client"
-require_relative "application/knife"
 require_relative "application/solo"
 require_relative "application/apply"

data/lib/chef/chef_fs/command_line.rb

@@ -19,6 +19,9 @@
 require_relative "file_system"
 require_relative "file_system/exceptions"
 require_relative "../util/diff"
+require "chef-utils/parallel_map" unless defined?(ChefUtils::ParallelMap)
+
+using ChefUtils::ParallelMap
 
 class Chef
   module ChefFS
@@ -140,7 +143,7 @@ class Chef
       end
 
       def self.diff(pattern, old_root, new_root, recurse_depth, get_content)
-        Chef::ChefFS::Parallelizer.parallelize(Chef::ChefFS::FileSystem.list_pairs(pattern, old_root, new_root)) do |old_entry, new_entry|
+        Chef::ChefFS::FileSystem.list_pairs(pattern, old_root, new_root).parallel_map do |old_entry, new_entry|
           diff_entries(old_entry, new_entry, recurse_depth, get_content)
         end.flatten(1)
       end
@@ -153,7 +156,7 @@ class Chef
             if recurse_depth == 0
               [ [ :common_subdirectories, old_entry, new_entry ] ]
             else
-              Chef::ChefFS::Parallelizer.parallelize(Chef::ChefFS::FileSystem.child_pairs(old_entry, new_entry)) do |old_child, new_child|
+              Chef::ChefFS::FileSystem.child_pairs(old_entry, new_entry).parallel_map do |old_child, new_child|
                 Chef::ChefFS::CommandLine.diff_entries(old_child, new_child, recurse_depth ? recurse_depth - 1 : nil, get_content)
               end.flatten(1)
             end

data/lib/chef/chef_fs/file_system.rb

@@ -18,7 +18,9 @@
 
 require_relative "path_utils"
 require_relative "file_system/exceptions"
-require_relative "parallelizer"
+require "chef-utils/parallel_map" unless defined?(ChefUtils::ParallelMap)
+
+using ChefUtils::ParallelMap
 
 class Chef
   module ChefFS
@@ -70,8 +72,8 @@ class Chef
 
               # Otherwise, go through all children and find any matches
             elsif entry.dir?
-              results = Parallelizer.parallelize(entry.children) { |child| Chef::ChefFS::FileSystem.list(child, pattern) }
-              results.flatten(1).each(&block)
+              results = entry.children.parallel_map { |child| Chef::ChefFS::FileSystem.list(child, pattern) }
+              results.flat_each(&block)
             end
           end
         end
@@ -138,7 +140,7 @@ class Chef
       def self.copy_to(pattern, src_root, dest_root, recurse_depth, options, ui = nil, format_path = nil)
         found_result = false
         error = false
-        parallel_do(list_pairs(pattern, src_root, dest_root)) do |src, dest|
+        list_pairs(pattern, src_root, dest_root).parallel_each do |src, dest|
           found_result = true
           new_dest_parent = get_or_create_parent(dest, options, ui, format_path)
           child_error = copy_entries(src, dest, new_dest_parent, recurse_depth, options, ui, format_path)
@@ -292,7 +294,7 @@ class Chef
                     end
                   end
                 else
-                  ui.output ("Not deleting extra entry #{dest_path} (purge is off)") if ui
+                  ui.output("Not deleting extra entry #{dest_path} (purge is off)") if ui
                 end
               end
 
@@ -319,7 +321,7 @@ class Chef
                   end
                   # Directory creation is recursive.
                   if recurse_depth != 0
-                    parallel_do(src_entry.children) do |src_child|
+                    src_entry.children.parallel_each do |src_child|
                       new_dest_child = new_dest_dir.child(src_child.name)
                       child_error = copy_entries(src_child, new_dest_child, new_dest_dir, recurse_depth ? recurse_depth - 1 : recurse_depth, options, ui, format_path)
                       error ||= child_error
@@ -356,7 +358,7 @@ class Chef
                 if dest_entry.dir?
                   # If both are directories, recurse into their children
                   if recurse_depth != 0
-                    parallel_do(child_pairs(src_entry, dest_entry)) do |src_child, dest_child|
+                    child_pairs(src_entry, dest_entry).parallel_each do |src_child, dest_child|
                       child_error = copy_entries(src_child, dest_child, dest_entry, recurse_depth ? recurse_depth - 1 : recurse_depth, options, ui, format_path)
                       error ||= child_error
                     end
@@ -423,9 +425,6 @@ class Chef
           parent
         end
 
-        def parallel_do(enum, options = {}, &block)
-          Chef::ChefFS::Parallelizer.parallel_do(enum, options, &block)
-        end
       end
     end
   end

data/lib/chef/client.rb

@@ -858,8 +858,8 @@ class Chef
 
     def profiling_prereqs!
       require "ruby-prof"
-    rescue LoadError
-      raise "You must have the ruby-prof gem installed in order to use --profile-ruby"
+    rescue LoadError => e
+      raise "You must have the ruby-prof gem installed in order to use --profile-ruby: #{e.message}"
     end
 
     def start_profiling

data/lib/chef/compliance/default_attributes.rb

@@ -1,5 +1,5 @@
 # Author:: Stephan Renatus <srenatus@chef.io>
-# Copyright:: (c) 2016-2019, Chef Software Inc. <legal@chef.io>
+# Copyright:: Copyright (c) Chef Software Inc. <legal@chef.io>
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -27,7 +27,7 @@ class Chef
 
       # Controls what is done with the resulting report after the Chef InSpec run.
       # Accepts a single string value or an array of multiple values.
-      # Accepted values: 'chef-server-automate', 'chef-automate', 'json-file', 'audit-enforcer'
+      # Accepted values: 'chef-server-automate', 'chef-automate', 'json-file', 'audit-enforcer', 'cli'
       "reporter" => "json-file",
 
       # Controls if Chef InSpec profiles should be fetched from Chef Automate or Chef Infra Server
@@ -38,11 +38,12 @@ class Chef
       # Allow for connections to HTTPS endpoints using self-signed ssl certificates.
       "insecure" => nil,
 
-      # Controls verbosity of Chef InSpec runner.
+      # Controls verbosity of Chef InSpec runner. See less output when true.
       "quiet" => true,
 
       # Chef Inspec Compliance profiles to be used for scan of node.
-      # See README.md for details
+      # See Compliance Phase documentation for further details:
+      # https://docs.chef.io/chef_compliance_phase/#compliance-phase-configuration
       "profiles" => {},
 
       # Extra inputs passed to Chef InSpec to allow finer-grained control over behavior.
@@ -87,7 +88,11 @@ class Chef
 
       # If enabled, a hash representation of the Chef Infra node object will be sent to Chef InSpec in an input
       # named `chef_node`.
-      "chef_node_attribute_enabled" => false
+      "chef_node_attribute_enabled" => false,
+
+      # Should the built-in compliance phase run. True and false force the behavior. Nil does magic based on if you have
+      # profiles defined but do not have the audit cookbook enabled.
+      "compliance_phase" => false
     )
   end
 end

data/lib/chef/compliance/fetcher/automate.rb

@@ -32,12 +32,12 @@ class Chef
             profile_fetch_url = target[:url]
           else
             # verifies that the target e.g base/ssh exists
-            base_path = "/compliance/profiles/#{uri.host}#{uri.path}"
-
+            profile = sanitize_profile_name(uri)
+            owner, id = profile.split("/")
             profile_path = if target.respond_to?(:key?) && target.key?(:version)
-                             "#{base_path}/version/#{target[:version]}/tar"
+                             "/compliance/profiles/#{owner}/#{id}/version/#{target[:version]}/tar"
                            else
-                             "#{base_path}/tar"
+                             "/compliance/profiles/#{owner}/#{id}/tar"
                            end
 
             url = URI(Chef::Config[:data_collector][:server_url])
@@ -46,13 +46,6 @@ class Chef
 
             config["token"] = Chef::Config[:data_collector][:token]
 
-            if config["token"].nil?
-              raise Inspec::FetcherFailure,
-                "No data-collector token set, which is required by the chef-automate fetcher. " \
-                "Set the `data_collector.token` configuration parameter in your client.rb " \
-                'or use the "chef-server-automate" reporter which does not require any ' \
-                "data-collector settings and uses #{ChefUtils::Dist::Server::PRODUCT} to fetch profiles."
-            end
           end
 
           new(profile_fetch_url, config)
@@ -60,6 +53,17 @@ class Chef
           nil
         end
 
+        # returns a parsed url for `admin/profile` or `compliance://admin/profile`
+        # TODO: remove in future, copied from inspec to support older versions of inspec
+        def self.sanitize_profile_name(profile)
+          uri = if URI(profile).scheme == "compliance"
+                  URI(profile)
+                else
+                  URI("compliance://#{profile}")
+                end
+          uri.to_s.sub(%r{^compliance:\/\/}, "")
+        end
+
         def to_s
           "#{ChefUtils::Dist::Automate::PRODUCT} for #{ChefUtils::Dist::Solo::PRODUCT} Fetcher"
         end

data/lib/chef/compliance/reporter/automate.rb

@@ -28,18 +28,28 @@ class Chef
           @token = Chef::Config[:data_collector][:token]
         end
 
-        # Method used in order to send the inspec report to the data_collector server
-        def send_report(report)
-          unless @entity_uuid && @run_id
-            Chef::Log.error "entity_uuid(#{@entity_uuid}) or run_id(#{@run_id}) can't be nil, not sending report to #{ChefUtils::Dist::Automate::PRODUCT}"
-            return false
+        def validate_config!
+          unless @entity_uuid
+            # TODO - this is a weird leakage of naming from the parent class
+            # but entity_uuid is never an attribute that the user can see;
+            # it is sourced from chef_guid, which we don't technically know about in this class -
+            # but telling the operator about a missing chef_guid is more helpful than telling
+            # them about a missing field they've never heard of. Better would be a dock link
+            # that described how to fix this situation.
+            raise "CMPL004: automate_reporter: chef_guid is not available and must be provided. Aborting because we cannot report the scan."
+          end
+
+          unless @run_id
+            raise "CMPL005: automate_reporter: run_id is not available, aborting because we cannot report the scan."
           end
 
           unless @url && @token
-            Chef::Log.warn "data_collector.token and data_collector.server_url must be defined in client.rb! Further information: https://docs.chef.io/chef_compliance_phase/#direct-reporting-to-chef-automate"
-            return false
+            raise "CMPL006: data_collector.token and data_collector.server_url must be configured in client.rb! Further information: https://docs.chef.io/chef_compliance_phase/#direct-reporting-to-chef-automate"
           end
+        end
 
+        # Method used in order to send the inspec report to the data_collector server
+        def send_report(report)
           headers = {
             "Content-Type" => "application/json",
             "x-data-collector-auth" => "version=1.0",

data/lib/chef/compliance/reporter/chef_server_automate.rb

@@ -30,11 +30,6 @@ class Chef
         end
 
         def send_report(report)
-          unless @entity_uuid && @run_id
-            Chef::Log.error "entity_uuid(#{@entity_uuid}) or run_id(#{@run_id}) can't be nil, not sending report to #{ChefUtils::Dist::Automate::PRODUCT}"
-            return false
-          end
-
           automate_report = truncate_controls_results(enriched_report(report), @control_results_limit)
 
           report_size = Chef::JSONCompat.to_json(automate_report, validate_utf8: false).bytesize
@@ -51,6 +46,16 @@ class Chef
           false
         end
 
+        def validate_config!
+          unless @entity_uuid
+            raise "CMPL007: chef_server_automate reporter: chef_guid is not available and must be provided. Aborting because we cannot report the scan"
+          end
+
+          unless @run_id
+            raise "CMPL008: chef_server_automate reporter: run_id is not available, aborting because we cannot report the scan."
+          end
+        end
+
         def http_client
           config = if @insecure
                      Chef::Config.merge(ssl_verify_mode: :verify_none)
@@ -80,7 +85,7 @@ class Chef
           when /404/
             Chef::Log.error "Object does not exist on remote server."
           when /413/
-            Chef::Log.error "You most likely hit the erchef request size in #{ChefUtils::Dist::Server::PRODUCT} that defaults to ~2MB. To increase this limit see the Compliance Phase troubleshooting documentation (http://docs.chef.io/chef_compliance_phase/#troubleshooting) or the Chef Infra Server configuration documentation (https://docs.chef.io/server/config_rb_server/)"
+            Chef::Log.error "You most likely hit the request size limit in #{ChefUtils::Dist::Server::PRODUCT} that defaults to ~2MB. To increase this limit see the Compliance Phase troubleshooting documentation (http://docs.chef.io/chef_compliance_phase/#troubleshooting) or the Chef Infra Server configuration documentation (https://docs.chef.io/server/config_rb_server/)"
           when /429/
             Chef::Log.error "This error typically means the data sent was larger than #{ChefUtils::Dist::Automate::PRODUCT}'s limit (4 MB). Run InSpec locally to identify any controls producing large diffs."
           end