chef 16.18.30 → 17.10.163

data/lib/chef/{knife/data_bag_list.rb → dsl/render_helpers.rb}

@@ -1,5 +1,4 @@
 #
-# Author:: Adam Jacob (<adam@chef.io>)
 # Copyright:: Copyright (c) Chef Software Inc.
 # License:: Apache License, Version 2.0
 #
@@ -14,29 +13,32 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
-#
-
-require_relative "../knife"
+require_relative "toml"
+require_relative "../json_compat"
+autoload :YAML, "yaml"
 
 class Chef
-  class Knife
-    class DataBagList < Knife
+  module DSL
+    module RenderHelpers
 
-      deps do
-        require_relative "../data_bag"
+      # pretty-print a hash as a JSON string
+      def render_json(hash)
+        JSON.pretty_generate(hash) + "\n"
       end
 
-      banner "knife data bag list (options)"
-      category "data bag"
-
-      option :with_uri,
-        short: "-w",
-        long: "--with-uri",
-        description: "Show corresponding URIs."
+      # pretty-print a hash as a TOML string
+      def render_toml(hash)
+        Chef::DSL::Toml::Dumper.new(hash).toml_str
+      end
 
-      def run
-        output(format_list_for_display(Chef::DataBag.list))
+      # pretty-print a hash as a YAML string
+      def render_yaml(hash)
+        yaml_content = hash.transform_keys(&:to_s).to_yaml
+        # above replaces first-level keys with strings, below the rest
+        yaml_content.gsub!(" :", " ")
       end
+
+      extend self
     end
   end
 end

data/lib/chef/dsl/secret.rb

@@ -0,0 +1,170 @@
+#
+# Author:: Marc Paradise (<marc@chef.io>)
+# Copyright:: Copyright (c) Chef Software Inc.
+# License:: Apache License, Version 2.0
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+require_relative "../secret_fetcher"
+
+class Chef
+  module DSL
+    module Secret
+
+      #
+      # This allows you to set the default secret service that is used when
+      # fetching secrets.
+      #
+      # @example
+      #
+      #   default_secret_service :hashi_vault
+      #   val1 = secret(name: "test1", config: { region: "us-west-1" })
+      #
+      # @example
+      #
+      #   default_secret_service #=> nil
+      #   default_secret_service :hashi_vault
+      #   default_secret_service #=> :hashi_vault
+      #
+      # @param [Symbol] service default secret service to use when fetching secrets
+      # @return [Symbol, nil] default secret service to use when fetching secrets
+      #
+      def default_secret_service(service = nil)
+        return run_context.default_secret_service if service.nil?
+        raise Chef::Exceptions::Secret::InvalidFetcherService.new("Unsupported secret service: #{service.inspect}", Chef::SecretFetcher::SECRET_FETCHERS) unless Chef::SecretFetcher::SECRET_FETCHERS.include?(service)
+
+        run_context.default_secret_service = service
+      end
+
+      #
+      # This allows you to set the secret service for the scope of the block
+      # passed into this method.
+      #
+      # @example
+      #
+      #   with_secret_service :hashi_vault do
+      #     val1 = secret(name: "test1", config: { region: "us-west-1" })
+      #     val2 = secret(name: "test2", config: { region: "us-west-1" })
+      #   end
+      #
+      # @example Combine with #with_secret_config
+      #
+      #   with_secret_service :hashi_vault do
+      #     with_secret_config region: "us-west-1" do
+      #       val1 = secret(name: "test1")
+      #       val2 = secret(name: "test2")
+      #     end
+      #   end
+      #
+      # @param [Symbol] service The default secret service to use when fetching secrets
+      #
+      def with_secret_service(service)
+        raise ArgumentError, "You must pass a block to #with_secret_service" unless block_given?
+
+        begin
+          old_service = default_secret_service
+          # Use "public" API for input validation
+          default_secret_service(service)
+          yield
+        ensure
+          # Use "private" API so we can set back to nil
+          run_context.default_secret_service = old_service
+        end
+      end
+
+      #
+      # This allows you to set the default secret config that is used when
+      # fetching secrets.
+      #
+      # @example
+      #
+      #   default_secret_config region: "us-west-1"
+      #   val1 = secret(name: "test1", service: :hashi_vault)
+      #
+      # @example
+      #
+      #   default_secret_config #=> {}
+      #   default_secret_service region: "us-west-1"
+      #   default_secret_service #=> { region: "us-west-1" }
+      #
+      # @param [Hash<Symbol,Object>] config The default configuration options to apply when fetching secrets
+      # @return [Hash<Symbol,Object>]
+      #
+      def default_secret_config(**config)
+        return run_context.default_secret_config if config.empty?
+
+        run_context.default_secret_config = config
+      end
+
+      #
+      # This allows you to set the secret config for the scope of the block
+      # passed into this method.
+      #
+      # @example
+      #
+      #   with_secret_config region: "us-west-1" do
+      #     val1 = secret(name: "test1", service: :hashi_vault)
+      #     val2 = secret(name: "test2", service: :hashi_vault)
+      #   end
+      #
+      # @param [Hash<Symbol,Object>] config The default configuration options to use when fetching secrets
+      #
+      def with_secret_config(**config)
+        raise ArgumentError, "You must pass a block to #with_secret_config" unless block_given?
+
+        begin
+          old_config = default_secret_config
+          # Use "public" API for input validation
+          default_secret_config(**config)
+          yield
+        ensure
+          # Use "private" API so we can set back to nil
+          run_context.default_secret_config = old_config
+        end
+      end
+
+      # Helper method which looks up a secret using the given service and configuration,
+      # and returns the retrieved secret value.
+      # This DSL providers a wrapper around [Chef::SecretFetcher]
+      #
+      # Use of the secret helper in the context of a resource block will automatically mark
+      # that resource as 'sensitive', preventing resource data from being logged.  See [Chef::Resource#sensitive].
+      #
+      # @option name [Object] The identifier or name for this secret
+      # @option version [Object] The secret version. If a service supports versions
+      #                          and no version is provided, the latest version will be fetched.
+      # @option service [Symbol] The service identifier for the service that will
+      #                         perform the secret lookup. See
+      #                         [Chef::SecretFetcher::SECRET_FETCHERS]
+      # @option config [Hash] The configuration that the named service expects
+      #
+      # @return result [Object] The response object type is determined by the fetcher but will usually be a string or a hash.
+      # See individual fetcher documentation to know what to expect for a given service.
+      #
+      # @example
+      #
+      # This example uses the built-in :example secret manager service, which
+      # accepts a hash of secrets.
+      #
+      #   value = secret(name: "test1", service: :example, config: { "test1" => "value1" })
+      #   log "My secret is #{value}"
+      #
+      #   value = secret(name: "test1", service: :aws_secrets_manager, version: "v1", config: { region: "us-west-1" })
+      #   log "My secret is #{value}"
+      def secret(name: nil, version: nil, service: default_secret_service, config: default_secret_config)
+        sensitive(true) if is_a?(Chef::Resource)
+        Chef::SecretFetcher.for_service(service, config, run_context).fetch(name, version)
+      end
+    end
+  end
+end

data/lib/chef/dsl/toml.rb

@@ -0,0 +1,116 @@
+require "date"
+
+# imported from https://github.com/chef-cookbooks/habitat
+class Chef
+  module DSL
+    module Toml
+      class Dumper
+        attr_reader :toml_str
+
+        def initialize(hash)
+          @toml_str = ""
+
+          visit(hash, [])
+        end
+
+        private
+
+        def visit(hash, prefix, extra_brackets = false)
+          simple_pairs, nested_pairs, table_array_pairs = sort_pairs hash
+
+          if prefix.any? && (simple_pairs.any? || hash.empty?)
+            print_prefix prefix, extra_brackets
+          end
+
+          dump_pairs simple_pairs, nested_pairs, table_array_pairs, prefix
+        end
+
+        def sort_pairs(hash)
+          nested_pairs = []
+          simple_pairs = []
+          table_array_pairs = []
+
+          hash.keys.sort.each do |key|
+            val = hash[key]
+            element = [key, val]
+
+            if val.is_a? Hash
+              nested_pairs << element
+            elsif val.is_a?(Array) && val.first.is_a?(Hash)
+              table_array_pairs << element
+            else
+              simple_pairs << element
+            end
+          end
+
+          [simple_pairs, nested_pairs, table_array_pairs]
+        end
+
+        def dump_pairs(simple, nested, table_array, prefix = [])
+          # First add simple pairs, under the prefix
+          dump_simple_pairs simple
+          dump_nested_pairs nested, prefix
+          dump_table_array_pairs table_array, prefix
+        end
+
+        def dump_simple_pairs(simple_pairs)
+          simple_pairs.each do |key, val|
+            key = quote_key(key) unless bare_key? key
+            @toml_str << "#{key} = #{to_toml(val)}\n"
+          end
+        end
+
+        def dump_nested_pairs(nested_pairs, prefix)
+          nested_pairs.each do |key, val|
+            key = quote_key(key) unless bare_key? key
+
+            visit val, prefix + [key], false
+          end
+        end
+
+        def dump_table_array_pairs(table_array_pairs, prefix)
+          table_array_pairs.each do |key, val|
+            key = quote_key(key) unless bare_key? key
+            aux_prefix = prefix + [key]
+
+            val.each do |child|
+              print_prefix aux_prefix, true
+              args = sort_pairs(child) << aux_prefix
+
+              dump_pairs(*args)
+            end
+          end
+        end
+
+        def print_prefix(prefix, array = false)
+          new_prefix = prefix.join(".")
+          new_prefix = "[#{new_prefix}]" if array
+
+          @toml_str += "[#{new_prefix}]\n"
+        end
+
+        def to_toml(obj)
+          if obj.is_a?(Time) || obj.is_a?(DateTime)
+            obj.strftime("%Y-%m-%dT%H:%M:%SZ")
+          elsif obj.is_a?(Date)
+            obj.strftime("%Y-%m-%d")
+          elsif obj.is_a? Regexp
+            obj.inspect.inspect
+          elsif obj.is_a? String
+            obj.inspect.gsub(/\\(#[$@{])/, '\1')
+          else
+            obj.inspect
+          end
+        end
+
+        def bare_key?(key)
+          !!key.to_s.match(/^[a-zA-Z0-9_-]*$/)
+        end
+
+        def quote_key(key)
+          '"' + key.gsub('"', '\\"') + '"'
+        end
+      end
+    end
+  end
+end

data/lib/chef/dsl/universal.rb

@@ -22,6 +22,10 @@ require_relative "data_query"
 require_relative "chef_vault"
 require_relative "registry_helper"
 require_relative "powershell"
+require_relative "secret"
+require_relative "reader_helpers"
+require_relative "render_helpers"
+require_relative "toml"
 require_relative "../mixin/powershell_exec"
 require_relative "../mixin/powershell_out"
 require_relative "../mixin/shell_out"
@@ -47,6 +51,9 @@ class Chef
       include Chef::DSL::ChefVault
       include Chef::DSL::RegistryHelper
       include Chef::DSL::Powershell
+      include Chef::DSL::ReaderHelpers
+      include Chef::DSL::RenderHelpers
+      include Chef::DSL::Secret
       include Chef::Mixin::PowershellExec
       include Chef::Mixin::PowershellOut
       include Chef::Mixin::ShellOut

data/lib/chef/dsl.rb

@@ -4,3 +4,4 @@ require_relative "dsl/data_query"
 require_relative "dsl/include_recipe"
 require_relative "dsl/include_attribute"
 require_relative "dsl/registry_helper"
+require_relative "dsl/secret"

data/lib/chef/event_dispatch/base.rb

@@ -164,7 +164,7 @@ class Chef
       # Called when LWRPs are finished loading
       def lwrp_load_complete; end
 
-      # Called when an ohai plugin file loading starts
+      # Called when ohai plugin file loading starts
       def ohai_plugin_load_start(file_count); end
 
       # Called when an ohai plugin file has been loaded
@@ -173,9 +173,51 @@ class Chef
       # Called when an ohai plugin file has an error on load.
       def ohai_plugin_file_load_failed(path, exception); end
 
-      # Called when an ohai plugin file loading has finished
+      # Called when ohai plugin file loading has finished
       def ohai_plugin_load_complete; end
 
+      # Called when compliance file loading starts
+      def compliance_load_start; end
+
+      # Called when compliance file loading ends
+      def compliance_load_complete; end
+
+      # Called when compliance profile loading starts
+      def profiles_load_start; end
+
+      # Called when compliance profile loading end
+      def profiles_load_complete; end
+
+      # Called when compliance input loading starts
+      def inputs_load_start; end
+
+      # Called when compliance input loading end
+      def inputs_load_complete; end
+
+      # Called when compliance waiver loading starts
+      def waivers_load_start; end
+
+      # Called when compliance waiver loading end
+      def waivers_load_complete; end
+
+      # Called when a compliance profile is found in a cookbook by the cookbook_compiler
+      def compliance_profile_loaded(profile); end
+
+      # Called when a compliance waiver is found in a cookbook by the cookbook_compiler
+      def compliance_waiver_loaded(waiver); end
+
+      # Called when a compliance waiver is found in a cookbook by the cookbook_compiler
+      def compliance_input_loaded(input); end
+
+      # Called when a compliance profile is enabled (by include_profile)
+      def compliance_profile_enabled(profile); end
+
+      # Called when a compliance waiver is enabled (by include_waiver)
+      def compliance_waiver_enabled(waiver); end
+
+      # Called when a compliance input is enabled (by include_input)
+      def compliance_input_enabled(input); end
+
       # Called before attribute files are loaded
       def attribute_load_start(attribute_file_count); end
 
@@ -221,7 +263,8 @@ class Chef
       # Called before convergence starts
       def converge_start(run_context); end
 
-      # Callback hook for handlers to register their interest in the action_collection
+      # Callback hook for handlers to grab a reference to the action_collection
+      # (sent before compiling cookbooks, consumers can also find it off the run_context.action_collection)
       def action_collection_registration(action_collection); end
 
       # Called when the converge phase is finished.

data/lib/chef/exceptions.rb

@@ -290,6 +290,36 @@ class Chef
 
     end
 
+    class Secret
+      class RetrievalError < RuntimeError; end
+      class ConfigurationInvalid < RuntimeError; end
+      class FetchFailed < RuntimeError; end
+      class MissingSecretName < RuntimeError; end
+      class InvalidSecretName < RuntimeError; end
+
+      class InvalidFetcherService < RuntimeError
+        def initialize(given, fetcher_service_names)
+          super("#{given} is not a supported secrets service.  Supported services are: :#{fetcher_service_names.join(" :")}")
+        end
+      end
+
+      class MissingFetcher < RuntimeError
+        def initialize(fetcher_service_names)
+          super("No secret service provided. Supported services are: :#{fetcher_service_names.join(" :")}")
+        end
+      end
+
+      class Azure
+        class IdentityNotFound < RuntimeError
+          def initialize
+            super("The managed identity could not be found. This could mean one of the following things:\n\n" \
+                  "  1. The VM has no system or user assigned identities.\n" \
+                  "  2. The managed identity object_id or client_id that was specified is not assigned to the VM.\n")
+          end
+        end
+      end
+    end
+
     # Exception class for collecting multiple failures. Used when running
     # delayed notifications so that chef can process each delayed
     # notification even if chef client or other notifications fail.

data/lib/chef/formatters/doc.rb

@@ -41,10 +41,11 @@ class Chef
       end
 
       def run_start(version, run_status)
-        puts_line "Starting #{ChefUtils::Dist::Infra::PRODUCT}, version #{version}"
+        puts_line "#{ChefUtils::Dist::Infra::PRODUCT}, version #{version}"
         puts_line "Patents: #{ChefUtils::Dist::Org::PATENTS}"
-        puts_line "Targeting node: #{Chef::Config.target_mode.host}" if Chef::Config.target_mode?
         puts_line "OpenSSL FIPS 140 mode enabled" if Chef::Config[:fips]
+        puts_line "Infra Phase starting"
+        puts_line "Targeting node: #{Chef::Config.target_mode.host}" if Chef::Config.target_mode?
       end
 
       def total_resources
@@ -56,7 +57,8 @@ class Chef
         # Print out deprecations.
         unless deprecations.empty?
           puts_line ""
-          puts_line "Deprecated features used!"
+          puts_line "Deprecation warnings that must be addressed before upgrading to Chef Infra #{Chef::VERSION.to_i + 1}:"
+          puts_line ""
           deprecations.each do |message, details|
             locations = details[:locations]
             if locations.size == 1
@@ -78,18 +80,18 @@ class Chef
           puts_line ""
         end
         if Chef::Config[:why_run]
-          puts_line "#{ChefUtils::Dist::Infra::PRODUCT} finished, #{@updated_resources}/#{total_resources} resources would have been updated"
+          puts_line "Infra Phase complete, #{@updated_resources}/#{total_resources} resources would have been updated"
         else
-          puts_line "#{ChefUtils::Dist::Infra::PRODUCT} finished, #{@updated_resources}/#{total_resources} resources updated in #{pretty_elapsed_time}"
+          puts_line "Infra Phase complete, #{@updated_resources}/#{total_resources} resources updated in #{pretty_elapsed_time}"
         end
       end
 
       def run_failed(exception)
         @end_time = Time.now
         if Chef::Config[:why_run]
-          puts_line "#{ChefUtils::Dist::Infra::PRODUCT} failed. #{@updated_resources} resources would have been updated"
+          puts_line "Infra Phase failed. #{@updated_resources} resources would have been updated"
         else
-          puts_line "#{ChefUtils::Dist::Infra::PRODUCT} failed. #{@updated_resources} resources updated in #{pretty_elapsed_time}"
+          puts_line "Infra Phase failed. #{@updated_resources} resources updated in #{pretty_elapsed_time}"
         end
       end
 
@@ -118,12 +120,12 @@ class Chef
       def node_load_completed(node, expanded_run_list, config); end
 
       def policyfile_loaded(policy)
-        puts_line "Using policy '#{policy["name"]}' at revision '#{policy["revision_id"]}'"
+        puts_line "Using Policyfile '#{policy["name"]}' at revision '#{policy["revision_id"]}'"
       end
 
       # Called before the cookbook collection is fetched from the server.
       def cookbook_resolution_start(expanded_run_list)
-        puts_line "resolving cookbooks for run list: #{expanded_run_list.inspect}"
+        puts_line "Resolving cookbooks for run list: #{expanded_run_list.inspect}"
       end
 
       # Called when there is an error getting the cookbook collection from the
@@ -148,7 +150,7 @@ class Chef
 
       # Called before cookbook sync starts
       def cookbook_sync_start(cookbook_count)
-        puts_line "Synchronizing Cookbooks:"
+        puts_line "Synchronizing cookbooks:"
         indent
       end
 
@@ -167,7 +169,7 @@ class Chef
 
       # Called when starting to collect gems from the cookbooks
       def cookbook_gem_start(gems)
-        puts_line "Installing Cookbook Gems:"
+        puts_line "Installing cookbook gem dependencies:"
         indent
       end
 
@@ -193,7 +195,7 @@ class Chef
 
       # Called when cookbook loading starts.
       def library_load_start(file_count)
-        puts_line "Compiling Cookbooks..."
+        puts_line "Compiling cookbooks..."
       end
 
       # Called after a file in a cookbook is loaded.
@@ -279,7 +281,7 @@ class Chef
       end
 
       def resource_bypassed(resource, action, provider)
-        puts " (Skipped: whyrun not supported by provider #{provider.class.name})", stream: resource
+        puts " (Skipped: Why-Run not supported by provider #{provider.class.name})", stream: resource
         unindent
       end
 
@@ -316,7 +318,7 @@ class Chef
       # Called when resource current state load is skipped due to the provider
       # not supporting whyrun mode.
       def resource_current_state_load_bypassed(resource, action, current_resource)
-        puts_line("* Whyrun not supported for #{resource}, bypassing load.", :yellow)
+        puts_line("* Why-Run not supported for #{resource}, bypassing load.", :yellow)
       end
 
       def stream_output(stream, output, options = {})
@@ -361,6 +363,52 @@ class Chef
         end
       end
 
+      # Called when compliance profile loading starts
+      def profiles_load_start
+        puts_line("Loading #{Inspec::Dist::PRODUCT_NAME} profile files:")
+      end
+
+      # Called when compliance input loading starts
+      def inputs_load_start
+        puts_line("Loading #{Inspec::Dist::PRODUCT_NAME} input files:")
+      end
+
+      # Called when compliance waiver loading starts
+      def waivers_load_start
+        puts_line("Loading #{Inspec::Dist::PRODUCT_NAME} waiver files:")
+      end
+
+      # Called when a compliance profile is found in a cookbook by the cookbook_compiler
+      def compliance_profile_loaded(profile)
+        start_line("  - #{profile.cookbook_name}::#{profile.pathname}", :cyan)
+        puts " (#{profile.version})", :cyan if profile.version
+      end
+
+      # Called when a compliance waiver is found in a cookbook by the cookbook_compiler
+      def compliance_input_loaded(input)
+        puts_line("  - #{input.cookbook_name}::#{input.pathname}", :cyan)
+      end
+
+      # Called when a compliance waiver is found in a cookbook by the cookbook_compiler
+      def compliance_waiver_loaded(waiver)
+        puts_line("  - #{waiver.cookbook_name}::#{waiver.pathname}", :cyan)
+      end
+
+      # Called when a compliance profile is enabled (by include_profile)
+      def compliance_profile_enabled(profile)
+        # puts_line("  * FIXME", :cyan)
+      end
+
+      # Called when a compliance waiver is enabled (by include_waiver)
+      def compliance_waiver_enabled(waiver)
+        # puts_line("  * FIXME", :cyan)
+      end
+
+      # Called when a compliance input is enabled (by include_input)
+      def compliance_input_enabled(input)
+        # puts_line("  * FIXME", :cyan)
+      end
+
       # (see Base#deprecation)
       def deprecation(deprecation, _location = nil)
         if Chef::Config[:treat_deprecation_warnings_as_errors]