chef 18.3.0 → 18.4.2

data/lib/chef/provider/package/chocolatey.rb

@@ -14,10 +14,15 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 #
+# This uses class variables on purpose to maintain a state cache between resources,
+# since they work on shared state
+#
+# rubocop:disable Style/ClassVars
 
 require_relative "../package"
 require_relative "../../resource/chocolatey_package"
 require_relative "../../win32/api/command_line_helper" if ChefUtils.windows?
+require "zip" unless defined?(Zip)
 
 class Chef
   class Provider
@@ -32,12 +37,16 @@ class Chef
         PATHFINDING_POWERSHELL_COMMAND = "[System.Environment]::GetEnvironmentVariable('ChocolateyInstall', 'MACHINE')".freeze
         CHOCO_MISSING_MSG = <<~EOS.freeze
           Could not locate your Chocolatey install. To install chocolatey, we recommend
-          the 'chocolatey' cookbook (https://github.com/chocolatey/chocolatey-cookbook).
+          the 'chocolatey_installer' resource.
           If Chocolatey is installed, ensure that the 'ChocolateyInstall' environment
           variable is correctly set. You can verify this with the PowerShell command
           '#{PATHFINDING_POWERSHELL_COMMAND}'.
         EOS
 
+        # initialize our cache on load
+        @@choco_available_packages = nil
+        @@choco_config = nil
+
         # Responsible for building the current_resource.
         #
         # @return [Chef::Resource::ChocolateyPackage] the current_resource
@@ -45,6 +54,10 @@ class Chef
           @current_resource = Chef::Resource::ChocolateyPackage.new(new_resource.name)
           current_resource.package_name(new_resource.package_name)
           current_resource.version(build_current_versions)
+          # Ensure that we have a working chocolatey executable - this used to be
+          # covered off by loading the resource, but since that's no longer required,
+          # we're going to put a quick check here to fail early!
+          choco_exe
           current_resource
         end
 
@@ -130,19 +143,48 @@ class Chef
         # install from, but like the rubygem provider's sources which are more like repos.
         def check_resource_semantics!; end
 
-        def self.get_choco_version
-          @get_choco_version ||= powershell_exec!("choco --version").result
+        def get_choco_version
+          @get_choco_version ||= powershell_exec!("#{choco_exe} --version").result
         end
 
         # Choco V2 uses 'Search' for remote repositories and 'List' for local packages
-        def self.query_command
+        def query_command
           return "list" if get_choco_version.match?(/^1/)
 
           "search"
         end
 
-        def query_command
-          self.class.query_command
+        # invalidate cache for testing purposes
+        def invalidate_cache
+          @@choco_config = nil
+        end
+
+        # This checks that the repo list has not changed between now and when we last checked
+        # the cache
+        def cache_is_valid?
+          return false if @@choco_config.nil? || (actual_config != @@choco_config)
+
+          true
+        end
+
+        # walk the collection to find peer resources to ensure that
+        # we get as much of the cache this run as possible.  Unified mode
+        # sub-resources will, of course, be incremental with this, since we can't
+        # grab them in advance, but even in that case we're still way, way
+        # more efficient with our queries...
+        #
+        # @return [Array] List of chocolatey packages referenced in the run list
+        def collect_package_requests(ignore_list: [])
+          return ["*"] if new_resource.bulk_query || Chef::Config[:always_use_bulk_chocolatey_package_list]
+
+          # Get to the root of the resource collection
+          rc = run_context.parent_run_context || run_context
+          rc = rc.parent_run_context while rc.parent_run_context
+
+          package_collection = package_name_array
+          package_collection += nested_package_resources(rc.resource_collection)
+          # downcase the array and uniq.  sorted for easier testing...
+          package_collection.uniq.sort.filter { |pkg| !ignore_list.include?(pkg) }
         end
 
         private
@@ -158,6 +200,22 @@ class Chef
           gem_v1 <=> gem_v2
         end
 
+        # Cache the configuration in order to ensure that we can check our
+        # package cache is valid for a run
+        def actual_config
+          config_path = ::File.join("#{choco_install_path}", "config", "chocolatey.config")
+          if ::File.exist?(config_path)
+            return ::File.read(config_path)
+          end
+
+          nil
+        end
+
+        # update the validity of the package cache
+        def set_package_cache
+          @@choco_config = actual_config
+        end
+
         # Magic to find where chocolatey is installed in the system, and to
         # return the full path of choco.exe
         #
@@ -175,9 +233,15 @@ class Chef
 
         # lets us mock out an incorrect value for testing.
         def choco_install_path
-          result = powershell_exec!(PATHFINDING_POWERSHELL_COMMAND).result
-          result = "" if result.empty?
-          result
+          @choco_install_path ||= begin
+            result = powershell_exec!(PATHFINDING_POWERSHELL_COMMAND).result
+            result = "" if result.empty?
+            result
+          end
+        end
+
+        def choco_lib_path
+          ::File.join(choco_install_path, "lib")
         end
 
         # Helper to dispatch a choco command through shell_out using the timeout
@@ -218,6 +282,17 @@ class Chef
           Hash[*lowercase_names(new_resource.package_name).zip(desired_versions).flatten]
         end
 
+        def nested_package_resources(res)
+          package_collection = []
+          res.each do |child_res|
+            package_collection += nested_package_resources(child_res.resources)
+            next unless child_res.is_a?(Chef::Resource::ChocolateyPackage)
+
+            package_collection += child_res.package_name.flatten
+          end
+          package_collection
+        end
+
         # Helper to construct optional args out of new_resource
         #
         # @param include_source [Boolean] should the source parameter be added
@@ -234,37 +309,127 @@ class Chef
         #
         # @return [Hash] name-to-version mapping of available packages
         def available_packages
-          return @available_packages if @available_packages
+          return @available_packages unless @available_packages.nil?
+
+          # @available_packages is per object - each resource is an object, meaning if you
+          # have a LOT of chocolatey package installs, then this quickly gets very slow.
+          # So we use @@choco_available_packages instead - BUT it's important to ensure that
+          # the cache is valid before you do this.  There are two cache items that can change:
+          # a) the sources - we check this with cache_is_valid?
+          if cache_is_valid? && @@choco_available_packages.is_a?(Hash) &&
+              @@choco_available_packages[new_resource.list_options]
+
+            # Ensure we have the package names, or else double check...
+            need_redo = false
+            package_name_array.each do |pkg|
+              need_redo = true unless @@choco_available_packages[new_resource.list_options][pkg.downcase]
+            end
+            return @@choco_available_packages[new_resource.list_options] unless need_redo
+          end
+          if new_resource.list_options
+            Chef::Log.info("Fetching chocolatey package list with options #{new_resource.list_options.inspect}")
+          else
+            Chef::Log.info("Fetching chocolatey package list")
+          end
 
-          @available_packages = {}
-          package_name_array.each do |pkg|
+          # Only reset the array if the cache is invalid - if we're just augmenting it, don't
+          # clear it
+          @@choco_available_packages = {} if @@choco_available_packages.nil? || !cache_is_valid?
+          if @@choco_available_packages[new_resource.list_options].nil?
+            @@choco_available_packages[new_resource.list_options] = {}
+          end
+
+          # Attempt to get everything we need in a single query.
+          # Grab 25 packages at a time at most, to avoid hurting servers too badly
+          #
+          # For v1 we actually used to grab the entire list of packages, but we found
+          # that it could cause undue load on really large package lists
+          collect_package_requests(
+            ignore_list: @@choco_available_packages[new_resource.list_options].keys
+          ).each_slice(25) do |pkg_set|
             available_versions =
               begin
-                cmd = [ query_command, "-r", pkg ]
-                cmd += common_options
-                cmd.push( new_resource.list_options ) if new_resource.list_options
-
-                raw = parse_list_output(*cmd)
-                raw.keys.each_with_object({}) do |name, available|
-                  available[name] = desired_name_versions[name] || raw[name]
-                end
+              cmd = [ query_command, "-r" ]
+
+              # Chocolatey doesn't actually take a wildcard for this query, however
+              # it will return all packages when using '*' as a query
+              unless pkg_set == ["*"]
+                cmd += pkg_set
               end
-            @available_packages.merge! available_versions
+              cmd += common_options
+              cmd.push( new_resource.list_options ) if new_resource.list_options
+
+              Chef::Log.debug("Choco List Command: #{cmd}")
+
+              raw = parse_list_output(*cmd)
+              raw.keys.each_with_object({}) do |name, available|
+                available[name] = desired_name_versions[name] || raw[name]
+              end
+            end
+            @@choco_available_packages[new_resource.list_options].merge!(available_versions)
           end
-          @available_packages
+          # Mark the cache as valid, with the required metadata
+          set_package_cache
+          # Why both?  So when we fail to find a package once, we don't try on every
+          # retry, even though it would be reasonable to do so if queried in another
+          # resource (because the chocolatey configuration may well have changed!)
+          @available_packages = @@choco_available_packages[new_resource.list_options]
         end
 
         # Installed packages in chocolatey as a Hash of names mapped to versions
-        # (names are downcased for case-insensitive matching)
+        # (names are downcased for case-insensitive matching).  Depending on the user
+        # preference, we get these either from the local database, or from the choco
+        # list command
         #
+        # @return [Hash] name-to-version mapping of installed packages
+        def installed_packages
+          if new_resource.use_choco_list == false || !Chef::Config[:always_use_choco_list]
+            installed_packages_via_choco
+          else
+            installed_packages_via_disk
+          end
+        end
+
         # Beginning with Choco 2.0, "list" returns local packages only while "search" returns packages from external package sources
         #
         # @return [Hash] name-to-version mapping of installed packages
-        def installed_packages
+        def installed_packages_via_choco
           @installed_packages ||= Hash[*parse_list_output("list", "-l", "-r").flatten]
           @installed_packages
         end
 
+        # Return packages sourced from the local disk - because this doesn't have
+        # shell out overhead, this ends up being a significant performance win
+        # vs calling choco list
+        #
+        # @return [Hash] name-to-version mapping of installed packages
+        def installed_packages_via_disk
+          @installed_packages ||= begin
+            targets = new_resource.name
+            target_dirs = []
+            # If we're using a single package name, have it at the head of the list
+            # so we can get more performance.  In either case, the
+            # array is filled by the call to `get_local_pkg_dirs` below - but
+            # that contains all possible package folders, and so we push our
+            # guess to the front as an optimization.
+            target_dirs << targets.first.downcase if targets.length == 1
+            if targets.downcase is_a?(String)
+              target_dirs << targets
+            end
+            target_dirs += get_local_pkg_dirs(choco_lib_path)
+            fetch_package_versions(choco_lib_path, target_dirs, targets)
+          end
+        end
+
+        # Grab the nupkg folder list
+        def get_local_pkg_dirs(base_dir)
+          return [] unless Dir.exist?(base_dir)
+
+          Dir.entries(base_dir).select do |dir|
+            ::File.directory?(::File.join(base_dir, dir)) && !dir.start_with?(".")
+          end
+        end
+
         # Helper to convert choco.exe list output to a Hash
         # (names are downcased for case-insensitive matching)
         #
@@ -296,7 +461,46 @@ class Chef
           args.push( [ "--password", new_resource.password ]) if new_resource.password
           args
         end
+
+        # Fetch the local package versions from chocolatey
+        def fetch_package_versions(base_dir, target_dirs, targets)
+          pkg_versions = {}
+          target_dirs.each do |dir|
+            pkg_versions.merge!(get_pkg_data(::File.join(base_dir, dir)))
+            # return early if we found the single package version we were looking for
+            return pkg_versions if targets.length == 1 && pkg_versions[targets.first]
+          end
+          pkg_versions
+        end
+
+        # Grab the locally installed packages from the nupkg list
+        # rather than shelling out to chocolatey
+        def get_pkg_data(path)
+          t = ::File.join(path, "*.nupkg").gsub("\\", "/")
+          targets = Dir.glob(t)
+
+          # Extract package version from the first nuspec file in this nupkg
+          targets.each do |target|
+            Zip::File.open(target) do |zip_file|
+              zip_file.each do |entry|
+                next unless entry.name.end_with?(".nuspec")
+
+                f = entry.get_input_stream
+                doc = REXML::Document.new(f.read.to_s)
+                f.close
+                id = doc.elements["package/metadata/id"]
+                version = doc.elements["package/metadata/version"]
+                return { id.text.to_s.downcase => version.text } if id && version
+              end
+            end
+          end
+          {}
+        rescue StandardError => e
+          Chef::Log.warn("Failed to get package info for #{path}: #{e}")
+          {}
+        end
       end
     end
   end
 end
+# rubocop:enable Style/ClassVars

data/lib/chef/provider/package/zypper.rb

@@ -132,6 +132,11 @@ class Chef
           end
           current_version ||= latest_version if is_installed
           current_version
+        rescue Mixlib::ShellOut::ShellCommandFailed => e
+          # zypper returns a '104' code if info is called for a non-existent package
+          return nil if e.message =~ /'104'/
+
+          raise
         end
 
         def resolve_available_version(package_name, new_version)

data/lib/chef/provider/powershell_script.rb

@@ -27,8 +27,13 @@ class Chef
       provides :powershell_script
 
       action :run do
-        validate_script_syntax!
-        super()
+        Chef::Log.debug("using inline impl: #{new_resource.use_inline_powershell}")
+        if new_resource.use_inline_powershell
+          run_using_powershell_exec
+        else
+          validate_script_syntax!
+          super()
+        end
       end
 
       # Set InputFormat to None as PowerShell will hang if STDIN is redirected
@@ -38,6 +43,7 @@ class Chef
       def command
         # Must use -File rather than -Command to launch the script
         # file created by the base class that contains the script
+
         # code -- otherwise, powershell.exe does not propagate the
         # error status of a failed Windows process that ran at the
         # end of the script, it gets changed to '1'.
@@ -52,6 +58,69 @@ class Chef
 
       protected
 
+      # Run the inline version of powershell, using powershell_exec rather than powershell_out - this should
+      # hopefully
+      def run_using_powershell_exec
+        # Because of the nature of powershell_exec, we can't easily stream data to the shell, so we just flat out
+        # disallow this combination
+        if new_resource.live_stream
+          raise "powershell_script does not support live_stream when inline powershell is enabled - please choose one or the other"
+        end
+
+        # This comes from the execute super resource - since we want to limit the scope of this to powershell,
+        # we use this here
+        if creates && sentinel_file.exist?
+          logger.debug("#{new_resource} sentinel file #{sentinel_file} exists - nothing to do")
+          return false
+        end
+
+        converge_by("execute direct powershell #{new_resource.name}") do
+          # For scoping - otherwise, we lose return_code/stdout/stderr when the timeout block is over
+          return_code = nil
+          stdout = nil
+          stderr = nil
+
+          r = powershell_exec!(powershell_wrapper_script, new_resource.interpreter.to_sym, timeout: new_resource.timeout)
+          # Split out the stdout/return code format if needed
+          return_code = r.result
+
+          # The script returns an array if there is stdout, or just a plain return value if
+          # there is not - we need to handle both of these cases for full coverage
+          if return_code.is_a?(Array)
+            # Why to_s?  Because if the only powershell output is an object,
+            # it'll be returned here
+            stdout = return_code[0].to_s
+            stderr = r.errors.join("\n")
+            return_code = return_code[-1]
+          else
+            stdout = ""
+            stderr = r.errors.join("\n")
+          end
+
+          Chef::Log.info("Powershell output: #{stdout}") unless stdout.empty?
+          Chef::Log.info("Powershell error: #{stderr}") unless stderr.empty?
+
+          # Check the return code, and validate.  This code is cribbed from execute.rb, and does exactly the same
+          # thing as it does there
+          valid_returns = new_resource.returns
+          valid_returns = [valid_returns] if valid_returns.is_a?(Integer)
+          unless valid_returns.include?(return_code)
+            # Handle sensitive results
+            if sensitive?
+              ex = ChefPowerShell::PowerShellExceptions::PowerShellCommandFailed.new("Command execution failed. STDOUT/STDERR suppressed for sensitive resource")
+              # Forcibly hide the exception cause chain here so we don't log the unredacted version
+              def ex.cause
+                nil
+              end
+              raise ex
+            else
+              raise ChefPowerShell::PowerShellExceptions::PowerShellCommandFailed.new("Powershell command returned #{return_code} - output was \"#{stdout}\", error output  was \"#{stderr}\"")
+            end
+          end
+          true
+        end
+      end
+
       def interpreter_path
         # Powershell.exe is always in "v1.0" folder (for backwards compatibility)
         # pwsh is the other interpreter and we will assume that it is on the path.
@@ -69,7 +138,7 @@ class Chef
       end
 
       def code
-        code = wrapper_script
+        code = powershell_wrapper_script
         logger.trace("powershell_script provider called with script code:\n\n#{new_resource.code}\n")
         logger.trace("powershell_script provider will execute transformed code:\n\n#{code}\n")
         code
@@ -129,7 +198,11 @@ class Chef
       # last process run in the script if it is the last command
       # executed, otherwise 0 or 1 based on whether $? is set to true
       # (success, where we return 0) or false (where we return 1).
-      def wrapper_script
+      #
+      # This is the regular powershell version of the above script - the difference
+      # is that regular powershell allows for hidden visibility variables, due to the
+      # very slightly different semantics.
+      def powershell_wrapper_script
         <<~EOH
           # Chef Client wrapper for powershell_script resources
 
@@ -148,12 +221,23 @@ class Chef
           # Catch any exceptions -- without this, exceptions will result
           # In a zero return code instead of the desired non-zero code
           # that indicates a failure
+
+#{if new_resource.use_inline_powershell
+    # Inline powershell doesn't allow for private visibility variables,
+    # and uses return instead of exit
+    <<-EOI
+          trap [Exception] {write-error ($_.Exception.Message);return 1}
+          $interpolatedexitcode = $#{new_resource.convert_boolean_return}
+    EOI
+  else
+    <<-EOI
           trap [Exception] {write-error ($_.Exception.Message);exit 1}
 
           # Variable state that should not be accessible to the user code
           new-variable -name interpolatedexitcode -visibility private -value $#{new_resource.convert_boolean_return}
           new-variable -name chefscriptresult -visibility private
-
+    EOI
+  end}
           # Initialize a variable we use to capture $? inside a block
           $global:lastcmdlet = $null
 
@@ -198,7 +282,13 @@ class Chef
           # status of PowerShell.exe will be $exitstatus. If it was
           # launched with -Command, it will be 0 if $exitstatus was 0,
           # 1 (i.e. failed) otherwise.
-          exit $exitstatus
+#{if new_resource.use_inline_powershell
+    # Inline powershell needs return, not exit
+    "return $exitstatus\n"
+  else
+    "exit $exitstatus\n"
+  end
+}
         EOH
       end
 

data/lib/chef/provider/service/systemd.rb

@@ -51,6 +51,7 @@ class Chef::Provider::Service::Systemd < Chef::Provider::Service::Simple
         current_resource.running(false)
         current_resource.enabled(false)
         current_resource.masked(false)
+        current_resource.static(false)
         current_resource.indirect(false)
       end
     else
@@ -59,6 +60,7 @@ class Chef::Provider::Service::Systemd < Chef::Provider::Service::Simple
 
     current_resource.enabled(is_enabled?)
     current_resource.masked(is_masked?)
+    current_resource.static(is_static?)
     current_resource.indirect(is_indirect?)
     current_resource
   end
@@ -168,20 +170,29 @@ class Chef::Provider::Service::Systemd < Chef::Provider::Service::Simple
     end
   end
 
-  def enable_service
-    if current_resource.masked || current_resource.indirect
-      logger.debug("#{new_resource} cannot be enabled: it is masked or indirect")
-      return
+  def enableable?(action)
+    if current_resource.masked
+      logger.debug("#{new_resource} cannot be #{action}d: it is masked")
+      return false
+    end
+    if current_resource.static
+      logger.debug("#{new_resource} cannot be #{action}d: it is static")
+      return false
+    end
+    if current_resource.indirect
+      logger.debug("#{new_resource} cannot be #{action}d: it is indirect")
+      return false
     end
+    true
+  end
+
+  def enable_service
+    # This function can safely assume that enableable? is true
     options, args = get_systemctl_options_args
     shell_out!(systemctl_path, args, "enable", new_resource.service_name, **options)
   end
 
   def disable_service
-    if current_resource.masked || current_resource.indirect
-      logger.debug("#{new_resource} cannot be disabled: it is masked or indirect")
-      return
-    end
     options, args = get_systemctl_options_args
     shell_out!(systemctl_path, args, "disable", new_resource.service_name, **options)
   end
@@ -217,6 +228,10 @@ class Chef::Provider::Service::Systemd < Chef::Provider::Service::Simple
     systemd_service_status["UnitFileState"] == "indirect"
   end
 
+  def is_static?
+    systemd_service_status["UnitFileState"] == "static"
+  end
+
   def is_masked?
     # Note: masked-runtime is excluded, because runtime is volatile, and
     # because masked-runtime is not masked.

data/lib/chef/provider/service/windows.rb

@@ -74,6 +74,7 @@ class Chef::Provider::Service::Windows < Chef::Provider::Service
       current_resource.run_as_user(config_info.service_start_name)    if config_info.service_start_name
       current_resource.display_name(config_info.display_name)         if config_info.display_name
       current_resource.delayed_start(current_delayed_start)           if current_delayed_start
+      current_resource.description(config_info.description)           if new_resource.description
     end
 
     current_resource

data/lib/chef/provider/service.rb

@@ -80,7 +80,19 @@ class Chef
         end
       end
 
+      # This is a hook for subclasses to be able to tell the super class that a
+      # service is or is not enable-able. For example, on systemd, static and
+      # indirect units are not enable-able.
+      #
+      # In addition, this method offloads the messaging to the user. If the
+      # method returns `false`, it should say why `action` couldn't be taken
+      def enableable?(action)
+        true
+      end
+
       action :enable do
+        return unless enableable?(:enable)
+
         if current_resource.enabled
           logger.debug("#{new_resource} already enabled - nothing to do")
         else
@@ -94,6 +106,8 @@ class Chef
       end
 
       action :disable do
+        return unless enableable?(:disable)
+
         if current_resource.enabled
           converge_by("disable service #{new_resource}") do
             disable_service

data/lib/chef/provider/user.rb

@@ -153,7 +153,11 @@ class Chef
           new_val = new_resource.send(user_attrib)
           cur_val = current_resource.send(user_attrib)
           if !new_val.nil? && new_val.to_s != cur_val.to_s
-            @change_desc << "change #{user_attrib} from #{cur_val} to #{new_val}"
+            if user_attrib.to_s == "password" && new_resource.sensitive
+              @change_desc << "change #{user_attrib} from ******** to ********"
+            else
+              @change_desc << "change #{user_attrib} from #{cur_val} to #{new_val}"
+            end
           end
         end
 

data/lib/chef/recipe.rb

@@ -70,20 +70,12 @@ class Chef
 
     # This was moved to Chef::Node#tag, redirecting here for compatibility
     def tag(*tags)
-      run_context.node.tag(*tags)
+      node.tag(*tags)
     end
 
-    # Removes the list of tags from the node.
-    #
-    # === Parameters
-    # tags<Array>:: A list of tags
-    #
-    # === Returns
-    # tags<Array>:: The current list of run_context.node.tags
+    # This was moved to Chef::Node#untag, redirecting here for compatibility
     def untag(*tags)
-      tags.each do |tag|
-        run_context.node.tags.delete(tag)
-      end
+      node.untag(*tags)
     end
 
     def from_yaml_file(filename)

data/lib/chef/resource/_rest_resource.rb

@@ -351,7 +351,7 @@ action_class do
     Chef.run_context.transport.connection
   end
 
-  # Remove all empty keys (recusively) from Hash.
+  # Remove all empty keys (recursively) from Hash.
   # @see https://stackoverflow.com/questions/56457020/#answer-56458673
   def deep_compact!(hsh)
     raise TypeError unless hsh.is_a? Hash