chef 15.11.3-universal-mingw32 → 16.1.16-universal-mingw32

data/lib/chef/provider/package/dnf/dnf_helper.py

@@ -17,6 +17,7 @@ def get_sack():
         conf = base.conf
         conf.read()
         conf.installroot = '/'
+        conf.assumeyes = True
         subst = conf.substitutions
         subst.update_from_etc(conf.installroot)
         try:
@@ -25,6 +26,17 @@ def get_sack():
         except AttributeError:
             pass
         base.read_all_repos()
+        repos = base.repos
+
+        if 'repos' in command:
+            for repo_pattern in command['repos']:
+                if 'enable' in repo_pattern:
+                    for repo in repos.get_matching(repo_pattern['enable']):
+                        repo.enable()
+                if 'disable' in repo_pattern:
+                    for repo in repos.get_matching(repo_pattern['disable']):
+                        repo.disable()
+
         try:
             base.configure_plugins()
         except AttributeError:
@@ -40,13 +52,38 @@ def flushcache():
         pass
     get_sack().load_system_repo(build_cache=True)
 
+def version_tuple(versionstr):
+    e = '0'
+    v = None
+    r = None
+    colon_index = versionstr.find(':')
+    if colon_index > 0:
+        e = str(versionstr[:colon_index])
+    dash_index = versionstr.find('-')
+    if dash_index > 0:
+        tmp = versionstr[colon_index + 1:dash_index]
+        if tmp != '':
+            v = tmp
+        arch_index = versionstr.find('.', dash_index)
+        if arch_index > 0:
+            r = versionstr[dash_index + 1:arch_index]
+        else:
+            r = versionstr[dash_index + 1:]
+    else:
+        tmp = versionstr[colon_index + 1:]
+        if tmp != '':
+            v = tmp
+    return (e, v, r)
+
 def versioncompare(versions):
     sack = get_sack()
     if (versions[0] is None) or (versions[1] is None):
-      sys.stdout.write('0\n')
+      outpipe.write('0\n')
+      outpipe.flush()
     else:
-      evr_comparison = sack.evr_cmp(versions[0], versions[1])
-      sys.stdout.write('{}\n'.format(evr_comparison))
+      evr_comparison = dnf.rpm.rpm.labelCompare(version_tuple(versions[0]), version_tuple(versions[1]))
+      outpipe.write('{}\n'.format(evr_comparison))
+      outpipe.flush()
 
 def query(command):
     sack = get_sack()
@@ -78,37 +115,59 @@ def query(command):
     pkgs = q.latest(1).run()
 
     if not pkgs:
-        sys.stdout.write('{} nil nil\n'.format(command['provides'].split().pop(0)))
+        outpipe.write('{} nil nil\n'.format(command['provides'].split().pop(0)))
+        outpipe.flush()
     else:
         # make sure we picked the package with the highest version
         pkgs.sort
         pkg = pkgs.pop()
-        sys.stdout.write('{} {}:{}-{} {}\n'.format(pkg.name, pkg.epoch, pkg.version, pkg.release, pkg.arch))
+        outpipe.write('{} {}:{}-{} {}\n'.format(pkg.name, pkg.epoch, pkg.version, pkg.release, pkg.arch))
+        outpipe.flush()
 
 # the design of this helper is that it should try to be 'brittle' and fail hard and exit in order
 # to keep process tables clean.  additional error handling should probably be added to the retry loop
 # on the ruby side.
 def exit_handler(signal, frame):
+    if base is not None:
+        base.close()
     sys.exit(0)
 
-signal.signal(signal.SIGINT, exit_handler)
-signal.signal(signal.SIGHUP, exit_handler)
-signal.signal(signal.SIGPIPE, exit_handler)
-
-while 1:
-    # kill self if we get orphaned (tragic)
-    ppid = os.getppid()
-    if ppid == 1:
-        sys.exit(0)
-    line = sys.stdin.readline()
-    command = json.loads(line)
-    if command['action'] == "whatinstalled":
-        query(command)
-    elif command['action'] == "whatavailable":
-        query(command)
-    elif command['action'] == "flushcache":
-        flushcache()
-    elif command['action'] == "versioncompare":
-        versioncompare(command['versions'])
-    else:
-        raise RuntimeError("bad command")
+def setup_exit_handler():
+    signal.signal(signal.SIGINT, exit_handler)
+    signal.signal(signal.SIGHUP, exit_handler)
+    signal.signal(signal.SIGPIPE, exit_handler)
+    signal.signal(signal.SIGQUIT, exit_handler)
+
+if len(sys.argv) < 3:
+  inpipe = sys.stdin
+  outpipe = sys.stdout
+else:
+  inpipe = os.fdopen(int(sys.argv[1]), "r")
+  outpipe = os.fdopen(int(sys.argv[2]), "w")
+
+try:
+    while 1:
+        # kill self if we get orphaned (tragic)
+        ppid = os.getppid()
+        if ppid == 1:
+            raise RuntimeError("orphaned")
+
+        setup_exit_handler()
+        line = inpipe.readline()
+
+        try:
+            command = json.loads(line)
+        except ValueError:
+            raise RuntimeError("bad json parse")
+
+        if command['action'] == "whatinstalled":
+            query(command)
+        elif command['action'] == "whatavailable":
+            query(command)
+        elif command['action'] == "versioncompare":
+            versioncompare(command['versions'])
+        else:
+            raise RuntimeError("bad command")
+finally:
+    if base is not None:
+        base.closeRpmDB()

data/lib/chef/provider/package/dnf/python_helper.rb

@@ -18,6 +18,7 @@
 require_relative "../../../mixin/which"
 require_relative "../../../mixin/shell_out"
 require_relative "version"
+require "singleton" unless defined?(Singleton)
 require "timeout" unless defined?(Timeout)
 
 class Chef
@@ -32,6 +33,8 @@ class Chef
           attr_accessor :stdin
           attr_accessor :stdout
           attr_accessor :stderr
+          attr_accessor :inpipe
+          attr_accessor :outpipe
           attr_accessor :wait_thr
 
           DNF_HELPER = ::File.expand_path(::File.join(::File.dirname(__FILE__), "dnf_helper.py")).freeze
@@ -50,16 +53,28 @@ class Chef
 
           def start
             ENV["PYTHONUNBUFFERED"] = "1"
-            @stdin, @stdout, @stderr, @wait_thr = Open3.popen3(dnf_command)
+            @inpipe, inpipe_write = IO.pipe
+            outpipe_read, @outpipe = IO.pipe
+            @stdin, @stdout, @stderr, @wait_thr = Open3.popen3("#{dnf_command} #{outpipe_read.fileno} #{inpipe_write.fileno}", outpipe_read.fileno => outpipe_read, inpipe_write.fileno => inpipe_write, close_others: false)
+            outpipe_read.close
+            inpipe_write.close
           end
 
           def reap
             unless wait_thr.nil?
-              Process.kill("KILL", wait_thr.pid) rescue nil
+              Process.kill("INT", wait_thr.pid) rescue nil
+              begin
+                Timeout.timeout(3) do
+                  wait_thr.value # this calls waitpid()
+                end
+              rescue Timeout::Error
+                Process.kill("KILL", wait_thr.pid) rescue nil
+              end
               stdin.close unless stdin.nil?
               stdout.close unless stdout.nil?
               stderr.close unless stderr.nil?
-              wait_thr.value # this calls waitpit()
+              inpipe.close unless inpipe.nil?
+              outpipe.close unless outpipe.nil?
             end
           end
 
@@ -68,26 +83,39 @@ class Chef
           end
 
           def compare_versions(version1, version2)
-            with_helper do
-              json = build_version_query("versioncompare", [version1, version2])
-              Chef::Log.trace "sending '#{json}' to python helper"
-              stdin.syswrite json + "\n"
-              stdout.sysread(4096).chomp.to_i
+            query("versioncompare", { "versions" => [version1, version2] }).to_i
+          end
+
+          def options_params(options)
+            options.each_with_object({}) do |opt, h|
+              if opt =~ /--enablerepo=(.+)/
+                $1.split(",").each do |repo|
+                  h["repos"] ||= []
+                  h["repos"].push( { "enable" => repo } )
+                end
+              end
+              if opt =~ /--disablerepo=(.+)/
+                $1.split(",").each do |repo|
+                  h["repos"] ||= []
+                  h["repos"].push( { "disable" => repo } )
+                end
+              end
             end
           end
 
           # @return Array<Version>
-          def query(action, provides, version = nil, arch = nil)
-            with_helper do
-              json = build_query(action, provides, version, arch)
-              Chef::Log.trace "sending '#{json}' to python helper"
-              stdin.syswrite json + "\n"
-              output = stdout.sysread(4096).chomp
-              Chef::Log.trace "got '#{output}' from python helper"
-              version = parse_response(output)
-              Chef::Log.trace "parsed #{version} from python helper"
-              version
-            end
+          # NB: "options" here is the dnf_package options hash and is deliberately not **opts
+          def package_query(action, provides, version: nil, arch: nil, options: {})
+            parameters = { "provides" => provides, "version" => version, "arch" => arch }
+            repo_opts = options_params(options || {})
+            parameters.merge!(repo_opts)
+            # XXX: for now we restart before and after every query with an enablerepo/disablerepo to clean the helpers internal state
+            restart unless repo_opts.empty?
+            query_output = query(action, parameters)
+            version = parse_response(query_output.lines.last)
+            Chef::Log.trace "parsed #{version} from python helper"
+            restart unless repo_opts.empty?
+            version
           end
 
           def restart
@@ -116,17 +144,28 @@ class Chef
             hash["version"] = version
           end
 
-          def build_query(action, provides, version, arch)
-            hash = { "action" => action }
-            hash["provides"] = provides
-            add_version(hash, version) unless version.nil?
-            hash["arch" ] = arch unless arch.nil?
-            FFI_Yajl::Encoder.encode(hash)
+          def query(action, parameters)
+            with_helper do
+              json = build_query(action, parameters)
+              Chef::Log.trace "sending '#{json}' to python helper"
+              outpipe.syswrite json + "\n"
+              output = inpipe.sysread(4096).chomp
+              Chef::Log.trace "got '#{output}' from python helper"
+              return output
+            end
           end
 
-          def build_version_query(action, versions)
+          def build_query(action, parameters)
             hash = { "action" => action }
-            hash["versions"] = versions
+            parameters.each do |param_name, param_value|
+              hash[param_name] = param_value unless param_value.nil?
+            end
+
+            # Special handling for certain action / param combos
+            if %i{whatinstalled whatavailable}.include?(action)
+              add_version(hash, parameters["version"]) unless parameters["version"].nil?
+            end
+
             FFI_Yajl::Encoder.encode(hash)
           end
 
@@ -135,12 +174,16 @@ class Chef
             array.each_slice(3).map { |x| Version.new(*x) }.first
           end
 
-          def drain_stderr
+          def drain_fds
             output = ""
-            output += stderr.sysread(4096).chomp until IO.select([stderr], nil, nil, 0).nil?
+            fds, = IO.select([stderr, stdout, inpipe], nil, nil, 0)
+            unless fds.nil?
+              fds.each do |fd|
+                output += fd.sysread(4096) rescue ""
+              end
+            end
             output
-          rescue
-            # we must rescue EOFError, and we don't much care about errors on stderr anyway
+          rescue => e
             output
           end
 
@@ -151,23 +194,23 @@ class Chef
               check
               ret = yield
             end
-            output = drain_stderr
+            output = drain_fds
             unless output.empty?
-              Chef::Log.trace "discarding output on stderr from python helper: #{output}"
+              Chef::Log.trace "discarding output on stderr/stdout from python helper: #{output}"
             end
             ret
           rescue EOFError, Errno::EPIPE, Timeout::Error, Errno::ESRCH => e
-            output = drain_stderr
+            output = drain_fds
             if ( max_retries -= 1 ) > 0
               unless output.empty?
-                Chef::Log.trace "discarding output on stderr from python helper: #{output}"
+                Chef::Log.trace "discarding output on stderr/stdout from python helper: #{output}"
               end
               restart
               retry
             else
               raise e if output.empty?
 
-              raise "dnf-helper.py had stderr output:\n\n#{output}"
+              raise "dnf-helper.py had stderr/stdout output:\n\n#{output}"
             end
           end
         end

data/lib/chef/provider/package/dnf/version.rb

@@ -33,13 +33,17 @@ class Chef
           end
 
           def to_s
-            "#{name}-#{version}.#{arch}"
+            "#{name}-#{version}.#{arch}" unless version.nil?
           end
 
           def version_with_arch
             "#{version}.#{arch}" unless version.nil?
           end
 
+          def name_with_arch
+            "#{name}.#{arch}" unless name.nil?
+          end
+
           def matches_name_and_arch?(other)
             other.version == version && other.arch == arch
           end

data/lib/chef/provider/package/dpkg.rb

@@ -149,7 +149,7 @@ class Chef
           resolved_source_array.all? { |s| s && ::File.exist?(s) }
         end
 
-        # Helper to return all the nanes of the missing sources for error messages.
+        # Helper to return all the names of the missing sources for error messages.
         #
         # @return [Array<String>] Array of missing sources
         def missing_sources

data/lib/chef/provider/package/freebsd/base.rb

@@ -58,7 +58,8 @@ class Chef
 
           def makefile_variable_value(variable, dir = nil)
             options = dir ? { cwd: dir } : {}
-            make_v = shell_out!("make", "-V", variable, options.merge!(env: nil, returns: [0, 1]))
+            options.merge!(env: nil, returns: [0, 1])
+            make_v = shell_out!("make", "-V", variable, **options)
             make_v.exitstatus == 0 ? make_v.stdout.strip.split($OUTPUT_RECORD_SEPARATOR).first : nil # $\ is the line separator, i.e. newline.
           end
         end

data/lib/chef/provider/package/homebrew.rb

@@ -24,56 +24,60 @@ class Chef
   class Provider
     class Package
       class Homebrew < Chef::Provider::Package
+        allow_nils
+        use_multipackage_api
 
-        provides :package, os: "darwin", override: true
+        provides :package, os: "darwin"
         provides :homebrew_package
 
         include Chef::Mixin::HomebrewUser
 
         def load_current_resource
-          self.current_resource = Chef::Resource::HomebrewPackage.new(new_resource.name)
+          @current_resource = Chef::Resource::HomebrewPackage.new(new_resource.name)
           current_resource.package_name(new_resource.package_name)
-          current_resource.version(current_installed_version)
-          logger.trace("#{new_resource} current version is #{current_resource.version}") if current_resource.version
-
-          @candidate_version = candidate_version
-
-          logger.trace("#{new_resource} candidate version is #{@candidate_version}") if @candidate_version
+          current_resource.version(get_current_versions)
+          logger.trace("#{new_resource} current package version(s): #{current_resource.version}") if current_resource.version
 
           current_resource
         end
 
-        def install_package(name, version)
-          unless current_resource.version == version
-            brew("install", options, name)
+        def candidate_version
+          package_name_array.map do |package_name|
+            available_version(package_name)
           end
         end
 
-        def upgrade_package(name, version)
-          current_version = current_resource.version
-
-          if current_version.nil? || current_version.empty?
-            install_package(name, version)
-          elsif current_version != version
-            brew("upgrade", options, name)
+        def get_current_versions
+          package_name_array.map do |package_name|
+            installed_version(package_name)
           end
         end
 
-        def remove_package(name, version)
-          if current_resource.version
-            brew("uninstall", options, name)
-          end
+        def install_package(names, versions)
+          brew_cmd_output("install", options, names.compact)
         end
 
-        # Homebrew doesn't really have a notion of purging, do a "force remove"
-        def purge_package(name, version)
-          if current_resource.version
-            brew("uninstall", "--force", options, name)
-          end
+        # upgrades are a bit harder in homebrew than other package formats. If you try to
+        # brew upgrade a package that isn't installed it will fail so if a user specifies
+        # the action of upgrade we need to figure out which packages need to be installed
+        # and which packages can be upgrades. We do this by checking if brew_info has an entry
+        # via the installed_version helper.
+        def upgrade_package(names, versions)
+          # @todo when we no longer support Ruby 2.6 this can be simplified to be a .filter_map
+          upgrade_pkgs = names.select { |x| x if installed_version(x) }.compact
+          install_pkgs = names.select { |x| x unless installed_version(x) }.compact
+
+          brew_cmd_output("upgrade", options, upgrade_pkgs) unless upgrade_pkgs.empty?
+          brew_cmd_output("install", options, install_pkgs) unless install_pkgs.empty?
         end
 
-        def brew(*args)
-          get_response_from_command("brew", *args)
+        def remove_package(names, versions)
+          brew_cmd_output("uninstall", options, names.compact)
+        end
+
+        # Homebrew doesn't really have a notion of purging, do a "force remove"
+        def purge_package(names, versions)
+          brew_cmd_output("uninstall", "--force", options, names.compact)
         end
 
         # We implement a querying method that returns the JSON-as-Hash
@@ -83,9 +87,50 @@ class Chef
         # information, but that is not any more robust than using the
         # command-line interface that returns the same thing.
         #
-        # https://github.com/Homebrew/homebrew/wiki/Querying-Brew
+        # https://docs.brew.sh/Querying-Brew
+        #
+        # @returns [Hash] a hash of package information where the key is the package name
         def brew_info
-          @brew_info ||= Chef::JSONCompat.from_json(brew("info", "--json=v1", new_resource.package_name)).first
+          @brew_info ||= begin
+            command_array = ["info", "--json=v1"].concat package_name_array
+            # convert the array of hashes into a hash where the key is the package name
+
+            cmd_output = brew_cmd_output(command_array, allow_failure: true)
+
+            if cmd_output.empty?
+              # we had some kind of failure so we need to iterate through each package to find them
+              package_name_array.each_with_object({}) do |package_name, hsh|
+                cmd_output = brew_cmd_output("info", "--json=v1", package_name, allow_failure: true)
+                if cmd_output.empty?
+                  hsh[package_name] = {}
+                else
+                  json = Chef::JSONCompat.from_json(cmd_output).first
+                  hsh[json["name"]] = json
+                end
+              end
+            else
+              Hash[Chef::JSONCompat.from_json(cmd_output).collect { |pkg| [pkg["name"], pkg] }]
+            end
+          end
+        end
+
+        #
+        # Return the package information given a package name or package alias
+        #
+        # @param [String] name_or_alias The name of the package or its alias
+        #
+        # @return [Hash] Package information
+        #
+        def package_info(package_name)
+          # return the package hash if it's in the brew info hash
+          return brew_info[package_name] if brew_info[package_name]
+
+          # check each item in the hash to see if we were passed an alias
+          brew_info.each_value do |p|
+            return p if p["aliases"].include?(package_name)
+          end
+
+          {}
         end
 
         # Some packages (formula) are "keg only" and aren't linked,
@@ -94,15 +139,20 @@ class Chef
         # "current" (as in latest). Otherwise, we will use the version
         # that brew thinks is linked as the current version.
         #
-        def current_installed_version
-          if brew_info["keg_only"]
-            if brew_info["installed"].empty?
+        # @param [String] package name
+        #
+        # @returns [String] package version
+        def installed_version(i)
+          p_data = package_info(i)
+
+          if p_data["keg_only"]
+            if p_data["installed"].empty?
               nil
             else
-              brew_info["installed"].last["version"]
+              p_data["installed"].last["version"]
             end
           else
-            brew_info["linked_keg"]
+            p_data["linked_keg"]
           end
         end
 
@@ -115,19 +165,33 @@ class Chef
         # forward project.
         #
         # https://github.com/Homebrew/homebrew/wiki/Acceptable-Formulae#stable-versions
-        def candidate_version
-          brew_info["versions"]["stable"]
-        end
+        #
+        # @param [String] package name
+        #
+        # @returns [String] package version
+        def available_version(i)
+          p_data = package_info(i)
 
-        private
+          # nothing is available
+          return nil if p_data.empty?
 
-        def get_response_from_command(*command)
+          p_data["versions"]["stable"]
+        end
+
+        def brew_cmd_output(*command, **options)
           homebrew_uid = find_homebrew_uid(new_resource.respond_to?(:homebrew_user) && new_resource.homebrew_user)
           homebrew_user = Etc.getpwuid(homebrew_uid)
 
-          logger.trace "Executing '#{command.join(" ")}' as user '#{homebrew_user.name}'"
+          logger.trace "Executing 'brew #{command.join(" ")}' as user '#{homebrew_user.name}'"
+
+          # allow the calling method to decide if the cmd should raise or not
+          # brew_info uses this when querying out available package info since a bad
+          # package name will raise and we want to surface a nil available package so that
+          # the package provider can magically handle that
+          shell_out_cmd = options[:allow_failure] ? :shell_out : :shell_out!
+
           # FIXME: this 1800 second default timeout should be deprecated
-          output = shell_out!(*command, timeout: 1800, user: homebrew_uid, environment: { "HOME" => homebrew_user.dir, "RUBYOPT" => nil, "TMPDIR" => nil })
+          output = send(shell_out_cmd, "brew", *command, timeout: 1800, user: homebrew_uid, environment: { "HOME" => homebrew_user.dir, "RUBYOPT" => nil, "TMPDIR" => nil })
           output.stdout.chomp
         end