chef-utils 16.16.13 → 17.0.242

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 631e152472419e3367b55807eacf3992125bc2f4039a69cbad751f3d02e65e5e
-  data.tar.gz: f59599d86976539841cfcec95135c94ebbd0b7f78af5b4263ed263539c65a0be
+  metadata.gz: 8b28b0dd9a34f1242fdeca73f39924515befdec0a2990e4bd56d4d94f071c2aa
+  data.tar.gz: 49cb4691092f6b0db0567a8975e72c3905682208ff0c089fe3149ef8ee8da4d0
 SHA512:
-  metadata.gz: 7b0a164279805db2f0fb5c4669e287d5de31f0edea5d5fd531e06697dbef66ee8d4e9ea00e3e5f31ae64dff8cb4f8d4a5201513c31946e8ae9bcc7d227d8c2ca
-  data.tar.gz: 19ad980f4a523f30263433d827515a6250309bf66d00d6f38d0a8225b0df9d704f535b205dea3d0d94522ea1791995f84146c47a4532c36e4f7741809b248e66
+  metadata.gz: 16f7f528de5d5fb631552951cef5f0e763bffce0c52bb2fb563f8d1cebd16269027dc3a2daf1f151bc58d5901adfb2309d93ecc3b6868e29f75be14b5538511b
+  data.tar.gz: 705ddfa785ecb1a7448a95f342a3718e5f90996131fe2678c6cd15875f3eee5d5cc7d5b7d38fabfa1500c07df26eac804261360cd13b205a7a24435a8a4ee922

data/chef-utils.gemspec

@@ -13,7 +13,7 @@ Gem::Specification.new do |spec|
   spec.homepage      = "https://github.com/chef/chef/tree/master/chef-utils"
   spec.license       = "Apache-2.0"
 
-  spec.required_ruby_version = ">= 2.6.0"
+  spec.required_ruby_version = ">= 2.6"
 
   spec.metadata = {
     "bug_tracker_uri" => "https://github.com/chef/chef/issues",
@@ -41,6 +41,10 @@ Gem::Specification.new do |spec|
   # ABSOLUTELY NO EXCEPTIONS
   #
 
+  # concurrent-ruby is: 1. lightweight, 2. has zero deps, 3. is external to chef
+  # this is used for the parallel_map enumerable extension for lightweight threading
+  spec.add_dependency "concurrent-ruby"
+
   spec.files = %w{Rakefile LICENSE} + Dir.glob("*.gemspec") +
     Dir.glob("{lib,spec}/**/*", File::FNM_DOTMATCH).reject { |f| File.directory?(f) }
 end

data/lib/chef-utils/dsl/cloud.rb

@@ -23,7 +23,7 @@ module ChefUtils
     module Cloud
       include Internal
 
-      # Determine if the current node is "in the cloud".
+      # Determine if the current node is running in a known cloud.
       #
       # @param [Chef::Node] node the node to check
       # @since 15.8
@@ -35,7 +35,18 @@ module ChefUtils
         !node["cloud"].nil?
       end
 
-      # Return true if the current current node is in EC2.
+      # Determine if the current node is running in Alibaba Cloud
+      #
+      # @param [Chef::Node] node the node to check
+      # @since 17.0
+      #
+      # @return [Boolean]
+      #
+      def alibaba?(node = __getnode)
+        node.key?("alibaba")
+      end
+
+      # Determine if the current node is running in AWS EC2.
       #
       # @param [Chef::Node] node the node to check
       # @since 15.8
@@ -46,7 +57,7 @@ module ChefUtils
         node.key?("ec2")
       end
 
-      # Return true if the current current node is in GCE.
+      # Determine if the current node running in Google Compute Engine (GCE).
       #
       # @param [Chef::Node] node the node to check
       # @since 15.8
@@ -57,7 +68,7 @@ module ChefUtils
         node.key?("gce")
       end
 
-      # Return true if the current current node is in Rackspace.
+      # Determine if the current node is running in Rackspace.
       #
       # @param [Chef::Node] node the node to check
       # @since 15.8
@@ -68,7 +79,7 @@ module ChefUtils
         node.key?("rackspace")
       end
 
-      # Return true if the current current node is in Eucalyptus.
+      # Determine if the current node is running in Eucalyptus.
       #
       # @param [Chef::Node] node the node to check
       # @since 15.8
@@ -81,7 +92,7 @@ module ChefUtils
       # chef-sugar backcompat method
       alias_method :euca?, :eucalyptus?
 
-      # Return true if the current current node is in Linode.
+      # Determine if the current node is running in Linode.
       #
       # @param [Chef::Node] node the node to check
       # @since 15.8
@@ -92,7 +103,7 @@ module ChefUtils
         node.key?("linode")
       end
 
-      # Return true if the current current node is in OpenStack.
+      # Determine if the current node is running in OpenStack.
       #
       # @param [Chef::Node] node the node to check
       # @since 15.8
@@ -103,7 +114,7 @@ module ChefUtils
         node.key?("openstack")
       end
 
-      # Return true if the current current node is in Azure.
+      # Determine if the current node is running in Microsoft Azure.
       #
       # @param [Chef::Node] node the node to check
       # @since 15.8
@@ -114,7 +125,7 @@ module ChefUtils
         node.key?("azure")
       end
 
-      # Return true if the current current node is in DigitalOcean.
+      # Determine if the current node is running in DigitalOcean.
       #
       # @param [Chef::Node] node the node to check
       # @since 15.8
@@ -127,7 +138,7 @@ module ChefUtils
       # chef-sugar backcompat method
       alias_method :digitalocean?, :digital_ocean?
 
-      # Return true if the current current node is in SoftLayer.
+      # Determine if the current node is running in SoftLayer (IBM Cloud).
       #
       # @param [Chef::Node] node the node to check
       # @since 15.8

data/lib/chef-utils/dsl/introspection.rb

@@ -29,6 +29,17 @@ module ChefUtils
     module Introspection
       include TrainHelpers
 
+      # Determine if the node is using the Chef Effortless pattern in which the Chef Infra Client is packaged using Chef Habitat
+      #
+      # @param [Chef::Node] node the node to check
+      # @since 17.0
+      #
+      # @return [Boolean]
+      #
+      def effortless?(node = __getnode)
+        !!(node && node.read("chef_packages", "chef", "chef_effortless"))
+      end
+
       # Determine if the node is a docker container.
       #
       # @param [Chef::Node] node the node to check

data/lib/chef-utils/dsl/platform.rb

@@ -126,7 +126,7 @@ module ChefUtils
       # Determine if the current node is CentOS Stream.
       #
       # @param [Chef::Node] node the node to check
-      # @since 16.13
+      # @since 17.0
       #
       # @return [Boolean]
       #

data/lib/chef-utils/dsl/windows.rb

@@ -58,7 +58,7 @@ module ChefUtils
         node["kernel"]["product_type"] == "Server"
       end
 
-      # Determine the current Windows NT version. The NT version often differs from the marketing version, but offers a good way to find desktop and server releases that are based on the same codebase. IE: NT 6.3 is Windows 8.1 and Windows 2012 R2.
+      # Determine the current Windows NT version. The NT version often differs from the marketing version, but offers a good way to find desktop and server releases that are based on the same codebase. For example NT 6.3 corresponds to Windows 8.1 and Windows 2012 R2.
       #
       # @param [Chef::Node] node the node to check
       # @since 15.8

data/lib/chef-utils/mash.rb

@@ -94,6 +94,10 @@ module ChefUtils
       end
     end
 
+    unless method_defined?(:regular_reader)
+      alias_method :regular_reader, :[]
+    end
+
     unless method_defined?(:regular_writer)
       alias_method :regular_writer, :[]=
     end
@@ -102,6 +106,11 @@ module ChefUtils
       alias_method :regular_update, :update
     end
 
+    # @param key<Object> The key to get.
+    def [](key)
+      regular_reader(key)
+    end
+
     # @param key<Object> The key to set.
     # @param value<Object>
     #   The value to set the key to.
@@ -112,6 +121,12 @@ module ChefUtils
       regular_writer(convert_key(key), convert_value(value))
     end
 
+    # internal API for use by Chef's deep merge cache
+    # @api private
+    def internal_get(key)
+      regular_reader(key)
+    end
+
     # internal API for use by Chef's deep merge cache
     # @api private
     def internal_set(key, value)

data/lib/chef-utils/parallel_map.rb

@@ -0,0 +1,131 @@
+# frozen_string_literal: true
+#
+# Copyright:: Copyright (c) Chef Software Inc.
+# License:: Apache License, Version 2.0
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+require "concurrent/executors"
+require "concurrent/future"
+require "singleton" unless defined?(Singleton)
+
+module ChefUtils
+  #
+  # This module contains ruby refinements that adds several methods to the Enumerable
+  # class which are useful for parallel processing.
+  #
+  module ParallelMap
+    refine Enumerable do
+
+      # Enumerates through the collection in parallel using the thread pool provided
+      # or the default thread pool.  By using the default thread pool this supports
+      # recursively calling the method without deadlocking while using a globally
+      # fixed number of workers.  This method supports lazy collections.  It returns
+      # synchronously, waiting until all the work is done.  Failures are only reported
+      # after the collection has executed and only the first exception is raised.
+      #
+      # (0..).lazy.parallel_map { |i| i*i }.first(5)
+      #
+      # @return [Array] output results
+      #
+      def parallel_map(pool: nil)
+        return self unless block_given?
+
+        pool ||= ChefUtils::DefaultThreadPool.instance.pool
+
+        futures = map do |item|
+          future = Concurrent::Future.execute(executor: pool) do
+            yield item
+          end
+        end
+
+        futures.map(&:value!)
+      end
+
+      # This has the same behavior as parallel_map but returns the enumerator instead of
+      # the return values.
+      #
+      # @return [Enumerable] the enumerable for method chaining
+      #
+      def parallel_each(pool: nil, &block)
+        return self unless block_given?
+
+        parallel_map(pool: pool, &block)
+
+        self
+      end
+
+      # The flat_each method is tightly coupled to the usage of parallel_map within the
+      # ChefFS implementation.  It is not itself a parallel method, but it is used to
+      # iterate through the 2nd level of nested structure, which is tied to the nested
+      # structures that ChefFS returns.
+      #
+      # This is different from Enumerable#flat_map because that behaves like map.flatten(1) while
+      # this behaves more like flatten(1).each.  We need this on an Enumerable, so we have no
+      # Enumerable#flatten method to call.
+      #
+      # [ [ 1, 2 ], [ 3, 4 ] ].flat_each(&block) calls block four times with 1, 2, 3, 4
+      #
+      # [ [ 1, 2 ], [ 3, 4 ] ].flat_map(&block) calls block twice with [1, 2] and [3,4]
+      #
+      def flat_each(&block)
+        map do |value|
+          if value.is_a?(Enumerable)
+            value.each(&block)
+          else
+            yield value
+          end
+        end
+      end
+    end
+  end
+
+  # The DefaultThreadPool has a fixed thread size and has no
+  # queue of work and the behavior on failure to find a thread is for the
+  # caller to run the work.  This contract means that the thread pool can
+  # be called recursively without deadlocking and while keeping the fixed
+  # number of threads (and not exponentially growing the thread pool with
+  # the depth of recursion).
+  #
+  class DefaultThreadPool
+    include Singleton
+
+    DEFAULT_THREAD_SIZE = 10
+
+    # Size of the thread pool, must be set before getting the thread pool or
+    # calling parallel_map/parallel_each.  Does not (but could be modified to)
+    # support dynamic resizing. To get fully synchronous behavior set this equal to
+    # zero rather than one since the caller will get work if the threads are
+    # busy.
+    #
+    # @return [Integer] number of threads
+    attr_accessor :threads
+
+    # Memoizing accessor for the thread pool
+    #
+    # @return [Concurrent::ThreadPoolExecutor] the thread pool
+    def pool
+      @pool ||= Concurrent::ThreadPoolExecutor.new(
+        min_threads: threads || DEFAULT_THREAD_SIZE,
+        max_threads: threads || DEFAULT_THREAD_SIZE,
+        max_queue: 0,
+        # "synchronous" redefines the 0 in max_queue to mean 'no queue' instead of 'infinite queue'
+        # it does not mean synchronous execution (no threads) but synchronous offload to the threads.
+        synchronous: true,
+        # this prevents deadlocks on recursive parallel usage
+        fallback_policy: :caller_runs
+      )
+    end
+  end
+end

data/lib/chef-utils/version.rb

@@ -16,5 +16,5 @@
 
 module ChefUtils
   CHEFUTILS_ROOT = File.expand_path("..", __dir__)
-  VERSION = "16.16.13"
+  VERSION = "17.0.242"
 end

data/spec/unit/dsl/cloud_spec.rb

@@ -45,6 +45,10 @@ RSpec.describe ChefUtils::DSL::Cloud do
     end
   end
 
+  context "on alibaba" do
+    cloud_reports_true_for(:cloud?, :alibaba?, node: { "alibaba" => {}, "cloud" => {} })
+  end
+
   context "on ec2" do
     cloud_reports_true_for(:cloud?, :ec2?, node: { "ec2" => {}, "cloud" => {} })
   end

data/spec/unit/dsl/introspection_spec.rb

@@ -32,6 +32,18 @@ RSpec.describe ChefUtils::DSL::Introspection do
 
   let(:test_instance) { IntrospectionTestClass.new(node) }
 
+  context "#effortless?" do
+    # FIXME: use a real VividMash for these tests instead of stubbing
+    it "is false by default" do
+      expect(node).to receive(:read).with("chef_packages", "chef", "chef_effortless").and_return(nil)
+      expect(ChefUtils.effortless?(node)).to be false
+    end
+    it "is true when ohai reports a effortless" do
+      expect(node).to receive(:read).with("chef_packages", "chef", "chef_effortless").and_return(true)
+      expect(ChefUtils.effortless?(node)).to be true
+    end
+  end
+
   context "#docker?" do
     # FIXME: use a real VividMash for these tests instead of stubbing
     it "is false by default" do

data/spec/unit/parallel_map_spec.rb

@@ -0,0 +1,156 @@
+# frozen_string_literal: true
+#
+# Copyright:: Copyright (c) Chef Software Inc.
+# License:: Apache License, Version 2.0
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+require "chef-utils/parallel_map"
+
+using ChefUtils::ParallelMap
+
+RSpec.describe ChefUtils::ParallelMap do
+
+  shared_examples_for "common parallel API tests" do
+
+    before(:each) do
+      ChefUtils::DefaultThreadPool.instance.instance_variable_set(:@pool, nil)
+      ChefUtils::DefaultThreadPool.instance.threads = threads
+    end
+
+    after(:each) do
+      ChefUtils::DefaultThreadPool.instance.instance_variable_set(:@pool, nil)
+    end
+
+    it "parallel_map runs in parallel" do
+      # this is implicitly also testing that we run in the caller when we exhaust threads by running threads+1
+      val = threads + 1
+      ret = []
+      start = Time.now
+      (1..val).parallel_map do |i|
+        loop do
+          if val == i
+            ret << i
+            val -= 1
+            break
+          end
+          # we spin for quite awhile to wait for very slow testers if we have to
+          if Time.now - start > 30
+            raise "timed out; deadlocked due to lack of parallelization?"
+          end
+
+          # need to sleep a tiny bit to let other threads schedule
+          sleep 0.000001
+        end
+      end
+      expected = (1..threads + 1).to_a.reverse
+      expect(ret).to eql(expected)
+    end
+
+    it "parallel_each runs in parallel" do
+      # this is implicitly also testing that we run in the caller when we exhaust threads by running threads+1
+      val = threads + 1
+      ret = []
+      start = Time.now
+      (1..val).parallel_each do |i|
+        loop do
+          if val == i
+            ret << i
+            val -= 1
+            break
+          end
+          # we spin for quite awhile to wait for very slow testers if we have to
+          if Time.now - start > 30
+            raise "timed out; deadlocked due to lack of parallelization?"
+          end
+
+          # need to sleep a tiny bit to let other threads schedule
+          sleep 0.000001
+        end
+      end
+      expected = (1..threads + 1).to_a.reverse
+      expect(ret).to eql(expected)
+    end
+
+    it "parallel_map throws exceptions" do
+      expect { (0..10).parallel_map { |i| raise "boom" } }.to raise_error(RuntimeError)
+    end
+
+    it "parallel_each throws exceptions" do
+      expect { (0..10).parallel_each { |i| raise "boom" } }.to raise_error(RuntimeError)
+    end
+
+    it "parallel_map runs" do
+      ans = Timeout.timeout(30) do
+        (1..10).parallel_map { |i| i }
+      end
+      expect(ans).to eql((1..10).to_a)
+    end
+
+    it "parallel_each runs" do
+      Timeout.timeout(30) do
+        (1..10).parallel_each { |i| i }
+      end
+    end
+
+    it "recursive parallel_map will not deadlock" do
+      ans = Timeout.timeout(30) do
+        (1..2).parallel_map { |i| (1..2).parallel_map { |i| i } }
+      end
+      expect(ans).to eql([[1, 2], [1, 2]])
+    end
+
+    it "recursive parallel_each will not deadlock" do
+      ans = Timeout.timeout(30) do
+        (1..2).parallel_each { |i| (1..2).parallel_each { |i| i } }
+      end
+    end
+
+    it "parallel_map is lazy" do
+      ans = Timeout.timeout(30) do
+        (1..).lazy.parallel_map { |i| i }.first(5)
+      end
+      expect(ans).to eql((1..5).to_a)
+    end
+
+    it "parallel_each is lazy" do
+      ans = Timeout.timeout(30) do
+        (1..).lazy.parallel_each { |i| i }.first(5)
+      end
+    end
+  end
+
+  context "with 10 threads" do
+    let(:threads) { 10 }
+    it_behaves_like "common parallel API tests"
+  end
+
+  context "with 0 threads" do
+    let(:threads) { 0 }
+    it_behaves_like "common parallel API tests"
+  end
+
+  context "with 1 threads" do
+    let(:threads) { 1 }
+    it_behaves_like "common parallel API tests"
+  end
+
+  context "flat_each" do
+    it "runs each over items which are nested one level" do
+      sum = 0
+      [ [ 1, 2 ], [3, 4]].flat_each { |i| sum += i }
+      expect(sum).to eql(10)
+    end
+  end
+end

metadata

@@ -1,15 +1,29 @@
 --- !ruby/object:Gem::Specification
 name: chef-utils
 version: !ruby/object:Gem::Version
-  version: 16.16.13
+  version: 17.0.242
 platform: ruby
 authors:
 - Chef Software, Inc
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2021-10-01 00:00:00.000000000 Z
-dependencies: []
+date: 2021-04-27 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: concurrent-ruby
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
 description: 
 email:
 - oss@chef.io
@@ -38,6 +52,7 @@ files:
 - lib/chef-utils/dsl/windows.rb
 - lib/chef-utils/internal.rb
 - lib/chef-utils/mash.rb
+- lib/chef-utils/parallel_map.rb
 - lib/chef-utils/version.rb
 - lib/chef-utils/version_string.rb
 - spec/spec_helper.rb
@@ -54,6 +69,7 @@ files:
 - spec/unit/dsl/which_spec.rb
 - spec/unit/dsl/windows_spec.rb
 - spec/unit/mash_spec.rb
+- spec/unit/parallel_map_spec.rb
 homepage: https://github.com/chef/chef/tree/master/chef-utils
 licenses:
 - Apache-2.0
@@ -71,14 +87,14 @@ required_ruby_version: !ruby/object:Gem::Requirement
   requirements:
   - - ">="
     - !ruby/object:Gem::Version
-      version: 2.6.0
+      version: '2.6'
 required_rubygems_version: !ruby/object:Gem::Requirement
   requirements:
   - - ">="
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.1.6
+rubygems_version: 3.2.15
 signing_key: 
 specification_version: 4
 summary: Basic utility functions for Core Chef Infra development