hybrid_platforms_conductor 32.14.0 → 32.16.3

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 99c918a62cd924ceaca95073e944df866b9b1ecce771dcb710122ec501cadf95
-  data.tar.gz: 959326a99c32ac8eb120af28c8b3c34304e2efa61df64f43a99545d0d33f88e3
+  metadata.gz: 22f6bf2ee85b7db6649e517503f00dd71f0604d4afb3a12bee347820af4d90e2
+  data.tar.gz: 7804ffa80bb56697c98970b1d781a5e3be12b27bc8c38a3df9eb26b666791ed8
 SHA512:
-  metadata.gz: 8e6c3a54a5c04f21aaeeba1a047366a7f427680077712742c0b2a7977cecfe14c4058fdd1e1c6c66842b603a7f7590bfea6eae0d975240e19a4ffa8214043c17
-  data.tar.gz: e8dd3ef50cec1a0dd7e94161159a30f359fca736bd6e7ab14ab916e2d743d48889bf891c3ca2bf626d3f087e1472ff43eb1abc0370e3fe63226cb0a8595950ad
+  metadata.gz: a89d17b28222ca1e32a132d42167a9b7b78e59b8ddb281f2805200e2a6a356e6547ee316fdacedc2c00d53ee14956027ea17b0a0424bd7fd8ad3260ba071d873
+  data.tar.gz: b29e79b9d21b8b9019307918c05c73c8d422c5fb942e6f3dc0c9fd317c293430e7a4658ed5551503f9bbf9fdf7bd17394904e0e7aabcb664101b555309c6b912

data/CHANGELOG.md

@@ -1,3 +1,63 @@
+# [v32.16.3](https://github.com/sweet-delights/hybrid-platforms-conductor/compare/v32.16.2...v32.16.3) (2021-06-01 11:19:50)
+
+## Global changes
+### Patches
+
+* [[Hotfix(platform_handler_serverless_chef)] Fix Chef packaging missing licenses accept](https://github.com/sweet-delights/hybrid-platforms-conductor/commit/69eafc1b23f77de65ba0dd68e5adb7fafb58157e)
+
+## Changes for platform_handler_serverless_chef
+### Patches
+
+* [[Hotfix(platform_handler_serverless_chef)] Fix Chef packaging missing licenses accept](https://github.com/sweet-delights/hybrid-platforms-conductor/commit/69eafc1b23f77de65ba0dd68e5adb7fafb58157e)
+
+# [v32.16.2](https://github.com/sweet-delights/hybrid-platforms-conductor/compare/v32.16.1...v32.16.2) (2021-06-01 09:52:22)
+
+## Global changes
+### Patches
+
+* [[Hotfix(platform_handler_serverless_chef)] Fix Chef Workstation bash steps when run by root](https://github.com/sweet-delights/hybrid-platforms-conductor/commit/1d99d1faa9cf66cd607eacc00205e312fd589715)
+
+## Changes for platform_handler_serverless_chef
+### Patches
+
+* [[Hotfix(platform_handler_serverless_chef)] Fix Chef Workstation bash steps when run by root](https://github.com/sweet-delights/hybrid-platforms-conductor/commit/1d99d1faa9cf66cd607eacc00205e312fd589715)
+
+# [v32.16.1](https://github.com/sweet-delights/hybrid-platforms-conductor/compare/v32.16.0...v32.16.1) (2021-06-01 09:21:47)
+
+## Global changes
+### Patches
+
+* [[Hotfix(platform_handler_serverless_chef)] Fix Chef Workstation installation step](https://github.com/sweet-delights/hybrid-platforms-conductor/commit/58320d5394fec3bca9e1d0ffaf06ade42f1fd160)
+
+## Changes for platform_handler_serverless_chef
+### Patches
+
+* [[Hotfix(platform_handler_serverless_chef)] Fix Chef Workstation installation step](https://github.com/sweet-delights/hybrid-platforms-conductor/commit/58320d5394fec3bca9e1d0ffaf06ade42f1fd160)
+
+# [v32.16.0](https://github.com/sweet-delights/hybrid-platforms-conductor/compare/v32.15.0...v32.16.0) (2021-05-31 17:55:49)
+
+## Global changes
+### Patches
+
+* [[Feature(cmd_runner)] Add option to force bash usage in run_cmd](https://github.com/sweet-delights/hybrid-platforms-conductor/commit/0711f1f33fc8f282b925726e0631ae78bf1337b5)
+
+## Changes for cmd_runner
+### Features
+
+* [[Feature(cmd_runner)] Add option to force bash usage in run_cmd](https://github.com/sweet-delights/hybrid-platforms-conductor/commit/0711f1f33fc8f282b925726e0631ae78bf1337b5)
+
+# [v32.15.0](https://github.com/sweet-delights/hybrid-platforms-conductor/compare/v32.14.0...v32.15.0) (2021-05-31 14:43:32)
+
+## Global changes
+### Patches
+
+* [[Feature(platform_handler_serverless_chef)] Use user-defined cookbook hpc_test to tune chef-client runs in test environments](https://github.com/sweet-delights/hybrid-platforms-conductor/commit/2cfa010997b562a599308ceaaf62a8740ff51468)
+
+## Changes for platform_handler_serverless_chef
+### Features
+
+* [[Feature(platform_handler_serverless_chef)] Use user-defined cookbook hpc_test to tune chef-client runs in test environments](https://github.com/sweet-delights/hybrid-platforms-conductor/commit/2cfa010997b562a599308ceaaf62a8740ff51468)
+
 # [v32.14.0](https://github.com/sweet-delights/hybrid-platforms-conductor/compare/v32.13.4...v32.14.0) (2021-05-31 09:05:45)
 
 ## Global changes

data/docs/plugins/platform_handler/serverless_chef.md

@@ -64,6 +64,12 @@ Services are being deployed by packaging the policy using [`chef install`](https
 
 Then packaged services are uploaded on the node to configured and deployment is done remotely using [Chef Infra Client in local mode](https://docs.chef.io/ctl_chef_client/#run-in-local-mode) on the remote node.
 
+## Test-wrapping cookbook `hpc_test`
+
+If you have a cookbook named `hpc_test` in your Chef repository, then this plugin will wrap any run-list run in a local environment mode (used in tests) with the recipes `hpc_test::before_run` and `hpc_test::after_run`.
+
+This way you can implement such recipes to adapt your Chef client runs to your test environment (like lack of connectivity, failing resources to be ignored...).
+
 ## Config DSL extension
 
 ### `helpers_including_recipes`

data/lib/hybrid_platforms_conductor/cmd_runner.rb

@@ -1,4 +1,5 @@
 require 'logger'
+require 'tempfile'
 require 'tty-command'
 require 'hybrid_platforms_conductor/logger_helpers'
 require 'hybrid_platforms_conductor/io_router'
@@ -66,6 +67,7 @@ module HybridPlatformsConductor
     #   * *timeout*: The command ended in timeout
     # * *timeout* (Integer or nil): Timeout to apply for the command to be run, or nil for no timeout [default: nil]
     # * *no_exception* (Boolean): If true, don't throw exception in case of error [default: false]
+    # * *force_bash* (Boolean): If true, then make sure command is invoked with bash instead of sh [default: false]
     # Result::
     # * Integer or Symbol: Exit status of the command, or Symbol in case of error. In case of dry-run mode the expected code is returned without executing anything.
     # * String: Standard output of the command
@@ -78,7 +80,8 @@ module HybridPlatformsConductor
       log_stderr_to_io: nil,
       expected_code: 0,
       timeout: nil,
-      no_exception: false
+      no_exception: false,
+      force_bash: false
     )
       expected_code = [expected_code] unless expected_code.is_a?(Array)
       if @dry_run
@@ -101,6 +104,14 @@ module HybridPlatformsConductor
             nil
           end
         start_time = Time.now if log_debug?
+        bash_file = nil
+        if force_bash
+          bash_file = Tempfile.new('hpc_bash')
+          bash_file.write(cmd)
+          bash_file.chmod 0700
+          bash_file.close
+          cmd = "/bin/bash -c #{bash_file.path}"
+        end
         begin
           # Make sure we keep a trace of stdout and stderr, even if it was not asked, just to use it in case of exceptions raised
           cmd_result_stdout = ''
@@ -141,6 +152,7 @@ module HybridPlatformsConductor
           cmd_stderr = "#{cmd_result_stderr.empty? ? '' : "#{cmd_result_stderr}\n"}#{$!}\n#{$!.backtrace.join("\n")}"
         ensure
           file_output.close unless file_output.nil?
+          bash_file.unlink unless bash_file.nil?
         end
         if log_debug?
           elapsed = Time.now - start_time

data/lib/hybrid_platforms_conductor/connector.rb

@@ -65,17 +65,19 @@ module HybridPlatformsConductor
     #
     # Parameters::
     # * *cmd* (String): The command to be run
+    # * *force_bash* (Boolean): If true, then make sure command is invoked with bash instead of sh [default: false]
     # Result::
     # * Integer: Exit code
     # * String: Standard output
     # * String: Error output
-    def run_cmd(cmd)
+    def run_cmd(cmd, force_bash: false)
       @cmd_runner.run_cmd(
         cmd,
         timeout: @timeout,
         log_to_stdout: false,
         log_stdout_to_io: @stdout_io,
-        log_stderr_to_io: @stderr_io
+        log_stderr_to_io: @stderr_io,
+        force_bash: force_bash
       )
     end
 

data/lib/hybrid_platforms_conductor/hpc_plugins/connector/local.rb

@@ -37,7 +37,7 @@ module HybridPlatformsConductor
         # Parameters::
         # * *bash_cmds* (String): Bash commands to execute
         def remote_bash(bash_cmds)
-          run_cmd "cd #{workspace_for(@node)} ; #{bash_cmds}"
+          run_cmd "cd #{workspace_for(@node)} ; #{bash_cmds}", force_bash: true
         end
 
         # Execute an interactive shell on the remote node

data/lib/hybrid_platforms_conductor/hpc_plugins/platform_handler/serverless_chef.rb

@@ -47,22 +47,44 @@ module HybridPlatformsConductor
         end
         self.extend_config_dsl_with MyDSLExtension, :init_serverless_chef
 
+        # Constructor
+        #
+        # Parameters::
+        # * *platform_type* (Symbol): Platform type
+        # * *repository_path* (String): Repository path
+        # * *logger* (Logger): Logger to be used [default: Logger.new(STDOUT)]
+        # * *logger_stderr* (Logger): Logger to be used for stderr [default: Logger.new(STDERR)]
+        # * *config* (Config): Config to be used. [default: Config.new]
+        # * *cmd_runner* (CmdRunner): Command executor to be used. [default: CmdRunner.new]
+        def initialize(
+          platform_type,
+          repository_path,
+          logger: Logger.new(STDOUT),
+          logger_stderr: Logger.new(STDERR),
+          config: Config.new,
+          cmd_runner: CmdRunner.new
+        )
+          super
+          # Mutex for getting the full recipes tree
+          @recipes_tree_mutex = Mutex.new
+        end
+
         # Setup the platform, install dependencies...
         # [API] - This method is optional.
         # [API] - @cmd_runner is accessible.
         def setup
           required_version = YAML.load_file("#{@repository_path}/chef_versions.yml")['workstation']
           Bundler.with_unbundled_env do
-            exit_status, stdout, _stderr = @cmd_runner.run_cmd '/opt/chef-workstation/bin/chef --version', expected_code: [0, 127]
+            exit_status, stdout, _stderr = @cmd_runner.run_cmd '/opt/chef-workstation/bin/chef --version', expected_code: [0, :command_error]
             existing_version =
-              if exit_status == 127
+              if exit_status == :command_error
                 'not installed'
               else
                 expected_match = stdout.match(/^Chef Workstation version: (.+)\.\d+$/)
                 expected_match.nil? ? 'unreadable' : expected_match[1]
               end
             log_debug "Current Chef version: #{existing_version}. Required version: #{required_version}"
-            @cmd_runner.run_cmd "curl -L https://omnitruck.chef.io/install.sh | sudo bash -s -- -P chef-workstation -v #{required_version}" unless existing_version == required_version
+            @cmd_runner.run_cmd "curl -L https://omnitruck.chef.io/install.sh | #{@cmd_runner.root? ? '' : 'sudo '}bash -s -- -P chef-workstation -v #{required_version}" unless existing_version == required_version
           end
         end
 
@@ -139,12 +161,25 @@ module HybridPlatformsConductor
             current_package_info = File.exist?(package_info_file) ? JSON.parse(File.read(package_info_file)).transform_keys(&:to_sym) : {}
             unless current_package_info == package_info
               Bundler.with_unbundled_env do
+                policy_file = "policyfiles/#{service}.rb"
+                if local_environment
+                  local_policy_file = "policyfiles/#{service}.local.rb"
+                  # In local mode, we always regenerate the lock file as we may modify the run list
+                  run_list = known_cookbook_paths.any? { |cookbook_path| File.exist?("#{@repository_path}/#{cookbook_path}/hpc_test/recipes/before_run.rb") } ? ['hpc_test::before_run'] : []
+                  dsl_parser = DslParser.new
+                  dsl_parser.parse("#{@repository_path}/#{policy_file}")
+                  run_list.concat dsl_parser.calls.find { |call_info| call_info[:method] == :run_list }[:args].flatten
+                  run_list << 'hpc_test::after_run' if known_cookbook_paths.any? { |cookbook_path| File.exist?("#{@repository_path}/#{cookbook_path}/hpc_test/recipes/after_run.rb") }
+                  File.write("#{@repository_path}/#{local_policy_file}", File.read("#{@repository_path}/#{policy_file}") + "\nrun_list #{run_list.map { |recipe| "'#{recipe}'" }.join(', ')}\n")
+                  policy_file = local_policy_file
+                end
+                lock_file = "#{File.dirname(policy_file)}/#{File.basename(policy_file, '.rb')}.lock.json"
                 # If the policy lock file does not exist, generate it
-                @cmd_runner.run_cmd "cd #{@repository_path} && /opt/chef-workstation/bin/chef install policyfiles/#{service}.rb" unless File.exist?("#{@repository_path}/policyfiles/#{service}.lock.json")
+                @cmd_runner.run_cmd "cd #{@repository_path} && /opt/chef-workstation/bin/chef install #{policy_file} --chef-license accept" unless File.exist?("#{@repository_path}/#{lock_file}")
                 extra_cp_data_bags = File.exist?("#{@repository_path}/data_bags") ? " && cp -ar data_bags/ #{package_dir}/" : ''
                 @cmd_runner.run_cmd "cd #{@repository_path} && \
-                  sudo rm -rf #{package_dir} && \
-                  /opt/chef-workstation/bin/chef export policyfiles/#{service}.rb #{package_dir}#{extra_cp_data_bags}"
+                  #{@cmd_runner.root? ? '' : 'sudo '}rm -rf #{package_dir} && \
+                  /opt/chef-workstation/bin/chef export #{policy_file} #{package_dir} --chef-license accept#{extra_cp_data_bags}"
               end
               unless @cmd_runner.dry_run
                 # Create secrets file
@@ -193,9 +228,15 @@ module HybridPlatformsConductor
             FileUtils.mkdir_p "#{package_dir}/nodes"
             File.write("#{package_dir}/nodes/#{node}.json", (known_nodes.include?(node) ? metadata_for(node) : {}).merge(@nodes_handler.metadata_of(node)).to_json)
           end
+          client_options = [
+            '--local-mode',
+            '--chef-license', 'accept',
+            '--json-attributes', "nodes/#{node}.json"
+          ]
+          client_options << '--why-run' if use_why_run
           if @nodes_handler.get_use_local_chef_of(node)
             # Just run the chef-client directly from the packaged repository
-            [{ bash: "cd #{package_dir} && sudo SSL_CERT_DIR=/etc/ssl/certs /opt/chef-workstation/bin/chef-client --local-mode --json-attributes nodes/#{node}.json#{use_why_run ? ' --why-run' : ''}" }]
+            [{ bash: "cd #{package_dir} && #{@cmd_runner.root? ? '' : 'sudo '}SSL_CERT_DIR=/etc/ssl/certs /opt/chef-workstation/bin/chef-client #{client_options.join(' ')}" }]
           else
             # Upload the package and run it from the node
             package_name = File.basename(package_dir)
@@ -211,7 +252,11 @@ module HybridPlatformsConductor
                   'set -o pipefail',
                   "if [ -n \"$(command -v apt)\" ]; then #{sudo}apt update && #{sudo}apt install -y curl build-essential ; else #{sudo}yum groupinstall 'Development Tools' && #{sudo}yum install -y curl ; fi",
                   'mkdir -p ./hpc_deploy',
-                  "curl --location https://omnitruck.chef.io/install.sh | tac | tac | #{sudo}bash -s -- -d /opt/artefacts -v #{required_chef_client_version} -s once"
+                  'rm -rf ./hpc_deploy/tmp',
+                  'mkdir -p ./hpc_deploy/tmp',
+                  'curl --location https://omnitruck.chef.io/install.sh --output ./hpc_deploy/install.sh',
+                  'chmod a+x ./hpc_deploy/install.sh',
+                  "#{sudo}TMPDIR=./hpc_deploy/tmp ./hpc_deploy/install.sh -d /opt/artefacts -v #{required_chef_client_version} -s once"
                 ]
               },
               {
@@ -219,10 +264,9 @@ module HybridPlatformsConductor
                 remote_bash: [
                   'set -e',
                   "cd ./hpc_deploy/#{package_name}",
-                  "#{sudo}SSL_CERT_DIR=/etc/ssl/certs /opt/chef/bin/chef-client --local-mode --chef-license=accept --json-attributes nodes/#{node}.json#{use_why_run ? ' --why-run' : ''}",
-                  'cd ..',
-                  "#{sudo}rm -rf #{package_name}"
-                ]
+                  "#{sudo}SSL_CERT_DIR=/etc/ssl/certs /opt/chef/bin/chef-client #{client_options.join(' ')}",
+                  'cd ..'
+                ] + (log_debug? ? [] : ["#{sudo}rm -rf ./hpc_deploy/#{package_name}"])
               }
             ]
           end
@@ -371,7 +415,7 @@ module HybridPlatformsConductor
           impacted_recipes.uniq!
           log_debug "* #{impacted_recipes.size} impacted recipes:\n#{impacted_recipes.map { |(cookbook, recipe)| "#{cookbook}::#{recipe}" }.sort.join("\n")}"
 
-          recipes_tree = RecipesTreeBuilder.new(@config, self).full_recipes_tree
+          recipes_tree = full_recipes_tree
           [
             impacted_nodes,
             (
@@ -419,6 +463,45 @@ module HybridPlatformsConductor
           @cookbook_paths
         end
 
+        # Get the run list of a given policy
+        #
+        # Parameters::
+        # * *policy* (String): Policy to get the run list from
+        # Result::
+        # * Array<[String or nil, Symbol, Symbol]>: Run list of the given policy, as [cookbook_dir, cookbook, recipe]
+        def policy_run_list(policy)
+          # Read the policy file
+          dsl_parser = DslParser.new
+          policy_file = "#{@repository_path}/policyfiles/#{policy}.rb"
+          dsl_parser.parse(policy_file)
+          run_list_call = dsl_parser.calls.find { |call_info| call_info[:method] == :run_list }
+          raise "Policy #{policy} has no run list defined in #{policy_file}" if run_list_call.nil?
+          run_list_call[:args].map { |recipe_def| decode_recipe(recipe_def) }
+        end
+
+        # Return the cookbook directory, cookbook name and recipe name from which a recipe definition is found.
+        # The following forms are handled:
+        # * cookbook
+        # * cookbook::recipe
+        # * recipe[cookbook]
+        # * recipe[cookbook::recipe]
+        #
+        # Parameters::
+        # * *recipe_def* (String): Recipe definition (cookbook or cookbook::recipe).
+        # Result::
+        # * String: The cookbook directory, or nil if unknown
+        # * Symbol: The cookbook name
+        # * Symbol: The recipe name
+        def decode_recipe(recipe_def)
+          recipe_def = $1 if recipe_def =~ /^recipe\[(.+)\]$/
+          cookbook, recipe = recipe_def.split('::').map(&:to_sym)
+          recipe = :default if recipe.nil?
+          # Find the cookbook it belongs to
+          cookbook_dir = known_cookbook_paths.find { |cookbook_path| File.exist?("#{@repository_path}/#{cookbook_path}/#{cookbook}") }
+          raise "Unknown recipe #{cookbook}::#{recipe} from cookbook #{@repository_path}/#{cookbook_dir}/#{cookbook}." if !cookbook_dir.nil? && !File.exist?("#{@repository_path}/#{cookbook_dir}/#{cookbook}/recipes/#{recipe}.rb")
+          return cookbook_dir, cookbook, recipe
+        end
+
         private
 
         # Return the JSON associated to a node
@@ -431,6 +514,18 @@ module HybridPlatformsConductor
           JSON.parse(File.read("#{@repository_path}/nodes/#{node}.json"))
         end
 
+        # Get the full recipes tree.
+        # Keep it in a cache for performance.
+        #
+        # Result::
+        # * Hash: The recipes tree. See RecipesTreeBuilder#full_recipes_tree for the detailed signature
+        def full_recipes_tree
+          @recipes_tree_mutex.synchronize do
+            @recipes_tree = RecipesTreeBuilder.new(@config, self).full_recipes_tree unless defined?(@recipes_tree)
+          end
+          @recipes_tree
+        end
+
       end
 
     end

data/lib/hybrid_platforms_conductor/hpc_plugins/platform_handler/serverless_chef/recipes_tree_builder.rb

@@ -36,12 +36,12 @@ module HybridPlatformsConductor
           def full_recipes_tree
             @recipes_tree = {}
             @platform.deployable_services.each do |service|
-              policy_run_list(service).each do |(cookbook_dir, cookbook, recipe)|
+              @platform.policy_run_list(service).each do |(cookbook_dir, cookbook, recipe)|
                 add_recipe_in_tree(cookbook_dir, cookbook, recipe)
               end
             end
             @platform.deployable_services.each do |service|
-              policy_run_list(service).each do |(cookbook_dir, cookbook, recipe)|
+              @platform.policy_run_list(service).each do |(cookbook_dir, cookbook, recipe)|
                 mark_recipe_used_by_policy(cookbook, recipe, service)
               end
             end
@@ -50,45 +50,6 @@ module HybridPlatformsConductor
 
           private
 
-          # Get the run list of a given policy
-          #
-          # Parameters::
-          # * *policy* (String): Policy to get the run list from
-          # Result::
-          # * Array<[String or nil, Symbol,Symbol]>: Run list of the given policy, as [cookbook_dir, cookbook, recipe]
-          def policy_run_list(policy)
-            # Read the policy file
-            dsl_parser = DslParser.new
-            policy_file = "#{@platform.repository_path}/policyfiles/#{policy}.rb"
-            dsl_parser.parse(policy_file)
-            run_list_call = dsl_parser.calls.find { |call_info| call_info[:method] == :run_list }
-            raise "Policy #{policy} has no run list defined in #{policy_file}" if run_list_call.nil?
-            run_list_call[:args].map { |recipe_def| decode_recipe(recipe_def) }
-          end
-
-          # Return the cookbook directory, cookbook name and recipe name from which a recipe definition is found.
-          # The following forms are handled:
-          # * cookbook
-          # * cookbook::recipe
-          # * recipe[cookbook]
-          # * recipe[cookbook::recipe]
-          #
-          # Parameters::
-          # * *recipe_def* (String): Recipe definition (cookbook or cookbook::recipe).
-          # Result::
-          # * String: The cookbook directory, or nil if unknown
-          # * Symbol: The cookbook name
-          # * Symbol: The recipe name
-          def decode_recipe(recipe_def)
-            recipe_def = $1 if recipe_def =~ /^recipe\[(.+)\]$/
-            cookbook, recipe = recipe_def.split('::').map(&:to_sym)
-            recipe = :default if recipe.nil?
-            # Find the cookbook it belongs to
-            cookbook_dir = @platform.known_cookbook_paths.find { |cookbook_path| File.exist?("#{@platform.repository_path}/#{cookbook_path}/#{cookbook}") }
-            raise "Unknown recipe #{cookbook}::#{recipe} from cookbook #{@platform.repository_path}/#{cookbook_dir}/#{cookbook}." if !cookbook_dir.nil? && !File.exist?("#{@platform.repository_path}/#{cookbook_dir}/#{cookbook}/recipes/#{recipe}.rb")
-            return cookbook_dir, cookbook, recipe
-          end
-
           # Fill the tree with a recipe and all its dependencies
           #
           # Parameters::
@@ -142,11 +103,11 @@ module HybridPlatformsConductor
             # Check for include_recipe
             used_recipes = recipe_content.
               scan(/include_recipe\s+["'](\w+(::\w+)?)["']/).
-              map { |(recipe_def, _sub_grp)| decode_recipe(recipe_def) }
+              map { |(recipe_def, _sub_grp)| @platform.decode_recipe(recipe_def) }
             # Check for some helpers we know include some recipes
             @config.known_helpers_including_recipes.each do |helper_name, used_recipes_by_helper|
               if recipe_content =~ Regexp.new(/(\W|^)#{Regexp.escape(helper_name)}(\W|$)/)
-                used_recipes.concat(used_recipes_by_helper.map { |recipe_def| decode_recipe(recipe_def) })
+                used_recipes.concat(used_recipes_by_helper.map { |recipe_def| @platform.decode_recipe(recipe_def) })
                 used_recipes.uniq!
               end
             end

data/lib/hybrid_platforms_conductor/hpc_plugins/provisioner/podman.rb

@@ -112,7 +112,7 @@ module HybridPlatformsConductor
         # Result::
         # * String: The Podman command
         def podman_cmd
-          @podman_cmd = @cmd_runner.root? ? 'podman' : 'sudo podman' unless defined?(@podman_cmd)
+          @podman_cmd = "#{@cmd_runner.root? ? '' : 'sudo '}podman" unless defined?(@podman_cmd)
           @podman_cmd
         end
 

data/lib/hybrid_platforms_conductor/version.rb

@@ -1,5 +1,5 @@
 module HybridPlatformsConductor
 
-  VERSION = '32.14.0'
+  VERSION = '32.16.3'
 
 end

data/spec/hybrid_platforms_conductor_test/api/cmd_runner_spec.rb

@@ -13,6 +13,13 @@ describe HybridPlatformsConductor::CmdRunner do
     end
   end
 
+  it 'runs a simple bash command and forces usage of bash' do
+    with_repository do |repository|
+      # Use set -o pipefail that does not work in /bin/sh
+      expect(test_cmd_runner.run_cmd "set -o pipefail ; echo TestStderr 1>&2 ; echo TestStdout", force_bash: true).to eq [0, "TestStdout\n", "TestStderr\n"]
+    end
+  end
+
   it 'runs a simple bash command and logs stdout and stderr to a file' do
     with_repository do |repository|
       test_cmd_runner.run_cmd "echo TestStderr 1>&2 ; sleep 1 ; echo TestStdout", log_to_file: "#{repository}/test_file"

data/spec/hybrid_platforms_conductor_test/api/platform_handlers/serverless_chef/packaging_spec.rb

@@ -7,24 +7,34 @@ describe HybridPlatformsConductor::HpcPlugins::PlatformHandler::ServerlessChef d
     # Parameters::
     # * *repository* (String): Repository to be packaged
     # * *policy* (String): Expected policy to be packaged [default: 'test_policy']
+    # * *policy_file* (String): Expected policy file used [default: "policyfiles/#{policy}.rb"]
     # * *install* (Boolean): Are we expecting the chef install stage? [default: true]
     # * *export* (Boolean): Are we expecting the chef export stage? [default: true]
     # * *data_bags* (Boolean): Do we expect data bags copy? [default: false]
+    # * *env* (String): Expected environment being packaged [default: 'prod']
     # * Proc: Code called with mock in place
     def with_packaging_mocked(
       repository,
       policy: 'test_policy',
+      policy_file: "policyfiles/#{policy}.rb",
       install: true,
       export: true,
-      data_bags: false
+      data_bags: false,
+      env: 'prod'
     )
       with_cmd_runner_mocked(
         if install
           [
             [
-              "cd #{repository} && /opt/chef-workstation/bin/chef install policyfiles/#{policy}.rb",
+              "cd #{repository} && /opt/chef-workstation/bin/chef install #{policy_file} --chef-license accept",
               proc do
-                File.write("#{repository}/policyfiles/#{policy}.lock.json", '{}')
+                # Mock the run_list stored in the lock file
+                File.write(
+                  "#{repository}/#{policy_file.gsub(/.rb$/, '.lock.json')}",
+                  {
+                    run_list: eval("[#{File.read("#{repository}/#{policy_file}").split("\n").select { |line| line =~ /^run_list.+$/ }.last.match(/^run_list(.+)$/)[1]}]").flatten
+                  }.to_json
+                )
                 [0, 'Chef install done', '']
               end
             ]
@@ -34,11 +44,12 @@ describe HybridPlatformsConductor::HpcPlugins::PlatformHandler::ServerlessChef d
         end +
         if export
           [
+            ['whoami', proc { [0, 'test_user', ''] }, { optional: true }],
             [
-              /^cd #{Regexp.escape(repository)} &&\s+sudo rm -rf dist\/prod\/#{Regexp.escape(policy)} &&\s+\/opt\/chef-workstation\/bin\/chef export policyfiles\/#{Regexp.escape(policy)}.rb dist\/prod\/#{Regexp.escape(policy)}#{data_bags ? " && cp -ar data_bags/ dist/prod/#{Regexp.escape(policy)}/" : ''}$/,
+              /^cd #{Regexp.escape(repository)} &&\s+sudo rm -rf dist\/#{Regexp.escape(env)}\/#{Regexp.escape(policy)} &&\s+\/opt\/chef-workstation\/bin\/chef export #{Regexp.escape(policy_file)} dist\/#{Regexp.escape(env)}\/#{Regexp.escape(policy)} --chef-license accept#{data_bags ? " && cp -ar data_bags/ dist/#{Regexp.escape(env)}/#{Regexp.escape(policy)}/" : ''}$/,
               proc do
-                FileUtils.mkdir_p "#{repository}/dist/prod/#{policy}"
-                FileUtils.cp_r("#{repository}/data_bags", "#{repository}/dist/prod/#{policy}/") if data_bags
+                FileUtils.mkdir_p "#{repository}/dist/#{env}/#{policy}"
+                FileUtils.cp_r("#{repository}/data_bags", "#{repository}/dist/#{env}/#{policy}/") if data_bags
                 [0, 'Chef export done', '']
               end
             ]
@@ -96,6 +107,26 @@ describe HybridPlatformsConductor::HpcPlugins::PlatformHandler::ServerlessChef d
         end
       end
 
+      it 'packages the repository for a given node and service in local mode' do
+        with_serverless_chef_platforms('1_node') do |platform, repository|
+          with_packaging_mocked(repository, policy_file: 'policyfiles/test_policy.local.rb', env: 'local') do
+            platform.package(services: { 'node' => %w[test_policy] }, secrets: {}, local_environment: true)
+            local_policy_file = "#{repository}/policyfiles/test_policy.local.lock.json"
+            expect(File.exist?(local_policy_file)).to eq true
+            expect(JSON.parse(File.read(local_policy_file))).to eq('run_list' => ['recipe[test_cookbook]'])
+          end
+        end
+      end
+
+      it 'packages the repository without resolving dependencies when the lock file already exists' do
+        with_serverless_chef_platforms('1_node') do |platform, repository|
+          File.write("#{repository}/policyfiles/test_policy.lock.json", '{}')
+          with_packaging_mocked(repository, install: false) do
+            platform.package(services: { 'node' => %w[test_policy] }, secrets: {}, local_environment: false)
+          end
+        end
+      end
+
       it 'does not package the repository twice for the same config' do
         with_serverless_chef_platforms('1_node', as_git: true) do |platform, repository|
           with_packaging_mocked(repository) do
@@ -208,6 +239,55 @@ describe HybridPlatformsConductor::HpcPlugins::PlatformHandler::ServerlessChef d
 
     end
 
+    context 'with a platform having hpc_test cookbook' do
+
+      it 'packages the repository with before_run and after_run recipes wrapping the run list' do
+        with_serverless_chef_platforms('hpc_test') do |platform, repository|
+          with_packaging_mocked(repository, policy_file: 'policyfiles/test_policy.local.rb', env: 'local') do
+            platform.package(services: { 'node' => %w[test_policy] }, secrets: {}, local_environment: true)
+            local_policy_file = "#{repository}/policyfiles/test_policy.local.lock.json"
+            expect(File.exist?(local_policy_file)).to eq true
+            expect(JSON.parse(File.read(local_policy_file))).to eq('run_list' => [
+              'hpc_test::before_run',
+              'recipe[test_cookbook]',
+              'hpc_test::after_run'
+            ])
+          end
+        end
+      end
+
+      it 'packages the repository with before_run only recipe' do
+        with_serverless_chef_platforms('hpc_test') do |platform, repository|
+          File.unlink "#{repository}/cookbooks/hpc_test/recipes/after_run.rb"
+          with_packaging_mocked(repository, policy_file: 'policyfiles/test_policy.local.rb', env: 'local') do
+            platform.package(services: { 'node' => %w[test_policy] }, secrets: {}, local_environment: true)
+            local_policy_file = "#{repository}/policyfiles/test_policy.local.lock.json"
+            expect(File.exist?(local_policy_file)).to eq true
+            expect(JSON.parse(File.read(local_policy_file))).to eq('run_list' => [
+              'hpc_test::before_run',
+              'recipe[test_cookbook]'
+            ])
+          end
+        end
+      end
+
+      it 'packages the repository with after_run only recipe' do
+        with_serverless_chef_platforms('hpc_test') do |platform, repository|
+          File.unlink "#{repository}/cookbooks/hpc_test/recipes/before_run.rb"
+          with_packaging_mocked(repository, policy_file: 'policyfiles/test_policy.local.rb', env: 'local') do
+            platform.package(services: { 'node' => %w[test_policy] }, secrets: {}, local_environment: true)
+            local_policy_file = "#{repository}/policyfiles/test_policy.local.lock.json"
+            expect(File.exist?(local_policy_file)).to eq true
+            expect(JSON.parse(File.read(local_policy_file))).to eq('run_list' => [
+              'recipe[test_cookbook]',
+              'hpc_test::after_run'
+            ])
+          end
+        end
+      end
+
+    end
+
   end
 
 end

data/spec/hybrid_platforms_conductor_test/api/platform_handlers/serverless_chef/services_deployment_spec.rb

@@ -37,7 +37,11 @@ describe HybridPlatformsConductor::HpcPlugins::PlatformHandler::ServerlessChef d
             'set -o pipefail',
             "if [ -n \"$(command -v apt)\" ]; then #{sudo}apt update && #{sudo}apt install -y curl build-essential ; else #{sudo}yum groupinstall 'Development Tools' && #{sudo}yum install -y curl ; fi",
             'mkdir -p ./hpc_deploy',
-            "curl --location https://omnitruck.chef.io/install.sh | tac | tac | #{sudo}bash -s -- -d /opt/artefacts -v 17.0 -s once"
+            'rm -rf ./hpc_deploy/tmp',
+            'mkdir -p ./hpc_deploy/tmp',
+            'curl --location https://omnitruck.chef.io/install.sh --output ./hpc_deploy/install.sh',
+            'chmod a+x ./hpc_deploy/install.sh',
+            "#{sudo}TMPDIR=./hpc_deploy/tmp ./hpc_deploy/install.sh -d /opt/artefacts -v 17.0 -s once"
           ]
         },
         {
@@ -45,9 +49,9 @@ describe HybridPlatformsConductor::HpcPlugins::PlatformHandler::ServerlessChef d
           remote_bash: [
             'set -e',
             "cd ./hpc_deploy/#{policy}",
-            "#{sudo}SSL_CERT_DIR=/etc/ssl/certs /opt/chef/bin/chef-client --local-mode --chef-license=accept --json-attributes nodes/#{node}.json#{check_mode ? ' --why-run' : ''}",
+            "#{sudo}SSL_CERT_DIR=/etc/ssl/certs /opt/chef/bin/chef-client --local-mode --chef-license accept --json-attributes nodes/#{node}.json#{check_mode ? ' --why-run' : ''}",
             'cd ..',
-            "#{sudo}rm -rf #{policy}"
+            "#{sudo}rm -rf ./hpc_deploy/#{policy}"
           ]
         }
       ]
@@ -237,7 +241,7 @@ describe HybridPlatformsConductor::HpcPlugins::PlatformHandler::ServerlessChef d
           )
           expect(platform.actions_to_deploy_on('local', 'test_policy_1', use_why_run: false)).to eq [
             {
-              bash: "cd #{repository}/dist/prod/test_policy_1 && sudo SSL_CERT_DIR=/etc/ssl/certs /opt/chef-workstation/bin/chef-client --local-mode --json-attributes nodes/local.json"
+              bash: "cd #{repository}/dist/prod/test_policy_1 && sudo SSL_CERT_DIR=/etc/ssl/certs /opt/chef-workstation/bin/chef-client --local-mode --chef-license accept --json-attributes nodes/local.json"
             }
           ]
         end

data/spec/hybrid_platforms_conductor_test/helpers/cmd_runner_helpers.rb

@@ -31,7 +31,7 @@ module HybridPlatformsConductorTest
         end
         # We need to protect the access to this array as the mocked commands can be called by competing threads
         remaining_expected_commands_mutex = Mutex.new
-        allow(cmd_runner).to receive(:run_cmd) do |cmd, log_to_file: nil, log_to_stdout: true, log_stdout_to_io: nil, log_stderr_to_io: nil, expected_code: 0, timeout: nil, no_exception: false|
+        allow(cmd_runner).to receive(:run_cmd) do |cmd, log_to_file: nil, log_to_stdout: true, log_stdout_to_io: nil, log_stderr_to_io: nil, expected_code: 0, timeout: nil, no_exception: false, force_bash: false|
           # Check the remaining expected commands
           found_command = nil
           found_command_code = nil

data/spec/hybrid_platforms_conductor_test/serverless_chef_repositories/hpc_test/chef_versions.yml

@@ -0,0 +1,3 @@
+---
+workstation: '21.5'
+client: '17.0'

data/spec/hybrid_platforms_conductor_test/serverless_chef_repositories/hpc_test/cookbooks/hpc_test/recipes/after_run.rb

@@ -0,0 +1 @@
+# Recipe

data/spec/hybrid_platforms_conductor_test/serverless_chef_repositories/hpc_test/cookbooks/hpc_test/recipes/before_run.rb

@@ -0,0 +1 @@
+# Recipe

data/spec/hybrid_platforms_conductor_test/serverless_chef_repositories/hpc_test/nodes/node.json

@@ -0,0 +1,10 @@
+{
+  "name": "node",
+  "normal": {
+    "description": "Single test node",
+    "image": "debian_9",
+    "private_ips": ["172.16.0.1"]
+  },
+  "policy_name": "test_policy",
+  "policy_group": "test_group"
+}

data/spec/hybrid_platforms_conductor_test/serverless_chef_repositories/hpc_test/policyfiles/test_policy.rb

@@ -0,0 +1,3 @@
+name File.basename(__FILE__, '.rb')
+default_source :supermarket
+run_list 'recipe[test_cookbook]'

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: hybrid_platforms_conductor
 version: !ruby/object:Gem::Version
-  version: 32.14.0
+  version: 32.16.3
 platform: ruby
 authors:
 - Muriel Salvan
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2021-05-31 00:00:00.000000000 Z
+date: 2021-06-01 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: range_operators
@@ -873,6 +873,11 @@ files:
 - spec/hybrid_platforms_conductor_test/serverless_chef_repositories/data_bags/data_bags/my_bag/my_item.json
 - spec/hybrid_platforms_conductor_test/serverless_chef_repositories/data_bags/nodes/node.json
 - spec/hybrid_platforms_conductor_test/serverless_chef_repositories/data_bags/policyfiles/test_policy.rb
+- spec/hybrid_platforms_conductor_test/serverless_chef_repositories/hpc_test/chef_versions.yml
+- spec/hybrid_platforms_conductor_test/serverless_chef_repositories/hpc_test/cookbooks/hpc_test/recipes/after_run.rb
+- spec/hybrid_platforms_conductor_test/serverless_chef_repositories/hpc_test/cookbooks/hpc_test/recipes/before_run.rb
+- spec/hybrid_platforms_conductor_test/serverless_chef_repositories/hpc_test/nodes/node.json
+- spec/hybrid_platforms_conductor_test/serverless_chef_repositories/hpc_test/policyfiles/test_policy.rb
 - spec/hybrid_platforms_conductor_test/serverless_chef_repositories/recipes/cookbooks/test_cookbook_1/recipes/default.rb
 - spec/hybrid_platforms_conductor_test/serverless_chef_repositories/recipes/cookbooks/test_cookbook_2/libraries/default.rb
 - spec/hybrid_platforms_conductor_test/serverless_chef_repositories/recipes/cookbooks/test_cookbook_2/recipes/default.rb