hybrid_platforms_conductor 33.0.3 → 33.2.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: eada2aa4c440c2fdfc69fc7f6ec20d2175d4d52d7405bc192ae32cf7a4850f38
-  data.tar.gz: a720caa3b3c584a5c354cb787a5374866bfa40f7740ecf727d14db9a271e09d5
+  metadata.gz: f311db8004c9c9bce51821d58c9cee66e36b73565a70e990d5d56269a758ef07
+  data.tar.gz: 232be58c9d26f1821afea89a55421514c8f97becd2b316acf62c2e51c5e29b8c
 SHA512:
-  metadata.gz: 5369c44dd7647622a49f8db8c01722ca0ba96ff7c5e7439fb3c4b208faea076513de0ec47fe3f5a6612807680ecff7b7e2962baf3da569c67c3fd8632dfb9a20
-  data.tar.gz: 882ca15df05a0a209a7de8f723cc5ddda0a3ea89ce3b5655d6b5c4968e01adc9aa07437bbd17dafe6363331f373776c84fca307e150a53c69ef800926e54ee92
+  metadata.gz: d2c2fe3c9d0f5fccceb5a2a2b9759316730c536a5de99e63b5ecc32ef857b4f33a5aef5c4494c4b42f54a1b17a620af7a06526a953a095a8ac3e8fdc7d6fcd8e
+  data.tar.gz: 6d15f57eae1683425b64871b0234a33a6a98cf3300a3c481d98b88a525809c8ad277836a228565675b5bcd76bc23a130b5392431a045bbe18ec6cd032eeea27d

data/CHANGELOG.md

@@ -1,3 +1,57 @@
+# [v33.2.1](https://github.com/sweet-delights/hybrid-platforms-conductor/compare/v33.2.0...v33.2.1) (2021-06-21 10:23:51)
+
+## Global changes
+### Patches
+
+* [[Hotfix(platform_handler_serverless_chef)] Corrected dry-run mode not working](https://github.com/sweet-delights/hybrid-platforms-conductor/commit/4800a0f4255c1999eed33651c1e66c445acd17bb)
+
+## Changes for platform_handler_serverless_chef
+### Patches
+
+* [[Hotfix(platform_handler_serverless_chef)] Corrected dry-run mode not working](https://github.com/sweet-delights/hybrid-platforms-conductor/commit/4800a0f4255c1999eed33651c1e66c445acd17bb)
+
+# [v33.2.0](https://github.com/sweet-delights/hybrid-platforms-conductor/compare/v33.1.1...v33.2.0) (2021-06-18 23:22:21)
+
+## Global changes
+### Patches
+
+* [[Feature(platform_handler_serverless_chef)] [#70] Install dependency gems from cookbook metadata before calling chef-client](https://github.com/sweet-delights/hybrid-platforms-conductor/commit/6dfe7aa053db63489f3d0a236433304606051ecd)
+
+## Changes for platform_handler_serverless_chef
+### Features
+
+* [[Feature(platform_handler_serverless_chef)] [#70] Install dependency gems from cookbook metadata before calling chef-client](https://github.com/sweet-delights/hybrid-platforms-conductor/commit/6dfe7aa053db63489f3d0a236433304606051ecd)
+
+# [v33.1.1](https://github.com/sweet-delights/hybrid-platforms-conductor/compare/v33.1.0...v33.1.1) (2021-06-18 13:19:39)
+
+### Patches
+
+* [[Fix] Corrected bugs on log remote_fs plugin not working in dry-run and topographer](https://github.com/sweet-delights/hybrid-platforms-conductor/commit/90dcde919b6fc99b8f79fc642fef8b4150a5a9d6)
+
+# [v33.1.0](https://github.com/sweet-delights/hybrid-platforms-conductor/compare/v33.0.4...v33.1.0) (2021-06-18 11:37:28)
+
+## Global changes
+### Patches
+
+* [[Feature(connector_local)] [#68] Add sudo support when copying files using the local connector](https://github.com/sweet-delights/hybrid-platforms-conductor/commit/7725beca4429de0e81bcac0c0ac4fe149e625da2)
+
+## Changes for connector_local
+### Features
+
+* [[Feature(connector_local)] [#68] Add sudo support when copying files using the local connector](https://github.com/sweet-delights/hybrid-platforms-conductor/commit/7725beca4429de0e81bcac0c0ac4fe149e625da2)
+
+# [v33.0.4](https://github.com/sweet-delights/hybrid-platforms-conductor/compare/v33.0.3...v33.0.4) (2021-06-18 10:09:57)
+
+## Global changes
+### Patches
+
+* [[Fix(cmd_runner)] [#69] Make sure commands are run in unbundled env](https://github.com/sweet-delights/hybrid-platforms-conductor/commit/0795c1ffdcf1b355c9dc91423bb6b7d88918fcf7)
+
+## Changes for cmd_runner
+### Patches
+
+* [[Fix(cmd_runner)] [#69] Make sure commands are run in unbundled env](https://github.com/sweet-delights/hybrid-platforms-conductor/commit/0795c1ffdcf1b355c9dc91423bb6b7d88918fcf7)
+
 # [v33.0.3](https://github.com/sweet-delights/hybrid-platforms-conductor/compare/v33.0.2...v33.0.3) (2021-06-17 12:42:32)
 
 ### Patches

data/lib/hybrid_platforms_conductor/cmd_runner.rb

@@ -130,18 +130,20 @@ module HybridPlatformsConductor
               (log_to_stdout ? [@logger_stderr] : []) +
               (file_output.nil? ? [] : [file_output])
           ) do
-            cmd_result = TTY::Command.new(
-              printer: :null,
-              pty: true,
-              timeout: timeout,
-              uuid: false
-            ).run!(cmd) do |stdout, stderr|
-              stdout_queue << stdout if stdout
-              stderr_queue << stderr if stderr
+            Bundler.with_unbundled_env do
+              cmd_result = TTY::Command.new(
+                printer: :null,
+                pty: true,
+                timeout: timeout,
+                uuid: false
+              ).run!(cmd) do |stdout, stderr|
+                stdout_queue << stdout if stdout
+                stderr_queue << stderr if stderr
+              end
+              exit_status = cmd_result.exit_status
+              cmd_stdout = cmd_result.out
+              cmd_stderr = cmd_result.err
             end
-            exit_status = cmd_result.exit_status
-            cmd_stdout = cmd_result.out
-            cmd_stderr = cmd_result.err
           end
         rescue TTY::Command::TimeoutExceeded
           exit_status = :timeout

data/lib/hybrid_platforms_conductor/deployer.rb

@@ -146,6 +146,7 @@ module HybridPlatformsConductor
             logger: @logger,
             logger_stderr: @logger_stderr,
             config: @config,
+            cmd_runner: @cmd_runner,
             nodes_handler: @nodes_handler,
             actions_executor: @actions_executor
           )

data/lib/hybrid_platforms_conductor/hpc_plugins/connector/local.rb

@@ -50,7 +50,9 @@ module HybridPlatformsConductor
         # [API] - @stdout_io can be used to send stdout output
         # [API] - @stderr_io can be used to send stderr output
         def remote_interactive
-          system "cd #{workspace_for(@node)} ; /bin/bash"
+          Bundler.with_unbundled_env do
+            system "cd #{workspace_for(@node)} ; /bin/bash"
+          end
         end
 
         # rubocop:disable Lint/UnusedMethodArgument
@@ -73,7 +75,11 @@ module HybridPlatformsConductor
         def remote_copy(from, to, sudo: false, owner: nil, group: nil)
           # If the destination is a relative path, prepend the workspace dir to it.
           to = "#{workspace_for(@node)}/#{to}" unless to.start_with?('/')
-          FileUtils.cp_r from, to
+          if sudo
+            run_cmd "#{@nodes_handler.sudo_on(@node)} cp -r \"#{from}\" \"#{to}\""
+          else
+            FileUtils.cp_r from, to unless @cmd_runner.dry_run
+          end
         end
         # rubocop:enable Lint/UnusedMethodArgument
 

data/lib/hybrid_platforms_conductor/hpc_plugins/log/remote_fs.rb

@@ -121,7 +121,9 @@ module HybridPlatformsConductor
           # ...
           # ===== STDERR =====
           # ...
-          if exit_status.is_a?(Symbol)
+          if @cmd_runner.dry_run
+            { error: 'No log to show in dry-run mode' }
+          elsif exit_status.is_a?(Symbol)
             { error: "Error: #{exit_status}\n#{stderr}" }
           else
             stdout_lines = stdout.split("\n")

data/lib/hybrid_platforms_conductor/hpc_plugins/platform_handler/serverless_chef.rb

@@ -74,18 +74,16 @@ module HybridPlatformsConductor
         # [API] - @cmd_runner is accessible.
         def setup
           required_version = YAML.load_file("#{@repository_path}/chef_versions.yml")['workstation']
-          Bundler.with_unbundled_env do
-            exit_status, stdout, _stderr = @cmd_runner.run_cmd '/opt/chef-workstation/bin/chef --version', expected_code: [0, :command_error]
-            existing_version =
-              if exit_status == :command_error
-                'not installed'
-              else
-                expected_match = stdout.match(/^Chef Workstation version: (.+)\.\d+$/)
-                expected_match.nil? ? 'unreadable' : expected_match[1]
-              end
-            log_debug "Current Chef version: #{existing_version}. Required version: #{required_version}"
-            @cmd_runner.run_cmd "curl -L https://omnitruck.chef.io/install.sh | #{@cmd_runner.root? ? '' : 'sudo '}bash -s -- -P chef-workstation -v #{required_version}" unless existing_version == required_version
-          end
+          exit_status, stdout, _stderr = @cmd_runner.run_cmd '/opt/chef-workstation/bin/chef --version', expected_code: [0, :command_error]
+          existing_version =
+            if exit_status == :command_error
+              'not installed'
+            else
+              expected_match = stdout.match(/^Chef Workstation version: (.+)\.\d+$/)
+              expected_match.nil? ? 'unreadable' : expected_match[1]
+            end
+          log_debug "Current Chef version: #{existing_version}. Required version: #{required_version}"
+          @cmd_runner.run_cmd "curl -L https://omnitruck.chef.io/install.sh | #{@cmd_runner.root? ? '' : 'sudo '}bash -s -- -P chef-workstation -v #{required_version}" unless existing_version == required_version
         end
 
         # Get the list of known nodes.
@@ -160,29 +158,34 @@ module HybridPlatformsConductor
             current_package_info = File.exist?(package_info_file) ? JSON.parse(File.read(package_info_file)).transform_keys(&:to_sym) : {}
             next if current_package_info == package_info
 
-            Bundler.with_unbundled_env do
-              policy_file = "policyfiles/#{service}.rb"
-              if local_environment
-                local_policy_file = "policyfiles/#{service}.local.rb"
-                # In local mode, we always regenerate the lock file as we may modify the run list
-                run_list = known_cookbook_paths.any? { |cookbook_path| File.exist?("#{@repository_path}/#{cookbook_path}/hpc_test/recipes/before_run.rb") } ? ['hpc_test::before_run'] : []
-                dsl_parser = DslParser.new
-                dsl_parser.parse("#{@repository_path}/#{policy_file}")
-                run_list.concat dsl_parser.calls.find { |call_info| call_info[:method] == :run_list }[:args].flatten
-                run_list << 'hpc_test::after_run' if known_cookbook_paths.any? { |cookbook_path| File.exist?("#{@repository_path}/#{cookbook_path}/hpc_test/recipes/after_run.rb") }
-                File.write("#{@repository_path}/#{local_policy_file}", File.read("#{@repository_path}/#{policy_file}") + "\nrun_list #{run_list.map { |recipe| "'#{recipe}'" }.join(', ')}\n")
-                policy_file = local_policy_file
-              end
-              lock_file = "#{File.dirname(policy_file)}/#{File.basename(policy_file, '.rb')}.lock.json"
-              # If the policy lock file does not exist, generate it
-              @cmd_runner.run_cmd "cd #{@repository_path} && /opt/chef-workstation/bin/chef install #{policy_file} --chef-license accept" unless File.exist?("#{@repository_path}/#{lock_file}")
-              extra_cp_data_bags = File.exist?("#{@repository_path}/data_bags") ? " && cp -ar data_bags/ #{package_dir}/" : ''
-              @cmd_runner.run_cmd "cd #{@repository_path} && \
-                #{@cmd_runner.root? ? '' : 'sudo '}rm -rf #{package_dir} && \
-                /opt/chef-workstation/bin/chef export #{policy_file} #{package_dir} --chef-license accept#{extra_cp_data_bags}"
+            policy_file = "policyfiles/#{service}.rb"
+            if local_environment
+              local_policy_file = "policyfiles/#{service}.local.rb"
+              # In local mode, we always regenerate the lock file as we may modify the run list
+              run_list = known_cookbook_paths.any? { |cookbook_path| File.exist?("#{@repository_path}/#{cookbook_path}/hpc_test/recipes/before_run.rb") } ? ['hpc_test::before_run'] : []
+              dsl_parser = DslParser.new
+              dsl_parser.parse("#{@repository_path}/#{policy_file}")
+              run_list.concat dsl_parser.calls.find { |call_info| call_info[:method] == :run_list }[:args].flatten
+              run_list << 'hpc_test::after_run' if known_cookbook_paths.any? { |cookbook_path| File.exist?("#{@repository_path}/#{cookbook_path}/hpc_test/recipes/after_run.rb") }
+              File.write("#{@repository_path}/#{local_policy_file}", File.read("#{@repository_path}/#{policy_file}") + "\nrun_list #{run_list.map { |recipe| "'#{recipe}'" }.join(', ')}\n")
+              policy_file = local_policy_file
             end
+            lock_file = "#{File.dirname(policy_file)}/#{File.basename(policy_file, '.rb')}.lock.json"
+            # If the policy lock file does not exist, generate it
+            @cmd_runner.run_cmd "cd #{@repository_path} && /opt/chef-workstation/bin/chef install #{policy_file} --chef-license accept" unless File.exist?("#{@repository_path}/#{lock_file}")
+            extra_cp_data_bags = File.exist?("#{@repository_path}/data_bags") ? " && cp -ar data_bags/ #{package_dir}/" : ''
+            @cmd_runner.run_cmd "cd #{@repository_path} && \
+              #{@cmd_runner.root? ? '' : 'sudo '}rm -rf #{package_dir} && \
+              /opt/chef-workstation/bin/chef export #{policy_file} #{package_dir} --chef-license accept#{extra_cp_data_bags}"
             next if @cmd_runner.dry_run
 
+            # Write the list of gems to be installed for this package
+            File.write(
+              "#{@repository_path}/#{package_dir}/gems.json",
+              Dir.glob("#{@repository_path}/#{package_dir}/cookbook_artifacts/*/metadata.json").map do |metadata|
+                JSON.parse(File.read(metadata))['gems']
+              end.flatten(1).to_json
+            )
             # Create secrets file
             secrets_file = "#{@repository_path}/#{package_dir}/data_bags/hpc_secrets/hpc_secrets.json"
             FileUtils.mkdir_p(File.dirname(secrets_file))
@@ -224,10 +227,13 @@ module HybridPlatformsConductor
         # * Array< Hash<Symbol,Object> >: List of actions to be done
         def actions_to_deploy_on(node, service, use_why_run: true)
           package_dir = "#{@repository_path}/dist/#{@local_env ? 'local' : 'prod'}/#{service}"
+          gems_to_install = []
           # Generate the nodes attributes file
           unless @cmd_runner.dry_run
             FileUtils.mkdir_p "#{package_dir}/nodes"
             File.write("#{package_dir}/nodes/#{node}.json", (known_nodes.include?(node) ? metadata_for(node) : {}).merge(@nodes_handler.metadata_of(node)).to_json)
+            # Get the gems to be installed
+            gems_to_install = JSON.parse(File.read("#{package_dir}/gems.json"))
           end
           client_options = [
             '--local-mode',
@@ -237,7 +243,19 @@ module HybridPlatformsConductor
           client_options << '--why-run' if use_why_run
           if @nodes_handler.get_use_local_chef_of(node)
             # Just run the chef-client directly from the packaged repository
-            [{ bash: "cd #{package_dir} && #{@cmd_runner.root? ? '' : 'sudo '}SSL_CERT_DIR=/etc/ssl/certs /opt/chef-workstation/bin/chef-client #{client_options.join(' ')}" }]
+            sudo_prefix = @cmd_runner.root? ? '' : 'sudo '
+            [
+              {
+                bash: [
+                  'set -e',
+                  "cd #{package_dir}"
+                ] +
+                  gems_to_install.map { |(gem_name, gem_version)| "#{sudo_prefix}SSL_CERT_DIR=/etc/ssl/certs /opt/chef-workstation/bin/chef gem install #{gem_name} --version \"#{gem_version}\"" } +
+                  [
+                    "#{sudo_prefix}SSL_CERT_DIR=/etc/ssl/certs /opt/chef-workstation/bin/chef-client #{client_options.join(' ')}"
+                  ]
+              }
+            ]
           else
             # Upload the package and run it from the node
             package_name = File.basename(package_dir)
@@ -265,10 +283,14 @@ module HybridPlatformsConductor
                 scp: { package_dir => './hpc_deploy' },
                 remote_bash: [
                   'set -e',
-                  "cd ./hpc_deploy/#{package_name}",
-                  "#{sudo}SSL_CERT_DIR=/etc/ssl/certs /opt/chef/bin/chef-client #{client_options.join(' ')}",
-                  'cd ..'
-                ] + (log_debug? ? [] : ["#{sudo}rm -rf ./hpc_deploy/#{package_name}"])
+                  "cd ./hpc_deploy/#{package_name}"
+                ] +
+                  gems_to_install.map { |(gem_name, gem_version)| "#{sudo}SSL_CERT_DIR=/etc/ssl/certs /opt/chef/embedded/bin/gem install #{gem_name} --version \"#{gem_version}\"" } +
+                  [
+                    "#{sudo}SSL_CERT_DIR=/etc/ssl/certs /opt/chef/bin/chef-client #{client_options.join(' ')}",
+                    'cd ..'
+                  ] +
+                  (log_debug? ? [] : ["#{sudo}rm -rf ./hpc_deploy/#{package_name}"])
               }
             ]
           end

data/lib/hybrid_platforms_conductor/log.rb

@@ -12,16 +12,19 @@ module HybridPlatformsConductor
     # * *logger* (Logger): Logger to be used [default: Logger.new(STDOUT)]
     # * *logger_stderr* (Logger): Logger to be used for stderr [default: Logger.new(STDERR)]
     # * *config* (Config): Config to be used. [default: Config.new]
+    # * *cmd_runner* (CmdRunner): CmdRunner to be used [default: CmdRunner.new]
     # * *nodes_handler* (NodesHandler): Nodes handler to be used. [default: NodesHandler.new]
     # * *actions_executor* (ActionsExecutor): Actions executor to be used. [default: ActionsExecutor.new]
     def initialize(
       logger: Logger.new($stdout),
       logger_stderr: Logger.new($stderr),
       config: Config.new,
+      cmd_runner: CmdRunner.new,
       nodes_handler: NodesHandler.new,
       actions_executor: ActionsExecutor.new
     )
       super(logger: logger, logger_stderr: logger_stderr, config: config)
+      @cmd_runner = cmd_runner
       @nodes_handler = nodes_handler
       @actions_executor = actions_executor
     end

data/lib/hybrid_platforms_conductor/topographer.rb

@@ -304,7 +304,8 @@ module HybridPlatformsConductor
 
     # Define clusters of ips with 24 bits ranges.
     def define_clusters_ip_24
-      @nodes_graph.each_key do |node_name|
+      # Clone keys as we modify the hash in the loop
+      @nodes_graph.keys.clone.each do |node_name|
         next unless @nodes_graph[node_name][:type] == :node && !@node_metadata[node_name][:private_ips].nil? && !@node_metadata[node_name][:private_ips].empty?
 
         ip_24 = "#{@node_metadata[node_name][:private_ips].first.split('.')[0..2].join('.')}.0/24"

data/lib/hybrid_platforms_conductor/version.rb

@@ -1,5 +1,5 @@
 module HybridPlatformsConductor
 
-  VERSION = '33.0.3'
+  VERSION = '33.2.1'
 
 end

data/spec/hybrid_platforms_conductor_test/api/actions_executor/connectors/local/remote_actions_spec.rb

@@ -87,6 +87,43 @@ describe HybridPlatformsConductor::ActionsExecutor do
         end
       end
 
+      it 'does not copy files remotely in dry-run mode' do
+        with_test_platform_for_remote_testing do
+          test_cmd_runner.dry_run = true
+          expect(FileUtils).not_to receive(:cp_r)
+          test_connector.remote_copy('/path/to/src.file', '/remote_path/to/dst.dir')
+        end
+      end
+
+      it 'copies files remotely with sudo' do
+        with_test_platform_for_remote_testing(
+          expected_cmds: [
+            [
+              'sudo -u root cp -r "/path/to/src.file" "/remote_path/to/dst.dir"',
+              proc { [0, '', ''] }
+            ]
+          ]
+        ) do
+          test_connector.remote_copy('/path/to/src.file', '/remote_path/to/dst.dir', sudo: true)
+        end
+      end
+
+      it 'copies files remotely with a different sudo' do
+        with_test_platform_for_remote_testing(
+          expected_cmds: [
+            [
+              'other_sudo --user root cp -r "/path/to/src.file" "/remote_path/to/dst.dir"',
+              proc { [0, '', ''] }
+            ]
+          ],
+          additional_config: <<~'EO_CONFIG'
+            sudo_for { |user| "other_sudo --user #{user}" }
+          EO_CONFIG
+        ) do
+          test_connector.remote_copy('/path/to/src.file', '/remote_path/to/dst.dir', sudo: true)
+        end
+      end
+
       it 'copies files remotely with timeout' do
         with_test_platform_for_remote_testing(
           timeout: 5
@@ -103,6 +140,35 @@ describe HybridPlatformsConductor::ActionsExecutor do
         end
       end
 
+      it 'copies relative files remotely with sudo' do
+        with_test_platform_for_remote_testing(
+          expected_cmds: [
+            [
+              'sudo -u root cp -r "/path/to/src.file" "/tmp/hpc_local_workspaces/node/to/dst.dir"',
+              proc { [0, '', ''] }
+            ]
+          ]
+        ) do
+          test_connector.remote_copy('/path/to/src.file', 'to/dst.dir', sudo: true)
+        end
+      end
+
+      it 'copies relative files remotely with a different sudo' do
+        with_test_platform_for_remote_testing(
+          expected_cmds: [
+            [
+              'other_sudo --user root cp -r "/path/to/src.file" "/tmp/hpc_local_workspaces/node/to/dst.dir"',
+              proc { [0, '', ''] }
+            ]
+          ],
+          additional_config: <<~'EO_CONFIG'
+            sudo_for { |user| "other_sudo --user #{user}" }
+          EO_CONFIG
+        ) do
+          test_connector.remote_copy('/path/to/src.file', 'to/dst.dir', sudo: true)
+        end
+      end
+
     end
 
   end

data/spec/hybrid_platforms_conductor_test/api/cmd_runner_spec.rb

@@ -45,6 +45,12 @@ describe HybridPlatformsConductor::CmdRunner do
     end
   end
 
+  it 'runs a command in an un-bundled environment' do
+    with_repository do
+      expect(test_cmd_runner.run_cmd('echo "${BUNDLE_GEMFILE}"')).to eq [0, "\n", '']
+    end
+  end
+
   it 'fails when the command does not exit 0' do
     with_repository do
       expect { test_cmd_runner.run_cmd 'exit 1' }.to raise_error(HybridPlatformsConductor::CmdRunner::UnexpectedExitCodeError, 'Command \'exit 1\' returned error code 1 (expected 0).')

data/spec/hybrid_platforms_conductor_test/api/platform_handlers/serverless_chef/packaging_spec.rb

@@ -12,6 +12,7 @@ describe HybridPlatformsConductor::HpcPlugins::PlatformHandler::ServerlessChef d
     # * *export* (Boolean): Are we expecting the chef export stage? [default: true]
     # * *data_bags* (Boolean): Do we expect data bags copy? [default: false]
     # * *env* (String): Expected environment being packaged [default: 'prod']
+    # * *cookbook_metadata* (Hash<String, Hash>): JSON metadata to generate for packaged cookbooks [default: {}]
     # * *block* (Proc): Code called with mock in place
     def with_packaging_mocked(
       repository,
@@ -21,6 +22,7 @@ describe HybridPlatformsConductor::HpcPlugins::PlatformHandler::ServerlessChef d
       export: true,
       data_bags: false,
       env: 'prod',
+      cookbook_metadata: {},
       &block
     )
       with_cmd_runner_mocked(
@@ -49,8 +51,14 @@ describe HybridPlatformsConductor::HpcPlugins::PlatformHandler::ServerlessChef d
             [
               %r{^cd #{Regexp.escape(repository)} &&\s+sudo rm -rf dist/#{Regexp.escape(env)}/#{Regexp.escape(policy)} &&\s+/opt/chef-workstation/bin/chef export #{Regexp.escape(policy_file)} dist/#{Regexp.escape(env)}/#{Regexp.escape(policy)} --chef-license accept#{data_bags ? " && cp -ar data_bags/ dist/#{Regexp.escape(env)}/#{Regexp.escape(policy)}/" : ''}$},
               proc do
-                FileUtils.mkdir_p "#{repository}/dist/#{env}/#{policy}"
-                FileUtils.cp_r("#{repository}/data_bags", "#{repository}/dist/#{env}/#{policy}/") if data_bags
+                package_dir = "#{repository}/dist/#{env}/#{policy}"
+                FileUtils.mkdir_p package_dir
+                FileUtils.cp_r("#{repository}/data_bags", "#{package_dir}/") if data_bags
+                cookbook_metadata.each do |cookbook, metadata|
+                  metadata_file = "#{package_dir}/cookbook_artifacts/#{cookbook}/metadata.json"
+                  FileUtils.mkdir_p File.dirname(metadata_file)
+                  File.write(metadata_file, metadata.to_json)
+                end
                 [0, 'Chef export done', '']
               end
             ]
@@ -80,6 +88,9 @@ describe HybridPlatformsConductor::HpcPlugins::PlatformHandler::ServerlessChef d
         with_serverless_chef_platforms('1_node') do |platform, repository|
           with_packaging_mocked(repository) do
             platform.package(services: { 'node' => %w[test_policy] }, secrets: {}, local_environment: false)
+            gems_file = "#{repository}/dist/prod/test_policy/gems.json"
+            expect(File.exist?(gems_file)).to eq true
+            expect(JSON.parse(File.read(gems_file))).to eq []
           end
         end
       end
@@ -212,6 +223,41 @@ describe HybridPlatformsConductor::HpcPlugins::PlatformHandler::ServerlessChef d
 
     end
 
+    context 'with a platform having several cookbooks' do
+
+      it 'generates the gems info to be installed' do
+        with_serverless_chef_platforms('several_cookbooks') do |platform, repository|
+          with_packaging_mocked(
+            repository,
+            policy: 'test_policy_1',
+            cookbook_metadata: {
+              'test_cookbook_1' => {
+                gems: [
+                  ['my_gem_1', '0.0.1'],
+                  ['my_gem_2', '0.0.2']
+                ]
+              },
+              'dependency_cookbook' => {
+                gems: [
+                  ['my_gem_3', '~> 1.3']
+                ]
+              }
+            }
+          ) do
+            platform.package(services: { 'node1' => %w[test_policy_1] }, secrets: {}, local_environment: false)
+            gems_file = "#{repository}/dist/prod/test_policy_1/gems.json"
+            expect(File.exist?(gems_file)).to eq true
+            expect(JSON.parse(File.read(gems_file)).sort).to eq [
+              ['my_gem_1', '0.0.1'],
+              ['my_gem_2', '0.0.2'],
+              ['my_gem_3', '~> 1.3']
+            ].sort
+          end
+        end
+      end
+
+    end
+
     context 'with a platform having data bags' do
 
       it 'packages data bags' do

data/spec/hybrid_platforms_conductor_test/api/platform_handlers/serverless_chef/services_deployment_spec.rb

@@ -6,9 +6,18 @@ describe HybridPlatformsConductor::HpcPlugins::PlatformHandler::ServerlessChef d
     #
     # Parameters::
     # * *repository* (String): The repository we package
+    # * *env* (String): The environment for which this repository is packaged [default: 'prod']
     # * *service* (String): The service being packaged in this repository [default: 'test_policy']
-    def mock_package(repository, service: 'test_policy')
-      FileUtils.mkdir_p "#{repository}/dist/prod/#{service}"
+    # * *gems* (Array<String, String>): The [<name>, <version>] gems info to be installed as generated by the packaging [default: []]
+    def mock_package(
+      repository,
+      env: 'prod',
+      service: 'test_policy',
+      gems: []
+    )
+      package_dir = "#{repository}/dist/#{env}/#{service}"
+      FileUtils.mkdir_p package_dir
+      File.write("#{package_dir}/gems.json", gems.to_json)
     end
 
     # Get expected actions to deploy a service on a given node
@@ -20,6 +29,7 @@ describe HybridPlatformsConductor::HpcPlugins::PlatformHandler::ServerlessChef d
     # * *env* (String): Environment expected to be packaged [default: 'prod']
     # * *policy* (String): Expected policy to be packaged [default: 'test_policy']
     # * *node* (String): Expected node to be deployed [default: 'node']
+    # * *gems_install_cmds* (Array<String>): Expected gem install commands [default: []]
     # Result::
     # * Array: Expected actions
     def expected_actions_to_deploy_chef(
@@ -28,7 +38,8 @@ describe HybridPlatformsConductor::HpcPlugins::PlatformHandler::ServerlessChef d
       sudo: 'sudo -u root ',
       env: 'prod',
       policy: 'test_policy',
-      node: 'node'
+      node: 'node',
+      gems_install_cmds: []
     )
       [
         {
@@ -48,11 +59,14 @@ describe HybridPlatformsConductor::HpcPlugins::PlatformHandler::ServerlessChef d
           scp: { "#{repository}/dist/#{env}/#{policy}" => './hpc_deploy' },
           remote_bash: [
             'set -e',
-            "cd ./hpc_deploy/#{policy}",
-            "#{sudo}SSL_CERT_DIR=/etc/ssl/certs /opt/chef/bin/chef-client --local-mode --chef-license accept --json-attributes nodes/#{node}.json#{check_mode ? ' --why-run' : ''}",
-            'cd ..',
-            "#{sudo}rm -rf ./hpc_deploy/#{policy}"
-          ]
+            "cd ./hpc_deploy/#{policy}"
+          ] +
+            gems_install_cmds.map { |gem_install_cmd| "#{sudo}SSL_CERT_DIR=/etc/ssl/certs /opt/chef/embedded/bin/#{gem_install_cmd}" } +
+            [
+              "#{sudo}SSL_CERT_DIR=/etc/ssl/certs /opt/chef/bin/chef-client --local-mode --chef-license accept --json-attributes nodes/#{node}.json#{check_mode ? ' --why-run' : ''}",
+              'cd ..',
+              "#{sudo}rm -rf ./hpc_deploy/#{policy}"
+            ]
         }
       ]
     end
@@ -109,6 +123,33 @@ describe HybridPlatformsConductor::HpcPlugins::PlatformHandler::ServerlessChef d
         end
       end
 
+      it 'returns actions to deploy on this node with gems to be installed' do
+        with_serverless_chef_platforms('1_node') do |platform, repository|
+          mock_package(
+            repository,
+            gems: [
+              ['my_gem_1', '0.0.1'],
+              ['my_gem_2', '0.0.2'],
+              ['my_gem_3', '~> 1.3']
+            ]
+          )
+          platform.prepare_for_deploy(
+            services: { 'node' => %w[test_policy] },
+            secrets: {},
+            local_environment: false,
+            why_run: false
+          )
+          expect(platform.actions_to_deploy_on('node', 'test_policy', use_why_run: false)).to eq expected_actions_to_deploy_chef(
+            repository,
+            gems_install_cmds: [
+              'gem install my_gem_1 --version "0.0.1"',
+              'gem install my_gem_2 --version "0.0.2"',
+              'gem install my_gem_3 --version "~> 1.3"'
+            ]
+          )
+        end
+      end
+
       it 'returns actions to deploy on this node with node attributes setup from metadata' do
         with_serverless_chef_platforms('1_node') do |platform, repository|
           test_nodes_handler.override_metadata_of 'node', :new_metadata, 'new_value'
@@ -161,7 +202,7 @@ describe HybridPlatformsConductor::HpcPlugins::PlatformHandler::ServerlessChef d
 
       it 'returns actions to deploy on this node using local mode' do
         with_serverless_chef_platforms('1_node') do |platform, repository|
-          mock_package(repository)
+          mock_package(repository, env: 'local')
           platform.prepare_for_deploy(
             services: { 'node' => %w[test_policy] },
             secrets: {},
@@ -174,7 +215,7 @@ describe HybridPlatformsConductor::HpcPlugins::PlatformHandler::ServerlessChef d
 
       it 'returns actions to deploy on this node in why-run mode and local mode' do
         with_serverless_chef_platforms('1_node') do |platform, repository|
-          mock_package(repository)
+          mock_package(repository, env: 'local')
           platform.prepare_for_deploy(
             services: { 'node' => %w[test_policy] },
             secrets: {},
@@ -219,7 +260,7 @@ describe HybridPlatformsConductor::HpcPlugins::PlatformHandler::ServerlessChef d
 
       it 'deploys services declared on 1 node on another node if asked' do
         with_serverless_chef_platforms('several_nodes') do |platform, repository|
-          mock_package(repository)
+          mock_package(repository, service: 'test_policy_1')
           platform.prepare_for_deploy(
             services: { 'node2' => %w[test_policy_1] },
             secrets: {},
@@ -230,9 +271,38 @@ describe HybridPlatformsConductor::HpcPlugins::PlatformHandler::ServerlessChef d
         end
       end
 
-      it 'deploys local nodes' do
+      it 'deploys using the local chef workstation when use_local_chef is set' do
         with_serverless_chef_platforms('several_nodes') do |platform, repository|
-          mock_package(repository)
+          mock_package(repository, service: 'test_policy_1')
+          platform.prepare_for_deploy(
+            services: { 'local' => %w[test_policy_1] },
+            secrets: {},
+            local_environment: false,
+            why_run: false
+          )
+          expect(platform.actions_to_deploy_on('local', 'test_policy_1', use_why_run: false)).to eq [
+            {
+              bash: [
+                'set -e',
+                "cd #{repository}/dist/prod/test_policy_1",
+                'sudo SSL_CERT_DIR=/etc/ssl/certs /opt/chef-workstation/bin/chef-client --local-mode --chef-license accept --json-attributes nodes/local.json'
+              ]
+            }
+          ]
+        end
+      end
+
+      it 'deploys using the local chef workstation with gems to be installed when use_local_chef is set' do
+        with_serverless_chef_platforms('several_nodes') do |platform, repository|
+          mock_package(
+            repository,
+            service: 'test_policy_1',
+            gems: [
+              ['my_gem_1', '0.0.1'],
+              ['my_gem_2', '0.0.2'],
+              ['my_gem_3', '~> 1.3']
+            ]
+          )
           platform.prepare_for_deploy(
             services: { 'local' => %w[test_policy_1] },
             secrets: {},
@@ -241,7 +311,14 @@ describe HybridPlatformsConductor::HpcPlugins::PlatformHandler::ServerlessChef d
           )
           expect(platform.actions_to_deploy_on('local', 'test_policy_1', use_why_run: false)).to eq [
             {
-              bash: "cd #{repository}/dist/prod/test_policy_1 && sudo SSL_CERT_DIR=/etc/ssl/certs /opt/chef-workstation/bin/chef-client --local-mode --chef-license accept --json-attributes nodes/local.json"
+              bash: [
+                'set -e',
+                "cd #{repository}/dist/prod/test_policy_1",
+                'sudo SSL_CERT_DIR=/etc/ssl/certs /opt/chef-workstation/bin/chef gem install my_gem_1 --version "0.0.1"',
+                'sudo SSL_CERT_DIR=/etc/ssl/certs /opt/chef-workstation/bin/chef gem install my_gem_2 --version "0.0.2"',
+                'sudo SSL_CERT_DIR=/etc/ssl/certs /opt/chef-workstation/bin/chef gem install my_gem_3 --version "~> 1.3"',
+                'sudo SSL_CERT_DIR=/etc/ssl/certs /opt/chef-workstation/bin/chef-client --local-mode --chef-license accept --json-attributes nodes/local.json'
+              ]
             }
           ]
         end
@@ -254,7 +331,7 @@ describe HybridPlatformsConductor::HpcPlugins::PlatformHandler::ServerlessChef d
       it 'deploys a service on a node belonging to another platform' do
         with_serverless_chef_platforms({ 'p1' => '1_node', 'p2' => 'several_nodes' }) do |repositories|
           platform_1, repository_1 = repositories.find { |platform, _repository| platform.name == 'p1' }
-          mock_package(repository_1)
+          mock_package(repository_1, service: 'test_policy_1')
           platform_1.prepare_for_deploy(
             services: { 'node2' => %w[test_policy_1] },
             secrets: {},

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: hybrid_platforms_conductor
 version: !ruby/object:Gem::Version
-  version: 33.0.3
+  version: 33.2.1
 platform: ruby
 authors:
 - Muriel Salvan
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2021-06-17 00:00:00.000000000 Z
+date: 2021-06-21 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: range_operators