hybrid_platforms_conductor 32.13.4 → 32.16.2

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 2e55b2875d12648408fdd837b525b0f96728211c114062b6de22aefc59baec16
-  data.tar.gz: cbe6360d9b3944604c4505ea90de60ee2dfe4bd1f6d51d7a287b43ad5e3f0ee4
+  metadata.gz: f78c78a80235a940f793af89344e6a972609f92d79ddce8acc6735b0e6b54ceb
+  data.tar.gz: 9c0bfef24e47646c61dd79899af40f33f063f90e73a626ed9c2617c7f7933593
 SHA512:
-  metadata.gz: c62d9ea4f8b68df98961b0cbf4e48f1ecfe3ab2b48bc30474f972c93d8affc40022f71a1226d804e3045acb19ebac4b46d2eb07ced1c87ba0513e1085a8d0ed9
-  data.tar.gz: 7b62f76dcdd766d4232c1e40ef6f1209d337b75378912f563d835c805c20b595063aaae9126c672e2674fca692927c262ada8157318d846ae397c799b977508a
+  metadata.gz: 60a24969416d8d674cad078642e575c1b22e3b4457dcabb5a436bf62a597d13388bf042851008d5a3b01a7de82ee6003c6856b90821adddc1436bf2a0ecbfdd1
+  data.tar.gz: 83cdf6e30fefad4d1d97cb1e1af84278e71aff6eadac4fe5d27232f70d6537a35a48d0e30494213a616796177fd1e6aa36f26bc6ac86d313b651ca2023ade1a0

data/CHANGELOG.md

@@ -1,3 +1,63 @@
+# [v32.16.2](https://github.com/sweet-delights/hybrid-platforms-conductor/compare/v32.16.1...v32.16.2) (2021-06-01 09:52:22)
+
+## Global changes
+### Patches
+
+* [[Hotfix(platform_handler_serverless_chef)] Fix Chef Workstation bash steps when run by root](https://github.com/sweet-delights/hybrid-platforms-conductor/commit/1d99d1faa9cf66cd607eacc00205e312fd589715)
+
+## Changes for platform_handler_serverless_chef
+### Patches
+
+* [[Hotfix(platform_handler_serverless_chef)] Fix Chef Workstation bash steps when run by root](https://github.com/sweet-delights/hybrid-platforms-conductor/commit/1d99d1faa9cf66cd607eacc00205e312fd589715)
+
+# [v32.16.1](https://github.com/sweet-delights/hybrid-platforms-conductor/compare/v32.16.0...v32.16.1) (2021-06-01 09:21:47)
+
+## Global changes
+### Patches
+
+* [[Hotfix(platform_handler_serverless_chef)] Fix Chef Workstation installation step](https://github.com/sweet-delights/hybrid-platforms-conductor/commit/58320d5394fec3bca9e1d0ffaf06ade42f1fd160)
+
+## Changes for platform_handler_serverless_chef
+### Patches
+
+* [[Hotfix(platform_handler_serverless_chef)] Fix Chef Workstation installation step](https://github.com/sweet-delights/hybrid-platforms-conductor/commit/58320d5394fec3bca9e1d0ffaf06ade42f1fd160)
+
+# [v32.16.0](https://github.com/sweet-delights/hybrid-platforms-conductor/compare/v32.15.0...v32.16.0) (2021-05-31 17:55:49)
+
+## Global changes
+### Patches
+
+* [[Feature(cmd_runner)] Add option to force bash usage in run_cmd](https://github.com/sweet-delights/hybrid-platforms-conductor/commit/0711f1f33fc8f282b925726e0631ae78bf1337b5)
+
+## Changes for cmd_runner
+### Features
+
+* [[Feature(cmd_runner)] Add option to force bash usage in run_cmd](https://github.com/sweet-delights/hybrid-platforms-conductor/commit/0711f1f33fc8f282b925726e0631ae78bf1337b5)
+
+# [v32.15.0](https://github.com/sweet-delights/hybrid-platforms-conductor/compare/v32.14.0...v32.15.0) (2021-05-31 14:43:32)
+
+## Global changes
+### Patches
+
+* [[Feature(platform_handler_serverless_chef)] Use user-defined cookbook hpc_test to tune chef-client runs in test environments](https://github.com/sweet-delights/hybrid-platforms-conductor/commit/2cfa010997b562a599308ceaaf62a8740ff51468)
+
+## Changes for platform_handler_serverless_chef
+### Features
+
+* [[Feature(platform_handler_serverless_chef)] Use user-defined cookbook hpc_test to tune chef-client runs in test environments](https://github.com/sweet-delights/hybrid-platforms-conductor/commit/2cfa010997b562a599308ceaaf62a8740ff51468)
+
+# [v32.14.0](https://github.com/sweet-delights/hybrid-platforms-conductor/compare/v32.13.4...v32.14.0) (2021-05-31 09:05:45)
+
+## Global changes
+### Patches
+
+* [[Feature(platform_handler_serverless_chef)] [#58] Add the serverless_chef platform handler](https://github.com/sweet-delights/hybrid-platforms-conductor/commit/6f84757096a802c79a18a6c9c1440b27e73decd1)
+
+## Changes for platform_handler_serverless_chef
+### Features
+
+* [[Feature(platform_handler_serverless_chef)] [#58] Add the serverless_chef platform handler](https://github.com/sweet-delights/hybrid-platforms-conductor/commit/6f84757096a802c79a18a6c9c1440b27e73decd1)
+
 # [v32.13.4](https://github.com/sweet-delights/hybrid-platforms-conductor/compare/v32.13.3...v32.13.4) (2021-05-11 14:00:47)
 
 ## Global changes

data/bin/get_impacted_nodes

@@ -10,7 +10,7 @@ executable = HybridPlatformsConductor::Executable.new(nodes_selection_options: f
   opts.on('-f', '--from-commit COMMIT_ID', "Specify the GIT commit from which we look for diffs. Defaults to #{git_from}.") do |commit_id|
     git_from = commit_id
   end
-  opts.on('-p', '--platform PLATFORM_NAME', "Specify the repository on which to perform the diff. Possible values are #{platforms_handler.known_platforms.join(', ')}") do |platform_name|
+  opts.on('-p', '--platform PLATFORM_NAME', "Specify the repository on which to perform the diff. Possible values are #{platforms_handler.known_platforms.map(&:name).join(', ')}") do |platform_name|
     platform = platform_name
   end
   opts.on('-s', '--smallest-test-sample', 'Display the minimal set of nodes to check that would validate all modifications.') do

data/bin/setup

@@ -7,5 +7,10 @@ platforms_handler = executable.platforms_handler
 executable.parse_options!
 
 platforms_handler.known_platforms.each do |platform|
-  platform.setup if platform.respond_to?(:setup)
+  if platform.respond_to?(:setup)
+    executable.out "===== Setup platform #{platform.name}..."
+    platform.setup
+    executable.out "===== Platform #{platform.name} setup successfully."
+    executable.out ''
+  end
 end

data/docs/plugins.md

@@ -107,6 +107,7 @@ Examples of platform handlers are:
 Check the [sample plugin file](../lib/hybrid_platforms_conductor/hpc_plugins/platform_handler/platform_handler_plugin.rb.sample) to know more about the API that needs to be implemented by such plugins.
 
 Plugins shipped by default:
+* [`serverless_chef`](plugins/platform_handler/serverless_chef.md)
 * [`yaml_inventory`](plugins/platform_handler/yaml_inventory.md)
 
 <a name="provisioner"></a>

data/docs/plugins/platform_handler/serverless_chef.md

@@ -0,0 +1,111 @@
+# PlatformHandler plugin: `serverless_chef`
+
+The `serverless_chef` platform handler is supporting a [Chef repository](https://docs.chef.io/chef_repo/), and deploying services from this repository with using a Chef Infra Server. It uses a client-only deployment process.
+
+The Chef repository concepts supported by this plugin are:
+* nodes,
+* policies,
+* data bags,
+* cookbooks,
+* knife configuration,
+* node attributes,
+* policy attributes.
+
+The Chef repository concepts not supported by this plugin are:
+* roles,
+* environments.
+
+## Requirements
+
+The platform repository has to contain a file named `chef_versions.yml` that will define the required Chef components' versions fro this Chef repository.
+
+Here is the structure of this Yaml file:
+* **`workstation`** (*String*): Version of the [Chef Workstation](https://downloads.chef.io/tools/workstation) to be installed locally during the [setup](/docs/executables/setup.md) phase.
+* **`client`** (*String*): Version of the [Chef Infra Client](https://docs.chef.io/chef_client_overview/) to be installed on nodes that will be deployed.
+
+Versions can be of the form `major.minor.patch` or only `major.minor` to benefit automatically from latest patch versions.
+
+Example of `chef_versions.yml`:
+```yaml
+---
+workstation: '21.5'
+client: '17.0'
+```
+
+## Inventory
+
+Inventory is read directly from the `nodes/*.json` files that are present in a Chef repository.
+
+Nodes are expected to use [policies](https://docs.chef.io/policy/) to know which service is to be deployed on a node.
+
+Metadata is taken from the normal attributes defined in the node's json file.
+
+Example of node json:
+```json
+{
+  "name": "test-node",
+  "normal": {
+    "description": "Single test node",
+    "image": "debian_9",
+    "private_ips": ["172.16.0.1"],
+    "metadata_property": "metadata_value"
+  },
+  "policy_name": "service_name",
+  "policy_group": "test_group"
+}
+```
+
+## Services
+
+Services available from a Chef repository are parsed from the [policy files](https://docs.chef.io/policyfile/) stored in `policyfiles/*.rb`.
+1 policy file is 1 service.
+
+Services are being deployed by packaging the policy using [`chef install`](https://docs.chef.io/workstation/ctl_chef/#chef-install), [`chef export`](https://docs.chef.io/workstation/ctl_chef/#chef-export), which ensures packaging processes optimal and independent from 1 policy to another.
+
+Then packaged services are uploaded on the node to configured and deployment is done remotely using [Chef Infra Client in local mode](https://docs.chef.io/ctl_chef_client/#run-in-local-mode) on the remote node.
+
+## Test-wrapping cookbook `hpc_test`
+
+If you have a cookbook named `hpc_test` in your Chef repository, then this plugin will wrap any run-list run in a local environment mode (used in tests) with the recipes `hpc_test::before_run` and `hpc_test::after_run`.
+
+This way you can implement such recipes to adapt your Chef client runs to your test environment (like lack of connectivity, failing resources to be ignored...).
+
+## Config DSL extension
+
+### `helpers_including_recipes`
+
+The `helpers_including_recipes` DSL helps understanding dependencies between cookbooks in your Chef repository.
+This is used only by the Hybrid Platform Conductor processes that compute which services are being impacted by some git diffs. For example the [`get_impacted_nodes` executable](/docs/executables/get_impacted_nodes.md), and is completely optional.
+
+This helper defines which recipes are being used by a given library helper.
+For example if you define a helper named `my_configs` in a library that will use internally a recipe named `my_cookbook::configs` you will need to declare this dependency using the `helpers_including_recipes` for Hybrid Platforms Conductor to know about this dependency.
+This way, if another recipe (let's say `my_cookbook::production`) uses this helper and a git diff reports a modification in the `my_cookbook::configs` recipe, every node using `my_cookbook::production` will be marked as impacted by such a git diff.
+
+The helper takes a Hash as parameters: for each helper name, it gives a list of used recipes.
+
+For example:
+```ruby
+helpers_including_recipes(
+  my_configs: %w[my_cookbook::configs]
+)
+```
+
+## Used credentials
+
+| Credential | Usage
+| --- | --- |
+
+## Used Metadata
+
+| Metadata | Type | Usage
+| --- | --- | --- |
+| `use_local_chef` | `Boolean` | If set to true, then run chef-client locally instead of deploying on a remote node |
+
+## Used environment variables
+
+| Variable | Usage
+| --- | --- |
+
+## External tools dependencies
+
+* `curl`: Used to install Chef Workstation.

data/lib/hybrid_platforms_conductor/cmd_runner.rb

@@ -1,4 +1,5 @@
 require 'logger'
+require 'tempfile'
 require 'tty-command'
 require 'hybrid_platforms_conductor/logger_helpers'
 require 'hybrid_platforms_conductor/io_router'
@@ -66,6 +67,7 @@ module HybridPlatformsConductor
     #   * *timeout*: The command ended in timeout
     # * *timeout* (Integer or nil): Timeout to apply for the command to be run, or nil for no timeout [default: nil]
     # * *no_exception* (Boolean): If true, don't throw exception in case of error [default: false]
+    # * *force_bash* (Boolean): If true, then make sure command is invoked with bash instead of sh [default: false]
     # Result::
     # * Integer or Symbol: Exit status of the command, or Symbol in case of error. In case of dry-run mode the expected code is returned without executing anything.
     # * String: Standard output of the command
@@ -78,7 +80,8 @@ module HybridPlatformsConductor
       log_stderr_to_io: nil,
       expected_code: 0,
       timeout: nil,
-      no_exception: false
+      no_exception: false,
+      force_bash: false
     )
       expected_code = [expected_code] unless expected_code.is_a?(Array)
       if @dry_run
@@ -101,6 +104,14 @@ module HybridPlatformsConductor
             nil
           end
         start_time = Time.now if log_debug?
+        bash_file = nil
+        if force_bash
+          bash_file = Tempfile.new('hpc_bash')
+          bash_file.write(cmd)
+          bash_file.chmod 0700
+          bash_file.close
+          cmd = "/bin/bash -c #{bash_file.path}"
+        end
         begin
           # Make sure we keep a trace of stdout and stderr, even if it was not asked, just to use it in case of exceptions raised
           cmd_result_stdout = ''
@@ -141,6 +152,7 @@ module HybridPlatformsConductor
           cmd_stderr = "#{cmd_result_stderr.empty? ? '' : "#{cmd_result_stderr}\n"}#{$!}\n#{$!.backtrace.join("\n")}"
         ensure
           file_output.close unless file_output.nil?
+          bash_file.unlink unless bash_file.nil?
         end
         if log_debug?
           elapsed = Time.now - start_time

data/lib/hybrid_platforms_conductor/connector.rb

@@ -65,17 +65,19 @@ module HybridPlatformsConductor
     #
     # Parameters::
     # * *cmd* (String): The command to be run
+    # * *force_bash* (Boolean): If true, then make sure command is invoked with bash instead of sh [default: false]
     # Result::
     # * Integer: Exit code
     # * String: Standard output
     # * String: Error output
-    def run_cmd(cmd)
+    def run_cmd(cmd, force_bash: false)
       @cmd_runner.run_cmd(
         cmd,
         timeout: @timeout,
         log_to_stdout: false,
         log_stdout_to_io: @stdout_io,
-        log_stderr_to_io: @stderr_io
+        log_stderr_to_io: @stderr_io,
+        force_bash: force_bash
       )
     end
 

data/lib/hybrid_platforms_conductor/deployer.rb

@@ -318,8 +318,9 @@ module HybridPlatformsConductor
           actions_executor: @actions_executor
         )
         instance.with_running_instance(stop_on_exit: true, destroy_on_exit: !reuse_instance, port: 22) do
-          # Test-provisioned nodes have SSH Session Exec capabilities
+          # Test-provisioned nodes have SSH Session Exec capabilities and are not local
           sub_executable.nodes_handler.override_metadata_of node, :ssh_session_exec, 'true'
+          sub_executable.nodes_handler.override_metadata_of node, :local_node, false
           # Test-provisioned nodes use default sudo
           sub_executable.config.sudo_procs.replace(sub_executable.config.sudo_procs.map do |sudo_proc_info|
             {

data/lib/hybrid_platforms_conductor/hpc_plugins/connector/local.rb

@@ -37,7 +37,7 @@ module HybridPlatformsConductor
         # Parameters::
         # * *bash_cmds* (String): Bash commands to execute
         def remote_bash(bash_cmds)
-          run_cmd "cd #{workspace_for(@node)} ; #{bash_cmds}"
+          run_cmd "cd #{workspace_for(@node)} ; #{bash_cmds}", force_bash: true
         end
 
         # Execute an interactive shell on the remote node

data/lib/hybrid_platforms_conductor/hpc_plugins/platform_handler/serverless_chef.rb

@@ -0,0 +1,535 @@
+require 'fileutils'
+require 'json'
+require 'yaml'
+require 'hybrid_platforms_conductor/platform_handler'
+require 'hybrid_platforms_conductor/hpc_plugins/platform_handler/serverless_chef/dsl_parser'
+require 'hybrid_platforms_conductor/hpc_plugins/platform_handler/serverless_chef/recipes_tree_builder'
+
+module HybridPlatformsConductor
+
+  module HpcPlugins
+
+    module PlatformHandler
+
+      # Handle a Chef repository without using a Chef Infra Server.
+      # Inventory is read from nodes/*.json.
+      # Services are defined from policy files in policyfiles/*.rb.
+      # Roles are not supported as they are considered made obsolete with the usage of policies by the Chef community.
+      # Required Chef versions are taken from a chef_versions.yml file containing the following keys:
+      # * *workstation* (String): The Chef Workstation version to be installed during setup (can be specified as major.minor only)
+      # * *client* (String): The Chef Infra Client version to be installed during nodes deployment (can be specified as major.minor only)
+      class ServerlessChef < HybridPlatformsConductor::PlatformHandler
+
+        # Add a Mixin to the DSL parsing the platforms configuration file.
+        # This can be used by any plugin to add plugin-specific configuration getters and setters, accessible later from NodesHandler instances.
+        # An optional initializer can also be given.
+        # [API] - Those calls are optional
+        module MyDSLExtension
+
+          # The list of library helpers we know include some recipes.
+          # This is used when parsing some recipe code: if such a helper is encountered then we assume a dependency on a given recipe.
+          # Hash< Symbol, Array<String> >: List of recipes definitions per helper name.
+          attr_reader :known_helpers_including_recipes
+
+          # Initialize the DSL
+          def init_serverless_chef
+            @known_helpers_including_recipes = {}
+          end
+
+          # Define helpers including recipes
+          #
+          # Parameters::
+          # * *included_recipes* (Hash< Symbol, Array<String> >): List of recipes definitions per helper name.
+          def helpers_including_recipes(included_recipes)
+            @known_helpers_including_recipes.merge!(included_recipes)
+          end
+
+        end
+        self.extend_config_dsl_with MyDSLExtension, :init_serverless_chef
+
+        # Constructor
+        #
+        # Parameters::
+        # * *platform_type* (Symbol): Platform type
+        # * *repository_path* (String): Repository path
+        # * *logger* (Logger): Logger to be used [default: Logger.new(STDOUT)]
+        # * *logger_stderr* (Logger): Logger to be used for stderr [default: Logger.new(STDERR)]
+        # * *config* (Config): Config to be used. [default: Config.new]
+        # * *cmd_runner* (CmdRunner): Command executor to be used. [default: CmdRunner.new]
+        def initialize(
+          platform_type,
+          repository_path,
+          logger: Logger.new(STDOUT),
+          logger_stderr: Logger.new(STDERR),
+          config: Config.new,
+          cmd_runner: CmdRunner.new
+        )
+          super
+          # Mutex for getting the full recipes tree
+          @recipes_tree_mutex = Mutex.new
+        end
+
+        # Setup the platform, install dependencies...
+        # [API] - This method is optional.
+        # [API] - @cmd_runner is accessible.
+        def setup
+          required_version = YAML.load_file("#{@repository_path}/chef_versions.yml")['workstation']
+          Bundler.with_unbundled_env do
+            exit_status, stdout, _stderr = @cmd_runner.run_cmd '/opt/chef-workstation/bin/chef --version', expected_code: [0, :command_error]
+            existing_version =
+              if exit_status == :command_error
+                'not installed'
+              else
+                expected_match = stdout.match(/^Chef Workstation version: (.+)\.\d+$/)
+                expected_match.nil? ? 'unreadable' : expected_match[1]
+              end
+            log_debug "Current Chef version: #{existing_version}. Required version: #{required_version}"
+            @cmd_runner.run_cmd "curl -L https://omnitruck.chef.io/install.sh | #{@cmd_runner.root? ? '' : 'sudo '}bash -s -- -P chef-workstation -v #{required_version}" unless existing_version == required_version
+          end
+        end
+
+        # Get the list of known nodes.
+        # [API] - This method is mandatory.
+        #
+        # Result::
+        # * Array<String>: List of node names
+        def known_nodes
+          Dir.glob("#{@repository_path}/nodes/*.json").map { |file| File.basename(file, '.json') }
+        end
+
+        # Get the metadata of a given node.
+        # [API] - This method is mandatory.
+        #
+        # Parameters::
+        # * *node* (String): Node to read metadata from
+        # Result::
+        # * Hash<Symbol,Object>: The corresponding metadata
+        def metadata_for(node)
+          (json_for(node)['normal'] || {}).transform_keys(&:to_sym)
+        end
+
+        # Return the services for a given node
+        # [API] - This method is mandatory.
+        #
+        # Parameters::
+        # * *node* (String): node to read configuration from
+        # Result::
+        # * Array<String>: The corresponding services
+        def services_for(node)
+          [json_for(node)['policy_name']]
+        end
+
+        # Get the list of services we can deploy
+        # [API] - This method is mandatory.
+        #
+        # Result::
+        # * Array<String>: The corresponding services
+        def deployable_services
+          Dir.glob("#{@repository_path}/policyfiles/*.rb").map { |file| File.basename(file, '.rb') }
+        end
+
+        # Package the repository, ready to be deployed on artefacts or directly to a node.
+        # [API] - This method is optional.
+        # [API] - @cmd_runner is accessible.
+        # [API] - @actions_executor is accessible.
+        #
+        # Parameters::
+        # * *services* (Hash< String, Array<String> >): Services to be deployed, per node
+        # * *secrets* (Hash): Secrets to be used for deployment
+        # * *local_environment* (Boolean): Are we deploying to a local environment?
+        def package(services:, secrets:, local_environment:)
+          # Make a stamp of the info that has been packaged, so that we don't package it again if useless
+          package_info = {
+            secrets: secrets,
+            commit: info[:commit].nil? ? Time.now.utc.strftime('%F %T') : info[:commit][:id],
+            other_files:
+              if info[:status].nil?
+                {}
+              else
+                Hash[
+                  (info[:status][:added_files] + info[:status][:changed_files] + info[:status][:untracked_files]).
+                    sort.
+                    map { |f| [f, File.mtime("#{@repository_path}/#{f}").strftime('%F %T')] }
+                ]
+              end,
+            deleted_files: info[:status].nil? ? [] : info[:status][:deleted_files].sort
+          }
+          # Each service is packaged individually.
+          services.values.flatten.sort.uniq.each do |service|
+            package_dir = "dist/#{local_environment ? 'local' : 'prod'}/#{service}"
+            package_info_file = "#{@repository_path}/#{package_dir}/hpc_package.info"
+            current_package_info = File.exist?(package_info_file) ? JSON.parse(File.read(package_info_file)).transform_keys(&:to_sym) : {}
+            unless current_package_info == package_info
+              Bundler.with_unbundled_env do
+                policy_file = "policyfiles/#{service}.rb"
+                if local_environment
+                  local_policy_file = "policyfiles/#{service}.local.rb"
+                  # In local mode, we always regenerate the lock file as we may modify the run list
+                  run_list = known_cookbook_paths.any? { |cookbook_path| File.exist?("#{@repository_path}/#{cookbook_path}/hpc_test/recipes/before_run.rb") } ? ['hpc_test::before_run'] : []
+                  dsl_parser = DslParser.new
+                  dsl_parser.parse("#{@repository_path}/#{policy_file}")
+                  run_list.concat dsl_parser.calls.find { |call_info| call_info[:method] == :run_list }[:args].flatten
+                  run_list << 'hpc_test::after_run' if known_cookbook_paths.any? { |cookbook_path| File.exist?("#{@repository_path}/#{cookbook_path}/hpc_test/recipes/after_run.rb") }
+                  File.write("#{@repository_path}/#{local_policy_file}", File.read("#{@repository_path}/#{policy_file}") + "\nrun_list #{run_list.map { |recipe| "'#{recipe}'" }.join(', ')}\n")
+                  policy_file = local_policy_file
+                end
+                lock_file = "#{File.dirname(policy_file)}/#{File.basename(policy_file, '.rb')}.lock.json"
+                # If the policy lock file does not exist, generate it
+                @cmd_runner.run_cmd "cd #{@repository_path} && /opt/chef-workstation/bin/chef install #{policy_file}" unless File.exist?("#{@repository_path}/#{lock_file}")
+                extra_cp_data_bags = File.exist?("#{@repository_path}/data_bags") ? " && cp -ar data_bags/ #{package_dir}/" : ''
+                @cmd_runner.run_cmd "cd #{@repository_path} && \
+                  #{@cmd_runner.root? ? '' : 'sudo '}rm -rf #{package_dir} && \
+                  /opt/chef-workstation/bin/chef export #{policy_file} #{package_dir}#{extra_cp_data_bags}"
+              end
+              unless @cmd_runner.dry_run
+                # Create secrets file
+                secrets_file = "#{@repository_path}/#{package_dir}/data_bags/hpc_secrets/hpc_secrets.json"
+                FileUtils.mkdir_p(File.dirname(secrets_file))
+                File.write(secrets_file, secrets.merge(id: 'hpc_secrets').to_json)
+                # Remember the package info
+                File.write(package_info_file, package_info.to_json)
+              end
+            end
+          end
+        end
+
+        # Prepare deployments.
+        # This method is called just before getting and executing the actions to be deployed.
+        # It is called once per platform.
+        # [API] - This method is optional.
+        # [API] - @cmd_runner is accessible.
+        # [API] - @actions_executor is accessible.
+        #
+        # Parameters::
+        # * *services* (Hash< String, Array<String> >): Services to be deployed, per node
+        # * *secrets* (Hash): Secrets to be used for deployment
+        # * *local_environment* (Boolean): Are we deploying to a local environment?
+        # * *why_run* (Boolean): Are we deploying in why-run mode?
+        def prepare_for_deploy(services:, secrets:, local_environment:, why_run:)
+          @local_env = local_environment
+        end
+
+        # Get the list of actions to perform to deploy on a given node.
+        # Those actions can be executed in parallel with other deployments on other nodes. They must be thread safe.
+        # [API] - This method is mandatory.
+        # [API] - @cmd_runner is accessible.
+        # [API] - @actions_executor is accessible.
+        #
+        # Parameters::
+        # * *node* (String): Node to deploy on
+        # * *service* (String): Service to be deployed
+        # * *use_why_run* (Boolean): Do we use a why-run mode? [default = true]
+        # Result::
+        # * Array< Hash<Symbol,Object> >: List of actions to be done
+        def actions_to_deploy_on(node, service, use_why_run: true)
+          package_dir = "#{@repository_path}/dist/#{@local_env ? 'local' : 'prod'}/#{service}"
+          # Generate the nodes attributes file
+          unless @cmd_runner.dry_run
+            FileUtils.mkdir_p "#{package_dir}/nodes"
+            File.write("#{package_dir}/nodes/#{node}.json", (known_nodes.include?(node) ? metadata_for(node) : {}).merge(@nodes_handler.metadata_of(node)).to_json)
+          end
+          client_options = [
+            '--local-mode',
+            '--chef-license', 'accept',
+            '--json-attributes', "nodes/#{node}.json"
+          ]
+          client_options << '--why-run' if use_why_run
+          if @nodes_handler.get_use_local_chef_of(node)
+            # Just run the chef-client directly from the packaged repository
+            [{ bash: "cd #{package_dir} && #{@cmd_runner.root? ? '' : 'sudo '}SSL_CERT_DIR=/etc/ssl/certs /opt/chef-workstation/bin/chef-client #{client_options.join(' ')}" }]
+          else
+            # Upload the package and run it from the node
+            package_name = File.basename(package_dir)
+            chef_versions_file = "#{@repository_path}/chef_versions.yml"
+            raise "Missing file #{chef_versions_file} specifying the Chef Infra Client version to be deployed" unless File.exist?(chef_versions_file)
+            required_chef_client_version = YAML.load_file(chef_versions_file)['client']
+            sudo = (@actions_executor.connector(:ssh).ssh_user == 'root' ? '' : "#{@nodes_handler.sudo_on(node)} ")
+            [
+              {
+                # Install dependencies
+                remote_bash: [
+                  'set -e',
+                  'set -o pipefail',
+                  "if [ -n \"$(command -v apt)\" ]; then #{sudo}apt update && #{sudo}apt install -y curl build-essential ; else #{sudo}yum groupinstall 'Development Tools' && #{sudo}yum install -y curl ; fi",
+                  'mkdir -p ./hpc_deploy',
+                  'rm -rf ./hpc_deploy/tmp',
+                  'mkdir -p ./hpc_deploy/tmp',
+                  'curl --location https://omnitruck.chef.io/install.sh --output ./hpc_deploy/install.sh',
+                  'chmod a+x ./hpc_deploy/install.sh',
+                  "#{sudo}TMPDIR=./hpc_deploy/tmp ./hpc_deploy/install.sh -d /opt/artefacts -v #{required_chef_client_version} -s once"
+                ]
+              },
+              {
+                scp: { package_dir => './hpc_deploy' },
+                remote_bash: [
+                  'set -e',
+                  "cd ./hpc_deploy/#{package_name}",
+                  "#{sudo}SSL_CERT_DIR=/etc/ssl/certs /opt/chef/bin/chef-client #{client_options.join(' ')}",
+                  'cd ..'
+                ] + (log_debug? ? [] : ["#{sudo}rm -rf ./hpc_deploy/#{package_name}"])
+              }
+            ]
+          end
+        end
+
+        # Parse stdout and stderr of a given deploy run and get the list of tasks with their status
+        # [API] - This method is mandatory.
+        #
+        # Parameters::
+        # * *stdout* (String): stdout to be parsed
+        # * *stderr* (String): stderr to be parsed
+        # Result::
+        # * Array< Hash<Symbol,Object> >: List of task properties. The following properties should be returned, among free ones:
+        #   * *name* (String): Task name
+        #   * *status* (Symbol): Task status. Should be one of:
+        #     * *:changed*: The task has been changed
+        #     * *:identical*: The task has not been changed
+        #   * *diffs* (String): Differences, if any
+        def parse_deploy_output(stdout, stderr)
+          tasks = []
+          current_task = nil
+          stdout.split("\n").each do |line|
+            # Remove control chars and spaces around
+            case line.gsub(/\e\[[^\x40-\x7E]*[\x40-\x7E]/, '').strip
+            when /^\* (\w+\[[^\]]+\]) action (.+)$/
+              # New task
+              task_name = $1
+              task_action = $2
+              current_task = {
+                name: task_name,
+                action: task_action,
+                status: :identical
+              }
+              tasks << current_task
+            when /^- (.+)$/
+              # Diff on the current task
+              diff_description = $1
+              unless current_task.nil?
+                current_task[:diffs] = '' unless current_task.key?(:diffs)
+                current_task[:diffs] << "#{diff_description}\n"
+                current_task[:status] = :changed
+              end
+            end
+          end
+          tasks
+        end
+
+        # Get the list of impacted nodes and services from a files diff.
+        # [API] - This method is optional
+        #
+        # Parameters::
+        # * *files_diffs* (Hash< String, Hash< Symbol, Object > >): List of diffs info, per file name having a diff. Diffs info have the following properties:
+        #   * *moved_to* (String): The new file path, in case it has been moved [optional]
+        #   * *diff* (String): The diff content
+        # Result::
+        # * Array<String>: The list of nodes impacted by this diff
+        # * Array<String>: The list of services impacted by this diff
+        # * Boolean: Are there some files that have a global impact (meaning all nodes are potentially impacted by this diff)?
+        def impacts_from(files_diffs)
+          impacted_nodes = []
+          impacted_services = []
+          # List of impacted [cookbook, recipe]
+          # Array< [Symbol, Symbol] >
+          impacted_recipes = []
+          impacted_global = false
+          files_diffs.keys.sort.each do |impacted_file|
+            if impacted_file =~ /^policyfiles\/([^\/]+)\.rb$/
+              log_debug "[#{impacted_file}] - Impacted service: #{$1}"
+              impacted_services << $1
+            elsif impacted_file =~ /^policyfiles\/([^\/]+)\.lock.json$/
+              log_debug "[#{impacted_file}] - Impacted service: #{$1}"
+              impacted_services << $1
+            elsif impacted_file =~ /^nodes\/([^\/]+)\.json/
+              log_debug "[#{impacted_file}] - Impacted node: #{$1}"
+              impacted_nodes << $1
+            else
+              cookbook_path = known_cookbook_paths.find { |cookbooks_path| impacted_file =~ /^#{Regexp.escape(cookbooks_path)}\/.+$/ }
+              if cookbook_path.nil?
+                # Global file
+                log_debug "[#{impacted_file}] - Global file impacted"
+                impacted_global = true
+              else
+                # File belonging to a cookbook
+                cookbook_name, file_path = impacted_file.match(/^#{cookbook_path}\/(\w+)\/(.+)$/)[1..2]
+                cookbook = cookbook_name.to_sym
+                # Small helper to register a recipe
+                register = proc do |source, recipe_name, cookbook_name: cookbook|
+                  cookbook_name = cookbook_name.to_sym if cookbook_name.is_a?(String)
+                  log_debug "[#{impacted_file}] - Impacted recipe from #{source}: #{cookbook_name}::#{recipe_name}"
+                  impacted_recipes << [cookbook_name, recipe_name.to_sym]
+                end
+                case file_path
+                when /recipes\/(.+)\.rb/
+                  register.call('direct', $1)
+                when /attributes\/.+\.rb/, 'metadata.rb'
+                  # Consider all recipes are impacted
+                  Dir.glob("#{@repository_path}/#{cookbook_path}/#{cookbook}/recipes/*.rb") do |recipe_path|
+                    register.call('attributes', File.basename(recipe_path, '.rb'))
+                  end
+                when /(templates|files)\/(.+)/
+                  # Find recipes using this file name
+                  included_file = File.basename($2)
+                  template_regexp = /["']#{Regexp.escape(included_file)}["']/
+                  Dir.glob("#{@repository_path}/#{cookbook_path}/#{cookbook}/recipes/*.rb") do |recipe_path|
+                    register.call("included file #{included_file}", File.basename(recipe_path, '.rb')) if File.read(recipe_path) =~ template_regexp
+                  end
+                when /resources\/(.+)/
+                  # Find any recipe using this resource
+                  included_resource = "#{cookbook}_#{File.basename($1, '.rb')}"
+                  resource_regexp = /(\W|^)#{Regexp.escape(included_resource)}(\W|$)/
+                  known_cookbook_paths.each do |cookbooks_path|
+                    Dir.glob("#{@repository_path}/#{cookbooks_path}/**/recipes/*.rb") do |recipe_path|
+                      if File.read(recipe_path) =~ resource_regexp
+                        cookbook_name, recipe_name = recipe_path.match(/#{cookbooks_path}\/(\w+)\/recipes\/(\w+)\.rb/)[1..2]
+                        register.call("included resource #{included_resource}", recipe_name, cookbook_name: cookbook_name)
+                      end
+                    end
+                  end
+                when /libraries\/(.+)/
+                  # Find any recipe using methods from this library
+                  lib_methods_regexps = File.read("#{@repository_path}/#{impacted_file}").scan(/(\W|^)def\s+(\w+)(\W|$)/).map { |_grp1, method_name, _grp2| /(\W|^)#{Regexp.escape(method_name)}(\W|$)/ }
+                  known_cookbook_paths.each do |cookbooks_path|
+                    Dir.glob("#{@repository_path}/#{cookbooks_path}/**/recipes/*.rb") do |recipe_path|
+                      file_content = File.read(recipe_path)
+                      found_lib_regexp = lib_methods_regexps.find { |regexp| file_content =~ regexp }
+                      unless found_lib_regexp.nil?
+                        cookbook_name, recipe_name = recipe_path.match(/#{cookbooks_path}\/(\w+)\/recipes\/(\w+)\.rb/)[1..2]
+                        register.call("included library helper #{found_lib_regexp.source[6..-7]}", recipe_name, cookbook_name: cookbook_name)
+                      end
+                    end
+                  end
+                when 'README.md', 'README.rdoc', 'CHANGELOG.md', '.rubocop.yml'
+                  # Ignore them
+                else
+                  log_warn "[#{impacted_file}] - Unknown impact for cookbook file belonging to #{cookbook}"
+                  # Consider all recipes are impacted by default
+                  Dir.glob("#{@repository_path}/#{cookbook_path}/#{cookbook}/recipes/*.rb") do |recipe_path|
+                    register.call('attributes', File.basename(recipe_path, '.rb'))
+                  end
+                end
+              end
+            end
+          end
+
+          # Devise the impacted services from the impacted recipes we just found.
+          impacted_recipes.uniq!
+          log_debug "* #{impacted_recipes.size} impacted recipes:\n#{impacted_recipes.map { |(cookbook, recipe)| "#{cookbook}::#{recipe}" }.sort.join("\n")}"
+
+          recipes_tree = full_recipes_tree
+          [
+            impacted_nodes,
+            (
+              impacted_services +
+                # Gather the list of services using the impacted recipes
+                impacted_recipes.map do |(cookbook, recipe)|
+                  recipe_info = recipes_tree.dig cookbook, recipe
+                  recipe_info.nil? ? [] : recipe_info[:used_by_policies]
+                end.flatten
+            ).sort.uniq,
+            impacted_global
+          ]
+        end
+
+        # Return the list of possible cookbook paths from this repository only.
+        # Returned paths are relative to the repository path.
+        #
+        # Result::
+        # * Array<String>: Known cookbook paths
+        def known_cookbook_paths
+          # Keep a cache of it for performance.
+          unless defined?(@cookbook_paths)
+            config_file = "#{@repository_path}/config.rb"
+            @cookbook_paths = (
+                ['cookbooks'] +
+                  if File.exist?(config_file)
+                    # Read the knife configuration to get cookbook paths
+                    dsl_parser = DslParser.new
+                    dsl_parser.parse(config_file)
+                    cookbook_path_call = dsl_parser.calls.find { |call_info| call_info[:method] == :cookbook_path }
+                    cookbook_path_call.nil? ? [] : cookbook_path_call[:args].first
+                  else
+                    []
+                  end
+              ).
+              map do |dir|
+                # Only keep dirs that actually exist and are part of our repository
+                full_path = dir.start_with?('/') ? dir : File.expand_path("#{@repository_path}/#{dir}")
+                full_path.start_with?(@repository_path) && File.exist?(full_path) ? full_path.gsub("#{@repository_path}/", '') : nil
+              end.
+              compact.
+              sort.
+              uniq
+          end
+          @cookbook_paths
+        end
+
+        # Get the run list of a given policy
+        #
+        # Parameters::
+        # * *policy* (String): Policy to get the run list from
+        # Result::
+        # * Array<[String or nil, Symbol, Symbol]>: Run list of the given policy, as [cookbook_dir, cookbook, recipe]
+        def policy_run_list(policy)
+          # Read the policy file
+          dsl_parser = DslParser.new
+          policy_file = "#{@repository_path}/policyfiles/#{policy}.rb"
+          dsl_parser.parse(policy_file)
+          run_list_call = dsl_parser.calls.find { |call_info| call_info[:method] == :run_list }
+          raise "Policy #{policy} has no run list defined in #{policy_file}" if run_list_call.nil?
+          run_list_call[:args].map { |recipe_def| decode_recipe(recipe_def) }
+        end
+
+        # Return the cookbook directory, cookbook name and recipe name from which a recipe definition is found.
+        # The following forms are handled:
+        # * cookbook
+        # * cookbook::recipe
+        # * recipe[cookbook]
+        # * recipe[cookbook::recipe]
+        #
+        # Parameters::
+        # * *recipe_def* (String): Recipe definition (cookbook or cookbook::recipe).
+        # Result::
+        # * String: The cookbook directory, or nil if unknown
+        # * Symbol: The cookbook name
+        # * Symbol: The recipe name
+        def decode_recipe(recipe_def)
+          recipe_def = $1 if recipe_def =~ /^recipe\[(.+)\]$/
+          cookbook, recipe = recipe_def.split('::').map(&:to_sym)
+          recipe = :default if recipe.nil?
+          # Find the cookbook it belongs to
+          cookbook_dir = known_cookbook_paths.find { |cookbook_path| File.exist?("#{@repository_path}/#{cookbook_path}/#{cookbook}") }
+          raise "Unknown recipe #{cookbook}::#{recipe} from cookbook #{@repository_path}/#{cookbook_dir}/#{cookbook}." if !cookbook_dir.nil? && !File.exist?("#{@repository_path}/#{cookbook_dir}/#{cookbook}/recipes/#{recipe}.rb")
+          return cookbook_dir, cookbook, recipe
+        end
+
+        private
+
+        # Return the JSON associated to a node
+        #
+        # Parameters::
+        # * *node* (String): The node to search for
+        # Result::
+        # * Hash: JSON object of this node
+        def json_for(node)
+          JSON.parse(File.read("#{@repository_path}/nodes/#{node}.json"))
+        end
+
+        # Get the full recipes tree.
+        # Keep it in a cache for performance.
+        #
+        # Result::
+        # * Hash: The recipes tree. See RecipesTreeBuilder#full_recipes_tree for the detailed signature
+        def full_recipes_tree
+          @recipes_tree_mutex.synchronize do
+            @recipes_tree = RecipesTreeBuilder.new(@config, self).full_recipes_tree unless defined?(@recipes_tree)
+          end
+          @recipes_tree
+        end
+
+      end
+
+    end
+
+  end
+
+end