hybrid_platforms_conductor 33.2.2 → 33.5.0

data/lib/hybrid_platforms_conductor/hpc_plugins/connector/local.rb

@@ -35,9 +35,11 @@ module HybridPlatformsConductor
         # [API] - @stderr_io can be used to send stderr output
         #
         # Parameters::
-        # * *bash_cmds* (String): Bash commands to execute
+        # * *bash_cmds* (String or SecretString): Bash commands to execute. Use #to_unprotected to access the real content (otherwise secrets are obfuscated).
         def remote_bash(bash_cmds)
-          run_cmd "cd #{workspace_for(@node)} ; #{bash_cmds}", force_bash: true
+          SecretString.protect("cd #{workspace_for(@node)} ; #{bash_cmds.to_unprotected}", silenced_str: "cd #{workspace_for(@node)} ; #{bash_cmds}") do |cmd|
+            run_cmd cmd, force_bash: true
+          end
         end
 
         # Execute an interactive shell on the remote node

data/lib/hybrid_platforms_conductor/hpc_plugins/connector/my_connector.rb.sample

@@ -104,7 +104,7 @@ module HybridPlatformsConductor
         # [API] - @stderr_io can be used to send stderr output
         #
         # Parameters::
-        # * *bash_cmds* (String): Bash commands to execute
+        # * *bash_cmds* (String or SecretString): Bash commands to execute. Use #to_unprotected to access the real content (otherwise secrets are obfuscated).
         def remote_bash(bash_cmds)
           MyConnectLib.connect_to(@nodes_handler.get_host_ip_of(@node)).run_bash(bash_cmds)
         end

data/lib/hybrid_platforms_conductor/hpc_plugins/connector/ssh.rb

@@ -236,31 +236,40 @@ module HybridPlatformsConductor
         # [API] - @stderr_io can be used to send stderr output
         #
         # Parameters::
-        # * *bash_cmds* (String): Bash commands to execute
+        # * *bash_cmds* (String or SecretString): Bash commands to execute. Use #to_unprotected to access the real content (otherwise secrets are obfuscated).
         def remote_bash(bash_cmds)
-          ssh_cmd =
+          SecretString.protect(
             if @nodes_handler.get_ssh_session_exec_of(@node) == false
               # When ExecSession is disabled we need to use stdin directly
-              "{ cat | #{ssh_exec} #{ssh_url} -T; } <<'HPC_EOF'\n#{bash_cmds}\nHPC_EOF"
+              "{ cat | #{ssh_exec} #{ssh_url} -T; } <<'HPC_EOF'\n#{bash_cmds.to_unprotected}\nHPC_EOF"
             else
-              "#{ssh_exec} #{ssh_url} /bin/bash <<'HPC_EOF'\n#{bash_cmds}\nHPC_EOF"
-            end
-          # Due to a limitation of Process.spawn, each individual argument is limited to 128KB of size.
-          # Therefore we need to make sure that if bash_cmds exceeds MAX_CMD_ARG_LENGTH bytes (considering EOF chars) then we use an intermediary shell script to store the commands.
-          if bash_cmds.size > MAX_CMD_ARG_LENGTH
-            # Write the commands in a file
-            temp_file = "#{Dir.tmpdir}/hpc_temp_cmds_#{Digest::MD5.hexdigest(bash_cmds)}.sh"
-            File.open(temp_file, 'w+') do |file|
-              file.write ssh_cmd
-              file.chmod 0o700
-            end
-            begin
-              run_cmd(temp_file)
-            ensure
-              File.unlink(temp_file)
+              "#{ssh_exec} #{ssh_url} /bin/bash <<'HPC_EOF'\n#{bash_cmds.to_unprotected}\nHPC_EOF"
+            end,
+            silenced_str:
+              if @nodes_handler.get_ssh_session_exec_of(@node) == false
+                # When ExecSession is disabled we need to use stdin directly
+                "{ cat | #{ssh_exec} #{ssh_url} -T; } <<'HPC_EOF'\n#{bash_cmds}\nHPC_EOF"
+              else
+                "#{ssh_exec} #{ssh_url} /bin/bash <<'HPC_EOF'\n#{bash_cmds}\nHPC_EOF"
+              end
+          ) do |ssh_cmd|
+            # Due to a limitation of Process.spawn, each individual argument is limited to 128KB of size.
+            # Therefore we need to make sure that if bash_cmds exceeds MAX_CMD_ARG_LENGTH bytes (considering EOF chars) then we use an intermediary shell script to store the commands.
+            if bash_cmds.to_unprotected.size > MAX_CMD_ARG_LENGTH
+              # Write the commands in a file
+              temp_file = "#{Dir.tmpdir}/hpc_temp_cmds_#{Digest::MD5.hexdigest(bash_cmds.to_unprotected)}.sh"
+              File.open(temp_file, 'w+') do |file|
+                file.write ssh_cmd.to_unprotected
+                file.chmod 0o700
+              end
+              begin
+                run_cmd(temp_file)
+              ensure
+                File.unlink(temp_file)
+              end
+            else
+              run_cmd ssh_cmd
             end
-          else
-            run_cmd ssh_cmd
           end
         end
 

data/lib/hybrid_platforms_conductor/hpc_plugins/platform_handler/serverless_chef.rb

@@ -263,7 +263,7 @@ module HybridPlatformsConductor
             raise "Missing file #{chef_versions_file} specifying the Chef Infra Client version to be deployed" unless File.exist?(chef_versions_file)
 
             required_chef_client_version = YAML.load_file(chef_versions_file)['client']
-            sudo = (@actions_executor.connector(:ssh).ssh_user == 'root' ? '' : "#{@nodes_handler.sudo_on(node)} ")
+            sudo = (@actions_executor.connector(:ssh).ssh_user == 'root' ? '' : "#{@nodes_handler.sudo_on(node)} -E ")
             [
               {
                 # Install dependencies

data/lib/hybrid_platforms_conductor/hpc_plugins/provisioner/proxmox.rb

@@ -263,6 +263,8 @@ module HybridPlatformsConductor
 
         private
 
+        include Credentials
+
         # Connect to the Proxmox API
         #
         # Parameters::
@@ -273,7 +275,7 @@ module HybridPlatformsConductor
           url = proxmox_test_info[:api_url]
           raise 'No Proxmox server defined' if url.nil?
 
-          Credentials.with_credentials_for(:proxmox, @logger, @logger_stderr, url: url) do |user, password|
+          with_credentials_for(:proxmox, resource: url) do |user, password|
             log_debug "[ #{@node}/#{@environment} ] - Connect to Proxmox #{url}"
             proxmox_logs = StringIO.new
             proxmox = ::Proxmox::Proxmox.new(
@@ -282,7 +284,7 @@ module HybridPlatformsConductor
               # cf https://pve.proxmox.com/wiki/Renaming_a_PVE_node
               URI.parse(url).host.downcase.split('.').first,
               user,
-              password,
+              password&.to_unprotected,
               ENV['hpc_realm_for_proxmox'] || 'pam',
               {
                 verify_ssl: false,
@@ -415,7 +417,7 @@ module HybridPlatformsConductor
             extra_files[config_file] = './proxmox/config'
             cmd << " --config ./proxmox/config/#{File.basename(config_file)}"
             stdout = nil
-            Credentials.with_credentials_for(:proxmox, @logger, @logger_stderr, url: proxmox_test_info[:api_url]) do |user, password|
+            with_credentials_for(:proxmox, resource: proxmox_test_info[:api_url]) do |user, password|
               # To avoid too fine concurrent accesses on the sync node file system, make sure all threads of our process wait for their turn to upload their files.
               # Otherwise there is a small probability that a directory scp makes previously copied files inaccessible for a short period of time.
               self.class.proxmox_waiter_files_mutex.synchronize do

data/lib/hybrid_platforms_conductor/hpc_plugins/provisioner/proxmox/reserve_proxmox_container

@@ -34,6 +34,7 @@
 #   * *hpc_password_for_proxmox*: Password to be used to query Proxmox API
 #   * *hpc_realm_for_proxmox*: Realm used to connect to the Proxmox API [default = 'pam']
 
+require 'English'
 require 'json'
 
 reserved_resource = nil

data/lib/hybrid_platforms_conductor/hpc_plugins/report/confluence.rb

@@ -14,6 +14,8 @@ module HybridPlatformsConductor
 
         extend_config_dsl_with CommonConfigDsl::Confluence, :init_confluence
 
+        include HybridPlatformsConductor::Confluence
+
         # Give the list of supported locales by this report generator
         # [API] - This method is mandatory.
         #
@@ -34,7 +36,7 @@ module HybridPlatformsConductor
           if confluence_info
             if confluence_info[:inventory_report_page_id]
               @nodes = nodes
-              HybridPlatformsConductor::Confluence.with_confluence(confluence_info[:url], @logger, @logger_stderr) do |confluence|
+              with_confluence(confluence_info[:url]) do |confluence|
                 confluence.update_page(confluence_info[:inventory_report_page_id], render('confluence_inventory'))
               end
               out "Inventory report Confluence page updated. Please visit #{confluence_info[:url]}/pages/viewpage.action?pageId=#{confluence_info[:inventory_report_page_id]}"

data/lib/hybrid_platforms_conductor/hpc_plugins/secrets_reader/keepass.rb

@@ -0,0 +1,174 @@
+require 'base64'
+require 'nokogiri'
+require 'tempfile'
+require 'keepass_kpscript'
+require 'zlib'
+require 'hybrid_platforms_conductor/credentials'
+require 'hybrid_platforms_conductor/safe_merge'
+require 'hybrid_platforms_conductor/secrets_reader'
+
+module HybridPlatformsConductor
+
+  module HpcPlugins
+
+    module SecretsReader
+
+      # Get secrets from a KeePass database
+      class Keepass < HybridPlatformsConductor::SecretsReader
+
+        include SafeMerge
+        include Credentials
+
+        # Extend the Config DSL
+        module ConfigDSLExtension
+
+          # List of defined KeePass secrets. Each info has the following properties:
+          # * *nodes_selectors_stack* (Array<Object>): Stack of nodes selectors impacted by this rule.
+          # * *database* (String): Database file path.
+          # * *group_path* (Array<String>): Group path to extract from.
+          # Array< Hash<Symbol, Object> >
+          attr_reader :keepass_secrets
+
+          # String: The KPScript command line
+          attr_reader :kpscript
+
+          # Mixin initializer
+          def init_keepass_config
+            @keepass_secrets = []
+            @kpscript = nil
+          end
+
+          # Set the KPScript command line
+          #
+          # Parameters::
+          # * *cmd* (String): KPScript command line
+          def use_kpscript_from(cmd)
+            @kpscript = cmd
+          end
+
+          # Set a KeePass database configuration
+          #
+          # Parameters::
+          # * *database* (String): Database file path.
+          # * *group_path* (Array<String>): Group path to extract from [default: []].
+          def secrets_from_keepass(database:, group_path: [])
+            @keepass_secrets << {
+              nodes_selectors_stack: current_nodes_selectors_stack,
+              database: database,
+              group_path: group_path
+            }
+          end
+
+        end
+
+        Config.extend_config_dsl_with ConfigDSLExtension, :init_keepass_config
+
+        # Return secrets for a given service to be deployed on a node.
+        # [API] - This method is mandatory
+        # [API] - The following API components are accessible:
+        # * *@config* (Config): Main configuration API.
+        # * *@cmd_runner* (CmdRunner): Command Runner API.
+        # * *@nodes_handler* (NodesHandler): Nodes handler API.
+        #
+        # Parameters::
+        # * *node* (String): Node to be deployed
+        # * *service* (String): Service to be deployed
+        # Result::
+        # * Hash: The secrets
+        def secrets_for(node, service)
+          secrets = {}
+          # As we are dealing with global secrets, cache the reading for performance between nodes and services.
+          # Keep secrets cache grouped by URL/ID
+          @secrets = {} unless defined?(@secrets)
+          @nodes_handler.select_confs_for_node(node, @config.keepass_secrets).each do |keepass_secrets_info|
+            secret_id = "#{keepass_secrets_info[:database]}:#{keepass_secrets_info[:group_path].join('/')}"
+            unless @secrets.key?(secret_id)
+              raise 'Missing KPScript configuration. Please use use_kpscript_from to set it.' if @config.kpscript.nil?
+
+              with_credentials_for(:keepass, resource: keepass_secrets_info[:database]) do |_user, password|
+                Tempfile.create('hpc_keepass') do |xml_file|
+                  key_file = ENV['hpc_key_file_for_keepass']
+                  password_enc = ENV['hpc_password_enc_for_keepass']
+                  keepass_credentials = {}
+                  keepass_credentials[:password] = password.to_unprotected if password
+                  keepass_credentials[:password_enc] = password_enc if password_enc
+                  keepass_credentials[:key_file] = key_file if key_file
+                  KeepassKpscript.
+                    use(@config.kpscript, debug: log_debug?).
+                    open(keepass_secrets_info[:database], **keepass_credentials).
+                    export('KeePass XML (2.x)', xml_file.path, group_path: keepass_secrets_info[:group_path].empty? ? nil : keepass_secrets_info[:group_path])
+                  @secrets[secret_id] = parse_xml_secrets(Nokogiri::XML(xml_file).at_xpath('KeePassFile/Root/Group'))
+                end
+              end
+            end
+            conflicting_path = safe_merge(secrets, @secrets[secret_id])
+            raise "Secret set at path #{conflicting_path.join('->')} by #{keepass_secrets_info[:database]}#{keepass_secrets_info[:group_path].empty? ? '' : " from group #{keepass_secrets_info[:group_path].join('/')}"} for service #{service} on node #{node} has conflicting values (#{log_debug? ? "#{@secrets[secret_id].dig(*conflicting_path)} != #{secrets.dig(*conflicting_path)}" : 'set debug for value details'})." unless conflicting_path.nil?
+          end
+          secrets
+        end
+
+        private
+
+        # List of fields to include in the secrets and their corresponding XML name
+        FIELDS = {
+          notes: 'Notes',
+          password: 'Password',
+          url: 'URL',
+          user_name: 'UserName'
+        }
+
+        # Parse XML secrets from a Nokogiri XML group node
+        #
+        # Parameters::
+        # * *group* (Nokogiri::XML::Element): The group to parse
+        # Result::
+        # * Hash: The JSON secrets parsed from this group
+        def parse_xml_secrets(group)
+          # Parse all entries
+          group.xpath('Entry').map do |entry|
+            [
+              entry.at_xpath('String/Key[contains(.,"Title")]/../Value').text,
+              FIELDS.map do |property, field|
+                value = entry.at_xpath("String/Key[contains(.,\"#{field}\")]/../Value")&.text
+                if value.nil? || value.empty?
+                  nil
+                else
+                  [
+                    property.to_s,
+                    value
+                  ]
+                end
+              end.compact.to_h.merge(
+                entry.xpath('Binary').map do |binary|
+                  binary_meta = group.document.at_xpath("KeePassFile/Meta/Binaries/Binary[@ID=#{Integer(binary.xpath('Value').attr('Ref').value)}]")
+                  binary_content = Base64.decode64(binary_meta.text)
+                  if binary_meta.attr('Compressed') == 'True'
+                    gz = Zlib::GzipReader.new(StringIO.new(binary_content))
+                    binary_content = gz.read
+                    gz.close
+                  end
+                  [
+                    binary.xpath('Key').text,
+                    binary_content
+                  ]
+                end.to_h
+              )
+            ]
+          end.to_h.merge(
+            # Add children groups
+            group.xpath('Group').map do |sub_group|
+              [
+                sub_group.at_xpath('Name').text,
+                parse_xml_secrets(sub_group)
+              ]
+            end.to_h
+          )
+        end
+
+      end
+
+    end
+
+  end
+
+end

data/lib/hybrid_platforms_conductor/hpc_plugins/secrets_reader/thycotic.rb

@@ -42,6 +42,8 @@ module HybridPlatformsConductor
 
         Config.extend_config_dsl_with ConfigDSLExtension, :init_thycotic_config
 
+        include HybridPlatformsConductor::Thycotic
+
         # Return secrets for a given service to be deployed on a node.
         # [API] - This method is mandatory
         # [API] - The following API components are accessible:
@@ -62,7 +64,7 @@ module HybridPlatformsConductor
           @nodes_handler.select_confs_for_node(node, @config.thycotic_secrets).each do |thycotic_secrets_info|
             server_id = "#{thycotic_secrets_info[:thycotic_url]}:#{thycotic_secrets_info[:secret_id]}"
             unless @secrets.key?(server_id)
-              HybridPlatformsConductor::Thycotic.with_thycotic(thycotic_secrets_info[:thycotic_url], @logger, @logger_stderr) do |thycotic|
+              with_thycotic(thycotic_secrets_info[:thycotic_url]) do |thycotic|
                 secret_file_item_id = thycotic.get_secret(thycotic_secrets_info[:secret_id]).dig(:secret, :items, :secret_item, :id)
                 raise "Unable to fetch secret file ID #{thycotic_secrets_info[:secret_id]} from #{thycotic_secrets_info[:thycotic_url]}" if secret_file_item_id.nil?
 

data/lib/hybrid_platforms_conductor/hpc_plugins/test/bitbucket_conf.rb

@@ -1,4 +1,5 @@
 require 'git'
+require 'hybrid_platforms_conductor/bitbucket'
 require 'hybrid_platforms_conductor/common_config_dsl/bitbucket'
 
 module HybridPlatformsConductor
@@ -12,9 +13,11 @@ module HybridPlatformsConductor
 
         extend_config_dsl_with CommonConfigDsl::Bitbucket, :init_bitbucket
 
+        include HybridPlatformsConductor::Bitbucket
+
         # Check my_test_plugin.rb.sample documentation for signature details.
         def test
-          @config.for_each_bitbucket_repo do |bitbucket, repo_info|
+          for_each_bitbucket_repo do |bitbucket, repo_info|
             # Test repo_info
             repo_id = "#{repo_info[:project]}/#{repo_info[:name]}"
             settings_pr = bitbucket.settings_pr(repo_info[:project], repo_info[:name])

data/lib/hybrid_platforms_conductor/hpc_plugins/test/github_ci.rb

@@ -1,3 +1,4 @@
+require 'hybrid_platforms_conductor/github'
 require 'hybrid_platforms_conductor/common_config_dsl/github'
 
 module HybridPlatformsConductor
@@ -11,9 +12,11 @@ module HybridPlatformsConductor
 
         extend_config_dsl_with CommonConfigDsl::Github, :init_github
 
+        include HybridPlatformsConductor::Github
+
         # Check my_test_plugin.rb.sample documentation for signature details.
         def test
-          @config.for_each_github_repo do |client, repo_info|
+          for_each_github_repo do |client, repo_info|
             log_debug "Checking CI for Github repository #{repo_info[:slug]}"
             last_status = client.repository_workflow_runs(repo_info[:slug])[:workflow_runs].
               select { |run| run[:head_branch] == 'master' }.

data/lib/hybrid_platforms_conductor/hpc_plugins/test/jenkins_ci_conf.rb

@@ -1,6 +1,7 @@
 require 'open-uri'
 require 'nokogiri'
 require 'hybrid_platforms_conductor/credentials'
+require 'hybrid_platforms_conductor/bitbucket'
 require 'hybrid_platforms_conductor/common_config_dsl/bitbucket'
 
 module HybridPlatformsConductor
@@ -14,15 +15,18 @@ module HybridPlatformsConductor
 
         extend_config_dsl_with CommonConfigDsl::Bitbucket, :init_bitbucket
 
+        include Credentials
+        include HybridPlatformsConductor::Bitbucket
+
         # Check my_test_plugin.rb.sample documentation for signature details.
         def test
-          @config.for_each_bitbucket_repo do |bitbucket, repo_info|
+          for_each_bitbucket_repo do |bitbucket, repo_info|
             if repo_info[:jenkins_ci_url].nil?
               error "Repository #{repo_info[:name]} does not have any Jenkins CI URL configured."
             else
-              Credentials.with_credentials_for(:jenkins_ci, @logger, @logger_stderr, url: repo_info[:jenkins_ci_url]) do |jenkins_user, jenkins_password|
+              with_credentials_for(:jenkins_ci, resource: repo_info[:jenkins_ci_url]) do |jenkins_user, jenkins_password|
                 # Get its config
-                doc = Nokogiri::XML(URI.parse("#{repo_info[:jenkins_ci_url]}/config.xml").open(http_basic_authentication: [jenkins_user, jenkins_password]).read)
+                doc = Nokogiri::XML(URI.parse("#{repo_info[:jenkins_ci_url]}/config.xml").open(http_basic_authentication: [jenkins_user, jenkins_password&.to_unprotected]).read)
                 # Check that this job builds the correct Bitbucket repository
                 assert_equal(
                   doc.xpath('/org.jenkinsci.plugins.workflow.multibranch.WorkflowMultiBranchProject/sources/data/jenkins.branch.BranchSource/source/serverUrl').text,

data/lib/hybrid_platforms_conductor/hpc_plugins/test/jenkins_ci_masters_ok.rb

@@ -1,5 +1,6 @@
 require 'json'
 require 'hybrid_platforms_conductor/credentials'
+require 'hybrid_platforms_conductor/bitbucket'
 require 'hybrid_platforms_conductor/common_config_dsl/bitbucket'
 
 module HybridPlatformsConductor
@@ -13,6 +14,9 @@ module HybridPlatformsConductor
 
         extend_config_dsl_with CommonConfigDsl::Bitbucket, :init_bitbucket
 
+        include Credentials
+        include HybridPlatformsConductor::Bitbucket
+
         SUCCESS_STATUSES = [
           # Add nil as the status of a currently running job (which is always the case for hybrid-platforms) is null
           nil,
@@ -23,17 +27,17 @@ module HybridPlatformsConductor
 
         # Check my_test_plugin.rb.sample documentation for signature details.
         def test
-          @config.for_each_bitbucket_repo do |_bitbucket, repo_info|
+          for_each_bitbucket_repo do |_bitbucket, repo_info|
             if repo_info[:jenkins_ci_url].nil?
               error "Repository #{repo_info[:name]} does not have any Jenkins CI URL configured."
             else
               master_info_url = "#{repo_info[:jenkins_ci_url]}/job/master/api/json"
-              Credentials.with_credentials_for(:jenkins_ci, @logger, @logger_stderr, url: master_info_url) do |jenkins_user, jenkins_password|
+              with_credentials_for(:jenkins_ci, resource: master_info_url) do |jenkins_user, jenkins_password|
                 # Get the master branch info from the API
-                master_info = JSON.parse(URI.parse(master_info_url).open(http_basic_authentication: [jenkins_user, jenkins_password]).read)
+                master_info = JSON.parse(URI.parse(master_info_url).open(http_basic_authentication: [jenkins_user, jenkins_password&.to_unprotected]).read)
                 # Get the last build's URL
                 last_build_info_url = "#{master_info['lastBuild']['url']}/api/json"
-                last_build_info = JSON.parse(URI.parse(last_build_info_url).open(http_basic_authentication: [jenkins_user, jenkins_password]).read)
+                last_build_info = JSON.parse(URI.parse(last_build_info_url).open(http_basic_authentication: [jenkins_user, jenkins_password&.to_unprotected]).read)
                 log_debug "Build info for #{master_info_url}:\n#{JSON.pretty_generate(last_build_info)}"
                 error "Last build for job #{repo_info[:project]}/#{repo_info[:name]} is in status #{last_build_info['result']}: #{master_info['lastBuild']['url']}" unless SUCCESS_STATUSES.include?(last_build_info['result'])
               rescue