hybrid_platforms_conductor 32.11.2 → 32.13.3

data/lib/hybrid_platforms_conductor/hpc_plugins/test_report/confluence.rb

@@ -53,7 +53,7 @@ module HybridPlatformsConductor
 
         private
 
-        TEMPLATES_PATH = File.expand_path("#{File.dirname(__FILE__)}/templates")
+        TEMPLATES_PATH = File.expand_path("#{__dir__}/templates")
 
         # Render a given ERB template into a String
         #

data/lib/hybrid_platforms_conductor/json_dumper.rb

@@ -56,7 +56,7 @@ module HybridPlatformsConductor
       nodes = @nodes_handler.select_nodes(nodes_selectors)
       unless @skip_run
         nodes.map { |node| @nodes_handler.platform_for(node) }.uniq.each.each do |platform_handler|
-          platform_handler.prepare_why_run_deploy_for_json_dump
+          platform_handler.prepare_why_run_deploy_for_json_dump if platform_handler.respond_to?(:prepare_why_run_deploy_for_json_dump)
         end
         @deployer.concurrent_execution = true
         @deployer.use_why_run = true

data/lib/hybrid_platforms_conductor/platform_handler.rb

@@ -142,7 +142,7 @@ module HybridPlatformsConductor
             git_status = git.status
             git_commit = git.log.first
             {
-              repo_name: File.basename(git.remotes.first.url).gsub(/\.git$/, ''),
+              repo_name: git.remotes.empty? ? File.basename(@repository_path) : File.basename(git.remotes.first.url).gsub(/\.git$/, ''),
               commit: {
                 id: git_commit.sha,
                 ref: git_commit.name,

data/lib/hybrid_platforms_conductor/services_handler.rb

@@ -108,23 +108,25 @@ module HybridPlatformsConductor
       local_environment:
     )
       platforms_for(services).each do |platform, platform_services|
-        platform_name = platform.name
-        # Compute the package ID that is unique to this packaging, so that we don't mix it with others if needed.
-        package_id = {
-          platform_name: platform_name,
-          services: Hash[platform_services.map { |node, node_services| [node, node_services.sort] }].sort,
-          secrets: secrets.sort,
-          local_environment: local_environment
-        }
-        if ServicesHandler.packaged_deployments.include?(package_id)
-          log_debug "Platform #{platform_name} has already been packaged for this deployment (package ID #{package_id}). Won't package it another time."
-        else
-          platform.package(
-            services: platform_services,
-            secrets: secrets,
+        if platform.respond_to?(:package)
+          platform_name = platform.name
+          # Compute the package ID that is unique to this packaging, so that we don't mix it with others if needed.
+          package_id = {
+            platform_name: platform_name,
+            services: Hash[platform_services.map { |node, node_services| [node, node_services.sort] }].sort,
+            secrets: secrets.sort,
             local_environment: local_environment
-          )
-          ServicesHandler.packaged_deployments << package_id
+          }
+          if ServicesHandler.packaged_deployments.include?(package_id)
+            log_debug "Platform #{platform_name} has already been packaged for this deployment (package ID #{package_id}). Won't package it another time."
+          else
+            platform.package(
+              services: platform_services,
+              secrets: secrets,
+              local_environment: local_environment
+            )
+            ServicesHandler.packaged_deployments << package_id
+          end
         end
       end
     end

data/lib/hybrid_platforms_conductor/tests_runner.rb

@@ -452,7 +452,6 @@ module HybridPlatformsConductor
           end]
           section "Run test commands on #{@test_cmds.keys.size} connected nodes (timeout to #{timeout} secs)" do
             start_time = Time.now
-            nbr_secs = nil
             @actions_executor.max_threads = @max_threads_connection_on_nodes
             @actions_result = @actions_executor.execute_actions(
               @test_cmds,

data/lib/hybrid_platforms_conductor/topographer.rb

@@ -110,7 +110,6 @@ module HybridPlatformsConductor
       @skip_run = false
 
       # Parse plugins
-      root_path = File.expand_path("#{File.dirname(__FILE__)}/..")
       @plugins = Hash[Dir.
         glob("#{File.dirname(__FILE__)}/topographer/plugins/*.rb").
         map do |file_name|

data/lib/hybrid_platforms_conductor/version.rb

@@ -1,5 +1,5 @@
 module HybridPlatformsConductor
 
-  VERSION = '32.11.2'
+  VERSION = '32.13.3'
 
 end

data/spec/hybrid_platforms_conductor_test/api/actions_executor/actions/remote_bash_spec.rb

@@ -79,6 +79,22 @@ describe HybridPlatformsConductor::ActionsExecutor do
       end
     end
 
+    it 'executes remote Bash code both from commands and a file in sequence' do
+      with_test_platform_for_action_plugins do |repository|
+        File.write("#{repository}/commands.txt", "bash_cmd3.bash\nbash_cmd4.bash")
+        test_actions_executor.execute_actions('node' => { remote_bash: [
+          'bash_cmd1.bash',
+          'bash_cmd2.bash',
+          { file: "#{repository}/commands.txt" }
+        ] })
+        expect(test_actions_executor.connector(:test_connector).calls).to eq [
+          [:connectable_nodes_from, ['node']],
+          [:with_connection_to, ['node'], { no_exception: true }],
+          [:remote_bash, "bash_cmd1.bash\nbash_cmd2.bash\nbash_cmd3.bash\nbash_cmd4.bash"]
+        ]
+      end
+    end
+
     it 'executes remote Bash code with environment variables set' do
       with_test_platform_for_action_plugins do
         test_actions_executor.execute_actions('node' => { remote_bash: {

data/spec/hybrid_platforms_conductor_test/api/actions_executor/connectors/local/connectable_nodes_spec.rb

@@ -0,0 +1,30 @@
+describe HybridPlatformsConductor::ActionsExecutor do
+
+  context 'checking connector plugin local' do
+
+    context 'checking connectable nodes selection' do
+
+      # Return the connector to be tested
+      #
+      # Result::
+      # * Connector: Connector to be tested
+      def test_connector
+        test_actions_executor.connector(:local)
+      end
+
+      it 'selects connectable nodes correctly' do
+        with_test_platform(nodes: {
+          'node1' => { meta: { host_ip: '192.168.42.42' } },
+          'node2' => {},
+          'node3' => { meta: { host_ip: '127.0.0.1', local_node: true } },
+          'node4' => { meta: { local_node: true } }
+        }) do
+          expect(test_connector.connectable_nodes_from(%w[node1 node2 node3 node4]).sort).to eq %w[node3 node4].sort
+        end
+      end
+
+    end
+
+  end
+
+end

data/spec/hybrid_platforms_conductor_test/api/actions_executor/connectors/local/remote_actions_spec.rb

@@ -0,0 +1,113 @@
+describe HybridPlatformsConductor::ActionsExecutor do
+
+  context 'checking connector plugin local' do
+
+    context 'checking remote actions' do
+
+      # Return the connector to be tested
+      #
+      # Result::
+      # * Connector: Connector to be tested
+      def test_connector
+        test_actions_executor.connector(:local)
+      end
+
+      # Get a test platform and the connector prepared the same way Actions Executor does before calling remote_* methods
+      #
+      # Parameters::
+      # * *expected_cmds* (Array< [String or Regexp, Proc] >): The expected commands that should be used, and their corresponding mocked code [default: []]
+      # * *expected_stdout* (String): Expected stdout after client code execution [default: '']
+      # * *expected_stderr* (String): Expected stderr after client code execution [default: '']
+      # * *timeout* (Integer or nil): Timeout to prepare the connector for [default: nil]
+      # * *password* (String or nil): Password to set for the node, or nil for none [default: nil]
+      # * *additional_config* (String): Additional config [default: '']
+      # * Proc: Client code to execute testing
+      def with_test_platform_for_remote_testing(
+        expected_cmds: [],
+        expected_stdout: '',
+        expected_stderr: '',
+        timeout: nil,
+        password: nil,
+        additional_config: ''
+      )
+        with_test_platform(
+          { nodes: { 'node' => { meta: { local_node: true } } } },
+          false,
+          additional_config
+        ) do
+          with_cmd_runner_mocked(expected_cmds) do
+            test_connector.with_connection_to(['node']) do
+              stdout = ''
+              stderr = ''
+              test_connector.prepare_for('node', timeout, stdout, stderr)
+              yield
+              expect(stdout).to eq expected_stdout
+              expect(stderr).to eq expected_stderr
+            end
+          end
+        end
+      end
+
+      it 'executes bash commands remotely' do
+        with_test_platform_for_remote_testing(
+          expected_cmds: [['cd /tmp/hpc_local_workspaces/node ; bash_cmd.bash', proc { [0, 'Bash commands executed on node', ''] }]],
+          expected_stdout: 'Bash commands executed on node'
+        ) do
+          test_connector.remote_bash('bash_cmd.bash')
+        end
+      end
+
+      it 'executes bash commands remotely with timeout' do
+        with_test_platform_for_remote_testing(
+          expected_cmds: [
+            [
+              'cd /tmp/hpc_local_workspaces/node ; bash_cmd.bash',
+              proc do |cmd, log_to_file: nil, log_to_stdout: true, log_stdout_to_io: nil, log_stderr_to_io: nil, expected_code: 0, timeout: nil, no_exception: false|
+                expect(timeout).to eq 5
+                [0, '', '']
+              end
+            ]
+          ],
+          timeout: 5
+        ) do
+          test_connector.remote_bash('bash_cmd.bash')
+        end
+      end
+
+      it 'executes interactive commands remotely' do
+        with_test_platform_for_remote_testing do
+          expect(test_connector).to receive(:system) do |cmd|
+            expect(cmd).to eq 'cd /tmp/hpc_local_workspaces/node ; /bin/bash'
+          end
+          test_connector.remote_interactive
+        end
+      end
+
+      it 'copies files remotely' do
+        with_test_platform_for_remote_testing do
+          expect(FileUtils).to receive(:cp_r).with('/path/to/src.file', '/remote_path/to/dst.dir')
+          test_connector.remote_copy('/path/to/src.file', '/remote_path/to/dst.dir')
+        end
+      end
+
+      it 'copies files remotely with timeout' do
+        with_test_platform_for_remote_testing(
+          timeout: 5
+        ) do
+          expect(FileUtils).to receive(:cp_r).with('/path/to/src.file', '/remote_path/to/dst.dir')
+          test_connector.remote_copy('/path/to/src.file', '/remote_path/to/dst.dir')
+        end
+      end
+
+      it 'copies relative files remotely' do
+        with_test_platform_for_remote_testing do
+          expect(FileUtils).to receive(:cp_r).with('/path/to/src.file', '/tmp/hpc_local_workspaces/node/to/dst.dir')
+          test_connector.remote_copy('/path/to/src.file', 'to/dst.dir')
+        end
+      end
+
+    end
+
+  end
+
+end

data/spec/hybrid_platforms_conductor_test/api/actions_executor/connectors/ssh/cli_options_spec.rb

@@ -97,11 +97,15 @@ describe HybridPlatformsConductor::ActionsExecutor do
         end
       end
 
-      it 'fails if no user name has been given, either through environment or command-line' do
+      it 'fails if no user name has been given, either through environment, command-line or using whoami' do
         ENV.delete 'hpc_ssh_user'
         ENV.delete 'USER'
         with_test_platform_for_cli do
-          expect { run 'run', '--node', 'node', '--command', 'echo Hello' }.to raise_error(RuntimeError, 'No SSH user name specified. Please use --ssh-user option or hpc_ssh_user environment variable to set it.')
+          with_cmd_runner_mocked([
+            ['whoami', proc { [0, '', ''] }]
+          ]) do
+            expect { run 'run', '--node', 'node', '--command', 'echo Hello' }.to raise_error(RuntimeError, 'No SSH user name specified. Please use --ssh-user option or hpc_ssh_user environment variable to set it.')
+          end
         end
       end
 

data/spec/hybrid_platforms_conductor_test/api/actions_executor/connectors/ssh/global_helpers_spec.rb

@@ -47,7 +47,7 @@ describe HybridPlatformsConductor::ActionsExecutor do
         end.join("\n") + "\n"
       end
 
-      it 'generates a global configuration with user from environment' do
+      it 'generates a global configuration with user from hpc_ssh_user environment variable' do
         with_test_platform do
           ENV['hpc_ssh_user'] = 'test_user'
           expect(ssh_config_for(nil)).to eq <<~EOS
@@ -59,6 +59,43 @@ describe HybridPlatformsConductor::ActionsExecutor do
         end
       end
 
+      it 'generates a global configuration with user from USER environment variable' do
+        with_test_platform do
+          ENV['USER'] = 'test_user'
+          expect(ssh_config_for(nil)).to eq <<~EOS
+            Host *
+              User test_user
+              ControlPath #{Dir.tmpdir}/hpc_ssh/hpc_ssh_mux_%h_%p_%r
+              PubkeyAcceptedKeyTypes +ssh-dss
+          EOS
+        end
+      end
+
+      it 'generates a global configuration with user taken from whoami when no env variable is set' do
+        with_test_platform do
+          original_user = ENV['USER']
+          begin
+            ENV.delete 'USER'
+            ENV.delete 'hpc_ssh_user'
+            with_cmd_runner_mocked(
+              [
+                ['ssh -V 2>&1', proc { [0, "OpenSSH_7.4p1 Debian-10+deb9u7, OpenSSL 1.0.2u  20 Dec 2019\n", ''] }],
+                ['whoami', proc { [0, 'test_whoami_user', ''] }]
+              ]
+            ) do
+              expect(ssh_config_for(nil)).to eq <<~EOS
+                Host *
+                  User test_whoami_user
+                  ControlPath #{Dir.tmpdir}/hpc_ssh/hpc_ssh_mux_%h_%p_%r
+                  PubkeyAcceptedKeyTypes +ssh-dss
+              EOS
+            end
+          ensure
+            ENV['USER'] = original_user
+          end
+        end
+      end
+
       it 'generates a global configuration with user from setting' do
         with_test_platform do
           test_connector.ssh_user = 'test_user'

data/spec/hybrid_platforms_conductor_test/api/actions_executor/connectors/ssh/remote_actions_spec.rb

@@ -6,7 +6,7 @@ describe HybridPlatformsConductor::ActionsExecutor do
 
       it 'executes bash commands remotely' do
         with_test_platform_for_remote_testing(
-          expected_cmds: [[/.+\/ssh hpc\.node \/bin\/bash <<'EOF'\nbash_cmd.bash\nEOF/, proc { [0, 'Bash commands executed on node', ''] }]],
+          expected_cmds: [[/.+\/ssh hpc\.node \/bin\/bash <<'HPC_EOF'\nbash_cmd.bash\nHPC_EOF/, proc { [0, 'Bash commands executed on node', ''] }]],
           expected_stdout: 'Bash commands executed on node'
         ) do
           test_connector.remote_bash('bash_cmd.bash')
@@ -17,7 +17,7 @@ describe HybridPlatformsConductor::ActionsExecutor do
         with_test_platform_for_remote_testing(
           expected_cmds: [
             [
-              /.+\/ssh hpc\.node \/bin\/bash <<'EOF'\nbash_cmd.bash\nEOF/,
+              /.+\/ssh hpc\.node \/bin\/bash <<'HPC_EOF'\nbash_cmd.bash\nHPC_EOF/,
               proc do |cmd, log_to_file: nil, log_to_stdout: true, log_stdout_to_io: nil, log_stderr_to_io: nil, expected_code: 0, timeout: nil, no_exception: false|
                 expect(timeout).to eq 5
                 [0, '', '']
@@ -76,7 +76,7 @@ describe HybridPlatformsConductor::ActionsExecutor do
             [
               /.+\/hpc_temp_cmds_.+\.sh$/,
               proc do |received_cmd|
-                expect(File.read(received_cmd)).to match /.+\/ssh hpc\.node \/bin\/bash <<'EOF'\n#{Regexp.escape(cmd)}\nEOF/
+                expect(File.read(received_cmd)).to match /.+\/ssh hpc\.node \/bin\/bash <<'HPC_EOF'\n#{Regexp.escape(cmd)}\nHPC_EOF/
                 [0, 'Bash commands executed on node', '']
               end
             ]
@@ -143,7 +143,7 @@ describe HybridPlatformsConductor::ActionsExecutor do
 
       it 'executes bash commands remotely without Session Exec capabilities' do
         with_test_platform_for_remote_testing(
-          expected_cmds: [[/^\{ cat \| .+\/ssh hpc\.node -T; } <<'EOF'\nbash_cmd.bash\nEOF$/, proc { [0, 'Bash commands executed on node', ''] }]],
+          expected_cmds: [[/^\{ cat \| .+\/ssh hpc\.node -T; } <<'HPC_EOF'\nbash_cmd.bash\nHPC_EOF$/, proc { [0, 'Bash commands executed on node', ''] }]],
           expected_stdout: 'Bash commands executed on node',
           session_exec: false
         ) do
@@ -165,6 +165,39 @@ describe HybridPlatformsConductor::ActionsExecutor do
         end
       end
 
+      it 'copies files remotely without Session Exec capabilities and with sudo' do
+        with_test_platform_for_remote_testing(
+          expected_cmds: [
+            [/^\{ cat \| .+\/ssh hpc\.node -T; } <<'HPC_EOF'\nmkdir -p hpc_tmp_scp\nHPC_EOF$/, proc { [0, '', ''] }],
+            [
+              /^scp -S .+\/ssh \/path\/to\/src.file hpc\.node:\.\/hpc_tmp_scp$/,
+              proc { [0, '', ''] }
+            ],
+            [/^\{ cat \| .+\/ssh hpc\.node -T; } <<'HPC_EOF'\nsudo -u root mv \.\/hpc_tmp_scp\/src\.file \/remote_path\/to\/dst\.dir\nHPC_EOF$/, proc { [0, '', ''] }]
+          ],
+          session_exec: false
+        ) do
+          test_connector.remote_copy('/path/to/src.file', '/remote_path/to/dst.dir', sudo: true)
+        end
+      end
+
+      it 'copies files remotely without Session Exec capabilities and with a different sudo' do
+        with_test_platform_for_remote_testing(
+          expected_cmds: [
+            [/^\{ cat \| .+\/ssh hpc\.node -T; } <<'HPC_EOF'\nmkdir -p hpc_tmp_scp\nHPC_EOF$/, proc { [0, '', ''] }],
+            [
+              /^scp -S .+\/ssh \/path\/to\/src.file hpc\.node:\.\/hpc_tmp_scp$/,
+              proc { [0, '', ''] }
+            ],
+            [/^\{ cat \| .+\/ssh hpc\.node -T; } <<'HPC_EOF'\nother_sudo --user root mv \.\/hpc_tmp_scp\/src\.file \/remote_path\/to\/dst\.dir\nHPC_EOF$/, proc { [0, '', ''] }]
+          ],
+          additional_config: 'sudo_for { |user| "other_sudo --user #{user}" }',
+          session_exec: false
+        ) do
+          test_connector.remote_copy('/path/to/src.file', '/remote_path/to/dst.dir', sudo: true)
+        end
+      end
+
     end
 
   end

data/spec/hybrid_platforms_conductor_test/docs_spec.rb

@@ -0,0 +1,10 @@
+describe 'Documentation' do
+
+  it 'makes sure all Markdown links are valid' do
+    check_stdout = `bundle exec tools/check_md README.md #{Dir.glob('docs/**/*.md').join(' ')}`.split("\n")
+    summary_idx = check_stdout.index { |line| line =~ /^\d+ errors:$/ }
+    expect(summary_idx).not_to eq(nil), "Could not parse check output: #{check_stdout.join("\n")}"
+    expect(check_stdout[summary_idx]).to eq('0 errors:'), "Invalid links found: #{check_stdout[summary_idx..-1].join("\n")}"
+  end
+
+end

data/tools/check_md

@@ -0,0 +1,89 @@
+#!/usr/bin/env ruby
+
+# Parsed information, per file name.
+# Information is:
+# * *anchors* (Array<String>): List of anchors
+# * *links* (Array<String>): List of links
+@info = {}
+
+# Get a file name from a link
+#
+# Parameters::
+# * *link* (String): The link to find the file from
+# * *file* (String): The file from which the link is taken
+# Result::
+# * String: Resulting file
+def link_to_file(link, file)
+  link_file = link.split('#').first
+  if link_file == ''
+    file
+  elsif link_file.start_with?('/')
+    link_file[1..-1]
+  else
+    File.expand_path("#{File.dirname(file)}/#{link_file}").gsub("#{Dir.pwd}/", '')
+  end
+end
+
+# Parse a Markdown file
+#
+# Parameters::
+# * *file* (String): Markdown file to be parsed
+def parse_md(file)
+  unless @info.key?(file)
+    puts "Parsing #{file}..."
+    content = File.read(file)
+    @info[file] = {
+      anchors: content.scan(/<a name="([^"]*)"><\/a>/).map { |(anchor)| anchor },
+      links: content.scan(/\[[^\]]*\]\(([^\)]*)\)/).map { |(link)| link }
+    }
+    # Parse linked files
+    @info[file][:links].each do |link|
+      puts "Found #{file} => #{link}"
+    end
+    @info[file][:links].each do |link|
+      unless link.start_with?('http')
+        linked_file = link_to_file(link, file)
+        parse_md(linked_file) if File.exist?(linked_file) && linked_file.end_with?('.md')
+      end
+    end
+  end
+end
+
+ARGV.each { |file| parse_md(file) }
+
+# Check links
+errors = []
+@info.sort_by { |file, _info| file }.each do |file, info|
+  puts "= Links from #{file}"
+  info[:links].sort.uniq.each do |link|
+    puts "[#{
+      if link.start_with?('http')
+        '*'
+      else
+        anchor = link.split('#')[1]
+        linked_file = link_to_file(link, file)
+        if @info.key?(linked_file)
+          if anchor.nil?
+            '*'
+          elsif @info[linked_file][:anchors].include?(anchor)
+            '*'
+          else
+            errors << "[#{file} -> #{link}] - Destination has no anchor named #{anchor}"
+            ' '
+          end
+        elsif File.exist?(linked_file)
+          '*'
+        else
+          errors << "[#{file} -> #{link}] - Destination file does not exist"
+          ' '
+        end
+      end
+    }] #{link}"
+  end
+end
+
+puts
+puts "#{errors.size} errors:"
+errors.each do |error|
+  puts error
+end