hybrid_platforms_conductor 33.2.2 → 33.5.0

data/spec/hybrid_platforms_conductor_test/api/actions_executor/connectors/ssh/remote_actions_spec.rb

@@ -13,6 +13,15 @@ describe HybridPlatformsConductor::ActionsExecutor do
         end
       end
 
+      it 'executes bash commands remotely from a SecretString' do
+        with_test_platform_for_remote_testing(
+          expected_cmds: [[%r{.+/ssh hpc\.node /bin/bash <<'HPC_EOF'\nbash_cmd.bash\nHPC_EOF}, proc { [0, 'Bash commands executed on node', ''] }]],
+          expected_stdout: 'Bash commands executed on node'
+        ) do
+          test_connector.remote_bash(SecretString.new('bash_cmd.bash', silenced_str: '__INVALID_BASH__'))
+        end
+      end
+
       it 'executes bash commands remotely with timeout' do
         with_test_platform_for_remote_testing(
           expected_cmds: [
@@ -88,6 +97,25 @@ describe HybridPlatformsConductor::ActionsExecutor do
         end
       end
 
+      it 'executes really big bash commands remotely using a SecretString' do
+        cmd = "echo #{'1' * 131_060}"
+        with_test_platform_for_remote_testing(
+          expected_cmds: [
+            [
+              %r{.+/hpc_temp_cmds_.+\.sh$},
+              proc do |received_cmd|
+                expect(File.read(received_cmd)).to match(%r{.+/ssh hpc\.node /bin/bash <<'HPC_EOF'\n#{Regexp.escape(cmd)}\nHPC_EOF})
+                [0, 'Bash commands executed on node', '']
+              end
+            ]
+          ],
+          expected_stdout: 'Bash commands executed on node'
+        ) do
+          # Use an argument that exceeds the max arg length limit
+          test_connector.remote_bash(SecretString.new(cmd, silenced_str: '__INVALID_BASH__'))
+        end
+      end
+
       it 'copies files remotely with sudo' do
         with_test_platform_for_remote_testing(
           expected_cmds: [
@@ -153,6 +181,16 @@ describe HybridPlatformsConductor::ActionsExecutor do
         end
       end
 
+      it 'executes bash commands remotely without Session Exec capabilities using a SecretString' do
+        with_test_platform_for_remote_testing(
+          expected_cmds: [[%r{^\{ cat \| .+/ssh hpc\.node -T; \} <<'HPC_EOF'\nbash_cmd.bash\nHPC_EOF$}, proc { [0, 'Bash commands executed on node', ''] }]],
+          expected_stdout: 'Bash commands executed on node',
+          session_exec: false
+        ) do
+          test_connector.remote_bash(SecretString.new('bash_cmd.bash', silenced_str: '__INVALID_BASH__'))
+        end
+      end
+
       it 'copies files remotely without Session Exec capabilities' do
         with_test_platform_for_remote_testing(
           expected_cmds: [

data/spec/hybrid_platforms_conductor_test/api/cmd_runner_spec.rb

@@ -7,6 +7,13 @@ describe HybridPlatformsConductor::CmdRunner do
     end
   end
 
+  it 'runs a simple bash command in a SecretString' do
+    with_repository do |repository|
+      test_cmd_runner.run_cmd SecretString.new("echo TestContent >#{repository}/test_file", silenced_str: '__INVALID_BASH__')
+      expect(File.read("#{repository}/test_file")).to eq "TestContent\n"
+    end
+  end
+
   it 'runs a simple bash command and returns exit code, stdout and stderr correctly' do
     with_repository do
       expect(test_cmd_runner.run_cmd('echo TestStderr 1>&2 ; echo TestStdout')).to eq [0, "TestStdout\n", "TestStderr\n"]
@@ -20,6 +27,13 @@ describe HybridPlatformsConductor::CmdRunner do
     end
   end
 
+  it 'runs a simple bash command and forces usage of bash in a SecretString' do
+    with_repository do
+      # Use set -o pipefail that does not work in /bin/sh
+      expect(test_cmd_runner.run_cmd(SecretString.new('set -o pipefail ; echo TestStderr 1>&2 ; echo TestStdout', silenced_str: '__INVALID_BASH__'), force_bash: true)).to eq [0, "TestStdout\n", "TestStderr\n"]
+    end
+  end
+
   it 'runs a simple bash command and logs stdout and stderr to a file' do
     with_repository do |repository|
       test_cmd_runner.run_cmd 'echo TestStderr 1>&2 ; sleep 1 ; echo TestStdout', log_to_file: "#{repository}/test_file"
@@ -47,7 +61,13 @@ describe HybridPlatformsConductor::CmdRunner do
 
   it 'runs a command in an un-bundled environment' do
     with_repository do
-      expect(test_cmd_runner.run_cmd('echo "${BUNDLE_GEMFILE}"')).to eq [0, "\n", '']
+      %w[
+        BUNDLE_GEMFILE
+        GEM_HOME
+        RUBYOPT
+      ].each do |var_to_check|
+        expect(test_cmd_runner.run_cmd("echo \"${#{var_to_check}}\"")).to eq [0, "\n", '']
+      end
     end
   end
 

data/spec/hybrid_platforms_conductor_test/api/config_spec.rb

@@ -19,10 +19,12 @@ describe HybridPlatformsConductor::Config do
   end
 
   it 'returns several defined OS images' do
-    with_platforms '
-      os_image :image_1, \'/path/to/image_1\'
-      os_image :image_2, \'/path/to/image_2\'
-    ' do
+    with_platforms(
+      <<~EO_CONFIG
+        os_image :image_1, '/path/to/image_1'
+        os_image :image_2, '/path/to/image_2'
+      EO_CONFIG
+    ) do
       expect(test_config.known_os_images.sort).to eq %i[image_1 image_2].sort
     end
   end
@@ -49,11 +51,13 @@ describe HybridPlatformsConductor::Config do
   end
 
   it 'includes several configuration files' do
-    with_platforms '
-      os_image :image_1, \'/path/to/image_1\'
-      include_config_from "#{__dir__}/my_conf_1.rb"
-      include_config_from "#{__dir__}/my_conf_2.rb"
-    ' do |hybrid_platforms_dir|
+    with_platforms(
+      <<~'EO_CONFIG'
+        os_image :image_1, '/path/to/image_1'
+        include_config_from "#{__dir__}/my_conf_1.rb"
+        include_config_from "#{__dir__}/my_conf_2.rb"
+      EO_CONFIG
+    ) do |hybrid_platforms_dir|
       File.write("#{hybrid_platforms_dir}/my_conf_1.rb", <<~'EO_CONFIG')
         os_image :image_4, '/path/to/image_4'
         include_config_from "#{__dir__}/my_conf_3.rb"
@@ -65,9 +69,7 @@ describe HybridPlatformsConductor::Config do
   end
 
   it 'applies nodes specific configuration to all nodes by default' do
-    with_platforms '
-      expect_tests_to_fail :my_test, \'Failure reason\'
-    ' do
+    with_platforms 'expect_tests_to_fail :my_test, \'Failure reason\'' do
       expect(test_config.expected_failures).to eq [
         {
           nodes_selectors_stack: [],
@@ -79,12 +81,14 @@ describe HybridPlatformsConductor::Config do
   end
 
   it 'filters nodes specific configuration to nodes sets in a scope' do
-    with_platforms '
-      for_nodes(%w[node1 node2 node3]) do
-        expect_tests_to_fail :my_test_1, \'Failure reason 1\'
-      end
-      expect_tests_to_fail :my_test_2, \'Failure reason 2\'
-    ' do
+    with_platforms(
+      <<~EO_CONFIG
+        for_nodes(%w[node1 node2 node3]) do
+          expect_tests_to_fail :my_test_1, 'Failure reason 1'
+        end
+        expect_tests_to_fail :my_test_2, 'Failure reason 2'
+      EO_CONFIG
+    ) do
       sort_proc = proc { |expected_failure_info| expected_failure_info[:reason] }
       expect(test_config.expected_failures.sort_by(&sort_proc)).to eq [
         {
@@ -102,14 +106,16 @@ describe HybridPlatformsConductor::Config do
   end
 
   it 'filters nodes specific configuration in a scoped stack' do
-    with_platforms '
-      for_nodes(%w[node1 node2 node3]) do
-        expect_tests_to_fail :my_test_1, \'Failure reason 1\'
-        for_nodes(%w[node2 node3 node4]) do
-          expect_tests_to_fail :my_test_2, \'Failure reason 2\'
+    with_platforms(
+      <<~EO_CONFIG
+        for_nodes(%w[node1 node2 node3]) do
+          expect_tests_to_fail :my_test_1, 'Failure reason 1'
+          for_nodes(%w[node2 node3 node4]) do
+            expect_tests_to_fail :my_test_2, 'Failure reason 2'
+          end
         end
-      end
-    ' do
+      EO_CONFIG
+    ) do
       sort_proc = proc { |expected_failure_info| expected_failure_info[:reason] }
       expect(test_config.expected_failures.sort_by(&sort_proc)).to eq [
         {
@@ -127,13 +133,15 @@ describe HybridPlatformsConductor::Config do
   end
 
   it 'returns the expected failures correctly' do
-    with_platforms '
-      expect_tests_to_fail :my_test_1, \'Failure reason 1\'
-      expect_tests_to_fail %i[my_test_2 my_test_3], \'Failure reason 23\'
-      for_nodes(%w[node1 node2 node3]) do
-        expect_tests_to_fail :my_test_4, \'Failure reason 4\'
-      end
-    ' do
+    with_platforms(
+      <<~EO_CONFIG
+        expect_tests_to_fail :my_test_1, 'Failure reason 1'
+        expect_tests_to_fail %i[my_test_2 my_test_3], 'Failure reason 23'
+        for_nodes(%w[node1 node2 node3]) do
+          expect_tests_to_fail :my_test_4, 'Failure reason 4'
+        end
+      EO_CONFIG
+    ) do
       sort_proc = proc { |expected_failure_info| expected_failure_info[:reason] }
       expect(test_config.expected_failures.sort_by(&sort_proc)).to eq [
         {
@@ -155,47 +163,15 @@ describe HybridPlatformsConductor::Config do
     end
   end
 
-  it 'returns the retriable errors correctly' do
-    with_platforms '
-      retry_deploy_for_errors_on_stdout \'Retry stdout global\'
-      retry_deploy_for_errors_on_stderr [
-        \'Retry stderr global\',
-        /.+Retry stderr regexp global/
-      ]
-      for_nodes(%w[node1 node2 node3]) do
-        retry_deploy_for_errors_on_stdout \'Retry stdout nodes\'
-        retry_deploy_for_errors_on_stderr \'Retry stderr nodes\'
-      end
-    ' do
-      sort_proc = proc { |retriable_error_info| ((retriable_error_info[:errors_on_stdout] || []) + (retriable_error_info[:errors_on_stderr] || [])).first.to_s }
-      expect(test_config.retriable_errors.sort_by(&sort_proc)).to eq [
-        {
-          nodes_selectors_stack: [],
-          errors_on_stdout: ['Retry stdout global']
-        },
-        {
-          nodes_selectors_stack: [],
-          errors_on_stderr: ['Retry stderr global', /.+Retry stderr regexp global/]
-        },
-        {
-          nodes_selectors_stack: [%w[node1 node2 node3]],
-          errors_on_stdout: ['Retry stdout nodes']
-        },
-        {
-          nodes_selectors_stack: [%w[node1 node2 node3]],
-          errors_on_stderr: ['Retry stderr nodes']
-        }
-      ].sort_by(&sort_proc)
-    end
-  end
-
   it 'returns the deployment schedules correctly' do
-    with_platforms '
-      deployment_schedule(IceCube::Schedule.new(Time.parse(\'2020-05-01 11:22:33 UTC\')))
-      for_nodes(%w[node1 node2 node3]) do
-        deployment_schedule(IceCube::Schedule.new(Time.parse(\'2020-05-02 22:33:44 UTC\')))
-      end
-    ' do
+    with_platforms(
+      <<~EO_CONFIG
+        deployment_schedule(IceCube::Schedule.new(Time.parse('2020-05-01 11:22:33 UTC')))
+        for_nodes(%w[node1 node2 node3]) do
+          deployment_schedule(IceCube::Schedule.new(Time.parse('2020-05-02 22:33:44 UTC')))
+        end
+      EO_CONFIG
+    ) do
       sort_proc = proc { |deployment_schedule_info| deployment_schedule_info[:schedule].to_ical }
       expect(test_config.deployment_schedules.sort_by(&sort_proc)).to eq [
         {

data/spec/hybrid_platforms_conductor_test/api/credentials_spec.rb

@@ -0,0 +1,251 @@
+describe HybridPlatformsConductor::Credentials do
+
+  # Create a container class for the credential Mixin to be tested, as a plugin as credentials can be used in any plugin.
+  let(:credential_tester_class) do
+    Class.new(HybridPlatformsConductor::Plugin) do
+      include HybridPlatformsConductor::Credentials
+    end
+  end
+
+  # Expect credentials to be as a given user and password
+  #
+  # Parameters::
+  # * *expected_user* (String or nil): The expected user
+  # * *expected_password* (String or nil): The expected password
+  # * *resource* (String or nil): The resource for which we query the credentials, or nil if none [default: nil]
+  def expect_credentials_to_be(expected_user, expected_password, resource: nil)
+    creds = {}
+    password_class = nil
+    credential_tester_class.new(logger: logger, logger_stderr: logger, config: test_config).instance_exec do
+      with_credentials_for(:test_credential, resource: resource) do |user, password|
+        password_class = password.class
+        creds = {
+          user: user,
+          # We clone the value as for security reasons it is removed when exiting the block
+          password: password&.to_unprotected.clone
+        }
+      end
+    end
+    # Make sure we always return a SecretString for the password
+    expect(password_class).to be SecretString unless password_class == NilClass
+    expect(creds).to eq(
+      user: expected_user,
+      password: expected_password
+    )
+  end
+
+  it 'returns no credentials when they are not set' do
+    with_platforms '' do
+      # Check that .netrc won't be read
+      expect(::Netrc).not_to receive(:read)
+      expect_credentials_to_be nil, nil
+    end
+  end
+
+  it 'returns credentials taken from environment variables' do
+    with_platforms '' do
+      ENV['hpc_user_for_test_credential'] = 'env_test_user'
+      ENV['hpc_password_for_test_credential'] = 'env_test_password'
+      begin
+        # Check that .netrc won't be read
+        expect(::Netrc).not_to receive(:read)
+        expect_credentials_to_be 'env_test_user', 'env_test_password'
+      ensure
+        ENV.delete('hpc_user_for_test_credential')
+        ENV.delete('hpc_password_for_test_credential')
+      end
+    end
+  end
+
+  it 'erases the value of the password taken from environment variable after usage' do
+    with_platforms '' do
+      ENV['hpc_user_for_test_credential'] = 'env_test_user'
+      ENV['hpc_password_for_test_credential'] = 'env_test_password'
+      begin
+        leaked_password = nil
+        credential_tester_class.new(logger: logger, logger_stderr: logger, config: test_config).instance_exec do
+          with_credentials_for(:test_credential) do |_user, password|
+            leaked_password = password
+          end
+        end
+        expect(leaked_password.to_unprotected).to eq "\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00"
+      ensure
+        ENV.delete('hpc_user_for_test_credential')
+        ENV.delete('hpc_password_for_test_credential')
+      end
+    end
+  end
+
+  it 'returns credentials taken from .netrc when a resource is specified' do
+    with_platforms '' do
+      expect(::Netrc).to receive(:read) do
+        mocked_netrc = instance_double(::Netrc)
+        expect(mocked_netrc).to receive(:[]).with('my_domain.com').and_return %w[test_user test_password]
+        expect(mocked_netrc).to receive(:instance_variable_get).with(:@data).and_return []
+        mocked_netrc
+      end
+      expect_credentials_to_be 'test_user', 'test_password', resource: 'http://My_Domain.com/path/to/resource'
+    end
+  end
+
+  it 'returns credentials taken from .netrc when a non-URL resource is specified' do
+    with_platforms '' do
+      expect(::Netrc).to receive(:read) do
+        mocked_netrc = instance_double(::Netrc)
+        expect(mocked_netrc).to receive(:[]).with('This is:not/ a URL!').and_return %w[test_user test_password]
+        expect(mocked_netrc).to receive(:instance_variable_get).with(:@data).and_return []
+        mocked_netrc
+      end
+      expect_credentials_to_be 'test_user', 'test_password', resource: 'This is:not/ a URL!'
+    end
+  end
+
+  it 'erases the value of the password taken from netrc after usage' do
+    with_platforms '' do
+      netrc_data = [['mocked_data']]
+      expect(::Netrc).to receive(:read) do
+        mocked_netrc = instance_double(::Netrc)
+        expect(mocked_netrc).to receive(:[]).with('my_domain.com').and_return %w[test_user test_password]
+        expect(mocked_netrc).to receive(:instance_variable_get).with(:@data).and_return netrc_data
+        mocked_netrc
+      end
+      leaked_password = nil
+      credential_tester_class.new(logger: logger, logger_stderr: logger, config: test_config).instance_exec do
+        with_credentials_for(:test_credential, resource: 'http://My_Domain.com/path/to/resource') do |_user, password|
+          leaked_password = password
+        end
+      end
+      expect(leaked_password).to eq "\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00"
+      expect(netrc_data).to eq [['GotYou!!!' * 100]]
+    end
+  end
+
+  it 'returns credentials taken from config' do
+    with_platforms(
+      <<~'EO_CONFIG'
+        credentials_for(:test_credential) do |resource, requester|
+          requester.call "user_for_#{resource}", "password_for_#{resource}"
+        end
+      EO_CONFIG
+    ) do
+      # Check that netrc is not called when config is used, and that env vars are ignored
+      ENV['hpc_user_for_test_credential'] = 'env_test_user'
+      ENV['hpc_password_for_test_credential'] = 'env_test_password'
+      begin
+        # Check that .netrc won't be read
+        expect(::Netrc).not_to receive(:read)
+        expect_credentials_to_be 'user_for_', 'password_for_'
+      ensure
+        ENV.delete('hpc_user_for_test_credential')
+        ENV.delete('hpc_password_for_test_credential')
+      end
+    end
+  end
+
+  it 'returns credentials taken from config for a given resource' do
+    with_platforms(
+      <<~'EO_CONFIG'
+        credentials_for(:test_credential) do |resource, requester|
+          requester.call "user_for_#{resource}", "password_for_#{resource}"
+        end
+      EO_CONFIG
+    ) do
+      # Check that netrc is not called when config is used, and that env vars are ignored
+      ENV['hpc_user_for_test_credential'] = 'env_test_user'
+      ENV['hpc_password_for_test_credential'] = 'env_test_password'
+      begin
+        # Check that .netrc won't be read
+        expect(::Netrc).not_to receive(:read)
+        expect_credentials_to_be 'user_for_test_resource', 'password_for_test_resource', resource: 'test_resource'
+      ensure
+        ENV.delete('hpc_user_for_test_credential')
+        ENV.delete('hpc_password_for_test_credential')
+      end
+    end
+  end
+
+  it 'returns credentials taken from config for a given resource even when they are nil' do
+    with_platforms(
+      <<~'EO_CONFIG'
+        credentials_for(:test_credential) do |resource, requester|
+          requester.call nil, nil
+        end
+      EO_CONFIG
+    ) do
+      # Check that netrc is not called when config is used, and that env vars are ignored
+      ENV['hpc_user_for_test_credential'] = 'env_test_user'
+      ENV['hpc_password_for_test_credential'] = 'env_test_password'
+      begin
+        # Check that .netrc won't be read
+        expect(::Netrc).not_to receive(:read)
+        expect_credentials_to_be nil, nil, resource: 'test_resource'
+      ensure
+        ENV.delete('hpc_user_for_test_credential')
+        ENV.delete('hpc_password_for_test_credential')
+      end
+    end
+  end
+
+  it 'returns credentials taken from config after filtering the resource name' do
+    with_platforms(
+      <<~'EO_CONFIG'
+        credentials_for(:test_credential, resource: 'another_resource') do |resource, requester|
+          requester.call "wrong_user_for_#{resource}", "wrong_password_for_#{resource}"
+        end
+        credentials_for(:test_credential, resource: /test_.*/) do |resource, requester|
+          requester.call "wrong_user_for_#{resource}", "wrong_password_for_#{resource}"
+        end
+        credentials_for(:test_credential, resource: /_resource/) do |resource, requester|
+          requester.call "correct_user_for_#{resource}", "correct_password_for_#{resource}"
+        end
+        credentials_for(:test_credential, resource: 'test_resource2') do |resource, requester|
+          requester.call "wrong_user_for_#{resource}", "wrong_password_for_#{resource}"
+        end
+      EO_CONFIG
+    ) do
+      expect_credentials_to_be 'correct_user_for_test_resource', 'correct_password_for_test_resource', resource: 'test_resource'
+    end
+  end
+
+  it 'returns credentials taken from config after filtering the resource name when no resource is given' do
+    with_platforms(
+      <<~'EO_CONFIG'
+        credentials_for(:test_credential, resource: 'another_resource') do |resource, requester|
+          requester.call "wrong_user_for_#{resource}", "wrong_password_for_#{resource}"
+        end
+        credentials_for(:test_credential, resource: /test_.*/) do |resource, requester|
+          requester.call "wrong_user_for_#{resource}", "wrong_password_for_#{resource}"
+        end
+        credentials_for(:test_credential) do |resource, requester|
+          requester.call "correct_user_for_#{resource}", "correct_password_for_#{resource}"
+        end
+        credentials_for(:test_credential, resource: /_resource/) do |resource, requester|
+          requester.call "wrong_user_for_#{resource}", "wrong_password_for_#{resource}"
+        end
+        credentials_for(:test_credential, resource: 'test_resource2') do |resource, requester|
+          requester.call "wrong_user_for_#{resource}", "wrong_password_for_#{resource}"
+        end
+      EO_CONFIG
+    ) do
+      expect_credentials_to_be 'correct_user_for_', 'correct_password_for_'
+    end
+  end
+
+  it 'fails if the requester is not called from config' do
+    with_platforms(
+      <<~'EO_CONFIG'
+        credentials_for(:test_credential) do |resource, requester|
+        end
+      EO_CONFIG
+    ) do
+      expect do
+        credential_tester_class.new(logger: logger, logger_stderr: logger, config: test_config).instance_exec do
+          with_credentials_for(:test_credential) do |_user, _password|
+            nil
+          end
+        end
+      end.to raise_error 'Requester not called by the credentials provider for test_credential (resource: ) - Please check the credentials_for code in your configuration.'
+    end
+  end
+
+end