hybrid_platforms_conductor 33.4.0 → 33.7.1
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/CHANGELOG.md +47 -0
- data/README.md +5 -5
- data/docs/config_dsl.md +7 -5
- data/docs/plugins/cmdb/host_keys.md +3 -1
- data/docs/plugins/connector/ssh.md +1 -0
- data/lib/hybrid_platforms_conductor/actions_executor.rb +29 -1
- data/lib/hybrid_platforms_conductor/bitbucket.rb +2 -2
- data/lib/hybrid_platforms_conductor/cmd_runner.rb +4 -4
- data/lib/hybrid_platforms_conductor/config.rb +2 -0
- data/lib/hybrid_platforms_conductor/confluence.rb +2 -2
- data/lib/hybrid_platforms_conductor/connector.rb +5 -2
- data/lib/hybrid_platforms_conductor/credentials.rb +20 -12
- data/lib/hybrid_platforms_conductor/deployer.rb +5 -7
- data/lib/hybrid_platforms_conductor/github.rb +1 -1
- data/lib/hybrid_platforms_conductor/hpc_plugins/action/bash.rb +1 -1
- data/lib/hybrid_platforms_conductor/hpc_plugins/action/remote_bash.rb +27 -17
- data/lib/hybrid_platforms_conductor/hpc_plugins/cmdb/host_keys.rb +13 -12
- data/lib/hybrid_platforms_conductor/hpc_plugins/connector/local.rb +6 -4
- data/lib/hybrid_platforms_conductor/hpc_plugins/connector/my_connector.rb.sample +1 -1
- data/lib/hybrid_platforms_conductor/hpc_plugins/connector/ssh.rb +37 -25
- data/lib/hybrid_platforms_conductor/hpc_plugins/log/remote_fs.rb +5 -6
- data/lib/hybrid_platforms_conductor/hpc_plugins/platform_handler/serverless_chef.rb +23 -14
- data/lib/hybrid_platforms_conductor/hpc_plugins/provisioner/docker.rb +1 -1
- data/lib/hybrid_platforms_conductor/hpc_plugins/provisioner/proxmox.rb +3 -2
- data/lib/hybrid_platforms_conductor/hpc_plugins/secrets_reader/keepass.rb +1 -1
- data/lib/hybrid_platforms_conductor/hpc_plugins/test/check_deploy_and_idempotence.rb +17 -3
- data/lib/hybrid_platforms_conductor/hpc_plugins/test/deploy_removes_root_access.rb +30 -10
- data/lib/hybrid_platforms_conductor/hpc_plugins/test/file_system.rb +1 -1
- data/lib/hybrid_platforms_conductor/hpc_plugins/test/hostname.rb +1 -2
- data/lib/hybrid_platforms_conductor/hpc_plugins/test/idempotence.rb +1 -1
- data/lib/hybrid_platforms_conductor/hpc_plugins/test/ip.rb +1 -2
- data/lib/hybrid_platforms_conductor/hpc_plugins/test/jenkins_ci_conf.rb +1 -1
- data/lib/hybrid_platforms_conductor/hpc_plugins/test/jenkins_ci_masters_ok.rb +2 -2
- data/lib/hybrid_platforms_conductor/hpc_plugins/test/local_users.rb +1 -2
- data/lib/hybrid_platforms_conductor/hpc_plugins/test/mounts.rb +1 -2
- data/lib/hybrid_platforms_conductor/hpc_plugins/test/orphan_files.rb +1 -2
- data/lib/hybrid_platforms_conductor/hpc_plugins/test/spectre.rb +1 -1
- data/lib/hybrid_platforms_conductor/hpc_plugins/test/vulnerabilities.rb +1 -2
- data/lib/hybrid_platforms_conductor/logger_helpers.rb +17 -0
- data/lib/hybrid_platforms_conductor/test.rb +21 -7
- data/lib/hybrid_platforms_conductor/tests_runner.rb +7 -6
- data/lib/hybrid_platforms_conductor/thycotic.rb +2 -2
- data/lib/hybrid_platforms_conductor/version.rb +1 -1
- data/spec/hybrid_platforms_conductor_test/api/actions_executor/actions/bash_spec.rb +15 -0
- data/spec/hybrid_platforms_conductor_test/api/actions_executor/actions/remote_bash_spec.rb +32 -0
- data/spec/hybrid_platforms_conductor_test/api/actions_executor/connectors/local/remote_actions_spec.rb +87 -0
- data/spec/hybrid_platforms_conductor_test/api/actions_executor/connectors/ssh/connections_spec.rb +30 -0
- data/spec/hybrid_platforms_conductor_test/api/actions_executor/connectors/ssh/global_helpers_spec.rb +10 -0
- data/spec/hybrid_platforms_conductor_test/api/actions_executor/connectors/ssh/remote_actions_spec.rb +38 -0
- data/spec/hybrid_platforms_conductor_test/api/actions_executor/helpers_spec.rb +195 -0
- data/spec/hybrid_platforms_conductor_test/api/cmd_runner_spec.rb +14 -0
- data/spec/hybrid_platforms_conductor_test/api/config_spec.rb +11 -0
- data/spec/hybrid_platforms_conductor_test/api/credentials_spec.rb +8 -4
- data/spec/hybrid_platforms_conductor_test/api/deployer/log_plugins/remote_fs_spec.rb +215 -0
- data/spec/hybrid_platforms_conductor_test/api/nodes_handler/cmdbs/host_keys_spec.rb +49 -10
- data/spec/hybrid_platforms_conductor_test/api/platform_handlers/serverless_chef/services_deployment_spec.rb +64 -16
- data/spec/hybrid_platforms_conductor_test/helpers/connector_ssh_helpers.rb +5 -3
- data/spec/hybrid_platforms_conductor_test/serverless_chef_repositories/1_local_node/chef_versions.yml +3 -0
- data/spec/hybrid_platforms_conductor_test/serverless_chef_repositories/1_local_node/nodes/node.json +15 -0
- data/spec/hybrid_platforms_conductor_test/serverless_chef_repositories/1_local_node/policyfiles/test_policy.rb +3 -0
- data/spec/hybrid_platforms_conductor_test/shared_examples/deployer.rb +134 -0
- data/spec/hybrid_platforms_conductor_test/test_connector.rb +2 -2
- metadata +20 -2
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA256:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: b7fbebabd55f289dc72c1882d4d02b1898035593a78cc1396e63e9f3f97405b4
|
4
|
+
data.tar.gz: 1b0394745277f497c7083828be1b813fe6ff60c0c6c14f22a0808e88c6d2911f
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: ae3267357ec72113fc39933fd63572dce1017d27162248e6416d7621f543701e20ddc5d155eb4cb7785f572a975304399a6a512940d8c8017af9389251cbc49b
|
7
|
+
data.tar.gz: b8eac4d69391f5ed8a5ce9e7c495549cbfac878a9d8847e568f17989a1ed944da7a16bcc9deeebacbee522dc6cd56b937baa0dc50b2ed1070f5a247a090bee9c
|
data/CHANGELOG.md
CHANGED
@@ -1,3 +1,50 @@
|
|
1
|
+
# [v33.7.1](https://github.com/sweet-delights/hybrid-platforms-conductor/compare/v33.7.0...v33.7.1) (2021-07-09 17:17:18)
|
2
|
+
|
3
|
+
## Global changes
|
4
|
+
### Patches
|
5
|
+
|
6
|
+
* [[Bugfix(platform_handler_serverless_chef)] [#93] Make sure chef runs use colors in their output](https://github.com/sweet-delights/hybrid-platforms-conductor/commit/14352425115cf46b92e4c747b2e34e3734314288)
|
7
|
+
|
8
|
+
## Changes for platform_handler_serverless_chef
|
9
|
+
### Patches
|
10
|
+
|
11
|
+
* [[Bugfix(platform_handler_serverless_chef)] [#93] Make sure chef runs use colors in their output](https://github.com/sweet-delights/hybrid-platforms-conductor/commit/14352425115cf46b92e4c747b2e34e3734314288)
|
12
|
+
|
13
|
+
# [v33.7.0](https://github.com/sweet-delights/hybrid-platforms-conductor/compare/v33.6.0...v33.7.0) (2021-07-09 16:32:25)
|
14
|
+
|
15
|
+
### Features
|
16
|
+
|
17
|
+
* [[Feature] [#91] Expose log_debug? method to the config DSL to adapt configuration in case we are in debug mode](https://github.com/sweet-delights/hybrid-platforms-conductor/commit/87f6f51df2d20c7861f9ddd59a8d7f68c27cdd74)
|
18
|
+
|
19
|
+
# [v33.6.0](https://github.com/sweet-delights/hybrid-platforms-conductor/compare/v33.5.1...v33.6.0) (2021-07-07 15:45:12)
|
20
|
+
|
21
|
+
## Global changes
|
22
|
+
### Patches
|
23
|
+
|
24
|
+
* [[Feature(connector_ssh)] [Feature(cmdb_host_keys)] [#78] Way to configure the SSH port being used with the ssh_port metadata](https://github.com/sweet-delights/hybrid-platforms-conductor/commit/f1be319eae30eedcf7f901d3ce0a14f6f4d1f2ea)
|
25
|
+
|
26
|
+
## Changes for cmdb_host_keys
|
27
|
+
### Features
|
28
|
+
|
29
|
+
* [[Feature(connector_ssh)] [Feature(cmdb_host_keys)] [#78] Way to configure the SSH port being used with the ssh_port metadata](https://github.com/sweet-delights/hybrid-platforms-conductor/commit/f1be319eae30eedcf7f901d3ce0a14f6f4d1f2ea)
|
30
|
+
|
31
|
+
## Changes for connector_ssh
|
32
|
+
### Features
|
33
|
+
|
34
|
+
* [[Feature(connector_ssh)] [Feature(cmdb_host_keys)] [#78] Way to configure the SSH port being used with the ssh_port metadata](https://github.com/sweet-delights/hybrid-platforms-conductor/commit/f1be319eae30eedcf7f901d3ce0a14f6f4d1f2ea)
|
35
|
+
|
36
|
+
# [v33.5.1](https://github.com/sweet-delights/hybrid-platforms-conductor/compare/v33.5.0...v33.5.1) (2021-07-07 12:01:32)
|
37
|
+
|
38
|
+
### Patches
|
39
|
+
|
40
|
+
* [[Bugfix] [#87] Make sure local nodes and root accounts are taken into account when getting sudo prefixes](https://github.com/sweet-delights/hybrid-platforms-conductor/commit/cb626f1c99a6f1a95c9c884c03bf3fd71045259c)
|
41
|
+
|
42
|
+
# [v33.5.0](https://github.com/sweet-delights/hybrid-platforms-conductor/compare/v33.4.0...v33.5.0) (2021-07-07 11:03:01)
|
43
|
+
|
44
|
+
### Features
|
45
|
+
|
46
|
+
* [[Feature] [Security] [#80] Use SecretString for credentials to avoid passwords and secrets accidental leakage](https://github.com/sweet-delights/hybrid-platforms-conductor/commit/0db5f529e539b81f86b9f618a63e62bdbd58cb38)
|
47
|
+
|
1
48
|
# [v33.4.0](https://github.com/sweet-delights/hybrid-platforms-conductor/compare/v33.3.0...v33.4.0) (2021-07-05 13:24:27)
|
2
49
|
|
3
50
|
### Features
|
data/README.md
CHANGED
@@ -84,9 +84,11 @@ Here are the various plugin categories:
|
|
84
84
|
* **[Actions](docs/plugins/action)** implement a given **action**. For example: bash code execution, ruby code execution, file transfer...
|
85
85
|
* **[Cmdbs](docs/plugins/cmdb)** parse **metadata** from various sources. For example: a database, Chef/Ansible inventory files, a configuration management database such as Consul...
|
86
86
|
* **[Connectors](docs/plugins/connector)** give ways to execute commands and transfer files on **nodes**. For example: using an SSH connection, a CLI for a Cloud provider...
|
87
|
+
* **[Logs](docs/plugins/log)** handle any deployment log storage strategy. They take deployment logs and send them wherever the plugins want.
|
87
88
|
* **[Platform handlers](docs/plugins/platform_handler)** handle any **platform** of a given **platform type**. They read **nodes**' inventory and **services** from **platforms**, and provide **actions** to deploy a **service** on a **node**.
|
88
89
|
* **[Provisioners](docs/plugins/provisioner)** provision (create, destroy, start, stop...) **nodes**. For example: using OpenShift, Proxmox, Docker, Podman...
|
89
90
|
* **[Reports](docs/plugins/report)** gather **platforms** and **nodes**' inventory information and publish them to some medium. For example: on command line, as a JSON file, on a content management system like Confluence or Mediawiki...
|
91
|
+
* **[Secrets readers](docs/plugins/secrets_reader)** reads secrets from various sources (vaults, APIs, local files...).
|
90
92
|
* **[Tests](docs/plugins/test)** define tests to be performed on **platforms** and **nodes**.
|
91
93
|
* **[Tests reports](docs/plugins/test_report)** publish tests results to some medium. For example: on command line, as a JSON file, on a content management system like Confluence or Mediawiki...
|
92
94
|
|
@@ -330,11 +332,9 @@ credentials_for(:bitbucket) do |resource, requester|
|
|
330
332
|
puts 'Input Bitbucket password...'
|
331
333
|
password = ''
|
332
334
|
$stdin.noecho { |io| io.sysread(256, password) }
|
333
|
-
|
334
|
-
|
335
|
-
requester.call 'my_bitbucket_name',
|
336
|
-
ensure
|
337
|
-
SecretString.erase(password)
|
335
|
+
password.chomp!
|
336
|
+
SecretString.protect(password) do |secret_password|
|
337
|
+
requester.call 'my_bitbucket_name', secret_password
|
338
338
|
end
|
339
339
|
end
|
340
340
|
```
|
data/docs/config_dsl.md
CHANGED
@@ -216,6 +216,10 @@ It takes the following parameters:
|
|
216
216
|
* a code block that will be called back by any process needing credentials matching the ID and resource specification.
|
217
217
|
The code block will be given both the resource name being accessed, and a requester object that needs to be given the corresponding user and password. When the requester finishes running, the credentials are not needed anymore and should be cleaned from memory to avoid vulnerabilities.
|
218
218
|
|
219
|
+
A secure way to five the password is to use a [`SecretString`](https://github.com/Muriel-Salvan/secret_string) class that will give the following guarantees:
|
220
|
+
* Avoid the password to leak inadvertently in logs, on-screen, files...
|
221
|
+
* Clear the password from memory as soon as it is not needed anymore.
|
222
|
+
|
219
223
|
Examples:
|
220
224
|
```ruby
|
221
225
|
# Using an environment variable as a password
|
@@ -228,11 +232,9 @@ credentials_for(:github) do |resource, requester|
|
|
228
232
|
puts 'Input Github password...'
|
229
233
|
password = ''
|
230
234
|
$stdin.noecho { |io| io.sysread(256, password) }
|
231
|
-
|
232
|
-
|
233
|
-
requester.call 'MyUserName',
|
234
|
-
ensure
|
235
|
-
SecretString.erase(password)
|
235
|
+
password.chomp!
|
236
|
+
SecretString.protect(password) do |secret_password|
|
237
|
+
requester.call 'MyUserName', secret_password
|
236
238
|
end
|
237
239
|
end
|
238
240
|
|
@@ -21,7 +21,9 @@ None
|
|
21
21
|
|
22
22
|
| Metadata | Type | Usage
|
23
23
|
| --- | --- | --- |
|
24
|
-
| `
|
24
|
+
| `host_ip` | `String` | Used to perform the `ssh-keyscan` |
|
25
|
+
| `hostname` | `String` | Used in place of the `host_ip` in case `host_ip` is not available |
|
26
|
+
| `ssh_port` | `Integer` | Port on which the `ssh-keyscan` will be performed (default: 22) |
|
25
27
|
|
26
28
|
## Used environment variables
|
27
29
|
|
@@ -70,6 +70,7 @@ end
|
|
70
70
|
| `host_keys` | `Array<String>` | The node's host keys used to generate a `known_hosts` file with those to avoid user confirmations when connecting. |
|
71
71
|
| `hostname` | `String` | Host name used to connect in case no IP address can be found in metadata. |
|
72
72
|
| `private_ips` | `Array<String>` | IP list to connect in case `host_ip` is not defined in metadata. |
|
73
|
+
| `ssh_port` | `Integer` | Port used to connect to SSH (default: 22). |
|
73
74
|
| `ssh_session_exec` | `Boolean` | If set to `false`, then consider that the node does not have any SSH SessionExec capabilities. This will make sure that remote command executions is done using stdin piping on interactive sessions instead of SSH commands execution. |
|
74
75
|
|
75
76
|
## Used environment variables
|
@@ -53,7 +53,8 @@ module HybridPlatformsConductor
|
|
53
53
|
logger_stderr: @logger_stderr,
|
54
54
|
config: @config,
|
55
55
|
cmd_runner: @cmd_runner,
|
56
|
-
nodes_handler: @nodes_handler
|
56
|
+
nodes_handler: @nodes_handler,
|
57
|
+
actions_executor: self
|
57
58
|
)
|
58
59
|
end
|
59
60
|
)
|
@@ -246,6 +247,33 @@ module HybridPlatformsConductor
|
|
246
247
|
@connector_plugins[connector_name]
|
247
248
|
end
|
248
249
|
|
250
|
+
# Is the access to a given node privileged?
|
251
|
+
# Take into account if remote actions are executed on a local node, and configurable sudos.
|
252
|
+
#
|
253
|
+
# Parameters::
|
254
|
+
# * *node* (String): Node on which we want privileged access
|
255
|
+
# Result::
|
256
|
+
# * Boolean: Is the access privileged?
|
257
|
+
def privileged_access?(node)
|
258
|
+
(@nodes_handler.get_local_node_of(node) ? @cmd_runner.whoami : connector(:ssh).ssh_user) == 'root'
|
259
|
+
end
|
260
|
+
|
261
|
+
# Get the sudo prefix to get privileged access.
|
262
|
+
# Take into account if remote actions are executed on a local node, and configurable sudos.
|
263
|
+
#
|
264
|
+
# Parameters::
|
265
|
+
# * *node* (String): Node on which we want privileged access
|
266
|
+
# * *forward_env* (Boolean): Do we need to forward environment in case of sudo? [default: false]
|
267
|
+
# Result::
|
268
|
+
# * String: Sudo prefix to be used (can be empty if root is being used)
|
269
|
+
def sudo_prefix(node, forward_env: false)
|
270
|
+
if privileged_access?(node)
|
271
|
+
''
|
272
|
+
else
|
273
|
+
"#{@nodes_handler.sudo_on(node)} #{forward_env ? '-E ' : ''}"
|
274
|
+
end
|
275
|
+
end
|
276
|
+
|
249
277
|
private
|
250
278
|
|
251
279
|
# Execute a list of actions for a node, and return exit codes, stdout and stderr of those actions.
|
@@ -77,7 +77,7 @@ module HybridPlatformsConductor
|
|
77
77
|
# Parameters::
|
78
78
|
# * *bitbucket_url* (String): The Bitbucket URL
|
79
79
|
# * *bitbucket_user_name* (String): Bitbucket user name to be used when querying the API
|
80
|
-
# * *bitbucket_password* (
|
80
|
+
# * *bitbucket_password* (SecretString): Bitbucket password to be used when querying the API
|
81
81
|
# * *logger* (Logger): Logger to be used [default = Logger.new(STDOUT)]
|
82
82
|
# * *logger_stderr* (Logger): Logger to be used for stderr [default = Logger.new(STDERR)]
|
83
83
|
def initialize(bitbucket_url, bitbucket_user_name, bitbucket_password, logger: Logger.new($stdout), logger_stderr: Logger.new($stderr))
|
@@ -148,7 +148,7 @@ module HybridPlatformsConductor
|
|
148
148
|
http_response = nil
|
149
149
|
loop do
|
150
150
|
begin
|
151
|
-
http_response = URI.parse(api_url).open(http_basic_authentication: [@bitbucket_user_name, @bitbucket_password])
|
151
|
+
http_response = URI.parse(api_url).open(http_basic_authentication: [@bitbucket_user_name, @bitbucket_password&.to_unprotected])
|
152
152
|
rescue
|
153
153
|
raise if retries.zero?
|
154
154
|
|
@@ -59,7 +59,7 @@ module HybridPlatformsConductor
|
|
59
59
|
# Raise an exception if the exit status is not the expected one.
|
60
60
|
#
|
61
61
|
# Parameters::
|
62
|
-
# * *cmd* (String): Command to be run
|
62
|
+
# * *cmd* (String or SecretString): Command to be run
|
63
63
|
# * *log_to_file* (String or nil): Log file capturing stdout or stderr (or nil for none). [default: nil]
|
64
64
|
# * *log_to_stdout* (Boolean): Do we send the output to stdout? [default: true]
|
65
65
|
# * *log_stdout_to_io* (IO or nil): IO to send command's stdout to, or nil for none. [default: nil]
|
@@ -108,7 +108,7 @@ module HybridPlatformsConductor
|
|
108
108
|
bash_file = nil
|
109
109
|
if force_bash
|
110
110
|
bash_file = Tempfile.new('hpc_bash')
|
111
|
-
bash_file.write(cmd)
|
111
|
+
bash_file.write(cmd.to_unprotected)
|
112
112
|
bash_file.chmod 0o700
|
113
113
|
bash_file.close
|
114
114
|
cmd = "/bin/bash -c #{bash_file.path}"
|
@@ -136,7 +136,7 @@ module HybridPlatformsConductor
|
|
136
136
|
pty: true,
|
137
137
|
timeout: timeout,
|
138
138
|
uuid: false
|
139
|
-
).run!(cmd) do |stdout, stderr|
|
139
|
+
).run!(cmd.to_unprotected) do |stdout, stderr|
|
140
140
|
stdout_queue << stdout if stdout
|
141
141
|
stderr_queue << stderr if stderr
|
142
142
|
end
|
@@ -162,7 +162,7 @@ module HybridPlatformsConductor
|
|
162
162
|
log_debug "Finished in #{elapsed} seconds with exit status #{exit_status} (#{(expected_code.include?(exit_status) ? 'success'.light_green : 'failure'.light_red).bold})"
|
163
163
|
end
|
164
164
|
unless expected_code.include?(exit_status)
|
165
|
-
error_title = "Command '#{cmd.split("\n").first}' returned error code #{exit_status} (expected #{expected_code.join(', ')})."
|
165
|
+
error_title = "Command '#{cmd.to_s.split("\n").first}' returned error code #{exit_status} (expected #{expected_code.join(', ')})."
|
166
166
|
if no_exception
|
167
167
|
# We consider the caller is responsible for logging what he wants about the details of the error (stdout and stderr)
|
168
168
|
log_error error_title
|
@@ -34,7 +34,7 @@ module HybridPlatformsConductor
|
|
34
34
|
# Parameters::
|
35
35
|
# * *confluence_url* (String): The Confluence URL
|
36
36
|
# * *confluence_user_name* (String): Confluence user name to be used when querying the API
|
37
|
-
# * *confluence_password* (
|
37
|
+
# * *confluence_password* (SecretString): Confluence password to be used when querying the API
|
38
38
|
# * *logger* (Logger): Logger to be used [default = Logger.new(STDOUT)]
|
39
39
|
# * *logger_stderr* (Logger): Logger to be used for stderr [default = Logger.new(STDERR)]
|
40
40
|
def initialize(confluence_url, confluence_user_name, confluence_password, logger: Logger.new($stdout), logger_stderr: Logger.new($stderr))
|
@@ -109,7 +109,7 @@ module HybridPlatformsConductor
|
|
109
109
|
page_url = URI.parse("#{@confluence_url}/#{api_path}")
|
110
110
|
Net::HTTP.start(page_url.host, page_url.port, use_ssl: true) do |http|
|
111
111
|
request = Net::HTTP.const_get(http_method.to_s.capitalize.to_sym).new(page_url.request_uri)
|
112
|
-
request.basic_auth @confluence_user_name, @confluence_password
|
112
|
+
request.basic_auth @confluence_user_name, @confluence_password&.to_unprotected
|
113
113
|
yield request if block_given?
|
114
114
|
response = http.request(request)
|
115
115
|
raise "Confluence page API request on #{page_url} returned an error: #{response.code}\n#{response.body}\n===== Request body =====\n#{request.body}" unless response.is_a?(Net::HTTPSuccess)
|
@@ -14,16 +14,19 @@ module HybridPlatformsConductor
|
|
14
14
|
# * *config* (Config): Config to be used. [default: Config.new]
|
15
15
|
# * *cmd_runner* (CmdRunner): Command executor to be used. [default: CmdRunner.new]
|
16
16
|
# * *nodes_handler* (NodesHandler): NodesHandler to be used. [default: NodesHandler.new]
|
17
|
+
# * *actions_executor* (ActionsExecutor): ActionsExecutor to be used. [default: ActionsExecutor.new]
|
17
18
|
def initialize(
|
18
19
|
logger: Logger.new($stdout),
|
19
20
|
logger_stderr: Logger.new($stderr),
|
20
21
|
config: Config.new,
|
21
22
|
cmd_runner: CmdRunner.new,
|
22
|
-
nodes_handler: NodesHandler.new
|
23
|
+
nodes_handler: NodesHandler.new,
|
24
|
+
actions_executor: ActionsExecutor.new
|
23
25
|
)
|
24
26
|
super(logger: logger, logger_stderr: logger_stderr, config: config)
|
25
27
|
@cmd_runner = cmd_runner
|
26
28
|
@nodes_handler = nodes_handler
|
29
|
+
@actions_executor = actions_executor
|
27
30
|
# If the connector has an initializer, use it
|
28
31
|
init if respond_to?(:init)
|
29
32
|
end
|
@@ -67,7 +70,7 @@ module HybridPlatformsConductor
|
|
67
70
|
# Handle the redirection of standard output and standard error to file and stdout depending on the context of the run.
|
68
71
|
#
|
69
72
|
# Parameters::
|
70
|
-
# * *cmd* (String): The command to be run
|
73
|
+
# * *cmd* (String or SecretString): The command to be run
|
71
74
|
# * *force_bash* (Boolean): If true, then make sure command is invoked with bash instead of sh [default: false]
|
72
75
|
# Result::
|
73
76
|
# * Integer: Exit code
|
@@ -1,4 +1,5 @@
|
|
1
1
|
require 'netrc'
|
2
|
+
require 'secret_string'
|
2
3
|
require 'uri'
|
3
4
|
require 'hybrid_platforms_conductor/logger_helpers'
|
4
5
|
|
@@ -65,8 +66,8 @@ module HybridPlatformsConductor
|
|
65
66
|
# * Proc: Client code called with credentials provided
|
66
67
|
# * Parameters::
|
67
68
|
# * *user* (String or nil): User name, or nil if none
|
68
|
-
# * *password* (
|
69
|
-
# !!! Never
|
69
|
+
# * *password* (SecretString or nil): Password, or nil if none.
|
70
|
+
# !!! Never clone this password in a scope broader than the client code itself !!!
|
70
71
|
def with_credentials_for(id, resource: nil)
|
71
72
|
# Get the credentials provider
|
72
73
|
provider = nil
|
@@ -81,7 +82,8 @@ module HybridPlatformsConductor
|
|
81
82
|
|
82
83
|
provider ||= proc do |requested_resource, requester|
|
83
84
|
# Check environment variables
|
84
|
-
user = ENV["hpc_user_for_#{id}"]
|
85
|
+
user = ENV["hpc_user_for_#{id}"]
|
86
|
+
# Clone the password as we are going to treat it as a secret string that will be wiped out
|
85
87
|
password = ENV["hpc_password_for_#{id}"].dup
|
86
88
|
if user.nil? || user.empty? || password.nil? || password.empty?
|
87
89
|
log_debug "[ Credentials for #{id} ] - Credentials not found from environment variables."
|
@@ -103,6 +105,7 @@ module HybridPlatformsConductor
|
|
103
105
|
# TODO: Add more credentials source if needed here
|
104
106
|
log_warn "[ Credentials for #{id} ] - Unable to get credentials for #{id} (Resource: #{requested_resource})."
|
105
107
|
else
|
108
|
+
# Clone in memory as we are going to wipe out ::Netrc's memory
|
106
109
|
user = netrc_user.dup
|
107
110
|
password = netrc_password.dup
|
108
111
|
log_debug "[ Credentials for #{id} ] - Credentials retrieved from .netrc using #{requested_resource}."
|
@@ -115,19 +118,18 @@ module HybridPlatformsConductor
|
|
115
118
|
data_string.replace('GotYou!!!' * 100)
|
116
119
|
end
|
117
120
|
end
|
118
|
-
# We do this assignment on purpose so that GC can remove sensitive data later
|
119
|
-
# rubocop:disable Lint/UselessAssignment
|
120
|
-
netrc = nil
|
121
|
-
# rubocop:enable Lint/UselessAssignment
|
122
121
|
end
|
123
122
|
end
|
124
123
|
else
|
125
124
|
log_debug "[ Credentials for #{id} ] - Credentials retrieved from environment variables."
|
126
125
|
end
|
127
|
-
|
128
|
-
|
129
|
-
|
130
|
-
|
126
|
+
if password.nil?
|
127
|
+
requester.call user, password
|
128
|
+
else
|
129
|
+
SecretString.protect(password) do |secret_password|
|
130
|
+
requester.call user, secret_password
|
131
|
+
end
|
132
|
+
end
|
131
133
|
end
|
132
134
|
|
133
135
|
requester_called = false
|
@@ -135,7 +137,13 @@ module HybridPlatformsConductor
|
|
135
137
|
resource,
|
136
138
|
proc do |user, password|
|
137
139
|
requester_called = true
|
138
|
-
|
140
|
+
if password.is_a?(String)
|
141
|
+
SecretString.protect(password) do |secret_password|
|
142
|
+
yield user, secret_password
|
143
|
+
end
|
144
|
+
else
|
145
|
+
yield user, password
|
146
|
+
end
|
139
147
|
end
|
140
148
|
)
|
141
149
|
|
@@ -544,15 +544,13 @@ module HybridPlatformsConductor
|
|
544
544
|
# Result::
|
545
545
|
# * Hash<String, [Integer or Symbol, String, String]>: Exit status code (or Symbol in case of error or dry run), standard output and error for each node.
|
546
546
|
def deploy(services)
|
547
|
-
# Get the ssh user directly from the connector
|
548
|
-
ssh_user = @actions_executor.connector(:ssh).ssh_user
|
549
|
-
|
550
547
|
# Deploy for real
|
551
548
|
@nodes_handler.prefetch_metadata_of services.keys, :image
|
552
549
|
outputs = @actions_executor.execute_actions(
|
553
550
|
services.map do |node, node_services|
|
554
551
|
image_id = @nodes_handler.get_image_of(node)
|
555
|
-
|
552
|
+
need_sudo = !@actions_executor.privileged_access?(node)
|
553
|
+
sudo = @actions_executor.sudo_prefix(node)
|
556
554
|
# Install corporate certificates if present
|
557
555
|
certificate_actions =
|
558
556
|
if @local_environment && ENV['hpc_certificates']
|
@@ -568,7 +566,7 @@ module HybridPlatformsConductor
|
|
568
566
|
{
|
569
567
|
scp: {
|
570
568
|
ENV['hpc_certificates'] => '/usr/local/share/ca-certificates',
|
571
|
-
:sudo =>
|
569
|
+
:sudo => need_sudo
|
572
570
|
},
|
573
571
|
remote_bash: "#{sudo}update-ca-certificates"
|
574
572
|
}
|
@@ -584,7 +582,7 @@ module HybridPlatformsConductor
|
|
584
582
|
cert_file,
|
585
583
|
'/etc/pki/ca-trust/source/anchors'
|
586
584
|
]
|
587
|
-
end.to_h.merge(sudo:
|
585
|
+
end.to_h.merge(sudo: need_sudo),
|
588
586
|
remote_bash: [
|
589
587
|
"#{sudo}update-ca-trust enable",
|
590
588
|
"#{sudo}update-ca-trust extract"
|
@@ -619,7 +617,7 @@ module HybridPlatformsConductor
|
|
619
617
|
services.keys.map do |node|
|
620
618
|
[
|
621
619
|
node,
|
622
|
-
{ remote_bash: "#{
|
620
|
+
{ remote_bash: "#{@actions_executor.sudo_prefix(node)}./mutex_dir unlock /tmp/hybrid_platforms_conductor_deploy_lock" }
|
623
621
|
]
|
624
622
|
end.to_h,
|
625
623
|
timeout: 10,
|
@@ -23,7 +23,7 @@ module HybridPlatformsConductor
|
|
23
23
|
c.api_endpoint = repo_info[:url]
|
24
24
|
end
|
25
25
|
with_credentials_for(:github, resource: repo_info[:url]) do |_github_user, github_token|
|
26
|
-
client = Octokit::Client.new(access_token: github_token)
|
26
|
+
client = Octokit::Client.new(access_token: github_token&.to_unprotected)
|
27
27
|
(repo_info[:repos] == :all ? client.repositories(repo_info[:user]).map { |repo| repo[:name] } : repo_info[:repos]).each do |name|
|
28
28
|
yield client, {
|
29
29
|
name: name,
|
@@ -15,30 +15,30 @@ module HybridPlatformsConductor
|
|
15
15
|
#
|
16
16
|
# Parameters::
|
17
17
|
# * *remote_bash* (Array or Object): List of commands (or single command) to be executed. Each command can be the following:
|
18
|
-
# * String: Simple bash command.
|
18
|
+
# * String or SecretString: Simple bash command.
|
19
19
|
# * Hash<Symbol, Object>: Information about the commands to execute. Can have the following properties:
|
20
|
-
# * *commands* (Array<String> or String): List of bash commands to execute (can be a single one) [default: ''].
|
20
|
+
# * *commands* (Array<String or SecretString> or String or SecretString): List of bash commands to execute (can be a single one) [default: ''].
|
21
21
|
# * *file* (String): Name of file from which commands should be taken [optional].
|
22
|
-
# * *env* (Hash<String, String>): Environment variables to be set before executing those commands [default: {}].
|
22
|
+
# * *env* (Hash<String, String or SecretString>): Environment variables to be set before executing those commands [default: {}].
|
23
23
|
def setup(remote_bash)
|
24
24
|
# Normalize the parameters.
|
25
25
|
# Array< Hash<Symbol,Object> >: Simple array of info:
|
26
|
-
# * *commands* (Array<String>): List of bash commands to execute.
|
27
|
-
# * *env* (Hash<String, String>): Environment variables to be set before executing those commands.
|
26
|
+
# * *commands* (Array<String or SecretString>): List of bash commands to execute.
|
27
|
+
# * *env* (Hash<String, String or SecretString>): Environment variables to be set before executing those commands.
|
28
28
|
@remote_bash = (remote_bash.is_a?(Array) ? remote_bash : [remote_bash]).map do |cmd_info|
|
29
|
-
if cmd_info.is_a?(
|
30
|
-
{
|
31
|
-
commands: [cmd_info],
|
32
|
-
env: {}
|
33
|
-
}
|
34
|
-
else
|
29
|
+
if cmd_info.is_a?(Hash)
|
35
30
|
commands = []
|
36
|
-
commands.concat(cmd_info[:commands].is_a?(
|
31
|
+
commands.concat(cmd_info[:commands].is_a?(Array) ? cmd_info[:commands] : [cmd_info[:commands]]) if cmd_info[:commands]
|
37
32
|
commands << File.read(cmd_info[:file]) if cmd_info[:file]
|
38
33
|
{
|
39
34
|
commands: commands,
|
40
35
|
env: cmd_info[:env] || {}
|
41
36
|
}
|
37
|
+
else
|
38
|
+
{
|
39
|
+
commands: [cmd_info],
|
40
|
+
env: {}
|
41
|
+
}
|
42
42
|
end
|
43
43
|
end
|
44
44
|
end
|
@@ -63,11 +63,21 @@ module HybridPlatformsConductor
|
|
63
63
|
# [API] - @stderr_io can be used to log stderr messages
|
64
64
|
# [API] - run_cmd(String) method can be used to execute a command. See CmdRunner#run_cmd to know about the result's signature.
|
65
65
|
def execute
|
66
|
-
|
67
|
-
|
68
|
-
|
69
|
-
|
70
|
-
|
66
|
+
# The commands or ENV variables can contain secrets, so make sure to protect all strings from secrets leaking
|
67
|
+
bash_cmds = @remote_bash.map do |cmd_info|
|
68
|
+
cmd_info[:env].map do |var_name, var_value|
|
69
|
+
SecretString.new("export #{var_name}='#{var_value.to_unprotected}'", silenced_str: "export #{var_name}='#{var_value}'")
|
70
|
+
end + cmd_info[:commands]
|
71
|
+
end.flatten
|
72
|
+
begin
|
73
|
+
SecretString.protect(bash_cmds.map(&:to_unprotected).join("\n"), silenced_str: bash_cmds.join("\n")) do |bash_str|
|
74
|
+
log_debug "[#{@node}] - Execute remote Bash commands \"#{bash_str}\"..."
|
75
|
+
@connector.remote_bash bash_str
|
76
|
+
end
|
77
|
+
ensure
|
78
|
+
# Make sure we erase all secret strings
|
79
|
+
bash_cmds.each(&:erase)
|
80
|
+
end
|
71
81
|
end
|
72
82
|
|
73
83
|
end
|