cem_acpt 0.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +7 -0
- data/.gitignore +11 -0
- data/.rspec +3 -0
- data/.travis.yml +6 -0
- data/CODEOWNERS +1 -0
- data/Gemfile +7 -0
- data/Gemfile.lock +68 -0
- data/README.md +146 -0
- data/Rakefile +6 -0
- data/bin/console +14 -0
- data/bin/setup +8 -0
- data/cem_acpt.gemspec +37 -0
- data/exe/cem_acpt +58 -0
- data/lib/cem_acpt/bootstrap/bootstrapper.rb +206 -0
- data/lib/cem_acpt/bootstrap/operating_system/rhel_family.rb +129 -0
- data/lib/cem_acpt/bootstrap/operating_system.rb +17 -0
- data/lib/cem_acpt/bootstrap.rb +12 -0
- data/lib/cem_acpt/context.rb +60 -0
- data/lib/cem_acpt/core_extensions.rb +111 -0
- data/lib/cem_acpt/image_name_builder.rb +104 -0
- data/lib/cem_acpt/logging.rb +193 -0
- data/lib/cem_acpt/platform/base/cmd.rb +65 -0
- data/lib/cem_acpt/platform/base.rb +78 -0
- data/lib/cem_acpt/platform/gcp/cmd.rb +313 -0
- data/lib/cem_acpt/platform/gcp/compute.rb +327 -0
- data/lib/cem_acpt/platform/gcp.rb +85 -0
- data/lib/cem_acpt/platform/vmpooler.rb +24 -0
- data/lib/cem_acpt/platform.rb +103 -0
- data/lib/cem_acpt/puppet_helpers.rb +38 -0
- data/lib/cem_acpt/runner.rb +304 -0
- data/lib/cem_acpt/shared_objects.rb +416 -0
- data/lib/cem_acpt/spec_helper_acceptance.rb +176 -0
- data/lib/cem_acpt/test_data.rb +157 -0
- data/lib/cem_acpt/utils.rb +70 -0
- data/lib/cem_acpt/version.rb +5 -0
- data/lib/cem_acpt.rb +27 -0
- data/sample_config.yaml +58 -0
- metadata +195 -0
@@ -0,0 +1,78 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
require 'securerandom'
|
4
|
+
require_relative File.join(__dir__, '..', 'image_name_builder')
|
5
|
+
require_relative File.join(__dir__, '..', 'logging')
|
6
|
+
|
7
|
+
module CemAcpt::Platform
|
8
|
+
# Base class for all platform classes. This class provides an API for
|
9
|
+
# interacting with the underlying platform.
|
10
|
+
class Base
|
11
|
+
include CemAcpt::Logging
|
12
|
+
|
13
|
+
attr_reader :config, :test_data, :local_port, :node_name, :image_name
|
14
|
+
|
15
|
+
# @param conf [CemAcpt::Config] the config object.
|
16
|
+
# @param single_test_data [Hash] the test data for the current test.
|
17
|
+
# @param local_port [Integer] the local port to use for the test.
|
18
|
+
def initialize(conf, single_test_data, local_port)
|
19
|
+
raise ArgumentError, 'single_test_data must be a Hash' unless single_test_data.is_a?(Hash)
|
20
|
+
|
21
|
+
@config = conf.get('node_data')
|
22
|
+
@test_data = single_test_data
|
23
|
+
@local_port = local_port
|
24
|
+
@node_name = @test_data[:node_name] || random_node_name
|
25
|
+
@image_name = conf.has?('image_name_builder') ? image_name_builder(conf, single_test_data) : @test_data[:image_name]
|
26
|
+
end
|
27
|
+
|
28
|
+
# Node should return a hash of all data about a created node.
|
29
|
+
def node
|
30
|
+
raise NotImplementedError, '#node must be implemented by subclass'
|
31
|
+
end
|
32
|
+
|
33
|
+
# Provision a node. Will be called asynchronously.
|
34
|
+
def provision
|
35
|
+
raise NotImplementedError, '#provision must be implemented by subclass'
|
36
|
+
end
|
37
|
+
|
38
|
+
# Destroy a node. Will be called asynchronously.
|
39
|
+
def destroy
|
40
|
+
raise NotImplementedError, '#destroy must be implemented by subclass'
|
41
|
+
end
|
42
|
+
|
43
|
+
# Tests to see if a node is ready to accept connections from the test suite.
|
44
|
+
def ready?
|
45
|
+
raise NotImplementedError, '#ready? must be implemented by subclass'
|
46
|
+
end
|
47
|
+
|
48
|
+
# Upload and install a Puppet module package on the node. Blocking call.
|
49
|
+
def install_puppet_module_package(_module_pkg_path, _remote_path)
|
50
|
+
raise NotImplementedError, '#install_puppet_module_package must be implemented by subclass'
|
51
|
+
end
|
52
|
+
|
53
|
+
# Generates a random node name.
|
54
|
+
def random_node_name
|
55
|
+
"acpt-test-#{SecureRandom.hex(10)}"
|
56
|
+
end
|
57
|
+
|
58
|
+
# Builds an image name if the config specifies to use the image name builder.
|
59
|
+
def image_name_builder(conf, tdata)
|
60
|
+
@image_name_builder ||= CemAcpt::ImageNameBuilder.new(conf).build(tdata)
|
61
|
+
end
|
62
|
+
|
63
|
+
# Returns a command provider specified by the Platform module of the specific platform.
|
64
|
+
def self.command_provider
|
65
|
+
raise NotImplementedError, '#command_provider must be implemented by subclass'
|
66
|
+
end
|
67
|
+
|
68
|
+
# Applies a Puppet manifest on the given node.
|
69
|
+
def self.apply_manifest(_instance_name, _manifest, _opts = {})
|
70
|
+
raise NotImplementedError, '#apply_manifest must be implemented by subclass'
|
71
|
+
end
|
72
|
+
|
73
|
+
# Runs a shell command on the given node.
|
74
|
+
def self.run_shell(_instance_name, _cmd, _opts = {})
|
75
|
+
raise NotImplementedError, '#run_shell must be implemented by subclass'
|
76
|
+
end
|
77
|
+
end
|
78
|
+
end
|
@@ -0,0 +1,313 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
require 'json'
|
4
|
+
require 'open3'
|
5
|
+
require_relative File.join(__dir__, '..', 'base', 'cmd.rb')
|
6
|
+
|
7
|
+
module CemAcpt::Platform::Gcp
|
8
|
+
# This class provides methods to run gcloud commands. It allows for default values to be
|
9
|
+
# set for the project, zone, and user and can also find these values from the local config.
|
10
|
+
# Additionally, this class provides a way to run SSH commands against GCP VMs using IAP.
|
11
|
+
class Cmd < CemAcpt::Platform::CmdBase
|
12
|
+
def initialize(project: nil, zone: nil, out_format: nil, filter: nil, user_name: nil, local_port: nil)
|
13
|
+
super
|
14
|
+
require 'net/ssh'
|
15
|
+
require 'net/ssh/proxy/command'
|
16
|
+
require 'net/scp'
|
17
|
+
|
18
|
+
@project = project unless project.nil?
|
19
|
+
@zone = zone unless zone.nil?
|
20
|
+
@default_out_format = out_format
|
21
|
+
@default_filter = filter
|
22
|
+
@user_name = user_name
|
23
|
+
@local_port = local_port
|
24
|
+
raise 'gcloud command not available' unless gcloud?
|
25
|
+
raise 'gcloud is not authenticated' unless authenticated?
|
26
|
+
end
|
27
|
+
|
28
|
+
# Returns either the project name passed in during object initialization
|
29
|
+
# or the project name from the local config.
|
30
|
+
def project
|
31
|
+
@project ||= project_from_config&.chomp
|
32
|
+
end
|
33
|
+
|
34
|
+
# Returns either the zone name passed in during object initialization
|
35
|
+
# or the zone name from the local config.
|
36
|
+
def zone
|
37
|
+
@zone ||= zone_from_config&.chomp
|
38
|
+
end
|
39
|
+
|
40
|
+
# Returns either the user name passed in during object initialization
|
41
|
+
# or queries the current active, authenticated user from gcloud and returns the name.
|
42
|
+
def user_name
|
43
|
+
@user_name ||= authenticated_user_name&.chomp
|
44
|
+
end
|
45
|
+
|
46
|
+
# Returns the format string passed in during object initialization or
|
47
|
+
# the default format string.
|
48
|
+
def format(out_format = nil)
|
49
|
+
out_format&.chomp || @default_out_format&.chomp
|
50
|
+
end
|
51
|
+
|
52
|
+
# Returns the filter string passed in during object initialization or
|
53
|
+
# the default filter string.
|
54
|
+
def filter(out_filter = nil)
|
55
|
+
out_filter&.chomp || @default_filter&.chomp
|
56
|
+
end
|
57
|
+
|
58
|
+
def local_port
|
59
|
+
@local_port ||= rand(49_512..65_535)
|
60
|
+
end
|
61
|
+
|
62
|
+
# Returns a formatted hash of ssh options to be used with Net::SSH.start.
|
63
|
+
# If you pass in a GCP VM instance name, this method will configure the
|
64
|
+
# IAP tunnel ProxyCommand to use. If you pass in an opts hash, it will
|
65
|
+
# merge the options with the default options.
|
66
|
+
def ssh_opts(instance_name: nil, use_proxy_command: true, opts: {})
|
67
|
+
base_opts = default_ssh_opts
|
68
|
+
if use_proxy_command
|
69
|
+
base_opts[:proxy] = proxy_command(instance_name, port: base_opts[:port])
|
70
|
+
base_opts[:host_key_alias] = vm_alias(instance_name)
|
71
|
+
end
|
72
|
+
base_opts.merge(opts).reject { |_, v| v.nil? }
|
73
|
+
end
|
74
|
+
|
75
|
+
# Runs `gcloud` commands locally.
|
76
|
+
def local_exec(command, out_format: 'json', out_filter: nil, project_flag: true)
|
77
|
+
cmd_parts = ['gcloud', command]
|
78
|
+
cmd_parts << "--project=#{project.chomp}" unless !project_flag || project.nil?
|
79
|
+
cmd_parts << "--format=#{format(out_format)}" if format(out_format)
|
80
|
+
cmd_parts << "--filter=\"#{filter(out_filter)}\"" if filter(out_filter)
|
81
|
+
final_command = cmd_parts.join(' ')
|
82
|
+
stdout, stderr, status = Open3.capture3(final_command)
|
83
|
+
raise "gcloud command '#{final_command}' failed: #{stderr}" unless status.success?
|
84
|
+
|
85
|
+
begin
|
86
|
+
return ::JSON.parse(stdout) if format(out_format) == 'json'
|
87
|
+
rescue ::JSON::ParserError
|
88
|
+
return stdout
|
89
|
+
end
|
90
|
+
|
91
|
+
stdout
|
92
|
+
end
|
93
|
+
|
94
|
+
# Stops a GCP VM instance.
|
95
|
+
def stop_instance(instance_name)
|
96
|
+
local_exec("compute instances stop #{instance_name}")
|
97
|
+
end
|
98
|
+
|
99
|
+
# Deletes a GCP VM instance.
|
100
|
+
def delete_instance(instance_name)
|
101
|
+
local_exec("compute instances delete #{instance_name} --quiet")
|
102
|
+
rescue StandardError
|
103
|
+
# Ignore errors when deleting instances.
|
104
|
+
end
|
105
|
+
|
106
|
+
# Returns a Hash describing a GCP VM instance.
|
107
|
+
def vm_describe(instance_name, out_format: 'json', out_filter: nil)
|
108
|
+
local_exec("compute instances describe #{instance_name}", out_format: out_format, out_filter: out_filter)
|
109
|
+
end
|
110
|
+
|
111
|
+
# This function runs the specified command as the currently authenticated user
|
112
|
+
# on the given CGP VM via SSH. Using `ssh` does not invoke the gcloud command
|
113
|
+
# and is dependent on the system's SSH configuration.
|
114
|
+
def ssh(instance_name, cmd, ignore_command_errors: false, use_proxy_command: true, opts: {})
|
115
|
+
Net::SSH.start(instance_name, user_name, ssh_opts(instance_name: instance_name, use_proxy_command: use_proxy_command, opts: opts)) do |ssh|
|
116
|
+
result = ssh.exec!(cmd)
|
117
|
+
if result.exitstatus != 0 && !ignore_command_errors
|
118
|
+
abridged_cmd = cmd.length > 100 ? "#{cmd[0..100]}..." : cmd
|
119
|
+
raise "Failed to run SSH command \"#{abridged_cmd}\" on #{instance_name}: #{result}"
|
120
|
+
end
|
121
|
+
result
|
122
|
+
end
|
123
|
+
end
|
124
|
+
|
125
|
+
# Uploads a file to the specified VM.
|
126
|
+
def scp_upload(instance_name, local, remote, use_proxy_command: true, scp_opts: {}, opts: {})
|
127
|
+
raise "Local file #{local} does not exist" unless File.exist?(local)
|
128
|
+
|
129
|
+
hostname = use_proxy_command ? vm_alias(instance_name) : instance_name
|
130
|
+
Net::SCP.start(hostname, user_name, ssh_opts(instance_name: instance_name, use_proxy_command: use_proxy_command, opts: opts)) do |scp|
|
131
|
+
scp.upload(local, remote, scp_opts).wait
|
132
|
+
end
|
133
|
+
end
|
134
|
+
|
135
|
+
# Downloads a file from the specified VM.
|
136
|
+
def scp_download(instance_name, remote, local, use_proxy_command: true, scp_opts: {}, opts: {})
|
137
|
+
raise "Local file #{local} does not exist" unless File.exist?(local)
|
138
|
+
|
139
|
+
hostname = use_proxy_command ? vm_alias(instance_name) : instance_name
|
140
|
+
Net::SCP.start(hostname, user_name, ssh_opts(instance_name: instance_name, use_proxy_command: use_proxy_command, opts: opts)) do |scp|
|
141
|
+
scp.download(remote, local, scp_opts).wait
|
142
|
+
end
|
143
|
+
end
|
144
|
+
|
145
|
+
def ssh_ready?(instance_name, timeout = 300, opts: {})
|
146
|
+
with_timed_retry(timeout) do
|
147
|
+
logger.debug("Testing SSH connection to #{instance_name}")
|
148
|
+
Net::SSH.start(instance_name, user_name, ssh_opts(instance_name: instance_name, opts: opts)) do |_|
|
149
|
+
true
|
150
|
+
end
|
151
|
+
end
|
152
|
+
end
|
153
|
+
|
154
|
+
# This function spawns a background thread to run a GCP IAP tunnel, run the given
|
155
|
+
# code block, then kill the thread. The code block will be yielded ssh_opts that
|
156
|
+
# are used to configure SSH connections over the IAP tunnel. The IAP tunnel is
|
157
|
+
# run in it's own thread and will not block the main thread. The IAP tunnel is
|
158
|
+
# killed when the code block exits. All SSH connections made in the code block
|
159
|
+
# must be made to the host '127.0.0.1' using the yielded ssh_opts and not the VM name.
|
160
|
+
# @param instance_name [String] The name of the GCP VM instance to connect to.
|
161
|
+
# @param instance_port [Integer] The port to connect to on the GCP VM instance.
|
162
|
+
# @param disable_connection_check [Boolean] If true, the connection check will be disabled.
|
163
|
+
def with_iap_tunnel(instance_name, instance_port = 22, disable_connection_check: false)
|
164
|
+
return unless block_given?
|
165
|
+
|
166
|
+
cmd = [
|
167
|
+
'compute start-iap-tunnel',
|
168
|
+
"#{instance_name} #{instance_port}",
|
169
|
+
"--local-host-port=localhost:#{local_port}",
|
170
|
+
]
|
171
|
+
cmd << '--disable-connection-check' if disable_connection_check
|
172
|
+
tunnel_ssh_opts = {
|
173
|
+
proxy: nil,
|
174
|
+
port: local_port,
|
175
|
+
forward_agent: false,
|
176
|
+
}
|
177
|
+
begin
|
178
|
+
thread = Thread.new do
|
179
|
+
local_exec(cmd.join(' '))
|
180
|
+
end
|
181
|
+
yield ssh_opts(use_proxy_command: false, opts: tunnel_ssh_opts)
|
182
|
+
thread.exit
|
183
|
+
rescue IOError
|
184
|
+
# Ignore errors when killing the thread.
|
185
|
+
ensure
|
186
|
+
thread.exit
|
187
|
+
end
|
188
|
+
end
|
189
|
+
|
190
|
+
def project_from_config
|
191
|
+
local_exec('config get-value project', out_format: nil, project_flag: false)
|
192
|
+
end
|
193
|
+
|
194
|
+
def zone_from_config
|
195
|
+
local_exec('config get-value compute/zone', out_format: nil, project_flag: false)
|
196
|
+
end
|
197
|
+
|
198
|
+
def run_shell(instance_name, cmd, opts = {})
|
199
|
+
ssh_opts = opts.key?(:ssh_opts) ? opts[:ssh_opts] : {}
|
200
|
+
command = [
|
201
|
+
'sudo -n -u root -i',
|
202
|
+
cmd,
|
203
|
+
]
|
204
|
+
ssh(
|
205
|
+
instance_name,
|
206
|
+
command.join(' '),
|
207
|
+
ignore_command_errors: true,
|
208
|
+
use_proxy_command: false,
|
209
|
+
opts: ssh_opts,
|
210
|
+
)
|
211
|
+
end
|
212
|
+
|
213
|
+
def apply_manifest(instance_name, manifest, opts = {})
|
214
|
+
require 'tempfile'
|
215
|
+
|
216
|
+
temp_manifest = Tempfile.new('acpt_manifest')
|
217
|
+
temp_manifest.write(manifest)
|
218
|
+
temp_manifest.close
|
219
|
+
begin
|
220
|
+
scp_upload(
|
221
|
+
instance_name,
|
222
|
+
temp_manifest.path,
|
223
|
+
'/tmp/acpt_manifest.pp',
|
224
|
+
opts: opts[:ssh_opts],
|
225
|
+
)
|
226
|
+
ensure
|
227
|
+
temp_manifest.unlink
|
228
|
+
end
|
229
|
+
apply_cmd = [
|
230
|
+
'sudo -n -u root -i',
|
231
|
+
"#{opts[:puppet_path]} apply /tmp/acpt_manifest.pp"
|
232
|
+
]
|
233
|
+
apply_cmd << '--trace' if opts[:apply][:trace]
|
234
|
+
apply_cmd << "--hiera_config=#{opts[:apply][:hiera_config]}" if opts[:apply][:hiera_config]
|
235
|
+
apply_cmd << '--debug' if opts[:apply][:debug]
|
236
|
+
apply_cmd << '--noop' if opts[:apply][:noop]
|
237
|
+
apply_cmd << '--detailed-exitcodes' if opts[:apply][:detailed_exitcodes]
|
238
|
+
|
239
|
+
ssh(
|
240
|
+
instance_name,
|
241
|
+
apply_cmd.join(' '),
|
242
|
+
ignore_command_errors: true,
|
243
|
+
opts: opts[:ssh_opts]
|
244
|
+
)
|
245
|
+
end
|
246
|
+
|
247
|
+
private
|
248
|
+
|
249
|
+
def vm_alias(vm)
|
250
|
+
"compute.#{vm_describe(vm)['id']}"
|
251
|
+
end
|
252
|
+
|
253
|
+
# Default options for Net::SSH
|
254
|
+
def default_ssh_opts
|
255
|
+
{
|
256
|
+
verify_host_key: :accept_new_or_local_tunnel,
|
257
|
+
keys: [File.join(ENV['HOME'], '.ssh', 'google_compute_engine')],
|
258
|
+
keys_only: true,
|
259
|
+
config: false,
|
260
|
+
check_host_ip: false,
|
261
|
+
non_interactive: true,
|
262
|
+
port: 22,
|
263
|
+
user: user_name,
|
264
|
+
user_known_hosts_file: File.join(ENV['HOME'], '.ssh', 'google_compute_known_hosts'),
|
265
|
+
}
|
266
|
+
end
|
267
|
+
|
268
|
+
# Returns a Proxy::Command object to use for the SSH option ProxyCommand.
|
269
|
+
# This works in the same way that `gcloud compute ssh --tunnel-through-iap` does.
|
270
|
+
def proxy_command(instance_name, port: 22, quiet: true, verbosity: nil)
|
271
|
+
proxy_command = [
|
272
|
+
'gcloud compute start-iap-tunnel',
|
273
|
+
"#{instance_name} #{port}",
|
274
|
+
'--listen-on-stdin',
|
275
|
+
"--project=#{project}",
|
276
|
+
"--zone=#{zone}",
|
277
|
+
]
|
278
|
+
proxy_command << '--no-user-output-enabled --quiet' if quiet
|
279
|
+
proxy_command << "--verbosity=#{verbosity}" unless quiet || verbosity.nil? || verbosity.empty?
|
280
|
+
Net::SSH::Proxy::Command.new(proxy_command.join(' '))
|
281
|
+
end
|
282
|
+
|
283
|
+
# This function checks to ensure that the gcloud tool is available on the system.
|
284
|
+
def gcloud?
|
285
|
+
ver = local_exec('--version', out_format: nil, project_flag: false)
|
286
|
+
!ver.nil? && !ver.empty?
|
287
|
+
rescue RuntimeError, Errno::ENOENT
|
288
|
+
false
|
289
|
+
end
|
290
|
+
|
291
|
+
# This function checks to ensure that a user is authenticated with gcloud.
|
292
|
+
def authenticated?
|
293
|
+
local_exec('auth list', project_flag: false).any? { |a| a['status'] == 'ACTIVE' }
|
294
|
+
rescue RuntimeError, Errno::ENOENT
|
295
|
+
false
|
296
|
+
end
|
297
|
+
|
298
|
+
# This function returns the currently authenticated user's name sanitized for use in SSH
|
299
|
+
def authenticated_user_name
|
300
|
+
uname = nil
|
301
|
+
begin
|
302
|
+
uname = local_exec('config get-value account', out_format: nil, project_flag: false)
|
303
|
+
rescue RuntimeError
|
304
|
+
auth_json = local_exec('auth list')
|
305
|
+
uname = auth_json.find { |x| x['status'] == 'ACTIVE' }['account']
|
306
|
+
end
|
307
|
+
|
308
|
+
raise 'failed to find authenticated user name' unless uname
|
309
|
+
|
310
|
+
uname.split(%r{[^a-zA-Z0-9_]}).join('_')
|
311
|
+
end
|
312
|
+
end
|
313
|
+
end
|