cem_acpt 0.1.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -0,0 +1,78 @@
1
+ # frozen_string_literal: true
2
+
3
+ require 'securerandom'
4
+ require_relative File.join(__dir__, '..', 'image_name_builder')
5
+ require_relative File.join(__dir__, '..', 'logging')
6
+
7
+ module CemAcpt::Platform
8
+ # Base class for all platform classes. This class provides an API for
9
+ # interacting with the underlying platform.
10
+ class Base
11
+ include CemAcpt::Logging
12
+
13
+ attr_reader :config, :test_data, :local_port, :node_name, :image_name
14
+
15
+ # @param conf [CemAcpt::Config] the config object.
16
+ # @param single_test_data [Hash] the test data for the current test.
17
+ # @param local_port [Integer] the local port to use for the test.
18
+ def initialize(conf, single_test_data, local_port)
19
+ raise ArgumentError, 'single_test_data must be a Hash' unless single_test_data.is_a?(Hash)
20
+
21
+ @config = conf.get('node_data')
22
+ @test_data = single_test_data
23
+ @local_port = local_port
24
+ @node_name = @test_data[:node_name] || random_node_name
25
+ @image_name = conf.has?('image_name_builder') ? image_name_builder(conf, single_test_data) : @test_data[:image_name]
26
+ end
27
+
28
+ # Node should return a hash of all data about a created node.
29
+ def node
30
+ raise NotImplementedError, '#node must be implemented by subclass'
31
+ end
32
+
33
+ # Provision a node. Will be called asynchronously.
34
+ def provision
35
+ raise NotImplementedError, '#provision must be implemented by subclass'
36
+ end
37
+
38
+ # Destroy a node. Will be called asynchronously.
39
+ def destroy
40
+ raise NotImplementedError, '#destroy must be implemented by subclass'
41
+ end
42
+
43
+ # Tests to see if a node is ready to accept connections from the test suite.
44
+ def ready?
45
+ raise NotImplementedError, '#ready? must be implemented by subclass'
46
+ end
47
+
48
+ # Upload and install a Puppet module package on the node. Blocking call.
49
+ def install_puppet_module_package(_module_pkg_path, _remote_path)
50
+ raise NotImplementedError, '#install_puppet_module_package must be implemented by subclass'
51
+ end
52
+
53
+ # Generates a random node name.
54
+ def random_node_name
55
+ "acpt-test-#{SecureRandom.hex(10)}"
56
+ end
57
+
58
+ # Builds an image name if the config specifies to use the image name builder.
59
+ def image_name_builder(conf, tdata)
60
+ @image_name_builder ||= CemAcpt::ImageNameBuilder.new(conf).build(tdata)
61
+ end
62
+
63
+ # Returns a command provider specified by the Platform module of the specific platform.
64
+ def self.command_provider
65
+ raise NotImplementedError, '#command_provider must be implemented by subclass'
66
+ end
67
+
68
+ # Applies a Puppet manifest on the given node.
69
+ def self.apply_manifest(_instance_name, _manifest, _opts = {})
70
+ raise NotImplementedError, '#apply_manifest must be implemented by subclass'
71
+ end
72
+
73
+ # Runs a shell command on the given node.
74
+ def self.run_shell(_instance_name, _cmd, _opts = {})
75
+ raise NotImplementedError, '#run_shell must be implemented by subclass'
76
+ end
77
+ end
78
+ end
@@ -0,0 +1,313 @@
1
+ # frozen_string_literal: true
2
+
3
+ require 'json'
4
+ require 'open3'
5
+ require_relative File.join(__dir__, '..', 'base', 'cmd.rb')
6
+
7
+ module CemAcpt::Platform::Gcp
8
+ # This class provides methods to run gcloud commands. It allows for default values to be
9
+ # set for the project, zone, and user and can also find these values from the local config.
10
+ # Additionally, this class provides a way to run SSH commands against GCP VMs using IAP.
11
+ class Cmd < CemAcpt::Platform::CmdBase
12
+ def initialize(project: nil, zone: nil, out_format: nil, filter: nil, user_name: nil, local_port: nil)
13
+ super
14
+ require 'net/ssh'
15
+ require 'net/ssh/proxy/command'
16
+ require 'net/scp'
17
+
18
+ @project = project unless project.nil?
19
+ @zone = zone unless zone.nil?
20
+ @default_out_format = out_format
21
+ @default_filter = filter
22
+ @user_name = user_name
23
+ @local_port = local_port
24
+ raise 'gcloud command not available' unless gcloud?
25
+ raise 'gcloud is not authenticated' unless authenticated?
26
+ end
27
+
28
+ # Returns either the project name passed in during object initialization
29
+ # or the project name from the local config.
30
+ def project
31
+ @project ||= project_from_config&.chomp
32
+ end
33
+
34
+ # Returns either the zone name passed in during object initialization
35
+ # or the zone name from the local config.
36
+ def zone
37
+ @zone ||= zone_from_config&.chomp
38
+ end
39
+
40
+ # Returns either the user name passed in during object initialization
41
+ # or queries the current active, authenticated user from gcloud and returns the name.
42
+ def user_name
43
+ @user_name ||= authenticated_user_name&.chomp
44
+ end
45
+
46
+ # Returns the format string passed in during object initialization or
47
+ # the default format string.
48
+ def format(out_format = nil)
49
+ out_format&.chomp || @default_out_format&.chomp
50
+ end
51
+
52
+ # Returns the filter string passed in during object initialization or
53
+ # the default filter string.
54
+ def filter(out_filter = nil)
55
+ out_filter&.chomp || @default_filter&.chomp
56
+ end
57
+
58
+ def local_port
59
+ @local_port ||= rand(49_512..65_535)
60
+ end
61
+
62
+ # Returns a formatted hash of ssh options to be used with Net::SSH.start.
63
+ # If you pass in a GCP VM instance name, this method will configure the
64
+ # IAP tunnel ProxyCommand to use. If you pass in an opts hash, it will
65
+ # merge the options with the default options.
66
+ def ssh_opts(instance_name: nil, use_proxy_command: true, opts: {})
67
+ base_opts = default_ssh_opts
68
+ if use_proxy_command
69
+ base_opts[:proxy] = proxy_command(instance_name, port: base_opts[:port])
70
+ base_opts[:host_key_alias] = vm_alias(instance_name)
71
+ end
72
+ base_opts.merge(opts).reject { |_, v| v.nil? }
73
+ end
74
+
75
+ # Runs `gcloud` commands locally.
76
+ def local_exec(command, out_format: 'json', out_filter: nil, project_flag: true)
77
+ cmd_parts = ['gcloud', command]
78
+ cmd_parts << "--project=#{project.chomp}" unless !project_flag || project.nil?
79
+ cmd_parts << "--format=#{format(out_format)}" if format(out_format)
80
+ cmd_parts << "--filter=\"#{filter(out_filter)}\"" if filter(out_filter)
81
+ final_command = cmd_parts.join(' ')
82
+ stdout, stderr, status = Open3.capture3(final_command)
83
+ raise "gcloud command '#{final_command}' failed: #{stderr}" unless status.success?
84
+
85
+ begin
86
+ return ::JSON.parse(stdout) if format(out_format) == 'json'
87
+ rescue ::JSON::ParserError
88
+ return stdout
89
+ end
90
+
91
+ stdout
92
+ end
93
+
94
+ # Stops a GCP VM instance.
95
+ def stop_instance(instance_name)
96
+ local_exec("compute instances stop #{instance_name}")
97
+ end
98
+
99
+ # Deletes a GCP VM instance.
100
+ def delete_instance(instance_name)
101
+ local_exec("compute instances delete #{instance_name} --quiet")
102
+ rescue StandardError
103
+ # Ignore errors when deleting instances.
104
+ end
105
+
106
+ # Returns a Hash describing a GCP VM instance.
107
+ def vm_describe(instance_name, out_format: 'json', out_filter: nil)
108
+ local_exec("compute instances describe #{instance_name}", out_format: out_format, out_filter: out_filter)
109
+ end
110
+
111
+ # This function runs the specified command as the currently authenticated user
112
+ # on the given CGP VM via SSH. Using `ssh` does not invoke the gcloud command
113
+ # and is dependent on the system's SSH configuration.
114
+ def ssh(instance_name, cmd, ignore_command_errors: false, use_proxy_command: true, opts: {})
115
+ Net::SSH.start(instance_name, user_name, ssh_opts(instance_name: instance_name, use_proxy_command: use_proxy_command, opts: opts)) do |ssh|
116
+ result = ssh.exec!(cmd)
117
+ if result.exitstatus != 0 && !ignore_command_errors
118
+ abridged_cmd = cmd.length > 100 ? "#{cmd[0..100]}..." : cmd
119
+ raise "Failed to run SSH command \"#{abridged_cmd}\" on #{instance_name}: #{result}"
120
+ end
121
+ result
122
+ end
123
+ end
124
+
125
+ # Uploads a file to the specified VM.
126
+ def scp_upload(instance_name, local, remote, use_proxy_command: true, scp_opts: {}, opts: {})
127
+ raise "Local file #{local} does not exist" unless File.exist?(local)
128
+
129
+ hostname = use_proxy_command ? vm_alias(instance_name) : instance_name
130
+ Net::SCP.start(hostname, user_name, ssh_opts(instance_name: instance_name, use_proxy_command: use_proxy_command, opts: opts)) do |scp|
131
+ scp.upload(local, remote, scp_opts).wait
132
+ end
133
+ end
134
+
135
+ # Downloads a file from the specified VM.
136
+ def scp_download(instance_name, remote, local, use_proxy_command: true, scp_opts: {}, opts: {})
137
+ raise "Local file #{local} does not exist" unless File.exist?(local)
138
+
139
+ hostname = use_proxy_command ? vm_alias(instance_name) : instance_name
140
+ Net::SCP.start(hostname, user_name, ssh_opts(instance_name: instance_name, use_proxy_command: use_proxy_command, opts: opts)) do |scp|
141
+ scp.download(remote, local, scp_opts).wait
142
+ end
143
+ end
144
+
145
+ def ssh_ready?(instance_name, timeout = 300, opts: {})
146
+ with_timed_retry(timeout) do
147
+ logger.debug("Testing SSH connection to #{instance_name}")
148
+ Net::SSH.start(instance_name, user_name, ssh_opts(instance_name: instance_name, opts: opts)) do |_|
149
+ true
150
+ end
151
+ end
152
+ end
153
+
154
+ # This function spawns a background thread to run a GCP IAP tunnel, run the given
155
+ # code block, then kill the thread. The code block will be yielded ssh_opts that
156
+ # are used to configure SSH connections over the IAP tunnel. The IAP tunnel is
157
+ # run in it's own thread and will not block the main thread. The IAP tunnel is
158
+ # killed when the code block exits. All SSH connections made in the code block
159
+ # must be made to the host '127.0.0.1' using the yielded ssh_opts and not the VM name.
160
+ # @param instance_name [String] The name of the GCP VM instance to connect to.
161
+ # @param instance_port [Integer] The port to connect to on the GCP VM instance.
162
+ # @param disable_connection_check [Boolean] If true, the connection check will be disabled.
163
+ def with_iap_tunnel(instance_name, instance_port = 22, disable_connection_check: false)
164
+ return unless block_given?
165
+
166
+ cmd = [
167
+ 'compute start-iap-tunnel',
168
+ "#{instance_name} #{instance_port}",
169
+ "--local-host-port=localhost:#{local_port}",
170
+ ]
171
+ cmd << '--disable-connection-check' if disable_connection_check
172
+ tunnel_ssh_opts = {
173
+ proxy: nil,
174
+ port: local_port,
175
+ forward_agent: false,
176
+ }
177
+ begin
178
+ thread = Thread.new do
179
+ local_exec(cmd.join(' '))
180
+ end
181
+ yield ssh_opts(use_proxy_command: false, opts: tunnel_ssh_opts)
182
+ thread.exit
183
+ rescue IOError
184
+ # Ignore errors when killing the thread.
185
+ ensure
186
+ thread.exit
187
+ end
188
+ end
189
+
190
+ def project_from_config
191
+ local_exec('config get-value project', out_format: nil, project_flag: false)
192
+ end
193
+
194
+ def zone_from_config
195
+ local_exec('config get-value compute/zone', out_format: nil, project_flag: false)
196
+ end
197
+
198
+ def run_shell(instance_name, cmd, opts = {})
199
+ ssh_opts = opts.key?(:ssh_opts) ? opts[:ssh_opts] : {}
200
+ command = [
201
+ 'sudo -n -u root -i',
202
+ cmd,
203
+ ]
204
+ ssh(
205
+ instance_name,
206
+ command.join(' '),
207
+ ignore_command_errors: true,
208
+ use_proxy_command: false,
209
+ opts: ssh_opts,
210
+ )
211
+ end
212
+
213
+ def apply_manifest(instance_name, manifest, opts = {})
214
+ require 'tempfile'
215
+
216
+ temp_manifest = Tempfile.new('acpt_manifest')
217
+ temp_manifest.write(manifest)
218
+ temp_manifest.close
219
+ begin
220
+ scp_upload(
221
+ instance_name,
222
+ temp_manifest.path,
223
+ '/tmp/acpt_manifest.pp',
224
+ opts: opts[:ssh_opts],
225
+ )
226
+ ensure
227
+ temp_manifest.unlink
228
+ end
229
+ apply_cmd = [
230
+ 'sudo -n -u root -i',
231
+ "#{opts[:puppet_path]} apply /tmp/acpt_manifest.pp"
232
+ ]
233
+ apply_cmd << '--trace' if opts[:apply][:trace]
234
+ apply_cmd << "--hiera_config=#{opts[:apply][:hiera_config]}" if opts[:apply][:hiera_config]
235
+ apply_cmd << '--debug' if opts[:apply][:debug]
236
+ apply_cmd << '--noop' if opts[:apply][:noop]
237
+ apply_cmd << '--detailed-exitcodes' if opts[:apply][:detailed_exitcodes]
238
+
239
+ ssh(
240
+ instance_name,
241
+ apply_cmd.join(' '),
242
+ ignore_command_errors: true,
243
+ opts: opts[:ssh_opts]
244
+ )
245
+ end
246
+
247
+ private
248
+
249
+ def vm_alias(vm)
250
+ "compute.#{vm_describe(vm)['id']}"
251
+ end
252
+
253
+ # Default options for Net::SSH
254
+ def default_ssh_opts
255
+ {
256
+ verify_host_key: :accept_new_or_local_tunnel,
257
+ keys: [File.join(ENV['HOME'], '.ssh', 'google_compute_engine')],
258
+ keys_only: true,
259
+ config: false,
260
+ check_host_ip: false,
261
+ non_interactive: true,
262
+ port: 22,
263
+ user: user_name,
264
+ user_known_hosts_file: File.join(ENV['HOME'], '.ssh', 'google_compute_known_hosts'),
265
+ }
266
+ end
267
+
268
+ # Returns a Proxy::Command object to use for the SSH option ProxyCommand.
269
+ # This works in the same way that `gcloud compute ssh --tunnel-through-iap` does.
270
+ def proxy_command(instance_name, port: 22, quiet: true, verbosity: nil)
271
+ proxy_command = [
272
+ 'gcloud compute start-iap-tunnel',
273
+ "#{instance_name} #{port}",
274
+ '--listen-on-stdin',
275
+ "--project=#{project}",
276
+ "--zone=#{zone}",
277
+ ]
278
+ proxy_command << '--no-user-output-enabled --quiet' if quiet
279
+ proxy_command << "--verbosity=#{verbosity}" unless quiet || verbosity.nil? || verbosity.empty?
280
+ Net::SSH::Proxy::Command.new(proxy_command.join(' '))
281
+ end
282
+
283
+ # This function checks to ensure that the gcloud tool is available on the system.
284
+ def gcloud?
285
+ ver = local_exec('--version', out_format: nil, project_flag: false)
286
+ !ver.nil? && !ver.empty?
287
+ rescue RuntimeError, Errno::ENOENT
288
+ false
289
+ end
290
+
291
+ # This function checks to ensure that a user is authenticated with gcloud.
292
+ def authenticated?
293
+ local_exec('auth list', project_flag: false).any? { |a| a['status'] == 'ACTIVE' }
294
+ rescue RuntimeError, Errno::ENOENT
295
+ false
296
+ end
297
+
298
+ # This function returns the currently authenticated user's name sanitized for use in SSH
299
+ def authenticated_user_name
300
+ uname = nil
301
+ begin
302
+ uname = local_exec('config get-value account', out_format: nil, project_flag: false)
303
+ rescue RuntimeError
304
+ auth_json = local_exec('auth list')
305
+ uname = auth_json.find { |x| x['status'] == 'ACTIVE' }['account']
306
+ end
307
+
308
+ raise 'failed to find authenticated user name' unless uname
309
+
310
+ uname.split(%r{[^a-zA-Z0-9_]}).join('_')
311
+ end
312
+ end
313
+ end