cem_acpt 0.2.4 → 0.6.0

Sign up to get free protection for your applications and to get access to all the features.
Files changed (58) hide show
  1. checksums.yaml +4 -4
  2. data/.github/workflows/spec.yml +30 -0
  3. data/Gemfile +4 -3
  4. data/Gemfile.lock +95 -43
  5. data/README.md +144 -83
  6. data/cem_acpt.gemspec +12 -7
  7. data/exe/cem_acpt +41 -7
  8. data/lib/cem_acpt/config.rb +340 -0
  9. data/lib/cem_acpt/core_extensions.rb +17 -61
  10. data/lib/cem_acpt/goss/api/action_response.rb +175 -0
  11. data/lib/cem_acpt/goss/api.rb +83 -0
  12. data/lib/cem_acpt/goss.rb +8 -0
  13. data/lib/cem_acpt/image_name_builder.rb +0 -9
  14. data/lib/cem_acpt/logging/formatter.rb +97 -0
  15. data/lib/cem_acpt/logging.rb +168 -142
  16. data/lib/cem_acpt/platform/base.rb +26 -37
  17. data/lib/cem_acpt/platform/gcp.rb +48 -62
  18. data/lib/cem_acpt/platform.rb +30 -28
  19. data/lib/cem_acpt/provision/terraform/linux.rb +47 -0
  20. data/lib/cem_acpt/provision/terraform/os_data.rb +72 -0
  21. data/lib/cem_acpt/provision/terraform/windows.rb +22 -0
  22. data/lib/cem_acpt/provision/terraform.rb +193 -0
  23. data/lib/cem_acpt/provision.rb +20 -0
  24. data/lib/cem_acpt/puppet_helpers.rb +0 -1
  25. data/lib/cem_acpt/test_data.rb +23 -13
  26. data/lib/cem_acpt/test_runner/log_formatter/goss_action_response.rb +104 -0
  27. data/lib/cem_acpt/test_runner/log_formatter.rb +10 -0
  28. data/lib/cem_acpt/test_runner.rb +170 -3
  29. data/lib/cem_acpt/utils/puppet.rb +29 -0
  30. data/lib/cem_acpt/utils/ssh.rb +197 -0
  31. data/lib/cem_acpt/utils/terminal.rb +27 -0
  32. data/lib/cem_acpt/utils.rb +4 -138
  33. data/lib/cem_acpt/version.rb +1 -1
  34. data/lib/cem_acpt.rb +73 -23
  35. data/lib/terraform/gcp/linux/goss/puppet_idempotent.yaml +10 -0
  36. data/lib/terraform/gcp/linux/goss/puppet_noop.yaml +12 -0
  37. data/lib/terraform/gcp/linux/main.tf +191 -0
  38. data/lib/terraform/gcp/linux/systemd/goss-acpt.service +8 -0
  39. data/lib/terraform/gcp/linux/systemd/goss-idempotent.service +8 -0
  40. data/lib/terraform/gcp/linux/systemd/goss-noop.service +8 -0
  41. data/lib/terraform/gcp/windows/.keep +0 -0
  42. data/sample_config.yaml +22 -21
  43. metadata +151 -51
  44. data/lib/cem_acpt/bootstrap/bootstrapper.rb +0 -206
  45. data/lib/cem_acpt/bootstrap/operating_system/rhel_family.rb +0 -129
  46. data/lib/cem_acpt/bootstrap/operating_system.rb +0 -17
  47. data/lib/cem_acpt/bootstrap.rb +0 -12
  48. data/lib/cem_acpt/context.rb +0 -153
  49. data/lib/cem_acpt/platform/base/cmd.rb +0 -71
  50. data/lib/cem_acpt/platform/gcp/cmd.rb +0 -353
  51. data/lib/cem_acpt/platform/gcp/compute.rb +0 -332
  52. data/lib/cem_acpt/platform/vmpooler.rb +0 -24
  53. data/lib/cem_acpt/rspec_utils.rb +0 -242
  54. data/lib/cem_acpt/shared_objects.rb +0 -537
  55. data/lib/cem_acpt/spec_helper_acceptance.rb +0 -184
  56. data/lib/cem_acpt/test_runner/run_handler.rb +0 -187
  57. data/lib/cem_acpt/test_runner/runner.rb +0 -210
  58. data/lib/cem_acpt/test_runner/runner_result.rb +0 -103
@@ -1,537 +0,0 @@
1
- # frozen_string_literal: true
2
-
3
- require 'concurrent-ruby'
4
- require 'deep_merge'
5
- require 'json'
6
- require 'yaml'
7
- require_relative 'core_extensions'
8
- require_relative 'logging'
9
-
10
- module CemAcpt
11
- using CemAcpt::CoreExtensions::ExtendedHash
12
-
13
- class ConfigImmutableError < RuntimeError; end
14
-
15
- # Holds module methods for writing various config objects
16
- # to files.
17
- module ConfigUtils
18
- class << self
19
- include CemAcpt::Logging
20
- end
21
-
22
- def self.object_to_file(obj, file_path)
23
- case File.extname(file_path)
24
- when '.json'
25
- raise "Object #{obj} is not serializable to JSON" unless obj.respond_to?(:to_json)
26
-
27
- logger.debug("Writing object to #{file_path}")
28
- File.write(file_path, obj.to_json)
29
- when '.yaml', '.yml'
30
- raise "Object #{obj} is not serializable to YAML" unless obj.respond_to?(:to_yaml)
31
-
32
- logger.debug("Writing object to #{file_path}")
33
- File.write(file_path, obj.to_yaml)
34
- else
35
- raise "File extension #{File.extname(file_path)} is not supported"
36
- end
37
- end
38
- end
39
-
40
- # Provides a thread-safe Config object.
41
- # Once the config is loaded by calling the `load` method, the object is
42
- # immutable and thread-safe as long as the internal hash is only accessed
43
- # by the `#get` and `#has?` methods. If `#load` is called more than once,
44
- # the object will raise an error.
45
- class Config
46
- include CemAcpt::Logging
47
-
48
- attr_reader :config_file
49
-
50
- def initialize
51
- @opts = {
52
- thread_pool: {
53
- min_threads: [2, Concurrent.processor_count - 1].max,
54
- max_threads: [2, Concurrent.processor_count - 1].max,
55
- max_queue: [2, Concurrent.processor_count - 1].max * 10,
56
- fallback_policy: :caller_runs,
57
- idletime: 1200,
58
- auto_terminate: true,
59
- },
60
- }
61
- end
62
-
63
- # Returns the value of the dot-separated key.
64
- # If the key is not found, returns nil.
65
- # @param dot_key [String] Dot-separated key
66
- # @return [Object] Value of the key
67
- def get(dot_key)
68
- @opts.dot_dig(dot_key)
69
- end
70
-
71
- # Checks if the dot-separated key exists in the config.
72
- # @param dot_key [String] Dot-separated key
73
- # @return [Boolean] True if the key exists, false otherwise
74
- def has?(dot_key)
75
- !!get(dot_key)
76
- end
77
-
78
- # Checks to see if cem_acpt is set to run in debug mode (log level is set to debug)
79
- # @return [TrueClass] If cem_acpt is running in debug mode
80
- # @return [FalseClass] If cem_acpt is not runnint in debug mode
81
- def debug_mode?
82
- @opts.dot_dig('log_level') == 'debug'
83
- end
84
-
85
- # Loads the config from specified file path. Config files must be in YAML
86
- # format. If 'opts' is specified, it will be merged with the config file.
87
- # Values in 'opts' will override values in the config file.
88
- # @param opts [Hash] Options to be merged with the config file
89
- # @param config_file [String] Path to the config file
90
- def load(opts: {}, config_file: './cem_acpt_config.yaml')
91
- raise ConfigImmutableError, 'Config is immutable, cannot load more than once' if frozen?
92
- raise ArgumentError, 'opts must be a Hash' unless opts.is_a?(Hash)
93
- raise ArgumentError, 'config_file must be a String' unless config_file.is_a?(String)
94
-
95
- @config_file = File.expand_path(config_file)
96
- @opts.deep_merge!(load_opts_from_file(File.expand_path(config_file)).deep_merge!(opts))
97
- @opts.format!
98
- @opts.freeze
99
- freeze
100
- end
101
-
102
- # Returns the config as a Hash.
103
- # @return [Hash] Config as a Hash
104
- def to_h
105
- ::Marshal.load(::Marshal.dump(@opts))
106
- end
107
-
108
- # Returns the config as a YAML string.
109
- # @return [String] Config as a YAML string
110
- def to_yaml
111
- to_h.to_yaml
112
- end
113
-
114
- # Returns the config as a JSON string.
115
- # @return [String] Config as a JSON string
116
- def to_json(*args)
117
- to_h.to_json(*args)
118
- end
119
-
120
- private
121
-
122
- # Loads the config from the specified file path.
123
- # @param config_file [String] Path to the config file
124
- # @return [Hash] Config hash
125
- def load_opts_from_file(config_file)
126
- return {} unless File.exist?(config_file)
127
-
128
- logger.debug("Loading config from #{config_file}")
129
- YAML.load_file(config_file)
130
- end
131
- end
132
-
133
- # Provides an object to hold NodeData configs.
134
- # Not thread-safe. Should not be mutated by itself,
135
- # but can be mutated safely by the NodeInventory object.
136
- # Currently unused.
137
- class NodeData
138
- include CemAcpt::Logging
139
-
140
- attr_reader :name, :data
141
-
142
- def initialize(name, data)
143
- data.format!
144
- @name = name
145
- @opts = data
146
- end
147
-
148
- def deep_merge(other)
149
- @opts.deep_merge(other.to_h)
150
- end
151
-
152
- def dot_dig(dot_key)
153
- @opts.dot_dig(dot_key)
154
- end
155
-
156
- def dot_store(dot_key, value)
157
- @opts.dot_store(dot_key, value)
158
- end
159
-
160
- def to_h
161
- @opts
162
- end
163
-
164
- def to_yaml
165
- to_h.to_yaml
166
- end
167
-
168
- def to_json(*args)
169
- to_h.to_json(*args)
170
- end
171
- end
172
-
173
- class NodeInventoryFileNotFoundError < StandardError; end
174
- class NodeInventoryFileLoadError < StandardError; end
175
- class NodeClaimedError < StandardError; end
176
- class NodeDoesNotExistError < StandardError; end
177
- class PropertyNotFoundError < StandardError; end
178
- class LockWaitTimeoutError < StandardError; end
179
- class LockNotRecognizedError < StandardError; end
180
-
181
- # Provides a thread-safe inventory of test nodes.
182
- class NodeInventory
183
- include CemAcpt::Logging
184
-
185
- attr_accessor :save_file_path
186
-
187
- def initialize
188
- @inventory = Concurrent::Map.new
189
- @lock = Concurrent::ReadWriteLock.new
190
- @claimed = Concurrent::Set.new
191
- @save_on_claim = false
192
- @save_file_path = 'spec/fixtures/node_inventory'
193
- @loaded_node_inv = nil
194
- end
195
-
196
- # When called, enables saving the inventory to a file on claim.
197
- def save_on_claim
198
- @save_on_claim = true
199
- end
200
-
201
- # Checks if save_on_claim is enabled.
202
- def save_on_claim?
203
- @save_on_claim
204
- end
205
-
206
- # Adds a new node to the inventory.
207
- # @param node_name [String] The name of the node to add.
208
- # @param node_data [Hash] The node data to add.
209
- def add(node_name, node_data)
210
- @inventory.put_if_absent(node_name, node_data)
211
- end
212
-
213
- def update(node_name, node_data)
214
- @inventory.replace_if_exists(node_name, node_data)
215
- end
216
-
217
- # Returns the node data for a given node.
218
- # @param node_name [String] The name of the node to get data for.
219
- def get(node_name)
220
- @inventory[node_name]
221
- end
222
-
223
- # Returns the node_name and node_data for a given node
224
- # that has a matching value for the given property.
225
- # @param property_path [String] The dot-separated property path to check.
226
- # @param value [Object] The value to check for.
227
- # @return [Array] Two-item frozen array of node name and node data.
228
- # @raise [NodeDoesNotExistError] If no nodes are found with the property equal to the value.
229
- def get_by_property(property_path, value)
230
- found = []
231
- @inventory.each_pair do |node_name, node_data|
232
- next unless node_data.dot_dig(property_path) == value
233
-
234
- found = [node_name, node_data].freeze
235
- end
236
- raise NodeDoesNotExistError, "No node found with property #{property_path} == #{value}" if found.empty?
237
-
238
- found
239
- end
240
-
241
- # Returns all property values of all nodes for the given property path
242
- # @param property_path [String] The dot-separated property path to check.
243
- # @return [Array] A frozen array of found property values.
244
- # @raise [PropertyNotFoundError] If no values are found on any nodes for the property path.
245
- def get_all_properties(property_path)
246
- found = []
247
- @inventory.each_pair do |_, node_data|
248
- prop = node_data.dot_dig(property_path)
249
- next unless prop
250
-
251
- found << prop
252
- end
253
- raise PropertyNotFoundError, "No property with path #{property_path} found in nodes" if found.empty?
254
-
255
- found.freeze
256
- end
257
-
258
- # Sets a specific property on a node in the inventory.
259
- # @param node_name [String] The name of the node to set the property on.
260
- # @param property_path [String] The dot-separated property path to set.
261
- # @param value [Object] The value to set.
262
- def set(node_name, property_path, value)
263
- @inventory[node_name].dot_store(property_path, value)
264
- end
265
-
266
- # Merges new node data in with the existing node data for the given node.
267
- # @param node_name [String] The name of the node to merge data for.
268
- # @param new_node_data [Hash] The new node data to merge.
269
- def merge!(node_name, new_node_data)
270
- @inventory.merge_pair(node_name, new_node_data) do |old_data|
271
- old_data.deep_merge(new_node_data)
272
- end
273
- end
274
-
275
- # Checks if a node is claimed.
276
- # @param node_name [String] The name of the node to check.
277
- # @return [Boolean] True if the node is claimed, false otherwise.
278
- def claimed?(node_name)
279
- raise NodeDoesNotExistError, "Node #{node_name} does not exist" unless @inventory.keys.include?(node_name)
280
-
281
- @claimed.include?(node_name)
282
- end
283
-
284
- # Claims a node, which returns the node data and adds that node to the
285
- # claimed set. Raises an error if the node is already claimed or does
286
- # not exist. This is used during acceptance testing to ensure that
287
- # nodes are not reused.
288
- # @param node_name [String] The name of the node to claim.
289
- # @return [String] The name of the node that was claimed.
290
- # @raise [NodeDoesNotExistError] If the node does not exist in the inventory.
291
- # @raise [NodeClaimedError] If the node is already claimed.
292
- def claim(node_name)
293
- with_lock_retry(:write) do
294
- unless @inventory.keys.include?(node_name)
295
- raise NodeDoesNotExistError, "Node #{node_name} does not exist in inventory"
296
- end
297
-
298
- if @claimed.include?(node_name)
299
- raise NodeClaimedError, "Node #{node_name} is already tainted and cannot be claimed"
300
- end
301
-
302
- @claimed.add(node_name)
303
- save_no_lock!(@save_file_path) if @save_on_claim
304
- node_name
305
- end
306
- end
307
-
308
- # Claims a node based on a property of the node data and a value that property should equal.
309
- # If a node is found but is already claimed, the method will continue to search for an
310
- # unclaimed node. Raises an error if no valid node is found. This is used during acceptance
311
- # testing to ensure that nodes are not reused.
312
- # @param property_path [String] The dot-separated property path to check.
313
- # @param value [Object] The value to check for.
314
- # @return [String] The name of the node that was claimed.
315
- # @raise [NodeDoesNotExistError] If no valid node is found.
316
- def claim_by_property(property_path, value)
317
- attempts ||= 1
318
- claim_name = nil
319
- @inventory.each_pair do |node_name, node_data|
320
- next if @claimed.include?(node_name)
321
-
322
- claim_name = node_name if node_data.dot_dig(property_path) == value
323
- end
324
- raise NodeDoesNotExistError, "No node found with property #{property_path} == #{value}" if claim_name.nil?
325
-
326
- claim(claim_name)
327
- rescue NodeDoesNotExistError => e
328
- # We sleep than retry three times to help mitigate race conditions
329
- if (attempts += 1) <= 3
330
- sleep(1)
331
- retry
332
- else
333
- raise e
334
- end
335
- end
336
-
337
- # Deletes a node from the inventory.
338
- # @param node_name [String] The name of the node to delete.
339
- def delete(node_name)
340
- @inventory.delete(node_name)
341
- end
342
-
343
- # Returns all node names in the current inventory.
344
- def nodes
345
- @inventory.keys
346
- end
347
-
348
- # Clears the inventory and removes claimed nodes. DOES NOT USE LOCK.
349
- # If this is called outside of a lock, it will not be thread-safe.
350
- def clear_no_lock!
351
- nodes.each { |node| delete(node) }
352
- @claimed.clear
353
- end
354
-
355
- # Clears the inventory and removes claimed nodes. Thread-safe.
356
- def clear!
357
- with_lock_retry(:write) do
358
- clear_no_lock!
359
- end
360
- end
361
-
362
- # Checks if the inventory contains a node with the given property.
363
- # @param node_name [String] The name of the node to check.
364
- # @param property_path [String] The dot-separated property path to check.
365
- def property?(node_name, property_path)
366
- !!@inventory[node_name].dot_dig(property_path)
367
- end
368
-
369
- # Returns a hash of the inventory.
370
- def to_h
371
- h = {}
372
- @inventory.each_pair do |node_name, node_data|
373
- h[node_name] = node_data.to_h
374
- end
375
- h[:claimed] = @claimed.to_a
376
- h
377
- end
378
-
379
- # Returns a YAML string of a specific node's node data
380
- def node_to_yaml(node_name)
381
- get(node_name).to_h.to_yaml
382
- end
383
-
384
- # Returns a JSON string of a specific node's node data
385
- def node_to_json(node_name, *args)
386
- get(node_name).to_h.to_json(*args)
387
- end
388
-
389
- # Returns a YAML string of the inventory.
390
- def to_yaml
391
- to_h.to_yaml
392
- end
393
-
394
- # Returns a JSON string of the inventory.
395
- def to_json(*args)
396
- to_h.to_json(*args)
397
- end
398
-
399
- # Saves the current node inventory to a file. DOES NOT USE LOCK.
400
- # If this is called outside of a lock, it will not be thread-safe.
401
- def save_no_lock!(file_path = @save_file_path)
402
- Dir.mkdir(file_path) unless Dir.exist?(file_path)
403
- @inventory.each_pair do |node_name, node_data|
404
- path = File.join(file_path, "#{node_name}.yaml")
405
- next if File.file?(path)
406
-
407
- File.open(path, 'w') do |f|
408
- f.write(node_data.to_h.to_yaml)
409
- end
410
- end
411
- end
412
-
413
- # Saves the current node inventory to a yaml file. Thread-safe.
414
- # @param file_path [String] The path to the file to save to.
415
- def save(file_path = @save_file_path)
416
- with_lock_retry(:write) do
417
- save_no_lock!(file_path)
418
- end
419
- end
420
-
421
- # Loads a node inventory from a file. DOES NOT USE LOCK.
422
- # If this is called outside of a lock, it will not be thread-safe.
423
- def load_no_lock!(file_path = @save_file_path)
424
- require 'net/ssh/proxy/command' # If ProxyCommand is used in ssh options, this is required.
425
- attempts ||= 1
426
- Dir.glob('*.yaml', base: file_path) do |file_name|
427
- node_name = File.basename(file_name, '.yaml')
428
- node_data = YAML.load_file(File.join(file_path, file_name))
429
- add(node_name, node_data) || update(node_name, node_data)
430
- end
431
- rescue StandardError => e
432
- raise e unless (attempts += 1) <= 3
433
-
434
- sleep(1)
435
- retry
436
- end
437
-
438
- def update_no_lock!; end
439
-
440
- # Loads a node inventory from a yaml file. Thread-safe.
441
- # @param file_path [String] The path to the file to load from.
442
- def load(file_path = @save_file_path)
443
- with_lock_retry(:write) do
444
- load_no_lock!(file_path)
445
- end
446
- end
447
-
448
- def clean_local_files(file_path = @save_file_path)
449
- Dir.glob('*.yaml', base: file_path) do |file_name|
450
- File.delete(File.expand_path(File.join(file_path, file_name)))
451
- end
452
- end
453
-
454
- private
455
-
456
- def with_lock_retry(lock_type, max_attempts: 3)
457
- raise 'No block given' unless block_given?
458
-
459
- attempts ||= 1
460
- case lock_type
461
- when :read
462
- @lock.with_read_lock { yield }
463
- when :write
464
- @lock.with_write_lock { yield }
465
- else
466
- raise LockNotRecognizedError, "Lock type #{lock_type} not recognized! Must be :read or :write!"
467
- end
468
- rescue Concurrent::ResourceLimitError => e
469
- if (attempts += 1) <= max_attempts
470
- sleep(1)
471
- logger.debug("Failed to acquire lock with error #{e.message}. Retrying...")
472
- retry
473
- else
474
- raise LockWaitTimeoutError, 'Lock could not be aquired'
475
- end
476
- end
477
- end
478
-
479
- class LocalPortAllocationError < StandardError; end
480
-
481
- # Handles local port allocation for things like local SSH tunnels.
482
- # This class is thread-safe as the port range is a Concurrent::Array,
483
- # which blocks when modified. When this class is initialized, it's
484
- # total assignable ports are set to all ports in the ephemeral port range.
485
- # When a port is allocated, it's removed from the port range.
486
- class LocalPortAllocator
487
- EPHEMERAL_PORT_RANGE = (49_152..65_535).to_a.freeze
488
-
489
- def initialize
490
- @port_range = Concurrent::Array.new(EPHEMERAL_PORT_RANGE.dup).shuffle!
491
- end
492
-
493
- # Returns an open port(s) in the ephemeral port range on the local machine.
494
- # If 'number_of_ports' is specified and greater than one, returns an array of ports.
495
- # If 'number_of_ports' is not specified or is one, returns a single port.
496
- # @param number_of_ports [Integer] the number of ports to allocate
497
- # @return [Integer, Array<Integer>] the port(s) allocated
498
- # @raise [ArgumentError] if number_of_ports is not an Integer or is less than one
499
- # @raise [LocalPortAllocationError] if there are no ports available
500
- def allocate(number_of_ports = 1)
501
- unless number_of_ports.is_a?(Integer) && number_of_ports.positive?
502
- raise ArgumentError, 'number_of_ports must be a positive integer'
503
- end
504
- raise LocalPortAllocationError, 'No ports available' if @port_range.empty?
505
-
506
- ports = []
507
- while ports.length < number_of_ports
508
- port = @port_range.pop
509
- next unless local_port_open?(port)
510
-
511
- ports << port
512
- end
513
- if ports.length == 1
514
- ports[0]
515
- else
516
- ports
517
- end
518
- end
519
-
520
- private
521
-
522
- # Checks if the given port is open on the local machine. This is done by
523
- # opening a socket on the port and checking if it raises an exception.
524
- # @param port [Integer] the port to check
525
- # @return [Boolean] true if the port is open, false otherwise
526
- def local_port_open?(port)
527
- require 'socket'
528
- socket = Socket.new(Socket::Constants::AF_INET, Socket::Constants::SOCK_STREAM, 0)
529
- socket.bind(Socket.pack_sockaddr_in(port, '0.0.0.0'))
530
- true
531
- rescue Errno::EADDRINUSE, Errno::CONNREFUSED
532
- false
533
- ensure
534
- socket&.close
535
- end
536
- end
537
- end