cem_acpt 0.1.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -0,0 +1,416 @@
1
+ # frozen_string_literal: true
2
+
3
+ require 'concurrent-ruby'
4
+ require 'deep_merge'
5
+ require 'json'
6
+ require 'ostruct'
7
+ require 'yaml'
8
+ require_relative 'core_extensions'
9
+ require_relative 'logging'
10
+
11
+ module CemAcpt
12
+ using CemAcpt::CoreExtensions::ExtendedHash
13
+
14
+ class ConfigImmutableError < RuntimeError; end
15
+
16
+ # Holds module methods for writing various config objects
17
+ # to files.
18
+ module ConfigUtils
19
+ class << self
20
+ include CemAcpt::Logging
21
+ end
22
+
23
+ def self.object_to_file(obj, file_path)
24
+ case File.extname(file_path)
25
+ when '.json'
26
+ raise "Object #{obj} is not serializable to JSON" unless obj.respond_to?(:to_json)
27
+
28
+ logger.debug("Writing object to #{file_path}")
29
+ File.write(file_path, obj.to_json)
30
+ when '.yaml', '.yml'
31
+ raise "Object #{obj} is not serializable to YAML" unless obj.respond_to?(:to_yaml)
32
+
33
+ logger.debug("Writing object to #{file_path}")
34
+ File.write(file_path, obj.to_yaml)
35
+ else
36
+ raise "File extension #{File.extname(file_path)} is not supported"
37
+ end
38
+ end
39
+ end
40
+
41
+ # Provides a thread-safe Config object.
42
+ # Once the config is loaded by calling the `load` method, the object is
43
+ # immutable and thread-safe as long as the internal hash is only accessed
44
+ # by the `#get` and `#has?` methods. If `#load` is called more than once,
45
+ # the object will raise an error.
46
+ class Config
47
+ include CemAcpt::Logging
48
+
49
+ attr_reader :config_file
50
+
51
+ def initialize
52
+ @opts = {}
53
+ end
54
+
55
+ # Returns the value of the dot-separated key.
56
+ # If the key is not found, returns nil.
57
+ # @param dot_key [String] Dot-separated key
58
+ # @return [Object] Value of the key
59
+ def get(dot_key)
60
+ @opts.dot_dig(dot_key)
61
+ end
62
+
63
+ # Checks if the dot-separated key exists in the config.
64
+ # @param dot_key [String] Dot-separated key
65
+ # @return [Boolean] True if the key exists, false otherwise
66
+ def has?(dot_key)
67
+ !!get(dot_key)
68
+ end
69
+
70
+ # Loads the config from specified file path. Config files must be in YAML
71
+ # format. If 'opts' is specified, it will be merged with the config file.
72
+ # Values in 'opts' will override values in the config file.
73
+ # @param opts [Hash] Options to be merged with the config file
74
+ # @param config_file [String] Path to the config file
75
+ def load(opts: {}, config_file: '~/.cem_acpt.yaml')
76
+ raise ConfigImmutableError, 'Config is immutable, cannot load more than once' if frozen?
77
+ raise ArgumentError, 'opts must be a Hash' unless opts.is_a?(Hash)
78
+ raise ArgumentError, 'config_file must be a String' unless config_file.is_a?(String)
79
+
80
+ @config_file = File.expand_path(config_file)
81
+ @opts = load_opts_from_file(File.expand_path(config_file)).deep_merge(opts)
82
+ @opts.format!
83
+ @opts.freeze
84
+ freeze
85
+ end
86
+
87
+ # Returns the config as a Hash.
88
+ # @return [Hash] Config as a Hash
89
+ def to_h
90
+ ::Marshal.load(::Marshal.dump(@opts))
91
+ end
92
+
93
+ # Returns the config as a YAML string.
94
+ # @return [String] Config as a YAML string
95
+ def to_yaml
96
+ to_h.to_yaml
97
+ end
98
+
99
+ # Returns the config as a JSON string.
100
+ # @return [String] Config as a JSON string
101
+ def to_json(*args)
102
+ to_h.to_json(*args)
103
+ end
104
+
105
+ private
106
+
107
+ # Loads the config from the specified file path.
108
+ # @param config_file [String] Path to the config file
109
+ # @return [Hash] Config hash
110
+ def load_opts_from_file(config_file)
111
+ return {} unless File.exist?(config_file)
112
+
113
+ logger.debug("Loading config from #{config_file}")
114
+ YAML.load_file(config_file)
115
+ end
116
+ end
117
+
118
+ # Provides an object to hold NodeData configs.
119
+ # Not thread-safe. Should not be mutated by itself,
120
+ # but can be mutated safely by the NodeInventory object.
121
+ # Currently unused.
122
+ class NodeData
123
+ include CemAcpt::Logging
124
+
125
+ attr_reader :name, :data
126
+
127
+ def initialize(name, data)
128
+ data.format!
129
+ @name = name
130
+ @opts = data
131
+ end
132
+
133
+ def deep_merge(other)
134
+ @opts.deep_merge(other.to_h)
135
+ end
136
+
137
+ def dot_dig(dot_key)
138
+ @opts.dot_dig(dot_key)
139
+ end
140
+
141
+ def dot_store(dot_key, value)
142
+ @opts.dot_store(dot_key, value)
143
+ end
144
+
145
+ def to_h
146
+ @opts
147
+ end
148
+
149
+ def to_yaml
150
+ to_h.to_yaml
151
+ end
152
+
153
+ def to_json(*args)
154
+ to_h.to_json(*args)
155
+ end
156
+ end
157
+
158
+ class NodeClaimedError < StandardError; end
159
+ class NodeDoesNotExistError < StandardError; end
160
+
161
+ # Provides a thread-safe inventory of test nodes.
162
+ class NodeInventory
163
+ include CemAcpt::Logging
164
+
165
+ attr_accessor :save_file_path
166
+
167
+ def initialize
168
+ @inventory = Concurrent::Map.new
169
+ @lock = Concurrent::ReadWriteLock.new
170
+ @claimed = Concurrent::Set.new
171
+ @save_on_claim = false
172
+ @save_file_path = 'spec/fixtures/node_inventory.yaml'
173
+ end
174
+
175
+ # When called, enables saving the inventory to a file on claim.
176
+ def save_on_claim
177
+ @save_on_claim = true
178
+ end
179
+
180
+ # Checks if save_on_claim is enabled.
181
+ def save_on_claim?
182
+ @save_on_claim
183
+ end
184
+
185
+ # Adds a new node to the inventory.
186
+ # @param node_name [String] The name of the node to add.
187
+ # @param node_data [Hash] The node data to add.
188
+ def add(node_name, node_data)
189
+ @inventory.put_if_absent(node_name, node_data)
190
+ end
191
+
192
+ # Returns the node data for a given node.
193
+ # @param node_name [String] The name of the node to get data for.
194
+ def get(node_name)
195
+ @inventory[node_name]
196
+ end
197
+
198
+ # Sets a specific property on a node in the inventory.
199
+ # @param node_name [String] The name of the node to set the property on.
200
+ # @param property_path [String] The dot-separated property path to set.
201
+ # @param value [Object] The value to set.
202
+ def set(node_name, property_path, value)
203
+ @inventory[node_name].dot_store(property_path, value)
204
+ end
205
+
206
+ # Merges new node data in with the existing node data for the given node.
207
+ # @param node_name [String] The name of the node to merge data for.
208
+ # @param new_node_data [Hash] The new node data to merge.
209
+ def merge!(node_name, new_node_data)
210
+ @inventory.merge_pair(node_name, new_node_data) do |old_data|
211
+ old_data.deep_merge(new_node_data)
212
+ end
213
+ end
214
+
215
+ # Checks if a node is claimed.
216
+ # @param node_name [String] The name of the node to check.
217
+ # @return [Boolean] True if the node is claimed, false otherwise.
218
+ def claimed?(node_name)
219
+ raise NodeDoesNotExistError, "Node #{node_name} does not exist" unless @inventory.keys.include?(node_name)
220
+
221
+ @claimed.include?(node_name)
222
+ end
223
+
224
+ # Claims a node, which returns the node data and adds that node to the
225
+ # claimed set. Raises an error if the node is already claimed or does
226
+ # not exist. This is used during acceptance testing to ensure that
227
+ # nodes are not reused.
228
+ # @param node_name [String] The name of the node to claim.
229
+ # @return [String] The name of the node that was claimed.
230
+ # @raise [NodeDoesNotExistError] If the node does not exist in the inventory.
231
+ # @raise [NodeClaimedError] If the node is already claimed.
232
+ def claim(node_name)
233
+ @lock.with_write_lock do
234
+ unless @inventory.keys.include?(node_name)
235
+ raise NodeDoesNotExistError, "Node #{node_name} does not exist in inventory"
236
+ end
237
+
238
+ if @claimed.include?(node_name)
239
+ raise NodeClaimedError, "Node #{node_name} is already tainted and cannot be claimed"
240
+ end
241
+
242
+ @claimed.add(node_name)
243
+ save_no_lock!(@save_file_path) if @save_on_claim
244
+ node_name
245
+ end
246
+ end
247
+
248
+ # Claims a node based on a property of the node data and a value that property should equal.
249
+ # If a node is found but is already claimed, the method will continue to search for an
250
+ # unclaimed node. Raises an error if no valid node is found. This is used during acceptance
251
+ # testing to ensure that nodes are not reused.
252
+ # @param property_path [String] The dot-separated property path to check.
253
+ # @param value [Object] The value to check for.
254
+ # @return [String] The name of the node that was claimed.
255
+ # @raise [NodeDoesNotExistError] If no valid node is found.
256
+ def claim_by_property(property_path, value)
257
+ claimed_set = @claimed.dup
258
+ claim_name = nil
259
+ @inventory.each_pair do |node_name, node_data|
260
+ next if claimed_set.include?(node_name)
261
+
262
+ claim_name = node_name if node_data.dot_dig(property_path) == value
263
+ end
264
+ return claim(claim_name) unless claim_name.nil?
265
+
266
+ raise NodeDoesNotExistError, "No node found with property #{property_path} == #{value}"
267
+ end
268
+
269
+ # Deletes a node from the inventory.
270
+ # @param node_name [String] The name of the node to delete.
271
+ def delete(node_name)
272
+ @inventory.delete(node_name)
273
+ end
274
+
275
+ # Returns all node names in the current inventory.
276
+ def nodes
277
+ @inventory.keys
278
+ end
279
+
280
+ # Clears the inventory and removes claimed nodes. DOES NOT USE LOCK.
281
+ # If this is called outside of a lock, it will not be thread-safe.
282
+ def clear_no_lock!
283
+ nodes.each { |node| delete(node) }
284
+ @claimed.clear
285
+ end
286
+
287
+ # Clears the inventory and removes claimed nodes. Thread-safe.
288
+ def clear!
289
+ @lock.with_write_lock do
290
+ clear_no_lock!
291
+ end
292
+ end
293
+
294
+ # Checks if the inventory contains a node with the given property.
295
+ # @param node_name [String] The name of the node to check.
296
+ # @param property_path [String] The dot-separated property path to check.
297
+ def property?(node_name, property_path)
298
+ !!@inventory[node_name].dot_dig(property_path)
299
+ end
300
+
301
+ # Returns a hash of the inventory.
302
+ def to_h
303
+ h = {}
304
+ @inventory.each_pair do |node_name, node_data|
305
+ h[node_name] = node_data.to_h
306
+ end
307
+ h[:claimed] = @claimed.to_a
308
+ h
309
+ end
310
+
311
+ # Returns a YAML string of the inventory.
312
+ def to_yaml
313
+ to_h.to_yaml
314
+ end
315
+
316
+ # Returns a JSON string of the inventory.
317
+ def to_json(*args)
318
+ to_h.to_json(*args)
319
+ end
320
+
321
+ # Saves the current node inventory to a file. DOES NOT USE LOCK.
322
+ # If this is called outside of a lock, it will not be thread-safe.
323
+ def save_no_lock!(file_path = @save_file_path)
324
+ File.open(file_path, 'w') do |file|
325
+ file.write(to_yaml)
326
+ end
327
+ end
328
+
329
+ # Saves the current node inventory to a yaml file. Thread-safe.
330
+ # @param file_path [String] The path to the file to save to.
331
+ def save(file_path = @save_file_path)
332
+ @lock.with_write_lock do
333
+ save_no_lock!(file_path)
334
+ end
335
+ end
336
+
337
+ # Loads a node inventory from a file. DOES NOT USE LOCK.
338
+ # If this is called outside of a lock, it will not be thread-safe.
339
+ def load_no_lock!(file_path = @save_file_path)
340
+ require 'net/ssh/proxy/command' # If ProxyCommand is used in ssh options, this is required.
341
+ clear_no_lock!
342
+ YAML.load_file(file_path).each_pair do |node_name, node_data|
343
+ if node_name == :claimed
344
+ @claimed = Set.new(node_data)
345
+ else
346
+ add(node_name, node_data)
347
+ end
348
+ end
349
+ end
350
+
351
+ # Loads a node inventory from a yaml file. Thread-safe.
352
+ # @param file_path [String] The path to the file to load from.
353
+ def load(file_path = @save_file_path)
354
+ @lock.with_write_lock do
355
+ load_no_lock!(file_path)
356
+ end
357
+ end
358
+ end
359
+
360
+ class LocalPortAllocationError < StandardError; end
361
+
362
+ # Handles local port allocation for things like local SSH tunnels.
363
+ # This class is thread-safe as the port range is a Concurrent::Array,
364
+ # which blocks when modified. When this class is initialized, it's
365
+ # total assignable ports are set to all ports in the ephemeral port range.
366
+ # When a port is allocated, it's removed from the port range.
367
+ class LocalPortAllocator
368
+ EPHEMERAL_PORT_RANGE = (49_152..65_535).to_a.freeze
369
+
370
+ def initialize
371
+ @port_range = Concurrent::Array.new(EPHEMERAL_PORT_RANGE.dup).shuffle!
372
+ end
373
+
374
+ # Returns an open port(s) in the ephemeral port range on the local machine.
375
+ # If 'number_of_ports' is specified and greater than one, returns an array of ports.
376
+ # If 'number_of_ports' is not specified or is one, returns a single port.
377
+ # @param number_of_ports [Integer] the number of ports to allocate
378
+ # @return [Integer, Array<Integer>] the port(s) allocated
379
+ # @raise [ArgumentError] if number_of_ports is not an Integer or is less than one
380
+ # @raise [LocalPortAllocationError] if there are no ports available
381
+ def allocate(number_of_ports = 1)
382
+ unless number_of_ports.is_a?(Integer) && number_of_ports.positive?
383
+ raise ArgumentError, 'number_of_ports must be a positive integer'
384
+ end
385
+ raise LocalPortAllocationError, 'No ports available' if @port_range.empty?
386
+
387
+ ports = []
388
+ while ports.length < number_of_ports
389
+ port = @port_range.pop
390
+ next unless local_port_open?(port)
391
+
392
+ ports << port
393
+ end
394
+ return ports[0] if ports.length == 1
395
+
396
+ ports
397
+ end
398
+
399
+ private
400
+
401
+ # Checks if the given port is open on the local machine. This is done by
402
+ # opening a socket on the port and checking if it raises an exception.
403
+ # @param port [Integer] the port to check
404
+ # @return [Boolean] true if the port is open, false otherwise
405
+ def local_port_open?(port)
406
+ require 'socket'
407
+ socket = Socket.new(Socket::Constants::AF_INET, Socket::Constants::SOCK_STREAM, 0)
408
+ socket.bind(Socket.pack_sockaddr_in(port, '0.0.0.0'))
409
+ true
410
+ rescue Errno::EADDRINUSE, Errno::CONNREFUSED
411
+ false
412
+ ensure
413
+ socket&.close
414
+ end
415
+ end
416
+ end
@@ -0,0 +1,176 @@
1
+ # frozen_string_literal: true
2
+
3
+ module CemAcpt
4
+ require_relative 'shared_objects'
5
+
6
+ # This methods is used in spec_helper_acceptance.rb to set up
7
+ # baseline configurations used in acceptance tests.
8
+ def self.configure_spec_helper!
9
+ require 'serverspec'
10
+ require 'yaml'
11
+
12
+ RSpec.configure do |config|
13
+ config.include CemAcpt::SpecHelperAcceptance
14
+ config.extend CemAcpt::SpecHelperAcceptance
15
+ config.add_setting :acpt_test_data, default: {}
16
+ config.add_setting :acpt_node_inventory
17
+ end
18
+
19
+ return unless RSpec.configuration.acpt_node_inventory.nil?
20
+
21
+ raise 'Node inventory file not found!' unless File.exist?('spec/fixtures/node_inventory.yaml')
22
+
23
+ node_inventory = NodeInventory.new
24
+ node_inventory.save_on_claim
25
+ node_inventory.save_file_path = 'spec/fixtures/node_inventory.yaml'
26
+ node_inventory.load
27
+ RSpec.configuration.acpt_node_inventory = node_inventory
28
+ end
29
+
30
+ # This module provides methods used in accpetance tests.
31
+ module SpecHelperAcceptance
32
+ # This method must be used inside of a `describe` block as the first
33
+ # statement. This prepares the ServerSpec configuration for the test node
34
+ # and sets up the test data.
35
+ def initialize_test_environment!
36
+ test_file = caller_locations.first.path
37
+
38
+ node_name = RSpec.configuration.acpt_node_inventory.claim_by_property('test_data.test_file', test_file)
39
+ node_data = RSpec.configuration.acpt_node_inventory.get(node_name)
40
+ backend = nil
41
+ host = nil
42
+ ssh_options = nil
43
+ puppet_path = nil
44
+
45
+ # Set remote communication variables based on transport type
46
+ case node_data[:transport]
47
+ when /ssh/
48
+ backend = :ssh
49
+ host = node_data[:node_name]
50
+ ssh_options = node_data[:ssh_opts]
51
+ sudo_options = '-n -u root -i'
52
+ puppet_path = '/opt/puppetlabs/bin/puppet'
53
+ when /winrm/
54
+ backend = :winrm
55
+ puppet_path = 'C:\Program Files\Puppet Labs\Puppet\bin\puppet.bat'
56
+ else
57
+ raise "Unknown transport: #{node[:transport]}"
58
+ end
59
+
60
+ # Set serverspec transport options and host for remote communication.
61
+ set :backend, backend
62
+ set :host, host
63
+ set(:ssh_options, ssh_options) if ssh_options
64
+ set(:sudo_options, sudo_options) if sudo_options
65
+ set(:os, family: 'windows') if backend == :winrm
66
+
67
+ # Get the command provider from the node's platform
68
+ # We add this as a RSpec config option so that we can use it in
69
+ # other functions.
70
+ require 'cem_acpt/platform'
71
+
72
+ acpt_test_data = {
73
+ test_file: test_file,
74
+ node_name: node_name,
75
+ node_data: node_data,
76
+ backend: backend,
77
+ platform: CemAcpt::Platform.get(node_data[:platform]),
78
+ puppet_path: puppet_path,
79
+ }
80
+ acpt_test_data[:host] = host if host
81
+ RSpec.configuration.acpt_test_data = acpt_test_data
82
+ end
83
+
84
+ # This method formats Puppet Apply options
85
+ def puppet_apply_options(opts = {})
86
+ if [opts[:catch_changes], opts[:expect_changes], opts[:catch_failures], opts[:expect_failures]].compact.length > 1
87
+ raise ArgumentError, 'Please specify only one of "catch_changes", "expect_changes", "catch_failures", or "expect_failures"'
88
+ end
89
+
90
+ apply_opts = { trace: true }.merge(opts)
91
+
92
+ if opts[:catch_changes]
93
+ apply_opts[:detailed_exit_codes] = true
94
+ apply_opts[:acceptable_exit_codes] = [0]
95
+ elsif opts[:catch_failures]
96
+ apply_opts[:detailed_exit_codes] = true
97
+ apply_opts[:acceptable_exit_codes] = [0, 2]
98
+ elsif opts[:expect_failures]
99
+ apply_opts[:detailed_exit_codes] = true
100
+ apply_opts[:acceptable_exit_codes] = [1, 4, 6]
101
+ elsif opts[:expect_changes]
102
+ apply_opts[:detailed_exit_codes] = true
103
+ apply_opts[:acceptable_exit_codes] = [2]
104
+ else
105
+ apply_opts[:detailed_exit_codes] = false
106
+ apply_opts[:acceptable_exit_codes] = [0]
107
+ end
108
+ apply_opts
109
+ end
110
+
111
+ # This methods handles formatting the output from Puppet apply.
112
+ def handle_puppet_apply_output(result, apply_opts)
113
+ exit_code = result.exitstatus
114
+ output = result.to_s
115
+ if apply_opts[:catch_changes] && !apply_opts[:acceptable_exit_codes].include?(exit_code)
116
+ failure = <<~ERROR
117
+ Apply manifest expected no changes. Puppet Apply returned exit code #{exit_code}
118
+ ====== Start output of Puppet apply with unexpected changes ======
119
+ #{output}
120
+ ====== End output of Puppet apply with unexpected changes ======
121
+ ERROR
122
+ raise failure
123
+ elsif !apply_opts[:acceptable_exit_codes].include?(exit_code)
124
+ failure = <<~ERROR
125
+ Apply manifest failed with exit code #{exit_code} (expected: #{apply_opts[:acceptable_exit_codes]})
126
+ ====== Start output of failed Puppet apply ======
127
+ #{output}
128
+ ====== End output of failed Puppet apply ======
129
+ ERROR
130
+ raise failure
131
+ end
132
+
133
+ yield result if block_given?
134
+
135
+ if ENV['RSPEC_DEBUG']
136
+ run_result = <<~RUNRES
137
+ apply manifest succeded with status #{exit_code}
138
+ ===== Start output of successful Puppet apply ======
139
+ #{output}
140
+ ===== End output of successful Puppet apply ======
141
+ RUNRES
142
+ puts run_result
143
+ end
144
+ result
145
+ end
146
+
147
+ # This method runs a shell command on the test node.
148
+ def run_shell(cmd, opts = {})
149
+ cmd = cmd.join(' ') if cmd.is_a?(Array)
150
+
151
+ host = RSpec.configuration.acpt_test_data[:node_name]
152
+ opts[:ssh_opts] = RSpec.configuration.acpt_test_data[:node_data][:ssh_opts]
153
+ RSpec.configuration.acpt_test_data[:platform].run_shell(host, cmd, opts)
154
+ end
155
+
156
+ # This method runs puppet apply on the test node using the provided manifest.
157
+ def apply_manifest(manifest, opts = {})
158
+ host = RSpec.configuration.acpt_test_data[:node_name]
159
+ opts = {
160
+ apply: puppet_apply_options(opts),
161
+ ssh_opts: RSpec.configuration.acpt_test_data[:node_data][:ssh_opts],
162
+ puppet_path: RSpec.configuration.acpt_test_data[:puppet_path],
163
+ }
164
+ result = RSpec.configuration.acpt_test_data[:platform].apply_manifest(host, manifest, opts)
165
+ handle_puppet_apply_output(result, opts[:apply])
166
+ end
167
+
168
+ # This method runs puppet apply on the test node using the provided manifest twice and
169
+ # asserts that the second run has no changes.
170
+ def idempotent_apply(manifest, opts = {})
171
+ opts.reject! { |k, _| %i[catch_changes expect_changes catch_failures expect_failures].include?(k) }
172
+ apply_manifest(manifest, opts.merge(catch_failures: true))
173
+ apply_manifest(manifest, opts.merge(catch_changes: true))
174
+ end
175
+ end
176
+ end