cem_acpt 0.2.5 → 0.6.0

Sign up to get free protection for your applications and to get access to all the features.
Files changed (58) hide show
  1. checksums.yaml +4 -4
  2. data/.github/workflows/spec.yml +30 -0
  3. data/Gemfile +4 -3
  4. data/Gemfile.lock +95 -43
  5. data/README.md +144 -83
  6. data/cem_acpt.gemspec +12 -7
  7. data/exe/cem_acpt +41 -7
  8. data/lib/cem_acpt/config.rb +340 -0
  9. data/lib/cem_acpt/core_extensions.rb +17 -61
  10. data/lib/cem_acpt/goss/api/action_response.rb +175 -0
  11. data/lib/cem_acpt/goss/api.rb +83 -0
  12. data/lib/cem_acpt/goss.rb +8 -0
  13. data/lib/cem_acpt/image_name_builder.rb +0 -9
  14. data/lib/cem_acpt/logging/formatter.rb +97 -0
  15. data/lib/cem_acpt/logging.rb +168 -142
  16. data/lib/cem_acpt/platform/base.rb +26 -37
  17. data/lib/cem_acpt/platform/gcp.rb +48 -62
  18. data/lib/cem_acpt/platform.rb +30 -28
  19. data/lib/cem_acpt/provision/terraform/linux.rb +47 -0
  20. data/lib/cem_acpt/provision/terraform/os_data.rb +72 -0
  21. data/lib/cem_acpt/provision/terraform/windows.rb +22 -0
  22. data/lib/cem_acpt/provision/terraform.rb +193 -0
  23. data/lib/cem_acpt/provision.rb +20 -0
  24. data/lib/cem_acpt/puppet_helpers.rb +0 -1
  25. data/lib/cem_acpt/test_data.rb +23 -13
  26. data/lib/cem_acpt/test_runner/log_formatter/goss_action_response.rb +104 -0
  27. data/lib/cem_acpt/test_runner/log_formatter.rb +10 -0
  28. data/lib/cem_acpt/test_runner.rb +170 -3
  29. data/lib/cem_acpt/utils/puppet.rb +29 -0
  30. data/lib/cem_acpt/utils/ssh.rb +197 -0
  31. data/lib/cem_acpt/utils/terminal.rb +27 -0
  32. data/lib/cem_acpt/utils.rb +4 -138
  33. data/lib/cem_acpt/version.rb +1 -1
  34. data/lib/cem_acpt.rb +73 -23
  35. data/lib/terraform/gcp/linux/goss/puppet_idempotent.yaml +10 -0
  36. data/lib/terraform/gcp/linux/goss/puppet_noop.yaml +12 -0
  37. data/lib/terraform/gcp/linux/main.tf +191 -0
  38. data/lib/terraform/gcp/linux/systemd/goss-acpt.service +8 -0
  39. data/lib/terraform/gcp/linux/systemd/goss-idempotent.service +8 -0
  40. data/lib/terraform/gcp/linux/systemd/goss-noop.service +8 -0
  41. data/lib/terraform/gcp/windows/.keep +0 -0
  42. data/sample_config.yaml +22 -21
  43. metadata +151 -51
  44. data/lib/cem_acpt/bootstrap/bootstrapper.rb +0 -206
  45. data/lib/cem_acpt/bootstrap/operating_system/rhel_family.rb +0 -129
  46. data/lib/cem_acpt/bootstrap/operating_system.rb +0 -17
  47. data/lib/cem_acpt/bootstrap.rb +0 -12
  48. data/lib/cem_acpt/context.rb +0 -153
  49. data/lib/cem_acpt/platform/base/cmd.rb +0 -71
  50. data/lib/cem_acpt/platform/gcp/cmd.rb +0 -345
  51. data/lib/cem_acpt/platform/gcp/compute.rb +0 -332
  52. data/lib/cem_acpt/platform/vmpooler.rb +0 -24
  53. data/lib/cem_acpt/rspec_utils.rb +0 -242
  54. data/lib/cem_acpt/shared_objects.rb +0 -537
  55. data/lib/cem_acpt/spec_helper_acceptance.rb +0 -184
  56. data/lib/cem_acpt/test_runner/run_handler.rb +0 -187
  57. data/lib/cem_acpt/test_runner/runner.rb +0 -210
  58. data/lib/cem_acpt/test_runner/runner_result.rb +0 -103
@@ -1,5 +1,6 @@
1
1
  # frozen_string_literal: true
2
2
 
3
+ require 'json'
3
4
  require 'securerandom'
4
5
  require_relative File.join(__dir__, '..', 'image_name_builder')
5
6
  require_relative File.join(__dir__, '..', 'logging')
@@ -10,69 +11,57 @@ module CemAcpt::Platform
10
11
  class Base
11
12
  include CemAcpt::Logging
12
13
 
13
- attr_reader :config, :test_data, :local_port, :node_name, :image_name
14
+ attr_reader :test_data, :node_name
14
15
 
15
16
  # @param conf [CemAcpt::Config] the config object.
16
17
  # @param single_test_data [Hash] the test data for the current test.
17
- # @param local_port [Integer] the local port to use for the test.
18
- def initialize(conf, single_test_data, local_port)
18
+ def initialize(config, single_test_data)
19
19
  raise ArgumentError, 'single_test_data must be a Hash' unless single_test_data.is_a?(Hash)
20
20
 
21
- @config = conf.get('node_data')
21
+ @config = config
22
22
  @test_data = single_test_data
23
- @local_port = local_port
24
23
  @node_name = @test_data[:node_name] || random_node_name
25
- @image_name = conf.has?('image_name_builder') ? image_name_builder(conf, single_test_data) : @test_data[:image_name]
26
24
  end
27
25
 
28
- # Node should return a hash of all data about a created node.
29
- def node
30
- raise NotImplementedError, '#node must be implemented by subclass'
26
+ def to_h
27
+ {
28
+ node_name: node_name,
29
+ image_name: image_name,
30
+ test_data: test_data,
31
+ platform_data: platform_data,
32
+ node_data: node_data,
33
+ }
31
34
  end
32
35
 
33
- # Provision a node. Will be called asynchronously.
34
- def provision
35
- raise NotImplementedError, '#provision must be implemented by subclass'
36
+ def to_json(*args)
37
+ to_h.to_json(*args)
36
38
  end
37
39
 
38
- # Destroy a node. Will be called asynchronously.
39
- def destroy
40
- raise NotImplementedError, '#destroy must be implemented by subclass'
40
+ # Data common to all nodes of the same platform.
41
+ def platform_data
42
+ raise NotImplementedError, 'common_data must be implemented by the specific platform module'
41
43
  end
42
44
 
43
- # Tests to see if a node is ready to accept connections from the test suite.
44
- def ready?
45
- raise NotImplementedError, '#ready? must be implemented by subclass'
45
+ # Data specific to the current node.
46
+ def node_data
47
+ raise NotImplementedError, 'node_data must be implemented by the specific platform module'
46
48
  end
47
49
 
48
- # Upload and install a Puppet module package on the node. Blocking call.
49
- def install_puppet_module_package(_module_pkg_path, _remote_path)
50
- raise NotImplementedError, '#install_puppet_module_package must be implemented by subclass'
50
+ # Generates or retrieves an image name from the test data.
51
+ def image_name
52
+ @image_name ||= (@config.has?('image_name_builder') ? image_name_builder(@config, test_data) : test_data[:image_name])
51
53
  end
52
54
 
55
+ private
56
+
53
57
  # Generates a random node name.
54
58
  def random_node_name
55
- "acpt-test-#{SecureRandom.hex(10)}"
59
+ "cem-acpt-#{SecureRandom.hex(12)}"
56
60
  end
57
61
 
58
62
  # Builds an image name if the config specifies to use the image name builder.
59
63
  def image_name_builder(conf, tdata)
60
64
  @image_name_builder ||= CemAcpt::ImageNameBuilder.new(conf).build(tdata)
61
65
  end
62
-
63
- # Returns a command provider specified by the Platform module of the specific platform.
64
- def self.command_provider
65
- raise NotImplementedError, '#command_provider must be implemented by subclass'
66
- end
67
-
68
- # Applies a Puppet manifest on the given node.
69
- def self.apply_manifest(_instance_name, _manifest, _opts = {})
70
- raise NotImplementedError, '#apply_manifest must be implemented by subclass'
71
- end
72
-
73
- # Runs a shell command on the given node.
74
- def self.run_shell(_instance_name, _cmd, _opts = {})
75
- raise NotImplementedError, '#run_shell must be implemented by subclass'
76
- end
77
66
  end
78
67
  end
@@ -1,85 +1,71 @@
1
1
  # frozen_string_literal: true
2
2
 
3
+ require 'json'
4
+
3
5
  # GCP platform implementation
4
6
  module Platform
5
- require_relative 'gcp/compute'
6
- require_relative 'gcp/cmd'
7
+ def platform_data
8
+ {
9
+ username: gcp_username,
10
+ credentials_file: gcp_credentials_file,
11
+ project: gcp_project,
12
+ region: gcp_region,
13
+ zone: gcp_zone,
14
+ subnetwork: gcp_subnetwork,
15
+ private_key: gcp_private_key,
16
+ public_key: gcp_public_key,
17
+ }
18
+ end
7
19
 
8
- # Returns information about the GCP instance
9
- def node
10
- @instance.info
20
+ def node_data
21
+ {
22
+ machine_type: gcp_machine_type,
23
+ image: image_name,
24
+ disk_size: gcp_disk_size,
25
+ test_name: @test_data[:test_name],
26
+ }
11
27
  end
12
28
 
13
- # Provision a GCP instance
14
- def provision
15
- creation_params = config.dup
16
- creation_params[:disk][:image_name] = image_name
17
- creation_params[:local_port] = local_port
18
- @instance = CemAcpt::Platform::Gcp::VM.new(
19
- node_name,
20
- components: creation_params,
21
- )
22
- @instance.configure!
23
- logger.debug("Creating with command: #{@instance.send(:create_cmd)}")
24
- @instance.create
29
+ def gcp_username
30
+ return @gcp_username if @gcp_username
31
+
32
+ @gcp_username = @config.get('platform.username')
33
+ @gcp_username ||= JSON.parse(`gcloud compute os-login describe-profile --format json`.chomp)['posixAccounts'].first['username']
25
34
  end
26
35
 
27
- # Destroy a GCP instance
28
- def destroy
29
- @instance.destroy
36
+ def gcp_credentials_file
37
+ @gcp_credentials_file ||= (@config.get('platform.credentials_file') || File.join(Dir.home, '.config', 'gcloud', 'application_default_credentials.json'))
30
38
  end
31
39
 
32
- # Returns true if the GCP instance is ready for use in the test suite
33
- def ready?
34
- logger.debug("Checking if #{node_name} is ready...")
35
- @instance.ready?
40
+ def gcp_project
41
+ @gcp_project ||= (@config.get('platform.project') || `gcloud config get-value project`.chomp)
36
42
  end
37
43
 
38
- # Runs the test suite against the GCP instance. Must be given a block.
39
- # If necessary, can pass information into the block to be used in the test suite.
40
- def run_tests(&block)
41
- logger.debug("Running tests for #{node_name}...")
42
- block.call @instance.cmd.env
44
+ def gcp_region
45
+ @gcp_region ||= (@config.get('platform.region') || `gcloud config get-value compute/region`.chomp)
43
46
  end
44
47
 
45
- # Uploads and installs a Puppet module package on the GCP instance.
46
- def install_puppet_module_package(module_pkg_path, remote_path = nil, puppet_path = '/opt/puppetlabs/bin/puppet')
47
- remote_path = remote_path.nil? ? File.join('/tmp', File.basename(module_pkg_path)) : remote_path
48
- logger.info("Uploading module package #{module_pkg_path} to #{remote_path} on #{node_name} and installing it...")
49
- logger.debug("Using puppet path: #{puppet_path}")
50
- @instance.install_puppet_module_package(module_pkg_path, remote_path, puppet_path)
51
- logger.info("Module package #{module_pkg_path} installed on #{node_name}")
48
+ def gcp_zone
49
+ @gcp_zone ||= (@config.get('platform.zone') || `gcloud config get-value compute/zone`.chomp)
52
50
  end
53
51
 
54
- # Extends the class with class methods from the SpecMethods module
55
- def self.included(base)
56
- base.extend(SpecMethods)
52
+ def gcp_subnetwork
53
+ @gcp_subnetwork ||= (@config.get('platform.subnetwork') || 'default')
57
54
  end
58
55
 
59
- # Holds class methods called from spec tests.
60
- module SpecMethods
61
- # Returns an instance of the GCP platform class command provider
62
- # @return [CemAcpt::Platform::Gcp::Cmd]
63
- def command_provider
64
- CemAcpt::Platform::Gcp::Cmd.new(out_format: 'json')
65
- end
56
+ def gcp_private_key
57
+ @gcp_private_key ||= (@test_data[:private_key] || File.join(Dir.home, '.ssh', 'google_compute_engine'))
58
+ end
66
59
 
67
- # Apllies the given Puppet manifest on the given instance
68
- # @param instance_name [String] the name of the instance to apply the manifest to
69
- # @param manifest [String] the Puppet manifest to apply
70
- # @param opts [Hash] options to pass to the apply command
71
- # @return [String] the output of the apply command
72
- def apply_manifest(instance_name, manifest, opts = {})
73
- command_provider.apply_manifest(instance_name, manifest, opts)
74
- end
60
+ def gcp_public_key
61
+ @gcp_public_key ||= (@test_data[:public_key] || File.join(Dir.home, '.ssh', 'google_compute_engine.pub'))
62
+ end
63
+
64
+ def gcp_machine_type
65
+ @gcp_machine_type ||= (@config.get('node_data.machine_type') || 'e2-small')
66
+ end
75
67
 
76
- # Runs a shell command on the given instance
77
- # @param instance_name [String] the name of the instance to run the command on
78
- # @param command [String] the command to run
79
- # @param opts [Hash] options to pass to the run_shell command
80
- # @return [String] the output of the run_shell command
81
- def run_shell(instance_name, cmd, opts = {})
82
- command_provider.run_shell(instance_name, cmd, opts)
83
- end
68
+ def gcp_disk_size
69
+ @gcp_disk_size ||= (@config.get('node_data.disk_size') || 40)
84
70
  end
85
71
  end
@@ -1,12 +1,11 @@
1
1
  # frozen_string_literal: true
2
2
 
3
3
  require 'concurrent-ruby'
4
+ require_relative 'logging'
4
5
 
5
6
  # CemAcpt::Platform manages creating and configring platform specific objects
6
7
  # for the acceptance test suites.
7
8
  module CemAcpt::Platform
8
- require_relative 'logging'
9
-
10
9
  class Error < StandardError; end
11
10
 
12
11
  PLATFORM_DIR = File.expand_path(File.join(__dir__, 'platform'))
@@ -19,16 +18,15 @@ module CemAcpt::Platform
19
18
  # @param platform [String] the name of the platform
20
19
  # @param config [CemAcpt::Config] the config object
21
20
  # @param test_data [Hash] the test data
22
- # @param local_port_allocator [CemAcpt::LocalPortAllocator] the local port allocator
23
- def use(platform, config, test_data, local_port_allocator)
21
+ def use(platform, config, test_data)
24
22
  raise Error, "Platform #{platform} is not supported" unless platforms.include?(platform)
25
23
  raise Error, 'test_data must be an Array' unless test_data.is_a?(Array)
26
24
 
27
25
  logger.info "Using #{platform} for #{test_data.length} tests..."
28
26
  test_data.each_with_object([]) do |single_test_data, ary|
29
- local_port = local_port_allocator.allocate
30
- logger.debug("Allocated local port #{local_port} for test #{single_test_data[:test_name]}")
31
- ary << new_platform_object(platform, config, single_test_data, local_port)
27
+ #local_port = local_port_allocator.allocate
28
+ #logger.debug("Allocated local port #{local_port} for test #{single_test_data[:test_name]}")
29
+ ary << new_platform_object(platform, config, single_test_data)
32
30
  end
33
31
  end
34
32
 
@@ -40,6 +38,22 @@ module CemAcpt::Platform
40
38
  platform_class(platform)
41
39
  end
42
40
 
41
+ # Returns an array of the names of the supported platforms.
42
+ # Supported platforms are discovered by looking for files in the
43
+ # platform directory, and platform names are the basename (no extension)
44
+ # of the files. We deliberately exclude the base class, as it is not
45
+ # a platform.
46
+ def platforms
47
+ return @platforms if defined?(@platforms)
48
+
49
+ @platforms = Dir.glob(File.join(PLATFORM_DIR, '*.rb')).map do |file|
50
+ File.basename(file, '.rb') unless file.end_with?('base.rb')
51
+ end
52
+ @platforms.compact!
53
+ logger.debug "Discovered platform(s): #{@platforms}"
54
+ @platforms
55
+ end
56
+
43
57
  private
44
58
 
45
59
  # Dynamically creates a new platform class if it doesn't exist
@@ -48,10 +62,10 @@ module CemAcpt::Platform
48
62
  # @param config [CemAcpt::Config] the config object.
49
63
  # @param single_test_data [Hash] the test data for a single test.
50
64
  # @return [CemAcpt::Platform::Base] an initialized platform class.
51
- def new_platform_object(platform, config, single_test_data, local_port)
65
+ def new_platform_object(platform, config, single_test_data)
52
66
  raise Error, 'single_test_data must be a Hash' unless single_test_data.is_a?(Hash)
53
67
 
54
- platform_class(platform).new(config, single_test_data, local_port)
68
+ platform_class(platform).new(config, single_test_data)
55
69
  end
56
70
 
57
71
  # Creates a new platform-specific Class object for the given platform.
@@ -59,12 +73,14 @@ module CemAcpt::Platform
59
73
  # @param platform [String] the name of the platform.
60
74
  # @return [Class] the platform-specific Object.
61
75
  def platform_class(platform)
76
+ class_name = platform.capitalize
62
77
  # We require the platform base class here so that we can use it as
63
78
  # a parent class for the platform-specific class.
64
79
  require_relative 'platform/base'
65
80
  # If the class has already been defined, we can just use it.
66
- if Object.const_defined?(platform.capitalize)
67
- klass = Object.const_get(platform.capitalize)
81
+ if Object.const_defined?(class_name)
82
+ logger.debug "Using existing platform class #{class_name}"
83
+ klass = Object.const_get(class_name)
68
84
  else
69
85
  # Otherwise, we need to create the class. We do this by setting
70
86
  # a new constant with the name of the platform capitalized, and
@@ -73,31 +89,17 @@ module CemAcpt::Platform
73
89
  # include and extend our class with the Platform module from the file,
74
90
  # include Logging and Concurrent::Async, and finally call the
75
91
  # initialize method on the class.
92
+ logger.debug "Creating platform class #{class_name}"
76
93
  klass = Object.const_set(
77
- platform.capitalize,
94
+ class_name,
78
95
  Class.new(CemAcpt::Platform::Base) do
79
96
  require_relative "platform/#{platform}"
80
97
  include Platform
81
98
  end,
82
99
  )
83
100
  end
101
+ logger.debug "Using platform class: #{klass.inspect}"
84
102
  klass
85
103
  end
86
-
87
- # Returns an array of the names of the supported platforms.
88
- # Supported platforms are discovered by looking for files in the
89
- # platform directory, and platform names are the basename (no extension)
90
- # of the files. We deliberately exclude the base class, as it is not
91
- # a platform.
92
- def platforms
93
- return @platforms if defined?(@platforms)
94
-
95
- @platforms = Dir.glob(File.join(PLATFORM_DIR, '*.rb')).map do |file|
96
- File.basename(file, '.rb') unless file.end_with?('base.rb')
97
- end
98
- @platforms.compact!
99
- logger.debug "Discovered platform(s): #{@platforms}"
100
- @platforms
101
- end
102
104
  end
103
105
  end
@@ -0,0 +1,47 @@
1
+ # frozen_string_literal: true
2
+
3
+ require_relative 'os_data'
4
+
5
+ module CemAcpt
6
+ module Provision
7
+ # Class provides methods for gathering provision data for Linux nodes
8
+ class Linux < OsData
9
+ def self.valid_names
10
+ %w[centos rhel oel alma]
11
+ end
12
+
13
+ def self.valid_versions
14
+ %w[7, 8]
15
+ end
16
+
17
+ def systemd_files
18
+ Dir.glob(File.join(provision_directory, 'systemd', '*.service')).map { |f| File.basename(f) }
19
+ end
20
+
21
+ def puppet_bin_path
22
+ '/opt/puppetlabs/puppet/bin/puppet'
23
+ end
24
+
25
+ def destination_provision_directory
26
+ '/opt/cem_acpt'
27
+ end
28
+
29
+ def provision_commands
30
+ commands = [
31
+ "sudo /opt/puppetlabs/puppet/bin/puppet module install #{destination_provision_directory}/#{remote_module_package_name}",
32
+ 'curl -fsSL https://goss.rocks/install | sudo sh',
33
+ ]
34
+ unless systemd_files.empty?
35
+ systemd_files.each do |file|
36
+ commands << "sudo cp /opt/cem_acpt/systemd/#{file} /etc/systemd/system/#{file}"
37
+ end
38
+ commands << 'sudo systemctl daemon-reload'
39
+ systemd_files.each do |file|
40
+ commands << "sudo systemctl start #{file} && sudo systemctl enable #{file}"
41
+ end
42
+ end
43
+ commands << "sudo #{puppet_bin_path} apply #{destination_provision_directory}/#{puppet_manifest_file}"
44
+ end
45
+ end
46
+ end
47
+ end
@@ -0,0 +1,72 @@
1
+ # frozen_string_literal: true
2
+
3
+ require_relative '../../logging'
4
+
5
+ module CemAcpt
6
+ module Provision
7
+ # Base class for OS-specific provisioning data
8
+ class OsData
9
+ include CemAcpt::Logging
10
+ extend CemAcpt::Logging
11
+
12
+ def self.use_for?(test_name)
13
+ name_ver = test_name.match(%r{^\w+_(\w+)-(\d+).*})
14
+ return false unless name_ver && name_ver.length == 3
15
+
16
+ if valid_versions.include?(name_ver[2]) || valid_versions.include?(name_ver[2].to_s)
17
+ return true if valid_names.include?(name_ver[1])
18
+ end
19
+
20
+ false
21
+ end
22
+
23
+ def self.valid_names
24
+ raise NotImplementedError
25
+ end
26
+
27
+ def self.valid_versions
28
+ raise NotImplementedError
29
+ end
30
+
31
+ attr_accessor :base_provision_directory
32
+
33
+ def initialize(config, provision_data)
34
+ @config = config
35
+ @provision_data = provision_data
36
+ @base_provision_directory = @config.get('terraform.dir')
37
+ end
38
+
39
+ def puppet_bin_path
40
+ raise NotImplementedError
41
+ end
42
+
43
+ def puppet_manifest_file
44
+ 'manifest.pp'
45
+ end
46
+
47
+ def remote_module_package_name
48
+ 'puppet-module.tar.gz'
49
+ end
50
+
51
+ def implementation_name
52
+ self.class.to_s.downcase.split('::').last
53
+ end
54
+
55
+ def provision_directory
56
+ File.join(base_provision_directory, implementation_name)
57
+ end
58
+
59
+ def destination_provision_directory
60
+ raise NotImplementedError
61
+ end
62
+
63
+ def provision_commands
64
+ raise NotImplementedError
65
+ end
66
+
67
+ def goss_files
68
+ Dir.glob(File.join(provision_directory, 'goss', '*.yaml')).map { |f| File.basename(f) }
69
+ end
70
+ end
71
+ end
72
+ end
@@ -0,0 +1,22 @@
1
+ # frozen_string_literal: true
2
+
3
+ require_relative 'os_data'
4
+
5
+ module CemAcpt
6
+ module Provision
7
+ # Class provides methods for gathering provision data for Windows nodes
8
+ class Windows < OsData
9
+ def self.valid_names
10
+ %w[windows]
11
+ end
12
+
13
+ def self.valid_versions
14
+ %w[2016 2019 2022]
15
+ end
16
+
17
+ def puppet_bin_path
18
+ 'C:/Program Files/Puppet Labs/Puppet/bin/puppet.bat'
19
+ end
20
+ end
21
+ end
22
+ end