cem_acpt 0.2.5 → 0.6.1

data/lib/cem_acpt/platform/gcp.rb

@@ -1,85 +1,71 @@
 # frozen_string_literal: true
 
+require 'json'
+
 # GCP platform implementation
 module Platform
-  require_relative 'gcp/compute'
-  require_relative 'gcp/cmd'
+  def platform_data
+    {
+      username: gcp_username,
+      credentials_file: gcp_credentials_file,
+      project: gcp_project,
+      region: gcp_region,
+      zone: gcp_zone,
+      subnetwork: gcp_subnetwork,
+      private_key: gcp_private_key,
+      public_key: gcp_public_key,
+    }
+  end
 
-  # Returns information about the GCP instance
-  def node
-    @instance.info
+  def node_data
+    {
+      machine_type: gcp_machine_type,
+      image: image_name,
+      disk_size: gcp_disk_size,
+      test_name: @test_data[:test_name],
+    }
   end
 
-  # Provision a GCP instance
-  def provision
-    creation_params = config.dup
-    creation_params[:disk][:image_name] = image_name
-    creation_params[:local_port] = local_port
-    @instance = CemAcpt::Platform::Gcp::VM.new(
-      node_name,
-      components: creation_params,
-    )
-    @instance.configure!
-    logger.debug("Creating with command: #{@instance.send(:create_cmd)}")
-    @instance.create
+  def gcp_username
+    return @gcp_username if @gcp_username
+
+    @gcp_username = @config.get('platform.username')
+    @gcp_username ||= JSON.parse(`gcloud compute os-login describe-profile --format json`.chomp)['posixAccounts'].first['username']
   end
 
-  # Destroy a GCP instance
-  def destroy
-    @instance.destroy
+  def gcp_credentials_file
+    @gcp_credentials_file ||= (@config.get('platform.credentials_file') || File.join(Dir.home, '.config', 'gcloud', 'application_default_credentials.json'))
   end
 
-  # Returns true if the GCP instance is ready for use in the test suite
-  def ready?
-    logger.debug("Checking if #{node_name} is ready...")
-    @instance.ready?
+  def gcp_project
+    @gcp_project ||= (@config.get('platform.project') || `gcloud config get-value project`.chomp)
   end
 
-  # Runs the test suite against the GCP instance. Must be given a block.
-  # If necessary, can pass information into the block to be used in the test suite.
-  def run_tests(&block)
-    logger.debug("Running tests for #{node_name}...")
-    block.call @instance.cmd.env
+  def gcp_region
+    @gcp_region ||= (@config.get('platform.region') || `gcloud config get-value compute/region`.chomp)
   end
 
-  # Uploads and installs a Puppet module package on the GCP instance.
-  def install_puppet_module_package(module_pkg_path, remote_path = nil, puppet_path = '/opt/puppetlabs/bin/puppet')
-    remote_path = remote_path.nil? ? File.join('/tmp', File.basename(module_pkg_path)) : remote_path
-    logger.info("Uploading module package #{module_pkg_path} to #{remote_path} on #{node_name} and installing it...")
-    logger.debug("Using puppet path: #{puppet_path}")
-    @instance.install_puppet_module_package(module_pkg_path, remote_path, puppet_path)
-    logger.info("Module package #{module_pkg_path} installed on #{node_name}")
+  def gcp_zone
+    @gcp_zone ||= (@config.get('platform.zone') || `gcloud config get-value compute/zone`.chomp)
   end
 
-  # Extends the class with class methods from the SpecMethods module
-  def self.included(base)
-    base.extend(SpecMethods)
+  def gcp_subnetwork
+    @gcp_subnetwork ||= (@config.get('platform.subnetwork') || 'default')
   end
 
-  # Holds class methods called from spec tests.
-  module SpecMethods
-    # Returns an instance of the GCP platform class command provider
-    # @return [CemAcpt::Platform::Gcp::Cmd]
-    def command_provider
-      CemAcpt::Platform::Gcp::Cmd.new(out_format: 'json')
-    end
+  def gcp_private_key
+    @gcp_private_key ||= (@test_data[:private_key] || File.join(Dir.home, '.ssh', 'google_compute_engine'))
+  end
 
-    # Apllies the given Puppet manifest on the given instance
-    # @param instance_name [String] the name of the instance to apply the manifest to
-    # @param manifest [String] the Puppet manifest to apply
-    # @param opts [Hash] options to pass to the apply command
-    # @return [String] the output of the apply command
-    def apply_manifest(instance_name, manifest, opts = {})
-      command_provider.apply_manifest(instance_name, manifest, opts)
-    end
+  def gcp_public_key
+    @gcp_public_key ||= (@test_data[:public_key] || File.join(Dir.home, '.ssh', 'google_compute_engine.pub'))
+  end
+
+  def gcp_machine_type
+    @gcp_machine_type ||= (@config.get('node_data.machine_type') || 'e2-small')
+  end
 
-    # Runs a shell command on the given instance
-    # @param instance_name [String] the name of the instance to run the command on
-    # @param command [String] the command to run
-    # @param opts [Hash] options to pass to the run_shell command
-    # @return [String] the output of the run_shell command
-    def run_shell(instance_name, cmd, opts = {})
-      command_provider.run_shell(instance_name, cmd, opts)
-    end
+  def gcp_disk_size
+    @gcp_disk_size ||= (@config.get('node_data.disk_size') || 40)
   end
 end

data/lib/cem_acpt/platform.rb

@@ -1,12 +1,11 @@
 # frozen_string_literal: true
 
 require 'concurrent-ruby'
+require_relative 'logging'
 
 # CemAcpt::Platform manages creating and configring platform specific objects
 # for the acceptance test suites.
 module CemAcpt::Platform
-  require_relative 'logging'
-
   class Error < StandardError; end
 
   PLATFORM_DIR = File.expand_path(File.join(__dir__, 'platform'))
@@ -19,16 +18,15 @@ module CemAcpt::Platform
     # @param platform [String] the name of the platform
     # @param config [CemAcpt::Config] the config object
     # @param test_data [Hash] the test data
-    # @param local_port_allocator [CemAcpt::LocalPortAllocator] the local port allocator
-    def use(platform, config, test_data, local_port_allocator)
+    def use(platform, config, test_data)
       raise Error, "Platform #{platform} is not supported" unless platforms.include?(platform)
       raise Error, 'test_data must be an Array' unless test_data.is_a?(Array)
 
       logger.info "Using #{platform} for #{test_data.length} tests..."
       test_data.each_with_object([]) do |single_test_data, ary|
-        local_port = local_port_allocator.allocate
-        logger.debug("Allocated local port #{local_port} for test #{single_test_data[:test_name]}")
-        ary << new_platform_object(platform, config, single_test_data, local_port)
+        #local_port = local_port_allocator.allocate
+        #logger.debug("Allocated local port #{local_port} for test #{single_test_data[:test_name]}")
+        ary << new_platform_object(platform, config, single_test_data)
       end
     end
 
@@ -40,6 +38,22 @@ module CemAcpt::Platform
       platform_class(platform)
     end
 
+    # Returns an array of the names of the supported platforms.
+    # Supported platforms are discovered by looking for files in the
+    # platform directory, and platform names are the basename (no extension)
+    # of the files. We deliberately exclude the base class, as it is not
+    # a platform.
+    def platforms
+      return @platforms if defined?(@platforms)
+
+      @platforms = Dir.glob(File.join(PLATFORM_DIR, '*.rb')).map do |file|
+        File.basename(file, '.rb') unless file.end_with?('base.rb')
+      end
+      @platforms.compact!
+      logger.debug "Discovered platform(s): #{@platforms}"
+      @platforms
+    end
+
     private
 
     # Dynamically creates a new platform class if it doesn't exist
@@ -48,10 +62,10 @@ module CemAcpt::Platform
     # @param config [CemAcpt::Config] the config object.
     # @param single_test_data [Hash] the test data for a single test.
     # @return [CemAcpt::Platform::Base] an initialized platform class.
-    def new_platform_object(platform, config, single_test_data, local_port)
+    def new_platform_object(platform, config, single_test_data)
       raise Error, 'single_test_data must be a Hash' unless single_test_data.is_a?(Hash)
 
-      platform_class(platform).new(config, single_test_data, local_port)
+      platform_class(platform).new(config, single_test_data)
     end
 
     # Creates a new platform-specific Class object for the given platform.
@@ -59,12 +73,14 @@ module CemAcpt::Platform
     # @param platform [String] the name of the platform.
     # @return [Class] the platform-specific Object.
     def platform_class(platform)
+      class_name = platform.capitalize
       # We require the platform base class here so that we can use it as
       # a parent class for the platform-specific class.
       require_relative 'platform/base'
       # If the class has already been defined, we can just use it.
-      if Object.const_defined?(platform.capitalize)
-        klass = Object.const_get(platform.capitalize)
+      if Object.const_defined?(class_name)
+        logger.debug "Using existing platform class #{class_name}"
+        klass = Object.const_get(class_name)
       else
         # Otherwise, we need to create the class. We do this by setting
         # a new constant with the name of the platform capitalized, and
@@ -73,31 +89,17 @@ module CemAcpt::Platform
         # include and extend our class with the Platform module from the file,
         # include Logging and Concurrent::Async, and finally call the
         # initialize method on the class.
+        logger.debug "Creating platform class #{class_name}"
         klass = Object.const_set(
-          platform.capitalize,
+          class_name,
           Class.new(CemAcpt::Platform::Base) do
             require_relative "platform/#{platform}"
             include Platform
           end,
         )
       end
+      logger.debug "Using platform class: #{klass.inspect}"
       klass
     end
-
-    # Returns an array of the names of the supported platforms.
-    # Supported platforms are discovered by looking for files in the
-    # platform directory, and platform names are the basename (no extension)
-    # of the files. We deliberately exclude the base class, as it is not
-    # a platform.
-    def platforms
-      return @platforms if defined?(@platforms)
-
-      @platforms = Dir.glob(File.join(PLATFORM_DIR, '*.rb')).map do |file|
-        File.basename(file, '.rb') unless file.end_with?('base.rb')
-      end
-      @platforms.compact!
-      logger.debug "Discovered platform(s): #{@platforms}"
-      @platforms
-    end
   end
 end

data/lib/cem_acpt/provision/terraform/linux.rb

@@ -0,0 +1,47 @@
+# frozen_string_literal: true
+
+require_relative 'os_data'
+
+module CemAcpt
+  module Provision
+    # Class provides methods for gathering provision data for Linux nodes
+    class Linux < OsData
+      def self.valid_names
+        %w[centos rhel oel alma]
+      end
+
+      def self.valid_versions
+        %w[7, 8]
+      end
+
+      def systemd_files
+        Dir.glob(File.join(provision_directory, 'systemd', '*.service')).map { |f| File.basename(f) }
+      end
+
+      def puppet_bin_path
+        '/opt/puppetlabs/puppet/bin/puppet'
+      end
+
+      def destination_provision_directory
+        '/opt/cem_acpt'
+      end
+
+      def provision_commands
+        commands = [
+          "sudo /opt/puppetlabs/puppet/bin/puppet module install #{destination_provision_directory}/#{remote_module_package_name}",
+          'curl -fsSL https://goss.rocks/install | sudo sh',
+        ]
+        unless systemd_files.empty?
+          systemd_files.each do |file|
+            commands << "sudo cp /opt/cem_acpt/systemd/#{file} /etc/systemd/system/#{file}"
+          end
+          commands << 'sudo systemctl daemon-reload'
+          systemd_files.each do |file|
+            commands << "sudo systemctl start #{file} && sudo systemctl enable #{file}"
+          end
+        end
+        commands << "sudo #{puppet_bin_path} apply #{destination_provision_directory}/#{puppet_manifest_file}"
+      end
+    end
+  end
+end

data/lib/cem_acpt/provision/terraform/os_data.rb

@@ -0,0 +1,72 @@
+# frozen_string_literal: true
+
+require_relative '../../logging'
+
+module CemAcpt
+  module Provision
+    # Base class for OS-specific provisioning data
+    class OsData
+      include CemAcpt::Logging
+      extend CemAcpt::Logging
+
+      def self.use_for?(test_name)
+        name_ver = test_name.match(%r{^\w+_(\w+)-(\d+).*})
+        return false unless name_ver && name_ver.length == 3
+
+        if valid_versions.include?(name_ver[2]) || valid_versions.include?(name_ver[2].to_s)
+          return true if valid_names.include?(name_ver[1])
+        end
+
+        false
+      end
+
+      def self.valid_names
+        raise NotImplementedError
+      end
+
+      def self.valid_versions
+        raise NotImplementedError
+      end
+
+      attr_accessor :base_provision_directory
+
+      def initialize(config, provision_data)
+        @config = config
+        @provision_data = provision_data
+        @base_provision_directory = @config.get('terraform.dir')
+      end
+
+      def puppet_bin_path
+        raise NotImplementedError
+      end
+
+      def puppet_manifest_file
+        'manifest.pp'
+      end
+
+      def remote_module_package_name
+        'puppet-module.tar.gz'
+      end
+
+      def implementation_name
+        self.class.to_s.downcase.split('::').last
+      end
+
+      def provision_directory
+        File.join(base_provision_directory, implementation_name)
+      end
+
+      def destination_provision_directory
+        raise NotImplementedError
+      end
+
+      def provision_commands
+        raise NotImplementedError
+      end
+
+      def goss_files
+        Dir.glob(File.join(provision_directory, 'goss', '*.yaml')).map { |f| File.basename(f) }
+      end
+    end
+  end
+end

data/lib/cem_acpt/provision/terraform/windows.rb

@@ -0,0 +1,22 @@
+# frozen_string_literal: true
+
+require_relative 'os_data'
+
+module CemAcpt
+  module Provision
+    # Class provides methods for gathering provision data for Windows nodes
+    class Windows < OsData
+      def self.valid_names
+        %w[windows]
+      end
+
+      def self.valid_versions
+        %w[2016 2019 2022]
+      end
+
+      def puppet_bin_path
+        'C:/Program Files/Puppet Labs/Puppet/bin/puppet.bat'
+      end
+    end
+  end
+end

data/lib/cem_acpt/provision/terraform.rb

@@ -0,0 +1,193 @@
+# frozen_string_literal: true
+
+require 'fileutils'
+require 'json'
+require 'ruby-terraform'
+require_relative '../logging'
+require_relative 'terraform/linux'
+require_relative 'terraform/windows'
+
+module CemAcpt
+  module Provision
+    class Terraform
+      DEFAULT_PLAN_NAME = 'testplan.tfplan'
+      include CemAcpt::Logging
+
+      attr_reader :environment, :working_dir
+
+      def initialize(config, provision_data)
+        @config = config
+        @provision_data = provision_data
+        @backend = new_backend(@provision_data[:test_data].first[:test_name])
+        @environment = new_environment(@config)
+        @working_dir = nil
+      end
+
+      def provision
+        logger.info('Terraform') { 'Provisioning nodes...' }
+        @working_dir = new_working_dir
+        validate_working_dir!
+
+        terraform_configure_logging
+        terraform_init
+        terraform_plan(formatted_vars, DEFAULT_PLAN_NAME)
+        terraform_apply(DEFAULT_PLAN_NAME)
+        JSON.parse(terraform_output('instance_name_ip', json: true))
+      end
+
+      def destroy
+        terraform_destroy(formatted_vars)
+        logger.verbose('Terraform') { "Deleting old working directory #{working_dir}" }
+        FileUtils.rm_rf(working_dir)
+        @working_dir = nil
+      end
+
+      private
+
+      def terraform
+        @terraform ||= RubyTerraform
+      end
+
+      def terraform_configure_logging
+        terraform.configure do |c|
+          c.logger = logger
+          c.stdout = c.logger
+          c.stderr = c.logger
+        end
+      end
+
+      def terraform_init
+        logger.debug('Terraform') { 'Initializing Terraform' }
+        terraform.init({ chdir: working_dir, input: false, no_color: true }, { environment: environment })
+      end
+
+      def terraform_plan(vars, plan_name = DEFAULT_PLAN_NAME)
+        logger.debug('Terraform') { "Creating Terraform plan '#{plan_name}'" }
+        logger.verbose('Terraform') { "Using vars:\n#{JSON.pretty_generate(vars)}" }
+        terraform.plan(
+          {
+            chdir: working_dir,
+            input: false,
+            no_color: true,
+            plan: plan_name,
+            vars: vars,
+          },
+          {
+            environment: environment,
+          },
+        )
+      end
+
+      def terraform_apply(plan_name = DEFAULT_PLAN_NAME)
+        logger.debug('Terraform') { "Running Terraform apply with the plan #{plan_name}" }
+        terraform.apply({ chdir: working_dir, input: false, no_color: true, plan: plan_name }, { environment: environment })
+      end
+
+      def terraform_output(name, json: true)
+        logger.debug('Terraform') { "Getting Terraform output #{name}" }
+        terraform.output({ chdir: working_dir, no_color: true, json: json, name: name }, { environment: environment })
+      end
+
+      def terraform_destroy(vars)
+        logger.debug('Terraform') { 'Destroying Terraform resources' }
+        terraform.destroy(
+          {
+            chdir: working_dir,
+            auto_approve: true,
+            input: false,
+            no_color: true,
+            vars: vars,
+          },
+          {
+            environment: environment,
+          },
+        )
+      end
+
+      def new_backend(test_name)
+        if CemAcpt::Provision::Linux.use_for?(test_name)
+          logger.info('Terraform') { 'Using Linux backend' }
+          logger.verbose('Terraform') { "Creating backend with provision_data:\n#{JSON.pretty_generate(@provision_data)}" }
+          CemAcpt::Provision::Linux.new(@config, @provision_data)
+        elsif CemAcpt::Provision::Windows.use_for?(test_name)
+          logger.info('Terraform') { 'Using Windows backend' }
+          logger.verbose('Terraform') { "Creating backend with provision_data:\n#{JSON.pretty_generate(@provision_data)}" }
+          CemAcpt::Provision::Windows.new(@config, @provision_data)
+        else
+          err_msg = [
+            "Test name #{test_name} does not match any known OS.",
+            "Known OSes are: #{CemAcpt::Provision::Linux.valid_names.join(', ')}",
+            "and #{CemAcpt::Provision::Windows.valid_names.join(', ')}.",
+            "Known versions are: #{CemAcpt::Provision::Linux.valid_versions.join(', ')}",
+            ", and #{CemAcpt::Provision::Windows.valid_versions.join(', ')}."
+          ].join(' ')
+          logger.error('Terraform') { err_msg }
+          raise ArgumentError, err_msg
+        end
+      end
+
+      def new_environment(config)
+        env = (config.get('terraform.environment') || {})
+        env['CLOUDSDK_PYTHON_SITEPACKAGES'] = '1' # This is needed for gcloud to use numpy
+        logger.verbose('Terraform') { "Using environment:\n#{JSON.pretty_generate(env)}" }
+        env
+      end
+
+      def new_working_dir
+        logger.debug('Terraform') { "Creating new working directory" }
+        base_dir = File.join(@config.get('terraform.dir'), @config.get('platform.name'))
+        logger.verbose('Terraform') { "Base directory defined as #{base_dir}" }
+        @backend.base_provision_directory = base_dir
+        logger.verbose('Terraform') { 'Base directory set in backend' }
+        work_dir = File.join(@config.get('terraform.dir'), "test_#{Time.now.to_i.to_s}")
+        logger.verbose('Terraform') { "Working directory defined as #{work_dir}" }
+        logger.verbose('Terraform') { "Copying backend provision directory #{@backend.provision_directory} to working directory" }
+        FileUtils.cp_r(@backend.provision_directory, work_dir)
+        logger.verbose('Terraform') { "Copied provision directory #{@backend.provision_directory} to #{work_dir}" }
+        FileUtils.cp(@provision_data[:module_package_path], work_dir)
+        logger.verbose('Terraform') { "Copied module package #{@provision_data[:module_package_path]} to #{work_dir}" }
+        work_dir
+      rescue StandardError => e
+        logger.error('Terraform') { 'Error creating working directory' }
+        raise e
+      end
+
+      def validate_working_dir!
+        logger.debug('Terraform') { "Validating working directory #{working_dir}" }
+        logger.verbose('Terraform') { "Content of #{working_dir}:\n#{Dir.glob(File.join(working_dir, '*')).join("\n")}" }
+        raise "Terraform working directory #{working_dir} does not exist" unless File.directory?(working_dir)
+        raise "Terraform working directory #{working_dir} does not contain a Terraform file" unless Dir.glob(File.join(working_dir, '*.tf')).any?
+        logger.info('Terraform') { "Using working directory: #{working_dir}" }
+      rescue StandardError => e
+        logger.error('Terraform') { 'Error validating working directory' }
+        raise e
+      end
+
+      def provision_node_data
+        node_data = @provision_data[:nodes].each_with_object({}) do |node, h|
+          h[node.node_name] = node.node_data.merge({
+            goss_file: node.test_data[:goss_file],
+            puppet_manifest: node.test_data[:puppet_manifest],
+            provision_dir_source: @backend.provision_directory,
+            provision_dir_dest: @backend.destination_provision_directory,
+            provision_commands: @backend.provision_commands,
+          })
+        end
+        node_data.to_json
+      rescue StandardError => e
+        logger.error('Terraform') { 'Error creating node data' }
+        raise e
+      end
+
+      def formatted_vars
+        @provision_data[:nodes].first.platform_data.merge({
+          puppet_module_package: @provision_data[:module_package_path],
+          node_data: provision_node_data,
+        })
+      rescue StandardError => e
+        logger.error('Terraform') { 'Error creating formatted vars' }
+        raise e
+      end
+    end
+  end
+end

data/lib/cem_acpt/provision.rb

@@ -0,0 +1,20 @@
+# frozen_string_literal: true
+
+require_relative 'logging'
+require_relative 'provision/terraform'
+
+module CemAcpt
+  module Provision
+    include CemAcpt::Logging
+
+    def self.new_provisioner(config, provision_data)
+      case config.get('provisioner')
+      when 'terraform'
+        logger.debug('Provision') { 'Using Terraform provisioner' }
+        CemAcpt::Provision::Terraform.new(config, provision_data)
+      else
+        raise ArgumentError, "Unknown provisioner #{config.get('provisioner')}"
+      end
+    end
+  end
+end

data/lib/cem_acpt/puppet_helpers.rb

@@ -28,7 +28,6 @@ module CemAcpt
 
         # Validates module metadata by raising exception if invalid
         _metadata = builder.metadata
-        logger.debug("Metadata for module #{builder.release_name} is valid")
 
         # Builds the module package
         logger.info("Building module package for #{builder.release_name}")