cem_acpt 0.6.5 → 0.7.0

data/lib/cem_acpt/image_builder.rb

@@ -0,0 +1,308 @@
+# frozen_string_literal: true
+
+require 'fileutils'
+require 'json'
+require 'ruby-terraform'
+require_relative 'logging'
+require_relative 'platform'
+require_relative 'utils'
+require_relative 'version'
+require_relative 'image_builder/exec'
+require_relative 'image_builder/provision_commands'
+
+module CemAcpt
+  # This module contains the classes and methods for building test node images
+  module ImageBuilder
+    def self.build_images(config)
+      TerraformBuilder.new(config).run
+    end
+
+    # This class builds test node images using Terraform
+    # NOTE: There is a huge amount of overlap between this class and
+    # CemAcpt::Provision::Terraform. This isn't ideal, but there are
+    # enough differences that a new class was needed. This should be
+    # refactored in the future.
+    class TerraformBuilder
+      DEFAULT_PLAN_NAME = 'testplan.tfplan'
+      DEFAULT_VARS_FILE = 'imagevars.json'
+      include CemAcpt::Logging
+
+      def initialize(config)
+        @config = config
+        @image_terraform_dir = File.join(config.get('terraform.dir'), 'image', config.get('platform.name'))
+        @exec = CemAcpt::ImageBuilder::Exec.new_exec(@config)
+        @environment = new_environment(@config)
+        @all_tfvars = { node_data: {} }
+        @linux_tfvars = { node_data: {} }
+        @windows_tfvars = { node_data: {} }
+        @duration = 0
+        @exit_code = 0
+        @private_key = nil
+        @public_key = nil
+      end
+
+      def run
+        @start_time = Time.now
+        logger.with_ci_group("CemAcptImage v#{CemAcpt::VERSION} run started at #{@start_time}") do
+          logger.info('CemAcptImage') { "Using working directory: #{@working_dir}..." }
+          keep_terminal_alive
+          @all_tfvars = new_tfvars(@config)
+          @working_dir = new_working_dir
+          @linux_tfvars, @windows_tfvars = divide_tfvars_by_os(@all_tfvars)
+          save_vars_to_file!('linux', @linux_tfvars)
+          save_vars_to_file!('windows', @windows_tfvars)
+          terraform_init
+          image_types = []
+          image_types << [@linux_tfvars, 'linux'] unless no_linux?
+          image_types << [@windows_tfvars, 'windows'] unless no_windows?
+          image_types.each do |tfvars, os_str|
+            terraform_plan(os_str, tfvars, DEFAULT_PLAN_NAME)
+            begin
+              terraform_apply(os_str, DEFAULT_PLAN_NAME)
+              output = JSON.parse(terraform_output(os_str, 'node-data', json: true))
+              output.each do |instance_name, data|
+                logger.info('CemAcptImage') { "Stopping instance #{instance_name}..." }
+                @exec.run('compute', 'instances', 'stop', instance_name)
+                deprecate_old_images_in_family(data['image_family'])
+                create_image_from_disk(data['disk_link'], image_name_from_image_family(data['image_family']), data['image_family'])
+              end
+            ensure
+              terraform_destroy(os_str, tfvars)
+            end
+          end
+        end
+      end
+
+      private
+
+      attr_reader :environment
+
+      def image_name_from_image_family(image_family)
+        "#{image_family}-v#{@start_time.to_i}"[0..64]
+      end
+
+      def deprecate_old_images_in_family(image_family)
+        logger.info('CemAcptImage') { "Deprecating old images in family #{image_family}..." }
+        images = @exec.run('compute', 'images', 'list', '--project', @config.get('platform.project'), '--filter', "family:#{image_family}")
+        images.each do |image|
+          next unless image['status'] == 'READY'
+
+          logger.info('CemAcptImage') { "Deprecating image #{image['name']}..." }
+          @exec.run('compute', 'images', 'deprecate', image['name'], '--state', 'DEPRECATED', '--delete-in', '1d', '--project', @config.get('platform.project'))
+        end
+      end
+
+      def create_image_from_disk(disk_link, image_name, image_family)
+        logger.info('CemAcptImage') { "Creating image from disk #{disk_link}..." }
+        @exec.run('compute', 'images', 'create', image_name, '--family', image_family,
+                  '--source-disk', disk_link, '--project', @config.get('platform.project'))
+        logger.info('CemAcptImage') { "Image #{image_name} created for family #{image_family}"}
+      end
+
+      def no_windows?
+        @windows_tfvars[:node_data].empty? || @config.get('cem_acpt_image.no_windows')
+      end
+
+      def no_linux?
+        @linux_tfvars[:node_data].empty? || @config.get('cem_acpt_image.no_linux')
+      end
+
+      def terraform
+        return @terraform if defined?(@terraform)
+
+        RubyTerraform.configure do |c|
+          c.logger = logger
+          c.stdout = logger
+          c.stderr = logger
+        end
+        @terraform = RubyTerraform
+        @terraform
+      end
+
+      def new_environment(config)
+        env = (config.get('terraform.environment') || {})
+        env['CLOUDSDK_PYTHON_SITEPACKAGES'] = '1' # This is needed for gcloud to use numpy
+        logger.verbose('ImageBuilder') { "Using environment:\n#{JSON.pretty_generate(env)}" }
+        env
+      end
+
+      def new_platform(config, **run_data)
+        CemAcpt::Platform.get(config.get('platform.name'), base_type: :base).new(config, **run_data)
+      end
+
+      # @return [Thread] The thread that keeps the terminal alive
+      def keep_terminal_alive
+        return unless @config.ci?
+
+        @keep_terminal_alive ||= CemAcpt::Utils::Terminal.keep_terminal_alive
+      end
+
+      def kill_keep_terminal_alive
+        return if @trap_context
+
+        keep_terminal_alive&.kill
+      end
+
+      # @return [Array<String>] The paths to the ssh private key, public key, and known hosts file
+      def new_ephemeral_ssh_keys
+        return [nil, nil, nil] if @config.get('no_ephemeral_ssh_key')
+
+        CemAcpt::Utils::SSH::Ephemeral.create
+      end
+
+      def clean_ephemeral_ssh_keys
+        return if @config.get('no_ephemeral_ssh_key')
+
+        CemAcpt::Utils::SSH::Ephemeral.clean
+      end
+
+      def no_os_str?(os_str)
+        return true if os_str == 'windows' && no_windows?
+        return true if os_str != 'windows' && no_linux?
+
+        false
+      end
+
+      def in_os_dir(os_str)
+        return unless block_given?
+
+        if no_os_str?(os_str)
+          logger.debug('ImageBuilder') { "Skipping #{os_str} because no_#{os_str} is true" }
+          return
+        end
+
+        os_dir = (os_str == 'windows') ? File.join(@working_dir, 'windows') : File.join(@working_dir, 'linux')
+        Dir.chdir(os_dir) do
+          yield
+        end
+      end
+
+      def terraform_init
+        raise 'Cannot initialize Terraform, both no_linux and no_windows are true' if no_linux? && no_windows?
+
+        logger.debug('ImageBuilder') { 'Initializing Terraform' }
+        in_os_dir('linux') do
+          terraform.init({ input: false, no_color: true })
+        end
+        in_os_dir('windows') do
+          terraform.init({ input: false, no_color: true })
+        end
+      end
+
+      def terraform_plan(os_dir, tfvars, plan_name)
+        in_os_dir(os_dir) do
+          logger.debug('ImageBuilder') { "Creating Terraform plan #{plan_name} for #{os_dir}" }
+          logger.verbose('ImageBuilder') { "Using vars:\n#{JSON.pretty_generate(tfvars)}" }
+          terraform.plan({ input: false, no_color: true, plan: plan_name, vars: tfvars }, { environment: environment })
+        end
+      end
+
+      def terraform_apply(os_dir, plan_name)
+        in_os_dir(os_dir) do
+          logger.debug('ImageBuilder') { "Applying Terraform plan #{plan_name} for #{os_dir}" }
+          terraform.apply({ input: false, no_color: true, plan: plan_name }, { environment: environment })
+        end
+      end
+
+      def terraform_output(os_dir, output_name, json: true)
+        in_os_dir(os_dir) do
+          logger.debug('ImageBuilder') { "Getting Terraform output #{output_name} for #{os_dir}" }
+          terraform.output({ no_color: true, json: json, name: output_name }, { environment: environment })
+        end
+      end
+
+      def terraform_destroy(os_dir, tfvars)
+        in_os_dir(os_dir) do
+          logger.debug('ImageBuilder') { "Destroying Terraform resources for #{os_dir}" }
+          logger.verbose('ImageBuilder') { "Using vars:\n#{JSON.pretty_generate(tfvars)}" }
+          terraform.destroy({ auto_approve: true, input: false, no_color: true, vars: tfvars }, { environment: environment })
+        end
+      end
+
+      def terraform_show(os_dir)
+        in_os_dir(os_dir) do
+          logger.debug('ImageBuilder') { "Showing Terraform state for #{os_dir}" }
+          terraform.show({ no_color: true }, { environment: environment })
+        end
+      end
+
+      # @return [Hash] A hash of the Terraform variables to use
+      def new_tfvars(config)
+        tfvars = { node_data: {} }
+        private_key, public_key, _ = new_ephemeral_ssh_keys
+        tfvars[:private_key] = private_key if private_key
+        tfvars[:public_key] = public_key if public_key
+        config.get('images').each do |image_name, image|
+          platform = new_platform(config, **tfvars)
+          tfvars.merge!(platform.platform_data)
+          provision_commands = CemAcpt::ImageBuilder::ProvisionCommands.provision_commands(
+            config,
+            image_name: image_name,
+            base_image: image[:base_image],
+            os: image[:os],
+            os_major_version: image[:os_major_version],
+            puppet_version: image[:puppet_version],
+          )
+          tfvars[:node_data][platform.node_name] = platform.node_data.merge(
+            image_family: image_name,
+            provision_commands: provision_commands,
+            base_image: image[:base_image],
+            windows_image: windows_image?(image[:os]),
+          )
+        end
+        logger.verbose('ImageBuilder') { "All Terraform variables:\n#{JSON.pretty_generate(tfvars)}" }
+        tfvars
+      end
+
+      def divide_tfvars_by_os(tfvars)
+        linux_node_data = tfvars[:node_data].reject { |_, data| data[:windows_image] }
+        windows_node_data = tfvars[:node_data].select { |_, data| data[:windows_image] }
+        linux_tfvars = tfvars.dup
+        linux_tfvars[:node_data] = linux_node_data
+        windows_tfvars = tfvars.dup
+        windows_tfvars[:node_data] = windows_node_data
+        logger.verbose('ImageBuilder') { "Linux Terraform variables:\n#{JSON.pretty_generate(linux_tfvars)}" }
+        logger.verbose('ImageBuilder') { "Windows Terraform variables:\n#{JSON.pretty_generate(windows_tfvars)}" }
+        [linux_tfvars, windows_tfvars]
+      end
+
+      def windows_image?(os_str)
+        os_str == 'windows'
+      end
+
+      def save_vars_to_file!(prefix, vars)
+        logger.debug('ImageBuilder') { "Saving vars to file #{File.join(@working_dir, "#{prefix}_#{DEFAULT_VARS_FILE}")}" }
+        File.write(File.join(@working_dir, "#{prefix}_#{DEFAULT_VARS_FILE}"), vars.to_json)
+      rescue StandardError => e
+        logger.error('ImageBuilder') { 'Error saving vars to file' }
+        raise e
+      end
+
+      def new_working_dir
+        unless Dir.exist?(@image_terraform_dir)
+          raise "Image Terraform directory #{@image_terraform_dir} does not exist"
+        end
+        logger.debug('ImageBuilder') { "Creating new working directory in #{@image_terraform_dir}" }
+        working_dir = File.join(@image_terraform_dir, "image_builder_#{Time.now.to_i}")
+        FileUtils.mkdir_p(working_dir)
+        logger.verbose('ImageBuilder') { "Created working directory #{working_dir}" }
+        FileUtils.cp_r(File.join(@image_terraform_dir, 'linux'), working_dir)
+        FileUtils.cp_r(File.join(@image_terraform_dir, 'windows'), working_dir)
+        logger.verbose('ImageBuilder') { "Copied Terraform files from #{@image_terraform_dir}/(linux|windows) to #{working_dir}" }
+        if File.exist?(@all_tfvars[:private_key])
+          FileUtils.cp(@all_tfvars[:private_key], working_dir)
+          @private_key = File.join(working_dir, File.basename(@all_tfvars[:private_key]))
+          logger.verbose('ImageBuilder') { "Copied private key #{@all_tfvars[:private_key]} to #{working_dir}" }
+        end
+        if File.exist?(@all_tfvars[:public_key])
+          FileUtils.cp(@all_tfvars[:public_key], working_dir)
+          @public_key = File.join(working_dir, File.basename(@all_tfvars[:public_key]))
+          logger.verbose('ImageBuilder') { "Copied public key #{@all_tfvars[:public_key]} to #{working_dir}" }
+        end
+
+        logger.verbose('ImageBuilder') { "Content of #{working_dir}:\n#{Dir.glob(File.join(working_dir, '*')).join("\n")}" }
+        working_dir
+      end
+    end
+  end
+end

data/lib/cem_acpt/image_name_builder.rb

@@ -1,7 +1,7 @@
 # frozen_string_literal: true
 
 module CemAcpt
-  require_relative 'core_extensions'
+  require_relative 'core_ext'
   require_relative 'logging'
 
   # Dynamically builds an image name based on parameters specified in the
@@ -20,7 +20,7 @@ module CemAcpt
   #   - 'validation_pattern'      - (Optional) A regex pattern to validate the image name against.
   class ImageNameBuilder
     include CemAcpt::Logging
-    using CemAcpt::CoreExtensions::ExtendedHash
+    using CemAcpt::CoreExt::ExtendedHash
 
     # Initializes the ImageNameBuilder.
     # @param config [CemAcpt::Config] The config to use.

data/lib/cem_acpt/platform/base.rb

@@ -6,33 +6,24 @@ require_relative File.join(__dir__, '..', 'image_name_builder')
 require_relative File.join(__dir__, '..', 'logging')
 
 module CemAcpt::Platform
-  # Base class for all platform classes. This class provides an API for
-  # interacting with the underlying platform.
   class Base
     include CemAcpt::Logging
 
-    attr_reader :test_data, :node_name
-
-    # @param conf [CemAcpt::Config] the config object.
-    # @param single_test_data [Hash] the test data for the current test.
-    # @param run_data [Hash] the run data for the current run minus the test data key.
-    def initialize(config, single_test_data, **run_data)
-      raise ArgumentError, 'single_test_data must be a Hash' unless single_test_data.is_a?(Hash)
-
+    def initialize(config, **run_data)
       @config = config
-      @test_data = single_test_data
-      @node_name = @test_data[:node_name] || random_node_name
       @run_data = run_data
     end
 
+    def node_name
+      @node_name ||= random_node_name
+    end
+
+    def image_name
+      @image_name ||= @run_data[:image_name]
+    end
+
     def to_h
-      {
-        node_name: node_name,
-        image_name: image_name,
-        test_data: test_data,
-        platform_data: platform_data,
-        node_data: node_data,
-      }
+      { node_name: node_name, platform_data: platform_data, node_data: node_data }
     end
 
     def to_json(*args)
@@ -41,7 +32,7 @@ module CemAcpt::Platform
 
     # Data common to all nodes of the same platform.
     def platform_data
-      raise NotImplementedError, 'common_data must be implemented by the specific platform module'
+      raise NotImplementedError, 'platform_data must be implemented by the specific platform module'
     end
 
     # Data specific to the current node.
@@ -49,17 +40,49 @@ module CemAcpt::Platform
       raise NotImplementedError, 'node_data must be implemented by the specific platform module'
     end
 
-    # Generates or retrieves an image name from the test data.
-    def image_name
-      @image_name ||= (@config.has?('image_name_builder') ? image_name_builder(@config, test_data) : test_data[:image_name])
-    end
-
     private
 
     # Generates a random node name.
     def random_node_name
       "cem-acpt-#{SecureRandom.hex(12)}"
     end
+  end
+
+  # Base class for all test node platform classes. This class provides an API for
+  # interacting with the underlying platform.
+  class TestBase < Base
+    attr_reader :test_data
+
+    # @param conf [CemAcpt::Config] the config object.
+    # @param single_test_data [Hash] the test data for the current test.
+    # @param run_data [Hash] the run data for the current run minus the test data key.
+    def initialize(config, single_test_data, **run_data)
+      super(config, **run_data)
+      raise ArgumentError, 'single_test_data must be a Hash' unless single_test_data.is_a?(Hash)
+
+      @test_data = single_test_data
+    end
+
+    def node_name
+      @node_name ||= (@test_data[:node_name] || random_node_name)
+    end
+
+    def to_h
+      {
+        node_name: node_name,
+        image_name: image_name,
+        test_data: test_data,
+        platform_data: platform_data,
+        node_data: node_data,
+      }
+    end
+
+    # Generates or retrieves an image name from the test data.
+    def image_name
+      @image_name ||= (@config.has?('image_name_builder') ? image_name_builder(@config, test_data) : test_data[:image_name])
+    end
+
+    private
 
     # Builds an image name if the config specifies to use the image name builder.
     def image_name_builder(conf, tdata)

data/lib/cem_acpt/platform/gcp.rb

@@ -18,12 +18,12 @@ module Platform
   end
 
   def node_data
-    {
-      machine_type: gcp_machine_type,
-      image: image_name,
-      disk_size: gcp_disk_size,
-      test_name: @test_data[:test_name],
-    }
+    nd = {}
+    nd[:machine_type] = gcp_machine_type
+    nd[:disk_size] = gcp_disk_size
+    nd[:image] = image_name if image_name
+    nd[:test_name] = @test_data[:test_name] if @test_data&.key?(:test_name)
+    nd
   end
 
   def gcp_username
@@ -62,7 +62,7 @@ module Platform
   end
 
   def gcp_machine_type
-    @gcp_machine_type ||= (@config.get('node_data.machine_type') || 'e2-small')
+    @gcp_machine_type ||= (@config.get('node_data.machine_type') || 'e2-medium')
   end
 
   def gcp_disk_size

data/lib/cem_acpt/platform.rb

@@ -9,6 +9,7 @@ module CemAcpt::Platform
   class Error < StandardError; end
 
   PLATFORM_DIR = File.expand_path(File.join(__dir__, 'platform'))
+  BASE_TYPES = %i[base test].freeze
 
   class << self
     include CemAcpt::Logging
@@ -26,16 +27,18 @@ module CemAcpt::Platform
 
       logger.info "Using #{platform} for #{run_data[:test_data].length} tests..."
       run_data[:test_data].dup.each_with_object([]) do |single_test_data, ary|
-        ary << new_platform_object(platform, config, single_test_data, **run_data.reject { |k, _| k == :test_data })
+        ary << new_test_platform_object(platform, config, single_test_data, **run_data.reject { |k, _| k == :test_data })
       end
     end
 
     # Returns an un-initialized platform specific Class of the given platform.
     # @param platform [String] the name of the platform
-    def get(platform)
+    # @param base_type [Symbol] the type of base class to use.
+    def get(platform, base_type: :test)
       raise Error, "Platform #{platform} is not supported" unless platforms.include?(platform)
+      raise Error, "Base type #{base_type} is not supported" unless BASE_TYPES.include?(base_type.to_sym)
 
-      platform_class(platform)
+      platform_class(base_type, platform)
     end
 
     # Returns an array of the names of the supported platforms.
@@ -62,17 +65,18 @@ module CemAcpt::Platform
     # @param config [CemAcpt::Config] the config object.
     # @param single_test_data [Hash] the test data for a single test.
     # @return [CemAcpt::Platform::Base] an initialized platform class.
-    def new_platform_object(platform, config, single_test_data, **run_data)
+    def new_test_platform_object(platform, config, single_test_data, **run_data)
       raise Error, 'single_test_data must be a Hash' unless single_test_data.is_a?(Hash)
 
-      platform_class(platform).new(config, single_test_data, **run_data)
+      platform_class(:test, platform).new(config, single_test_data, **run_data)
     end
 
     # Creates a new platform-specific Class object for the given platform.
     # Does not initialize the class.
+    # @param base_type [Symbol] the type of base class to use.
     # @param platform [String] the name of the platform.
     # @return [Class] the platform-specific Object.
-    def platform_class(platform)
+    def platform_class(base_type, platform)
       class_name = platform.capitalize
       # We require the platform base class here so that we can use it as
       # a parent class for the platform-specific class.
@@ -90,9 +94,17 @@ module CemAcpt::Platform
         # include Logging and Concurrent::Async, and finally call the
         # initialize method on the class.
         logger.debug "Creating platform class #{class_name}"
+        baseklass = case base_type.to_sym
+                    when :base
+                      CemAcpt::Platform::Base
+                    when :test
+                      CemAcpt::Platform::TestBase
+                    else
+                      raise Error, "Base type #{base_type} is not supported"
+                    end
         klass = Object.const_set(
           class_name,
-          Class.new(CemAcpt::Platform::Base) do
+          Class.new(baseklass) do
             require_relative "platform/#{platform}"
             include Platform
           end,

data/lib/cem_acpt/provision/terraform.rb

@@ -156,7 +156,7 @@ module CemAcpt
         logger.verbose('Terraform') { "Base directory defined as #{base_dir}" }
         @backend.base_provision_directory = base_dir
         logger.verbose('Terraform') { 'Base directory set in backend' }
-        work_dir = File.join(@config.get('terraform.dir'), "test_#{Time.now.to_i.to_s}")
+        work_dir = File.join(@config.get('terraform.dir'), "test_#{Time.now.to_i}")
         logger.verbose('Terraform') { "Working directory defined as #{work_dir}" }
         logger.verbose('Terraform') { "Copying backend provision directory #{@backend.provision_directory} to working directory" }
         FileUtils.cp_r(@backend.provision_directory, work_dir)

data/lib/cem_acpt/test_data.rb

@@ -1,7 +1,7 @@
 # frozen_string_literal: true
 
 require 'pathname'
-require_relative 'core_extensions'
+require_relative 'core_ext'
 require_relative 'logging'
 
 module CemAcpt
@@ -18,7 +18,7 @@ module CemAcpt
     # Fetcher provides the methods for extracting and formatting test data.
     class Fetcher
       include CemAcpt::Logging
-      using CemAcpt::CoreExtensions::ExtendedHash
+      using CemAcpt::CoreExt::ExtendedHash
 
       attr_reader :acpt_test_dir, :acceptance_tests
 

data/lib/cem_acpt/test_runner.rb

@@ -1,6 +1,5 @@
 # frozen_string_literal: true
 
-require_relative 'core_extensions'
 require_relative 'goss'
 require_relative 'logging'
 require_relative 'platform'
@@ -57,8 +56,8 @@ module CemAcpt
             logger.info('CemAcpt') { 'Provisioned test nodes...' }
             logger.debug('CemAcpt') { "Instance names and IPs: #{@instance_names_ips}" }
             @results = run_tests(@instance_names_ips.map { |_, v| v['ip'] },
-                                config.get('actions.only'),
-                                config.get('actions.except'))
+                                 config.get('actions.only'),
+                                 config.get('actions.except'))
           end
         end
       ensure

data/lib/cem_acpt/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module CemAcpt
-  VERSION = '0.6.5'
+  VERSION = '0.7.0'
 end