cem_acpt 0.1.0

data/lib/cem_acpt/platform/gcp/compute.rb

@@ -0,0 +1,327 @@
+# frozen_string_literal: true
+
+module CemAcpt::Platform::Gcp
+  require_relative 'cmd'
+  require_relative File.join(__dir__, '..', '..', 'logging.rb')
+
+  module Helper
+    def add_cmd(cmd_obj)
+      @cmd = cmd_obj
+    end
+
+    def cmd
+      @cmd ||= CemAcpt::Platform::Gcp::Cmd.new(out_format: 'json')
+    end
+
+    def cmd_flag_vars?(*vars)
+      vars.none? { |v| v.nil? || (v.empty? if v.respond_to?(:empty?)) }
+    end
+  end
+
+  # This class represents a GCP project
+  class Project
+    include Helper
+
+    def initialize(name: nil, zone: nil)
+      @name = name
+      @zone = zone
+    end
+
+    def name
+      @name ||= cmd.project_from_config
+    end
+
+    def zone
+      @zone ||= cmd.zone_from_config
+    end
+
+    def cmd_flag
+      "--project=#{name}"
+    end
+  end
+
+  # This class represents a GCP VM service account
+  class ServiceAccount
+    include Helper
+
+    attr_reader :name, :scopes
+
+    def initialize(name: nil, scopes: ['devstorage.read_only', 'logging.write', 'monitoring.write', 'servicecontrol', 'service.management.readonly', 'trace.append' ])
+      @name = name
+      @scopes = scopes.map { |s| "https://www.googleapis.com/auth/#{s}" }
+    end
+
+    def cmd_flag
+      "--service-account=#{name} --scopes=#{scopes.join(',')}" if cmd_flag_vars?(name, scopes)
+    end
+  end
+
+  # This class represents a GCP VM disk
+  class Disk
+    include Helper
+
+    attr_reader :project, :name, :size, :image_name
+
+    def initialize(project: nil, name: 'disk-1', image_name: '', size: 20, type: 'pd-standard')
+      @project = project
+      @name = name
+      @image_name = image_name
+      @size = size
+      @type = type
+    end
+
+    def cmd_flag
+      return unless cmd_flag_vars?(name, image, size, type)
+
+      '--create-disk=auto-delete=yes,boot=yes,' \
+      "device-name=#{name},image=#{image}," \
+      "mode=rw,size=#{size},type=#{type}"
+    end
+
+    def type
+      type_path(@type)
+    end
+
+    def image
+      @image ||= image_path(@image_name)
+    end
+
+    private
+
+    def image_path(image_str)
+      raise ArgumentError, 'image must be a non-empty string' unless image_str.is_a?(String) && !image_str.empty?
+
+      data = cmd.local_exec('compute images list', out_filter: "name ~ '#{image_str}$'").first
+      if data.nil? || data.empty?
+        data = cmd.local_exec('compute images list', out_filter: "family ~ '#{image_str}$'").first
+      end
+      raise "Image #{image_str} not found" unless data
+      raise "Image status invalid for image #{data['selfLink']}: #{data[status]}" unless data['status'] == 'READY'
+
+      data['selfLink'].split('v1/')[-1]
+    end
+
+    def type_path(type)
+      raise ArgumentError, 'type must be a non-empty string' unless type.is_a?(String) && !type.empty?
+
+      "projects/#{project.name}/zones/#{project.zone}/diskTypes/#{type}"
+    end
+  end
+
+  # This class represents a GCP VM network interface
+  class NetworkInterface
+    include Helper
+
+    attr_reader :tier, :subnetwork
+
+    def initialize(tier: 'STANDARD', subnetwork: 'default')
+      @tier = tier.upcase
+      @subnetwork = subnetwork
+    end
+
+    def cmd_flag
+      "--network-interface=network-tier=#{tier},subnet=#{subnetwork}" if cmd_flag_vars?(tier, subnetwork)
+    end
+  end
+
+  # This class represents GCP VM metadata
+  class Metadata
+    include Helper
+
+    def initialize(**data)
+      @data = data.transform_keys(&:to_s)
+    end
+
+    def data
+      format_data(@data)
+    end
+
+    def cmd_flag
+      return unless cmd_flag_vars?(data)
+
+      data_from_file = data.select { |k, _v| k == 'ssh-keys' }.map { |k, v| "#{k}=#{v}" }.join(',')
+      normalized_data = data.reject { |k, _v| k == 'ssh-keys' }.map { |k, v| "#{k}=#{v}" }.join(',')
+      cmd = []
+      cmd << "--metadata-from-file=#{data_from_file}" unless data_from_file.empty?
+      cmd << "--metadata=#{normalized_data}" unless normalized_data.empty?
+      cmd.join(' ')
+    end
+
+    private
+
+    def format_data(data)
+      default_data = { 'enable-oslogin' => 'TRUE' }
+      if data['ephemeral_ssh_key']
+        args = {
+          username: (data.dig('ephemeral_ssh_key', 'username') || cmd.user_name),
+        }
+        args[:lifetime] = data.dig('ephemeral_ssh_key', 'lifetime') if data.dig('ephemeral_ssh_key', 'lifetime')
+        args[:keydir] = data.dig('ephemeral_ssh_key', 'keydir') if data.dig('ephemeral_ssh_key', 'keydir')
+        priv_key_path, ssh_keys = ephemeral_ssh_key(**args)
+        default_data['priv_key_local_path'] = priv_key_path
+        default_data['ssh-keys'] = ssh_keys
+        data.delete('ephemeral_ssh_key')
+      end
+      default_data.merge(data)
+    end
+
+    def ephemeral_ssh_key(username: nil, lifetime: 2, keydir: '/tmp/acpt_ssh_keys')
+      raise ArgumentError, 'username must be a non-empty string' unless username.is_a?(String) && !username.empty?
+
+      keypath = File.join(keydir, SecureRandom.hex(10))
+      Dir.mkdir(keydir) unless Dir.exist?(keydir)
+      lifetime_in_seconds = (lifetime.to_i * 60 * 60)
+      iso8601_expiration = (Time.now.utc + lifetime_in_seconds).strftime('%Y-%m-%dT%H:%M:%S%z')
+      keygen_cmd = [
+        'ssh-keygen',
+        '-t ecdsa',
+        '-b 521',
+        "-C 'google-ssh {\"userName\":\"#{username}\",\"expireOn\":\"#{iso8601_expiration}\"}'",
+        "-f #{keypath}",
+        '-N ""',
+        '-q',
+      ].join(' ')
+      _, stderr, status = Open3.capture3(keygen_cmd)
+      raise stderr unless status.success?
+
+      File.write("#{keypath}.pub", "#{username}:#{File.read("#{keypath}.pub")}")
+      [keypath, "#{keypath}.pub"]
+    rescue StandardError => e
+      raise "Failed to generate ephemeral SSH key: #{e} #{e.backtrace}"
+    end
+  end
+
+  # This class represents a GCP VM. It is composed of various component classes.
+  class VM
+    include Helper
+    include CemAcpt::Logging
+
+    attr_accessor :name, :project, :service_account, :disk, :network_interface, :machine_type, :metadata
+    attr_reader :cmd
+
+    def initialize(name, components: {})
+      @name = name
+      @components = components
+      @machine_type = components[:machine_type]
+      @cmd = CemAcpt::Platform::Gcp::Cmd.new(
+        project: components[:project][:name],
+        zone: components[:project][:zone],
+        out_format: 'json',
+        local_port: components[:local_port],
+      )
+      @configured = false
+    end
+
+    def configure!
+      return @configured if @configured
+
+      @project = Project.new(**@components[:project])
+      @project.add_cmd(@cmd)
+      @service_account = @components.key?(:service_account) ? ServiceAccount.new(**@components[:service_account]) : nil
+      @disk = Disk.new(project: @project, **@components[:disk])
+      @disk.add_cmd(@cmd)
+      @network_interface = NetworkInterface.new(**@components[:network_interface])
+      @machine_type = @components[:machine_type]
+      @metadata = Metadata.new(**@components[:metadata])
+      @metadata.add_cmd(@cmd)
+      @configured = true
+    end
+
+    def info
+      describe_cmd = "compute instances describe #{name}"
+      logger.debug("Gathering info for VM #{name} with gcloud command: #{describe_cmd}")
+      data = @cmd.local_exec(describe_cmd, out_format: 'json')
+      opts = @cmd.ssh_opts(instance_name: name)
+      {
+        node_data: data,
+        transport: :ssh,
+        ssh_opts: opts,
+      }
+    end
+
+    def create
+      @cmd.local_exec(create_cmd)
+    rescue StandardError => e
+      raise "Failed to create VM #{name} with command #{create_cmd}: #{e}"
+    end
+
+    def ready?
+      logger.debug("Checking if VM #{name} is ready")
+      instance_status = @cmd.local_exec('compute instances list', out_filter: "NAME = #{name}").first['status']
+      logger.debug("Instance #{name} status: #{instance_status}")
+      return false unless instance_status == 'RUNNING'
+
+      logger.debug("Checking instance #{name} SSH connectivity")
+      @cmd.ssh_ready?(name)
+    rescue StandardError
+      false
+    end
+
+    def destroy
+      @cmd.delete_instance(name)
+    end
+
+    def install_puppet_module_package(module_pkg_path, remote_path, puppet_path)
+      @cmd.scp_upload(@name, module_pkg_path, remote_path)
+      @cmd.ssh(@name, "sudo #{puppet_path} module install #{remote_path}")
+    end
+
+    private
+
+    # Determines whether the given instance name is a fully qualified
+    # hostname or not.
+    def hostname?(instance_name)
+      instance_name.include?('.')
+    end
+
+    # Validates a GCP instance name.
+    def validate_instance_name(instance_name)
+      if instance_name.nil? || !instance_name.is_a?(String) || instance_name.empty?
+        raise ArgumentError, 'Instance name must be a non-empty string'
+      end
+      raise ArgumentError, 'Instance name must be less than 253 characters' if instance_name.length > 253
+
+      return true unless hostname?(instance_name)
+
+      labels = instance_name.split('.')
+      labels.each do |label|
+        unless label.match?(%r{^[a-z]([-a-z0-9]*[a-z0-9])?$})
+          raise ArgumentError, "Instance name portion #{label} contains invalid characters"
+        end
+        raise ArgumentError, "Instance name portion #{label} is too long" if label.length > 63
+      end
+      true
+    end
+
+    def create_cmd
+      validate_instance_name(name)
+      if hostname?(name)
+        instance_name = name.split('.')[0]
+        hostname = name
+      else
+        instance_name = name
+        hostname = nil
+      end
+
+      cmd = [
+        'compute',
+        'instances',
+        'create',
+        instance_name,
+        '--async',
+        "--machine-type=#{machine_type}",
+        '--maintenance-policy=MIGRATE',
+        '--no-shielded-secure-boot',
+        '--shielded-vtpm',
+        '--shielded-integrity-monitoring',
+        '--reservation-affinity=any',
+        '--tags=acpt-test-node',
+      ]
+      [disk&.cmd_flag, network_interface&.cmd_flag, metadata&.cmd_flag, service_account&.cmd_flag].each do |flag|
+        cmd << flag unless flag.nil?
+      end
+      cmd << "--hostname=#{hostname}" unless hostname.nil?
+      cmd.join(' ')
+    end
+  end
+end

data/lib/cem_acpt/platform/gcp.rb

@@ -0,0 +1,85 @@
+# frozen_string_literal: true
+
+# GCP platform implementation
+module Platform
+  require_relative 'gcp/compute'
+  require_relative 'gcp/cmd'
+
+  # Returns information about the GCP instance
+  def node
+    @instance.info
+  end
+
+  # Provision a GCP instance
+  def provision
+    creation_params = config.dup
+    creation_params[:disk][:image_name] = image_name
+    creation_params[:local_port] = local_port
+    @instance = CemAcpt::Platform::Gcp::VM.new(
+      node_name,
+      components: creation_params,
+    )
+    @instance.configure!
+    logger.debug("Creating with command: #{@instance.send(:create_cmd)}")
+    @instance.create
+  end
+
+  # Destroy a GCP instance
+  def destroy
+    @instance.destroy
+  end
+
+  # Returns true if the GCP instance is ready for use in the test suite
+  def ready?
+    logger.debug("Checking if #{node_name} is ready...")
+    @instance.ready?
+  end
+
+  # Runs the test suite against the GCP instance. Must be given a block.
+  # If necessary, can pass information into the block to be used in the test suite.
+  def run_tests(&block)
+    logger.debug("Running tests for #{node_name}...")
+    block.call
+  end
+
+  # Uploads and installs a Puppet module package on the GCP instance.
+  def install_puppet_module_package(module_pkg_path, remote_path = nil, puppet_path = '/opt/puppetlabs/bin/puppet')
+    remote_path = remote_path.nil? ? File.join('/tmp', File.basename(module_pkg_path)) : remote_path
+    logger.info("Uploading module package #{module_pkg_path} to #{remote_path} on #{node_name} and installing it...")
+    logger.debug("Using puppet path: #{puppet_path}")
+    @instance.install_puppet_module_package(module_pkg_path, remote_path, puppet_path)
+    logger.info("Module package #{module_pkg_path} installed on #{node_name}")
+  end
+
+  # Extends the class with class methods from the SpecMethods module
+  def self.included(base)
+    base.extend(SpecMethods)
+  end
+
+  # Holds class methods called from spec tests.
+  module SpecMethods
+    # Returns an instance of the GCP platform class command provider
+    # @return [CemAcpt::Platform::Gcp::Cmd]
+    def command_provider
+      CemAcpt::Platform::Gcp::Cmd.new(out_format: 'json')
+    end
+
+    # Apllies the given Puppet manifest on the given instance
+    # @param instance_name [String] the name of the instance to apply the manifest to
+    # @param manifest [String] the Puppet manifest to apply
+    # @param opts [Hash] options to pass to the apply command
+    # @return [String] the output of the apply command
+    def apply_manifest(instance_name, manifest, opts = {})
+      CemAcpt::Platform::Gcp::Cmd.new(out_format: 'json').apply_manifest(instance_name, manifest, opts)
+    end
+
+    # Runs a shell command on the given instance
+    # @param instance_name [String] the name of the instance to run the command on
+    # @param command [String] the command to run
+    # @param opts [Hash] options to pass to the run_shell command
+    # @return [String] the output of the run_shell command
+    def run_shell(instance_name, cmd, opts = {})
+      CemAcpt::Platform::Gcp::Cmd.new(out_format: 'json').run_shell(instance_name, cmd, opts)
+    end
+  end
+end

data/lib/cem_acpt/platform/vmpooler.rb

@@ -0,0 +1,24 @@
+# frozen_string_literal: true
+
+# Currently a placeholder for future functionality
+module Platform
+  def node
+    logger.debug 'VMPooler node'
+  end
+
+  def provision
+    logger.debug 'VMPooler provision'
+  end
+
+  def destroy
+    logger.debug 'VMPooler destroy'
+  end
+
+  def ready?
+    logger.debug 'VMPooler ready?'
+  end
+
+  def install_puppet_module_package
+    logger.debug 'VMPooler install_puppet_module_package'
+  end
+end

data/lib/cem_acpt/platform.rb

@@ -0,0 +1,103 @@
+# frozen_string_literal: true
+
+require 'concurrent-ruby'
+
+# CemAcpt::Platform manages creating and configring platform specific objects
+# for the acceptance test suites.
+module CemAcpt::Platform
+  require_relative 'logging'
+
+  class Error < StandardError; end
+
+  PLATFORM_DIR = File.expand_path(File.join(__dir__, 'platform'))
+
+  class << self
+    include CemAcpt::Logging
+
+    # Creates a new platform specific object of the given platform for each
+    # item in the test data.
+    # @param platform [String] the name of the platform
+    # @param config [CemAcpt::Config] the config object
+    # @param test_data [Hash] the test data
+    # @param local_port_allocator [CemAcpt::LocalPortAllocator] the local port allocator
+    def use(platform, config, test_data, local_port_allocator)
+      raise Error, "Platform #{platform} is not supported" unless platforms.include?(platform)
+      raise Error, 'test_data must be an Array' unless test_data.is_a?(Array)
+
+      logger.info "Using #{platform} for #{test_data.length} tests..."
+      test_data.each_with_object([]) do |single_test_data, ary|
+        local_port = local_port_allocator.allocate
+        logger.debug("Allocated local port #{local_port} for test #{single_test_data[:test_name]}")
+        ary << new_platform_object(platform, config, single_test_data, local_port)
+      end
+    end
+
+    # Returns an un-initialized platform specific Class of the given platform.
+    # @param platform [String] the name of the platform
+    def get(platform)
+      raise Error, "Platform #{platform} is not supported" unless platforms.include?(platform)
+
+      platform_class(platform)
+    end
+
+    private
+
+    # Dynamically creates a new platform class if it doesn't exist
+    # and returns a new instance of it.
+    # @param platform [String] the name of the platform.
+    # @param config [CemAcpt::Config] the config object.
+    # @param single_test_data [Hash] the test data for a single test.
+    # @return [CemAcpt::Platform::Base] an initialized platform class.
+    def new_platform_object(platform, config, single_test_data, local_port)
+      raise Error, 'single_test_data must be a Hash' unless single_test_data.is_a?(Hash)
+
+      platform_class(platform).new(config, single_test_data, local_port)
+    end
+
+    # Creates a new platform-specific Class object for the given platform.
+    # Does not initialize the class.
+    # @param platform [String] the name of the platform.
+    # @return [Class] the platform-specific Object.
+    def platform_class(platform)
+      # We require the platform base class here so that we can use it as
+      # a parent class for the platform-specific class.
+      require_relative 'platform/base'
+      # If the class has already been defined, we can just use it.
+      if Object.const_defined?(platform.capitalize)
+        klass = Object.const_get(platform.capitalize)
+      else
+        # Otherwise, we need to create the class. We do this by setting
+        # a new constant with the name of the platform capitalized, and
+        # associate that constant with a new instance of Class that inherits
+        # from the platform base class. We then require the platform file,
+        # include and extend our class with the Platform module from the file,
+        # include Logging and Concurrent::Async, and finally call the
+        # initialize method on the class.
+        klass = Object.const_set(
+          platform.capitalize,
+          Class.new(CemAcpt::Platform::Base) do
+            require_relative "platform/#{platform}"
+            include Platform
+          end,
+        )
+      end
+      klass
+    end
+
+    # Returns an array of the names of the supported platforms.
+    # Supported platforms are discovered by looking for files in the
+    # platform directory, and platform names are the basename (no extension)
+    # of the files. We deliberately exclude the base class, as it is not
+    # a platform.
+    def platforms
+      return @platforms if defined?(@platforms)
+
+      @platforms = Dir.glob(File.join(PLATFORM_DIR, '*.rb')).map do |file|
+        File.basename(file, '.rb') unless file.end_with?('base.rb')
+      end
+      @platforms.compact!
+      logger.debug "Discovered platform(s): #{@platforms}"
+      @platforms
+    end
+  end
+end

data/lib/cem_acpt/puppet_helpers.rb

@@ -0,0 +1,38 @@
+# frozen_string_literal: true
+
+require_relative 'logging'
+
+module CemAcpt
+  # Holds modules that provide helper methods for various Puppet-related
+  # tasks.
+  module PuppetHelpers
+    # Provides helper methods for Puppet Modules.
+    module Module
+      class << self
+        include CemAcpt::Logging
+      end
+
+      # Builds a Puppet module package.
+      # @param module_dir [String] Path to the module directory. If target_dir
+      #   is specified as a relative path, it will be relative to the module dir.
+      # @param target_dir [String] Path to the target directory where the package
+      #   will be built. This defaults to the relative path 'pkg/'.
+      # @param should_log [Boolean] Whether or not to log the build process.
+      # @return [String] Path to the built package.
+      def self.build_module_package(module_dir, target_dir = nil, should_log: false)
+        require 'puppet/modulebuilder'
+
+        builder_logger = should_log ? logger : nil
+        builder = Puppet::Modulebuilder::Builder.new(File.expand_path(module_dir), target_dir, builder_logger)
+
+        # Validates module metadata by raising exception if invalid
+        _metadata = builder.metadata
+        logger.debug("Metadata for module #{builder.release_name} is valid")
+
+        # Builds the module package
+        logger.info("Building module package for #{builder.release_name}")
+        builder.build
+      end
+    end
+  end
+end