cem_acpt 0.2.4 → 0.6.0

data/lib/cem_acpt/shared_objects.rb

@@ -1,537 +0,0 @@
-# frozen_string_literal: true
-
-require 'concurrent-ruby'
-require 'deep_merge'
-require 'json'
-require 'yaml'
-require_relative 'core_extensions'
-require_relative 'logging'
-
-module CemAcpt
-  using CemAcpt::CoreExtensions::ExtendedHash
-
-  class ConfigImmutableError < RuntimeError; end
-
-  # Holds module methods for writing various config objects
-  # to files.
-  module ConfigUtils
-    class << self
-      include CemAcpt::Logging
-    end
-
-    def self.object_to_file(obj, file_path)
-      case File.extname(file_path)
-      when '.json'
-        raise "Object #{obj} is not serializable to JSON" unless obj.respond_to?(:to_json)
-
-        logger.debug("Writing object to #{file_path}")
-        File.write(file_path, obj.to_json)
-      when '.yaml', '.yml'
-        raise "Object #{obj} is not serializable to YAML" unless obj.respond_to?(:to_yaml)
-
-        logger.debug("Writing object to #{file_path}")
-        File.write(file_path, obj.to_yaml)
-      else
-        raise "File extension #{File.extname(file_path)} is not supported"
-      end
-    end
-  end
-
-  # Provides a thread-safe Config object.
-  # Once the config is loaded by calling the `load` method, the object is
-  # immutable and thread-safe as long as the internal hash is only accessed
-  # by the `#get` and `#has?` methods. If `#load` is called more than once,
-  # the object will raise an error.
-  class Config
-    include CemAcpt::Logging
-
-    attr_reader :config_file
-
-    def initialize
-      @opts = {
-        thread_pool: {
-          min_threads: [2, Concurrent.processor_count - 1].max,
-          max_threads: [2, Concurrent.processor_count - 1].max,
-          max_queue: [2, Concurrent.processor_count - 1].max * 10,
-          fallback_policy: :caller_runs,
-          idletime: 1200,
-          auto_terminate: true,
-        },
-      }
-    end
-
-    # Returns the value of the dot-separated key.
-    # If the key is not found, returns nil.
-    # @param dot_key [String] Dot-separated key
-    # @return [Object] Value of the key
-    def get(dot_key)
-      @opts.dot_dig(dot_key)
-    end
-
-    # Checks if the dot-separated key exists in the config.
-    # @param dot_key [String] Dot-separated key
-    # @return [Boolean] True if the key exists, false otherwise
-    def has?(dot_key)
-      !!get(dot_key)
-    end
-
-    # Checks to see if cem_acpt is set to run in debug mode (log level is set to debug)
-    # @return [TrueClass] If cem_acpt is running in debug mode
-    # @return [FalseClass] If cem_acpt is not runnint in debug mode
-    def debug_mode?
-      @opts.dot_dig('log_level') == 'debug'
-    end
-
-    # Loads the config from specified file path. Config files must be in YAML
-    # format. If 'opts' is specified, it will be merged with the config file.
-    # Values in 'opts' will override values in the config file.
-    # @param opts [Hash] Options to be merged with the config file
-    # @param config_file [String] Path to the config file
-    def load(opts: {}, config_file: './cem_acpt_config.yaml')
-      raise ConfigImmutableError, 'Config is immutable, cannot load more than once' if frozen?
-      raise ArgumentError, 'opts must be a Hash' unless opts.is_a?(Hash)
-      raise ArgumentError, 'config_file must be a String' unless config_file.is_a?(String)
-
-      @config_file = File.expand_path(config_file)
-      @opts.deep_merge!(load_opts_from_file(File.expand_path(config_file)).deep_merge!(opts))
-      @opts.format!
-      @opts.freeze
-      freeze
-    end
-
-    # Returns the config as a Hash.
-    # @return [Hash] Config as a Hash
-    def to_h
-      ::Marshal.load(::Marshal.dump(@opts))
-    end
-
-    # Returns the config as a YAML string.
-    # @return [String] Config as a YAML string
-    def to_yaml
-      to_h.to_yaml
-    end
-
-    # Returns the config as a JSON string.
-    # @return [String] Config as a JSON string
-    def to_json(*args)
-      to_h.to_json(*args)
-    end
-
-    private
-
-    # Loads the config from the specified file path.
-    # @param config_file [String] Path to the config file
-    # @return [Hash] Config hash
-    def load_opts_from_file(config_file)
-      return {} unless File.exist?(config_file)
-
-      logger.debug("Loading config from #{config_file}")
-      YAML.load_file(config_file)
-    end
-  end
-
-  # Provides an object to hold NodeData configs.
-  # Not thread-safe. Should not be mutated by itself,
-  # but can be mutated safely by the NodeInventory object.
-  # Currently unused.
-  class NodeData
-    include CemAcpt::Logging
-
-    attr_reader :name, :data
-
-    def initialize(name, data)
-      data.format!
-      @name = name
-      @opts = data
-    end
-
-    def deep_merge(other)
-      @opts.deep_merge(other.to_h)
-    end
-
-    def dot_dig(dot_key)
-      @opts.dot_dig(dot_key)
-    end
-
-    def dot_store(dot_key, value)
-      @opts.dot_store(dot_key, value)
-    end
-
-    def to_h
-      @opts
-    end
-
-    def to_yaml
-      to_h.to_yaml
-    end
-
-    def to_json(*args)
-      to_h.to_json(*args)
-    end
-  end
-
-  class NodeInventoryFileNotFoundError < StandardError; end
-  class NodeInventoryFileLoadError < StandardError; end
-  class NodeClaimedError < StandardError; end
-  class NodeDoesNotExistError < StandardError; end
-  class PropertyNotFoundError < StandardError; end
-  class LockWaitTimeoutError < StandardError; end
-  class LockNotRecognizedError < StandardError; end
-
-  # Provides a thread-safe inventory of test nodes.
-  class NodeInventory
-    include CemAcpt::Logging
-
-    attr_accessor :save_file_path
-
-    def initialize
-      @inventory = Concurrent::Map.new
-      @lock = Concurrent::ReadWriteLock.new
-      @claimed = Concurrent::Set.new
-      @save_on_claim = false
-      @save_file_path = 'spec/fixtures/node_inventory'
-      @loaded_node_inv = nil
-    end
-
-    # When called, enables saving the inventory to a file on claim.
-    def save_on_claim
-      @save_on_claim = true
-    end
-
-    # Checks if save_on_claim is enabled.
-    def save_on_claim?
-      @save_on_claim
-    end
-
-    # Adds a new node to the inventory.
-    # @param node_name [String] The name of the node to add.
-    # @param node_data [Hash] The node data to add.
-    def add(node_name, node_data)
-      @inventory.put_if_absent(node_name, node_data)
-    end
-
-    def update(node_name, node_data)
-      @inventory.replace_if_exists(node_name, node_data)
-    end
-
-    # Returns the node data for a given node.
-    # @param node_name [String] The name of the node to get data for.
-    def get(node_name)
-      @inventory[node_name]
-    end
-
-    # Returns the node_name and node_data for a given node
-    # that has a matching value for the given property.
-    # @param property_path [String] The dot-separated property path to check.
-    # @param value [Object] The value to check for.
-    # @return [Array] Two-item frozen array of node name and node data.
-    # @raise [NodeDoesNotExistError] If no nodes are found with the property equal to the value.
-    def get_by_property(property_path, value)
-      found = []
-      @inventory.each_pair do |node_name, node_data|
-        next unless node_data.dot_dig(property_path) == value
-
-        found = [node_name, node_data].freeze
-      end
-      raise NodeDoesNotExistError, "No node found with property #{property_path} == #{value}" if found.empty?
-
-      found
-    end
-
-    # Returns all property values of all nodes for the given property path
-    # @param property_path [String] The dot-separated property path to check.
-    # @return [Array] A frozen array of found property values.
-    # @raise [PropertyNotFoundError] If no values are found on any nodes for the property path.
-    def get_all_properties(property_path)
-      found = []
-      @inventory.each_pair do |_, node_data|
-        prop = node_data.dot_dig(property_path)
-        next unless prop
-
-        found << prop
-      end
-      raise PropertyNotFoundError, "No property with path #{property_path} found in nodes" if found.empty?
-
-      found.freeze
-    end
-
-    # Sets a specific property on a node in the inventory.
-    # @param node_name [String] The name of the node to set the property on.
-    # @param property_path [String] The dot-separated property path to set.
-    # @param value [Object] The value to set.
-    def set(node_name, property_path, value)
-      @inventory[node_name].dot_store(property_path, value)
-    end
-
-    # Merges new node data in with the existing node data for the given node.
-    # @param node_name [String] The name of the node to merge data for.
-    # @param new_node_data [Hash] The new node data to merge.
-    def merge!(node_name, new_node_data)
-      @inventory.merge_pair(node_name, new_node_data) do |old_data|
-        old_data.deep_merge(new_node_data)
-      end
-    end
-
-    # Checks if a node is claimed.
-    # @param node_name [String] The name of the node to check.
-    # @return [Boolean] True if the node is claimed, false otherwise.
-    def claimed?(node_name)
-      raise NodeDoesNotExistError, "Node #{node_name} does not exist" unless @inventory.keys.include?(node_name)
-
-      @claimed.include?(node_name)
-    end
-
-    # Claims a node, which returns the node data and adds that node to the
-    # claimed set. Raises an error if the node is already claimed or does
-    # not exist. This is used during acceptance testing to ensure that
-    # nodes are not reused.
-    # @param node_name [String] The name of the node to claim.
-    # @return [String] The name of the node that was claimed.
-    # @raise [NodeDoesNotExistError] If the node does not exist in the inventory.
-    # @raise [NodeClaimedError] If the node is already claimed.
-    def claim(node_name)
-      with_lock_retry(:write) do
-        unless @inventory.keys.include?(node_name)
-          raise NodeDoesNotExistError, "Node #{node_name} does not exist in inventory"
-        end
-
-        if @claimed.include?(node_name)
-          raise NodeClaimedError, "Node #{node_name} is already tainted and cannot be claimed"
-        end
-
-        @claimed.add(node_name)
-        save_no_lock!(@save_file_path) if @save_on_claim
-        node_name
-      end
-    end
-
-    # Claims a node based on a property of the node data and a value that property should equal.
-    # If a node is found but is already claimed, the method will continue to search for an
-    # unclaimed node. Raises an error if no valid node is found. This is used during acceptance
-    # testing to ensure that nodes are not reused.
-    # @param property_path [String] The dot-separated property path to check.
-    # @param value [Object] The value to check for.
-    # @return [String] The name of the node that was claimed.
-    # @raise [NodeDoesNotExistError] If no valid node is found.
-    def claim_by_property(property_path, value)
-      attempts ||= 1
-      claim_name = nil
-      @inventory.each_pair do |node_name, node_data|
-        next if @claimed.include?(node_name)
-
-        claim_name = node_name if node_data.dot_dig(property_path) == value
-      end
-      raise NodeDoesNotExistError, "No node found with property #{property_path} == #{value}" if claim_name.nil?
-
-      claim(claim_name)
-    rescue NodeDoesNotExistError => e
-      # We sleep than retry three times to help mitigate race conditions
-      if (attempts += 1) <= 3
-        sleep(1)
-        retry
-      else
-        raise e
-      end
-    end
-
-    # Deletes a node from the inventory.
-    # @param node_name [String] The name of the node to delete.
-    def delete(node_name)
-      @inventory.delete(node_name)
-    end
-
-    # Returns all node names in the current inventory.
-    def nodes
-      @inventory.keys
-    end
-
-    # Clears the inventory and removes claimed nodes. DOES NOT USE LOCK.
-    # If this is called outside of a lock, it will not be thread-safe.
-    def clear_no_lock!
-      nodes.each { |node| delete(node) }
-      @claimed.clear
-    end
-
-    # Clears the inventory and removes claimed nodes. Thread-safe.
-    def clear!
-      with_lock_retry(:write) do
-        clear_no_lock!
-      end
-    end
-
-    # Checks if the inventory contains a node with the given property.
-    # @param node_name [String] The name of the node to check.
-    # @param property_path [String] The dot-separated property path to check.
-    def property?(node_name, property_path)
-      !!@inventory[node_name].dot_dig(property_path)
-    end
-
-    # Returns a hash of the inventory.
-    def to_h
-      h = {}
-      @inventory.each_pair do |node_name, node_data|
-        h[node_name] = node_data.to_h
-      end
-      h[:claimed] = @claimed.to_a
-      h
-    end
-
-    # Returns a YAML string of a specific node's node data
-    def node_to_yaml(node_name)
-      get(node_name).to_h.to_yaml
-    end
-
-    # Returns a JSON string of a specific node's node data
-    def node_to_json(node_name, *args)
-      get(node_name).to_h.to_json(*args)
-    end
-
-    # Returns a YAML string of the inventory.
-    def to_yaml
-      to_h.to_yaml
-    end
-
-    # Returns a JSON string of the inventory.
-    def to_json(*args)
-      to_h.to_json(*args)
-    end
-
-    # Saves the current node inventory to a file. DOES NOT USE LOCK.
-    # If this is called outside of a lock, it will not be thread-safe.
-    def save_no_lock!(file_path = @save_file_path)
-      Dir.mkdir(file_path) unless Dir.exist?(file_path)
-      @inventory.each_pair do |node_name, node_data|
-        path = File.join(file_path, "#{node_name}.yaml")
-        next if File.file?(path)
-
-        File.open(path, 'w') do |f|
-          f.write(node_data.to_h.to_yaml)
-        end
-      end
-    end
-
-    # Saves the current node inventory to a yaml file. Thread-safe.
-    # @param file_path [String] The path to the file to save to.
-    def save(file_path = @save_file_path)
-      with_lock_retry(:write) do
-        save_no_lock!(file_path)
-      end
-    end
-
-    # Loads a node inventory from a file. DOES NOT USE LOCK.
-    # If this is called outside of a lock, it will not be thread-safe.
-    def load_no_lock!(file_path = @save_file_path)
-      require 'net/ssh/proxy/command' # If ProxyCommand is used in ssh options, this is required.
-      attempts ||= 1
-      Dir.glob('*.yaml', base: file_path) do |file_name|
-        node_name = File.basename(file_name, '.yaml')
-        node_data = YAML.load_file(File.join(file_path, file_name))
-        add(node_name, node_data) || update(node_name, node_data)
-      end
-    rescue StandardError => e
-      raise e unless (attempts += 1) <= 3
-
-      sleep(1)
-      retry
-    end
-
-    def update_no_lock!; end
-
-    # Loads a node inventory from a yaml file. Thread-safe.
-    # @param file_path [String] The path to the file to load from.
-    def load(file_path = @save_file_path)
-      with_lock_retry(:write) do
-        load_no_lock!(file_path)
-      end
-    end
-
-    def clean_local_files(file_path = @save_file_path)
-      Dir.glob('*.yaml', base: file_path) do |file_name|
-        File.delete(File.expand_path(File.join(file_path, file_name)))
-      end
-    end
-
-    private
-
-    def with_lock_retry(lock_type, max_attempts: 3)
-      raise 'No block given' unless block_given?
-
-      attempts ||= 1
-      case lock_type
-      when :read
-        @lock.with_read_lock { yield }
-      when :write
-        @lock.with_write_lock { yield }
-      else
-        raise LockNotRecognizedError, "Lock type #{lock_type} not recognized! Must be :read or :write!"
-      end
-    rescue Concurrent::ResourceLimitError => e
-      if (attempts += 1) <= max_attempts
-        sleep(1)
-        logger.debug("Failed to acquire lock with error #{e.message}. Retrying...")
-        retry
-      else
-        raise LockWaitTimeoutError, 'Lock could not be aquired'
-      end
-    end
-  end
-
-  class LocalPortAllocationError < StandardError; end
-
-  # Handles local port allocation for things like local SSH tunnels.
-  # This class is thread-safe as the port range is a Concurrent::Array,
-  # which blocks when modified. When this class is initialized, it's
-  # total assignable ports are set to all ports in the ephemeral port range.
-  # When a port is allocated, it's removed from the port range.
-  class LocalPortAllocator
-    EPHEMERAL_PORT_RANGE = (49_152..65_535).to_a.freeze
-
-    def initialize
-      @port_range = Concurrent::Array.new(EPHEMERAL_PORT_RANGE.dup).shuffle!
-    end
-
-    # Returns an open port(s) in the ephemeral port range on the local machine.
-    # If 'number_of_ports' is specified and greater than one, returns an array of ports.
-    # If 'number_of_ports' is not specified or is one, returns a single port.
-    # @param number_of_ports [Integer] the number of ports to allocate
-    # @return [Integer, Array<Integer>] the port(s) allocated
-    # @raise [ArgumentError] if number_of_ports is not an Integer or is less than one
-    # @raise [LocalPortAllocationError] if there are no ports available
-    def allocate(number_of_ports = 1)
-      unless number_of_ports.is_a?(Integer) && number_of_ports.positive?
-        raise ArgumentError, 'number_of_ports must be a positive integer'
-      end
-      raise LocalPortAllocationError, 'No ports available' if @port_range.empty?
-
-      ports = []
-      while ports.length < number_of_ports
-        port = @port_range.pop
-        next unless local_port_open?(port)
-
-        ports << port
-      end
-      if ports.length == 1
-        ports[0]
-      else
-        ports
-      end
-    end
-
-    private
-
-    # Checks if the given port is open on the local machine. This is done by
-    # opening a socket on the port and checking if it raises an exception.
-    # @param port [Integer] the port to check
-    # @return [Boolean] true if the port is open, false otherwise
-    def local_port_open?(port)
-      require 'socket'
-      socket = Socket.new(Socket::Constants::AF_INET, Socket::Constants::SOCK_STREAM, 0)
-      socket.bind(Socket.pack_sockaddr_in(port, '0.0.0.0'))
-      true
-    rescue Errno::EADDRINUSE, Errno::CONNREFUSED
-      false
-    ensure
-      socket&.close
-    end
-  end
-end