cem_acpt 0.2.6-universal-java-17

data/lib/cem_acpt/shared_objects.rb

@@ -0,0 +1,537 @@
+# frozen_string_literal: true
+
+require 'concurrent-ruby'
+require 'deep_merge'
+require 'json'
+require 'yaml'
+require_relative 'core_extensions'
+require_relative 'logging'
+
+module CemAcpt
+  using CemAcpt::CoreExtensions::ExtendedHash
+
+  class ConfigImmutableError < RuntimeError; end
+
+  # Holds module methods for writing various config objects
+  # to files.
+  module ConfigUtils
+    class << self
+      include CemAcpt::Logging
+    end
+
+    def self.object_to_file(obj, file_path)
+      case File.extname(file_path)
+      when '.json'
+        raise "Object #{obj} is not serializable to JSON" unless obj.respond_to?(:to_json)
+
+        logger.debug("Writing object to #{file_path}")
+        File.write(file_path, obj.to_json)
+      when '.yaml', '.yml'
+        raise "Object #{obj} is not serializable to YAML" unless obj.respond_to?(:to_yaml)
+
+        logger.debug("Writing object to #{file_path}")
+        File.write(file_path, obj.to_yaml)
+      else
+        raise "File extension #{File.extname(file_path)} is not supported"
+      end
+    end
+  end
+
+  # Provides a thread-safe Config object.
+  # Once the config is loaded by calling the `load` method, the object is
+  # immutable and thread-safe as long as the internal hash is only accessed
+  # by the `#get` and `#has?` methods. If `#load` is called more than once,
+  # the object will raise an error.
+  class Config
+    include CemAcpt::Logging
+
+    attr_reader :config_file
+
+    def initialize
+      @opts = {
+        thread_pool: {
+          min_threads: [2, Concurrent.processor_count - 1].max,
+          max_threads: [2, Concurrent.processor_count - 1].max,
+          max_queue: [2, Concurrent.processor_count - 1].max * 10,
+          fallback_policy: :caller_runs,
+          idletime: 1200,
+          auto_terminate: true,
+        },
+      }
+    end
+
+    # Returns the value of the dot-separated key.
+    # If the key is not found, returns nil.
+    # @param dot_key [String] Dot-separated key
+    # @return [Object] Value of the key
+    def get(dot_key)
+      @opts.dot_dig(dot_key)
+    end
+
+    # Checks if the dot-separated key exists in the config.
+    # @param dot_key [String] Dot-separated key
+    # @return [Boolean] True if the key exists, false otherwise
+    def has?(dot_key)
+      !!get(dot_key)
+    end
+
+    # Checks to see if cem_acpt is set to run in debug mode (log level is set to debug)
+    # @return [TrueClass] If cem_acpt is running in debug mode
+    # @return [FalseClass] If cem_acpt is not runnint in debug mode
+    def debug_mode?
+      @opts.dot_dig('log_level') == 'debug'
+    end
+
+    # Loads the config from specified file path. Config files must be in YAML
+    # format. If 'opts' is specified, it will be merged with the config file.
+    # Values in 'opts' will override values in the config file.
+    # @param opts [Hash] Options to be merged with the config file
+    # @param config_file [String] Path to the config file
+    def load(opts: {}, config_file: './cem_acpt_config.yaml')
+      raise ConfigImmutableError, 'Config is immutable, cannot load more than once' if frozen?
+      raise ArgumentError, 'opts must be a Hash' unless opts.is_a?(Hash)
+      raise ArgumentError, 'config_file must be a String' unless config_file.is_a?(String)
+
+      @config_file = File.expand_path(config_file)
+      @opts.deep_merge!(load_opts_from_file(File.expand_path(config_file)).deep_merge!(opts))
+      @opts.format!
+      @opts.freeze
+      freeze
+    end
+
+    # Returns the config as a Hash.
+    # @return [Hash] Config as a Hash
+    def to_h
+      ::Marshal.load(::Marshal.dump(@opts))
+    end
+
+    # Returns the config as a YAML string.
+    # @return [String] Config as a YAML string
+    def to_yaml
+      to_h.to_yaml
+    end
+
+    # Returns the config as a JSON string.
+    # @return [String] Config as a JSON string
+    def to_json(*args)
+      to_h.to_json(*args)
+    end
+
+    private
+
+    # Loads the config from the specified file path.
+    # @param config_file [String] Path to the config file
+    # @return [Hash] Config hash
+    def load_opts_from_file(config_file)
+      return {} unless File.exist?(config_file)
+
+      logger.debug("Loading config from #{config_file}")
+      YAML.load_file(config_file)
+    end
+  end
+
+  # Provides an object to hold NodeData configs.
+  # Not thread-safe. Should not be mutated by itself,
+  # but can be mutated safely by the NodeInventory object.
+  # Currently unused.
+  class NodeData
+    include CemAcpt::Logging
+
+    attr_reader :name, :data
+
+    def initialize(name, data)
+      data.format!
+      @name = name
+      @opts = data
+    end
+
+    def deep_merge(other)
+      @opts.deep_merge(other.to_h)
+    end
+
+    def dot_dig(dot_key)
+      @opts.dot_dig(dot_key)
+    end
+
+    def dot_store(dot_key, value)
+      @opts.dot_store(dot_key, value)
+    end
+
+    def to_h
+      @opts
+    end
+
+    def to_yaml
+      to_h.to_yaml
+    end
+
+    def to_json(*args)
+      to_h.to_json(*args)
+    end
+  end
+
+  class NodeInventoryFileNotFoundError < StandardError; end
+  class NodeInventoryFileLoadError < StandardError; end
+  class NodeClaimedError < StandardError; end
+  class NodeDoesNotExistError < StandardError; end
+  class PropertyNotFoundError < StandardError; end
+  class LockWaitTimeoutError < StandardError; end
+  class LockNotRecognizedError < StandardError; end
+
+  # Provides a thread-safe inventory of test nodes.
+  class NodeInventory
+    include CemAcpt::Logging
+
+    attr_accessor :save_file_path
+
+    def initialize
+      @inventory = Concurrent::Map.new
+      @lock = Concurrent::ReadWriteLock.new
+      @claimed = Concurrent::Set.new
+      @save_on_claim = false
+      @save_file_path = 'spec/fixtures/node_inventory'
+      @loaded_node_inv = nil
+    end
+
+    # When called, enables saving the inventory to a file on claim.
+    def save_on_claim
+      @save_on_claim = true
+    end
+
+    # Checks if save_on_claim is enabled.
+    def save_on_claim?
+      @save_on_claim
+    end
+
+    # Adds a new node to the inventory.
+    # @param node_name [String] The name of the node to add.
+    # @param node_data [Hash] The node data to add.
+    def add(node_name, node_data)
+      @inventory.put_if_absent(node_name, node_data)
+    end
+
+    def update(node_name, node_data)
+      @inventory.replace_if_exists(node_name, node_data)
+    end
+
+    # Returns the node data for a given node.
+    # @param node_name [String] The name of the node to get data for.
+    def get(node_name)
+      @inventory[node_name]
+    end
+
+    # Returns the node_name and node_data for a given node
+    # that has a matching value for the given property.
+    # @param property_path [String] The dot-separated property path to check.
+    # @param value [Object] The value to check for.
+    # @return [Array] Two-item frozen array of node name and node data.
+    # @raise [NodeDoesNotExistError] If no nodes are found with the property equal to the value.
+    def get_by_property(property_path, value)
+      found = []
+      @inventory.each_pair do |node_name, node_data|
+        next unless node_data.dot_dig(property_path) == value
+
+        found = [node_name, node_data].freeze
+      end
+      raise NodeDoesNotExistError, "No node found with property #{property_path} == #{value}" if found.empty?
+
+      found
+    end
+
+    # Returns all property values of all nodes for the given property path
+    # @param property_path [String] The dot-separated property path to check.
+    # @return [Array] A frozen array of found property values.
+    # @raise [PropertyNotFoundError] If no values are found on any nodes for the property path.
+    def get_all_properties(property_path)
+      found = []
+      @inventory.each_pair do |_, node_data|
+        prop = node_data.dot_dig(property_path)
+        next unless prop
+
+        found << prop
+      end
+      raise PropertyNotFoundError, "No property with path #{property_path} found in nodes" if found.empty?
+
+      found.freeze
+    end
+
+    # Sets a specific property on a node in the inventory.
+    # @param node_name [String] The name of the node to set the property on.
+    # @param property_path [String] The dot-separated property path to set.
+    # @param value [Object] The value to set.
+    def set(node_name, property_path, value)
+      @inventory[node_name].dot_store(property_path, value)
+    end
+
+    # Merges new node data in with the existing node data for the given node.
+    # @param node_name [String] The name of the node to merge data for.
+    # @param new_node_data [Hash] The new node data to merge.
+    def merge!(node_name, new_node_data)
+      @inventory.merge_pair(node_name, new_node_data) do |old_data|
+        old_data.deep_merge(new_node_data)
+      end
+    end
+
+    # Checks if a node is claimed.
+    # @param node_name [String] The name of the node to check.
+    # @return [Boolean] True if the node is claimed, false otherwise.
+    def claimed?(node_name)
+      raise NodeDoesNotExistError, "Node #{node_name} does not exist" unless @inventory.keys.include?(node_name)
+
+      @claimed.include?(node_name)
+    end
+
+    # Claims a node, which returns the node data and adds that node to the
+    # claimed set. Raises an error if the node is already claimed or does
+    # not exist. This is used during acceptance testing to ensure that
+    # nodes are not reused.
+    # @param node_name [String] The name of the node to claim.
+    # @return [String] The name of the node that was claimed.
+    # @raise [NodeDoesNotExistError] If the node does not exist in the inventory.
+    # @raise [NodeClaimedError] If the node is already claimed.
+    def claim(node_name)
+      with_lock_retry(:write) do
+        unless @inventory.keys.include?(node_name)
+          raise NodeDoesNotExistError, "Node #{node_name} does not exist in inventory"
+        end
+
+        if @claimed.include?(node_name)
+          raise NodeClaimedError, "Node #{node_name} is already tainted and cannot be claimed"
+        end
+
+        @claimed.add(node_name)
+        save_no_lock!(@save_file_path) if @save_on_claim
+        node_name
+      end
+    end
+
+    # Claims a node based on a property of the node data and a value that property should equal.
+    # If a node is found but is already claimed, the method will continue to search for an
+    # unclaimed node. Raises an error if no valid node is found. This is used during acceptance
+    # testing to ensure that nodes are not reused.
+    # @param property_path [String] The dot-separated property path to check.
+    # @param value [Object] The value to check for.
+    # @return [String] The name of the node that was claimed.
+    # @raise [NodeDoesNotExistError] If no valid node is found.
+    def claim_by_property(property_path, value)
+      attempts ||= 1
+      claim_name = nil
+      @inventory.each_pair do |node_name, node_data|
+        next if @claimed.include?(node_name)
+
+        claim_name = node_name if node_data.dot_dig(property_path) == value
+      end
+      raise NodeDoesNotExistError, "No node found with property #{property_path} == #{value}" if claim_name.nil?
+
+      claim(claim_name)
+    rescue NodeDoesNotExistError => e
+      # We sleep than retry three times to help mitigate race conditions
+      if (attempts += 1) <= 3
+        sleep(1)
+        retry
+      else
+        raise e
+      end
+    end
+
+    # Deletes a node from the inventory.
+    # @param node_name [String] The name of the node to delete.
+    def delete(node_name)
+      @inventory.delete(node_name)
+    end
+
+    # Returns all node names in the current inventory.
+    def nodes
+      @inventory.keys
+    end
+
+    # Clears the inventory and removes claimed nodes. DOES NOT USE LOCK.
+    # If this is called outside of a lock, it will not be thread-safe.
+    def clear_no_lock!
+      nodes.each { |node| delete(node) }
+      @claimed.clear
+    end
+
+    # Clears the inventory and removes claimed nodes. Thread-safe.
+    def clear!
+      with_lock_retry(:write) do
+        clear_no_lock!
+      end
+    end
+
+    # Checks if the inventory contains a node with the given property.
+    # @param node_name [String] The name of the node to check.
+    # @param property_path [String] The dot-separated property path to check.
+    def property?(node_name, property_path)
+      !!@inventory[node_name].dot_dig(property_path)
+    end
+
+    # Returns a hash of the inventory.
+    def to_h
+      h = {}
+      @inventory.each_pair do |node_name, node_data|
+        h[node_name] = node_data.to_h
+      end
+      h[:claimed] = @claimed.to_a
+      h
+    end
+
+    # Returns a YAML string of a specific node's node data
+    def node_to_yaml(node_name)
+      get(node_name).to_h.to_yaml
+    end
+
+    # Returns a JSON string of a specific node's node data
+    def node_to_json(node_name, *args)
+      get(node_name).to_h.to_json(*args)
+    end
+
+    # Returns a YAML string of the inventory.
+    def to_yaml
+      to_h.to_yaml
+    end
+
+    # Returns a JSON string of the inventory.
+    def to_json(*args)
+      to_h.to_json(*args)
+    end
+
+    # Saves the current node inventory to a file. DOES NOT USE LOCK.
+    # If this is called outside of a lock, it will not be thread-safe.
+    def save_no_lock!(file_path = @save_file_path)
+      Dir.mkdir(file_path) unless Dir.exist?(file_path)
+      @inventory.each_pair do |node_name, node_data|
+        path = File.join(file_path, "#{node_name}.yaml")
+        next if File.file?(path)
+
+        File.open(path, 'w') do |f|
+          f.write(node_data.to_h.to_yaml)
+        end
+      end
+    end
+
+    # Saves the current node inventory to a yaml file. Thread-safe.
+    # @param file_path [String] The path to the file to save to.
+    def save(file_path = @save_file_path)
+      with_lock_retry(:write) do
+        save_no_lock!(file_path)
+      end
+    end
+
+    # Loads a node inventory from a file. DOES NOT USE LOCK.
+    # If this is called outside of a lock, it will not be thread-safe.
+    def load_no_lock!(file_path = @save_file_path)
+      require 'net/ssh/proxy/command' # If ProxyCommand is used in ssh options, this is required.
+      attempts ||= 1
+      Dir.glob('*.yaml', base: file_path) do |file_name|
+        node_name = File.basename(file_name, '.yaml')
+        node_data = YAML.load_file(File.join(file_path, file_name))
+        add(node_name, node_data) || update(node_name, node_data)
+      end
+    rescue StandardError => e
+      raise e unless (attempts += 1) <= 3
+
+      sleep(1)
+      retry
+    end
+
+    def update_no_lock!; end
+
+    # Loads a node inventory from a yaml file. Thread-safe.
+    # @param file_path [String] The path to the file to load from.
+    def load(file_path = @save_file_path)
+      with_lock_retry(:write) do
+        load_no_lock!(file_path)
+      end
+    end
+
+    def clean_local_files(file_path = @save_file_path)
+      Dir.glob('*.yaml', base: file_path) do |file_name|
+        File.delete(File.expand_path(File.join(file_path, file_name)))
+      end
+    end
+
+    private
+
+    def with_lock_retry(lock_type, max_attempts: 3)
+      raise 'No block given' unless block_given?
+
+      attempts ||= 1
+      case lock_type
+      when :read
+        @lock.with_read_lock { yield }
+      when :write
+        @lock.with_write_lock { yield }
+      else
+        raise LockNotRecognizedError, "Lock type #{lock_type} not recognized! Must be :read or :write!"
+      end
+    rescue Concurrent::ResourceLimitError => e
+      if (attempts += 1) <= max_attempts
+        sleep(1)
+        logger.debug("Failed to acquire lock with error #{e.message}. Retrying...")
+        retry
+      else
+        raise LockWaitTimeoutError, 'Lock could not be aquired'
+      end
+    end
+  end
+
+  class LocalPortAllocationError < StandardError; end
+
+  # Handles local port allocation for things like local SSH tunnels.
+  # This class is thread-safe as the port range is a Concurrent::Array,
+  # which blocks when modified. When this class is initialized, it's
+  # total assignable ports are set to all ports in the ephemeral port range.
+  # When a port is allocated, it's removed from the port range.
+  class LocalPortAllocator
+    EPHEMERAL_PORT_RANGE = (49_152..65_535).to_a.freeze
+
+    def initialize
+      @port_range = Concurrent::Array.new(EPHEMERAL_PORT_RANGE.dup).shuffle!
+    end
+
+    # Returns an open port(s) in the ephemeral port range on the local machine.
+    # If 'number_of_ports' is specified and greater than one, returns an array of ports.
+    # If 'number_of_ports' is not specified or is one, returns a single port.
+    # @param number_of_ports [Integer] the number of ports to allocate
+    # @return [Integer, Array<Integer>] the port(s) allocated
+    # @raise [ArgumentError] if number_of_ports is not an Integer or is less than one
+    # @raise [LocalPortAllocationError] if there are no ports available
+    def allocate(number_of_ports = 1)
+      unless number_of_ports.is_a?(Integer) && number_of_ports.positive?
+        raise ArgumentError, 'number_of_ports must be a positive integer'
+      end
+      raise LocalPortAllocationError, 'No ports available' if @port_range.empty?
+
+      ports = []
+      while ports.length < number_of_ports
+        port = @port_range.pop
+        next unless local_port_open?(port)
+
+        ports << port
+      end
+      if ports.length == 1
+        ports[0]
+      else
+        ports
+      end
+    end
+
+    private
+
+    # Checks if the given port is open on the local machine. This is done by
+    # opening a socket on the port and checking if it raises an exception.
+    # @param port [Integer] the port to check
+    # @return [Boolean] true if the port is open, false otherwise
+    def local_port_open?(port)
+      require 'socket'
+      socket = Socket.new(Socket::Constants::AF_INET, Socket::Constants::SOCK_STREAM, 0)
+      socket.bind(Socket.pack_sockaddr_in(port, '0.0.0.0'))
+      true
+    rescue Errno::EADDRINUSE, Errno::CONNREFUSED
+      false
+    ensure
+      socket&.close
+    end
+  end
+end