hybrid_platforms_conductor 32.3.6

data/lib/hybrid_platforms_conductor/hpc_plugins/provisioner/proxmox/proxmox_waiter.rb

@@ -0,0 +1,707 @@
+# Require tmpdir before futex, as this Rubygem has a bug missing its require.
+require 'tmpdir'
+require 'futex'
+require 'json'
+require 'proxmox'
+require 'time'
+
+# Serve Proxmox reservation requests, like a waiter in a restaurant ;-)
+# Multi-process safe.
+class ProxmoxWaiter
+
+  # Integer: Timeout in seconds to get the futex
+  # Take into account that some processes can be lengthy while the futex is taken:
+  # * POST/DELETE operations in the Proxmox API requires tasks to be performed which can take a few seconds, depending on the load.
+  # * Proxmox API sometimes fails to respond when containers are being locked temporarily (we have a 30 secs timeout for each one).
+  FUTEX_TIMEOUT = 600
+
+  # Integer: Maximum timeout in seconds before retrying getting the Futex when we are not first in the queue (a rand will be applied to it)
+  RETRY_QUEUE_WAIT = 30
+
+  # Constructor
+  #
+  # Parameters::
+  # * *config_file* (String): Path to a JSON file containing a configuration for ProxmoxWaiter.
+  #   Here is the file structure:
+  #   * *proxmox_api_url* (String): Proxmox API URL.
+  #   * *futex_file* (String): Path to the file serving as a futex.
+  #   * *logs_dir* (String): Path to the directory containing logs [default: '.']
+  #   * *pve_nodes* (Array<String>): List of PVE nodes allowed to spawn new containers [default: all]
+  #   * *vm_ips_list* (Array<String>): The list of IPs that are available for the Proxomx containers.
+  #   * *vm_ids_range* ([Integer, Integer]): Minimum and maximum reservable VM ID
+  #   * *coeff_ram_consumption* (Integer): Importance coefficient to assign to the RAM consumption when selecting available PVE nodes
+  #   * *coeff_disk_consumption* (Integer): Importance coefficient to assign to the disk consumption when selecting available PVE nodes
+  #   * *expiration_period_secs* (Integer): Number of seconds defining the expiration period
+  #   * *expire_stopped_vm_timeout_secs* (Integer): Number of seconds before defining stopped VMs as expired
+  #   * *limits* (Hash): Limits to be taken into account while reserving resources. Each property is optional and no property means no limit.
+  #     * *nbr_vms_max* (Integer): Max number of VMs we can reserve.
+  #     * *cpu_loads_thresholds* ([Float, Float, Float]): CPU load thresholds from which a PVE node should not be used (as soon as 1 of the value is greater than 1 of those thresholds, discard the node).
+  #     * *ram_percent_used_max* (Float): Max percentage (between 0 and 1) of RAM that can be reserved on a PVE node.
+  #     * *disk_percent_used_max* (Float): Max percentage (between 0 and 1) of disk that can be reserved on a PVE node.
+  # * *proxmox_user* (String): Proxmox user to be used to connect to the API.
+  # * *proxmox_password* (String): Proxmox password to be used to connect to the API.
+  # * *proxmox_realm* (String): Proxmox realm to be used to connect to the API.
+  def initialize(config_file, proxmox_user, proxmox_password, proxmox_realm)
+    @config = JSON.parse(File.read(config_file))
+    @proxmox_user = proxmox_user
+    @proxmox_password = proxmox_password
+    @proxmox_realm = proxmox_realm
+    # Keep a memory of non-debug stopped containers, so that we can guess if they are expired or not after some time.
+    # Time when we noticed a given container is stopped, per creation date, per VM ID, per PVE node
+    # We add the creation date as a VM ID can be reused (with a different creation date) and we want to make sure we don't think a newly created VM is here for longer that it should.
+    # Hash< String,   Hash< Integer, Hash< String,        Time                 > > >
+    # Hash< pve_node, Hash< vm_id,   Hash< creation_date, time_seen_as_stopped > > >
+    @non_debug_stopped_containers = {}
+    @log_file = "#{@config['logs_dir'] || '.'}/proxmox_waiter_#{Time.now.utc.strftime('%Y%m%d%H%M%S')}_pid_#{Process.pid}_#{File.basename(config_file, '.json')}.log"
+    FileUtils.mkdir_p File.dirname(@log_file)
+  end
+
+  # Reserve resources for a new container.
+  # Check resources availability.
+  #
+  # Parameters::
+  # * *vm_info* (Hash<String,Object>): The VM info to be created, using the same properties as LXC container creation through Proxmox API.
+  # Result::
+  # * Hash<Symbol, Object> or Symbol: Reserved resource info, or Symbol in case of error.
+  #   The following properties are set as resource info:
+  #   * *pve_node* (String): Node on which the container has been created.
+  #   * *vm_id* (Integer): The VM ID
+  #   * *vm_ip* (String): The VM IP
+  #   Possible error codes returned are:
+  #   * *not_enough_resources*: There is no available free resources to be reserved
+  #   * *no_available_ip*: There is no available IP to be reserved
+  #   * *no_available_vm_id*: There is no available VM ID to be reserved
+  #   * *exceeded_number_of_vms*: There is already too many VMs running
+  def create(vm_info)
+    log "Ask to create #{vm_info}"
+    # Extract the required resources from the desired VM info
+    nbr_cpus = vm_info['cpulimit']
+    ram_mb = vm_info['memory']
+    disk_gb = Integer(vm_info['rootfs'].split(':').last)
+    reserved_resource = nil
+    start do
+      pve_node_scores = pve_scores_for(nbr_cpus, ram_mb, disk_gb)
+      # Check if we are not exceeding hard-limits:
+      # * the number of vms to be created
+      # * the free IPs
+      # * the free VM IDs
+      # In such case, even when free resources on PVE nodes are enough to host the new container, we still need to clean-up before.
+      nbr_vms = nbr_vms_handled_by_us
+      if nbr_vms >= @config['limits']['nbr_vms_max'] || free_ips.empty? || free_vm_ids.empty?
+        log 'Hitting at least 1 hard-limit. Check if we can destroy expired containers.'
+        log "[ Hard limit reached ] - Already #{nbr_vms} are created (max is #{@config['limits']['nbr_vms_max']})." if nbr_vms >= @config['limits']['nbr_vms_max']
+        log '[ Hard limit reached ] - No more available IPs.' if free_ips.empty?
+        log '[ Hard limit reached ] - No more available VM IDs.' if free_vm_ids.empty?
+        clean_up_done = false
+        # Check if we can remove some expired ones
+        @config['pve_nodes'].each do |pve_node|
+          if api_get("nodes/#{pve_node}/lxc").any? { |lxc_info| is_vm_expired?(pve_node, Integer(lxc_info['vmid'])) }
+            destroy_expired_vms_on(pve_node)
+            clean_up_done = true
+          end
+        end
+        if clean_up_done
+          nbr_vms = nbr_vms_handled_by_us
+          if nbr_vms >= @config['limits']['nbr_vms_max']
+            log "[ Hard limit reached ] - Still too many running VMs after clean-up: #{nbr_vms}."
+            reserved_resource = :exceeded_number_of_vms
+          elsif free_ips.empty?
+            log '[ Hard limit reached ] - Still no available IP'
+            reserved_resource = :no_available_ip
+          elsif free_vm_ids.empty?
+            log '[ Hard limit reached ] - Still no available VM ID'
+            reserved_resource = :no_available_vm_id
+          end
+        else
+          log 'Could not find any expired VM to destroy.'
+          # There was nothing to clean. So wait for other processes to destroy their containers.
+          reserved_resource =
+            if nbr_vms >= @config['limits']['nbr_vms_max']
+              :exceeded_number_of_vms
+            elsif free_ips.empty?
+              :no_available_ip
+            else
+              :no_available_vm_id
+            end
+        end
+      end
+      if reserved_resource.nil?
+        # Select the best node, first keeping expired VMs if possible.
+        # This is the index of the scores to be checked: if we can choose without recycling VMs, do it by considering score index 0.
+        score_idx =
+          if pve_node_scores.all? { |_pve_node, pve_node_scores| pve_node_scores[0].nil? }
+            # No node was available without removing expired VMs.
+            # Therefore we consider only scores without expired VMs.
+            log 'No PVE node has enough free resources without removing eventual expired VMs'
+            1
+          else
+            0
+          end
+        selected_pve_node, selected_pve_node_score = pve_node_scores.inject([nil, nil]) do |(best_pve_node, best_score), (pve_node, pve_node_scores)|
+          if pve_node_scores[score_idx].nil? ||
+            (!best_score.nil? && pve_node_scores[score_idx] >= best_score)
+            [best_pve_node, best_score]
+          else
+            [pve_node, pve_node_scores[score_idx]]
+          end
+        end
+        if selected_pve_node.nil?
+          # No PVE node can host our request.
+          log 'Could not find any PVE node with enough free resources'
+          reserved_resource = :not_enough_resources
+        else
+          log "[ #{selected_pve_node} ] - PVE node selected with score #{selected_pve_node_score}"
+          # We know on which PVE node we can instantiate our new container.
+          # We have to purge expired VMs on this PVE node before reserving a new creation.
+          destroy_expired_vms_on(selected_pve_node) if score_idx == 1
+          # Now select the correct VM ID and VM IP.
+          vm_id_or_error, ip = reserve_on(selected_pve_node, nbr_cpus, ram_mb, disk_gb)
+          if ip.nil?
+            # We have an error
+            reserved_resource = vm_id_or_error
+          else
+            # Create the container for real
+            completed_vm_info = vm_info.dup
+            completed_vm_info['vmid'] = vm_id_or_error
+            completed_vm_info['net0'] = "#{completed_vm_info['net0']},ip=#{ip}/32"
+            completed_vm_info['description'] = "#{completed_vm_info['description']}creation_date: #{Time.now.utc.strftime('%FT%T')}\n"
+            log "[ #{selected_pve_node}/#{vm_id_or_error} ] - Create LXC container"
+            wait_for_proxmox_task(selected_pve_node, @proxmox.post("nodes/#{selected_pve_node}/lxc", completed_vm_info))
+            reserved_resource = {
+              pve_node: selected_pve_node,
+              vm_id: vm_id_or_error,
+              vm_ip: ip
+            }
+          end
+        end
+      end
+    end
+    reserved_resource
+  end
+
+  # Destroy a VM.
+  #
+  # Parameters::
+  # * *vm_info* (Hash<String,Object>): The VM info to be destroyed:
+  #   * *vm_id* (Integer): The VM ID
+  #   * *node* (String): The node for which this VM has been created
+  #   * *environment* (String): The environment for which this VM has been created
+  # Result::
+  # * Hash<Symbol, Object> or Symbol: Released resource info, or Symbol in case of error.
+  #   The following properties are set as resource info:
+  #   * *pve_node* (String): Node on which the container has been released (if found).
+  #   Possible error codes returned are:
+  #   None
+  def destroy(vm_info)
+    log "Ask to destroy #{vm_info}"
+    found_pve_node = nil
+    start do
+      vm_id_str = vm_info['vm_id'].to_s
+      # Destroy the VM ID
+      # Find which PVE node hosts this VM
+      unless @config['pve_nodes'].any? do |pve_node|
+          api_get("nodes/#{pve_node}/lxc").any? do |lxc_info|
+            if lxc_info['vmid'] == vm_id_str
+              # Make sure this VM is still used for the node and environment we want.
+              # It could have been deleted manually and re-affected to another node/environment automatically, and in this case we should not remove it.
+              metadata = vm_metadata(pve_node, vm_info['vm_id'])
+              if metadata[:node] == vm_info['node'] && metadata[:environment] == vm_info['environment']
+                destroy_vm_on(pve_node, vm_info['vm_id'])
+                found_pve_node = pve_node
+                true
+              else
+                log "[ #{pve_node}/#{vm_info['vm_id']} ] - This container is not hosting the node/environment to be destroyed: #{metadata[:node]}/#{metadata[:environment]} != #{vm_info['node']}/#{vm_info['environment']}"
+                false
+              end
+            else
+              false
+            end
+          end
+        end
+        log "Could not find any PVE node hosting VM #{vm_info['vm_id']}"
+      end
+    end
+    reserved_resource = {}
+    reserved_resource[:pve_node] = found_pve_node unless found_pve_node.nil?
+    reserved_resource
+  end
+
+  private
+
+  # Log a message to stdout and in the log file
+  #
+  # Parameters::
+  # * *msg* (String): Message to log
+  def log(msg)
+    puts msg
+    File.open(@log_file, 'a') { |f| f.puts "[ #{Time.now.utc.strftime('%F %T.%L')} ] - [ PID #{Process.pid} ] - #{msg}" }
+  end
+
+  # Get the access queue from a file.
+  # Handle the case of missing file.
+  #
+  # Parameters::
+  # * *queue_file* (String): The file holding the queue
+  # Result::
+  # * Array<Integer>: PIDs queue
+  def read_access_queue(queue_file)
+    (File.exist?(queue_file) ? File.read(queue_file).split("\n").map { |line| Integer(line) } : [])
+  end
+
+  # Write the access queue to a file.
+  #
+  # Parameters::
+  # * *queue_file* (String): The file holding the queue
+  # * *access_queue* (Array<Integer>): PIDs queue
+  def write_access_queue(queue_file, access_queue)
+    File.write(queue_file, access_queue.join("\n"))
+  end
+
+  # Get an exclusive (based on PID) access using a futex-protected queue
+  #
+  # Parameters::
+  # * *futex_file* (String): Name of the file to be used as a futex
+  # * Prox: Code called with access authorized
+  def with_futex_queue_access_on(futex_file)
+    pid = Process.pid
+    queue_futex_file = "#{futex_file}.queue"
+    # Register ourselves in the queue (at the end)
+    Futex.new(queue_futex_file, timeout: FUTEX_TIMEOUT).open do
+      access_queue = read_access_queue(queue_futex_file)
+      log "[ Futex queue ] - Register our PID in the queue: #{access_queue.join(', ')}"
+      write_access_queue(queue_futex_file, access_queue + [pid])
+    end
+    # Loop until we are first ones in the queue
+    retry_futex_queue = true
+    while retry_futex_queue
+      Futex.new(futex_file, timeout: FUTEX_TIMEOUT).open do
+        # Check if we are the first one in the queue
+        Futex.new(queue_futex_file, timeout: FUTEX_TIMEOUT).open do
+          access_queue = read_access_queue(queue_futex_file)
+          idx = access_queue.index(pid)
+          log "[ Futex queue ] - We are ##{idx} in the queue: #{access_queue.join(', ')}"
+          if idx.nil?
+            # We disappeared from the queue!
+            log '[ Futex queue ] - !!! Somebody removed use from the queue. Add our PID back.'
+            write_access_queue(queue_futex_file, access_queue + [pid])
+          elsif idx == 0
+            # Access granted
+            log '[ Futex queue ] - Exclusive access granted'
+            write_access_queue(queue_futex_file, access_queue[1..-1])
+            retry_futex_queue = false
+          else
+            # Just check that the first PID still exists, otherwise remove it from the queue.
+            # This way we avoid starvation in case of killed processes.
+            first_pid = access_queue.first
+            first_pid_exist =
+              begin
+                Process.getpgid(first_pid)
+                true
+              rescue Errno::ESRCH
+                false
+              end
+            unless first_pid_exist
+              log "[ Futex queue ] - !!! First PID #{first_pid} does not exist - remove it from the queue"
+              write_access_queue(queue_futex_file, access_queue[1..-1])
+            end
+          end
+        end
+        yield unless retry_futex_queue
+      end
+      sleep(rand(RETRY_QUEUE_WAIT) + 1) if retry_futex_queue
+    end
+  end
+
+  # Grab the lock to start a new atomic session.
+  # Make sure the lock is released at the end of the session.
+  #
+  # Parameters::
+  # * Proc: Client code with the session started.
+  #   The following instance variables are set:
+  #   * *@expiration_date* (Time): The expiration date to be considered when selecting expired VMs
+  #   * *@proxmox* (Proxmox): The Proxmox instance
+  def start
+    with_futex_queue_access_on(@config['futex_file']) do
+      # Connect to Proxmox's API
+      @proxmox = Proxmox::Proxmox.new(
+        "#{@config['proxmox_api_url']}/api2/json/",
+        # Proxmox uses the hostname as the node name so make the default API node derived from the URL.
+        # cf https://pve.proxmox.com/wiki/Renaming_a_PVE_node
+        URI.parse(@config['proxmox_api_url']).host.downcase.split('.').first,
+        @proxmox_user,
+        @proxmox_password,
+        @proxmox_realm,
+        { verify_ssl: false }
+      )
+      # Cache of get queries to the API
+      @gets_cache = {}
+      # Check connectivity before going further
+      begin
+        nodes_info = api_get('nodes')
+        # Get the list of PVE nodes by default
+        @config['pve_nodes'] = nodes_info.map { |node_info| node_info['node'] } unless @config['pve_nodes']
+      rescue
+        raise "Unable to connect to Proxmox API #{@config['proxmox_api_url']} with user #{@proxmox_user}: #{$!}"
+      end
+      @expiration_date = Time.now.utc - @config['expiration_period_secs']
+      log "Consider expiration date #{@expiration_date.strftime('%F %T')}"
+      begin
+        yield
+      ensure
+        @expiration_date = nil
+        @proxmox = nil
+      end
+    end
+  end
+
+  # Compute scores if we were to allocate resources for each possible PVE node.
+  # Those scores can help in choosing the best PVE node to host those resources.
+  # The best score is the smallest one.
+  # The score is computed by simulating resources' consumptions on the node if our container was to be installed in this node.
+  # The score uses coefficients as to better weigh some criterias more than others (all configured in the config file).
+  # 2 scores are gathered: 1 with the current PVE node's VMs, and 1 with the node having expired VMs removed.
+  # If a score is nil, it means the node can't be used (for example when a hard limit has been hit).
+  # Prerequisites:
+  # * This method should be called in a #start block
+  #
+  # Parameters::
+  # * *nbr_cpus* (Integer): Wanted CPUs
+  # * *ram_mb* (Integer): Wanted MB of RAM
+  # * *disk_gb* (Integer): Wanted GB of disk
+  # Result::
+  # * Hash<String, [Float or nil, Float or nil]>: The set of 2 scores, per PVE node name
+  def pve_scores_for(nbr_cpus, ram_mb, disk_gb)
+    Hash[@config['pve_nodes'].map do |pve_node|
+      # Get some resource usages stats from the node directly
+      status_info = api_get("nodes/#{pve_node}/status")
+      load_average = status_info['loadavg'].map { |load_str| Float(load_str) }
+      log "[ #{pve_node} ] - Load average: #{load_average.join(', ')}"
+      [
+        pve_node,
+        # If CPU load is too high, don't select the node anyway.
+        if load_average.zip(@config['limits']['cpu_loads_thresholds']).all? { |load_current, load_limit| load_current <= load_limit }
+          storage_info = api_get("nodes/#{pve_node}/storage").find { |search_storage_info| search_storage_info['storage'] == 'local-lvm' }
+          disk_gb_total = storage_info['total'] / (1024 * 1024 * 1024)
+          ram_mb_total = status_info['memory']['total'] / (1024 * 1024)
+          # Used resources is the sum of the allocated resource for each VM in this PVE node.
+          # It is not forcefully the currently used resource.
+          # This way we are sure to keep the allocated resources intact for containers not handled by this script.
+          disk_gb_used = 0
+          ram_mb_used = 0
+          # Store the resources used by containers we can recycle in separate variables.
+          expired_disk_gb_used = 0
+          expired_ram_mb_used = 0
+          found_vm_ids = api_get("nodes/#{pve_node}/lxc").map do |lxc_info|
+            vm_id = Integer(lxc_info['vmid'])
+            # Some times the Proxmox API returns maxdisk as a String (but not always) even if it is documented as Integer here: https://pve.proxmox.com/pve-docs/api-viewer/#/nodes/{node}/lxc.
+            # TODO: Remove the Integer conversion when Proxmox API will be fixed.
+            lxc_disk_gb_used = Integer(lxc_info['maxdisk']) / (1024 * 1024 * 1024)
+            lxc_ram_mb_used = lxc_info['maxmem'] / (1024 * 1024)
+            if is_vm_expired?(pve_node, vm_id)
+              expired_disk_gb_used += lxc_disk_gb_used
+              expired_ram_mb_used += lxc_ram_mb_used
+            else
+              disk_gb_used += lxc_disk_gb_used
+              ram_mb_used += lxc_ram_mb_used
+            end
+            vm_id.to_s
+          end
+          log "[ #{pve_node} ] - RAM MB usage: #{ram_mb_used + expired_ram_mb_used} / #{ram_mb_total} (#{expired_ram_mb_used} MB from expired containers)"
+          log "[ #{pve_node} ] - Disk GB usage: #{disk_gb_used + expired_disk_gb_used} / #{disk_gb_total} (#{expired_disk_gb_used} GB from expired containers)"
+          # Evaluate the expected percentages of resources' usage if we were to add our new container to this PVE node.
+          expected_ram_percent_used = (ram_mb_used + expired_ram_mb_used + ram_mb).to_f / ram_mb_total
+          expected_disk_percent_used = (disk_gb_used + expired_disk_gb_used + disk_gb).to_f / disk_gb_total
+          expected_ram_percent_used_without_expired = (ram_mb_used + ram_mb).to_f / ram_mb_total
+          expected_disk_percent_used_without_expired = (disk_gb_used + disk_gb).to_f / disk_gb_total
+          # If we break the limits, don't select this node.
+          # Otherwise, store the scores, taking into account coefficients to then choose among possible PVE nodes.
+          [
+            if expected_ram_percent_used <= @config['limits']['ram_percent_used_max'] &&
+              expected_disk_percent_used <= @config['limits']['disk_percent_used_max']
+              expected_ram_percent_used * @config['coeff_ram_consumption'] + expected_disk_percent_used * @config['coeff_disk_consumption']
+            else
+              nil
+            end,
+            if expected_ram_percent_used_without_expired <= @config['limits']['ram_percent_used_max'] &&
+              expected_disk_percent_used_without_expired <= @config['limits']['disk_percent_used_max']
+              expected_ram_percent_used_without_expired * @config['coeff_ram_consumption'] + expected_disk_percent_used_without_expired * @config['coeff_disk_consumption']
+            else
+              nil
+            end
+          ]
+        else
+          # CPU load is too high. Don't select this node.
+          log "[ #{pve_node} ] - Load average is too high for this PVE node to be selected (thresholds: : #{@config['limits']['cpu_loads_thresholds'].join(', ')})"
+          [nil, nil]
+        end
+      ]
+    end]
+  end
+
+  # Is a given VM expired?
+  #
+  # Parameters::
+  # * *pve_node* (String): The PVE node hosting this VM
+  # * *vm_id* (Integer): The VM ID
+  # Result::
+  # * Boolean: Is the given VM expired?
+  def is_vm_expired?(pve_node, vm_id)
+    if vm_id.between?(*@config['vm_ids_range'])
+      # Get its reservation date from the notes
+      metadata = vm_metadata(pve_node, vm_id)
+      if metadata[:creation_date].nil? || Time.parse("#{metadata[:creation_date]} UTC") < @expiration_date
+        log "[ #{pve_node}/#{vm_id} ] - [ Expired ] - Creation date is #{metadata[:creation_date]}"
+        true
+      else
+        state = vm_state(pve_node, vm_id)
+        if state == 'running' || metadata[:debug] == 'true'
+          # Just in case it was previously remembered as a non-debug stopped container, clear it.
+          @non_debug_stopped_containers[pve_node].delete(vm_id) if @non_debug_stopped_containers.key?(pve_node)
+          log "[ #{pve_node}/#{vm_id} ] - State is #{state} and debug is #{metadata[:debug]}"
+          false
+        else
+          # Check if it is not a left-over from a crash.
+          # If it stays not running for long and is not meant for debug purposes, then it is also considered expired.
+          # For this, remember previously seen containers that were stopped
+          first_time_seen_as_stopped = @non_debug_stopped_containers.dig pve_node, vm_id, metadata[:creation_date]
+          if first_time_seen_as_stopped.nil?
+            # It is the first time we see it stopped.
+            # Remember it and consider it as non-expired.
+            @non_debug_stopped_containers[pve_node] = {} unless @non_debug_stopped_containers.key?(pve_node)
+            @non_debug_stopped_containers[pve_node][vm_id] = {} unless @non_debug_stopped_containers[pve_node].key?(vm_id)
+            @non_debug_stopped_containers[pve_node][vm_id][metadata[:creation_date]] = Time.now
+            log "[ #{pve_node}/#{vm_id} ] - Discovered non-debug container (created on #{metadata[:creation_date]}) as stopped"
+            false
+          elsif Time.now - first_time_seen_as_stopped >= @config['expire_stopped_vm_timeout_secs']
+            # If it is stopped from more than the timeout, then consider it expired
+            log "[ #{pve_node}/#{vm_id} ] - [ Expired ] - Non-debug container (created on #{metadata[:creation_date]}) is stopped since #{first_time_seen_as_stopped.strftime('%F %T')} (more than #{@config['expire_stopped_vm_timeout_secs']} seconds ago)"
+            true
+          else
+            log "[ #{pve_node}/#{vm_id} ] - Non-debug container (created on #{metadata[:creation_date]}) is stopped since #{first_time_seen_as_stopped.strftime('%F %T')} (less than #{@config['expire_stopped_vm_timeout_secs']} seconds ago)"
+            false
+          end
+        end
+      end
+    else
+      log "[ #{pve_node}/#{vm_id} ] - Container is not part of our VM ID range."
+      false
+    end
+  end
+
+  # Get the metadata we associate to VMs.
+  # It can be empty if no metadata found.
+  #
+  # Parameters::
+  # * *pve_node* (String): The PVE node hosting this VM
+  # * *vm_id* (Integer): The VM ID
+  # Result::
+  # * Hash<Symbol, String>: The metadata
+  def vm_metadata(pve_node, vm_id)
+    lxc_config = api_get("nodes/#{pve_node}/lxc/#{vm_id}/config")
+    vm_description_lines = (lxc_config['description'] || '').split("\n")
+    hpc_marker_idx = vm_description_lines.index('===== HPC info =====')
+    if hpc_marker_idx.nil?
+      {}
+    else
+      Hash[vm_description_lines[hpc_marker_idx + 1..-1].map do |line|
+        property, value = line.split(': ')
+        [property.to_sym, value]
+      end]
+    end
+  end
+
+  # Count the number of VMs handled by us currently existing.
+  #
+  # Result::
+  # * Integer: Number of VMs handled by us
+  def nbr_vms_handled_by_us
+    @config['pve_nodes'].map do |pve_node|
+      api_get("nodes/#{pve_node}/lxc").select { |lxc_info| Integer(lxc_info['vmid']).between?(*@config['vm_ids_range']) }.size
+    end.sum
+  end
+
+  # Reserve resources for a new container on a PVE node, and assign a new VM ID and IP to it.
+  # Prerequisites:
+  # * This method should be called in a #start block
+  #
+  # Parameters::
+  # * *pve_node* (String): Node on which we reserve the resources.
+  # * *nbr_cpus* (Integer): Wanted CPUs
+  # * *ram_mb* (Integer): Wanted MB of RAM
+  # * *disk_gb* (Integer): Wanted GB of disk
+  # Result::
+  # * [Integer, String] or Symbol: Reserved resource info ([vm_id, ip]), or Symbol in case of error.
+  #   Possible error codes returned are:
+  #   * *no_available_ip*: There is no available IP to be reserved
+  #   * *no_available_vm_id*: There is no available VM ID to be reserved
+  def reserve_on(pve_node, nbr_cpus, ram_mb, disk_gb)
+    # We select a new VM ID and VM IP.
+    selected_vm_ip = free_ips.first
+    if selected_vm_ip.nil?
+      # No available IP for now.
+      :no_available_ip
+    else
+      selected_vm_id = free_vm_ids.first
+      if selected_vm_id.nil?
+        # No available ID for now.
+        :no_available_vm_id
+      else
+        # Success
+        log "[ #{pve_node}/#{selected_vm_id} ] - New LXC container reserved with IP #{selected_vm_ip}"
+        [selected_vm_id, selected_vm_ip]
+      end
+    end
+  end
+
+  # Destroy expired VMs on a PVE node.
+  # Only consider VMs that fall in the config VM ID range and are expired.
+  #
+  # Parameters::
+  # * *pve_node* (String): PVE node to delete expired VMs from.
+  def destroy_expired_vms_on(pve_node)
+    api_get("nodes/#{pve_node}/lxc").each do |lxc_info|
+      vm_id = Integer(lxc_info['vmid'])
+      destroy_vm_on(pve_node, vm_id) if is_vm_expired?(pve_node, vm_id)
+    end
+    # Invalidate the API cache for anything related to this PVE node
+    pve_node_paths_regexp = /^nodes\/#{Regexp.escape(pve_node)}\/.+$/
+    @gets_cache.delete_if { |path, _result| path =~ pve_node_paths_regexp }
+  end
+
+  # Destroy a VM on a PVE node.
+  # Stop it if needed before destroy.
+  #
+  # Parameters::
+  # * *pve_node* (String): PVE node hosting the VM
+  # * *vm_id* (Integer): The VM ID to destroy
+  def destroy_vm_on(pve_node, vm_id)
+    if vm_state(pve_node, vm_id) == 'running'
+      log "[ #{pve_node}/#{vm_id} ] - Stop LXC container"
+      wait_for_proxmox_task(pve_node, @proxmox.post("nodes/#{pve_node}/lxc/#{vm_id}/status/stop"))
+    end
+    log "[ #{pve_node}/#{vm_id} ] - Destroy LXC container"
+    wait_for_proxmox_task(pve_node, @proxmox.delete("nodes/#{pve_node}/lxc/#{vm_id}"))
+  end
+
+  # Return the list of available IPs
+  #
+  # Result::
+  # * Array<String>: List of available IPs
+  def free_ips
+    # Consider all nodes and all IPs to ensure we won't create any conflict, even outside our allowed range
+    @config['vm_ips_list'] -
+      api_get('nodes').map do |pve_node_info|
+        pve_node = pve_node_info['node']
+        api_get("nodes/#{pve_node}/lxc").map do |lxc_info|
+          ip_of(pve_node, Integer(lxc_info['vmid']))
+        end.compact
+      end.flatten
+  end
+
+  # Return the list of available VM IDs
+  #
+  # Result::
+  # * Array<Integer>: List of available VM IDs
+  def free_vm_ids
+    Range.new(*@config['vm_ids_range']).to_a -
+      api_get('nodes').map do |pve_node_info|
+        api_get("nodes/#{pve_node_info['node']}/lxc").map { |lxc_info| Integer(lxc_info['vmid']) }
+      end.flatten
+  end
+
+  # Wait for a given Proxmox task completion
+  #
+  # Parameters::
+  # * *pve_node* (String): The PVE node on which the task is run
+  # * *task* (String): The task ID
+  def wait_for_proxmox_task(pve_node, task)
+    raise "Invalid task: #{task}" if task[0..3] == 'NOK:'
+    while task_status(pve_node, task) == 'running'
+      log "[ #{pve_node} ] - Wait for Proxmox task #{task} to complete..."
+      sleep 1
+    end
+    log "[ #{pve_node} ] - Proxmox task #{task} completed."
+  end
+
+  # Get task status
+  #
+  # Parameters::
+  # * *pve_node* (String): Node on which the task status is to be queried
+  # * *task* (String): Task ID to query
+  # Result::
+  # * String: The task status
+  def task_status(pve_node, task)
+    status_info = @proxmox.get("nodes/#{pve_node}/tasks/#{task}/status")
+    "#{status_info['status']}#{status_info['exitstatus'] ? ":#{status_info['exitstatus']}" : ''}"
+  end
+
+  # Get a path from the API it returns its JSON result.
+  # Keep a cache of it, whose lifespan is this ProxmoxWaiter instance.
+  #
+  # Parameters::
+  # * *path* (String): API path to query
+  def api_get(path)
+    @gets_cache[path] = @proxmox.get(path) unless @gets_cache.key?(path)
+    @gets_cache[path]
+  end
+
+  # Get the state of a VM
+  #
+  # Parameters::
+  # * *pve_node* (String): The PVE node having the container
+  # * *vm_id* (Integer): The VM ID
+  # Result::
+  # * String: The state
+  def vm_state(pve_node, vm_id)
+    api_get("nodes/#{pve_node}/lxc/#{vm_id}/status/current")['status']
+  end
+
+  # Timeout in seconds before giving up on a lock
+  LOCK_TIMEOUT = 30
+
+  # Get the IP address of a given LXC container
+  #
+  # Parameters::
+  # * *pve_node* (String): The PVE node having the container
+  # * *vm_id* (Integer): The VM ID
+  # Result::
+  # * String or nil: The corresponding IP address, or nil if not found (could be that the container has disappeared, as this method is used also for containers not part of our sync node)
+  def ip_of(pve_node, vm_id)
+    ip_found = nil
+    config_path = "nodes/#{pve_node}/lxc/#{vm_id}/config"
+    lxc_config = nil
+    begin_time = Time.now
+    loop do
+      lxc_config = api_get(config_path)
+      if lxc_config.is_a?(String)
+        log "[ #{pve_node}/#{vm_id} ] - Error while checking its config: #{lxc_config}. Might be that the VM has disappeared."
+        lxc_config = { 'lock' => "Error: #{lxc_config}" }
+      elsif lxc_config.key?('lock')
+        # The node is currently doing some task. Wait for the lock to be released.
+        log "[ #{pve_node}/#{vm_id} ] - Node is being locked (reason: #{lxc_config['lock']}). Wait for the lock to be released..."
+        sleep 1
+      else
+        break
+      end
+      # Make sure we don't cache the error or the lock
+      @gets_cache.delete(config_path)
+      if Time.now - begin_time > LOCK_TIMEOUT
+        log "[ #{pve_node}/#{vm_id} ] - !!! Timeout while waiting for to be unlocked (reason: #{lxc_config['lock']})."
+        break
+      end
+    end
+    if lxc_config['net0'].nil?
+      log "[ #{pve_node}/#{vm_id} ] - !!! Config does not contain net0 information: #{lxc_config}"
+    else
+      lxc_config['net0'].split(',').each do |net_info|
+        property, value = net_info.split('=')
+        if property == 'ip'
+          ip_found = value.split('/').first
+          break
+        end
+      end
+    end
+    ip_found
+  end
+
+end