hybrid_platforms_conductor 32.8.2 → 32.11.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: f190c223603682dd67a933cb989ef89858d71459ebb1d962bfffda9def7666f2
-  data.tar.gz: 4fd85a4b961dc8ba8cc2c83b3f9749b49a021203c37afbe91db5123e54068007
+  metadata.gz: 2d3d8ff58b63ff1dbcadaeba8bea4c5e6469ace3ef68d9b39ab1eabc8f68a209
+  data.tar.gz: 3b02a1707c32c436e6d353775e38d116bba42e16cc35a2854a69f052377c85a4
 SHA512:
-  metadata.gz: e77c67013ea488956dd55927a06b22733efc7f283aa234bd3527b02a8044517447177a686c4964e27c7ff62afd9513de34efe33321ee6fdb16f7e3d6cd0dd06c
-  data.tar.gz: 0e041f3b998df727e83ec6580b09a02e4ee22c246e23a2f7ef5e3b64a3d60fd39745c1ed2241846467c14ba9d11b6dc8e8955929b6940f7ef87132369ecc9c80
+  metadata.gz: f32880a9ab54d037caf3f9400bac31fcbc8993618c0859d22ff4d07585b8962683e295ee4a09847606a377aa343c78b09cddbc20026e34baeb03265409c66271
+  data.tar.gz: 2545b32a9aa07ea5cad6a01ba418d19eaa2ce28dcdd66146af07b98de6da44c30a9da8c35de33a4b054078f80f8813ceb05c12899b2f0c323b59ef383dc267da

data/lib/hybrid_platforms_conductor/common_config_dsl/idempotence_tests.rb

@@ -11,13 +11,24 @@ module HybridPlatformsConductor
       # Array< Hash<Symbol, Object> >
       attr_reader :ignored_idempotence_tasks
 
-      # Initialize the DSL 
+      # List of ignored tasks info. Each info has the following properties:
+      # * *nodes_selectors_stack* (Array<Object>): Stack of nodes selectors impacted by this rule
+      # * *ignored_tasks* (Hash<String, String>): List of task names for which we ignore divergence errors, with the corresponding descriptive reason for ignore.
+      # Array< Hash<Symbol, Object> >
+      attr_reader :ignored_divergent_tasks
+
+      # Initialize the DSL
       def init_idempotence_tests
         # List of ignored tasks info. Each info has the following properties:
         # * *nodes_selectors_stack* (Array<Object>): Stack of nodes selectors impacted by this rule
         # * *ignored_tasks* (Hash<String, String>): List of task names for which we ignore idempotence errors, with the corresponding descriptive reason for ignore.
         # Array< Hash<Symbol, Object> >
         @ignored_idempotence_tasks = []
+        # List of ignored tasks info. Each info has the following properties:
+        # * *nodes_selectors_stack* (Array<Object>): Stack of nodes selectors impacted by this rule
+        # * *ignored_tasks* (Hash<String, String>): List of task names for which we ignore divergence errors, with the corresponding descriptive reason for ignore.
+        # Array< Hash<Symbol, Object> >
+        @ignored_divergent_tasks = []
       end
 
       # Ignore idempotence errors on a set of tasks
@@ -31,6 +42,17 @@ module HybridPlatformsConductor
         }
       end
 
+      # Ignore idempotence errors on a set of tasks
+      #
+      # Parameters::
+      # * *tasks_to_ignore* (Hash<String, String>): Set of tasks to ignore, along with the reason
+      def ignore_divergent_tasks(tasks_to_ignore)
+        @ignored_divergent_tasks << {
+          ignored_tasks: tasks_to_ignore,
+          nodes_selectors_stack: current_nodes_selectors_stack,
+        }
+      end
+
     end
 
   end

data/lib/hybrid_platforms_conductor/hpc_plugins/provisioner/proxmox.rb

@@ -18,11 +18,19 @@ module HybridPlatformsConductor
         attr_accessor *%i[logger logger_stderr]
 
         def check_response(response)
-          log_debug "Response from Proxmox API: #{response}"
-          log_warn "Response from Proxmox API: #{response}" if response.code >= 400 && !log_debug?
+          msg = "Response from Proxmox API: #{response} - #{response.net_http_res.message}"
+          log_debug msg
+          log_warn msg if response.code >= 400 && !log_debug?
           super
         end
 
+        # Re-authenticate the Proxmox instance
+        # This can be useful when the API returns errors due to invalidated tokens
+        def reauthenticate
+          log_debug 'Force re-authentication to Proxmox'
+          @auth_params = create_ticket
+        end
+
       end
       ::Proxmox::Proxmox.prepend ProxmoxPatches
 
@@ -74,13 +82,13 @@ module HybridPlatformsConductor
           # First check if we already have a test container that corresponds to this node and environment
           @lxc_details = nil
           with_proxmox do |proxmox|
-            proxmox.get('nodes').each do |node_info|
+            proxmox_get(proxmox, 'nodes').each do |node_info|
               if proxmox_test_info[:test_config][:pve_nodes].include?(node_info['node']) && node_info['status'] == 'online'
-                proxmox.get("nodes/#{node_info['node']}/lxc").each do |lxc_info|
+                proxmox_get(proxmox, "nodes/#{node_info['node']}/lxc").each do |lxc_info|
                   vm_id = Integer(lxc_info['vmid'])
                   if vm_id.between?(*proxmox_test_info[:test_config][:vm_ids_range])
                     # Check if the description contains our ID
-                    lxc_config = proxmox.get("nodes/#{node_info['node']}/lxc/#{vm_id}/config")
+                    lxc_config = proxmox_get(proxmox, "nodes/#{node_info['node']}/lxc/#{vm_id}/config")
                     vm_description_lines = (lxc_config['description'] || '').split("\n")
                     hpc_marker_idx = vm_description_lines.index('===== HPC info =====')
                     unless hpc_marker_idx.nil?
@@ -222,8 +230,8 @@ module HybridPlatformsConductor
             with_proxmox do |proxmox|
               vm_id_str = @lxc_details[:vm_id].to_s
               status =
-                if proxmox.get("nodes/#{@lxc_details[:pve_node]}/lxc").any? { |data_info| data_info['vmid'] == vm_id_str }
-                  status_info = proxmox.get("nodes/#{@lxc_details[:pve_node]}/lxc/#{@lxc_details[:vm_id]}/status/current")
+                if proxmox_get(proxmox, "nodes/#{@lxc_details[:pve_node]}/lxc").any? { |data_info| data_info['vmid'] == vm_id_str }
+                  status_info = proxmox_get(proxmox, "nodes/#{@lxc_details[:pve_node]}/lxc/#{@lxc_details[:vm_id]}/status/current")
                   # Careful that it is possible that somebody destroyed the VM and so its status is missing
                   status = status_info.key?('status') ? status_info['status'].to_sym : :missing
                   status = :exited if status == :stopped
@@ -292,11 +300,29 @@ module HybridPlatformsConductor
           end
         end
 
-        # Maximum number of retries to perform on the Proxmox API.
-        NBR_RETRIES_MAX = 5
-
-        # Minimum seconds to wait between retries
-        RETRY_WAIT_TIME_SECS = 5
+        # Perform a get operation on the API
+        # Protect the get API methods with a retry mechanism in case of 5xx errors.
+        #
+        # Parameters::
+        # * *proxmox* (Proxmox): The Proxmox instance
+        # * *path* (String): Path to get
+        # Result::
+        # * Object: API response
+        def proxmox_get(proxmox, path)
+          response = nil
+          idx_try = 0
+          loop do
+            response = proxmox.get(path)
+            break if !(response.is_a?(String)) || !(response =~ /^NOK: error code = 5\d\d$/)
+            log_warn "[ #{@node}/#{@environment} ] - Proxmox API call get #{path} returned error #{response} (attempt ##{idx_try}/#{proxmox_test_info[:api_max_retries]})"
+            raise "[ #{@node}/#{@environment} ] - Proxmox API call get #{path} returns #{response} continuously (tried #{idx_try + 1} times)" if idx_try >= proxmox_test_info[:api_max_retries]
+            idx_try += 1
+            # We have to reauthenticate: error 500 raised by Proxmox are often due to token being invalidated wrongly
+            proxmox.reauthenticate
+            sleep proxmox_test_info[:api_wait_between_retries_secs] + rand(5)
+          end
+          response
+        end
 
         # Run a Proxmox task.
         # Handle a retry mechanism in case of 5xx errors.
@@ -313,11 +339,13 @@ module HybridPlatformsConductor
           while task.nil? do
             task = proxmox.send(http_method, "nodes/#{pve_node}/#{sub_path}", *args)
             if task =~ /^NOK: error code = 5\d\d$/
-              log_warn "[ #{@node}/#{@environment} ] - Proxmox API call #{http_method} nodes/#{pve_node}/#{sub_path} #{args} returned error #{task} (attempt ##{idx_try}/#{NBR_RETRIES_MAX})"
+              log_warn "[ #{@node}/#{@environment} ] - Proxmox API call #{http_method} nodes/#{pve_node}/#{sub_path} #{args} returned error #{task} (attempt ##{idx_try}/#{proxmox_test_info[:api_max_retries]})"
               task = nil
+              break if idx_try >= proxmox_test_info[:api_max_retries]
               idx_try += 1
-              break if idx_try == NBR_RETRIES_MAX
-              sleep RETRY_WAIT_TIME_SECS + rand(5)
+              # We have to reauthenticate: error 500 raised by Proxmox are often due to token being invalidated wrongly
+              proxmox.reauthenticate
+              sleep proxmox_test_info[:api_wait_between_retries_secs] + rand(5)
             end
           end
           if task.nil?
@@ -358,7 +386,7 @@ module HybridPlatformsConductor
         # Result::
         # * String: The task status
         def task_status(proxmox, pve_node, task)
-          status_info = proxmox.get("nodes/#{pve_node}/tasks/#{task}/status")
+          status_info = proxmox_get(proxmox, "nodes/#{pve_node}/tasks/#{task}/status")
           "#{status_info['status']}#{status_info['exitstatus'] ? ":#{status_info['exitstatus']}" : ''}"
         end
 
@@ -377,7 +405,9 @@ module HybridPlatformsConductor
             (proxmox_test_info[:test_config].merge(
               proxmox_api_url: proxmox_test_info[:api_url],
               futex_file: '/tmp/hpc_proxmox_allocations.futex',
-              logs_dir: '/tmp/hpc_proxmox_waiter_logs'
+              logs_dir: '/tmp/hpc_proxmox_waiter_logs',
+              api_max_retries: proxmox_test_info[:api_max_retries],
+              api_wait_between_retries_secs: proxmox_test_info[:api_wait_between_retries_secs]
             )).to_json
           )
           result = nil
@@ -486,7 +516,7 @@ module HybridPlatformsConductor
         # So remaining length is 255 - 13 = 242 characters.
         MAX_FILE_ID_SIZE = 242
 
-        # Get an ID unique for theis node/environment and that can be used in file names.
+        # Get an ID unique for this node/environment and that can be used in file names.
         #
         # Result::
         # * String: ID
@@ -506,6 +536,8 @@ module HybridPlatformsConductor
         # Result::
         # * Hash<Symbol,Object>: Configuration of the Proxmox instance to be used:
         #   * *api_url* (String): The Proxmox API URL
+        #   * *api_max_retries* (Integer): Max number of API retries
+        #   * *api_wait_between_retries_secs* (Integer): Number of seconds to wait between API retries
         #   * *sync_node* (String): Node to be used to synchronize Proxmox resources acquisition
         #   * *test_config* (Hash<Symbol,Object>): The test configuration. Check ProxmoxWaiter#initialize (config_file structure) method to get details.
         #   * *vm_config* (Hash<Symbol,Object>): Extra configuration of a created container. Check #request_lxc_creation_for results to get details.

data/lib/hybrid_platforms_conductor/hpc_plugins/provisioner/proxmox/proxmox_waiter.rb

@@ -26,6 +26,8 @@ class ProxmoxWaiter
   #   * *proxmox_api_url* (String): Proxmox API URL.
   #   * *futex_file* (String): Path to the file serving as a futex.
   #   * *logs_dir* (String): Path to the directory containing logs [default: '.']
+  #   * *api_max_retries* (Integer): Max number of API retries
+  #   * *api_wait_between_retries_secs* (Integer): Number of seconds to wait between API retries
   #   * *pve_nodes* (Array<String>): List of PVE nodes allowed to spawn new containers [default: all]
   #   * *vm_ips_list* (Array<String>): The list of IPs that are available for the Proxomx containers.
   #   * *vm_ids_range* ([Integer, Integer]): Minimum and maximum reservable VM ID
@@ -603,10 +605,35 @@ class ProxmoxWaiter
   # Result::
   # * Array<Integer>: List of available VM IDs
   def free_vm_ids
-    Range.new(*@config['vm_ids_range']).to_a -
+    vm_ids = Range.new(*@config['vm_ids_range']).to_a -
       api_get('nodes').map do |pve_node_info|
         api_get("nodes/#{pve_node_info['node']}/lxc").map { |lxc_info| Integer(lxc_info['vmid']) }
       end.flatten
+    # Make sure the vm_ids that are available don't have any leftovers in the cgroups.
+    # This can happen with some Proxmox bugs, and make the API returns 500 errors.
+    # cf. https://forum.proxmox.com/threads/lxc-console-cleanup-error.38293/
+    # TODO: Remove this when Proxmox will have solved the issue with leftovers of destroyed vms.
+    (vm_ids.map(&:to_s) & vm_ids_in_cgroups).each do |vm_id_str|
+      # We are having a vm_id that is available but still has some leftovers in cgroups.
+      # Clean those to avoid 500 errors in API.
+      log "Found VMID #{vm_id_str} with leftovers in cgroups. Cleaning those."
+      Dir.glob("/sys/fs/cgroup/*/lxc/#{vm_id_str}") do |cgroup_dir|
+        log "Removing #{cgroup_dir}"
+        FileUtils.rm_rf cgroup_dir
+      end
+    end
+    vm_ids
+  end
+
+  # Return the list of VM IDs present in cgroups
+  #
+  # Result::
+  # * Array<String>: List of VM IDs as strings (as some are not Integers like '1010-1')
+  def vm_ids_in_cgroups
+    Dir.glob('/sys/fs/cgroup/*/lxc/*').map do |file|
+      basename = File.basename(file)
+      basename =~ /^\d.+$/ ? basename : nil
+    end.compact.sort.uniq
   end
 
   # Wait for a given Proxmox task completion
@@ -637,11 +664,26 @@ class ProxmoxWaiter
 
   # Get a path from the API it returns its JSON result.
   # Keep a cache of it, whose lifespan is this ProxmoxWaiter instance.
+  # Have a retry mechanism to make sure eventual non-deterministic 5xx errors are not an issue.
   #
   # Parameters::
   # * *path* (String): API path to query
-  def api_get(path)
-    @gets_cache[path] = @proxmox.get(path) unless @gets_cache.key?(path)
+  # Result::
+  # * Object: The API response
+  def api_get(path, nbr_retries: 3, wait_between_retry_secs: 10)
+    unless @gets_cache.key?(path)
+      idx_try = 0
+      loop do
+        @gets_cache[path] = @proxmox.get(path)
+        break unless @gets_cache[path].is_a?(String) && @gets_cache[path] =~ /^NOK: error code = 5\d\d$/
+        raise "Proxmox API get #{path} returns #{@gets_cache[path]} continuously (tried #{idx_try + 1} times)" if idx_try >= @config['api_max_retries']
+        idx_try += 1
+        # We have to reauthenticate: error 500 raised by Proxmox are often due to token being invalidated wrongly
+        # TODO: Provide a way to do it properly in the official gem
+        @proxmox.instance_variable_set(:@auth_params, @proxmox.send(:create_ticket))
+        sleep @config['api_wait_between_retries_secs']
+      end
+    end
     @gets_cache[path]
   end
 

data/lib/hybrid_platforms_conductor/hpc_plugins/test/check_deploy_and_idempotence.rb

@@ -81,7 +81,10 @@ module HybridPlatformsConductor
                   exit_status, stdout, stderr = deployer.deploy_on(@node)[@node]
                   assert_equal exit_status, 0, "Check-node after deployment returned error code #{exit_status}", log_debug? ? nil : deployer.stdouts_to_s
                   # Check that the output of the check-node returns no changes.
-                  ignored_tasks = @nodes_handler.select_confs_for_node(@node, @config.ignored_idempotence_tasks).inject({}) do |merged_ignored_tasks, conf|
+                  ignored_tasks = (
+                    @nodes_handler.select_confs_for_node(@node, @config.ignored_idempotence_tasks) +
+                      @nodes_handler.select_confs_for_node(@node, @config.ignored_divergent_tasks)
+                  ).inject({}) do |merged_ignored_tasks, conf|
                     merged_ignored_tasks.merge(conf[:ignored_tasks])
                   end
                   @deployer.parse_deploy_output(@node, stdout, stderr).each do |task_info|

data/lib/hybrid_platforms_conductor/hpc_plugins/test/divergence.rb

@@ -11,8 +11,23 @@ module HybridPlatformsConductor
 
         # Check my_test_plugin.rb.sample documentation for signature details.
         def test_on_check_node(stdout, stderr, exit_status)
+          # Check that the output of the check-node returns no changes.
+          ignored_tasks = @nodes_handler.select_confs_for_node(@node, @config.ignored_divergent_tasks).inject({}) do |merged_ignored_tasks, conf|
+            merged_ignored_tasks.merge(conf[:ignored_tasks])
+          end
           @deployer.parse_deploy_output(@node, stdout, stderr).each do |task_info|
-            error "Task #{task_info[:name]} has diverged", JSON.pretty_generate(task_info) if task_info[:status] == :changed
+            if task_info[:status] == :changed
+              if ignored_tasks.key?(task_info[:name])
+                # It was expected that this task is not idempotent
+                log_debug "Task #{task_info[:name]} was expected to be divergent. Reason: #{ignored_tasks[task_info[:name]]}"
+              else
+                extra_details = task_info.slice(*(task_info.keys - %i[name status diffs]))
+                error_details = []
+                error_details << "----- Changes:\n#{task_info[:diffs].strip}\n-----" if task_info[:diffs]
+                error_details << "----- Additional details:\n#{JSON.pretty_generate(extra_details)}\n-----" unless extra_details.empty?
+                error "Task #{task_info[:name]} has diverged", error_details.empty? ? nil : error_details.join("\n")
+              end
+            end
           end
         end
 

data/lib/hybrid_platforms_conductor/hpc_plugins/test/executables.rb

@@ -11,24 +11,38 @@ module HybridPlatformsConductor
 
         # Check my_test_plugin.rb.sample documentation for signature details.
         def test
-          example_platform = PlatformsHandler.new.known_platforms.first
-          example_node = example_platform.known_nodes.first
-        	[
-            "#{CmdRunner.executables_prefix}check-node --node #{example_node} --show-commands",
-            "#{CmdRunner.executables_prefix}deploy --node #{example_node} --show-commands --why-run",
+          tests = [
             "#{CmdRunner.executables_prefix}dump_nodes_json --help",
             "#{CmdRunner.executables_prefix}free_ips",
             "#{CmdRunner.executables_prefix}free_veids",
-            "#{CmdRunner.executables_prefix}get_impacted_nodes --platform #{example_platform.name} --show-commands",
-            "#{CmdRunner.executables_prefix}last_deploys --node #{example_node} --show-commands",
-            "#{CmdRunner.executables_prefix}nodes_to_deploy --node #{example_node} --show-commands",
-            "#{CmdRunner.executables_prefix}report --node #{example_node} --format stdout",
-            "#{CmdRunner.executables_prefix}run --node #{example_node} --show-commands --interactive",
             "#{CmdRunner.executables_prefix}setup --help",
             "#{CmdRunner.executables_prefix}ssh_config",
-            "#{CmdRunner.executables_prefix}test --help",
-            "#{CmdRunner.executables_prefix}topograph --from \"--node #{example_node}\" --to \"--node #{example_node}\" --skip-run --output graphviz:graph.gv"
-          ].each do |cmd|
+            "#{CmdRunner.executables_prefix}test --help"
+          ]
+          example_platform = PlatformsHandler.new(
+            logger: @logger,
+            logger_stderr: @logger_stderr,
+            config: @config,
+            cmd_runner: @cmd_runner
+          ).known_platforms.first
+          unless example_platform.nil?
+            tests.concat [
+              "#{CmdRunner.executables_prefix}get_impacted_nodes --platform #{example_platform.name} --show-commands",
+            ]
+            example_node = example_platform.known_nodes.first
+            unless example_node.nil?
+              tests.concat [
+                "#{CmdRunner.executables_prefix}check-node --node #{example_node} --show-commands",
+                "#{CmdRunner.executables_prefix}deploy --node #{example_node} --show-commands --why-run",
+                "#{CmdRunner.executables_prefix}last_deploys --node #{example_node} --show-commands",
+                "#{CmdRunner.executables_prefix}nodes_to_deploy --node #{example_node} --show-commands",
+                "#{CmdRunner.executables_prefix}report --node #{example_node} --format stdout",
+                "#{CmdRunner.executables_prefix}run --node #{example_node} --show-commands --interactive",
+                "#{CmdRunner.executables_prefix}topograph --from \"--node #{example_node}\" --to \"--node #{example_node}\" --skip-run --output graphviz:graph.gv"
+              ]
+            end
+          end
+        	tests.sort.each do |cmd|
             log_debug "Testing #{cmd}"
             exit_status, stdout, _stderr = @cmd_runner.run_cmd "#{cmd} 2>&1", no_exception: true, log_to_stdout: log_debug?
             assert_equal(exit_status, 0, "Command #{cmd} returned code #{exit_status}:\n#{stdout}")

data/lib/hybrid_platforms_conductor/hpc_plugins/test/idempotence.rb

@@ -43,7 +43,10 @@ module HybridPlatformsConductor
                   assert_equal tested_node, @node, "Wrong node being tested: #{tested_node} should be #{@node}"
                   assert_equal exit_status, 0, "Check-node returned error code #{exit_status}"
                   # Check that the output of the check-node returns no changes.
-                  ignored_tasks = @nodes_handler.select_confs_for_node(@node, @config.ignored_idempotence_tasks).inject({}) do |merged_ignored_tasks, conf|
+                  ignored_tasks = (
+                    @nodes_handler.select_confs_for_node(@node, @config.ignored_idempotence_tasks) +
+                      @nodes_handler.select_confs_for_node(@node, @config.ignored_divergent_tasks)
+                  ).inject({}) do |merged_ignored_tasks, conf|
                     merged_ignored_tasks.merge(conf[:ignored_tasks])
                   end
                   @deployer.parse_deploy_output(@node, stdout, stderr).each do |task_info|

data/lib/hybrid_platforms_conductor/version.rb

@@ -1,5 +1,5 @@
 module HybridPlatformsConductor
 
-  VERSION = '32.8.2'
+  VERSION = '32.11.1'
 
 end

data/spec/hybrid_platforms_conductor_test/api/deployer/provisioners/proxmox/reserve_proxmox_container/retries_spec.rb

@@ -10,6 +10,7 @@ describe HybridPlatformsConductor::HpcPlugins::Provisioner::Proxmox do
         with_sync_node do
           mock_proxmox(mocked_pve_nodes: [{ 'pve_node_name' => {} }] * 5)
           expect(call_reserve_proxmox_container(2, 128 * 1024, 4, max_retries: 5)).to eq(error: 'not_enough_resources')
+          expect_proxmox_actions_to_be []
         end
       end
 
@@ -25,6 +26,69 @@ describe HybridPlatformsConductor::HpcPlugins::Provisioner::Proxmox do
             vm_id: 1000,
             vm_ip: '192.168.0.100'
           )
+          expect_proxmox_actions_to_be [
+            [
+              :post,
+              'nodes/pve_node_name/lxc',
+              {
+                'ostemplate' => 'test_template.iso',
+                'hostname' => 'test.hostname.my-domain.com',
+                'description' => /node: test_node\nenvironment: test_env/,
+                'cores' => 2,
+                'cpulimit' => 2,
+                'memory' => 1024,
+                'rootfs' => 'local-lvm:4',
+                'net0' => 'name=eth0,bridge=vmbr0,gw=172.16.16.16,ip=192.168.0.100/32',
+                'vmid' => 1000
+              }
+            ]
+          ]
+        end
+      end
+
+      it 'retries a few times before ending in error for a 5xx API error' do
+        with_sync_node do
+          mock_proxmox(mocked_pve_nodes: [{ 'pve_node_name' => { error_strings: ['NOK: error code = 500'] * 5 } }])
+          result = call_reserve_proxmox_container(2, 1024, 4, config: { api_max_retries: 4 })
+          expect(result[:error]).not_to eq nil
+          expect(result[:error]).to match /Unhandled exception from reserve_proxmox_container: Proxmox API get nodes\/pve_node_name\/lxc returns NOK: error code = 500 continuously \(tried 5 times\)/
+          expect_proxmox_actions_to_be [
+            [:create_ticket],
+            [:create_ticket],
+            [:create_ticket],
+            [:create_ticket]
+          ]
+        end
+      end
+
+      it 'retries API errors a few times until it gets resolved' do
+        with_sync_node do
+          mock_proxmox(mocked_pve_nodes: [{ 'pve_node_name' => { error_strings: ['NOK: error code = 500'] * 3 } }])
+          expect(call_reserve_proxmox_container(2, 1024, 4, config: { api_max_retries: 4 })).to eq(
+            pve_node: 'pve_node_name',
+            vm_id: 1000,
+            vm_ip: '192.168.0.100'
+          )
+          expect_proxmox_actions_to_be [
+            [:create_ticket],
+            [:create_ticket],
+            [:create_ticket],
+            [
+              :post,
+              'nodes/pve_node_name/lxc',
+              {
+                'ostemplate' => 'test_template.iso',
+                'hostname' => 'test.hostname.my-domain.com',
+                'description' => /node: test_node\nenvironment: test_env/,
+                'cores' => 2,
+                'cpulimit' => 2,
+                'memory' => 1024,
+                'rootfs' => 'local-lvm:4',
+                'net0' => 'name=eth0,bridge=vmbr0,gw=172.16.16.16,ip=192.168.0.100/32',
+                'vmid' => 1000
+              }
+            ]
+          ]
         end
       end
 

data/spec/hybrid_platforms_conductor_test/api/deployer/provisioners/proxmox/reserve_proxmox_container/vm_ids_assignment_spec.rb

@@ -60,6 +60,98 @@ describe HybridPlatformsConductor::HpcPlugins::Provisioner::Proxmox do
         end
       end
 
+      it 'makes sure to remove cgroup files that are leftovers of removed containers' do
+        with_sync_node(leftovers: [
+          '/sys/fs/cgroup/memory/lxc/1003'
+        ]) do
+          mock_proxmox(mocked_pve_nodes: {
+            'pve_node_name' => {
+              lxc_containers: {
+                1000 => { ip: '192.168.1.100' },
+                1001 => { ip: '192.168.1.101' }
+              }
+            }
+          })
+          expect(call_reserve_proxmox_container(2, 1024, 1, config: { vm_ids_range: [1000, 1100] })).to eq(
+            pve_node: 'pve_node_name',
+            vm_id: 1002,
+            vm_ip: '192.168.0.100'
+          )
+        end
+      end
+
+      it 'makes sure to remove cgroup files that are leftovers of removed containers even when they are reusing the VM ID' do
+        with_sync_node(leftovers: [
+          '/sys/fs/cgroup/memory/lxc/1002'
+        ]) do
+          mock_proxmox(mocked_pve_nodes: {
+            'pve_node_name' => {
+              lxc_containers: {
+                1000 => { ip: '192.168.1.100' },
+                1001 => { ip: '192.168.1.101' }
+              }
+            }
+          })
+          expect(call_reserve_proxmox_container(2, 1024, 1, config: { vm_ids_range: [1000, 1100] })).to eq(
+            pve_node: 'pve_node_name',
+            vm_id: 1002,
+            vm_ip: '192.168.0.100'
+          )
+        end
+      end
+
+      it 'makes sure to remove cgroup files that are leftovers of removed containers when several cgroups contain files' do
+        with_sync_node(leftovers: [
+          '/sys/fs/cgroup/memory/lxc/1003',
+          '/sys/fs/cgroup/network/lxc/1003',
+          '/sys/fs/cgroup/cpu/lxc/1003'
+        ]) do
+          mock_proxmox(mocked_pve_nodes: {
+            'pve_node_name' => {
+              lxc_containers: {
+                1000 => { ip: '192.168.1.100' },
+                1001 => { ip: '192.168.1.101' }
+              }
+            }
+          })
+          expect(call_reserve_proxmox_container(2, 1024, 1, config: { vm_ids_range: [1000, 1100] })).to eq(
+            pve_node: 'pve_node_name',
+            vm_id: 1002,
+            vm_ip: '192.168.0.100'
+          )
+        end
+      end
+
+      it 'makes sure to remove only cgroup files that are leftovers of removed containers inside our VM ID range' do
+        with_sync_node(
+          leftovers: [
+            '/sys/fs/cgroup/memory/lxc/100',
+            '/sys/fs/cgroup/memory/lxc/1001',
+            '/sys/fs/cgroup/memory/lxc/1002',
+            '/sys/fs/cgroup/memory/lxc/1003'
+          ],
+          expect_remaining_leftovers: [
+            '/sys/fs/cgroup/memory/lxc/100',
+            '/sys/fs/cgroup/memory/lxc/1001'
+          ]
+        ) do
+          mock_proxmox(mocked_pve_nodes: {
+            'pve_node_name' => {
+              lxc_containers: {
+                100 => { ip: '192.168.1.10' },
+                1000 => { ip: '192.168.1.100' },
+                1001 => { ip: '192.168.1.101' }
+              }
+            }
+          })
+          expect(call_reserve_proxmox_container(2, 1024, 1, config: { vm_ids_range: [1000, 1100] })).to eq(
+            pve_node: 'pve_node_name',
+            vm_id: 1002,
+            vm_ip: '192.168.0.100'
+          )
+        end
+      end
+
     end
 
   end

data/spec/hybrid_platforms_conductor_test/api/deployer/provisioners/proxmox/start_spec.rb

@@ -39,16 +39,7 @@ describe HybridPlatformsConductor::HpcPlugins::Provisioner::Proxmox do
           mock_proxmox_to_start_node(nbr_api_errors: 3)
         ]
         instance.create
-        # To speed up the test, alter the wait time between retries.
-        old_wait_secs = HybridPlatformsConductor::HpcPlugins::Provisioner::Proxmox.const_get(:RETRY_WAIT_TIME_SECS)
-        begin
-          HybridPlatformsConductor::HpcPlugins::Provisioner::Proxmox.send(:remove_const, :RETRY_WAIT_TIME_SECS)
-          HybridPlatformsConductor::HpcPlugins::Provisioner::Proxmox.const_set(:RETRY_WAIT_TIME_SECS, 1)
-          instance.start
-        ensure
-          HybridPlatformsConductor::HpcPlugins::Provisioner::Proxmox.send(:remove_const, :RETRY_WAIT_TIME_SECS)
-          HybridPlatformsConductor::HpcPlugins::Provisioner::Proxmox.const_set(:RETRY_WAIT_TIME_SECS, old_wait_secs)
-        end
+        instance.start
       end
     end
 
@@ -58,19 +49,10 @@ describe HybridPlatformsConductor::HpcPlugins::Provisioner::Proxmox do
           # 1 - The info on existing containers
           mock_proxmox_to_get_nodes_info,
           # 2 - The start of the container - fail too many times
-          mock_proxmox_to_start_node(nbr_api_errors: HybridPlatformsConductor::HpcPlugins::Provisioner::Proxmox.const_get(:NBR_RETRIES_MAX), task_status: nil)
+          mock_proxmox_to_start_node(nbr_api_errors: 4, task_status: nil)
         ]
         instance.create
-        # To speed up the test, alter the wait time between retries.
-        old_wait_secs = HybridPlatformsConductor::HpcPlugins::Provisioner::Proxmox.const_get(:RETRY_WAIT_TIME_SECS)
-        begin
-          HybridPlatformsConductor::HpcPlugins::Provisioner::Proxmox.send(:remove_const, :RETRY_WAIT_TIME_SECS)
-          HybridPlatformsConductor::HpcPlugins::Provisioner::Proxmox.const_set(:RETRY_WAIT_TIME_SECS, 1)
-          expect { instance.start }.to raise_error '[ node/test ] - Proxmox API call post nodes/pve_node_name/lxc/1024/status/start [] is constantly failing. Giving up.'
-        ensure
-          HybridPlatformsConductor::HpcPlugins::Provisioner::Proxmox.send(:remove_const, :RETRY_WAIT_TIME_SECS)
-          HybridPlatformsConductor::HpcPlugins::Provisioner::Proxmox.const_set(:RETRY_WAIT_TIME_SECS, old_wait_secs)
-        end
+        expect { instance.start }.to raise_error '[ node/test ] - Proxmox API call post nodes/pve_node_name/lxc/1024/status/start [] is constantly failing. Giving up.'
       end
     end
 

data/spec/hybrid_platforms_conductor_test/api/deployer/provisioners/proxmox/state_spec.rb

@@ -23,6 +23,32 @@ describe HybridPlatformsConductor::HpcPlugins::Provisioner::Proxmox do
       end
     end
 
+    it 'retries calls to the API when getting back errors 5xx' do
+      with_test_proxmox_platform do |instance|
+        mock_proxmox_calls_with [
+          # 1 - The info on existing containers
+          mock_proxmox_to_get_nodes_info,
+          # 2 - The status of the container
+          mock_proxmox_to_status_node(nbr_api_errors: 3)
+        ]
+        instance.create
+        expect(instance.state).to eq :created
+      end
+    end
+
+    it 'fails to get an instance\'s status when the Proxmox API fails too many times' do
+      with_test_proxmox_platform do |instance|
+        mock_proxmox_calls_with [
+          # 1 - The info on existing containers
+          mock_proxmox_to_get_nodes_info,
+          # 2 - The status of the container
+          mock_proxmox_to_status_node(nbr_api_errors: 4, status: nil)
+        ]
+        instance.create
+        expect { instance.state }.to raise_error '[ node/test ] - Proxmox API call get nodes/pve_node_name/lxc returns NOK: error code = 500 continuously (tried 4 times)'
+      end
+    end
+
   end
 
 end

data/spec/hybrid_platforms_conductor_test/helpers/provisioner_proxmox_helpers.rb

@@ -23,6 +23,8 @@ module HybridPlatformsConductorTest
             test_platform path: '#{repository}'
             proxmox(
               api_url: 'https://my-proxmox.my-domain.com:8006',
+              api_max_retries: 3,
+              api_wait_between_retries_secs: 0,
               sync_node: 'node',
               test_config: {
                 pve_nodes: ['pve_node_name'],
@@ -75,12 +77,20 @@ module HybridPlatformsConductorTest
       # * *proxmox_password* (String or nil): Proxmox password used to connect to Proxmox API [default: nil]
       # * *proxmox_realm* (String or nil): Proxmox realm used to connect to Proxmox API [default: 'pam']
       # * *nodes_info* (Array<Hash>): Nodes info returned by the Proxmox API [default: []]
+      # * *nbr_api_errors* (Integer): Number of API errors 500 to mock before getting a successful query [defaults: 0]
       # * *extra_expects* (Proc or nil): Code called for additional expectations on the proxmox instance, or nil if none [default: nil]
       #   * Parameters::
       #     * *proxmox* (Double): The mocked Proxmox instance
       # Result::
       # * Proc: Code called in place of Proxmox.new. Signature is the same as Proxmox.new.
-      def mock_proxmox_to_get_nodes_info(proxmox_user: nil, proxmox_password: nil, proxmox_realm: 'pam', nodes_info: [], extra_expects: nil)
+      def mock_proxmox_to_get_nodes_info(
+        proxmox_user: nil,
+        proxmox_password: nil,
+        proxmox_realm: 'pam',
+        nodes_info: [],
+        nbr_api_errors: 0,
+        extra_expects: nil
+      )
         proc do |url, pve_node, user, password, realm, options|
           expect(url).to eq 'https://my-proxmox.my-domain.com:8006/api2/json/'
           expect(pve_node).to eq 'my-proxmox'
@@ -97,8 +107,10 @@ module HybridPlatformsConductorTest
             # Nothing
           end
           # Mock checking existing nodes
-          expect(proxmox).to receive(:get).with('nodes') do
-            nodes_info
+          idx_try = 0
+          expect(proxmox).to receive(:get).exactly(nbr_api_errors + 1).times.with('nodes') do
+            idx_try += 1
+            idx_try <= nbr_api_errors ? 'NOK: error code = 500' : nodes_info
           end
           extra_expects.call(proxmox) unless extra_expects.nil?
           proxmox
@@ -144,6 +156,7 @@ module HybridPlatformsConductorTest
             idx_try += 1
             idx_try <= nbr_api_errors ? 'NOK: error code = 500' : task_name
           end
+          expect(proxmox).to receive(:reauthenticate).exactly(nbr_api_errors - (task_status.nil? ? 1 : 0)).times
           # Mock checking task status
           unless task_status.nil?
             # Mock checking task status
@@ -243,13 +256,15 @@ module HybridPlatformsConductorTest
       # Parameters::
       # * *proxmox_user* (String or nil): Proxmox user used to connect to Proxmox API [default: nil]
       # * *proxmox_password* (String or nil): Proxmox password used to connect to Proxmox API [default: nil]
-      # * *status* (String): Mocked status [default: 'created']
+      # * *status* (String or nil): Mocked status, or nil if it should not be asked [default: 'created']
+      # * *nbr_api_errors* (Integer): Number of API errors 500 to mock before getting a successful query [defaults: 0]
       # Result::
       # * Proc: Code called in place of Proxmox.new. Signature is the same as Proxmox.new.
       def mock_proxmox_to_status_node(
         proxmox_user: nil,
         proxmox_password: nil,
-        task_status: 'OK'
+        status: 'created',
+        nbr_api_errors: 0
       )
         proc do |url, pve_node, user, password, realm, options|
           expect(url).to eq 'https://my-proxmox.my-domain.com:8006/api2/json/'
@@ -267,17 +282,26 @@ module HybridPlatformsConductorTest
             # Nothing
           end
           # Mock getting status of a container
-          expect(proxmox).to receive(:get).with('nodes/pve_node_name/lxc') do
-            [
+          idx_try = 0
+          expect(proxmox).to receive(:get).exactly(nbr_api_errors + (status.nil? ? 0 : 1)).times.with('nodes/pve_node_name/lxc') do
+            idx_try += 1
+            if idx_try <= nbr_api_errors
+              'NOK: error code = 500'
+            else
+              [
+                {
+                  'vmid' => '1024'
+                }
+              ]
+            end
+          end
+          expect(proxmox).to receive(:reauthenticate).exactly(nbr_api_errors - (status.nil? ? 1 : 0)).times
+          unless status.nil?
+            expect(proxmox).to receive(:get).with('nodes/pve_node_name/lxc/1024/status/current') do
               {
-                'vmid' => '1024'
+                'status' => status
               }
-            ]
-          end
-          expect(proxmox).to receive(:get).with('nodes/pve_node_name/lxc/1024/status/current') do
-            {
-              'status' => 'created'
-            }
+            end
           end
           proxmox
         end
@@ -548,13 +572,17 @@ module HybridPlatformsConductorTest
                   ]
                 when /^nodes\/([^\/]+)\/lxc$/
                   pve_node_name = $1
-                  pve_nodes[pve_node_name][:lxc_containers].map do |vm_id, vm_info|
-                    {
-                      'vmid' => vm_id.to_s,
-                      'maxdisk' => vm_info[:maxdisk],
-                      'maxmem' => vm_info[:maxmem],
-                      'cpus' => vm_info[:cpus]
-                    }
+                  if pve_nodes[pve_node_name][:error_strings].nil? || pve_nodes[pve_node_name][:error_strings].empty?
+                    pve_nodes[pve_node_name][:lxc_containers].map do |vm_id, vm_info|
+                      {
+                        'vmid' => vm_id.to_s,
+                        'maxdisk' => vm_info[:maxdisk],
+                        'maxmem' => vm_info[:maxmem],
+                        'cpus' => vm_info[:cpus]
+                      }
+                    end
+                  else
+                    pve_nodes[pve_node_name][:error_strings].shift
                   end
                 when /^nodes\/([^\/]+)\/lxc\/([^\/]+)\/config$/
                   pve_node_name = $1
@@ -615,6 +643,10 @@ module HybridPlatformsConductorTest
                   raise "Unknown Proxmox API post call: #{path}. Please adapt the test framework."
                 end
               end
+              # Mock create_ticket
+              allow(proxmox).to receive(:create_ticket) do
+                @proxmox_actions << [:create_ticket]
+              end
               proxmox
             end
           end,
@@ -625,11 +657,36 @@ module HybridPlatformsConductorTest
       # Prepare a repository to test reserve_proxmox_container
       #
       # Parameters::
+      # * *leftovers* (Array<String>): List of leftover files among cgroups [default: []]
+      # * *expect_remaining_leftovers* (Array<String>): List of leftover files among cgroups that should remain after run [default: []]
       # * Proc: Code to be called with repository setup
-      def with_sync_node
+      def with_sync_node(leftovers: [], expect_remaining_leftovers: [])
         with_repository('sync_node') do |repository|
+          # Mock the cgroup file system of the sync node
+          remaining_leftovers = leftovers.clone
+          allow(Dir).to receive(:glob).and_wrap_original do |original_glob, dir, &block|
+            case dir
+            when '/sys/fs/cgroup/*/lxc/*'
+              block.nil? ? remaining_leftovers : remaining_leftovers.each(&block)
+            when /^\/sys\/fs\/cgroup\/\*\/lxc\/(.+)$/
+              vm_id_str = $1
+              file_pattern = /^\/sys\/fs\/cgroup\/.+\/lxc\/#{Regexp.escape(vm_id_str)}$/
+              matched_files = remaining_leftovers.select { |file| file =~ file_pattern }
+              block.nil? ? matched_files : matched_files.each(&block)
+            else
+              original_glob.call(dir, &block)
+            end
+          end
+          allow(FileUtils).to receive(:rm_rf).and_wrap_original do |original_rm_rf, path|
+            if path.start_with?('/sys/fs/cgroup')
+              remaining_leftovers.delete_if { |file| file.start_with?(path) }
+            else
+              original_rm_rf.call(path)
+            end
+          end
           @repository = repository
           yield
+          expect(remaining_leftovers.sort).to eq expect_remaining_leftovers.sort
         end
       end
 
@@ -642,14 +699,26 @@ module HybridPlatformsConductorTest
       # * *wait_before_retry* (Integer): Specify the number of seconds to wait before retry [default: 0]
       # * *create* (Hash or nil): Create file content, or nil if none [default: nil]
       # * *destroy* (Hash or nil): Destroy file content, or nil if none [default: nil]
+      # * *api_max_retries* (Integer): Max number of API retries [default: 3]
+      # * *api_wait_between_retries_secs* (Integer): Number of seconds to wait between API retries [default: 0]
       # Result::
       # * Hash: JSON result of the call
-      def call_reserve_proxmox_container_with(config: {}, max_retries: 1, wait_before_retry: 0, create: nil, destroy: nil)
+      def call_reserve_proxmox_container_with(
+        config: {},
+        max_retries: 1,
+        wait_before_retry: 0,
+        create: nil,
+        destroy: nil,
+        api_max_retries: 3,
+        api_wait_between_retries_secs: 0
+      )
         # Make sure we set default values in the config
         config = {
           proxmox_api_url: 'https://my-proxmox.my-domain.com:8006',
           futex_file: "#{@repository}/proxmox/allocations.futex",
           logs_dir: "#{Dir.tmpdir}/hpc_test_proxmox_waiter_logs",
+          api_max_retries: api_max_retries,
+          api_wait_between_retries_secs: api_wait_between_retries_secs,
           pve_nodes: ['pve_node_name'],
           vm_ips_list: %w[
             192.168.0.100
@@ -716,7 +785,14 @@ module HybridPlatformsConductorTest
       # * *wait_before_retry* (Integer): Specify the number of seconds to wait before retry [default: 0]
       # Result::
       # * Hash: JSON result of the call
-      def call_reserve_proxmox_container(cpus, ram_mb, disk_gb, config: {}, max_retries: 1, wait_before_retry: 0)
+      def call_reserve_proxmox_container(
+        cpus,
+        ram_mb,
+        disk_gb,
+        config: {},
+        max_retries: 1,
+        wait_before_retry: 0
+      )
         call_reserve_proxmox_container_with(
           config: config,
           max_retries: max_retries,
@@ -763,7 +839,13 @@ module HybridPlatformsConductorTest
       # Parameters::
       # * *expected_proxmox_actions* (Array<Array>): Expected Proxmox actions
       def expect_proxmox_actions_to_be(expected_proxmox_actions)
-        expect(@proxmox_actions.size).to eq expected_proxmox_actions.size
+        expect(@proxmox_actions.size).to eq(expected_proxmox_actions.size), <<~EOS
+          Expected #{expected_proxmox_actions.size} Proxmox actions, but got #{@proxmox_actions.size} instead:
+          ----- Received:
+          #{@proxmox_actions.map(&:inspect).join("\n")}
+          ----- Expected:
+          #{expected_proxmox_actions.map(&:inspect).join("\n")}
+        EOS
         @proxmox_actions.zip(expected_proxmox_actions).each do |proxmox_action, expected_proxmox_action|
           expect(proxmox_action.size).to eq expected_proxmox_action.size
           expect(proxmox_action[0..1]).to eq expected_proxmox_action[0..1]

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: hybrid_platforms_conductor
 version: !ruby/object:Gem::Version
-  version: 32.8.2
+  version: 32.11.1
 platform: ruby
 authors:
 - Muriel Salvan
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2021-03-15 00:00:00.000000000 Z
+date: 2021-04-05 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: range_operators
@@ -281,20 +281,20 @@ description: Provides a complete toolset to help DevOps maintain, deploy, monito
 email:
 - muriel@x-aeon.com
 executables:
-- topograph
-- test
-- free_veids
-- free_ips
-- nodes_to_deploy
-- last_deploys
-- check-node
 - run
-- report
 - get_impacted_nodes
-- ssh_config
+- dump_nodes_json
+- check-node
+- nodes_to_deploy
+- free_veids
+- free_ips
+- test
 - deploy
+- report
+- topograph
 - setup
-- dump_nodes_json
+- last_deploys
+- ssh_config
 extensions: []
 extra_rdoc_files: []
 files: