hybrid_platforms_conductor 32.9.0 → 32.11.2

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 378b6da76cab8e20f60c0517f7bbef90933b5d573069437ee16d913ca81c0689
-  data.tar.gz: 1924dd81e2740b50ee55758271282729d6df1fd6a45b555f96c321d1b40e54e3
+  metadata.gz: 2162d052fbbe0833d8a26d94d9fc8a490e9c57b0fd35ecd5656be8d3a52a2893
+  data.tar.gz: 40d6251c2dac02c7588ea158174ecb6da4f2381712e8696b5395fdfcb05555ce
 SHA512:
-  metadata.gz: f8a72778375c154e42829a430bec151a7553b79874d1a878e129d80eed22b0f77f094751ee370c84441e7ed658d9e7b87e80c8f950ba64b491e1e62eec1f4a2f
-  data.tar.gz: e437a3f4871bde44d6bcbcd8a588dd83ff33c0374cee3769eada500a3acf04606e37144f98d2a49aaf69e0ff8cdd935d3eaa396fea5bb19ac485184df883db24
+  metadata.gz: 572f8136afc8153bcb645c2612b128b7c6aeb2b44fc795fa1785f0dc6bad06298171466b37aef5d03ad38d6f49cb39cab3c268ce6edc0807cb125328e24c6cba
+  data.tar.gz: 993478c8a81af0e5bfce3a74cfe5ce9c8bb4259d0f6063e41912ea905d493a2e6cca801525d45ffd080ba1570d80e2c0097e56aac33ee694526e53e6ee9685ee

data/lib/hybrid_platforms_conductor/common_config_dsl/idempotence_tests.rb

@@ -11,13 +11,24 @@ module HybridPlatformsConductor
       # Array< Hash<Symbol, Object> >
       attr_reader :ignored_idempotence_tasks
 
-      # Initialize the DSL 
+      # List of ignored tasks info. Each info has the following properties:
+      # * *nodes_selectors_stack* (Array<Object>): Stack of nodes selectors impacted by this rule
+      # * *ignored_tasks* (Hash<String, String>): List of task names for which we ignore divergence errors, with the corresponding descriptive reason for ignore.
+      # Array< Hash<Symbol, Object> >
+      attr_reader :ignored_divergent_tasks
+
+      # Initialize the DSL
       def init_idempotence_tests
         # List of ignored tasks info. Each info has the following properties:
         # * *nodes_selectors_stack* (Array<Object>): Stack of nodes selectors impacted by this rule
         # * *ignored_tasks* (Hash<String, String>): List of task names for which we ignore idempotence errors, with the corresponding descriptive reason for ignore.
         # Array< Hash<Symbol, Object> >
         @ignored_idempotence_tasks = []
+        # List of ignored tasks info. Each info has the following properties:
+        # * *nodes_selectors_stack* (Array<Object>): Stack of nodes selectors impacted by this rule
+        # * *ignored_tasks* (Hash<String, String>): List of task names for which we ignore divergence errors, with the corresponding descriptive reason for ignore.
+        # Array< Hash<Symbol, Object> >
+        @ignored_divergent_tasks = []
       end
 
       # Ignore idempotence errors on a set of tasks
@@ -31,6 +42,17 @@ module HybridPlatformsConductor
         }
       end
 
+      # Ignore idempotence errors on a set of tasks
+      #
+      # Parameters::
+      # * *tasks_to_ignore* (Hash<String, String>): Set of tasks to ignore, along with the reason
+      def ignore_divergent_tasks(tasks_to_ignore)
+        @ignored_divergent_tasks << {
+          ignored_tasks: tasks_to_ignore,
+          nodes_selectors_stack: current_nodes_selectors_stack,
+        }
+      end
+
     end
 
   end

data/lib/hybrid_platforms_conductor/hpc_plugins/provisioner/proxmox.rb

@@ -18,11 +18,19 @@ module HybridPlatformsConductor
         attr_accessor *%i[logger logger_stderr]
 
         def check_response(response)
-          log_debug "Response from Proxmox API: #{response}"
-          log_warn "Response from Proxmox API: #{response}" if response.code >= 400 && !log_debug?
+          msg = "Response from Proxmox API: #{response} - #{response.net_http_res.message}"
+          log_debug msg
+          log_warn msg if response.code >= 400 && !log_debug?
           super
         end
 
+        # Re-authenticate the Proxmox instance
+        # This can be useful when the API returns errors due to invalidated tokens
+        def reauthenticate
+          log_debug 'Force re-authentication to Proxmox'
+          @auth_params = create_ticket
+        end
+
       end
       ::Proxmox::Proxmox.prepend ProxmoxPatches
 
@@ -309,6 +317,8 @@ module HybridPlatformsConductor
             log_warn "[ #{@node}/#{@environment} ] - Proxmox API call get #{path} returned error #{response} (attempt ##{idx_try}/#{proxmox_test_info[:api_max_retries]})"
             raise "[ #{@node}/#{@environment} ] - Proxmox API call get #{path} returns #{response} continuously (tried #{idx_try + 1} times)" if idx_try >= proxmox_test_info[:api_max_retries]
             idx_try += 1
+            # We have to reauthenticate: error 500 raised by Proxmox are often due to token being invalidated wrongly
+            proxmox.reauthenticate
             sleep proxmox_test_info[:api_wait_between_retries_secs] + rand(5)
           end
           response
@@ -333,6 +343,8 @@ module HybridPlatformsConductor
               task = nil
               break if idx_try >= proxmox_test_info[:api_max_retries]
               idx_try += 1
+              # We have to reauthenticate: error 500 raised by Proxmox are often due to token being invalidated wrongly
+              proxmox.reauthenticate
               sleep proxmox_test_info[:api_wait_between_retries_secs] + rand(5)
             end
           end

data/lib/hybrid_platforms_conductor/hpc_plugins/provisioner/proxmox/proxmox_waiter.rb

@@ -605,10 +605,35 @@ class ProxmoxWaiter
   # Result::
   # * Array<Integer>: List of available VM IDs
   def free_vm_ids
-    Range.new(*@config['vm_ids_range']).to_a -
+    vm_ids = Range.new(*@config['vm_ids_range']).to_a -
       api_get('nodes').map do |pve_node_info|
         api_get("nodes/#{pve_node_info['node']}/lxc").map { |lxc_info| Integer(lxc_info['vmid']) }
       end.flatten
+    # Make sure the vm_ids that are available don't have any leftovers in the cgroups.
+    # This can happen with some Proxmox bugs, and make the API returns 500 errors.
+    # cf. https://forum.proxmox.com/threads/lxc-console-cleanup-error.38293/
+    # TODO: Remove this when Proxmox will have solved the issue with leftovers of destroyed vms.
+    (vm_ids.map(&:to_s) & vm_ids_in_cgroups).each do |vm_id_str|
+      # We are having a vm_id that is available but still has some leftovers in cgroups.
+      # Clean those to avoid 500 errors in API.
+      log "Found VMID #{vm_id_str} with leftovers in cgroups. Cleaning those."
+      Dir.glob("/sys/fs/cgroup/*/lxc/#{vm_id_str}") do |cgroup_dir|
+        log "Removing #{cgroup_dir}"
+        FileUtils.rm_rf cgroup_dir
+      end
+    end
+    vm_ids
+  end
+
+  # Return the list of VM IDs present in cgroups
+  #
+  # Result::
+  # * Array<String>: List of VM IDs as strings (as some are not Integers like '1010-1')
+  def vm_ids_in_cgroups
+    Dir.glob('/sys/fs/cgroup/*/lxc/*').map do |file|
+      basename = File.basename(file)
+      basename =~ /^\d.+$/ ? basename : nil
+    end.compact.sort.uniq
   end
 
   # Wait for a given Proxmox task completion
@@ -653,6 +678,9 @@ class ProxmoxWaiter
         break unless @gets_cache[path].is_a?(String) && @gets_cache[path] =~ /^NOK: error code = 5\d\d$/
         raise "Proxmox API get #{path} returns #{@gets_cache[path]} continuously (tried #{idx_try + 1} times)" if idx_try >= @config['api_max_retries']
         idx_try += 1
+        # We have to reauthenticate: error 500 raised by Proxmox are often due to token being invalidated wrongly
+        # TODO: Provide a way to do it properly in the official gem
+        @proxmox.instance_variable_set(:@auth_params, @proxmox.send(:create_ticket))
         sleep @config['api_wait_between_retries_secs']
       end
     end

data/lib/hybrid_platforms_conductor/hpc_plugins/report/templates/confluence_inventory.html.erb

@@ -28,6 +28,6 @@
 </table>
 <p>
   <br/>
-  Generated at <%= Time.now.utc.strftime('%F %T') %> UTC by <a href="https://www.site.my_company.net/git/projects/PROJECTrepos/hybrid_platforms_conductor/browse">Hybrid Platforms Conductor</a>.
+  Generated at <%= Time.now.utc.strftime('%F %T') %> UTC by <a href="https://github.com/sweet-delights/hybrid-platforms-conductor">Hybrid Platforms Conductor</a>.
   <br/>
 </p>

data/lib/hybrid_platforms_conductor/hpc_plugins/test/check_deploy_and_idempotence.rb

@@ -81,7 +81,10 @@ module HybridPlatformsConductor
                   exit_status, stdout, stderr = deployer.deploy_on(@node)[@node]
                   assert_equal exit_status, 0, "Check-node after deployment returned error code #{exit_status}", log_debug? ? nil : deployer.stdouts_to_s
                   # Check that the output of the check-node returns no changes.
-                  ignored_tasks = @nodes_handler.select_confs_for_node(@node, @config.ignored_idempotence_tasks).inject({}) do |merged_ignored_tasks, conf|
+                  ignored_tasks = (
+                    @nodes_handler.select_confs_for_node(@node, @config.ignored_idempotence_tasks) +
+                      @nodes_handler.select_confs_for_node(@node, @config.ignored_divergent_tasks)
+                  ).inject({}) do |merged_ignored_tasks, conf|
                     merged_ignored_tasks.merge(conf[:ignored_tasks])
                   end
                   @deployer.parse_deploy_output(@node, stdout, stderr).each do |task_info|

data/lib/hybrid_platforms_conductor/hpc_plugins/test/divergence.rb

@@ -11,8 +11,23 @@ module HybridPlatformsConductor
 
         # Check my_test_plugin.rb.sample documentation for signature details.
         def test_on_check_node(stdout, stderr, exit_status)
+          # Check that the output of the check-node returns no changes.
+          ignored_tasks = @nodes_handler.select_confs_for_node(@node, @config.ignored_divergent_tasks).inject({}) do |merged_ignored_tasks, conf|
+            merged_ignored_tasks.merge(conf[:ignored_tasks])
+          end
           @deployer.parse_deploy_output(@node, stdout, stderr).each do |task_info|
-            error "Task #{task_info[:name]} has diverged", JSON.pretty_generate(task_info) if task_info[:status] == :changed
+            if task_info[:status] == :changed
+              if ignored_tasks.key?(task_info[:name])
+                # It was expected that this task is not idempotent
+                log_debug "Task #{task_info[:name]} was expected to be divergent. Reason: #{ignored_tasks[task_info[:name]]}"
+              else
+                extra_details = task_info.slice(*(task_info.keys - %i[name status diffs]))
+                error_details = []
+                error_details << "----- Changes:\n#{task_info[:diffs].strip}\n-----" if task_info[:diffs]
+                error_details << "----- Additional details:\n#{JSON.pretty_generate(extra_details)}\n-----" unless extra_details.empty?
+                error "Task #{task_info[:name]} has diverged", error_details.empty? ? nil : error_details.join("\n")
+              end
+            end
           end
         end
 

data/lib/hybrid_platforms_conductor/hpc_plugins/test/executables.rb

@@ -11,24 +11,38 @@ module HybridPlatformsConductor
 
         # Check my_test_plugin.rb.sample documentation for signature details.
         def test
-          example_platform = PlatformsHandler.new.known_platforms.first
-          example_node = example_platform.known_nodes.first
-        	[
-            "#{CmdRunner.executables_prefix}check-node --node #{example_node} --show-commands",
-            "#{CmdRunner.executables_prefix}deploy --node #{example_node} --show-commands --why-run",
+          tests = [
             "#{CmdRunner.executables_prefix}dump_nodes_json --help",
             "#{CmdRunner.executables_prefix}free_ips",
             "#{CmdRunner.executables_prefix}free_veids",
-            "#{CmdRunner.executables_prefix}get_impacted_nodes --platform #{example_platform.name} --show-commands",
-            "#{CmdRunner.executables_prefix}last_deploys --node #{example_node} --show-commands",
-            "#{CmdRunner.executables_prefix}nodes_to_deploy --node #{example_node} --show-commands",
-            "#{CmdRunner.executables_prefix}report --node #{example_node} --format stdout",
-            "#{CmdRunner.executables_prefix}run --node #{example_node} --show-commands --interactive",
             "#{CmdRunner.executables_prefix}setup --help",
             "#{CmdRunner.executables_prefix}ssh_config",
-            "#{CmdRunner.executables_prefix}test --help",
-            "#{CmdRunner.executables_prefix}topograph --from \"--node #{example_node}\" --to \"--node #{example_node}\" --skip-run --output graphviz:graph.gv"
-          ].each do |cmd|
+            "#{CmdRunner.executables_prefix}test --help"
+          ]
+          example_platform = PlatformsHandler.new(
+            logger: @logger,
+            logger_stderr: @logger_stderr,
+            config: @config,
+            cmd_runner: @cmd_runner
+          ).known_platforms.first
+          unless example_platform.nil?
+            tests.concat [
+              "#{CmdRunner.executables_prefix}get_impacted_nodes --platform #{example_platform.name} --show-commands",
+            ]
+            example_node = example_platform.known_nodes.first
+            unless example_node.nil?
+              tests.concat [
+                "#{CmdRunner.executables_prefix}check-node --node #{example_node} --show-commands",
+                "#{CmdRunner.executables_prefix}deploy --node #{example_node} --show-commands --why-run",
+                "#{CmdRunner.executables_prefix}last_deploys --node #{example_node} --show-commands",
+                "#{CmdRunner.executables_prefix}nodes_to_deploy --node #{example_node} --show-commands",
+                "#{CmdRunner.executables_prefix}report --node #{example_node} --format stdout",
+                "#{CmdRunner.executables_prefix}run --node #{example_node} --show-commands --interactive",
+                "#{CmdRunner.executables_prefix}topograph --from \"--node #{example_node}\" --to \"--node #{example_node}\" --skip-run --output graphviz:graph.gv"
+              ]
+            end
+          end
+        	tests.sort.each do |cmd|
             log_debug "Testing #{cmd}"
             exit_status, stdout, _stderr = @cmd_runner.run_cmd "#{cmd} 2>&1", no_exception: true, log_to_stdout: log_debug?
             assert_equal(exit_status, 0, "Command #{cmd} returned code #{exit_status}:\n#{stdout}")

data/lib/hybrid_platforms_conductor/hpc_plugins/test/idempotence.rb

@@ -43,7 +43,10 @@ module HybridPlatformsConductor
                   assert_equal tested_node, @node, "Wrong node being tested: #{tested_node} should be #{@node}"
                   assert_equal exit_status, 0, "Check-node returned error code #{exit_status}"
                   # Check that the output of the check-node returns no changes.
-                  ignored_tasks = @nodes_handler.select_confs_for_node(@node, @config.ignored_idempotence_tasks).inject({}) do |merged_ignored_tasks, conf|
+                  ignored_tasks = (
+                    @nodes_handler.select_confs_for_node(@node, @config.ignored_idempotence_tasks) +
+                      @nodes_handler.select_confs_for_node(@node, @config.ignored_divergent_tasks)
+                  ).inject({}) do |merged_ignored_tasks, conf|
                     merged_ignored_tasks.merge(conf[:ignored_tasks])
                   end
                   @deployer.parse_deploy_output(@node, stdout, stderr).each do |task_info|

data/lib/hybrid_platforms_conductor/hpc_plugins/test_report/templates/confluence.html.erb

@@ -237,6 +237,6 @@
 </table>
 <p>
   <br/>
-  Generated at <%= Time.now.utc.strftime('%F %T') %> UTC by <a href="https://www.site.my_company.net/git/projects/PROJECTrepos/hybrid_platforms_conductor/browse">Hybrid Platforms Conductor</a>.
+  Generated at <%= Time.now.utc.strftime('%F %T') %> UTC by <a href="https://github.com/sweet-delights/hybrid-platforms-conductor">Hybrid Platforms Conductor</a>.
   <br/>
 </p>

data/lib/hybrid_platforms_conductor/version.rb

@@ -1,5 +1,5 @@
 module HybridPlatformsConductor
 
-  VERSION = '32.9.0'
+  VERSION = '32.11.2'
 
 end

data/spec/hybrid_platforms_conductor_test/api/deployer/provisioners/proxmox/reserve_proxmox_container/retries_spec.rb

@@ -10,6 +10,7 @@ describe HybridPlatformsConductor::HpcPlugins::Provisioner::Proxmox do
         with_sync_node do
           mock_proxmox(mocked_pve_nodes: [{ 'pve_node_name' => {} }] * 5)
           expect(call_reserve_proxmox_container(2, 128 * 1024, 4, max_retries: 5)).to eq(error: 'not_enough_resources')
+          expect_proxmox_actions_to_be []
         end
       end
 
@@ -25,6 +26,23 @@ describe HybridPlatformsConductor::HpcPlugins::Provisioner::Proxmox do
             vm_id: 1000,
             vm_ip: '192.168.0.100'
           )
+          expect_proxmox_actions_to_be [
+            [
+              :post,
+              'nodes/pve_node_name/lxc',
+              {
+                'ostemplate' => 'test_template.iso',
+                'hostname' => 'test.hostname.my-domain.com',
+                'description' => /node: test_node\nenvironment: test_env/,
+                'cores' => 2,
+                'cpulimit' => 2,
+                'memory' => 1024,
+                'rootfs' => 'local-lvm:4',
+                'net0' => 'name=eth0,bridge=vmbr0,gw=172.16.16.16,ip=192.168.0.100/32',
+                'vmid' => 1000
+              }
+            ]
+          ]
         end
       end
 
@@ -34,6 +52,12 @@ describe HybridPlatformsConductor::HpcPlugins::Provisioner::Proxmox do
           result = call_reserve_proxmox_container(2, 1024, 4, config: { api_max_retries: 4 })
           expect(result[:error]).not_to eq nil
           expect(result[:error]).to match /Unhandled exception from reserve_proxmox_container: Proxmox API get nodes\/pve_node_name\/lxc returns NOK: error code = 500 continuously \(tried 5 times\)/
+          expect_proxmox_actions_to_be [
+            [:create_ticket],
+            [:create_ticket],
+            [:create_ticket],
+            [:create_ticket]
+          ]
         end
       end
 
@@ -45,6 +69,26 @@ describe HybridPlatformsConductor::HpcPlugins::Provisioner::Proxmox do
             vm_id: 1000,
             vm_ip: '192.168.0.100'
           )
+          expect_proxmox_actions_to_be [
+            [:create_ticket],
+            [:create_ticket],
+            [:create_ticket],
+            [
+              :post,
+              'nodes/pve_node_name/lxc',
+              {
+                'ostemplate' => 'test_template.iso',
+                'hostname' => 'test.hostname.my-domain.com',
+                'description' => /node: test_node\nenvironment: test_env/,
+                'cores' => 2,
+                'cpulimit' => 2,
+                'memory' => 1024,
+                'rootfs' => 'local-lvm:4',
+                'net0' => 'name=eth0,bridge=vmbr0,gw=172.16.16.16,ip=192.168.0.100/32',
+                'vmid' => 1000
+              }
+            ]
+          ]
         end
       end
 

data/spec/hybrid_platforms_conductor_test/api/deployer/provisioners/proxmox/reserve_proxmox_container/vm_ids_assignment_spec.rb

@@ -60,6 +60,98 @@ describe HybridPlatformsConductor::HpcPlugins::Provisioner::Proxmox do
         end
       end
 
+      it 'makes sure to remove cgroup files that are leftovers of removed containers' do
+        with_sync_node(leftovers: [
+          '/sys/fs/cgroup/memory/lxc/1003'
+        ]) do
+          mock_proxmox(mocked_pve_nodes: {
+            'pve_node_name' => {
+              lxc_containers: {
+                1000 => { ip: '192.168.1.100' },
+                1001 => { ip: '192.168.1.101' }
+              }
+            }
+          })
+          expect(call_reserve_proxmox_container(2, 1024, 1, config: { vm_ids_range: [1000, 1100] })).to eq(
+            pve_node: 'pve_node_name',
+            vm_id: 1002,
+            vm_ip: '192.168.0.100'
+          )
+        end
+      end
+
+      it 'makes sure to remove cgroup files that are leftovers of removed containers even when they are reusing the VM ID' do
+        with_sync_node(leftovers: [
+          '/sys/fs/cgroup/memory/lxc/1002'
+        ]) do
+          mock_proxmox(mocked_pve_nodes: {
+            'pve_node_name' => {
+              lxc_containers: {
+                1000 => { ip: '192.168.1.100' },
+                1001 => { ip: '192.168.1.101' }
+              }
+            }
+          })
+          expect(call_reserve_proxmox_container(2, 1024, 1, config: { vm_ids_range: [1000, 1100] })).to eq(
+            pve_node: 'pve_node_name',
+            vm_id: 1002,
+            vm_ip: '192.168.0.100'
+          )
+        end
+      end
+
+      it 'makes sure to remove cgroup files that are leftovers of removed containers when several cgroups contain files' do
+        with_sync_node(leftovers: [
+          '/sys/fs/cgroup/memory/lxc/1003',
+          '/sys/fs/cgroup/network/lxc/1003',
+          '/sys/fs/cgroup/cpu/lxc/1003'
+        ]) do
+          mock_proxmox(mocked_pve_nodes: {
+            'pve_node_name' => {
+              lxc_containers: {
+                1000 => { ip: '192.168.1.100' },
+                1001 => { ip: '192.168.1.101' }
+              }
+            }
+          })
+          expect(call_reserve_proxmox_container(2, 1024, 1, config: { vm_ids_range: [1000, 1100] })).to eq(
+            pve_node: 'pve_node_name',
+            vm_id: 1002,
+            vm_ip: '192.168.0.100'
+          )
+        end
+      end
+
+      it 'makes sure to remove only cgroup files that are leftovers of removed containers inside our VM ID range' do
+        with_sync_node(
+          leftovers: [
+            '/sys/fs/cgroup/memory/lxc/100',
+            '/sys/fs/cgroup/memory/lxc/1001',
+            '/sys/fs/cgroup/memory/lxc/1002',
+            '/sys/fs/cgroup/memory/lxc/1003'
+          ],
+          expect_remaining_leftovers: [
+            '/sys/fs/cgroup/memory/lxc/100',
+            '/sys/fs/cgroup/memory/lxc/1001'
+          ]
+        ) do
+          mock_proxmox(mocked_pve_nodes: {
+            'pve_node_name' => {
+              lxc_containers: {
+                100 => { ip: '192.168.1.10' },
+                1000 => { ip: '192.168.1.100' },
+                1001 => { ip: '192.168.1.101' }
+              }
+            }
+          })
+          expect(call_reserve_proxmox_container(2, 1024, 1, config: { vm_ids_range: [1000, 1100] })).to eq(
+            pve_node: 'pve_node_name',
+            vm_id: 1002,
+            vm_ip: '192.168.0.100'
+          )
+        end
+      end
+
     end
 
   end

data/spec/hybrid_platforms_conductor_test/api/deployer/provisioners/proxmox/state_spec.rb

@@ -23,20 +23,6 @@ describe HybridPlatformsConductor::HpcPlugins::Provisioner::Proxmox do
       end
     end
 
-
-    it '' do
-      with_test_proxmox_platform do |instance|
-        mock_proxmox_calls_with [
-          # 1 - The info on existing containers
-          mock_proxmox_to_get_nodes_info,
-          # 2 - The start of the container - fail a few times
-          mock_proxmox_to_start_node(nbr_api_errors: 2)
-        ]
-        instance.create
-        instance.start
-      end
-    end
-
     it 'retries calls to the API when getting back errors 5xx' do
       with_test_proxmox_platform do |instance|
         mock_proxmox_calls_with [

data/spec/hybrid_platforms_conductor_test/helpers/provisioner_proxmox_helpers.rb

@@ -156,6 +156,7 @@ module HybridPlatformsConductorTest
             idx_try += 1
             idx_try <= nbr_api_errors ? 'NOK: error code = 500' : task_name
           end
+          expect(proxmox).to receive(:reauthenticate).exactly(nbr_api_errors - (task_status.nil? ? 1 : 0)).times
           # Mock checking task status
           unless task_status.nil?
             # Mock checking task status
@@ -294,6 +295,7 @@ module HybridPlatformsConductorTest
               ]
             end
           end
+          expect(proxmox).to receive(:reauthenticate).exactly(nbr_api_errors - (status.nil? ? 1 : 0)).times
           unless status.nil?
             expect(proxmox).to receive(:get).with('nodes/pve_node_name/lxc/1024/status/current') do
               {
@@ -641,6 +643,10 @@ module HybridPlatformsConductorTest
                   raise "Unknown Proxmox API post call: #{path}. Please adapt the test framework."
                 end
               end
+              # Mock create_ticket
+              allow(proxmox).to receive(:create_ticket) do
+                @proxmox_actions << [:create_ticket]
+              end
               proxmox
             end
           end,
@@ -651,11 +657,36 @@ module HybridPlatformsConductorTest
       # Prepare a repository to test reserve_proxmox_container
       #
       # Parameters::
+      # * *leftovers* (Array<String>): List of leftover files among cgroups [default: []]
+      # * *expect_remaining_leftovers* (Array<String>): List of leftover files among cgroups that should remain after run [default: []]
       # * Proc: Code to be called with repository setup
-      def with_sync_node
+      def with_sync_node(leftovers: [], expect_remaining_leftovers: [])
         with_repository('sync_node') do |repository|
+          # Mock the cgroup file system of the sync node
+          remaining_leftovers = leftovers.clone
+          allow(Dir).to receive(:glob).and_wrap_original do |original_glob, dir, &block|
+            case dir
+            when '/sys/fs/cgroup/*/lxc/*'
+              block.nil? ? remaining_leftovers : remaining_leftovers.each(&block)
+            when /^\/sys\/fs\/cgroup\/\*\/lxc\/(.+)$/
+              vm_id_str = $1
+              file_pattern = /^\/sys\/fs\/cgroup\/.+\/lxc\/#{Regexp.escape(vm_id_str)}$/
+              matched_files = remaining_leftovers.select { |file| file =~ file_pattern }
+              block.nil? ? matched_files : matched_files.each(&block)
+            else
+              original_glob.call(dir, &block)
+            end
+          end
+          allow(FileUtils).to receive(:rm_rf).and_wrap_original do |original_rm_rf, path|
+            if path.start_with?('/sys/fs/cgroup')
+              remaining_leftovers.delete_if { |file| file.start_with?(path) }
+            else
+              original_rm_rf.call(path)
+            end
+          end
           @repository = repository
           yield
+          expect(remaining_leftovers.sort).to eq expect_remaining_leftovers.sort
         end
       end
 
@@ -808,7 +839,13 @@ module HybridPlatformsConductorTest
       # Parameters::
       # * *expected_proxmox_actions* (Array<Array>): Expected Proxmox actions
       def expect_proxmox_actions_to_be(expected_proxmox_actions)
-        expect(@proxmox_actions.size).to eq expected_proxmox_actions.size
+        expect(@proxmox_actions.size).to eq(expected_proxmox_actions.size), <<~EOS
+          Expected #{expected_proxmox_actions.size} Proxmox actions, but got #{@proxmox_actions.size} instead:
+          ----- Received:
+          #{@proxmox_actions.map(&:inspect).join("\n")}
+          ----- Expected:
+          #{expected_proxmox_actions.map(&:inspect).join("\n")}
+        EOS
         @proxmox_actions.zip(expected_proxmox_actions).each do |proxmox_action, expected_proxmox_action|
           expect(proxmox_action.size).to eq expected_proxmox_action.size
           expect(proxmox_action[0..1]).to eq expected_proxmox_action[0..1]

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: hybrid_platforms_conductor
 version: !ruby/object:Gem::Version
-  version: 32.9.0
+  version: 32.11.2
 platform: ruby
 authors:
 - Muriel Salvan
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2021-03-22 00:00:00.000000000 Z
+date: 2021-04-05 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: range_operators
@@ -281,20 +281,20 @@ description: Provides a complete toolset to help DevOps maintain, deploy, monito
 email:
 - muriel@x-aeon.com
 executables:
-- topograph
-- test
-- free_veids
-- free_ips
-- nodes_to_deploy
-- last_deploys
-- check-node
 - run
-- report
 - get_impacted_nodes
-- ssh_config
+- dump_nodes_json
+- check-node
+- nodes_to_deploy
+- free_veids
+- free_ips
+- test
 - deploy
+- report
+- topograph
 - setup
-- dump_nodes_json
+- last_deploys
+- ssh_config
 extensions: []
 extra_rdoc_files: []
 files: