hybrid_platforms_conductor 32.9.1 → 32.10.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 4f1beb4de64ac4dcdc1de3e5063980ea4dab1b8ceef204baae0a9f4bf209f1f3
-  data.tar.gz: 0c5e65553da39646554d737c8a690939497a8c89f602a4eae01a4987fff994b7
+  metadata.gz: 7d350aa81076d2c5a6837b7184bf903640fb0be35aae2b22f596b870b56a53b8
+  data.tar.gz: d75bebd5e1f6b12b30ea6f62c01c835bc9f89fba9ed3a208b3fd2dd0bc021328
 SHA512:
-  metadata.gz: 3129d3b37140c6488b2b6a8b6c1c8cb22ce745181a643925d9bb2fdc77de97a331660c8d5193dde940535d58b545576796d22a29008d181c73bbf7bb384c2251
-  data.tar.gz: 187c6215c37e89b924693c1819fea12f8cc444858ac575f9445b535ae9866a396b01dbc5baaf85ca4e8163d60c8865c931eca319c4824148fdd06029920b9710
+  metadata.gz: 37c23ac804c359f5875cb9f77f3e639b1711dc967ae664554d50b6e8aeefe26f59385a786c8ff31f56f5a3fc9eb55aaf65cfdc2946fa4c7e8b4b4372fa1ac3e0
+  data.tar.gz: a558f9c2b88b4a69110a870ba27549315823f2fab6c91c61068f3be1b10c7f2482d18b33883b0762d9cf2ac787fe61616b81d08535b1bc5d352e29e914a0519c

data/lib/hybrid_platforms_conductor/hpc_plugins/provisioner/proxmox.rb

@@ -18,15 +18,16 @@ module HybridPlatformsConductor
         attr_accessor *%i[logger logger_stderr]
 
         def check_response(response)
-          log_debug "Response from Proxmox API: #{response}"
-          log_warn "Response from Proxmox API: #{response}" if response.code >= 400 && !log_debug?
+          msg = "Response from Proxmox API: #{response} - #{response.net_http_res.message}"
+          log_debug msg
+          log_warn msg if response.code >= 400 && !log_debug?
           super
         end
 
         # Re-authenticate the Proxmox instance
         # This can be useful when the API returns errors due to invalidated tokens
         def reauthenticate
-          log_info 'Force re-authentication to Proxmox'
+          log_debug 'Force re-authentication to Proxmox'
           @auth_params = create_ticket
         end
 

data/lib/hybrid_platforms_conductor/hpc_plugins/provisioner/proxmox/proxmox_waiter.rb

@@ -605,10 +605,35 @@ class ProxmoxWaiter
   # Result::
   # * Array<Integer>: List of available VM IDs
   def free_vm_ids
-    Range.new(*@config['vm_ids_range']).to_a -
+    vm_ids = Range.new(*@config['vm_ids_range']).to_a -
       api_get('nodes').map do |pve_node_info|
         api_get("nodes/#{pve_node_info['node']}/lxc").map { |lxc_info| Integer(lxc_info['vmid']) }
       end.flatten
+    # Make sure the vm_ids that are available don't have any leftovers in the cgroups.
+    # This can happen with some Proxmox bugs, and make the API returns 500 errors.
+    # cf. https://forum.proxmox.com/threads/lxc-console-cleanup-error.38293/
+    # TODO: Remove this when Proxmox will have solved the issue with leftovers of destroyed vms.
+    (vm_ids.map(&:to_s) & vm_ids_in_cgroups).each do |vm_id_str|
+      # We are having a vm_id that is available but still has some leftovers in cgroups.
+      # Clean those to avoid 500 errors in API.
+      log "Found VMID #{vm_id_str} with leftovers in cgroups. Cleaning those."
+      Dir.glob("/sys/fs/cgroup/*/lxc/#{vm_id_str}") do |cgroup_dir|
+        log "Removing #{cgroup_dir}"
+        FileUtils.rm_rf cgroup_dir
+      end
+    end
+    vm_ids
+  end
+
+  # Return the list of VM IDs present in cgroups
+  #
+  # Result::
+  # * Array<String>: List of VM IDs as strings (as some are not Integers like '1010-1')
+  def vm_ids_in_cgroups
+    Dir.glob('/sys/fs/cgroup/*/lxc/*').map do |file|
+      basename = File.basename(file)
+      basename =~ /^\d.+$/ ? basename : nil
+    end.compact.sort.uniq
   end
 
   # Wait for a given Proxmox task completion

data/lib/hybrid_platforms_conductor/version.rb

@@ -1,5 +1,5 @@
 module HybridPlatformsConductor
 
-  VERSION = '32.9.1'
+  VERSION = '32.10.0'
 
 end

data/spec/hybrid_platforms_conductor_test/api/deployer/provisioners/proxmox/reserve_proxmox_container/vm_ids_assignment_spec.rb

@@ -60,6 +60,98 @@ describe HybridPlatformsConductor::HpcPlugins::Provisioner::Proxmox do
         end
       end
 
+      it 'makes sure to remove cgroup files that are leftovers of removed containers' do
+        with_sync_node(leftovers: [
+          '/sys/fs/cgroup/memory/lxc/1003'
+        ]) do
+          mock_proxmox(mocked_pve_nodes: {
+            'pve_node_name' => {
+              lxc_containers: {
+                1000 => { ip: '192.168.1.100' },
+                1001 => { ip: '192.168.1.101' }
+              }
+            }
+          })
+          expect(call_reserve_proxmox_container(2, 1024, 1, config: { vm_ids_range: [1000, 1100] })).to eq(
+            pve_node: 'pve_node_name',
+            vm_id: 1002,
+            vm_ip: '192.168.0.100'
+          )
+        end
+      end
+
+      it 'makes sure to remove cgroup files that are leftovers of removed containers even when they are reusing the VM ID' do
+        with_sync_node(leftovers: [
+          '/sys/fs/cgroup/memory/lxc/1002'
+        ]) do
+          mock_proxmox(mocked_pve_nodes: {
+            'pve_node_name' => {
+              lxc_containers: {
+                1000 => { ip: '192.168.1.100' },
+                1001 => { ip: '192.168.1.101' }
+              }
+            }
+          })
+          expect(call_reserve_proxmox_container(2, 1024, 1, config: { vm_ids_range: [1000, 1100] })).to eq(
+            pve_node: 'pve_node_name',
+            vm_id: 1002,
+            vm_ip: '192.168.0.100'
+          )
+        end
+      end
+
+      it 'makes sure to remove cgroup files that are leftovers of removed containers when several cgroups contain files' do
+        with_sync_node(leftovers: [
+          '/sys/fs/cgroup/memory/lxc/1003',
+          '/sys/fs/cgroup/network/lxc/1003',
+          '/sys/fs/cgroup/cpu/lxc/1003'
+        ]) do
+          mock_proxmox(mocked_pve_nodes: {
+            'pve_node_name' => {
+              lxc_containers: {
+                1000 => { ip: '192.168.1.100' },
+                1001 => { ip: '192.168.1.101' }
+              }
+            }
+          })
+          expect(call_reserve_proxmox_container(2, 1024, 1, config: { vm_ids_range: [1000, 1100] })).to eq(
+            pve_node: 'pve_node_name',
+            vm_id: 1002,
+            vm_ip: '192.168.0.100'
+          )
+        end
+      end
+
+      it 'makes sure to remove only cgroup files that are leftovers of removed containers inside our VM ID range' do
+        with_sync_node(
+          leftovers: [
+            '/sys/fs/cgroup/memory/lxc/100',
+            '/sys/fs/cgroup/memory/lxc/1001',
+            '/sys/fs/cgroup/memory/lxc/1002',
+            '/sys/fs/cgroup/memory/lxc/1003'
+          ],
+          expect_remaining_leftovers: [
+            '/sys/fs/cgroup/memory/lxc/100',
+            '/sys/fs/cgroup/memory/lxc/1001'
+          ]
+        ) do
+          mock_proxmox(mocked_pve_nodes: {
+            'pve_node_name' => {
+              lxc_containers: {
+                100 => { ip: '192.168.1.10' },
+                1000 => { ip: '192.168.1.100' },
+                1001 => { ip: '192.168.1.101' }
+              }
+            }
+          })
+          expect(call_reserve_proxmox_container(2, 1024, 1, config: { vm_ids_range: [1000, 1100] })).to eq(
+            pve_node: 'pve_node_name',
+            vm_id: 1002,
+            vm_ip: '192.168.0.100'
+          )
+        end
+      end
+
     end
 
   end

data/spec/hybrid_platforms_conductor_test/helpers/provisioner_proxmox_helpers.rb

@@ -657,11 +657,36 @@ module HybridPlatformsConductorTest
       # Prepare a repository to test reserve_proxmox_container
       #
       # Parameters::
+      # * *leftovers* (Array<String>): List of leftover files among cgroups [default: []]
+      # * *expect_remaining_leftovers* (Array<String>): List of leftover files among cgroups that should remain after run [default: []]
       # * Proc: Code to be called with repository setup
-      def with_sync_node
+      def with_sync_node(leftovers: [], expect_remaining_leftovers: [])
         with_repository('sync_node') do |repository|
+          # Mock the cgroup file system of the sync node
+          remaining_leftovers = leftovers.clone
+          allow(Dir).to receive(:glob).and_wrap_original do |original_glob, dir, &block|
+            case dir
+            when '/sys/fs/cgroup/*/lxc/*'
+              block.nil? ? remaining_leftovers : remaining_leftovers.each(&block)
+            when /^\/sys\/fs\/cgroup\/\*\/lxc\/(.+)$/
+              vm_id_str = $1
+              file_pattern = /^\/sys\/fs\/cgroup\/.+\/lxc\/#{Regexp.escape(vm_id_str)}$/
+              matched_files = remaining_leftovers.select { |file| file =~ file_pattern }
+              block.nil? ? matched_files : matched_files.each(&block)
+            else
+              original_glob.call(dir, &block)
+            end
+          end
+          allow(FileUtils).to receive(:rm_rf).and_wrap_original do |original_rm_rf, path|
+            if path.start_with?('/sys/fs/cgroup')
+              remaining_leftovers.delete_if { |file| file.start_with?(path) }
+            else
+              original_rm_rf.call(path)
+            end
+          end
           @repository = repository
           yield
+          expect(remaining_leftovers.sort).to eq expect_remaining_leftovers.sort
         end
       end
 

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: hybrid_platforms_conductor
 version: !ruby/object:Gem::Version
-  version: 32.9.1
+  version: 32.10.0
 platform: ruby
 authors:
 - Muriel Salvan
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2021-03-23 00:00:00.000000000 Z
+date: 2021-03-24 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: range_operators
@@ -281,20 +281,20 @@ description: Provides a complete toolset to help DevOps maintain, deploy, monito
 email:
 - muriel@x-aeon.com
 executables:
-- topograph
-- test
+- setup
 - free_veids
-- free_ips
-- nodes_to_deploy
+- dump_nodes_json
+- topograph
 - last_deploys
-- check-node
-- run
 - report
 - get_impacted_nodes
 - ssh_config
+- test
+- nodes_to_deploy
+- free_ips
+- check-node
 - deploy
-- setup
-- dump_nodes_json
+- run
 extensions: []
 extra_rdoc_files: []
 files: