hybrid_platforms_conductor 32.3.6

data/lib/hybrid_platforms_conductor/hpc_plugins/provisioner/proxmox/reserve_proxmox_container

@@ -0,0 +1,122 @@
+#!/usr/bin/env ruby
+# This scripts creates and destroys containers in an atomic way in a Proxmox cluster.
+# It handles:
+# * An intelligent resources allocation algorithm to balance usage among cluster PVE nodes.
+# * A retry mechanism if the resources are fully reserved.
+# * An expiration mechanism that removes expired containers before assigning new ones to protect resources.
+# It guarantees that:
+# * Any IP and VM ID reserved through this script will not be reserved by another concurrent process (unless the container becomes expired).
+# * If creation is granted, creating this container will not exceed resources limits specified in the config.
+# * The returned reserved VM ID and IP are already created.
+
+# Usage: ./reserve_proxmox_container [options]
+#     -c, --create FILE                Create a VM with the configuration stored in a given JSON file
+#     -d, --destroy FILE               Destroy a VM with the configuration stored in a given JSON file
+#     -h, --help                       Display help and exit
+#     -m, --max-retries NBR            Set the maximum number of retries to attempt (default: 240)
+#     -o, --config FILE                Set the path to the file storing configuration (default: ./config.json)
+#     -w, --wait-before-retry SECS     Set the number of seconds to wait between retries (default: 4)
+
+# Result:
+# There can be some logs but the script ends with a line containing '===== JSON =====' followed by parseable JSON.
+# Resulting JSON has the following structure in case of success:
+# * *pve_node* (String): Name of the pve node on which the creation has been reserved
+# * *vm_id* (Integer): VM ID reserved
+# * *vm_ip* (String): VM IP reserved
+# Resulting JSON has the following structure in case of error:
+# * *error* (String): The error message
+
+# Uses:
+# * A config file (given through --config option) that describes the available resources as JSON.
+#   Check ProxmoxWaiter#initialize signature for the structure's description of this file.
+# * The environment variables:
+#   * *hpc_user_for_proxmox*: User to be used to query Proxmox API
+#   * *hpc_password_for_proxmox*: Password to be used to query Proxmox API
+#   * *hpc_realm_for_proxmox*: Realm used to connect to the Proxmox API [default = 'pam']
+
+require 'json'
+
+reserved_resource = nil
+
+begin
+
+  # As this script is meant to be run in an SSH session, make sure we automatically flush stdout/stderr.
+  # Otherwise it will wait for the script completion before dumping everything, which is annoying as we want to track when it waits for hours.
+  $stdout.sync = true
+  $stderr.sync = true
+
+  require 'optparse'
+  require 'time'
+  require 'tmpdir'
+  require "#{__dir__}/proxmox_waiter"
+
+  # Define here default values
+  create_info = nil
+  destroy_info = nil
+  wait_time_before_retry_secs = 60
+  # Don't hesitate to retry for a few hours, as resources are taken by tests that can run for a few hours sometimes.
+  max_retries = 240
+  config_file = "#{__dir__}/config.json"
+
+  OptionParser.new do |opts|
+    opts.banner = "Usage: #{$0} [options]"
+    opts.on('-c', '--create FILE', 'Create a VM with the configuration stored in a given JSON file') do |file|
+      create_info = JSON.parse(File.read(file))
+    end
+    opts.on('-d', '--destroy FILE', 'Destroy a VM with the configuration stored in a given JSON file') do |file|
+      destroy_info = JSON.parse(File.read(file))
+    end
+    opts.on('-h', '--help', 'Display help and exit') do
+      puts opts
+      exit 0
+    end
+    opts.on('-m', '--max-retries NBR', "Set the maximum number of retries to attempt (default: #{max_retries})") do |nbr|
+      max_retries = Integer(nbr)
+    end
+    opts.on('-o', '--config FILE', "Set the path to the file storing configuration (default: #{config_file})") do |file|
+      config_file = file
+    end
+    opts.on('-w', '--wait-before-retry SECS', "Set the number of seconds to wait between retries (default: #{wait_time_before_retry_secs})") do |secs|
+      wait_time_before_retry_secs = Integer(secs)
+    end
+  end.parse!
+
+  raise 'Either use --create or --destroy options. Not both.' if create_info && destroy_info
+
+  proxmox_waiter = ProxmoxWaiter.new(
+    config_file,
+    ENV['hpc_user_for_proxmox'],
+    ENV['hpc_password_for_proxmox'],
+    ENV['hpc_realm_for_proxmox'] || 'pam'
+  )
+
+  idx_retry = 0
+  while idx_retry < max_retries
+    reserved_resource =
+      if destroy_info.nil?
+        puts "[ #{Time.now.utc.strftime('%F %T')} ] - Create #{create_info} in a Proxmox container (attempt ##{idx_retry}/#{max_retries})"
+        proxmox_waiter.create(create_info)
+      else
+        puts "[ #{Time.now.utc.strftime('%F %T')} ] - Destroy #{destroy_info} (attempt ##{idx_retry}/#{max_retries})"
+        proxmox_waiter.destroy(destroy_info)
+      end
+    if reserved_resource.is_a?(Symbol)
+      # We got an error (missing resources, IPs...).
+      # Try again.
+      puts "Could not perform the operation because of error: #{reserved_resource}. Waiting #{wait_time_before_retry_secs} seconds before trying again."
+      reserved_resource = { error: reserved_resource }
+      sleep wait_time_before_retry_secs
+    else
+      break
+    end
+    idx_retry += 1
+  end
+
+rescue
+  reserved_resource = {
+    error: "Unhandled exception from reserve_proxmox_container: #{$!}\n#{$!.backtrace.join("\n")}"
+  }
+end
+
+puts '===== JSON ====='
+puts reserved_resource.to_json

data/lib/hybrid_platforms_conductor/hpc_plugins/report/confluence.rb

@@ -0,0 +1,69 @@
+require 'erubis'
+require 'hybrid_platforms_conductor/report'
+require 'hybrid_platforms_conductor/confluence'
+require 'hybrid_platforms_conductor/common_config_dsl/confluence'
+
+module HybridPlatformsConductor
+
+  module HpcPlugins
+
+    module Report
+
+      # Export in the Mediawiki format
+      class Confluence < HybridPlatformsConductor::Report
+
+        self.extend_config_dsl_with CommonConfigDsl::Confluence, :init_confluence
+
+        # Give the list of supported locales by this report generator
+        # [API] - This method is mandatory.
+        #
+        # Result::
+        # * Array<Symbol>: List of supported locales
+        def self.supported_locales
+          [:en]
+        end
+
+        # Create a report for a list of nodes, in a given locale
+        # [API] - This method is mandatory.
+        #
+        # Parameters::
+        # * *nodes* (Array<String>): List of nodes
+        # * *locale_code* (Symbol): The locale code
+        def report_for(nodes, locale_code)
+          confluence_info = @config.confluence_info
+          if confluence_info
+            if confluence_info[:inventory_report_page_id]
+              @nodes = nodes
+              HybridPlatformsConductor::Confluence.with_confluence(confluence_info[:url], @logger, @logger_stderr) do |confluence|
+                confluence.update_page(confluence_info[:inventory_report_page_id], render('confluence_inventory'))
+              end
+              out "Inventory report Confluence page updated. Please visit #{confluence_info[:url]}/pages/viewpage.action?pageId=#{confluence_info[:inventory_report_page_id]}"
+            else
+              log_warn 'No inventory_report_page_id in the Confluence information defined. Ignoring the Confluence report.'
+            end
+          else
+            log_warn 'No Confluence information defined. Ignoring the Confluence report.'
+          end
+        end
+
+        private
+
+        TEMPLATES_PATH = File.expand_path("#{File.dirname(__FILE__)}/templates")
+
+        # Render a given ERB template into a String
+        #
+        # Parameters::
+        # * *template* (String): Template name
+        # Result::
+        # * String: Rendered template
+        def render(template)
+          Erubis::Eruby.new(File.read("#{TEMPLATES_PATH}/#{template}.html.erb")).result(binding)
+        end
+
+      end
+
+    end
+
+  end
+
+end

data/lib/hybrid_platforms_conductor/hpc_plugins/report/mediawiki.rb

@@ -0,0 +1,164 @@
+require 'hybrid_platforms_conductor/report'
+require 'time'
+
+module HybridPlatformsConductor
+
+  module HpcPlugins
+
+    module Report
+
+      # Export in the Mediawiki format
+      class Mediawiki < HybridPlatformsConductor::Report
+
+        TRANSLATIONS = {
+          en: {
+            alias: 'Alias',
+            comment: 'Comment',
+            daily_backup: 'Daily backup',
+            direct_deploy: 'Direct deployment',
+            encrypted_storage: 'Encrypted storage',
+            failover_ips: 'Failover IPs',
+            false: 'No',
+            fqdn: 'FQDN',
+            geom_mirror: 'GEOM mirror',
+            gui: 'GUI',
+            handled_by_chef: 'Handled by Chef',
+            hosted_on: 'Hosted on',
+            hostname: 'Hostname',
+            image: 'Image',
+            kernel: 'Kernel',
+            location: 'Location',
+            missing_chef_recipes: 'Missing Chef recipes',
+            missing_industrialization: 'Missing industrialization',
+            non_encrypted_storage: 'Non-encrypted storage',
+            openvz_specs: 'OpenVZ specifications',
+            os: 'OS',
+            xae_ip: 'XAE IP',
+            xae_location: 'XAE location',
+            xae_physical_ref: 'XAE physical reference',
+            xae_reference: 'XAE reference',
+            private_ips: 'Private IPs',
+            product: 'Product',
+            public_ips: 'Public IPs',
+            raid: 'RAID setup',
+            ripe_ips: 'RIPE IPs',
+            server_type: 'Server type',
+            'sub-product': 'Sub-Product',
+            true: 'Yes',
+            unattended_upgrades: 'Unattended upgrades',
+            veid: 'VEID',
+            vlan: 'VLAN',
+            vrack: 'VRack'
+          }
+        }
+
+        # Give the list of supported locales by this report generator
+        # [API] - This method is mandatory.
+        #
+        # Result::
+        # * Array<Symbol>: List of supported locales
+        def self.supported_locales
+          TRANSLATIONS.keys
+        end
+
+        # Create a report for a list of nodes, in a given locale
+        # [API] - This method is mandatory.
+        #
+        # Parameters::
+        # * *nodes* (Array<String>): List of nodes
+        # * *locale_code* (Symbol): The locale code
+        def report_for(nodes, locale_code)
+          output = ''
+          locale = TRANSLATIONS[locale_code]
+
+          output << <<~EOS
+            Back to the [[Hadoop]] / [[Impala]] / [[XAE_Network_Topology]] portal pages
+
+            This page has been generated using <code>./bin/report --format mediawiki</code> on #{Time.now.utc.strftime('%F %T')} UTC.
+
+          EOS
+
+          # Get all confs
+          # Use the translations' keys to know all properties we want to display
+          all_properties = (%i[physical_node cluster private_ips description] + locale.keys).uniq
+          @nodes_handler.prefetch_metadata_of nodes, locale.keys
+          nodes.
+            map do |node|
+              { node: node }.merge(Hash[all_properties.map { |property| [property, @nodes_handler.metadata_of(node, property)] }])
+            end.
+            # Group them by physical / VMs
+            group_by do |node_info|
+              # Consume the info to not display it again later
+              physical_node = node_info.delete(:physical_node)
+              !physical_node.nil? && physical_node
+            end.
+            each do |physical, nodes_for_physical|
+              output << "= #{physical ? 'Physical' : 'Virtual'} nodes =\n\n"
+              # Group them by location
+              nodes_for_physical.
+                group_by do |node_info|
+                  # Consume the info to not display it again later
+                  cluster = node_info.delete(:cluster)
+                  cluster.nil? ? '' : cluster
+                end.
+                sort.
+                each do |cluster, nodes_for_cluster|
+                  output << "== #{cluster.empty? ? 'Independent nodes' : "Belonging to cluster #{cluster}"} ==\n\n"
+                  # Group them by IP range (24 bits)
+                  nodes_for_cluster.
+                    group_by { |node_info| node_info[:private_ips].nil? || node_info[:private_ips].empty? ? [] : node_info[:private_ips].first.split('.')[0..2].map(&:to_i) }.
+                    sort.
+                    each do |ip_range, nodes_for_ip_range|
+                      output << "=== #{ip_range.empty? ? 'No IP' : "#{ip_range.join('.')}/24"} ===\n\n"
+                      nodes_for_ip_range.
+                        sort_by { |node_info| node_info[:node] }.
+                        each do |node_info|
+                          output << "* '''#{node_info.delete(:node)}'''#{node_info[:private_ips].nil? || node_info[:private_ips].empty? ? '' : " - #{node_info[:private_ips].first}"} - #{node_info.delete(:description)}\n"
+                          node_info.delete(:private_ips) if !node_info[:private_ips].nil? && node_info[:private_ips].size == 1
+                          node_info.sort.each do |property, value|
+                            unless value.nil?
+                              raise "Missing translation of key: #{property}. Please edit TRANSLATIONS[:#{locale_code}]." unless locale.key?(property)
+                              formatted_value =
+                                if value.is_a?(Array)
+                                  "\n#{value.map { |item| "::* #{item}" }.join("\n")}"
+                                elsif value.is_a?(Hash)
+                                  "\n#{value.map { |item, value| "::* #{item}: #{value}" }.join("\n")}"
+                                elsif value.is_a?(TrueClass)
+                                  locale[:true]
+                                elsif value.is_a?(FalseClass)
+                                  locale[:false]
+                                else
+                                  value.to_str
+                                end
+                              output << ": #{locale[property]}: #{formatted_value}\n"
+                            end
+                          end
+                          output << "\n\n"
+                        end
+                    end
+                end
+            end
+
+          output << <<~EOS
+            Back to the [[Hadoop]] / [[Impala]] / [[XAE_Network_Topology]] portal pages
+
+            [[Category:My Project]]
+            [[Category:Hadoop]]
+            [[Category:NoSQL]]
+            [[Category:Hosting]]
+            [[Category:XAE]]
+            [[Category:Server]]
+            [[Category:Configuration]]
+            [[Category:Chef]]
+          EOS
+
+          out output
+        end
+
+      end
+
+    end
+
+  end
+
+end

data/lib/hybrid_platforms_conductor/hpc_plugins/report/my_report_plugin.rb.sample

@@ -0,0 +1,88 @@
+# This file is an example of a Reports plugin that can be used to dump information about the platforms.
+# The MyReportPlugin example contains example of code that could be used to write a plugin for a new kind of report.
+require 'hybrid_platforms_conductor/report'
+
+module HybridPlatformsConductor
+
+  module HpcPlugins
+
+    module Report
+
+      # Brief description of the plugin here.
+      # The name should match the file name.
+      # It should always inherit from ReportPlugin.
+      # In all instance methods of a platform handler, the following variables are accessible:
+      # * @nodes_handler (NodesHandler): Nodes handler used to access information about the platforms
+      class MyReportPlugin < HybridPlatformsConductor::Report
+
+        # Are dependencies met before using this plugin?
+        # [API] - This method is optional
+        #
+        # Result::
+        # * Boolean: Are dependencies met before using this plugin?
+        def self.valid?
+          true
+        end
+
+        # Add a Mixin to the DSL parsing the platforms configuration file.
+        # This can be used by any plugin to add plugin-specific configuration getters and setters, accessible later from NodesHandler instances.
+        # An optional initializer can also be given.
+        # [API] - Those calls are optional
+        module MyDSLExtension
+
+          attr_accessor :my_property
+
+          # Initialize the DSL
+          def init_my_dsl_extension
+            @my_property = 42
+          end
+
+        end
+        self.extend_config_dsl_with MyDSLExtension, :init_my_dsl_extension
+
+        # Give the list of supported locales by this report generator
+        # [API] - This method is mandatory.
+        #
+        # Result::
+        # * Array<Symbol>: List of supported locales
+        def self.supported_locales
+          # This method has to publish the list of translations it accepts.
+          [
+            :fr,
+            :en
+          ]
+        end
+
+        # Create a report for a list of nodes, in a given locale
+        # [API] - This method is mandatory.
+        #
+        # Parameters::
+        # * *nodes* (Array<String>): List of nodes
+        # * *locale_code* (Symbol): The locale code
+        def report_for(nodes, locale_code)
+          # This method simply provides a report for a given list of nodes in the desired locale.
+          # The locale will be one of the supported ones.
+          output = ''
+          case locale_code
+          when :fr
+            output << 'Un beau rapport'
+          when :en
+            output << 'A nice report'
+          end
+          @platforms_handler.known_platforms.each do |platform|
+            output << "= Report for platform #{platform.repository_path} of type #{platform.platform_type}:"
+            platform.known_nodes.each do |node|
+              output << "== Report for node #{node}:"
+              output << @nodes_handler.get_description_of node
+            end
+          end
+          out output
+        end
+
+      end
+
+    end
+
+  end
+
+end

data/lib/hybrid_platforms_conductor/hpc_plugins/report/stdout.rb

@@ -0,0 +1,61 @@
+require 'hybrid_platforms_conductor/report'
+require 'terminal-table'
+
+module HybridPlatformsConductor
+
+  module HpcPlugins
+
+    module Report
+
+      # Export on stdout
+      class Stdout < HybridPlatformsConductor::Report
+
+        # Give the list of supported locales by this report generator
+        # [API] - This method is mandatory.
+        #
+        # Result::
+        # * Array<Symbol>: List of supported locales
+        def self.supported_locales
+          [:en]
+        end
+
+        # Create a report for a list of nodes, in a given locale
+        # [API] - This method is mandatory.
+        #
+        # Parameters::
+        # * *nodes* (Array<String>): List of nodes
+        # * *locale_code* (Symbol): The locale code
+        def report_for(nodes, locale_code)
+          @nodes_handler.prefetch_metadata_of nodes, %i[hostname host_ip physical image description services]
+          out(Terminal::Table.new(headings: [
+            'Node',
+            'Platform',
+            'Host name',
+            'IP',
+            'Physical?',
+            'OS',
+            'Description',
+            'Services'
+          ]) do |table|
+            nodes.sort.each do |node|
+              table << [
+                node,
+                @platforms_handler.known_platforms.find { |platform| platform.known_nodes.include?(node) }&.name,
+                @nodes_handler.get_hostname_of(node),
+                @nodes_handler.get_host_ip_of(node),
+                @nodes_handler.get_physical_of(node) ? 'Yes' : 'No',
+                @nodes_handler.get_image_of(node),
+                @nodes_handler.get_description_of(node),
+                (@nodes_handler.get_services_of(node) || []).sort.join(', ')
+              ]
+            end
+          end)
+        end
+
+      end
+
+    end
+
+  end
+
+end