hybrid_platforms_conductor 32.11.1 → 32.13.2

data/examples/tutorial/05_extend_with_plugins/dev_docker_image/Dockerfile

@@ -0,0 +1,20 @@
+# syntax=docker/dockerfile:1
+# Pull the image containing Go
+FROM debian:buster
+
+# Install sshd
+RUN apt-get update && apt-get install -y openssh-server
+RUN mkdir /var/run/sshd
+# Activate root login
+RUN sed -i 's/#PermitRootLogin prohibit-password/PermitRootLogin yes/' /etc/ssh/sshd_config
+# Speed-up considerably ssh performance and avoid huge lags and timeouts without DNS
+RUN sed -i 's/#UseDNS yes/UseDNS no/' /etc/ssh/sshd_config
+EXPOSE 22
+
+# Upload our root key for key authentication of root
+COPY hpc_root.key.pub /root/.ssh/authorized_keys
+RUN chmod 700 /root/.ssh
+RUN chmod 400 /root/.ssh/authorized_keys
+
+# Startup command
+CMD ["/usr/sbin/sshd", "-D"]

data/examples/tutorial/05_extend_with_plugins/dev_docker_image/hpc_root.key

@@ -0,0 +1,27 @@
+-----BEGIN OPENSSH PRIVATE KEY-----
+b3BlbnNzaC1rZXktdjEAAAAABG5vbmUAAAAEbm9uZQAAAAAAAAABAAABFwAAAAdzc2gtcn
+NhAAAAAwEAAQAAAQEAwrphsyDcvMR21e1np5SFNlBXy3fOfVPZKbqWICnDoMLgwyQASubN
+bYfywZ61qEOBkMfUxkUbIa/sqSDgACczOYPjL3zfadoL0/0J2QcJjSjljGpiiQfX+WFyZ+
++0SpDBGQQdMtPsezCf8yYUOCOe3oeXke3i5fXddl4b0KSPsqk0ETfrNOMw6NZ/mdzhlGxn
+U/UaJoO28f4ggZVGwyEUzr68ccWSDXgpnHwULMnD93FyTQyXn5V80Bob5x3dW5hxl1JP+t
+ppLVuEeAs3Yhz9Z8WOwDzut+K5OjvKDiVcRAD+bng5STw5/1m5Ac3H8ynl5NGX0uspiSfM
+FKXYsAW0zwAAA8i31y2Tt9ctkwAAAAdzc2gtcnNhAAABAQDCumGzINy8xHbV7WenlIU2UF
+fLd859U9kpupYgKcOgwuDDJABK5s1th/LBnrWoQ4GQx9TGRRshr+ypIOAAJzM5g+MvfN9p
+2gvT/QnZBwmNKOWMamKJB9f5YXJn77RKkMEZBB0y0+x7MJ/zJhQ4I57eh5eR7eLl9d12Xh
+vQpI+yqTQRN+s04zDo1n+Z3OGUbGdT9Romg7bx/iCBlUbDIRTOvrxxxZINeCmcfBQsycP3
+cXJNDJeflXzQGhvnHd1bmHGXUk/62mktW4R4CzdiHP1nxY7APO634rk6O8oOJVxEAP5ueD
+lJPDn/WbkBzcfzKeXk0ZfS6ymJJ8wUpdiwBbTPAAAAAwEAAQAAAQBXxLAyj9ZOUvMJG1dE
+BkJAp9fagqaUgI16LHpsBscBF7U832YuXHctLkBxmrpgHAUzH468yvsepO+zj3jHOOPywY
+pNpigAyuHX685bfruEX3GPZ4gAIYPsCEsLo9QMY9BVGTCb7z9Aizsc2gX1A8EgpdbgVGuX
+oMjwCBzbXsDSEnzdq4JQLtE90p1pC1ofu10F//Y9qPVQRamLtLXseZXEPi4avhjbf7zDOA
+u8CXje/K06IhJS/+9g/YK3oz3gdZoFu5KDRIHXjoJuSv/ZpAGjpBRgOMyBuajr0Uhp7k0B
+jMvfZE57gOpyqYVtVeuzHz1sETPM88xHjDNxZYzmyv8JAAAAgQCFGKZ+gFF3ngYcFMS5x5
+brtwda6cYzFEpTssfejomFyB36S6tfDhpCVNPQLfZxTGHD5LD/nod8vIY27LA8wAl+cQw5
+J94sOhU015FaO8wCmNVtBXiijhqRE4tSBbPRz99Q8VdFSH2bNU/rHpSDghHVm2w66wN245
+odaXVHgKoCdQAAAIEA/WfU39M4o+nYani9JKA3eADfrPzCIbL5/zsVJ2YIabdl93OWZZ3n
+J+WBZnxRw9sWWJ60TYL9eAaMIVmeRDVQDfRVxJm6cqqfyo6DmaSee2s/Zv3HvE95sIG4wN
+LDuMs9lx+rrUkz4EcezjyfwHmHZn04nSoxg5Q7LebKXcPnh10AAACBAMS4wg431yvCzgrE
+Zy7vZIQgFw/zDSwpqX0RX1//MvFuGVYY0JCUsEgoEfUFzCK8969YOSNavTZKg2HSUczJhV
+8KX2+UIIiyYPQMg0F0JF+nG//FDC2yiJMWCf9ijD5g0ccSM4PG+kAfWDNPZiCQdn47ZtGL
+OB3a2eyfiTcPg0YbAAAAEWFkbWluQGV4YW1wbGUuY29tAQ==
+-----END OPENSSH PRIVATE KEY-----

data/examples/tutorial/05_extend_with_plugins/dev_docker_image/hpc_root.key.pub

@@ -0,0 +1 @@
+ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQDCumGzINy8xHbV7WenlIU2UFfLd859U9kpupYgKcOgwuDDJABK5s1th/LBnrWoQ4GQx9TGRRshr+ypIOAAJzM5g+MvfN9p2gvT/QnZBwmNKOWMamKJB9f5YXJn77RKkMEZBB0y0+x7MJ/zJhQ4I57eh5eR7eLl9d12XhvQpI+yqTQRN+s04zDo1n+Z3OGUbGdT9Romg7bx/iCBlUbDIRTOvrxxxZINeCmcfBQsycP3cXJNDJeflXzQGhvnHd1bmHGXUk/62mktW4R4CzdiHP1nxY7APO634rk6O8oOJVxEAP5ueDlJPDn/WbkBzcfzKeXk0ZfS6ymJJ8wUpdiwBbTP admin@example.com

data/examples/tutorial/05_extend_with_plugins/my-platforms/Gemfile

@@ -0,0 +1,4 @@
+source 'http://rubygems.org'
+
+gem 'hybrid_platforms_conductor'
+gem 'my_hpc_plugins', path: "#{Dir.home}/hpc_tutorial/my_hpc_plugins"

data/examples/tutorial/05_extend_with_plugins/my-platforms/hpc_config.rb

@@ -0,0 +1,13 @@
+yaml_inventory_platform path: "#{Dir.home}/hpc_tutorial/my-service-conf-repo"
+for_nodes('web1') do
+  expect_tests_to_fail %i[connection], 'web1 is temporarily down - will bring it up later'
+end
+# Select only the nodes implementing our web-hello service (that is all the webN nodes)
+for_nodes [{ service: 'web-hello' }] do
+  # On our web servers we should have users used by our services
+  check_local_users_do_exist %w[sshd www-data]
+  # Make sure we have no leftovers of obsolete users
+  check_local_users_do_not_exist %w[dangerous_user obsolete_user]
+end
+os_image :debian_10, "#{hybrid_platforms_dir}/images/debian_10"
+json_bash_platform path: "#{Dir.home}/hpc_tutorial/dev-servers-conf-repo"

data/examples/tutorial/05_extend_with_plugins/my-platforms/images/debian_10/Dockerfile

@@ -0,0 +1,13 @@
+# syntax=docker/dockerfile:1
+FROM debian:buster
+
+RUN apt-get update && apt-get install -y openssh-server
+RUN mkdir /var/run/sshd
+# Activate root login with test password
+RUN echo 'root:root_pwd' | chpasswd
+RUN sed -i 's/#PermitRootLogin prohibit-password/PermitRootLogin yes/' /etc/ssh/sshd_config
+# Speed-up considerably ssh performance and avoid huge lags and timeouts without DNS
+RUN sed -i 's/#UseDNS yes/UseDNS no/' /etc/ssh/sshd_config
+EXPOSE 22
+
+CMD ["/usr/sbin/sshd", "-D"]

data/examples/tutorial/05_extend_with_plugins/my-platforms/my_commands.bash

@@ -0,0 +1,2 @@
+echo Hostname here is
+hostname

data/examples/tutorial/05_extend_with_plugins/my-service-conf-repo/inventory.yaml

@@ -0,0 +1,100 @@
+---
+local:
+  metadata:
+    description: The local environment
+    local_node: true
+    service_port: 1107
+    service_timeout: 60
+  services:
+    - my-service
+web1:
+  metadata:
+    description: Web service nbr 1
+    image: debian_10
+    hostname: web1.hpc_tutorial.org
+    planet: Mercury
+    service_port: 1201
+    service_timeout: 60
+  services:
+    - web-hello
+    - my-service
+web2:
+  metadata:
+    description: Web service nbr 2
+    image: debian_10
+    hostname: web2.hpc_tutorial.org
+    planet: Venus
+    service_port: 1202
+    service_timeout: 60
+  services:
+    - web-hello
+    - my-service
+web3:
+  metadata:
+    description: Web service nbr 3
+    image: debian_10
+    hostname: web3.hpc_tutorial.org
+    planet: Earth
+    service_port: 1203
+    service_timeout: 60
+  services:
+    - web-hello
+    - my-service
+web4:
+  metadata:
+    description: Web service nbr 4
+    image: debian_10
+    hostname: web4.hpc_tutorial.org
+    planet: Mars
+    service_port: 1204
+    service_timeout: 60
+  services:
+    - web-hello
+    - my-service
+web5:
+  metadata:
+    description: Web service nbr 5
+    image: debian_10
+    hostname: web5.hpc_tutorial.org
+    planet: Jupiter
+    service_port: 1205
+    service_timeout: 60
+  services:
+    - web-hello
+    - my-service
+web6:
+  metadata:
+    description: Web service nbr 6
+    image: debian_10
+    hostname: web6.hpc_tutorial.org
+  services:
+    - web-hello
+web7:
+  metadata:
+    description: Web service nbr 7
+    image: debian_10
+    hostname: web7.hpc_tutorial.org
+  services:
+    - web-hello
+web8:
+  metadata:
+    description: Web service nbr 8
+    image: debian_10
+    hostname: web8.hpc_tutorial.org
+  services:
+    - web-hello
+web9:
+  metadata:
+    description: Web service nbr 9
+    image: debian_10
+    hostname: web9.hpc_tutorial.org
+  services:
+    - web-hello
+web10:
+  metadata:
+    description: Web service nbr 10
+    image: debian_10
+    hostname: web10.hpc_tutorial.org
+  # The list of service names this node should have
+  services:
+    - web-hello

data/examples/tutorial/05_extend_with_plugins/my-service-conf-repo/my-service.conf.erb

@@ -0,0 +1,3 @@
+service-port: <%= @service_port %>
+service-timeout: <%= @service_timeout %>
+service-logs: stdout

data/examples/tutorial/05_extend_with_plugins/my-service-conf-repo/service_my-service.rb

@@ -0,0 +1,58 @@
+# Get the wanted content of the configuration file as a String, based on the node's metadata
+#
+# Parameters::
+# * *node* (String): Node for which we configure our service
+# Result::
+# * String: The wanted content
+def wanted_conf_for(node)
+  # We will access the node's metadata using the NodesHandler API, through the @nodes_handler object
+  @service_port = @nodes_handler.get_service_port_of(node)
+  @service_timeout = @nodes_handler.get_service_timeout_of(node)
+  # We use erubis to generate the configuration from our eRuby template, and return it directly
+  Erubis::Eruby.new(File.read("#{@platform_handler.repository_path}/my-service.conf.erb")).result(binding)
+end
+
+# Get actions to check the node's service against the wanted content
+#
+# Parameters::
+# * *node* (String): Node on which we check the service
+# Result::
+# * Array< Hash<Symbol,Object> >: The list of actions
+def check(node)
+  # We first dump the wanted content in a temporary file and then we diff it.
+  [
+    {
+      remote_bash: <<~EOS
+        cat <<EOF >/tmp/my-service.conf.wanted
+        #{wanted_conf_for(node)}
+        EOF
+        echo Diffs on my-service.conf:
+        if test -f ~/hpc_tutorial/node/my-service.conf; then
+          diff ~/hpc_tutorial/node/my-service.conf /tmp/my-service.conf.wanted || true
+        else
+          echo "Create file from scratch"
+          cat /tmp/my-service.conf.wanted
+        fi
+      EOS
+    }
+  ]
+end
+
+# Get actions to deploy the node's service against the wanted content
+#
+# Parameters::
+# * *node* (String): Node on which we deploy the service
+# Result::
+# * Array< Hash<Symbol,Object> >: The list of actions
+def deploy(node)
+  # We first check, as this will display diffs and prepare the file to be copied.
+  # And then we really deploy the file on our node.
+  check(node) + [
+    {
+      remote_bash: <<~EOS
+        mkdir -p ~/hpc_tutorial/node
+        cp /tmp/my-service.conf.wanted ~/hpc_tutorial/node/my-service.conf
+      EOS
+    }
+  ]
+end

data/examples/tutorial/05_extend_with_plugins/my-service-conf-repo/service_web-hello.rb

@@ -0,0 +1,43 @@
+# Get actions to check the node's service against the wanted content
+#
+# Parameters::
+# * *node* (String): Node on which we check the service
+# Result::
+# * Array< Hash<Symbol,Object> >: The list of actions
+def check(node)
+  # We first dump the wanted content in a temporary file and then we diff it.
+  # We will access the node's planet, hostname and IP from its metadata using the NodesHandler API, through the @nodes_handler object
+  [
+    {
+      remote_bash: <<~EOS
+        echo 'Hello #{@nodes_handler.get_planet_of(node) || 'World'} from #{@nodes_handler.get_hostname_of(node)} (#{@nodes_handler.get_host_ip_of(node)})' >/tmp/hello_world.txt.wanted
+        echo Diffs on hello_world.txt:
+        if test -f /root/hello_world.txt; then
+          diff /root/hello_world.txt /tmp/hello_world.txt.wanted || true
+        else
+          echo "Create hello_world.txt from scratch"
+          cat /tmp/hello_world.txt.wanted
+        fi
+      EOS
+    }
+  ]
+end
+
+# Get actions to deploy the node's service against the wanted content
+#
+# Parameters::
+# * *node* (String): Node on which we deploy the service
+# Result::
+# * Array< Hash<Symbol,Object> >: The list of actions
+def deploy(node)
+  # We first check, as this will display diffs and prepare the file to be copied.
+  # And then we really deploy the file on our node.
+  check(node) + [
+    {
+      remote_bash: <<~EOS
+        mkdir -p ~/hpc_tutorial/node
+        cp /tmp/hello_world.txt.wanted /root/hello_world.txt
+      EOS
+    }
+  ]
+end

data/examples/tutorial/05_extend_with_plugins/my_hpc_plugins/lib/my_hpc_plugins/hpc_plugins/platform_handler/json_bash.rb

@@ -0,0 +1,115 @@
+require 'json'
+require 'hybrid_platforms_conductor/platform_handler'
+
+module MyHpcPlugins
+
+  module HpcPlugins
+
+    module PlatformHandler
+
+      # A nice platform handler to handle platforms of our team, using json inventory and bash scripts.
+      class JsonBash < HybridPlatformsConductor::PlatformHandler
+
+        # Get the list of known nodes.
+        # [API] - This method is mandatory.
+        #
+        # Result::
+        # * Array<String>: List of node names
+        def known_nodes
+          # This method is used to get the list of nodes that are handled by the platform
+          # In our case we read our json file to get this information, and use just the first part of the hostname as the node's name.
+          JSON.parse(File.read("#{repository_path}/hosts.json")).keys.map { |hostname| hostname.split('.').first }
+        end
+
+        # Get the metadata of a given node.
+        # [API] - This method is mandatory.
+        #
+        # Parameters::
+        # * *node* (String): Node to read metadata from
+        # Result::
+        # * Hash<Symbol,Object>: The corresponding metadata
+        def metadata_for(node)
+          # All nodes handled by this platform are running a debian buster image and we derive their name from their hostname.
+          {
+            hostname: "#{node}.hpc_tutorial.org",
+            image: 'debian_10'
+          }
+        end
+
+        # Return the services for a given node
+        # [API] - This method is mandatory.
+        #
+        # Parameters::
+        # * *node* (String): node to read configuration from
+        # Result::
+        # * Array<String>: The corresponding services
+        def services_for(node)
+          # This info is taken from our JSON inventory file
+          [JSON.parse(File.read("#{repository_path}/hosts.json"))["#{node}.hpc_tutorial.org"]]
+        end
+
+        # Get the list of services we can deploy
+        # [API] - This method is mandatory.
+        #
+        # Result::
+        # * Array<String>: The corresponding services
+        def deployable_services
+          # This info is taken by listing existing bash scripts
+          Dir.glob("#{repository_path}/install-*.bash").map { |file| File.basename(file).match(/install-(.*)\.bash/)[1] }
+        end
+
+        # Get the list of actions to perform to deploy on a given node.
+        # Those actions can be executed in parallel with other deployments on other nodes. They must be thread safe.
+        # [API] - This method is mandatory.
+        # [API] - @cmd_runner is accessible.
+        # [API] - @actions_executor is accessible.
+        #
+        # Parameters::
+        # * *node* (String): Node to deploy on
+        # * *service* (String): Service to be deployed
+        # * *use_why_run* (Boolean): Do we use a why-run mode? [default = true]
+        # Result::
+        # * Array< Hash<Symbol,Object> >: List of actions to be done
+        def actions_to_deploy_on(node, service, use_why_run: true)
+          # This method returns all the actions to execute to deploy on a node.
+          # The use_why_run switch is on if the deployment should just be simulated.
+          # Those actions (bash commands, scp of files, ruby code...) should be thread safe as they can be executed in parallel with other deployment actions for other nodes in case of a concurrent deployment on several nodes.
+          # In our case it's very simple: we just call our bash script on the node's hostname.
+          [{ bash: "#{repository_path}/install-#{service}.bash #{@nodes_handler.get_hostname_of(node)} #{use_why_run ? 'check' : ''}" }]
+        end
+
+        # Parse stdout and stderr of a given deploy run and get the list of tasks with their status
+        # [API] - This method is mandatory.
+        #
+        # Parameters::
+        # * *stdout* (String): stdout to be parsed
+        # * *stderr* (String): stderr to be parsed
+        # Result::
+        # * Array< Hash<Symbol,Object> >: List of task properties. The following properties should be returned, among free ones:
+        #   * *name* (String): Task name
+        #   * *status* (Symbol): Task status. Should be one of:
+        #     * *:changed*: The task has been changed
+        #     * *:identical*: The task has not been changed
+        #   * *diffs* (String): Differences, if any
+        def parse_deploy_output(stdout, stderr)
+          # In our case our bash scripts return the last line as a status, so use it.
+          [{
+            name: 'Install tool',
+            status:
+              case stdout.split("\n").last
+              when 'OK'
+                :identical
+              else
+                :changed
+              end,
+            diffs: stdout
+          }]
+        end
+
+      end
+
+    end
+
+  end
+
+end

data/examples/tutorial/05_extend_with_plugins/my_hpc_plugins/lib/my_hpc_plugins/hpc_plugins/report/web_report.rb

@@ -0,0 +1,52 @@
+require 'hybrid_platforms_conductor/report'
+
+module MyHpcPlugins
+
+  module HpcPlugins
+
+    module Report
+
+      # Publish reports to our web reporting tool
+      class WebReport < HybridPlatformsConductor::Report
+
+        # Give the list of supported locales by this report generator
+        # [API] - This method is mandatory.
+        #
+        # Result::
+        # * Array<Symbol>: List of supported locales
+        def self.supported_locales
+          # This method has to publish the list of translations it accepts.
+          [:en]
+        end
+
+        # Create a report for a list of nodes, in a given locale
+        # [API] - This method is mandatory.
+        #
+        # Parameters::
+        # * *nodes* (Array<String>): List of nodes
+        # * *locale_code* (Symbol): The locale code
+        def report_for(nodes, locale_code)
+          # This method simply provides a report for a given list of nodes in the desired locale.
+          # The locale will be one of the supported ones.
+          # Generate the report in a file to be uploaded on web10.
+          File.write(
+            '/tmp/web_report.txt',
+            @platforms_handler.known_platforms.map do |platform|
+              "= Inventory for platform #{platform.repository_path} of type #{platform.platform_type}:\n" +
+                platform.known_nodes.map do |node|
+                  "* Node #{node} (IP: #{@nodes_handler.get_host_ip_of(node)}, Hostname: #{@nodes_handler.get_hostname_of(node)})."
+                end.join("\n")
+            end.join("\n")
+          )
+          # Upload the file on our web10 instance
+          system 'scp -o StrictHostKeyChecking=no /tmp/web_report.txt root@web10.hpc_tutorial.org:/root/hello_world.txt'
+          out 'Upload successful'
+        end
+
+      end
+
+    end
+
+  end
+
+end