hybrid_platforms_conductor 32.13.0 → 32.14.0

data/examples/tutorial/04_test/web_docker_image/hello_world.txt

@@ -0,0 +1 @@
+Hello World!

data/examples/tutorial/04_test/web_docker_image/hpc_root.key

@@ -0,0 +1,27 @@
+-----BEGIN OPENSSH PRIVATE KEY-----
+b3BlbnNzaC1rZXktdjEAAAAABG5vbmUAAAAEbm9uZQAAAAAAAAABAAABFwAAAAdzc2gtcn
+NhAAAAAwEAAQAAAQEAu5oVFddWtHFKA8+PnRssWieXrnMUPM8+qTdQrxoMaNacFWN+pBv3
+Nj61GphCANoEPBvgcm7ltgZa1EumAZh4NJFi7AqM4Z2AFsqsvzRS1HNFRClEwr576iRkMc
+ECIr0JmxeOscVcDMx0/9uFSfON7DiuStOOpIJIiQfjtFBsLFpzdfsVfZ3zc18Hxugqsbq0
+t0bsTZdjRKnIYeKTSXKbF2n9b/dAf2tBtpywpun1WXQul7w1z20m9zv955k4kq3rAhSIPH
+aOSE+QCXlICYqwmP2KVKCG6XEqSWjF4ujzcE7PlKIK1gsXdfum6GL23X8k8njBIjRzq6a0
+Uabp65tvVQAAA8hgF2y/YBdsvwAAAAdzc2gtcnNhAAABAQC7mhUV11a0cUoDz4+dGyxaJ5
+eucxQ8zz6pN1CvGgxo1pwVY36kG/c2PrUamEIA2gQ8G+BybuW2BlrUS6YBmHg0kWLsCozh
+nYAWyqy/NFLUc0VEKUTCvnvqJGQxwQIivQmbF46xxVwMzHT/24VJ843sOK5K046kgkiJB+
+O0UGwsWnN1+xV9nfNzXwfG6CqxurS3RuxNl2NEqchh4pNJcpsXaf1v90B/a0G2nLCm6fVZ
+dC6XvDXPbSb3O/3nmTiSresCFIg8do5IT5AJeUgJirCY/YpUoIbpcSpJaMXi6PNwTs+Uog
+rWCxd1+6boYvbdfyTyeMEiNHOrprRRpunrm29VAAAAAwEAAQAAAQEAj5IE7vkkvuUweqgf
+RWGlWL2SJIyngT4tuzy8/7SUqKsNOoUxRxCr6ZSl7tt1L8eplALVi135aIuGeZAHVJNF8x
+0Mso4EcSmi/E5RU+cMONCawuY+XDeBB5igIT+PjdxT/KBTsRLUNmtGYwSyBf1hsNVT9ang
+MWAPeOdTyQv/LRUe+O/ZKQ/h/X51BEwSXqz32RmlmZdMnDacIJ0NWdbcbAEtB4JT83S20d
+bycQNOxGVmyevOGjyTXfUX6FhtYT7A7zzqNUL8UstSdajWeTZvVGd0C7Ob1kxaU3tnAwFr
+0sHTPHX7FQQFGhiU8gBZHOJ/PQzo+qqcCSsM3FcJlpVw5QAAAIAPYGXt7diWFYEhZVfj3n
+KA0lX7LjQms0en4CoSkN4BELQJnaaqlTkrqN7PNL/WbmeavzDqQMk1jb9Q2+gra826THX5
+GGtbFPFb9TSwBNi+zsyVURKEzqNYSw28ytKIdj98VW1qqQoolgTc5qoXG1JW5f5bEPbzUT
+LXPP5j+YvuCgAAAIEA8NVJc5vXzJxbdgBxOPPHaPCAZ28V15Ggy+Bj1K/9G6FHqa/SFhy0
+tCpBDNB/POX2UXCQDLteTIDodj9h2Yat6PJP9RKYCC9F5+/fU/0Y/tvD/p/uNhwEuqToex
+gBUoL+r7/G3qYK13n8AKwrvb2wiYgP3aUVW7AyOier1MKt4OsAAACBAMdqmcj+5fNSTuOg
+iPgGylUSIYnJ0w2jjZIaX96aat/w+SgciLhenbwpcf/9E8MDYKbXyhArdQvOgkEVogEI4s
+PrFG66ogawPuRUUsuZBhm/b535EEvNAlkC8fMPVuKfOIsHJs8Cwcy7QPZSZu5pyKznpnCT
+UlYGT6aB0GGoeeC/AAAAEWFkbWluQGV4YW1wbGUuY29tAQ==
+-----END OPENSSH PRIVATE KEY-----

data/examples/tutorial/04_test/web_docker_image/hpc_root.key.pub

@@ -0,0 +1 @@
+ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQC7mhUV11a0cUoDz4+dGyxaJ5eucxQ8zz6pN1CvGgxo1pwVY36kG/c2PrUamEIA2gQ8G+BybuW2BlrUS6YBmHg0kWLsCozhnYAWyqy/NFLUc0VEKUTCvnvqJGQxwQIivQmbF46xxVwMzHT/24VJ843sOK5K046kgkiJB+O0UGwsWnN1+xV9nfNzXwfG6CqxurS3RuxNl2NEqchh4pNJcpsXaf1v90B/a0G2nLCm6fVZdC6XvDXPbSb3O/3nmTiSresCFIg8do5IT5AJeUgJirCY/YpUoIbpcSpJaMXi6PNwTs+UogrWCxd1+6boYvbdfyTyeMEiNHOrprRRpunrm29V admin@example.com

data/examples/tutorial/04_test/web_docker_image/main.go

@@ -0,0 +1,43 @@
+package main
+
+import (
+    "fmt"
+    "io/ioutil"
+    "log"
+    "net/http"
+    "os"
+)
+
+const homepageEndPoint = "/"
+
+// StartWebServer the webserver
+func StartWebServer() {
+    http.HandleFunc(homepageEndPoint, handleHomepage)
+    port := os.Getenv("PORT")
+    if len(port) == 0 {
+        panic("Environment variable PORT is not set")
+    }
+
+    log.Printf("Starting web server to listen on endpoints [%s] and port %s",
+        homepageEndPoint, port)
+    if err := http.ListenAndServe(":"+port, nil); err != nil {
+        panic(err)
+    }
+}
+
+func handleHomepage(w http.ResponseWriter, r *http.Request) {
+    urlPath := r.URL.Path
+    log.Printf("Web request received on url path %s", urlPath)
+    content, content_err := ioutil.ReadFile("/root/hello_world.txt")
+    if content_err != nil {
+        fmt.Printf("Failed to read message to display, err: %s", content_err)
+    }
+    _, write_err := w.Write(content)
+    if write_err != nil {
+        fmt.Printf("Failed to write response, err: %s", write_err)
+    }
+}
+
+func main() {
+    StartWebServer()
+}

data/examples/tutorial/04_test/web_docker_image/start.sh

@@ -0,0 +1,7 @@
+#!/bin/bash
+
+# Start sshd as a daemon
+/usr/sbin/sshd
+
+# Start web server
+sh -c /codebase/bin/server

data/examples/tutorial/04_test/web_docker_image/test.bash

@@ -0,0 +1,6 @@
+#!/bin/bash
+
+for ((i=1;i<=10;i++));
+do 
+   echo "Container web$i: $(curl http://web$i.hpc_tutorial.org 2>/dev/null)"
+done

data/examples/tutorial/05_extend_with_plugins/dev-servers-conf-repo/hosts.json

@@ -0,0 +1,12 @@
+{
+  "dev1.hpc_tutorial.org": "python",
+  "dev2.hpc_tutorial.org": "python",
+  "dev3.hpc_tutorial.org": "python",
+  "dev4.hpc_tutorial.org": "python",
+  "dev5.hpc_tutorial.org": "python",
+  "dev6.hpc_tutorial.org": "gcc",
+  "dev7.hpc_tutorial.org": "gcc",
+  "dev8.hpc_tutorial.org": "gcc",
+  "dev9.hpc_tutorial.org": "gcc",
+  "dev10.hpc_tutorial.org": "gcc"
+}

data/examples/tutorial/05_extend_with_plugins/dev-servers-conf-repo/install-gcc.bash

@@ -0,0 +1,14 @@
+hostname=${1}
+check_flag=${2:-deploy}
+if [ "${check_flag}" = "check" ]; then
+  # Check if gcc is installed
+  if ssh -o StrictHostKeyChecking=no root@${hostname} 'gcc --version' 2>/dev/null; then
+    echo 'OK'
+  else
+    echo 'Missing'
+  fi
+else
+  # Install gcc
+  ssh -o StrictHostKeyChecking=no root@${hostname} 'apt install -y gcc' 2>/dev/null
+  echo 'Installed'
+fi

data/examples/tutorial/05_extend_with_plugins/dev-servers-conf-repo/install-python.bash

@@ -0,0 +1,14 @@
+hostname=${1}
+check_flag=${2:-deploy}
+if [ "${check_flag}" = "check" ]; then
+  # Check if python3 is installed
+  if ssh -o StrictHostKeyChecking=no root@${hostname} 'python3 --version' 2>/dev/null; then
+    echo 'OK'
+  else
+    echo 'Missing'
+  fi
+else
+  # Install python3
+  ssh -o StrictHostKeyChecking=no root@${hostname} 'apt install -y python3-pip' 2>/dev/null
+  echo 'Installed'
+fi

data/examples/tutorial/05_extend_with_plugins/dev_docker_image/Dockerfile

@@ -0,0 +1,20 @@
+# syntax=docker/dockerfile:1
+# Pull the image containing Go
+FROM debian:buster
+
+# Install sshd
+RUN apt-get update && apt-get install -y openssh-server
+RUN mkdir /var/run/sshd
+# Activate root login
+RUN sed -i 's/#PermitRootLogin prohibit-password/PermitRootLogin yes/' /etc/ssh/sshd_config
+# Speed-up considerably ssh performance and avoid huge lags and timeouts without DNS
+RUN sed -i 's/#UseDNS yes/UseDNS no/' /etc/ssh/sshd_config
+EXPOSE 22
+
+# Upload our root key for key authentication of root
+COPY hpc_root.key.pub /root/.ssh/authorized_keys
+RUN chmod 700 /root/.ssh
+RUN chmod 400 /root/.ssh/authorized_keys
+
+# Startup command
+CMD ["/usr/sbin/sshd", "-D"]

data/examples/tutorial/05_extend_with_plugins/dev_docker_image/hpc_root.key

@@ -0,0 +1,27 @@
+-----BEGIN OPENSSH PRIVATE KEY-----
+b3BlbnNzaC1rZXktdjEAAAAABG5vbmUAAAAEbm9uZQAAAAAAAAABAAABFwAAAAdzc2gtcn
+NhAAAAAwEAAQAAAQEAwrphsyDcvMR21e1np5SFNlBXy3fOfVPZKbqWICnDoMLgwyQASubN
+bYfywZ61qEOBkMfUxkUbIa/sqSDgACczOYPjL3zfadoL0/0J2QcJjSjljGpiiQfX+WFyZ+
++0SpDBGQQdMtPsezCf8yYUOCOe3oeXke3i5fXddl4b0KSPsqk0ETfrNOMw6NZ/mdzhlGxn
+U/UaJoO28f4ggZVGwyEUzr68ccWSDXgpnHwULMnD93FyTQyXn5V80Bob5x3dW5hxl1JP+t
+ppLVuEeAs3Yhz9Z8WOwDzut+K5OjvKDiVcRAD+bng5STw5/1m5Ac3H8ynl5NGX0uspiSfM
+FKXYsAW0zwAAA8i31y2Tt9ctkwAAAAdzc2gtcnNhAAABAQDCumGzINy8xHbV7WenlIU2UF
+fLd859U9kpupYgKcOgwuDDJABK5s1th/LBnrWoQ4GQx9TGRRshr+ypIOAAJzM5g+MvfN9p
+2gvT/QnZBwmNKOWMamKJB9f5YXJn77RKkMEZBB0y0+x7MJ/zJhQ4I57eh5eR7eLl9d12Xh
+vQpI+yqTQRN+s04zDo1n+Z3OGUbGdT9Romg7bx/iCBlUbDIRTOvrxxxZINeCmcfBQsycP3
+cXJNDJeflXzQGhvnHd1bmHGXUk/62mktW4R4CzdiHP1nxY7APO634rk6O8oOJVxEAP5ueD
+lJPDn/WbkBzcfzKeXk0ZfS6ymJJ8wUpdiwBbTPAAAAAwEAAQAAAQBXxLAyj9ZOUvMJG1dE
+BkJAp9fagqaUgI16LHpsBscBF7U832YuXHctLkBxmrpgHAUzH468yvsepO+zj3jHOOPywY
+pNpigAyuHX685bfruEX3GPZ4gAIYPsCEsLo9QMY9BVGTCb7z9Aizsc2gX1A8EgpdbgVGuX
+oMjwCBzbXsDSEnzdq4JQLtE90p1pC1ofu10F//Y9qPVQRamLtLXseZXEPi4avhjbf7zDOA
+u8CXje/K06IhJS/+9g/YK3oz3gdZoFu5KDRIHXjoJuSv/ZpAGjpBRgOMyBuajr0Uhp7k0B
+jMvfZE57gOpyqYVtVeuzHz1sETPM88xHjDNxZYzmyv8JAAAAgQCFGKZ+gFF3ngYcFMS5x5
+brtwda6cYzFEpTssfejomFyB36S6tfDhpCVNPQLfZxTGHD5LD/nod8vIY27LA8wAl+cQw5
+J94sOhU015FaO8wCmNVtBXiijhqRE4tSBbPRz99Q8VdFSH2bNU/rHpSDghHVm2w66wN245
+odaXVHgKoCdQAAAIEA/WfU39M4o+nYani9JKA3eADfrPzCIbL5/zsVJ2YIabdl93OWZZ3n
+J+WBZnxRw9sWWJ60TYL9eAaMIVmeRDVQDfRVxJm6cqqfyo6DmaSee2s/Zv3HvE95sIG4wN
+LDuMs9lx+rrUkz4EcezjyfwHmHZn04nSoxg5Q7LebKXcPnh10AAACBAMS4wg431yvCzgrE
+Zy7vZIQgFw/zDSwpqX0RX1//MvFuGVYY0JCUsEgoEfUFzCK8969YOSNavTZKg2HSUczJhV
+8KX2+UIIiyYPQMg0F0JF+nG//FDC2yiJMWCf9ijD5g0ccSM4PG+kAfWDNPZiCQdn47ZtGL
+OB3a2eyfiTcPg0YbAAAAEWFkbWluQGV4YW1wbGUuY29tAQ==
+-----END OPENSSH PRIVATE KEY-----

data/examples/tutorial/05_extend_with_plugins/dev_docker_image/hpc_root.key.pub

@@ -0,0 +1 @@
+ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQDCumGzINy8xHbV7WenlIU2UFfLd859U9kpupYgKcOgwuDDJABK5s1th/LBnrWoQ4GQx9TGRRshr+ypIOAAJzM5g+MvfN9p2gvT/QnZBwmNKOWMamKJB9f5YXJn77RKkMEZBB0y0+x7MJ/zJhQ4I57eh5eR7eLl9d12XhvQpI+yqTQRN+s04zDo1n+Z3OGUbGdT9Romg7bx/iCBlUbDIRTOvrxxxZINeCmcfBQsycP3cXJNDJeflXzQGhvnHd1bmHGXUk/62mktW4R4CzdiHP1nxY7APO634rk6O8oOJVxEAP5ueDlJPDn/WbkBzcfzKeXk0ZfS6ymJJ8wUpdiwBbTP admin@example.com

data/examples/tutorial/05_extend_with_plugins/my-platforms/Gemfile

@@ -0,0 +1,4 @@
+source 'http://rubygems.org'
+
+gem 'hybrid_platforms_conductor'
+gem 'my_hpc_plugins', path: "#{Dir.home}/hpc_tutorial/my_hpc_plugins"

data/examples/tutorial/05_extend_with_plugins/my-platforms/hpc_config.rb

@@ -0,0 +1,13 @@
+yaml_inventory_platform path: "#{Dir.home}/hpc_tutorial/my-service-conf-repo"
+for_nodes('web1') do
+  expect_tests_to_fail %i[connection], 'web1 is temporarily down - will bring it up later'
+end
+# Select only the nodes implementing our web-hello service (that is all the webN nodes)
+for_nodes [{ service: 'web-hello' }] do
+  # On our web servers we should have users used by our services
+  check_local_users_do_exist %w[sshd www-data]
+  # Make sure we have no leftovers of obsolete users
+  check_local_users_do_not_exist %w[dangerous_user obsolete_user]
+end
+os_image :debian_10, "#{hybrid_platforms_dir}/images/debian_10"
+json_bash_platform path: "#{Dir.home}/hpc_tutorial/dev-servers-conf-repo"

data/examples/tutorial/05_extend_with_plugins/my-platforms/images/debian_10/Dockerfile

@@ -0,0 +1,13 @@
+# syntax=docker/dockerfile:1
+FROM debian:buster
+
+RUN apt-get update && apt-get install -y openssh-server
+RUN mkdir /var/run/sshd
+# Activate root login with test password
+RUN echo 'root:root_pwd' | chpasswd
+RUN sed -i 's/#PermitRootLogin prohibit-password/PermitRootLogin yes/' /etc/ssh/sshd_config
+# Speed-up considerably ssh performance and avoid huge lags and timeouts without DNS
+RUN sed -i 's/#UseDNS yes/UseDNS no/' /etc/ssh/sshd_config
+EXPOSE 22
+
+CMD ["/usr/sbin/sshd", "-D"]

data/examples/tutorial/05_extend_with_plugins/my-platforms/my_commands.bash

@@ -0,0 +1,2 @@
+echo Hostname here is
+hostname

data/examples/tutorial/05_extend_with_plugins/my-service-conf-repo/inventory.yaml

@@ -0,0 +1,100 @@
+---
+local:
+  metadata:
+    description: The local environment
+    local_node: true
+    service_port: 1107
+    service_timeout: 60
+  services:
+    - my-service
+web1:
+  metadata:
+    description: Web service nbr 1
+    image: debian_10
+    hostname: web1.hpc_tutorial.org
+    planet: Mercury
+    service_port: 1201
+    service_timeout: 60
+  services:
+    - web-hello
+    - my-service
+web2:
+  metadata:
+    description: Web service nbr 2
+    image: debian_10
+    hostname: web2.hpc_tutorial.org
+    planet: Venus
+    service_port: 1202
+    service_timeout: 60
+  services:
+    - web-hello
+    - my-service
+web3:
+  metadata:
+    description: Web service nbr 3
+    image: debian_10
+    hostname: web3.hpc_tutorial.org
+    planet: Earth
+    service_port: 1203
+    service_timeout: 60
+  services:
+    - web-hello
+    - my-service
+web4:
+  metadata:
+    description: Web service nbr 4
+    image: debian_10
+    hostname: web4.hpc_tutorial.org
+    planet: Mars
+    service_port: 1204
+    service_timeout: 60
+  services:
+    - web-hello
+    - my-service
+web5:
+  metadata:
+    description: Web service nbr 5
+    image: debian_10
+    hostname: web5.hpc_tutorial.org
+    planet: Jupiter
+    service_port: 1205
+    service_timeout: 60
+  services:
+    - web-hello
+    - my-service
+web6:
+  metadata:
+    description: Web service nbr 6
+    image: debian_10
+    hostname: web6.hpc_tutorial.org
+  services:
+    - web-hello
+web7:
+  metadata:
+    description: Web service nbr 7
+    image: debian_10
+    hostname: web7.hpc_tutorial.org
+  services:
+    - web-hello
+web8:
+  metadata:
+    description: Web service nbr 8
+    image: debian_10
+    hostname: web8.hpc_tutorial.org
+  services:
+    - web-hello
+web9:
+  metadata:
+    description: Web service nbr 9
+    image: debian_10
+    hostname: web9.hpc_tutorial.org
+  services:
+    - web-hello
+web10:
+  metadata:
+    description: Web service nbr 10
+    image: debian_10
+    hostname: web10.hpc_tutorial.org
+  # The list of service names this node should have
+  services:
+    - web-hello

data/examples/tutorial/05_extend_with_plugins/my-service-conf-repo/my-service.conf.erb

@@ -0,0 +1,3 @@
+service-port: <%= @service_port %>
+service-timeout: <%= @service_timeout %>
+service-logs: stdout

data/examples/tutorial/05_extend_with_plugins/my-service-conf-repo/service_my-service.rb

@@ -0,0 +1,58 @@
+# Get the wanted content of the configuration file as a String, based on the node's metadata
+#
+# Parameters::
+# * *node* (String): Node for which we configure our service
+# Result::
+# * String: The wanted content
+def wanted_conf_for(node)
+  # We will access the node's metadata using the NodesHandler API, through the @nodes_handler object
+  @service_port = @nodes_handler.get_service_port_of(node)
+  @service_timeout = @nodes_handler.get_service_timeout_of(node)
+  # We use erubis to generate the configuration from our eRuby template, and return it directly
+  Erubis::Eruby.new(File.read("#{@platform_handler.repository_path}/my-service.conf.erb")).result(binding)
+end
+
+# Get actions to check the node's service against the wanted content
+#
+# Parameters::
+# * *node* (String): Node on which we check the service
+# Result::
+# * Array< Hash<Symbol,Object> >: The list of actions
+def check(node)
+  # We first dump the wanted content in a temporary file and then we diff it.
+  [
+    {
+      remote_bash: <<~EOS
+        cat <<EOF >/tmp/my-service.conf.wanted
+        #{wanted_conf_for(node)}
+        EOF
+        echo Diffs on my-service.conf:
+        if test -f ~/hpc_tutorial/node/my-service.conf; then
+          diff ~/hpc_tutorial/node/my-service.conf /tmp/my-service.conf.wanted || true
+        else
+          echo "Create file from scratch"
+          cat /tmp/my-service.conf.wanted
+        fi
+      EOS
+    }
+  ]
+end
+
+# Get actions to deploy the node's service against the wanted content
+#
+# Parameters::
+# * *node* (String): Node on which we deploy the service
+# Result::
+# * Array< Hash<Symbol,Object> >: The list of actions
+def deploy(node)
+  # We first check, as this will display diffs and prepare the file to be copied.
+  # And then we really deploy the file on our node.
+  check(node) + [
+    {
+      remote_bash: <<~EOS
+        mkdir -p ~/hpc_tutorial/node
+        cp /tmp/my-service.conf.wanted ~/hpc_tutorial/node/my-service.conf
+      EOS
+    }
+  ]
+end

data/examples/tutorial/05_extend_with_plugins/my-service-conf-repo/service_web-hello.rb

@@ -0,0 +1,43 @@
+# Get actions to check the node's service against the wanted content
+#
+# Parameters::
+# * *node* (String): Node on which we check the service
+# Result::
+# * Array< Hash<Symbol,Object> >: The list of actions
+def check(node)
+  # We first dump the wanted content in a temporary file and then we diff it.
+  # We will access the node's planet, hostname and IP from its metadata using the NodesHandler API, through the @nodes_handler object
+  [
+    {
+      remote_bash: <<~EOS
+        echo 'Hello #{@nodes_handler.get_planet_of(node) || 'World'} from #{@nodes_handler.get_hostname_of(node)} (#{@nodes_handler.get_host_ip_of(node)})' >/tmp/hello_world.txt.wanted
+        echo Diffs on hello_world.txt:
+        if test -f /root/hello_world.txt; then
+          diff /root/hello_world.txt /tmp/hello_world.txt.wanted || true
+        else
+          echo "Create hello_world.txt from scratch"
+          cat /tmp/hello_world.txt.wanted
+        fi
+      EOS
+    }
+  ]
+end
+
+# Get actions to deploy the node's service against the wanted content
+#
+# Parameters::
+# * *node* (String): Node on which we deploy the service
+# Result::
+# * Array< Hash<Symbol,Object> >: The list of actions
+def deploy(node)
+  # We first check, as this will display diffs and prepare the file to be copied.
+  # And then we really deploy the file on our node.
+  check(node) + [
+    {
+      remote_bash: <<~EOS
+        mkdir -p ~/hpc_tutorial/node
+        cp /tmp/hello_world.txt.wanted /root/hello_world.txt
+      EOS
+    }
+  ]
+end