hybrid_platforms_conductor 32.3.6

data/lib/hybrid_platforms_conductor/hpc_plugins/platform_handler/platform_handler_plugin.rb.sample

@@ -0,0 +1,292 @@
+require 'hybrid_platforms_conductor/platform_handler'
+
+module HybridPlatformsConductor
+
+  module HpcPlugins
+
+    module PlatformHandler
+
+      # Brief description of the plugin here.
+      # The name should match the file name.
+      # It should always inherit from PlatformHandler.
+      # In all instance methods of a platform handler, the following variables are accessible:
+      # * @platform_type (Symbol): The platform type. It is the name of the plugin (:platform_handler_plugin)
+      # * @repository_path (String): The repository on which this platform handler instance will perform (/path/to/repository)
+      # * @nodes_handler (NodesHandler): Nodes handler used to access information about the platforms
+      class PlatformHandlerPlugin < HybridPlatformsConductor::PlatformHandler
+
+        # Are dependencies met before using this plugin?
+        # [API] - This method is optional
+        #
+        # Result::
+        # * Boolean: Are dependencies met before using this plugin?
+        def self.valid?
+          true
+        end
+
+        # Add a Mixin to the DSL parsing the platforms configuration file.
+        # This can be used by any plugin to add plugin-specific configuration getters and setters, accessible later from NodesHandler instances.
+        # An optional initializer can also be given.
+        # [API] - Those calls are optional
+        module MyDSLExtension
+
+          attr_accessor :my_property
+
+          # Initialize the DSL
+          def init_my_dsl_extension
+            @my_property = 42
+          end
+
+        end
+        self.extend_config_dsl_with MyDSLExtension, :init_my_dsl_extension
+
+        # Register test classes at the platform level.
+        # This will be called for each platform of this type.
+        # [API] - This method is optional
+        #
+        # Result::
+        # * Hash<Symbol,Class>: A list of tests classes (that should inherit from Tests::Test), per test name
+        def tests
+          # If this plugin defines some tests that are specific to this platform, then return the test classes here.
+          # If no tests are written, then no need to define this method at all.
+          # The documentation of those test classes can be found in the file my_test_plugin.rb.sample.
+          # Additionnally to generic test classes, the ones that are defined from specific platform handler plugins (like here) have the following differences:
+          # * Global tests (using the test instance method) are run once per platform of the matching kind.
+          # * Node-specific tests (using the test_for_node, test_on_node and test_on_check_node methods) are run only on nodes belonging to a platform of the matching kind.
+          # * All instance methods have access to the variable @repository_path that can be used to reference the specific repository for which tests are being run.
+          {
+            "#{name}_my_test1": HybridPlatformsConductor::Tests::MyTestClass1,
+            "#{name}_my_test2": HybridPlatformsConductor::Tests::MyTestClass2
+          }
+        end
+
+        # Initialize a new instance of this platform handler.
+        # [API] - This method is optional.
+        # [API] - @cmd_runner is accessible.
+        def init
+          # This method is called when initializing a new instance of this platform handler, for a given repository.
+          @my_instance_variable = :initial_value
+        end
+
+        # Setup the platform, install dependencies...
+        # [API] - This method is mandatory.
+        # [API] - @cmd_runner is accessible.
+        def setup
+          # This method is called by the setup executable.
+          # It should contain any command that sets up a repository.
+          @cmd_runner.run_cmd "cd #{@repository_path} && bundle install --path vendor/bundle --binstubs"
+        end
+
+        # Get the list of known nodes.
+        # [API] - This method is mandatory.
+        #
+        # Result::
+        # * Array<String>: List of node names
+        def known_nodes
+          # This method is used to get the list of nodes that are handled by the platform
+          [
+            'my_node_1',
+            'my_node_2',
+            'my_node_3'
+          ]
+        end
+
+        # Get the list of known nodes lists names.
+        # [API] - This method is optional.
+        #
+        # Result::
+        # * Array<String>: List of nodes lists' names
+        def known_nodes_lists
+          # This method is used to get the list of nodes lists that are handled by the platform.
+          # A nodes list is just a group of nodes selectors that can then be used to ease command line interfaces.
+          [
+            'my_nodes_list_1',
+            'my_nodes_list_2'
+          ]
+        end
+
+        # Get the list of nodes selectors belonging to a nodes list
+        # [API] - This method is optional unless known_nodes_lists has been defined.
+        #
+        # Parameters::
+        # * *nodes_list* (String): Name of the nodes list
+        # Result::
+        # * Array<Object>: List of nodes selectors
+        def nodes_selectors_from_nodes_list(nodes_list)
+          # If known_nodes_lists has been defined, then this method should return the list of nodes selectors of a given nodes list.
+          case nodes_list
+          when 'my_nodes_list_1'
+            [
+              'my_node_1',
+              'my_node_2'
+            ]
+          when 'my_nodes_list_2'
+            [
+              '/my_node.*/'
+            ]
+          else
+            raise "Unknown list name #{nodes_list}"
+          end
+        end
+
+        # Get the metadata of a given node.
+        # [API] - This method is mandatory.
+        #
+        # Parameters::
+        # * *node* (String): Node to read metadata from
+        # Result::
+        # * Hash<Symbol,Object>: The corresponding metadata
+        def metadata_for(node)
+          {
+            french_name: "Le noeud #{node}"
+          }
+        end
+
+        # Return the services for a given node
+        # [API] - This method is mandatory.
+        #
+        # Parameters::
+        # * *node* (String): node to read configuration from
+        # Result::
+        # * Array<String>: The corresponding services
+        def services_for(node)
+          ['hadoop_worker']
+        end
+
+        # Get the list of services we can deploy
+        # [API] - This method is mandatory.
+        #
+        # Result::
+        # * Array<String>: The corresponding services
+        def deployable_services
+          ['hadoop_worker']
+        end
+
+        # Package the repository, ready to be deployed on artefacts or directly to a node.
+        # [API] - This method is mandatory.
+        # [API] - @cmd_runner is accessible.
+        # [API] - @actions_executor is accessible.
+        #
+        # Parameters::
+        # * *services* (Hash< String, Array<String> >): Services to be deployed, per node
+        # * *secrets* (Hash): Secrets to be used for deployment
+        # * *local_environment* (Boolean): Are we deploying to a local environment?
+        def package(services:, secrets:, local_environment:)
+          # This method should take all actions to prepare the repository to be deployed on nodes later.
+          File.write('temporary_secrets_to_be_deployed.json', secrets)
+          # Usually it is meant to package the deployment scripts.
+          @cmd_runner.run_cmd "cd #{@repository_path} && ./scripts/package_in_debian_format.sh"
+        end
+
+        # Prepare deployments.
+        # This method is called just before getting and executing the actions to be deployed.
+        # It is called once per platform.
+        # [API] - This method is optional.
+        # [API] - @cmd_runner is accessible.
+        # [API] - @actions_executor is accessible.
+        #
+        # Parameters::
+        # * *services* (Hash< String, Array<String> >): Services to be deployed, per node
+        # * *secrets* (Hash): Secrets to be used for deployment
+        # * *local_environment* (Boolean): Are we deploying to a local environment?
+        # * *why_run* (Boolean): Are we deploying in why-run mode?
+        def prepare_for_deploy(services:, secrets:, local_environment:, why_run:)
+          # This method is used if some actions are needed before deploying for real.
+          # The why_run switch is on if the deployment should just be simulated.
+          ENV['deploy_time'] = Time.now.to_s
+        end
+
+        # Get the list of actions to perform to deploy on a given node.
+        # Those actions can be executed in parallel with other deployments on other nodes. They must be thread safe.
+        # [API] - This method is mandatory.
+        # [API] - @cmd_runner is accessible.
+        # [API] - @actions_executor is accessible.
+        #
+        # Parameters::
+        # * *node* (String): Node to deploy on
+        # * *service* (String): Service to be deployed
+        # * *use_why_run* (Boolean): Do we use a why-run mode? [default = true]
+        # Result::
+        # * Array< Hash<Symbol,Object> >: List of actions to be done
+        def actions_to_deploy_on(node, service, use_why_run: true)
+          # This method returns all the actions to execute to deploy on a node.
+          # The use_why_run switch is on if the deployment should just be simulated.
+          # Those actions (bash commands, scp of files, ruby code...) should be thread safe as they can be executed in parallel with other deployment actions for other nodes in case of a concurrent deployment on several nodes.
+          # The complete description of an action can be found in actions_executor.rb file, in the execute_actions_on method description.
+          [
+            {
+              scp: {
+                "#{File.dirname(__FILE__)}/platform_handler_plugin/scripts" => './deploy_scripts',
+                'temporary_secrets_to_be_deployed.json' => './temporary_secrets_to_be_deployed.json'
+              },
+              remote_bash: [
+                'cd ./deploy_scripts',
+                "./deploy_service #{service}",
+                'rm temporary_secrets_to_be_deployed.json'
+              ]
+            }
+          ]
+        end
+
+        # Prepare a why-run deployment so that a JSON file describing the nodes will be output in the run_logs.
+        # [API] - This method is mandatory.
+        # [API] - @cmd_runner is accessible.
+        # [API] - @actions_executor is accessible.
+        # [API] - @deployer is accessible.
+        def prepare_why_run_deploy_for_json_dump
+          # This method is used to prepare a deployment action that should dump JSON info from the node itself.
+          @dump_json_mode = true
+        end
+
+        # Parse stdout and stderr of a given deploy run and get the list of tasks with their status
+        # [API] - This method is mandatory.
+        #
+        # Parameters::
+        # * *stdout* (String): stdout to be parsed
+        # * *stderr* (String): stderr to be parsed
+        # Result::
+        # * Array< Hash<Symbol,Object> >: List of task properties. The following properties should be returned, among free ones:
+        #   * *name* (String): Task name
+        #   * *status* (Symbol): Task status. Should be on of:
+        #     * *:changed*: The task has been changed
+        #     * *:identical*: The task has not been changed
+        #   * *diffs* (String): Differences, if any
+        def parse_deploy_output(stdout, stderr)
+          tasks = []
+          stdout.split("\n").each do |line|
+            if line =~ /Task (.+) status - (.+)/
+              tasks << {
+                name: $1,
+                status: $2 == 'ok' ? :identical : :changed
+              }
+            end
+          end
+          tasks
+        end
+
+        # Get the list of impacted nodes and services from a files diff.
+        # [API] - This method is optional
+        #
+        # Parameters::
+        # * *files_diffs* (Hash< String, Hash< Symbol, Object > >): List of diffs info, per file name having a diff. Diffs info have the following properties:
+        #   * *moved_to* (String): The new file path, in case it has been moved [optional]
+        #   * *diff* (String): The diff content
+        # Result::
+        # * Array<String>: The list of nodes impacted by this diff
+        # * Array<String>: The list of services impacted by this diff
+        # * Boolean: Are there some files that have a global impact (meaning all nodes are potentially impacted by this diff)?
+        def impacts_from(files_diffs)
+          [
+            files_diffs.keys.select { |file| file.match(/nodes\/(.+)/)[1] }.compact.uniq,
+            files_diffs.keys.select { |file| file.match(/services\/(.+)/)[1] }.compact.uniq,
+            false
+          ]
+        end
+
+      end
+
+    end
+
+  end
+
+end

data/lib/hybrid_platforms_conductor/hpc_plugins/provisioner/docker.rb

@@ -0,0 +1,148 @@
+require 'docker-api'
+require 'hybrid_platforms_conductor/provisioner'
+
+module HybridPlatformsConductor
+
+  module HpcPlugins
+
+    module Provisioner
+
+      # Provision Docker containers
+      class Docker < HybridPlatformsConductor::Provisioner
+
+        # Are dependencies met before using this plugin?
+        # [API] - This method is optional
+        #
+        # Result::
+        # * Boolean: Are dependencies met before using this plugin?
+        def self.valid?
+          docker_ok = false
+          begin
+            ::Docker.validate_version!
+            docker_ok = true
+          rescue
+            log_error "[ #{@node}/#{@environment} ] - Docker is not installed correctly. Please install it. Error: #{$!}"
+          end
+          docker_ok
+        end
+
+        # Create an instance.
+        # Reuse an existing one if it already exists.
+        # [API] - This method is mandatory
+        def create
+          # Get the image name for this node
+          image = @nodes_handler.get_image_of(@node).to_sym
+          # Find if we have such an image registered
+          if @config.known_os_images.include?(image)
+            # Build the image if it does not exist
+            image_tag = "hpc_image_#{image}"
+            docker_image = nil
+            image_futex_file = "#{Dir.tmpdir}/hpc_docker_image_futexes/#{image_tag}"
+            FileUtils.mkdir_p File.dirname(image_futex_file)
+            Futex.new(image_futex_file).open do
+              docker_image = ::Docker::Image.all.find { |search_image| !search_image.info['RepoTags'].nil? && search_image.info['RepoTags'].include?("#{image_tag}:latest") }
+              unless docker_image
+                log_debug "[ #{@node}/#{@environment} ] - Creating Docker image #{image_tag}..."
+                Excon.defaults[:read_timeout] = 600
+                docker_image = ::Docker::Image.build_from_dir(@config.os_image_dir(image))
+                docker_image.tag repo: image_tag
+              end
+            end
+            container_name = "hpc_docker_container_#{@node}_#{@environment}"
+            container_futex_file = "#{Dir.tmpdir}/hpc_docker_container_futexes/#{image_tag}"
+            FileUtils.mkdir_p File.dirname(container_futex_file)
+            Futex.new(container_futex_file).open do
+              old_docker_container = ::Docker::Container.all(all: true).find { |container| container.info['Names'].include? "/#{container_name}" }
+              @container =
+                if old_docker_container
+                  old_docker_container
+                else
+                  log_debug "[ #{@node}/#{@environment} ] - Creating Docker container #{container_name}..."
+                  # We add the SYS_PTRACE capability as some images need to restart services (for example postfix) and those services need the rights to ls in /proc/{PID}/exe to check if a status is running. Without SYS_PTRACE such ls returns permission denied and the service can't be stopped (as init.d always returns it as stopped even when running).
+                  # We add the privileges as some containers need to install and configure the udev package, which needs RW access to /sys.
+                  # We add the bind to cgroup volume to be able to test systemd specifics (enabling/disabling services for example).
+                  ::Docker::Container.create(
+                    name: container_name,
+                    image: image_tag,
+                    CapAdd: 'SYS_PTRACE',
+                    Privileged: true,
+                    Binds: ['/sys/fs/cgroup:/sys/fs/cgroup:ro'],
+                    # Some playbooks need the hostname to be set to a correct FQDN
+                    Hostname: "#{@node}.testdomain"
+                  )
+                end
+            end
+          else
+            raise "[ #{@node}/#{@environment} ] - Unknown OS image #{image} defined for node #{@node}"
+          end
+        end
+
+        # Start an instance
+        # Prerequisite: create has been called before
+        # [API] - This method is mandatory
+        def start
+          log_debug "[ #{@node}/#{@environment} ] - Start Docker Container #{@container.refresh!.info['Name'][1..-1]} ..."
+          @container.start
+        end
+
+        # Stop an instance
+        # Prerequisite: create has been called before
+        # [API] - This method is mandatory
+        def stop
+          log_debug "[ #{@node}/#{@environment} ] - Stop Docker Container #{@container.refresh!.info['Name'][1..-1]} ..."
+          @container.stop
+        end
+
+        # Destroy an instance
+        # Prerequisite: create has been called before
+        # [API] - This method is mandatory
+        def destroy
+          log_debug "[ #{@node}/#{@environment} ] - Destroy Docker Container #{@container.refresh!.info['Name'][1..-1]} ..."
+          @container.remove
+          @container = nil
+        end
+
+        # Return the state of an instance
+        # [API] - This method is mandatory
+        #
+        # Result::
+        # * Symbol: The state the instance is in. Possible values are:
+        #   * *:missing*: The instance does not exist
+        #   * *:created*: The instance has been created but is not running
+        #   * *:running*: The instance is running
+        #   * *:exited*: The instance has run and is now stopped
+        #   * *:error*: The instance is in error
+        def state
+          if !defined?(@container) || @container.nil?
+            :missing
+          else
+            begin
+              @container.refresh!.info['State']['Status'].to_sym
+            rescue
+              log_error "[ #{@node}/#{@environment} ] - Error while reading state of Docker container: #{$!}"
+              :error
+            end
+          end
+        end
+
+        # Return the IP address of an instance.
+        # Prerequisite: create has been called before.
+        # [API] - This method is optional
+        #
+        # Result::
+        # * String or nil: The instance IP address, or nil if this information is not relevant
+        def ip
+          # Get its IP that could have changed upon restart
+          # cf https://github.com/moby/moby/issues/2801
+          # Make sure we refresh its info before querying it, as we could hit a cache of a previous IP.
+          @container.refresh!
+          @container.json['NetworkSettings']['IPAddress']
+        end
+
+      end
+
+    end
+
+  end
+
+end

data/lib/hybrid_platforms_conductor/hpc_plugins/provisioner/my_provisioner.rb.sample

@@ -0,0 +1,103 @@
+require 'hybrid_platforms_conductor/provisioner'
+
+module HybridPlatformsConductor
+
+  module HpcPlugins
+
+    module Provisioner
+
+      class MyProvisioner < HybridPlatformsConductor::Provisioner
+
+        # Are dependencies met before using this plugin?
+        # [API] - This method is optional
+        #
+        # Result::
+        # * Boolean: Are dependencies met before using this plugin?
+        def self.valid?
+          true
+        end
+
+        # Add a Mixin to the DSL parsing the platforms configuration file.
+        # This can be used by any plugin to add plugin-specific configuration getters and setters, accessible later from NodesHandler instances.
+        # An optional initializer can also be given.
+        # [API] - Those calls are optional
+        module MyDSLExtension
+
+          attr_accessor :my_property
+
+          # Initialize the DSL
+          def init_my_dsl_extension
+            @my_property = 42
+          end
+
+        end
+        self.extend_config_dsl_with MyDSLExtension, :init_my_dsl_extension
+
+        # Create an instance.
+        # Reuse an existing one if it already exists.
+        # [API] - This method is mandatory
+        def create
+          @vm = InfraAPI.create_vm_for(@nodes_handler.get_image_of(@node))
+        end
+
+        # Start an instance
+        # Prerequisite: create has been called before
+        # [API] - This method is mandatory
+        def start
+          @vm.start
+        end
+
+        # Stop an instance
+        # Prerequisite: create has been called before
+        # [API] - This method is mandatory
+        def stop
+          @vm.stop
+        end
+
+        # Destroy an instance
+        # Prerequisite: create has been called before
+        # [API] - This method is mandatory
+        def destroy
+          @vm.remove
+        end
+
+        # Return the state of an instance
+        # [API] - This method is mandatory
+        #
+        # Result::
+        # * Symbol: The state the instance is in. Possible values are:
+        #   * *:missing*: The instance does not exist
+        #   * *:created*: The instance has been created but is not running
+        #   * *:running*: The instance is running
+        #   * *:exited*: The instance has run and is now stopped
+        #   * *:error*: The instance is in error
+        def state
+          @vm.state.to_sym
+        end
+
+        # Return the IP address of an instance.
+        # Prerequisite: create has been called before.
+        # [API] - This method is optional
+        #
+        # Result::
+        # * String or nil: The instance IP address, or nil if this information is not relevant
+        def ip
+          @vm.ip_address
+        end
+
+        # Return the default timeout to apply when waiting for an instance to be started/stopped...
+        # [API] - This method is optional
+        #
+        # Result::
+        # * Integer: The timeout in seconds
+        def default_timeout
+          3600
+        end
+
+      end
+
+    end
+
+  end
+
+end