hybrid_platforms_conductor 32.3.6

data/lib/hybrid_platforms_conductor/common_config_dsl/idempotence_tests.rb

@@ -0,0 +1,38 @@
+module HybridPlatformsConductor
+
+  module CommonConfigDsl
+
+    # Config DSL configuring idempotence testing (used by different test plugins)
+    module IdempotenceTests
+
+      # List of ignored tasks info. Each info has the following properties:
+      # * *nodes_selectors_stack* (Array<Object>): Stack of nodes selectors impacted by this rule
+      # * *ignored_tasks* (Hash<String, String>): List of task names for which we ignore idempotence errors, with the corresponding descriptive reason for ignore.
+      # Array< Hash<Symbol, Object> >
+      attr_reader :ignored_idempotence_tasks
+
+      # Initialize the DSL 
+      def init_idempotence_tests
+        # List of ignored tasks info. Each info has the following properties:
+        # * *nodes_selectors_stack* (Array<Object>): Stack of nodes selectors impacted by this rule
+        # * *ignored_tasks* (Hash<String, String>): List of task names for which we ignore idempotence errors, with the corresponding descriptive reason for ignore.
+        # Array< Hash<Symbol, Object> >
+        @ignored_idempotence_tasks = []
+      end
+
+      # Ignore idempotence errors on a set of tasks
+      #
+      # Parameters::
+      # * *tasks_to_ignore* (Hash<String, String>): Set of tasks to ignore, along with the reason
+      def ignore_idempotence_tasks(tasks_to_ignore)
+        @ignored_idempotence_tasks << {
+          ignored_tasks: tasks_to_ignore,
+          nodes_selectors_stack: current_nodes_selectors_stack,
+        }
+      end
+
+    end
+
+  end
+
+end

data/lib/hybrid_platforms_conductor/config.rb

@@ -0,0 +1,263 @@
+require 'cleanroom'
+require 'git'
+require 'ice_cube'
+require 'hybrid_platforms_conductor/plugins'
+
+module HybridPlatformsConductor
+
+  # Object used to access the whole configuration
+  class Config
+
+    include LoggerHelpers, Cleanroom
+
+    class << self
+
+      # Array<Symbol>: List of mixin initializers to call
+      attr_accessor :mixin_initializers
+
+      # Extend the config DSL used when parsing the hpc_config.rb file with a given Mixin.
+      # This can be used by any plugin to add plugin-specific configuration in the hpc_config.rb file.
+      #
+      # Parameters::
+      # * *mixin* (Module): Mixin to add to the Platforms DSL
+      # * *init_method* (Symbol or nil): The initializer method of this Mixin, or nil if none [default = nil]
+      def extend_config_dsl_with(mixin, init_method = nil)
+        include mixin
+        @mixin_initializers << init_method unless init_method.nil?
+        mixin.instance_methods.each do |method_name|
+          expose method_name unless method_name == init_method
+        end
+      end
+
+    end
+    @mixin_initializers = []
+
+    # Directory of the definition of the platforms
+    #   String
+    attr_reader :hybrid_platforms_dir
+    expose :hybrid_platforms_dir
+
+    # List of expected failures info. Each info has the following properties:
+    # * *nodes_selectors_stack* (Array<Object>): Stack of nodes selectors impacted by this expected failure
+    # * *tests* (Array<Symbol>): List of tests impacted by this expected failre
+    # * *reason* (String): Reason for this expected failure
+    # Array<Hash,Symbol,Object>
+    attr_reader :expected_failures
+
+    # List of retriable errors. Each info has the following properties:
+    # * *nodes_selectors_stack* (Array<Object>): Stack of nodes selectors impacted by those errors
+    # * *errors_on_stdout* (Array<String or Regexp>): List of errors match (as exact string match or using a regexp) to check against stdout
+    # * *errors_on_stderr* (Array<String or Regexp>): List of errors match (as exact string match or using a regexp) to check against stderr
+    attr_reader :retriable_errors
+
+    # List of deployment schedules. Each info has the following properties:
+    # * *nodes_selectors_stack* (Array<Object>): Stack of nodes selectors impacted by this rule
+    # * *schedule* (IceCube::Schedule): The deployment schedule
+    attr_reader :deployment_schedules
+
+    # Constructor
+    #
+    # Parameters::
+    # * *logger* (Logger): Logger to be used [default = Logger.new(STDOUT)]
+    # * *logger_stderr* (Logger): Logger to be used for stderr [default = Logger.new(STDERR)]
+    def initialize(logger: Logger.new(STDOUT), logger_stderr: Logger.new(STDERR))
+      init_loggers(logger, logger_stderr)
+      @hybrid_platforms_dir = File.expand_path(ENV['hpc_platforms'].nil? ? '.' : ENV['hpc_platforms'])
+      # Stack of the nodes selectors scopes
+      # Array< Object >
+      @nodes_selectors_stack = []
+      # List of OS image directories, per image name
+      # Hash<Symbol, String>
+      @os_images = {}
+      # Plugin ID of the tests provisioner
+      # Symbol
+      @tests_provisioner = :docker
+      # List of expected failures info. Each info has the following properties:
+      # * *nodes_selectors_stack* (Array<Object>): Stack of nodes selectors impacted by this expected failure
+      # * *tests* (Array<Symbol>): List of tests impacted by this expected failre
+      # * *reason* (String): Reason for this expected failure
+      # Array<Hash,Symbol,Object>
+      @expected_failures = []
+      # List of retriable errors. Each info has the following properties:
+      # * *nodes_selectors_stack* (Array<Object>): Stack of nodes selectors impacted by those errors
+      # * *errors_on_stdout* (Array<String or Regexp>): List of errors match (as exact string match or using a regexp) to check against stdout
+      # * *errors_on_stderr* (Array<String or Regexp>): List of errors match (as exact string match or using a regexp) to check against stderr
+      @retriable_errors = []
+      # List of deployment schedules. Each info has the following properties:
+      # * *nodes_selectors_stack* (Array<Object>): Stack of nodes selectors impacted by this rule
+      # * *schedule* (IceCube::Schedule): The deployment schedule
+      @deployment_schedules = []
+      # Make sure plugins can decorate our DSL with their owns additions as well
+      # Therefore we parse all possible plugin types
+      Dir.glob("#{__dir__}/hpc_plugins/*").each do |plugin_dir|
+        Plugins.new(File.basename(plugin_dir).to_sym, logger: @logger, logger_stderr: @logger_stderr)
+      end
+      # Call initializers if needed
+      Config.mixin_initializers.each do |mixin_init_method|
+        self.send(mixin_init_method)
+      end
+      include_config_from "#{@hybrid_platforms_dir}/hpc_config.rb"
+    end
+
+    # Include configuration from a DSL config file
+    #
+    # Parameters::
+    # * *dsl_file* (String): Path to the DSL file
+    def include_config_from(dsl_file)
+      log_debug "Include config from #{dsl_file}"
+      self.evaluate_file(dsl_file)
+    end
+    expose :include_config_from
+
+    # Register a new OS image
+    #
+    # Parameters::
+    # * *image* (Symbol): Name of the Docker image
+    # * *dir* (String): Directory containing the Dockerfile defining the image
+    def os_image(image, dir)
+      raise "OS image #{image} already defined to #{@os_images[image]}" if @os_images.key?(image)
+      @os_images[image] = dir
+    end
+    expose :os_image
+
+    # Set which provisioner should be used for tests
+    #
+    # Parameters::
+    # * *provisioner* (Symbol): Plugin ID of the provisioner to be used for tests
+    def tests_provisioner(provisioner)
+      @tests_provisioner = provisioner
+    end
+    expose :tests_provisioner
+
+    # Limit the scope of configuration to a given set of nodes
+    #
+    # Parameters::
+    # * *nodes_selectors* (Object): Nodes selectors, as defined by the NodesHandler#select_nodes method (check its signature for details)
+    # Proc: DSL code called in the context of those selected nodes
+    def for_nodes(nodes_selectors)
+      @nodes_selectors_stack << nodes_selectors
+      begin
+        yield
+      ensure
+        @nodes_selectors_stack.pop
+      end
+    end
+    expose :for_nodes
+
+    # Mark some tests as expected failures.
+    #
+    # Parameters::
+    # * *tests* (Symbol or Array<Symbol>): List of tests expected to fail.
+    # * *reason* (String): Descriptive reason for the failure
+    def expect_tests_to_fail(tests, reason)
+      @expected_failures << {
+        tests: tests.is_a?(Array) ? tests : [tests],
+        nodes_selectors_stack: current_nodes_selectors_stack,
+        reason: reason
+      }
+    end
+    expose :expect_tests_to_fail
+
+    # Mark some errors on stdout to be retriable during a deploy
+    #
+    # Parameters::
+    # * *errors* (String, Regexp or Array<String or Regexp>): Single (or list of) errors matching pattern (either as exact string match or using a regexp).
+    def retry_deploy_for_errors_on_stdout(errors)
+      @retriable_errors << {
+        errors_on_stdout: errors.is_a?(Array) ? errors : [errors],
+        nodes_selectors_stack: current_nodes_selectors_stack
+      }
+    end
+    expose :retry_deploy_for_errors_on_stdout
+
+    # Mark some errors on stderr to be retriable during a deploy
+    #
+    # Parameters::
+    # * *errors* (String, Regexp or Array<String or Regexp>): Single (or list of) errors matching pattern (either as exact string match or using a regexp).
+    def retry_deploy_for_errors_on_stderr(errors)
+      @retriable_errors << {
+        errors_on_stderr: errors.is_a?(Array) ? errors : [errors],
+        nodes_selectors_stack: current_nodes_selectors_stack
+      }
+    end
+    expose :retry_deploy_for_errors_on_stderr
+
+    # Set a deployment schedule
+    #
+    # Parameters::
+    # * *schedule* (IceCube::Schedule): The deployment schedule
+    def deployment_schedule(schedule)
+      @deployment_schedules << {
+        schedule: schedule,
+        nodes_selectors_stack: current_nodes_selectors_stack
+      }
+    end
+    expose :deployment_schedule
+
+    # Helper to get a daily schedule at a given time
+    #
+    # Parameters::
+    # * *time* (String): Time (UTC) for the daily schedule
+    # * *duration* (Integer): Number of seconds of duration [default: 3000]
+    # Result::
+    # * IceCube::Schedule: Corresponding schedule
+    def daily_at(time, duration: 3000)
+      IceCube::Schedule.new(Time.parse("2020-01-01 #{time} UTC"), duration: duration) do |s|
+        s.add_recurrence_rule(IceCube::Rule.daily)
+      end
+    end
+    expose :daily_at
+
+    # Helper to get a weekly schedule at a given day and time
+    #
+    # Parameters::
+    # * *days* (Symbol or Array<Symbol>): Days for the weekly schedule (see IceCube::Rule documentation to know day names)
+    # * *time* (String): Time (UTC) for the weekly schedule
+    # * *duration* (Integer): Number of seconds of duration [default: 3000]
+    # Result::
+    # * IceCube::Schedule: Corresponding schedule
+    def weekly_at(days, time, duration: 3000)
+      days = [days] unless days.is_a?(Array)
+      IceCube::Schedule.new(Time.parse("2020-01-01 #{time} UTC"), duration: duration) do |s|
+        s.add_recurrence_rule(IceCube::Rule.weekly.day(*days))
+      end
+    end
+    expose :weekly_at
+
+    # Get the current nodes selector stack.
+    #
+    # Result::
+    # * Array<Object>: Nodes selectors stack
+    def current_nodes_selectors_stack
+      @nodes_selectors_stack.clone
+    end
+
+    # Get the list of known Docker images
+    #
+    # Result::
+    # * Array<Symbol>: List of known Docker images
+    def known_os_images
+      @os_images.keys
+    end
+
+    # Get the directory containing a Docker image
+    #
+    # Parameters::
+    # * *image* (Symbol): Image name
+    # Result::
+    # * String: Directory containing the Dockerfile of the image
+    def os_image_dir(image)
+      @os_images[image]
+    end
+
+    # Name of the provisioner to be used for tests
+    #
+    # Result::
+    # * Symbol: Provisioner to be used for tests
+    def tests_provisioner_id
+      @tests_provisioner
+    end
+
+  end
+
+end

data/lib/hybrid_platforms_conductor/confluence.rb

@@ -0,0 +1,119 @@
+require 'json'
+require 'net/http'
+require 'nokogiri'
+require 'uri'
+require 'hybrid_platforms_conductor/logger_helpers'
+require 'hybrid_platforms_conductor/credentials'
+
+module HybridPlatformsConductor
+
+  # Object used to access Confluence API
+  class Confluence
+
+    include LoggerHelpers
+
+    # Provide a Confluence connector, and make sure the password is being cleaned when exiting.
+    #
+    # Parameters::
+    # * *confluence_url* (String): The Confluence URL
+    # * *logger* (Logger): Logger to be used
+    # * *logger_stderr* (Logger): Logger to be used for stderr
+    # * Proc: Code called with the Confluence instance.
+    #   * *confluence* (Confluence): The Confluence instance to use.
+    def self.with_confluence(confluence_url, logger, logger_stderr)
+      Credentials.with_credentials_for(:confluence, logger, logger_stderr, url: confluence_url) do |confluence_user, confluence_password|
+        yield Confluence.new(confluence_url, confluence_user, confluence_password, logger: logger, logger_stderr: logger_stderr)
+      end
+    end
+
+    # Constructor
+    #
+    # Parameters::
+    # * *confluence_url* (String): The Confluence URL
+    # * *confluence_user_name* (String): Confluence user name to be used when querying the API
+    # * *confluence_password* (String): Confluence password to be used when querying the API
+    # * *logger* (Logger): Logger to be used [default = Logger.new(STDOUT)]
+    # * *logger_stderr* (Logger): Logger to be used for stderr [default = Logger.new(STDERR)]
+    def initialize(confluence_url, confluence_user_name, confluence_password, logger: Logger.new(STDOUT), logger_stderr: Logger.new(STDERR))
+      init_loggers(logger, logger_stderr)
+      @confluence_url = confluence_url
+      @confluence_user_name = confluence_user_name
+      @confluence_password = confluence_password
+    end
+
+    # Return a Confluence storage format content from a page ID
+    #
+    # Parameters::
+    # * *page_id* (String): Confluence page ID
+    # Result::
+    # * Nokogiri::HTML: Storage format content, as a Nokogiri object
+    def page_storage_format(page_id)
+      Nokogiri::HTML(call_api("plugins/viewstorage/viewpagestorage.action?pageId=#{page_id}").body)
+    end
+
+    # Return some info of a given page ID
+    #
+    # Parameters::
+    # * *page_id* (String): Confluence page ID
+    # Result::
+    # * Hash: Page information, as returned by the Confluence API
+    def page_info(page_id)
+      JSON.parse(call_api("rest/api/content/#{page_id}").body)
+    end
+
+    # Update a Confluence page to a new content.
+    #
+    # Parameters::
+    # * *page_id* (String): Confluence page ID
+    # * *content* (String): New content
+    # * *version* (String or nil): New version, or nil to automatically increase last existing version [default: nil]
+    def update_page(page_id, content, version: nil)
+      info = page_info(page_id)
+      version = info['version']['number'] + 1 if version.nil?
+      log_debug "Update Confluence page #{page_id}..."
+      call_api("rest/api/content/#{page_id}", :put) do |request|
+        request['Content-Type'] = 'application/json'
+        request.body = {
+          type: 'page',
+          title: info['title'],
+          body: {
+            storage: {
+              value: content,
+              representation: 'storage'
+            }
+          },
+          version: { number: version }
+        }.to_json
+      end
+    end
+
+    private
+
+    # Call the Confluence API for a given URL and HTTP verb.
+    # Provide a simple way to tweak the request with an optional proc.
+    # Automatically handles authentication, base URL and error handling.
+    #
+    # Parameters::
+    # * *api_path* (String): The API path to query
+    # * *http_method* (Symbol): HTTP method to be used to create the request [default = :get]
+    # * Proc: Optional code called to alter the request
+    #   * Parameters::
+    #     * *request* (Net::HTTPRequest): The request
+    # Result::
+    # * Net::HTTPResponse: The corresponding response
+    def call_api(api_path, http_method = :get)
+      response = nil
+      page_url = URI.parse("#{@confluence_url}/#{api_path}")
+      Net::HTTP.start(page_url.host, page_url.port, use_ssl: true) do |http|
+        request = Net::HTTP.const_get(http_method.to_s.capitalize.to_sym).new(page_url.request_uri)
+        request.basic_auth @confluence_user_name, @confluence_password
+        yield request if block_given?
+        response = http.request(request)
+        raise "Confluence page API request on #{page_url} returned an error: #{response.code}\n#{response.body}\n===== Request body =====\n#{request.body}" unless response.is_a?(Net::HTTPSuccess)
+      end
+      response
+    end
+
+  end
+
+end

data/lib/hybrid_platforms_conductor/connector.rb

@@ -0,0 +1,84 @@
+require 'hybrid_platforms_conductor/logger_helpers'
+require 'hybrid_platforms_conductor/plugin'
+
+module HybridPlatformsConductor
+
+  # Base class for any connector
+  class Connector < Plugin
+
+    # Constructor
+    #
+    # Parameters::
+    # * *logger* (Logger): Logger to be used [default: Logger.new(STDOUT)]
+    # * *logger_stderr* (Logger): Logger to be used for stderr [default: Logger.new(STDERR)]
+    # * *config* (Config): Config to be used. [default: Config.new]
+    # * *cmd_runner* (CmdRunner): Command executor to be used. [default: CmdRunner.new]
+    # * *nodes_handler* (NodesHandler): NodesHandler to be used. [default: NodesHandler.new]
+    def initialize(
+      logger: Logger.new(STDOUT),
+      logger_stderr: Logger.new(STDERR),
+      config: Config.new,
+      cmd_runner: CmdRunner.new,
+      nodes_handler: NodesHandler.new
+    )
+      super(logger: logger, logger_stderr: logger_stderr, config: config)
+      @cmd_runner = cmd_runner
+      @nodes_handler = nodes_handler
+      # If the connector has an initializer, use it
+      init if respond_to?(:init)
+    end
+
+    # Prepare a connector to be run for a given node in a given context.
+    # It is required to call this method before using the following methods:
+    # * remote_bash
+    # * run_cmd
+    #
+    # Paramaters::
+    # * *node* (String): The node this connector is currently targeting
+    # * *timeout* (Integer or nil): Timeout this connector's process should have (in seconds), or nil if none
+    # * *stdout_io* (IO): IO to log stdout to
+    # * *stderr_io* (IO): IO to log stderr to
+    def prepare_for(node, timeout, stdout_io, stderr_io)
+      @node = node
+      @timeout = timeout
+      @stdout_io = stdout_io
+      @stderr_io = stderr_io
+    end
+
+    # Prepare connections to a given set of nodes.
+    # Useful to prefetch metadata or open bulk connections.
+    #
+    # Parameters::
+    # * *nodes* (Array<String>): Nodes to prepare the connection to
+    # * *no_exception* (Boolean): Should we still continue if some nodes have connection errors? [default: false]
+    # * Proc: Code called with the connections prepared.
+    #   * Parameters::
+    #     * *connected_nodes* (Array<String>): The list of connected nodes (should be equal to nodes unless no_exception == true and some nodes failed to connect)
+    def with_connection_to(nodes, no_exception: false)
+      yield nodes
+    end
+
+    private
+
+    # Run a command.
+    # Handle the redirection of standard output and standard error to file and stdout depending on the context of the run.
+    #
+    # Parameters::
+    # * *cmd* (String): The command to be run
+    # Result::
+    # * Integer: Exit code
+    # * String: Standard output
+    # * String: Error output
+    def run_cmd(cmd)
+      @cmd_runner.run_cmd(
+        cmd,
+        timeout: @timeout,
+        log_to_stdout: false,
+        log_stdout_to_io: @stdout_io,
+        log_stderr_to_io: @stderr_io
+      )
+    end
+
+  end
+
+end