hybrid_platforms_conductor 32.3.6

data/lib/hybrid_platforms_conductor/parallel_threads.rb

@@ -0,0 +1,97 @@
+require 'thread'
+
+module HybridPlatformsConductor
+
+  # Provide utilities to handle parallel threads
+  module ParallelThreads
+
+    # Iterate over a list of objects.
+    # Provide a mechanism to multithread this iteration (in such case the iterating code has to be thread-safe).
+    # In case of multithreaded run, a progress bar is being displayed.
+    #
+    # Parameters::
+    # * *list* (Array<Object>): List of objects to iterate over
+    # * *parallel* (Boolean): Iterate in a multithreaded way? [default: false]
+    # * *nbr_threads_max* (Integer or nil): Maximum number of threads to be used in case of parallel, or nil for no limit [default: nil]
+    # * *progress* (String or nil): Name of a progress bar to follow the progression, or nil for no progress bar [default: 'Progress']
+    # * Proc: The code called for each node being iterated on.
+    #   * Parameters::
+    #     * *element* (Object): The object
+    def for_each_element_in(list, parallel: false, nbr_threads_max: nil, progress: 'Process')
+      if parallel
+        # Threads to wait for
+        threads_to_join = []
+        # Spread elements evenly among the threads.
+        # Use a shared pool of elements to be handled by threads.
+        pools = {
+          to_process: list.dup,
+          processing: [],
+          processed: []
+        }
+        nbr_total = list.size
+        # Protect access to the pools using a mutex
+        pools_semaphore = Mutex.new
+        # Spawn the threads, each one responsible for handling its list
+        (nbr_threads_max.nil? || nbr_threads_max > nbr_total ? nbr_total : nbr_threads_max).times do
+          threads_to_join << Thread.new do
+            # As exceptions are logged anyway whatever the Thread setting is, only turn it on for debug.
+            # That will keep tests execution cleaner.
+            Thread.current.report_on_exception = log_debug?
+            begin
+              loop do
+                # Modify the list while processing it, so that reporting can be done.
+                element = nil
+                pools_semaphore.synchronize do
+                  element = pools[:to_process].shift
+                  pools[:processing] << element unless element.nil?
+                end
+                break if element.nil?
+                begin
+                  yield element
+                ensure
+                  pools_semaphore.synchronize do
+                    pools[:processing].delete(element)
+                    pools[:processed] << element
+                  end
+                end
+              end
+            rescue
+              log_error "Unhandled exception occurred in thread #{Thread.current.object_id}: #{$!}\n#{$!.backtrace.join("\n")}"
+              raise
+            end
+          end
+        end
+        if progress
+          # Here the main thread just reports progression
+          nbr_to_process = nil
+          nbr_processing = nil
+          nbr_processed = nil
+          with_progress_bar(nbr_total, name: progress) do |progress_bar|
+            loop do
+              pools_semaphore.synchronize do
+                nbr_to_process = pools[:to_process].size
+                nbr_processing = pools[:processing].size
+                nbr_processed = pools[:processed].size
+              end
+              progress_bar.title = "Queue: #{nbr_to_process} - Processing: #{nbr_processing} - Done: #{nbr_processed} - Total: #{nbr_total}"
+              progress_bar.progress = nbr_processed
+              break if nbr_processed == nbr_total
+              sleep 0.5
+            end
+          end
+        end
+        # Wait for threads to be joined
+        threads_to_join.each do |thread|
+          thread.join
+        end
+      else
+        # Execute synchronously
+        list.each do |element|
+          yield element
+        end
+      end
+    end
+
+  end
+
+end

data/lib/hybrid_platforms_conductor/platform_handler.rb

@@ -0,0 +1,188 @@
+require 'json'
+require 'hybrid_platforms_conductor/plugin'
+
+module HybridPlatformsConductor
+
+  # Common ancestor to any platform handler
+  class PlatformHandler < Plugin
+
+    # Make it so that we can sort lists of platforms
+    include Comparable
+
+    # Callback called when a subclass inherits this class.
+    #
+    # Parameters::
+    # * *subclass* (Class): The inheriting class
+    def self.inherited(subclass)
+      # Make sure we define automatically a helper for such a platform
+      mixin = Module.new
+      platform_type = subclass.name.split('::').last.gsub(/([a-z\d])([A-Z])/, '\1_\2').downcase.to_sym
+      mixin.define_method("#{platform_type}_platform".to_sym) do |path: nil, git: nil, branch: 'master', &platform_config_code|
+        repository_path =
+          if !path.nil?
+            path
+          elsif !git.nil?
+            # Clone in a local repository
+            local_repository_path = "#{@git_platforms_dir}/#{File.basename(git)[0..-File.extname(git).size - 1]}"
+            unless File.exist?(local_repository_path)
+              branch = "refs/heads/#{branch}" unless branch.include?('/')
+              local_ref = "refs/remotes/origin/#{branch.split('/').last}"
+              section "Cloning #{git} (#{branch} => #{local_ref}) into #{local_repository_path}" do
+                git_repo = Git.init(local_repository_path, )
+                git_repo.add_remote('origin', git).fetch(ref: "#{branch}:#{local_ref}")
+                git_repo.checkout local_ref
+              end
+            end
+            local_repository_path
+          else
+            raise 'The platform has to be defined with either a path or a git URL'
+          end
+        @platform_dirs[platform_type] = [] unless @platform_dirs.key?(platform_type)
+        @platform_dirs[platform_type] << repository_path
+        platform_config_code.call(repository_path) unless platform_config_code.nil?
+      end
+      # Register this new mixin in the Config DSL
+      extend_config_dsl_with(mixin)
+      super
+    end
+
+    # Repository path
+    #   String
+    attr_reader :repository_path
+
+    # Platform type
+    #   Symbol
+    attr_reader :platform_type
+
+    # Before deploying, need to set some components in case the plugins need them
+    attr_accessor :nodes_handler, :actions_executor
+
+    # Constructor
+    #
+    # Parameters::
+    # * *platform_type* (Symbol): Platform type
+    # * *repository_path* (String): Repository path
+    # * *logger* (Logger): Logger to be used [default: Logger.new(STDOUT)]
+    # * *logger_stderr* (Logger): Logger to be used for stderr [default: Logger.new(STDERR)]
+    # * *config* (Config): Config to be used. [default: Config.new]
+    # * *cmd_runner* (CmdRunner): Command executor to be used. [default: CmdRunner.new]
+    def initialize(
+      platform_type,
+      repository_path,
+      logger: Logger.new(STDOUT),
+      logger_stderr: Logger.new(STDERR),
+      config: Config.new,
+      cmd_runner: CmdRunner.new
+    )
+      super(logger: logger, logger_stderr: logger_stderr, config: config)
+      @platform_type = platform_type
+      @repository_path = repository_path
+      @cmd_runner = cmd_runner
+      self.init if self.respond_to?(:init)
+    end
+
+    # Return the name of the platform
+    #
+    # Result::
+    # * String: Name of the platform
+    def name
+      info[:repo_name]
+    end
+
+    # Get the list of impacted nodes and services from a files diff.
+    # [API] - This is the default implementation, and is meant to be overriden by Platform Handlers.
+    #
+    # Parameters::
+    # * *files_diffs* (Hash< String, Hash< Symbol, Object > >): List of diffs info, per file name having a diff. Diffs info have the following properties:
+    #   * *moved_to* (String): The new file path, in case it has been moved [optional]
+    #   * *diff* (String): The diff content
+    # Result::
+    # * Array<String>: The list of nodes impacted by this diff
+    # * Array<String>: The list of services impacted by this diff
+    # * Boolean: Are there some files that have a global impact (meaning all nodes are potentially impacted by this diff)?
+    def impacts_from(files_diffs)
+      # By default, consider all nodes of the platform are impacted by whatever diff.
+      [
+        [],
+        [],
+        true
+      ]
+    end
+
+    # Get some information from this platform.
+    # This information identifies the code level that is currently checked out.
+    #
+    # Result::
+    # * Hash<Symbol,Object>: Description of this platform:
+    #   * *repo_name* (String): The repository name
+    #   * *commit* (Hash<Symbol,Object>): Information on the checked out Git commit
+    #     * *id* (String): Commit ID
+    #     * *ref* (String): Associated reference
+    #     * *message* (String): Associated message
+    #     * *date* (Time): Commit date in UTC
+    #     * *author* (Hash<Symbol,Object>): Information on the author:
+    #       * *name* (String): Name of the commit author
+    #       * *email* (String): Email of the commit author
+    #  * *status* (Hash<Symbol,Object>): Information on the checked out Git status
+    #    * *changed_files* (Array<String>): List of changed files
+    #    * *added_files* (Array<String>): List of added files
+    #    * *deleted_files* (Array<String>): List of deleted files
+    #    * *untracked_files* (Array<String>): List of untracked files
+    def info
+      # Keep info in a memory cache, so that we don't query git for nothing
+      unless defined?(@info)
+        git = nil
+        begin
+          git = Git.open(@repository_path)
+        rescue
+          log_debug "Platform #{@repository_path} is not a git repository"
+        end
+        @info =
+          if git
+            git_status = git.status
+            git_commit = git.log.first
+            {
+              repo_name: File.basename(git.remotes.first.url).gsub(/\.git$/, ''),
+              commit: {
+                id: git_commit.sha,
+                ref: git_commit.name,
+                message: git_commit.message,
+                date: git_commit.date.utc,
+                author: {
+                  name: git_commit.author.name,
+                  email: git_commit.author.email
+                }
+              },
+              status: {
+                changed_files: git_status.changed.keys,
+                added_files: git_status.added.keys,
+                deleted_files: git_status.deleted.keys,
+                untracked_files: git_status.untracked.keys
+              }
+            }
+          else
+            {
+              repo_name: File.basename(@repository_path)
+            }
+          end
+      end
+      @info
+    end
+
+    # Order relation
+    #
+    # Parameters::
+    # * *other* (Object): Other object to compare to
+    # Result::
+    # * Integer: -1, 0, or +1 depending on whether the receiver is less than, equal to, or greater than the other object
+    def <=>(other)
+      if other.is_a?(PlatformHandler)
+        name <=> other.name
+      else
+        super
+      end
+    end
+
+  end
+
+end

data/lib/hybrid_platforms_conductor/platforms_handler.rb

@@ -0,0 +1,118 @@
+require 'hybrid_platforms_conductor/logger_helpers'
+require 'hybrid_platforms_conductor/platform_handler'
+
+module HybridPlatformsConductor
+
+  # Provide an API to access information given by Platform Handlers
+  class PlatformsHandler
+
+    module ConfigDSLExtension
+
+      # List of platforms repository directories, per platform type
+      #   Hash<Symbol, Array<String> >
+      attr_reader :platform_dirs
+
+      # Mixin initializer
+      def init_platforms_handler
+        @platform_dirs = {}
+        # Directory in which platforms are cloned
+        @git_platforms_dir = "#{@hybrid_platforms_dir}/cloned_platforms"
+      end
+
+    end
+    Config.extend_config_dsl_with ConfigDSLExtension, :init_platforms_handler
+
+    include LoggerHelpers
+
+    # Constructor
+    #
+    # Parameters::
+    # * *logger* (Logger): Logger to be used [default: Logger.new(STDOUT)]
+    # * *logger_stderr* (Logger): Logger to be used for stderr [default: Logger.new(STDERR)]
+    # * *config* (Config): Config to be used. [default: Config.new]
+    # * *cmd_runner* (CmdRunner): Command executor to be used. [default: CmdRunner.new]
+    def initialize(
+      logger: Logger.new(STDOUT),
+      logger_stderr: Logger.new(STDERR),
+      config: Config.new,
+      cmd_runner: CmdRunner.new
+    )
+      init_loggers(logger, logger_stderr)
+      @config = config
+      @cmd_runner = cmd_runner
+      @platform_types = Plugins.new(:platform_handler, logger: @logger, logger_stderr: @logger_stderr)
+      # Keep a list of instantiated platform handlers per platform type
+      # Hash<Symbol, Array<PlatformHandler> >
+      @platform_handlers = {}
+      # Read all platforms from the config
+      @config.platform_dirs.each do |platform_type, repositories|
+        repositories.each do |repository_path|
+          platform_handler = @platform_types[platform_type].new(
+            platform_type,
+            repository_path,
+            logger: @logger,
+            logger_stderr: @logger_stderr,
+            config: @config,
+            cmd_runner: @cmd_runner
+          )
+          # Check that this platform has unique name
+          raise "Platform name #{platform_handler.name} is declared several times." if @platform_handlers.values.flatten.any? { |known_platform| known_platform.name == platform_handler.name }
+          @platform_handlers[platform_type] = [] unless @platform_handlers.key?(platform_type)
+          @platform_handlers[platform_type] << platform_handler
+        end
+      end
+    end
+
+    # The list of registered platform handler classes, per platform type.
+    #
+    # Result::
+    # * Hash<Symbol,Class>: The list of registered platform handler classes, per platform type.
+    def platform_types
+      @platform_types.to_hash
+    end
+
+    # Get the list of known platforms
+    #
+    # Parameters::
+    # * *platform_type* (Symbol or nil): Filter only platforms of a given platform type, or nil for all platforms [default: nil]
+    # Result::
+    # * Array<PlatformHandler>: List of platform handlers
+    def known_platforms(platform_type: nil)
+      (platform_type.nil? ? @platform_handlers.keys : [platform_type]).map { |platform_type| (@platform_handlers[platform_type] || []) }.flatten
+    end
+
+    # Return the platform handler for a given platform name
+    #
+    # Parameters::
+    # * *platform_name* (String): The platform name
+    # Result::
+    # * PlatformHandler or nil: Corresponding platform handler, or nil if none
+    def platform(platform_name)
+      @platform_handlers.values.flatten.find { |known_platform| known_platform.name == platform_name }
+    end
+
+    # Inject dependencies that can't be set at initialization time.
+    # This is due to the fact that a PlatformHandler is a single plugin handling both inventory and services from a single repository.
+    # If we split those plugins into an inventory-type plugin and service-type plugin, each part could be initialized without having cyclic dependencies.
+    # The inventory-type part does not need NodesHandler nor ActionsExecutor (as it would be used by NodesHandler).
+    # The service-type part would use NodesHandler and ActionsExecutor given to its initializer.
+    # TODO: Split this plugin type in 2 to avoid this late dependency injection.
+    #
+    # Parameters::
+    # * *nodes_handler* (NodesHandler): Nodes handler to be used. [default: NodesHandler.new]
+    # * *actions_executor* (ActionsExecutor): Actions Executor to be used. [default: ActionsExecutor.new]
+    def inject_dependencies(
+      nodes_handler: NodesHandler.new,
+      actions_executor: ActionsExecutor.new
+    )
+      @nodes_handler = nodes_handler
+      @actions_executor = actions_executor
+      @platform_handlers.values.flatten.each do |platform|
+        platform.nodes_handler = @nodes_handler
+        platform.actions_executor = @actions_executor
+      end
+    end
+
+  end
+
+end

data/lib/hybrid_platforms_conductor/plugin.rb

@@ -0,0 +1,53 @@
+module HybridPlatformsConductor
+
+  # Base class for all plugins
+  class Plugin
+
+    include LoggerHelpers
+
+    class << self
+
+      include LoggerHelpers
+
+      # Class loggers
+      attr_accessor :logger, :logger_stderr
+
+      # Are dependencies met before using this plugin?
+      # This method can be overriden by any plugin
+      #
+      # Result::
+      # * Boolean: Are dependencies met before using this plugin?
+      def valid?
+        true
+      end
+
+      # Extend the config DSL used when parsing the hpc_config.rb file with a given Mixin.
+      # This can be used by any plugin to add plugin-specific configuration in the hpc_config.rb file.
+      #
+      # Parameters::
+      # * *mixin* (Module): Mixin to add to the Platforms DSL
+      # * *init_method* (Symbol or nil): The initializer method of this Mixin, or nil if none [default = nil]
+      def extend_config_dsl_with(mixin, init_method = nil)
+        Config.extend_config_dsl_with(mixin, init_method)
+      end
+
+    end
+
+    # Constructor
+    #
+    # Parameters::
+    # * *logger* (Logger): Logger to be used [default: Logger.new(STDOUT)]
+    # * *logger_stderr* (Logger): Logger to be used for stderr [default: Logger.new(STDERR)]
+    # * *config* (Config): Config to be used. [default: Config.new]
+    def initialize(
+      logger: Logger.new(STDOUT),
+      logger_stderr: Logger.new(STDERR),
+      config: Config.new
+    )
+      init_loggers(logger, logger_stderr)
+      @config = config
+    end
+
+  end
+
+end