hybrid_platforms_conductor 32.3.6

data/lib/hybrid_platforms_conductor/cmd_runner.rb

@@ -0,0 +1,188 @@
+require 'logger'
+require 'tty-command'
+require 'hybrid_platforms_conductor/logger_helpers'
+require 'hybrid_platforms_conductor/io_router'
+
+module HybridPlatformsConductor
+
+  class CmdRunner
+
+    class UnexpectedExitCodeError < StandardError
+    end
+
+    class TimeoutError < StandardError
+    end
+
+    include LoggerHelpers
+
+    # Return the executables prefix to use to execute commands
+    #
+    # Result::
+    # * String: The executable prefix
+    def self.executables_prefix
+      $0.include?('/') ? "#{File.dirname($0)}/" : ''
+    end
+
+    # Dry-run switch. When true, then commands are just printed out without being executed.
+    #   Boolean
+    attr_accessor :dry_run
+
+    # Constructor
+    #
+    # Parameters::
+    # * *logger* (Logger): Logger to be used [default = Logger.new(STDOUT)]
+    # * *logger_stderr* (Logger): Logger to be used for stderr [default = Logger.new(STDERR)]
+    def initialize(logger: Logger.new(STDOUT), logger_stderr: Logger.new(STDERR))
+      init_loggers(logger, logger_stderr)
+      @dry_run = false
+    end
+
+    # Complete an option parser with options meant to control this Actions Executor
+    #
+    # Parameters::
+    # * *options_parser* (OptionParser): The option parser to complete
+    # * *parallel* (Boolean): Do we activate options regarding parallel execution? [default = true]
+    def options_parse(options_parser, parallel: true)
+      options_parser.separator ''
+      options_parser.separator 'Command runner options:'
+      options_parser.on('-s', '--show-commands', 'Display the commands that would be run instead of running them') do
+        @dry_run = true
+      end
+    end
+
+    # Run an external command.
+    # Handle dry-run mode, timeout, and check for an expected return code.
+    # Raise an exception if the exit status is not the expected one.
+    #
+    # Parameters::
+    # * *cmd* (String): Command to be run
+    # * *log_to_file* (String or nil): Log file capturing stdout or stderr (or nil for none). [default: nil]
+    # * *log_to_stdout* (Boolean): Do we send the output to stdout? [default: true]
+    # * *log_stdout_to_io* (IO or nil): IO to send command's stdout to, or nil for none. [default: nil]
+    # * *log_stderr_to_io* (IO or nil): IO to send command's stderr to, or nil for none. [default: nil]
+    # * *expected_code* (Integer, Symbol or Array<Integer or Symbol>): Return codes (or single one) that is expected [default: 0]
+    #   Symbol error codes can be used:
+    #   * *command_error*: The command could not be executed
+    #   * *timeout*: The command ended in timeout
+    # * *timeout* (Integer or nil): Timeout to apply for the command to be run, or nil for no timeout [default: nil]
+    # * *no_exception* (Boolean): If true, don't throw exception in case of error [default: false]
+    # Result::
+    # * Integer or Symbol: Exit status of the command, or Symbol in case of error. In case of dry-run mode the expected code is returned without executing anything.
+    # * String: Standard output of the command
+    # * String: Standard error output of the command (can be a descriptive message of the error in case of error)
+    def run_cmd(
+      cmd,
+      log_to_file: nil,
+      log_to_stdout: true,
+      log_stdout_to_io: nil,
+      log_stderr_to_io: nil,
+      expected_code: 0,
+      timeout: nil,
+      no_exception: false
+    )
+      expected_code = [expected_code] unless expected_code.is_a?(Array)
+      if @dry_run
+        out cmd
+        return expected_code.first, '', ''
+      else
+        log_debug "#{timeout.nil? ? '' : "[ Timeout #{timeout} ] - "}#{cmd.light_cyan.bold}"
+        exit_status = nil
+        cmd_stdout = nil
+        cmd_stderr = nil
+        file_output =
+          if log_to_file
+            if File.exist?(log_to_file)
+              File.open(log_to_file, 'a')
+            else
+              FileUtils.mkdir_p(File.dirname(log_to_file))
+              File.open(log_to_file, 'w')
+            end
+          else
+            nil
+          end
+        start_time = Time.now if log_debug?
+        begin
+          # Make sure we keep a trace of stdout and stderr, even if it was not asked, just to use it in case of exceptions raised
+          cmd_result_stdout = ''
+          cmd_result_stderr = ''
+          # Route IOs
+          stdout_queue = Queue.new
+          stderr_queue = Queue.new
+          IoRouter.with_io_router(
+            stdout_queue => [cmd_result_stdout] +
+              (log_stdout_to_io ? [log_stdout_to_io] : []) +
+              (log_to_stdout ? [@logger] : []) +
+              (file_output.nil? ? [] : [file_output]),
+            stderr_queue => [cmd_result_stderr] +
+              (log_stderr_to_io ? [log_stderr_to_io] : []) +
+              (log_to_stdout ? [@logger_stderr] : []) +
+              (file_output.nil? ? [] : [file_output])
+          ) do
+            cmd_result = TTY::Command.new(
+              printer: :null,
+              pty: true,
+              timeout: timeout,
+              uuid: false
+            ).run!(cmd) do |stdout, stderr|
+              stdout_queue << stdout if stdout
+              stderr_queue << stderr if stderr
+            end
+            exit_status = cmd_result.exit_status
+            cmd_stdout = cmd_result.out
+            cmd_stderr = cmd_result.err
+          end
+        rescue TTY::Command::TimeoutExceeded
+          exit_status = :timeout
+          cmd_stdout = cmd_result_stdout
+          cmd_stderr = "#{cmd_result_stderr.empty? ? '' : "#{cmd_result_stderr}\n"}Timeout of #{timeout} triggered"
+        rescue
+          exit_status = :command_error
+          cmd_stdout = cmd_result_stdout
+          cmd_stderr = "#{cmd_result_stderr.empty? ? '' : "#{cmd_result_stderr}\n"}#{$!}\n#{$!.backtrace.join("\n")}"
+        ensure
+          file_output.close unless file_output.nil?
+        end
+        if log_debug?
+          elapsed = Time.now - start_time
+          log_debug "Finished in #{elapsed} seconds with exit status #{exit_status} (#{(expected_code.include?(exit_status) ? 'success'.light_green : 'failure'.light_red).bold})"
+        end
+        unless expected_code.include?(exit_status)
+          error_title = "Command '#{cmd.split("\n").first}' returned error code #{exit_status} (expected #{expected_code.join(', ')})."
+          if no_exception
+            # We consider the caller is responsible for logging what he wants about the details of the error (stdout and stderr)
+            log_error error_title
+          else
+            # The exception won't contain stdout and stderr details (unless output to stdout was on), so dump them now
+            log_error "#{error_title}#{log_to_stdout ? '' : "\n----- Command STDOUT:\n#{cmd_stdout}\n----- Command STDERR:\n#{cmd_stderr}"}"
+            raise exit_status == :timeout ? TimeoutError : UnexpectedExitCodeError, error_title
+          end
+        end
+        return exit_status, cmd_stdout, cmd_stderr
+      end
+    end
+
+    # Is the current user root?
+    # Look into the environment to decide.
+    #
+    # Result::
+    # Boolean: Is the current user root?
+    def root?
+      whoami == 'root'
+    end
+
+    # Who is the local user?
+    # Keep a cache of it.
+    #
+    # Result::
+    # String: Name of the local user
+    def whoami
+      unless defined?(@whoami)
+        _exit_status, stdout, _stderr = run_cmd 'whoami', log_to_stdout: log_debug?
+        @whoami = stdout.strip
+      end
+      @whoami
+    end
+
+  end
+
+end

data/lib/hybrid_platforms_conductor/cmdb.rb

@@ -0,0 +1,34 @@
+require 'hybrid_platforms_conductor/logger_helpers'
+require 'hybrid_platforms_conductor/plugin'
+
+module HybridPlatformsConductor
+
+  # Base class for any CMDB plugin
+  class Cmdb < Plugin
+
+    # Constructor
+    #
+    # Parameters::
+    # * *logger* (Logger): Logger to be used [default: Logger.new(STDOUT)]
+    # * *logger_stderr* (Logger): Logger to be used for stderr [default: Logger.new(STDERR)]
+    # * *config* (Config): Config to be used. [default: Config.new]
+    # * *cmd_runner* (CmdRunner): Command Runner to be used. [default: CmdRunner.new]
+    # * *platforms_handler* (PlatformsHandler): Platforms Handler to be used. [default: PlatformsHandler.new]
+    # * *nodes_handler* (NodesHandler): Nodes Handler to be used. [default: NodesHandler.new]
+    def initialize(
+      logger: Logger.new(STDOUT),
+      logger_stderr: Logger.new(STDERR),
+      config: Config.new,
+      cmd_runner: CmdRunner.new,
+      platforms_handler: PlatformsHandler.new,
+      nodes_handler: NodesHandler.new
+    )
+      super(logger: logger, logger_stderr: logger_stderr, config: config)
+      @cmd_runner = cmd_runner
+      @platforms_handler = platforms_handler
+      @nodes_handler = nodes_handler
+    end
+
+  end
+
+end

data/lib/hybrid_platforms_conductor/common_config_dsl/bitbucket.rb

@@ -0,0 +1,78 @@
+require 'hybrid_platforms_conductor/bitbucket'
+
+module HybridPlatformsConductor
+
+  module CommonConfigDsl
+
+    module Bitbucket
+
+      # Initialize the DSL 
+      def init_bitbucket
+        # List of Bitbucket repositories definitions
+        # Array< Hash<Symbol, Object> >
+        # Each definition is just mapping the signature of #bitbucket_repos
+        @bitbucket_repos = []
+      end
+
+      # Register new Bitbucket repositories
+      #
+      # Parameters::
+      # * *url* (String): URL to the Bitbucket server
+      # * *project* (String): Project name from the Bitbucket server, storing repositories
+      # * *repos* (Array<String> or Symbol): List of repository names from this project, or :all for all [default: :all]
+      # * *jenkins_ci_url* (String or nil): Corresponding Jenkins CI URL, or nil if none [default: nil]
+      # * *checks* (Hash<Symbol, Object>): Checks definition to be perform on those repositories (see the #for_each_bitbucket_repo to know the structure) [default: {}]
+      def bitbucket_repos(url:, project:, repos: :all, jenkins_ci_url: nil, checks: {})
+        @bitbucket_repos << {
+          url: url,
+          project: project,
+          repos: repos,
+          jenkins_ci_url: jenkins_ci_url,
+          checks: checks
+        }
+      end
+
+      # Iterate over each Bitbucket repository
+      #
+      # Parameters::
+      # * Proc: Code called for each Bitbucket repository:
+      #   * Parameters::
+      #     * *bitbucket* (Bitbucket): The Bitbucket instance used to query the API for this repository
+      #     * *repo_info* (Hash<Symbol, Object>): The repository info:
+      #       * *name* (String): Repository name.
+      #       * *project* (String): Project name.
+      #       * *url* (String): Project Git URL.
+      #       * *jenkins_ci_url* (String or nil): Corresponding Jenkins CI URL, or nil if none.
+      #       * *checks* (Hash<Symbol, Object>): Checks to be performed on this repository:
+      #         * *branch_permissions* (Array< Hash<Symbol, Object> >): List of branch permissions to check [optional]
+      #           * *type* (String): Type of branch permissions to check. Examples of values are 'fast-forward-only', 'no-deletes', 'pull-request-only'.
+      #           * *branch* (String): Branch on which those permissions apply.
+      #           * *exempted_users* (Array<String>): List of exempted users for this permission [default: []]
+      #           * *exempted_groups* (Array<String>): List of exempted groups for this permission [default: []]
+      #           * *exempted_keys* (Array<String>): List of exempted access keys for this permission [default: []]
+      #         * *pr_settings* (Hash<Symbol, Object>): PR specific settings to check [optional]
+      #           * *required_approvers* (Integer): Number of required approvers [optional]
+      #           * *required_builds* (Integer): Number of required successful builds [optional]
+      #           * *default_merge_strategy* (String): Name of the default merge strategy. Example: 'rebase-no-ff' [optional]
+      #           * *mandatory_default_reviewers* (Array<String>): List of mandatory reviewers to check [default: []]
+      def for_each_bitbucket_repo
+        @bitbucket_repos.each do |bitbucket_repo_info|
+          HybridPlatformsConductor::Bitbucket.with_bitbucket(bitbucket_repo_info[:url], @logger, @logger_stderr) do |bitbucket|
+            (bitbucket_repo_info[:repos] == :all ? bitbucket.repos(bitbucket_repo_info[:project])['values'].map { |repo_info| repo_info['slug'] } : bitbucket_repo_info[:repos]).each do |name|
+              yield bitbucket, {
+                name: name,
+                project: bitbucket_repo_info[:project],
+                url: "#{bitbucket_repo_info[:url]}/scm/#{bitbucket_repo_info[:project].downcase}/#{name}.git",
+                jenkins_ci_url: bitbucket_repo_info[:jenkins_ci_url].nil? ? nil : "#{bitbucket_repo_info[:jenkins_ci_url]}/job/#{name}",
+                checks: bitbucket_repo_info[:checks]
+              }
+            end
+          end
+        end
+      end
+
+    end
+
+  end
+
+end

data/lib/hybrid_platforms_conductor/common_config_dsl/confluence.rb

@@ -0,0 +1,43 @@
+module HybridPlatformsConductor
+
+  module CommonConfigDsl
+
+    module Confluence
+
+      # Initialize the DSL 
+      def init_confluence
+        # Confluence configuration (can be nil if none)
+        # Hash<Symbol, Object> or nil. See #confluence_info to know details.
+        @confluence = nil
+      end
+
+      # Register a Confluence server
+      #
+      # Parameters::
+      # * *url* (String): URL to the Confluence server
+      # * *inventory_report_page_id* (String or nil): Confluence page id used for inventory reports, or nil if none [default: nil]
+      # * *tests_report_page_id* (String or nil): Confluence page id used for test reports, or nil if none [default: nil]
+      def confluence(url:, inventory_report_page_id: nil, tests_report_page_id: nil)
+        @confluence = {
+          url: url,
+          inventory_report_page_id: inventory_report_page_id,
+          tests_report_page_id: tests_report_page_id
+        }
+      end
+
+      # Return the Confluence information
+      #
+      # Result::
+      # * Hash<Symbol, Object> or nil: The Confluence information, or nil if none
+      #   * *url* (String): The Confluence URL.
+      #   * *inventory_report_page_id* (String or nil): Confluence page id used for inventory reports, or nil if none.
+      #   * *tests_report_page_id* (String or nil): Confluence page id used for test reports, or nil if none.
+      def confluence_info
+        @confluence
+      end
+
+    end
+
+  end
+
+end

data/lib/hybrid_platforms_conductor/common_config_dsl/file_system_tests.rb

@@ -0,0 +1,110 @@
+module HybridPlatformsConductor
+
+  module CommonConfigDsl
+
+    # Config DSL configuring file system testing (used by different test plugins)
+    module FileSystemTests
+
+      # List of paths rules to be checked. Each info has the following properties:
+      # * *nodes_selectors_stack* (Array<Object>): Stack of nodes selectors impacted by this rule
+      # * *paths* (Array<String>): List of paths to check.
+      # * *state* (Symbol): State those paths should be in. Possible states are:
+      #   * *present*: Paths should exist
+      #   * *absent*: Paths should not exist
+      # * *context* (Hash<Symbol,Object>): Context on which those paths are checked. Possible properties are
+      #   * *sudo_user* (String or nil): Sudo user to be used to perform checks, or nil if none [default: nil]
+      #   * *file_system_type* (Symbol): File system to be checked [default: :local]. Possible values:
+      #     * *local*: Local file system
+      #     * *hdfs*: HDFS file system
+      # Array< Hash<Symbol, Object> >
+      attr_reader :fs_paths_rules
+
+      # Initialize the DSL 
+      def init_file_system_tests
+        @fs_paths_rules = []
+        @context = {
+          file_system_type: :local
+        }
+      end
+
+      # Give a list of paths to check for absence
+      #
+      # Parameters::
+      # * *paths* (String or Array<String>): List of (or single) paths
+      def check_files_do_not_exist(*paths)
+        @fs_paths_rules << {
+          paths: paths.flatten,
+          nodes_selectors_stack: current_nodes_selectors_stack,
+          context: @context.clone,
+          state: :absent
+        }
+      end
+
+      # Give a list of paths to check for presence
+      #
+      # Parameters::
+      # * *paths* (String or Array<String>): List of (or single) paths
+      def check_files_do_exist(*paths)
+        @fs_paths_rules << {
+          paths: paths.flatten,
+          nodes_selectors_stack: current_nodes_selectors_stack,
+          context: @context.clone,
+          state: :present              
+        }
+      end
+
+      # Set the rules to be in a context of HDFS checking
+      #
+      # Parameters::
+      # * *with_sudo* (String or nil): Sudo user to be used to perform HDFS commands, or nil for none [default: nil]
+      # * Proc: Configuration code called within this context
+      def on_hdfs(with_sudo: nil)
+        old_context = @context.clone
+        begin
+          @context[:file_system_type] = :hdfs
+          @context[:sudo_user] = with_sudo unless with_sudo.nil?
+          yield
+        ensure
+          @context = old_context
+        end
+      end
+
+      # Aggregate a list of paths rules for a given file system type, per path to be checked
+      #
+      # Parameters::
+      # * *nodes_handler* (NodesHandler): NodesHandler to be sued to resolve nodes selections
+      # * *node* (String): Node for which we select rules
+      # * *file_system_type* (Symbol): File system type to be selected [default: :local]
+      # Result::
+      # * Hash<String, Hash<Symbol,Object> >: Rule infos, per path. Each info has the following properties:
+      #   * *state* (Symbol): State the path should be in
+      #   * *context* (Hash): Associated context, as defined by the configuration
+      def aggregate_files_rules(nodes_handler, node, file_system_type: :local)
+        nodes_handler.
+          select_confs_for_node(node, fs_paths_rules).
+          inject({}) do |merged_paths, paths_info|
+            if paths_info[:context][:file_system_type] == file_system_type
+              merged_paths.merge(Hash[paths_info[:paths].map do |path|
+                [
+                  path,
+                  {
+                    state: paths_info[:state],
+                    context: paths_info[:context]
+                  }
+                ]
+              end]) do |path, rule_info_1, rule_info_2|
+                # Just check that configuration is not inconsistent
+                raise "Inconsistent rule for #{file_system_type} file system checks in configuration for #{node}: #{path} is marked as being both #{rule_info_1[:state]} and #{rule_info_2[:state]}" if rule_info_1[:state] != rule_info_2[:state]
+                rule_info_2
+              end
+            else
+              merged_paths
+            end
+          end
+      end
+
+    end
+
+  end
+
+end