hybrid_platforms_conductor 32.3.6

data/lib/hybrid_platforms_conductor/hpc_plugins/test_report/templates/_confluence_errors_status.html.erb

@@ -0,0 +1,46 @@
+<%# Partial template used to display a status.
+  # Expects the following variables to be set:
+  # * *@status_test_name* (String): The test name
+  # * *@status_test_criteria* (Hash<Symbol,Object>): The test criteria (options to give to the signatures of methods retrieving test data, like errors_for)
+  # * *@max_errors* (Integer): Number max of errors to be displayed
+%>
+<% if should_have_been_tested?(@status_test_name, **@status_test_criteria) %>
+  <%
+    errors = errors_for(@status_test_name, **@status_test_criteria)
+    color, title =
+      if missing_tests_for(@status_test_name, **@status_test_criteria)
+        ['Grey', 'Not run']
+      elsif errors.empty?
+        ['Green', 'Success']
+      elsif select_tests(@status_test_name, **@status_test_criteria).any? { |test| test.expected_failure.nil? && !test.errors.empty? }
+        ['Red', 'Errors']
+      else
+        ['Yellow', 'Expected errors']
+      end
+  %>
+  <div class="content-wrapper">
+    <p>
+      <ac:structured-macro ac:name="status" ac:schema-version="1">
+        <ac:parameter ac:name="colour"><%= color %></ac:parameter>
+        <ac:parameter ac:name="title"><%= title %></ac:parameter>
+      </ac:structured-macro>
+      <% unless errors.empty? %>
+        <ac:structured-macro ac:name="expand" ac:schema-version="1">
+          <ac:parameter ac:name="title">Errors</ac:parameter>
+          <ac:rich-text-body>
+            <div style="text-align: left;">
+              <ul>
+                <% errors[0..@max_errors - 1].each do |error| %>
+                  <li><%= CGI.escapeHTML(error[0..@max_error_message_length - 1]) %></li>
+                <% end %>
+                <% if errors.size > @max_errors %>
+                  <li>...</li>
+                <% end %>
+              </ul>
+            </div>
+          </ac:rich-text-body>
+        </ac:structured-macro>
+      <% end %>
+    </p>
+  </div>
+<% end %>

data/lib/hybrid_platforms_conductor/hpc_plugins/test_report/templates/_confluence_gauge.html.erb

@@ -0,0 +1,49 @@
+<%# Partial template used to display a gauge.
+  # Expects the following variables to be set:
+  # * *@gauge_success* (Integer): Number of tests in success
+  # * *@gauge_unexpected_error* (Integer): Number of tests in unexpected error
+  # * *@gauge_expected_error* (Integer): Number of tests in expected error
+  # * *@gauge_not_run* (Integer): Number of tests not run
+%>
+<% gauge_total = @gauge_success + @gauge_unexpected_error + @gauge_expected_error + @gauge_not_run %>
+<% unless gauge_total == 0 %>
+  <%= (@gauge_success*100.0/gauge_total).to_i %> %
+  <br/>
+  <ac:structured-macro ac:name="chart" ac:schema-version="1">
+    <ac:parameter ac:name="imageFormat">png</ac:parameter>
+    <ac:parameter ac:name="showShapes">false</ac:parameter>
+    <ac:parameter ac:name="forgive">false</ac:parameter>
+    <ac:parameter ac:name="legend">false</ac:parameter>
+    <ac:parameter ac:name="width">200</ac:parameter>
+    <ac:parameter ac:name="dataOrientation">vertical</ac:parameter>
+    <ac:parameter ac:name="colors">#14892C,#FFD351,#D04437,#CCCCCC</ac:parameter>
+    <ac:parameter ac:name="height">200</ac:parameter>
+    <ac:parameter ac:name="pieSectionLabel">%1%</ac:parameter>
+    <ac:rich-text-body>
+      <table>
+        <tbody>
+          <tr>
+            <th>Label</th>
+            <th>Value</th>
+          </tr>
+          <tr>
+            <td>Success</td>
+            <td><%= @gauge_success %></td>
+          </tr>
+          <tr>
+            <td>Expected error</td>
+            <td><%= @gauge_expected_error %></td>
+          </tr>
+          <tr>
+            <td>Unexpected error</td>
+            <td><%= @gauge_unexpected_error %></td>
+          </tr>
+          <tr>
+            <td>Missing</td>
+            <td><%= @gauge_not_run %></td>
+          </tr>
+        </tbody>
+      </table>
+    </ac:rich-text-body>
+  </ac:structured-macro>
+<% end %>

data/lib/hybrid_platforms_conductor/hpc_plugins/test_report/templates/confluence.html.erb

@@ -0,0 +1,242 @@
+<ac:structured-macro ac:name="toc" ac:schema-version="1"/>
+<h1>Summary</h1>
+<table>
+  <tbody>
+    <% # Compute the global counters of tests
+       global_info = classify_tests(global_tests)
+       platform_info = classify_tests(platform_tests)
+       node_info = classify_tests(node_tests)
+       success_percentage = 100.0 * (global_info[:success].size + platform_info[:success].size + node_info[:success].size).to_f / (global_tests.size + platform_tests.size + node_tests.size)
+    %>
+    <tr>
+      <th style="text-align: center; vertical-align: middle;">Global-level tests</th>
+      <th style="text-align: center; vertical-align: middle;">Platforms/repositories tests</th>
+      <th style="text-align: center; vertical-align: middle;">Nodes tests</th>
+    </tr>
+    <tr>
+      <td style="text-align: center; vertical-align: middle;">
+        <ac:structured-macro ac:name="status" ac:schema-version="1">
+          <ac:parameter ac:name="colour"><%= status_color(global_info[:status]) %></ac:parameter>
+        </ac:structured-macro>
+      </td>
+      <td style="text-align: center; vertical-align: middle;">
+        <ac:structured-macro ac:name="status" ac:schema-version="1">
+          <ac:parameter ac:name="colour"><%= status_color(platform_info[:status]) %></ac:parameter>
+        </ac:structured-macro>
+      </td>
+      <td style="text-align: center; vertical-align: middle;">
+        <ac:structured-macro ac:name="status" ac:schema-version="1">
+          <ac:parameter ac:name="colour"><%= status_color(node_info[:status]) %></ac:parameter>
+        </ac:structured-macro>
+      </td>
+    </tr>
+    <tr>
+      <td style="text-align: center; vertical-align: middle;"><%= render_gauge(global_info) %></td>
+      <td style="text-align: center; vertical-align: middle;"><%= render_gauge(platform_info) %></td>
+      <td style="text-align: center; vertical-align: middle;"><%= render_gauge(node_info) %></td>
+    </tr>
+  </tbody>
+</table>
+<h1>Evolution</h1>
+<ac:structured-macro ac:name="chart" ac:schema-version="1">
+  <ac:parameter ac:name="imageFormat">png</ac:parameter>
+  <ac:parameter ac:name="timeSeries">true</ac:parameter>
+  <ac:parameter ac:name="dateFormat">yyyy-MM-dd HH:mm:ss</ac:parameter>
+  <ac:parameter ac:name="forgive">false</ac:parameter>
+  <ac:parameter ac:name="timePeriod">Second</ac:parameter>
+  <ac:parameter ac:name="width">800</ac:parameter>
+  <ac:parameter ac:name="dataOrientation">vertical</ac:parameter>
+  <ac:parameter ac:name="type">timeSeries</ac:parameter>
+  <ac:parameter ac:name="yLabel">% success</ac:parameter>
+  <ac:parameter ac:name="height">300</ac:parameter>
+  <ac:rich-text-body>
+    <table>
+      <tbody>
+        <tr>
+          <th>Time</th>
+          <th>% success</th>
+        </tr>
+        <% (@previous_success_percentages + [[Time.now.utc, success_percentage]]).each do |time, percentage| %>
+          <tr>
+            <td><%= time.strftime('%F %T') %></td>
+            <td><%= percentage %></td>
+          </tr>
+        <% end %>
+      </tbody>
+    </table>
+  </ac:rich-text-body>
+</ac:structured-macro>
+<h1>Platforms used for those tests</h1>
+<table>
+  <tbody>
+    <tr>
+      <th>Platform</th>
+      <th>Commit title</th>
+      <th>Commit date</th>
+      <th>Commit hash</th>
+      <th>Commit author</th>
+    </tr>
+    <% @tests.map { |test| test.platform }.uniq.compact.sort.each do |platform| %>
+      <% platform_info = platform.info %>
+      <tr>
+        <td><strong><%= platform.name %></strong></td>
+        <td><%= CGI.escapeHTML(platform_info[:commit][:message]) %></td>
+        <td><%= platform_info[:commit][:date].strftime('%F %T') %></td>
+        <td><%= platform_info[:commit][:id] %></td>
+        <td><%= CGI.escapeHTML(platform_info[:commit][:author][:name]) %> (<%= CGI.escapeHTML(platform_info[:commit][:author][:email]) %>)</td>
+      </tr>
+    <% end %>
+  </tbody>
+</table>
+<h1>Stats by nodes list</h1>
+<table>
+  <tbody>
+    <tr>
+      <th style="vertical-align: middle;">Nodes list</th>
+      <th style="text-align: center; vertical-align: middle;">Status</th>
+      <th style="text-align: center; vertical-align: middle;"></th>
+    </tr>
+    <% nodes_by_nodes_list.each do |nodes_list, nodes_info| %>
+      <tr>
+        <td style="vertical-align: middle;">
+          <strong><%= nodes_list %></strong>
+          <br/>
+          <ac:structured-macro ac:name="status" ac:schema-version="1">
+            <ac:parameter ac:name="colour"><%=
+              if nodes_info[:tested_nodes].empty?
+                'Grey'
+              elsif nodes_info[:tested_nodes_in_error].empty?
+                'Green'
+              else
+                'Red'
+              end %></ac:parameter>
+          </ac:structured-macro>
+        </td>
+        <td style="text-align: center; vertical-align: middle;">
+          Total: <%= nodes_info[:nodes].size %> (<%= nodes_info[:tested_nodes].size %> tested) / Errors: <%= nodes_info[:tested_nodes_in_error].size %> (<%= nodes_info[:tested_nodes_in_error_as_expected].size %> expected)
+          <br/>
+          <% unless nodes_info[:nodes].empty? %>
+            <table>
+              <tbody>
+                <tr>
+                  <% ((nodes_info[:tested_nodes_in_error_as_expected].size * @nbr_cells_in_status_bars.to_f) / nodes_info[:nodes].size).round.times do %>
+                    <td bgcolor="#FFD351">
+                      <br/>
+                    </td>
+                  <% end %>
+                  <% (((nodes_info[:tested_nodes_in_error].size - nodes_info[:tested_nodes_in_error_as_expected].size).abs * @nbr_cells_in_status_bars.to_f) / nodes_info[:nodes].size).round.times do %>
+                    <td bgcolor="#D04437">
+                      <br/>
+                    </td>
+                  <% end %>
+                  <% (((nodes_info[:tested_nodes].size - nodes_info[:tested_nodes_in_error].size) * @nbr_cells_in_status_bars.to_f) / nodes_info[:nodes].size).round.times do %>
+                    <td bgcolor="#14892C">
+                      <br/>
+                    </td>
+                  <% end %>
+                  <% (((nodes_info[:nodes].size - nodes_info[:tested_nodes].size) * @nbr_cells_in_status_bars.to_f) / nodes_info[:nodes].size).round.times do %>
+                    <td bgcolor="#CCCCCC">
+                      <br/>
+                    </td>
+                  <% end %>
+                </tr>
+              </tbody>
+            </table>
+          <% end %>
+        </td>
+        <td style="text-align: center; vertical-align: middle;">
+          <% unless (nodes_info[:tested_nodes_in_error] - nodes_info[:tested_nodes_in_error_as_expected]).empty? %>
+            <ac:structured-macro ac:name="expand" ac:schema-version="1">
+              <ac:parameter ac:name="title">Unexpected nodes in error</ac:parameter>
+              <ac:rich-text-body>
+                <div style="text-align: left;">
+                  <ul>
+                    <% (nodes_info[:tested_nodes_in_error] - nodes_info[:tested_nodes_in_error_as_expected]).sort.each do |node| %>
+                      <li><%= node %></li>
+                    <% end %>
+                  </ul>
+                </div>
+              </ac:rich-text-body>
+            </ac:structured-macro>
+          <% end %>
+        </td>
+      </tr>
+    <% end %>
+  </tbody>
+</table>
+<h1>Global tests</h1>
+<table>
+  <tbody>
+    <tr>
+      <th style="text-align: center; vertical-align: middle;">Test</th>
+      <% @global_test_names.each do |test_name| %>
+        <th style="text-align: center; vertical-align: middle;"><%= test_name %></th>
+      <% end %>
+    </tr>
+    <tr>
+      <td style="text-align: center; vertical-align: middle;"></td>
+      <% @global_test_names.each do |test_name| %>
+        <td style="text-align: center; vertical-align: middle;">
+          <%= render_status(test_name, {}) %>
+        </td>
+      <% end %>
+    </tr>
+  </tbody>
+</table>
+<h1>Platform tests</h1>
+<table>
+  <tbody>
+    <tr>
+      <th style="vertical-align: middle;">Platforms \ Test</th>
+      <% @platform_test_names.each do |test_name| %>
+        <th style="text-align: center; vertical-align: middle;"><%= test_name %></th>
+      <% end %>
+    </tr>
+    <% @tested_platforms.each do |platform| %>
+      <tr>
+        <td style="vertical-align: middle;">
+          <ac:structured-macro ac:name="status" ac:schema-version="1">
+            <ac:parameter ac:name="colour"><%= group_errors(platform_tests.select { |test| test.platform == platform }, :test_name).empty? ? 'Green' : 'Red' %></ac:parameter>
+          </ac:structured-macro>
+          <strong><%= platform.name %></strong>
+        </td>
+        <% @platform_test_names.each do |test_name| %>
+          <td style="text-align: center; vertical-align: middle;">
+            <%= render_status(test_name, { platform: platform }) %>
+          </td>
+        <% end %>
+      </tr>
+    <% end %>
+  </tbody>
+</table>
+<h1>Node tests</h1>
+<table>
+  <tbody>
+    <tr>
+      <th style="vertical-align: middle;">Nodes \ Test</th>
+      <% @node_test_names.each do |test_name| %>
+        <th style="text-align: center; vertical-align: middle;"><%= test_name %></th>
+      <% end %>
+    </tr>
+    <% @tested_nodes.each do |node| %>
+      <tr>
+        <td style="vertical-align: middle;">
+          <ac:structured-macro ac:name="status" ac:schema-version="1">
+            <ac:parameter ac:name="colour"><%= group_errors(node_tests.select { |test| test.node == node }, :test_name).empty? ? 'Green' : 'Red' %></ac:parameter>
+          </ac:structured-macro>
+          <strong><%= node %></strong>
+        </td>
+        <% @node_test_names.each do |test_name| %>
+          <td style="text-align: center; vertical-align: middle;">
+            <%= render_status(test_name, { node: node }) %>
+          </td>
+        <% end %>
+      </tr>
+    <% end %>
+  </tbody>
+</table>
+<p>
+  <br/>
+  Generated at <%= Time.now.utc.strftime('%F %T') %> UTC by <a href="https://www.site.my_company.net/git/projects/PROJECTrepos/hybrid_platforms_conductor/browse">Hybrid Platforms Conductor</a>.
+  <br/>
+</p>

data/lib/hybrid_platforms_conductor/io_router.rb

@@ -0,0 +1,70 @@
+module HybridPlatformsConductor
+
+  # Simple router of IO and queue streams from som inputs to outputs, asynchronous
+  class IoRouter
+
+    # Create an IO router and make sure it is freed when client code has finished
+    #
+    # Parameters::
+    # * *routes* (Hash<IO or Queue, Array<IO> >):  List of destination IOs that should receive content per source IO.
+    # * Proc: Client code
+    def self.with_io_router(routes)
+      io_router = IoRouter.new(routes)
+      begin
+        io_router.start
+        yield
+      ensure
+        io_router.stop
+      end
+    end
+
+    # Constructor
+    #
+    # Parameters::
+    # * *routes* (Hash<IO or Queue, Array<IO> >):  List of destination IOs that should receive content per source IO.
+    def initialize(routes)
+      @routes = routes
+      @reading_thread = nil
+    end
+
+    # Start routing messages asynchronously
+    def start
+      raise 'IO router is already started. Can\'t start it again.' unless @reading_thread.nil?
+      @end_read = false
+      # Create a thread to handle routes asynchronously
+      @reading_thread = Thread.new do
+        loop do
+          need_to_stop = @end_read.clone
+          data_found = false
+          @routes.each do |src_io, dst_ios|
+            if src_io.is_a?(Queue)
+              queue_size = src_io.size
+              if queue_size > 0
+                # There is data to be read from src_io
+                data_found = true
+                data_chunk_str = queue_size.times.map { src_io.pop }.join
+                dst_ios.each do |dst_io|
+                  dst_io << data_chunk_str
+                  dst_io.flush if dst_io.respond_to?(:flush)
+                end
+              end
+            else
+              raise "Unknown type of source IO: #{src_io}"
+            end
+          end
+          break if need_to_stop && !data_found
+          sleep 0.1
+        end
+      end
+    end
+
+    # Stop routing messages asynchronously
+    def stop
+      raise 'IO router is not started. Can\'t stop it.' if @reading_thread.nil?
+      @end_read = true
+      @reading_thread.join
+    end
+
+  end
+
+end

data/lib/hybrid_platforms_conductor/json_dumper.rb

@@ -0,0 +1,88 @@
+require 'fileutils'
+require 'logger'
+require 'hybrid_platforms_conductor/logger_helpers'
+require 'hybrid_platforms_conductor/nodes_handler'
+require 'hybrid_platforms_conductor/deployer'
+
+module HybridPlatformsConductor
+
+  # Gives ways to dump nodes info into JSON files
+  class JsonDumper
+
+    include LoggerHelpers
+
+    # The output JSON directory
+    #   String
+    attr_accessor :dump_dir
+
+    # Constructor
+    #
+    # Parameters::
+    # * *logger* (Logger): Logger to be used [default = Logger.new(STDOUT)]
+    # * *logger_stderr* (Logger): Logger to be used for stderr [default = Logger.new(STDERR)]
+    # * *config* (Config): Config to be used. [default = Config.new]
+    # * *nodes_handler* (NodesHandler): Nodes handler to be used. [default = NodesHandler.new]
+    # * *deployer* (Deployer): Deployer to be used. [default = Deployer.new]
+    def initialize(logger: Logger.new(STDOUT), logger_stderr: Logger.new(STDERR), config: Config.new, nodes_handler: NodesHandler.new, deployer: Deployer.new)
+      init_loggers(logger, logger_stderr)
+      @config = config
+      @nodes_handler = nodes_handler
+      @deployer = deployer
+      # Default values
+      @skip_run = false
+      @dump_dir = 'nodes_json'
+    end
+
+    # Complete an option parser with ways to tune the way to dump nodes json
+    #
+    # Parameters::
+    # * *options_parser* (OptionParser): The option parser to complete
+    def options_parse(options_parser)
+      options_parser.separator ''
+      options_parser.separator 'JSON dump options:'
+      options_parser.on('-k', '--skip-run', 'Skip the actual gathering of dumps in run_logs. If set, the current run_logs content will be used.') do
+        @skip_run = true
+      end
+      options_parser.on('-j', '--json-dir DIRECTORY', "Specify the output directory in which JSON files are being written. Defaults to #{@dump_dir}.") do |dir|
+        @dump_dir = dir
+      end
+    end
+
+    # Dump JSON files containing description of the given nodes
+    #
+    # Parameters::
+    # * *nodes_selectors* (Array<object>): List of nodes selectors to dump files for
+    def dump_json_for(nodes_selectors)
+      nodes = @nodes_handler.select_nodes(nodes_selectors)
+      unless @skip_run
+        nodes.map { |node| @nodes_handler.platform_for(node) }.uniq.each.each do |platform_handler|
+          platform_handler.prepare_why_run_deploy_for_json_dump
+        end
+        @deployer.concurrent_execution = true
+        @deployer.use_why_run = true
+        @deployer.deploy_on(nodes)
+      end
+      # Parse the logs
+      FileUtils.mkdir_p @dump_dir
+      nodes.each do |node|
+        stdout_file_name = "#{@config.hybrid_platforms_dir}/run_logs/#{node}.stdout"
+        if File.exist?(stdout_file_name)
+          stdout = File.read(stdout_file_name).split("\n")
+          dump_begin_idx = stdout.index('===== Node JSON dump BEGIN =====')
+          dump_end_idx = stdout.index('===== Node JSON dump END =====')
+          if dump_begin_idx.nil? || dump_end_idx.nil?
+            out "[ #{node} ] - Error while dumping JSON. Check #{stdout_file_name}"
+          else
+            json_file_name = "#{@dump_dir}/#{node}.json"
+            File.write(json_file_name, stdout[dump_begin_idx+1..dump_end_idx-1].join("\n"))
+            out "[ #{node} ] - OK. Check #{json_file_name}"
+          end
+        else
+          out "[ #{node} ] - Error while dumping JSON. File #{stdout_file_name} does not exist."
+        end
+      end
+    end
+
+  end
+
+end