gitlab_quality-test_tooling 0.1.0 → 0.2.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 01e27d782b61b17f8388d031534df9096ed36ed3d1c2e94734fb22875f5a47f5
-  data.tar.gz: a6df70390360e39d3b33df544c7e39d9838f16682763ed620120285886439d84
+  metadata.gz: 1826436013184c9312e5ba0bf45ed49eec2380743d81fa1afb31342559a799a2
+  data.tar.gz: d7fa98724fcf619d807ff4c61756ff119af3ad04c6b9f21306aaab59b872bcbe
 SHA512:
-  metadata.gz: 58640b757e621d232e5951cde8fe76fa8f3df7dfd0d5dcae326e0e1546fba6337218176ba1af0ec451a8dc2ea6e0616fd5f8ce66e8c9ab4645889c7768cc0bcd
-  data.tar.gz: e9a55097a4a8deb1a65f181e6674d89517a2a625012c0967ced81564a2cf52eff1529cdc58372a631f0d5bbb0b8684cb17b4518b22b92c77351a98e4d349deef
+  metadata.gz: 8c3bb098432bd3992ad6654c6540f75521e74648dd68c48be4b14962624ad835c199872cc1edf2d7a5173ec358ea3a629d612533853c2ad57a106c830b4c9ff6
+  data.tar.gz: 1d5adf1e7cc98306bfc3d1477aee4310ac6e267da9ed73a26203d9cdba45dd0a8b927c4ecd49f37543a8771c661eaeaa035105393957e288a62cd82165955be6

data/.rubocop.yml

@@ -19,6 +19,12 @@ AllCops:
   MaxFilesInCache: 1_000_000
   NewCops: enable
 
+CodeReuse/ActiveRecord:
+  Enabled: false
+
+Gemspec/RequiredRubyVersion:
+  Enabled: false
+
 Gemspec/RequireMFA:
   Enabled: false
 
@@ -49,12 +55,11 @@ Style/ModuleFunction:
 Style/SignalException:
   Enabled: false
 
+Layout/LineLength:
+  Max: 180
+
 Layout/SpaceBeforeFirstArg:
   Enabled: false
 
 RSpec/MultipleMemoizedHelpers:
   Enabled: false
-
-CodeReuse/ActiveRecord:
-  Exclude:
-    - 'spec/**/*_spec.rb'

data/Gemfile.lock

@@ -1,7 +1,7 @@
 PATH
   remote: .
   specs:
-    gitlab_quality-test_tooling (0.1.0)
+    gitlab_quality-test_tooling (0.2.0)
       activesupport (~> 6.1)
       gitlab (~> 4.18.0)
       http (~> 5.0)

data/Guardfile

@@ -42,28 +42,6 @@ guard :rspec, cmd: "bundle exec rspec" do
   ruby = dsl.ruby
   dsl.watch_spec_files_for(ruby.lib_files)
 
-  # Rails files
-  rails = dsl.rails(view_extensions: %w[erb haml slim])
-  dsl.watch_spec_files_for(rails.app_files)
-  dsl.watch_spec_files_for(rails.views)
-
-  watch(rails.controllers) do |m|
-    [
-      rspec.spec.call("routing/#{m[1]}_routing"),
-      rspec.spec.call("controllers/#{m[1]}_controller"),
-      rspec.spec.call("acceptance/#{m[1]}")
-    ]
-  end
-
-  # Rails config changes
-  watch(rails.spec_helper)     { rspec.spec_dir }
-  watch(rails.routes)          { "#{rspec.spec_dir}/routing" }
-  watch(rails.app_controller)  { "#{rspec.spec_dir}/controllers" }
-
-  # Capybara features specs
-  watch(rails.view_dirs)     { |m| rspec.spec.call("features/#{m[1]}") }
-  watch(rails.layouts)       { |m| rspec.spec.call("features/#{m[1]}") }
-
   # Turnip features and steps
   watch(%r{^spec/acceptance/(.+)\.feature$})
   watch(%r{^spec/acceptance/steps/(.+)_steps\.rb$}) do |m|

data/exe/generate-test-session

@@ -0,0 +1,49 @@
+#!/usr/bin/env ruby
+# frozen_string_literal: true
+
+require "bundler/setup"
+require "optparse"
+
+require_relative "../lib/gitlab_quality/test_tooling"
+
+params = {}
+
+options = OptionParser.new do |opts|
+  opts.banner = "Usage: #{$PROGRAM_NAME} [options]"
+
+  opts.on('-i', '--input-files FILES', String, 'Generate test session report') do |input_files|
+    params[:input_files] = input_files
+  end
+
+  opts.on('-p', '--project PROJECT', String, 'Can be an integer or a group/project string') do |project|
+    params[:project] = project
+  end
+
+  opts.on('-t', '--token ACCESS_TOKEN', String, 'A valid access token with Reporter permission in PROJECT') do |token|
+    params[:token] = GitlabQuality::TestTooling::Runtime::TokenFinder.find_token!(token)
+  end
+
+  opts.on('--dry-run', "Perform a dry-run (don't create or update issues or test cases)") do
+    params[:dry_run] = true
+  end
+
+  opts.on_tail('-v', '--version', 'Show the version') do
+    require_relative "../lib/gitlab_quality/test_tooling/version"
+    puts "#{$PROGRAM_NAME} : #{GitlabQuality::TestTooling::VERSION}"
+    exit
+  end
+
+  opts.on_tail('-h', '--help', 'Show the usage') do
+    puts opts
+    exit
+  end
+
+  opts.parse(ARGV)
+end
+
+if params.any?
+  GitlabQuality::TestTooling::Report::GenerateTestSession.new(**params).invoke!
+else
+  puts options
+  exit 1
+end

data/exe/post-to-slack

@@ -0,0 +1,57 @@
+#!/usr/bin/env ruby
+# frozen_string_literal: true
+
+require "bundler/setup"
+require "optparse"
+
+require_relative "../lib/gitlab_quality/test_tooling"
+
+params = {}
+
+options = OptionParser.new do |opts|
+  opts.banner = "Usage: #{$PROGRAM_NAME} [options]"
+
+  opts.on('-w', '--slack-webhook-url SLACK_WEBHOOK_URL', String, 'Slack webhook URL') do |slack_webhook_url|
+    params[:slack_webhook_url] = slack_webhook_url
+  end
+
+  opts.on('-c', '--channel CHANNEL', String, 'Slack channel to post the message to') do |channel|
+    params[:channel] = channel
+  end
+
+  opts.on('-m', '--message MESSAGE', String, 'Post message to Slack') do |message|
+    params[:message] = message
+  end
+
+  opts.on('-t', '--include-summary-table FILES', String, 'Create a results summary table to post to Slack') do |files|
+    params[:message] += "\n\n#{GitlabQuality::TestTooling::SummaryTable.create(input_files: files)}"
+  end
+
+  opts.on('-u', '--username USERNAME', String, 'Username to use for the Slack message') do |username|
+    params[:username] = username
+  end
+
+  opts.on('-i', '--icon-emoji ICON_EMOJI', String, 'Icon emoji to use for the Slack message') do |icon_emoji|
+    params[:icon_emoji] = icon_emoji
+  end
+
+  opts.on_tail('-v', '--version', 'Show the version') do
+    require_relative "../lib/gitlab_quality/test_tooling/version"
+    puts "#{$PROGRAM_NAME} : #{GitlabQuality::TestTooling::VERSION}"
+    exit
+  end
+
+  opts.on_tail('-h', '--help', 'Show the usage') do
+    puts opts
+    exit
+  end
+
+  opts.parse(ARGV)
+end
+
+if params.any?
+  GitlabQuality::TestTooling::Slack::PostToSlack.new(**params).invoke!
+else
+  puts options
+  exit 1
+end

data/exe/prepare-stage-reports

@@ -0,0 +1,38 @@
+#!/usr/bin/env ruby
+# frozen_string_literal: true
+
+require "bundler/setup"
+require "optparse"
+
+require_relative "../lib/gitlab_quality/test_tooling"
+
+params = {}
+
+options = OptionParser.new do |opts|
+  opts.banner = "Usage: #{$PROGRAM_NAME} [options]"
+
+  opts.on('-j', '--junit-files FILES', String,
+    'Prepare separate reports for each Stage from the provided JUnit XML files') do |junit_files|
+    params[:junit_files] = junit_files
+  end
+
+  opts.on_tail('-v', '--version', 'Show the version') do
+    require_relative "../lib/gitlab_quality/test_tooling/version"
+    puts "#{$PROGRAM_NAME} : #{GitlabQuality::TestTooling::VERSION}"
+    exit
+  end
+
+  opts.on_tail('-h', '--help', 'Show the usage') do
+    puts opts
+    exit
+  end
+
+  opts.parse(ARGV)
+end
+
+if params.any?
+  GitlabQuality::TestTooling::Report::PrepareStageReports.new(**params).invoke!
+else
+  puts options
+  exit 1
+end

data/exe/relate-failure-issue

@@ -0,0 +1,59 @@
+#!/usr/bin/env ruby
+# frozen_string_literal: true
+
+require "bundler/setup"
+require "optparse"
+
+require_relative "../lib/gitlab_quality/test_tooling"
+
+params = {}
+
+options = OptionParser.new do |opts|
+  opts.banner = "Usage: #{$PROGRAM_NAME} [options]"
+
+  opts.on('-i', '--input-files FILES', String,
+    'Relate test failures to failure issues from RSpec JSON files') do |input_files|
+    params[:input_files] = input_files
+  end
+
+  opts.on('--max-diff-ratio DIFF_RATO', Float, 'Max stacktrace diff ratio for QA failure issues detection') do |value|
+    params[:max_diff_ratio] = value
+  end
+
+  opts.on('-p', '--project PROJECT', String, 'Can be an integer or a group/project string') do |project|
+    params[:project] = project
+  end
+
+  opts.on('-t', '--token ACCESS_TOKEN', String, 'A valid access token with Reporter permission in PROJECT') do |token|
+    params[:token] = GitlabQuality::TestTooling::Runtime::TokenFinder.find_token!(token)
+  end
+
+  opts.on('--system-log-files SYSTEM_LOG_FILES', String,
+    'Include errors from system logs in failure issues') do |system_log_files|
+    params[:system_logs] = system_log_files
+  end
+
+  opts.on('--dry-run', "Perform a dry-run (don't create or update issues or test cases)") do
+    params[:dry_run] = true
+  end
+
+  opts.on_tail('-v', '--version', 'Show the version') do
+    require_relative "../lib/gitlab_quality/test_tooling/version"
+    puts "#{$PROGRAM_NAME} : #{GitlabQuality::TestTooling::VERSION}"
+    exit
+  end
+
+  opts.on_tail('-h', '--help', 'Show the usage') do
+    puts opts
+    exit
+  end
+
+  opts.parse(ARGV)
+end
+
+if params.any?
+  GitlabQuality::TestTooling::Report::RelateFailureIssue.new(**params).invoke!
+else
+  puts options
+  exit 1
+end

data/exe/report-results

@@ -0,0 +1,56 @@
+#!/usr/bin/env ruby
+# frozen_string_literal: true
+
+require "bundler/setup"
+require "optparse"
+
+require_relative "../lib/gitlab_quality/test_tooling"
+
+params = {}
+
+options = OptionParser.new do |opts|
+  opts.banner = "Usage: #{$PROGRAM_NAME} [options]"
+
+  opts.on('-i', '--input-files FILES', String,
+    'Report test results from JUnit or JSON files in GitLab test cases and results issues') do |input_files|
+    params[:input_files] = input_files
+  end
+
+  opts.on('--test-case-project TEST_CASE_PROJECT', String,
+    'Can be an integer or a group/project string') do |test_case_project|
+    params[:test_case_project] = test_case_project
+  end
+
+  opts.on('--results-issue-project RESULTS_ISSUE_PROJECT', String,
+    'Can be an integer or a group/project string') do |results_issue_project|
+    params[:results_issue_project] = results_issue_project
+  end
+
+  opts.on('-t', '--token ACCESS_TOKEN', String, 'A valid access token') do |token|
+    params[:token] = GitlabQuality::TestTooling::Runtime::TokenFinder.find_token!(token)
+  end
+
+  opts.on('--dry-run', "Perform a dry-run (don't create or update issues or test cases)") do
+    params[:dry_run] = true
+  end
+
+  opts.on_tail('-v', '--version', 'Show the version') do
+    require_relative "../lib/gitlab_quality/test_tooling/version"
+    puts "#{$PROGRAM_NAME} : #{GitlabQuality::TestTooling::VERSION}"
+    exit
+  end
+
+  opts.on_tail('-h', '--help', 'Show the usage') do
+    puts opts
+    exit
+  end
+
+  opts.parse(ARGV)
+end
+
+if params.any?
+  GitlabQuality::TestTooling::Report::ReportResults.new(**params).invoke!
+else
+  puts options
+  exit 1
+end

data/exe/update-screenshot-paths

@@ -0,0 +1,38 @@
+#!/usr/bin/env ruby
+# frozen_string_literal: true
+
+require "bundler/setup"
+require "optparse"
+
+require_relative "../lib/gitlab_quality/test_tooling"
+
+params = {}
+
+options = OptionParser.new do |opts|
+  opts.banner = "Usage: #{$PROGRAM_NAME} [options]"
+
+  opts.on('--junit-files FILES', String,
+    "Update the path to screenshots to container's host from the provided JUnit XML files") do |_files|
+    params[:junit_files] = junit_files
+  end
+
+  opts.on_tail('-v', '--version', 'Show the version') do
+    require_relative "../lib/gitlab_quality/test_tooling/version"
+    puts "#{$PROGRAM_NAME} : #{GitlabQuality::TestTooling::VERSION}"
+    exit
+  end
+
+  opts.on_tail('-h', '--help', 'Show the usage') do
+    puts opts
+    exit
+  end
+
+  opts.parse(ARGV)
+end
+
+if params.any?
+  GitlabQuality::TestTooling::Report::UpdateScreenshotPath.new(**params).invoke!
+else
+  puts options
+  exit 1
+end

data/lib/gitlab_quality/test_tooling/gitlab_issue_client.rb

@@ -0,0 +1,194 @@
+# frozen_string_literal: true
+
+require 'gitlab'
+
+module Gitlab
+  # Monkey patch the Gitlab client to use the correct API path and add required methods
+  class Client
+    def team_member(project, id)
+      get("/projects/#{url_encode(project)}/members/all/#{id}")
+    end
+
+    def issue_discussions(project, issue_id, options = {})
+      get("/projects/#{url_encode(project)}/issues/#{issue_id}/discussions", query: options)
+    end
+
+    def add_note_to_issue_discussion_as_thread(project, issue_id, discussion_id, options = {})
+      post("/projects/#{url_encode(project)}/issues/#{issue_id}/discussions/#{discussion_id}/notes", query: options)
+    end
+  end
+end
+
+module GitlabQuality
+  module TestTooling
+    # The GitLab client is used for API access: https://github.com/NARKOZ/gitlab
+    class GitlabIssueClient
+      MAINTAINER_ACCESS_LEVEL = 40
+      RETRY_BACK_OFF_DELAY = 60
+      MAX_RETRY_ATTEMPTS = 3
+
+      def initialize(token:, project:)
+        @token = token
+        @project = project
+        @retry_backoff = 0
+
+        configure_gitlab_client
+      end
+
+      def assert_user_permission!
+        handle_gitlab_client_exceptions do
+          user = Gitlab.user
+          member = Gitlab.team_member(project, user.id)
+
+          abort_not_permitted if member.access_level < MAINTAINER_ACCESS_LEVEL
+        end
+      rescue Gitlab::Error::NotFound
+        abort_not_permitted
+      end
+
+      def find_issues(iid: nil, options: {}, &select)
+        select ||= :itself
+
+        handle_gitlab_client_exceptions do
+          break [Gitlab.issue(project, iid)].select(&select) if iid
+
+          Gitlab.issues(project, options)
+            .auto_paginate
+            .select(&select)
+        end
+      end
+
+      def find_issue_discussions(iid:)
+        handle_gitlab_client_exceptions do
+          Gitlab.issue_discussions(project, iid, order_by: 'created_at', sort: 'asc').auto_paginate
+        end
+      end
+
+      def create_issue(title:, description:, labels:, issue_type: 'issue')
+        attrs = { issue_type: issue_type, description: description, labels: labels }
+
+        handle_gitlab_client_exceptions do
+          Gitlab.create_issue(project, title, attrs)
+        end
+      end
+
+      def edit_issue(iid:, options: {})
+        handle_gitlab_client_exceptions do
+          Gitlab.edit_issue(project, iid, options)
+        end
+      end
+
+      def find_issue_notes(iid:)
+        handle_gitlab_client_exceptions do
+          Gitlab.issue_notes(project, iid, order_by: 'created_at', sort: 'asc')&.auto_paginate
+        end
+      end
+
+      def create_issue_note(iid:, note:)
+        handle_gitlab_client_exceptions do
+          Gitlab.create_issue_note(project, iid, note)
+        end
+      end
+
+      def edit_issue_note(issue_iid:, note_id:, note:)
+        handle_gitlab_client_exceptions do
+          Gitlab.edit_issue_note(project, issue_iid, note_id, note)
+        end
+      end
+
+      def add_note_to_issue_discussion_as_thread(iid:, discussion_id:, body:)
+        handle_gitlab_client_exceptions do
+          Gitlab.add_note_to_issue_discussion_as_thread(project, iid, discussion_id, body: body)
+        end
+      end
+
+      def find_user_id(username:)
+        handle_gitlab_client_exceptions do
+          user = Gitlab.users(username: username)&.first
+          user['id'] unless user.nil?
+        end
+      end
+
+      def upload_file(file_fullpath:)
+        ignore_gitlab_client_exceptions do
+          Gitlab.upload_file(project, file_fullpath)
+        end
+      end
+
+      def ignore_gitlab_client_exceptions
+        yield
+      rescue StandardError, SystemCallError, OpenSSL::SSL::SSLError, Net::OpenTimeout, Net::ReadTimeout,
+        Gitlab::Error::Error => e
+        puts "Ignoring the following error: #{e}"
+      end
+
+      def handle_gitlab_client_exceptions # rubocop:disable Metrics/AbcSize
+        yield
+      rescue Gitlab::Error::NotFound
+        # This error could be raised in assert_user_permission!
+        # If so, we want it to terminate at that point
+        raise
+      rescue SystemCallError, OpenSSL::SSL::SSLError, Net::OpenTimeout, Net::ReadTimeout,
+        Gitlab::Error::InternalServerError, Gitlab::Error::Parsing => e
+        @retry_backoff += RETRY_BACK_OFF_DELAY
+
+        raise if @retry_backoff > RETRY_BACK_OFF_DELAY * MAX_RETRY_ATTEMPTS
+
+        warn_exception(e)
+        warn("Sleeping for #{@retry_backoff} seconds before retrying...")
+        sleep @retry_backoff
+
+        retry
+      rescue StandardError => e
+        pipeline = Runtime::Env.pipeline_from_project_name
+        channel = case pipeline
+                  when "canary"
+                    "qa-production"
+                  when "staging-canary"
+                    "qa-staging"
+                  else
+                    "qa-#{pipeline}"
+                  end
+        error_msg = warn_exception(e)
+
+        return unless Runtime::Env.ci_commit_ref_name == Runtime::Env.default_branch
+
+        slack_options = {
+          slack_webhook_url: ENV.fetch('CI_SLACK_WEBHOOK_URL', nil),
+          channel: channel,
+          username: "GitLab QA Bot",
+          icon_emoji: ':ci_failing:',
+          message: <<~MSG
+            An unexpected error occurred while reporting test results in issues.
+            The error occurred in job: #{Runtime::Env.ci_job_url}
+            `#{error_msg}`
+          MSG
+        }
+        puts "Posting Slack message to channel: #{channel}"
+
+        GitlabQuality::TestTooling::Slack::PostToSlack.new(**slack_options).invoke!
+      end
+
+      private
+
+      attr_reader :token, :project
+
+      def configure_gitlab_client
+        Gitlab.configure do |config|
+          config.endpoint = Runtime::Env.gitlab_api_base
+          config.private_token = token
+        end
+      end
+
+      def abort_not_permitted
+        abort "You must have at least Reporter access to the project to use this feature."
+      end
+
+      def warn_exception(error)
+        error_msg = "#{error.class.name} #{error.message}"
+        warn(error_msg)
+        error_msg
+      end
+    end
+  end
+end

data/lib/gitlab_quality/test_tooling/gitlab_issue_dry_client.rb

@@ -0,0 +1,26 @@
+# frozen_string_literal: true
+
+module GitlabQuality
+  module TestTooling
+    class GitlabIssueDryClient < GitlabIssueClient
+      def create_issue(title:, description:, labels:, issue_type: 'issue')
+        attrs = { description: description, labels: labels }
+
+        puts "The following #{issue_type} would have been created:"
+        puts "project: #{project}, title: #{title}, attrs: #{attrs}"
+      end
+
+      def edit_issue(iid:, options: {})
+        puts "The #{project}##{iid} issue would have been updated with: #{options}"
+      end
+
+      def create_issue_note(iid:, note:)
+        puts "The following note would have been posted on #{project}##{iid} issue: #{note}"
+      end
+
+      def add_note_to_issue_discussion_as_thread(iid:, discussion_id:, body:)
+        puts "The following discussion note would have been posted on #{project}##{iid} (discussion #{discussion_id}) issue: #{body}"
+      end
+    end
+  end
+end

data/lib/gitlab_quality/test_tooling/report/concerns/find_set_dri.rb

@@ -0,0 +1,51 @@
+# frozen_string_literal: true
+
+require 'json'
+
+module GitlabQuality
+  module TestTooling
+    module Report
+      module Concerns
+        module FindSetDri
+          def set_dri_via_group(product_group, test)
+            parse_json_with_sets
+            fetch_stage_sets(test)
+
+            return @sets.sample['username'] if @stage_sets.empty?
+
+            fetch_group_sets(product_group)
+
+            if @group_sets.empty?
+              @stage_sets.sample['username']
+            else
+              @group_sets.sample['username']
+            end
+          end
+
+          private
+
+          def parse_json_with_sets
+            response = Support::HttpRequest.make_http_request(
+              url: 'https://gitlab-org.gitlab.io/gitlab-roulette/roulette.json'
+            )
+            @sets = JSON.parse(response.body).select { |user| user['role'].include?('software-engineer-in-test') }
+          end
+
+          def fetch_stage_sets(test)
+            @stage_sets = @sets.select do |user|
+              user['role'].include?(test.stage.split("_").map(&:capitalize).join(" "))
+            end
+          end
+
+          def fetch_group_sets(product_group)
+            @group_sets = @stage_sets.select do |user|
+              user['role'].include?(product_group.split("_").map do |word|
+                                      word == 'and' ? word : word.capitalize
+                                    end.join(" "))
+            end
+          end
+        end
+      end
+    end
+  end
+end