gitlab_quality-test_tooling 1.11.0 → 1.17.0

data/lib/gitlab_quality/test_tooling/gitlab_client/commits_client.rb

@@ -0,0 +1,20 @@
+# frozen_string_literal: true
+
+module GitlabQuality
+  module TestTooling
+    module GitlabClient
+      class CommitsClient < GitlabClient
+        def create(branch_name, file_path, new_content, message)
+          commit = handle_gitlab_client_exceptions do
+            client.create_commit(project, branch_name, message, [
+              { action: :update, file_path: file_path, content: new_content }
+            ])
+          end
+
+          Runtime::Logger.debug("Created commit #{commit['id']} (#{commit['web_url']}) on #{branch_name}") if commit
+          commit
+        end
+      end
+    end
+  end
+end

data/lib/gitlab_quality/test_tooling/gitlab_client/commits_dry_client.rb

@@ -0,0 +1,14 @@
+# frozen_string_literal: true
+
+module GitlabQuality
+  module TestTooling
+    module GitlabClient
+      class CommitsDryClient < CommitsClient
+        def create(branch_name, file_path, new_content, message)
+          puts "A commit would have been created on branch_name: #{branch_name}, file_path: #{file_path}, message: #{message} and content:"
+          puts new_content
+        end
+      end
+    end
+  end
+end

data/lib/gitlab_quality/test_tooling/gitlab_client/gitlab_client.rb

@@ -15,24 +15,32 @@ module GitlabQuality
           @retry_backoff = 0
         end
 
-        def handle_gitlab_client_exceptions # rubocop:disable Metrics/AbcSize
+        def handle_gitlab_client_exceptions
           yield
         rescue Gitlab::Error::NotFound
           # This error could be raised in assert_user_permission!
           # If so, we want it to terminate at that point
           raise
         rescue SystemCallError, OpenSSL::SSL::SSLError, Net::OpenTimeout, Net::ReadTimeout,
-          Gitlab::Error::InternalServerError, Gitlab::Error::Parsing => e
+          Gitlab::Error::InternalServerError, Gitlab::Error::BadRequest, Gitlab::Error::ResponseError, Gitlab::Error::Parsing => e
           @retry_backoff += RETRY_BACK_OFF_DELAY
 
           raise if @retry_backoff > RETRY_BACK_OFF_DELAY * MAX_RETRY_ATTEMPTS
 
-          warn_exception(e)
+          warn("#{error.class.name} #{error.message}")
           warn("Sleeping for #{@retry_backoff} seconds before retrying...")
           sleep @retry_backoff
 
           retry
         rescue StandardError => e
+          post_exception_to_slack(e) if Runtime::Env.ci_commit_ref_name == Runtime::Env.default_branch
+
+          raise e
+        end
+
+        def post_exception_to_slack(error)
+          return unless ENV['CI_SLACK_WEBHOOK_URL']
+
           pipeline = Runtime::Env.pipeline_from_project_name
           channel = case pipeline
                     when "canary"
@@ -42,9 +50,6 @@ module GitlabQuality
                     else
                       "qa-#{pipeline}"
                     end
-          error_msg = warn_exception(e)
-
-          return unless Runtime::Env.ci_commit_ref_name == Runtime::Env.default_branch
 
           slack_options = {
             slack_webhook_url: ENV.fetch('CI_SLACK_WEBHOOK_URL', nil),
@@ -54,7 +59,7 @@ module GitlabQuality
             message: <<~MSG
               An unexpected error occurred while reporting test results in issues.
               The error occurred in job: #{Runtime::Env.ci_job_url}
-              `#{error_msg}`
+              `#{error.class.name} #{error.message}`
             MSG
           }
           puts "Posting Slack message to channel: #{channel}"
@@ -79,12 +84,6 @@ module GitlabQuality
             private_token: token
           )
         end
-
-        def warn_exception(error)
-          error_msg = "#{error.class.name} #{error.message}"
-          warn(error_msg)
-          error_msg
-        end
       end
     end
   end

data/lib/gitlab_quality/test_tooling/gitlab_client/issues_client.rb

@@ -48,6 +48,12 @@ module GitlabQuality
           end
         end
 
+        def find_issue_notes(iid:)
+          handle_gitlab_client_exceptions do
+            client.issue_notes(project, iid, order_by: 'created_at', sort: 'asc').auto_paginate
+          end
+        end
+
         def find_issue_discussions(iid:)
           handle_gitlab_client_exceptions do
             client.issue_discussions(project, iid, order_by: 'created_at', sort: 'asc').auto_paginate
@@ -75,12 +81,6 @@ module GitlabQuality
           end
         end
 
-        def find_issue_notes(iid:)
-          handle_gitlab_client_exceptions do
-            client.issue_notes(project, iid, order_by: 'created_at', sort: 'asc')&.auto_paginate
-          end
-        end
-
         def create_issue_note(iid:, note:)
           handle_gitlab_client_exceptions do
             client.create_issue_note(project, iid, note)

data/lib/gitlab_quality/test_tooling/gitlab_client/merge_requests_client.rb

@@ -1,28 +1,36 @@
 # frozen_string_literal: true
 
-require 'gitlab'
-
 module GitlabQuality
   module TestTooling
     module GitlabClient
       class MergeRequestsClient < GitlabClient
         def find_merge_request_changes(merge_request_iid:)
-          client.merge_request_changes(project, merge_request_iid)
+          handle_gitlab_client_exceptions do
+            client.merge_request_changes(project, merge_request_iid)
+          end
         end
 
-        def create_merge_request(title:, source_branch:, target_branch:, description:, labels:)
+        def create_merge_request(title:, source_branch:, target_branch:, description:, labels:, assignee_id: nil, reviewer_ids: [])
+          attrs = {
+            source_branch: source_branch,
+            target_branch: target_branch,
+            description: description,
+            labels: labels,
+            assignee_id: assignee_id,
+            squash: true,
+            remove_source_branch: true,
+            reviewer_ids: reviewer_ids
+          }.compact
+
           merge_request = handle_gitlab_client_exceptions do
             client.create_merge_request(project,
               title,
-              source_branch: source_branch,
-              target_branch: target_branch,
-              description: description,
-              labels: labels,
-              squash: true,
-              remove_source_branch: true)
+              attrs)
           end
 
           Runtime::Logger.debug("Created merge request #{merge_request['iid']} (#{merge_request['web_url']})") if merge_request
+
+          merge_request
         end
 
         def find(iid: nil, options: {}, &select)

data/lib/gitlab_quality/test_tooling/gitlab_client/merge_requests_dry_client.rb

@@ -26,9 +26,11 @@ module GitlabQuality
           puts "The following note would have been updated id: #{id} with body: #{note} for mr_iid: #{merge_request_iid}"
         end
 
-        def create_merge_request(title:, source_branch:, target_branch:, description:, labels:)
+        def create_merge_request(title:, source_branch:, target_branch:, description:, labels:, assignee_id:, reviewer_ids:)
           puts "A merge request would be created with title: #{title} " \
-               "source_branch: #{source_branch} target_branch: #{target_branch} description: #{description} labels: #{labels}"
+               "source_branch: #{source_branch} target_branch: #{target_branch} " \
+               "description: #{description} labels: #{labels}, assignee_id: #{assignee_id}" \
+               "reviewer_ids: #{reviewer_ids}"
         end
       end
     end

data/lib/gitlab_quality/test_tooling/gitlab_client/repository_files_client.rb

@@ -0,0 +1,23 @@
+# frozen_string_literal: true
+
+module GitlabQuality
+  module TestTooling
+    module GitlabClient
+      class RepositoryFilesClient < GitlabClient
+        attr_reader :file_path
+
+        def initialize(file_path:, **kwargs)
+          @file_path = file_path
+
+          super
+        end
+
+        def file_contents
+          handle_gitlab_client_exceptions do
+            client.file_contents(project, file_path.gsub(%r{^/}, ""))
+          end
+        end
+      end
+    end
+  end
+end

data/lib/gitlab_quality/test_tooling/knapsack_reports/spec_run_time.rb

@@ -0,0 +1,85 @@
+# frozen_string_literal: true
+
+module GitlabQuality
+  module TestTooling
+    module KnapsackReports
+      class SpecRunTime
+        attr_reader :file, :expected, :actual, :expected_suite_duration, :actual_suite_duration
+
+        ACTUAL_TO_EXPECTED_SPEC_RUN_TIME_RATIO_THRESHOLD = 1.5 # actual run time is longer than expected by 50% +
+        SPEC_WEIGHT_PERCENTAGE_TRESHOLD = 15 # a spec file takes 15%+ of the total test suite run time
+        SUITE_DURATION_THRESHOLD = 70 * 60 # if test suite takes more than 70 minutes, job risks timing out
+
+        def initialize(file:, expected:, actual:, expected_suite_duration:, actual_suite_duration:)
+          @file = file
+          @expected = expected.to_f
+          @actual = actual.to_f
+          @expected_suite_duration = expected_suite_duration.to_f
+          @actual_suite_duration = actual_suite_duration.to_f
+        end
+
+        def should_report?
+          # guideline proposed in https://gitlab.com/gitlab-org/quality/engineering-productivity/team/-/issues/354
+          exceed_actual_to_expected_ratio_threshold? && test_suite_bottleneck?
+        end
+
+        def ci_pipeline_url_markdown
+          "[#{ci_pipeline_id}](#{ci_pipeline_url})"
+        end
+
+        def ci_pipeline_created_at
+          ENV.fetch('CI_PIPELINE_CREATED_AT', nil)
+        end
+
+        def ci_job_link_markdown
+          "[#{ci_job_name}](#{ci_job_url})"
+        end
+
+        def file_link_markdown
+          "[#{file}](#{file_link})"
+        end
+
+        def actual_percentage
+          (actual / actual_suite_duration * 100).round(2)
+        end
+
+        def name
+          nil
+        end
+
+        private
+
+        def exceed_actual_to_expected_ratio_threshold?
+          actual / expected >= ACTUAL_TO_EXPECTED_SPEC_RUN_TIME_RATIO_THRESHOLD
+        end
+
+        def test_suite_bottleneck?
+          # now we only report bottlenecks when they risk causing job timeouts
+          return unless actual_suite_duration > SUITE_DURATION_THRESHOLD
+
+          actual_percentage > SPEC_WEIGHT_PERCENTAGE_TRESHOLD
+        end
+
+        def ci_job_url
+          ENV.fetch('CI_JOB_URL', nil)
+        end
+
+        def ci_job_name
+          ENV.fetch('CI_JOB_NAME_SLUG', nil)
+        end
+
+        def ci_pipeline_id
+          ENV.fetch('CI_PIPELINE_IID', nil)
+        end
+
+        def ci_pipeline_url
+          ENV.fetch('CI_PIPELINE_URL', nil)
+        end
+
+        def file_link
+          "#{Runtime::Env.file_base_url}#{file}"
+        end
+      end
+    end
+  end
+end

data/lib/gitlab_quality/test_tooling/knapsack_reports/spec_run_time_report.rb

@@ -0,0 +1,60 @@
+# frozen_string_literal: true
+
+require 'json'
+
+module GitlabQuality
+  module TestTooling
+    module KnapsackReports
+      class SpecRunTimeReport
+        attr_reader :expected_report, :actual_report
+
+        def initialize(expected_report_path:, actual_report_path:)
+          @expected_report = parse(expected_report_path)
+          @actual_report = parse(actual_report_path)
+        end
+
+        def filtered_report
+          @filtered_report = actual_report.keys.filter_map do |spec_file|
+            expected_run_time = expected_report[spec_file]
+            actual_run_time = actual_report[spec_file]
+
+            if expected_run_time.nil?
+              puts "#{spec_file} missing from the expected Knapsack report, skipping."
+              next
+            end
+
+            spec_run_time = SpecRunTime.new(
+              file: spec_file,
+              expected: expected_run_time,
+              actual: actual_run_time,
+              expected_suite_duration: expected_test_suite_run_time_total,
+              actual_suite_duration: actual_test_suite_run_time_total
+            )
+
+            spec_run_time if spec_run_time.should_report?
+          end
+        end
+
+        private
+
+        def parse(report_path)
+          JSON.parse(File.read(report_path))
+        end
+
+        def expected_test_suite_run_time_total
+          @expected_test_suite_run_time_total ||=
+            expected_report.reduce(0) do |total_run_time, (_spec_file, run_time)|
+              total_run_time + run_time
+            end
+        end
+
+        def actual_test_suite_run_time_total
+          @actual_test_suite_run_time_total ||=
+            actual_report.reduce(0) do |total_run_time, (_spec_file, run_time)|
+              total_run_time + run_time
+            end
+        end
+      end
+    end
+  end
+end

data/lib/gitlab_quality/test_tooling/report/concerns/issue_reports.rb

@@ -1,13 +1,16 @@
 # frozen_string_literal: true
 
+require 'active_support/core_ext/object/blank'
+
 module GitlabQuality
   module TestTooling
     module Report
       module Concerns
         module IssueReports
-          JOB_URL_REGEX                = %r{(?<job_url>https://(?<host>[\w.]+)/(?<project_path>[\w\-./]+)/-/jobs/\d+)}
+          JOB_URL_REGEX = %r{(?<job_url>https://(?<host>[\w.]+)/(?<project_path>[\w\-./]+)/-/jobs/\d+)}
           FAILED_JOB_DESCRIPTION_REGEX = /First happened in #{JOB_URL_REGEX}\./m
-          REPORT_ITEM_REGEX            = /^1\. \d{4}-\d{2}-\d{2}: #{JOB_URL_REGEX} \((?<pipeline_url>.+)\)$/
+          REPORT_ITEM_REGEX = /^1\. \d{4}-\d{2}-\d{2}: #{JOB_URL_REGEX} \((?<pipeline_url>\S+)\)/
+          LATEST_REPORTS_TO_SHOW = 10
 
           def initial_reports_section(test)
             <<~REPORTS
@@ -17,22 +20,21 @@ module GitlabQuality
             REPORTS
           end
 
-          def add_report_to_issue_description(issue, test)
-            issue_description = issue.description
-
-            # We include the number of reports in the header, for visibility.
-            new_issue_description =
-              if issue_description.include?('### Reports')
-                # We count the number of existing reports.
-                reports_count = issue_description
-                  .scan(REPORT_ITEM_REGEX)
-                  .size.to_i + 1
-                issue_description.sub(/^### Reports.*$/, "### Reports (#{reports_count})")
-              else # For issue with the legacy format, we add the Reports section
-                update_legacy_issue_description(issue_description)
-              end
+          def increment_reports(
+            current_reports_content:,
+            test:,
+            reports_section_header: '### Reports',
+            item_extra_content: nil,
+            reports_extra_content: nil)
+            preserved_content = current_reports_content.split(reports_section_header).first&.strip
+            reports = report_lines(current_reports_content) + [report_list_item(test, item_extra_content: item_extra_content)]
 
-            [new_issue_description, report_list_item(test)].join("\n")
+            [
+              preserved_content,
+              "#{reports_section_header} (#{reports.size})",
+              reports_list(reports),
+              reports_extra_content
+            ].reject(&:blank?).compact.join("\n\n")
           end
 
           def failed_issue_job_url(issue)
@@ -49,8 +51,28 @@ module GitlabQuality
 
           private
 
-          def report_list_item(test)
-            "1. #{Time.new.utc.strftime('%F')}: #{test.ci_job_url} (#{ENV.fetch('CI_PIPELINE_URL', 'pipeline url is missing')})"
+          def report_lines(content)
+            content.lines.grep(REPORT_ITEM_REGEX).map(&:strip)
+          end
+
+          def reports_list(reports)
+            sorted_reports = reports.sort.reverse
+
+            if sorted_reports.size > LATEST_REPORTS_TO_SHOW
+              [
+                "Last 10 reports:",
+                sorted_reports[...LATEST_REPORTS_TO_SHOW].join("\n"),
+                "<details><summary>See #{sorted_reports.size - LATEST_REPORTS_TO_SHOW} more reports</summary>",
+                sorted_reports[LATEST_REPORTS_TO_SHOW..].join("\n"),
+                "</details>"
+              ].join("\n\n")
+            else
+              sorted_reports.join("\n")
+            end
+          end
+
+          def report_list_item(test, item_extra_content: nil)
+            "1. #{Time.new.utc.strftime('%F')}: #{test.ci_job_url} (#{ENV.fetch('CI_PIPELINE_URL', 'pipeline url is missing')}) #{item_extra_content}".strip
           end
 
           def job_urls_from_description(issue_description, regex)
@@ -60,15 +82,6 @@ module GitlabQuality
             end
           end
 
-          def update_legacy_issue_description(issue_description)
-            test_captures = issue_description.scan(JOB_URL_REGEX)
-            reports_count = test_captures.size.to_i + 1
-
-            updated_description = "#{issue_description}\n\n### Reports (#{reports_count})\n"
-            updated_description = [updated_description, *test_captures_to_report_items(test_captures)].join("\n") unless test_captures.empty?
-            updated_description
-          end
-
           def test_captures_to_report_items(test_captures)
             test_captures.map do |ci_job_url, _, _|
               report_list_item(GitlabQuality::TestTooling::TestResult::JsonTestResult.new(

data/lib/gitlab_quality/test_tooling/report/concerns/utils.rb

@@ -16,7 +16,7 @@ module GitlabQuality
           end
 
           def new_issue_title(test)
-            "#{partial_file_path(test.file)} | #{search_safe(test.name)}".strip
+            "[Test] #{partial_file_path(test.file)} | #{search_safe(test.name)}".strip
           end
 
           def partial_file_path(path)
@@ -45,6 +45,28 @@ module GitlabQuality
 
             @pipeline ||= Runtime::Env.pipeline_from_project_name
           end
+
+          def readable_duration(duration_in_seconds)
+            minutes = (duration_in_seconds / 60).to_i
+            seconds = (duration_in_seconds % 60).round(2)
+
+            min_output = normalize_duration_output(minutes, 'minute')
+            sec_output = normalize_duration_output(seconds, 'second')
+
+            "#{min_output} #{sec_output}".strip
+          end
+
+          private
+
+          def normalize_duration_output(number, unit)
+            if number <= 0
+              ""
+            elsif number <= 1
+              "#{number} #{unit}"
+            else
+              "#{number} #{unit}s"
+            end
+          end
         end
       end
     end

data/lib/gitlab_quality/test_tooling/report/flaky_test_issue.rb

@@ -13,20 +13,36 @@ module GitlabQuality
       #   - Find issue by test hash
       #   - Reopen issue if it already exists, but is closed
       class FlakyTestIssue < ReportAsIssue
+        include Concerns::GroupAndCategoryLabels
         include Concerns::IssueReports
 
-        NEW_ISSUE_LABELS      = Set.new(['type::maintenance', 'priority::3', 'severity::3', 'failure::flaky-test']).freeze
-        FOUND_IN_MR_LABEL     = 'found:in MR'
-        FOUND_IN_MASTER_LABEL = 'found:master'
-
-        def initialize(token:, input_files:, project: nil, merge_request_iid: nil, confidential: false, dry_run: false, **_kwargs)
+        IDENTITY_LABELS       = ['test', 'failure::flaky-test', 'automation:bot-authored'].freeze
+        NEW_ISSUE_LABELS      = Set.new(['type::maintenance', 'priority::3', 'severity::3', *IDENTITY_LABELS]).freeze
+        SEARCH_LABELS         = ['test'].freeze
+        FOUND_IN_MR_LABEL     = '~"found:in MR"'
+        FOUND_IN_MASTER_LABEL = '~"found:master"'
+        REPORT_SECTION_HEADER = '### Flakiness reports'
+        REPORTS_DOCUMENTATION = <<~DOC
+          Flaky tests were detected, please refer to the [Flaky tests documentation](https://docs.gitlab.com/ee/development/testing_guide/flaky_tests.html)
+          to learn more about how to reproduce them.
+        DOC
+
+        def initialize(
+          token:,
+          input_files:,
+          base_issue_labels: nil,
+          confidential: false,
+          dry_run: false,
+          project: nil,
+          **_kwargs)
           super(token: token, input_files: input_files, project: project, confidential: confidential, dry_run: dry_run)
-          @merge_request_iid = merge_request_iid
+
+          @base_issue_labels = Set.new(base_issue_labels)
         end
 
         private
 
-        attr_reader :merge_request_iid
+        attr_reader :base_issue_labels
 
         def run!
           puts "Reporting flaky tests in `#{files.join(',')}` as issues in project `#{project}` via the API at `#{Runtime::Env.gitlab_api_base}`."
@@ -34,62 +50,91 @@ module GitlabQuality
           TestResults::Builder.new(files).test_results_per_file do |test_results|
             puts "=> Reporting #{test_results.count} tests in #{test_results.path}"
 
-            test_results.each do |test|
-              next if test.status != 'passed' # We only want failed tests that passed in the end
-
-              create_flaky_issue(test)
-            end
+            process_test_results(test_results)
           end
         end
 
-        def new_issue_title(test)
-          "Flaky test in #{super}"
-        end
+        def process_test_results(test_results)
+          test_results.each do |test|
+            next unless test_is_applicable?(test)
 
-        def new_issue_labels(_test)
-          found_label =
-            if !merge_request_iid || merge_request_iid.empty?
-              FOUND_IN_MASTER_LABEL
-            else
-              FOUND_IN_MR_LABEL
-            end
+            puts " => Reporting flakiness for test '#{test.name}'..."
 
-          NEW_ISSUE_LABELS + [found_label]
+            issues = find_issues_by_hash(test_hash(test), state: 'opened', labels: SEARCH_LABELS)
+            issues << create_issue(test) if issues.empty?
+
+            update_reports(issues, test)
+            collect_issues(test, issues)
+          end
         end
 
-        def new_issue_description(test)
-          super + [
-            "Flaky tests were detected, please refer to the [Flaky tests documentation](https://docs.gitlab.com/ee/development/testing_guide/flaky_tests.html) " \
-            "to learn more about how to reproduce them.",
-            initial_reports_section(test)
-          ].compact.join("\n\n")
+        def test_is_applicable?(test)
+          test.status == 'passed' # We only want failed tests that passed in the end
         end
 
-        def create_flaky_issue(test)
-          puts "   => Finding existing issues for flaky test '#{test.name}' (run time: #{test.run_time} seconds)..."
+        def up_to_date_labels(test:, issue: nil, new_labels: Set.new)
+          (base_issue_labels + super).to_a
+        end
 
-          issues = find_issues_by_hash(test_hash(test))
+        def update_reports(issues, test)
           issues.each do |issue|
-            puts "   => Existing issue link #{issue.web_url}."
+            puts "   => Adding the flaky test to the existing issue: #{issue.web_url}"
+            add_report_to_issue(issue: issue, test: test, related_issues: (issues - [issue]))
+          end
+        end
 
-            puts "   => Adding the flaky test to the existing issue..."
-            add_report_to_issue(issue, test)
+        def add_report_to_issue(issue:, test:, related_issues:)
+          reports_note = existing_reports_note(issue: issue)
+          note_body = [
+            report_body(reports_note: reports_note, test: test),
+            identity_labels_quick_action,
+            relate_issues_quick_actions(related_issues)
+          ].join("\n")
+
+          if reports_note
+            gitlab.edit_issue_note(
+              issue_iid: issue.iid,
+              note_id: reports_note.id,
+              note: note_body
+            )
+          else
+            gitlab.create_issue_note(iid: issue.iid, note: note_body)
+          end
+        end
 
-            if issue.state == 'closed'
-              puts "   => Issue is closed. Reopening it."
-              reopen_issue(issue)
-            end
+        def existing_reports_note(issue:)
+          gitlab.find_issue_notes(iid: issue.iid).find do |note|
+            note.body.start_with?(REPORT_SECTION_HEADER)
           end
+        end
 
-          create_issue(test) unless issues.any?
+        def report_body(reports_note:, test:)
+          increment_reports(
+            current_reports_content: reports_note&.body.to_s,
+            test: test,
+            reports_section_header: REPORT_SECTION_HEADER,
+            item_extra_content: found_label,
+            reports_extra_content: REPORTS_DOCUMENTATION
+          )
+        end
+
+        def found_label
+          if ENV.key?('CI_MERGE_REQUEST_IID')
+            FOUND_IN_MR_LABEL
+          else
+            FOUND_IN_MASTER_LABEL
+          end
         end
 
-        def add_report_to_issue(issue, test)
-          gitlab.edit_issue(iid: issue.iid, options: { description: add_report_to_issue_description(issue, test) })
+        def identity_labels_quick_action
+          labels_list = IDENTITY_LABELS.map { |label| %(~"#{label}") }.join(' ')
+          %(/label #{labels_list})
         end
 
-        def reopen_issue(issue)
-          gitlab.edit_issue(iid: issue.iid, options: { state_event: 'reopen' })
+        def relate_issues_quick_actions(issues)
+          issues.map do |issue|
+            "/relate #{issue.web_url}"
+          end.join("\n")
         end
       end
     end