gitlab-qa 6.7.0 → 6.8.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 1615e54144d0b669245a15b6a5874efc1d00157fe859dbc6ff2e723380e9da10
-  data.tar.gz: f3ccd0528987c838c793d1bbc2f742833584756bf7a87c58c29b60fa2f7b6ac3
+  metadata.gz: daf050b7d871f8eefdc1fa68b052608c11ee186b927fe89accfceca15291ed45
+  data.tar.gz: 4510e98db8e41c11a1585070a39cd19769384917cff551195942955974dcd78d
 SHA512:
-  metadata.gz: 47fce0af7a769ccb9dca14c87f3d1f169495324f753f3db60bb8aa10fc6e05a4612f0bc1704e884dca3d7349bd6d0972d5ca77421bba71f60fd03258e24bc52b
-  data.tar.gz: f1d65176693a0091989392eec19425a34d2cbc1ee92cb428178ee22ca438b19e7246fd1306bfdaaa73f5d531a6c01ad5a6b2ee1324ca72ef0ba9b255ae4782f3
+  metadata.gz: 3a3a6dc22aac1a77fe46f1379a43b489d08cc771b38ea629f9ff3a8f24fd9886f767ed3b3ca8aa18bfeabadc9019aefe2933261c969167e6c8e95f7f75e2c6a1
+  data.tar.gz: c612e3b42691a28864bcf703fdf89cbbf57f75817341f867d1e8486a25adcbfcf262f3506157841d734813e699e628620dd524af22b4f8da17c689a96c6116cf

data/.gitlab-ci.yml

@@ -2,6 +2,7 @@ stages:
   - check
   - release
   - test
+  - report
   - notify
 
 default:
@@ -48,6 +49,10 @@ variables:
   QA_CAN_TEST_GIT_PROTOCOL_V2: "true"
   QA_CAN_TEST_PRAEFECT: "false"
   QA_TESTCASES_REPORTING_PROJECT: "gitlab-org/quality/testcases"
+  QA_FAILURES_REPORTING_PROJECT: "gitlab-org/gitlab"
+  # The --dry-run or --max-diff-ratio option can be set to modify the behavior of `exe/gitlab-qa-report --relate-failure-issue` without releasing a new gem version.
+  QA_FAILURES_REPORTER_OPTIONS: "--dry-run"
+  QA_TESTCASE_SESSIONS_PROJECT: "gitlab-org/quality/testcase-sessions"
 
 .check-base:
   stage: check
@@ -95,6 +100,7 @@ release:
     - exe/gitlab-qa-report --update-screenshot-path "gitlab-qa-run-*/**/rspec-*.xml"
     - export GITLAB_QA_ACCESS_TOKEN="$GITLAB_QA_PRODUCTION_ACCESS_TOKEN"
     - if [ "$TOP_UPSTREAM_SOURCE_REF" == "master" ] || [[ "$TOP_UPSTREAM_SOURCE_JOB" == https://ops.gitlab.net* ]]; then exe/gitlab-qa-report --report-in-issues "gitlab-qa-run-*/**/rspec-*.json" --project "$QA_TESTCASES_REPORTING_PROJECT" || true; fi
+    - if [ "$TOP_UPSTREAM_SOURCE_REF" == "master" ] || [[ "$TOP_UPSTREAM_SOURCE_JOB" == https://ops.gitlab.net* ]]; then exe/gitlab-qa-report --relate-failure-issue "gitlab-qa-run-*/**/rspec-*.json" --project "$QA_FAILURES_REPORTING_PROJECT" $QA_FAILURES_REPORTER_OPTIONS || true; fi
     - exit $test_run_exit_code
 
 .ce-qa:
@@ -333,6 +339,7 @@ ce:update:
     - exe/gitlab-qa Test::Omnibus::Update ${RELEASE:=CE} ${RELEASE:=CE} -- $RSPEC_REPORT_OPTS || test_run_exit_code=$?
     - export GITLAB_QA_ACCESS_TOKEN="$GITLAB_QA_PRODUCTION_ACCESS_TOKEN"
     - if [ "$TOP_UPSTREAM_SOURCE_REF" == "master" ] || [[ "$TOP_UPSTREAM_SOURCE_JOB" == https://ops.gitlab.net* ]]; then exe/gitlab-qa-report --report-in-issues "gitlab-qa-run-*/**/rspec-*.json" --project "$QA_TESTCASES_REPORTING_PROJECT" || true; fi
+    - if [ "$TOP_UPSTREAM_SOURCE_REF" == "master" ] || [[ "$TOP_UPSTREAM_SOURCE_JOB" == https://ops.gitlab.net* ]]; then exe/gitlab-qa-report --relate-failure-issue "gitlab-qa-run-*/**/rspec-*.json" --project "$QA_FAILURES_REPORTING_PROJECT" $QA_FAILURES_REPORTER_OPTIONS || true; fi
     - exit $test_run_exit_code
   extends:
     - .test
@@ -347,6 +354,7 @@ ce:update-quarantine:
     - exe/gitlab-qa Test::Omnibus::Update ${RELEASE:=CE} ${RELEASE:=CE} -- --tag quarantine --tag ~orchestrated $RSPEC_REPORT_OPTS || test_run_exit_code=$?
     - export GITLAB_QA_ACCESS_TOKEN="$GITLAB_QA_PRODUCTION_ACCESS_TOKEN"
     - if [ "$TOP_UPSTREAM_SOURCE_REF" == "master" ] || [[ "$TOP_UPSTREAM_SOURCE_JOB" == https://ops.gitlab.net* ]]; then exe/gitlab-qa-report --report-in-issues "gitlab-qa-run-*/**/rspec-*.json" --project "$QA_TESTCASES_REPORTING_PROJECT" || true; fi
+    - if [ "$TOP_UPSTREAM_SOURCE_REF" == "master" ] || [[ "$TOP_UPSTREAM_SOURCE_JOB" == https://ops.gitlab.net* ]]; then exe/gitlab-qa-report --relate-failure-issue "gitlab-qa-run-*/**/rspec-*.json" --project "$QA_FAILURES_REPORTING_PROJECT" $QA_FAILURES_REPORTER_OPTIONS || true; fi
     - exit $test_run_exit_code
   extends:
     - .test
@@ -360,6 +368,7 @@ ee:update:
     - exe/gitlab-qa Test::Omnibus::Update ${RELEASE:=EE} ${RELEASE:=EE} -- $RSPEC_REPORT_OPTS || test_run_exit_code=$?
     - export GITLAB_QA_ACCESS_TOKEN="$GITLAB_QA_PRODUCTION_ACCESS_TOKEN"
     - if [ "$TOP_UPSTREAM_SOURCE_REF" == "master" ] || [[ "$TOP_UPSTREAM_SOURCE_JOB" == https://ops.gitlab.net* ]]; then exe/gitlab-qa-report --report-in-issues "gitlab-qa-run-*/**/rspec-*.json" --project "$QA_TESTCASES_REPORTING_PROJECT" || true; fi
+    - if [ "$TOP_UPSTREAM_SOURCE_REF" == "master" ] || [[ "$TOP_UPSTREAM_SOURCE_JOB" == https://ops.gitlab.net* ]]; then exe/gitlab-qa-report --relate-failure-issue "gitlab-qa-run-*/**/rspec-*.json" --project "$QA_FAILURES_REPORTING_PROJECT" $QA_FAILURES_REPORTER_OPTIONS || true; fi
     - exit $test_run_exit_code
   extends:
     - .test
@@ -374,6 +383,7 @@ ee:update-quarantine:
     - exe/gitlab-qa Test::Omnibus::Update ${RELEASE:=EE} ${RELEASE:=EE} -- --tag quarantine --tag ~orchestrated $RSPEC_REPORT_OPTS || test_run_exit_code=$?
     - export GITLAB_QA_ACCESS_TOKEN="$GITLAB_QA_PRODUCTION_ACCESS_TOKEN"
     - if [ "$TOP_UPSTREAM_SOURCE_REF" == "master" ] || [[ "$TOP_UPSTREAM_SOURCE_JOB" == https://ops.gitlab.net* ]]; then exe/gitlab-qa-report --report-in-issues "gitlab-qa-run-*/**/rspec-*.json" --project "$QA_TESTCASES_REPORTING_PROJECT" || true; fi
+    - if [ "$TOP_UPSTREAM_SOURCE_REF" == "master" ] || [[ "$TOP_UPSTREAM_SOURCE_JOB" == https://ops.gitlab.net* ]]; then exe/gitlab-qa-report --relate-failure-issue "gitlab-qa-run-*/**/rspec-*.json" --project "$QA_FAILURES_REPORTING_PROJECT" $QA_FAILURES_REPORTER_OPTIONS || true; fi
     - exit $test_run_exit_code
   extends:
     - .test
@@ -941,6 +951,22 @@ staging:
     - .only-qa
   allow_failure: true
 
+generate_test_session:
+  stage: report
+  rules:
+    - if: '$TOP_UPSTREAM_SOURCE_JOB && $TOP_UPSTREAM_SOURCE_REF == "master"'
+      when: always
+    - if: '$TOP_UPSTREAM_SOURCE_JOB =~ /\Ahttps:\/\/ops.gitlab.net\//'
+      when: always
+  artifacts:
+    when: always
+    expire_in: 10d
+    paths:
+      - REPORT_ISSUE_URL
+  script:
+    - export GITLAB_QA_ACCESS_TOKEN="$GITLAB_QA_PRODUCTION_ACCESS_TOKEN"
+    - exe/gitlab-qa-report --generate-test-session "gitlab-qa-run-*/**/rspec-*.json" --project "$QA_TESTCASE_SESSIONS_PROJECT"
+
 .notify_upstream_commit:
   stage: notify
   image: ruby:2.6
@@ -966,7 +992,7 @@ notify_upstream_commit:failure:
 .notify_slack:
   image: alpine
   stage: notify
-  dependencies: []
+  dependencies: ['generate_test_session']
   cache: {}
   before_script:
     - apk update && apk add git curl bash
@@ -985,4 +1011,4 @@ notify_slack:
     - echo "RELEASE is ${RELEASE}"
     - echo "CI_PIPELINE_URL is $CI_PIPELINE_URL"
     - echo "TOP_UPSTREAM_SOURCE_JOB is $TOP_UPSTREAM_SOURCE_JOB"
-    - bin/slack $NOTIFY_CHANNEL "☠️ QA against $RELEASE failed! ☠️ See $CI_PIPELINE_URL (triggered from $TOP_UPSTREAM_SOURCE_JOB)" ci_failing
+    - 'bin/slack $NOTIFY_CHANNEL "☠️ QA against $RELEASE failed! ☠️ See the test session report: $(cat REPORT_ISSUE_URL), and pipeline: $CI_PIPELINE_URL (triggered from $TOP_UPSTREAM_SOURCE_JOB)" ci_failing'

data/lib/gitlab/qa.rb

@@ -99,12 +99,15 @@ module Gitlab
 
     module Report
       autoload :GitlabIssueClient, 'gitlab/qa/report/gitlab_issue_client'
+      autoload :GitlabIssueDryClient, 'gitlab/qa/report/gitlab_issue_dry_client'
       autoload :BaseTestResults, 'gitlab/qa/report/base_test_results'
+      autoload :RelateFailureIssue, 'gitlab/qa/report/relate_failure_issue'
       autoload :JsonTestResults, 'gitlab/qa/report/json_test_results'
       autoload :JUnitTestResults, 'gitlab/qa/report/junit_test_results'
       autoload :PrepareStageReports, 'gitlab/qa/report/prepare_stage_reports'
       autoload :ReportAsIssue, 'gitlab/qa/report/report_as_issue'
       autoload :ResultsInIssues, 'gitlab/qa/report/results_in_issues'
+      autoload :GenerateTestSession, 'gitlab/qa/report/generate_test_session'
       autoload :SummaryTable, 'gitlab/qa/report/summary_table'
       autoload :TestResult, 'gitlab/qa/report/test_result'
       autoload :UpdateScreenshotPath, 'gitlab/qa/report/update_screenshot_path'

data/lib/gitlab/qa/report/base_test_results.rb

@@ -6,8 +6,11 @@ module Gitlab
       class BaseTestResults
         include Enumerable
 
+        attr_reader :path
+
         def initialize(path)
-          @results = parse(path)
+          @path = path
+          @results = parse
           @testcases = process
         end
 
@@ -15,7 +18,7 @@ module Gitlab
           testcases.each(&block)
         end
 
-        def write(path)
+        def write
           raise NotImplementedError
         end
 
@@ -23,7 +26,7 @@ module Gitlab
 
         attr_reader :results, :testcases
 
-        def parse(path)
+        def parse
           raise NotImplementedError
         end
 

data/lib/gitlab/qa/report/generate_test_session.rb

@@ -0,0 +1,203 @@
+# frozen_string_literal: true
+
+module Gitlab
+  module QA
+    module Report
+      class GenerateTestSession < ReportAsIssue
+        private
+
+        def run!
+          puts "Generating test results in `#{files.join(',')}` as issues in project `#{project}` via the API at `#{Runtime::Env.gitlab_api_base}`."
+
+          tests = Dir.glob(files).flat_map do |path|
+            puts "Loading tests in #{path}"
+
+            Report::JsonTestResults.new(path).to_a
+          end
+
+          issue = gitlab.create_issue(
+            title: "Test session report | #{Runtime::Env.deploy_environment}",
+            description: generate_description(tests),
+            labels: ['Quality', 'QA', 'triage report']
+          )
+
+          File.write('REPORT_ISSUE_URL', issue.web_url)
+        end
+
+        def generate_description(tests)
+          <<~MARKDOWN
+          ## Session summary
+
+          * Deploy version: #{Runtime::Env.deploy_version}
+          * Pipeline: [#{Runtime::Env.ci_pipeline_id}](#{Runtime::Env.ci_pipeline_url})
+          #{generate_summary(tests: tests)}
+
+          #{generate_stages_listing(tests)}
+
+          ## Release QA issue
+
+          * #{Runtime::Env.qa_issue_url}
+          MARKDOWN
+        end
+
+        def generate_summary(tests:, tests_by_status: nil)
+          tests_by_status ||= tests.group_by(&:status)
+          total = tests.size
+          passed = tests_by_status['passed']&.size || 0
+          failed = tests_by_status['failed']&.size || 0
+          others = total - passed - failed
+
+          <<~MARKDOWN.chomp
+          * Total #{total} tests
+          * Passed #{passed} tests
+          * Failed #{failed} tests
+          * #{others} other tests (usually skipped)
+          MARKDOWN
+        end
+
+        def generate_stages_listing(tests)
+          generate_tests_by_stage(tests).map do |stage, tests_for_stage|
+            tests_by_status = tests_for_stage.group_by(&:status)
+
+            <<~MARKDOWN.chomp
+            ### #{stage&.capitalize || 'Unknown'}
+
+            #{generate_summary(
+              tests: tests_for_stage, tests_by_status: tests_by_status)}
+
+            #{generate_testcase_listing_by_status(
+              tests: tests_for_stage, tests_by_status: tests_by_status)}
+            MARKDOWN
+          end.join("\n\n")
+        end
+
+        def generate_tests_by_stage(tests)
+          # https://about.gitlab.com/handbook/product/product-categories/#devops-stages
+          ordering = %w[
+            manage
+            plan
+            create
+            verify
+            package
+            release
+            configure
+            monitor
+            secure
+            defend
+            growth
+            fulfillment
+            enablement
+          ]
+
+          tests.sort_by do |test|
+            ordering.index(test.stage) || ordering.size
+          end.group_by(&:stage)
+        end
+
+        def generate_testcase_listing_by_status(tests:, tests_by_status:)
+          failed_tests = tests_by_status['failed']
+          passed_tests = tests_by_status['passed']
+          other_tests = tests.reject do |test|
+            test.status == 'failed' || test.status == 'passed'
+          end
+
+          [
+            (failed_listings(failed_tests) if failed_tests),
+            (passed_listings(passed_tests) if passed_tests),
+            (other_listings(other_tests) if other_tests.any?)
+          ].compact.join("\n\n")
+        end
+
+        def failed_listings(failed_tests)
+          generate_testcase_listing(failed_tests)
+        end
+
+        def passed_listings(passed_tests)
+          <<~MARKDOWN.chomp
+            <details><summary>Passed tests:</summary>
+
+            #{generate_testcase_listing(passed_tests)}
+
+            </details>
+          MARKDOWN
+        end
+
+        def other_listings(other_tests)
+          <<~MARKDOWN.chomp
+            <details><summary>Other tests:</summary>
+
+            #{generate_testcase_listing(other_tests)}
+
+            </details>
+          MARKDOWN
+        end
+
+        def generate_testcase_listing(tests)
+          body = tests.group_by(&:testcase).map do |testcase, tests_with_same_testcase|
+            tests_with_same_testcase.sort_by!(&:name)
+            [
+              generate_test_text(testcase, tests_with_same_testcase),
+              generate_test_job(tests_with_same_testcase),
+              generate_test_status(tests_with_same_testcase),
+              generate_test_actions(tests_with_same_testcase)
+            ].join(' | ')
+          end.join("\n")
+
+          <<~MARKDOWN.chomp
+          | Test | Job | Status | Action |
+          | - | - | - | - |
+          #{body}
+          MARKDOWN
+        end
+
+        def generate_test_text(testcase, tests_with_same_testcase)
+          text = tests_with_same_testcase.map(&:name).uniq.join(', ')
+
+          if testcase
+            "[#{text}](#{testcase})"
+          else
+            text
+          end
+        end
+
+        def generate_test_job(tests_with_same_testcase)
+          tests_with_same_testcase.map(&:ci_job_url).uniq.map do |ci_job_url|
+            ci_job_id = ci_job_url[/\d+\z/]
+
+            "[#{ci_job_id}](#{ci_job_url})"
+          end.join(', ')
+        end
+
+        def generate_test_status(tests_with_same_testcase)
+          tests_with_same_testcase.map(&:status).uniq.map do |status|
+            %(~"#{status}")
+          end.join(', ')
+        end
+
+        def generate_test_actions(tests_with_same_testcase)
+          # All failed tests would be grouped together, meaning that
+          # if one failed, all the tests here would be failed too.
+          # So this check is safe. Same applies to 'passed'.
+          # But all other status might be mixing together,
+          # we cannot assume other statuses.
+          if tests_with_same_testcase.first.status == 'failed'
+            tests_having_failure_issue =
+              tests_with_same_testcase.select(&:failure_issue)
+
+            if tests_having_failure_issue.any?
+              items = tests_having_failure_issue.map do |test|
+                "<li>[ ] [failure issue](#{test.failure_issue})</li>"
+              end.join(' ')
+
+              "<ul>#{items}</ul>"
+            else
+              '<ul><li>[ ] failure issue exists or was created</li></ul>'
+            end
+          else
+            '-'
+          end
+        end
+      end
+    end
+  end
+end

data/lib/gitlab/qa/report/gitlab_issue_client.rb

@@ -45,7 +45,7 @@ module Gitlab
           abort_not_permitted
         end
 
-        def find_issues(iid:, options: {}, &select)
+        def find_issues(iid: nil, options: {}, &select)
           select ||= :itself
 
           handle_gitlab_client_exceptions do
@@ -57,15 +57,17 @@ module Gitlab
           end
         end
 
+        def find_issue_discussions(iid:)
+          handle_gitlab_client_exceptions do
+            Gitlab.issue_discussions(project, iid, order_by: 'created_at', sort: 'asc')
+          end
+        end
+
         def create_issue(title:, description:, labels:)
-          puts "Creating issue..."
+          attrs = { description: description, labels: labels }
 
           handle_gitlab_client_exceptions do
-            Gitlab.create_issue(
-              project,
-              title,
-              { description: description, labels: labels }
-            )
+            Gitlab.create_issue(project, title, attrs)
           end
         end
 
@@ -75,6 +77,18 @@ module Gitlab
           end
         end
 
+        def create_issue_note(iid:, note:)
+          handle_gitlab_client_exceptions do
+            Gitlab.create_issue_note(project, iid, note)
+          end
+        end
+
+        def add_note_to_issue_discussion_as_thread(iid:, discussion_id:, body:)
+          handle_gitlab_client_exceptions do
+            Gitlab.add_note_to_issue_discussion_as_thread(project, iid, discussion_id, body: body)
+          end
+        end
+
         def handle_gitlab_client_exceptions
           yield
         rescue Gitlab::Error::NotFound

data/lib/gitlab/qa/report/gitlab_issue_dry_client.rb

@@ -0,0 +1,28 @@
+# frozen_string_literal: true
+
+module Gitlab
+  module QA
+    module Report
+      class GitlabIssueDryClient < GitlabIssueClient
+        def create_issue(title:, description:, labels:)
+          attrs = { description: description, labels: labels }
+
+          puts "The following issue would have been created:"
+          puts "project: #{project}, title: #{title}, attrs: #{attrs}"
+        end
+
+        def edit_issue(iid:, options: {})
+          puts "The #{project}##{iid} issue would have been updated with: #{options}"
+        end
+
+        def create_issue_note(iid:, note:)
+          puts "The following note would have been posted on #{project}##{iid} issue: #{note}"
+        end
+
+        def add_note_to_issue_discussion_as_thread(iid:, discussion_id:, body:)
+          puts "The following discussion note would have been posted on #{project}##{iid} (discussion #{discussion_id}) issue: #{body}"
+        end
+      end
+    end
+  end
+end

data/lib/gitlab/qa/report/json_test_results.rb

@@ -6,7 +6,7 @@ module Gitlab
   module QA
     module Report
       class JsonTestResults < BaseTestResults
-        def write(path)
+        def write
           json = results.merge('examples' => testcases.map(&:report))
 
           File.write(path, JSON.pretty_generate(json))
@@ -14,7 +14,7 @@ module Gitlab
 
         private
 
-        def parse(path)
+        def parse
           JSON.parse(File.read(path))
         end
 

data/lib/gitlab/qa/report/junit_test_results.rb

@@ -6,13 +6,13 @@ module Gitlab
   module QA
     module Report
       class JUnitTestResults < BaseTestResults
-        def write(path)
+        def write
           # Ignore it for now
         end
 
         private
 
-        def parse(path)
+        def parse
           Nokogiri::XML.parse(File.read(path))
         end
 

data/lib/gitlab/qa/report/relate_failure_issue.rb

@@ -0,0 +1,139 @@
+# frozen_string_literal: true
+
+require 'nokogiri'
+require 'active_support/core_ext/enumerable'
+require 'rubygems/text'
+
+module Gitlab
+  module QA
+    module Report
+      # Uses the API to create or update GitLab issues with the results of tests from RSpec report files.
+      class RelateFailureIssue < ReportAsIssue
+        DEFAULT_MAX_DIFF_RATIO_FOR_DETECTION = 0.05
+        STACKTRACE_REGEX = %r{### Stack trace\s*(```)\s*.*(Failure/Error: .+)(\1)}m.freeze
+        NEW_ISSUE_LABELS = Set.new(%w[QA Quality test failure::investigating priority::2]).freeze
+
+        MultipleIssuesFound = Class.new(StandardError)
+
+        def initialize(max_diff_ratio: DEFAULT_MAX_DIFF_RATIO_FOR_DETECTION, **kwargs)
+          super
+          @max_diff_ratio = max_diff_ratio.to_f
+        end
+
+        private
+
+        attr_reader :max_diff_ratio
+
+        def run!
+          puts "Reporting test failures in `#{files.join(',')}` as issues in project `#{project}` via the API at `#{Runtime::Env.gitlab_api_base}`."
+
+          test_results_per_file do |test_results|
+            puts "=> Reporting tests in #{test_results.path}"
+
+            test_results.each do |test|
+              next if test.failures.empty?
+
+              relate_test_to_issue(test)
+            end
+
+            test_results.write
+          end
+        end
+
+        def relate_test_to_issue(test)
+          puts "  => Searching issues for test '#{test.name}'..."
+
+          begin
+            issue = find_or_create_issue(test)
+            return unless issue
+
+            post_failed_job_note(issue, test)
+            puts "  => Marked #{issue.web_url} as related to #{test.testcase}."
+          rescue MultipleIssuesFound => e
+            warn(e.message)
+          end
+        end
+
+        def find_or_create_issue(test)
+          issue, diff_ratio = find_failure_issue(test)
+
+          if issue
+            puts "  => Found issue #{issue.web_url} for test '#{test.name}' with a diff ratio of #{(diff_ratio * 100).round(2)}%."
+          else
+            issue = create_issue(test)
+            puts "  => Created new issue: #{issue.web_url} for test '#{test.name}'." if issue
+          end
+
+          issue
+        end
+
+        def failure_issues(test)
+          gitlab.find_issues(options: { state: 'opened', labels: 'QA' }).select do |issue|
+            issue_title = issue.title.strip
+            issue_title.include?(test.name) || issue_title.include?(partial_file_path(test.file))
+          end
+        end
+
+        def find_relevant_failure_issues(test) # rubocop:disable Metrics/AbcSize
+          ld = Class.new.extend(Gem::Text).method(:levenshtein_distance)
+          first_test_failure_stacktrace = test.failures.first['message_lines'].join("\n")
+
+          # Search with the `search` param returns 500 errors, so we filter by ~QA and then filter further in Ruby
+          failure_issues(test).each_with_object({}) do |issue, memo|
+            relevant_issue_stacktrace = find_issue_stacktrace(issue)
+            unless relevant_issue_stacktrace
+              puts "  => [DEBUG] Found issue #{issue.web_url} but stacktrace doesn't match."
+              next
+            end
+
+            distance = ld.call(first_test_failure_stacktrace, relevant_issue_stacktrace)
+            diff_ratio = (distance.to_f / first_test_failure_stacktrace.size).round(3)
+            if diff_ratio <= max_diff_ratio
+              memo[issue] = diff_ratio
+            else
+              puts "  => [DEBUG] Found issue #{issue.web_url} but stacktrace is too different (#{(diff_ratio * 100).round(2)}%)."
+            end
+          end
+        end
+
+        def find_issue_stacktrace(issue)
+          issue_stacktrace_match = issue.description.match(STACKTRACE_REGEX)
+
+          if issue_stacktrace_match
+            issue_stacktrace_match[2].gsub(/^#.*$/, '').strip
+          else
+            puts "\n  => Stacktrace couldn't be found for #{issue.web_url}:\n\n#{issue.description}\n\n----------------------------------\n"
+          end
+        end
+
+        def find_failure_issue(test)
+          relevant_issues = find_relevant_failure_issues(test)
+
+          return nil if relevant_issues.empty?
+
+          best_matching_issue, smaller_diff_ratio = relevant_issues.min_by { |_, diff_ratio| diff_ratio }
+
+          unless relevant_issues.values.count(smaller_diff_ratio) == 1 # rubocop:disable Style/IfUnlessModifier
+            raise(MultipleIssuesFound, %(Too many issues found for test '#{test.name}' (`#{test.file}`)!))
+          end
+
+          test.failure_issue ||= best_matching_issue.web_url
+
+          [best_matching_issue, smaller_diff_ratio]
+        end
+
+        def new_issue_description(test)
+          super + "\n\n### Stack trace\n\n```\n#{test.failures.first['message_lines'].join("\n")}\n```"
+        end
+
+        def new_issue_labels(test)
+          NEW_ISSUE_LABELS + up_to_date_labels(test: test)
+        end
+
+        def post_failed_job_note(issue, test)
+          gitlab.create_issue_note(iid: issue.iid, note: "/relate #{test.testcase}")
+        end
+      end
+    end
+  end
+end

data/lib/gitlab/qa/report/report_as_issue.rb

@@ -1,13 +1,17 @@
 # frozen_string_literal: true
 
+require 'set'
+
 module Gitlab
   module QA
     module Report
       class ReportAsIssue
-        def initialize(token:, input_files:, project: nil)
-          @gitlab = GitlabIssueClient.new(token: token, project: project)
-          @files = Array(input_files)
+        MAX_TITLE_LENGTH = 255
+
+        def initialize(token:, input_files:, project: nil, dry_run: false, **kwargs)
           @project = project
+          @gitlab = (dry_run ? GitlabIssueDryClient : GitlabIssueClient).new(token: token, project: project)
+          @files = Array(input_files)
         end
 
         def invoke!
@@ -24,6 +28,14 @@ module Gitlab
           raise NotImplementedError
         end
 
+        def new_issue_description(test)
+          "### Full description\n\n#{search_safe(test.name)}\n\n### File path\n\n#{test.file}"
+        end
+
+        def new_issue_labels(test)
+          []
+        end
+
         def validate_input!
           assert_project!
           assert_input_files!(files)
@@ -41,6 +53,92 @@ module Gitlab
 
           abort "Please provide valid JUnit report files. No files were found matching `#{files.join(',')}`"
         end
+
+        def test_results_per_file
+          Dir.glob(files).each do |path|
+            extension = File.extname(path)
+
+            test_results =
+              case extension
+              when '.json'
+                Report::JsonTestResults.new(path)
+              when '.xml'
+                Report::JUnitTestResults.new(path)
+              else
+                raise "Unknown extension #{extension}"
+              end
+
+            yield test_results
+          end
+        end
+
+        def create_issue(test)
+          gitlab.create_issue(
+            title: title_from_test(test),
+            description: new_issue_description(test),
+            labels: new_issue_labels(test)
+          )
+        end
+
+        def issue_labels(issue)
+          issue&.labels&.to_set || Set.new
+        end
+
+        def update_labels(issue, test)
+          new_labels = up_to_date_labels(test: test, issue: issue)
+
+          return if issue_labels(issue) == new_labels
+
+          gitlab.edit_issue(iid: issue.iid, options: { labels: new_labels.to_a })
+        end
+
+        def up_to_date_labels(test:, issue: nil)
+          labels = issue_labels(issue)
+          labels << "Enterprise Edition" if ee_test?(test)
+          quarantine_job? ? labels << "quarantine" : labels.delete("quarantine")
+
+          labels
+        end
+
+        def ee_test?(test)
+          test.file =~ %r{features/ee/(api|browser_ui)}
+        end
+
+        def quarantine_job?
+          Runtime::Env.ci_job_name&.include?('quarantine')
+        end
+
+        def partial_file_path(path)
+          path.match(/((api|browser_ui).*)/i)[1]
+        end
+
+        def title_from_test(test)
+          title = "#{partial_file_path(test.file)} | #{search_safe(test.name)}".strip
+
+          return title unless title.length > MAX_TITLE_LENGTH
+
+          "#{title[0...MAX_TITLE_LENGTH - 3]}..."
+        end
+
+        def search_safe(value)
+          value.delete('"')
+        end
+
+        def pipeline
+          # Gets the name of the pipeline the test was run in, to be used as the key of a scoped label
+          #
+          # Tests can be run in several pipelines:
+          #   gitlab-qa, nightly, master, staging, canary, production, preprod, and MRs
+          #
+          # Some of those run in their own project, so CI_PROJECT_NAME is the name we need. Those are:
+          #   nightly, staging, canary, production, and preprod
+          #
+          # MR, master, and gitlab-qa tests run in gitlab-qa, but we only want to report tests run on master
+          # because the other pipelines will be monitored by the author of the MR that triggered them.
+          # So we assume that we're reporting a master pipeline if the project name is 'gitlab-qa'.
+
+          @pipeline ||= Runtime::Env.pipeline_from_project_name
+        end
       end
     end
   end

data/lib/gitlab/qa/report/results_in_issues.rb

@@ -8,31 +8,19 @@ module Gitlab
     module Report
       # Uses the API to create or update GitLab issues with the results of tests from RSpec report files.
       class ResultsInIssues < ReportAsIssue
-        MAX_TITLE_LENGTH = 255
-
         private
 
         def run!
           puts "Reporting test results in `#{files.join(',')}` as issues in project `#{project}` via the API at `#{Runtime::Env.gitlab_api_base}`."
 
-          Dir.glob(files).each do |path|
-            puts "Reporting tests in #{path}"
-            extension = File.extname(path)
-
-            case extension
-            when '.json'
-              test_results = Report::JsonTestResults.new(path)
-            when '.xml'
-              test_results = Report::JUnitTestResults.new(path)
-            else
-              raise "Unknown extension #{extension}"
-            end
+          test_results_per_file do |test_results|
+            puts "Reporting tests in #{test_results.path}"
 
             test_results.each do |test|
               report_test(test)
             end
 
-            test_results.write(path)
+            test_results.write
           end
         end
 
@@ -72,12 +60,14 @@ module Gitlab
           issues.first
         end
 
-        def create_issue(test)
-          gitlab.create_issue(
-            title: title_from_test(test),
-            description: "### Full description\n\n#{search_safe(test.name)}\n\n### File path\n\n#{test.file}",
-            labels: 'status::automated'
-          )
+        def new_issue_labels(test)
+          %w[status::automated]
+        end
+
+        def up_to_date_labels(test:, issue: nil)
+          labels = super
+          labels.delete_if { |label| label.start_with?("#{pipeline}::") }
+          labels << (test.failures.empty? ? "#{pipeline}::passed" : "#{pipeline}::failed")
         end
 
         def iid_from_testcase_url(url)
@@ -85,23 +75,7 @@ module Gitlab
         end
 
         def search_term(test)
-          %("#{test.file}" "#{search_safe(test.name)}")
-        end
-
-        def title_from_test(test)
-          title = "#{partial_file_path(test.file)} | #{search_safe(test.name)}".strip
-
-          return title unless title.length > MAX_TITLE_LENGTH
-
-          "#{title[0...MAX_TITLE_LENGTH - 3]}..."
-        end
-
-        def partial_file_path(path)
-          path.match(/((api|browser_ui).*)/i)[1]
-        end
-
-        def search_safe(value)
-          value.delete('"')
+          %("#{partial_file_path(test.file)}" "#{search_safe(test.name)}")
         end
 
         def note_status(issue, test)
@@ -109,13 +83,11 @@ module Gitlab
 
           note = note_content(test)
 
-          gitlab.handle_gitlab_client_exceptions do
-            Gitlab.issue_discussions(project, issue.iid, order_by: 'created_at', sort: 'asc').each do |discussion|
-              return add_note_to_discussion(issue.iid, discussion.id) if new_note_matches_discussion?(note, discussion)
-            end
-
-            Gitlab.create_issue_note(project, issue.iid, note)
+          gitlab.find_issue_discussions(iid: issue.iid).each do |discussion|
+            return gitlab.add_note_to_issue_discussion_as_thread(iid: issue.iid, discussion_id: discussion.id, body: failure_summary) if new_note_matches_discussion?(note, discussion)
           end
+
+          gitlab.create_issue_note(iid: issue.iid, note: note)
         end
 
         def note_content(test)
@@ -143,10 +115,6 @@ module Gitlab
           summary.join(' ')
         end
 
-        def quarantine_job?
-          Runtime::Env.ci_job_name&.include?('quarantine')
-        end
-
         def new_note_matches_discussion?(note, discussion)
           note_error = error_and_stack_trace(note)
           discussion_error = error_and_stack_trace(discussion.notes.first['body'])
@@ -163,42 +131,6 @@ module Gitlab
 
           result
         end
-
-        def add_note_to_discussion(issue_iid, discussion_id)
-          gitlab.handle_gitlab_client_exceptions do
-            Gitlab.add_note_to_issue_discussion_as_thread(project, issue_iid, discussion_id, body: failure_summary)
-          end
-        end
-
-        def update_labels(issue, test)
-          labels = issue.labels
-          labels.delete_if { |label| label.start_with?("#{pipeline}::") }
-          labels << (test.failures.empty? ? "#{pipeline}::passed" : "#{pipeline}::failed")
-          labels << "Enterprise Edition" if ee_test?(test)
-          quarantine_job? ? labels << "quarantine" : labels.delete("quarantine")
-
-          gitlab.edit_issue(iid: issue.iid, options: { labels: labels })
-        end
-
-        def ee_test?(test)
-          test.file =~ %r{features/ee/(api|browser_ui)}
-        end
-
-        def pipeline
-          # Gets the name of the pipeline the test was run in, to be used as the key of a scoped label
-          #
-          # Tests can be run in several pipelines:
-          #   gitlab-qa, nightly, master, staging, canary, production, preprod, and MRs
-          #
-          # Some of those run in their own project, so CI_PROJECT_NAME is the name we need. Those are:
-          #   nightly, staging, canary, production, and preprod
-          #
-          # MR, master, and gitlab-qa tests run in gitlab-qa, but we only want to report tests run on master
-          # because the other pipelines will be monitored by the author of the MR that triggered them.
-          # So we assume that we're reporting a master pipeline if the project name is 'gitlab-qa'.
-
-          Runtime::Env.pipeline_from_project_name
-        end
       end
     end
   end

data/lib/gitlab/qa/report/test_result.rb

@@ -19,6 +19,10 @@ module Gitlab
           self.failures = failures_from_exceptions
         end
 
+        def stage
+          @stage ||= file[%r{(?:api|browser_ui)/(?:(?:\d+_)?(\w+))}, 1]
+        end
+
         def name
           raise NotImplementedError
         end
@@ -46,8 +50,16 @@ module Gitlab
             report['file_path']
           end
 
+          def status
+            report['status']
+          end
+
+          def ci_job_url
+            report['ci_job_url']
+          end
+
           def skipped
-            report['status'] == 'pending'
+            status == 'pending'
           end
 
           def testcase
@@ -58,6 +70,14 @@ module Gitlab
             report['testcase'] = new_testcase
           end
 
+          def failure_issue
+            report['failure_issue']
+          end
+
+          def failure_issue=(new_failure_issue)
+            report['failure_issue'] = new_failure_issue
+          end
+
           private
 
           # rubocop:disable Metrics/AbcSize
@@ -71,6 +91,7 @@ module Gitlab
 
               {
                 'message' => "#{exception['class']}: #{exception['message']}",
+                'message_lines' => exception['message_lines'],
                 'stacktrace' => "#{exception['message_lines'].join("\n")}\n#{exception['backtrace'].slice(0..spec_file_first_index).join("\n")}"
               }
             end

data/lib/gitlab/qa/reporter.rb

@@ -4,6 +4,7 @@ module Gitlab
   module QA
     class Reporter
       # rubocop:disable Metrics/AbcSize
+      # rubocop:disable Metrics/PerceivedComplexity
       def self.invoke(args)
         report_options = {}
         slack_options = {}
@@ -21,11 +22,25 @@ module Gitlab
             report_options[:input_files] = files if files
           end
 
-          opts.on('-p', '--project PROJECT_ID', String, 'A valid project ID. Can be an integer or a group/project string. Required by --report-in-issues') do |value|
+          opts.on('--relate-failure-issue FILES', String, 'Relate test failures to failure issues from RSpec JSON files') do |files|
+            report_options[:relate_failure_issue] = true
+            report_options[:input_files] = files if files
+          end
+
+          opts.on('--max-diff-ratio DIFF_RATO', Float, 'Max stacktrace diff ratio for QA failure issues detection. Used by with --relate-failure-issue') do |value|
+            report_options[:max_diff_ratio] = value
+          end
+
+          opts.on('-p', '--project PROJECT_ID', String, 'A valid project ID. Can be an integer or a group/project string. Required by --report-in-issues and --relate-failure-issue') do |value|
             report_options[:project] = value
           end
 
-          opts.on('-t', '--token ACCESS_TOKEN', String, 'A valid access token. Used by --report-in-issues') do |value|
+          opts.on('--generate-test-session FILES', String, 'Generate test session report') do |files|
+            report_options[:generate_test_session] = true
+            report_options[:input_files] = files if files
+          end
+
+          opts.on('-t', '--token ACCESS_TOKEN', String, 'A valid access token. Required by --report-in-issues and --relate-failure-issue') do |value|
             report_options[:token] = value
           end
 
@@ -45,6 +60,10 @@ module Gitlab
             report_options[:files] = files
           end
 
+          opts.on('--dry-run', "Perform a dry-run (don't create or update issues)") do |files|
+            report_options[:dry_run] = true
+          end
+
           opts.on_tail('-v', '--version', 'Show the version') do
             require 'gitlab/qa/version'
             puts "#{$PROGRAM_NAME} : #{VERSION}"
@@ -63,10 +82,18 @@ module Gitlab
           if report_options.delete(:prepare_stage_reports)
             Gitlab::QA::Report::PrepareStageReports.new(**report_options).invoke!
 
+          elsif report_options.delete(:relate_failure_issue)
+            report_options[:token] = Runtime::TokenFinder.find_token!(report_options[:token])
+            Gitlab::QA::Report::RelateFailureIssue.new(**report_options).invoke!
+
           elsif report_options.delete(:report_in_issues)
             report_options[:token] = Runtime::TokenFinder.find_token!(report_options[:token])
             Gitlab::QA::Report::ResultsInIssues.new(**report_options).invoke!
 
+          elsif report_options.delete(:generate_test_session)
+            report_options[:token] = Runtime::TokenFinder.find_token!(report_options[:token])
+            Gitlab::QA::Report::GenerateTestSession.new(**report_options).invoke!
+
           elsif slack_options.delete(:post_to_slack)
             Gitlab::QA::Slack::PostToSlack.new(**slack_options).invoke!
 
@@ -79,6 +106,7 @@ module Gitlab
           exit 1
         end
       end
+      # rubocop:enable Metrics/PerceivedComplexity
       # rubocop:enable Metrics/AbcSize
     end
   end

data/lib/gitlab/qa/runtime/env.rb

@@ -62,7 +62,6 @@ module Gitlab
           'KNAPSACK_TEST_FILE_PATTERN' => :knapsack_test_file_pattern,
           'KNAPSACK_TEST_DIR' => :knapsack_test_dir,
           'CI' => :ci,
-          'CI_JOB_ID' => :ci_job_id,
           'CI_JOB_URL' => :ci_job_url,
           'CI_RUNNER_ID' => :ci_runner_id,
           'CI_SERVER_HOST' => :ci_server_host,
@@ -91,7 +90,10 @@ module Gitlab
           attr_writer(method_name)
 
           define_method(method_name) do
-            ENV[env_name] || instance_variable_get("@#{method_name}")
+            ENV[env_name] ||
+              if instance_variable_defined?("@#{method_name}")
+                instance_variable_get("@#{method_name}")
+              end
           end
         end
 
@@ -123,12 +125,28 @@ module Gitlab
           ENV['CI_PIPELINE_SOURCE']
         end
 
+        def ci_pipeline_url
+          ENV['CI_PIPELINE_URL']
+        end
+
+        def ci_pipeline_id
+          ENV['CI_PIPELINE_ID']
+        end
+
         def ci_project_name
           ENV['CI_PROJECT_NAME']
         end
 
         def pipeline_from_project_name
-          ci_project_name.to_s.start_with?('gitlab-qa') ? 'master' : ci_project_name
+          if ci_project_name.to_s.start_with?('gitlab-qa')
+            if ENV['TOP_UPSTREAM_SOURCE_JOB'].to_s.start_with?('https://ops.gitlab.net')
+              'staging-orchestrated'
+            else
+              'master'
+            end
+          else
+            ci_project_name
+          end
         end
 
         def run_id
@@ -147,6 +165,14 @@ module Gitlab
           ENV['GITLAB_QA_CONTAINER_REGISTRY_ACCESS_TOKEN']
         end
 
+        def qa_issue_url
+          ENV['GITLAB_QA_ISSUE_URL']
+        end
+
+        def deploy_environment
+          ENV['DEPLOY_ENVIRONMENT'] || pipeline_from_project_name
+        end
+
         def host_artifacts_dir
           @host_artifacts_dir ||= File.join(ENV['QA_ARTIFACTS_DIR'] || '/tmp/gitlab-qa', Runtime::Env.run_id)
         end

data/lib/gitlab/qa/version.rb

@@ -1,5 +1,5 @@
 module Gitlab
   module QA
-    VERSION = '6.7.0'.freeze
+    VERSION = '6.8.0'.freeze
   end
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: gitlab-qa
 version: !ruby/object:Gem::Version
-  version: 6.7.0
+  version: 6.8.0
 platform: ruby
 authors:
 - Grzegorz Bizon
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2020-11-02 00:00:00.000000000 Z
+date: 2020-11-10 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: climate_control
@@ -259,10 +259,13 @@ files:
 - lib/gitlab/qa/docker/volumes.rb
 - lib/gitlab/qa/release.rb
 - lib/gitlab/qa/report/base_test_results.rb
+- lib/gitlab/qa/report/generate_test_session.rb
 - lib/gitlab/qa/report/gitlab_issue_client.rb
+- lib/gitlab/qa/report/gitlab_issue_dry_client.rb
 - lib/gitlab/qa/report/json_test_results.rb
 - lib/gitlab/qa/report/junit_test_results.rb
 - lib/gitlab/qa/report/prepare_stage_reports.rb
+- lib/gitlab/qa/report/relate_failure_issue.rb
 - lib/gitlab/qa/report/report_as_issue.rb
 - lib/gitlab/qa/report/results_in_issues.rb
 - lib/gitlab/qa/report/summary_table.rb