gitlab-qa 6.4.0 → 6.8.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 12e0b622362bae1054406a72f79f42a29582f9ebca8162a2c3f2d8128c9938f6
-  data.tar.gz: fbaa6130f08f1a082cb90e4e1a5c65d7fb253b9f311cbaeb5937b3aa752ce20b
+  metadata.gz: daf050b7d871f8eefdc1fa68b052608c11ee186b927fe89accfceca15291ed45
+  data.tar.gz: 4510e98db8e41c11a1585070a39cd19769384917cff551195942955974dcd78d
 SHA512:
-  metadata.gz: 9bdd569f223141416086e574911e71fc2350b55b487644ccdc68524cee0da8ee0c5ddaa9c26be1877cc2b218045b736a3212b71a26188f3770ac8355bfd8cf69
-  data.tar.gz: e4a83ae9c48728baef2ea881f1abcd50476ce96e59ef1579b788eb6c58812c9aa27a40397caaf7293c71dd5bf1e6614e5607d2d41d438447c3badeafc9cd4595
+  metadata.gz: 3a3a6dc22aac1a77fe46f1379a43b489d08cc771b38ea629f9ff3a8f24fd9886f767ed3b3ca8aa18bfeabadc9019aefe2933261c969167e6c8e95f7f75e2c6a1
+  data.tar.gz: c612e3b42691a28864bcf703fdf89cbbf57f75817341f867d1e8486a25adcbfcf262f3506157841d734813e699e628620dd524af22b4f8da17c689a96c6116cf

data/.gitlab-ci.yml

@@ -2,10 +2,11 @@ stages:
   - check
   - release
   - test
+  - report
   - notify
 
 default:
-  image: registry.gitlab.com/gitlab-org/gitlab-build-images:gitlab-qa-ruby-2.6
+  image: registry.gitlab.com/gitlab-org/gitlab-build-images:gitlab-qa-ruby-2.7
   tags:
     - gitlab-org
   cache:
@@ -48,6 +49,10 @@ variables:
   QA_CAN_TEST_GIT_PROTOCOL_V2: "true"
   QA_CAN_TEST_PRAEFECT: "false"
   QA_TESTCASES_REPORTING_PROJECT: "gitlab-org/quality/testcases"
+  QA_FAILURES_REPORTING_PROJECT: "gitlab-org/gitlab"
+  # The --dry-run or --max-diff-ratio option can be set to modify the behavior of `exe/gitlab-qa-report --relate-failure-issue` without releasing a new gem version.
+  QA_FAILURES_REPORTER_OPTIONS: "--dry-run"
+  QA_TESTCASE_SESSIONS_PROJECT: "gitlab-org/quality/testcase-sessions"
 
 .check-base:
   stage: check
@@ -95,6 +100,7 @@ release:
     - exe/gitlab-qa-report --update-screenshot-path "gitlab-qa-run-*/**/rspec-*.xml"
     - export GITLAB_QA_ACCESS_TOKEN="$GITLAB_QA_PRODUCTION_ACCESS_TOKEN"
     - if [ "$TOP_UPSTREAM_SOURCE_REF" == "master" ] || [[ "$TOP_UPSTREAM_SOURCE_JOB" == https://ops.gitlab.net* ]]; then exe/gitlab-qa-report --report-in-issues "gitlab-qa-run-*/**/rspec-*.json" --project "$QA_TESTCASES_REPORTING_PROJECT" || true; fi
+    - if [ "$TOP_UPSTREAM_SOURCE_REF" == "master" ] || [[ "$TOP_UPSTREAM_SOURCE_JOB" == https://ops.gitlab.net* ]]; then exe/gitlab-qa-report --relate-failure-issue "gitlab-qa-run-*/**/rspec-*.json" --project "$QA_FAILURES_REPORTING_PROJECT" $QA_FAILURES_REPORTER_OPTIONS || true; fi
     - exit $test_run_exit_code
 
 .ce-qa:
@@ -333,6 +339,7 @@ ce:update:
     - exe/gitlab-qa Test::Omnibus::Update ${RELEASE:=CE} ${RELEASE:=CE} -- $RSPEC_REPORT_OPTS || test_run_exit_code=$?
     - export GITLAB_QA_ACCESS_TOKEN="$GITLAB_QA_PRODUCTION_ACCESS_TOKEN"
     - if [ "$TOP_UPSTREAM_SOURCE_REF" == "master" ] || [[ "$TOP_UPSTREAM_SOURCE_JOB" == https://ops.gitlab.net* ]]; then exe/gitlab-qa-report --report-in-issues "gitlab-qa-run-*/**/rspec-*.json" --project "$QA_TESTCASES_REPORTING_PROJECT" || true; fi
+    - if [ "$TOP_UPSTREAM_SOURCE_REF" == "master" ] || [[ "$TOP_UPSTREAM_SOURCE_JOB" == https://ops.gitlab.net* ]]; then exe/gitlab-qa-report --relate-failure-issue "gitlab-qa-run-*/**/rspec-*.json" --project "$QA_FAILURES_REPORTING_PROJECT" $QA_FAILURES_REPORTER_OPTIONS || true; fi
     - exit $test_run_exit_code
   extends:
     - .test
@@ -347,6 +354,7 @@ ce:update-quarantine:
     - exe/gitlab-qa Test::Omnibus::Update ${RELEASE:=CE} ${RELEASE:=CE} -- --tag quarantine --tag ~orchestrated $RSPEC_REPORT_OPTS || test_run_exit_code=$?
     - export GITLAB_QA_ACCESS_TOKEN="$GITLAB_QA_PRODUCTION_ACCESS_TOKEN"
     - if [ "$TOP_UPSTREAM_SOURCE_REF" == "master" ] || [[ "$TOP_UPSTREAM_SOURCE_JOB" == https://ops.gitlab.net* ]]; then exe/gitlab-qa-report --report-in-issues "gitlab-qa-run-*/**/rspec-*.json" --project "$QA_TESTCASES_REPORTING_PROJECT" || true; fi
+    - if [ "$TOP_UPSTREAM_SOURCE_REF" == "master" ] || [[ "$TOP_UPSTREAM_SOURCE_JOB" == https://ops.gitlab.net* ]]; then exe/gitlab-qa-report --relate-failure-issue "gitlab-qa-run-*/**/rspec-*.json" --project "$QA_FAILURES_REPORTING_PROJECT" $QA_FAILURES_REPORTER_OPTIONS || true; fi
     - exit $test_run_exit_code
   extends:
     - .test
@@ -360,6 +368,7 @@ ee:update:
     - exe/gitlab-qa Test::Omnibus::Update ${RELEASE:=EE} ${RELEASE:=EE} -- $RSPEC_REPORT_OPTS || test_run_exit_code=$?
     - export GITLAB_QA_ACCESS_TOKEN="$GITLAB_QA_PRODUCTION_ACCESS_TOKEN"
     - if [ "$TOP_UPSTREAM_SOURCE_REF" == "master" ] || [[ "$TOP_UPSTREAM_SOURCE_JOB" == https://ops.gitlab.net* ]]; then exe/gitlab-qa-report --report-in-issues "gitlab-qa-run-*/**/rspec-*.json" --project "$QA_TESTCASES_REPORTING_PROJECT" || true; fi
+    - if [ "$TOP_UPSTREAM_SOURCE_REF" == "master" ] || [[ "$TOP_UPSTREAM_SOURCE_JOB" == https://ops.gitlab.net* ]]; then exe/gitlab-qa-report --relate-failure-issue "gitlab-qa-run-*/**/rspec-*.json" --project "$QA_FAILURES_REPORTING_PROJECT" $QA_FAILURES_REPORTER_OPTIONS || true; fi
     - exit $test_run_exit_code
   extends:
     - .test
@@ -374,6 +383,7 @@ ee:update-quarantine:
     - exe/gitlab-qa Test::Omnibus::Update ${RELEASE:=EE} ${RELEASE:=EE} -- --tag quarantine --tag ~orchestrated $RSPEC_REPORT_OPTS || test_run_exit_code=$?
     - export GITLAB_QA_ACCESS_TOKEN="$GITLAB_QA_PRODUCTION_ACCESS_TOKEN"
     - if [ "$TOP_UPSTREAM_SOURCE_REF" == "master" ] || [[ "$TOP_UPSTREAM_SOURCE_JOB" == https://ops.gitlab.net* ]]; then exe/gitlab-qa-report --report-in-issues "gitlab-qa-run-*/**/rspec-*.json" --project "$QA_TESTCASES_REPORTING_PROJECT" || true; fi
+    - if [ "$TOP_UPSTREAM_SOURCE_REF" == "master" ] || [[ "$TOP_UPSTREAM_SOURCE_JOB" == https://ops.gitlab.net* ]]; then exe/gitlab-qa-report --relate-failure-issue "gitlab-qa-run-*/**/rspec-*.json" --project "$QA_FAILURES_REPORTING_PROJECT" $QA_FAILURES_REPORTER_OPTIONS || true; fi
     - exit $test_run_exit_code
   extends:
     - .test
@@ -729,6 +739,44 @@ ee:packages-quarantine:
   variables:
     QA_SCENARIO: "Test::Integration::Packages"
 
+ce:actioncable:
+  extends:
+    - .test
+    - .high-capacity
+    - .ce-qa
+    - .rspec-report-opts
+  variables:
+    QA_SCENARIO: "Test::Integration::Actioncable"
+
+ce:actioncable-quarantine:
+  extends:
+    - .test
+    - .high-capacity
+    - .ce-qa
+    - .quarantine
+    - .rspec-report-opts
+  variables:
+    QA_SCENARIO: "Test::Integration::Actioncable"
+
+ee:actioncable:
+  extends:
+    - .test
+    - .high-capacity
+    - .ee-qa
+    - .rspec-report-opts
+  variables:
+    QA_SCENARIO: "Test::Integration::Actioncable"
+
+ee:actioncable-quarantine:
+  extends:
+    - .test
+    - .high-capacity
+    - .ee-qa
+    - .quarantine
+    - .rspec-report-opts
+  variables:
+    QA_SCENARIO: "Test::Integration::Actioncable"
+
 ee:elasticsearch:
   extends:
     - .test
@@ -903,6 +951,22 @@ staging:
     - .only-qa
   allow_failure: true
 
+generate_test_session:
+  stage: report
+  rules:
+    - if: '$TOP_UPSTREAM_SOURCE_JOB && $TOP_UPSTREAM_SOURCE_REF == "master"'
+      when: always
+    - if: '$TOP_UPSTREAM_SOURCE_JOB =~ /\Ahttps:\/\/ops.gitlab.net\//'
+      when: always
+  artifacts:
+    when: always
+    expire_in: 10d
+    paths:
+      - REPORT_ISSUE_URL
+  script:
+    - export GITLAB_QA_ACCESS_TOKEN="$GITLAB_QA_PRODUCTION_ACCESS_TOKEN"
+    - exe/gitlab-qa-report --generate-test-session "gitlab-qa-run-*/**/rspec-*.json" --project "$QA_TESTCASE_SESSIONS_PROJECT"
+
 .notify_upstream_commit:
   stage: notify
   image: ruby:2.6
@@ -928,7 +992,7 @@ notify_upstream_commit:failure:
 .notify_slack:
   image: alpine
   stage: notify
-  dependencies: []
+  dependencies: ['generate_test_session']
   cache: {}
   before_script:
     - apk update && apk add git curl bash
@@ -947,4 +1011,4 @@ notify_slack:
     - echo "RELEASE is ${RELEASE}"
     - echo "CI_PIPELINE_URL is $CI_PIPELINE_URL"
     - echo "TOP_UPSTREAM_SOURCE_JOB is $TOP_UPSTREAM_SOURCE_JOB"
-    - bin/slack $NOTIFY_CHANNEL "☠️ QA against $RELEASE failed! ☠️ See $CI_PIPELINE_URL (triggered from $TOP_UPSTREAM_SOURCE_JOB)" ci_failing
+    - 'bin/slack $NOTIFY_CHANNEL "☠️ QA against $RELEASE failed! ☠️ See the test session report: $(cat REPORT_ISSUE_URL), and pipeline: $CI_PIPELINE_URL (triggered from $TOP_UPSTREAM_SOURCE_JOB)" ci_failing'

data/.gitlab/merge_request_templates/Release.md

@@ -32,4 +32,4 @@ with the latest commit from https://gitlab.com/gitlab-org/gitlab-qa/commits/mast
 - Checklist after merging:
   - [ ] [Create a tag for the new release version](docs/release_process.md#how-to).
 
-/label ~Quality ~backstage
+/label ~Quality ~"feature::maintenance"

data/.rubocop.yml

@@ -1,3 +1,5 @@
+inherit_from: .rubocop_todo.yml
+
 require:
   # Due to a probably bug in rubocop (https://github.com/bbatsov/rubocop/issues/5251)
   # we need to require rubocop-rspec a second time so that RSpec/FilePath does

data/.rubocop_todo.yml

@@ -0,0 +1,98 @@
+# This configuration was generated by
+# `rubocop --auto-gen-config`
+# on 2020-10-31 07:26:03 -0700 using RuboCop version 0.82.0.
+# The point is for the user to remove these configuration records
+# one by one as the offenses are removed from the code base.
+# Note that changes in the inspected code, or installation of new
+# versions of RuboCop, may require this file to be generated again.
+
+# Offense count: 51
+# Cop supports --auto-correct.
+# Configuration parameters: AutoCorrect, AllowHeredoc, AllowURI, URISchemes, IgnoreCopDirectives, IgnoredPatterns.
+# URISchemes: http, https
+Layout/LineLength:
+  Max: 210
+
+# Offense count: 2
+# Cop supports --auto-correct.
+Lint/RedundantCopDisableDirective:
+  Exclude:
+    - 'lib/gitlab/qa/component/staging.rb'
+    - 'lib/gitlab/qa/runtime/scenario.rb'
+
+# Offense count: 1
+# Cop supports --auto-correct.
+# Configuration parameters: PreferredName.
+Naming/RescuedExceptionsVariableName:
+  Exclude:
+    - 'lib/gitlab/qa/component/staging.rb'
+
+# Offense count: 1
+# Cop supports --auto-correct.
+Performance/RegexpMatch:
+  Exclude:
+    - 'lib/gitlab/qa/component/gitlab.rb'
+
+# Offense count: 3
+# Cop supports --auto-correct.
+RSpec/EmptyLineAfterLetBlock:
+  Exclude:
+    - 'spec/gitlab/qa/support/dev_eeqa_image_spec.rb'
+
+# Offense count: 4
+# Cop supports --auto-correct.
+# Configuration parameters: CustomTransform, IgnoredWords.
+RSpec/ExampleWording:
+  Exclude:
+    - 'spec/gitlab/qa/component/gitlab_spec.rb'
+
+# Offense count: 2
+RSpec/LeakyConstantDeclaration:
+  Exclude:
+    - 'spec/gitlab/qa/scenario/test/instance/deployment_base_spec.rb'
+
+# Offense count: 93
+# Cop supports --auto-correct.
+# Configuration parameters: EnforcedStyle.
+# SupportedStyles: always, always_true, never
+Style/FrozenStringLiteralComment:
+  Enabled: false
+
+# Offense count: 1
+# Cop supports --auto-correct.
+Style/HashTransformKeys:
+  Exclude:
+    - 'lib/gitlab/qa/docker/volumes.rb'
+
+# Offense count: 6
+# Cop supports --auto-correct.
+Style/IfUnlessModifier:
+  Exclude:
+    - 'lib/gitlab/qa/component/gitlab.rb'
+    - 'lib/gitlab/qa/release.rb'
+    - 'lib/gitlab/qa/runtime/env.rb'
+    - 'lib/gitlab/qa/runtime/scenario.rb'
+    - 'lib/gitlab/qa/scenario/test/omnibus/upgrade.rb'
+    - 'lib/gitlab/qa/support/http_request.rb'
+
+# Offense count: 1
+Style/MethodMissingSuper:
+  Exclude:
+    - 'lib/gitlab/qa/runtime/scenario.rb'
+
+# Offense count: 1
+Style/MissingRespondToMissing:
+  Exclude:
+    - 'lib/gitlab/qa/runtime/scenario.rb'
+
+# Offense count: 5
+# Cop supports --auto-correct.
+# Configuration parameters: EnforcedStyle.
+# SupportedStyles: literals, strict
+Style/MutableConstant:
+  Exclude:
+    - 'db/migrate/**/*'
+    - 'db/post_migrate/**/*'
+    - 'db/geo/migrate/**/*'
+    - 'lib/gitlab/qa/release.rb'
+    - 'lib/gitlab/qa/report/update_screenshot_path.rb'

data/docs/what_tests_can_be_run.md

@@ -545,6 +545,23 @@ $ export EE_LICENSE=$(cat /path/to/GitLab.gitlab_license)
 $ gitlab-qa Test::Integration::Jira EE
 ```
 
+### `Test::Integration::Actioncable CE|EE|<full image address>`
+
+This tests the real-time assignees feature by setting
+`actioncable['enable'] = true` in the Omnibus configuration
+before starting the GitLab container.
+
+To run tests against the GitLab container, a GitLab QA (`gitlab/gitlab-qa`)
+container is spun up and tests are run from it by running the
+`Test::Instance::All` scenario with the `--tag actioncable` RSpec parameter,
+which runs only the tests with `:actioncable` metadata.
+
+Example:
+
+```
+$ gitlab-qa Test::Integration::Actioncable CE
+```
+
 ### `Test::Instance::Any CE|EE|<full image address>:nightly|latest|any_tag http://your.instance.gitlab`
 
 This tests that a live GitLab instance works as expected by running tests

data/gitlab-qa.gemspec

@@ -20,12 +20,12 @@ Gem::Specification.new do |spec|
 
   # Some dependencies are pinned, to prevent new cops from breaking the CI pipelines
   spec.add_development_dependency 'climate_control', '~> 0.2'
-  spec.add_development_dependency 'gitlab-styles', '2.4.0'
+  spec.add_development_dependency 'gitlab-styles', '~> 4.3.0'
   spec.add_development_dependency 'pry', '~> 0.11'
   spec.add_development_dependency 'rake', '~> 12.2'
   spec.add_development_dependency 'rspec', '~> 3.7'
-  spec.add_development_dependency 'rubocop', '~> 0.54.0'
-  spec.add_development_dependency 'rubocop-rspec', '1.20.1'
+  spec.add_development_dependency 'rubocop', '~> 0.82.0'
+  spec.add_development_dependency 'rubocop-rspec', '~> 1.36'
   spec.add_development_dependency 'webmock', '3.7.0'
   spec.add_runtime_dependency 'activesupport', '~>  6.0.2'
   spec.add_runtime_dependency 'gitlab', '~>  4.11.0'

data/lib/gitlab/qa.rb

@@ -39,6 +39,7 @@ module Gitlab
         end
 
         module Integration
+          autoload :Actioncable, 'gitlab/qa/scenario/test/integration/actioncable'
           autoload :Geo, 'gitlab/qa/scenario/test/integration/geo'
           autoload :LDAP, 'gitlab/qa/scenario/test/integration/ldap'
           autoload :LDAPNoTLS, 'gitlab/qa/scenario/test/integration/ldap_no_tls'
@@ -98,12 +99,15 @@ module Gitlab
 
     module Report
       autoload :GitlabIssueClient, 'gitlab/qa/report/gitlab_issue_client'
+      autoload :GitlabIssueDryClient, 'gitlab/qa/report/gitlab_issue_dry_client'
       autoload :BaseTestResults, 'gitlab/qa/report/base_test_results'
+      autoload :RelateFailureIssue, 'gitlab/qa/report/relate_failure_issue'
       autoload :JsonTestResults, 'gitlab/qa/report/json_test_results'
       autoload :JUnitTestResults, 'gitlab/qa/report/junit_test_results'
       autoload :PrepareStageReports, 'gitlab/qa/report/prepare_stage_reports'
       autoload :ReportAsIssue, 'gitlab/qa/report/report_as_issue'
       autoload :ResultsInIssues, 'gitlab/qa/report/results_in_issues'
+      autoload :GenerateTestSession, 'gitlab/qa/report/generate_test_session'
       autoload :SummaryTable, 'gitlab/qa/report/summary_table'
       autoload :TestResult, 'gitlab/qa/report/test_result'
       autoload :UpdateScreenshotPath, 'gitlab/qa/report/update_screenshot_path'

data/lib/gitlab/qa/component/base.rb

@@ -61,7 +61,7 @@ module Gitlab
         end
 
         def prepare_network
-          if runner_network && !docker.network_exists?(runner_network)
+          if runner_network && !docker.network_exists?(runner_network) # rubocop:disable Style/IfUnlessModifier
             docker.network_create("--driver=bridge --internal #{runner_network}")
           end
 

data/lib/gitlab/qa/component/postgresql.rb

@@ -30,7 +30,8 @@ module Gitlab
           begin
             run_psql 'template1'
           rescue StandardError
-            retry if Time.now - start < 30
+            sleep 5
+            retry if Time.now - start < 60
             raise
           end
         end

data/lib/gitlab/qa/docker/shellout.rb

@@ -27,7 +27,7 @@ module Gitlab
               end
             end
 
-            if wait.value.exited? && wait.value.exitstatus.nonzero?
+            if wait.value.exited? && wait.value.exitstatus.nonzero? # rubocop:disable Style/IfUnlessModifier
               raise StatusError, "Docker command `#{@command.mask_secrets}` failed!"
             end
           end

data/lib/gitlab/qa/report/base_test_results.rb

@@ -6,8 +6,11 @@ module Gitlab
       class BaseTestResults
         include Enumerable
 
+        attr_reader :path
+
         def initialize(path)
-          @results = parse(path)
+          @path = path
+          @results = parse
           @testcases = process
         end
 
@@ -15,7 +18,7 @@ module Gitlab
           testcases.each(&block)
         end
 
-        def write(path)
+        def write
           raise NotImplementedError
         end
 
@@ -23,7 +26,7 @@ module Gitlab
 
         attr_reader :results, :testcases
 
-        def parse(path)
+        def parse
           raise NotImplementedError
         end
 

data/lib/gitlab/qa/report/generate_test_session.rb

@@ -0,0 +1,203 @@
+# frozen_string_literal: true
+
+module Gitlab
+  module QA
+    module Report
+      class GenerateTestSession < ReportAsIssue
+        private
+
+        def run!
+          puts "Generating test results in `#{files.join(',')}` as issues in project `#{project}` via the API at `#{Runtime::Env.gitlab_api_base}`."
+
+          tests = Dir.glob(files).flat_map do |path|
+            puts "Loading tests in #{path}"
+
+            Report::JsonTestResults.new(path).to_a
+          end
+
+          issue = gitlab.create_issue(
+            title: "Test session report | #{Runtime::Env.deploy_environment}",
+            description: generate_description(tests),
+            labels: ['Quality', 'QA', 'triage report']
+          )
+
+          File.write('REPORT_ISSUE_URL', issue.web_url)
+        end
+
+        def generate_description(tests)
+          <<~MARKDOWN
+          ## Session summary
+
+          * Deploy version: #{Runtime::Env.deploy_version}
+          * Pipeline: [#{Runtime::Env.ci_pipeline_id}](#{Runtime::Env.ci_pipeline_url})
+          #{generate_summary(tests: tests)}
+
+          #{generate_stages_listing(tests)}
+
+          ## Release QA issue
+
+          * #{Runtime::Env.qa_issue_url}
+          MARKDOWN
+        end
+
+        def generate_summary(tests:, tests_by_status: nil)
+          tests_by_status ||= tests.group_by(&:status)
+          total = tests.size
+          passed = tests_by_status['passed']&.size || 0
+          failed = tests_by_status['failed']&.size || 0
+          others = total - passed - failed
+
+          <<~MARKDOWN.chomp
+          * Total #{total} tests
+          * Passed #{passed} tests
+          * Failed #{failed} tests
+          * #{others} other tests (usually skipped)
+          MARKDOWN
+        end
+
+        def generate_stages_listing(tests)
+          generate_tests_by_stage(tests).map do |stage, tests_for_stage|
+            tests_by_status = tests_for_stage.group_by(&:status)
+
+            <<~MARKDOWN.chomp
+            ### #{stage&.capitalize || 'Unknown'}
+
+            #{generate_summary(
+              tests: tests_for_stage, tests_by_status: tests_by_status)}
+
+            #{generate_testcase_listing_by_status(
+              tests: tests_for_stage, tests_by_status: tests_by_status)}
+            MARKDOWN
+          end.join("\n\n")
+        end
+
+        def generate_tests_by_stage(tests)
+          # https://about.gitlab.com/handbook/product/product-categories/#devops-stages
+          ordering = %w[
+            manage
+            plan
+            create
+            verify
+            package
+            release
+            configure
+            monitor
+            secure
+            defend
+            growth
+            fulfillment
+            enablement
+          ]
+
+          tests.sort_by do |test|
+            ordering.index(test.stage) || ordering.size
+          end.group_by(&:stage)
+        end
+
+        def generate_testcase_listing_by_status(tests:, tests_by_status:)
+          failed_tests = tests_by_status['failed']
+          passed_tests = tests_by_status['passed']
+          other_tests = tests.reject do |test|
+            test.status == 'failed' || test.status == 'passed'
+          end
+
+          [
+            (failed_listings(failed_tests) if failed_tests),
+            (passed_listings(passed_tests) if passed_tests),
+            (other_listings(other_tests) if other_tests.any?)
+          ].compact.join("\n\n")
+        end
+
+        def failed_listings(failed_tests)
+          generate_testcase_listing(failed_tests)
+        end
+
+        def passed_listings(passed_tests)
+          <<~MARKDOWN.chomp
+            <details><summary>Passed tests:</summary>
+
+            #{generate_testcase_listing(passed_tests)}
+
+            </details>
+          MARKDOWN
+        end
+
+        def other_listings(other_tests)
+          <<~MARKDOWN.chomp
+            <details><summary>Other tests:</summary>
+
+            #{generate_testcase_listing(other_tests)}
+
+            </details>
+          MARKDOWN
+        end
+
+        def generate_testcase_listing(tests)
+          body = tests.group_by(&:testcase).map do |testcase, tests_with_same_testcase|
+            tests_with_same_testcase.sort_by!(&:name)
+            [
+              generate_test_text(testcase, tests_with_same_testcase),
+              generate_test_job(tests_with_same_testcase),
+              generate_test_status(tests_with_same_testcase),
+              generate_test_actions(tests_with_same_testcase)
+            ].join(' | ')
+          end.join("\n")
+
+          <<~MARKDOWN.chomp
+          | Test | Job | Status | Action |
+          | - | - | - | - |
+          #{body}
+          MARKDOWN
+        end
+
+        def generate_test_text(testcase, tests_with_same_testcase)
+          text = tests_with_same_testcase.map(&:name).uniq.join(', ')
+
+          if testcase
+            "[#{text}](#{testcase})"
+          else
+            text
+          end
+        end
+
+        def generate_test_job(tests_with_same_testcase)
+          tests_with_same_testcase.map(&:ci_job_url).uniq.map do |ci_job_url|
+            ci_job_id = ci_job_url[/\d+\z/]
+
+            "[#{ci_job_id}](#{ci_job_url})"
+          end.join(', ')
+        end
+
+        def generate_test_status(tests_with_same_testcase)
+          tests_with_same_testcase.map(&:status).uniq.map do |status|
+            %(~"#{status}")
+          end.join(', ')
+        end
+
+        def generate_test_actions(tests_with_same_testcase)
+          # All failed tests would be grouped together, meaning that
+          # if one failed, all the tests here would be failed too.
+          # So this check is safe. Same applies to 'passed'.
+          # But all other status might be mixing together,
+          # we cannot assume other statuses.
+          if tests_with_same_testcase.first.status == 'failed'
+            tests_having_failure_issue =
+              tests_with_same_testcase.select(&:failure_issue)
+
+            if tests_having_failure_issue.any?
+              items = tests_having_failure_issue.map do |test|
+                "<li>[ ] [failure issue](#{test.failure_issue})</li>"
+              end.join(' ')
+
+              "<ul>#{items}</ul>"
+            else
+              '<ul><li>[ ] failure issue exists or was created</li></ul>'
+            end
+          else
+            '-'
+          end
+        end
+      end
+    end
+  end
+end