gitlab_quality-test_tooling 2.10.0 → 2.15.3

data/lib/gitlab_quality/test_tooling/gitlab_client/issues_client.rb

@@ -72,6 +72,12 @@ module GitlabQuality
           end
         end
 
+        def related_merge_requests(iid:)
+          handle_gitlab_client_exceptions do
+            client.related_merge_requests(project, iid).auto_paginate
+          end
+        end
+
         def find_issue_discussions(iid:)
           handle_gitlab_client_exceptions do
             client.issue_discussions(project, iid, order_by: 'created_at', sort: 'asc').auto_paginate
@@ -169,7 +175,7 @@ module GitlabQuality
         def find_commit_parent(project, sha)
           handle_gitlab_client_exceptions do
             # In a merged results commit, the first parent is the one from
-            # the main branch, and the second parent is from the branch
+            # the default branch, and the second parent is from the branch
             # itself (more likely to have caused the issue)
             client.commit(project, sha).parent_ids.last
           end

data/lib/gitlab_quality/test_tooling/gitlab_client/issues_dry_client.rb

@@ -4,7 +4,7 @@ module GitlabQuality
   module TestTooling
     module GitlabClient
       class IssuesDryClient < IssuesClient
-        def create_issue(title:, description:, labels:, issue_type: 'issue', confidential: false)
+        def create_issue(title:, description:, labels:, issue_type: 'issue', _assignee_id: nil, _due_date: nil, confidential: false)
           attrs = { description: description, labels: labels, confidential: confidential }
 
           puts "The following #{issue_type} would have been created:"

data/lib/gitlab_quality/test_tooling/gitlab_client/merge_requests_client.rb

@@ -1,5 +1,16 @@
 # frozen_string_literal: true
 
+require 'gitlab'
+
+module Gitlab
+  # Monkey patch the Gitlab client to allow passing query options
+  class Client
+    def merge_request_diffs(project, merge_request_iid, options = {})
+      get("/projects/#{url_encode(project)}/merge_requests/#{merge_request_iid}/diffs", query: options).auto_paginate
+    end
+  end
+end
+
 module GitlabQuality
   module TestTooling
     module GitlabClient
@@ -10,6 +21,12 @@ module GitlabQuality
           end
         end
 
+        def merge_request_diffs(merge_request_iid:)
+          handle_gitlab_client_exceptions do
+            client.merge_request_diffs(project, merge_request_iid, per_page: 100)
+          end
+        end
+
         def create_merge_request(title:, source_branch:, target_branch:, description:, labels:, assignee_id: nil, reviewer_ids: [])
           attrs = {
             source_branch: source_branch,
@@ -33,6 +50,10 @@ module GitlabQuality
           merge_request
         end
 
+        def merge_request(id:, options: {})
+          client.merge_request(project, id, options)
+        end
+
         def find(iid: nil, options: {}, &select)
           select ||= :itself
 

data/lib/gitlab_quality/test_tooling/gitlab_client/merge_requests_dry_client.rb

@@ -4,16 +4,6 @@ module GitlabQuality
   module TestTooling
     module GitlabClient
       class MergeRequestsDryClient < MergeRequestsClient
-        def find_merge_request_changes(merge_request_iid:)
-          puts "Finding changes for merge_request_id #{merge_request_iid}"
-          puts "project: #{project}"
-        end
-
-        def merge_request_changed_files(merge_request_iid:)
-          puts "Changed files for #{merge_request_iid}"
-          []
-        end
-
         def find_note(body:, merge_request_iid:)
           puts "Find note for #{merge_request_iid} with body: #{body} for mr_iid: #{merge_request_iid}"
         end

data/lib/gitlab_quality/test_tooling/gitlab_client/work_items_client.rb

@@ -3,15 +3,14 @@
 module GitlabQuality
   module TestTooling
     module GitlabClient
-      # The GitLab client is used for API access: https://github.com/NARKOZ/gitlab
       class WorkItemsClient < GitlabGraphqlClient
-        def work_item(workitem_iid: nil)
+        def work_item(workitem_iid:, widgets: [:notes, :linked_items, :labels, :hierarchy])
           query = <<~GQL
             query {
               namespace(fullPath: "#{group}") {
                 workItem(iid: "#{workitem_iid}") {
                   #{work_item_fields}
-                  #{work_item_widgets}
+                  #{work_item_widgets(widgets)}
                 }
               }
             }
@@ -19,7 +18,7 @@ module GitlabQuality
           post(query)[:workItem]
         end
 
-        def group_work_items(labels: [], cursor: '', state: 'opened', created_after: nil, extras: [])
+        def group_work_items(labels: [], cursor: '', state: 'opened', created_after: nil, extras: [:work_item_fields])
           query = <<~GQL
             query {
               group(fullPath: "#{group}") {
@@ -73,6 +72,18 @@ module GitlabQuality
           post(query)
         end
 
+        def update_note(note_id:, body:)
+          query = <<~GQL
+            mutation UpdateNote {
+                updateNote(input: { body: "#{body}", id: "#{note_id}" }) {
+                    clientMutationId
+                    errors
+                }
+            }
+          GQL
+          post(query)
+        end
+
         def create_linked_items(work_item_id:, item_ids:, link_type:)
           query = <<~GQL
             mutation WorkItemAddLinkedItems {
@@ -160,50 +171,76 @@ module GitlabQuality
           GQL
         end
 
-        def work_item_widgets
+        def work_item_widget_notes
           <<~GQL
-            widgets(onlyTypes: [LINKED_ITEMS, NOTES, LABELS, HIERARCHY]) {
-              ... on WorkItemWidgetNotes {
-                discussions(filter: ONLY_COMMENTS) {
-                  nodes {
-                          notes {
-                            nodes {
-                            #{note_fields}
-                            }
+            ... on WorkItemWidgetNotes {
+              discussions(filter: ONLY_COMMENTS) {
+                nodes {
+                        notes {
+                          nodes {
+                          #{note_fields}
                           }
                         }
-                }
-              }
-              ... on WorkItemWidgetLinkedItems {
-                  linkedItems {
-                      nodes {
-                          linkType
-                          workItem {
-                              #{work_item_fields}
-                          }
                       }
-                  }
               }
+            }
+          GQL
+        end
 
-              ... on WorkItemWidgetLabels{
-                labels{
-                   nodes{
-                       title
-                   }
+        def work_item_widget_linked_items
+          <<~GQL
+            ... on WorkItemWidgetLinkedItems {
+                linkedItems {
+                    nodes {
+                        linkType
+                        workItem {
+                            #{work_item_fields}
+                        }
+                    }
                 }
+            }
+          GQL
+        end
+
+        def work_item_widget_labels
+          <<~GQL
+            ... on WorkItemWidgetLabels{
+              labels{
+                 nodes{
+                     title
+                 }
               }
+            }
+          GQL
+        end
 
-              ... on WorkItemWidgetHierarchy {
-                children {
-                    nodes{
-                     #{work_item_fields}
-                    }
-                }
+        def work_item_widget_hierarchy
+          <<~GQL
+            ... on WorkItemWidgetHierarchy {
+              children {
+                  nodes{
+                   #{work_item_fields}
+                  }
               }
             }
           GQL
         end
 
+        def work_item_widgets(widgets = [])
+          <<~GQL
+            widgets(onlyTypes: [#{types_for_widgets(widgets)}]) {
+              #{work_item_widget_notes if widgets.include?(:notes)}
+              #{work_item_widget_linked_items if widgets.include?(:linked_items)}
+              #{work_item_widget_labels if widgets.include?(:labels)}
+              #{work_item_widget_hierarchy if widgets.include?(:hierarchy)}
+            }
+          GQL
+        end
+
+        def types_for_widgets(widgets = [])
+          widgets.map(&:upcase).join(', ')
+        end
+
         # https://docs.gitlab.com/api/graphql/reference/#note
         def note_fields
           <<~GQL

data/lib/gitlab_quality/test_tooling/report/concerns/results_reporter.rb

@@ -36,7 +36,7 @@ module GitlabQuality
           end
 
           def new_issue_labels(_test)
-            %w[Quality status::automated]
+            %w[E2E status::automated]
           end
 
           def up_to_date_labels(test:, issue: nil, new_labels: Set.new)

data/lib/gitlab_quality/test_tooling/report/concerns/utils.rb

@@ -43,9 +43,9 @@ module GitlabQuality
             # Some of those run in their own project, so CI_PROJECT_NAME is the name we need. Those are:
             #   nightly, staging, canary, production, and preprod
             #
-            # MR, master/main, and gitlab tests run in gitlab-qa, but we only want to report tests run on
-            # master/main because the other pipelines will be monitored by the author of the MR that triggered them.
-            # So we assume that we're reporting a master/main pipeline if the project name is 'gitlab'.
+            # MR, master, and gitlab tests run in gitlab-qa, but we only want to report tests run on
+            # master because the other pipelines will be monitored by the author of the MR that triggered them.
+            # So we assume that we're reporting a master pipeline if the project name is 'gitlab'.
 
             @pipeline ||= Runtime::Env.pipeline_from_project_name
           end

data/lib/gitlab_quality/test_tooling/report/feature_readiness/report_on_epic.rb

@@ -0,0 +1,174 @@
+# frozen_string_literal: true
+
+require 'pp'
+require 'stringio'
+
+module GitlabQuality
+  module TestTooling
+    module Report
+      module FeatureReadiness
+        class ReportOnEpic
+          FEATURE_READINESS_REPORT_COMMENT_ID = '<!-- FEATURE READINESS REPORT COMMENT -->'
+
+          class << self
+            include GitlabQuality::TestTooling::FeatureReadiness::Concerns::WorkItemConcern
+
+            def report(analyzed_epic, work_item_client)
+              must_haves_report_rows = generate_report_rows(analyzed_epic, :must_haves)
+              should_haves_report_rows = generate_report_rows(analyzed_epic, :should_haves)
+
+              existing_note = existing_note_containing_text(FEATURE_READINESS_REPORT_COMMENT_ID, analyzed_epic[:epic_iid], work_item_client)
+
+              if existing_note
+                work_item_client.update_note(note_id: existing_note[:id],
+                  body: comment({ must_haves: must_haves_report_rows, should_haves: should_haves_report_rows }, analyzed_epic).tr('"', "'"))
+              else
+                work_item_client.create_discussion(id: analyzed_epic[:epic_id],
+                  note: comment({ must_haves: must_haves_report_rows, should_haves: should_haves_report_rows }, analyzed_epic).tr('"', "'"))
+              end
+            end
+
+            private
+
+            def generate_report_rows(epic, type)
+              status_checks = check_statuses(epic)
+              create_rows(epic, type, status_checks)
+            end
+
+            def create_rows(epic, type, status_checks)
+              if type == :must_haves
+                [
+                  create_documentation_row(epic, status_checks),
+                  create_feature_flag_row(epic, status_checks),
+                  create_unit_tests_coverage_row(status_checks)
+
+                ]
+              else
+                [
+                  create_feature_tests_row(epic, status_checks),
+                  create_e2e_tests_row(epic, status_checks)
+                ]
+              end
+            end
+
+            def create_documentation_row(epic, status_checks)
+              ["Documentation added?", status_icon(status_checks[:has_docs]),
+                prepend_text('Added in:', format_links(epic[:doc_mrs]))]
+            end
+
+            def create_feature_flag_row(epic, status_checks)
+              ["Feature Flag added?", status_icon(status_checks[:feature_flag_added]),
+                prepend_text('Added in:', format_links(epic[:feature_flag_mrs]))]
+            end
+
+            def create_feature_tests_row(epic, status_checks)
+              ["Feature tests added?", status_icon(status_checks[:has_feature_specs]),
+                format_links(epic[:feature_spec_mrs])]
+            end
+
+            def create_e2e_tests_row(epic, status_checks)
+              ["End-to-end tests added?", status_icon(status_checks[:has_e2e_specs]),
+                format_links(epic[:e2e_spec_mrs])]
+            end
+
+            def create_unit_tests_coverage_row(status_checks)
+              ["Unit tests coverage complete?", status_icon(status_checks[:has_complete_unit_tests]),
+                prepend_text('Coverage missing for:', format_links(status_checks[:missing_specs]))]
+            end
+
+            def prepend_text(prepend_text, text)
+              return "#{prepend_text} #{text}" unless text.empty?
+
+              text
+            end
+
+            def check_statuses(epic)
+              {
+                has_docs: epic[:doc_mrs].any?,
+                feature_flag_added: epic[:feature_flag_mrs].any?,
+                has_feature_specs: epic[:feature_spec_mrs].any?,
+                has_e2e_specs: epic[:e2e_spec_mrs].any?,
+                missing_specs: missing_spec_mrs(epic),
+                has_complete_unit_tests: missing_spec_mrs(epic).empty?
+              }
+            end
+
+            def comment(rows, epic)
+              # Generate markdown table
+              must_haves_table_rows = rows[:must_haves].map do |description, status, links|
+                "| #{description} | #{status} | #{links} |"
+              end.join("\n")
+
+              should_haves_table_rows = rows[:should_haves].map do |description, status, links|
+                "| #{description} | #{status} | #{links} |"
+              end.join("\n")
+
+              <<~COMMENT
+                #{FEATURE_READINESS_REPORT_COMMENT_ID}
+
+                # :vertical_traffic_light: Feature Readiness Evaluation Report
+
+                ### :octagonal_sign: Must haves
+
+                | Evaluation | Result | Notes |
+                |------------|--------|-------|
+                #{must_haves_table_rows}
+
+                ### :warning: Should haves
+
+                | Evaluation | Result | Notes |
+                |------------|--------|-------|
+                #{should_haves_table_rows}
+
+                #{data(epic)}
+
+                ---
+
+                _Please note that this automation is under testing. Please add any feedback on [this issue](https://gitlab.com/gitlab-org/quality/quality-engineering/team-tasks/-/issues/3587)._
+
+              COMMENT
+            end
+
+            def status_icon(condition)
+              condition ? ':white_check_mark:' : ':x:'
+            end
+
+            def format_links(data)
+              return '' if data.empty?
+
+              data.map do |item|
+                item.map { |key, url| "[#{key}](#{url})" }.first
+              end.join(", ")
+            end
+
+            def missing_spec_mrs(epic)
+              epic[:issues].flat_map do |issue|
+                issue[:merge_requests].flat_map do |mr|
+                  next [] unless mr[:files_with_missing_specs]&.any?
+
+                  mr[:files_with_missing_specs].map do |file|
+                    { file => mr[:merge_request_web_url] }
+                  end
+                end.compact
+              end
+            end
+
+            def data(epic)
+              output = StringIO.new
+              PP.pp(epic, output)
+              <<~DATA
+                <details><summary>Expand for data</summary>
+
+                ```ruby
+                #{output.string}
+                ```
+
+                </details>
+              DATA
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/gitlab_quality/test_tooling/report/generate_test_session.rb

@@ -33,7 +33,7 @@ module GitlabQuality
           issue = gitlab.create_issue(
             title: "#{Time.now.strftime('%Y-%m-%d')} Test session report | #{Runtime::Env.qa_run_type}",
             description: generate_description(tests),
-            labels: ['automation:bot-authored', 'Quality', 'QA', 'triage report', pipeline_name_label],
+            labels: ['automation:bot-authored', 'E2E', 'triage report', pipeline_name_label],
             confidential: confidential
           )
 

data/lib/gitlab_quality/test_tooling/report/relate_failure_issue.rb

@@ -23,16 +23,23 @@ module GitlabQuality
         FAILURE_STACKTRACE_REGEX = %r{(?:(?:.*Failure/Error:(?<stacktrace>.+))|(?<stacktrace>.+))}m
         ISSUE_STACKTRACE_REGEX = /### Stack trace\s*(```)#{FAILURE_STACKTRACE_REGEX}(```)\n*\n###/m
 
-        NEW_ISSUE_LABELS = Set.new(%w[test failure::new priority::2 automation:bot-authored]).freeze
+        NEW_ISSUE_LABELS = Set.new(%w[test failure::new priority::2 automation:bot-authored type::maintenance]).freeze
         SCREENSHOT_IGNORED_ERRORS = ['500 Internal Server Error', 'fabricate_via_api!', 'Error Code 500'].freeze
-
+        FAILURE_ISSUE_GUIDE_URL = "https://handbook.gitlab.com/handbook/engineering/testing/guide-to-e2e-test-failure-issues/"
+        FAILURE_ISSUE_HANDBOOK_GUIDE = "**:rotating_light: [End-to-End Test Failure Issue Debugging Guide](#{FAILURE_ISSUE_GUIDE_URL}) :rotating_light:**\n".freeze
         # Map commits to security fork (gitlab-org/security/gitlab) for gitlab-org/gitlab since security patches exist
         # there before being released to the public repository
         DIFF_PROJECT_MAPPINGS = {
+          'gitlab-org/quality/e2e-test-issues' => 'gitlab-org/security/gitlab',
           'gitlab-org/gitlab' => 'gitlab-org/security/gitlab',
           'gitlab-org/customers-gitlab-com' => 'gitlab-org/customers-gitlab-com'
         }.freeze
 
+        # Don't use the E2E test issues project for commit parent
+        COMMIT_PROJECT_MAPPINGS = {
+          'gitlab-org/quality/e2e-test-issues' => 'gitlab-org/gitlab'
+        }.freeze
+
         # The project contains record of the deployments we use to determine the commit diff
         OPS_RELEASES_METADATA_PROJECT = 'gitlab-org/release/metadata'
 
@@ -179,7 +186,9 @@ module GitlabQuality
           end
 
           created_issue = super
-          test.failure_issue ||= created_issue.web_url
+
+          # On a dry run, created_issue may not be populated
+          test.failure_issue ||= created_issue&.web_url
 
           created_issue
         end
@@ -300,7 +309,7 @@ module GitlabQuality
         end
 
         def new_issue_description(test)
-          super + [
+          FAILURE_ISSUE_HANDBOOK_GUIDE + super + [
             "\n#{commit_diff_section}",
             "### Stack trace",
             "```\n#{test.full_stacktrace}\n```",
@@ -318,10 +327,10 @@ module GitlabQuality
           initialize_gitlab_ops_client
 
           if Runtime::Env.ci_pipeline_url.include?('ops.gitlab.net')
-            pipeline = ops_gitlab_client.find_pipeline(project, Runtime::Env.ci_pipeline_id.to_i)
+            pipeline = ops_gitlab_client.find_pipeline(Runtime::Env.ci_project_path, Runtime::Env.ci_pipeline_id.to_i)
             generate_ops_gitlab_diff(pipeline)
           else
-            pipeline = gitlab.find_pipeline(project, Runtime::Env.ci_pipeline_id.to_i)
+            pipeline = gitlab.find_pipeline(Runtime::Env.ci_project_path, Runtime::Env.ci_pipeline_id.to_i)
             generate_gitlab_diff(pipeline)
           end
         end
@@ -342,6 +351,8 @@ module GitlabQuality
         end
 
         def fetch_deployment_info(pipeline)
+          return 'No pipeline name set.' unless Runtime::Env.ci_pipeline_name
+
           pipeline_deploy_version = Runtime::Env.ci_pipeline_name.match(/(\d+\.\d+\.\d+)(?:-|$)/)&.captures&.first
           deployments = fetch_deployments(ops_gitlab_client, pipeline)
           found_deployment = find_matching_deployment(pipeline_deploy_version, deployments)
@@ -427,7 +438,8 @@ module GitlabQuality
 
         def generate_gitlab_diff(pipeline)
           pipeline_sha = pipeline.sha
-          parent_sha = gitlab.find_commit_parent(project, pipeline_sha)
+          commit_project = COMMIT_PROJECT_MAPPINGS.fetch(project, project)
+          parent_sha = gitlab.find_commit_parent(commit_project, pipeline_sha)
           diff_project = if DIFF_PROJECT_MAPPINGS.key?(project)
                            DIFF_PROJECT_MAPPINGS[project]
                          else
@@ -542,6 +554,8 @@ module GitlabQuality
         end
 
         def screenshot_artifact_url(test)
+          return "" unless test.screenshot_image
+
           ci_job_url = test.ci_job_url
           screenshot_path = test.screenshot_image[%r{qa/.*$}]
 

data/lib/gitlab_quality/test_tooling/runtime/env.rb

@@ -46,7 +46,7 @@ module GitlabQuality
         end
 
         def default_branch
-          env_var_value_if_defined('QA_DEFAULT_BRANCH') || 'main'
+          'master'
         end
 
         def ci_api_v4_url

data/lib/gitlab_quality/test_tooling/test_meta/processor/add_to_blocking_processor.rb

@@ -97,7 +97,7 @@ module GitlabQuality
 
                   This MR was created based on data from reliable e2e test report: #{context.report_issue}
 
-                  /label ~"Quality" ~"QA" ~"type::maintenance"
+                  /label ~"E2E" ~"type::maintenance"
                   /label ~"devops::#{devops_stage}"
                   #{context.label_from_product_group(product_group)}
 

data/lib/gitlab_quality/test_tooling/test_meta/processor/add_to_quarantine_processor.rb

@@ -137,7 +137,7 @@ module GitlabQuality
                 - [ ] To ensure a faster turnaround, ask in the `#quality_maintainers` Slack channel for someone to review and merge the merge request, rather than assigning it directly.
 
                 <!-- Base labels. -->
-                /label ~"Quality" ~"QA" ~"type::maintenance" ~"maintenance::pipelines"
+                /label ~"E2E" ~"type::maintenance" ~"maintenance::pipelines"
 
                 <!--
                 Choose the stage that appears in the test path, e.g. ~"devops::create" for

data/lib/gitlab_quality/test_tooling/test_result/base_test_result.rb

@@ -14,7 +14,8 @@ module GitlabQuality
           "Error reference number: 502",
           "(502): `GitLab is not responding`",
           "<head><title>502 Bad Gateway</title></head>",
-          "14:connections to all backends failing"
+          "14:connections to all backends failing",
+          "gitlab_canary=true cookie was set in browser but 'Next' badge was not shown on UI"
         ].freeze
 
         SHARED_EXAMPLES_CALLERS = %w[include_examples it_behaves_like].freeze
@@ -114,7 +115,7 @@ module GitlabQuality
         end
 
         def file_base_url
-          @file_base_url ||= "https://gitlab.com/#{project}/-/blob/#{ref}/"
+          @file_base_url ||= "https://gitlab.com/#{project == 'gitlab-org/quality/e2e-test-issues' ? 'gitlab-org/gitlab' : project}/-/blob/#{ref}/"
         end
 
         def test_file_link
@@ -143,15 +144,23 @@ module GitlabQuality
         def files_client
           @files_client ||= GitlabClient::RepositoryFilesClient.new(
             token: token,
-            project: project,
+            project: mapped_project,
             file_path: file,
-            ref: ref)
+            ref: ref.nil? || ref.empty? ? 'master' : ref)
         end
 
         private
 
         attr_reader :token, :project, :ref
 
+        def mapped_project
+          if project == 'gitlab-org/quality/e2e-test-issues'
+            'gitlab-org/gitlab'
+          else
+            project
+          end
+        end
+
         def screenshot
           report.fetch('screenshot', nil)
         end

data/lib/gitlab_quality/test_tooling/version.rb

@@ -2,6 +2,6 @@
 
 module GitlabQuality
   module TestTooling
-    VERSION = "2.10.0"
+    VERSION = "2.15.3"
   end
 end