gitlab_quality-test_tooling 1.11.0 → 1.17.0

data/lib/gitlab_quality/test_tooling/test_meta/processor/add_to_quarantine_processor.rb

@@ -0,0 +1,199 @@
+# frozen_string_literal: true
+
+module GitlabQuality
+  module TestTooling
+    module TestMeta
+      module Processor
+        class AddToQuarantineProcessor < MetaProcessor
+          QUARANTINE_METADATA = <<~META
+            ,
+            %{indentation}quarantine: {
+            %{indentation}  issue: '%{issue_url}',
+            %{indentation}  type: %{quarantine_type}
+            %{indentation}}%{suffix}
+          META
+
+          class << self
+            # Execute the processor
+            #
+            # @param [Hash<String,String>] spec the spec to update
+            # @option spec [String] :file_path the path to the spec file
+            # @option spec [String] :stage the stage of the test
+            # @option spec [String] :failure_issue the issue url of the failure
+            # @option spec [String] :name the name of the example
+            # @param [TestMetaUpdater] context instance of TestMetaUpdater
+            def execute(spec, context) # rubocop:disable Metrics/AbcSize
+              @context = context
+              @existing_mrs = nil
+              @file_path = spec["file_path"]
+              testcase = spec["testcase"]
+              devops_stage = spec["stage"]
+              product_group = spec["product_group"]
+              @failure_issue_url = spec["failure_issue"]
+              @example_name = spec["name"]
+              @issue_id = failure_issue_url.split('/').last # split url segment, last segment of path is the issue id
+              @mr_title = format("%{prefix} %{example_name}", prefix: '[QUARANTINE]', example_name: example_name).truncate(72, omission: '')
+              @failure_issue = context.fetch_issue(iid: issue_id)
+
+              @file_contents = context.get_file_contents(file_path)
+
+              new_content, @changed_line_no = add_quarantine_metadata
+
+              return unless proceed_with_merge_request?
+
+              branch = context.create_branch("#{issue_id}-quarantine-#{SecureRandom.hex(4)}", example_name, context.ref)
+
+              context.commit_changes(branch, <<~COMMIT_MESSAGE, file_path, new_content)
+                Quarantine end-to-end test
+
+                #{"Quarantine #{example_name}".truncate(72)}
+              COMMIT_MESSAGE
+
+              gitlab_bot_user_id = context.user_id_for_username(Runtime::Env.gitlab_bot_username)
+
+              merge_request = context.create_merge_request(mr_title, branch, gitlab_bot_user_id) do
+                <<~MARKDOWN
+                  ## What does this MR do?
+
+                  Quarantines the test [`#{example_name}`](https://gitlab.com/#{context.project}/-/blob/#{context.ref}/#{file_path}#L#{changed_line_no + 1})
+
+                  This test was identified in the reliable e2e test report: #{context.report_issue}
+
+                  [Testcase link](#{testcase})
+
+                  [Spec metrics link](#{context.single_spec_metrics_link(example_name)})
+
+                  ### E2E Test Failure issue(s)
+
+                  #{failure_issue_url}
+
+                  ### Check-list
+
+                  - [ ] General code guidelines check-list
+                    - [ ] [Code review guidelines](https://docs.gitlab.com/ee/development/code_review.html)
+                    - [ ] [Style guides](https://docs.gitlab.com/ee/development/contributing/style_guides.html)
+                  - [ ] Quarantine test check-list
+                    - [ ] Follow the [Quarantining Tests guide](https://about.gitlab.com/handbook/engineering/infrastructure/test-platform/debugging-qa-test-failures/#quarantining-tests).
+                    - [ ] Confirm the test has a [`quarantine:` tag with the specified quarantine type](https://about.gitlab.com/handbook/engineering/infrastructure/test-platform/debugging-qa-test-failures/#quarantined-test-types).
+                    - [ ] Note if the test should be [quarantined for a specific environment](https://docs.gitlab.com/ee/development/testing_guide/end_to_end/execution_context_selection.html#quarantine-a-test-for-a-specific-environment).
+                    - [ ] (Optionally) In case of an emergency (e.g. blocked deployments), consider adding labels to pick into auto-deploy (~"Pick into auto-deploy" ~"priority::1" ~"severity::1").
+                  - [ ] To ensure a faster turnaround, ask in the `#quality_maintainers` Slack channel for someone to review and merge the merge request, rather than assigning it directly.
+
+                  <!-- Base labels. -->
+                  /label ~"Quality" ~"QA" ~"type::maintenance" ~"maintenance::pipelines"
+
+                  <!--
+                  Choose the stage that appears in the test path, e.g. ~"devops::create" for
+                  `qa/specs/features/browser_ui/3_create/web_ide/add_file_template_spec.rb`.
+                  -->
+                  /label ~"devops::#{devops_stage}"
+                  #{context.label_from_product_group(product_group)}
+
+                  <div align="center">
+                  (This MR was automatically generated by [`gitlab_quality-test_tooling`](https://gitlab.com/gitlab-org/ruby/gems/gitlab_quality-test_tooling) at #{Time.now.utc})
+                  </div>
+                MARKDOWN
+              end
+
+              if merge_request
+                context.add_processed_record({ file_path => changed_line_no })
+                Runtime::Logger.info("  Created MR for quarantine: #{merge_request.web_url}")
+              end
+
+              merge_request
+            end
+
+            # Performs post processing. Takes a list of MRs and posts them in a note on report_issue and Slack
+            #
+            # @param [Gitlab::ObjectifiedHash] merge_requests
+            def post_process(merge_requests)
+              web_urls = merge_requests.compact.map { |mr| "- #{mr.web_url}\n" }.join
+
+              return if web_urls.empty?
+
+              context.post_note_on_report_issue(<<~ISSUE_NOTE)
+
+                The following merge requests have been created to quarantine the unstable tests:
+
+                #{web_urls}
+              ISSUE_NOTE
+
+              context.post_message_on_slack(<<~MSG)
+                *Action Required!* The following merge requests have been created to quarantine the unstable tests identified
+                in the reliable test report: #{context.report_issue}
+
+                #{web_urls}
+
+                Maintainers are requested to review and merge. Thank you.
+              MSG
+            end
+
+            private
+
+            attr_reader :context, :file_path, :file_contents, :failure_issue_url, :example_name,
+              :issue_id, :mr_title, :failure_issue, :changed_line_no
+
+            # Checks if the failure issue is closed or if there is already an MR open
+            #
+            # @return [Boolean]
+            def proceed_with_merge_request? # rubocop:disable Metrics/AbcSize
+              if context.issue_is_closed?(failure_issue)
+                Runtime::Logger.info("  Failure issue '#{failure_issue_url}' is closed. Will not proceed with creating MR.")
+                return false
+              elsif context.record_processed?(file_path, changed_line_no)
+                Runtime::Logger.info("  Record already processed for #{file_path}:#{changed_line_no}. Will not proceed with creating MR.")
+                return false
+              elsif existing_mrs&.any?
+                Runtime::Logger.info("  An open MR already exists for '#{example_name}':  #{existing_mrs.first['web_url']}. Will not proceed with creating MR.")
+                return false
+              end
+
+              true
+            end
+
+            # Add quarantine metadata to the file content and replace it
+            #
+            # @return [Array<String, Integer>] first value holds the new content, the second value holds the line number of the test
+            def add_quarantine_metadata # rubocop:disable Metrics/AbcSize
+              matched_lines = context.find_example_match_lines(file_contents, example_name)
+
+              context.update_matched_line(matched_lines.last, file_contents.dup) do |line|
+                indentation = context.indentation(line)
+
+                if line.sub(DESCRIPTION_REGEX, '').include?(',') && line.split.last != 'do'
+                  line[line.rindex(',')] = format(QUARANTINE_METADATA.rstrip, issue_url: failure_issue_url, indentation: indentation, suffix: ',', quarantine_type: quarantine_type)
+                else
+                  line[line.rindex(' ')] = format(QUARANTINE_METADATA.rstrip, issue_url: failure_issue_url, indentation: indentation, suffix: ' ', quarantine_type: quarantine_type)
+                end
+
+                line
+              end
+            end
+
+            # Returns the quarantine type based on the failure scoped label
+            #
+            # @return [String]
+            def quarantine_type
+              case context.issue_scoped_label(failure_issue, 'failure')&.split('::')&.last
+              when 'new', 'investigating'
+                ':investigating'
+              when 'broken-test'
+                ':broken'
+              when 'bug'
+                ':bug'
+              when 'flaky-test'
+                ':flaky'
+              when 'stale-test'
+                ':stale'
+              when 'test-environment'
+                ':test_environment'
+              else
+                ':investigating'
+              end
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/gitlab_quality/test_tooling/test_meta/processor/meta_processor.rb

@@ -0,0 +1,44 @@
+# frozen_string_literal: true
+
+require 'active_support/core_ext/string/filters'
+
+module GitlabQuality
+  module TestTooling
+    module TestMeta
+      module Processor
+        class MetaProcessor
+          class << self
+            DESCRIPTION_REGEX = /('.*?')|(".*?")/
+
+            def execute
+              raise 'method not implemented'
+            end
+
+            def post_process
+              raise 'method not implemented'
+            end
+
+            private_class_method :new
+
+            # Fetch existing MRs for given mr title
+            #
+            # @return [Array<Gitlab::ObjectifiedHash>]
+            def existing_mrs
+              @existing_mrs ||= context.existing_merge_requests(title: mr_title)
+            end
+
+            # Returns the index of the end of test description
+            #
+            # @param [String] line The line containing the test description
+            # @return [Integer]
+            def end_of_description_index(line)
+              description_length = line.match(DESCRIPTION_REGEX)[0].length
+              description_start_index = line.index(DESCRIPTION_REGEX)
+              description_start_index + description_length
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/gitlab_quality/test_tooling/test_meta/test_meta_updater.rb

@@ -0,0 +1,313 @@
+# frozen_string_literal: true
+
+require 'json'
+
+module GitlabQuality
+  module TestTooling
+    module TestMeta
+      class TestMetaUpdater
+        include TestTooling::Concerns::FindSetDri
+
+        attr_reader :project, :ref, :report_issue, :processed_records
+
+        TEST_PLATFORM_MAINTAINERS_SLACK_CHANNEL_ID = 'C0437FV9KBN' # test-platform-maintainers
+
+        def initialize(token:, project:, specs_file:, processor:, ref: 'master', dry_run: false)
+          @specs_file = specs_file
+          @token = token
+          @project = project
+          @ref = ref
+          @dry_run = dry_run
+          @processor = processor
+          @processed_records = {}
+        end
+
+        def invoke!
+          JSON.parse(File.read(specs_file)).tap do |contents|
+            @report_issue = contents['report_issue']
+
+            results = []
+            contents['specs'].each do |spec|
+              results << processor.execute(spec, self)
+            end
+            processor.post_process(results)
+          end
+        end
+
+        # Add processed records
+        #
+        # @param [Hash<String,Integer>] record the processed record
+        # @option record [String] :file_path the path to the spec file
+        # @option spec [Intenger] :changed_line_no the line number change in file_path
+        # @return [Hash<String,Integer>] processed_records
+        def add_processed_record(record)
+          @processed_records.merge!(record)
+        end
+
+        # Fetch contents of file from the repository
+        #
+        # @param [String] file_path path to the file
+        # @return [String] contents of the file
+        def get_file_contents(file_path)
+          repository_files = GitlabClient::RepositoryFilesClient.new(token: token, project: project, file_path: file_path)
+          repository_files.file_contents
+        end
+
+        # Find all lines that contain any part of the example name
+        #
+        # @param [String] content the content of the spec file
+        # @param [String] example_name the name of example to find
+        # @return [Array<String, Integer>] first value holds the matched line, the second value holds the line number of matched line
+        def find_example_match_lines(content, example_name)
+          lines = content.split("\n")
+
+          matched_lines = []
+
+          lines.each_with_index do |line, line_index|
+            string_within_quotes = spec_desc_string_within_quotes(line)
+
+            matched_lines << [line, line_index] if string_within_quotes && example_name.include?(string_within_quotes)
+          rescue StandardError => e
+            puts "Error: #{e}"
+          end
+
+          matched_lines
+        end
+
+        # Update the provided matched_line with content from the block if given
+        #
+        # @param [Array<String, Integer>] matched_line first value holds the line content, the second value holds the line number
+        # @param [String] content full orignal content of the spec file
+        # @return [Array<String, Integer>] first value holds the new content, the second value holds the line number of the test
+        def update_matched_line(matched_line, content)
+          lines = content.split("\n")
+
+          begin
+            resulting_line = block_given? ? yield(matched_line[0]) : matched_line[0]
+            lines[matched_line[1]] = resulting_line
+          rescue StandardError => e
+            puts "Error: #{e}"
+          end
+
+          [lines.join("\n") << "\n", matched_line[1]]
+        end
+
+        # Create a branch from the ref
+        #
+        # @param [String] name_prefix the prefix to attach to the branch name
+        # @param [String] example_name the example
+        # @return [Gitlab::ObjectifiedHash] the new branch
+        def create_branch(name_prefix, example_name, ref)
+          branch_name = [name_prefix, example_name.gsub(/\W/, '-')]
+          @branches_client ||= (dry_run ? GitlabClient::BranchesDryClient : GitlabClient::BranchesClient).new(token: token, project: project)
+          @branches_client.create(branch_name.join('-'), ref)
+        end
+
+        # Commit changes to a branch
+        #
+        # @param [Gitlab::ObjectifiedHash] branch the branch to commit to
+        # @param [String] message the message to commit
+        # @param [String] new_content the new content to commit
+        # @return [Gitlab::ObjectifiedHash] the commit
+        def commit_changes(branch, message, file_path, new_content)
+          @commits_client ||= (dry_run ? GitlabClient::CommitsDryClient : GitlabClient::CommitsClient)
+                                .new(token: token, project: project)
+          @commits_client.create(branch['name'], file_path, new_content, message)
+        end
+
+        # Create a Merge Request with a given branch
+        #
+        # @param [String] title_prefix the prefix of the title
+        # @param [String] example_name the example
+        # @param [Gitlab::ObjectifiedHash] branch the branch
+        # @param [Integer] assignee_id
+        # @param [Array<Integer>] reviewer_ids
+        # @param [String] labels comma seperated list of labels
+        # @return [Gitlab::ObjectifiedHash] the created merge request
+        def create_merge_request(title, branch, assignee_id = nil, reviewer_ids = [], labels = '')
+          description = yield
+
+          merge_request_client.create_merge_request(
+            title: title,
+            source_branch: branch['name'],
+            target_branch: ref,
+            description: description,
+            labels: labels,
+            assignee_id: assignee_id,
+            reviewer_ids: reviewer_ids)
+        end
+
+        # Check if issue is closed
+        #
+        # @param [Gitlab::ObjectifiedHash] issue the issue
+        # @return [Boolean] True or False
+        def issue_is_closed?(issue)
+          issue['state'] == 'closed'
+        end
+
+        # Get scoped label from issue
+        #
+        # @param [Gitlab::ObjectifiedHash] issue the issue
+        # @param [String] scope
+        # @return [String] scoped label
+        def issue_scoped_label(issue, scope)
+          issue['labels'].detect { |label| label.match(/#{scope}::/) }
+        end
+
+        # Fetch an issue
+        #
+        # @param [String] iid: The iid of the issue
+        # @return [Gitlab::ObjectifiedHash]
+        def fetch_issue(iid:)
+          issue_client.find_issues(iid: iid).first
+        end
+
+        # Post note on report_issue
+        #
+        # @param [String] note the note to post
+        # @return [Gitlab::ObjectifiedHash]
+        def post_note_on_report_issue(note)
+          iid = report_issue&.split('/')&.last # split url segment, last segment of path is the issue id
+          if iid
+            issue_client.create_issue_note(iid: iid, note: note)
+          else
+            Runtime::Logger.info("#{self.class.name}##{__method__} Note was NOT posted on report issue: #{report_issue}")
+          end
+        end
+
+        # Post a note of merge reqest
+        #
+        # @param [String] note
+        # @param [Integer] merge_request_iid
+        # @return [Gitlab::ObjectifiedHash]
+        def post_note_on_merge_request(note, merge_request_iid)
+          merge_request_client.create_note(note: note, merge_request_iid: merge_request_iid)
+        end
+
+        # Fetch the id for the dri of the product group and stage
+        # The first item returned is the id of the assignee and the second item is the handle
+        #
+        # @param [String] product_group
+        # @param [String] devops_stage
+        # @return [Array<Integer, String>]
+        def fetch_dri_id(product_group, devops_stage)
+          assignee_handle = ENV.fetch('QA_TEST_DRI_HANDLE', nil) || set_dri_via_group(product_group, devops_stage)
+
+          [user_id_for_username(assignee_handle), assignee_handle]
+        end
+
+        # Fetch id for the given GitLab username/handle
+        #
+        # @param [String] username
+        # @return [Integer]
+        def user_id_for_username(username)
+          issue_client.find_user_id(username: username)
+        end
+
+        # Post a message on Slack
+        #
+        # @param [String] message the message to post
+        # @return [HTTP::Response]
+        def post_message_on_slack(message)
+          channel = ENV.fetch('SLACK_QA_CHANNEL', nil) || TEST_PLATFORM_MAINTAINERS_SLACK_CHANNEL_ID
+          slack_options = {
+            slack_webhook_url: ENV.fetch('CI_SLACK_WEBHOOK_URL', nil),
+            channel: channel,
+            username: "GitLab Quality Test Tooling",
+            icon_emoji: ':warning:',
+            message: message
+          }
+          puts "Posting Slack message to channel: #{channel}"
+
+          (dry_run ? GitlabQuality::TestTooling::Slack::PostToSlackDry : GitlabQuality::TestTooling::Slack::PostToSlack).new(**slack_options).invoke!
+        end
+
+        # Provide indentaiton based on the given line
+        #
+        # @param[String] line the line to use for indentation
+        # @return[String] indentation
+        def indentation(line)
+          # Indent the same number of spaces as the current line
+          no_of_spaces = line[/\A */].size
+          # If the first char on current line is not a quote, add two more spaces
+          no_of_spaces += /['"]/.match?(line.lstrip[0]) ? 0 : 2
+
+          " " * no_of_spaces
+        end
+
+        # Returns and existing merge request with the given title
+        #
+        # @param [String] title: Title of the merge request
+        # @return [Array<Gitlab::ObjectifiedHash>] Merge requests
+        def existing_merge_requests(title:)
+          merge_request_client.find(options: { search: title, in: 'title', state: 'opened' })
+        end
+
+        # Checks if changes has already been made to given file and line number
+        #
+        # @param [String] file_path path to the file
+        # @param [Integer] changed_line_no updated line number
+        # @return [Boolean]
+        def record_processed?(file_path, changed_line_no)
+          processed_records[file_path] && processed_records[file_path] == changed_line_no
+        end
+
+        # Infers product group label from the provided product group
+        #
+        # @param [String] product_group product group
+        # @return [String]
+        def label_from_product_group(product_group)
+          label = labels_inference.infer_labels_from_product_group(product_group).to_a.first
+
+          label ? %(/label ~"#{label}") : ''
+        end
+
+        # Returns the link to the Grafana dashboard for single spec metrics
+        #
+        # @param [String] example_name the full example name
+        # @return [String]
+        def single_spec_metrics_link(example_name)
+          base_url = "https://dashboards.quality.gitlab.net/d/cW0UMgv7k/single-spec-metrics?orgId=1&var-run_type=All&var-name="
+          base_url + CGI.escape(example_name)
+        end
+
+        private
+
+        attr_reader :token, :specs_file, :dry_run, :processor
+
+        # Returns any test description string within single or double quotes
+        #
+        # @param [String] line the line to check for any quoted string
+        # @return [String] the match or nil if no match
+        def spec_desc_string_within_quotes(line)
+          match = line.match(/(?:it|describe|context|\s)+ ['"]([^'"]*)['"]/)
+          match ? match[1] : nil
+        end
+
+        # Returns the GitlabIssueClient or GitlabIssueDryClient based on the value of dry_run
+        #
+        # @return [GitlabIssueDryClient | GitlabIssueClient]
+        def issue_client
+          @issue_client ||= (dry_run ? GitlabClient::IssuesDryClient : GitlabClient::IssuesClient).new(token: token, project: project)
+        end
+
+        # Returns the MergeRequestDryClient or MergeRequest based on the value of dry_run
+        #
+        # @return [MergeRequestDryClient | MergeRequest]
+        def merge_request_client
+          @merge_request_client ||= (dry_run ? GitlabClient::MergeRequestsDryClient : GitlabClient::MergeRequestsClient).new(
+            token: token,
+            project: project
+          )
+        end
+
+        # Returns a cached instance of GitlabQuality::TestTooling::LabelsInference
+        #
+        #  @return [GitlabQuality::TestTooling::LabelsInference]
+        def labels_inference
+          @labels_inference ||= GitlabQuality::TestTooling::LabelsInference.new
+        end
+      end
+    end
+  end
+end

data/lib/gitlab_quality/test_tooling/version.rb

@@ -2,6 +2,6 @@
 
 module GitlabQuality
   module TestTooling
-    VERSION = "1.11.0"
+    VERSION = "1.17.0"
   end
 end