gitlab_quality-test_tooling 0.6.2 → 0.7.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 80c8edb6daa8bf924d294ef291d82fd60c8545898aaa0eac7e71b98dad4dc1c2
-  data.tar.gz: 9e414922f797ce645e951f3fc26adb394a7a7f48f2c01bf7ddd7d218a1229486
+  metadata.gz: c393338cd20306bc419cf1bbb0c4dd3e09f51312f0c6bbc2a58e60b062a8170d
+  data.tar.gz: 3e43e01fc39e8c1544f9fdf552f061b98c7056710377ffeaa5a229f2ca08d587
 SHA512:
-  metadata.gz: 573774ab0d7be7d23d2c262f0246377b824604240930ba451b088af914e7b166b8844e63425f3583a16f45bdc73be3779e5e5eb081218ff934fcb2e87698a8bf
-  data.tar.gz: f0ad9eb31e28e2c9f21c317b6d84e78645c36f5fff30baa8ada905d17f202710e087125a373f8fc38dd06612c08764839d2a35aaa835d133211219ee2778c37a
+  metadata.gz: b78a87a21945eaacdf4e7ff609b248cb427c7e4c3ccf19c8f4784aa19f0219856144ff2c35079f9556907308f11a72fdf10c60ab3e1f79799f6eede53556bf2b
+  data.tar.gz: 7362e44b09a26d0b4893a05cafc819f9aaff457a7f9dc0c39f645851b2b3362872e2acf14ed3b0e1e35a00f17e02753ff2b42badf4de27672433d6288f39828e

data/Gemfile.lock

@@ -1,7 +1,7 @@
 PATH
   remote: .
   specs:
-    gitlab_quality-test_tooling (0.6.2)
+    gitlab_quality-test_tooling (0.7.0)
       activesupport (~> 6.1)
       gitlab (~> 4.19)
       http (~> 5.0)

data/lib/gitlab_quality/test_tooling/gitlab_issue_client.rb

@@ -61,8 +61,14 @@ module GitlabQuality
         end
       end
 
-      def create_issue(title:, description:, labels:, issue_type: 'issue')
-        attrs = { issue_type: issue_type, description: description, labels: labels }
+      def create_issue(title:, description:, labels:, issue_type: 'issue', assignee_id: nil, due_date: nil)
+        attrs = {
+          issue_type: issue_type,
+          description: description,
+          labels: labels,
+          assignee_id: assignee_id,
+          due_date: due_date
+        }.compact
 
         handle_gitlab_client_exceptions do
           client.create_issue(project, title, attrs)

data/lib/gitlab_quality/test_tooling/labels_inference.rb

@@ -0,0 +1,53 @@
+#!/usr/bin/env ruby
+# frozen_string_literal: true
+
+require 'json'
+require 'httparty'
+
+module GitlabQuality
+  module TestTooling
+    class LabelsInference
+      WWW_GITLAB_COM_SITE = 'https://about.gitlab.com'
+      WWW_GITLAB_COM_GROUPS_JSON = "#{WWW_GITLAB_COM_SITE}/groups.json".freeze
+      WWW_GITLAB_COM_CATEGORIES_JSON = "#{WWW_GITLAB_COM_SITE}/categories.json".freeze
+      FEATURE_CATEGORY_METADATA_REGEX = /(?<=feature_category: :)\w+/
+
+      def infer_labels_from_product_group(product_group)
+        [groups_mapping.dig(product_group, 'label')].compact.to_set
+      end
+
+      def infer_labels_from_feature_category(feature_category)
+        [
+          categories_mapping.dig(feature_category, 'label'),
+          *infer_labels_from_product_group(categories_mapping.dig(feature_category, 'group'))
+        ].compact.to_set
+      end
+
+      private
+
+      def categories_mapping
+        @categories_mapping ||= self.class.fetch_json(WWW_GITLAB_COM_CATEGORIES_JSON)
+      end
+
+      def groups_mapping
+        @groups_mapping ||= self.class.fetch_json(WWW_GITLAB_COM_GROUPS_JSON)
+      end
+
+      def self.fetch_json(json_url)
+        json = with_retries { HTTParty.get(json_url, format: :plain) }
+        JSON.parse(json)
+      rescue JSON::ParserError
+        Runtime::Logger.debug("#{self.class.name}##{__method__} attempted to parse invalid JSON:\n\n#{json}")
+        {}
+      end
+
+      def self.with_retries(attempts: 3)
+        yield
+      rescue Errno::ECONNRESET, OpenSSL::SSL::SSLError, Net::OpenTimeout
+        retry if (attempts -= 1).positive?
+        raise
+      end
+      private_class_method :with_retries
+    end
+  end
+end

data/lib/gitlab_quality/test_tooling/report/relate_failure_issue.rb

@@ -13,8 +13,7 @@ module GitlabQuality
       # - Takes the JSON test run reports, e.g. `$CI_PROJECT_DIR/gitlab-qa-run-*/**/rspec-*.json`
       # - Takes a project where failure issues should be created
       # - Find issue by title (with test description or test file), then further filter by stack trace, then pick the better-matching one
-      # - Post a note to the latest failed job, e.g. https://gitlab.com/gitlab-org/gitlab/-/issues/408333#note_1361882769
-      # - Update labels
+      # - Add the failed job to the issue description, and update labels
       class RelateFailureIssue < ReportAsIssue
         include Concerns::FindSetDri
 
@@ -22,19 +21,20 @@ module GitlabQuality
         SPAM_THRESHOLD_FOR_FAILURE_ISSUES = 3
         FAILURE_STACKTRACE_REGEX = %r{(?:(?:.*Failure/Error:(?<stacktrace>.+))|(?<stacktrace>.+))}m
         ISSUE_STACKTRACE_REGEX = /### Stack trace\s*(```)#{FAILURE_STACKTRACE_REGEX}(```)/m
-        FAILED_JOB_DESCRIPTION_REGEX = %r{First happened in https?://\S+\.}m
-        FAILED_JOB_NOTE_REGEX = %r{Failed most recently in \D+ pipeline: https?://\S+}
+        JOB_URL_REGEX = %r{(?<job_url>https://(?<host>[\w.]+)/(?<project_path>[\w\-./]+)/-/jobs/\d+)}
+        FAILED_JOB_DESCRIPTION_REGEX = /First happened in #{JOB_URL_REGEX}\./m
+        REPORT_ITEM_REGEX = /^1\. \d{4}-\d{2}-\d{2}: #{JOB_URL_REGEX} \((?<pipeline_url>.+)\)$/
         NEW_ISSUE_LABELS = Set.new(%w[test failure::new priority::2]).freeze
         IGNORE_EXCEPTIONS = ['Net::ReadTimeout', '403 Forbidden - Your account has been blocked'].freeze
         SCREENSHOT_IGNORED_ERRORS = ['500 Internal Server Error', 'fabricate_via_api!', 'Error Code 500'].freeze
 
         MultipleIssuesFound = Class.new(StandardError)
 
-        def initialize(max_diff_ratio: DEFAULT_MAX_DIFF_RATIO_FOR_DETECTION, system_logs: [], base_issue_labels: [], **kwargs)
+        def initialize(max_diff_ratio: DEFAULT_MAX_DIFF_RATIO_FOR_DETECTION, system_logs: [], base_issue_labels: Set.new, **kwargs)
           super
           @max_diff_ratio = max_diff_ratio.to_f
           @system_logs = Dir.glob(system_logs)
-          @base_issue_labels = base_issue_labels || []
+          @base_issue_labels = Set.new(base_issue_labels)
           @issue_type = 'issue'
           @commented_issue_list = Set.new
         end
@@ -58,35 +58,36 @@ module GitlabQuality
         end
 
         def relate_failure_to_issue(test)
-          puts "  => Searching issues for test '#{test.name}'..."
+          puts " => Relating issues for test '#{test.name}'..."
 
           begin
-            issue, issue_already_commented = find_and_link_issue(test)
-            return create_issue(test) unless issue || test.quarantine?
+            issue = find_issue_and_update_reports(test)
 
-            update_labels(issue, test) unless issue_already_commented
+            create_issue(test) unless issue || test.quarantine?
           rescue MultipleIssuesFound => e
             warn(e.message)
           end
         end
 
-        def find_and_link_issue(test)
+        def find_issue_and_update_reports(test)
           issue, diff_ratio = find_failure_issue(test)
-          return [false, true] unless issue
+          return unless issue
 
-          issue_already_commented = issue_already_commented?(issue)
-          if issue_already_commented
-            puts " => Failure already commented on issue."
+          failure_already_reported = failure_already_reported?(issue, test)
+          if failure_already_reported
+            puts "  => Failure already reported on issue."
           else
             puts "  => Found issue #{issue.web_url} for test '#{test.name}' with a diff ratio of #{(diff_ratio * 100).round(2)}%."
-            post_or_update_failed_job_note(issue, test)
+            update_reports(issue, test)
             @commented_issue_list.add(issue.web_url)
           end
 
-          [issue, issue_already_commented]
+          issue
         end
 
-        def issue_already_commented?(issue)
+        def failure_already_reported?(issue, test)
+          @commented_issue_list.add(issue.web_url) if failed_issue_job_urls(issue).include?(test.ci_job_url)
+
           @commented_issue_list.include?(issue.web_url)
         end
 
@@ -94,33 +95,23 @@ module GitlabQuality
           similar_issues = pipeline_issues_with_similar_stacktrace(test)
 
           if similar_issues.size >= SPAM_THRESHOLD_FOR_FAILURE_ISSUES
-            puts " => Similar failure issues have already been opened for same pipeline environment"
-            puts " => Will not create new issue for this failing spec"
+            puts "  => Similar failure issues have already been opened for the same pipeline environment, we won't create new issue"
             similar_issues.each do |similar_issue|
-              puts "Please check issue: #{similar_issue.web_url}"
-
-              unless existing_failure_note(similar_issue, test.ci_job_url)
-                gitlab.create_issue_note(iid: similar_issue.iid,
-                  note: "This failed job is most likely related: #{test.ci_job_url}")
-              end
+              puts "  => Please check issue: #{similar_issue.web_url}"
+              update_reports(similar_issue, test)
             end
             return
           end
 
-          issue = super
-          puts "for test '#{test.name}'."
-
-          post_or_update_failed_job_note(issue, test)
-
-          assign_dri(issue, test)
-
-          issue
+          super
         end
 
         def pipeline_issues_with_similar_stacktrace(test)
-          gitlab.find_issues(options: { state: 'opened', labels: (base_issue_labels + %w[test failure::new]).join(','),
+          search_labels = (base_issue_labels + Set.new(%w[test failure::new])).to_a
+          gitlab.find_issues(options: { state: 'opened', labels: search_labels,
                                         created_after: past_timestamp(2) }).select do |issue|
             job_url_from_issue = failed_issue_job_url(issue)
+
             next if pipeline != pipeline_env_from_job_url(job_url_from_issue)
 
             stack_trace_from_issue = cleaned_stack_trace_from_issue(issue)
@@ -131,20 +122,22 @@ module GitlabQuality
         end
 
         def failed_issue_job_url(issue)
-          existing_note = existing_failure_note(issue)
-          if existing_note
-            job_url_string = existing_note.body
-            matched = job_url_string.match(FAILED_JOB_NOTE_REGEX)
-          else
-            job_url_string = issue.description
-            matched = job_url_string.match(FAILED_JOB_DESCRIPTION_REGEX)
-          end
+          job_urls_from_description(issue.description, REPORT_ITEM_REGEX).last ||
+            # Legacy format
+            job_urls_from_description(issue.description, FAILED_JOB_DESCRIPTION_REGEX).last
+        end
 
-          return unless matched
+        def failed_issue_job_urls(issue)
+          job_urls_from_description(issue.description, REPORT_ITEM_REGEX) +
+            # Legacy format
+            job_urls_from_description(issue.description, FAILED_JOB_DESCRIPTION_REGEX)
+        end
 
-          job_url = matched[0].chop.split.last
-          puts "=> Found failed job url in the issue: #{job_url}"
-          job_url
+        def job_urls_from_description(issue_description, regex)
+          issue_description.lines.filter_map do |line|
+            match = line.match(regex)
+            match[:job_url] if match
+          end
         end
 
         def pipeline_env_from_job_url(job_url)
@@ -163,7 +156,8 @@ module GitlabQuality
         end
 
         def failure_issues(test)
-          gitlab.find_issues(options: { state: 'opened', labels: base_issue_labels + %w[test] }).select do |issue|
+          search_labels = (base_issue_labels + Set.new(%w[test])).to_a
+          gitlab.find_issues(options: { state: 'opened', labels: search_labels }).select do |issue|
             issue_title = issue.title.strip
             issue_title.include?(test.name) || issue_title.include?(partial_file_path(test.file))
           end
@@ -235,15 +229,14 @@ module GitlabQuality
           stacktrace_match = stacktrace.match(regex)
 
           if stacktrace_match
-            stacktrace_match[:stacktrace].split('First happened in')[0].gsub(/^\s*#.*$/, '').gsub(/^[[:space:]]+/,
-              '').strip
+            stacktrace_match[:stacktrace].gsub(/^\s*#.*$/, '').gsub(/^[[:space:]]+/, '').strip
           else
-            puts "  => [DEBUG] Stacktrace doesn't match the expected regex (#{regex}):\n----------------\n#{stacktrace}\n----------------\n"
+            puts "  => [DEBUG] Stacktrace doesn't match the regex (#{regex}):\n----------------\n#{stacktrace}\n----------------\n"
           end
         end
 
         def remove_unique_resource_names(stacktrace)
-          stacktrace.gsub(/qa-(test|user)-[a-z0-9-]+/, '<unique-test-resource>').gsub(
+          stacktrace.gsub(/(QA User |qa-(test|user)-)[a-z0-9-]+/, '<unique-test-resource>').gsub(
             /(?:-|_)(?:\d+[a-z]|[a-z]+\d)[a-z\d]{4,}/, '<unique-hash>')
         end
 
@@ -268,10 +261,9 @@ module GitlabQuality
           super + [
             "\n### Stack trace",
             "```\n#{full_stacktrace(test)}\n```",
-            "First happened in #{test.ci_job_url}.",
-            ("Related test case: #{test.testcase}." if test.testcase),
             screenshot_section(test),
-            system_log_errors_section(test)
+            system_log_errors_section(test),
+            reports_section(test)
           ].compact.join("\n\n")
         end
 
@@ -286,40 +278,87 @@ module GitlabQuality
             ).system_logs_summary_markdown
           end
 
-          puts "  => No system logs or correlation id provided, skipping this section in issue description" if section.empty?
+          if section.empty?
+            puts "  => No system logs or correlation id provided, skipping this section in issue description"
+            return
+          end
 
           section
         end
 
+        def reports_section(test)
+          <<~REPORTS
+          ### Reports (1)
+
+          #{report_list_item(test)}
+          REPORTS
+        end
+
+        def report_list_item(test)
+          "1. #{Time.new.utc.strftime('%F')}: #{test.ci_job_url} (#{ENV.fetch('CI_PIPELINE_URL', 'pipeline url is missing')})"
+        end
+
+        def labels_inference
+          @labels_inference ||= GitlabQuality::TestTooling::LabelsInference.new
+        end
+
         def new_issue_labels(test)
-          up_to_date_labels(test: test, new_labels: NEW_ISSUE_LABELS)
+          puts "  => [DEBUG] product_group: #{test.product_group}; feature_category: #{test.feature_category}"
+          new_labels = NEW_ISSUE_LABELS +
+            labels_inference.infer_labels_from_product_group(test.product_group) +
+            labels_inference.infer_labels_from_feature_category(test.feature_category)
+          up_to_date_labels(test: test, new_labels: new_labels)
         end
 
         def up_to_date_labels(test:, issue: nil, new_labels: Set.new)
-          base_issue_labels + (super << pipeline_name_label).to_a
+          (Set.new(base_issue_labels) + (super << pipeline_name_label)).to_a
         end
 
-        def post_or_update_failed_job_note(issue, test)
-          current_note = "Failed most recently in #{pipeline} pipeline: #{test.ci_job_url}"
-          existing_note = existing_failure_note(issue)
+        def new_issue_assignee_id(test)
+          return unless test.product_group?
 
-          if existing_note
-            gitlab.edit_issue_note(issue_iid: issue.iid, note_id: existing_note.id, note: current_note)
-          else
-            gitlab.create_issue_note(iid: issue.iid, note: current_note)
-          end
+          dri = set_dri_via_group(test.product_group, test)
+          puts "  => Assigning #{dri} as DRI for the issue."
 
-          puts "  => Linked #{test.ci_job_url} to #{issue.web_url}."
+          gitlab.find_user_id(username: dri)
         end
 
-        def new_issue_title(test)
-          "Failure in #{super}"
+        def new_issue_due_date(test)
+          return unless test.product_group?
+
+          Date.today + 1.month
         end
 
-        def existing_failure_note(issue, content = "Failed most recently in")
-          gitlab.find_issue_notes(iid: issue.iid)&.find do |note|
-            note.body.include?(content)
-          end
+        def update_reports(issue, test)
+          gitlab.edit_issue(iid: issue.iid, options: {
+            description: up_to_date_issue_description(issue.description, test),
+            labels: up_to_date_labels(test: test, issue: issue)
+          })
+          puts "  => Added a report in '#{issue.title}': #{issue.web_url}!"
+        end
+
+        def up_to_date_issue_description(issue_description, test)
+          # We include the number of reports in the header, for visibility.
+          new_issue_description =
+            if issue_description.include?('### Reports')
+              # We count the number of existing reports.
+              reports_count = issue_description
+                .scan(REPORT_ITEM_REGEX)
+                .size.to_i + 1
+              issue_description.sub(/^### Reports.*$/, "### Reports (#{reports_count})")
+            else # For issue with the legacy format, we add the Reports section
+              reports_count = issue_description
+                .scan(JOB_URL_REGEX)
+                .size.to_i + 1
+
+              "#{issue_description}\n\n### Reports (#{reports_count})"
+            end
+
+          "#{new_issue_description}\n#{report_list_item(test)}"
+        end
+
+        def new_issue_title(test)
+          "Failure in #{super}"
         end
 
         def screenshot_section(test)
@@ -331,18 +370,7 @@ module GitlabQuality
           relative_url = gitlab.upload_file(file_fullpath: test.failure_screenshot)
           return unless relative_url
 
-          "### Screenshot: #{relative_url.markdown}"
-        end
-
-        def assign_dri(issue, test)
-          if test.product_group?
-            dri = set_dri_via_group(test.product_group, test)
-            dri_id = gitlab.find_user_id(username: dri)
-            gitlab.edit_issue(iid: issue.iid, options: { assignee_id: dri_id, due_date: Date.today + 1.month })
-            puts "  => Assigning #{dri} as DRI for the issue."
-          else
-            puts "  => No product group metadata found for test '#{test.name}'"
-          end
+          "### Screenshot\n\n#{relative_url.markdown}"
         end
 
         # Checks if a test failure should be reported.

data/lib/gitlab_quality/test_tooling/report/report_as_issue.rb

@@ -8,6 +8,8 @@ module GitlabQuality
       class ReportAsIssue
         include Concerns::Utils
 
+        FILE_BASE_URL = "https://gitlab.com/gitlab-org/gitlab/-/blob/master/"
+
         def initialize(token:, input_files:, project: nil, dry_run: false, **_kwargs)
           @project = project
           @gitlab = (dry_run ? GitlabIssueDryClient : GitlabIssueClient).new(token: token, project: project)
@@ -38,16 +40,31 @@ module GitlabQuality
 
           | Field | Value |
           | ------ | ------ |
-          | File | `#{test.file}` |
+          | File | #{test_file_link(test)} |
           | Description | `#{test.name}` |
           | Hash | `#{failed_test_hash(test)}` |
+          #{"| Test case | #{test.testcase} |" if test.testcase}
           DESCRIPTION
         end
 
+        def test_file_link(test)
+          path_prefix = test.file.start_with?('qa/') ? 'qa/' : ''
+
+          "[`#{path_prefix}#{test.file}`](#{FILE_BASE_URL}#{path_prefix}#{test.file})"
+        end
+
         def new_issue_labels(_test)
           []
         end
 
+        def new_issue_assignee_id(_test)
+          nil
+        end
+
+        def new_issue_due_date(_test)
+          nil
+        end
+
         def validate_input!
           assert_project!
           assert_input_files!(files)
@@ -67,14 +84,17 @@ module GitlabQuality
         end
 
         def create_issue(test)
-          issue = gitlab.create_issue(
+          attrs = {
             title: title_from_test(test),
             description: new_issue_description(test),
             labels: new_issue_labels(test).to_a,
-            issue_type: issue_type
-          )
+            issue_type: issue_type,
+            assignee_id: new_issue_assignee_id(test),
+            due_date: new_issue_due_date(test)
+          }.compact
+          issue = gitlab.create_issue(**attrs)
 
-          new_link = issue_type == 'test_case' ? issue.web_url.sub('/issues/', '/quality/test_cases/') : issue.web_url
+          new_link = issue_type == 'test_case' ? issue&.web_url&.sub('/issues/', '/quality/test_cases/') : issue&.web_url
 
           puts "Created new #{issue_type}: #{new_link}"
 
@@ -95,7 +115,7 @@ module GitlabQuality
 
         def up_to_date_labels(test:, issue: nil, new_labels: Set.new)
           labels = issue_labels(issue)
-          labels |= new_labels
+          labels |= new_labels.to_set
           ee_test?(test) ? labels << 'Enterprise Edition' : labels.delete('Enterprise Edition')
 
           if test.quarantine?

data/lib/gitlab_quality/test_tooling/test_results/test_result.rb

@@ -108,7 +108,11 @@ module GitlabQuality
           end
 
           def product_group
-            report['product_group'] if product_group?
+            report['product_group']
+          end
+
+          def feature_category
+            report['feature_category']
           end
 
           private

data/lib/gitlab_quality/test_tooling/version.rb

@@ -2,6 +2,6 @@
 
 module GitlabQuality
   module TestTooling
-    VERSION = "0.6.2"
+    VERSION = "0.7.0"
   end
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: gitlab_quality-test_tooling
 version: !ruby/object:Gem::Version
-  version: 0.6.2
+  version: 0.7.0
 platform: ruby
 authors:
 - GitLab Quality
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2023-06-06 00:00:00.000000000 Z
+date: 2023-06-09 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: climate_control
@@ -344,6 +344,7 @@ files:
 - lib/gitlab_quality/test_tooling.rb
 - lib/gitlab_quality/test_tooling/gitlab_issue_client.rb
 - lib/gitlab_quality/test_tooling/gitlab_issue_dry_client.rb
+- lib/gitlab_quality/test_tooling/labels_inference.rb
 - lib/gitlab_quality/test_tooling/report/concerns/find_set_dri.rb
 - lib/gitlab_quality/test_tooling/report/concerns/results_reporter.rb
 - lib/gitlab_quality/test_tooling/report/concerns/utils.rb