gitlab_quality-test_tooling 2.1.0 → 2.3.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 69b4478a8c7af86d2dae63c79d0924037feb673486f1ae6f15d5617121a0b9f4
-  data.tar.gz: 9a7f797d54687a515d834cfe0585bca7b87f5a28cd33000d5c06ebea292de56e
+  metadata.gz: 349ae941a991eb8ad7c3c863cc0b840c84541d32ff742948bf06796be3b5263a
+  data.tar.gz: f23b333320ea5b242d7bda47f895fa6d56c8c63f2206657353749980a7470b6e
 SHA512:
-  metadata.gz: 50206a9f212b2d78ab38b8433ffb5f22864e8a322b6065b6492a9023104e2caf0097042c8935c913f7d16de982fd3fb04a8438d294865f61caac84eee60d943e
-  data.tar.gz: 1267891667d8bf81617843d65ea6cd9bee2aef24007a2dc53ba92d970529f771dcbf8b8209aba7d59531f40ffac2e2c549444441dcae123c8e70bac1c0d417e7
+  metadata.gz: 489db862072fc1f574e35de9f5eae7c8adc0909a5c1c76d3e7d30d9f0c8633b9af0e5c0b7e26cc80802bf2ca8afb599ed151133ce1fb13b2fbe719a01a9e9ede
+  data.tar.gz: 4cba020c783aabde06a85dc48bd68b95f97b5df9fbe0471ff85f7e1d7788912404a2e996a1a95b54e6d276fc0e1c48f7d9b6202fb6f3231f85e5a4bd0da85a29

data/Gemfile.lock

@@ -1,7 +1,7 @@
 PATH
   remote: .
   specs:
-    gitlab_quality-test_tooling (2.1.0)
+    gitlab_quality-test_tooling (2.3.0)
       activesupport (>= 7.0, < 7.2)
       amatch (~> 0.4.1)
       fog-google (~> 1.24, >= 1.24.1)

data/README.md

@@ -191,6 +191,16 @@ Usage: exe/existing-test-health-issue [options]
     -h, --help                       Show the usage
 ```
 
+### `exe/detect-infrastructure-failures`
+
+```shell
+Purpose: Checks wether a job failed on a known infrastructure error by parsing its trace.
+Usage: exe/detect-infrastructure-failures [options]
+   -j, --job-id JOB_ID              A valid Job ID
+   -p, --project PROJECT            Can be an integer or a group/project string
+   -t, --token TOKEN                A valid access token with `api` scope and Maintainer permission in PROJECT
+```
+
 ### `exe/flaky-test-issues`
 
 ```shell

data/exe/detect-infrastructure-failures

@@ -0,0 +1,31 @@
+#!/usr/bin/env ruby
+# frozen_string_literal: true
+
+require "bundler/setup"
+require "optparse"
+
+require_relative "../lib/gitlab_quality/test_tooling"
+
+params = {}
+
+OptionParser.new do |opts|
+  opts.banner = "Usage: #{$PROGRAM_NAME} [options]"
+
+  opts.on('-j', '--job-id JOB_ID', String, 'A valid job ID') do |job_id|
+    params[:job_id] = job_id
+  end
+
+  opts.on('-p', '--project PROJECT', String, 'Can be an integer or a group/project string') do |project|
+    params[:project] = project
+  end
+
+  opts.on('-t', '--token TOKEN', String, 'A valid access token with `api` scope and Maintainer permission in PROJECT') do |token|
+    params[:token] = token
+  end
+
+  opts.parse(ARGV)
+end
+
+raise ArgumentError, "Missing argument(s). Required arguments are: --job-id, --project, --token" if params.empty? || ([:job_id, :project, :token] - params.keys).any?
+
+exit GitlabQuality::TestTooling::JobTraceAnalyzer.new(**params).found_infrastructure_error? ? 0 : 1

data/lib/gitlab_quality/test_tooling/gitlab_client/job_client.rb

@@ -0,0 +1,29 @@
+# frozen_string_literal: true
+
+require 'gitlab'
+
+module GitlabQuality
+  module TestTooling
+    module GitlabClient
+      class JobClient < GitlabClient
+        attr_reader :job_id
+
+        def initialize(token:, project:, job_id:)
+          super
+
+          @job_id = job_id
+        end
+
+        def job_trace
+          trace = ''
+
+          ignore_gitlab_client_exceptions do
+            trace = client.job_trace(project, job_id)
+          end
+
+          trace
+        end
+      end
+    end
+  end
+end

data/lib/gitlab_quality/test_tooling/job_trace_analyzer.rb

@@ -0,0 +1,68 @@
+# frozen_string_literal: true
+
+require_relative 'gitlab_client/job_client'
+
+module GitlabQuality
+  module TestTooling
+    class JobTraceAnalyzer
+      attr_reader :project, :token, :job_id
+
+      TRANSIENT_ROOT_CAUSE_TO_TRACE_MAP =
+        {
+          failed_to_pull_image: ['job failed: failed to pull image'],
+          gitlab_com_overloaded: ['gitlab is currently unable to handle this request due to load'],
+          runner_disk_full: [
+            'no space left on device',
+            'Check free disk space'
+          ],
+          job_timeout: [
+            'ERROR: Job failed: execution took longer than',
+            'Rspec suite is exceeding the 80 minute limit and is forced to exit with error'
+          ],
+          gitaly: ['gitaly spawn failed'],
+          infrastructure: [
+            'the requested url returned error: 5', # any 5XX error code should be transient
+            'error: downloading artifacts from coordinator',
+            'error: uploading artifacts as "archive" to coordinator',
+            '500 Internal Server Error',
+            "Internal Server Error 500",
+            '502 Bad Gateway',
+            '503 Service Unavailable',
+            'Error: EEXIST: file already exists',
+            'Failed to connect to 127.0.0.1',
+            "Failed to open TCP connection to",
+            'connection reset by peer',
+            'segmentation fault',
+            'no space left on device',
+            'Check free disk space',
+            'CLUSTERDOWN'
+          ],
+          flaky_test: [
+            "We have detected a PG::QueryCanceled error in the specs, so we're failing early"
+          ]
+        }.freeze
+
+      def initialize(project:, token:, job_id:)
+        @project = project
+        @token = token
+        @job_id = job_id
+      end
+
+      def found_infrastructure_error?
+        TRANSIENT_ROOT_CAUSE_TO_TRACE_MAP[:infrastructure].any? do |search_string|
+          found = job_trace.downcase.include?(search_string.downcase)
+
+          puts "Found infrastructure error stacktrace: #{search_string}" if found
+
+          found
+        end
+      end
+
+      private
+
+      def job_trace
+        @job_trace ||= GitlabClient::JobClient.new(project: project, token: token, job_id: job_id).job_trace
+      end
+    end
+  end
+end

data/lib/gitlab_quality/test_tooling/report/concerns/issue_reports.rb

@@ -11,13 +11,15 @@ module GitlabQuality
           FAILED_JOB_DESCRIPTION_REGEX = /First happened in #{JOB_URL_REGEX}\./m
           REPORT_ITEM_REGEX = /^1\. (?<report_date>\d{4}-\d{2}-\d{2}): #{JOB_URL_REGEX} \((?<pipeline_url>\S+)\) ?(?<extra_content>.*)$/
           LATEST_REPORTS_TO_SHOW = 10
+          DISPLAYED_HISTORY_REPORTS_THRESHOLD = 510 # https://gitlab.com/gitlab-org/quality/engineering-productivity/team/-/issues/587
           DAILY_REPORTS_THRESHOLDS = 10
 
           class ReportsList
-            def initialize(preserved_content:, section_header:, reports:, extra_content:)
+            def initialize(preserved_content:, section_header:, reports:, total_reports_count:, extra_content:)
               @preserved_content = preserved_content
               @section_header = section_header
               @reports = reports
+              @total_reports_count = total_reports_count
               @extra_content = extra_content
             end
 
@@ -31,14 +33,14 @@ module GitlabQuality
             end
 
             def reports_count
-              reports.size
+              total_reports_count
             end
 
             def to_s
               [
                 preserved_content,
                 "#{section_header} (#{reports_count})",
-                reports_list,
+                reports_list(total_reports_count),
                 extra_content
               ].reject(&:blank?).compact.join("\n\n")
             end
@@ -53,15 +55,16 @@ module GitlabQuality
 
             private
 
-            attr_reader :preserved_content, :section_header, :reports, :extra_content
+            attr_reader :preserved_content, :section_header, :reports, :total_reports_count, :extra_content
 
-            def reports_list
+            def reports_list(total_reports_count)
               if sorted_reports.size > LATEST_REPORTS_TO_SHOW
+                max_displayed_reports = DISPLAYED_HISTORY_REPORTS_THRESHOLD - LATEST_REPORTS_TO_SHOW
                 [
                   "Last #{LATEST_REPORTS_TO_SHOW} reports:",
-                  sorted_reports[...LATEST_REPORTS_TO_SHOW].join("\n"),
-                  "<details><summary>See #{sorted_reports.size - LATEST_REPORTS_TO_SHOW} more reports</summary>",
-                  sorted_reports[LATEST_REPORTS_TO_SHOW..].join("\n"),
+                  displayed_reports[...LATEST_REPORTS_TO_SHOW].join("\n"),
+                  "<details><summary>With #{total_reports_count - LATEST_REPORTS_TO_SHOW} more reports (displaying up to #{max_displayed_reports} reports) </summary>",
+                  displayed_reports[LATEST_REPORTS_TO_SHOW..].join("\n"),
                   "</details>"
                 ].join("\n\n")
               else
@@ -70,7 +73,12 @@ module GitlabQuality
             end
 
             def sorted_reports
-              @sorted_reports ||= reports.sort.reverse
+              @sorted_reports ||=
+                reports.sort_by { |report| [report.report_date, report.job_id, report.to_s] }.reverse
+            end
+
+            def displayed_reports
+              sorted_reports[0...DISPLAYED_HISTORY_REPORTS_THRESHOLD]
             end
           end
 
@@ -88,8 +96,8 @@ module GitlabQuality
               "1. #{report_date}: #{job_url} (#{pipeline_url}) #{extra_content}".strip
             end
 
-            def <=>(other)
-              to_s <=> other.to_s
+            def job_id
+              job_url.split('/').last
             end
 
             private
@@ -122,14 +130,33 @@ module GitlabQuality
             preserved_content = current_reports_content.split(reports_section_header).first&.strip
             reports = report_lines(current_reports_content) + [ReportsList.report_list_item(test, item_extra_content: item_extra_content)]
 
+            total_reports_count = increment_total_reports_count(
+              reports_section_header: reports_section_header,
+              content: current_reports_content,
+              reports: reports
+            )
+
             ReportsList.new(
               preserved_content: preserved_content,
               section_header: reports_section_header,
               reports: reports,
+              total_reports_count: total_reports_count,
               extra_content: reports_extra_content
             )
           end
 
+          def increment_total_reports_count(reports_section_header:, content:, reports:)
+            reports_count = reports.count
+
+            return reports_count if reports_count < DISPLAYED_HISTORY_REPORTS_THRESHOLD
+
+            count_match = content.match(/#{Regexp.escape(reports_section_header)} \((\d+)\)/)
+
+            return reports_count unless count_match
+
+            count_match[1].to_i + 1
+          end
+
           def failed_issue_job_url(issue)
             job_urls_from_description(issue.description, REPORT_ITEM_REGEX).last ||
               # Legacy format

data/lib/gitlab_quality/test_tooling/test_results/json_test_results.rb

@@ -16,9 +16,14 @@ module GitlabQuality
 
         def parse
           JSON.parse(File.read(path))
+        rescue JSON::ParserError
+          Runtime::Logger.debug("#{self.class.name}##{__method__} attempted to parse invalid JSON at path: #{path}")
+          {}
         end
 
         def process
+          return [] if results.empty?
+
           results['examples'].map do |test|
             GitlabQuality::TestTooling::TestResult::JsonTestResult.new(report: test, project: project, token: token)
           end

data/lib/gitlab_quality/test_tooling/version.rb

@@ -2,6 +2,6 @@
 
 module GitlabQuality
   module TestTooling
-    VERSION = "2.1.0"
+    VERSION = "2.3.0"
   end
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: gitlab_quality-test_tooling
 version: !ruby/object:Gem::Version
-  version: 2.1.0
+  version: 2.3.0
 platform: ruby
 authors:
 - GitLab Quality
 autorequire:
 bindir: exe
 cert_chain: []
-date: 2024-10-22 00:00:00.000000000 Z
+date: 2024-11-28 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: climate_control
@@ -414,6 +414,7 @@ description: A collection of test-related tools.
 email:
 - quality@gitlab.com
 executables:
+- detect-infrastructure-failures
 - existing-test-health-issue
 - failed-test-issues
 - flaky-test-issues
@@ -444,6 +445,7 @@ files:
 - LICENSE.txt
 - README.md
 - Rakefile
+- exe/detect-infrastructure-failures
 - exe/existing-test-health-issue
 - exe/failed-test-issues
 - exe/flaky-test-issues
@@ -468,10 +470,12 @@ files:
 - lib/gitlab_quality/test_tooling/gitlab_client/gitlab_client.rb
 - lib/gitlab_quality/test_tooling/gitlab_client/issues_client.rb
 - lib/gitlab_quality/test_tooling/gitlab_client/issues_dry_client.rb
+- lib/gitlab_quality/test_tooling/gitlab_client/job_client.rb
 - lib/gitlab_quality/test_tooling/gitlab_client/jobs_client.rb
 - lib/gitlab_quality/test_tooling/gitlab_client/merge_requests_client.rb
 - lib/gitlab_quality/test_tooling/gitlab_client/merge_requests_dry_client.rb
 - lib/gitlab_quality/test_tooling/gitlab_client/repository_files_client.rb
+- lib/gitlab_quality/test_tooling/job_trace_analyzer.rb
 - lib/gitlab_quality/test_tooling/knapsack_reports/spec_run_time.rb
 - lib/gitlab_quality/test_tooling/knapsack_reports/spec_run_time_report.rb
 - lib/gitlab_quality/test_tooling/labels_inference.rb