gitlab_quality-test_tooling 2.2.0 → 2.3.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 7437a045ff423ca3c6585be2aa13b11d97fd94abb5d8ffd95a14d11f48970b68
-  data.tar.gz: fd91b826d48c661a7b05f196857ec95c7fc59d478e8718ffa9ca5d534f38ffe5
+  metadata.gz: 349ae941a991eb8ad7c3c863cc0b840c84541d32ff742948bf06796be3b5263a
+  data.tar.gz: f23b333320ea5b242d7bda47f895fa6d56c8c63f2206657353749980a7470b6e
 SHA512:
-  metadata.gz: 4de31f3a187af7c92df5b015c631e5fa1278fa6a3d62f8111294ce07c245236b9e30e78a6760859eee4d024ee0de62dd544201e2d3e4beec73ef7be00f443def
-  data.tar.gz: 1d44821f07f2d61ac2d1b31b0837149bcf015070ffa1c4e6791d4685c3be322ab0991a28bf1551130eaf274045d58855a405b680948673f78aa208c9a34cab77
+  metadata.gz: 489db862072fc1f574e35de9f5eae7c8adc0909a5c1c76d3e7d30d9f0c8633b9af0e5c0b7e26cc80802bf2ca8afb599ed151133ce1fb13b2fbe719a01a9e9ede
+  data.tar.gz: 4cba020c783aabde06a85dc48bd68b95f97b5df9fbe0471ff85f7e1d7788912404a2e996a1a95b54e6d276fc0e1c48f7d9b6202fb6f3231f85e5a4bd0da85a29

data/Gemfile.lock

@@ -1,7 +1,7 @@
 PATH
   remote: .
   specs:
-    gitlab_quality-test_tooling (2.2.0)
+    gitlab_quality-test_tooling (2.3.0)
       activesupport (>= 7.0, < 7.2)
       amatch (~> 0.4.1)
       fog-google (~> 1.24, >= 1.24.1)

data/README.md

@@ -191,6 +191,16 @@ Usage: exe/existing-test-health-issue [options]
     -h, --help                       Show the usage
 ```
 
+### `exe/detect-infrastructure-failures`
+
+```shell
+Purpose: Checks wether a job failed on a known infrastructure error by parsing its trace.
+Usage: exe/detect-infrastructure-failures [options]
+   -j, --job-id JOB_ID              A valid Job ID
+   -p, --project PROJECT            Can be an integer or a group/project string
+   -t, --token TOKEN                A valid access token with `api` scope and Maintainer permission in PROJECT
+```
+
 ### `exe/flaky-test-issues`
 
 ```shell

data/exe/detect-infrastructure-failures

@@ -0,0 +1,31 @@
+#!/usr/bin/env ruby
+# frozen_string_literal: true
+
+require "bundler/setup"
+require "optparse"
+
+require_relative "../lib/gitlab_quality/test_tooling"
+
+params = {}
+
+OptionParser.new do |opts|
+  opts.banner = "Usage: #{$PROGRAM_NAME} [options]"
+
+  opts.on('-j', '--job-id JOB_ID', String, 'A valid job ID') do |job_id|
+    params[:job_id] = job_id
+  end
+
+  opts.on('-p', '--project PROJECT', String, 'Can be an integer or a group/project string') do |project|
+    params[:project] = project
+  end
+
+  opts.on('-t', '--token TOKEN', String, 'A valid access token with `api` scope and Maintainer permission in PROJECT') do |token|
+    params[:token] = token
+  end
+
+  opts.parse(ARGV)
+end
+
+raise ArgumentError, "Missing argument(s). Required arguments are: --job-id, --project, --token" if params.empty? || ([:job_id, :project, :token] - params.keys).any?
+
+exit GitlabQuality::TestTooling::JobTraceAnalyzer.new(**params).found_infrastructure_error? ? 0 : 1

data/lib/gitlab_quality/test_tooling/gitlab_client/job_client.rb

@@ -0,0 +1,29 @@
+# frozen_string_literal: true
+
+require 'gitlab'
+
+module GitlabQuality
+  module TestTooling
+    module GitlabClient
+      class JobClient < GitlabClient
+        attr_reader :job_id
+
+        def initialize(token:, project:, job_id:)
+          super
+
+          @job_id = job_id
+        end
+
+        def job_trace
+          trace = ''
+
+          ignore_gitlab_client_exceptions do
+            trace = client.job_trace(project, job_id)
+          end
+
+          trace
+        end
+      end
+    end
+  end
+end

data/lib/gitlab_quality/test_tooling/job_trace_analyzer.rb

@@ -0,0 +1,68 @@
+# frozen_string_literal: true
+
+require_relative 'gitlab_client/job_client'
+
+module GitlabQuality
+  module TestTooling
+    class JobTraceAnalyzer
+      attr_reader :project, :token, :job_id
+
+      TRANSIENT_ROOT_CAUSE_TO_TRACE_MAP =
+        {
+          failed_to_pull_image: ['job failed: failed to pull image'],
+          gitlab_com_overloaded: ['gitlab is currently unable to handle this request due to load'],
+          runner_disk_full: [
+            'no space left on device',
+            'Check free disk space'
+          ],
+          job_timeout: [
+            'ERROR: Job failed: execution took longer than',
+            'Rspec suite is exceeding the 80 minute limit and is forced to exit with error'
+          ],
+          gitaly: ['gitaly spawn failed'],
+          infrastructure: [
+            'the requested url returned error: 5', # any 5XX error code should be transient
+            'error: downloading artifacts from coordinator',
+            'error: uploading artifacts as "archive" to coordinator',
+            '500 Internal Server Error',
+            "Internal Server Error 500",
+            '502 Bad Gateway',
+            '503 Service Unavailable',
+            'Error: EEXIST: file already exists',
+            'Failed to connect to 127.0.0.1',
+            "Failed to open TCP connection to",
+            'connection reset by peer',
+            'segmentation fault',
+            'no space left on device',
+            'Check free disk space',
+            'CLUSTERDOWN'
+          ],
+          flaky_test: [
+            "We have detected a PG::QueryCanceled error in the specs, so we're failing early"
+          ]
+        }.freeze
+
+      def initialize(project:, token:, job_id:)
+        @project = project
+        @token = token
+        @job_id = job_id
+      end
+
+      def found_infrastructure_error?
+        TRANSIENT_ROOT_CAUSE_TO_TRACE_MAP[:infrastructure].any? do |search_string|
+          found = job_trace.downcase.include?(search_string.downcase)
+
+          puts "Found infrastructure error stacktrace: #{search_string}" if found
+
+          found
+        end
+      end
+
+      private
+
+      def job_trace
+        @job_trace ||= GitlabClient::JobClient.new(project: project, token: token, job_id: job_id).job_trace
+      end
+    end
+  end
+end

data/lib/gitlab_quality/test_tooling/version.rb

@@ -2,6 +2,6 @@
 
 module GitlabQuality
   module TestTooling
-    VERSION = "2.2.0"
+    VERSION = "2.3.0"
   end
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: gitlab_quality-test_tooling
 version: !ruby/object:Gem::Version
-  version: 2.2.0
+  version: 2.3.0
 platform: ruby
 authors:
 - GitLab Quality
 autorequire:
 bindir: exe
 cert_chain: []
-date: 2024-11-20 00:00:00.000000000 Z
+date: 2024-11-28 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: climate_control
@@ -414,6 +414,7 @@ description: A collection of test-related tools.
 email:
 - quality@gitlab.com
 executables:
+- detect-infrastructure-failures
 - existing-test-health-issue
 - failed-test-issues
 - flaky-test-issues
@@ -444,6 +445,7 @@ files:
 - LICENSE.txt
 - README.md
 - Rakefile
+- exe/detect-infrastructure-failures
 - exe/existing-test-health-issue
 - exe/failed-test-issues
 - exe/flaky-test-issues
@@ -468,10 +470,12 @@ files:
 - lib/gitlab_quality/test_tooling/gitlab_client/gitlab_client.rb
 - lib/gitlab_quality/test_tooling/gitlab_client/issues_client.rb
 - lib/gitlab_quality/test_tooling/gitlab_client/issues_dry_client.rb
+- lib/gitlab_quality/test_tooling/gitlab_client/job_client.rb
 - lib/gitlab_quality/test_tooling/gitlab_client/jobs_client.rb
 - lib/gitlab_quality/test_tooling/gitlab_client/merge_requests_client.rb
 - lib/gitlab_quality/test_tooling/gitlab_client/merge_requests_dry_client.rb
 - lib/gitlab_quality/test_tooling/gitlab_client/repository_files_client.rb
+- lib/gitlab_quality/test_tooling/job_trace_analyzer.rb
 - lib/gitlab_quality/test_tooling/knapsack_reports/spec_run_time.rb
 - lib/gitlab_quality/test_tooling/knapsack_reports/spec_run_time_report.rb
 - lib/gitlab_quality/test_tooling/labels_inference.rb