gitlab_quality-test_tooling 2.2.0 → 2.4.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 7437a045ff423ca3c6585be2aa13b11d97fd94abb5d8ffd95a14d11f48970b68
-  data.tar.gz: fd91b826d48c661a7b05f196857ec95c7fc59d478e8718ffa9ca5d534f38ffe5
+  metadata.gz: 8d991bf33f977c242a64aeaef438f73085f6aeaf7575674ace10a36ea62484be
+  data.tar.gz: 7e2b37d11e14b21d2acb9b8bcd49094331d0fd06e799cf041a494413295367d3
 SHA512:
-  metadata.gz: 4de31f3a187af7c92df5b015c631e5fa1278fa6a3d62f8111294ce07c245236b9e30e78a6760859eee4d024ee0de62dd544201e2d3e4beec73ef7be00f443def
-  data.tar.gz: 1d44821f07f2d61ac2d1b31b0837149bcf015070ffa1c4e6791d4685c3be322ab0991a28bf1551130eaf274045d58855a405b680948673f78aa208c9a34cab77
+  metadata.gz: d73508a586cacef8d59d4e40d39ae198d61f22b900f0995705ddf5753467b23676eb71adb90b08e4a9064aa94f250edb78665dd371edef1e735f36e57ccbc547
+  data.tar.gz: f858ca470f52e7b5dfe86e49016f5f45ab19132c3946ea5f3d9c445ca739da336bfa5ee0d36fab79d88b8d6d26af5392812cef400a75b52cd9dc9a3081d06627

data/Gemfile.lock

@@ -1,7 +1,7 @@
 PATH
   remote: .
   specs:
-    gitlab_quality-test_tooling (2.2.0)
+    gitlab_quality-test_tooling (2.4.0)
       activesupport (>= 7.0, < 7.2)
       amatch (~> 0.4.1)
       fog-google (~> 1.24, >= 1.24.1)

data/README.md

@@ -191,6 +191,16 @@ Usage: exe/existing-test-health-issue [options]
     -h, --help                       Show the usage
 ```
 
+### `exe/detect-infrastructure-failures`
+
+```shell
+Purpose: Checks wether a job failed on a known infrastructure error by parsing its trace.
+Usage: exe/detect-infrastructure-failures [options]
+   -j, --job-id JOB_ID              A valid Job ID
+   -p, --project PROJECT            Can be an integer or a group/project string
+   -t, --token TOKEN                A valid access token with `api` scope and Maintainer permission in PROJECT
+```
+
 ### `exe/flaky-test-issues`
 
 ```shell

data/exe/detect-infrastructure-failures

@@ -0,0 +1,31 @@
+#!/usr/bin/env ruby
+# frozen_string_literal: true
+
+require "bundler/setup"
+require "optparse"
+
+require_relative "../lib/gitlab_quality/test_tooling"
+
+params = {}
+
+OptionParser.new do |opts|
+  opts.banner = "Usage: #{$PROGRAM_NAME} [options]"
+
+  opts.on('-j', '--job-id JOB_ID', String, 'A valid job ID') do |job_id|
+    params[:job_id] = job_id
+  end
+
+  opts.on('-p', '--project PROJECT', String, 'Can be an integer or a group/project string') do |project|
+    params[:project] = project
+  end
+
+  opts.on('-t', '--token TOKEN', String, 'A valid access token with `api` scope and Maintainer permission in PROJECT') do |token|
+    params[:token] = token
+  end
+
+  opts.parse(ARGV)
+end
+
+raise ArgumentError, "Missing argument(s). Required arguments are: --job-id, --project, --token" if params.empty? || ([:job_id, :project, :token] - params.keys).any?
+
+exit GitlabQuality::TestTooling::JobTraceAnalyzer.new(**params).found_infrastructure_error? ? 0 : 1

data/lib/gitlab_quality/test_tooling/gitlab_client/job_client.rb

@@ -0,0 +1,29 @@
+# frozen_string_literal: true
+
+require 'gitlab'
+
+module GitlabQuality
+  module TestTooling
+    module GitlabClient
+      class JobClient < GitlabClient
+        attr_reader :job_id
+
+        def initialize(token:, project:, job_id:)
+          super
+
+          @job_id = job_id
+        end
+
+        def job_trace
+          trace = ''
+
+          ignore_gitlab_client_exceptions do
+            trace = client.job_trace(project, job_id)
+          end
+
+          trace
+        end
+      end
+    end
+  end
+end

data/lib/gitlab_quality/test_tooling/job_trace_analyzer.rb

@@ -0,0 +1,130 @@
+# frozen_string_literal: true
+
+require_relative 'gitlab_client/job_client'
+
+module GitlabQuality
+  module TestTooling
+    class JobTraceAnalyzer
+      attr_reader :project, :token, :job_id
+
+      FailureTraceDefinition = Struct.new(:type, :trace_start, :trace_end, :language, :label, keyword_init: true)
+      FAILURE_TRACE_DEFINITIONS = [
+        FailureTraceDefinition.new(
+          type: :rspec,
+          trace_start: "Failures:\n",
+          trace_end: "[TEST PROF INFO]",
+          language: :ruby,
+          label: '~backend'
+        ),
+        FailureTraceDefinition.new(
+          type: :jest,
+          trace_start: "Summary of all failing tests\n",
+          trace_end: "\nRan all test suites.",
+          language: :javascript,
+          label: '~frontend'
+        ),
+        FailureTraceDefinition.new(
+          type: :workhorse,
+          trace_start: "make: Entering directory '/builds/gitlab-org/gitlab/workhorse'",
+          trace_end: "make: Leaving directory '/builds/gitlab-org/gitlab/workhorse'",
+          language: :go,
+          label: '~workhorse'
+        ),
+        FailureTraceDefinition.new(
+          type: :rubocop,
+          trace_start: "Running RuboCop in graceful mode:",
+          trace_end: "section_end",
+          language: :ruby,
+          label: '~backend'
+        )
+      ].freeze
+
+      TRANSIENT_ROOT_CAUSE_TO_TRACE_MAP =
+        {
+          failed_to_pull_image: ['job failed: failed to pull image'],
+          gitlab_com_overloaded: ['gitlab is currently unable to handle this request due to load'],
+          runner_disk_full: [
+            'no space left on device',
+            'Check free disk space'
+          ],
+          job_timeout: [
+            'ERROR: Job failed: execution took longer than',
+            'Rspec suite is exceeding the 80 minute limit and is forced to exit with error'
+          ],
+          gitaly: ['gitaly spawn failed'],
+          infrastructure: [
+            'the requested url returned error: 5', # any 5XX error code should be transient
+            'error: downloading artifacts from coordinator',
+            'error: uploading artifacts as "archive" to coordinator',
+            '500 Internal Server Error',
+            "Internal Server Error 500",
+            '502 Bad Gateway',
+            '503 Service Unavailable',
+            'Error: EEXIST: file already exists',
+            'Failed to connect to 127.0.0.1',
+            "Failed to open TCP connection to",
+            'connection reset by peer',
+            'segmentation fault',
+            'no space left on device',
+            'Check free disk space',
+            'CLUSTERDOWN'
+          ],
+          flaky_test: [
+            "We have detected a PG::QueryCanceled error in the specs, so we're failing early"
+          ]
+        }.freeze
+
+      AFTER_SCRIPT_TRACE_START_MARKER = 'Running after_script'
+
+      def initialize(project:, token:, job_id:)
+        @project = project
+        @token = token
+        @job_id = job_id
+      end
+
+      def found_infrastructure_error?
+        trace_to_search = failure_summary || main_trace
+
+        TRANSIENT_ROOT_CAUSE_TO_TRACE_MAP[:infrastructure].any? do |search_string|
+          found = trace_to_search.downcase.include?(search_string.downcase)
+
+          puts "Found infrastructure error stacktrace: #{search_string}" if found
+
+          found
+        end
+      end
+
+      private
+
+      def detected_failure_trace_definition
+        return @detected_failure_trace_definition if defined?(@detected_failure_trace_definition)
+
+        @detected_failure_trace_definition = FAILURE_TRACE_DEFINITIONS.find do |failure_trace_definition|
+          job_trace.include?(failure_trace_definition.trace_start) &&
+            job_trace.include?(failure_trace_definition.trace_end)
+        end
+      end
+
+      def job_trace
+        @job_trace ||= GitlabClient::JobClient.new(project: project, token: token, job_id: job_id).job_trace
+      end
+
+      def main_trace
+        return job_trace unless job_trace.include?(AFTER_SCRIPT_TRACE_START_MARKER)
+
+        job_trace.split(AFTER_SCRIPT_TRACE_START_MARKER).first
+      end
+
+      def failure_summary
+        return unless detected_failure_trace_definition
+
+        @failure_summary ||= main_trace
+          .split(detected_failure_trace_definition.trace_start)
+          .last
+          .split(detected_failure_trace_definition.trace_end)
+          .first
+          .chomp
+      end
+    end
+  end
+end

data/lib/gitlab_quality/test_tooling/version.rb

@@ -2,6 +2,6 @@
 
 module GitlabQuality
   module TestTooling
-    VERSION = "2.2.0"
+    VERSION = "2.4.0"
   end
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: gitlab_quality-test_tooling
 version: !ruby/object:Gem::Version
-  version: 2.2.0
+  version: 2.4.0
 platform: ruby
 authors:
 - GitLab Quality
 autorequire:
 bindir: exe
 cert_chain: []
-date: 2024-11-20 00:00:00.000000000 Z
+date: 2024-12-05 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: climate_control
@@ -414,6 +414,7 @@ description: A collection of test-related tools.
 email:
 - quality@gitlab.com
 executables:
+- detect-infrastructure-failures
 - existing-test-health-issue
 - failed-test-issues
 - flaky-test-issues
@@ -444,6 +445,7 @@ files:
 - LICENSE.txt
 - README.md
 - Rakefile
+- exe/detect-infrastructure-failures
 - exe/existing-test-health-issue
 - exe/failed-test-issues
 - exe/flaky-test-issues
@@ -468,10 +470,12 @@ files:
 - lib/gitlab_quality/test_tooling/gitlab_client/gitlab_client.rb
 - lib/gitlab_quality/test_tooling/gitlab_client/issues_client.rb
 - lib/gitlab_quality/test_tooling/gitlab_client/issues_dry_client.rb
+- lib/gitlab_quality/test_tooling/gitlab_client/job_client.rb
 - lib/gitlab_quality/test_tooling/gitlab_client/jobs_client.rb
 - lib/gitlab_quality/test_tooling/gitlab_client/merge_requests_client.rb
 - lib/gitlab_quality/test_tooling/gitlab_client/merge_requests_dry_client.rb
 - lib/gitlab_quality/test_tooling/gitlab_client/repository_files_client.rb
+- lib/gitlab_quality/test_tooling/job_trace_analyzer.rb
 - lib/gitlab_quality/test_tooling/knapsack_reports/spec_run_time.rb
 - lib/gitlab_quality/test_tooling/knapsack_reports/spec_run_time_report.rb
 - lib/gitlab_quality/test_tooling/labels_inference.rb