gitlab_quality-test_tooling 3.16.0 → 3.18.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 4c662ad8ff8b0b48b868eb201c12bd783c4a446bf0647d01e42778d59fd9f334
-  data.tar.gz: 3b26b7d715a0d55314e35194f490bef33aefc590356095bfb247e3f81cc2da5b
+  metadata.gz: 3770b676c5ff745323a47749eb04f305d435cff9769a9b567a1cc76289d5b2bb
+  data.tar.gz: 9cab37f5980a19ce80980dec4f97d867ca7bfe6e5524b383c71ea8d01509bbc8
 SHA512:
-  metadata.gz: b09d7935e13222a40f137f51a4ddc0164e3da841b9fc30eba901f8cead0f237a5ff1cc6c8df44a121e42d207fe0b1442e62ab3bc415f477a95a95ae05553c8c7
-  data.tar.gz: 2d9f33720064de7b9f8b78c00fafe7ff0b582e7554b7c7f7d89e690b080cf5e71aa515407e3825ce110f48221f3761816268841c594c823b2539576530a4e938
+  metadata.gz: 42a46bd31d9d7bdf985ad8d430c02e0d7037e116b426ff6358f089ae619f08680cf091758cbe6b5b229612be29e691a5ee760a57fb87b0da91324860b3939590
+  data.tar.gz: 71e1eda39f86d10c71d4d14770a925eaad6dc09d019c1cbcf7d769c6e23a73de79cce7fab02412b53ddfee8d4f6e0fa56fa55ae9880636491ffa9aa1024121ae

data/AGENTS.md

@@ -1,6 +1,6 @@
 # gitlab_quality-test_tooling
 
-Ruby gem providing test tooling for GitLab CI — metrics export, test quarantine, failure reporting, flaky test tracking, and code coverage analysis.
+Ruby gem providing test tooling for GitLab CI — test quarantine, failure reporting, flaky test tracking, and code coverage analysis.
 
 ## Tech Stack
 
@@ -70,11 +70,14 @@ Reference: https://docs.gitlab.com/ee/development/changelog.html
 
 ## Key Modules
 
-- **TestMetricsExporter** — Serializes RSpec results to ClickHouse (exception classes, failure messages, CI metadata)
-- **Report** — Creates/updates GitLab issues for test failures, flaky tests, slow tests
+- **Report** — Creates/updates GitLab issues for test failures, flaky tests, slow tests. `RelateFailureIssue` can also trigger a GitLab Duo Agent Platform workflow to investigate newly-created failure issues (see `--duo-analysis` on `exe/relate-failure-issue`); the agent posts its analysis back as an issue comment.
 - **TestQuarantine** — RSpec formatter that skips quarantined tests
 - **CodeCoverage** — Coverage analysis and responsibility patterns
-- **ClickHouse::Client** — Batch HTTP client for pushing metrics
+- **ClickHouse::Client** — HTTP query/insert client for ClickHouse, used by the code coverage subsystem
+
+## Review Guidance
+
+- `RelateFailureIssue#trigger_duo_analysis` deliberately rescues all `StandardError` (best-effort): a Duo/API failure must never block failure-issue creation. Preserve the broad rescue — don't narrow or remove it.
 
 ## Release Process
 

data/Gemfile.lock

@@ -1,7 +1,7 @@
 PATH
   remote: .
   specs:
-    gitlab_quality-test_tooling (3.16.0)
+    gitlab_quality-test_tooling (3.18.0)
       activesupport (>= 7.0)
       amatch (~> 0.4.1)
       fog-google (~> 1.24, >= 1.24.1)

data/README.md

@@ -93,6 +93,7 @@ Usage: exe/relate-failure-issue [options]
                                      Labels to exclude when searching for existing issues
         --confidential               Makes created new issues confidential
         --dry-run                    Perform a dry-run (don't create or update issues)
+        --duo-analysis MODES         Start a GitLab Duo workflow to investigate failure issues (comma-separated modes; supported: create). Off by default
     -v, --version                    Show the version
     -h, --help                       Show the usage
 ```
@@ -263,6 +264,12 @@ Usage: exe/feature-readiness-evaluation [options]
 
 ### `exe/test-coverage`
 
+Runs in one of three modes:
+
+- **Full coverage-metrics export** (default, no `--per-test-coverage` / `--run-aggregation`): reads the LCOV report, Crystalball test map, and test reports, then exports the coverage-metrics and test-file-mapping tables. Requires the full argument set.
+- **Per-test coverage** (`--per-test-coverage GLOB`): inserts per-test rows into `code_coverage.test_coverage_per_file` and runs the daily `test_health_risk` aggregation unless `--skip-aggregation`. Needs only the `--clickhouse-*` options; `--test-reports` and `--jest-quarantine-file` are optional enrichment.
+- **Aggregation only** (`--run-aggregation`): runs the daily `test_health_risk` aggregation and exits. Needs only the `--clickhouse-*` options. Pair with `--skip-aggregation` on batched per-test inserts to aggregate once at the end instead of once per batch.
+
 ```shell
 Purpose: Export test coverage metrics to ClickHouse
 Usage: exe/test-coverage [options]
@@ -280,6 +287,10 @@ Options:
                                      ClickHouse shared database name
         --responsibility-patterns PATH
                                      Path to YAML file with responsibility classification patterns
+        --per-test-coverage GLOB     Per-test coverage mode (see above). Glob for per-test coverage JSON/NDJSON files.
+        --jest-quarantine-file PATH  Optional. Path to quarantined_vue3_specs.txt, ingested before the aggregation.
+        --skip-aggregation           Optional. With --per-test-coverage, insert rows without running the aggregation.
+        --run-aggregation            Run only the daily test_health_risk aggregation, then exit.
 
 Environment variables:
   GLCI_CLICKHOUSE_METRICS_PASSWORD   ClickHouse password (required, not passed via CLI for security)

data/exe/relate-failure-issue

@@ -66,6 +66,16 @@ options = OptionParser.new do |opts|
     params[:environment_issues_output_file] = file
   end
 
+  opts.on('--duo-analysis MODES', Array,
+    'Start a GitLab Duo Agent Platform workflow to investigate failure issues (the agent posts its ' \
+      'analysis back as a comment). Comma-separated modes; currently supported: create. Off by default.') do |modes|
+    supported = %w[create]
+    invalid = modes - supported
+    raise OptionParser::InvalidArgument, "#{invalid.join(',')} (supported: #{supported.join(',')})" if invalid.any?
+
+    params[:duo_analysis] = modes
+  end
+
   opts.on_tail('-v', '--version', 'Show the version') do
     require_relative "../lib/gitlab_quality/test_tooling/version"
     puts "#{$PROGRAM_NAME} : #{GitlabQuality::TestTooling::VERSION}"

data/exe/test-coverage

@@ -21,12 +21,12 @@ require_relative '../lib/gitlab_quality/test_tooling/code_coverage/test_report'
 require_relative '../lib/gitlab_quality/test_tooling/code_coverage/test_map'
 require_relative '../lib/gitlab_quality/test_tooling/code_coverage/test_file_mapping_data'
 require_relative '../lib/gitlab_quality/test_tooling/code_coverage/per_test_coverage_data'
+require_relative '../lib/gitlab_quality/test_tooling/code_coverage/per_test_coverage_exporter'
 require_relative '../lib/gitlab_quality/test_tooling/code_coverage/source_file_classifier'
 require_relative '../lib/gitlab_quality/test_tooling/code_coverage/responsibility_classifier'
 require_relative '../lib/gitlab_quality/test_tooling/code_coverage/responsibility_patterns_config'
 
 params = {}
-required_params = [:test_reports, :coverage_report, :test_map, :clickhouse_url, :clickhouse_database, :clickhouse_username, :clickhouse_shared_database, :responsibility_patterns]
 
 options = OptionParser.new do |opts|
   opts.banner = "Usage: #{$PROGRAM_NAME} [options]"
@@ -68,19 +68,35 @@ options = OptionParser.new do |opts|
   end
 
   opts.on('--per-test-coverage GLOB',
-    'Optional. Glob pattern for per-test coverage JSON files. ' \
-      'When provided, populates code_coverage.test_coverage_per_file and runs the ' \
-      'daily test_health_risk aggregation. (e.g., "tmp/per-test-coverage-*.json")') do |pattern|
+    'Per-test coverage mode. Glob for per-test coverage JSON/NDJSON files. ' \
+      'Inserts rows into code_coverage.test_coverage_per_file and runs the daily ' \
+      'test_health_risk aggregation unless --skip-aggregation. Needs only the ' \
+      '--clickhouse-* options; --test-reports and --jest-quarantine-file are ' \
+      'optional enrichment. (e.g., "tmp/per-test-coverage-*.ndjson")') do |pattern|
     params[:per_test_coverage] = pattern
   end
 
   opts.on('--jest-quarantine-file PATH',
     'Optional. Path to quarantined_vue3_specs.txt. When provided alongside ' \
       '--per-test-coverage, populates code_coverage.jest_quarantined_tests_today ' \
-      'before the nightly aggregation so Jest tests count toward is_quarantined.') do |path|
+      'before the aggregation so Jest tests count toward is_quarantined.') do |path|
     params[:jest_quarantine_file] = path
   end
 
+  opts.on('--skip-aggregation',
+    'Optional. With --per-test-coverage, insert rows (and ingest the jest ' \
+      'quarantine file) without running the test_health_risk aggregation. Use ' \
+      'when streaming many batches, then run --run-aggregation once at the end.') do
+    params[:skip_aggregation] = true
+  end
+
+  opts.on('--run-aggregation',
+    'Run only the daily test_health_risk aggregation over ' \
+      'code_coverage.test_coverage_per_file, then exit. Needs only the ' \
+      '--clickhouse-* options.') do
+    params[:run_aggregation] = true
+  end
+
   opts.separator ""
   opts.separator "Environment variables:"
   opts.separator "  GLCI_CLICKHOUSE_METRICS_PASSWORD   ClickHouse password (required, not passed via CLI for security)"
@@ -102,31 +118,74 @@ options = OptionParser.new do |opts|
   opts.parse(ARGV)
 end
 
-if params.any? && (required_params - params.keys).none?
-  clickhouse_password = ENV.fetch('GLCI_CLICKHOUSE_METRICS_PASSWORD', nil)
-  if clickhouse_password.to_s.strip.empty?
-    puts "Error: GLCI_CLICKHOUSE_METRICS_PASSWORD environment variable must be set and not empty"
-    exit 1
+mode =
+  if params[:run_aggregation]
+    puts "Warning: --per-test-coverage is ignored in --run-aggregation mode." if params[:per_test_coverage]
+    :aggregation
+  elsif params[:per_test_coverage]
+    :per_test
+  else
+    :full_export
   end
 
-  [:clickhouse_url, :clickhouse_database, :clickhouse_username, :clickhouse_shared_database].each do |param|
-    if params[param].to_s.strip.empty?
-      puts "Error: --#{param.to_s.tr('_', '-')} cannot be empty"
-      exit 1
-    end
-  end
+# Per-test and aggregation-only modes are self-contained: they touch only the
+# ClickHouse connection, not the LCOV report, Crystalball test map, or
+# responsibility patterns the full coverage-metrics export needs.
+clickhouse_required = [:clickhouse_url, :clickhouse_database, :clickhouse_username]
+full_export_required =
+  [:test_reports, :coverage_report, :test_map, :responsibility_patterns, :clickhouse_shared_database] + clickhouse_required
+required_params = mode == :full_export ? full_export_required : clickhouse_required
 
-  begin
-    uri = URI.parse(params[:clickhouse_url])
-    unless uri.is_a?(URI::HTTP) || uri.is_a?(URI::HTTPS)
-      puts "Error: --clickhouse-url must be a valid HTTP or HTTPS URL"
-      exit 1
-    end
-  rescue URI::InvalidURIError
-    puts "Error: --clickhouse-url is not a valid URL format"
+unless params.any? && (required_params - params.keys).none?
+  puts "Missing argument(s). Required arguments are: #{required_params}\nPassed arguments are: #{params}\n"
+  puts options
+  exit 1
+end
+
+clickhouse_password = ENV.fetch('GLCI_CLICKHOUSE_METRICS_PASSWORD', nil)
+if clickhouse_password.to_s.strip.empty?
+  puts "Error: GLCI_CLICKHOUSE_METRICS_PASSWORD environment variable must be set and not empty"
+  exit 1
+end
+
+non_empty_params = mode == :full_export ? clickhouse_required + [:clickhouse_shared_database] : clickhouse_required
+non_empty_params.each do |param|
+  next unless params[param].to_s.strip.empty?
+
+  puts "Error: --#{param.to_s.tr('_', '-')} cannot be empty"
+  exit 1
+end
+
+begin
+  uri = URI.parse(params[:clickhouse_url])
+  unless uri.is_a?(URI::HTTP) || uri.is_a?(URI::HTTPS)
+    puts "Error: --clickhouse-url must be a valid HTTP or HTTPS URL"
     exit 1
   end
+rescue URI::InvalidURIError
+  puts "Error: --clickhouse-url is not a valid URL format"
+  exit 1
+end
 
+clickhouse_data = {
+  url: params[:clickhouse_url],
+  database: params[:clickhouse_database],
+  username: params[:clickhouse_username],
+  password: clickhouse_password
+}
+
+case mode
+when :aggregation
+  GitlabQuality::TestTooling::CodeCoverage::ClickHouse::TestHealthRiskAggregator.new(**clickhouse_data).run
+when :per_test
+  GitlabQuality::TestTooling::CodeCoverage::PerTestCoverageExporter.new(
+    coverage_glob: params[:per_test_coverage],
+    clickhouse: clickhouse_data,
+    test_reports: params[:test_reports],
+    jest_quarantine_file: params[:jest_quarantine_file],
+    skip_aggregation: params.fetch(:skip_aggregation, false)
+  ).run
+when :full_export
   artifacts = GitlabQuality::TestTooling::CodeCoverage::Artifacts.new(
     coverage_report: params[:coverage_report],
     test_map: params[:test_map],
@@ -181,13 +240,6 @@ if params.any? && (required_params - params.keys).none?
     test_classifications
   )
 
-  clickhouse_data = {
-    url: params[:clickhouse_url],
-    database: params[:clickhouse_database],
-    username: params[:clickhouse_username],
-    password: clickhouse_password
-  }
-
   shared_clickhouse_data = {
     url: params[:clickhouse_url],
     database: params[:clickhouse_shared_database],
@@ -210,36 +262,4 @@ if params.any? && (required_params - params.keys).none?
   )
   test_file_mappings_table = GitlabQuality::TestTooling::CodeCoverage::ClickHouse::TestFileMappingsTable.new(**shared_clickhouse_data)
   test_file_mappings_table.push(test_file_mapping_data.as_db_table)
-
-  # Per-test coverage export (optional). Only runs when --per-test-coverage
-  # was provided AND at least one matching artifact exists.
-  if params[:per_test_coverage]
-    per_test_files = Dir.glob(params[:per_test_coverage])
-    if per_test_files.any?
-      per_test_data = GitlabQuality::TestTooling::CodeCoverage::PerTestCoverageData.new(
-        per_test_files,
-        tests_to_categories: tests_to_categories,
-        feature_categories_to_teams: category_owners.feature_categories_to_teams,
-        captured_sha: ENV.fetch('CI_COMMIT_SHA', '')
-      )
-      per_test_coverage_table = GitlabQuality::TestTooling::CodeCoverage::ClickHouse::PerTestCoverageTable.new(**clickhouse_data)
-      per_test_coverage_table.push(per_test_data.as_db_table)
-
-      if params[:jest_quarantine_file] && File.exist?(params[:jest_quarantine_file])
-        jest_qt = GitlabQuality::TestTooling::CodeCoverage::ClickHouse::JestQuarantinedTestsTable.new(**clickhouse_data)
-        jest_qt.populate(quarantine_file: params[:jest_quarantine_file])
-      elsif params[:jest_quarantine_file]
-        puts "Jest quarantine file not found at #{params[:jest_quarantine_file]}; skipping jest quarantine ingestion."
-      end
-
-      aggregator = GitlabQuality::TestTooling::CodeCoverage::ClickHouse::TestHealthRiskAggregator.new(**clickhouse_data)
-      aggregator.run
-    else
-      puts "No per-test coverage artifacts matched #{params[:per_test_coverage]}; skipping per-test export and aggregation."
-    end
-  end
-else
-  puts "Missing argument(s). Required arguments are: #{required_params}\nPassed arguments are: #{params}\n"
-  puts options
-  exit 1
 end

data/lib/gitlab_quality/test_tooling/code_coverage/per_test_coverage_exporter.rb

@@ -0,0 +1,127 @@
+# frozen_string_literal: true
+
+require 'json'
+require 'logger'
+
+module GitlabQuality
+  module TestTooling
+    module CodeCoverage
+      # Standalone per-test coverage export: inserts per-test, per-source-file
+      # rows into `code_coverage.test_coverage_per_file` and, unless skipped,
+      # runs the daily `test_health_risk` aggregation.
+      #
+      # Unlike the full coverage-metrics export, this path needs only ClickHouse
+      # credentials and the per-test coverage glob, not the LCOV report, the
+      # Crystalball test map, or the responsibility patterns. Category enrichment
+      # (feature_category/group/stage/section per row) is optional: pass
+      # `test_reports` to map test files to feature categories via the test
+      # report JSON, otherwise those columns stay blank.
+      #
+      # `skip_aggregation` lets a batched caller insert without re-running the
+      # table-wide aggregation on every batch. The streaming export invokes this
+      # once per artifact batch with `skip_aggregation: true`, then runs the
+      # aggregation once at the end.
+      class PerTestCoverageExporter
+        # @param clickhouse [Hash] connection params (:url, :database, :username, :password)
+        def initialize(
+          coverage_glob:, clickhouse:,
+          test_reports: nil, jest_quarantine_file: nil,
+          captured_sha: ENV.fetch('CI_COMMIT_SHA', ''), skip_aggregation: false, logger: nil)
+          @coverage_glob = coverage_glob
+          @clickhouse = clickhouse
+          @test_reports = test_reports
+          @jest_quarantine_file = jest_quarantine_file
+          @captured_sha = captured_sha.to_s
+          @skip_aggregation = skip_aggregation
+          @logger = logger || ::Logger.new($stdout)
+        end
+
+        # @return [void]
+        def run
+          coverage_files = Dir.glob(coverage_glob)
+          if coverage_files.empty?
+            logger.info(
+              "#{ClickHouse::LOG_PREFIX} No per-test coverage artifacts matched #{coverage_glob}; nothing to export."
+            )
+            return
+          end
+
+          insert(coverage_files)
+          ingest_jest_quarantine
+          aggregate unless skip_aggregation
+        end
+
+        private
+
+        attr_reader :coverage_glob, :clickhouse, :test_reports, :jest_quarantine_file,
+          :captured_sha, :skip_aggregation, :logger
+
+        def insert(coverage_files)
+          tests_to_categories, feature_categories_to_teams = resolve_categories
+
+          data = PerTestCoverageData.new(
+            coverage_files,
+            tests_to_categories: tests_to_categories,
+            feature_categories_to_teams: feature_categories_to_teams,
+            captured_sha: captured_sha
+          )
+
+          ClickHouse::PerTestCoverageTable.new(**clickhouse_credentials).push(data.as_db_table)
+        end
+
+        def ingest_jest_quarantine
+          return unless jest_quarantine_file
+
+          unless File.exist?(jest_quarantine_file)
+            logger.info(
+              "#{ClickHouse::LOG_PREFIX} Jest quarantine file not found at #{jest_quarantine_file}; " \
+                "skipping jest quarantine ingestion."
+            )
+            return
+          end
+
+          ClickHouse::JestQuarantinedTestsTable.new(**clickhouse_credentials)
+            .populate(quarantine_file: jest_quarantine_file)
+        end
+
+        def aggregate
+          ClickHouse::TestHealthRiskAggregator.new(**clickhouse_credentials).run
+        end
+
+        # `tests_to_categories` comes from the test report JSON; without it, rows
+        # carry blank categories. `CategoryOwners` fetches stages.yml over HTTP,
+        # so only build it when there are reports to enrich.
+        def resolve_categories
+          return [{}, {}] unless test_reports
+
+          # Comma-separated patterns, mirroring Artifacts#test_reports.
+          patterns = test_reports.split(',').map(&:strip).reject(&:empty?)
+          report_files = Dir.glob(patterns)
+          return [{}, {}] if report_files.empty?
+
+          [tests_to_categories_from(report_files), CategoryOwners.new.feature_categories_to_teams]
+        end
+
+        def tests_to_categories_from(report_files)
+          report_files.each_with_object({}) do |file, combined|
+            # Category enrichment is optional, so a malformed or vanished report
+            # shouldn't abort the per-test insert (the rows are the point). Warn
+            # and skip the file, leaving those tests with blank categories.
+            begin
+              report = TestReport.new(JSON.parse(File.read(file)))
+            rescue JSON::ParserError, Errno::ENOENT => e
+              logger.warn("#{ClickHouse::LOG_PREFIX} Skipping unreadable test report #{file}: #{e.message}")
+              next
+            end
+
+            combined.merge!(report.tests_to_categories) { |_, old_val, new_val| (old_val + new_val).uniq }
+          end
+        end
+
+        def clickhouse_credentials
+          clickhouse.merge(logger: logger)
+        end
+      end
+    end
+  end
+end

data/lib/gitlab_quality/test_tooling/gitlab_client/issues_client.rb

@@ -43,6 +43,8 @@ module GitlabQuality
       # The GitLab client is used for API access: https://github.com/NARKOZ/gitlab
       class IssuesClient < GitlabClient
         REPORTER_ACCESS_LEVEL = 20
+        DUO_WORKFLOW_TIMEOUT = 30
+        DUO_WORKFLOW_DEFINITION = 'developer/v1'
 
         def assert_user_permission!
           handle_gitlab_client_exceptions do
@@ -123,6 +125,28 @@ module GitlabQuality
           end
         end
 
+        # Starts a GitLab Duo Agent Platform workflow (Flows API).
+        # The agent investigates and posts its own analysis back, so no follow-up call is needed.
+        #
+        # This is a best-effort helper: it deliberately does NOT use handle_gitlab_client_exceptions
+        # (which would retry with long backoffs and post failures to Slack) and uses a short timeout
+        # so a slow or unavailable Duo endpoint can never stall the reporting job. The caller is
+        # expected to rescue any error.
+        #
+        # project_id accepts either a numeric ID or a namespace path (e.g. 'gitlab-org/quality/...'),
+        # matching the GitLab API's project-identifier convention; the path form is verified working.
+        def start_duo_workflow(goal:, workflow_definition: DUO_WORKFLOW_DEFINITION, project_id: project)
+          client.post('/ai/duo_workflows/workflows',
+            timeout: DUO_WORKFLOW_TIMEOUT,
+            headers: { 'Content-Type' => 'application/json' },
+            body: {
+              project_id: project_id,
+              goal: goal,
+              workflow_definition: workflow_definition,
+              start_workflow: true
+            }.to_json)
+        end
+
         def edit_issue_note(issue_iid:, note_id:, note:)
           handle_gitlab_client_exceptions do
             client.edit_issue_note(project, issue_iid, note_id, note)

data/lib/gitlab_quality/test_tooling/report/relate_failure_issue.rb

@@ -83,12 +83,13 @@ module GitlabQuality
           @metrics_files = Array(metrics_files)
           @group_similar = group_similar
           @environment_issues_output_file = environment_issues_output_file
+          @duo_analysis_modes = Array(kwargs[:duo_analysis])
         end
 
         private
 
         attr_reader :max_diff_ratio, :system_logs, :base_issue_labels, :exclude_labels_for_search, :metrics_files, :ops_gitlab_client, :group_similar,
-          :environment_issues_output_file
+          :environment_issues_output_file, :duo_analysis_modes
 
         def run!
           puts "Reporting test failures in `#{files.join(',')}` as issues in project `#{project}` via the API at `#{Runtime::Env.gitlab_api_base}`."
@@ -294,9 +295,47 @@ module GitlabQuality
           # On a dry run, created_issue may not be populated
           test.failure_issue ||= created_issue&.web_url
 
+          trigger_duo_analysis(created_issue, test) if duo_analysis_modes.include?('create')
+
           created_issue
         end
 
+        # Best-effort: start a GitLab Duo workflow to investigate the issue; the agent comments its
+        # analysis back. A failure here must never block issue creation or the pipeline.
+        def trigger_duo_analysis(issue, test)
+          return unless issue&.web_url
+
+          gitlab.start_duo_workflow(goal: duo_analysis_goal(issue, test))
+          puts "  => Triggered Duo failure analysis workflow for #{issue.web_url}"
+        rescue StandardError => e
+          warn("  => Could not trigger Duo failure analysis for #{issue.web_url} (ignored): #{e.class}: #{e.message}")
+        end
+
+        def duo_analysis_goal(issue, test)
+          <<~GOAL.chomp
+            Investigate the newly-reported end-to-end test failure described in this issue: #{issue.web_url}
+            (failing spec: `#{test.name}`). Read the issue for the failing spec and stack trace.
+
+            Investigate the `gitlab-org/gitlab` project (where the product code and E2E specs live —
+            note this is NOT the project hosting this issue): inspect its recent commits and merge
+            requests on the default branch using GitLab tools, and consult the Orbit knowledge graph
+            via MCP tools if it is available. Using those, identify the most likely cause of this
+            failure. Then post a comment on that issue with your analysis.
+
+            The failure may be caused by a product code change, or it may be environmental — for
+            example a flaky test, an infrastructure/environment issue, or a CI runner problem. Decide
+            which is more likely:
+
+            - If a specific change or merge request looks responsible, name the likely author in your
+              comment and ask them to confirm. Frame this as advisory ("this looks likely related —
+              please confirm"), never as an assertion of blame.
+            - If the failure looks environmental, flaky, or otherwise not attributable to a specific
+              change, explain why — but do not ping or attribute it to anyone.
+
+            This is an automated, best-effort request to speed up triage of newly-detected failures.
+          GOAL
+        end
+
         def pipeline_issues_with_similar_stacktrace(test)
           search_labels = (base_issue_labels + Set.new(%w[test failure::new])).to_a
           not_labels = exclude_labels_for_search.to_a
@@ -428,6 +467,10 @@ module GitlabQuality
         end
 
         def generate_diff_link
+          # Outside CI (e.g. local runs) there is no pipeline to diff against, so skip gracefully
+          # rather than crashing on a nil pipeline URL or an unsupported project.
+          return "No commit diff available (no CI pipeline context)." unless Runtime::Env.ci_pipeline_url
+
           initialize_gitlab_ops_client
 
           if Runtime::Env.ci_pipeline_url.include?('ops.gitlab.net')

data/lib/gitlab_quality/test_tooling/version.rb

@@ -2,6 +2,6 @@
 
 module GitlabQuality
   module TestTooling
-    VERSION = "3.16.0"
+    VERSION = "3.18.0"
   end
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: gitlab_quality-test_tooling
 version: !ruby/object:Gem::Version
-  version: 3.16.0
+  version: 3.18.0
 platform: ruby
 authors:
 - GitLab Quality
 autorequire:
 bindir: exe
 cert_chain: []
-date: 2026-05-26 00:00:00.000000000 Z
+date: 2026-06-15 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: climate_control
@@ -502,6 +502,7 @@ files:
 - lib/gitlab_quality/test_tooling/code_coverage/coverage_data.rb
 - lib/gitlab_quality/test_tooling/code_coverage/lcov_file.rb
 - lib/gitlab_quality/test_tooling/code_coverage/per_test_coverage_data.rb
+- lib/gitlab_quality/test_tooling/code_coverage/per_test_coverage_exporter.rb
 - lib/gitlab_quality/test_tooling/code_coverage/responsibility_classifier.rb
 - lib/gitlab_quality/test_tooling/code_coverage/responsibility_patterns_config.rb
 - lib/gitlab_quality/test_tooling/code_coverage/rspec_report.rb
@@ -594,11 +595,6 @@ files:
 - lib/gitlab_quality/test_tooling/test_meta/test_meta_updater.rb
 - lib/gitlab_quality/test_tooling/test_metric/json_test_metric.rb
 - lib/gitlab_quality/test_tooling/test_metrics/json_test_metric_collection.rb
-- lib/gitlab_quality/test_tooling/test_metrics_exporter/client.rb
-- lib/gitlab_quality/test_tooling/test_metrics_exporter/config.rb
-- lib/gitlab_quality/test_tooling/test_metrics_exporter/config_helper.rb
-- lib/gitlab_quality/test_tooling/test_metrics_exporter/formatter.rb
-- lib/gitlab_quality/test_tooling/test_metrics_exporter/test_metrics.rb
 - lib/gitlab_quality/test_tooling/test_quarantine/quarantine_formatter.rb
 - lib/gitlab_quality/test_tooling/test_quarantine/quarantine_helper.rb
 - lib/gitlab_quality/test_tooling/test_result/base_test_result.rb

data/lib/gitlab_quality/test_tooling/test_metrics_exporter/client.rb

@@ -1,50 +0,0 @@
-# frozen_string_literal: true
-
-require "httparty"
-require "json"
-
-module GitlabQuality
-  module TestTooling
-    module TestMetricsExporter
-      class Client
-        ResponseError = Class.new(StandardError)
-
-        TESTS_PATH = "/api/v1/tests"
-        # Observer enforces a per-request cap; batch above this size to avoid silent failures.
-        MAX_BATCH_SIZE = 10_000
-
-        def initialize(url:, token:)
-          @url = url
-          @token = token
-        end
-
-        # POST array of test metric records to the observer service.
-        # Wraps each batch as { "tests" => [...] } and splits oversized payloads
-        # into chunks of at most MAX_BATCH_SIZE records.
-        #
-        # @param tests [Array<Hash>]
-        # @return [Boolean] true when every batch succeeds (or input is empty)
-        # @raise [ResponseError] on the first non-2xx batch response
-        def post_tests(tests)
-          tests.each_slice(MAX_BATCH_SIZE) do |batch|
-            response = HTTParty.post(
-              "#{url.to_s.chomp('/')}#{TESTS_PATH}",
-              body: { tests: batch }.to_json,
-              headers: {
-                "X-Gitlab-Token" => token,
-                "Content-Type" => "application/json"
-              }
-            )
-            raise ResponseError, "Observer request failed with status #{response.code}: #{response.body}" unless response.success?
-          end
-
-          true
-        end
-
-        private
-
-        attr_reader :url, :token
-      end
-    end
-  end
-end

data/lib/gitlab_quality/test_tooling/test_metrics_exporter/config.rb

@@ -1,88 +0,0 @@
-# frozen_string_literal: true
-
-require "logger"
-require "singleton"
-
-module GitlabQuality
-  module TestTooling
-    module TestMetricsExporter
-      class Config
-        include Singleton
-
-        class << self
-          def configuration
-            Config.instance
-          end
-
-          def configure
-            yield(configuration)
-          end
-        end
-
-        attr_accessor :run_type,
-          :observer_url,
-          :observer_token
-        attr_writer :extra_rspec_metadata_keys,
-          :skip_record_proc,
-          :test_retried_proc,
-          :custom_metrics_proc,
-          :spec_file_path_prefix,
-          :logger
-
-        # Whether observer export is configured
-        #
-        # Export is considered enabled when all required attributes are set
-        #
-        # @return [Boolean]
-        def observer_configured?
-          [observer_url, observer_token].none? { |value| value.nil? || value.to_s.empty? }
-        end
-
-        # Extra rspec metadata keys to include in exported metrics
-        #
-        # @return [Array<Symbol>]
-        def extra_rspec_metadata_keys
-          @extra_rspec_metadata_keys ||= []
-        end
-
-        # Extra path prefix for constructing full file path within mono-repository setups
-        #
-        # @return [String]
-        def spec_file_path_prefix
-          @spec_file_path_prefix ||= ""
-        end
-
-        # A lambda that determines whether to skip recording a test result
-        #
-        # This is useful when you would want to skip initial failure when retrying specs is set up in a separate process
-        # and you want to avoid duplicate records
-        #
-        # @return [Proc]
-        def skip_record_proc
-          @skip_record_proc ||= ->(_example) { false }
-        end
-
-        # A lambda that determines whether a test was retried or not
-        #
-        # @return [Proc]
-        def test_retried_proc
-          @test_retried_proc ||= ->(_example) { false }
-        end
-
-        # A lambda that return hash with additional custom metrics
-        #
-        # @return [Proc]
-        def custom_metrics_proc
-          @custom_metrics_proc ||= ->(_example) { {} }
-        end
-
-        # Logger instance
-        #
-        # @return [Logger]
-        def logger
-          @logger ||= Logger.new($stdout, level: Logger::INFO)
-        end
-      end
-    end
-  end
-end

data/lib/gitlab_quality/test_tooling/test_metrics_exporter/config_helper.rb

@@ -1,115 +0,0 @@
-# frozen_string_literal: true
-
-require 'logger'
-require 'active_support/core_ext/object/blank'
-
-require_relative 'config'
-require_relative 'formatter'
-
-module GitlabQuality
-  module TestTooling
-    module TestMetricsExporter
-      class ConfigHelper
-        REQUIRED_OBSERVER_ENV_VARS = %w[
-          GLCI_OBSERVER_URL
-          GLCI_OBSERVER_AUTH_TOKEN
-        ].freeze
-
-        class << self
-          def configure!(run_type = test_run_type)
-            return unless ENV.fetch("CI", nil) && ENV.fetch("GLCI_EXPORT_TEST_METRICS", "true") == "true" && run_type
-
-            RSpec.configure do |rspec_config|
-              next if rspec_config.dry_run?
-
-              Config.configure do |exporter_config|
-                self.logger = exporter_config.logger
-                next warn_missing_observer_variables unless observer_env_vars_present?
-
-                yield(exporter_config) if block_given?
-                configure_exporter!(exporter_config, run_type)
-
-                rspec_config.add_formatter Formatter
-
-                logger.info("Test metrics export is enabled for run type: #{run_type}")
-              end
-            end
-          end
-
-          private
-
-          attr_writer :logger
-
-          def logger
-            @logger ||= Logger.new($stdout)
-          end
-
-          def observer_env_vars_present?
-            REQUIRED_OBSERVER_ENV_VARS.all? { |var| ENV.fetch(var, nil) && !ENV[var].empty? }
-          end
-
-          def configure_exporter!(config, run_type)
-            config.run_type = run_type
-            config.custom_metrics_proc = custom_metrics_proc
-
-            configure_observer!(config)
-          end
-
-          def configure_observer!(config)
-            config.observer_url = observer_url
-            config.observer_token = observer_token
-          end
-
-          def warn_missing_observer_variables
-            missing = REQUIRED_OBSERVER_ENV_VARS.reject { |var| ENV.fetch(var, nil) && !ENV[var].empty? }
-            logger.warn("Test metrics export is enabled but missing environment variables: #{missing.join(', ')}")
-          end
-
-          def custom_metrics_proc
-            proc do |_example|
-              { pipeline_type: pipeline_type, ci_pipeline_id: ci_pipeline_id }
-            end
-          end
-
-          def default_branch?
-            ENV["CI_COMMIT_REF_NAME"] == ENV["CI_DEFAULT_BRANCH"]
-          end
-
-          def pipeline_type
-            @pipeline_type ||= if default_branch? && ENV["SCHEDULE_TYPE"].present?
-                                 "default_branch_scheduled_pipeline"
-                               elsif default_branch?
-                                 "default_branch_pipeline"
-                               elsif ENV["CI_COMMIT_REF_NAME"]&.match?(/^[\d-]+-stable-ee$/)
-                                 "stable_branch_pipeline"
-                               elsif ENV["CI_MERGE_REQUEST_TARGET_BRANCH_NAME"]&.match?(/^[\d-]+-stable-ee$/)
-                                 "backport_merge_request_pipeline"
-                               elsif ENV["CI_MERGE_REQUEST_IID"].present?
-                                 "merge_request_pipeline"
-                               elsif ENV["CI_PIPELINE_SOURCE"] == "pipeline"
-                                 "downstream_pipeline"
-                               else
-                                 "unknown"
-                               end
-          end
-
-          def test_run_type
-            @run_type ||= ENV.fetch("GLCI_TEST_METRICS_RUN_TYPE", nil)
-          end
-
-          def observer_url
-            ENV.fetch("GLCI_OBSERVER_URL", nil)
-          end
-
-          def observer_token
-            ENV.fetch("GLCI_OBSERVER_AUTH_TOKEN", nil)
-          end
-
-          def ci_pipeline_id
-            (ENV["PARENT_PIPELINE_ID"] || ENV.fetch("CI_PIPELINE_ID", nil)).to_i
-          end
-        end
-      end
-    end
-  end
-end

data/lib/gitlab_quality/test_tooling/test_metrics_exporter/formatter.rb

@@ -1,67 +0,0 @@
-# frozen_string_literal: true
-
-require "rspec/core/formatters/base_formatter"
-
-require_relative "test_metrics"
-require_relative "client"
-
-module GitlabQuality
-  module TestTooling
-    module TestMetricsExporter
-      class Formatter < RSpec::Core::Formatters::BaseFormatter
-        RSpec::Core::Formatters.register(self, :stop)
-
-        LOG_PREFIX = "[MetricsExporter]"
-
-        def stop(notification)
-          logger.debug("#{LOG_PREFIX} Starting test metrics export")
-          data = notification.examples.filter_map do |example|
-            next if config.skip_record_proc.call(example)
-
-            TestMetrics.new(example, time).data
-          end
-          return logger.warn("#{LOG_PREFIX} No test execution records found, metrics will not be exported!") if data.empty?
-
-          push_to_observer(data)
-        end
-
-        private
-
-        def config
-          Config.configuration
-        end
-
-        def logger
-          config.logger
-        end
-
-        # Single common timestamp for all exported example metrics to keep data points consistently grouped
-        #
-        # @return [String]
-        def time
-          return @time if @time
-
-          ci_created_at = ENV.fetch("CI_PIPELINE_CREATED_AT", nil)
-          @time = (ci_created_at ? Time.strptime(ci_created_at, '%Y-%m-%dT%H:%M:%S%z') : Time.now.utc).strftime('%Y-%m-%dT%H:%M:%S.%6N')
-        end
-
-        # Push data to observer service
-        #
-        # @param data [Array<Hash>]
-        # @return [void]
-        def push_to_observer(data)
-          return logger.debug("#{LOG_PREFIX} Observer configuration missing, skipping export!") unless config.observer_configured?
-
-          observer_client.post_tests(data)
-          logger.info("#{LOG_PREFIX} Successfully pushed #{data.size} entries to Observer!")
-        rescue StandardError => e
-          logger.error("#{LOG_PREFIX} Error occurred while pushing metrics to Observer: #{e.message}")
-        end
-
-        def observer_client
-          Client.new(url: config.observer_url, token: config.observer_token)
-        end
-      end
-    end
-  end
-end

data/lib/gitlab_quality/test_tooling/test_metrics_exporter/test_metrics.rb

@@ -1,228 +0,0 @@
-# frozen_string_literal: true
-
-require 'time'
-
-module GitlabQuality
-  module TestTooling
-    module TestMetricsExporter
-      class TestMetrics
-        def initialize(example, timestamp)
-          @example = example
-          @timestamp = timestamp
-        end
-
-        # Test data hash
-        #
-        # @return [Hash]
-        def data
-          {
-            timestamp: timestamp,
-            **rspec_metrics,
-            **ci_metrics,
-            **custom_metrics
-          }.compact
-        end
-
-        private
-
-        attr_reader :example, :timestamp
-
-        # Exporter configuration
-        #
-        # @return [Config]
-        def config
-          Config.configuration
-        end
-
-        # Rspec related metrics
-        #
-        # @return [Hash]
-        def rspec_metrics # rubocop:disable Metrics/AbcSize
-          {
-            id: without_relative_path(example.id),
-            name: example.full_description,
-            hash: OpenSSL::Digest.hexdigest("SHA256", "#{file_path}#{example.full_description}")[..40],
-            file_path: file_path,
-            status: example.execution_result.status,
-            run_time: (example.execution_result.run_time * 1000).round,
-            location: example_location,
-            # TODO: remove exception_class once migration to exception_classes is fully complete on clickhouse side
-            exception_class: example.execution_result.exception&.class&.to_s,
-            exception_classes: exception_classes.map { |e| e.class.to_s }.uniq,
-            failure_exception: failure_exception,
-            quarantined: quarantined?,
-            quarantine_issue_url: quarantine_issue_url || "",
-            feature_category: example.metadata[:feature_category] || "",
-            test_retried: config.test_retried_proc.call(example),
-            run_type: run_type,
-            spec_file_path_prefix: config.spec_file_path_prefix
-          }
-        end
-
-        # CI related metrics
-        #
-        # @return [Hash]
-        def ci_metrics
-          {
-            ci_project_id: env("CI_PROJECT_ID")&.to_i,
-            ci_project_path: env("CI_PROJECT_PATH"),
-            ci_job_name: ci_job_name,
-            ci_job_id: env('CI_JOB_ID')&.to_i,
-            ci_pipeline_id: env('CI_PIPELINE_ID')&.to_i,
-            ci_merge_request_iid: (env('CI_MERGE_REQUEST_IID') || env('TOP_UPSTREAM_MERGE_REQUEST_IID'))&.to_i,
-            ci_branch: env("CI_COMMIT_REF_NAME"),
-            ci_target_branch: env("CI_MERGE_REQUEST_TARGET_BRANCH_NAME"),
-            ci_server_url: env("CI_SERVER_URL")
-          }
-        end
-
-        # Additional custom metrics
-        #
-        # @return [Hash]
-        def custom_metrics
-          metrics = example.metadata
-            .slice(*config.extra_rspec_metadata_keys)
-            .merge(config.custom_metrics_proc.call(example))
-
-          metrics.each_with_object({}) do |(k, value), custom_metrics|
-            custom_metrics[k.to_sym] = metrics_value(value)
-          end
-        end
-
-        # Checks if spec is quarantined
-        #
-        # @return [String]
-        def quarantined?
-          return false unless example.metadata.key?(:quarantine)
-
-          # if quarantine key is present and status is pending, consider it quarantined
-          example.execution_result.status == :pending
-        end
-
-        # Extract quarantine issue URL from metadata
-        #
-        # @return [String, nil]
-        def quarantine_issue_url
-          return nil unless example.metadata.key?(:quarantine)
-
-          metadata = example.metadata[:quarantine]
-          case metadata
-          when String
-            # Direct URL: quarantine: 'https://gitlab.com/.../issues/123'
-            metadata if metadata.start_with?('http')
-          when Hash
-            # Hash format: quarantine: { issue: 'https://...', reason: '...' }
-            issue = metadata[:issue] || metadata['issue']
-            # Handle array of URLs (take the first one)
-            issue.is_a?(Array) ? issue.first : issue
-          end
-        end
-
-        # Base ci job name
-        #
-        # @return [String]
-        def ci_job_name
-          env("CI_JOB_NAME")&.gsub(%r{ \d{1,2}/\d{1,2}}, '')
-        end
-
-        # Example location
-        #
-        # @return [String]
-        def example_location
-          return @example_location if @example_location
-
-          # ensures that location will be correct even in case of shared examples
-          file = example
-                 .metadata
-                 .fetch(:shared_group_inclusion_backtrace)
-                 .last
-                 &.formatted_inclusion_location
-
-          return without_relative_path(example.location) unless file
-
-          @example_location = without_relative_path(file)
-        end
-
-        # File path based on actual test location, not shared example location
-        #
-        # @return [String]
-        def file_path
-          @file_path ||= example_location.gsub(/:\d+$/, "")
-        end
-
-        # Failure exception classes
-        #
-        # @return [Array<Exception>]
-        def exception_classes
-          exception = example.execution_result.exception
-          return [] unless exception
-          return [exception] unless exception.respond_to?(:all_exceptions)
-
-          exception.all_exceptions.flatten
-        end
-
-        # Truncated exception message
-        #
-        # For MultipleExceptionError, returns the first wrapped exception's message
-        # instead of the unhelpful wrapper class name.
-        #
-        # @return [String]
-        def failure_exception
-          exception = example.execution_result.exception
-          return unless exception
-
-          source = if exception.respond_to?(:all_exceptions)
-                     exception.all_exceptions.flatten.first || exception
-                   else
-                     exception
-                   end
-
-          source.to_s.tr("\n", " ").slice(0, 1000)
-        end
-
-        # Test run type | suite name
-        #
-        # @return [String]
-        def run_type
-          config.run_type || ci_job_name || "unknown"
-        end
-
-        # Return non empty environment variable value
-        #
-        # @param [String] name
-        # @return [String, nil]
-        def env(name)
-          return unless ENV[name] && !ENV[name].empty?
-
-          ENV.fetch(name)
-        end
-
-        # Metrics value cast to a valid type
-        #
-        # @param value [Object]
-        # @return [Object]
-        def metrics_value(value)
-          return value if value.is_a?(Numeric) || value.is_a?(String) || bool?(value) || value.nil?
-
-          value.to_s
-        end
-
-        # Value is a true or false
-        #
-        # @param val [Object]
-        # @return [Boolean]
-        def bool?(val)
-          [true, false].include?(val)
-        end
-
-        # Path without leading ./
-        #
-        # @param path [String]
-        # @return [String]
-        def without_relative_path(path)
-          path.gsub(%r{^\./}, "")
-        end
-      end
-    end
-  end
-end