gitlab-exporter 16.8.0 → 16.10.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 8b320ac3c29627e5bca0700ab6007fb3681a286679032bccae4595af29e764fc
-  data.tar.gz: ab52c7b430a87959228fdb4483c2bf3a26f9838c983a5292dd27bb6b72123a8d
+  metadata.gz: 3a3b7fa869eab272c352626dfd0f8acc2cbcc3fb2df46da92ea0387471282d2b
+  data.tar.gz: fc2f8a0a213b2a97a92ed5f417a5856f181c9f422cbbc4640c9cce45d86e4fcf
 SHA512:
-  metadata.gz: dbd696eb2e666142c275e46aff04eb9beda6739863fe505249f0074d1d8d1145fe6b41573e0f3521feee2bcde7dd2ad33e8fc76b1abc3f75e34e23d95d94258f
-  data.tar.gz: 2d426ce702f2d360b5aa8cc5ae51f4924295e717615c6da3adbd9ef1e46c241ed40e151082918fa7c963c7367c857730e1fafb41ffc328bc6b3d14879615abcf
+  metadata.gz: 2f6603c161ae6c9346d5b3ecd919f4df54bceb8232e588db181d1142eedc2e90231039b15a357ceed1ae62ae735b7540567e2717bbacadcea5e83ddd5b366fb2
+  data.tar.gz: 760580b43d2290a933842a4ffbd4182fe60660b401d03e9df6a7c8d85f6c3363054124b9aba2fd5a4983fb1989a11f2df28cf2f3cbdf7ca67e653bde4be95bf7

data/Gemfile.lock

@@ -1,12 +1,12 @@
 PATH
   remote: .
   specs:
-    gitlab-exporter (16.8.0)
+    gitlab-exporter (16.10.0)
       connection_pool (= 2.5.5)
       deep_merge (~> 1.2.2)
-      faraday (= 2.14.1)
+      faraday (= 2.14.3)
       pg (= 1.6.3)
-      puma (= 8.0.1)
+      puma (= 8.0.2)
       quantile (= 0.2.1)
       redis (= 4.8.1)
       redis-namespace (= 1.11.0)
@@ -22,7 +22,7 @@ GEM
     connection_pool (2.5.5)
     deep_merge (1.2.2)
     diff-lcs (1.5.0)
-    faraday (2.14.1)
+    faraday (2.14.3)
       faraday-net_http (>= 2.0, < 3.5)
       json
       logger
@@ -40,7 +40,7 @@ GEM
       racc
     pg (1.6.3)
     prism (1.9.0)
-    puma (8.0.1)
+    puma (8.0.2)
       nio4r (~> 2.0)
     quantile (0.2.1)
     racc (1.8.1)

data/gitlab-exporter.gemspec

@@ -24,9 +24,9 @@ Gem::Specification.new do |s|
 
   s.add_runtime_dependency "connection_pool", "2.5.5"
   s.add_runtime_dependency "deep_merge", "~> 1.2.2"
-  s.add_runtime_dependency "faraday", "2.14.1"
+  s.add_runtime_dependency "faraday", "2.14.3"
   s.add_runtime_dependency "pg", "1.6.3"
-  s.add_runtime_dependency "puma", "8.0.1"
+  s.add_runtime_dependency "puma", "8.0.2"
   s.add_runtime_dependency "quantile", "0.2.1"
   s.add_runtime_dependency "redis", "4.8.1"
   s.add_runtime_dependency "redis-namespace", "1.11.0"

data/lib/gitlab_exporter/database/bloat_btree.sql

@@ -1,101 +1,147 @@
 -- Originally from: https://github.com/ioguix/pgsql-bloat-estimation/blob/master/btree/btree_bloat.sql
--- WARNING: executed with a non-superuser role, the query inspect only index on tables you are granted to read.
+--
+-- Rewritten to use CROSS JOIN LATERAL stages so this query stays in sync with the
+-- `postgres_index_bloat_estimate(p_schema, p_idxname)` function used by the main GitLab
+-- application's reindexing pipeline:
+-- https://gitlab.com/gitlab-org/gitlab/-/blob/master/db/migrate/20260526120000_add_postgres_index_bloat_estimate_function.rb
+--
+-- Expanding the index columns inside a LATERAL join (instead of putting a set-returning
+-- function such as generate_series in the SELECT list) avoids the ProjectSet barrier that
+-- prevents the planner from pushing predicates down. See:
+-- https://gitlab.com/gitlab-org/gitlab/-/work_items/569363
+-- https://gitlab.com/gitlab-org/gitlab/-/work_items/599210
+--
+-- WARNING: executed with a non-superuser role, the query inspects only indexes on tables you are granted to read.
 -- WARNING: rows with is_na = 't' are known to have bad statistics ("name" type is not supported).
--- This query is compatible with PostgreSQL 8.2 and after
-SELECT current_database(), nspname AS schemaname, tblname, idxname AS object_name, bs*(relpages)::bigint AS real_size,
-  bs*(relpages-est_pages)::bigint AS extra_size,
-  100 * (relpages-est_pages)::float / relpages AS extra_ratio,
-  fillfactor,
-  CASE WHEN relpages > est_pages_ff
-    THEN bs*(relpages-est_pages_ff)
+-- This query is compatible with PostgreSQL 9.3 and after (LATERAL joins).
+--
+-- Note: unlike the previous ioguix-style query, this form does NOT need to materialize
+-- pg_stats to stay fast on PostgreSQL 17. The old query joined pg_stats once as a single
+-- large join that the PG17 planner badly underestimated (see f18ac4b); here pg_stats is
+-- looked up per index inside the LATERAL, resolving through the pg_statistic indexes, so
+-- there is no single join for the planner to misestimate.
+SELECT
+  current_database(),
+  n.nspname AS schemaname,
+  ct.relname AS tblname,
+  ci.relname AS object_name,
+  c.bs * ci.relpages::bigint AS real_size,
+  c.bs * (ci.relpages - bloat.est_pages)::bigint AS extra_size,
+  100 * (ci.relpages - bloat.est_pages)::float / ci.relpages AS extra_ratio,
+  c.fillfactor,
+  CASE WHEN ci.relpages > bloat.est_pages_ff
+    THEN c.bs * (ci.relpages - bloat.est_pages_ff)
     ELSE 0
   END AS bloat_size,
-  100 * (relpages-est_pages_ff)::float / relpages AS bloat_ratio,
-  is_na
-  -- , 100-(pst).avg_leaf_density AS pst_avg_bloat, est_pages, index_tuple_hdr_bm, maxalign, pagehdr, nulldatawidth, nulldatahdrwidth, reltuples, relpages -- (DEBUG INFO)
-FROM (
-  SELECT coalesce(1 +
-         ceil(reltuples/floor((bs-pageopqdata-pagehdr)/(4+nulldatahdrwidth)::float)), 0 -- ItemIdData size + computed avg size of a tuple (nulldatahdrwidth)
-      ) AS est_pages,
-      coalesce(1 +
-         ceil(reltuples/floor((bs-pageopqdata-pagehdr)*fillfactor/(100*(4+nulldatahdrwidth)::float))), 0
-      ) AS est_pages_ff,
-      bs, nspname, tblname, idxname, relpages, fillfactor, is_na
-      -- , pgstatindex(idxoid) AS pst, index_tuple_hdr_bm, maxalign, pagehdr, nulldatawidth, nulldatahdrwidth, reltuples -- (DEBUG INFO)
-  FROM (
-      SELECT maxalign, bs, nspname, tblname, idxname, reltuples, relpages, idxoid, fillfactor,
-            ( index_tuple_hdr_bm +
-                maxalign - CASE -- Add padding to the index tuple header to align on MAXALIGN
-                  WHEN index_tuple_hdr_bm%maxalign = 0 THEN maxalign
-                  ELSE index_tuple_hdr_bm%maxalign
-                END
-              + nulldatawidth + maxalign - CASE -- Add padding to the data to align on MAXALIGN
-                  WHEN nulldatawidth = 0 THEN 0
-                  WHEN nulldatawidth::integer%maxalign = 0 THEN maxalign
-                  ELSE nulldatawidth::integer%maxalign
-                END
-            )::numeric AS nulldatahdrwidth, pagehdr, pageopqdata, is_na
-            -- , index_tuple_hdr_bm, nulldatawidth -- (DEBUG INFO)
-      FROM (
-          SELECT n.nspname, i.tblname, i.idxname, i.reltuples, i.relpages,
-              i.idxoid, i.fillfactor, current_setting('block_size')::numeric AS bs,
-              CASE -- MAXALIGN: 4 on 32bits, 8 on 64bits (and mingw32 ?)
-                WHEN version() ~ 'mingw32' OR version() ~ '64-bit|x86_64|ppc64|ia64|amd64' THEN 8
-                ELSE 4
-              END AS maxalign,
-              /* per page header, fixed size: 20 for 7.X, 24 for others */
-              24 AS pagehdr,
-              /* per page btree opaque data */
-              16 AS pageopqdata,
-              /* per tuple header: add IndexAttributeBitMapData if some cols are null-able */
-              CASE WHEN max(coalesce(s.null_frac,0)) = 0
-                  THEN 2 -- IndexTupleData size
-                  ELSE 2 + (( 32 + 8 - 1 ) / 8) -- IndexTupleData size + IndexAttributeBitMapData size ( max num filed per index + 8 - 1 /8)
-              END AS index_tuple_hdr_bm,
-              /* data len: we remove null values save space using it fractionnal part from stats */
-              sum( (1-coalesce(s.null_frac, 0)) * coalesce(s.avg_width, 1024)) AS nulldatawidth,
-              max( CASE WHEN i.atttypid = 'pg_catalog.name'::regtype THEN 1 ELSE 0 END ) > 0 AS is_na
-          FROM (
-              SELECT ct.relname AS tblname, ct.relnamespace, ic.idxname, ic.attpos, ic.indkey, ic.indkey[ic.attpos], ic.reltuples, ic.relpages, ic.tbloid, ic.idxoid, ic.fillfactor,
-                  coalesce(a1.attnum, a2.attnum) AS attnum, coalesce(a1.attname, a2.attname) AS attname, coalesce(a1.atttypid, a2.atttypid) AS atttypid,
-                  CASE WHEN a1.attnum IS NULL
-                  THEN ic.idxname
-                  ELSE ct.relname
-                  END AS attrelname
-              FROM (
-                  SELECT idxname, reltuples, relpages, tbloid, idxoid, fillfactor, indkey,
-                      pg_catalog.generate_series(1,indnatts) AS attpos
-                  FROM (
-                      SELECT ci.relname AS idxname, ci.reltuples, ci.relpages, i.indrelid AS tbloid,
-                          i.indexrelid AS idxoid,
-                          coalesce(substring(
-                              array_to_string(ci.reloptions, ' ')
-                              from 'fillfactor=([0-9]+)')::smallint, 90) AS fillfactor,
-                          i.indnatts,
-                          pg_catalog.string_to_array(pg_catalog.textin(
-                              pg_catalog.int2vectorout(i.indkey)),' ')::int[] AS indkey
-                      FROM pg_catalog.pg_index i
-                      JOIN pg_catalog.pg_class ci ON ci.oid = i.indexrelid
-                      WHERE ci.relam=(SELECT oid FROM pg_am WHERE amname = 'btree')
-                      AND ci.relpages > 0
-                  ) AS idx_data
-              ) AS ic
-              JOIN pg_catalog.pg_class ct ON ct.oid = ic.tbloid
-              LEFT JOIN pg_catalog.pg_attribute a1 ON
-                  ic.indkey[ic.attpos] <> 0
-                  AND a1.attrelid = ic.tbloid
-                  AND a1.attnum = ic.indkey[ic.attpos]
-              LEFT JOIN pg_catalog.pg_attribute a2 ON
-                  ic.indkey[ic.attpos] = 0
-                  AND a2.attrelid = ic.idxoid
-                  AND a2.attnum = ic.attpos
-            ) i
-            JOIN pg_catalog.pg_namespace n ON n.oid = i.relnamespace
-            JOIN pg_catalog.pg_stats s ON s.schemaname = n.nspname
-                                      AND s.tablename = i.attrelname
-                                      AND s.attname = i.attname
-            GROUP BY 1,2,3,4,5,6,7,8,9,10,11
-      ) AS rows_data_stats
-  ) AS rows_hdr_pdg_stats
-) AS relation_stats
-ORDER BY nspname, tblname, idxname;
+  100 * (ci.relpages - bloat.est_pages_ff)::float / ci.relpages AS bloat_ratio,
+  bloat.is_na
+FROM pg_catalog.pg_class ci
+JOIN pg_catalog.pg_index i     ON i.indexrelid = ci.oid
+JOIN pg_catalog.pg_class ct    ON ct.oid = i.indrelid
+JOIN pg_catalog.pg_namespace n ON n.oid = ct.relnamespace
+
+-- Per-index system constants and fillfactor.
+CROSS JOIN LATERAL (
+    SELECT
+        current_setting('block_size')::numeric AS bs,
+        CASE -- MAXALIGN: 4 on 32bits, 8 on 64bits (and mingw32 ?)
+            WHEN version() ~ 'mingw32'::text
+              OR version() ~ '64-bit|x86_64|ppc64|ia64|amd64'::text THEN 8
+            ELSE 4
+        END AS maxalign,
+        24 AS pagehdr,     -- per page header, fixed size: 24 for 8.3 and after
+        16 AS pageopqdata, -- per page btree opaque data
+        COALESCE(
+            substring(array_to_string(ci.reloptions, ' '::text) FROM 'fillfactor=([0-9]+)')::smallint::integer,
+            90
+        ) AS fillfactor
+) c
+
+-- Aggregate the per-column statistics for this index, expanding the index columns
+-- through a LATERAL generate_series rather than a SELECT-list set-returning function.
+CROSS JOIN LATERAL (
+    SELECT
+        max(CASE WHEN COALESCE(a1.atttypid, a2.atttypid) = 'pg_catalog.name'::regtype THEN 1 ELSE 0 END) > 0 AS is_na,
+        /* per tuple header: add IndexAttributeBitMapData if some cols are null-able */
+        CASE WHEN max(COALESCE(s_t.null_frac, s_x.null_frac, 0::real)) = 0::double precision
+            THEN 2 -- IndexTupleData size
+            ELSE 2 + (32 + 8 - 1) / 8 -- IndexTupleData size + IndexAttributeBitMapData size
+        END AS index_tuple_hdr_bm,
+        /* data len: we remove null values, saving space, using their fractional part from stats */
+        sum((1::double precision - COALESCE(s_t.null_frac, s_x.null_frac, 0::real))
+            * COALESCE(s_t.avg_width, s_x.avg_width, 1024)::double precision) AS nulldatawidth
+    FROM generate_series(1, i.indnatts::integer) AS gs(attpos)
+    -- Decode the indkey int2vector into an int[] once per index (not once per column).
+    CROSS JOIN LATERAL (
+        SELECT string_to_array(textin(int2vectorout(i.indkey)), ' '::text)::integer[] AS indkey_arr
+    ) ik
+    LEFT JOIN pg_catalog.pg_attribute a1
+           ON ik.indkey_arr[gs.attpos] <> 0
+          AND a1.attrelid = i.indrelid
+          AND a1.attnum   = ik.indkey_arr[gs.attpos]
+    LEFT JOIN pg_catalog.pg_attribute a2
+           ON ik.indkey_arr[gs.attpos] = 0
+          AND a2.attrelid = ci.oid
+          AND a2.attnum   = gs.attpos
+    LEFT JOIN pg_catalog.pg_stats s_t
+           ON s_t.schemaname = n.nspname
+          AND s_t.tablename  = ct.relname
+          AND s_t.attname    = a1.attname
+    LEFT JOIN pg_catalog.pg_stats s_x
+           ON s_x.schemaname = n.nspname
+          AND s_x.tablename  = ci.relname
+          AND s_x.attname    = a2.attname
+) agg
+
+-- Index tuple header width, aligned on MAXALIGN.
+CROSS JOIN LATERAL (
+    SELECT
+        (
+            (agg.index_tuple_hdr_bm + c.maxalign
+             - CASE -- Add padding to the index tuple header to align on MAXALIGN
+                 WHEN (agg.index_tuple_hdr_bm % c.maxalign) = 0 THEN c.maxalign
+                 ELSE agg.index_tuple_hdr_bm % c.maxalign
+               END)::double precision
+          + agg.nulldatawidth
+          + c.maxalign::double precision
+          - CASE -- Add padding to the data to align on MAXALIGN
+              WHEN agg.nulldatawidth = 0::double precision THEN 0
+              WHEN (agg.nulldatawidth::integer % c.maxalign) = 0 THEN c.maxalign
+              ELSE agg.nulldatawidth::integer % c.maxalign
+            END::double precision
+        )::numeric AS nulldatahdrwidth
+) hdr
+
+-- Estimated number of pages, with and without the fillfactor applied.
+CROSS JOIN LATERAL (
+    SELECT
+        agg.is_na,
+        -- NULLIF guards against division by zero: floor(...) reaches 0 once
+        -- nulldatahdrwidth exceeds the usable page size (an index whose columns are
+        -- entirely unanalyzed and fall back to the 1024-byte default width). Without
+        -- it, PostgreSQL raises a hard error that would fail the whole probe; instead
+        -- the NULL flows through the outer COALESCE to est_pages = 0.
+        COALESCE(
+            1::double precision + ceil(
+                ci.reltuples / NULLIF(floor(
+                    (c.bs - c.pageopqdata::numeric - c.pagehdr::numeric)::double precision
+                    / (4::numeric + hdr.nulldatahdrwidth)::double precision
+                ), 0)
+            ),
+            0::double precision
+        ) AS est_pages,
+        COALESCE(
+            1::double precision + ceil(
+                ci.reltuples / NULLIF(floor(
+                    ((c.bs - c.pageopqdata::numeric - c.pagehdr::numeric) * c.fillfactor::numeric)::double precision
+                    / (100::double precision * (4::numeric + hdr.nulldatahdrwidth)::double precision)
+                ), 0)
+            ),
+            0::double precision
+        ) AS est_pages_ff
+) bloat
+
+WHERE ci.relkind = 'i'
+  AND ci.relam = (SELECT oid FROM pg_am WHERE amname = 'btree'::name)
+  AND ci.relpages > 0
+ORDER BY n.nspname, ct.relname, ci.relname;

data/lib/gitlab_exporter/database/bloat_table.sql

@@ -1,7 +1,14 @@
 -- Originally from: https://github.com/ioguix/pgsql-bloat-estimation/blob/master/table/table_bloat.sql
 /* WARNING: executed with a non-superuser role, the query inspect only tables and materialized view (9.3+) you are granted to read.
-* This query is compatible with PostgreSQL 9.0 and more
+* This query is compatible with PostgreSQL 12 and later
 */
+-- pg_stats is materialized so the planner sees the true row count before planning the join,
+-- avoiding a catastrophic cardinality underestimate on PostgreSQL 17.
+WITH pg_stats_cached AS MATERIALIZED (
+  SELECT schemaname, tablename, attname, null_frac, avg_width
+  FROM pg_stats
+  WHERE inherited = false
+)
 SELECT current_database(), schemaname, tblname AS object_name, bs*tblpages AS real_size,
   (tblpages-est_tblpages)*bs AS extra_size,
   CASE WHEN tblpages - est_tblpages > 0
@@ -49,8 +56,8 @@ FROM (
       FROM pg_attribute AS att
         JOIN pg_class AS tbl ON att.attrelid = tbl.oid
         JOIN pg_namespace AS ns ON ns.oid = tbl.relnamespace
-        LEFT JOIN pg_stats AS s ON s.schemaname=ns.nspname
-          AND s.tablename = tbl.relname AND s.inherited=false AND s.attname=att.attname
+        LEFT JOIN pg_stats_cached AS s ON s.schemaname=ns.nspname
+          AND s.tablename = tbl.relname AND s.attname=att.attname
         LEFT JOIN pg_class AS toast ON tbl.reltoastrelid = toast.oid
       WHERE NOT att.attisdropped
         AND tbl.relkind in ('r','m')

data/lib/gitlab_exporter/database/zoekt.rb

@@ -2,6 +2,7 @@ module GitLab
   module Exporter
     module Database
       # A helper class to collect zoekt metrics.
+      # rubocop:disable Metrics/ClassLength
       class ZoektCollector < Base
         # Query to get processing zoekt_tasks distribution by zoekt_node_id
         ZOEKT_TASKS_PROCESSING_QUERY = <<~SQL.freeze
@@ -108,6 +109,40 @@ module GitLab
             zoekt_nodes ON zoekt_indices.zoekt_node_id = zoekt_nodes.id
         SQL
 
+        ZOEKT_NODE_ENABLED_NAMESPACES_QUERY = <<~SQL.freeze
+          SELECT
+            zn.id AS node_id,
+            zn.metadata ->> 'name' AS node_name,
+            COUNT(DISTINCT zi.zoekt_enabled_namespace_id) AS count
+          FROM zoekt_nodes zn
+          LEFT JOIN zoekt_indices zi ON zi.zoekt_node_id = zn.id
+          GROUP BY zn.id, zn.metadata ->> 'name'
+        SQL
+
+        ZOEKT_NODE_TASKS_QUERY = <<~SQL.freeze
+          SELECT
+            zn.id AS node_id,
+            zn.metadata ->> 'name' AS node_name,
+            zt.state,
+            COUNT(*) AS count
+          FROM zoekt_nodes zn
+          LEFT JOIN zoekt_tasks zt ON zt.zoekt_node_id = zn.id
+          GROUP BY zn.id, zn.metadata ->> 'name', zt.state
+        SQL
+
+        ZOEKT_INDICES_STALE_USED_STORAGE_QUERY = <<~SQL.freeze
+          SELECT COUNT(*) AS count FROM zoekt_indices WHERE last_indexed_at >= used_storage_bytes_updated_at
+        SQL
+
+        ZOEKT_TASK_STATES = {
+          0 => "pending",
+          1 => "processing",
+          10 => "done",
+          250 => "skipped",
+          255 => "failed",
+          256 => "orphaned"
+        }.freeze
+
         def run
           return {} unless zoekt_indexing_enabled?
 
@@ -119,7 +154,10 @@ module GitLab
             repositories_state_query_result: execute(ZOEKT_REPOSITORIES_STATE_QUERY),
             indices_state_query_result: execute(ZOEKT_INDICES_STATE_QUERY),
             indices_watermark_query_result: execute(ZOEKT_INDICES_WATERMARK_QUERY),
-            indices_storage_query_result: execute(ZOEKT_INDICES_STORAGE_QUERY)
+            indices_storage_query_result: execute(ZOEKT_INDICES_STORAGE_QUERY),
+            node_enabled_namespaces_query_result: execute(ZOEKT_NODE_ENABLED_NAMESPACES_QUERY),
+            node_tasks_query_result: execute(ZOEKT_NODE_TASKS_QUERY),
+            indices_stale_used_storage_query_result: execute(ZOEKT_INDICES_STALE_USED_STORAGE_QUERY)
           }.compact
         end
 
@@ -181,6 +219,7 @@ module GitLab
           nil
         end
       end
+      # rubocop:enable Metrics/ClassLength
 
       # The prober which is called when gathering metrics
       # rubocop:disable Metrics/ClassLength
@@ -245,6 +284,24 @@ module GitLab
           "gauge"
         )
 
+        PrometheusMetrics.describe(
+          "search_zoekt_node_enabled_namespaces",
+          "Number of enabled namespaces per Zoekt node",
+          "gauge"
+        )
+
+        PrometheusMetrics.describe(
+          "search_zoekt_node_tasks",
+          "Number of Zoekt indexing tasks on a node, broken down by state",
+          "gauge"
+        )
+
+        PrometheusMetrics.describe(
+          "search_zoekt_indices_with_stale_used_storage_bytes",
+          "Number of Zoekt indices whose used_storage_bytes value has not been updated since the last index run",
+          "gauge"
+        )
+
         def initialize(metrics: PrometheusMetrics.new, **opts)
           @metrics = metrics
           @collector = opts[:collector] || ZoektCollector.new(**opts)
@@ -269,6 +326,9 @@ module GitLab
           process_indices_state_results(results[:indices_state_query_result])
           process_indices_watermark_results(results[:indices_watermark_query_result])
           process_indices_storage_results(results[:indices_storage_query_result])
+          process_node_enabled_namespaces_results(results[:node_enabled_namespaces_query_result])
+          process_node_tasks_results(results[:node_tasks_query_result])
+          process_indices_stale_used_storage_results(results[:indices_stale_used_storage_query_result])
         end
 
         def process_task_results(results)
@@ -363,23 +423,11 @@ module GitLab
         end
 
         def add_zoekt_repositories_state_to_metric(row)
-          @metrics.add(
-            "search_zoekt_repositories_states_total",
-            row["count"].to_i,
-            **{
-              state: row["state"]
-            }.compact
-          )
+          @metrics.add("search_zoekt_repositories_states_total", row["count"].to_i, **{ state: row["state"] }.compact)
         end
 
         def add_zoekt_indices_state_to_metric(row)
-          @metrics.add(
-            "search_zoekt_indices_states",
-            row["count"].to_i,
-            **{
-              state: row["state"]
-            }.compact
-          )
+          @metrics.add("search_zoekt_indices_states", row["count"].to_i, **{ state: row["state"] }.compact)
         end
 
         def add_zoekt_indices_watermark_to_metric(row)
@@ -392,26 +440,56 @@ module GitLab
           )
         end
 
-        # rubocop:disable Metrics/MethodLength
-        def add_zoekt_indices_storage_to_metric(row)
+        def process_node_enabled_namespaces_results(results)
+          results&.each { |row| add_zoekt_node_enabled_namespaces_to_metric(row) }
+        end
+
+        def process_node_tasks_results(results)
+          results&.each { |row| add_zoekt_node_tasks_to_metric(row) }
+        end
+
+        def process_indices_stale_used_storage_results(results)
+          results&.each { |row| add_zoekt_indices_stale_used_storage_to_metric(row) }
+        end
+
+        def add_zoekt_node_enabled_namespaces_to_metric(row)
           @metrics.add(
-            "search_zoekt_indices_reserved_storage_bytes",
-            row["reserved_storage_bytes"].to_i,
+            "search_zoekt_node_enabled_namespaces",
+            row["count"].to_i,
             **{
-              zoekt_index_id: row["id"],
-              zoekt_node_name: row["node_name"]
+              node_id: row["node_id"],
+              node_name: row["node_name"]
             }.compact
           )
+        end
+
+        def add_zoekt_node_tasks_to_metric(row)
+          state_int = row["state"]&.to_i
+          return if row["state"].nil?
+
+          state_label = ZoektCollector::ZOEKT_TASK_STATES[state_int]
+          return if state_label.nil?
+
           @metrics.add(
-            "search_zoekt_indices_used_storage_bytes",
-            row["used_storage_bytes"].to_i,
+            "search_zoekt_node_tasks",
+            row["count"].to_i,
             **{
-              zoekt_index_id: row["id"],
-              zoekt_node_name: row["node_name"]
+              node_id: row["node_id"],
+              node_name: row["node_name"],
+              state: state_label
             }.compact
           )
         end
-        # rubocop:enable Metrics/MethodLength
+
+        def add_zoekt_indices_stale_used_storage_to_metric(row)
+          @metrics.add("search_zoekt_indices_with_stale_used_storage_bytes", row["count"].to_i)
+        end
+
+        def add_zoekt_indices_storage_to_metric(row)
+          payload = { zoekt_index_id: row["id"], zoekt_node_name: row["node_name"] }.compact
+          @metrics.add("search_zoekt_indices_reserved_storage_bytes", row["reserved_storage_bytes"].to_i, **payload)
+          @metrics.add("search_zoekt_indices_used_storage_bytes", row["used_storage_bytes"].to_i, **payload)
+        end
 
         def write_to(target)
           target.write(@metrics.to_s)

data/lib/gitlab_exporter/version.rb

@@ -1,5 +1,5 @@
 module GitLab
   module Exporter
-    VERSION = "16.8.0".freeze
+    VERSION = "16.10.0".freeze
   end
 end

data/spec/database/zoekt_spec.rb

@@ -36,6 +36,23 @@ describe GitLab::Exporter::Database::ZoektCollector do
         "node_name" => "zoekt-2" }
     ]
   end
+  let(:zoekt_node_enabled_namespaces_query) { described_class::ZOEKT_NODE_ENABLED_NAMESPACES_QUERY }
+  let(:zoekt_node_enabled_namespaces_query_results) do
+    [
+      { "node_id" => "1", "node_name" => "zoekt-1", "count" => "3" },
+      { "node_id" => "2", "node_name" => "zoekt-2", "count" => "5" }
+    ]
+  end
+  let(:zoekt_node_tasks_query) { described_class::ZOEKT_NODE_TASKS_QUERY }
+  let(:zoekt_node_tasks_query_results) do
+    [
+      { "node_id" => "1", "node_name" => "zoekt-1", "state" => "0", "count" => "4" },
+      { "node_id" => "1", "node_name" => "zoekt-1", "state" => "1", "count" => "2" },
+      { "node_id" => "2", "node_name" => "zoekt-2", "state" => "10", "count" => "7" }
+    ]
+  end
+  let(:zoekt_indices_stale_used_storage_query) { described_class::ZOEKT_INDICES_STALE_USED_STORAGE_QUERY }
+  let(:zoekt_indices_stale_used_storage_query_results) { [{ "count" => "6" }] }
 
   let(:zoekt_repository_schema_version_query_results) { [{ "count" => "1" }] }
 
@@ -80,7 +97,10 @@ describe GitLab::Exporter::Database::ZoektCollector do
         repositories_state_query_result: zoekt_repositories_state_query_results,
         indices_state_query_result: zoekt_indices_state_query_results,
         indices_watermark_query_result: zoekt_indices_watermark_query_results,
-        indices_storage_query_result: zoekt_indices_storage_query_results
+        indices_storage_query_result: zoekt_indices_storage_query_results,
+        node_enabled_namespaces_query_result: zoekt_node_enabled_namespaces_query_results,
+        node_tasks_query_result: zoekt_node_tasks_query_results,
+        indices_stale_used_storage_query_result: zoekt_indices_stale_used_storage_query_results
       }
     end
 
@@ -122,6 +142,12 @@ describe GitLab::Exporter::Database::ZoektCollector do
                                                    .and_return(zoekt_indices_watermark_query_results)
         expect(connection).to receive(:exec_params).with(zoekt_indices_storage_query, [])
                                                    .and_return(zoekt_indices_storage_query_results)
+        expect(connection).to receive(:exec_params).with(zoekt_node_enabled_namespaces_query, [])
+                                                   .and_return(zoekt_node_enabled_namespaces_query_results)
+        expect(connection).to receive(:exec_params).with(zoekt_node_tasks_query, [])
+                                                   .and_return(zoekt_node_tasks_query_results)
+        expect(connection).to receive(:exec_params).with(zoekt_indices_stale_used_storage_query, [])
+                                                   .and_return(zoekt_indices_stale_used_storage_query_results)
 
         expect(collector.run).to eq(result)
       end
@@ -144,6 +170,12 @@ describe GitLab::Exporter::Database::ZoektCollector do
                                                     .and_raise(PG::UndefinedTable)
           allow(connection).to receive(:exec_params).with(zoekt_indices_storage_query, [])
                                                     .and_raise(PG::UndefinedTable)
+          allow(connection).to receive(:exec_params).with(zoekt_node_enabled_namespaces_query, [])
+                                                    .and_raise(PG::UndefinedTable)
+          allow(connection).to receive(:exec_params).with(zoekt_node_tasks_query, [])
+                                                    .and_raise(PG::UndefinedTable)
+          allow(connection).to receive(:exec_params).with(zoekt_indices_stale_used_storage_query, [])
+                                                    .and_raise(PG::UndefinedTable)
 
           expect(collector.run).to eq({})
         end
@@ -167,6 +199,12 @@ describe GitLab::Exporter::Database::ZoektCollector do
                                                      .and_raise(PG::UndefinedColumn)
           expect(connection).to receive(:exec_params).with(zoekt_indices_storage_query, [])
                                                      .and_raise(PG::UndefinedColumn)
+          expect(connection).to receive(:exec_params).with(zoekt_node_enabled_namespaces_query, [])
+                                                     .and_raise(PG::UndefinedColumn)
+          expect(connection).to receive(:exec_params).with(zoekt_node_tasks_query, [])
+                                                     .and_raise(PG::UndefinedColumn)
+          expect(connection).to receive(:exec_params).with(zoekt_indices_stale_used_storage_query, [])
+                                                     .and_raise(PG::UndefinedColumn)
 
           expect(collector.run).to eq({ task_processing_query_result: zoekt_tasks_processing_query_results })
         end
@@ -257,6 +295,19 @@ describe GitLab::Exporter::Database::ZoektProber do
           "node_name" => "zoekt-1" },
         { "id" => "2", "reserved_storage_bytes" => "2000000", "used_storage_bytes" => "1500000",
           "node_name" => "zoekt-2" }
+      ],
+      node_enabled_namespaces_query_result: [
+        { "node_id" => "1", "node_name" => "zoekt-1", "count" => "3" },
+        { "node_id" => "2", "node_name" => "zoekt-2", "count" => "5" }
+      ],
+      node_tasks_query_result: [
+        { "node_id" => "1", "node_name" => "zoekt-1", "state" => "0", "count" => "4" },
+        { "node_id" => "1", "node_name" => "zoekt-1", "state" => "1", "count" => "2" },
+        { "node_id" => "2", "node_name" => "zoekt-2", "state" => "10", "count" => "7" },
+        { "node_id" => "2", "node_name" => "zoekt-2", "state" => nil, "count" => "1" }
+      ],
+      indices_stale_used_storage_query_result: [
+        { "count" => "6" }
       ]
     }
   end
@@ -361,7 +412,73 @@ describe GitLab::Exporter::Database::ZoektProber do
           )
       end
 
+      data[:node_enabled_namespaces_query_result].each do |ns_data|
+        expect(metrics).to receive(:add)
+          .with(
+            "search_zoekt_node_enabled_namespaces",
+            ns_data["count"].to_i,
+            node_id: ns_data["node_id"],
+            node_name: ns_data["node_name"]
+          )
+      end
+
+      # Only rows with a known state integer are emitted; the nil-state row is skipped.
+      task_state_map = GitLab::Exporter::Database::ZoektCollector::ZOEKT_TASK_STATES
+      data[:node_tasks_query_result].each do |task_data|
+        next if task_data["state"].nil?
+
+        state_label = task_state_map[task_data["state"].to_i]
+        next if state_label.nil?
+
+        expect(metrics).to receive(:add)
+          .with(
+            "search_zoekt_node_tasks",
+            task_data["count"].to_i,
+            node_id: task_data["node_id"],
+            node_name: task_data["node_name"],
+            state: state_label
+          )
+      end
+
+      data[:indices_stale_used_storage_query_result].each do |stale_data|
+        expect(metrics).to receive(:add)
+          .with(
+            "search_zoekt_indices_with_stale_used_storage_bytes",
+            stale_data["count"].to_i
+          )
+      end
+
       probe_db
     end
+
+    context "when node_tasks_query_result contains nil state rows" do
+      it "skips rows where state is nil" do
+        nil_state_data = data.merge(
+          node_tasks_query_result: [{ "node_id" => "1", "node_name" => "zoekt-1", "state" => nil, "count" => "3" }],
+          node_enabled_namespaces_query_result: [],
+          indices_stale_used_storage_query_result: []
+        )
+        allow(collector).to receive(:run).and_return(nil_state_data)
+
+        expect(metrics).not_to receive(:add).with("search_zoekt_node_tasks", anything, anything)
+
+        described_class.new(metrics: metrics, collector: collector, **opts).probe_db
+      end
+    end
+
+    context "when node_tasks_query_result contains an unknown state integer" do
+      it "skips rows where state integer is not in ZOEKT_TASK_STATES" do
+        unknown_state_data = data.merge(
+          node_tasks_query_result: [{ "node_id" => "1", "node_name" => "zoekt-1", "state" => "99", "count" => "2" }],
+          node_enabled_namespaces_query_result: [],
+          indices_stale_used_storage_query_result: []
+        )
+        allow(collector).to receive(:run).and_return(unknown_state_data)
+
+        expect(metrics).not_to receive(:add).with("search_zoekt_node_tasks", anything, anything)
+
+        described_class.new(metrics: metrics, collector: collector, **opts).probe_db
+      end
+    end
   end
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: gitlab-exporter
 version: !ruby/object:Gem::Version
-  version: 16.8.0
+  version: 16.10.0
 platform: ruby
 authors:
 - Pablo Carranza
@@ -44,14 +44,14 @@ dependencies:
     requirements:
     - - '='
       - !ruby/object:Gem::Version
-        version: 2.14.1
+        version: 2.14.3
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - '='
       - !ruby/object:Gem::Version
-        version: 2.14.1
+        version: 2.14.3
 - !ruby/object:Gem::Dependency
   name: pg
   requirement: !ruby/object:Gem::Requirement
@@ -72,14 +72,14 @@ dependencies:
     requirements:
     - - '='
       - !ruby/object:Gem::Version
-        version: 8.0.1
+        version: 8.0.2
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - '='
       - !ruby/object:Gem::Version
-        version: 8.0.1
+        version: 8.0.2
 - !ruby/object:Gem::Dependency
   name: quantile
   requirement: !ruby/object:Gem::Requirement