gitlab-exporter 16.8.0 → 16.9.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 8b320ac3c29627e5bca0700ab6007fb3681a286679032bccae4595af29e764fc
-  data.tar.gz: ab52c7b430a87959228fdb4483c2bf3a26f9838c983a5292dd27bb6b72123a8d
+  metadata.gz: 77bae663d5dff12133db411a27154cb55c100caf0bec1ecdd4dd4e8102c88994
+  data.tar.gz: d98c9128dc327d8757d4bb8d021cf863274f59588767ff877ce5ebe4c3274636
 SHA512:
-  metadata.gz: dbd696eb2e666142c275e46aff04eb9beda6739863fe505249f0074d1d8d1145fe6b41573e0f3521feee2bcde7dd2ad33e8fc76b1abc3f75e34e23d95d94258f
-  data.tar.gz: 2d426ce702f2d360b5aa8cc5ae51f4924295e717615c6da3adbd9ef1e46c241ed40e151082918fa7c963c7367c857730e1fafb41ffc328bc6b3d14879615abcf
+  metadata.gz: ff4b8dc86a0765ab98cf1e7ac97ee7016e09e1ec1e0a08db4366bdd9a66ae75dc34cb9d41ad5ed28668a6d279f641db780526ac3619b04b114f4c68aa29d8a74
+  data.tar.gz: a76a114f309436e969865d25cb3a0e7aa4b94d8239cb4d7f775958ef971d3a6747380d002adf8999d3a8be545ad3692980ae980dd5ed6e1a8533cc3f7bc87173

data/Gemfile.lock

@@ -1,10 +1,10 @@
 PATH
   remote: .
   specs:
-    gitlab-exporter (16.8.0)
+    gitlab-exporter (16.9.0)
       connection_pool (= 2.5.5)
       deep_merge (~> 1.2.2)
-      faraday (= 2.14.1)
+      faraday (= 2.14.3)
       pg (= 1.6.3)
       puma (= 8.0.1)
       quantile (= 0.2.1)
@@ -22,7 +22,7 @@ GEM
     connection_pool (2.5.5)
     deep_merge (1.2.2)
     diff-lcs (1.5.0)
-    faraday (2.14.1)
+    faraday (2.14.3)
       faraday-net_http (>= 2.0, < 3.5)
       json
       logger

data/gitlab-exporter.gemspec

@@ -24,7 +24,7 @@ Gem::Specification.new do |s|
 
   s.add_runtime_dependency "connection_pool", "2.5.5"
   s.add_runtime_dependency "deep_merge", "~> 1.2.2"
-  s.add_runtime_dependency "faraday", "2.14.1"
+  s.add_runtime_dependency "faraday", "2.14.3"
   s.add_runtime_dependency "pg", "1.6.3"
   s.add_runtime_dependency "puma", "8.0.1"
   s.add_runtime_dependency "quantile", "0.2.1"

data/lib/gitlab_exporter/database/bloat_btree.sql

@@ -1,7 +1,13 @@
 -- Originally from: https://github.com/ioguix/pgsql-bloat-estimation/blob/master/btree/btree_bloat.sql
 -- WARNING: executed with a non-superuser role, the query inspect only index on tables you are granted to read.
 -- WARNING: rows with is_na = 't' are known to have bad statistics ("name" type is not supported).
--- This query is compatible with PostgreSQL 8.2 and after
+-- This query is compatible with PostgreSQL 12 and after
+-- pg_stats is materialized so the planner sees the true row count before planning the join,
+-- avoiding a catastrophic cardinality underestimate on PostgreSQL 17.
+WITH pg_stats_cached AS MATERIALIZED (
+  SELECT schemaname, tablename, attname, null_frac, avg_width
+  FROM pg_catalog.pg_stats
+)
 SELECT current_database(), nspname AS schemaname, tblname, idxname AS object_name, bs*(relpages)::bigint AS real_size,
   bs*(relpages-est_pages)::bigint AS extra_size,
   100 * (relpages-est_pages)::float / relpages AS extra_ratio,
@@ -91,9 +97,9 @@ FROM (
                   AND a2.attnum = ic.attpos
             ) i
             JOIN pg_catalog.pg_namespace n ON n.oid = i.relnamespace
-            JOIN pg_catalog.pg_stats s ON s.schemaname = n.nspname
-                                      AND s.tablename = i.attrelname
-                                      AND s.attname = i.attname
+            JOIN pg_stats_cached s ON s.schemaname = n.nspname
+                                  AND s.tablename = i.attrelname
+                                  AND s.attname = i.attname
             GROUP BY 1,2,3,4,5,6,7,8,9,10,11
       ) AS rows_data_stats
   ) AS rows_hdr_pdg_stats

data/lib/gitlab_exporter/database/bloat_table.sql

@@ -1,7 +1,14 @@
 -- Originally from: https://github.com/ioguix/pgsql-bloat-estimation/blob/master/table/table_bloat.sql
 /* WARNING: executed with a non-superuser role, the query inspect only tables and materialized view (9.3+) you are granted to read.
-* This query is compatible with PostgreSQL 9.0 and more
+* This query is compatible with PostgreSQL 12 and later
 */
+-- pg_stats is materialized so the planner sees the true row count before planning the join,
+-- avoiding a catastrophic cardinality underestimate on PostgreSQL 17.
+WITH pg_stats_cached AS MATERIALIZED (
+  SELECT schemaname, tablename, attname, null_frac, avg_width
+  FROM pg_stats
+  WHERE inherited = false
+)
 SELECT current_database(), schemaname, tblname AS object_name, bs*tblpages AS real_size,
   (tblpages-est_tblpages)*bs AS extra_size,
   CASE WHEN tblpages - est_tblpages > 0
@@ -49,8 +56,8 @@ FROM (
       FROM pg_attribute AS att
         JOIN pg_class AS tbl ON att.attrelid = tbl.oid
         JOIN pg_namespace AS ns ON ns.oid = tbl.relnamespace
-        LEFT JOIN pg_stats AS s ON s.schemaname=ns.nspname
-          AND s.tablename = tbl.relname AND s.inherited=false AND s.attname=att.attname
+        LEFT JOIN pg_stats_cached AS s ON s.schemaname=ns.nspname
+          AND s.tablename = tbl.relname AND s.attname=att.attname
         LEFT JOIN pg_class AS toast ON tbl.reltoastrelid = toast.oid
       WHERE NOT att.attisdropped
         AND tbl.relkind in ('r','m')

data/lib/gitlab_exporter/database/zoekt.rb

@@ -2,6 +2,7 @@ module GitLab
   module Exporter
     module Database
       # A helper class to collect zoekt metrics.
+      # rubocop:disable Metrics/ClassLength
       class ZoektCollector < Base
         # Query to get processing zoekt_tasks distribution by zoekt_node_id
         ZOEKT_TASKS_PROCESSING_QUERY = <<~SQL.freeze
@@ -108,6 +109,40 @@ module GitLab
             zoekt_nodes ON zoekt_indices.zoekt_node_id = zoekt_nodes.id
         SQL
 
+        ZOEKT_NODE_ENABLED_NAMESPACES_QUERY = <<~SQL.freeze
+          SELECT
+            zn.id AS node_id,
+            zn.metadata ->> 'name' AS node_name,
+            COUNT(DISTINCT zi.zoekt_enabled_namespace_id) AS count
+          FROM zoekt_nodes zn
+          LEFT JOIN zoekt_indices zi ON zi.zoekt_node_id = zn.id
+          GROUP BY zn.id, zn.metadata ->> 'name'
+        SQL
+
+        ZOEKT_NODE_TASKS_QUERY = <<~SQL.freeze
+          SELECT
+            zn.id AS node_id,
+            zn.metadata ->> 'name' AS node_name,
+            zt.state,
+            COUNT(*) AS count
+          FROM zoekt_nodes zn
+          LEFT JOIN zoekt_tasks zt ON zt.zoekt_node_id = zn.id
+          GROUP BY zn.id, zn.metadata ->> 'name', zt.state
+        SQL
+
+        ZOEKT_INDICES_STALE_USED_STORAGE_QUERY = <<~SQL.freeze
+          SELECT COUNT(*) AS count FROM zoekt_indices WHERE last_indexed_at >= used_storage_bytes_updated_at
+        SQL
+
+        ZOEKT_TASK_STATES = {
+          0 => "pending",
+          1 => "processing",
+          10 => "done",
+          250 => "skipped",
+          255 => "failed",
+          256 => "orphaned"
+        }.freeze
+
         def run
           return {} unless zoekt_indexing_enabled?
 
@@ -119,7 +154,10 @@ module GitLab
             repositories_state_query_result: execute(ZOEKT_REPOSITORIES_STATE_QUERY),
             indices_state_query_result: execute(ZOEKT_INDICES_STATE_QUERY),
             indices_watermark_query_result: execute(ZOEKT_INDICES_WATERMARK_QUERY),
-            indices_storage_query_result: execute(ZOEKT_INDICES_STORAGE_QUERY)
+            indices_storage_query_result: execute(ZOEKT_INDICES_STORAGE_QUERY),
+            node_enabled_namespaces_query_result: execute(ZOEKT_NODE_ENABLED_NAMESPACES_QUERY),
+            node_tasks_query_result: execute(ZOEKT_NODE_TASKS_QUERY),
+            indices_stale_used_storage_query_result: execute(ZOEKT_INDICES_STALE_USED_STORAGE_QUERY)
           }.compact
         end
 
@@ -181,6 +219,7 @@ module GitLab
           nil
         end
       end
+      # rubocop:enable Metrics/ClassLength
 
       # The prober which is called when gathering metrics
       # rubocop:disable Metrics/ClassLength
@@ -245,6 +284,24 @@ module GitLab
           "gauge"
         )
 
+        PrometheusMetrics.describe(
+          "search_zoekt_node_enabled_namespaces",
+          "Number of enabled namespaces per Zoekt node",
+          "gauge"
+        )
+
+        PrometheusMetrics.describe(
+          "search_zoekt_node_tasks",
+          "Number of Zoekt indexing tasks on a node, broken down by state",
+          "gauge"
+        )
+
+        PrometheusMetrics.describe(
+          "search_zoekt_indices_with_stale_used_storage_bytes",
+          "Number of Zoekt indices whose used_storage_bytes value has not been updated since the last index run",
+          "gauge"
+        )
+
         def initialize(metrics: PrometheusMetrics.new, **opts)
           @metrics = metrics
           @collector = opts[:collector] || ZoektCollector.new(**opts)
@@ -269,6 +326,9 @@ module GitLab
           process_indices_state_results(results[:indices_state_query_result])
           process_indices_watermark_results(results[:indices_watermark_query_result])
           process_indices_storage_results(results[:indices_storage_query_result])
+          process_node_enabled_namespaces_results(results[:node_enabled_namespaces_query_result])
+          process_node_tasks_results(results[:node_tasks_query_result])
+          process_indices_stale_used_storage_results(results[:indices_stale_used_storage_query_result])
         end
 
         def process_task_results(results)
@@ -363,23 +423,11 @@ module GitLab
         end
 
         def add_zoekt_repositories_state_to_metric(row)
-          @metrics.add(
-            "search_zoekt_repositories_states_total",
-            row["count"].to_i,
-            **{
-              state: row["state"]
-            }.compact
-          )
+          @metrics.add("search_zoekt_repositories_states_total", row["count"].to_i, **{ state: row["state"] }.compact)
         end
 
         def add_zoekt_indices_state_to_metric(row)
-          @metrics.add(
-            "search_zoekt_indices_states",
-            row["count"].to_i,
-            **{
-              state: row["state"]
-            }.compact
-          )
+          @metrics.add("search_zoekt_indices_states", row["count"].to_i, **{ state: row["state"] }.compact)
         end
 
         def add_zoekt_indices_watermark_to_metric(row)
@@ -392,26 +440,56 @@ module GitLab
           )
         end
 
-        # rubocop:disable Metrics/MethodLength
-        def add_zoekt_indices_storage_to_metric(row)
+        def process_node_enabled_namespaces_results(results)
+          results&.each { |row| add_zoekt_node_enabled_namespaces_to_metric(row) }
+        end
+
+        def process_node_tasks_results(results)
+          results&.each { |row| add_zoekt_node_tasks_to_metric(row) }
+        end
+
+        def process_indices_stale_used_storage_results(results)
+          results&.each { |row| add_zoekt_indices_stale_used_storage_to_metric(row) }
+        end
+
+        def add_zoekt_node_enabled_namespaces_to_metric(row)
           @metrics.add(
-            "search_zoekt_indices_reserved_storage_bytes",
-            row["reserved_storage_bytes"].to_i,
+            "search_zoekt_node_enabled_namespaces",
+            row["count"].to_i,
             **{
-              zoekt_index_id: row["id"],
-              zoekt_node_name: row["node_name"]
+              node_id: row["node_id"],
+              node_name: row["node_name"]
             }.compact
           )
+        end
+
+        def add_zoekt_node_tasks_to_metric(row)
+          state_int = row["state"]&.to_i
+          return if row["state"].nil?
+
+          state_label = ZoektCollector::ZOEKT_TASK_STATES[state_int]
+          return if state_label.nil?
+
           @metrics.add(
-            "search_zoekt_indices_used_storage_bytes",
-            row["used_storage_bytes"].to_i,
+            "search_zoekt_node_tasks",
+            row["count"].to_i,
             **{
-              zoekt_index_id: row["id"],
-              zoekt_node_name: row["node_name"]
+              node_id: row["node_id"],
+              node_name: row["node_name"],
+              state: state_label
             }.compact
           )
         end
-        # rubocop:enable Metrics/MethodLength
+
+        def add_zoekt_indices_stale_used_storage_to_metric(row)
+          @metrics.add("search_zoekt_indices_with_stale_used_storage_bytes", row["count"].to_i)
+        end
+
+        def add_zoekt_indices_storage_to_metric(row)
+          payload = { zoekt_index_id: row["id"], zoekt_node_name: row["node_name"] }.compact
+          @metrics.add("search_zoekt_indices_reserved_storage_bytes", row["reserved_storage_bytes"].to_i, **payload)
+          @metrics.add("search_zoekt_indices_used_storage_bytes", row["used_storage_bytes"].to_i, **payload)
+        end
 
         def write_to(target)
           target.write(@metrics.to_s)

data/lib/gitlab_exporter/version.rb

@@ -1,5 +1,5 @@
 module GitLab
   module Exporter
-    VERSION = "16.8.0".freeze
+    VERSION = "16.9.0".freeze
   end
 end

data/spec/database/zoekt_spec.rb

@@ -36,6 +36,23 @@ describe GitLab::Exporter::Database::ZoektCollector do
         "node_name" => "zoekt-2" }
     ]
   end
+  let(:zoekt_node_enabled_namespaces_query) { described_class::ZOEKT_NODE_ENABLED_NAMESPACES_QUERY }
+  let(:zoekt_node_enabled_namespaces_query_results) do
+    [
+      { "node_id" => "1", "node_name" => "zoekt-1", "count" => "3" },
+      { "node_id" => "2", "node_name" => "zoekt-2", "count" => "5" }
+    ]
+  end
+  let(:zoekt_node_tasks_query) { described_class::ZOEKT_NODE_TASKS_QUERY }
+  let(:zoekt_node_tasks_query_results) do
+    [
+      { "node_id" => "1", "node_name" => "zoekt-1", "state" => "0", "count" => "4" },
+      { "node_id" => "1", "node_name" => "zoekt-1", "state" => "1", "count" => "2" },
+      { "node_id" => "2", "node_name" => "zoekt-2", "state" => "10", "count" => "7" }
+    ]
+  end
+  let(:zoekt_indices_stale_used_storage_query) { described_class::ZOEKT_INDICES_STALE_USED_STORAGE_QUERY }
+  let(:zoekt_indices_stale_used_storage_query_results) { [{ "count" => "6" }] }
 
   let(:zoekt_repository_schema_version_query_results) { [{ "count" => "1" }] }
 
@@ -80,7 +97,10 @@ describe GitLab::Exporter::Database::ZoektCollector do
         repositories_state_query_result: zoekt_repositories_state_query_results,
         indices_state_query_result: zoekt_indices_state_query_results,
         indices_watermark_query_result: zoekt_indices_watermark_query_results,
-        indices_storage_query_result: zoekt_indices_storage_query_results
+        indices_storage_query_result: zoekt_indices_storage_query_results,
+        node_enabled_namespaces_query_result: zoekt_node_enabled_namespaces_query_results,
+        node_tasks_query_result: zoekt_node_tasks_query_results,
+        indices_stale_used_storage_query_result: zoekt_indices_stale_used_storage_query_results
       }
     end
 
@@ -122,6 +142,12 @@ describe GitLab::Exporter::Database::ZoektCollector do
                                                    .and_return(zoekt_indices_watermark_query_results)
         expect(connection).to receive(:exec_params).with(zoekt_indices_storage_query, [])
                                                    .and_return(zoekt_indices_storage_query_results)
+        expect(connection).to receive(:exec_params).with(zoekt_node_enabled_namespaces_query, [])
+                                                   .and_return(zoekt_node_enabled_namespaces_query_results)
+        expect(connection).to receive(:exec_params).with(zoekt_node_tasks_query, [])
+                                                   .and_return(zoekt_node_tasks_query_results)
+        expect(connection).to receive(:exec_params).with(zoekt_indices_stale_used_storage_query, [])
+                                                   .and_return(zoekt_indices_stale_used_storage_query_results)
 
         expect(collector.run).to eq(result)
       end
@@ -144,6 +170,12 @@ describe GitLab::Exporter::Database::ZoektCollector do
                                                     .and_raise(PG::UndefinedTable)
           allow(connection).to receive(:exec_params).with(zoekt_indices_storage_query, [])
                                                     .and_raise(PG::UndefinedTable)
+          allow(connection).to receive(:exec_params).with(zoekt_node_enabled_namespaces_query, [])
+                                                    .and_raise(PG::UndefinedTable)
+          allow(connection).to receive(:exec_params).with(zoekt_node_tasks_query, [])
+                                                    .and_raise(PG::UndefinedTable)
+          allow(connection).to receive(:exec_params).with(zoekt_indices_stale_used_storage_query, [])
+                                                    .and_raise(PG::UndefinedTable)
 
           expect(collector.run).to eq({})
         end
@@ -167,6 +199,12 @@ describe GitLab::Exporter::Database::ZoektCollector do
                                                      .and_raise(PG::UndefinedColumn)
           expect(connection).to receive(:exec_params).with(zoekt_indices_storage_query, [])
                                                      .and_raise(PG::UndefinedColumn)
+          expect(connection).to receive(:exec_params).with(zoekt_node_enabled_namespaces_query, [])
+                                                     .and_raise(PG::UndefinedColumn)
+          expect(connection).to receive(:exec_params).with(zoekt_node_tasks_query, [])
+                                                     .and_raise(PG::UndefinedColumn)
+          expect(connection).to receive(:exec_params).with(zoekt_indices_stale_used_storage_query, [])
+                                                     .and_raise(PG::UndefinedColumn)
 
           expect(collector.run).to eq({ task_processing_query_result: zoekt_tasks_processing_query_results })
         end
@@ -257,6 +295,19 @@ describe GitLab::Exporter::Database::ZoektProber do
           "node_name" => "zoekt-1" },
         { "id" => "2", "reserved_storage_bytes" => "2000000", "used_storage_bytes" => "1500000",
           "node_name" => "zoekt-2" }
+      ],
+      node_enabled_namespaces_query_result: [
+        { "node_id" => "1", "node_name" => "zoekt-1", "count" => "3" },
+        { "node_id" => "2", "node_name" => "zoekt-2", "count" => "5" }
+      ],
+      node_tasks_query_result: [
+        { "node_id" => "1", "node_name" => "zoekt-1", "state" => "0", "count" => "4" },
+        { "node_id" => "1", "node_name" => "zoekt-1", "state" => "1", "count" => "2" },
+        { "node_id" => "2", "node_name" => "zoekt-2", "state" => "10", "count" => "7" },
+        { "node_id" => "2", "node_name" => "zoekt-2", "state" => nil, "count" => "1" }
+      ],
+      indices_stale_used_storage_query_result: [
+        { "count" => "6" }
       ]
     }
   end
@@ -361,7 +412,73 @@ describe GitLab::Exporter::Database::ZoektProber do
           )
       end
 
+      data[:node_enabled_namespaces_query_result].each do |ns_data|
+        expect(metrics).to receive(:add)
+          .with(
+            "search_zoekt_node_enabled_namespaces",
+            ns_data["count"].to_i,
+            node_id: ns_data["node_id"],
+            node_name: ns_data["node_name"]
+          )
+      end
+
+      # Only rows with a known state integer are emitted; the nil-state row is skipped.
+      task_state_map = GitLab::Exporter::Database::ZoektCollector::ZOEKT_TASK_STATES
+      data[:node_tasks_query_result].each do |task_data|
+        next if task_data["state"].nil?
+
+        state_label = task_state_map[task_data["state"].to_i]
+        next if state_label.nil?
+
+        expect(metrics).to receive(:add)
+          .with(
+            "search_zoekt_node_tasks",
+            task_data["count"].to_i,
+            node_id: task_data["node_id"],
+            node_name: task_data["node_name"],
+            state: state_label
+          )
+      end
+
+      data[:indices_stale_used_storage_query_result].each do |stale_data|
+        expect(metrics).to receive(:add)
+          .with(
+            "search_zoekt_indices_with_stale_used_storage_bytes",
+            stale_data["count"].to_i
+          )
+      end
+
       probe_db
     end
+
+    context "when node_tasks_query_result contains nil state rows" do
+      it "skips rows where state is nil" do
+        nil_state_data = data.merge(
+          node_tasks_query_result: [{ "node_id" => "1", "node_name" => "zoekt-1", "state" => nil, "count" => "3" }],
+          node_enabled_namespaces_query_result: [],
+          indices_stale_used_storage_query_result: []
+        )
+        allow(collector).to receive(:run).and_return(nil_state_data)
+
+        expect(metrics).not_to receive(:add).with("search_zoekt_node_tasks", anything, anything)
+
+        described_class.new(metrics: metrics, collector: collector, **opts).probe_db
+      end
+    end
+
+    context "when node_tasks_query_result contains an unknown state integer" do
+      it "skips rows where state integer is not in ZOEKT_TASK_STATES" do
+        unknown_state_data = data.merge(
+          node_tasks_query_result: [{ "node_id" => "1", "node_name" => "zoekt-1", "state" => "99", "count" => "2" }],
+          node_enabled_namespaces_query_result: [],
+          indices_stale_used_storage_query_result: []
+        )
+        allow(collector).to receive(:run).and_return(unknown_state_data)
+
+        expect(metrics).not_to receive(:add).with("search_zoekt_node_tasks", anything, anything)
+
+        described_class.new(metrics: metrics, collector: collector, **opts).probe_db
+      end
+    end
   end
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: gitlab-exporter
 version: !ruby/object:Gem::Version
-  version: 16.8.0
+  version: 16.9.0
 platform: ruby
 authors:
 - Pablo Carranza
@@ -44,14 +44,14 @@ dependencies:
     requirements:
     - - '='
       - !ruby/object:Gem::Version
-        version: 2.14.1
+        version: 2.14.3
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - '='
       - !ruby/object:Gem::Version
-        version: 2.14.1
+        version: 2.14.3
 - !ruby/object:Gem::Dependency
   name: pg
   requirement: !ruby/object:Gem::Requirement