gitlab-exporter 15.2.0 → 15.3.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: ce8ca39488a8899bf54a56909d58d146630d65b42e15125abe04b71b195184a0
-  data.tar.gz: 964aa5174ccc4741d8aec5d4f04113fe3300308be52df082b7d3822a3b9486a3
+  metadata.gz: c199d67a010ca81ea54be6868ef3224c15633ae3a96f93c796d5388e979fda02
+  data.tar.gz: 57c2b7b3cff080ebb2b271781620fcf909374947baa2e427919abc2d4d912faf
 SHA512:
-  metadata.gz: cbade2952b3dcec413afad922cd0d092e8a37be8096f520ef3d46a94d129a76e9d2d3e48f5ba97c8c9fb855fe0f707c5ea0ec4c18cb2a9aa2059c069d1e24598
-  data.tar.gz: bbc7876726057eff433549331304a604160f5a377f7e4185d91b7d1c066196ac14791a24cf67e203e7cfd87b6cda4e70da5264c27b14d82ef356d1b1fa03510b
+  metadata.gz: b84c5bf72aeddf20385b464c489bd347ed0f74888b68ed2a573437d8a6dc79ef642d50b7726c067392dedf295c3a7ada74417a2dba79ff8434e93ac4814cc18b
+  data.tar.gz: cb981b9dd8f5fe66f1f6bf5a0331a94c79365ca772b274497434aa1401c920fbca197c1cb8218191189eef9bd724794c7a57bfa82e395b03bd828239ab5e7a0f

data/Gemfile.lock

@@ -1,7 +1,7 @@
 PATH
   remote: .
   specs:
-    gitlab-exporter (15.2.0)
+    gitlab-exporter (15.3.0)
       connection_pool (= 2.2.5)
       deep_merge (~> 1.2.2)
       faraday (>= 1.8.0, <= 2.8.1)

data/README.md

@@ -31,6 +31,8 @@ metrics.
       (table bloat) and keys `bloat_ratio bloat_size extra_size real_size` (see below)
     * [Remote mirrors](lib/gitlab_exporter/database/remote_mirrors.rb) --
       `project_remote_mirror_last_successful_update_time_seconds`, `project_remote_mirror_last_update_time_seconds`
+    * [Zoekt](lib/gitlab_exporter/database/zoekt.rb) --
+      `search_zoekt_task_processing_queue_size`
 1. Git
     * [git pull/push timings](lib/gitlab_exporter/git.rb) --
       `git_pull_time_milliseconds`, `git_push_time_milliseconds`

data/config/gitlab-exporter.yml.example

@@ -78,6 +78,9 @@ probes:
         <<: *db_common_opts
         project_ids:
         - 1
+    zoekt:
+      class_name: Database::ZoektProber
+      <<: *db_common
 
   process: &process
     methods:

data/lib/gitlab_exporter/database/zoekt.rb

@@ -0,0 +1,98 @@
+module GitLab
+  module Exporter
+    module Database
+      # A helper class to collect zoekt metrics.
+      class ZoektCollector < Base
+        QUERY = <<~SQL.freeze
+          WITH task_counts AS (
+            SELECT 
+              zoekt_node_id,
+              COUNT(*) AS count
+            FROM 
+              zoekt_tasks
+            WHERE 
+              perform_at <= $1
+              AND state IN (0, 1)
+            GROUP BY 
+              zoekt_node_id
+          )
+          SELECT 
+            n.id AS node_id,
+            n.metadata ->> 'name' AS node_name,
+            COALESCE(tc.count, 0) AS task_count
+          FROM 
+            zoekt_nodes n
+          LEFT JOIN 
+            task_counts tc ON n.id = tc.zoekt_node_id
+        SQL
+
+        ZOEKT_ENABLED_QUERY = <<~SQL.freeze
+          SELECT
+            zoekt_settings ->> 'zoekt_indexing_enabled' AS zoekt_indexing_enabled 
+          FROM application_settings
+          ORDER BY ID DESC
+          LIMIT 1
+        SQL
+
+        def run
+          return unless zoekt_indexing_enabled?
+
+          execute(QUERY, [Time.now.utc])
+        end
+
+        private
+
+        def zoekt_indexing_enabled?
+          @zoekt_indexing_enabled ||=
+            begin
+              with_connection_pool do |conn|
+                conn.exec(ZOEKT_ENABLED_QUERY).first["zoekt_indexing_enabled"] == "true"
+              end
+            rescue PG::UndefinedColumn
+              false
+            end
+        end
+
+        def execute(query, params)
+          with_connection_pool do |conn|
+            conn.exec_params(query, params)
+          end
+        rescue PG::UndefinedTable, PG::UndefinedColumn
+          nil
+        end
+      end
+
+      # The prober which is called when gathering metrics
+      class ZoektProber
+        PrometheusMetrics.describe("search_zoekt_task_processing_queue_size",
+                                   "Number of tasks waiting to be processed by Zoekt",
+                                   "gauge")
+
+        def initialize(metrics: PrometheusMetrics.new, **opts)
+          @metrics = metrics
+          @collector = opts[:collector] || ZoektCollector.new(**opts)
+        end
+
+        def probe_db
+          results = @collector.run
+          results.to_a.each do |row|
+            @metrics.add(
+              "search_zoekt_task_processing_queue_size",
+              row["task_count"].to_i,
+              node_name: row["node_name"],
+              node_id: row["node_id"]
+            )
+          end
+
+          self
+        rescue PG::ConnectionBad
+          self
+        end
+
+        def write_to(target)
+          target.write(@metrics.to_s)
+        end
+      end
+    end
+  end
+end

data/lib/gitlab_exporter/database.rb

@@ -9,6 +9,7 @@ module GitLab
       autoload :BloatProber,          "gitlab_exporter/database/bloat"
       autoload :RemoteMirrorsProber,  "gitlab_exporter/database/remote_mirrors"
       autoload :PgSequencesProber,    "gitlab_exporter/database/pg_sequences"
+      autoload :ZoektProber,          "gitlab_exporter/database/zoekt"
     end
   end
 end

data/lib/gitlab_exporter/prometheus.rb

@@ -16,17 +16,24 @@ module GitLab
       end
 
       class << self
-        def describe(name, description)
+        def describe(name, description, metric_type = nil)
           @metric_descriptions ||= {}
           @metric_descriptions[name] = description
+          @metric_types ||= {}
+          @metric_types[name] = metric_type
         end
 
         def description(name)
           @metric_descriptions && @metric_descriptions[name]
         end
 
+        def metric_type(name)
+          @metric_types && @metric_types[name]
+        end
+
         def clear_descriptions
           @metric_descriptions = {}
+          @metric_types = {}
         end
       end
 
@@ -48,10 +55,11 @@ module GitLab
         buffer = ""
         @metrics.each do |name, measurements|
           buffer << "# HELP #{name} #{self.class.description(name)}\n" if self.class.description(name)
+          buffer << "# TYPE #{name} #{self.class.metric_type(name)}\n" if self.class.metric_type(name)
 
           measurements.each do |measurement|
             buffer << name.to_s
-            labels = (measurement[:labels] || {}).map { |label, value| "#{label}=\"#{value}\"" }.join(",")
+            labels = labels(measurement[:labels])
             buffer << "{#{labels}}" unless labels.empty?
             buffer << " #{measurement[:value]}"
             buffer << " #{measurement[:timestamp]}" if @include_timestamp
@@ -63,6 +71,10 @@ module GitLab
 
       private
 
+      def labels(measurement_labels = {})
+        measurement_labels.map { |label, value| "#{label}=\"#{value}\"" }.join(",")
+      end
+
       def add_quantiles_to_metrics
         @quantiles.each do |data, measurements|
           estimator = Quantile::Estimator.new

data/lib/gitlab_exporter/version.rb

@@ -1,5 +1,5 @@
 module GitLab
   module Exporter
-    VERSION = "15.2.0".freeze
+    VERSION = "15.3.0".freeze
   end
 end

data/spec/database/zoekt_spec.rb

@@ -0,0 +1,137 @@
+require "spec_helper"
+require "gitlab_exporter/database/zoekt"
+
+describe GitLab::Exporter::Database::ZoektCollector do
+  let(:connection_pool) { double("connection pool") }
+  let(:connection) { double("connection") }
+  let(:query) { described_class::QUERY }
+  let(:zoekt_enabled_query) { described_class::ZOEKT_ENABLED_QUERY }
+
+  subject(:collector) { described_class.new(connection_string: "host=localhost") }
+
+  before do
+    allow(collector).to receive(:connection_pool).and_return(connection_pool)
+    allow(connection_pool).to receive(:with).and_yield(connection)
+    # allow(Time).to receive(:now).and_return(time_now)
+  end
+
+  def stub_zoekt_enabled
+    allow(connection).to receive(:exec).with(zoekt_enabled_query).and_return([{ "zoekt_indexing_enabled" => "true" }])
+  end
+
+  def stub_zoekt_not_enabled
+    allow(connection).to receive(:exec).with(zoekt_enabled_query).and_return([{ "zoekt_indexing_enabled" => "false" }])
+  end
+
+  describe "#run" do
+    let(:zoekt_enabled_results) {}
+    let(:frozen_time) { Time.new(2023, 1, 1, 0, 0, 0, 0) }
+    let(:query_results) do
+      [
+        { "node_id" => "1", "node_name" => "zoekt-1", "task_count" => "5" },
+        { "node_id" => "2", "node_name" => "zoekt-2", "task_count" => "10" }
+      ]
+    end
+
+    before do
+      allow(Time).to receive(:now).and_return(frozen_time)
+    end
+
+    context "when zoekt is not enabled" do
+      before do
+        stub_zoekt_not_enabled
+      end
+
+      it "returns nil" do
+        allow(connection).to receive(:exec_params).with(query, frozen_time).and_raise(PG::UndefinedTable)
+
+        expect(collector.run).to be_nil
+      end
+    end
+
+    context "when zoekt is enabled" do
+      before do
+        stub_zoekt_enabled
+      end
+
+      it "executes the query with the current time" do
+        expect(connection).to receive(:exec_params).with(query, [frozen_time]).and_return(query_results)
+
+        expect(collector.run).to eq(query_results)
+      end
+
+      context "when PG::UndefinedTable is raised" do
+        it "returns nil" do
+          allow(connection).to receive(:exec_params).with(query, [frozen_time]).and_raise(PG::UndefinedTable)
+
+          expect(collector.run).to be_nil
+        end
+      end
+
+      context "when PG::UndefinedColumn is raised" do
+        it "returns nil" do
+          allow(connection).to receive(:exec_params).with(query, [frozen_time]).and_raise(PG::UndefinedColumn)
+
+          expect(collector.run).to be_nil
+        end
+      end
+    end
+  end
+
+  describe "#zoekt_indexing_enabled?" do
+    it "returns true when zoekt is enabled" do
+      expect(connection).to receive(:exec)
+        .with(zoekt_enabled_query).and_return([{ "zoekt_indexing_enabled" => "true" }])
+
+      expect(subject.send(:zoekt_indexing_enabled?)).to eq(true)
+    end
+
+    it "returns false when PG::UndefinedColumn is raised" do
+      expect(connection).to receive(:exec).with(zoekt_enabled_query).and_raise(PG::UndefinedColumn)
+
+      expect(subject.send(:zoekt_indexing_enabled?)).to eq(false)
+    end
+  end
+end
+
+describe GitLab::Exporter::Database::ZoektProber do
+  let(:opts) { { connection_string: "host=localhost" } }
+  let(:metrics) { double("PrometheusMetrics", add: nil) }
+  let(:collector) { double(GitLab::Exporter::Database::ZoektCollector, run: data) }
+  let(:data) do
+    [
+      { "node_id" => "1", "node_name" => "zoekt-1", "task_count" => "5" },
+      { "node_id" => "2", "node_name" => "zoekt-2", "task_count" => "10" }
+    ]
+  end
+
+  describe "#probe_db" do
+    before do
+      allow(collector).to receive(:connected?).and_return(true)
+    end
+
+    subject(:probe_db) do
+      described_class.new(metrics: metrics, collector: collector, **opts).probe_db
+    end
+
+    it "invokes the collector" do
+      expect(collector).to receive(:run)
+
+      probe_db
+    end
+
+    it "adds metrics for each node" do
+      expect(collector).to receive(:run).and_return(data)
+
+      data.each do |node_data|
+        expect(metrics).to receive(:add)
+          .with("search_zoekt_task_processing_queue_size",
+                node_data["task_count"].to_i,
+                node_name: node_data["node_name"],
+                node_id: node_data["node_id"])
+      end
+
+      probe_db
+    end
+  end
+end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: gitlab-exporter
 version: !ruby/object:Gem::Version
-  version: 15.2.0
+  version: 15.3.0
 platform: ruby
 authors:
 - Pablo Carranza
@@ -231,6 +231,7 @@ files:
 - lib/gitlab_exporter/database/remote_mirrors.rb
 - lib/gitlab_exporter/database/row_count.rb
 - lib/gitlab_exporter/database/tuple_stats.rb
+- lib/gitlab_exporter/database/zoekt.rb
 - lib/gitlab_exporter/elasticsearch.rb
 - lib/gitlab_exporter/git.rb
 - lib/gitlab_exporter/memstats.rb
@@ -250,6 +251,7 @@ files:
 - spec/database/ci_builds_spec.rb
 - spec/database/pg_sequences_spec.rb
 - spec/database/row_count_spec.rb
+- spec/database/zoekt_spec.rb
 - spec/elasticsearch_spec.rb
 - spec/fixtures/config.yml
 - spec/fixtures/smaps/sample.txt
@@ -290,6 +292,7 @@ test_files:
 - spec/database/ci_builds_spec.rb
 - spec/database/pg_sequences_spec.rb
 - spec/database/row_count_spec.rb
+- spec/database/zoekt_spec.rb
 - spec/elasticsearch_spec.rb
 - spec/fixtures/config.yml
 - spec/fixtures/smaps/sample.txt