gitlab-exporter 17.0.1 → 17.0.2

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 15b0fdb50984859832e6a21dcb76c31dfbfb68f28e5601f2c1389782ea4b5f2f
-  data.tar.gz: a02d8bda949c2d5f94981d50123ff845e73d958306d0bcf9e3ca0cfd4c8107e5
+  metadata.gz: 0f38401d83375e243787b0f692035d11cc9cc9e5a3bde34ca9d468261be12a41
+  data.tar.gz: 4f8f1dbed6854da6003209745214b6687f405145b79c5dc2a0ee06d0bbedec4f
 SHA512:
-  metadata.gz: ab40fc6975a31c6175cb3139dd1e576a06f7f1a6b598efc387f76228e68413a47b7902d7aca1682128ee77dd76ee4f2e178fec7bed8dc027676ec2e614289490
-  data.tar.gz: e749cdfd3a8db77f8a594ad7007db5dc919003defa0afb7c3691a461b78d0bcdf459e8bce6d5106809f0cdc567323a968fe1d8de88c990a2c5074339b6c38343
+  metadata.gz: 7c88e061a15dcc9f640c29a910de6d81eedb345c113bce789b08a44a252ddc708bdb082e5aa7ee881eb383dcf51c72d34fd7ec8c00353fe210e0b97915e05ab4
+  data.tar.gz: 9a26c277997d178c65607aadbba2b82b931bc1375a1f732c35665639cfb1a404869409272f85b640df8a5f101cf25eddee985c1f20dbb8d75f6904aaffd1e7c6

data/Gemfile.lock

@@ -1,7 +1,7 @@
 PATH
   remote: .
   specs:
-    gitlab-exporter (17.0.1)
+    gitlab-exporter (17.0.2)
       base64 (= 0.3.0)
       connection_pool (= 2.5.5)
       deep_merge (~> 1.2.2)

data/README.md

@@ -23,9 +23,10 @@ metrics.
      `gitlab_pg_sequences_min_value`, `gitlab_pg_sequences_max_value`, `gitlab_pg_sequences_current_value`
     * [Row count queries](lib/gitlab_exporter/database/row_count.rb) --
       `gitlab_database_rows`
-    * [CI builds](lib/gitlab_exporter/database/ci_builds.rb) --
+    * [CI builds](lib/gitlab_exporter/database/ci_builds.rb) (**deprecated**) --
       `ci_pending_builds`, `ci_created_builds`, `ci_stale_builds`,
-      `ci_running_builds`
+      `ci_running_builds`, `ci_unarchived_traces`
+      (see [Deprecated: CI builds metrics](#deprecated-ci-builds-metrics))
     * [Bloat](lib/gitlab_exporter/database/bloat.rb) --
       `gitlab_database_bloat_$type_$key` with type `btree` (index bloat) or `table`
       (table bloat) and keys `bloat_ratio bloat_size extra_size real_size` (see below)
@@ -133,6 +134,57 @@ Also see the [original documentation](https://github.com/ioguix/pgsql-bloat-esti
 Note that all metrics returned are estimates without an upper bound for
 the error.
 
+### Deprecated: CI builds metrics
+
+The [CI builds probe](lib/gitlab_exporter/database/ci_builds.rb) is deprecated and
+will be removed in a future major version. It collects its metrics by querying
+`p_ci_builds` directly, which puts avoidable load on tables that job scheduling
+already loads heavily, and it has had almost no maintenance since 2021. GitLab
+Rails and GitLab Runner instrument most of the same information by counting
+events as they happen; GitLab.com stopped using this probe for CI state years
+ago.
+
+The probe prints a deprecation notice on every run, and each of its metrics
+carries `(deprecated, ...)` in its `# HELP` line, so the warning reaches
+Prometheus and Grafana and not only the exporter's log.
+
+Replacements, where one exists:
+
+| Deprecated metric | Replacement |
+| --- | --- |
+| `ci_pending_builds` | `gitlab_ci_current_queue_size` (Rails, gauge, by `runner_type`) |
+| `ci_running_builds` | `gitlab_runner_jobs` (Runner, gauge, by `runner`, `state`, `stage`) |
+| `ci_created_builds` | None. A `created` build is not queued yet, so nothing counts it. |
+| `ci_stale_builds` | None. The nearest signal is `gitlab_ci_queue_operations_total{operation="build_status_stale"}` (Rails, counter), which counts stale rows met while queueing rather than how many are stale now. |
+| `ci_unarchived_traces` | None. |
+
+The replacements are not a like-for-like relabel: the Rails queue metrics are
+keyed by `runner_type` and the Runner metrics by runner, so neither carries the
+`namespace` label this probe reports. Queue health is better served by the Rails
+metrics that have no exporter equivalent at all -- `job_queue_duration_seconds`
+(how long jobs wait), `job_register_attempts_total` and
+`job_register_attempts_failed_total`, and `gitlab_ci_queue_operations_total`.
+Those four are always instrumented.
+
+**`gitlab_ci_current_queue_size`, and every other `gitlab_ci_queue_*` metric, is
+behind the `gitlab_ci_builds_queuing_metrics` ops feature flag, which is disabled
+by default.** Enable it before pointing a dashboard at them, or the replacement
+panels stay empty:
+
+```ruby
+Feature.enable(:gitlab_ci_builds_queuing_metrics)
+```
+
+Metric definitions live in
+[`lib/gitlab/ci/queue/metrics.rb`](https://gitlab.com/gitlab-org/gitlab/-/blob/master/lib/gitlab/ci/queue/metrics.rb)
+(Rails) and
+[`commands/builds_helper.go`](https://gitlab.com/gitlab-org/gitlab-runner/-/blob/main/commands/builds_helper.go)
+(Runner). See
+[GitLab Prometheus metrics](https://docs.gitlab.com/administration/monitoring/prometheus/gitlab_metrics/)
+for enabling the Rails exporter, and
+[Monitoring runners](https://docs.gitlab.com/runner/monitoring/) for the
+Runner's.
+
 ## Contributing
 
 gitlab-exporter is an open source project and we are very happy to accept community contributions. Please refer to [CONTRIBUTING.md](/CONTRIBUTING.md) for details.

data/lib/gitlab_exporter/database/ci_builds.rb

@@ -294,12 +294,53 @@ module GitLab
 
       # The prober which is called when gathering metrics
       class CiBuildsProber
+        # The stdout notice in #probe_db reaches whoever reads the exporter's log,
+        # which is not who reads the metrics. HELP is the one channel that travels
+        # with the sample to the scraper, so the deprecation rides there too: it
+        # survives into Prometheus and shows up in Grafana's metric browser.
+        DEPRECATED = "(deprecated, to be removed in a future major version)".freeze
+        REPLACEMENT = "See the CI builds entry in README.md for replacements.".freeze
+
+        PrometheusMetrics.describe(
+          "ci_created_builds",
+          "#{DEPRECATED} Number of CI builds in the created state, by namespace. #{REPLACEMENT}",
+          "gauge"
+        )
+
+        PrometheusMetrics.describe(
+          "ci_pending_builds",
+          "#{DEPRECATED} Number of CI builds in the pending state, by namespace. #{REPLACEMENT}",
+          "gauge"
+        )
+
+        PrometheusMetrics.describe(
+          "ci_stale_builds",
+          "#{DEPRECATED} Number of running CI builds not updated for over an hour. #{REPLACEMENT}",
+          "gauge"
+        )
+
+        PrometheusMetrics.describe(
+          "ci_running_builds",
+          "#{DEPRECATED} Number of running CI builds, by runner and namespace. #{REPLACEMENT}",
+          "gauge"
+        )
+
+        PrometheusMetrics.describe(
+          "ci_unarchived_traces",
+          "#{DEPRECATED} Number of finished CI builds whose trace is not archived. #{REPLACEMENT}",
+          "gauge"
+        )
+
         def initialize(metrics: PrometheusMetrics.new, **opts)
           @metrics = metrics
           @collector = CiBuildsCollector.new(**opts)
         end
 
         def probe_db
+          puts "[DEPRECATED] probe_db and CiBuildsProber are now considered obsolete"\
+            " and will be removed in a future major version,"\
+            " please use the CI/CD queuing metrics emitted by GitLab Rails instead"
+
           @results = @collector.run
 
           ci_builds_metrics(@results[:created_builds], "ci_created_builds") if @results[:created_builds]

data/lib/gitlab_exporter/version.rb

@@ -1,5 +1,5 @@
 module GitLab
   module Exporter
-    VERSION = "17.0.1".freeze
+    VERSION = "17.0.2".freeze
   end
 end

data/spec/database/ci_builds_spec.rb

@@ -361,3 +361,55 @@ describe GitLab::Exporter::Database do
     end
   end
 end
+
+# Asserting on the notice keeps a later refactor of #probe_db from dropping it
+# silently, which would leave operators unwarned before the probe goes (issue #119).
+describe GitLab::Exporter::Database::CiBuildsProber do
+  let(:prober) do
+    described_class.new(connection_string: "host=localhost",
+                        metrics: GitLab::Exporter::PrometheusMetrics.new(include_timestamp: false))
+  end
+
+  let(:metric_names) do
+    %w[ci_created_builds ci_pending_builds ci_stale_builds ci_running_builds ci_unarchived_traces]
+  end
+
+  it "warns that the probe is deprecated and names the replacement" do
+    allow_any_instance_of(GitLab::Exporter::Database::CiBuildsCollector).to receive(:run)
+      .and_return(pending_builds: [], stale_builds: 0, per_runner: [], unarchived_traces: 0)
+
+    expect { prober.probe_db }.to output(/\[DEPRECATED\].*CiBuildsProber.*GitLab Rails/m).to_stdout
+  end
+
+  # The stdout notice never reaches the scraper. HELP does, so every metric the
+  # probe can emit has to carry the deprecation, not just the ones with a
+  # replacement.
+  it "describes every metric it can emit as deprecated" do
+    metric_names.each do |name|
+      expect(GitLab::Exporter::PrometheusMetrics.description(name))
+        .to start_with("(deprecated"), "#{name} is emitted without a deprecated HELP line"
+      expect(GitLab::Exporter::PrometheusMetrics.metric_type(name)).to eq("gauge")
+    end
+  end
+
+  it "renders the deprecation into the scrape output" do
+    allow_any_instance_of(GitLab::Exporter::Database::CiBuildsCollector).to receive(:run)
+      .and_return(pending_builds: [{ namespace: "1", shared_runners: "yes", value: 12 }],
+                  stale_builds: 3, per_runner: [], unarchived_traces: 0)
+
+    output = StringIO.new
+    capture_stdout do prober.probe_db.write_to(output) end
+
+    expect(output.string).to include("# HELP ci_pending_builds (deprecated")
+    expect(output.string).to include("# TYPE ci_pending_builds gauge")
+    expect(output.string).to include(%(ci_pending_builds{namespace="1",shared_runners="yes"} 12.0))
+  end
+
+  def capture_stdout
+    original = $stdout
+    $stdout = StringIO.new
+    yield
+  ensure
+    $stdout = original
+  end
+end

data/spec/prometheus_metrics_spec.rb

@@ -29,15 +29,29 @@ describe GitLab::Exporter::PrometheusMetrics do
 
     allow(Time).to receive(:now).and_return(time)
 
-    described_class.describe("mymetric", "description")
-    described_class.describe("missingmetric", "otherdescription")
-    subject.add("mymetric", 1.3, mylabel: "x", myotherlabel: "y")
-
-    expect(subject.to_s).to eq(<<~METRICS)
+    # Descriptions are class-level state, and probes register theirs when their
+    # file loads, which happens once. Describing on a subclass keeps this example
+    # out of the shared registry: clearing it here would drop those registrations
+    # for every later example, and nothing can put them back.
+    metrics_class = Class.new(described_class)
+    metrics = metrics_class.new
+
+    metrics_class.describe("mymetric", "description")
+    metrics_class.describe("missingmetric", "otherdescription")
+    metrics.add("mymetric", 1.3, mylabel: "x", myotherlabel: "y")
+
+    expect(metrics.to_s).to eq(<<~METRICS)
       # HELP mymetric description
       mymetric{mylabel="x",myotherlabel="y"} 1.3 #{(time.to_f * 1000).to_i}
     METRICS
+  end
+
+  it "clears descriptions" do
+    metrics_class = Class.new(described_class)
+    metrics_class.describe("mymetric", "description", "gauge")
 
-    described_class.clear_descriptions
+    expect { metrics_class.clear_descriptions }
+      .to change { metrics_class.description("mymetric") }.from("description").to(nil)
+    expect(metrics_class.metric_type("mymetric")).to be_nil
   end
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: gitlab-exporter
 version: !ruby/object:Gem::Version
-  version: 17.0.1
+  version: 17.0.2
 platform: ruby
 authors:
 - Pablo Carranza