gitlab-exporter 10.4.0 → 11.1.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: ace0ad673d8d36d1a6dbbacefb7f6aefde7d795e76eda8d3788c4cb4f5eb5247
-  data.tar.gz: ac42d2e6deaaa3fee45ed74ec1cec81f402abba0c4f14798878b2b006c72400b
+  metadata.gz: b3eef01acf595b16deb86b2578776e3487b513865fca93ede7684ba982932706
+  data.tar.gz: e5194e0230ac9672b9c30fbd1bf1be3b11333d894a21505dd97e39891f72b960
 SHA512:
-  metadata.gz: d3ac36c2b2ec584cb8ac60355649eb79d5a3d96e85a266f14794627fc3a9fa61a5cfb231f6c33e044cbb2e71d7965c50463b6673f80c69de4ebca58d658280e6
-  data.tar.gz: 43323ec5663db9dd865e84b3c7907ba43f805a958904cfef57d7566b6259a31cc1a7aa96c494ffb976b4b04148c2fbfe63bd688844672ad1938c41bf05181a4b
+  metadata.gz: 60c79e4a4e550adf557129aa278986726370e21f77ca5f67864f23fbb181f5c904d02f5015d51d8317c748b9a72b74cfd59a5f14b4a0af0ffce6b03883ceef4a
+  data.tar.gz: 5bd99ea95151fd95a5f72597474ec6c81e3fd9e1f0284f5d85b7595257f24cddde5b0a315a6705a6258b8b7d0ebcbe5178f6816978ebc3b2c47a435578d58ee0

data/.gitlab-ci.yml

@@ -6,6 +6,10 @@ include:
   - template: Security/SAST.gitlab-ci.yml  # https://gitlab.com/gitlab-org/gitlab-foss/blob/master/lib/gitlab/ci/templates/Security/SAST.gitlab-ci.yml
   - template: Security/Secret-Detection.gitlab-ci.yml # https://gitlab.com/gitlab-org/gitlab-foss/-/blob/master/lib/gitlab/ci/templates/Security/Secret-Detection.gitlab-ci.yml
 
+stages:
+  - test
+  - dast
+
 default:
   image: ruby:2.7
   cache:

data/Gemfile.lock

@@ -1,7 +1,7 @@
 PATH
   remote: .
   specs:
-    gitlab-exporter (10.4.0)
+    gitlab-exporter (11.1.0)
       connection_pool (= 2.2.5)
       pg (= 1.2.3)
       puma (= 5.3.2)

data/README.md

@@ -34,7 +34,7 @@ metrics.
       `git_pull_time_milliseconds`, `git_push_time_milliseconds`
     * git processes stats (see Process below)
 1. [Sidekiq](lib/gitlab_exporter/sidekiq.rb)
-	* Stats
+	* Stats (probe_stats)
 	  * `sidekiq_jobs_processed_total`
 	  * `sidekiq_jobs_failed_total`
 	  * `sidekiq_jobs_enqueued_size`
@@ -44,14 +44,21 @@ metrics.
 	  * `sidekiq_default_queue_latency_seconds`
 	  * `sidekiq_processes_size`
 	  * `sidekiq_workers_size`
-	* Queues
+	* Queues (probe_queues)
 	  * `sidekiq_queue_size`
 	  * `sidekiq_queue_paused`
 	  * `sidekiq_queue_latency_seconds`
-	* Jobs
+	* Jobs (probe_jobs_limit)
 	  * `sidekiq_enqueued_jobs`
+  * Workers (probe_workers)
 	  * `sidekiq_running_jobs`
+  * Retries (probe_retries)
 	  * `sidekiq_to_be_retried_jobs`
+  * Future Sets (probe_future_sets)
+    * `sidekiq_schedule_set_processing_delay_seconds`
+    * `sidekiq_schedule_set_backlog_count`
+    * `sidekiq_retry_set_processing_delay_seconds`
+    * `sidekiq_retry_set_backlog_count`
 
 ### Setup with GitLab Development Kit
 

data/config/gitlab-exporter.yml.example

@@ -78,8 +78,6 @@ probes:
     opts:
       - pid_or_pattern: "sidekiq .* \\[.*?\\]"
         name: sidekiq
-      - pid_or_pattern: "unicorn.* worker\\[.*?\\]"
-        name: unicorn
       - pid_or_pattern: "git-upload-pack --stateless-rpc"
         name: git_upload_pack
         quantiles: true

data/lib/gitlab_exporter/cli.rb

@@ -217,7 +217,7 @@ module GitLab
             opts.on("--pid=123", "Process ID") do |val|
               @pid = val
             end
-            opts.on("--pattern=unicorn", "Process command pattern") do |val|
+            opts.on("--pattern=worker", "Process command pattern") do |val|
               @pattern = val
             end
             opts.on("--name=NAME", "Process name to be used in metrics") do |val|
@@ -275,7 +275,7 @@ module GitLab
           ::GitLab::Exporter::SidekiqProber.new(redis_url: @redis_url)
                                            .probe_stats
                                            .probe_queues
-                                           .probe_jobs
+                                           .probe_jobs_limit
                                            .probe_workers
                                            .probe_retries
                                            .write_to(@target)

data/lib/gitlab_exporter/prometheus.rb

@@ -15,6 +15,21 @@ module GitLab
         @include_timestamp = include_timestamp
       end
 
+      class << self
+        def describe(name, description)
+          @metric_descriptions ||= {}
+          @metric_descriptions[name] = description
+        end
+
+        def description(name)
+          @metric_descriptions && @metric_descriptions[name]
+        end
+
+        def clear_descriptions
+          @metric_descriptions = {}
+        end
+      end
+
       def add(name, value, quantile = false, **labels)
         fail "value '#{value}' must be a number" unless value.is_a?(Numeric)
 
@@ -32,6 +47,8 @@ module GitLab
 
         buffer = ""
         @metrics.each do |name, measurements|
+          buffer << "# HELP #{name} #{self.class.description(name)}\n" if self.class.description(name)
+
           measurements.each do |measurement|
             buffer << name.to_s
             labels = (measurement[:labels] || {}).map { |label, value| "#{label}=\"#{value}\"" }.join(",")

data/lib/gitlab_exporter/sidekiq.rb

@@ -1,4 +1,5 @@
 require "sidekiq/api"
+require "sidekiq/scheduled"
 require "digest"
 
 module GitLab
@@ -7,8 +8,12 @@ module GitLab
     #
     # It takes the Redis URL Sidekiq is connected to
     class SidekiqProber
-      QUEUE_JOB_STATS_SCRIPT = File.read(File.expand_path("#{__FILE__}/../sidekiq_queue_job_stats.lua")).freeze
-      QUEUE_JOB_STATS_SHA    = Digest::SHA1.hexdigest(QUEUE_JOB_STATS_SCRIPT).freeze
+      # The maximum depth (from the head) of each queue to probe. Probing the
+      # entirety of a very large queue will take longer and run the risk of
+      # timing out. But when we have a very large queue, we are most in need of
+      # reliable metrics. This trades off completeness for predictability by
+      # only taking a limited amount of items from the head of the queue.
+      PROBE_JOBS_LIMIT = 1_000
 
       POOL_SIZE = 3
 
@@ -17,6 +22,9 @@ module GitLab
       # needed to be re-initialized
       POOL_TIMEOUT = 90
 
+      PrometheusMetrics.describe("sidekiq_enqueued_jobs",
+                                 "Total number of jobs enqueued by class name. Only inspects the first #{PROBE_JOBS_LIMIT} jobs per queue.") # rubocop:disable Layout/LineLength
+
       def self.connection_pool
         @@connection_pool ||= Hash.new do |h, connection_hash| # rubocop:disable Style/ClassVars
           config = connection_hash.merge(pool_timeout: POOL_TIMEOUT, size: POOL_SIZE)
@@ -63,17 +71,51 @@ module GitLab
       end
 
       def probe_jobs
+        puts "[REMOVED] probe_jobs is now considered obsolete and does not emit any metrics,"\
+             " please use probe_jobs_limit instead"
+
+        self
+      end
+
+      def probe_future_sets
+        now = Time.now.to_f
+        with_sidekiq do
+          Sidekiq.redis do |conn|
+            Sidekiq::Scheduled::SETS.each do |set|
+              # Default to 0; if all jobs are due in the future, there is no "negative" delay.
+              delay = 0
+
+              _job, timestamp = conn.zrangebyscore(set, "-inf", now.to_s, limit: [0, 1], withscores: true).first
+              delay = now - timestamp if timestamp
+
+              @metrics.add("sidekiq_#{set}_set_processing_delay_seconds", delay)
+
+              # zcount is O(log(N)) (prob. binary search), so is still quick even with large sets
+              @metrics.add("sidekiq_#{set}_set_backlog_count",
+                           conn.zcount(set, "-inf", now.to_s))
+            end
+          end
+        end
+      end
+
+      # Count worker classes present in Sidekiq queues. This only looks at the
+      # first PROBE_JOBS_LIMIT jobs in each queue. This means that we run a
+      # single LRANGE command for each queue, which does not block other
+      # commands. For queues over PROBE_JOBS_LIMIT in size, this means that we
+      # will not have completely accurate statistics, but the probe performance
+      # will also not degrade as the queue gets larger.
+      def probe_jobs_limit
         with_sidekiq do
-          job_stats = {}
+          job_stats = Hash.new(0)
 
           Sidekiq::Queue.all.each do |queue|
             Sidekiq.redis do |conn|
-              stats = conn.evalsha(QUEUE_JOB_STATS_SHA, ["queue:#{queue.name}"])
-              job_stats.merge!(stats.to_h)
+              conn.lrange("queue:#{queue.name}", 0, PROBE_JOBS_LIMIT).each do |job|
+                job_class = Sidekiq.load_json(job)["class"]
+
+                job_stats[job_class] += 1
+              end
             end
-          rescue Redis::CommandError # Could happen if the script exceeded the maximum run time (5 seconds by default)
-            # FIXME: Should we call SCRIPT KILL?
-            return self
           end
 
           job_stats.each do |class_name, count|
@@ -169,14 +211,9 @@ module GitLab
       def connected?
         return @connected unless @connected.nil?
 
-        # This is also a good "connected check"
         Sidekiq.redis do |conn|
-          # Using administrative commands on conn directly (which is a Redis::Namespace)
-          # will be removed in redis-namespace 2.0.
-          conn.redis.script(:load, QUEUE_JOB_STATS_SCRIPT) unless conn.redis.script(:exists, QUEUE_JOB_STATS_SHA)
+          @connected = (conn.ping == "PONG")
         end
-
-        @connected = true
       rescue Redis::BaseConnectionError => e
         @logger&.error "Error connecting to the Redis: #{e}"
         @connected = false

data/lib/gitlab_exporter/version.rb

@@ -1,5 +1,5 @@
 module GitLab
   module Exporter
-    VERSION = "10.4.0".freeze
+    VERSION = "11.1.0".freeze
   end
 end

data/spec/prometheus_metrics_spec.rb

@@ -23,4 +23,21 @@ describe GitLab::Exporter::PrometheusMetrics do
       subject.add("mymetric", "invalid", mylabel: "x", myotherlabel: "y").to_s
     }.to raise_error(RuntimeError)
   end
+
+  it "supports described metrics" do
+    time = Time.now
+
+    allow(Time).to receive(:now).and_return(time)
+
+    described_class.describe("mymetric", "description")
+    described_class.describe("missingmetric", "otherdescription")
+    subject.add("mymetric", 1.3, mylabel: "x", myotherlabel: "y")
+
+    expect(subject.to_s).to eq(<<~METRICS)
+      # HELP mymetric description
+      mymetric{mylabel="x",myotherlabel="y"} 1.3 #{(time.to_f * 1000).to_i}
+    METRICS
+
+    described_class.clear_descriptions
+  end
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: gitlab-exporter
 version: !ruby/object:Gem::Version
-  version: 10.4.0
+  version: 11.1.0
 platform: ruby
 authors:
 - Pablo Carranza
@@ -189,7 +189,6 @@ files:
 - lib/gitlab_exporter/prometheus.rb
 - lib/gitlab_exporter/ruby.rb
 - lib/gitlab_exporter/sidekiq.rb
-- lib/gitlab_exporter/sidekiq_queue_job_stats.lua
 - lib/gitlab_exporter/util.rb
 - lib/gitlab_exporter/version.rb
 - lib/gitlab_exporter/web_exporter.rb

data/lib/gitlab_exporter/sidekiq_queue_job_stats.lua

@@ -1,42 +0,0 @@
---
--- Adapted from https://github.com/mperham/sidekiq/blob/2f9258e4fe77991c526f7a65c92bcf792eef8338/lib/sidekiq/api.rb#L231
---
-local queue_name      = KEYS[1]
-local initial_size    = redis.call('llen', queue_name)
-local deleted_size    = 0
-local page            = 0
-local page_size       = 2000
-local temp_job_stats  = {}
-local final_job_stats = {}
-
-while true do
-  local range_start = page * page_size - deleted_size
-  local range_end   = range_start + page_size - 1
-  local entries     = redis.call('lrange', queue_name, range_start, range_end)
-
-  if #entries == 0 then
-    break
-  end
-
-  page = page + 1
-
-  for index, entry in next, entries do
-    local class = cjson.decode(entry)['class']
-    if class ~= nil then
-      if temp_job_stats[class] ~= nil then
-        temp_job_stats[class] = temp_job_stats[class] + 1
-      else
-        temp_job_stats[class] = 1
-      end
-    end
-  end
-
-  deleted_size = initial_size - redis.call('llen', queue_name)
-end
-
-for class, count in next, temp_job_stats do
-  local stat_entry = {class, count}
-  table.insert(final_job_stats, stat_entry)
-end
-
-return final_job_stats