gitlab-exporter 7.1.2 → 10.0.0

data/lib/gitlab_exporter/database/bloat_table.sql

@@ -1,5 +1,5 @@
 -- Originally from: https://github.com/ioguix/pgsql-bloat-estimation/blob/master/table/table_bloat.sql
-/* WARNING: executed with a non-superuser role, the query inspect only tables you are granted to read.
+/* WARNING: executed with a non-superuser role, the query inspect only tables and materialized view (9.3+) you are granted to read.
 * This query is compatible with PostgreSQL 9.0 and more
 */
 SELECT current_database(), schemaname, tblname AS object_name, bs*tblpages AS real_size,
@@ -16,12 +16,12 @@ SELECT current_database(), schemaname, tblname AS object_name, bs*tblpages AS re
     THEN 100 * (tblpages - est_tblpages_ff)/tblpages::float
     ELSE 0
   END AS bloat_ratio, is_na
-  -- , (pst).free_percent + (pst).dead_tuple_percent AS real_frag
+  -- , tpl_hdr_size, tpl_data_size, (pst).free_percent + (pst).dead_tuple_percent AS real_frag -- (DEBUG INFO)
 FROM (
   SELECT ceil( reltuples / ( (bs-page_hdr)/tpl_size ) ) + ceil( toasttuples / 4 ) AS est_tblpages,
     ceil( reltuples / ( (bs-page_hdr)*fillfactor/(tpl_size*100) ) ) + ceil( toasttuples / 4 ) AS est_tblpages_ff,
     tblpages, fillfactor, bs, tblid, schemaname, tblname, heappages, toastpages, is_na
-    -- , stattuple.pgstattuple(tblid) AS pst
+    -- , tpl_hdr_size, tpl_data_size, pgstattuple(tblid) AS pst -- (DEBUG INFO)
   FROM (
     SELECT
       ( 4 + tpl_hdr_size + tpl_data_size + (2*ma)
@@ -29,6 +29,7 @@ FROM (
         - CASE WHEN ceil(tpl_data_size)::int%ma = 0 THEN ma ELSE ceil(tpl_data_size)::int%ma END
       ) AS tpl_size, bs - page_hdr AS size_per_block, (heappages + toastpages) AS tblpages, heappages,
       toastpages, reltuples, toasttuples, bs, page_hdr, tblid, schemaname, tblname, fillfactor, is_na
+      -- , tpl_hdr_size, tpl_data_size
     FROM (
       SELECT
         tbl.oid AS tblid, ns.nspname AS schemaname, tbl.relname AS tblname, tbl.reltuples,
@@ -40,24 +41,24 @@ FROM (
         current_setting('block_size')::numeric AS bs,
         CASE WHEN version()~'mingw32' OR version()~'64-bit|x86_64|ppc64|ia64|amd64' THEN 8 ELSE 4 END AS ma,
         24 AS page_hdr,
-        23 + CASE WHEN MAX(coalesce(null_frac,0)) > 0 THEN ( 7 + count(*) ) / 8 ELSE 0::int END
-          + CASE WHEN tbl.relhasoids THEN 4 ELSE 0 END AS tpl_hdr_size,
-        sum( (1-coalesce(s.null_frac, 0)) * coalesce(s.avg_width, 1024) ) AS tpl_data_size,
+        23 + CASE WHEN MAX(coalesce(s.null_frac,0)) > 0 THEN ( 7 + count(s.attname) ) / 8 ELSE 0::int END
+           + CASE WHEN bool_or(att.attname = 'oid' and att.attnum < 0) THEN 4 ELSE 0 END AS tpl_hdr_size,
+        sum( (1-coalesce(s.null_frac, 0)) * coalesce(s.avg_width, 0) ) AS tpl_data_size,
         bool_or(att.atttypid = 'pg_catalog.name'::regtype)
-          OR count(att.attname) <> count(s.attname) AS is_na
+          OR sum(CASE WHEN att.attnum > 0 THEN 1 ELSE 0 END) <> count(s.attname) AS is_na
       FROM pg_attribute AS att
         JOIN pg_class AS tbl ON att.attrelid = tbl.oid
         JOIN pg_namespace AS ns ON ns.oid = tbl.relnamespace
         LEFT JOIN pg_stats AS s ON s.schemaname=ns.nspname
           AND s.tablename = tbl.relname AND s.inherited=false AND s.attname=att.attname
         LEFT JOIN pg_class AS toast ON tbl.reltoastrelid = toast.oid
-      WHERE att.attnum > 0 AND NOT att.attisdropped
-        AND tbl.relkind = 'r'
-      GROUP BY 1,2,3,4,5,6,7,8,9,10, tbl.relhasoids
+      WHERE NOT att.attisdropped
+        AND tbl.relkind in ('r','m')
+      GROUP BY 1,2,3,4,5,6,7,8,9,10
       ORDER BY 2,3
     ) AS s
   ) AS s2
 ) AS s3
-WHERE NOT is_na
+-- WHERE NOT is_na
 --   AND tblpages*((pst).free_percent + (pst).dead_tuple_percent)::float4/100 >= 1
-AND   schemaname= 'public';
+ORDER BY schemaname, tblname;

data/lib/gitlab_exporter/database/ci_builds.rb

@@ -391,6 +391,7 @@ module GitLab
 
         def include_bool_if_row_defined(row, field)
           return {} unless row[field.to_s]
+
           { field => row[field.to_s] == "t" ? "yes" : "no" }
         end
 
@@ -470,7 +471,7 @@ module GitLab
 
         def add_ci_created_pending_builds(metric_name, value, labels)
           add_metric_with_namespace_label(metric_name,
-                                          [:namespace, :shared_runners, :has_minutes],
+                                          %i[namespace shared_runners has_minutes],
                                           value,
                                           labels)
         end
@@ -505,8 +506,8 @@ module GitLab
         def add_ci_running_builds(value, labels)
           add_metric_with_namespace_label(
             "ci_running_builds",
-            [:runner, :namespace, :runner_type, :scheduled,
-             :triggered, :mirror, :mirror_trigger_builds, :has_minutes],
+            %i[runner namespace runner_type scheduled
+               triggered mirror mirror_trigger_builds has_minutes],
             value,
             labels
           )

data/lib/gitlab_exporter/database/row_count.rb

@@ -29,7 +29,7 @@ module GitLab
 
         MIRROR_QUERY = {
           select: :projects,
-          joins:  <<~SQL,
+          joins: <<~SQL,
             INNER JOIN project_mirror_data ON project_mirror_data.project_id = projects.id
             INNER JOIN namespaces AS root_namespaces ON root_namespaces.id = (
               WITH RECURSIVE "base_and_ancestors" AS (
@@ -112,9 +112,9 @@ module GitLab
               ON users.id = u.user_id",
             where: "user_type IS NULL",
             fields: {
-              admin:        {},
-              external:     {},
-              state:        {},
+              admin: {},
+              external: {},
+              state: {},
               access_level: { definition: "COALESCE(u.access_level, 0)" }
             }
           },
@@ -122,14 +122,14 @@ module GitLab
             select: :projects,
             fields: {
               visibility_level: {},
-              archived:         {}
+              archived: {}
             }
           },
           groups: {
             select: :namespaces,
             fields: {
               visibility_level: {},
-              root:             { definition: "(parent_id IS NULL)" }
+              root: { definition: "(parent_id IS NULL)" }
             }
           }
         }.freeze
@@ -175,30 +175,12 @@ module GitLab
 
         def execute(query)
           with_connection_pool do |conn|
-            conn.exec(query).map_types!(type_map_for_results(conn))
+            conn.exec(query)
           end
         rescue PG::UndefinedTable, PG::UndefinedColumn
           nil
         end
 
-        def type_map_for_results(conn)
-          @type_map_for_results ||= begin
-            tm = PG::BasicTypeMapForResults.new(conn)
-
-            # Remove warning message:
-            # Warning: no type cast defined for type "name" with oid 19.
-            # Please cast this type explicitly to TEXT to be safe for future changes.
-            # Warning: no type cast defined for type "regproc" with oid 24.
-            # Please cast this type explicitly to TEXT to be safe for future changes.
-            [{ "type": "text", "oid": 19 }, { "type": "int4", "oid": 24 }].each do |value|
-              old_coder = tm.coders.find { |c| c.name == value[:type] }
-              tm.add_coder(old_coder.dup.tap { |c| c.oid = value[:oid] })
-            end
-
-            tm
-          end
-        end
-
         # Not private so I can test it without meta programming tricks
         def construct_query(query)
           query_string = "SELECT COUNT(*)"

data/lib/gitlab_exporter/database/tuple_stats.rb

@@ -5,7 +5,7 @@ module GitLab
       #
       # It takes a connection string (e.g. "dbname=test port=5432")
       class TupleStatsCollector < Base
-        COLUMNS = %w(relname seq_tup_read idx_tup_fetch n_tup_ins n_tup_upd n_tup_del n_tup_hot_upd n_dead_tup seq_scan)
+        COLUMNS = %w[relname seq_tup_read idx_tup_fetch n_tup_ins n_tup_upd n_tup_del n_tup_hot_upd n_dead_tup seq_scan]
                   .join(",")
         QUERY   = <<-SQL.freeze
           SELECT #{COLUMNS}
@@ -35,9 +35,11 @@ module GitLab
 
           result.each do |table_name, tuple_stats|
             tuple_stats.each do |column_name, value|
+              next if value.is_a?(Numeric)
+
               @metrics.add("gitlab_database_stat_table_#{column_name}",
                            value.to_f,
-                           table_name: table_name) unless value.is_a?(Numeric)
+                           table_name: table_name)
             end
           end
 

data/lib/gitlab_exporter/git.rb

@@ -13,6 +13,7 @@ module GitLab
     class Git
       def initialize(repo)
         fail "Repository #{repo} does not exists" unless Dir.exist? repo
+
         @repo = repo
         @tracker = TimeTracker.new
       end
@@ -35,6 +36,7 @@ module GitLab
       def execute(command)
         result = CommandResult.new(*Open3.capture2e(command, chdir: @repo))
         fail "Command #{command} failed with status #{result.status}\n#{result.stdout}" if result.failed?
+
         result
       end
     end
@@ -91,12 +93,16 @@ module GitLab
       end
 
       def probe_git # rubocop:disable Metrics/MethodLength
+        puts "[DEPRECATED] probe_git and GitProcessProber are now considered obsolete"\
+          " and will be removed in future major versions,"\
+          " please use git metrics produced by Gitaly instead"
+
         counts = Hash.new(0)
 
         Utils.pgrep("^git ").each do |pid|
           process_cmd = begin
                           File.read("/proc/#{pid}/cmdline")
-                        rescue
+                        rescue StandardError
                           "" # Process file is gone (race condition)
                         end
           subcommand  = self.class.extract_subcommand(process_cmd)
@@ -131,6 +137,7 @@ module GitLab
 
       def self.extract_subcommand(cmd)
         return if cmd.empty?
+
         cmd_splitted = cmd.split("\u0000") # cmdline does not return it space-separated
 
         cmd_splitted.shift # Because it's "git"

data/lib/gitlab_exporter/memstats.rb

@@ -87,7 +87,7 @@ module GitLab
               end
             end
           end
-        rescue => e
+        rescue StandardError => e
           puts "Error: #{e}"
           @valid = false
           create_memstats_not_available(totals)

data/lib/gitlab_exporter/memstats/mapping.rb

@@ -25,7 +25,7 @@ module GitLab
       # Locked:                0 kB
       # VmFlags: rd ex mr mw me dw sd
       class Mapping
-        FIELDS = %w(size rss shared_clean shared_dirty private_clean private_dirty swap pss).freeze
+        FIELDS = %w[size rss shared_clean shared_dirty private_clean private_dirty swap pss].freeze
 
         attr_reader :address_start
         attr_reader :address_end

data/lib/gitlab_exporter/process.rb

@@ -71,6 +71,10 @@ module GitLab
       end
 
       def probe_stat
+        puts "[DEPRECATED] probe_stat and ProcessProber are now considered obsolete"\
+          " and will be removed in future major versions,"\
+          " please use metrics produced by application processes instead"
+
         @pids.each do |pid|
           stats = ProcessStats.new(pid)
           next unless stats.valid?
@@ -88,12 +92,20 @@ module GitLab
       end
 
       def probe_count
+        puts "[DEPRECATED] probe_count and ProcessProber are now considered obsolete"\
+          " and will be removed in future major versions,"\
+          " please use metrics produced by application processes instead"
+
         @metrics.add("process_count", @pids.count, name: @name.downcase)
 
         self
       end
 
       def probe_smaps
+        puts "[DEPRECATED] probe_smaps and ProcessProber are now considered obsolete"\
+          " and will be removed in future major versions,"\
+          " please use metrics produced by application processes instead"
+
         @pids.each do |pid|
           stats = ::GitLab::Exporter::MemStats::Aggregator.new(pid)
 
@@ -105,9 +117,7 @@ module GitLab
           ::GitLab::Exporter::MemStats::Mapping::FIELDS.each do |field|
             value = stats.totals[field]
 
-            if value >= 0
-              @metrics.add("process_smaps_#{field}_bytes", value * 1024, @use_quantiles, **labels)
-            end
+            @metrics.add("process_smaps_#{field}_bytes", value * 1024, @use_quantiles, **labels) if value >= 0
           end
         end
 

data/lib/gitlab_exporter/ruby.rb

@@ -0,0 +1,21 @@
+# frozen_string_literal: true
+
+module GitLab
+  module Exporter
+    # Probes a current process GC for info then writes metrics to a target
+    class RubyProber
+      def initialize(options, metrics: PrometheusMetrics.new, logger: nil) # rubocop:disable Lint/UnusedMethodArgument
+        @metrics = metrics
+        @use_quantiles = options.fetch(:quantiles, false)
+      end
+
+      def probe_gc
+        GC.stat.each do |stat, value|
+          @metrics.add("ruby_gc_stat_#{stat}", value.to_i, @use_quantiles)
+        end
+
+        self
+      end
+    end
+  end
+end

data/lib/gitlab_exporter/sidekiq.rb

@@ -10,56 +10,63 @@ module GitLab
       QUEUE_JOB_STATS_SCRIPT = File.read(File.expand_path("#{__FILE__}/../sidekiq_queue_job_stats.lua")).freeze
       QUEUE_JOB_STATS_SHA    = Digest::SHA1.hexdigest(QUEUE_JOB_STATS_SCRIPT).freeze
 
+      POOL_SIZE = 3
+
+      # This timeout is configured to higher interval than scrapping
+      # of Prometheus to ensure that connection is kept instead of
+      # needed to be re-initialized
+      POOL_TIMEOUT = 90
+
+      def self.connection_pool
+        @@connection_pool ||= Hash.new do |h, connection_hash| # rubocop:disable Style/ClassVars
+          config = connection_hash.merge(pool_timeout: POOL_TIMEOUT, size: POOL_SIZE)
+
+          h[connection_hash] = Sidekiq::RedisConnection.create(config)
+        end
+      end
+
       def initialize(opts, metrics: PrometheusMetrics.new, logger: nil)
         @opts    = opts
         @metrics = metrics
         @logger  = logger
-
-        Sidekiq.configure_client do |config|
-          config.redis = redis_options
-        end
-
-        ensure_queue_job_stats_script_loaded
       end
 
       def probe_stats
-        return self unless connected?
-
-        stats = Sidekiq::Stats.new
-
-        @metrics.add("sidekiq_jobs_processed_total", stats.processed)
-        @metrics.add("sidekiq_jobs_failed_total", stats.failed)
-        @metrics.add("sidekiq_jobs_enqueued_size", stats.enqueued)
-        @metrics.add("sidekiq_jobs_scheduled_size", stats.scheduled_size)
-        @metrics.add("sidekiq_jobs_retry_size", stats.retry_size)
-        @metrics.add("sidekiq_jobs_dead_size", stats.dead_size)
-
-        @metrics.add("sidekiq_default_queue_latency_seconds", stats.default_queue_latency)
-        @metrics.add("sidekiq_processes_size", stats.processes_size)
-        @metrics.add("sidekiq_workers_size", stats.workers_size)
+        with_sidekiq do
+          stats = Sidekiq::Stats.new
+
+          @metrics.add("sidekiq_jobs_processed_total", stats.processed)
+          @metrics.add("sidekiq_jobs_failed_total", stats.failed)
+          @metrics.add("sidekiq_jobs_enqueued_size", stats.enqueued)
+          @metrics.add("sidekiq_jobs_scheduled_size", stats.scheduled_size)
+          @metrics.add("sidekiq_jobs_retry_size", stats.retry_size)
+          @metrics.add("sidekiq_jobs_dead_size", stats.dead_size)
+
+          @metrics.add("sidekiq_default_queue_latency_seconds", stats.default_queue_latency)
+          @metrics.add("sidekiq_processes_size", stats.processes_size)
+          @metrics.add("sidekiq_workers_size", stats.workers_size)
+        end
 
         self
       end
 
       def probe_queues
-        return self unless connected?
-
-        Sidekiq::Queue.all.each do |queue|
-          @metrics.add("sidekiq_queue_size", queue.size, name: queue.name)
-          @metrics.add("sidekiq_queue_latency_seconds", queue.latency, name: queue.name)
-          @metrics.add("sidekiq_queue_paused", queue.paused? ? 1 : 0, name: queue.name)
+        with_sidekiq do
+          Sidekiq::Queue.all.each do |queue|
+            @metrics.add("sidekiq_queue_size", queue.size, name: queue.name)
+            @metrics.add("sidekiq_queue_latency_seconds", queue.latency, name: queue.name)
+            @metrics.add("sidekiq_queue_paused", queue.paused? ? 1 : 0, name: queue.name)
+          end
         end
 
         self
       end
 
-      def probe_jobs # rubocop:disable Metrics/MethodLength
-        return self unless connected?
-
-        job_stats = {}
+      def probe_jobs
+        with_sidekiq do
+          job_stats = {}
 
-        Sidekiq::Queue.all.each do |queue|
-          begin
+          Sidekiq::Queue.all.each do |queue|
             Sidekiq.redis do |conn|
               stats = conn.evalsha(QUEUE_JOB_STATS_SHA, ["queue:#{queue.name}"])
               job_stats.merge!(stats.to_h)
@@ -68,44 +75,44 @@ module GitLab
             # FIXME: Should we call SCRIPT KILL?
             return self
           end
-        end
 
-        job_stats.each do |class_name, count|
-          @metrics.add("sidekiq_enqueued_jobs", count, name: class_name)
+          job_stats.each do |class_name, count|
+            @metrics.add("sidekiq_enqueued_jobs", count, name: class_name)
+          end
         end
 
         self
       end
 
       def probe_workers
-        return self unless connected?
-
-        worker_stats = Hash.new(0)
+        with_sidekiq do
+          worker_stats = Hash.new(0)
 
-        Sidekiq::Workers.new.map do |_pid, _tid, work|
-          job_klass = work["payload"]["class"]
+          Sidekiq::Workers.new.map do |_pid, _tid, work|
+            job_klass = work["payload"]["class"]
 
-          worker_stats[job_klass] += 1
-        end
+            worker_stats[job_klass] += 1
+          end
 
-        worker_stats.each do |class_name, count|
-          @metrics.add("sidekiq_running_jobs", count, name: class_name)
+          worker_stats.each do |class_name, count|
+            @metrics.add("sidekiq_running_jobs", count, name: class_name)
+          end
         end
 
         self
       end
 
       def probe_retries
-        return self unless connected?
+        with_sidekiq do
+          retry_stats = Hash.new(0)
 
-        retry_stats = Hash.new(0)
-
-        Sidekiq::RetrySet.new.map do |job|
-          retry_stats[job.klass] += 1
-        end
+          Sidekiq::RetrySet.new.map do |job|
+            retry_stats[job.klass] += 1
+          end
 
-        retry_stats.each do |class_name, count|
-          @metrics.add("sidekiq_to_be_retried_jobs", count, name: class_name)
+          retry_stats.each do |class_name, count|
+            @metrics.add("sidekiq_to_be_retried_jobs", count, name: class_name)
+          end
         end
 
         self
@@ -115,9 +122,9 @@ module GitLab
         puts "[DEPRECATED] probe_dead is now considered obsolete and will be removed in future major versions,"\
              " please use probe_stats instead"
 
-        return self unless connected?
-
-        @metrics.add("sidekiq_dead_jobs", Sidekiq::Stats.new.dead_size)
+        with_sidekiq do
+          @metrics.add("sidekiq_dead_jobs", Sidekiq::Stats.new.dead_size)
+        end
 
         self
       end
@@ -128,6 +135,19 @@ module GitLab
 
       private
 
+      def with_sidekiq
+        # TODO: this is not concurrent safe as we change global context
+        # It means that we are unable to use many different sidekiq's
+        # which is not a problem as of now
+        Sidekiq.configure_client do |config|
+          config.redis = self.class.connection_pool[redis_options]
+        end
+
+        return unless connected?
+
+        yield
+      end
+
       def redis_options
         options = {
           url: @opts[:redis_url],
@@ -147,25 +167,19 @@ module GitLab
       end
 
       def connected?
-        @connected ||= begin
-                         Sidekiq.redis do |conn|
-                           conn.get("foo")
-                         end
-                         true
-                       end
-      rescue Redis::CannotConnectError, Redis::TimeoutError # rubocop:disable Lint/HandleExceptions
-        # Maybe we're trying connecting to a slave
-      end
-
-      def ensure_queue_job_stats_script_loaded
-        return unless connected?
+        return @connected unless @connected.nil?
 
+        # This is also a good "connected check"
         Sidekiq.redis do |conn|
           # Using administrative commands on conn directly (which is a Redis::Namespace)
           # will be removed in redis-namespace 2.0.
-          next if conn.redis.script(:exists, QUEUE_JOB_STATS_SHA)
-          conn.redis.script(:load, QUEUE_JOB_STATS_SCRIPT)
+          conn.redis.script(:load, QUEUE_JOB_STATS_SCRIPT) unless conn.redis.script(:exists, QUEUE_JOB_STATS_SHA)
         end
+
+        @connected = true
+      rescue Redis::BaseConnectionError => e
+        @logger&.error "Error connecting to the Redis: #{e}"
+        @connected = false
       end
     end
   end