gitlab-monitor 4.0.0

data/lib/gitlab_monitor/database.rb

@@ -0,0 +1,13 @@
+module GitLab
+  module Monitor
+    # Database-related classes
+    module Database
+      autoload :Base,                 "gitlab_monitor/database/base"
+      autoload :CiBuildsProber,       "gitlab_monitor/database/ci_builds"
+      autoload :TuplesProber,         "gitlab_monitor/database/tuple_stats"
+      autoload :RowCountProber,       "gitlab_monitor/database/row_count"
+      autoload :BloatProber,          "gitlab_monitor/database/bloat"
+      autoload :RemoteMirrorsProber,  "gitlab_monitor/database/remote_mirrors"
+    end
+  end
+end

data/lib/gitlab_monitor/database/base.rb

@@ -0,0 +1,44 @@
+require "pg"
+require "connection_pool"
+
+module GitLab
+  module Monitor
+    module Database
+      # An abstract class for interacting with DB
+      #
+      # It takes a connection string (e.g. "dbname=test port=5432")
+      class Base
+        def self.connection_pool
+          @connection_pool ||= Hash.new do |h, connection_string|
+            h[connection_string] = ConnectionPool.new(size: 3, timeout: 5) do
+              PG.connect(connection_string)
+            end
+          end
+        end
+
+        def initialize(args)
+          @connection_string = args[:connection_string]
+        end
+
+        def run
+          fail NotImplemented
+        end
+
+        def connection_pool
+          self.class.connection_pool[@connection_string]
+        end
+
+        def with_connection_pool
+          connection_pool.with do |conn|
+            begin
+              yield conn
+            rescue PG::UnableToSend => e
+              conn.reset
+              raise e
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/gitlab_monitor/database/bloat.rb

@@ -0,0 +1,74 @@
+module GitLab
+  module Monitor
+    module Database
+      # Helper to collect bloat metrics.
+      class BloatCollector < Base
+        def run(type = :btree)
+          execute(self.class.query_for(type)).each_with_object({}) do |row, h|
+            h[row["object_name"]] = row
+          end
+        end
+
+        private
+
+        def execute(query)
+          with_connection_pool do |conn|
+            conn.exec(query)
+          end
+        end
+
+        class << self
+          def query_for(type)
+            @queries ||= {}
+
+            return @queries[type] if @queries[type]
+
+            file = File.join(__dir__, "bloat_#{type}.sql")
+            fail "Unknown bloat query file: #{file}" unless File.exist?(file)
+
+            @queries[type] = File.read(file)
+          end
+        end
+      end
+
+      # Prober class to gather bloat metrics
+      class BloatProber
+        METRIC_KEYS = %w(bloat_ratio bloat_size extra_size real_size).freeze
+
+        attr_reader :metrics, :collector, :bloat_types
+
+        def initialize(opts,
+                       metrics: PrometheusMetrics.new,
+                       collector: BloatCollector.new(connection_string: opts[:connection_string]))
+          @metrics = metrics
+          @collector = collector
+          @bloat_types = opts[:bloat_types] || %i(btree table)
+        end
+
+        def probe_db
+          bloat_types.each do |type|
+            probe_for_type(type)
+          end
+        end
+
+        def write_to(target)
+          target.write(metrics.to_s)
+        end
+
+        private
+
+        def probe_for_type(type)
+          collector.run(type).each do |query_name, data|
+            METRIC_KEYS.each do |key|
+              metrics.add("gitlab_database_bloat_#{type}_#{key}", data[key], query_name: query_name)
+            end
+          end
+
+          self
+        rescue PG::ConnectionBad
+          self
+        end
+      end
+    end
+  end
+end

data/lib/gitlab_monitor/database/bloat_btree.sql

@@ -0,0 +1,84 @@
+-- Originally from: https://github.com/ioguix/pgsql-bloat-estimation/blob/master/btree/btree_bloat.sql
+-- WARNING: executed with a non-superuser role, the query inspect only index on tables you are granted to read.
+-- WARNING: rows with is_na = 't' are known to have bad statistics ("name" type is not supported).
+-- This query is compatible with PostgreSQL 8.2 and after
+SELECT current_database(), nspname AS schemaname, tblname, idxname AS object_name, bs*(relpages)::bigint AS real_size,
+  bs*(relpages-est_pages)::bigint AS extra_size,
+  100 * (relpages-est_pages)::float / relpages AS extra_ratio,
+  fillfactor,
+  CASE WHEN relpages > est_pages_ff
+    THEN bs*(relpages-est_pages_ff)
+    ELSE 0
+  END AS bloat_size,
+  100 * (relpages-est_pages_ff)::float / relpages AS bloat_ratio,
+  is_na
+  -- , 100-(sub.pst).avg_leaf_density, est_pages, index_tuple_hdr_bm, maxalign, pagehdr, nulldatawidth, nulldatahdrwidth, sub.reltuples, sub.relpages -- (DEBUG INFO)
+FROM (
+  SELECT coalesce(1 +
+       ceil(reltuples/floor((bs-pageopqdata-pagehdr)/(4+nulldatahdrwidth)::float)), 0 -- ItemIdData size + computed avg size of a tuple (nulldatahdrwidth)
+    ) AS est_pages,
+    coalesce(1 +
+       ceil(reltuples/floor((bs-pageopqdata-pagehdr)*fillfactor/(100*(4+nulldatahdrwidth)::float))), 0
+    ) AS est_pages_ff,
+    bs, nspname, table_oid, tblname, idxname, relpages, fillfactor, is_na
+    -- , stattuple.pgstatindex(quote_ident(nspname)||'.'||quote_ident(idxname)) AS pst, index_tuple_hdr_bm, maxalign, pagehdr, nulldatawidth, nulldatahdrwidth, reltuples -- (DEBUG INFO)
+  FROM (
+    SELECT maxalign, bs, nspname, tblname, idxname, reltuples, relpages, relam, table_oid, fillfactor,
+      ( index_tuple_hdr_bm +
+          maxalign - CASE -- Add padding to the index tuple header to align on MAXALIGN
+            WHEN index_tuple_hdr_bm%maxalign = 0 THEN maxalign
+            ELSE index_tuple_hdr_bm%maxalign
+          END
+        + nulldatawidth + maxalign - CASE -- Add padding to the data to align on MAXALIGN
+            WHEN nulldatawidth = 0 THEN 0
+            WHEN nulldatawidth::integer%maxalign = 0 THEN maxalign
+            ELSE nulldatawidth::integer%maxalign
+          END
+      )::numeric AS nulldatahdrwidth, pagehdr, pageopqdata, is_na
+      -- , index_tuple_hdr_bm, nulldatawidth -- (DEBUG INFO)
+    FROM (
+      SELECT
+        i.nspname, i.tblname, i.idxname, i.reltuples, i.relpages, i.relam, a.attrelid AS table_oid,
+        current_setting('block_size')::numeric AS bs, fillfactor,
+        CASE -- MAXALIGN: 4 on 32bits, 8 on 64bits (and mingw32 ?)
+          WHEN version() ~ 'mingw32' OR version() ~ '64-bit|x86_64|ppc64|ia64|amd64' THEN 8
+          ELSE 4
+        END AS maxalign,
+        /* per page header, fixed size: 20 for 7.X, 24 for others */
+        24 AS pagehdr,
+        /* per page btree opaque data */
+        16 AS pageopqdata,
+        /* per tuple header: add IndexAttributeBitMapData if some cols are null-able */
+        CASE WHEN max(coalesce(s.null_frac,0)) = 0
+          THEN 2 -- IndexTupleData size
+          ELSE 2 + (( 32 + 8 - 1 ) / 8) -- IndexTupleData size + IndexAttributeBitMapData size ( max num filed per index + 8 - 1 /8)
+        END AS index_tuple_hdr_bm,
+        /* data len: we remove null values save space using it fractionnal part from stats */
+        sum( (1-coalesce(s.null_frac, 0)) * coalesce(s.avg_width, 1024)) AS nulldatawidth,
+        max( CASE WHEN a.atttypid = 'pg_catalog.name'::regtype THEN 1 ELSE 0 END ) > 0 AS is_na
+      FROM pg_attribute AS a
+        JOIN (
+          SELECT nspname, tbl.relname AS tblname, idx.relname AS idxname, idx.reltuples, idx.relpages, idx.relam,
+            indrelid, indexrelid, indkey::smallint[] AS attnum,
+            coalesce(substring(
+              array_to_string(idx.reloptions, ' ')
+               from 'fillfactor=([0-9]+)')::smallint, 90) AS fillfactor
+          FROM pg_index
+            JOIN pg_class idx ON idx.oid=pg_index.indexrelid
+            JOIN pg_class tbl ON tbl.oid=pg_index.indrelid
+            JOIN pg_namespace ON pg_namespace.oid = idx.relnamespace
+          WHERE pg_index.indisvalid AND tbl.relkind = 'r' AND idx.relpages > 0
+        ) AS i ON a.attrelid = i.indexrelid
+        JOIN pg_stats AS s ON s.schemaname = i.nspname
+          AND ((s.tablename = i.tblname AND s.attname = pg_catalog.pg_get_indexdef(a.attrelid, a.attnum, TRUE)) -- stats from tbl
+          OR   (s.tablename = i.idxname AND s.attname = a.attname))-- stats from functionnal cols
+        JOIN pg_type AS t ON a.atttypid = t.oid
+      WHERE a.attnum > 0
+      GROUP BY 1, 2, 3, 4, 5, 6, 7, 8, 9
+    ) AS s1
+  ) AS s2
+    JOIN pg_am am ON s2.relam = am.oid WHERE am.amname = 'btree'
+) AS sub
+WHERE NOT is_na
+AND   nspname = 'public'
+ORDER BY 2,3,4;

data/lib/gitlab_monitor/database/bloat_table.sql

@@ -0,0 +1,63 @@
+-- Originally from: https://github.com/ioguix/pgsql-bloat-estimation/blob/master/table/table_bloat.sql
+/* WARNING: executed with a non-superuser role, the query inspect only tables you are granted to read.
+* This query is compatible with PostgreSQL 9.0 and more
+*/
+SELECT current_database(), schemaname, tblname AS object_name, bs*tblpages AS real_size,
+  (tblpages-est_tblpages)*bs AS extra_size,
+  CASE WHEN tblpages - est_tblpages > 0
+    THEN 100 * (tblpages - est_tblpages)/tblpages::float
+    ELSE 0
+  END AS extra_ratio, fillfactor,
+  CASE WHEN tblpages - est_tblpages_ff > 0
+    THEN (tblpages-est_tblpages_ff)*bs
+    ELSE 0
+  END AS bloat_size,
+  CASE WHEN tblpages - est_tblpages_ff > 0
+    THEN 100 * (tblpages - est_tblpages_ff)/tblpages::float
+    ELSE 0
+  END AS bloat_ratio, is_na
+  -- , (pst).free_percent + (pst).dead_tuple_percent AS real_frag
+FROM (
+  SELECT ceil( reltuples / ( (bs-page_hdr)/tpl_size ) ) + ceil( toasttuples / 4 ) AS est_tblpages,
+    ceil( reltuples / ( (bs-page_hdr)*fillfactor/(tpl_size*100) ) ) + ceil( toasttuples / 4 ) AS est_tblpages_ff,
+    tblpages, fillfactor, bs, tblid, schemaname, tblname, heappages, toastpages, is_na
+    -- , stattuple.pgstattuple(tblid) AS pst
+  FROM (
+    SELECT
+      ( 4 + tpl_hdr_size + tpl_data_size + (2*ma)
+        - CASE WHEN tpl_hdr_size%ma = 0 THEN ma ELSE tpl_hdr_size%ma END
+        - CASE WHEN ceil(tpl_data_size)::int%ma = 0 THEN ma ELSE ceil(tpl_data_size)::int%ma END
+      ) AS tpl_size, bs - page_hdr AS size_per_block, (heappages + toastpages) AS tblpages, heappages,
+      toastpages, reltuples, toasttuples, bs, page_hdr, tblid, schemaname, tblname, fillfactor, is_na
+    FROM (
+      SELECT
+        tbl.oid AS tblid, ns.nspname AS schemaname, tbl.relname AS tblname, tbl.reltuples,
+        tbl.relpages AS heappages, coalesce(toast.relpages, 0) AS toastpages,
+        coalesce(toast.reltuples, 0) AS toasttuples,
+        coalesce(substring(
+          array_to_string(tbl.reloptions, ' ')
+          FROM 'fillfactor=([0-9]+)')::smallint, 100) AS fillfactor,
+        current_setting('block_size')::numeric AS bs,
+        CASE WHEN version()~'mingw32' OR version()~'64-bit|x86_64|ppc64|ia64|amd64' THEN 8 ELSE 4 END AS ma,
+        24 AS page_hdr,
+        23 + CASE WHEN MAX(coalesce(null_frac,0)) > 0 THEN ( 7 + count(*) ) / 8 ELSE 0::int END
+          + CASE WHEN tbl.relhasoids THEN 4 ELSE 0 END AS tpl_hdr_size,
+        sum( (1-coalesce(s.null_frac, 0)) * coalesce(s.avg_width, 1024) ) AS tpl_data_size,
+        bool_or(att.atttypid = 'pg_catalog.name'::regtype)
+          OR count(att.attname) <> count(s.attname) AS is_na
+      FROM pg_attribute AS att
+        JOIN pg_class AS tbl ON att.attrelid = tbl.oid
+        JOIN pg_namespace AS ns ON ns.oid = tbl.relnamespace
+        LEFT JOIN pg_stats AS s ON s.schemaname=ns.nspname
+          AND s.tablename = tbl.relname AND s.inherited=false AND s.attname=att.attname
+        LEFT JOIN pg_class AS toast ON tbl.reltoastrelid = toast.oid
+      WHERE att.attnum > 0 AND NOT att.attisdropped
+        AND tbl.relkind = 'r'
+      GROUP BY 1,2,3,4,5,6,7,8,9,10, tbl.relhasoids
+      ORDER BY 2,3
+    ) AS s
+  ) AS s2
+) AS s3
+WHERE NOT is_na
+--   AND tblpages*((pst).free_percent + (pst).dead_tuple_percent)::float4/100 >= 1
+AND   schemaname= 'public';

data/lib/gitlab_monitor/database/ci_builds.rb

@@ -0,0 +1,527 @@
+module GitLab
+  module Monitor
+    module Database
+      # A helper class to collect CI builds metrics.
+      class CiBuildsCollector < Base # rubocop:disable Metrics/ClassLength
+        SET_RANDOM_PAGE_COST = "SET LOCAL random_page_cost TO 1".freeze
+
+        BUILDS_QUERY_EE =
+          <<~SQL.freeze
+            SELECT
+              projects.namespace_id,
+              ci_builds.status,
+              projects.shared_runners_enabled,
+              (COALESCE(namespaces.shared_runners_minutes_limit, application_settings.shared_runners_minutes, 0) = 0 OR
+                 COALESCE(namespace_statistics.shared_runners_seconds, 0) < COALESCE(namespaces.shared_runners_minutes_limit, application_settings.shared_runners_minutes, 0) * 60) as has_minutes,
+              COUNT(*) AS count
+            FROM ci_builds
+            JOIN projects
+              ON projects.id = ci_builds.project_id
+            JOIN namespaces
+              ON namespaces.id = projects.namespace_id
+            LEFT JOIN namespace_statistics
+              ON namespace_statistics.namespace_id = namespaces.id
+            JOIN application_settings
+              ON application_settings.id = 1
+            WHERE ci_builds.type = 'Ci::Build'
+              AND ci_builds.status = '%s'
+              AND projects.pending_delete = 'f'
+            GROUP BY
+              projects.namespace_id,
+              ci_builds.status,
+              projects.shared_runners_enabled,
+              namespaces.shared_runners_minutes_limit,
+              namespace_statistics.shared_runners_seconds,
+              application_settings.shared_runners_minutes
+          SQL
+
+        BUILDS_QUERY_CE =
+          <<~SQL.freeze
+            SELECT
+              projects.namespace_id,
+              ci_builds.status,
+              projects.shared_runners_enabled,
+              COUNT(*) AS count
+            FROM ci_builds
+            JOIN projects
+              ON projects.id = ci_builds.project_id
+            WHERE ci_builds.type = 'Ci::Build'
+              AND ci_builds.status = '%s'
+              AND projects.pending_delete = 'f'
+            GROUP BY
+              projects.namespace_id,
+              ci_builds.status,
+              projects.shared_runners_enabled
+          SQL
+
+        STALE_BUILDS_QUERY =
+          <<~SQL.freeze
+            SELECT
+              COUNT(*) AS count
+            FROM ci_builds
+            JOIN projects
+              ON projects.id = ci_builds.project_id
+            WHERE ci_builds.type = 'Ci::Build'
+              AND ci_builds.status = 'running'
+              AND ci_builds.updated_at < NOW() - INTERVAL '1 hour'
+              AND projects.pending_delete = 'f'
+          SQL
+
+        PER_RUNNER_QUERY_EE =
+          <<~SQL.freeze
+            SELECT
+              ci_builds.runner_id,
+              ci_runners.is_shared,
+              projects.namespace_id,
+              projects.mirror,
+              projects.mirror_trigger_builds,
+              ci_pipelines.pipeline_schedule_id,
+              ci_builds.trigger_request_id,
+              (COALESCE(namespaces.shared_runners_minutes_limit, application_settings.shared_runners_minutes, 0) = 0 OR
+                 COALESCE(namespace_statistics.shared_runners_seconds, 0) < COALESCE(namespaces.shared_runners_minutes_limit, application_settings.shared_runners_minutes, 0) * 60) as has_minutes,
+              COUNT(*) AS count
+            FROM ci_builds
+            JOIN ci_runners
+              ON ci_runners.id = ci_builds.runner_id
+            JOIN projects
+              ON projects.id = ci_builds.project_id
+            JOIN ci_pipelines
+              ON ci_pipelines.id = ci_builds.commit_id
+            JOIN namespaces
+              ON namespaces.id = projects.namespace_id
+            LEFT JOIN namespace_statistics
+              ON namespace_statistics.namespace_id = namespaces.id
+            JOIN application_settings
+              ON application_settings.id = 1
+            WHERE ci_builds.type = 'Ci::Build'
+              AND ci_builds.status = 'running'
+              AND projects.pending_delete = 'f'
+            GROUP BY
+              ci_builds.runner_id,
+              ci_runners.is_shared,
+              projects.namespace_id,
+              projects.mirror,
+              projects.mirror_trigger_builds,
+              ci_pipelines.pipeline_schedule_id,
+              ci_builds.trigger_request_id,
+              namespaces.shared_runners_minutes_limit,
+              namespace_statistics.shared_runners_seconds,
+              application_settings.shared_runners_minutes
+          SQL
+
+        PER_RUNNER_QUERY_CE =
+          <<~SQL.freeze
+            SELECT
+              ci_builds.runner_id,
+              ci_runners.is_shared,
+              projects.namespace_id,
+              ci_pipelines.pipeline_schedule_id,
+              ci_builds.trigger_request_id,
+              COUNT(*) AS count
+            FROM ci_builds
+            JOIN ci_runners
+              ON ci_runners.id = ci_builds.runner_id
+            JOIN projects
+              ON projects.id = ci_builds.project_id
+            JOIN ci_pipelines
+              ON ci_pipelines.id = ci_builds.commit_id
+            WHERE ci_builds.type = 'Ci::Build'
+              AND ci_builds.status = 'running'
+              AND projects.pending_delete = 'f'
+            GROUP BY
+              ci_builds.runner_id,
+              ci_runners.is_shared,
+              projects.namespace_id,
+              ci_pipelines.pipeline_schedule_id,
+              ci_builds.trigger_request_id
+          SQL
+
+        MIRROR_COLUMN_QUERY =
+          <<~SQL.freeze
+            SELECT EXISTS (SELECT 1 FROM information_schema.columns WHERE table_name='projects' AND column_name='mirror')
+          SQL
+
+        REPEATED_COMMANDS_QUERY_EE =
+          <<~SQL.freeze
+            SELECT
+              subquery.namespace_id,
+              subquery.shared_runners_enabled,
+              subquery.project_id,
+              subquery.status,
+              subquery.has_minutes,
+              MAX(subquery.count) as count
+            FROM (
+              SELECT
+                projects.namespace_id,
+                projects.shared_runners_enabled,
+                ci_builds.project_id,
+                ci_builds.commit_id,
+                ci_builds.status,
+                (COALESCE(namespaces.shared_runners_minutes_limit, application_settings.shared_runners_minutes, 0) = 0 OR
+                   COALESCE(namespace_statistics.shared_runners_seconds, 0) < COALESCE(namespaces.shared_runners_minutes_limit, application_settings.shared_runners_minutes, 0) * 60) as has_minutes,
+                COUNT(*) AS count
+              FROM ci_builds
+              JOIN projects
+                ON projects.id = ci_builds.project_id
+              JOIN namespaces
+                ON namespaces.id = projects.namespace_id
+              LEFT JOIN namespace_statistics
+                ON namespace_statistics.namespace_id = namespaces.id
+              JOIN application_settings
+                ON application_settings.id = 1
+              WHERE ci_builds.type = 'Ci::Build'
+                AND ci_builds.status IN ('running', 'pending')
+              GROUP BY
+                projects.namespace_id,
+                projects.shared_runners_enabled,
+                ci_builds.project_id,
+                ci_builds.commit_id,
+                ci_builds.status,
+                ci_builds.commands,
+                namespaces.shared_runners_minutes_limit,
+                namespace_statistics.shared_runners_seconds,
+                application_settings.shared_runners_minutes
+              HAVING COUNT(*) > %d
+            ) AS subquery
+            GROUP BY
+              subquery.namespace_id,
+              subquery.shared_runners_enabled,
+              subquery.project_id,
+              subquery.commit_id,
+              subquery.status,
+              subquery.has_minutes
+          SQL
+
+        REPEATED_COMMANDS_QUERY_CE =
+          <<~SQL.freeze
+            SELECT
+              subquery.namespace_id,
+              subquery.shared_runners_enabled,
+              subquery.project_id,
+              subquery.status,
+              MAX(subquery.count) as count
+            FROM (
+              SELECT
+                projects.namespace_id,
+                projects.shared_runners_enabled,
+                ci_builds.project_id,
+                ci_builds.commit_id,
+                ci_builds.status,
+                COUNT(*) AS count
+              FROM ci_builds
+              JOIN projects
+                ON projects.id = ci_builds.project_id
+              JOIN namespaces
+                ON namespaces.id = projects.namespace_id
+              WHERE ci_builds.type = 'Ci::Build'
+                AND ci_builds.status IN ('running', 'pending')
+              GROUP BY
+                projects.namespace_id,
+                projects.shared_runners_enabled,
+                ci_builds.project_id,
+                ci_builds.commit_id,
+                ci_builds.status,
+                ci_builds.commands
+              HAVING COUNT(*) > %d
+            ) AS subquery
+            GROUP BY
+              subquery.namespace_id,
+              subquery.shared_runners_enabled,
+              subquery.project_id,
+              subquery.commit_id,
+              subquery.status
+          SQL
+
+        UNARCHIVED_TRACES_QUERY =
+          <<~SQL.freeze
+            SELECT
+              COUNT(*) as count
+            FROM ci_builds
+            JOIN ci_build_trace_chunks
+              ON ci_build_trace_chunks.build_id = ci_builds.id
+            LEFT JOIN ci_job_artifacts
+              ON ci_job_artifacts.job_id = ci_builds.id
+              AND ci_job_artifacts.file_type = 3
+            WHERE ci_builds.type = 'Ci::Build'
+              AND ci_builds.status IN ('success', 'failed', 'canceled')
+              AND ci_builds.finished_at < '%s'
+              AND ci_job_artifacts.job_id IS NULL
+          SQL
+
+        STATUS_CREATED = "created".freeze
+        STATUS_PENDING = "pending".freeze
+
+        DEFAULT_UNARCHIVED_TRACES_OFFSET_MINUTES = 1440
+
+        def initialize(opts)
+          super(opts)
+
+          @allowed_repeated_commands_count = opts[:allowed_repeated_commands_count]
+          @created_builds_counting_disabled = opts[:created_builds_counting_disabled]
+          @unarchived_traces_offset_minutes = opts[:unarchived_traces_offset_minutes]
+        end
+
+        def run
+          results = {}
+          results[:created_builds] = builds(STATUS_CREATED) unless @created_builds_counting_disabled
+          results[:pending_builds] = builds(STATUS_PENDING)
+          results[:stale_builds] = stale_builds
+          results[:per_runner] = per_runner_builds
+          results[:repeated_commands] = repeated_commands
+          results[:unarchived_traces] = unarchived_traces
+          results
+        end
+
+        private
+
+        def builds(status)
+          results = []
+
+          query = mirror_column? ? BUILDS_QUERY_EE : BUILDS_QUERY_CE
+          query = query % [status] # rubocop:disable Style/FormatString
+          exec_query_with_custom_random_page_cost(query).each do |row|
+            results << transform_builds_row_to_values(row)
+          end
+
+          results
+        rescue PG::UndefinedTable, PG::UndefinedColumn
+          results
+        end
+
+        def transform_builds_row_to_values(row)
+          values = { namespace: row["namespace_id"].to_s,
+                     shared_runners: row["shared_runners_enabled"] == "t" ? "yes" : "no",
+                     value: row["count"].to_i }
+          include_ee_fields(values, row)
+        end
+
+        def stale_builds
+          with_connection_pool do |conn|
+            conn.exec(STALE_BUILDS_QUERY)[0]["count"].to_i
+          end
+        rescue PG::UndefinedTable, PG::UndefinedColumn
+          0
+        end
+
+        def per_runner_builds
+          results = []
+
+          query = mirror_column? ? PER_RUNNER_QUERY_EE : PER_RUNNER_QUERY_CE
+          exec_query_with_custom_random_page_cost(query).each do |row|
+            results << transform_per_runners_builds_row_to_values(row)
+          end
+
+          results
+        rescue PG::UndefinedTable, PG::UndefinedColumn
+          []
+        end
+
+        def transform_per_runners_builds_row_to_values(row)
+          values = { runner: row["runner_id"].to_s,
+                     shared_runner: row["is_shared"] == "t" ? "yes" : "no",
+                     namespace: row["namespace_id"].to_s,
+                     scheduled: row["pipeline_schedule_id"] ? "yes" : "no",
+                     triggered: row["trigger_request_id"] ? "yes" : "no",
+                     value: row["count"].to_i }
+          include_ee_fields(values, row)
+        end
+
+        def repeated_commands
+          results = []
+
+          query = mirror_column? ? REPEATED_COMMANDS_QUERY_EE : REPEATED_COMMANDS_QUERY_CE
+          query = query % [allowed_repeated_commands_count] # rubocop:disable Style/FormatString
+          exec_query_with_custom_random_page_cost(query).each do |row|
+            results << transform_repeated_commands_row_to_values(row)
+          end
+
+          results
+        rescue PG::UndefinedTable, PG::UndefinedColumn
+          []
+        end
+
+        def allowed_repeated_commands_count
+          @allowed_repeated_commands_count ||= 2
+        end
+
+        def transform_repeated_commands_row_to_values(row)
+          values = { namespace: row["namespace_id"].to_s,
+                     project: row["project_id"].to_s,
+                     shared_runners: row["shared_runners_enabled"] == "t" ? "yes" : "no",
+                     status: row["status"].to_s,
+                     value: row["count"].to_i }
+
+          include_has_minutes_field(values, row)
+        end
+
+        def unarchived_traces
+          time = Time.now - (unarchived_traces_offset_minutes * 60)
+          query = UNARCHIVED_TRACES_QUERY % [time.strftime("%F %T")] # rubocop:disable Style/FormatString
+
+          with_connection_pool do |conn|
+            conn.exec(query)[0]["count"].to_i
+          end
+        rescue PG::UndefinedTable, PG::UndefinedColumn
+          0
+        end
+
+        def unarchived_traces_offset_minutes
+          @unarchived_traces_offset_minutes ||= DEFAULT_UNARCHIVED_TRACES_OFFSET_MINUTES
+        end
+
+        def include_ee_fields(values, row)
+          values.merge!(include_bool_if_row_defined(row, :mirror))
+          values.merge!(include_bool_if_row_defined(row, :mirror_trigger_builds))
+          include_has_minutes_field(values, row)
+        end
+
+        def include_has_minutes_field(values, row)
+          values.merge!(include_bool_if_row_defined(row, :has_minutes))
+          values
+        end
+
+        def include_bool_if_row_defined(row, field)
+          return {} unless row[field.to_s]
+          { field => row[field.to_s] == "t" ? "yes" : "no" }
+        end
+
+        def exec_query_with_custom_random_page_cost(query)
+          with_connection_pool do |conn|
+            conn.transaction do |trans|
+              trans.exec(SET_RANDOM_PAGE_COST)
+              trans.exec(query)
+            end
+          end
+        end
+
+        def mirror_column?
+          @mirror_column ||=
+            begin
+              with_connection_pool do |conn|
+                conn.exec(MIRROR_COLUMN_QUERY)[0]["exists"] == "t"
+              end
+            rescue PG::UndefinedColumn
+              false
+            end
+        end
+      end
+
+      # The prober which is called when gathering metrics
+      class CiBuildsProber
+        def initialize(opts, metrics: PrometheusMetrics.new)
+          @metrics = metrics
+
+          collector_opts = { connection_string: opts[:connection_string],
+                             allowed_repeated_commands_count: opts[:allowed_repeated_commands_count],
+                             created_builds_counting_disabled: opts[:created_builds_counting_disabled],
+                             unarchived_traces_offset_minutes: opts[:unarchived_traces_offset_minutes] }
+          @collector = CiBuildsCollector.new(collector_opts)
+        end
+
+        def probe_db
+          @results = @collector.run
+
+          ci_builds_metrics(@results[:created_builds], "ci_created_builds") if @results[:created_builds]
+          ci_builds_metrics(@results[:pending_builds], "ci_pending_builds")
+          ci_stale_builds_metrics
+          metrics_per_runner
+          repeated_commands_metrics
+          unarchived_traces_metrics
+
+          self
+        rescue PG::ConnectionBad
+          self
+        end
+
+        def write_to(target)
+          target.write(@metrics.to_s)
+        end
+
+        private
+
+        def ci_builds_metrics(results_list, metric_name)
+          other_values = {}
+
+          results_list.each do |metric|
+            # If we have a low value, put the value into an "other" bucket.
+            if metric[:value] < 10
+              key = { shared_runners: metric[:shared_runners] }
+              key[:has_minutes] = metric[:has_minutes] if metric[:has_minutes]
+
+              other_values[key] ||= 0
+              other_values[key] += metric[:value]
+            else
+              add_ci_created_pending_builds(metric_name, metric[:value], metric)
+            end
+          end
+
+          # Add metrics for the "other" bucket.
+          other_values.each { |key, value| add_ci_created_pending_builds(metric_name, value, key) }
+        end
+
+        def add_ci_created_pending_builds(metric_name, value, labels)
+          add_metric_with_namespace_label(metric_name,
+                                          [:namespace, :shared_runners, :has_minutes],
+                                          value,
+                                          labels)
+        end
+
+        def ci_stale_builds_metrics
+          @metrics.add("ci_stale_builds", @results[:stale_builds])
+        end
+
+        def metrics_per_runner
+          other_values = {}
+
+          @results[:per_runner].each do |metric|
+            # If we have a low value, put the value into an "other" bucket.
+            if metric[:value] < 10
+              key = { runner: metric[:runner], shared_runner: metric[:shared_runner],
+                      scheduled: metric[:scheduled], triggered: metric[:triggered] }
+              key[:mirror] = metric[:mirror] if metric[:mirror]
+              key[:mirror_trigger_builds] = metric[:mirror_trigger_builds] if metric[:mirror_trigger_builds]
+              key[:has_minutes] = metric[:has_minutes] if metric[:has_minutes]
+
+              other_values[key] ||= 0
+              other_values[key] += metric[:value]
+            else
+              add_ci_running_builds(metric[:value], metric)
+            end
+          end
+
+          # Add metrics for the "other" bucket.
+          other_values.each { |key, value| add_ci_running_builds(value, key) }
+        end
+
+        def add_ci_running_builds(value, labels)
+          add_metric_with_namespace_label(
+            "ci_running_builds",
+            [:runner, :namespace, :shared_runner, :scheduled,
+             :triggered, :mirror, :mirror_trigger_builds, :has_minutes],
+            value,
+            labels
+          )
+        end
+
+        def add_metric_with_namespace_label(metric_name, allowed_labels, value, labels)
+          labels[:namespace] = "" unless labels[:namespace]
+
+          selected_labels = labels.select { |k, _| allowed_labels.include?(k) }.sort.to_h
+          @metrics.add(metric_name, value, selected_labels)
+        end
+
+        def repeated_commands_metrics
+          @results[:repeated_commands].each do |metric|
+            value = metric.delete(:value)
+
+            @metrics.add("ci_repeated_commands_builds", value, metric)
+          end
+        end
+
+        def unarchived_traces_metrics
+          @metrics.add("ci_unarchived_traces", @results[:unarchived_traces])
+        end
+      end
+    end
+  end
+end