gitlab-exporter 7.0.5 → 7.2.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: de89f0131cde7415ce666ec09c4da388b53f4cbf57b8e7872f3e5538be257e30
-  data.tar.gz: 4b84a64c0d3b26306628ec04dc2ce7b56b36eda349385a43855b1a968c2a8560
+  metadata.gz: 86ad2e8d49582c70ba5faa5e7a8744a0e812f5682447cc1e1f000e050a3b9451
+  data.tar.gz: 2a0f878e42abef28992284e0e372db960831025b7ff3cf71c676369210d05fea
 SHA512:
-  metadata.gz: a292b9c5c4f24fab3014e6bc9ae5f3b86c86ff47d5034fce9280d1fa75372cbcd18e5d75a6e130c9de77fc662b1ef2fa6d1dd9f390a004236c727e311a9e02a3
-  data.tar.gz: 6f1309c2abeeeca930d0485d66b9b714006f1d97814141fba7f6a2b4db063ee41f0181857de0044435d8e3b14d10e844d2ed595a5ce2505fe63a02ef6abce466
+  metadata.gz: 28d6f61e6df16ac17ba9e25ddea5d6a39c65940796898a2a6bad66c352ad47a1341593e3bb8dd7edaefd5281a787612beda9ff6f3f3700c6c458ad345fef055d
+  data.tar.gz: f7b2ff3ddd97529571e0517f4b8f9c7d42fe512f5d786a9c60dfae6337cafc56f5ae8eecaf98e3d85d0cb987e66363102bfec40a6ff4411524359cdf888f9d3e

data/.gitlab-ci.yml

@@ -1,17 +1,26 @@
+include:
+  - template: Security/DAST.gitlab-ci.yml  # https://gitlab.com/gitlab-org/gitlab-foss/blob/master/lib/gitlab/ci/templates/Security/DAST.gitlab-ci.yml
+  - template: Security/Container-Scanning.gitlab-ci.yml  # https://gitlab.com/gitlab-org/gitlab-foss/blob/master/lib/gitlab/ci/templates/Security/Container-Scanning.gitlab-ci.yml
+  - template: Security/Dependency-Scanning.gitlab-ci.yml  # https://gitlab.com/gitlab-org/gitlab-foss/blob/master/lib/gitlab/ci/templates/Security/Dependency-Scanning.gitlab-ci.yml
+  - template: Security/License-Scanning.gitlab-ci.yml  # https://gitlab.com/gitlab-org/gitlab-foss/blob/master/lib/gitlab/ci/templates/Security/License-Scanning.gitlab-ci.yml
+  - template: Security/SAST.gitlab-ci.yml  # https://gitlab.com/gitlab-org/gitlab-foss/blob/master/lib/gitlab/ci/templates/Security/SAST.gitlab-ci.yml
+  - template: Security/Secret-Detection.gitlab-ci.yml # https://gitlab.com/gitlab-org/gitlab-foss/-/blob/master/lib/gitlab/ci/templates/Security/Secret-Detection.gitlab-ci.yml
+
 default:
   image: "ruby:2.3"
-  before_script:
-    - git config --global user.email "bot@gitlab.com"
-    - git config --global user.name "Bot User"
-    - bundle install -j $(nproc) --path vendor
   cache:
     paths:
       - vendor
   tags:
     - gitlab-org
 
+.before_scripts: &before_scripts
+  - git config --global user.email "bot@gitlab.com"
+  - git config --global user.name "Bot User"
+  - bundle install -j $(nproc) --path vendor
+
 workflow:
-  rules:
+  rules: &workflow_rules
     # For merge requests, create a pipeline.
     - if: '$CI_MERGE_REQUEST_IID'
     # For `master` branch, create a pipeline (this includes on schedules, pushes, merges, etc.).
@@ -22,15 +31,18 @@ workflow:
 rspec:
   script:
     - bundle exec rspec spec -f d -c
+  before_script: *before_scripts
 
 rubocop:
   script:
     - bundle exec rubocop
+  before_script: *before_scripts
 
-include:
-  - template: Security/DAST.gitlab-ci.yml  # https://gitlab.com/gitlab-org/gitlab-foss/blob/master/lib/gitlab/ci/templates/Security/DAST.gitlab-ci.yml
-  - template: Security/Container-Scanning.gitlab-ci.yml  # https://gitlab.com/gitlab-org/gitlab-foss/blob/master/lib/gitlab/ci/templates/Security/Container-Scanning.gitlab-ci.yml
-  - template: Security/Dependency-Scanning.gitlab-ci.yml  # https://gitlab.com/gitlab-org/gitlab-foss/blob/master/lib/gitlab/ci/templates/Security/Dependency-Scanning.gitlab-ci.yml
-  - template: Security/License-Scanning.gitlab-ci.yml  # https://gitlab.com/gitlab-org/gitlab-foss/blob/master/lib/gitlab/ci/templates/Security/License-Scanning.gitlab-ci.yml
-  - template: Security/SAST.gitlab-ci.yml  # https://gitlab.com/gitlab-org/gitlab-foss/blob/master/lib/gitlab/ci/templates/Security/SAST.gitlab-ci.yml
+license_scanning:
+  rules: *workflow_rules
+
+gemnasium-dependency_scanning:
+  rules: *workflow_rules
 
+secret_detection:
+  rules: *workflow_rules

data/Gemfile.lock

@@ -1,7 +1,7 @@
 PATH
   remote: .
   specs:
-    gitlab-exporter (7.0.5)
+    gitlab-exporter (7.2.0)
       connection_pool (~> 2.2.1)
       pg (~> 1.1)
       quantile (~> 0.2.0)
@@ -14,19 +14,20 @@ GEM
   remote: https://rubygems.org/
   specs:
     ast (2.4.0)
-    connection_pool (2.2.2)
+    connection_pool (2.2.3)
     diff-lcs (1.3)
-    mustermann (1.0.3)
+    mustermann (1.1.1)
+      ruby2_keywords (~> 0.0.1)
     parser (2.5.1.0)
       ast (~> 2.4.0)
-    pg (1.2.2)
+    pg (1.2.3)
     powerpack (0.1.1)
     quantile (0.2.1)
-    rack (2.0.7)
-    rack-protection (2.0.7)
+    rack (2.2.3)
+    rack-protection (2.0.8.1)
       rack
     rainbow (2.1.0)
-    redis (4.1.3)
+    redis (4.1.4)
     redis-namespace (1.6.0)
       redis (>= 3.0.4)
     rspec (3.7.0)
@@ -49,17 +50,18 @@ GEM
       ruby-progressbar (~> 1.7)
       unicode-display_width (~> 1.0, >= 1.0.1)
     ruby-progressbar (1.8.1)
-    sidekiq (5.2.7)
+    ruby2_keywords (0.0.2)
+    sidekiq (5.2.9)
       connection_pool (~> 2.2, >= 2.2.2)
-      rack (>= 1.5.0)
+      rack (~> 2.0)
       rack-protection (>= 1.5.0)
-      redis (>= 3.3.5, < 5)
-    sinatra (2.0.7)
+      redis (>= 3.3.5, < 4.2)
+    sinatra (2.0.8.1)
       mustermann (~> 1.0)
       rack (~> 2.0)
-      rack-protection (= 2.0.7)
+      rack-protection (= 2.0.8.1)
       tilt (~> 2.0)
-    tilt (2.0.9)
+    tilt (2.0.10)
     unicode-display_width (1.6.0)
 
 PLATFORMS

data/lib/gitlab_exporter/database/base.rb

@@ -16,8 +16,9 @@ module GitLab
           end
         end
 
-        def initialize(args)
+        def initialize(args, logger: nil)
           @connection_string = args[:connection_string]
+          @logger = logger
         end
 
         def run
@@ -33,10 +34,14 @@ module GitLab
             begin
               yield conn
             rescue PG::UnableToSend => e
+              @logger.warn "Error sending to the database: #{e}" if @logger
               conn.reset
               raise e
             end
           end
+        rescue PG::Error => e
+          @logger.error "Error connecting to the database: #{e}" if @logger
+          raise e
         end
       end
     end

data/lib/gitlab_exporter/database/bloat.rb

@@ -3,6 +3,8 @@ module GitLab
     module Database
       # Helper to collect bloat metrics.
       class BloatCollector < Base
+        attr_writer :logger
+
         def run(type = :btree)
           execute(self.class.query_for(type)).each_with_object({}) do |row, h|
             h[row["object_name"]] = row
@@ -39,9 +41,11 @@ module GitLab
 
         def initialize(opts,
                        metrics: PrometheusMetrics.new,
-                       collector: BloatCollector.new(connection_string: opts[:connection_string]))
+                       collector: BloatCollector.new(connection_string: opts[:connection_string]),
+                       logger: nil)
           @metrics = metrics
           @collector = collector
+          @collector.logger = logger
           @bloat_types = opts[:bloat_types] || %i(btree table)
         end
 

data/lib/gitlab_exporter/database/bloat_btree.sql

@@ -12,73 +12,90 @@ SELECT current_database(), nspname AS schemaname, tblname, idxname AS object_nam
   END AS bloat_size,
   100 * (relpages-est_pages_ff)::float / relpages AS bloat_ratio,
   is_na
-  -- , 100-(sub.pst).avg_leaf_density, est_pages, index_tuple_hdr_bm, maxalign, pagehdr, nulldatawidth, nulldatahdrwidth, sub.reltuples, sub.relpages -- (DEBUG INFO)
+  -- , 100-(pst).avg_leaf_density AS pst_avg_bloat, est_pages, index_tuple_hdr_bm, maxalign, pagehdr, nulldatawidth, nulldatahdrwidth, reltuples, relpages -- (DEBUG INFO)
 FROM (
   SELECT coalesce(1 +
-       ceil(reltuples/floor((bs-pageopqdata-pagehdr)/(4+nulldatahdrwidth)::float)), 0 -- ItemIdData size + computed avg size of a tuple (nulldatahdrwidth)
-    ) AS est_pages,
-    coalesce(1 +
-       ceil(reltuples/floor((bs-pageopqdata-pagehdr)*fillfactor/(100*(4+nulldatahdrwidth)::float))), 0
-    ) AS est_pages_ff,
-    bs, nspname, table_oid, tblname, idxname, relpages, fillfactor, is_na
-    -- , stattuple.pgstatindex(quote_ident(nspname)||'.'||quote_ident(idxname)) AS pst, index_tuple_hdr_bm, maxalign, pagehdr, nulldatawidth, nulldatahdrwidth, reltuples -- (DEBUG INFO)
+         ceil(reltuples/floor((bs-pageopqdata-pagehdr)/(4+nulldatahdrwidth)::float)), 0 -- ItemIdData size + computed avg size of a tuple (nulldatahdrwidth)
+      ) AS est_pages,
+      coalesce(1 +
+         ceil(reltuples/floor((bs-pageopqdata-pagehdr)*fillfactor/(100*(4+nulldatahdrwidth)::float))), 0
+      ) AS est_pages_ff,
+      bs, nspname, tblname, idxname, relpages, fillfactor, is_na
+      -- , pgstatindex(idxoid) AS pst, index_tuple_hdr_bm, maxalign, pagehdr, nulldatawidth, nulldatahdrwidth, reltuples -- (DEBUG INFO)
   FROM (
-    SELECT maxalign, bs, nspname, tblname, idxname, reltuples, relpages, relam, table_oid, fillfactor,
-      ( index_tuple_hdr_bm +
-          maxalign - CASE -- Add padding to the index tuple header to align on MAXALIGN
-            WHEN index_tuple_hdr_bm%maxalign = 0 THEN maxalign
-            ELSE index_tuple_hdr_bm%maxalign
-          END
-        + nulldatawidth + maxalign - CASE -- Add padding to the data to align on MAXALIGN
-            WHEN nulldatawidth = 0 THEN 0
-            WHEN nulldatawidth::integer%maxalign = 0 THEN maxalign
-            ELSE nulldatawidth::integer%maxalign
-          END
-      )::numeric AS nulldatahdrwidth, pagehdr, pageopqdata, is_na
-      -- , index_tuple_hdr_bm, nulldatawidth -- (DEBUG INFO)
-    FROM (
-      SELECT
-        i.nspname, i.tblname, i.idxname, i.reltuples, i.relpages, i.relam, a.attrelid AS table_oid,
-        current_setting('block_size')::numeric AS bs, fillfactor,
-        CASE -- MAXALIGN: 4 on 32bits, 8 on 64bits (and mingw32 ?)
-          WHEN version() ~ 'mingw32' OR version() ~ '64-bit|x86_64|ppc64|ia64|amd64' THEN 8
-          ELSE 4
-        END AS maxalign,
-        /* per page header, fixed size: 20 for 7.X, 24 for others */
-        24 AS pagehdr,
-        /* per page btree opaque data */
-        16 AS pageopqdata,
-        /* per tuple header: add IndexAttributeBitMapData if some cols are null-able */
-        CASE WHEN max(coalesce(s.null_frac,0)) = 0
-          THEN 2 -- IndexTupleData size
-          ELSE 2 + (( 32 + 8 - 1 ) / 8) -- IndexTupleData size + IndexAttributeBitMapData size ( max num filed per index + 8 - 1 /8)
-        END AS index_tuple_hdr_bm,
-        /* data len: we remove null values save space using it fractionnal part from stats */
-        sum( (1-coalesce(s.null_frac, 0)) * coalesce(s.avg_width, 1024)) AS nulldatawidth,
-        max( CASE WHEN a.atttypid = 'pg_catalog.name'::regtype THEN 1 ELSE 0 END ) > 0 AS is_na
-      FROM pg_attribute AS a
-        JOIN (
-          SELECT nspname, tbl.relname AS tblname, idx.relname AS idxname, idx.reltuples, idx.relpages, idx.relam,
-            indrelid, indexrelid, indkey::smallint[] AS attnum,
-            coalesce(substring(
-              array_to_string(idx.reloptions, ' ')
-               from 'fillfactor=([0-9]+)')::smallint, 90) AS fillfactor
-          FROM pg_index
-            JOIN pg_class idx ON idx.oid=pg_index.indexrelid
-            JOIN pg_class tbl ON tbl.oid=pg_index.indrelid
-            JOIN pg_namespace ON pg_namespace.oid = idx.relnamespace
-          WHERE pg_index.indisvalid AND tbl.relkind = 'r' AND idx.relpages > 0
-        ) AS i ON a.attrelid = i.indexrelid
-        JOIN pg_stats AS s ON s.schemaname = i.nspname
-          AND ((s.tablename = i.tblname AND s.attname = pg_catalog.pg_get_indexdef(a.attrelid, a.attnum, TRUE)) -- stats from tbl
-          OR   (s.tablename = i.idxname AND s.attname = a.attname))-- stats from functionnal cols
-        JOIN pg_type AS t ON a.atttypid = t.oid
-      WHERE a.attnum > 0
-      GROUP BY 1, 2, 3, 4, 5, 6, 7, 8, 9
-    ) AS s1
-  ) AS s2
-    JOIN pg_am am ON s2.relam = am.oid WHERE am.amname = 'btree'
-) AS sub
-WHERE NOT is_na
-AND   nspname = 'public'
-ORDER BY 2,3,4;
+      SELECT maxalign, bs, nspname, tblname, idxname, reltuples, relpages, idxoid, fillfactor,
+            ( index_tuple_hdr_bm +
+                maxalign - CASE -- Add padding to the index tuple header to align on MAXALIGN
+                  WHEN index_tuple_hdr_bm%maxalign = 0 THEN maxalign
+                  ELSE index_tuple_hdr_bm%maxalign
+                END
+              + nulldatawidth + maxalign - CASE -- Add padding to the data to align on MAXALIGN
+                  WHEN nulldatawidth = 0 THEN 0
+                  WHEN nulldatawidth::integer%maxalign = 0 THEN maxalign
+                  ELSE nulldatawidth::integer%maxalign
+                END
+            )::numeric AS nulldatahdrwidth, pagehdr, pageopqdata, is_na
+            -- , index_tuple_hdr_bm, nulldatawidth -- (DEBUG INFO)
+      FROM (
+          SELECT n.nspname, i.tblname, i.idxname, i.reltuples, i.relpages,
+              i.idxoid, i.fillfactor, current_setting('block_size')::numeric AS bs,
+              CASE -- MAXALIGN: 4 on 32bits, 8 on 64bits (and mingw32 ?)
+                WHEN version() ~ 'mingw32' OR version() ~ '64-bit|x86_64|ppc64|ia64|amd64' THEN 8
+                ELSE 4
+              END AS maxalign,
+              /* per page header, fixed size: 20 for 7.X, 24 for others */
+              24 AS pagehdr,
+              /* per page btree opaque data */
+              16 AS pageopqdata,
+              /* per tuple header: add IndexAttributeBitMapData if some cols are null-able */
+              CASE WHEN max(coalesce(s.null_frac,0)) = 0
+                  THEN 2 -- IndexTupleData size
+                  ELSE 2 + (( 32 + 8 - 1 ) / 8) -- IndexTupleData size + IndexAttributeBitMapData size ( max num filed per index + 8 - 1 /8)
+              END AS index_tuple_hdr_bm,
+              /* data len: we remove null values save space using it fractionnal part from stats */
+              sum( (1-coalesce(s.null_frac, 0)) * coalesce(s.avg_width, 1024)) AS nulldatawidth,
+              max( CASE WHEN i.atttypid = 'pg_catalog.name'::regtype THEN 1 ELSE 0 END ) > 0 AS is_na
+          FROM (
+              SELECT ct.relname AS tblname, ct.relnamespace, ic.idxname, ic.attpos, ic.indkey, ic.indkey[ic.attpos], ic.reltuples, ic.relpages, ic.tbloid, ic.idxoid, ic.fillfactor,
+                  coalesce(a1.attnum, a2.attnum) AS attnum, coalesce(a1.attname, a2.attname) AS attname, coalesce(a1.atttypid, a2.atttypid) AS atttypid,
+                  CASE WHEN a1.attnum IS NULL
+                  THEN ic.idxname
+                  ELSE ct.relname
+                  END AS attrelname
+              FROM (
+                  SELECT idxname, reltuples, relpages, tbloid, idxoid, fillfactor, indkey,
+                      pg_catalog.generate_series(1,indnatts) AS attpos
+                  FROM (
+                      SELECT ci.relname AS idxname, ci.reltuples, ci.relpages, i.indrelid AS tbloid,
+                          i.indexrelid AS idxoid,
+                          coalesce(substring(
+                              array_to_string(ci.reloptions, ' ')
+                              from 'fillfactor=([0-9]+)')::smallint, 90) AS fillfactor,
+                          i.indnatts,
+                          pg_catalog.string_to_array(pg_catalog.textin(
+                              pg_catalog.int2vectorout(i.indkey)),' ')::int[] AS indkey
+                      FROM pg_catalog.pg_index i
+                      JOIN pg_catalog.pg_class ci ON ci.oid = i.indexrelid
+                      WHERE ci.relam=(SELECT oid FROM pg_am WHERE amname = 'btree')
+                      AND ci.relpages > 0
+                  ) AS idx_data
+              ) AS ic
+              JOIN pg_catalog.pg_class ct ON ct.oid = ic.tbloid
+              LEFT JOIN pg_catalog.pg_attribute a1 ON
+                  ic.indkey[ic.attpos] <> 0
+                  AND a1.attrelid = ic.tbloid
+                  AND a1.attnum = ic.indkey[ic.attpos]
+              LEFT JOIN pg_catalog.pg_attribute a2 ON
+                  ic.indkey[ic.attpos] = 0
+                  AND a2.attrelid = ic.idxoid
+                  AND a2.attnum = ic.attpos
+            ) i
+            JOIN pg_catalog.pg_namespace n ON n.oid = i.relnamespace
+            JOIN pg_catalog.pg_stats s ON s.schemaname = n.nspname
+                                      AND s.tablename = i.attrelname
+                                      AND s.attname = i.attname
+            GROUP BY 1,2,3,4,5,6,7,8,9,10,11
+      ) AS rows_data_stats
+  ) AS rows_hdr_pdg_stats
+) AS relation_stats
+ORDER BY nspname, tblname, idxname;

data/lib/gitlab_exporter/database/bloat_table.sql

@@ -1,5 +1,5 @@
 -- Originally from: https://github.com/ioguix/pgsql-bloat-estimation/blob/master/table/table_bloat.sql
-/* WARNING: executed with a non-superuser role, the query inspect only tables you are granted to read.
+/* WARNING: executed with a non-superuser role, the query inspect only tables and materialized view (9.3+) you are granted to read.
 * This query is compatible with PostgreSQL 9.0 and more
 */
 SELECT current_database(), schemaname, tblname AS object_name, bs*tblpages AS real_size,
@@ -16,12 +16,12 @@ SELECT current_database(), schemaname, tblname AS object_name, bs*tblpages AS re
     THEN 100 * (tblpages - est_tblpages_ff)/tblpages::float
     ELSE 0
   END AS bloat_ratio, is_na
-  -- , (pst).free_percent + (pst).dead_tuple_percent AS real_frag
+  -- , tpl_hdr_size, tpl_data_size, (pst).free_percent + (pst).dead_tuple_percent AS real_frag -- (DEBUG INFO)
 FROM (
   SELECT ceil( reltuples / ( (bs-page_hdr)/tpl_size ) ) + ceil( toasttuples / 4 ) AS est_tblpages,
     ceil( reltuples / ( (bs-page_hdr)*fillfactor/(tpl_size*100) ) ) + ceil( toasttuples / 4 ) AS est_tblpages_ff,
     tblpages, fillfactor, bs, tblid, schemaname, tblname, heappages, toastpages, is_na
-    -- , stattuple.pgstattuple(tblid) AS pst
+    -- , tpl_hdr_size, tpl_data_size, pgstattuple(tblid) AS pst -- (DEBUG INFO)
   FROM (
     SELECT
       ( 4 + tpl_hdr_size + tpl_data_size + (2*ma)
@@ -29,6 +29,7 @@ FROM (
         - CASE WHEN ceil(tpl_data_size)::int%ma = 0 THEN ma ELSE ceil(tpl_data_size)::int%ma END
       ) AS tpl_size, bs - page_hdr AS size_per_block, (heappages + toastpages) AS tblpages, heappages,
       toastpages, reltuples, toasttuples, bs, page_hdr, tblid, schemaname, tblname, fillfactor, is_na
+      -- , tpl_hdr_size, tpl_data_size
     FROM (
       SELECT
         tbl.oid AS tblid, ns.nspname AS schemaname, tbl.relname AS tblname, tbl.reltuples,
@@ -40,24 +41,24 @@ FROM (
         current_setting('block_size')::numeric AS bs,
         CASE WHEN version()~'mingw32' OR version()~'64-bit|x86_64|ppc64|ia64|amd64' THEN 8 ELSE 4 END AS ma,
         24 AS page_hdr,
-        23 + CASE WHEN MAX(coalesce(null_frac,0)) > 0 THEN ( 7 + count(*) ) / 8 ELSE 0::int END
-          + CASE WHEN tbl.relhasoids THEN 4 ELSE 0 END AS tpl_hdr_size,
-        sum( (1-coalesce(s.null_frac, 0)) * coalesce(s.avg_width, 1024) ) AS tpl_data_size,
+        23 + CASE WHEN MAX(coalesce(s.null_frac,0)) > 0 THEN ( 7 + count(s.attname) ) / 8 ELSE 0::int END
+           + CASE WHEN bool_or(att.attname = 'oid' and att.attnum < 0) THEN 4 ELSE 0 END AS tpl_hdr_size,
+        sum( (1-coalesce(s.null_frac, 0)) * coalesce(s.avg_width, 0) ) AS tpl_data_size,
         bool_or(att.atttypid = 'pg_catalog.name'::regtype)
-          OR count(att.attname) <> count(s.attname) AS is_na
+          OR sum(CASE WHEN att.attnum > 0 THEN 1 ELSE 0 END) <> count(s.attname) AS is_na
       FROM pg_attribute AS att
         JOIN pg_class AS tbl ON att.attrelid = tbl.oid
         JOIN pg_namespace AS ns ON ns.oid = tbl.relnamespace
         LEFT JOIN pg_stats AS s ON s.schemaname=ns.nspname
           AND s.tablename = tbl.relname AND s.inherited=false AND s.attname=att.attname
         LEFT JOIN pg_class AS toast ON tbl.reltoastrelid = toast.oid
-      WHERE att.attnum > 0 AND NOT att.attisdropped
-        AND tbl.relkind = 'r'
-      GROUP BY 1,2,3,4,5,6,7,8,9,10, tbl.relhasoids
+      WHERE NOT att.attisdropped
+        AND tbl.relkind in ('r','m')
+      GROUP BY 1,2,3,4,5,6,7,8,9,10
       ORDER BY 2,3
     ) AS s
   ) AS s2
 ) AS s3
-WHERE NOT is_na
+-- WHERE NOT is_na
 --   AND tblpages*((pst).free_percent + (pst).dead_tuple_percent)::float4/100 >= 1
-AND   schemaname= 'public';
+ORDER BY schemaname, tblname;

data/lib/gitlab_exporter/database/ci_builds.rb

@@ -21,8 +21,7 @@ module GitLab
               ON namespaces.id = projects.namespace_id
             LEFT JOIN namespace_statistics
               ON namespace_statistics.namespace_id = namespaces.id
-            JOIN application_settings
-              ON application_settings.id = 1
+            JOIN application_settings ON (TRUE)
             WHERE ci_builds.type = 'Ci::Build'
               AND ci_builds.status = '%s'
               -- The created_at filter has been introduced for performance reasons only
@@ -95,8 +94,7 @@ module GitLab
               ON namespaces.id = projects.namespace_id
             LEFT JOIN namespace_statistics
               ON namespace_statistics.namespace_id = namespaces.id
-            JOIN application_settings
-              ON application_settings.id = 1
+            JOIN application_settings ON (TRUE)
             WHERE ci_builds.type = 'Ci::Build'
               AND ci_builds.status = 'running'
               -- The created_at filter has been introduced for performance reasons only
@@ -175,8 +173,7 @@ module GitLab
                 ON namespaces.id = projects.namespace_id
               LEFT JOIN namespace_statistics
                 ON namespace_statistics.namespace_id = namespaces.id
-              JOIN application_settings
-                ON application_settings.id = 1
+              JOIN application_settings ON (TRUE)
               WHERE ci_builds.type = 'Ci::Build'
                 AND ci_builds.status IN ('running', 'pending')
                 -- The created_at filter has been introduced for performance reasons only
@@ -265,8 +262,8 @@ module GitLab
 
         DEFAULT_UNARCHIVED_TRACES_OFFSET_MINUTES = 1440
 
-        def initialize(opts)
-          super(opts)
+        def initialize(opts, logger: nil)
+          super(opts, logger: logger)
 
           @allowed_repeated_commands_count = opts[:allowed_repeated_commands_count]
           @created_builds_counting_disabled = opts[:created_builds_counting_disabled]
@@ -420,14 +417,14 @@ module GitLab
 
       # The prober which is called when gathering metrics
       class CiBuildsProber
-        def initialize(opts, metrics: PrometheusMetrics.new)
+        def initialize(opts, metrics: PrometheusMetrics.new, logger: nil)
           @metrics = metrics
 
           collector_opts = { connection_string: opts[:connection_string],
                              allowed_repeated_commands_count: opts[:allowed_repeated_commands_count],
                              created_builds_counting_disabled: opts[:created_builds_counting_disabled],
                              unarchived_traces_offset_minutes: opts[:unarchived_traces_offset_minutes] }
-          @collector = CiBuildsCollector.new(collector_opts)
+          @collector = CiBuildsCollector.new(collector_opts, logger: logger)
         end
 
         def probe_db

data/lib/gitlab_exporter/database/remote_mirrors.rb

@@ -35,7 +35,7 @@ module GitLab
 
       # The prober which is called when gathering metrics
       class RemoteMirrorsProber
-        def initialize(opts, metrics: PrometheusMetrics.new)
+        def initialize(opts, metrics: PrometheusMetrics.new, logger: nil) # rubocop:disable Lint/UnusedMethodArgument
           @metrics = metrics
           @collector = RemoteMirrorsCollector.new(
             connection_string: opts[:connection_string],

data/lib/gitlab_exporter/database/row_count.rb

@@ -175,6 +175,14 @@ module GitLab
 
         def execute(query)
           with_connection_pool do |conn|
+            conn.exec(query).map_types!(type_map_for_results(conn))
+          end
+        rescue PG::UndefinedTable, PG::UndefinedColumn
+          nil
+        end
+
+        def type_map_for_results(conn)
+          @type_map_for_results ||= begin
             tm = PG::BasicTypeMapForResults.new(conn)
 
             # Remove warning message:
@@ -187,10 +195,8 @@ module GitLab
               tm.add_coder(old_coder.dup.tap { |c| c.oid = value[:oid] })
             end
 
-            conn.exec(query).map_types!(tm)
+            tm
           end
-        rescue PG::UndefinedTable, PG::UndefinedColumn
-          nil
         end
 
         # Not private so I can test it without meta programming tricks
@@ -212,7 +218,7 @@ module GitLab
 
       # The prober which is called when gathering metrics
       class RowCountProber
-        def initialize(opts, metrics: PrometheusMetrics.new)
+        def initialize(opts, metrics: PrometheusMetrics.new, logger: nil) # rubocop:disable Lint/UnusedMethodArgument
           @metrics = metrics
           @collector = RowCountCollector.new(
             connection_string: opts[:connection_string],

data/lib/gitlab_exporter/database/tuple_stats.rb

@@ -25,9 +25,9 @@ module GitLab
 
       # Probes the DB specified by opts[:connection_string] for tuple stats, then converts them to metrics
       class TuplesProber
-        def initialize(opts, metrics: PrometheusMetrics.new)
+        def initialize(opts, metrics: PrometheusMetrics.new, logger: nil)
           @metrics   = metrics
-          @collector = TupleStatsCollector.new(connection_string: opts[:connection_string])
+          @collector = TupleStatsCollector.new(connection_string: opts[:connection_string], logger: logger)
         end
 
         def probe_db

data/lib/gitlab_exporter/git.rb

@@ -62,7 +62,7 @@ module GitLab
     # Optionally takes a metrics object which by default is a PrometheusMetrics, useful to change the
     # metrics writer to something else.
     class GitProber
-      def initialize(opts, metrics: PrometheusMetrics.new)
+      def initialize(opts, metrics: PrometheusMetrics.new, logger: nil) # rubocop:disable Lint/UnusedMethodArgument
         @metrics = metrics
         @labels = opts[:labels] || {}
         @git = Git.new(opts[:source])
@@ -85,7 +85,7 @@ module GitLab
 
     # A special prober for git processes
     class GitProcessProber
-      def initialize(opts, metrics: PrometheusMetrics.new)
+      def initialize(opts, metrics: PrometheusMetrics.new, logger: nil) # rubocop:disable Lint/UnusedMethodArgument
         @opts    = opts
         @metrics = metrics
       end

data/lib/gitlab_exporter/prober.rb

@@ -2,9 +2,10 @@ module GitLab
   module Exporter
     # A class to combine multiple probers into one
     class Prober
-      def initialize(prober_opts, metrics: PrometheusMetrics.new)
+      def initialize(prober_opts, metrics: PrometheusMetrics.new, logger: nil)
         @prober_opts = prober_opts
         @metrics     = metrics
+        @logger      = logger
 
         resolve_prober_classes
       end
@@ -12,7 +13,7 @@ module GitLab
       def probe_all
         @prober_opts.each do |_probe_name, params|
           Utils.wrap_in_array(params[:opts]).each do |opts|
-            prober = params[:class].new(opts, metrics: @metrics)
+            prober = params[:class].new(opts, metrics: @metrics, logger: @logger)
             params[:methods].each do |meth|
               prober.send(meth)
             end

data/lib/gitlab_exporter/process.rb

@@ -59,7 +59,7 @@ module GitLab
 
     # Probes a process for info then writes metrics to a target
     class ProcessProber
-      def initialize(options, metrics: PrometheusMetrics.new)
+      def initialize(options, metrics: PrometheusMetrics.new, logger: nil) # rubocop:disable Lint/UnusedMethodArgument
         @metrics = metrics
         @name    = options[:name]
         @pids    = if options[:pid_or_pattern] =~ /^\d+$/

data/lib/gitlab_exporter/sidekiq.rb

@@ -10,9 +10,10 @@ module GitLab
       QUEUE_JOB_STATS_SCRIPT = File.read(File.expand_path("#{__FILE__}/../sidekiq_queue_job_stats.lua")).freeze
       QUEUE_JOB_STATS_SHA    = Digest::SHA1.hexdigest(QUEUE_JOB_STATS_SCRIPT).freeze
 
-      def initialize(opts, metrics: PrometheusMetrics.new)
+      def initialize(opts, metrics: PrometheusMetrics.new, logger: nil)
         @opts    = opts
         @metrics = metrics
+        @logger  = logger
 
         Sidekiq.configure_client do |config|
           config.redis = redis_options

data/lib/gitlab_exporter/version.rb

@@ -1,5 +1,5 @@
 module GitLab
   module Exporter
-    VERSION = "7.0.5".freeze
+    VERSION = "7.2.0".freeze
   end
 end

data/lib/gitlab_exporter/web_exporter.rb

@@ -42,6 +42,11 @@ module GitLab
 
           memory_threshold = (config[:server] && config[:server][:memory_threshold]) || 1024
           use MemoryKillerMiddleware, memory_threshold
+          use Rack::Logger
+        end
+
+        def logger
+          request.logger
         end
 
         def setup_server(config)
@@ -62,7 +67,7 @@ module GitLab
 
             get "/#{probe_name}" do
               content_type "text/plain; version=0.0.4"
-              prober = Prober.new(opts, metrics: PrometheusMetrics.new(include_timestamp: false))
+              prober = Prober.new(opts, metrics: PrometheusMetrics.new(include_timestamp: false), logger: logger)
 
               prober.probe_all
               prober.write_to(response)

data/spec/database/bloat_spec.rb

@@ -43,7 +43,11 @@ describe GitLab::Exporter::Database::BloatProber do
   end
 
   describe "#probe_db" do
-    subject { described_class.new(opts, metrics: metrics, collector: collector).probe_db }
+    subject { described_class.new(opts, metrics: metrics, collector: collector, logger: STDOUT).probe_db }
+
+    before do
+      expect(collector).to receive(:logger=).with(STDOUT)
+    end
 
     it "invokes the collector for each bloat type" do
       expect(collector).to receive(:run).with(:btree)

metadata

@@ -1,11 +1,11 @@
 --- !ruby/object:Gem::Specification
 name: gitlab-exporter
 version: !ruby/object:Gem::Version
-  version: 7.0.5
+  version: 7.2.0
 platform: ruby
 authors:
 - Pablo Carranza
-autorequire: 
+autorequire:
 bindir: bin
 cert_chain: []
 date: 2016-07-27 00:00:00.000000000 Z
@@ -191,7 +191,7 @@ homepage: http://gitlab.com
 licenses:
 - MIT
 metadata: {}
-post_install_message: 
+post_install_message:
 rdoc_options: []
 require_paths:
 - lib
@@ -207,7 +207,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
 requirements: []
 rubygems_version: 3.0.3
-signing_key: 
+signing_key:
 specification_version: 4
 summary: GitLab metrics exporter
 test_files: