gitlab-exporter 7.1.2 → 7.2.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 350884c8d1f03a58cad2630e0076aa641d22d60aed65a5d2342cb7f4508ec655
-  data.tar.gz: d1857b8e6a8f842b438d12810c110ff4b8bbdf5226b5a6c5859dd949c12d1dd6
+  metadata.gz: 86ad2e8d49582c70ba5faa5e7a8744a0e812f5682447cc1e1f000e050a3b9451
+  data.tar.gz: 2a0f878e42abef28992284e0e372db960831025b7ff3cf71c676369210d05fea
 SHA512:
-  metadata.gz: 9b62aaf4f95b4da82b1e5bc86058b97c363a1b43ae7026589e30da6af47ccf69261ec279094574927180b270be358afbba8304d10bff3fe8fb047fb54b6be538
-  data.tar.gz: 34d27b77c5b04c4a079d184ff30db9b75f37471f9ac0e5199790ce550ee360e52d6f81d2b22b6f159a780ff7525b866b79f8ab0c06265d6251f697177b86fc83
+  metadata.gz: 28d6f61e6df16ac17ba9e25ddea5d6a39c65940796898a2a6bad66c352ad47a1341593e3bb8dd7edaefd5281a787612beda9ff6f3f3700c6c458ad345fef055d
+  data.tar.gz: f7b2ff3ddd97529571e0517f4b8f9c7d42fe512f5d786a9c60dfae6337cafc56f5ae8eecaf98e3d85d0cb987e66363102bfec40a6ff4411524359cdf888f9d3e

data/Gemfile.lock

@@ -1,7 +1,7 @@
 PATH
   remote: .
   specs:
-    gitlab-exporter (7.1.2)
+    gitlab-exporter (7.2.0)
       connection_pool (~> 2.2.1)
       pg (~> 1.1)
       quantile (~> 0.2.0)

data/lib/gitlab_exporter/database/bloat_btree.sql

@@ -12,73 +12,90 @@ SELECT current_database(), nspname AS schemaname, tblname, idxname AS object_nam
   END AS bloat_size,
   100 * (relpages-est_pages_ff)::float / relpages AS bloat_ratio,
   is_na
-  -- , 100-(sub.pst).avg_leaf_density, est_pages, index_tuple_hdr_bm, maxalign, pagehdr, nulldatawidth, nulldatahdrwidth, sub.reltuples, sub.relpages -- (DEBUG INFO)
+  -- , 100-(pst).avg_leaf_density AS pst_avg_bloat, est_pages, index_tuple_hdr_bm, maxalign, pagehdr, nulldatawidth, nulldatahdrwidth, reltuples, relpages -- (DEBUG INFO)
 FROM (
   SELECT coalesce(1 +
-       ceil(reltuples/floor((bs-pageopqdata-pagehdr)/(4+nulldatahdrwidth)::float)), 0 -- ItemIdData size + computed avg size of a tuple (nulldatahdrwidth)
-    ) AS est_pages,
-    coalesce(1 +
-       ceil(reltuples/floor((bs-pageopqdata-pagehdr)*fillfactor/(100*(4+nulldatahdrwidth)::float))), 0
-    ) AS est_pages_ff,
-    bs, nspname, table_oid, tblname, idxname, relpages, fillfactor, is_na
-    -- , stattuple.pgstatindex(quote_ident(nspname)||'.'||quote_ident(idxname)) AS pst, index_tuple_hdr_bm, maxalign, pagehdr, nulldatawidth, nulldatahdrwidth, reltuples -- (DEBUG INFO)
+         ceil(reltuples/floor((bs-pageopqdata-pagehdr)/(4+nulldatahdrwidth)::float)), 0 -- ItemIdData size + computed avg size of a tuple (nulldatahdrwidth)
+      ) AS est_pages,
+      coalesce(1 +
+         ceil(reltuples/floor((bs-pageopqdata-pagehdr)*fillfactor/(100*(4+nulldatahdrwidth)::float))), 0
+      ) AS est_pages_ff,
+      bs, nspname, tblname, idxname, relpages, fillfactor, is_na
+      -- , pgstatindex(idxoid) AS pst, index_tuple_hdr_bm, maxalign, pagehdr, nulldatawidth, nulldatahdrwidth, reltuples -- (DEBUG INFO)
   FROM (
-    SELECT maxalign, bs, nspname, tblname, idxname, reltuples, relpages, relam, table_oid, fillfactor,
-      ( index_tuple_hdr_bm +
-          maxalign - CASE -- Add padding to the index tuple header to align on MAXALIGN
-            WHEN index_tuple_hdr_bm%maxalign = 0 THEN maxalign
-            ELSE index_tuple_hdr_bm%maxalign
-          END
-        + nulldatawidth + maxalign - CASE -- Add padding to the data to align on MAXALIGN
-            WHEN nulldatawidth = 0 THEN 0
-            WHEN nulldatawidth::integer%maxalign = 0 THEN maxalign
-            ELSE nulldatawidth::integer%maxalign
-          END
-      )::numeric AS nulldatahdrwidth, pagehdr, pageopqdata, is_na
-      -- , index_tuple_hdr_bm, nulldatawidth -- (DEBUG INFO)
-    FROM (
-      SELECT
-        i.nspname, i.tblname, i.idxname, i.reltuples, i.relpages, i.relam, a.attrelid AS table_oid,
-        current_setting('block_size')::numeric AS bs, fillfactor,
-        CASE -- MAXALIGN: 4 on 32bits, 8 on 64bits (and mingw32 ?)
-          WHEN version() ~ 'mingw32' OR version() ~ '64-bit|x86_64|ppc64|ia64|amd64' THEN 8
-          ELSE 4
-        END AS maxalign,
-        /* per page header, fixed size: 20 for 7.X, 24 for others */
-        24 AS pagehdr,
-        /* per page btree opaque data */
-        16 AS pageopqdata,
-        /* per tuple header: add IndexAttributeBitMapData if some cols are null-able */
-        CASE WHEN max(coalesce(s.null_frac,0)) = 0
-          THEN 2 -- IndexTupleData size
-          ELSE 2 + (( 32 + 8 - 1 ) / 8) -- IndexTupleData size + IndexAttributeBitMapData size ( max num filed per index + 8 - 1 /8)
-        END AS index_tuple_hdr_bm,
-        /* data len: we remove null values save space using it fractionnal part from stats */
-        sum( (1-coalesce(s.null_frac, 0)) * coalesce(s.avg_width, 1024)) AS nulldatawidth,
-        max( CASE WHEN a.atttypid = 'pg_catalog.name'::regtype THEN 1 ELSE 0 END ) > 0 AS is_na
-      FROM pg_attribute AS a
-        JOIN (
-          SELECT nspname, tbl.relname AS tblname, idx.relname AS idxname, idx.reltuples, idx.relpages, idx.relam,
-            indrelid, indexrelid, indkey::smallint[] AS attnum,
-            coalesce(substring(
-              array_to_string(idx.reloptions, ' ')
-               from 'fillfactor=([0-9]+)')::smallint, 90) AS fillfactor
-          FROM pg_index
-            JOIN pg_class idx ON idx.oid=pg_index.indexrelid
-            JOIN pg_class tbl ON tbl.oid=pg_index.indrelid
-            JOIN pg_namespace ON pg_namespace.oid = idx.relnamespace
-          WHERE pg_index.indisvalid AND tbl.relkind = 'r' AND idx.relpages > 0
-        ) AS i ON a.attrelid = i.indexrelid
-        JOIN pg_stats AS s ON s.schemaname = i.nspname
-          AND ((s.tablename = i.tblname AND s.attname = pg_catalog.pg_get_indexdef(a.attrelid, a.attnum, TRUE)) -- stats from tbl
-          OR   (s.tablename = i.idxname AND s.attname = a.attname))-- stats from functionnal cols
-        JOIN pg_type AS t ON a.atttypid = t.oid
-      WHERE a.attnum > 0
-      GROUP BY 1, 2, 3, 4, 5, 6, 7, 8, 9
-    ) AS s1
-  ) AS s2
-    JOIN pg_am am ON s2.relam = am.oid WHERE am.amname = 'btree'
-) AS sub
-WHERE NOT is_na
-AND   nspname = 'public'
-ORDER BY 2,3,4;
+      SELECT maxalign, bs, nspname, tblname, idxname, reltuples, relpages, idxoid, fillfactor,
+            ( index_tuple_hdr_bm +
+                maxalign - CASE -- Add padding to the index tuple header to align on MAXALIGN
+                  WHEN index_tuple_hdr_bm%maxalign = 0 THEN maxalign
+                  ELSE index_tuple_hdr_bm%maxalign
+                END
+              + nulldatawidth + maxalign - CASE -- Add padding to the data to align on MAXALIGN
+                  WHEN nulldatawidth = 0 THEN 0
+                  WHEN nulldatawidth::integer%maxalign = 0 THEN maxalign
+                  ELSE nulldatawidth::integer%maxalign
+                END
+            )::numeric AS nulldatahdrwidth, pagehdr, pageopqdata, is_na
+            -- , index_tuple_hdr_bm, nulldatawidth -- (DEBUG INFO)
+      FROM (
+          SELECT n.nspname, i.tblname, i.idxname, i.reltuples, i.relpages,
+              i.idxoid, i.fillfactor, current_setting('block_size')::numeric AS bs,
+              CASE -- MAXALIGN: 4 on 32bits, 8 on 64bits (and mingw32 ?)
+                WHEN version() ~ 'mingw32' OR version() ~ '64-bit|x86_64|ppc64|ia64|amd64' THEN 8
+                ELSE 4
+              END AS maxalign,
+              /* per page header, fixed size: 20 for 7.X, 24 for others */
+              24 AS pagehdr,
+              /* per page btree opaque data */
+              16 AS pageopqdata,
+              /* per tuple header: add IndexAttributeBitMapData if some cols are null-able */
+              CASE WHEN max(coalesce(s.null_frac,0)) = 0
+                  THEN 2 -- IndexTupleData size
+                  ELSE 2 + (( 32 + 8 - 1 ) / 8) -- IndexTupleData size + IndexAttributeBitMapData size ( max num filed per index + 8 - 1 /8)
+              END AS index_tuple_hdr_bm,
+              /* data len: we remove null values save space using it fractionnal part from stats */
+              sum( (1-coalesce(s.null_frac, 0)) * coalesce(s.avg_width, 1024)) AS nulldatawidth,
+              max( CASE WHEN i.atttypid = 'pg_catalog.name'::regtype THEN 1 ELSE 0 END ) > 0 AS is_na
+          FROM (
+              SELECT ct.relname AS tblname, ct.relnamespace, ic.idxname, ic.attpos, ic.indkey, ic.indkey[ic.attpos], ic.reltuples, ic.relpages, ic.tbloid, ic.idxoid, ic.fillfactor,
+                  coalesce(a1.attnum, a2.attnum) AS attnum, coalesce(a1.attname, a2.attname) AS attname, coalesce(a1.atttypid, a2.atttypid) AS atttypid,
+                  CASE WHEN a1.attnum IS NULL
+                  THEN ic.idxname
+                  ELSE ct.relname
+                  END AS attrelname
+              FROM (
+                  SELECT idxname, reltuples, relpages, tbloid, idxoid, fillfactor, indkey,
+                      pg_catalog.generate_series(1,indnatts) AS attpos
+                  FROM (
+                      SELECT ci.relname AS idxname, ci.reltuples, ci.relpages, i.indrelid AS tbloid,
+                          i.indexrelid AS idxoid,
+                          coalesce(substring(
+                              array_to_string(ci.reloptions, ' ')
+                              from 'fillfactor=([0-9]+)')::smallint, 90) AS fillfactor,
+                          i.indnatts,
+                          pg_catalog.string_to_array(pg_catalog.textin(
+                              pg_catalog.int2vectorout(i.indkey)),' ')::int[] AS indkey
+                      FROM pg_catalog.pg_index i
+                      JOIN pg_catalog.pg_class ci ON ci.oid = i.indexrelid
+                      WHERE ci.relam=(SELECT oid FROM pg_am WHERE amname = 'btree')
+                      AND ci.relpages > 0
+                  ) AS idx_data
+              ) AS ic
+              JOIN pg_catalog.pg_class ct ON ct.oid = ic.tbloid
+              LEFT JOIN pg_catalog.pg_attribute a1 ON
+                  ic.indkey[ic.attpos] <> 0
+                  AND a1.attrelid = ic.tbloid
+                  AND a1.attnum = ic.indkey[ic.attpos]
+              LEFT JOIN pg_catalog.pg_attribute a2 ON
+                  ic.indkey[ic.attpos] = 0
+                  AND a2.attrelid = ic.idxoid
+                  AND a2.attnum = ic.attpos
+            ) i
+            JOIN pg_catalog.pg_namespace n ON n.oid = i.relnamespace
+            JOIN pg_catalog.pg_stats s ON s.schemaname = n.nspname
+                                      AND s.tablename = i.attrelname
+                                      AND s.attname = i.attname
+            GROUP BY 1,2,3,4,5,6,7,8,9,10,11
+      ) AS rows_data_stats
+  ) AS rows_hdr_pdg_stats
+) AS relation_stats
+ORDER BY nspname, tblname, idxname;

data/lib/gitlab_exporter/database/bloat_table.sql

@@ -1,5 +1,5 @@
 -- Originally from: https://github.com/ioguix/pgsql-bloat-estimation/blob/master/table/table_bloat.sql
-/* WARNING: executed with a non-superuser role, the query inspect only tables you are granted to read.
+/* WARNING: executed with a non-superuser role, the query inspect only tables and materialized view (9.3+) you are granted to read.
 * This query is compatible with PostgreSQL 9.0 and more
 */
 SELECT current_database(), schemaname, tblname AS object_name, bs*tblpages AS real_size,
@@ -16,12 +16,12 @@ SELECT current_database(), schemaname, tblname AS object_name, bs*tblpages AS re
     THEN 100 * (tblpages - est_tblpages_ff)/tblpages::float
     ELSE 0
   END AS bloat_ratio, is_na
-  -- , (pst).free_percent + (pst).dead_tuple_percent AS real_frag
+  -- , tpl_hdr_size, tpl_data_size, (pst).free_percent + (pst).dead_tuple_percent AS real_frag -- (DEBUG INFO)
 FROM (
   SELECT ceil( reltuples / ( (bs-page_hdr)/tpl_size ) ) + ceil( toasttuples / 4 ) AS est_tblpages,
     ceil( reltuples / ( (bs-page_hdr)*fillfactor/(tpl_size*100) ) ) + ceil( toasttuples / 4 ) AS est_tblpages_ff,
     tblpages, fillfactor, bs, tblid, schemaname, tblname, heappages, toastpages, is_na
-    -- , stattuple.pgstattuple(tblid) AS pst
+    -- , tpl_hdr_size, tpl_data_size, pgstattuple(tblid) AS pst -- (DEBUG INFO)
   FROM (
     SELECT
       ( 4 + tpl_hdr_size + tpl_data_size + (2*ma)
@@ -29,6 +29,7 @@ FROM (
         - CASE WHEN ceil(tpl_data_size)::int%ma = 0 THEN ma ELSE ceil(tpl_data_size)::int%ma END
       ) AS tpl_size, bs - page_hdr AS size_per_block, (heappages + toastpages) AS tblpages, heappages,
       toastpages, reltuples, toasttuples, bs, page_hdr, tblid, schemaname, tblname, fillfactor, is_na
+      -- , tpl_hdr_size, tpl_data_size
     FROM (
       SELECT
         tbl.oid AS tblid, ns.nspname AS schemaname, tbl.relname AS tblname, tbl.reltuples,
@@ -40,24 +41,24 @@ FROM (
         current_setting('block_size')::numeric AS bs,
         CASE WHEN version()~'mingw32' OR version()~'64-bit|x86_64|ppc64|ia64|amd64' THEN 8 ELSE 4 END AS ma,
         24 AS page_hdr,
-        23 + CASE WHEN MAX(coalesce(null_frac,0)) > 0 THEN ( 7 + count(*) ) / 8 ELSE 0::int END
-          + CASE WHEN tbl.relhasoids THEN 4 ELSE 0 END AS tpl_hdr_size,
-        sum( (1-coalesce(s.null_frac, 0)) * coalesce(s.avg_width, 1024) ) AS tpl_data_size,
+        23 + CASE WHEN MAX(coalesce(s.null_frac,0)) > 0 THEN ( 7 + count(s.attname) ) / 8 ELSE 0::int END
+           + CASE WHEN bool_or(att.attname = 'oid' and att.attnum < 0) THEN 4 ELSE 0 END AS tpl_hdr_size,
+        sum( (1-coalesce(s.null_frac, 0)) * coalesce(s.avg_width, 0) ) AS tpl_data_size,
         bool_or(att.atttypid = 'pg_catalog.name'::regtype)
-          OR count(att.attname) <> count(s.attname) AS is_na
+          OR sum(CASE WHEN att.attnum > 0 THEN 1 ELSE 0 END) <> count(s.attname) AS is_na
       FROM pg_attribute AS att
         JOIN pg_class AS tbl ON att.attrelid = tbl.oid
         JOIN pg_namespace AS ns ON ns.oid = tbl.relnamespace
         LEFT JOIN pg_stats AS s ON s.schemaname=ns.nspname
           AND s.tablename = tbl.relname AND s.inherited=false AND s.attname=att.attname
         LEFT JOIN pg_class AS toast ON tbl.reltoastrelid = toast.oid
-      WHERE att.attnum > 0 AND NOT att.attisdropped
-        AND tbl.relkind = 'r'
-      GROUP BY 1,2,3,4,5,6,7,8,9,10, tbl.relhasoids
+      WHERE NOT att.attisdropped
+        AND tbl.relkind in ('r','m')
+      GROUP BY 1,2,3,4,5,6,7,8,9,10
       ORDER BY 2,3
     ) AS s
   ) AS s2
 ) AS s3
-WHERE NOT is_na
+-- WHERE NOT is_na
 --   AND tblpages*((pst).free_percent + (pst).dead_tuple_percent)::float4/100 >= 1
-AND   schemaname= 'public';
+ORDER BY schemaname, tblname;

data/lib/gitlab_exporter/version.rb

@@ -1,5 +1,5 @@
 module GitLab
   module Exporter
-    VERSION = "7.1.2".freeze
+    VERSION = "7.2.0".freeze
   end
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: gitlab-exporter
 version: !ruby/object:Gem::Version
-  version: 7.1.2
+  version: 7.2.0
 platform: ruby
 authors:
 - Pablo Carranza