gitlab-exporter 6.1.0 → 7.0.4

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 61203f2c5d53c61f05151255076c5004c65a254c2445d632ae8c7262ae0e1f82
-  data.tar.gz: 66b13da650dd0e66b3a4d639bf1738de51474e047e0df878893a5b675d158827
+  metadata.gz: 871a04f90ba88d0584d1378778d84fa495e3d258824ca5af41c89eddcc7f6d68
+  data.tar.gz: d394f2f50050049ac4d1156ee6bfb79f5fcf422980989599afabf3eec4f2b210
 SHA512:
-  metadata.gz: 7ab02d67cc6e2bf4e443e73f49138eb7549961d535a1c7425e7f9a7872ec7095f6ad77522105eee8111c674cbdebbfec304296953c884dd006832d452079414c
-  data.tar.gz: 81bf3b39c8cb1462dabbf51be56057efb9fddfbe0db6da4d6f1f7827f71f4a8cebf04e5890caf07ccaa588b873897dfef32e91d7276361a6fe629349eb2691b0
+  metadata.gz: 8f2fa49fd5423c9fcf0d47512949b8ec93d95b2c1b48e48693fe1ad273af8053cdae42392b48740bc925e4d8e701254f815089a25f1311c9ecdba7e139e5d3f8
+  data.tar.gz: f9e8f5e8d42ba861bd297d0b0b9caae6b0729007ac3674fd637be7d091c8ca300cb0ff8cf6b084a07a7903e479381e2d58dceb770dfb829df585fd0d6395b6d8

data/.gitlab-ci.yml

@@ -1,13 +1,23 @@
-image: "ruby:2.3"
+default:
+  image: "ruby:2.3"
+  before_script:
+    - git config --global user.email "bot@gitlab.com"
+    - git config --global user.name "Bot User"
+    - bundle install -j $(nproc) --path vendor
+  cache:
+    paths:
+      - vendor
+  tags:
+    - gitlab-org
 
-before_script:
-  - git config --global user.email "bot@gitlab.com"
-  - git config --global user.name "Bot User"
-  - bundle install -j $(nproc) --path vendor
-
-cache:
-  paths:
-    - vendor
+workflow:
+  rules:
+    # For merge requests, create a pipeline.
+    - if: '$CI_MERGE_REQUEST_IID'
+    # For `master` branch, create a pipeline (this includes on schedules, pushes, merges, etc.).
+    - if: '$CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH'
+    # For tags, create a pipeline.
+    - if: '$CI_COMMIT_TAG'
 
 rspec:
   script:

data/Gemfile.lock

@@ -1,7 +1,7 @@
 PATH
   remote: .
   specs:
-    gitlab-exporter (6.1.0)
+    gitlab-exporter (7.0.4)
       connection_pool (~> 2.2.1)
       pg (~> 1.1)
       quantile (~> 0.2.0)
@@ -20,14 +20,14 @@ GEM
       ruby2_keywords (~> 0.0.1)
     parser (2.5.1.0)
       ast (~> 2.4.0)
-    pg (1.2.2)
+    pg (1.2.3)
     powerpack (0.1.1)
     quantile (0.2.1)
-    rack (2.2.2)
+    rack (2.0.9)
     rack-protection (2.0.8.1)
       rack
     rainbow (2.1.0)
-    redis (4.1.3)
+    redis (4.1.4)
     redis-namespace (1.6.0)
       redis (>= 3.0.4)
     rspec (3.7.0)
@@ -51,9 +51,9 @@ GEM
       unicode-display_width (~> 1.0, >= 1.0.1)
     ruby-progressbar (1.8.1)
     ruby2_keywords (0.0.2)
-    sidekiq (5.2.7)
+    sidekiq (5.2.8)
       connection_pool (~> 2.2, >= 2.2.2)
-      rack (>= 1.5.0)
+      rack (< 2.1.0)
       rack-protection (>= 1.5.0)
       redis (>= 3.3.5, < 5)
     sinatra (2.0.8.1)

data/lib/gitlab_exporter/database/row_count.rb

@@ -8,19 +8,38 @@ module GitLab
       # This class works under the assumption you do COUNT(*) queries, define
       # queries in the QUERIES constant. If in doubt how these work, read
       # #construct_query
+      # rubocop:disable Metrics/ClassLength
       class RowCountCollector < Base
+        # We ignore mirrors with a next_execution_timestamp before
+        # 2020-03-28 because this is when we stopped processing mirrors
+        # for private projects on the free plan. Skipping those can
+        # significantly improve query performance:
+        # https://gitlab.com/gitlab-org/gitlab/-/issues/216252#note_334514544
         WHERE_MIRROR_ENABLED = <<~SQL.freeze
           projects.mirror = true
+          AND projects.archived = false
           AND project_mirror_data.retry_count <= 14
-          AND (projects.visibility_level = 20 OR plans.name IN ('early_adopter', 'bronze', 'silver', 'gold'))
+          AND (
+            (projects.visibility_level = 20 AND root_namespaces.visibility_level = 20)
+            OR
+            plans.name IN ('early_adopter', 'bronze', 'silver', 'gold')
+          )
+          AND project_mirror_data.next_execution_timestamp > '2020-03-28'
         SQL
 
         MIRROR_QUERY = {
           select: :projects,
           joins:  <<~SQL,
             INNER JOIN project_mirror_data ON project_mirror_data.project_id = projects.id
-            INNER JOIN namespaces ON projects.namespace_id = namespaces.id
-            LEFT JOIN plans ON namespaces.plan_id = plans.id
+            INNER JOIN namespaces AS root_namespaces ON root_namespaces.id = (
+              WITH RECURSIVE "base_and_ancestors" AS (
+                (SELECT "namespaces".* FROM "namespaces" WHERE "namespaces"."id" = projects.namespace_id)
+                UNION
+                (SELECT "namespaces".* FROM "namespaces", "base_and_ancestors" WHERE "namespaces"."id" = "base_and_ancestors"."parent_id")
+              ) SELECT "namespaces".id FROM "base_and_ancestors" AS "namespaces" WHERE "namespaces"."parent_id" IS NULL
+            )
+            LEFT JOIN gitlab_subscriptions ON gitlab_subscriptions.namespace_id = root_namespaces.id
+            LEFT JOIN plans ON plans.id = gitlab_subscriptions.hosted_plan_id
           SQL
           check: "SELECT 1 FROM information_schema.tables WHERE table_name='plans'"
         }.freeze
@@ -91,7 +110,7 @@ module GitLab
                 GROUP BY members.user_id
               ) AS u
               ON users.id = u.user_id",
-            where: "ghost IS NULL AND bot_type IS NULL",
+            where: "user_type IS NULL",
             fields: {
               admin:        {},
               external:     {},
@@ -189,6 +208,7 @@ module GitLab
           query_string << ";"
         end
       end
+      # rubocop:enable Metrics/ClassLength
 
       # The prober which is called when gathering metrics
       class RowCountProber

data/lib/gitlab_exporter/version.rb

@@ -1,5 +1,5 @@
 module GitLab
   module Exporter
-    VERSION = "6.1.0".freeze
+    VERSION = "7.0.4".freeze
   end
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: gitlab-exporter
 version: !ruby/object:Gem::Version
-  version: 6.1.0
+  version: 7.0.4
 platform: ruby
 authors:
 - Pablo Carranza