gitlab-exporter 7.0.0 → 7.0.5

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 63b7f7660a9e551506b9e1f9ef5ae509c1af8c93da24e2a9ab32ee1363d29c7b
-  data.tar.gz: f410a4e7c189cbaa3ad2c169955ce2098397f86bd93061fcf6a9b85353ade364
+  metadata.gz: de89f0131cde7415ce666ec09c4da388b53f4cbf57b8e7872f3e5538be257e30
+  data.tar.gz: 4b84a64c0d3b26306628ec04dc2ce7b56b36eda349385a43855b1a968c2a8560
 SHA512:
-  metadata.gz: 7c93c493c8da03f94b1cd9786bebc4c83564cb60774a2786b7450faaeec6335d707c3e8f0ab1e6e2bc18466db598441cdfaccd6a4bb7482bdb232412e134cbbd
-  data.tar.gz: 6c406c6bffa5c8c7150e638943e895b3e9272122b5ab81a38cbdedff6a70279af72cc91a0a3de92f0459cb3621cb867927d253ca15e6e2cfc141ee909c79c120
+  metadata.gz: a292b9c5c4f24fab3014e6bc9ae5f3b86c86ff47d5034fce9280d1fa75372cbcd18e5d75a6e130c9de77fc662b1ef2fa6d1dd9f390a004236c727e311a9e02a3
+  data.tar.gz: 6f1309c2abeeeca930d0485d66b9b714006f1d97814141fba7f6a2b4db063ee41f0181857de0044435d8e3b14d10e844d2ed595a5ce2505fe63a02ef6abce466

data/.gitlab-ci.yml

@@ -1,13 +1,23 @@
-image: "ruby:2.3"
+default:
+  image: "ruby:2.3"
+  before_script:
+    - git config --global user.email "bot@gitlab.com"
+    - git config --global user.name "Bot User"
+    - bundle install -j $(nproc) --path vendor
+  cache:
+    paths:
+      - vendor
+  tags:
+    - gitlab-org
 
-before_script:
-  - git config --global user.email "bot@gitlab.com"
-  - git config --global user.name "Bot User"
-  - bundle install -j $(nproc) --path vendor
-
-cache:
-  paths:
-    - vendor
+workflow:
+  rules:
+    # For merge requests, create a pipeline.
+    - if: '$CI_MERGE_REQUEST_IID'
+    # For `master` branch, create a pipeline (this includes on schedules, pushes, merges, etc.).
+    - if: '$CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH'
+    # For tags, create a pipeline.
+    - if: '$CI_COMMIT_TAG'
 
 rspec:
   script:
@@ -16,3 +26,11 @@ rspec:
 rubocop:
   script:
     - bundle exec rubocop
+
+include:
+  - template: Security/DAST.gitlab-ci.yml  # https://gitlab.com/gitlab-org/gitlab-foss/blob/master/lib/gitlab/ci/templates/Security/DAST.gitlab-ci.yml
+  - template: Security/Container-Scanning.gitlab-ci.yml  # https://gitlab.com/gitlab-org/gitlab-foss/blob/master/lib/gitlab/ci/templates/Security/Container-Scanning.gitlab-ci.yml
+  - template: Security/Dependency-Scanning.gitlab-ci.yml  # https://gitlab.com/gitlab-org/gitlab-foss/blob/master/lib/gitlab/ci/templates/Security/Dependency-Scanning.gitlab-ci.yml
+  - template: Security/License-Scanning.gitlab-ci.yml  # https://gitlab.com/gitlab-org/gitlab-foss/blob/master/lib/gitlab/ci/templates/Security/License-Scanning.gitlab-ci.yml
+  - template: Security/SAST.gitlab-ci.yml  # https://gitlab.com/gitlab-org/gitlab-foss/blob/master/lib/gitlab/ci/templates/Security/SAST.gitlab-ci.yml
+

data/Gemfile.lock

@@ -1,7 +1,7 @@
 PATH
   remote: .
   specs:
-    gitlab-exporter (7.0.0)
+    gitlab-exporter (7.0.5)
       connection_pool (~> 2.2.1)
       pg (~> 1.1)
       quantile (~> 0.2.0)
@@ -16,15 +16,14 @@ GEM
     ast (2.4.0)
     connection_pool (2.2.2)
     diff-lcs (1.3)
-    mustermann (1.1.1)
-      ruby2_keywords (~> 0.0.1)
+    mustermann (1.0.3)
     parser (2.5.1.0)
       ast (~> 2.4.0)
-    pg (1.2.3)
+    pg (1.2.2)
     powerpack (0.1.1)
     quantile (0.2.1)
-    rack (2.0.9)
-    rack-protection (2.0.8.1)
+    rack (2.0.7)
+    rack-protection (2.0.7)
       rack
     rainbow (2.1.0)
     redis (4.1.3)
@@ -50,18 +49,17 @@ GEM
       ruby-progressbar (~> 1.7)
       unicode-display_width (~> 1.0, >= 1.0.1)
     ruby-progressbar (1.8.1)
-    ruby2_keywords (0.0.2)
-    sidekiq (5.2.8)
+    sidekiq (5.2.7)
       connection_pool (~> 2.2, >= 2.2.2)
-      rack (< 2.1.0)
+      rack (>= 1.5.0)
       rack-protection (>= 1.5.0)
       redis (>= 3.3.5, < 5)
-    sinatra (2.0.8.1)
+    sinatra (2.0.7)
       mustermann (~> 1.0)
       rack (~> 2.0)
-      rack-protection (= 2.0.8.1)
+      rack-protection (= 2.0.7)
       tilt (~> 2.0)
-    tilt (2.0.10)
+    tilt (2.0.9)
     unicode-display_width (1.6.0)
 
 PLATFORMS

data/lib/gitlab_exporter/database/ci_builds.rb

@@ -144,9 +144,9 @@ module GitLab
               ci_builds.trigger_request_id
           SQL
 
-        MIRROR_COLUMN_QUERY =
+        EE_CHECK_QUERY =
           <<~SQL.freeze
-            SELECT EXISTS (SELECT 1 FROM information_schema.columns WHERE table_name='projects' AND column_name='mirror')
+            SELECT COUNT(*) FROM licenses
           SQL
 
         REPEATED_COMMANDS_QUERY_EE =
@@ -289,7 +289,7 @@ module GitLab
         def builds(status)
           results = []
 
-          query = mirror_column? ? BUILDS_QUERY_EE : BUILDS_QUERY_CE
+          query = ee? ? BUILDS_QUERY_EE : BUILDS_QUERY_CE
           query = query % [status] # rubocop:disable Style/FormatString
           exec_query_with_custom_random_page_cost(query).each do |row|
             results << transform_builds_row_to_values(row)
@@ -318,7 +318,7 @@ module GitLab
         def per_runner_builds
           results = []
 
-          query = mirror_column? ? PER_RUNNER_QUERY_EE : PER_RUNNER_QUERY_CE
+          query = ee? ? PER_RUNNER_QUERY_EE : PER_RUNNER_QUERY_CE
           exec_query_with_custom_random_page_cost(query).each do |row|
             results << transform_per_runners_builds_row_to_values(row)
           end
@@ -341,7 +341,7 @@ module GitLab
         def repeated_commands
           results = []
 
-          query = mirror_column? ? REPEATED_COMMANDS_QUERY_EE : REPEATED_COMMANDS_QUERY_CE
+          query = ee? ? REPEATED_COMMANDS_QUERY_EE : REPEATED_COMMANDS_QUERY_CE
           query = query % [allowed_repeated_commands_count] # rubocop:disable Style/FormatString
           exec_query_with_custom_random_page_cost(query).each do |row|
             results << transform_repeated_commands_row_to_values(row)
@@ -406,11 +406,11 @@ module GitLab
           end
         end
 
-        def mirror_column?
-          @mirror_column ||=
+        def ee?
+          @ee ||=
             begin
               with_connection_pool do |conn|
-                conn.exec(MIRROR_COLUMN_QUERY)[0]["exists"] == "t"
+                conn.exec(EE_CHECK_QUERY)[0]["count"].to_i > 0 # rubocop:disable Style/NumericPredicate
               end
             rescue PG::UndefinedColumn
               false

data/lib/gitlab_exporter/database/row_count.rb

@@ -10,10 +10,21 @@ module GitLab
       # #construct_query
       # rubocop:disable Metrics/ClassLength
       class RowCountCollector < Base
+        # We ignore mirrors with a next_execution_timestamp before
+        # 2020-03-28 because this is when we stopped processing mirrors
+        # for private projects on the free plan. Skipping those can
+        # significantly improve query performance:
+        # https://gitlab.com/gitlab-org/gitlab/-/issues/216252#note_334514544
         WHERE_MIRROR_ENABLED = <<~SQL.freeze
           projects.mirror = true
+          AND projects.archived = false
           AND project_mirror_data.retry_count <= 14
-          AND (projects.visibility_level = 20 OR plans.name IN ('early_adopter', 'bronze', 'silver', 'gold'))
+          AND (
+            (projects.visibility_level = 20 AND root_namespaces.visibility_level = 20)
+            OR
+            plans.name IN ('early_adopter', 'bronze', 'silver', 'gold')
+          )
+          AND project_mirror_data.next_execution_timestamp > '2020-03-28'
         SQL
 
         MIRROR_QUERY = {
@@ -99,7 +110,7 @@ module GitLab
                 GROUP BY members.user_id
               ) AS u
               ON users.id = u.user_id",
-            where: "ghost IS NULL AND bot_type IS NULL",
+            where: "user_type IS NULL",
             fields: {
               admin:        {},
               external:     {},

data/lib/gitlab_exporter/version.rb

@@ -1,5 +1,5 @@
 module GitLab
   module Exporter
-    VERSION = "7.0.0".freeze
+    VERSION = "7.0.5".freeze
   end
 end

data/spec/database/ci_builds_spec.rb

@@ -9,7 +9,7 @@ describe GitLab::Exporter::Database do
   let(:stale_builds_query) { "SELECT NOT UPDATED RUNNING" }
   let(:per_runner_query_ee) { "SELECT ALL RUNNING PER RUNNER EE" }
   let(:per_runner_query_ce) { "SELECT ALL RUNNING PER RUNNER CE" }
-  let(:mirror_column_query) { "SELECT DOES MIRROR COLUMN EXISTS" }
+  let(:ee_check_query) { "SELECT COUNT(*) FROM licenses" }
   let(:repeated_commands_query_ee) { "SELECT EE REPEATED COMNANDS %d" }
   let(:repeated_commands_query_ce) { "SELECT CE REPEATED COMNANDS %d" }
   let(:unarchived_traces_query) { "SELECT UNARCHIVED TRACES %s LIST" }
@@ -22,11 +22,11 @@ describe GitLab::Exporter::Database do
   let(:unarchived_traces_offset_minutes) { 60 }
 
   def stub_ee
-    allow(connection).to receive(:exec).with(mirror_column_query).and_return([{ "exists" => "t" }])
+    allow(connection).to receive(:exec).with(ee_check_query).and_return([{ "count" => 1 }])
   end
 
   def stub_ce
-    allow(connection).to receive(:exec).with(mirror_column_query).and_return([{ "exists" => "f" }])
+    allow(connection).to receive(:exec).with(ee_check_query).and_return([{ "count" => 0 }])
   end
 
   def builds_query_row_ee(shared_runners_enabled, status, namespace_id, has_minutes, count)
@@ -86,7 +86,7 @@ describe GitLab::Exporter::Database do
     stub_const("GitLab::Exporter::Database::CiBuildsCollector::STALE_BUILDS_QUERY", stale_builds_query)
     stub_const("GitLab::Exporter::Database::CiBuildsCollector::PER_RUNNER_QUERY_EE", per_runner_query_ee)
     stub_const("GitLab::Exporter::Database::CiBuildsCollector::PER_RUNNER_QUERY_CE", per_runner_query_ce)
-    stub_const("GitLab::Exporter::Database::CiBuildsCollector::MIRROR_COLUMN_QUERY", mirror_column_query)
+    stub_const("GitLab::Exporter::Database::CiBuildsCollector::EE_CHECK_QUERY", ee_check_query)
     stub_const("GitLab::Exporter::Database::CiBuildsCollector::REPEATED_COMMANDS_QUERY_EE", repeated_commands_query_ee)
     stub_const("GitLab::Exporter::Database::CiBuildsCollector::REPEATED_COMMANDS_QUERY_CE", repeated_commands_query_ce)
     stub_const("GitLab::Exporter::Database::CiBuildsCollector::UNARCHIVED_TRACES_QUERY", unarchived_traces_query)

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: gitlab-exporter
 version: !ruby/object:Gem::Version
-  version: 7.0.0
+  version: 7.0.5
 platform: ruby
 authors:
 - Pablo Carranza