gitlab_quality-test_tooling 1.35.0 → 1.37.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: af317c2353ed0cd5ac69cda95756183c9811e4b9983397e5658972cc2ec80853
-  data.tar.gz: 4ef83ce3161fe6560b6e3e9d7426673e50b80291973ccbff0a5051184dcc13a4
+  metadata.gz: 6cad17948e73dda3f8281aaedb34969d1178904f4951e54034b38438c473baa8
+  data.tar.gz: 49d38db27d6617815fcec94ebce2610f6ac47faec14f622cf58f2baaae6add55
 SHA512:
-  metadata.gz: 8a015884914a10e72b3ad2bada2ba48444a15dd701a752c54029fa5acf9906e3384ac0ca9d991c04657b31a9fc25d6ec5708ed128e232ac6c5e8a4e76563c1db
-  data.tar.gz: bec57b7c3f65ead2655c992ee9ac972551bbcff9454b81b2b76a5ae11479e7f441717244a3b853c857e3f5ac4377eafbfd116272cea75feca20fc7b1c0d43397
+  metadata.gz: 22e114b48a3260dc61887d5bea9efd7574e4584d0a85fa452574ff89eab73ab26b287c83303997c8696331a11748a80d58e51ade1729fd646aa8a32ff69909b3
+  data.tar.gz: 5937ae1ed046b9cc123e5d7c2157cc809bb4edffd35189a89aec90bd95f947bc9026960c806783c9f809918f469fb27961ac938bb46a2a73e94d0341a726986a

data/Gemfile.lock

@@ -1,9 +1,10 @@
 PATH
   remote: .
   specs:
-    gitlab_quality-test_tooling (1.35.0)
+    gitlab_quality-test_tooling (1.37.0)
       activesupport (>= 7.0, < 7.2)
       amatch (~> 0.4.1)
+      fog-google (~> 1.24, >= 1.24.1)
       gitlab (~> 4.19)
       http (~> 5.0)
       influxdb-client (~> 3.1)
@@ -39,6 +40,7 @@ GEM
     bigdecimal (3.1.8)
     binding_of_caller (1.0.0)
       debug_inspector (>= 0.0.1)
+    builder (3.3.0)
     byebug (11.1.3)
     claide (1.1.0)
     claide-plugins (0.9.2)
@@ -71,11 +73,13 @@ GEM
       danger
       gitlab (~> 4.2, >= 4.2.0)
     debug_inspector (1.2.0)
+    declarative (0.0.20)
     diff-lcs (1.5.0)
     docile (1.4.0)
     domain_name (0.6.20240107)
     drb (2.2.1)
     e2mmap (0.1.0)
+    excon (0.111.0)
     faraday (2.9.0)
       faraday-net_http (>= 2.0, < 3.2)
     faraday-http-cache (2.5.1)
@@ -86,6 +90,30 @@ GEM
     ffi-compiler (1.0.1)
       ffi (>= 1.0.0)
       rake
+    fog-core (2.4.0)
+      builder
+      excon (~> 0.71)
+      formatador (>= 0.2, < 2.0)
+      mime-types
+    fog-google (1.24.1)
+      addressable (>= 2.7.0)
+      fog-core (< 2.5)
+      fog-json (~> 1.2)
+      fog-xml (~> 0.1.0)
+      google-apis-compute_v1 (~> 0.53)
+      google-apis-dns_v1 (~> 0.28)
+      google-apis-iamcredentials_v1 (~> 0.15)
+      google-apis-monitoring_v3 (~> 0.37)
+      google-apis-pubsub_v1 (~> 0.30)
+      google-apis-sqladmin_v1beta4 (~> 0.38)
+      google-apis-storage_v1 (>= 0.19, < 1)
+      google-cloud-env (>= 1.2, < 3.0)
+    fog-json (1.2.0)
+      fog-core
+      multi_json (~> 1.10)
+    fog-xml (0.1.4)
+      fog-core
+      nokogiri (>= 1.5.11, < 2.0.0)
     formatador (1.1.0)
     git (1.19.1)
       addressable (~> 2.8)
@@ -103,6 +131,37 @@ GEM
       rubocop-performance (~> 1.15)
       rubocop-rails (~> 2.17)
       rubocop-rspec (~> 2.22)
+    google-apis-compute_v1 (0.102.0)
+      google-apis-core (>= 0.15.0, < 2.a)
+    google-apis-core (0.15.0)
+      addressable (~> 2.5, >= 2.5.1)
+      googleauth (~> 1.9)
+      httpclient (>= 2.8.1, < 3.a)
+      mini_mime (~> 1.0)
+      representable (~> 3.0)
+      retriable (>= 2.0, < 4.a)
+      rexml
+    google-apis-dns_v1 (0.44.0)
+      google-apis-core (>= 0.15.0, < 2.a)
+    google-apis-iamcredentials_v1 (0.21.0)
+      google-apis-core (>= 0.15.0, < 2.a)
+    google-apis-monitoring_v3 (0.65.0)
+      google-apis-core (>= 0.15.0, < 2.a)
+    google-apis-pubsub_v1 (0.54.0)
+      google-apis-core (>= 0.15.0, < 2.a)
+    google-apis-sqladmin_v1beta4 (0.74.0)
+      google-apis-core (>= 0.15.0, < 2.a)
+    google-apis-storage_v1 (0.41.0)
+      google-apis-core (>= 0.15.0, < 2.a)
+    google-cloud-env (2.1.1)
+      faraday (>= 1.0, < 3.a)
+    googleauth (1.11.0)
+      faraday (>= 1.0, < 3.a)
+      google-cloud-env (~> 2.1)
+      jwt (>= 1.4, < 3.0)
+      multi_json (~> 1.11)
+      os (>= 0.9, < 2.0)
+      signet (>= 0.16, < 2.a)
     guard (2.18.1)
       formatador (>= 0.2.4)
       listen (>= 2.7, < 4.0)
@@ -129,11 +188,14 @@ GEM
     httparty (0.21.0)
       mini_mime (>= 1.0.0)
       multi_xml (>= 0.5.2)
+    httpclient (2.8.3)
     i18n (1.14.5)
       concurrent-ruby (~> 1.0)
     influxdb-client (3.1.0)
     jaro_winkler (1.5.6)
     json (2.7.1)
+    jwt (2.8.2)
+      base64
     kramdown (2.4.0)
       rexml
     kramdown-parser-gfm (1.1.0)
@@ -147,11 +209,15 @@ GEM
       rake (~> 13.0)
     lumberjack (1.2.10)
     method_source (1.0.0)
+    mime-types (3.5.2)
+      mime-types-data (~> 3.2015)
+    mime-types-data (3.2024.0702)
     mini_mime (1.1.5)
     mini_portile2 (2.8.5)
     minitest (5.23.0)
     mize (0.4.1)
       protocol (~> 2.0)
+    multi_json (1.15.0)
     multi_xml (0.6.0)
     mutex_m (0.2.0)
     nap (1.1.0)
@@ -169,6 +235,7 @@ GEM
       faraday (>= 1, < 3)
       sawyer (~> 0.9)
     open4 (1.3.4)
+    os (1.1.4)
     parallel (1.24.0)
     parser (3.3.0.5)
       ast (~> 2.4.1)
@@ -196,6 +263,11 @@ GEM
     rbs (2.8.4)
     rchardet (1.8.0)
     regexp_parser (2.9.0)
+    representable (3.2.0)
+      declarative (< 0.1.0)
+      trailblazer-option (>= 0.1.1, < 0.2.0)
+      uber (< 0.2.0)
+    retriable (3.1.2)
     reverse_markdown (2.1.1)
       nokogiri
     rexml (3.2.6)
@@ -262,6 +334,11 @@ GEM
       faraday (>= 0.17.3, < 3)
     sexp_processor (4.17.1)
     shellany (0.0.1)
+    signet (0.19.0)
+      addressable (~> 2.8)
+      faraday (>= 0.17.5, < 3.a)
+      jwt (>= 1.5, < 3.0)
+      multi_json (~> 1.10)
     simplecov (0.22.0)
       docile (~> 1.1)
       simplecov-html (~> 0.11)
@@ -296,8 +373,10 @@ GEM
     timecop (0.9.8)
     tins (1.32.1)
       sync
+    trailblazer-option (0.1.2)
     tzinfo (2.0.6)
       concurrent-ruby (~> 1.0)
+    uber (0.1.0)
     unicode-display_width (2.5.0)
     unparser (0.6.12)
       diff-lcs (~> 1.3)
@@ -330,4 +409,4 @@ DEPENDENCIES
   webmock (= 3.7.0)
 
 BUNDLED WITH
-   2.5.13
+   2.5.15

data/lib/gitlab_quality/test_tooling/gitlab_client/gitlab_client.rb

@@ -44,11 +44,11 @@ module GitlabQuality
           pipeline = Runtime::Env.pipeline_from_project_name
           channel = case pipeline
                     when "canary"
-                      "qa-production"
-                    when "staging-canary"
-                      "qa-staging"
+                      "e2e-run-production"
+                    when "staging", "staging-canary"
+                      "e2e-run-staging"
                     else
-                      "qa-#{pipeline}"
+                      "e2e-run-#{pipeline}"
                     end
 
           slack_options = {

data/lib/gitlab_quality/test_tooling/test_metrics_exporter/config.rb

@@ -21,11 +21,15 @@ module GitlabQuality
         attr_accessor :influxdb_url,
           :influxdb_token,
           :influxdb_bucket,
+          :gcs_bucket,
+          :gcs_project_id,
+          :gcs_credentials,
+          :gcs_metrics_file_name,
+          :test_metric_file_name,
           :run_type
 
         attr_writer :custom_keys_tags,
-          :custom_keys_fields,
-          :test_metric_file_name
+          :custom_keys_fields
 
         def custom_keys_tags
           @custom_keys_tags || []
@@ -34,10 +38,6 @@ module GitlabQuality
         def custom_keys_fields
           @custom_keys_fields || []
         end
-
-        def test_metric_file_name
-          @test_metric_file_name || "test_metrics.json"
-        end
       end
     end
   end

data/lib/gitlab_quality/test_tooling/test_metrics_exporter/formatter.rb

@@ -29,15 +29,27 @@ module GitlabQuality
           Config.configuration
         end
 
+        # rubocop:disable Metrics/AbcSize
         def setup_test_metrics_exporter(examples)
           @log_test_metrics = LogTestMetrics.new(
             examples: examples,
+            run_type: config.run_type
+          )
+
+          @log_test_metrics.configure_influxdb_client(
             influxdb_url: config.influxdb_url,
             influxdb_token: config.influxdb_token,
-            influxdb_bucket: config.influxdb_bucket,
-            run_type: config.run_type
+            influxdb_bucket: config.influxdb_bucket
+          )
+
+          @log_test_metrics.configure_gcs_client(
+            gcs_bucket: config.gcs_bucket,
+            gcs_project_id: config.gcs_project_id,
+            gcs_credentials: config.gcs_credentials,
+            gcs_metrics_file_name: config.gcs_metrics_file_name
           )
         end
+        # rubocop:enable Metrics/AbcSize
       end
     end
   end

data/lib/gitlab_quality/test_tooling/test_metrics_exporter/log_test_metrics.rb

@@ -5,19 +5,33 @@ module GitlabQuality
     module TestMetricsExporter
       class LogTestMetrics
         include TestMetrics
-        include InfluxdbTools
+        include Support::InfluxdbTools
+        include Support::GcsTools
 
         CUSTOM_METRICS_KEY = :custom_test_metrics
 
-        def initialize(examples:, influxdb_url: nil, influxdb_token: nil, influxdb_bucket: nil, run_type: nil)
+        def initialize(
+          examples:,
+          run_type: nil
+        )
           @examples = examples
+          @run_type = run_type
+        end
+
+        def configure_influxdb_client(influxdb_url: nil, influxdb_token: nil, influxdb_bucket: nil)
           @influxdb_url = influxdb_url
           @influxdb_token = influxdb_token
           @influxdb_bucket = influxdb_bucket
-          @run_type = run_type
         end
 
-        # Push test execution metrics to influxdb
+        def configure_gcs_client(gcs_bucket: nil, gcs_project_id: nil, gcs_credentials: nil, gcs_metrics_file_name: nil)
+          @gcs_bucket = gcs_bucket
+          @gcs_project_id = gcs_project_id
+          @gcs_credentials = gcs_credentials
+          @gcs_metrics_file_name = gcs_metrics_file_name
+        end
+
+        # Push test execution metrics
         #
         # @param [Array<String>] custom_keys_tags
         # @param [Array<String>] custom_keys_fields
@@ -25,10 +39,8 @@ module GitlabQuality
         def push_test_metrics(custom_keys_tags: nil, custom_keys_fields: nil)
           @test_metrics ||= examples.filter_map { |example| parse_test_results(example, custom_keys_tags, custom_keys_fields) }
 
-          write_api(url: influxdb_url, token: influxdb_token, bucket: influxdb_bucket).write(data: test_metrics)
-          Runtime::Logger.debug("Pushed #{test_metrics.length} test execution entries to influxdb")
-        rescue StandardError => e
-          Runtime::Logger.error("Failed to push test execution metrics to influxdb, error: #{e}")
+          push_test_metrics_to_influxdb
+          push_test_metrics_to_gcs
         end
 
         # Save metrics in json file
@@ -43,14 +55,43 @@ module GitlabQuality
           @test_metrics ||= examples.filter_map { |example| parse_test_results(example, custom_keys_tags, custom_keys_fields) }
           file = "tmp/#{file_name}"
 
-          File.write(file, test_metrics.to_json) && Runtime::Logger.debug("Saved test metrics to #{file}")
+          File.write(file, test_metrics.to_json) && Runtime::Logger.info("Saved test metrics to #{file}")
         rescue StandardError => e
           Runtime::Logger.error("Failed to save test execution metrics, error: #{e}")
         end
 
         private
 
-        attr_reader :examples, :test_metrics, :influxdb_url, :influxdb_token, :influxdb_bucket, :run_type
+        attr_reader :examples, :test_metrics, :influxdb_url, :influxdb_token, :influxdb_bucket, :run_type, :gcs_bucket,
+          :gcs_project_id, :gcs_credentials, :gcs_metrics_file_name
+
+        # Push test execution metrics to Influxdb
+        #
+        # @return [nil]
+        def push_test_metrics_to_influxdb
+          write_api(url: influxdb_url, token: influxdb_token, bucket: influxdb_bucket).write(data: test_metrics)
+          Runtime::Logger.info("Pushed #{test_metrics.length} test execution entries to Influxdb")
+        rescue StandardError => e
+          Runtime::Logger.error("Failed to push test execution metrics to Influxdb, error: #{e}")
+        end
+
+        # Push test execution metrics to GCS
+        #
+        # @return [nil]
+        def push_test_metrics_to_gcs
+          gcs_client(project_id: gcs_project_id, credentials: gcs_credentials)
+            .put_object(
+              gcs_bucket || raise("Missing GCS bucket name"),
+              gcs_metrics_file_name,
+              test_metrics.to_json,
+              force: true,
+              content_type: 'application/json'
+            )
+
+          Runtime::Logger.info("Pushed #{test_metrics.length} test execution entries to GCS")
+        rescue StandardError => e
+          Runtime::Logger.error("Failed to push test execution metrics to GCS, error: #{e}")
+        end
 
         # Transform example to influxdb compatible metrics data
         # https://github.com/influxdata/influxdb-client-ruby#data-format

data/lib/gitlab_quality/test_tooling/test_metrics_exporter/support/gcs_tools.rb

@@ -0,0 +1,36 @@
+# frozen_string_literal: true
+
+require 'fog/google'
+
+module GitlabQuality
+  module TestTooling
+    module TestMetricsExporter
+      module Support
+        module GcsTools
+          # GCS Client
+          #
+          # @param project_id [String]
+          # @param credentials [String]
+          # @return [Fog::Storage::Google]
+          def gcs_client(project_id:, credentials:)
+            Fog::Storage::Google.new(
+              google_project: project_id || raise("Missing Google project_id"),
+              **gcs_creds(credentials)
+            )
+          end
+
+          # GCS Credentials
+          #
+          # @param credentials [String]
+          # @return [Hash]
+          def gcs_creds(credentials)
+            json_key = credentials || raise('Missing Google credentials')
+            return { google_json_key_location: json_key } if File.exist?(json_key)
+
+            { google_json_key_string: json_key }
+          end
+        end
+      end
+    end
+  end
+end

data/lib/gitlab_quality/test_tooling/test_metrics_exporter/support/influxdb_tools.rb

@@ -0,0 +1,33 @@
+# frozen_string_literal: true
+
+require 'influxdb-client'
+
+module GitlabQuality
+  module TestTooling
+    module TestMetricsExporter
+      module Support
+        module InfluxdbTools
+          # InfluxDb client
+          #
+          # @return [InfluxDB2::Client]
+          def influx_client(url:, token:, bucket:)
+            @influx_client ||= InfluxDB2::Client.new(
+              url || raise('Missing influxdb_url'),
+              token || raise('Missing influxdb_token'),
+              bucket: bucket || raise('Missing influxdb_bucket'),
+              org: "gitlab-qa",
+              precision: InfluxDB2::WritePrecision::NANOSECOND
+            )
+          end
+
+          # Write client
+          #
+          # @return [WriteApi]
+          def write_api(url:, token:, bucket:)
+            @write_api ||= influx_client(url: url, token: token, bucket: bucket).create_write_api
+          end
+        end
+      end
+    end
+  end
+end

data/lib/gitlab_quality/test_tooling/version.rb

@@ -2,6 +2,6 @@
 
 module GitlabQuality
   module TestTooling
-    VERSION = "1.35.0"
+    VERSION = "1.37.0"
   end
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: gitlab_quality-test_tooling
 version: !ruby/object:Gem::Version
-  version: 1.35.0
+  version: 1.37.0
 platform: ruby
 authors:
 - GitLab Quality
 autorequire:
 bindir: exe
 cert_chain: []
-date: 2024-08-22 00:00:00.000000000 Z
+date: 2024-08-30 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: climate_control
@@ -268,6 +268,26 @@ dependencies:
     - - "~>"
       - !ruby/object:Gem::Version
         version: '3.1'
+- !ruby/object:Gem::Dependency
+  name: fog-google
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.24'
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 1.24.1
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.24'
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 1.24.1
 - !ruby/object:Gem::Dependency
   name: nokogiri
   requirement: !ruby/object:Gem::Requirement
@@ -479,8 +499,9 @@ files:
 - lib/gitlab_quality/test_tooling/test_metrics/json_test_metric_collection.rb
 - lib/gitlab_quality/test_tooling/test_metrics_exporter/config.rb
 - lib/gitlab_quality/test_tooling/test_metrics_exporter/formatter.rb
-- lib/gitlab_quality/test_tooling/test_metrics_exporter/influxdb_tools.rb
 - lib/gitlab_quality/test_tooling/test_metrics_exporter/log_test_metrics.rb
+- lib/gitlab_quality/test_tooling/test_metrics_exporter/support/gcs_tools.rb
+- lib/gitlab_quality/test_tooling/test_metrics_exporter/support/influxdb_tools.rb
 - lib/gitlab_quality/test_tooling/test_metrics_exporter/test_metrics.rb
 - lib/gitlab_quality/test_tooling/test_result/base_test_result.rb
 - lib/gitlab_quality/test_tooling/test_result/j_unit_test_result.rb

data/lib/gitlab_quality/test_tooling/test_metrics_exporter/influxdb_tools.rb

@@ -1,31 +0,0 @@
-# frozen_string_literal: true
-
-require 'influxdb-client'
-
-module GitlabQuality
-  module TestTooling
-    module TestMetricsExporter
-      module InfluxdbTools
-        # InfluxDb client
-        #
-        # @return [InfluxDB2::Client]
-        def influx_client(url:, token:, bucket:)
-          @influx_client ||= InfluxDB2::Client.new(
-            url || raise('Missing influxdb_url'),
-            token || raise('Missing influxdb_token'),
-            bucket: bucket || raise('Missing influxdb_bucket'),
-            org: "gitlab-qa",
-            precision: InfluxDB2::WritePrecision::NANOSECOND
-          )
-        end
-
-        # Write client
-        #
-        # @return [WriteApi]
-        def write_api(url:, token:, bucket:)
-          @write_api ||= influx_client(url: url, token: token, bucket: bucket).create_write_api
-        end
-      end
-    end
-  end
-end