gitlab-qa 8.4.2 → 8.6.0

data/support/manifests/suggested_reviewer/recommender-bot.yaml

@@ -0,0 +1,242 @@
+---
+apiVersion: v1
+kind: Secret
+metadata:
+  name: gcp-credentials
+type: Opaque
+data:
+  gcp_json: "ewogICJjbGllbnRfaWQiOiAiMTIzNDU2Nzg5YWJjZGVmZy5hcHBzLmdvb2dsZXVzZXJjb250ZW50LmNvbSIsCiAgImNsaWVudF9zZWNyZXQiOiAiZC1GTHlld3VxeWV1cXciLAogICJxdW90YV9wcm9qZWN0X2lkIjogInN1Z2dlc3RlZC1yZXZpZXdlci0xNTA0MDBlNiIsCiAgInJlZnJlc2hfdG9rZW4iOiAiMS8vMTIzNDU2NzhhYmNkZWZnIiwKICAidHlwZSI6ICJhdXRob3JpemVkX3VzZXIiCn0K"
+
+---
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: recommender-bot-envoy-sidecar-config
+  labels:
+    app: recommender-bot
+data:
+  envoy.yaml: |
+    static_resources:
+      listeners:
+        - name: "recommender-bot-http-listener"
+          address:
+            socket_address: { address: "0.0.0.0", port_value: 8282 }
+          filter_chains:
+            - filters:
+              - name: "envoy.filters.network.http_connection_manager"
+                typed_config:
+                  "@type": type.googleapis.com/envoy.extensions.filters.network.http_connection_manager.v3.HttpConnectionManager
+                  stat_prefix: "grpc_json"
+                  codec_type: "AUTO"
+                  route_config:
+                    name: "local_route"
+                    virtual_hosts:
+                      - name: "local_service"
+                        domains: [ "*" ]
+                        routes:
+                          - match: { prefix: "/", grpc: {} }
+                            route: { cluster: "recommender-bot", timeout: 60s }
+                  http_filters:
+                    - name: "envoy.filters.http.grpc_json_transcoder"
+                      typed_config:
+                        "@type": type.googleapis.com/envoy.extensions.filters.http.grpc_json_transcoder.v3.GrpcJsonTranscoder
+                        proto_descriptor: "/data/protos/recommender-bot.protoset.pb"
+                        services: [ "bot.RecommenderService" ]
+                        print_options:
+                          add_whitespace: true
+                          always_print_primitive_fields: true
+                          always_print_enums_as_ints: false
+                          preserve_proto_field_names: false
+                        convert_grpc_status: true
+                        request_validation_options:
+                          reject_unknown_method: true
+                          reject_unknown_query_parameters: true
+                    - name: "envoy.filters.http.router"
+        - name: "recommender-bot-http-auth-listener"
+          address:
+            socket_address: { address: "0.0.0.0", port_value: 8484 }
+          filter_chains:
+            - filters:
+              - name: "envoy.filters.network.http_connection_manager"
+                typed_config:
+                  "@type": type.googleapis.com/envoy.extensions.filters.network.http_connection_manager.v3.HttpConnectionManager
+                  stat_prefix: "grpc_json"
+                  codec_type: "AUTO"
+                  route_config:
+                    name: "local_route"
+                    virtual_hosts:
+                      - name: "local_service"
+                        domains: [ "*" ]
+                        routes:
+                          - match: { prefix: "/", grpc: {} }
+                            route: { cluster: "recommender-bot", timeout: 60s }
+                  http_filters:
+                    - name: "envoy.filters.http.grpc_json_transcoder"
+                      typed_config:
+                        "@type": type.googleapis.com/envoy.extensions.filters.http.grpc_json_transcoder.v3.GrpcJsonTranscoder
+                        proto_descriptor: "/data/protos/recommender-bot.protoset.pb"
+                        services: [ "bot.RecommenderService" ]
+                        print_options:
+                          add_whitespace: true
+                          always_print_primitive_fields: true
+                          always_print_enums_as_ints: false
+                          preserve_proto_field_names: false
+                        convert_grpc_status: true
+                        request_validation_options:
+                          reject_unknown_method: true
+                          reject_unknown_query_parameters: true
+                    - name: "envoy.filters.http.ext_authz"
+                      typed_config:
+                        "@type": type.googleapis.com/envoy.extensions.filters.http.ext_authz.v3.ExtAuthz
+                        http_service:
+                          server_uri:
+                            uri: "http://authenticator-service:8080"
+                            cluster: "ext-auth"
+                            timeout: 60s
+                          authorization_request:
+                            allowed_headers:
+                              patterns:
+                                - { exact: "Job-Token", ignore_case: true }
+                                - { exact: "Project-Id", ignore_case: true }
+                        failure_mode_allow: false
+                    - name: "envoy.filters.http.router"
+
+      clusters:
+        - name: "recommender-bot"
+          connect_timeout: "1.25s"
+          type: "STATIC"
+          lb_policy: "ROUND_ROBIN"
+          dns_lookup_family: V4_ONLY
+          typed_extension_protocol_options:
+            envoy.extensions.upstreams.http.v3.HttpProtocolOptions:
+              "@type": type.googleapis.com/envoy.extensions.upstreams.http.v3.HttpProtocolOptions
+              explicit_http_config:
+                http2_protocol_options: { }
+          load_assignment:
+            cluster_name: "recommender-bot"
+            endpoints:
+              - lb_endpoints:
+                  - endpoint:
+                      address:
+                        socket_address:
+                          address: "127.0.0.1"
+                          port_value: 8080
+        - name: "ext-auth"
+          connect_timeout: "1.25s"
+          type: "LOGICAL_DNS"
+          lb_policy: "ROUND_ROBIN"
+          load_assignment:
+            cluster_name: "ext-auth"
+            endpoints:
+              - lb_endpoints:
+                  - endpoint:
+                      address:
+                        socket_address:
+                          address: "authenticator-service"
+                          port_value: 8080
+
+---
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: recommender-bot-deployment
+  labels:
+    app: recommender-bot
+spec:
+  replicas: 1
+  selector:
+    matchLabels:
+      app: recommender-bot
+  template:
+    metadata:
+      labels:
+        app: recommender-bot
+    spec:
+      initContainers:
+        - name: protosets
+          image: recommender-bot-service:dev
+          imagePullPolicy: Never
+          command: [sh, -c]
+          args: [cp /app/recommender-bot.protoset.pb /data/protos]
+          volumeMounts:
+            - name: grpc-protosets
+              mountPath: /data/protos
+      containers:
+        - name: recommender-bot-envoy-sidecar
+          image: envoyproxy/envoy-alpine:v1.19-latest
+          imagePullPolicy: IfNotPresent
+          ports:
+            - containerPort: 8282
+            - containerPort: 8484
+          volumeMounts:
+            - name: recommender-bot-envoy-sidecar-config
+              mountPath: /etc/envoy
+            - name: grpc-protosets
+              mountPath: /data/protos
+              readOnly: true
+        - name: recommender-bot
+          image: recommender-bot-service:dev
+          imagePullPolicy: Never
+          ports:
+            - containerPort: 8080
+          env:
+            - name: RECOMMENDER_BOT_SERVICE_PORT
+              value: "8080"
+            - name: RECOMMENDER_BOT_GITLAB_TOKEN
+              valueFrom:
+                secretKeyRef:
+                  name: recommender-bot-gitlab-token
+                  key: token
+            - name: RECOMMENDER_BOT_GRPC_SERVICE_URL
+              value: recommender-service:8080
+            - name: RECOMMENDER_BOT_PG_CONN_STRING
+              valueFrom:
+                secretKeyRef:
+                  name: recommender-bot-pg-conn-string
+                  key: pg-conn
+            - name: PUBSUB_EMULATOR_HOST
+              value: pubsub:8432
+            - name: GOOGLE_APPLICATION_CREDENTIALS
+              value: /data/gcp/gcp_credentials.json
+            - name: RECOMMENDER_BOT_PUBSUB_PROJECT_ID
+              value: project-test
+            - name: RECOMMENDER_BOT_PUBSUB_TOPIC_RECOMMENDATIONS
+              value: gitlab.merge-request-recommendations-test.1
+          volumeMounts:
+            - name: gcp-secrets
+              mountPath: /data/gcp
+              readOnly: true
+      volumes:
+        - name: grpc-protosets
+          emptyDir: {}
+        - name: recommender-bot-envoy-sidecar-config
+          configMap:
+            name: recommender-bot-envoy-sidecar-config
+        - name: gcp-secrets
+          secret:
+            secretName: gcp-credentials
+            items:
+              - key: gcp_json
+                path: gcp_credentials.json
+
+---
+apiVersion: v1
+kind: Service
+metadata:
+  name: recommender-bot-service
+  labels:
+    app: recommender-bot
+spec:
+  type: NodePort
+  ports:
+    - name: http-plain
+      port: 8282
+      targetPort: 8282
+    - name: http-auth
+      port: 8484
+      targetPort: 8484
+    - name: grpc
+      port: 8080
+      targetPort: 8080
+  selector:
+    app: recommender-bot

data/support/manifests/suggested_reviewer/recommender.yaml

@@ -0,0 +1,52 @@
+---
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: recommender-deployment
+  labels:
+    app: recommender
+spec:
+  replicas: 1
+  selector:
+    matchLabels:
+      app: recommender
+  template:
+    metadata:
+      labels:
+        app: recommender
+    spec:
+      containers:
+        - name: recommender
+          image: recommender-service:dev
+          imagePullPolicy: Never
+          ports:
+            - containerPort: 8080
+          volumeMounts:
+            - name: models-mount
+              mountPath: /app/data/models
+          env:
+            - name: RECOMMENDER_SERVICE_PORT
+              value: "8080"
+            - name: RECOMMENDER_MODELS_STORAGE
+              value: local
+            - name: GRPC_VERBOSITY
+              value: debug
+      volumes:
+        - name: models-mount
+          hostPath:
+            path: /tmp/gitlab-qa/suggested_reviewer/data/models
+
+---
+apiVersion: v1
+kind: Service
+metadata:
+  name: recommender-service
+  labels:
+    app: recommender
+spec:
+  type: ClusterIP
+  ports:
+    - port: 8080
+      targetPort: 8080
+  selector:
+    app: recommender

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: gitlab-qa
 version: !ruby/object:Gem::Version
-  version: 8.4.2
+  version: 8.6.0
 platform: ruby
 authors:
 - GitLab Quality
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2022-09-07 00:00:00.000000000 Z
+date: 2022-10-11 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: climate_control
@@ -313,7 +313,6 @@ files:
 - lib/gitlab/qa/component/base.rb
 - lib/gitlab/qa/component/elasticsearch.rb
 - lib/gitlab/qa/component/gitlab.rb
-- lib/gitlab/qa/component/internet_tunnel.rb
 - lib/gitlab/qa/component/jira.rb
 - lib/gitlab/qa/component/ldap.rb
 - lib/gitlab/qa/component/mail_hog.rb
@@ -327,13 +326,14 @@ files:
 - lib/gitlab/qa/component/specs.rb
 - lib/gitlab/qa/component/staging.rb
 - lib/gitlab/qa/component/staging_ref.rb
+- lib/gitlab/qa/component/suggested_reviewer.rb
 - lib/gitlab/qa/component/telegraf.rb
 - lib/gitlab/qa/docker/command.rb
 - lib/gitlab/qa/docker/engine.rb
-- lib/gitlab/qa/docker/shellout.rb
 - lib/gitlab/qa/docker/volumes.rb
 - lib/gitlab/qa/release.rb
 - lib/gitlab/qa/report/base_test_results.rb
+- lib/gitlab/qa/report/find_set_dri.rb
 - lib/gitlab/qa/report/generate_test_session.rb
 - lib/gitlab/qa/report/gitlab_issue_client.rb
 - lib/gitlab/qa/report/gitlab_issue_dry_client.rb
@@ -391,7 +391,6 @@ files:
 - lib/gitlab/qa/scenario/test/integration/instance_saml.rb
 - lib/gitlab/qa/scenario/test/integration/integrations.rb
 - lib/gitlab/qa/scenario/test/integration/jira.rb
-- lib/gitlab/qa/scenario/test/integration/kubernetes.rb
 - lib/gitlab/qa/scenario/test/integration/ldap.rb
 - lib/gitlab/qa/scenario/test/integration/ldap_no_server.rb
 - lib/gitlab/qa/scenario/test/integration/ldap_no_tls.rb
@@ -407,24 +406,33 @@ files:
 - lib/gitlab/qa/scenario/test/integration/saml.rb
 - lib/gitlab/qa/scenario/test/integration/service_ping_disabled.rb
 - lib/gitlab/qa/scenario/test/integration/smtp.rb
-- lib/gitlab/qa/scenario/test/integration/ssh_tunnel.rb
+- lib/gitlab/qa/scenario/test/integration/suggested_reviewer.rb
 - lib/gitlab/qa/scenario/test/omnibus/image.rb
 - lib/gitlab/qa/scenario/test/omnibus/update.rb
 - lib/gitlab/qa/scenario/test/omnibus/update_from_previous.rb
 - lib/gitlab/qa/scenario/test/omnibus/upgrade.rb
 - lib/gitlab/qa/scenario/test/sanity/version.rb
+- lib/gitlab/qa/service/cluster_provider/base.rb
+- lib/gitlab/qa/service/cluster_provider/k3d.rb
+- lib/gitlab/qa/service/kubernetes_cluster.rb
 - lib/gitlab/qa/slack/post_to_slack.rb
-- lib/gitlab/qa/support/dev_ee_qa_image.rb
 - lib/gitlab/qa/support/get_request.rb
 - lib/gitlab/qa/support/gitlab_upgrade_path.rb
 - lib/gitlab/qa/support/gitlab_version_info.rb
 - lib/gitlab/qa/support/http_request.rb
 - lib/gitlab/qa/support/invalid_response_error.rb
+- lib/gitlab/qa/support/shell_command.rb
+- lib/gitlab/qa/support/shellout.rb
 - lib/gitlab/qa/test_logger.rb
 - lib/gitlab/qa/version.rb
 - scripts/generate-qa-jobs.rb
 - support/data/admin_access_token_seed.rb
 - support/data/license_usage_seed.rb
+- support/manifests/suggested_reviewer/authenticator.yaml
+- support/manifests/suggested_reviewer/postgres.yaml
+- support/manifests/suggested_reviewer/pubsub.yaml
+- support/manifests/suggested_reviewer/recommender-bot.yaml
+- support/manifests/suggested_reviewer/recommender.yaml
 - tls_certificates/authority/ca.crt
 - tls_certificates/authority/ca.key
 - tls_certificates/authority/ca.pem

data/lib/gitlab/qa/component/internet_tunnel.rb

@@ -1,76 +0,0 @@
-# frozen_string_literal: true
-
-require 'tempfile'
-
-module Gitlab
-  module QA
-    module Component
-      class InternetTunnel < Base
-        DOCKER_IMAGE = 'gitlab/ssh-tunnel'
-        DOCKER_IMAGE_TAG = '1.0.0'
-
-        attr_writer :gitlab_hostname
-
-        def initialize
-          super
-
-          key_dir = ENV['CI_PROJECT_DIR'] || Dir.tmpdir
-          @ssh_key = Tempfile.new('tunnel-ssh-private-key', key_dir)
-          @ssh_key.write(ENV.fetch('TUNNEL_SSH_PRIVATE_KEY'))
-          @ssh_key.close
-
-          File.chmod(0o600, @ssh_key.path)
-
-          @volumes['/root/.ssh/id_rsa'] = @ssh_key.path
-        end
-
-        def instance
-          raise 'Please provide a block!' unless block_given?
-
-          super
-        end
-
-        def url
-          "https://#{subdomain}.#{tunnel_server_hostname}"
-        end
-
-        private
-
-        def name
-          @name ||= "ssh-tunnel-#{SecureRandom.hex(4)}"
-        end
-
-        def tunnel_server_hostname
-          ENV.fetch("TUNNEL_SERVER_HOSTNAME")
-        end
-
-        def subdomain
-          @subdomain ||= rand(30_000..49_000)
-        end
-
-        def start
-          raise "Must set gitlab_hostname" unless @gitlab_hostname
-
-          @docker.run(
-            image: DOCKER_IMAGE,
-            tag: DOCKER_IMAGE_TAG,
-            args: ["-o StrictHostKeyChecking=no -N -R #{subdomain}:#{@gitlab_hostname}:80 #{ENV.fetch('TUNNEL_SSH_USER')}@#{tunnel_server_hostname}"]) do |command|
-            command << '-d '
-            command << "--name #{name}"
-            command << "--net #{network}"
-
-            @volumes.to_h.each do |to, from|
-              command.volume(from, to, 'Z')
-            end
-          end
-        end
-
-        def teardown
-          super
-
-          @ssh_key.unlink
-        end
-      end
-    end
-  end
-end

data/lib/gitlab/qa/docker/shellout.rb

@@ -1,77 +0,0 @@
-# frozen_string_literal: true
-
-require 'open3'
-require 'active_support/core_ext/string/filters'
-
-module Gitlab
-  module QA
-    module Docker
-      class Shellout
-        using Rainbow
-
-        StatusError = Class.new(StandardError)
-
-        def initialize(command)
-          @command = command
-          @output = []
-          @logger = Runtime::Logger.logger
-        end
-
-        attr_reader :command, :output
-
-        def execute! # rubocop:disable Metrics/AbcSize
-          raise StatusError, 'Command already executed' if output.any?
-
-          logger.info("Docker shell command: `#{command.mask_secrets.cyan}`")
-
-          Open3.popen2e(@command.to_s) do |_in, out, wait|
-            out.each do |line|
-              output.push(line)
-
-              if stream_progress
-                print "."
-              elsif command.stream_output
-                puts line
-              end
-
-              yield line, wait if block_given?
-            end
-            puts if stream_progress && !output.empty?
-
-            fail! if wait.value.exited? && wait.value.exitstatus.nonzero?
-
-            logger.debug("Docker shell command output:\n#{string_output}") unless command.stream_output || output.empty?
-          end
-
-          string_output
-        end
-
-        private
-
-        attr_reader :logger
-
-        # Raise error and print output to error log level
-        #
-        # @return [void]
-        def fail!
-          logger.error("Docker shell command output:\n#{string_output}") unless command.stream_output
-          raise StatusError, "Docker command `#{command.mask_secrets.truncate(100)}` failed! " + "✘".red
-        end
-
-        # Stream only command execution progress and log output when command finished
-        #
-        # @return [Boolean]
-        def stream_progress
-          !(Runtime::Env.ci || command.stream_output)
-        end
-
-        # Stringified command output
-        #
-        # @return [String]
-        def string_output
-          output.join.chomp
-        end
-      end
-    end
-  end
-end

data/lib/gitlab/qa/scenario/test/integration/kubernetes.rb

@@ -1,56 +0,0 @@
-# frozen_string_literal: true
-
-require 'yaml'
-
-module Gitlab
-  module QA
-    module Scenario
-      module Test
-        module Integration
-          class Kubernetes < Scenario::Template
-            def perform(release, *rspec_args)
-              Runtime::Env.require_kubernetes_environment!
-
-              Component::Gitlab.perform do |gitlab|
-                gitlab.release = release
-                gitlab.network = 'test'
-
-                Component::InternetTunnel.perform do |tunnel_gitlab|
-                  Component::InternetTunnel.perform do |tunnel_registry|
-                    tunnel_gitlab.gitlab_hostname = gitlab.hostname
-                    tunnel_gitlab.network = 'test'
-                    tunnel_registry.gitlab_hostname = gitlab.hostname
-                    tunnel_registry.network = 'test'
-
-                    gitlab.omnibus_configuration << <<~OMNIBUS
-                      external_url '#{tunnel_gitlab.url}';
-                      nginx['listen_port'] = 80;
-                      nginx['listen_https'] = false;
-
-                      registry_external_url '#{tunnel_registry.url}';
-                      registry_nginx['listen_port'] = 80;
-                      registry_nginx['listen_https'] = false;
-                    OMNIBUS
-
-                    tunnel_gitlab.instance do
-                      tunnel_registry.instance do
-                        gitlab.instance do
-                          Component::Specs.perform do |specs|
-                            specs.suite = 'Test::Integration::Kubernetes'
-                            specs.release = gitlab.release
-                            specs.network = gitlab.network
-                            specs.args = [tunnel_gitlab.url, *rspec_args]
-                          end
-                        end
-                      end
-                    end
-                  end
-                end
-              end
-            end
-          end
-        end
-      end
-    end
-  end
-end

data/lib/gitlab/qa/scenario/test/integration/ssh_tunnel.rb

@@ -1,56 +0,0 @@
-# frozen_string_literal: true
-
-module Gitlab
-  module QA
-    module Scenario
-      module Test
-        module Integration
-          class SSHTunnel < Scenario::Template
-            def perform(release, *rspec_args)
-              Runtime::Env.override_default_password!
-              Runtime::Env.require_ssh_tunnel!
-
-              Component::Gitlab.perform do |gitlab|
-                gitlab.release = release
-                gitlab.network = 'test'
-
-                Component::InternetTunnel.perform do |tunnel_gitlab|
-                  Component::InternetTunnel.perform do |tunnel_registry|
-                    tunnel_gitlab.gitlab_hostname = gitlab.hostname
-                    tunnel_gitlab.network = 'test'
-                    tunnel_registry.gitlab_hostname = gitlab.hostname
-                    tunnel_registry.network = 'test'
-
-                    gitlab.omnibus_configuration << <<~OMNIBUS
-                      external_url '#{tunnel_gitlab.url}';
-                      nginx['listen_port'] = 80;
-                      nginx['listen_https'] = false;
-                      gitlab_rails['initial_root_password'] = '#{Runtime::Env.require_initial_password!}';
-
-                      registry_external_url '#{tunnel_registry.url}';
-                      registry_nginx['listen_port'] = 80;
-                      registry_nginx['listen_https'] = false;
-                    OMNIBUS
-
-                    tunnel_gitlab.instance do
-                      tunnel_registry.instance do
-                        gitlab.instance do
-                          Component::Specs.perform do |specs|
-                            specs.suite = 'Test::Integration::SSHTunnel'
-                            specs.release = gitlab.release
-                            specs.network = gitlab.network
-                            specs.args = [tunnel_gitlab.url, *rspec_args]
-                          end
-                        end
-                      end
-                    end
-                  end
-                end
-              end
-            end
-          end
-        end
-      end
-    end
-  end
-end