gitlab-qa 8.4.2 → 8.5.0

data/lib/gitlab/qa/support/shell_command.rb

@@ -0,0 +1,98 @@
+# frozen_string_literal: true
+
+require 'open3'
+require 'active_support/core_ext/string/filters'
+
+module Gitlab
+  module QA
+    module Support
+      class ShellCommand
+        using Rainbow
+
+        StatusError = Class.new(StandardError)
+
+        # Shell command
+        #
+        # @param [<String, Array>] command
+        # @param [<String, Array>] mask_secrets
+        # @param [Boolean] stream_output stream command output to stdout directly instead of logger
+        def initialize(command = nil, stdin_data: nil, mask_secrets: nil, stream_output: false)
+          @command = command
+          @mask_secrets = Array(mask_secrets)
+          @stream_output = stream_output
+          @output = []
+          @logger = Runtime::Logger.logger
+          @stdin_data = stdin_data
+        end
+
+        attr_reader :command, :output, :stream_output
+
+        def execute! # rubocop:disable Metrics/AbcSize
+          raise StatusError, 'Command already executed' if output.any?
+
+          logger.info("Shell command: `#{mask_secrets(command).cyan}`")
+
+          Open3.popen2e(@command.to_s) do |stdin, out, wait|
+            if @stdin_data
+              stdin.puts(@stdin_data)
+              stdin.close
+            end
+
+            out.each do |line|
+              output.push(line)
+
+              if stream_progress
+                print "."
+              elsif stream_output
+                puts line
+              end
+
+              yield line, wait if block_given?
+            end
+            puts if stream_progress && !output.empty?
+
+            fail! if wait.value.exited? && wait.value.exitstatus.nonzero?
+
+            logger.debug("Shell command output:\n#{string_output}") unless stream_output || output.empty?
+          end
+
+          string_output
+        end
+
+        private
+
+        attr_reader :logger
+
+        # Raise error and print output to error log level
+        #
+        # @return [void]
+        def fail!
+          logger.error("Shell command output:\n#{string_output}") unless stream_output
+          raise StatusError, "Command `#{mask_secrets(command).truncate(100)}` failed! " + "✘".red
+        end
+
+        # Stream only command execution progress and log output when command finished
+        #
+        # @return [Boolean]
+        def stream_progress
+          !(Runtime::Env.ci || stream_output)
+        end
+
+        # Stringified command output
+        #
+        # @return [String]
+        def string_output
+          mask_secrets(output.join.chomp)
+        end
+
+        # Returns a masked string
+        #
+        # @param [String] input the string to mask
+        # @return [String] The masked string
+        def mask_secrets(input)
+          @mask_secrets.each_with_object(+input) { |secret, s| s.gsub!(secret, '*****') }.to_s
+        end
+      end
+    end
+  end
+end

data/lib/gitlab/qa/support/shellout.rb

@@ -0,0 +1,16 @@
+# frozen_string_literal: true
+
+module Gitlab
+  module QA
+    module Support
+      module Shellout
+        module_function
+
+        def shell(command = nil, stdin_data: nil, mask_secrets: nil, stream_output: false, &block)
+          Support::ShellCommand.new(
+            command, stdin_data: stdin_data, mask_secrets: mask_secrets, stream_output: stream_output).execute!(&block)
+        end
+      end
+    end
+  end
+end

data/lib/gitlab/qa/version.rb

@@ -2,6 +2,6 @@
 
 module Gitlab
   module QA
-    VERSION = '8.4.2'
+    VERSION = '8.5.0'
   end
 end

data/lib/gitlab/qa.rb

@@ -10,7 +10,6 @@ module Gitlab
     loader.ignore("#{__dir__}/qa/version.rb")
 
     loader.inflector.inflect(
-      'dev_ee_qa_image' => 'DevEEQAImage',
       'results_in_testcases' => 'ResultsInTestCases',
       'postgresql' => 'PostgreSQL',
       'registry_with_cdn' => 'RegistryWithCDN',

data/support/manifests/suggested_reviewer/authenticator.yaml

@@ -0,0 +1,41 @@
+---
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: authenticator-deployment
+  labels:
+    app: authenticator
+spec:
+  replicas: 1
+  selector:
+    matchLabels:
+      app: authenticator
+  template:
+    metadata:
+      labels:
+        app: authenticator
+    spec:
+      imagePullSecrets:
+        - name: gitlab-registry
+      containers:
+        - name: authenticator
+          image: registry.gitlab.com/gitlab-org/modelops/applied-ml/review-recommender/authenticator:0.1.0
+          imagePullPolicy: Always
+          ports:
+            - containerPort: 8080
+          args: []
+          env:
+            - name: AUTHENTICATOR_SERVICE_PORT
+              value: "8080"
+---
+apiVersion: v1
+kind: Service
+metadata:
+  name: authenticator-service
+spec:
+  type: NodePort
+  ports:
+    - port: 8080
+      targetPort: 8080
+  selector:
+    app: authenticator

data/support/manifests/suggested_reviewer/postgres.yaml

@@ -0,0 +1,84 @@
+---
+apiVersion: v1
+kind: Secret
+metadata:
+  name: postgres-secret
+type: kubernetes.io/basic-auth
+stringData:
+  username: reviewer-recommender
+  password: ml4dawin
+
+---
+apiVersion: v1
+kind: PersistentVolumeClaim
+metadata:
+  name: postgres-pvc
+spec:
+  storageClassName: local-path
+  accessModes:
+    - ReadWriteOnce
+  resources:
+    requests:
+      storage: 1Gi
+
+---
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: postgres-deployment
+spec:
+  replicas: 1
+  selector:
+    matchLabels:
+      app: postgres
+  template:
+    metadata:
+      labels:
+        app: postgres
+    spec:
+      containers:
+        - name: postgres
+          image: postgres:13-alpine
+          imagePullPolicy: IfNotPresent
+          ports:
+            - containerPort: 5432
+          volumeMounts:
+            - name: postgres-pv
+              mountPath: /var/lib/postgresql/data
+              subPath: postgres
+          resources:
+            limits:
+              memory: 256Mi
+              cpu: 250m
+          env:
+            - name: POSTGRES_DB
+              value: reviewer-recommender
+            - name: POSTGRES_USER
+              valueFrom:
+                secretKeyRef:
+                  name: postgres-secret
+                  key: username
+            - name: POSTGRES_PASSWORD
+              valueFrom:
+                secretKeyRef:
+                  name: postgres-secret
+                  key: password
+      volumes:
+        - name: postgres-pv
+          persistentVolumeClaim:
+            claimName: postgres-pvc
+
+---
+apiVersion: v1
+kind: Service
+metadata:
+  name: postgres
+  labels:
+    app: postgres
+spec:
+  type: ClusterIP
+  selector:
+    app: postgres
+  ports:
+    - port: 5432
+      targetPort: 5432

data/support/manifests/suggested_reviewer/pubsub.yaml

@@ -0,0 +1,41 @@
+---
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: pubsub-deployment
+  labels:
+    app: pubsub
+spec:
+  replicas: 1
+  selector:
+    matchLabels:
+      app: pubsub
+  template:
+    metadata:
+      labels:
+        app: pubsub
+    spec:
+      containers:
+        - name: pubsub
+          image: singularities/pubsub-emulator:latest
+          imagePullPolicy: IfNotPresent
+          env:
+            - name: PUBSUB_PROJECT_ID
+              value: project-test
+            - name: PUBSUB_LISTEN_ADDRESS
+              value: 0.0.0.0:8432
+
+---
+apiVersion: v1
+kind: Service
+metadata:
+  name: pubsub
+  labels:
+    app: pubsub
+spec:
+  type: ClusterIP
+  selector:
+    app: pubsub
+  ports:
+    - port: 8432
+      targetPort: 8432

data/support/manifests/suggested_reviewer/recommender-bot.yaml

@@ -0,0 +1,242 @@
+---
+apiVersion: v1
+kind: Secret
+metadata:
+  name: gcp-credentials
+type: Opaque
+data:
+  gcp_json: "ewogICJjbGllbnRfaWQiOiAiMTIzNDU2Nzg5YWJjZGVmZy5hcHBzLmdvb2dsZXVzZXJjb250ZW50LmNvbSIsCiAgImNsaWVudF9zZWNyZXQiOiAiZC1GTHlld3VxeWV1cXciLAogICJxdW90YV9wcm9qZWN0X2lkIjogInN1Z2dlc3RlZC1yZXZpZXdlci0xNTA0MDBlNiIsCiAgInJlZnJlc2hfdG9rZW4iOiAiMS8vMTIzNDU2NzhhYmNkZWZnIiwKICAidHlwZSI6ICJhdXRob3JpemVkX3VzZXIiCn0K"
+
+---
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: recommender-bot-envoy-sidecar-config
+  labels:
+    app: recommender-bot
+data:
+  envoy.yaml: |
+    static_resources:
+      listeners:
+        - name: "recommender-bot-http-listener"
+          address:
+            socket_address: { address: "0.0.0.0", port_value: 8282 }
+          filter_chains:
+            - filters:
+              - name: "envoy.filters.network.http_connection_manager"
+                typed_config:
+                  "@type": type.googleapis.com/envoy.extensions.filters.network.http_connection_manager.v3.HttpConnectionManager
+                  stat_prefix: "grpc_json"
+                  codec_type: "AUTO"
+                  route_config:
+                    name: "local_route"
+                    virtual_hosts:
+                      - name: "local_service"
+                        domains: [ "*" ]
+                        routes:
+                          - match: { prefix: "/", grpc: {} }
+                            route: { cluster: "recommender-bot", timeout: 60s }
+                  http_filters:
+                    - name: "envoy.filters.http.grpc_json_transcoder"
+                      typed_config:
+                        "@type": type.googleapis.com/envoy.extensions.filters.http.grpc_json_transcoder.v3.GrpcJsonTranscoder
+                        proto_descriptor: "/data/protos/recommender-bot.protoset.pb"
+                        services: [ "bot.RecommenderService" ]
+                        print_options:
+                          add_whitespace: true
+                          always_print_primitive_fields: true
+                          always_print_enums_as_ints: false
+                          preserve_proto_field_names: false
+                        convert_grpc_status: true
+                        request_validation_options:
+                          reject_unknown_method: true
+                          reject_unknown_query_parameters: true
+                    - name: "envoy.filters.http.router"
+        - name: "recommender-bot-http-auth-listener"
+          address:
+            socket_address: { address: "0.0.0.0", port_value: 8484 }
+          filter_chains:
+            - filters:
+              - name: "envoy.filters.network.http_connection_manager"
+                typed_config:
+                  "@type": type.googleapis.com/envoy.extensions.filters.network.http_connection_manager.v3.HttpConnectionManager
+                  stat_prefix: "grpc_json"
+                  codec_type: "AUTO"
+                  route_config:
+                    name: "local_route"
+                    virtual_hosts:
+                      - name: "local_service"
+                        domains: [ "*" ]
+                        routes:
+                          - match: { prefix: "/", grpc: {} }
+                            route: { cluster: "recommender-bot", timeout: 60s }
+                  http_filters:
+                    - name: "envoy.filters.http.grpc_json_transcoder"
+                      typed_config:
+                        "@type": type.googleapis.com/envoy.extensions.filters.http.grpc_json_transcoder.v3.GrpcJsonTranscoder
+                        proto_descriptor: "/data/protos/recommender-bot.protoset.pb"
+                        services: [ "bot.RecommenderService" ]
+                        print_options:
+                          add_whitespace: true
+                          always_print_primitive_fields: true
+                          always_print_enums_as_ints: false
+                          preserve_proto_field_names: false
+                        convert_grpc_status: true
+                        request_validation_options:
+                          reject_unknown_method: true
+                          reject_unknown_query_parameters: true
+                    - name: "envoy.filters.http.ext_authz"
+                      typed_config:
+                        "@type": type.googleapis.com/envoy.extensions.filters.http.ext_authz.v3.ExtAuthz
+                        http_service:
+                          server_uri:
+                            uri: "http://authenticator-service:8080"
+                            cluster: "ext-auth"
+                            timeout: 60s
+                          authorization_request:
+                            allowed_headers:
+                              patterns:
+                                - { exact: "Job-Token", ignore_case: true }
+                                - { exact: "Project-Id", ignore_case: true }
+                        failure_mode_allow: false
+                    - name: "envoy.filters.http.router"
+
+      clusters:
+        - name: "recommender-bot"
+          connect_timeout: "1.25s"
+          type: "STATIC"
+          lb_policy: "ROUND_ROBIN"
+          dns_lookup_family: V4_ONLY
+          typed_extension_protocol_options:
+            envoy.extensions.upstreams.http.v3.HttpProtocolOptions:
+              "@type": type.googleapis.com/envoy.extensions.upstreams.http.v3.HttpProtocolOptions
+              explicit_http_config:
+                http2_protocol_options: { }
+          load_assignment:
+            cluster_name: "recommender-bot"
+            endpoints:
+              - lb_endpoints:
+                  - endpoint:
+                      address:
+                        socket_address:
+                          address: "127.0.0.1"
+                          port_value: 8080
+        - name: "ext-auth"
+          connect_timeout: "1.25s"
+          type: "LOGICAL_DNS"
+          lb_policy: "ROUND_ROBIN"
+          load_assignment:
+            cluster_name: "ext-auth"
+            endpoints:
+              - lb_endpoints:
+                  - endpoint:
+                      address:
+                        socket_address:
+                          address: "authenticator-service"
+                          port_value: 8080
+
+---
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: recommender-bot-deployment
+  labels:
+    app: recommender-bot
+spec:
+  replicas: 1
+  selector:
+    matchLabels:
+      app: recommender-bot
+  template:
+    metadata:
+      labels:
+        app: recommender-bot
+    spec:
+      initContainers:
+        - name: protosets
+          image: recommender-bot-service:dev
+          imagePullPolicy: Never
+          command: [sh, -c]
+          args: [cp /app/recommender-bot.protoset.pb /data/protos]
+          volumeMounts:
+            - name: grpc-protosets
+              mountPath: /data/protos
+      containers:
+        - name: recommender-bot-envoy-sidecar
+          image: envoyproxy/envoy-alpine:v1.19-latest
+          imagePullPolicy: IfNotPresent
+          ports:
+            - containerPort: 8282
+            - containerPort: 8484
+          volumeMounts:
+            - name: recommender-bot-envoy-sidecar-config
+              mountPath: /etc/envoy
+            - name: grpc-protosets
+              mountPath: /data/protos
+              readOnly: true
+        - name: recommender-bot
+          image: recommender-bot-service:dev
+          imagePullPolicy: Never
+          ports:
+            - containerPort: 8080
+          env:
+            - name: RECOMMENDER_BOT_SERVICE_PORT
+              value: "8080"
+            - name: RECOMMENDER_BOT_GITLAB_TOKEN
+              valueFrom:
+                secretKeyRef:
+                  name: recommender-bot-gitlab-token
+                  key: token
+            - name: RECOMMENDER_BOT_GRPC_SERVICE_URL
+              value: recommender-service:8080
+            - name: RECOMMENDER_BOT_PG_CONN_STRING
+              valueFrom:
+                secretKeyRef:
+                  name: recommender-bot-pg-conn-string
+                  key: pg-conn
+            - name: PUBSUB_EMULATOR_HOST
+              value: pubsub:8432
+            - name: GOOGLE_APPLICATION_CREDENTIALS
+              value: /data/gcp/gcp_credentials.json
+            - name: RECOMMENDER_BOT_PUBSUB_PROJECT_ID
+              value: project-test
+            - name: RECOMMENDER_BOT_PUBSUB_TOPIC_RECOMMENDATIONS
+              value: gitlab.merge-request-recommendations-test.1
+          volumeMounts:
+            - name: gcp-secrets
+              mountPath: /data/gcp
+              readOnly: true
+      volumes:
+        - name: grpc-protosets
+          emptyDir: {}
+        - name: recommender-bot-envoy-sidecar-config
+          configMap:
+            name: recommender-bot-envoy-sidecar-config
+        - name: gcp-secrets
+          secret:
+            secretName: gcp-credentials
+            items:
+              - key: gcp_json
+                path: gcp_credentials.json
+
+---
+apiVersion: v1
+kind: Service
+metadata:
+  name: recommender-bot-service
+  labels:
+    app: recommender-bot
+spec:
+  type: NodePort
+  ports:
+    - name: http-plain
+      port: 8282
+      targetPort: 8282
+    - name: http-auth
+      port: 8484
+      targetPort: 8484
+    - name: grpc
+      port: 8080
+      targetPort: 8080
+  selector:
+    app: recommender-bot

data/support/manifests/suggested_reviewer/recommender.yaml

@@ -0,0 +1,52 @@
+---
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: recommender-deployment
+  labels:
+    app: recommender
+spec:
+  replicas: 1
+  selector:
+    matchLabels:
+      app: recommender
+  template:
+    metadata:
+      labels:
+        app: recommender
+    spec:
+      containers:
+        - name: recommender
+          image: recommender-service:dev
+          imagePullPolicy: Never
+          ports:
+            - containerPort: 8080
+          volumeMounts:
+            - name: models-mount
+              mountPath: /app/data/models
+          env:
+            - name: RECOMMENDER_SERVICE_PORT
+              value: "8080"
+            - name: RECOMMENDER_MODELS_STORAGE
+              value: local
+            - name: GRPC_VERBOSITY
+              value: debug
+      volumes:
+        - name: models-mount
+          hostPath:
+            path: /tmp/gitlab-qa/suggested_reviewer/data/models
+
+---
+apiVersion: v1
+kind: Service
+metadata:
+  name: recommender-service
+  labels:
+    app: recommender
+spec:
+  type: ClusterIP
+  ports:
+    - port: 8080
+      targetPort: 8080
+  selector:
+    app: recommender

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: gitlab-qa
 version: !ruby/object:Gem::Version
-  version: 8.4.2
+  version: 8.5.0
 platform: ruby
 authors:
 - GitLab Quality
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2022-09-07 00:00:00.000000000 Z
+date: 2022-09-28 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: climate_control
@@ -327,10 +327,10 @@ files:
 - lib/gitlab/qa/component/specs.rb
 - lib/gitlab/qa/component/staging.rb
 - lib/gitlab/qa/component/staging_ref.rb
+- lib/gitlab/qa/component/suggested_reviewer.rb
 - lib/gitlab/qa/component/telegraf.rb
 - lib/gitlab/qa/docker/command.rb
 - lib/gitlab/qa/docker/engine.rb
-- lib/gitlab/qa/docker/shellout.rb
 - lib/gitlab/qa/docker/volumes.rb
 - lib/gitlab/qa/release.rb
 - lib/gitlab/qa/report/base_test_results.rb
@@ -408,23 +408,33 @@ files:
 - lib/gitlab/qa/scenario/test/integration/service_ping_disabled.rb
 - lib/gitlab/qa/scenario/test/integration/smtp.rb
 - lib/gitlab/qa/scenario/test/integration/ssh_tunnel.rb
+- lib/gitlab/qa/scenario/test/integration/suggested_reviewer.rb
 - lib/gitlab/qa/scenario/test/omnibus/image.rb
 - lib/gitlab/qa/scenario/test/omnibus/update.rb
 - lib/gitlab/qa/scenario/test/omnibus/update_from_previous.rb
 - lib/gitlab/qa/scenario/test/omnibus/upgrade.rb
 - lib/gitlab/qa/scenario/test/sanity/version.rb
+- lib/gitlab/qa/service/cluster_provider/base.rb
+- lib/gitlab/qa/service/cluster_provider/k3d.rb
+- lib/gitlab/qa/service/kubernetes_cluster.rb
 - lib/gitlab/qa/slack/post_to_slack.rb
-- lib/gitlab/qa/support/dev_ee_qa_image.rb
 - lib/gitlab/qa/support/get_request.rb
 - lib/gitlab/qa/support/gitlab_upgrade_path.rb
 - lib/gitlab/qa/support/gitlab_version_info.rb
 - lib/gitlab/qa/support/http_request.rb
 - lib/gitlab/qa/support/invalid_response_error.rb
+- lib/gitlab/qa/support/shell_command.rb
+- lib/gitlab/qa/support/shellout.rb
 - lib/gitlab/qa/test_logger.rb
 - lib/gitlab/qa/version.rb
 - scripts/generate-qa-jobs.rb
 - support/data/admin_access_token_seed.rb
 - support/data/license_usage_seed.rb
+- support/manifests/suggested_reviewer/authenticator.yaml
+- support/manifests/suggested_reviewer/postgres.yaml
+- support/manifests/suggested_reviewer/pubsub.yaml
+- support/manifests/suggested_reviewer/recommender-bot.yaml
+- support/manifests/suggested_reviewer/recommender.yaml
 - tls_certificates/authority/ca.crt
 - tls_certificates/authority/ca.key
 - tls_certificates/authority/ca.pem