gitlab-qa 8.4.2 → 8.6.0

data/lib/gitlab/qa/service/cluster_provider/k3d.rb

@@ -0,0 +1,141 @@
+# frozen_string_literal: true
+
+module Gitlab
+  module QA
+    module Service
+      module ClusterProvider
+        class K3d < Base
+          def create_registry_mirror
+            return if registry_mirror_exists?('registry-gitlab')
+
+            Runtime::Env.require_qa_container_registry_access_token!
+
+            shell <<~CMD
+              k3d registry create registry-gitlab \
+                -p 5000 \
+                --proxy-remote-url https://#{GITLAB_REGISTRY} \
+                --proxy-password #{Runtime::Env.gitlab_username} \
+                --proxy-username #{Runtime::Env.qa_container_registry_access_token} \
+                -v tmp/registry-gitlab:/var/lib/registry
+            CMD
+
+            File.write('tmp/registry-mirror.yml', registry_mirror)
+
+            create_args << %w[--registry-use k3d-registry-gitlab:5000 --registry-config tmp/registry-mirror.yml]
+          end
+
+          def validate_dependencies
+            find_executable('k3d') || raise("You must first install `k3d` executable to run these tests.")
+          end
+
+          def setup
+            shell "k3d cluster create #{cluster_name} #{create_args&.join(' ')}"
+
+            install_local_storage
+          end
+
+          def teardown
+            shell "k3d cluster delete #{cluster_name}"
+          end
+
+          private
+
+          def registry_mirror_exists?(name)
+            shell('k3d registry list').include?("k3d-#{name}")
+          end
+
+          def retry_until(max_attempts: 10, wait: 1)
+            max_attempts.times do
+              result = yield
+              return result if result
+
+              sleep wait
+            end
+
+            raise "Retried #{max_attempts} times. Aborting"
+          end
+
+          def install_local_storage
+            shell('kubectl apply -f -', stdin_data: local_storage_config)
+          end
+
+          # See https://github.com/rancher/k3d/issues/67
+          def local_storage_config
+            <<~YAML
+              ---
+              apiVersion: v1
+              kind: ServiceAccount
+              metadata:
+                name: storage-provisioner
+                namespace: kube-system
+              ---
+              apiVersion: rbac.authorization.k8s.io/v1
+              kind: ClusterRoleBinding
+              metadata:
+                name: storage-provisioner
+              roleRef:
+                apiGroup: rbac.authorization.k8s.io
+                kind: ClusterRole
+                name: system:persistent-volume-provisioner
+              subjects:
+                - kind: ServiceAccount
+                  name: storage-provisioner
+                  namespace: kube-system
+              ---
+              apiVersion: v1
+              kind: Pod
+              metadata:
+                name: storage-provisioner
+                namespace: kube-system
+              spec:
+                serviceAccountName: storage-provisioner
+                tolerations:
+                - effect: NoExecute
+                  key: node.kubernetes.io/not-ready
+                  operator: Exists
+                  tolerationSeconds: 300
+                - effect: NoExecute
+                  key: node.kubernetes.io/unreachable
+                  operator: Exists
+                  tolerationSeconds: 300
+                hostNetwork: true
+                containers:
+                - name: storage-provisioner
+                  image: gcr.io/k8s-minikube/storage-provisioner:v1.8.1
+                  command: ["/storage-provisioner"]
+                  imagePullPolicy: IfNotPresent
+                  volumeMounts:
+                  - mountPath: /tmp
+                    name: tmp
+                volumes:
+                - name: tmp
+                  hostPath:
+                    path: /tmp
+                    type: Directory
+              ---
+              kind: StorageClass
+              apiVersion: storage.k8s.io/v1
+              metadata:
+                name: standard
+                namespace: kube-system
+                annotations:
+                  storageclass.kubernetes.io/is-default-class: "true"
+                labels:
+                  addonmanager.kubernetes.io/mode: EnsureExists
+              provisioner: k8s.io/minikube-hostpath
+            YAML
+          end
+
+          def registry_mirror
+            <<~YAML
+              mirrors:
+                "registry.gitlab.com":
+                  endpoint:
+                    - http://k3d-registry-gitlab:5000
+            YAML
+          end
+        end
+      end
+    end
+  end
+end

data/lib/gitlab/qa/service/kubernetes_cluster.rb

@@ -0,0 +1,62 @@
+# frozen_string_literal: true
+
+require 'mkmf'
+
+module Gitlab
+  module QA
+    module Service
+      class KubernetesCluster
+        include Support::Shellout
+
+        attr_reader :provider
+
+        def initialize(provider_class: QA::Service::ClusterProvider::K3d)
+          @provider = provider_class.new
+        end
+
+        def create!
+          validate_dependencies
+
+          @provider.validate_dependencies
+          @provider.setup
+
+          self
+        end
+
+        def remove!
+          @provider.teardown
+        end
+
+        def cluster_name
+          @provider.cluster_name
+        end
+
+        def to_s
+          cluster_name
+        end
+
+        def create_registry_mirror
+          @provider.create_registry_mirror
+        end
+
+        def create_secret(secret, secret_name)
+          shell("kubectl create secret generic #{secret_name} --from-literal=token='#{secret}'", mask_secrets: [secret])
+        end
+
+        def apply_manifest(manifest)
+          shell('kubectl apply -f -', stdin_data: manifest)
+        end
+
+        private
+
+        def admin_user
+          @admin_user ||= "#{@provider.cluster_name}-admin"
+        end
+
+        def validate_dependencies
+          find_executable('kubectl') || raise("You must first install `kubectl` executable to run these tests.")
+        end
+      end
+    end
+  end
+end

data/lib/gitlab/qa/support/shell_command.rb

@@ -0,0 +1,98 @@
+# frozen_string_literal: true
+
+require 'open3'
+require 'active_support/core_ext/string/filters'
+
+module Gitlab
+  module QA
+    module Support
+      class ShellCommand
+        using Rainbow
+
+        StatusError = Class.new(StandardError)
+
+        # Shell command
+        #
+        # @param [<String, Array>] command
+        # @param [<String, Array>] mask_secrets
+        # @param [Boolean] stream_output stream command output to stdout directly instead of logger
+        def initialize(command = nil, stdin_data: nil, mask_secrets: nil, stream_output: false)
+          @command = command
+          @mask_secrets = Array(mask_secrets)
+          @stream_output = stream_output
+          @output = []
+          @logger = Runtime::Logger.logger
+          @stdin_data = stdin_data
+        end
+
+        attr_reader :command, :output, :stream_output
+
+        def execute! # rubocop:disable Metrics/AbcSize
+          raise StatusError, 'Command already executed' if output.any?
+
+          logger.info("Shell command: `#{mask_secrets(command).cyan}`")
+
+          Open3.popen2e(@command.to_s) do |stdin, out, wait|
+            if @stdin_data
+              stdin.puts(@stdin_data)
+              stdin.close
+            end
+
+            out.each do |line|
+              output.push(line)
+
+              if stream_progress
+                print "."
+              elsif stream_output
+                puts line
+              end
+
+              yield line, wait if block_given?
+            end
+            puts if stream_progress && !output.empty?
+
+            fail! if wait.value.exited? && wait.value.exitstatus.nonzero?
+
+            logger.debug("Shell command output:\n#{string_output}") unless stream_output || output.empty?
+          end
+
+          string_output
+        end
+
+        private
+
+        attr_reader :logger
+
+        # Raise error and print output to error log level
+        #
+        # @return [void]
+        def fail!
+          logger.error("Shell command output:\n#{string_output}") unless stream_output
+          raise StatusError, "Command `#{mask_secrets(command).truncate(100)}` failed! " + "✘".red
+        end
+
+        # Stream only command execution progress and log output when command finished
+        #
+        # @return [Boolean]
+        def stream_progress
+          !(Runtime::Env.ci || stream_output)
+        end
+
+        # Stringified command output
+        #
+        # @return [String]
+        def string_output
+          mask_secrets(output.join.chomp)
+        end
+
+        # Returns a masked string
+        #
+        # @param [String] input the string to mask
+        # @return [String] The masked string
+        def mask_secrets(input)
+          @mask_secrets.each_with_object(+input) { |secret, s| s.gsub!(secret, '*****') }.to_s
+        end
+      end
+    end
+  end
+end

data/lib/gitlab/qa/support/shellout.rb

@@ -0,0 +1,16 @@
+# frozen_string_literal: true
+
+module Gitlab
+  module QA
+    module Support
+      module Shellout
+        module_function
+
+        def shell(command = nil, stdin_data: nil, mask_secrets: nil, stream_output: false, &block)
+          Support::ShellCommand.new(
+            command, stdin_data: stdin_data, mask_secrets: mask_secrets, stream_output: stream_output).execute!(&block)
+        end
+      end
+    end
+  end
+end

data/lib/gitlab/qa/version.rb

@@ -2,6 +2,6 @@
 
 module Gitlab
   module QA
-    VERSION = '8.4.2'
+    VERSION = '8.6.0'
   end
 end

data/lib/gitlab/qa.rb

@@ -10,7 +10,6 @@ module Gitlab
     loader.ignore("#{__dir__}/qa/version.rb")
 
     loader.inflector.inflect(
-      'dev_ee_qa_image' => 'DevEEQAImage',
       'results_in_testcases' => 'ResultsInTestCases',
       'postgresql' => 'PostgreSQL',
       'registry_with_cdn' => 'RegistryWithCDN',

data/support/manifests/suggested_reviewer/authenticator.yaml

@@ -0,0 +1,41 @@
+---
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: authenticator-deployment
+  labels:
+    app: authenticator
+spec:
+  replicas: 1
+  selector:
+    matchLabels:
+      app: authenticator
+  template:
+    metadata:
+      labels:
+        app: authenticator
+    spec:
+      imagePullSecrets:
+        - name: gitlab-registry
+      containers:
+        - name: authenticator
+          image: registry.gitlab.com/gitlab-org/modelops/applied-ml/review-recommender/authenticator:0.1.0
+          imagePullPolicy: Always
+          ports:
+            - containerPort: 8080
+          args: []
+          env:
+            - name: AUTHENTICATOR_SERVICE_PORT
+              value: "8080"
+---
+apiVersion: v1
+kind: Service
+metadata:
+  name: authenticator-service
+spec:
+  type: NodePort
+  ports:
+    - port: 8080
+      targetPort: 8080
+  selector:
+    app: authenticator

data/support/manifests/suggested_reviewer/postgres.yaml

@@ -0,0 +1,84 @@
+---
+apiVersion: v1
+kind: Secret
+metadata:
+  name: postgres-secret
+type: kubernetes.io/basic-auth
+stringData:
+  username: reviewer-recommender
+  password: ml4dawin
+
+---
+apiVersion: v1
+kind: PersistentVolumeClaim
+metadata:
+  name: postgres-pvc
+spec:
+  storageClassName: local-path
+  accessModes:
+    - ReadWriteOnce
+  resources:
+    requests:
+      storage: 1Gi
+
+---
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: postgres-deployment
+spec:
+  replicas: 1
+  selector:
+    matchLabels:
+      app: postgres
+  template:
+    metadata:
+      labels:
+        app: postgres
+    spec:
+      containers:
+        - name: postgres
+          image: postgres:13-alpine
+          imagePullPolicy: IfNotPresent
+          ports:
+            - containerPort: 5432
+          volumeMounts:
+            - name: postgres-pv
+              mountPath: /var/lib/postgresql/data
+              subPath: postgres
+          resources:
+            limits:
+              memory: 256Mi
+              cpu: 250m
+          env:
+            - name: POSTGRES_DB
+              value: reviewer-recommender
+            - name: POSTGRES_USER
+              valueFrom:
+                secretKeyRef:
+                  name: postgres-secret
+                  key: username
+            - name: POSTGRES_PASSWORD
+              valueFrom:
+                secretKeyRef:
+                  name: postgres-secret
+                  key: password
+      volumes:
+        - name: postgres-pv
+          persistentVolumeClaim:
+            claimName: postgres-pvc
+
+---
+apiVersion: v1
+kind: Service
+metadata:
+  name: postgres
+  labels:
+    app: postgres
+spec:
+  type: ClusterIP
+  selector:
+    app: postgres
+  ports:
+    - port: 5432
+      targetPort: 5432

data/support/manifests/suggested_reviewer/pubsub.yaml

@@ -0,0 +1,41 @@
+---
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: pubsub-deployment
+  labels:
+    app: pubsub
+spec:
+  replicas: 1
+  selector:
+    matchLabels:
+      app: pubsub
+  template:
+    metadata:
+      labels:
+        app: pubsub
+    spec:
+      containers:
+        - name: pubsub
+          image: singularities/pubsub-emulator:latest
+          imagePullPolicy: IfNotPresent
+          env:
+            - name: PUBSUB_PROJECT_ID
+              value: project-test
+            - name: PUBSUB_LISTEN_ADDRESS
+              value: 0.0.0.0:8432
+
+---
+apiVersion: v1
+kind: Service
+metadata:
+  name: pubsub
+  labels:
+    app: pubsub
+spec:
+  type: ClusterIP
+  selector:
+    app: pubsub
+  ports:
+    - port: 8432
+      targetPort: 8432