gitlab_quality-test_tooling 0.1.0 → 0.2.1

data/lib/gitlab_quality/test_tooling/gitlab_issue_client.rb

@@ -0,0 +1,194 @@
+# frozen_string_literal: true
+
+require 'gitlab'
+
+module Gitlab
+  # Monkey patch the Gitlab client to use the correct API path and add required methods
+  class Client
+    def team_member(project, id)
+      get("/projects/#{url_encode(project)}/members/all/#{id}")
+    end
+
+    def issue_discussions(project, issue_id, options = {})
+      get("/projects/#{url_encode(project)}/issues/#{issue_id}/discussions", query: options)
+    end
+
+    def add_note_to_issue_discussion_as_thread(project, issue_id, discussion_id, options = {})
+      post("/projects/#{url_encode(project)}/issues/#{issue_id}/discussions/#{discussion_id}/notes", query: options)
+    end
+  end
+end
+
+module GitlabQuality
+  module TestTooling
+    # The GitLab client is used for API access: https://github.com/NARKOZ/gitlab
+    class GitlabIssueClient
+      MAINTAINER_ACCESS_LEVEL = 40
+      RETRY_BACK_OFF_DELAY = 60
+      MAX_RETRY_ATTEMPTS = 3
+
+      def initialize(token:, project:)
+        @token = token
+        @project = project
+        @retry_backoff = 0
+
+        configure_gitlab_client
+      end
+
+      def assert_user_permission!
+        handle_gitlab_client_exceptions do
+          user = Gitlab.user
+          member = Gitlab.team_member(project, user.id)
+
+          abort_not_permitted if member.access_level < MAINTAINER_ACCESS_LEVEL
+        end
+      rescue Gitlab::Error::NotFound
+        abort_not_permitted
+      end
+
+      def find_issues(iid: nil, options: {}, &select)
+        select ||= :itself
+
+        handle_gitlab_client_exceptions do
+          break [Gitlab.issue(project, iid)].select(&select) if iid
+
+          Gitlab.issues(project, options)
+            .auto_paginate
+            .select(&select)
+        end
+      end
+
+      def find_issue_discussions(iid:)
+        handle_gitlab_client_exceptions do
+          Gitlab.issue_discussions(project, iid, order_by: 'created_at', sort: 'asc').auto_paginate
+        end
+      end
+
+      def create_issue(title:, description:, labels:, issue_type: 'issue')
+        attrs = { issue_type: issue_type, description: description, labels: labels }
+
+        handle_gitlab_client_exceptions do
+          Gitlab.create_issue(project, title, attrs)
+        end
+      end
+
+      def edit_issue(iid:, options: {})
+        handle_gitlab_client_exceptions do
+          Gitlab.edit_issue(project, iid, options)
+        end
+      end
+
+      def find_issue_notes(iid:)
+        handle_gitlab_client_exceptions do
+          Gitlab.issue_notes(project, iid, order_by: 'created_at', sort: 'asc')&.auto_paginate
+        end
+      end
+
+      def create_issue_note(iid:, note:)
+        handle_gitlab_client_exceptions do
+          Gitlab.create_issue_note(project, iid, note)
+        end
+      end
+
+      def edit_issue_note(issue_iid:, note_id:, note:)
+        handle_gitlab_client_exceptions do
+          Gitlab.edit_issue_note(project, issue_iid, note_id, note)
+        end
+      end
+
+      def add_note_to_issue_discussion_as_thread(iid:, discussion_id:, body:)
+        handle_gitlab_client_exceptions do
+          Gitlab.add_note_to_issue_discussion_as_thread(project, iid, discussion_id, body: body)
+        end
+      end
+
+      def find_user_id(username:)
+        handle_gitlab_client_exceptions do
+          user = Gitlab.users(username: username)&.first
+          user['id'] unless user.nil?
+        end
+      end
+
+      def upload_file(file_fullpath:)
+        ignore_gitlab_client_exceptions do
+          Gitlab.upload_file(project, file_fullpath)
+        end
+      end
+
+      def ignore_gitlab_client_exceptions
+        yield
+      rescue StandardError, SystemCallError, OpenSSL::SSL::SSLError, Net::OpenTimeout, Net::ReadTimeout,
+        Gitlab::Error::Error => e
+        puts "Ignoring the following error: #{e}"
+      end
+
+      def handle_gitlab_client_exceptions # rubocop:disable Metrics/AbcSize
+        yield
+      rescue Gitlab::Error::NotFound
+        # This error could be raised in assert_user_permission!
+        # If so, we want it to terminate at that point
+        raise
+      rescue SystemCallError, OpenSSL::SSL::SSLError, Net::OpenTimeout, Net::ReadTimeout,
+        Gitlab::Error::InternalServerError, Gitlab::Error::Parsing => e
+        @retry_backoff += RETRY_BACK_OFF_DELAY
+
+        raise if @retry_backoff > RETRY_BACK_OFF_DELAY * MAX_RETRY_ATTEMPTS
+
+        warn_exception(e)
+        warn("Sleeping for #{@retry_backoff} seconds before retrying...")
+        sleep @retry_backoff
+
+        retry
+      rescue StandardError => e
+        pipeline = Runtime::Env.pipeline_from_project_name
+        channel = case pipeline
+                  when "canary"
+                    "qa-production"
+                  when "staging-canary"
+                    "qa-staging"
+                  else
+                    "qa-#{pipeline}"
+                  end
+        error_msg = warn_exception(e)
+
+        return unless Runtime::Env.ci_commit_ref_name == Runtime::Env.default_branch
+
+        slack_options = {
+          slack_webhook_url: ENV.fetch('CI_SLACK_WEBHOOK_URL', nil),
+          channel: channel,
+          username: "GitLab QA Bot",
+          icon_emoji: ':ci_failing:',
+          message: <<~MSG
+            An unexpected error occurred while reporting test results in issues.
+            The error occurred in job: #{Runtime::Env.ci_job_url}
+            `#{error_msg}`
+          MSG
+        }
+        puts "Posting Slack message to channel: #{channel}"
+
+        GitlabQuality::TestTooling::Slack::PostToSlack.new(**slack_options).invoke!
+      end
+
+      private
+
+      attr_reader :token, :project
+
+      def configure_gitlab_client
+        Gitlab.configure do |config|
+          config.endpoint = Runtime::Env.gitlab_api_base
+          config.private_token = token
+        end
+      end
+
+      def abort_not_permitted
+        abort "You must have at least Reporter access to the project to use this feature."
+      end
+
+      def warn_exception(error)
+        error_msg = "#{error.class.name} #{error.message}"
+        warn(error_msg)
+        error_msg
+      end
+    end
+  end
+end

data/lib/gitlab_quality/test_tooling/gitlab_issue_dry_client.rb

@@ -0,0 +1,26 @@
+# frozen_string_literal: true
+
+module GitlabQuality
+  module TestTooling
+    class GitlabIssueDryClient < GitlabIssueClient
+      def create_issue(title:, description:, labels:, issue_type: 'issue')
+        attrs = { description: description, labels: labels }
+
+        puts "The following #{issue_type} would have been created:"
+        puts "project: #{project}, title: #{title}, attrs: #{attrs}"
+      end
+
+      def edit_issue(iid:, options: {})
+        puts "The #{project}##{iid} issue would have been updated with: #{options}"
+      end
+
+      def create_issue_note(iid:, note:)
+        puts "The following note would have been posted on #{project}##{iid} issue: #{note}"
+      end
+
+      def add_note_to_issue_discussion_as_thread(iid:, discussion_id:, body:)
+        puts "The following discussion note would have been posted on #{project}##{iid} (discussion #{discussion_id}) issue: #{body}"
+      end
+    end
+  end
+end

data/lib/gitlab_quality/test_tooling/report/concerns/find_set_dri.rb

@@ -0,0 +1,51 @@
+# frozen_string_literal: true
+
+require 'json'
+
+module GitlabQuality
+  module TestTooling
+    module Report
+      module Concerns
+        module FindSetDri
+          def set_dri_via_group(product_group, test)
+            parse_json_with_sets
+            fetch_stage_sets(test)
+
+            return @sets.sample['username'] if @stage_sets.empty?
+
+            fetch_group_sets(product_group)
+
+            if @group_sets.empty?
+              @stage_sets.sample['username']
+            else
+              @group_sets.sample['username']
+            end
+          end
+
+          private
+
+          def parse_json_with_sets
+            response = Support::HttpRequest.make_http_request(
+              url: 'https://gitlab-org.gitlab.io/gitlab-roulette/roulette.json'
+            )
+            @sets = JSON.parse(response.body).select { |user| user['role'].include?('software-engineer-in-test') }
+          end
+
+          def fetch_stage_sets(test)
+            @stage_sets = @sets.select do |user|
+              user['role'].include?(test.stage.split("_").map(&:capitalize).join(" "))
+            end
+          end
+
+          def fetch_group_sets(product_group)
+            @group_sets = @stage_sets.select do |user|
+              user['role'].include?(product_group.split("_").map do |word|
+                                      word == 'and' ? word : word.capitalize
+                                    end.join(" "))
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/gitlab_quality/test_tooling/report/concerns/results_reporter.rb

@@ -0,0 +1,75 @@
+# frozen_string_literal: true
+
+require 'active_support/core_ext/enumerable'
+
+module GitlabQuality
+  module TestTooling
+    module Report
+      module Concerns
+        module ResultsReporter
+          include Concerns::Utils
+
+          TEST_CASE_RESULTS_SECTION_TEMPLATE = "\n\n### DO NOT EDIT BELOW THIS LINE\n\nActive and historical test results:"
+
+          def find_issue(test)
+            issues = search_for_issues(test)
+
+            warn(%(Too many #{issue_type}s found with the file path "#{test.file}" and name "#{test.name}")) if issues.many?
+
+            puts "Found existing #{issue_type}: #{issues.first.web_url}" unless issues.empty?
+
+            issues.first
+          end
+
+          def find_issue_by_iid(iid)
+            issues = gitlab.find_issues(iid: iid) do |issue|
+              issue.state == 'opened' && issue.issue_type == issue_type
+            end
+
+            warn(%(#{issue_type} iid "#{iid}" not valid)) if issues.empty?
+
+            issues.first
+          end
+
+          def issue_title_needs_updating?(issue, test)
+            issue.title.strip != title_from_test(test) && !%w[canary production preprod release].include?(pipeline)
+          end
+
+          def new_issue_labels(_test)
+            %w[Quality status::automated]
+          end
+
+          def up_to_date_labels(test:, issue: nil, new_labels: Set.new)
+            labels = super
+            labels |= new_issue_labels(test).to_set
+            labels.delete_if { |label| label.start_with?("#{pipeline}::") }
+            labels << (test.failures.empty? ? "#{pipeline}::passed" : "#{pipeline}::failed")
+          end
+
+          def update_issue_title(issue, test)
+            old_title = issue.title.strip
+            new_title = title_from_test(test)
+
+            warn(%(#{issue_type} title needs to be updated from '#{old_title}' to '#{new_title}'))
+
+            new_description = updated_description(issue, test)
+
+            gitlab.edit_issue(iid: issue.iid, options: { title: new_title, description: new_description })
+          end
+
+          private
+
+          def search_term(test)
+            %("#{partial_file_path(test.file)}" "#{search_safe(test.name)}")
+          end
+
+          def search_for_issues(test)
+            gitlab.find_issues(options: { search: search_term(test) }) do |issue|
+              issue.state == 'opened' && issue.issue_type == issue_type && issue.title.strip == title_from_test(test)
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/gitlab_quality/test_tooling/report/concerns/utils.rb

@@ -0,0 +1,49 @@
+# frozen_string_literal: true
+
+module GitlabQuality
+  module TestTooling
+    module Report
+      module Concerns
+        module Utils
+          MAX_TITLE_LENGTH = 255
+
+          def partial_file_path(path)
+            path.match(/((ee|api|browser_ui).*)/i)[1]
+          end
+
+          def new_issue_title(test)
+            "#{partial_file_path(test.file)} | #{search_safe(test.name)}".strip
+          end
+
+          def title_from_test(test)
+            title = new_issue_title(test)
+
+            return title unless title.length > MAX_TITLE_LENGTH
+
+            "#{title[0...MAX_TITLE_LENGTH - 3]}..."
+          end
+
+          def search_safe(value)
+            value.delete('"')
+          end
+
+          def pipeline
+            # Gets the name of the pipeline the test was run in, to be used as the key of a scoped label
+            #
+            # Tests can be run in several pipelines:
+            #   gitlab, nightly, staging, canary, production, preprod, MRs, and the default branch (master/main)
+            #
+            # Some of those run in their own project, so CI_PROJECT_NAME is the name we need. Those are:
+            #   nightly, staging, canary, production, and preprod
+            #
+            # MR, master/main, and gitlab tests run in gitlab-qa, but we only want to report tests run on
+            # master/main because the other pipelines will be monitored by the author of the MR that triggered them.
+            # So we assume that we're reporting a master/main pipeline if the project name is 'gitlab'.
+
+            @pipeline ||= Runtime::Env.pipeline_from_project_name
+          end
+        end
+      end
+    end
+  end
+end

data/lib/gitlab_quality/test_tooling/report/generate_test_session.rb

@@ -0,0 +1,275 @@
+# frozen_string_literal: true
+
+require 'erb'
+require 'date'
+
+module GitlabQuality
+  module TestTooling
+    module Report
+      class GenerateTestSession < ReportAsIssue
+        def initialize(**kwargs)
+          super
+          @issue_type = 'issue'
+        end
+
+        private
+
+        # rubocop:disable Metrics/AbcSize
+        def run!
+          puts "Generating test results in `#{files.join(',')}` as issues in project `#{project}` via the API at `#{Runtime::Env.gitlab_api_base}`."
+
+          tests = Dir.glob(files).flat_map do |path|
+            puts "Loading tests in #{path}"
+
+            TestResults::JsonTestResults.new(path).to_a
+          end
+
+          issue = gitlab.create_issue(
+            title: "#{Time.now.strftime('%Y-%m-%d')} Test session report | #{Runtime::Env.qa_run_type}",
+            description: generate_description(tests),
+            labels: ['Quality', 'QA', 'triage report', pipeline_name_label]
+          )
+
+          # Workaround for https://gitlab.com/gitlab-org/gitlab/-/issues/295493
+          unless Runtime::Env.qa_issue_url.to_s.empty?
+            gitlab.create_issue_note(
+              iid: issue.iid,
+              note: "/relate #{Runtime::Env.qa_issue_url}")
+          end
+
+          File.write('REPORT_ISSUE_URL', issue.web_url)
+        end
+        # rubocop:enable Metrics/AbcSize
+
+        def generate_description(tests)
+          <<~MARKDOWN.rstrip
+          ## Session summary
+
+          * Deploy version: #{Runtime::Env.deploy_version}
+          * Deploy environment: #{Runtime::Env.deploy_environment}
+          * Pipeline: #{Runtime::Env.pipeline_from_project_name} [#{Runtime::Env.ci_pipeline_id}](#{Runtime::Env.ci_pipeline_url})
+          #{generate_summary(tests: tests)}
+
+          #{generate_failed_jobs_listing}
+
+          #{generate_stages_listing(tests)}
+
+          #{generate_qa_issue_relation}
+
+          #{generate_link_to_dashboard}
+          MARKDOWN
+        end
+
+        def generate_summary(tests:, tests_by_status: nil)
+          tests_by_status ||= tests.group_by(&:status)
+          total = tests.size
+          passed = tests_by_status['passed']&.size || 0
+          failed = tests_by_status['failed']&.size || 0
+          others = total - passed - failed
+
+          <<~MARKDOWN.chomp
+          * Total #{total} tests
+          * Passed #{passed} tests
+          * Failed #{failed} tests
+          * #{others} other tests (usually skipped)
+          MARKDOWN
+        end
+
+        def generate_failed_jobs_listing
+          failed_jobs = []
+
+          client = Gitlab.client(
+            endpoint: Runtime::Env.ci_api_v4_url,
+            private_token: Runtime::Env.gitlab_ci_api_token)
+
+          gitlab.handle_gitlab_client_exceptions do
+            failed_jobs = client.pipeline_jobs(
+              Runtime::Env.ci_project_id,
+              Runtime::Env.ci_pipeline_id,
+              scope: 'failed')
+          end
+
+          listings = failed_jobs.map do |job|
+            allowed_to_fail = ' (allowed to fail)' if job.allow_failure
+
+            "* [#{job.name}](#{job.web_url})#{allowed_to_fail}"
+          end.join("\n")
+
+          <<~MARKDOWN.chomp if failed_jobs.any?
+          ## Failed jobs
+
+          #{listings}
+          MARKDOWN
+        end
+
+        def generate_stages_listing(tests)
+          generate_tests_by_stage(tests).map do |stage, tests_for_stage|
+            tests_by_status = tests_for_stage.group_by(&:status)
+
+            <<~MARKDOWN.chomp
+            ### #{stage&.capitalize || 'Unknown'}
+
+            #{generate_summary(
+              tests: tests_for_stage, tests_by_status: tests_by_status)}
+
+            #{generate_testcase_listing_by_status(
+              tests: tests_for_stage, tests_by_status: tests_by_status)}
+            MARKDOWN
+          end.join("\n\n")
+        end
+
+        def generate_tests_by_stage(tests)
+          # https://about.gitlab.com/handbook/product/product-categories/#devops-stages
+          ordering = %w[
+            manage
+            plan
+            create
+            verify
+            package
+            release
+            configure
+            monitor
+            secure
+            defend
+            growth
+            fulfillment
+            enablement
+          ]
+
+          tests.sort_by do |test|
+            ordering.index(test.stage) || ordering.size
+          end.group_by(&:stage)
+        end
+
+        def generate_testcase_listing_by_status(tests:, tests_by_status:)
+          failed_tests = tests_by_status['failed']
+          passed_tests = tests_by_status['passed']
+          other_tests = tests.reject do |test|
+            test.status == 'failed' || test.status == 'passed'
+          end
+
+          [
+            (failed_listings(failed_tests) if failed_tests),
+            (passed_listings(passed_tests) if passed_tests),
+            (other_listings(other_tests) if other_tests.any?)
+          ].compact.join("\n\n")
+        end
+
+        def failed_listings(failed_tests)
+          generate_testcase_listing(failed_tests)
+        end
+
+        def passed_listings(passed_tests)
+          <<~MARKDOWN.chomp
+            <details><summary>Passed tests:</summary>
+
+            #{generate_testcase_listing(passed_tests, passed: true)}
+
+            </details>
+          MARKDOWN
+        end
+
+        def other_listings(other_tests)
+          <<~MARKDOWN.chomp
+            <details><summary>Other tests:</summary>
+
+            #{generate_testcase_listing(other_tests)}
+
+            </details>
+          MARKDOWN
+        end
+
+        def generate_testcase_listing(tests, passed: false)
+          body = tests.group_by(&:testcase).map do |testcase, tests_with_same_testcase|
+            tests_with_same_testcase.sort_by!(&:name)
+            [
+              generate_test_text(testcase, tests_with_same_testcase, passed),
+              generate_test_job(tests_with_same_testcase),
+              generate_test_status(tests_with_same_testcase),
+              generate_test_actions(tests_with_same_testcase)
+            ].join(' | ')
+          end.join("\n")
+
+          <<~MARKDOWN.chomp
+          | Test | Job | Status | Action |
+          | - | - | - | - |
+          #{body}
+          MARKDOWN
+        end
+
+        def generate_test_text(testcase, tests_with_same_testcase, passed)
+          text = tests_with_same_testcase.map(&:name).uniq.join(', ')
+          encoded_text = ERB::Util.url_encode(text)
+
+          if testcase && !passed
+            # Workaround for reducing system notes on testcase issues
+            # The first regex extracts the link to the issues list page from a link to a single issue show page by removing the issue id.
+            "[#{text}](#{testcase.match(%r{[\s\S]+/[^/\d]+})}?state=opened&search=#{encoded_text})"
+          else
+            text
+          end
+        end
+
+        def generate_test_job(tests_with_same_testcase)
+          tests_with_same_testcase.map do |test|
+            ci_job_id = test.ci_job_url[/\d+\z/]
+
+            "[#{ci_job_id}](#{test.ci_job_url})#{' ~"quarantine"' if test.quarantine?}"
+          end.uniq.join(', ')
+        end
+
+        def generate_test_status(tests_with_same_testcase)
+          tests_with_same_testcase.map(&:status).uniq.map do |status|
+            %(~"#{status}")
+          end.join(', ')
+        end
+
+        def generate_test_actions(tests_with_same_testcase)
+          # All failed tests would be grouped together, meaning that
+          # if one failed, all the tests here would be failed too.
+          # So this check is safe. Same applies to 'passed'.
+          # But all other status might be mixing together,
+          # we cannot assume other statuses.
+          if tests_with_same_testcase.first.status == 'failed'
+            tests_having_failure_issue =
+              tests_with_same_testcase.select(&:failure_issue)
+
+            if tests_having_failure_issue.any?
+              items = tests_having_failure_issue.uniq(&:failure_issue).map do |test|
+                "<li>[ ] [failure issue](#{test.failure_issue})</li>"
+              end.join(' ')
+
+              "<ul>#{items}</ul>"
+            else
+              '<ul><li>[ ] failure issue exists or was created</li></ul>'
+            end
+          else
+            '-'
+          end
+        end
+
+        def generate_qa_issue_relation
+          return unless Runtime::Env.qa_issue_url
+
+          <<~MARKDOWN.chomp
+          ## Release QA issue
+
+          * #{Runtime::Env.qa_issue_url}
+
+          /relate #{Runtime::Env.qa_issue_url}
+          MARKDOWN
+        end
+
+        def generate_link_to_dashboard
+          return unless Runtime::Env.qa_run_type
+
+          <<~MARKDOWN.chomp
+          ## Link to Grafana dashboard for run-type of #{Runtime::Env.qa_run_type}
+
+          * https://dashboards.quality.gitlab.net/d/kuNYMgDnz/test-run-metrics?orgId=1&refresh=1m&var-run_type=#{Runtime::Env.qa_run_type}
+          MARKDOWN
+        end
+      end
+    end
+  end
+end