gitlab_quality-test_tooling 0.1.0 → 0.2.0

data/lib/gitlab_quality/test_tooling/report/results_in_issues.rb

@@ -0,0 +1,130 @@
+# frozen_string_literal: true
+
+module GitlabQuality
+  module TestTooling
+    module Report
+      # Uses the API to create or update GitLab test result issues with the results of tests from RSpec report files.
+      class ResultsInIssues < ReportAsIssue
+        include Concerns::ResultsReporter
+
+        def initialize(**kwargs)
+          super
+          @issue_type = 'issue'
+        end
+
+        def get_related_issue(testcase, test)
+          issue = find_linked_results_issue_by_iid(testcase, test)
+          is_new = false
+
+          if issue
+            issue = update_issue_title(issue, test) if issue_title_needs_updating?(issue, test)
+          else
+            puts "No valid issue link found"
+            issue = find_or_create_results_issue(test)
+            is_new = true
+          end
+
+          [issue, is_new]
+        end
+
+        def update_issue(issue, test)
+          new_labels = issue_labels(issue)
+          new_labels |= ['Testcase Linked']
+
+          labels_updated = update_labels(issue, test, new_labels)
+          note_posted = note_status(issue, test)
+
+          if labels_updated || note_posted
+            puts "Issue updated: #{issue.web_url}"
+          else
+            puts "Test passed, no results issue update needed."
+          end
+        end
+
+        private
+
+        def find_linked_results_issue_by_iid(testcase, _test)
+          iid = issue_iid_from_testcase(testcase)
+
+          return unless iid
+
+          find_issue_by_iid(iid)
+        end
+
+        def find_or_create_results_issue(test)
+          find_issue(test) || create_issue(test)
+        end
+
+        def issue_iid_from_testcase(testcase)
+          results = testcase.description.partition(TEST_CASE_RESULTS_SECTION_TEMPLATE).last if testcase.description.include?(TEST_CASE_RESULTS_SECTION_TEMPLATE)
+
+          return puts "No issue link found" unless results
+
+          issue_iid = results.split('/').last
+
+          issue_iid&.to_i
+        end
+
+        def note_status(issue, test)
+          return false if test.skipped
+          return false if test.failures.empty?
+
+          note = note_content(test)
+
+          gitlab.find_issue_discussions(iid: issue.iid).each do |discussion|
+            if new_note_matches_discussion?(
+              note, discussion)
+              return gitlab.add_note_to_issue_discussion_as_thread(iid: issue.iid, discussion_id: discussion.id,
+                body: failure_summary)
+            end
+          end
+
+          gitlab.create_issue_note(iid: issue.iid, note: note)
+
+          true
+        end
+
+        def note_content(test)
+          errors = test.failures.each_with_object([]) do |failure, text|
+            text << <<~TEXT
+              Error:
+              ```
+              #{failure['message']}
+              ```
+
+              Stacktrace:
+              ```
+              #{failure['stacktrace']}
+              ```
+            TEXT
+          end.join("\n\n")
+
+          "#{failure_summary}\n\n#{errors}"
+        end
+
+        def failure_summary
+          summary = [":x: ~\"#{pipeline}::failed\""]
+          summary << "in job `#{Runtime::Env.ci_job_name}` in #{Runtime::Env.ci_job_url}"
+          summary.join(' ')
+        end
+
+        def new_note_matches_discussion?(note, discussion)
+          note_error = error_and_stack_trace(note)
+          discussion_error = error_and_stack_trace(discussion.notes.first['body'])
+
+          return false if note_error.empty? || discussion_error.empty?
+
+          note_error == discussion_error
+        end
+
+        def error_and_stack_trace(text)
+          text.strip[/Error:(.*)/m, 1].to_s
+        end
+
+        def updated_description(_issue, test)
+          new_issue_description(test)
+        end
+      end
+    end
+  end
+end

data/lib/gitlab_quality/test_tooling/report/results_in_testcases.rb

@@ -0,0 +1,113 @@
+# frozen_string_literal: true
+
+require 'erb'
+
+module GitlabQuality
+  module TestTooling
+    module Report
+      # Uses the API to create or update GitLab test cases with the results of tests from RSpec report files.
+      class ResultsInTestCases < ReportAsIssue
+        include Concerns::ResultsReporter
+
+        attr_reader :issue_type, :gitlab
+
+        def initialize(**kwargs)
+          super
+          @issue_type = 'test_case'
+        end
+
+        def find_or_create_testcase(test)
+          find_testcase(test) || create_issue(test)
+        end
+
+        def add_issue_link_to_testcase(testcase, issue, test)
+          results_section = testcase.description.include?(TEST_CASE_RESULTS_SECTION_TEMPLATE) ? '' : TEST_CASE_RESULTS_SECTION_TEMPLATE
+
+          gitlab.edit_issue(iid: testcase.iid,
+            options: { description: (testcase.description + results_section + "\n\n#{issue.web_url}") })
+          # We are using test.testcase for the url here instead of testcase.web_url since it has the updated test case path
+          puts "Added results issue #{issue.web_url} link to test case #{test.testcase}"
+        end
+
+        def update_testcase(testcase, test)
+          puts "Test case labels updated." if update_labels(testcase, test)
+          puts "Test case quarantine section updated." if update_quarantine_link(testcase, test)
+        end
+
+        private
+
+        def find_testcase(test)
+          testcase = find_testcase_by_iid(test)
+
+          if testcase
+            testcase = update_issue_title(testcase, test) if issue_title_needs_updating?(testcase, test)
+          else
+            testcase = find_issue(test)
+          end
+
+          testcase
+        end
+
+        def find_testcase_by_iid(test)
+          iid = testcase_iid_from_url(test.testcase)
+
+          return unless iid
+
+          find_issue_by_iid(iid)
+        end
+
+        def testcase_iid_from_url(url)
+          return warn(%(\nPlease update #{url} to test case url")) if url&.include?('/-/issues/')
+
+          url && url.split('/').last.to_i
+        end
+
+        def new_issue_description(test)
+          quarantine_section = test.quarantine? && test.quarantine_issue ? "\n\n### Quarantine issue\n\n#{test.quarantine_issue}" : ''
+
+          "#{super}#{quarantine_section}\n\n#{execution_graph_section(test)}"
+        end
+
+        def execution_graph_section(test)
+          formatted_title = ERB::Util.url_encode(test.name)
+
+          <<~MKDOWN.strip
+            ### Executions
+
+            All Environments:
+            <img src="https://dashboards.quality.gitlab.net/render/d-solo/cW0UMgv7k/spec-health?orgId=1&var-run_type=All&var-name=#{formatted_title}&panelId=4&width=1000&height=500" />
+          MKDOWN
+        end
+
+        def updated_description(testcase, test)
+          historical_results_section = testcase.description.match(/### DO NOT EDIT BELOW THIS LINE[\s\S]+/)
+
+          "#{new_issue_description(test)}\n\n#{historical_results_section}"
+        end
+
+        def issue_title_needs_updating?(testcase, test)
+          super || (!testcase.description.include?(execution_graph_section(test)) && !%w[canary production preprod
+            release].include?(pipeline))
+        end
+
+        def quarantine_link_needs_updating?(testcase, test)
+          if test.quarantine? && test.quarantine_issue
+            return false if testcase.description.include?(test.quarantine_issue)
+          else
+            return false unless testcase.description.include?('Quarantine issue')
+          end
+
+          true
+        end
+
+        def update_quarantine_link(testcase, test)
+          return unless quarantine_link_needs_updating?(testcase, test)
+
+          new_description = updated_description(testcase, test)
+
+          gitlab.edit_issue(iid: testcase.iid, options: { description: new_description })
+        end
+      end
+    end
+  end
+end

data/lib/gitlab_quality/test_tooling/report/update_screenshot_path.rb

@@ -0,0 +1,81 @@
+# frozen_string_literal: true
+
+require 'nokogiri'
+require 'json'
+require 'active_support/core_ext/object/blank'
+
+module GitlabQuality
+  module TestTooling
+    module Report
+      class UpdateScreenshotPath
+        def initialize(junit_files:)
+          @junit_files = junit_files
+        end
+
+        REGEX = %r{(?<gitlab_qa_run>gitlab-qa-run-.*?(?=/))/(?<gitlab_ce_ee_qa>gitlab-(?:ee|ce)-qa-.*?(?=/))}
+        CONTAINER_PATH = File.join('/home', 'gitlab', 'qa', 'tmp').freeze
+
+        def invoke!
+          Dir.glob(junit_files).each do |junit_file|
+            match_data = junit_file.match(REGEX)
+            next unless match_data
+
+            host_relative_path = "#{match_data[:gitlab_qa_run]}/#{match_data[:gitlab_ce_ee_qa]}"
+
+            rewrite_schreenshot_paths_in_junit_file(junit_file, host_relative_path)
+            rewrite_schreenshot_paths_in_json_file(junit_file.gsub('.xml', '.json'), host_relative_path)
+          end
+        end
+
+        private
+
+        attr_reader :junit_files
+
+        def rewrite_schreenshot_paths_in_junit_file(junit_file, host_relative_path)
+          File.write(
+            junit_file,
+            rewrite_each_junit_screenshot_path(junit_file, host_relative_path).to_s
+          )
+
+          puts "Saved #{junit_file}"
+        end
+
+        def rewrite_schreenshot_paths_in_json_file(json_file, host_relative_path)
+          File.write(
+            json_file,
+            JSON.pretty_generate(
+              rewrite_each_json_screenshot_path(json_file, host_relative_path)
+            )
+          )
+
+          puts "Saved #{json_file}"
+        end
+
+        def rewrite_each_junit_screenshot_path(junit_file, host_relative_path)
+          Nokogiri::XML(File.open(junit_file)).tap do |report|
+            report.xpath('//system-out').each do |system_out|
+              system_out.content = remove_container_absolute_path_prefix(system_out.content, host_relative_path)
+            end
+          end
+        end
+
+        def rewrite_each_json_screenshot_path(json_file, host_relative_path)
+          JSON.parse(File.read(json_file)).tap do |report|
+            examples = report['examples']
+
+            examples.each do |example|
+              next unless example['screenshot'].present?
+
+              example['screenshot']['image'] =
+                remove_container_absolute_path_prefix(example.dig('screenshot', 'image'), host_relative_path)
+            end
+          end
+        end
+
+        def remove_container_absolute_path_prefix(image_container_absolute_path, host_relative_path)
+          image_container_absolute_path.gsub(CONTAINER_PATH, host_relative_path)
+        end
+      end
+    end
+  end
+end

data/lib/gitlab_quality/test_tooling/runtime/env.rb

@@ -0,0 +1,113 @@
+# frozen_string_literal: true
+
+require 'active_support/core_ext/object/blank'
+require 'securerandom'
+
+module GitlabQuality
+  module TestTooling
+    module Runtime
+      module Env
+        extend self
+        using Rainbow
+
+        ENV_VARIABLES = {
+          'GITLAB_API_BASE' => :api_base,
+          'GITLAB_QA_ISSUE_URL' => :qa_issue_url,
+          'GITLAB_CI_API_TOKEN' => :gitlab_ci_api_token,
+          'CI_COMMIT_REF_NAME' => :ci_commit_ref_name,
+          'CI_JOB_NAME' => :ci_job_name,
+          'CI_JOB_URL' => :ci_job_url,
+          'CI_PROJECT_ID' => :ci_project_id,
+          'CI_PROJECT_NAME' => :ci_project_name,
+          'CI_PIPELINE_ID' => :ci_pipeline_id,
+          'CI_PIPELINE_URL' => :ci_pipeline_url,
+          'CI_API_V4_URL' => :ci_api_v4_url,
+          'SLACK_QA_CHANNEL' => :slack_qa_channel,
+          'DEPLOY_VERSION' => :deploy_version
+        }.freeze
+
+        ENV_VARIABLES.each do |env_name, method_name|
+          define_method(method_name) do
+            env_var_value_if_defined(env_name) || (instance_variable_get("@#{method_name}") if instance_variable_defined?("@#{method_name}"))
+          end
+        end
+
+        def log_level
+          env_var_value_if_defined('QA_LOG_LEVEL')&.upcase || 'INFO'
+        end
+
+        def log_path
+          env_var_value_if_defined('QA_LOG_PATH') || host_artifacts_dir
+        end
+
+        def default_branch
+          env_var_value_if_defined('QA_DEFAULT_BRANCH') || 'main'
+        end
+
+        def gitlab_api_base
+          env_var_value_if_defined('GITLAB_API_BASE') || 'https://gitlab.com/api/v4'
+        end
+
+        def pipeline_from_project_name
+          ci_project_name.to_s.start_with?('gitlab') ? default_branch : ci_project_name
+        end
+
+        def run_id
+          @run_id ||= "gitlab-qa-run-#{Time.now.strftime('%Y-%m-%d-%H-%M-%S')}-#{SecureRandom.hex(4)}"
+        end
+
+        def colorized_logs?
+          enabled?(ENV.fetch('COLORIZED_LOGS', nil), default: false)
+        end
+
+        def deploy_environment
+          env_var_value_if_defined('DEPLOY_ENVIRONMENT') || pipeline_from_project_name
+        end
+
+        def host_artifacts_dir
+          @host_artifacts_dir ||= File.join(
+            env_var_value_if_defined('QA_ARTIFACTS_DIR') || '/tmp/gitlab-qa', Runtime::Env.run_id
+          )
+        end
+
+        def qa_run_type
+          return env_var_value_if_defined('QA_RUN_TYPE') if env_var_value_valid?('QA_RUN_TYPE')
+
+          live_envs = %w[staging staging-canary staging-ref canary preprod production]
+          return unless live_envs.include?(ci_project_name)
+
+          test_subset = if env_var_value_if_defined('NO_ADMIN') == 'true'
+                          'sanity-no-admin'
+                        elsif env_var_value_if_defined('SMOKE_ONLY') == 'true'
+                          'sanity'
+                        else
+                          'full'
+                        end
+
+          "#{ci_project_name}-#{test_subset}"
+        end
+
+        private
+
+        def enabled?(value, default: true)
+          return default if value.nil?
+
+          (value =~ /^(false|no|0)$/i) != 0
+        end
+
+        def env_var_value_valid?(variable)
+          !ENV[variable].blank?
+        end
+
+        def env_var_value_if_defined(variable)
+          return ENV.fetch(variable) if env_var_value_valid?(variable)
+        end
+
+        def env_var_name_if_defined(variable)
+          # Pass through the variables if they are defined and not empty in the environment
+          return "$#{variable}" if env_var_value_valid?(variable)
+        end
+      end
+    end
+  end
+end

data/lib/gitlab_quality/test_tooling/runtime/logger.rb

@@ -0,0 +1,92 @@
+# frozen_string_literal: true
+
+require 'forwardable'
+require 'fileutils'
+require 'rainbow'
+require 'active_support/logger'
+
+module GitlabQuality
+  module TestTooling
+    module Runtime
+      class Logger
+        extend SingleForwardable
+
+        def_delegators :logger, :debug, :info, :warn, :error, :fatal, :unknown
+
+        TIME_FORMAT = "%Y-%m-%d %H:%M:%S"
+        LEVEL_COLORS = {
+          "DEBUG" => :magenta,
+          "INFO" => :green,
+          "WARN" => :yellow,
+          "ERROR" => :red,
+          "FATAL" => :indianred
+        }.freeze
+
+        Rainbow.enabled = Runtime::Env.colorized_logs?
+
+        class << self
+          # Combined logger instance
+          #
+          # @param [String] source
+          # @return [ActiveSupport::Logger]
+          def logger(source: 'Gitlab QA')
+            @logger ||= begin
+              log_path = Env.log_path
+              ::FileUtils.mkdir_p(log_path)
+
+              console_log = console_logger(source: source, level: Env.log_level)
+              file_log = file_logger(source: source, path: log_path)
+
+              console_log.extend(ActiveSupport::Logger.broadcast(file_log))
+            end
+          end
+
+          private
+
+          # Console logger instance
+          #
+          # @param [String] source
+          # @param [<Symbol, String>] level
+          # @return [ActiveSupport::Logger]
+          def console_logger(source:, level:)
+            ActiveSupport::Logger.new($stdout, level: level, datetime_format: TIME_FORMAT).tap do |logger|
+              logger.formatter = proc do |severity, datetime, _progname, msg|
+                msg_prefix = message_prefix(datetime, source, severity)
+
+                Rainbow(msg_prefix).public_send(LEVEL_COLORS.fetch(severity, :silver)) + "#{msg}\n" # rubocop:disable GitlabSecurity/PublicSend
+              end
+            end
+          end
+
+          # File logger
+          #
+          # @param [String] source
+          # @param [String] path
+          # @return [ActiveSupport::Logger]
+          def file_logger(source:, path:)
+            log_file = "#{path}/#{source.downcase.tr(' ', '-')}.log"
+
+            ActiveSupport::Logger.new(log_file, level: :debug, datetime_format: TIME_FORMAT).tap do |logger|
+              logger.formatter = proc do |severity, datetime, _progname, msg|
+                msg_prefix = message_prefix(datetime, source, severity)
+
+                "#{msg_prefix}#{msg}\n".gsub(/\e\[(\d+)(?:;\d+)*m/, "")
+              end
+            end
+          end
+
+          # Log message prefix
+          #
+          # @note when outputted, the date will be formatted as "Jun 07 2022 11:30:00 UTC"
+          # @param [DateTime] date
+          # @param [String] source
+          # @param [String] severity
+          # @return [String]
+          def message_prefix(date, source, severity)
+            "[#{date.strftime('%h %d %Y %H:%M:%S %Z')} (#{source})] #{severity.ljust(5)} -- "
+          end
+        end
+      end
+    end
+  end
+end

data/lib/gitlab_quality/test_tooling/runtime/token_finder.rb

@@ -0,0 +1,44 @@
+# frozen_string_literal: true
+
+module GitlabQuality
+  module TestTooling
+    module Runtime
+      class TokenFinder
+        def self.find_token!(token, suffix: nil)
+          new(token, suffix).find_token!
+        end
+
+        attr_reader :token, :suffix
+
+        def initialize(token, suffix)
+          @token = token
+          @suffix = suffix
+        end
+
+        def find_token!
+          find_token_from_attrs || find_token_from_env || find_token_from_file
+        end
+
+        def find_token_from_attrs
+          token
+        end
+
+        def find_token_from_env
+          Env.gitlab_ci_api_token
+        end
+
+        def find_token_from_file
+          @token_from_file ||= File.read(token_file_path).strip
+        rescue Errno::ENOENT
+          fail "Please provide a valid access token with the `-t/--token` option, the `GITLAB_CI_API_TOKEN` environment variable, or in the `#{token_file_path}` file!"
+        end
+
+        private
+
+        def token_file_path
+          @token_file_path ||= File.expand_path("../api_token#{"_#{suffix}" if suffix}", __dir__)
+        end
+      end
+    end
+  end
+end

data/lib/gitlab_quality/test_tooling/slack/post_to_slack.rb

@@ -0,0 +1,36 @@
+# frozen_string_literal: true
+
+module GitlabQuality
+  module TestTooling
+    module Slack
+      class PostToSlack
+        def initialize(slack_webhook_url:, channel:, message:, username:, icon_emoji:)
+          @slack_webhook_url = slack_webhook_url
+          @channel = channel
+          @message = message
+          @username = username
+          @icon_emoji = icon_emoji
+        end
+
+        def invoke!
+          params = {}
+          params['channel'] = channel
+          params['username'] = username
+          params['icon_emoji'] = icon_emoji
+          params['text'] = message
+
+          Support::HttpRequest.make_http_request(
+            method: 'post',
+            url: slack_webhook_url,
+            params: params,
+            show_response: true
+          )
+        end
+
+        private
+
+        attr_reader :slack_webhook_url, :channel, :message, :username, :icon_emoji
+      end
+    end
+  end
+end

data/lib/gitlab_quality/test_tooling/summary_table.rb

@@ -0,0 +1,41 @@
+# frozen_string_literal: true
+
+require 'nokogiri'
+require 'table_print'
+
+module GitlabQuality
+  module TestTooling
+    class SummaryTable
+      def self.create(input_files:)
+        "```\n#{TablePrint::Printer.table_print(collect_results(input_files))}```\n"
+      end
+
+      # rubocop:disable Metrics/AbcSize
+      def self.collect_results(input_files)
+        stage_wise_results = []
+
+        Dir.glob(input_files).each do |report_file|
+          stage_hash = {}
+          stage_hash["Dev Stage"] = File.basename(report_file, ".*").capitalize
+
+          report_stats = Nokogiri::XML(File.open(report_file)).children[0].attributes
+
+          stage_hash["Total"] = report_stats["tests"].value
+          stage_hash["Failures"] = report_stats["failures"].value
+          stage_hash["Errors"] = report_stats["errors"].value
+          stage_hash["Skipped"] = report_stats["skipped"].value
+          stage_hash["Result"] = result_emoji(report_stats)
+
+          stage_wise_results << stage_hash
+        end
+
+        stage_wise_results
+      end
+      # rubocop:enable Metrics/AbcSize
+
+      def self.result_emoji(report_stats)
+        report_stats["failures"].value.to_i.positive? || report_stats["errors"].value.to_i.positive? ? "❌" : "✅"
+      end
+    end
+  end
+end

data/lib/gitlab_quality/test_tooling/support/http_request.rb

@@ -0,0 +1,34 @@
+# frozen_string_literal: true
+
+require 'http'
+require 'json'
+
+module GitlabQuality
+  module TestTooling
+    module Support
+      class HttpRequest
+        # rubocop:disable Metrics/AbcSize
+        def self.make_http_request(method: 'get', url: nil, params: {}, headers: {}, show_response: false, fail_on_error: true)
+          raise "URL not defined for making request. Exiting..." unless url
+
+          res = HTTP.follow.method(method).call(url, json: params, headers: headers)
+
+          if show_response
+            if res.content_type.mime_type == "application/json"
+              res_body = JSON.parse(res.body.to_s)
+              pp res_body
+            else
+              res_body = res.body.to_s
+              puts res_body
+            end
+          end
+
+          raise "#{method.upcase} request failed!\nCode: #{res.code}\nResponse: #{res.body}\n" if fail_on_error && !res.status.success?
+
+          res
+        end
+        # rubocop:enable Metrics/AbcSize
+      end
+    end
+  end
+end