gitlab_quality-test_tooling 0.1.0 → 0.2.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 01e27d782b61b17f8388d031534df9096ed36ed3d1c2e94734fb22875f5a47f5
-  data.tar.gz: a6df70390360e39d3b33df544c7e39d9838f16682763ed620120285886439d84
+  metadata.gz: db0c200678a2bb9d3eea994129fd3db411c9af0647225f451d548b02005e69e0
+  data.tar.gz: 356fcd2142b1f62507e9b4ff76903d21dfbee980a8a41662a2cd02255bc2831f
 SHA512:
-  metadata.gz: 58640b757e621d232e5951cde8fe76fa8f3df7dfd0d5dcae326e0e1546fba6337218176ba1af0ec451a8dc2ea6e0616fd5f8ce66e8c9ab4645889c7768cc0bcd
-  data.tar.gz: e9a55097a4a8deb1a65f181e6674d89517a2a625012c0967ced81564a2cf52eff1529cdc58372a631f0d5bbb0b8684cb17b4518b22b92c77351a98e4d349deef
+  metadata.gz: 2fceda952ea6665034e2c0fd4f55fbbadb3226b953613a9bc3cb3b2c3b0332b59f2043a1277a73749c8fc449bb0ea5a919b1cf278bf6a4ad92978ffa6e748fe5
+  data.tar.gz: cdca3c8f902068f205379836272146a61e361ca4dcd13d86b7f22372fb0a7549ed798c40e0419400ba7a2cffff3cacb726339557ac108c1b55e0edc71b72e308

data/.rubocop.yml

@@ -19,6 +19,12 @@ AllCops:
   MaxFilesInCache: 1_000_000
   NewCops: enable
 
+CodeReuse/ActiveRecord:
+  Enabled: false
+
+Gemspec/RequiredRubyVersion:
+  Enabled: false
+
 Gemspec/RequireMFA:
   Enabled: false
 
@@ -49,12 +55,11 @@ Style/ModuleFunction:
 Style/SignalException:
   Enabled: false
 
+Layout/LineLength:
+  Max: 180
+
 Layout/SpaceBeforeFirstArg:
   Enabled: false
 
 RSpec/MultipleMemoizedHelpers:
   Enabled: false
-
-CodeReuse/ActiveRecord:
-  Exclude:
-    - 'spec/**/*_spec.rb'

data/Gemfile.lock

@@ -1,7 +1,7 @@
 PATH
   remote: .
   specs:
-    gitlab_quality-test_tooling (0.1.0)
+    gitlab_quality-test_tooling (0.2.1)
       activesupport (~> 6.1)
       gitlab (~> 4.18.0)
       http (~> 5.0)

data/Guardfile

@@ -42,28 +42,6 @@ guard :rspec, cmd: "bundle exec rspec" do
   ruby = dsl.ruby
   dsl.watch_spec_files_for(ruby.lib_files)
 
-  # Rails files
-  rails = dsl.rails(view_extensions: %w[erb haml slim])
-  dsl.watch_spec_files_for(rails.app_files)
-  dsl.watch_spec_files_for(rails.views)
-
-  watch(rails.controllers) do |m|
-    [
-      rspec.spec.call("routing/#{m[1]}_routing"),
-      rspec.spec.call("controllers/#{m[1]}_controller"),
-      rspec.spec.call("acceptance/#{m[1]}")
-    ]
-  end
-
-  # Rails config changes
-  watch(rails.spec_helper)     { rspec.spec_dir }
-  watch(rails.routes)          { "#{rspec.spec_dir}/routing" }
-  watch(rails.app_controller)  { "#{rspec.spec_dir}/controllers" }
-
-  # Capybara features specs
-  watch(rails.view_dirs)     { |m| rspec.spec.call("features/#{m[1]}") }
-  watch(rails.layouts)       { |m| rspec.spec.call("features/#{m[1]}") }
-
   # Turnip features and steps
   watch(%r{^spec/acceptance/(.+)\.feature$})
   watch(%r{^spec/acceptance/steps/(.+)_steps\.rb$}) do |m|

data/README.md

@@ -1,28 +1,169 @@
 # `GitlabQuality::TestTooling`
 
-TODO: Delete this and the text below, and describe your gem
-
-Welcome to your new gem! In this directory, you'll find the files you need to be able to package up your Ruby library into a gem. Put your Ruby code in the file `lib/gitlab_quality/test_tooling`. To experiment with that code, run `bin/console` for an interactive prompt.
+This gem provides test tooling that can be used by different projects or different part of the same project, mostly in CI scripts.
 
 ## Installation
 
-Install the gem and add to the application's Gemfile by executing:
+Add this line to your application's Gemfile:
+
+```ruby
+gem 'gitlab_quality-test_tooling', require: false
+```
+
+And then execute:
 
-    $ bundle add gitlab_quality-test_tooling
+```sh
+$ bundle install
+```
 
-If bundler is not being used to manage dependencies, install the gem by executing:
+Or install it yourself as:
 
-    $ gem install gitlab_quality-test_tooling
+```sh
+$ gem install gitlab_quality-test_tooling
+```
 
 ## Usage
 
-TODO: Write usage instructions here
+The gem provides the following executables.
+
+### `generate-test-session`
+
+```shell
+Purpose: Generate test session report based on RSpec report files (JSON or JUnit XML)
+Usage: exe/generate-test-session [options]
+    -i, --input-files FILES          RSpec report files (JSON or JUnit XML)
+    -p, --project PROJECT            Can be an integer or a group/project string
+    -t, --token ACCESS_TOKEN         A valid access token with Reporter permission in PROJECT
+        --dry-run                    Perform a dry-run (don't create or update issues or test cases)
+    -v, --version                    Show the version
+    -h, --help                       Show the usage
+```
+
+### `post-to-slack`
+
+```shell
+$ exe/post-to-slack -h
+Purpose: Post a message to Slack, and optionally add a test summary table based on RSpec report files (JUnit XML)
+Usage: exe/post-to-slack [options]
+    -w SLACK_WEBHOOK_URL,            Slack webhook URL
+        --slack-webhook-url
+    -c, --channel CHANNEL            Slack channel to post the message to
+    -m, --message MESSAGE            Post message to Slack
+    -t FILES,                        Add a test summary table based on RSpec report files (JUnit XML)
+        --include-summary-table
+    -u, --username USERNAME          Username to use for the Slack message
+    -i, --icon-emoji ICON_EMOJI      Icon emoji to use for the Slack message
+    -v, --version                    Show the version
+    -h, --help                       Show the usage
+```
+
+### `prepare-stage-reports`
+
+```shell
+$ exe/prepare-stage-reports -h
+Purpose: Prepare separate reports for each DevOps stage from the provided RSpec report files (JUnit XML)
+Usage: exe/prepare-stage-reports [options]
+    -j, --junit-files FILES          RSpec report files (JUnit XML)
+    -v, --version                    Show the version
+    -h, --help                       Show the usage
+```
+
+### `relate-failure-issue`
+
+```shell
+$ exe/relate-failure-issue -h
+Purpose: Relate test failures to failure issues from RSpec report files (JSON or JUnit XML)
+Usage: exe/relate-failure-issue [options]
+    -i, --input-files FILES          RSpec report files (JSON or JUnit XML)
+        --max-diff-ratio DIFF_RATO   Max stacktrace diff ratio for QA failure issues detection
+    -p, --project PROJECT            Can be an integer or a group/project string
+    -t, --token ACCESS_TOKEN         A valid access token with Reporter permission in PROJECT
+        --system-log-files SYSTEM_LOG_FILES
+                                     Include errors from system logs in failure issues
+        --dry-run                    Perform a dry-run (don't create or update issues)
+    -v, --version                    Show the version
+    -h, --help                       Show the usage
+```
+
+### `report-results`
+
+```shell
+$ exe/report-results -h
+Purpose: Report test results from RSpec report files (JSON or JUnit XML) in GitLab test cases and result issues
+Usage: exe/report-results [options]
+    -i, --input-files FILES          RSpec report files (JSON or JUnit XML)
+        --test-case-project TEST_CASE_PROJECT
+                                     Can be an integer or a group/project string
+        --results-issue-project RESULTS_ISSUE_PROJECT
+                                     Can be an integer or a group/project string
+    -t, --token ACCESS_TOKEN         A valid access token
+        --dry-run                    Perform a dry-run (don't create or update issues or test cases)
+    -v, --version                    Show the version
+    -h, --help                       Show the usage
+```
+
+### `update-screenshot-paths`
+
+```shell
+$ exe/update-screenshot-paths -h
+Purpose: Update the path to screenshots to container's host from RSpec report files (JSON or JUnit XML)
+Usage: exe/update-screenshot-paths [options]
+    -i, --input-files FILES          RSpec report files (JSON or JUnit XML)
+    -v, --version                    Show the version
+    -h, --help                       Show the usage
+```
 
 ## Development
 
+### Initial setup
+
 After checking out the repo, run `bin/setup` to install dependencies. Then, run `rake spec` to run the tests. You can also run `bin/console` for an interactive prompt that will allow you to experiment.
 
-To install this gem onto your local machine, run `bundle exec rake install`. To release a new version, update the version number in `version.rb`, and then run `bundle exec rake release`, which will create a git tag for the version, push git commits and the created tag, and push the `.gem` file to [rubygems.org](https://rubygems.org).
+To install this gem onto your local machine, run `bundle exec rake install`.
+
+### Activate lefthook locally
+
+```shell
+lefthook install
+```
+
+### Testing unreleased changes in merge requests
+
+To test an unreleased change in an actual merge request, you can create a merge request that will install the unreleased version of `gitlab_quality-test_tooling`. Bundler can install gems by specifying a repository and a revision from Git.
+
+For example, to test `gitlab_quality-test_tooling` changes from the `your-branch-name` branch in [`gitlab-org/gitlab`](https://gitlab.com/gitlab-org/gitlab), in the `Gemfile`:
+
+```ruby
+group :development, :test, :danger do
+  gem 'gitlab_quality-test_tooling', '~> 3.4.3', require: false,
+  git: 'https://gitlab.com/gitlab-org/ruby/gems/gitlab_quality-test_tooling.git',
+  ref: 'your-branch-name'
+end
+```
+
+See an [example](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/92580) for more details.
+
+## Release
+
+[Automated gem release process](https://gitlab.com/gitlab-org/quality/pipeline-common#release-process) is used to release new version of `gitlab_quality-test_tooling` through pipelines, and this will:
+
+- Publish the gem: https://rubygems.org/gems/gitlab_quality-test_tooling
+- Add a release in the `gitlab_quality-test_tooling` project: https://gitlab.com/gitlab-org/ruby/gems/gitlab_quality-test_tooling/-/releases
+- Populate the release log with the API contents. For example: https://gitlab.com/api/v4/projects/19861191/repository/changelog?version=3.4.4
+
+### Before release
+
+Make sure to include a changelog entry in your commit message and read the [changelog entries section](https://docs.gitlab.com/ee/development/changelog.html).
+
+If you forget to set the changelog entry in your commit messages, you can also edit the release notes after it's being released.
+
+### Steps to release
+
+Use a `Release` merge request template and create a merge requet to update the version number in `version.rb`, and get the merge request merged by a maintainer.
+
+This will then be packaged into a gem and pushed to [rubygems.org](https://rubygems.org) by the CI/CD.
+
+For example: [Bump version to 3.4.3](https://gitlab.com/gitlab-org/ruby/gems/gitlab-dangerfiles/-/merge_requests/149).
 
 ## License
 

data/exe/generate-test-session

@@ -0,0 +1,50 @@
+#!/usr/bin/env ruby
+# frozen_string_literal: true
+
+require "bundler/setup"
+require "optparse"
+
+require_relative "../lib/gitlab_quality/test_tooling"
+
+params = {}
+
+options = OptionParser.new do |opts|
+  opts.banner = "Usage: #{$PROGRAM_NAME} [options]"
+
+  opts.on('-i', '--input-files FILES', String, 'RSpec report files (JSON or JUnit XML)') do |input_files|
+    params[:input_files] = input_files
+  end
+
+  opts.on('-p', '--project PROJECT', String, 'Can be an integer or a group/project string') do |project|
+    params[:project] = project
+  end
+
+  opts.on('-t', '--token ACCESS_TOKEN', String, 'A valid access token with Reporter permission in PROJECT') do |token|
+    params[:token] = GitlabQuality::TestTooling::Runtime::TokenFinder.find_token!(token)
+  end
+
+  opts.on('--dry-run', "Perform a dry-run (don't create or update issues or test cases)") do
+    params[:dry_run] = true
+  end
+
+  opts.on_tail('-v', '--version', 'Show the version') do
+    require_relative "../lib/gitlab_quality/test_tooling/version"
+    puts "#{$PROGRAM_NAME} : #{GitlabQuality::TestTooling::VERSION}"
+    exit
+  end
+
+  opts.on_tail('-h', '--help', 'Show the usage') do
+    puts "Purpose: Generate test session report based on RSpec report files (JSON or JUnit XML)"
+    puts opts
+    exit
+  end
+
+  opts.parse(ARGV)
+end
+
+if params.any?
+  GitlabQuality::TestTooling::Report::GenerateTestSession.new(**params).invoke!
+else
+  puts options
+  exit 1
+end

data/exe/post-to-slack

@@ -0,0 +1,58 @@
+#!/usr/bin/env ruby
+# frozen_string_literal: true
+
+require "bundler/setup"
+require "optparse"
+
+require_relative "../lib/gitlab_quality/test_tooling"
+
+params = {}
+
+options = OptionParser.new do |opts|
+  opts.banner = "Usage: #{$PROGRAM_NAME} [options]"
+
+  opts.on('-w', '--slack-webhook-url SLACK_WEBHOOK_URL', String, 'Slack webhook URL') do |slack_webhook_url|
+    params[:slack_webhook_url] = slack_webhook_url
+  end
+
+  opts.on('-c', '--channel CHANNEL', String, 'Slack channel to post the message to') do |channel|
+    params[:channel] = channel
+  end
+
+  opts.on('-m', '--message MESSAGE', String, 'Post message to Slack') do |message|
+    params[:message] = message
+  end
+
+  opts.on('-t', '--include-summary-table FILES', String, 'Add a test summary table based on RSpec report files (JUnit XML)') do |files|
+    params[:message] += "\n\n#{GitlabQuality::TestTooling::SummaryTable.create(input_files: files)}"
+  end
+
+  opts.on('-u', '--username USERNAME', String, 'Username to use for the Slack message') do |username|
+    params[:username] = username
+  end
+
+  opts.on('-i', '--icon-emoji ICON_EMOJI', String, 'Icon emoji to use for the Slack message') do |icon_emoji|
+    params[:icon_emoji] = icon_emoji
+  end
+
+  opts.on_tail('-v', '--version', 'Show the version') do
+    require_relative "../lib/gitlab_quality/test_tooling/version"
+    puts "#{$PROGRAM_NAME} : #{GitlabQuality::TestTooling::VERSION}"
+    exit
+  end
+
+  opts.on_tail('-h', '--help', 'Show the usage') do
+    puts "Purpose: Post a message to Slack, and optionally add a test summary table based on RSpec report files (JUnit XML)"
+    puts opts
+    exit
+  end
+
+  opts.parse(ARGV)
+end
+
+if params.any?
+  GitlabQuality::TestTooling::Slack::PostToSlack.new(**params).invoke!
+else
+  puts options
+  exit 1
+end

data/exe/prepare-stage-reports

@@ -0,0 +1,38 @@
+#!/usr/bin/env ruby
+# frozen_string_literal: true
+
+require "bundler/setup"
+require "optparse"
+
+require_relative "../lib/gitlab_quality/test_tooling"
+
+params = {}
+
+options = OptionParser.new do |opts|
+  opts.banner = "Usage: #{$PROGRAM_NAME} [options]"
+
+  opts.on('-j', '--junit-files FILES', String, 'RSpec report files (JUnit XML)') do |junit_files|
+    params[:junit_files] = junit_files
+  end
+
+  opts.on_tail('-v', '--version', 'Show the version') do
+    require_relative "../lib/gitlab_quality/test_tooling/version"
+    puts "#{$PROGRAM_NAME} : #{GitlabQuality::TestTooling::VERSION}"
+    exit
+  end
+
+  opts.on_tail('-h', '--help', 'Show the usage') do
+    puts "Purpose: Prepare separate reports for each DevOps stage from the provided RSpec report files (JUnit XML)"
+    puts opts
+    exit
+  end
+
+  opts.parse(ARGV)
+end
+
+if params.any?
+  GitlabQuality::TestTooling::Report::PrepareStageReports.new(**params).invoke!
+else
+  puts options
+  exit 1
+end

data/exe/relate-failure-issue

@@ -0,0 +1,59 @@
+#!/usr/bin/env ruby
+# frozen_string_literal: true
+
+require "bundler/setup"
+require "optparse"
+
+require_relative "../lib/gitlab_quality/test_tooling"
+
+params = {}
+
+options = OptionParser.new do |opts|
+  opts.banner = "Usage: #{$PROGRAM_NAME} [options]"
+
+  opts.on('-i', '--input-files FILES', String, 'RSpec report files (JSON or JUnit XML)') do |input_files|
+    params[:input_files] = input_files
+  end
+
+  opts.on('--max-diff-ratio DIFF_RATO', Float, 'Max stacktrace diff ratio for QA failure issues detection') do |value|
+    params[:max_diff_ratio] = value
+  end
+
+  opts.on('-p', '--project PROJECT', String, 'Can be an integer or a group/project string') do |project|
+    params[:project] = project
+  end
+
+  opts.on('-t', '--token ACCESS_TOKEN', String, 'A valid access token with Reporter permission in PROJECT') do |token|
+    params[:token] = GitlabQuality::TestTooling::Runtime::TokenFinder.find_token!(token)
+  end
+
+  opts.on('--system-log-files SYSTEM_LOG_FILES', String,
+    'Include errors from system logs in failure issues') do |system_log_files|
+    params[:system_logs] = system_log_files
+  end
+
+  opts.on('--dry-run', "Perform a dry-run (don't create or update issues)") do
+    params[:dry_run] = true
+  end
+
+  opts.on_tail('-v', '--version', 'Show the version') do
+    require_relative "../lib/gitlab_quality/test_tooling/version"
+    puts "#{$PROGRAM_NAME} : #{GitlabQuality::TestTooling::VERSION}"
+    exit
+  end
+
+  opts.on_tail('-h', '--help', 'Show the usage') do
+    puts "Purpose: Relate test failures to failure issues from RSpec report files (JSON or JUnit XML)"
+    puts opts
+    exit
+  end
+
+  opts.parse(ARGV)
+end
+
+if params.any?
+  GitlabQuality::TestTooling::Report::RelateFailureIssue.new(**params).invoke!
+else
+  puts options
+  exit 1
+end

data/exe/report-results

@@ -0,0 +1,56 @@
+#!/usr/bin/env ruby
+# frozen_string_literal: true
+
+require "bundler/setup"
+require "optparse"
+
+require_relative "../lib/gitlab_quality/test_tooling"
+
+params = {}
+
+options = OptionParser.new do |opts|
+  opts.banner = "Usage: #{$PROGRAM_NAME} [options]"
+
+  opts.on('-i', '--input-files FILES', String, 'RSpec report files (JSON or JUnit XML)') do |input_files|
+    params[:input_files] = input_files
+  end
+
+  opts.on('--test-case-project TEST_CASE_PROJECT', String,
+    'Can be an integer or a group/project string') do |test_case_project|
+    params[:test_case_project] = test_case_project
+  end
+
+  opts.on('--results-issue-project RESULTS_ISSUE_PROJECT', String,
+    'Can be an integer or a group/project string') do |results_issue_project|
+    params[:results_issue_project] = results_issue_project
+  end
+
+  opts.on('-t', '--token ACCESS_TOKEN', String, 'A valid access token') do |token|
+    params[:token] = GitlabQuality::TestTooling::Runtime::TokenFinder.find_token!(token)
+  end
+
+  opts.on('--dry-run', "Perform a dry-run (don't create or update issues or test cases)") do
+    params[:dry_run] = true
+  end
+
+  opts.on_tail('-v', '--version', 'Show the version') do
+    require_relative "../lib/gitlab_quality/test_tooling/version"
+    puts "#{$PROGRAM_NAME} : #{GitlabQuality::TestTooling::VERSION}"
+    exit
+  end
+
+  opts.on_tail('-h', '--help', 'Show the usage') do
+    puts "Purpose: Report test results from RSpec report files (JSON or JUnit XML) in GitLab test cases and result issues"
+    puts opts
+    exit
+  end
+
+  opts.parse(ARGV)
+end
+
+if params.any?
+  GitlabQuality::TestTooling::Report::ReportResults.new(**params).invoke!
+else
+  puts options
+  exit 1
+end

data/exe/update-screenshot-paths

@@ -0,0 +1,38 @@
+#!/usr/bin/env ruby
+# frozen_string_literal: true
+
+require "bundler/setup"
+require "optparse"
+
+require_relative "../lib/gitlab_quality/test_tooling"
+
+params = {}
+
+options = OptionParser.new do |opts|
+  opts.banner = "Usage: #{$PROGRAM_NAME} [options]"
+
+  opts.on('-i', '--input-files FILES', String, 'RSpec report files (JSON or JUnit XML)') do |input_files|
+    params[:junit_files] = input_files
+  end
+
+  opts.on_tail('-v', '--version', 'Show the version') do
+    require_relative "../lib/gitlab_quality/test_tooling/version"
+    puts "#{$PROGRAM_NAME} : #{GitlabQuality::TestTooling::VERSION}"
+    exit
+  end
+
+  opts.on_tail('-h', '--help', 'Show the usage') do
+    puts "Purpose: Update the path to screenshots to container's host from RSpec report files (JSON or JUnit XML)"
+    puts opts
+    exit
+  end
+
+  opts.parse(ARGV)
+end
+
+if params.any?
+  GitlabQuality::TestTooling::Report::UpdateScreenshotPath.new(**params).invoke!
+else
+  puts options
+  exit 1
+end