confiner 0.2.3 → 0.3.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 6e9601fa7e82b448423188a4dde817864c6f7f57f1a1efd9aae67c1b2e8708c7
-  data.tar.gz: 23b2a999e52c58e5081fc9f4857e45a5a0c1783c0b1eca373f4ad4b57394a97f
+  metadata.gz: dcdbf54c2eb2a930389aafac447765559b7d0303bfe98eb2c1fdf54399ee03e7
+  data.tar.gz: a475b92cef079c62dd767b155f9653222f8d804e393db5cfb5e044d08aae85e7
 SHA512:
-  metadata.gz: 48ad12ecefe1473ec4d481f3ebce5d325b9a0a052af1705a9a69621ea78a6d2fb5461fa4a068a83808943dc055f71fec936f54764894bc7a94de6e810e6bbb68
-  data.tar.gz: f74349e0a4003152ab16e7e86b4c34b77c58376ad7550fdf1796d4f7bd9633f0fc177abc56133607c4ce3c6108ca43b4598eacaf44eaa029c6127e301f80410b
+  metadata.gz: 5ab67f9debe3399df85e64a246481d2b4fa50a5056de956af84203ab34e818d5b4d61869526a4b9ff84eb7cb4a7d3317d7967d6afcfee18c903385f5f82f9595
+  data.tar.gz: 06bc73016ec031a83b158cda3d91442d9cb6c24e42495365fe9c9010e05d12a9fc420f09108cfc1b35e70d3a53d12e4ae11075501c25a56035befb614f3233dd

data/lib/confiner/cli.rb

@@ -8,19 +8,19 @@ module Confiner
     include Logger
 
     attr_accessor :action, :plugins
-    attr_reader :parser, :plugin_options
+    attr_reader :parser, :plugin_arguments
 
     def initialize(*options)
-      @plugins = []
-      @plugin_options = []
-      @rules_files = []
+      @plugin_arguments = [] # arguments to be passed to the plugin(s)
+      @rules_files = [] # which files were loaded
       @rules = []
+      @plugin_options = { debug: false, dry_run: false }
 
       # default logging to standard out
       Logger.log_to = $stdout
 
       if options.include?('--')
-        @plugin_options = options[options.index('--')..]
+        @plugin_arguments = options[options.index('--')..]
         options = options[0..options.index('--')]
       end
 
@@ -64,6 +64,14 @@ module Confiner
           exit(0)
         end
 
+        opts.on('--dry-run', 'Dry run') do
+          @plugin_options[:dry_run] = true
+        end
+
+        opts.on('--debug', 'Toggle debug mode') do
+          @plugin_options[:debug] = true
+        end
+
         opts.on('-o OUTPUT', '--output-to OUTPUT', 'File to output the log to') do |output_to|
           Logger.log_to = output_to
         end
@@ -111,9 +119,9 @@ module Confiner
       log :rule, rule.keys.map { |k| "\t#{k}=#{rule[k]}" }.join(',')
 
       rule['plugin']['args'] ||= {}
-      rule['plugin']['args'].transform_keys!(&:to_sym) # 2.5 compatability
+      rule['plugin']['args'].transform_keys!(&:to_sym) # ruby 2.5 compatability
 
-      plugin = Plugins.const_get(translate_plugin_name(rule['plugin']['name'])).new(**rule['plugin']['args'])
+      plugin = Plugins.const_get(translate_plugin_name(rule['plugin']['name'])).new(@plugin_options, **rule['plugin']['args'])
 
       # perform verification of actions before execution
       rule['actions'].each do |action|

data/lib/confiner/plugin.rb

@@ -7,6 +7,11 @@ module Confiner
 
     attr_reader :examples
 
+    DEFAULT_PLUGIN_ARGS = {
+      debug: false,  # Used for debugging logging
+      dry_run: false # Used for when external requests are inhibited
+    }.freeze
+
     class << self
       # Define arguments that the plugin will accept
       # @param [Array<Symbol, Hash>] args the arguments that this plugin accepts
@@ -35,7 +40,10 @@ module Confiner
       end
     end
 
-    def initialize(**args)
+    def initialize(options, **args)
+      @options = DEFAULT_PLUGIN_ARGS.merge(options)
+      @options = Struct.new(*@options.keys).new(*@options.values)
+
       args.each do |k, v|
         v = ENV[v[1..]] if v[0] == '$' # get environment variable
 

data/lib/confiner/plugins/gitlab.rb

@@ -6,20 +6,46 @@ module Confiner
   module Plugins
     class Gitlab < Plugin
       arguments :private_token,                           # the GitLab API Private-Token to use
-                :project_id,                              # the project where the pipelines are fetch from
+                :project_id,                              # the project where the pipelines are fetched from
                 :target_project,                          # where failure issues will be searched, and where an MR will be filed
                 :failure_issue_labels,                    # what labels to search for when searching for the failure issue
-                :failure_issue_prefix,                    # what prefix an issue has in GitLab Issues to search for the failure issue
+                :failure_issue_prefix,                    # what prefix an issue has in GitLab Issues to search for the failure issue (comma separated)
+                :environment => {},                       # metadata about the environment the tests are running and which will be confined
                 :timeout => 10,                           # the timeout that HTTParty will consume (timeout of requests)
                 :threshold => 3,                          # the failure / pass threshold
                 :endpoint => 'https://gitlab.com/api/v4', # the GitLab API Endpoint (e.g. https://gitlab.com/api/v4)
                 :pwd => '.',                              # the path of the working directory for the examples
-                :ref => 'main'                            # the default Git ref used when updating
+                :ref => 'main',                           # the default Git ref used when updating
+                :job_pattern => '.+'                      # the regex pattern to match names of jobs in GitLab (Job = Suite Name)
 
       MERGE_REQUEST_TITLE = '[QUARANTINE] %s'
-      QUARANTINE_METADATA = %(, quarantine: { issue: '%s', type: :investigating })
-
-      def initialize(**args)
+      QUARANTINE_METADATA = %(, quarantine: { issue: '%{issue_url}', type: :investigating })
+      ONLY_QUARANTINE_METADATA = %(, quarantine: { issue: '%{issue_url}', type: :investigating, only: { %{pattern} } })
+
+      # GitLab Confiner Plugin
+      # @param [Hash] args the arguments for GitLab
+      # @option args [String] :private_token the private token for GitLab to connect to
+      # @option args [String] :project_id the project id (or name) where the pipelines are fetched from
+      # @option args [String] :target_project where failure issues will be searched, and where an MR will be filed
+      # @option args [String] :failure_issue_labels what labels to search for when searching for the failure issue (comma separated)
+      # @option args [String] :failure_issue_prefix what prefix an issue has in GitLab Issues to search for the failure issue
+      #
+      # @option args [Hash] :environment metadata about the environment the tests are running and which will be confined
+      # @option :environment [String] :name the name of the environment
+      # @option :environment [String] :pattern the pattern of how to quarantine/dequarantine
+      #
+      # @option args [Integer] :timeout the timeout that HTTParty will consume (timeout of requests)
+      # @option args [Integer] :threshold the failure / pass threshold
+      # @option args [String] :endpoint the GitLab API Endpoint (e.g. https://gitlab.com/api/v4)
+      # @option args [String] :pwd the path of the working directory for where the tests are located
+      # @option args [String] :ref the default Git ref used when updating
+      # @option args [String] :job_pattern the regex pattern to match names of jobs in GitLab (Job = Suite Name)
+      #
+      # @example
+      #   gitlab = Confiner::Plugins::Gitlab.new({ debug: true }, private_token: 'ABC-123', project_id: 'my-group/my-project', target_project: 'my-group/my-project', failure_issue_labels: 'QA,test', failure_issue_prefix: 'Failure in ', timeout: 10, threshold: 3, endpoint: 'https://gitlab.com/api/v4', pwd: 'qa', ref: 'master')
+      #   gitlab.quarantine
+      #   gitlab.dequarantine
+      def initialize(options, **args)
         super
 
         ENV['GITLAB_API_HTTPARTY_OPTIONS'] = ENV.fetch('GITLAB_API_HTTPARTY_OPTIONS') { "{read_timeout: #{timeout}}" }
@@ -33,19 +59,23 @@ module Confiner
       def quarantine
         log :gitlab, 'Beginning Quarantine Process', 2
 
+        if environment&.any?
+          raise ArgumentError, 'Specify both environment[name] and environment[pattern]' unless environment['name'] && environment['pattern']
+        end
+
         # store the examples from the pipelines
         @examples = get_examples
 
         quarantines = @examples.select(&:failed?).map(&:name).uniq.each_with_object([]) do |failed_example, quarantines|
           # count the number of failures consecutively for this example
 
-          number_of_failures = @examples.select { _1.name == failed_example && _1.failed? }.size
-          if number_of_failures >= threshold
+          fails = @examples.select { _1.name == failed_example && _1.failed? }
+          if fails.size >= threshold
             quarantines << failed_example
 
             example = @examples.find { _1.name == failed_example }
 
-            log :quarantine, "Quarantining #{failed_example} (#{number_of_failures} >= #{threshold})", 3
+            log :quarantine, "Quarantining #{failed_example} (#{fails.size} >= #{threshold})", 3
 
             # check to see if there is a merge request first
             # if there is no merge request...
@@ -60,9 +90,45 @@ module Confiner
               file_contents = get_example_file_contents(example)
               new_contents, changed_line_no = add_quarantine_metadata(file_contents, example, failure_issue)
 
-              branch = create_branch(failure_issue, example)
-              commit_changes(branch, example, new_contents)
-              create_merge_request(example, branch, changed_line_no, failure_issue)
+              log(:debug, new_contents, 4) if @options.debug
+
+              if @options.dry_run
+                log :dry_run, 'Skipping creating branch, committing and filing merge request', 4
+              else
+                branch = create_branch(failure_issue, 'quarantine', example)
+                commit_changes(branch, <<~COMMIT_MESSAGE, example, new_contents)
+                  Quarantine end-to-end test
+
+                  Quarantine #{example.name}
+                COMMIT_MESSAGE
+
+                create_merge_request('[QUARANTINE]', example, branch) do
+                  markdown_occurrences = []
+
+                  fails.each do |fail|
+                    markdown_occurrences << "1. [#{fail.occurrence[:job]}](#{fail.occurrence[:pipeline_url]})"
+                  end
+
+                  meets_or_exceeds = fails.size > threshold ? 'exceeds' : 'meets'
+
+                  <<~MARKDOWN
+                    ## What does this MR do?
+
+                    Quarantines the test `#{example.name}` (https://gitlab.com/#{target_project}/-/blob/#{ref}/#{example.file}#L#{changed_line_no})
+
+                    This test has been found by [Confiner](https://gitlab.com/gitlab-org/quality/confiner) to have failed
+                    #{fails.size} times, which #{meets_or_exceeds} the threshold of #{threshold} times.
+
+                    #{markdown_occurrences.join("\n")}
+
+                    > #{failure_issue['web_url']}
+
+                    <div align="center">
+                    (This MR was automatically generated by [Confiner](https://gitlab.com/gitlab-org/quality/confiner) at #{Time.now.utc})
+                    </div>
+                  MARKDOWN
+                end
+              end
 
               log :quarantine, "Done Quarantining #{failed_example}", 3
             rescue => e
@@ -82,7 +148,84 @@ module Confiner
 
       # Dequarantine Action - Automatically Dequarantine tests
       def dequarantine
+        log :gitlab, 'Beginning Dequarantine Process', 2
+
+        @examples = get_examples
+
+        dequarantines = @examples.select(&:passed?).map(&:name).uniq.each_with_object([]) do |passed_example, dequarantines|
+          passes = @examples.select { _1.name == passed_example && _1.passed? }
+          fails = @examples.select { _1.name == passed_example && _1.failed? }
+
+          if passes.size >= threshold
+            next log(:dequarantine, "Detected #{fails.size} failures for #{passed_example}, thus, not de-quarantining", 3) unless fails.size.zero?
+
+            dequarantines << passed_example
+
+            example = @examples.find { _1.name == passed_example }
+
+            log :dequarantine, "Dequarantining #{example} (#{passes.size} >= #{threshold})", 3
+
+            begin
+              file_contents = get_example_file_contents(example)
+              new_contents, changed_line_no, failure_issue = remove_quarantine_metadata(file_contents, example)
+
+              next log(:warn, <<~MESSAGE.tr("\n", ' '), 4) if file_contents == new_contents
+                Unable to dequarantine. This is likely due to the quarantine being applied to the outer context.
+                See https://gitlab.com/gitlab-org/quality/confiner/-/issues/3
+              MESSAGE
+
+              if @options.dry_run
+                log :dry_run, 'Skipping creating branch, committing and filing merge request', 4
+              else
+                branch = create_branch(failure_issue, 'dequarantine', example)
+                commit_changes(branch, <<~COMMIT_MESSAGE, example, new_contents)
+                  Dequarantine end-to-end test
 
+                  Dequarantine #{example.name}
+                COMMIT_MESSAGE
+
+                create_merge_request('[DEQUARANTINE]',example, branch) do
+                  markdown_occurrences = []
+
+                  passes.each do |pass|
+                    markdown_occurrences << "1. [#{pass.occurrence[:job]}](#{pass.occurrence[:pipeline_url]})"
+                  end
+
+                  meets_or_exceeds = passes.size > threshold ? 'exceeds' : 'meets'
+
+                  <<~MARKDOWN
+                    ## What does this MR do?
+
+                    Dequarantines the test `#{example.name}` (https://gitlab.com/#{target_project}/-/blob/#{ref}/#{example.file}#L#{changed_line_no})
+
+                    This quarantined test has been found by [Confiner](https://gitlab.com/gitlab-org/quality/confiner) to have passed
+                    #{passes.size} times consecutively, which #{meets_or_exceeds} the threshold of #{threshold} times.
+
+                    #{markdown_occurrences.join("\n")}
+
+                    > #{failure_issue}
+
+                    <div align="center">
+                    (This MR was automatically generated by [Confiner](https://gitlab.com/gitlab-org/quality/confiner) at #{Time.now.utc})
+                    </div>
+                  MARKDOWN
+                end
+              end
+
+              log :dequarantine, "Done Dequarantining #{passed_example}", 3
+            rescue => e
+              log :fatal, "There was an issue dequarantining #{example}. Error was #{e.message}\n#{e.backtrace}"
+            end
+          end
+        end
+
+        if dequarantines.any?
+          log :dequarantine, "Found #{dequarantines.size} candidates to be dequarantined", 3
+        else
+          log :dequarantine, 'Found no candidates to be dequarantined', 3
+        end
+
+        log :gitlab, 'Done with Dequarantine Process', 2
       end
 
       private
@@ -102,13 +245,16 @@ module Confiner
       # @param [Integer] threshold the amount of pipelines to fetch
       # @note The threshold defaults to default threshold multiplied by two to the amount of pipelines
       # @return [Array<Example>] array of examples
-      def get_examples(threshold: self.threshold * 2)
+      def get_examples(threshold: self.threshold * 2, job_pattern: Regexp.new(self.job_pattern))
         examples = []
 
         get_last_n_runs(threshold: threshold).each do |run|
           run.each do |pipeline|
             # fetch the pipeline test report
             @gitlab_client.pipeline_test_report(project_id, pipeline['id'])['test_suites'].each do |suite|
+              # skip if the job name does not match the job_pattern argument
+              next log(:skip, "Skipping #{suite['name']}", 4) unless suite['name'].match(job_pattern)
+
               log :suite, "Suite: #{suite['name']} (#{pipeline['web_url']})", 4
               suite['test_cases'].each do |example|
                 examples << Example.new(**example, occurrence: { job: suite['name'], pipeline_url: pipeline['web_url'] })
@@ -157,44 +303,56 @@ module Confiner
       # @param [Gitlab::ObjectifiedHash] failure_issue the failure issue
       # @return [Array<String, Integer>] first value holds the new content, the second value holds the line number of the test
       def add_quarantine_metadata(content, example, failure_issue)
-        content_to_return = content.dup
+        find_example_match_in_contents(content, example) do |line|
+          if line.include?(',')
+            line[line.index(',')] = if environment['name'] && environment['pattern']
+                                      ONLY_QUARANTINE_METADATA % { issue_url: failure_issue['web_url'], pattern: environment['pattern'] }
+                                    else
+                                      QUARANTINE_METADATA % { issue_url: failure_issue['web_url'] }
+                                    end << ','
+          else
+            line[line.rindex(' ')] = if environment['name'] && environment['pattern']
+                                       ONLY_QUARANTINE_METADATA % { issue_url: failure_issue['web_url'], pattern: environment['pattern'] }
+                                     else
+                                       QUARANTINE_METADATA % { issue_url: failure_issue['web_url'] }
+                                     end << ' '
+          end
 
-        best_match = -1
-        lines = content_to_return.split("\n")
-        example.name.split.reverse.each do |word|
-          # given a full-descriptive RSpec example name:
-          #   Plan Group Iterations creates a group iteration
-          # scan these words backwards, incrementing the index until we find the best match
-          new_match = content_to_return.index(word)
-          best_match = new_match if new_match > best_match
+          line
         end
+      end
 
-        # the matched line where the example is
-        match = content_to_return[best_match..].split("\n").first
-        matched_line_no = 0
-
-        lines.each_with_index do |line, line_no|
-          if line.match?(match)
-            matched_line_no = line_no + 1
-
-            log :rspec, "Found match on line #{example.file}:#{matched_line_no}", 4
-
-            if line.include?(',')
-              line[line.index(',')] = QUARANTINE_METADATA % failure_issue['web_url'] << ','
+      # Add dequarantine metadata to the file content and replace it
+      # @param [String] content the content to
+      # @param [Example] example the example to find and replace
+      # @return [Array<(String, Integer, String)>]
+      def remove_quarantine_metadata(content, example)
+        issue = +''
+
+        [
+          find_example_match_in_contents(content, example) do |line|
+            if line.include?('quarantine:')
+              issue = line[/issue:( *)?'(.+)',/, 2]# pluck the issue URL from the line before removing
+              line.gsub(/,( *)quarantine:( *)?{.+}/, '')
             else
-              line[line.rindex(' ')] = QUARANTINE_METADATA % failure_issue['web_url'] << ' '
+              line
             end
-          end
-        end
+          end,
 
-        [lines.join("\n") << "\n", matched_line_no]
+          issue
+        ].flatten
       end
 
       # Create a branch from the ref
-      # @param [Gitlab::ObjectifiedHash] failure_issue the existing failure issue
+      # @param [Gitlab::ObjectifiedHash, String] failure_issue the existing failure issue fetched from the GitLab API, or the full URL
+      # @param [String] name_prefix the prefix to attach to the branch name
+      # @param [Confiner::Example] example the example
       # @return [Gitlab::ObjectifiedHash] the new branch
-      def create_branch(failure_issue, example)
-        branch = @gitlab_client.create_branch(target_project, "#{failure_issue['iid']}-quarantine-#{example.name.gsub(/\W/, '-')}", ref)
+      def create_branch(failure_issue, name_prefix, example)
+        issue_id = failure_issue.is_a?(String)? failure_issue.split('/').last : failure_issue['iid'] # split url segment, last segment of path is the issue id
+        branch_name = [issue_id, name_prefix, environment['name'] || '', example.name.gsub(/\W/, '-')]
+
+        branch = @gitlab_client.create_branch(target_project, branch_name.join('-'), ref)
 
         log :branch, "Created branch #{branch['name']} (#{branch['web_url']})", 4
 
@@ -203,16 +361,11 @@ module Confiner
 
       # Commit changes to a branch
       # @param [Gitlab::ObjectifiedHash] branch the branch to commit to
+      # @param [String] message the message to commit
       # @param [Example] example the example
       # @param [Gitlab::ObjectifiedHash] new_content the new content to commit
       # @return [Gitlab::ObjectifiedHash] the commit
-      def commit_changes(branch, example, new_content)
-        message = <<~COMMIT
-          Quarantine end-to-end test
-
-          Quarantine #{example.name}
-        COMMIT
-
+      def commit_changes(branch, message, example, new_content)
         commit = @gitlab_client.create_commit(target_project, branch['name'], message, [
           { action: :update, file_path: example.file, content: new_content}
         ])
@@ -223,45 +376,65 @@ module Confiner
       end
 
       # Create a Merge Request with a given branch
+      # @param [String] title_prefix the prefix of the title
       # @param [Example] example the example
       # @param [Gitlab::ObjectifiedHash] branch the branch
-      # @param [Integer] changed_line_number the line that was changed to quarantine
-      # @param [Gitlab::ObjectifiedHash] failure_issue the failure issue
       # @return [Gitlab::ObjectifiedHash] the created merge request
-      def create_merge_request(example, branch, changed_line_number, failure_issue)
-        occurrences = @examples.select { _1.name == example.name && _1.failed? }
-        markdown_occurrences = []
+      def create_merge_request(title_prefix, example, branch, &block)
+        description = block.call
 
-        @examples.map do |eggsample|
-          if eggsample.name == example.name && eggsample.failed?
-            markdown_occurrences << "1. [#{eggsample.occurrence[:job]}](#{eggsample.occurrence[:pipeline_url]})"
-          end
-        end
+        environment_name = environment['name'] ? "[#{environment['name']}]" : ''
+        merge_request = @gitlab_client.create_merge_request(target_project,
+          "%{prefix} %{environment_name} %{example_name}" % { prefix: title_prefix, environment_name: environment_name, example_name: example.name },
+
+          source_branch: branch['name'],
+          target_branch: ref,
+          description: description,
+          labels: failure_issue_labels,
+          squash: true,
+          remove_source_branch: true)
 
-        meets_or_exceeds_threshold = occurrences.size > threshold ? 'exceeds' : 'meets'
+        log :mr, "Created merge request #{merge_request['iid']} (#{merge_request['web_url']})", 4
 
-        description = <<~MARKDOWN
-          ## What does this MR do?
+        merge_request
+      end
 
-          Quarantines the test `#{example.name}` (https://gitlab.com/#{target_project}/-/blob/#{ref}/#{example.file}#L#{changed_line_number})
+      private
 
-          This test has been found by [Confiner](https://gitlab.com/gitlab-org/quality/confiner) to have failed
-          #{occurrences.size} times, which #{meets_or_exceeds_threshold} the threshold of #{threshold} times.
+      # Find an example in file contents and transform the line
+      # @param [String] content the file contents
+      # @param [Confiner::Example] example the name of the example
+      # @yield [String] the line of the contents found
+      # @return [Array<String, Integer>] first item return is the new file contents. second item is the line number where the match was found
+      def find_example_match_in_contents(content, example)
+        content_to_return = content.dup
 
-          #{markdown_occurrences.join("\n")}
+        best_match = -1
+        lines = content_to_return.split("\n")
+        example.name.split.reverse.each do |word|
+          # given a full-descriptive RSpec example name:
+          #   Plan Group Iterations creates a group iteration
+          # scan these words backwards, incrementing the index until we find the best match
+          new_match = content_to_return.index(word) || -1
+          best_match = new_match if new_match > best_match
+        end
 
-          > #{failure_issue['web_url']}
+        # the matched line where the example is
+        match = content_to_return[best_match..].split("\n").first
+        matched_line_no = 0
 
-          <div align="center">
-          (This MR was automatically generated by [Confiner](https://gitlab.com/gitlab-org/quality/confiner) at #{Time.now.utc})
-          </div>
-        MARKDOWN
+        lines.each_with_index do |line, line_no|
+          if line.match?(match)
+            matched_line_no = line_no + 1
 
-        merge_request = @gitlab_client.create_merge_request(target_project, MERGE_REQUEST_TITLE % example.name, source_branch: branch['name'], target_branch: ref, description: description, labels: failure_issue_labels, squash: true, remove_source_branch: true)
+            log :rspec, "Found match on line #{example.file}:#{matched_line_no}", 4
 
-        log :mr, "Created merge request #{merge_request['iid']} (#{merge_request['web_url']})", 4
+            resulting_line = block_given? ? yield(line) : line
+            lines[line_no] = resulting_line
+          end
+        end
 
-        merge_request
+        [lines.join("\n") << "\n", matched_line_no]
       end
     end
   end

data/lib/confiner/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Confiner
-  VERSION = '0.2.3'
+  VERSION = '0.3.0'
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: confiner
 version: !ruby/object:Gem::Version
-  version: 0.2.3
+  version: 0.3.0
 platform: ruby
 authors:
 - GitLab Quality
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2022-03-07 00:00:00.000000000 Z
+date: 2022-03-22 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rspec