confiner 0.1.0

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA256:
+  metadata.gz: 75209090e950ca1c9174bf10ef33e4c33cfbaaaa0d0f77f6cbc55735b7be90b7
+  data.tar.gz: c187a788996ec52e9fecd5d42de5933d298345752a4d14c060914ec21826d859
+SHA512:
+  metadata.gz: 0e5738fd37387d6c45921c696ac32629afa04da3ac91cf42545fedda2d14fc017caf1319d753344598976f8d3204ccb0ba398c4035e809b0c16adf0a2081dd3a
+  data.tar.gz: b8b248eb4f1956f38d3279dde330b8b7d8ee62980796fa89fb5b8b5cb821fb4b77fe7f07bd1888dd802cbffe15fde5243ad76227e2c597e9204b444ea4d32045

data/bin/confiner

@@ -0,0 +1,6 @@
+#!/usr/bin/env ruby
+# frozen_string_literal: true
+
+require_relative '../lib/confiner'
+
+Confiner::Cli.run(*ARGV)

data/lib/confiner/cli.rb

@@ -0,0 +1,174 @@
+# frozen_string_literal: true
+
+require 'optparse'
+require 'yaml'
+
+module Confiner
+  class Cli
+    include Logger
+
+    attr_accessor :action, :plugins
+    attr_reader :parser, :plugin_options
+
+    def initialize(*options)
+      @plugins = []
+      @plugin_options = []
+      @rules_files = []
+      @rules = []
+
+      # default logging to standard out
+      Logger.log_to $stdout
+
+      if options.include?('--')
+        @plugin_options = options[options.index('--')..]
+        options = options[0..options.index('--')]
+      end
+
+      @parser = OptionParser.new do |opts|
+        opts.banner = <<~USAGE
+          Usage: #{$PROGRAM_NAME} [options] [-- plugin_options]
+
+          Examples:
+            Run all rules within .confiner directory outputting to a log file:
+              #{$PROGRAM_NAME} -o confiner.log
+            Run a specific rule file overriding specific arguments for plugins:
+              #{$PROGRAM_NAME} -r rules.yml -- --arg1=foo --arg2=bar
+            Run all all_rules within a specific directory:
+              #{$PROGRAM_NAME} -r ./all_rules
+
+          Options:
+        USAGE
+
+        opts.on('-h', '--help', 'Show the help') do
+          puts opts
+          exit 0
+        end
+
+        opts.on('-r RULES', '--rules RULES', 'Path to rule yaml file or directory of rules') do |rule|
+          rule.strip! # strip any trailing/leading spacess
+          rules = File.expand_path(rule)
+
+          raise "Rule file or directory `#{rules}` does not exist" unless File.exist?(rules)
+
+          @rules = if File.directory?(rules)
+                     Dir[File.join(rules, '**', '*.yml')].each_with_object([]) do |definitions, all_rules|
+                       all_rules << load_yaml(definitions)
+                     end
+                   else
+                     [load_yaml(rules)]
+                   end
+        end
+
+        opts.on('-v', '--version', 'Show the version') do
+          $stdout.puts "#{$PROGRAM_NAME} version #{VERSION}"
+          exit(0)
+        end
+
+        opts.on('-o OUTPUT', '--output-to OUTPUT', 'File to output the log to') do |output_to|
+          log_to(output_to)
+        end
+      end
+
+      @parser.parse!(options)
+
+      log :confiner, 'Program Start'
+
+      if @rules.empty?
+        # load any and all rules within .confiner
+        raise 'No rules to run. Are you missing a .confiner directory or -r argument?' unless Dir.exist?('.confiner')
+
+        @rules = Dir[File.join('.confiner', '**', '*.yml')].each_with_object([]) do |definitions, rules|
+          rules << load_yaml(definitions)
+        end
+      end
+
+      log :rules, 'Using rule files:'
+      @rules_files.each do |file|
+        log :loaded, file, 2
+      end
+    end
+
+    # Run the confiner
+    def run
+      @rules.each do |rules|
+        rules.each do |rule|
+          process_rule(rule)
+        end
+      end
+
+      log :confiner, 'Done'
+    end
+
+    def self.run(*argv)
+      new(*argv).run
+    end
+
+    private
+
+    # Process a singular rule
+    # @param [Hash] rule
+    def process_rule(rule)
+      log :rule, rule.keys.map { |k| "\t#{k}=#{rule[k]}" }.join(',')
+
+      rule['plugin']['args'].transform_keys!(&:to_sym) # 2.5 compatability
+
+      plugin = Plugins.const_get(translate_plugin_name(rule['plugin']['name'])).new(**rule['plugin']['args'])
+
+      # perform verification of actions before execution
+      rule['actions'].each do |action|
+        raise "YAML is invalid. Action `#{action}` does not exist." unless plugin.respond_to?(action)
+      end
+
+      # execute each action
+      rule['actions'].each do |action|
+        plugin.run(action) { |p| p.public_send(action) }
+      end
+    end
+
+    # Ensure that the rules are well-formed
+    def validate_rules(rules, file)
+      raise "YAML is invalid. Rules must be an array (from #{file})." unless rules.is_a? Array
+
+      rules.each do |rule|
+        # name is required
+        raise "YAML is invalid. Rule must have a name. (from #{file})" unless rule['name']
+
+        # actions are required
+        raise "YAML is invalid. Rule `#{rule['name']}` must have actions and it must be an Array (from #{file})" unless rule['actions']&.is_a? Array
+
+        # plugin is required and must be well-formed
+        raise "YAML is invalid. Rule `#{rule['name']}` must have a plugin and it must have a name (from #{file})" unless rule['plugin'] && rule['plugin']['name']
+
+        # Plugin must exist
+        plugin = begin
+                  Plugins.const_get(translate_plugin_name(rule['plugin']['name']))
+                rescue NameError
+                  raise "YAML is invalid. Rule `#{rule['name']}` does not have plugin `#{rule['plugin']['name']}` (from #{file})"
+                end
+
+        # Validate the actions
+        rule['actions'].each do |action|
+          begin
+            plugin.instance_method(action)
+          rescue NameError
+            raise "YAML is invalid. Rule `#{rule['name']}` plugin `#{rule['plugin']['name']}` has no action `#{action}` (from #{file})"
+          end
+        end
+      end
+    end
+
+    # Translate a plugin name from snake_case to PascalCase
+    def translate_plugin_name(plugin_name)
+      plugin_name.split('_').map(&:capitalize).join
+    end
+
+    # Load yaml file and validate all rules
+    def load_yaml(file)
+      raise 'File is not a YAML file' unless File.extname(file).match(/yml|yaml/i)
+
+      @rules_files << file
+
+      validate_rules(YAML.load_file(file), file)
+    end
+  end
+end

data/lib/confiner/example.rb

@@ -0,0 +1,38 @@
+# frozen_string_literal: true
+
+module Confiner
+  # Representation of an example (test)
+  class Example
+    attr_accessor :status, :name, :classname, :file, :occurrence
+
+    def initialize(**attributes)
+      @status = attributes.fetch('status') { attributes.fetch(:status) }
+      @name = attributes.fetch('name') { attributes.fetch(:name) }
+      @classname = attributes.fetch('classname') { attributes.fetch(:classname) }
+      @file = attributes.fetch('file') { attributes.fetch(:file) }
+      @occurrence = attributes.fetch('occurrence') { attributes.fetch(:occurrence) }
+
+      yield(self) if block_given?
+    end
+
+    # Check if this example had passed
+    # @return [Boolean] true if the example had passed
+    def passed?
+      status == 'success'
+    end
+
+    # Check if this example had failed
+    # @return [Boolean] true if the example had failed
+    def failed?
+      status == 'failed'
+    end
+
+    # Check if this example had been skipped
+    # @return [Boolean] true if the example had been skipped
+    def skipped?
+      status == 'skipped'
+    end
+
+    def to_s; name; end
+  end
+end

data/lib/confiner/logger.rb

@@ -0,0 +1,26 @@
+# frozen_string_literal: true
+
+module Confiner
+  module Logger
+    # Log something with a specific level
+    def log(level, message, indentation = 1)
+      raise ArgumentError, 'Level must be less than 12 characters' if level.size > 12
+
+      output = "(#{Time.now.strftime('%F %H:%M:%S')})\t\e[0;35m#{level.to_s.upcase}#{' ' * (12 - level.size)}\e[m#{"\t" * indentation}#{message}"
+
+      Logger.log_to(File.open(Logger.log_to.strip, 'a+t')) if Logger.log_to.is_a?(String)
+      Logger.log_to.puts(output)
+    end
+
+    # Where to output the log
+    def self.log_to(file = $stdout)
+      @out_file ||= file
+    end
+
+    def run(action)
+      log :plugin, "#{self.class}##{action}"
+      super
+      log :plugin, "#{self.class}##{action} Done"
+    end
+  end
+end

data/lib/confiner/plugin.rb

@@ -0,0 +1,46 @@
+# frozen_string_literal: true
+
+module Confiner
+  # Plugin for Confiner
+  class Plugin
+    prepend Logger
+
+    attr_reader :examples
+
+    class << self
+      # Define arguments that the plugin will accept
+      # @param [Array<Symbol, Hash>] args the arguments that this plugin accepts
+      # @option args [Symbol] :argument the argument to pass with no default
+      # @option args [Hash] :argument the argument to pass with a default value
+      # @note the arguments should be well-formed in the .yml rule file
+      def arguments(*args)
+        @arguments ||= args
+
+        args.each do |arg|
+          if arg.is_a? Hash
+            arg.each do |a, default_value|
+              attr_writer a
+
+              define_method(a) do
+                instance_variable_get("@#{a}") || instance_variable_set("@#{a}", default_value)
+              end
+            end
+          else
+            attr_accessor arg
+          end
+        end
+      end
+    end
+
+    def initialize(**args)
+      args.each do |k, v|
+        self.public_send(:"#{k}=", v)
+      end
+    end
+
+    # Run the plugin
+    def run(action, &block)
+      yield self
+    end
+  end
+end

data/lib/confiner/plugins/debug.rb

@@ -0,0 +1,17 @@
+# frozen_string_literal: true
+
+module Confiner
+  module Plugins
+    class Debug < Plugin
+      arguments :arg1, :arg2
+
+      def action1
+        log :debug, arg1
+      end
+
+      def action2
+        log :debug, arg2
+      end
+    end
+  end
+end

data/lib/confiner/plugins/gitlab.rb

@@ -0,0 +1,256 @@
+# frozen_string_literal: true
+
+require 'gitlab'
+
+module Confiner
+  module Plugins
+    class Gitlab < Plugin
+      arguments :private_token,                           # the GitLab API Private-Token to use
+                :project_id,                              # the project where the pipelines are fetch from
+                :target_project,                          # where failure issues will be searched, and where an MR will be filed
+                :failure_issue_labels,                    # what labels to search for when searching for the failure issue
+                :failure_issue_prefix,                    # what prefix an issue has in GitLab Issues to search for the failure issue
+                :timeout => 10,                           # the timeout that HTTParty will consume (timeout of requests)
+                :threshold => 3,                          # the failure / pass threshold
+                :endpoint => 'https://gitlab.com/api/v4', # the GitLab API Endpoint (e.g. https://gitlab.com/api/v4)
+                :pwd => '.',                              # the path of the working directory for the examples
+                :ref => 'master'                          # the default Git ref used when updating
+
+      MERGE_REQUEST_TITLE = '[QUARANTINE] %s'
+      QUARANTINE_METADATA = %(, quarantine: { issue: '%s', type: :investigating })
+
+      def initialize(**args)
+        super
+
+        ENV['GITLAB_API_HTTPARTY_OPTIONS'] = ENV.fetch('GITLAB_API_HTTPARTY_OPTIONS') { "{read_timeout: #{timeout}}" }
+
+        @gitlab_client = ::Gitlab.client(private_token: private_token, endpoint: endpoint)
+      end
+
+      # Quarantine Action - Automatically Quarantine tests
+      def quarantine
+        log :gitlab, 'Beginning Quarantine Process', 2
+
+        # store the examples from the pipelines
+        @examples = get_examples
+
+        @examples.select(&:failed?).map(&:name).uniq.each do |failed_example|
+          # count the number of failures consecutively for this example
+
+          number_of_failures = @examples.select { _1.name == failed_example && _1.failed? }.size
+          if number_of_failures >= threshold
+            example = @examples.find { _1.name == failed_example }
+
+            log :quarantine, "Quarantining #{failed_example} (#{number_of_failures} >= #{threshold})", 3
+
+            # check to see if there is a merge request first
+            # if there is no merge request...
+            #   - Check for an existing issue
+            #   - Check for an existing Quarantine MR
+            #   - Add a quarantine tag: `it 'should be quarantined', quarantine: { issue: 'https://issue', type: :investigating }`
+            #   - File the merge request
+
+            begin
+              # begin the quarantining process
+              failure_issue = get_failure_issue_for_example(example)
+              file_contents = get_example_file_contents(example)
+              new_contents, changed_line_no = add_quarantine_metadata(file_contents, example, failure_issue)
+
+              branch = create_branch(failure_issue, example)
+              commit_changes(branch, example, new_contents)
+              create_merge_request(example, branch, changed_line_no, failure_issue)
+
+              log :quarantine, "Done Quarantining #{failed_example}", 3
+            rescue => e
+              log :fatal, "There was an issue quarantining #{example}. Error was #{e.message}\n#{e.backtrace}"
+            end
+          end
+        end
+
+        log :gitlab, 'Done with Quarantine Process', 2
+      end
+
+      # Dequarantine Action - Automatically Dequarantine tests
+      def dequarantine
+
+      end
+
+      private
+
+      # Get last n amount of runs
+      # @param [Integer] threshold the amount of pipelines to fetch
+      #   For instance, if threshold: 3, the amount of pipelines will return 6.  (3x Successful, 3x Failed)
+      # @return [Array<Gitlab::ObjectifiedHash>] an array of pipelines returned from GitLab
+      def get_last_n_runs(threshold:)
+        pipelines = [] # collection of both passing and failing pipelines
+
+        pipelines << @gitlab_client.pipelines(project_id, per_page: threshold, status: :success)
+        pipelines << @gitlab_client.pipelines(project_id, per_page: threshold, status: :failed)
+      end
+
+      # Get examples from pipelines
+      # @param [Integer] threshold the amount of pipelines to fetch
+      # @note The threshold defaults to default threshold multiplied by two to the amount of pipelines
+      # @return [Array<Example>] array of examples
+      def get_examples(threshold: self.threshold * 2)
+        examples = []
+
+        get_last_n_runs(threshold: threshold).each do |run|
+          run.each do |pipeline|
+            # fetch the pipeline test report
+            @gitlab_client.pipeline_test_report(project_id, pipeline['id'])['test_suites'].each do |suite|
+              log :suite, "Suite: #{suite['name']} (#{pipeline['web_url']})", 4
+              suite['test_cases'].each do |example|
+                examples << Example.new(**example, occurrence: { job: suite['name'], pipeline_url: pipeline['web_url'] })
+                log :test, example['name'], 5
+              end
+            end
+          end
+        end
+
+        examples
+      end
+
+      # Query the GitLab Issues API to fetch QA failure issues
+      # @param [String,Example] example the name of the example to search for
+      # @option example [String] the name of the example to search for
+      # @option example [Example] an instance of Example
+      def get_failure_issue_for_example(example)
+        issues = @gitlab_client.issues(target_project,
+          labels: failure_issue_labels,
+          state: :opened,
+          search: "#{failure_issue_prefix}#{example}",
+          in: :title,
+          per_page: 1
+        )
+
+        if issues.any?
+          log :issue, "Found issue #{issues.first['web_url']} for `#{example}`", 4
+
+          issues.first
+        else
+          log :fatal, "No failure issue exists for `#{example}`. Skipping."
+        end
+      end
+
+      # Get the file contents of an example
+      # @param [Example] example the example to get the contents of
+      def get_example_file_contents(example)
+        example.file = example.file.sub('./', File.join(pwd, '/')) unless pwd == '.'
+
+        @gitlab_client.file_contents(target_project, example.file)
+      end
+
+      # Add quarantine metadata to the file content and replace it
+      # @param [String] content the content to
+      # @param [Example] example the example to find and replace
+      # @param [Gitlab::ObjectifiedHash] failure_issue the failure issue
+      # @return [Array<String, Integer>] first value holds the new content, the second value holds the line number of the test
+      def add_quarantine_metadata(content, example, failure_issue)
+        content_to_return = content.dup
+
+        best_match = -1
+        lines = content_to_return.split("\n")
+        example.name.split.reverse.each do |word|
+          # given a full-descriptive RSpec example name:
+          #   Plan Group Iterations creates a group iteration
+          # scan these words backwards, incrementing the index until we find the best match
+          new_match = content_to_return.index(word)
+          best_match = new_match if new_match > best_match
+        end
+
+        # the matched line where the example is
+        match = content_to_return[best_match..].split("\n").first
+        matched_line_no = 0
+
+        lines.each_with_index do |line, line_no|
+          if line.match?(match)
+            matched_line_no = line_no + 1
+
+            log :rspec, "Found match on line #{example.file}:#{matched_line_no}", 4
+
+            if line.include?(',')
+              line[line.index(',')] = QUARANTINE_METADATA % failure_issue['web_url'] << ','
+            else
+              line[line.rindex(' ')] = QUARANTINE_METADATA % failure_issue['web_url'] << ' '
+            end
+          end
+        end
+
+        [lines.join("\n") << "\n", matched_line_no]
+      end
+
+      # Create a branch from the ref
+      # @param [Gitlab::ObjectifiedHash] failure_issue the existing failure issue
+      # @return [Gitlab::ObjectifiedHash] the new branch
+      def create_branch(failure_issue, example)
+        branch = @gitlab_client.create_branch(target_project, "#{failure_issue['iid']}-quarantine-#{example.name.tr(' ', '-')}", ref)
+
+        log :branch, "Created branch #{branch['name']} (#{branch['web_url']})", 4
+
+        branch
+      end
+
+      # Commit changes to a branch
+      # @param [Gitlab::ObjectifiedHash] branch the branch to commit to
+      # @param [Example] example the example
+      # @param [Gitlab::ObjectifiedHash] new_content the new content to commit
+      # @return [Gitlab::ObjectifiedHash] the commit
+      def commit_changes(branch, example, new_content)
+        message = <<~COMMIT
+          Quarantine end-to-end test
+
+          Quarantine #{example.name}
+        COMMIT
+
+        commit = @gitlab_client.create_commit(target_project, branch['name'], message, [
+          { action: :update, file_path: example.file, content: new_content}
+        ])
+
+        log :commit, "Created commit #{commit['id']} (#{commit['web_url']}) on #{branch['name']}", 4
+
+        commit
+      end
+
+      # Create a Merge Request with a given branch
+      # @param [Example] example the example
+      # @param [Gitlab::ObjectifiedHash] branch the branch
+      # @param [Integer] changed_line_number the line that was changed to quarantine
+      # @param [Gitlab::ObjectifiedHash] failure_issue the failure issue
+      # @return [Gitlab::ObjectifiedHash] the created merge request
+      def create_merge_request(example, branch, changed_line_number, failure_issue)
+        occurrences = @examples.select { _1.name == example.name && _1.failed? }
+        markdown_occurrences = []
+
+        @examples.map do |eggsample|
+          if eggsample.name == example.name && eggsample.failed?
+            markdown_occurrences << "1. [#{eggsample.occurrence[:job]}](#{eggsample.occurrence[:pipeline_url]})"
+          end
+        end
+
+        description = <<~MARKDOWN
+          ## What does this MR do?
+
+          Quarantines the test `#{example.name}` (#{example.file}:#{changed_line_number})
+
+          This test has been found by [Confiner](https://gitlab.com/gitlab-org/quality/confiner) to have been failing for
+          more than (or equal to) #{threshold} times.  This test has failed #{occurrences.size} times.
+
+          #{markdown_occurrences.join("\n")}
+
+          > #{failure_issue['web_url']}
+
+          <div align="center">
+          (This MR was automatically generated by [Confiner](https://gitlab.com/gitlab-org/quality/confiner) at #{Time.now.utc})
+          </div>
+        MARKDOWN
+
+        merge_request = @gitlab_client.create_merge_request(target_project, MERGE_REQUEST_TITLE % example.name, source_branch: branch['name'], target_branch: ref, description: description, labels: failure_issue_labels, squash: true, remove_source_branch: true)
+
+        log :mr, "Created merge request #{merge_request['iid']} (#{merge_request['web_url']})", 4
+
+        merge_request
+      end
+    end
+  end
+end

data/lib/confiner.rb

@@ -0,0 +1,11 @@
+# frozen_string_literal: true
+
+require 'zeitwerk'
+
+loader = Zeitwerk::Loader.new
+loader.push_dir(__dir__)
+loader.setup
+
+module Confiner
+  VERSION = '0.1.0'
+end

metadata

@@ -0,0 +1,94 @@
+--- !ruby/object:Gem::Specification
+name: confiner
+version: !ruby/object:Gem::Version
+  version: 0.1.0
+platform: ruby
+authors:
+- GitLab Quality
+autorequire:
+bindir: bin
+cert_chain: []
+date: 2022-01-06 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: rspec
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 3.10.0
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 3.10.0
+- !ruby/object:Gem::Dependency
+  name: zeitwerk
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 2.5.1
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 2.5.1
+- !ruby/object:Gem::Dependency
+  name: gitlab
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 4.17.0
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 4.17.0
+description:
+email:
+- quality+confiner@gitlab.com
+executables:
+- confiner
+extensions: []
+extra_rdoc_files: []
+files:
+- bin/confiner
+- lib/confiner.rb
+- lib/confiner/cli.rb
+- lib/confiner/example.rb
+- lib/confiner/logger.rb
+- lib/confiner/plugin.rb
+- lib/confiner/plugins/debug.rb
+- lib/confiner/plugins/gitlab.rb
+homepage: https://gitlab.com/gitlab-org/quality/confiner
+licenses:
+- MIT
+metadata: {}
+post_install_message:
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '2.5'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubygems_version: 3.2.22
+signing_key:
+specification_version: 4
+summary: Yaml rule-based confinement
+test_files: []