dri 0.10.2 → 0.11.0

data/lib/dri/commands/fetch/failures.rb

@@ -1,38 +1,39 @@
 # frozen_string_literal: true
 
+require "influxdb-client"
+
 require_relative '../../command'
 require_relative '../../utils/table'
 require_relative '../../utils/constants'
+require_relative '../../support/influxdb_tools'
 
 module Dri
   module Commands
     class Fetch
-      class Failures < Dri::Command
+      class Failures < Dri::Command # rubocop:disable Metrics/ClassLength
         include Dri::Utils::Table
         include Dri::Utils::Constants::Triage::Labels
+        include Dri::Support::InfluxdbTools
         using Refinements
 
         def initialize(options)
           @options = options
-          @start_date = @options[:start_date] ? Date.parse(@options[:start_date]) : Date.today
+          @start_date = @options[:start_date] ? Date.parse(@options[:start_date]) : Date.today - 1
           @end_date = @options[:end_date] ? Date.parse(@options[:end_date]) : Date.today
           @cutoff_time = @options[:cutoff] ? Time.parse(options[:cutoff]).utc : nil
         end
 
-        def execute(input: $stdin, output: $stdout) # rubocop:disable Metrics/AbcSize, Metrics/CyclomaticComplexity, Metrics/PerceivedComplexity, Metrics/MethodLength
+        def execute(_input: $stdin, output: $stdout) # rubocop:disable Metrics/AbcSize, Metrics/CyclomaticComplexity, Metrics/PerceivedComplexity, Metrics/MethodLength
           verify_config_exists
 
-          urgent_environments = %w[canary canary.staging]
-
           title = add_color('Title', :bright_yellow)
           triaged = add_color('Triaged?', :bright_yellow)
           environment = add_color('Environment', :bright_yellow)
-          author = add_color('Author', :bright_yellow)
           url = add_color('URL', :bright_yellow)
           updated_at = add_color('Updated at', :bright_yellow)
 
           sorted_failures = []
-          labels = { title: title, triaged: triaged, environment: environment, author: author, url: url,
+          labels = { title: title, triaged: triaged, environment: environment, url: url,
                      updated_at: updated_at }
           triaged_counter = 0
 
@@ -57,10 +58,30 @@ module Dri
               exit 0
             end
 
+            blocker_set = Set.new
+
+            blocker_set = Set.new
+
+            if query_api
+              begin
+                blockers = query_api.query(query: query_reliables_smokes)
+
+                blockers.each do |table|
+                  table.records.each do |record|
+                    blocker_set.add(record.values['name'])
+                  end
+                end
+              rescue StandardError => e
+                logger.error "An error occurred querying for reliable and smoke failures: #{e.message}"
+              end
+            end
+
             failures.each do |failure|
               project_id = failure.project_id
               title = failure.title.truncate(60)
-              author = failure.to_h.dig('author', 'username').truncate(25)
+
+              title = bold("*#{failure.title.truncate(60)}*") if blocker?(failure.title, blocker_set)
+
               url = failure.web_url
               updated_at = failure.updated_at
               triaged = add_color('x', :red)
@@ -69,9 +90,6 @@ module Dri
 
                 env == 'gitlab.com' ? 'production' : env
               end
-              urgent = urgent_environments.all? { |env| envs.include?(env) }
-
-              next if @options[:urgent] && !urgent
 
               emoji_awards = api_client.fetch_awarded_emojis(failure.iid, project_id: project_id).find do |e|
                 e.name == emoji && e.to_h.dig('user', 'username') == username
@@ -82,33 +100,74 @@ module Dri
                 triaged_counter += 1
               end
 
-              sorted_failures << { title: title, triaged: triaged, environment: envs.first || 'none', author: author,
+              sorted_failures << { title: title, triaged: triaged, environment: envs.first || 'none',
                                    url: url, updated_at: updated_at }
             end
 
             sorted_failures.sort_by! { |failure| failure[@options[:sort_by]&.to_sym || :environment] }
           end
 
-          msg = if @options[:urgent]
-                  <<~MSG
-                    Found: #{sorted_failures.size} urgent failures, occurring in both canary.gitlab.com and canary.staging.gitlab.com.
-                  MSG
-                else
-                  <<~MSG
-                    Found: #{sorted_failures.size} failures, of these #{triaged_counter} have been triaged with a #{emoji}.
-                  MSG
-                end
+          msg = <<~MSG
+            Found: #{sorted_failures.size} failures, of these #{triaged_counter} have been triaged with a #{emoji}.
+            Smoke and Reliable failures are marked with *Failure..* in bold.
+          MSG
 
           terminal_width = TTY::Screen.width
 
-          if terminal_width <= 210 # adjust the desired minimum width
+          if terminal_width <= 210
             labels.delete(:updated_at)
             sorted_failures.map { |failure| failure.delete(:updated_at) }
           end
 
-          print_table(labels.values, sorted_failures.map(&:values), alignments: [:left, :center, :center, :left, :left])
+          print_table(
+            labels.values,
+            sorted_failures.map(&:values),
+            alignments: [:left, :center, :center, :left]
+          )
           output.puts(msg)
         end
+
+        private
+
+        def blocker?(title, set)
+          title_part = title.split('|').last
+          return true if title_part&.strip && set.include?(title_part.strip)
+
+          false
+        end
+
+        def query_reliables_smokes
+          <<~QUERY
+            from(bucket: "#{Support::InfluxdbTools::INFLUX_MAIN_TEST_METRICS_BUCKET}")
+              |> range(start: -1d)
+              |> filter(fn: (r) => r._measurement == "test-stats")
+              |> filter(fn: (r) => r.run_type == "staging-full" or
+                r.run_type == "staging-sanity" or
+                r.run_type == "production-full" or
+                r.run_type == "production-sanity" or
+                r.run_type == "package-and-qa" or
+                r.run_type == "nightly" or
+                r.run_type == "e2e-test-on-gdk"
+              )
+              |> filter(fn: (r) => r.job_name != "airgapped" and
+                r.job_name != "instance-image-slow-network" and
+                r.job_name != "nplus1-instance-image"
+              )
+              |> filter(fn: (r) => r.status != "pending" and
+                r.merge_request == "false" and
+                r.quarantined == "false" and
+                r.smoke == "true" or
+                r.reliable == "true"
+              )
+              |> filter(fn: (r) => r["_field"] == "job_url" or
+                r["_field"] == "failure_exception" or
+                r["_field"] == "id"
+              )
+              |> pivot(rowKey: ["_time"], columnKey: ["_field"], valueColumn: "_value")
+              |> group(columns: ["name"])
+              |> distinct(column: "name")
+          QUERY
+        end
       end
     end
   end

data/lib/dri/commands/fetch/featureflags.rb

@@ -17,7 +17,7 @@ module Dri
           @today_iso_format = Time.now.strftime('%Y-%m-%dT00:00:00Z')
         end
 
-        def execute(input: $stdin, output: $stdout) # rubocop:disable Metrics/AbcSize, Metrics/CyclomaticComplexity, Metrics/PerceivedComplexity
+        def execute(_input: $stdin, output: $stdout) # rubocop:disable Metrics/AbcSize, Metrics/CyclomaticComplexity, Metrics/PerceivedComplexity
           verify_config_exists
 
           summary = add_color('Summary', :bright_yellow)

data/lib/dri/commands/fetch/pipelines.rb

@@ -20,7 +20,7 @@ module Dri
           @options = options
         end
 
-        def execute(input: $stdin, output: $stdout)
+        def execute(*)
           verify_config_exists
           logger.info "Fetching pipelines' status, this might take a while..."
           logger.warn "This command needs a large window to correctly print the table"
@@ -149,8 +149,8 @@ module Dri
         # @param [Integer] pipeline_id
         # @param [Boolean] sanity
         def allure_report(pipeline_name:, pipeline_id:, sanity:)
-          "https://storage.googleapis.com/gitlab-qa-allure-reports/#{allure_bucket_name(pipeline_name, sanity)}"\
-           "/master/#{pipeline_id}/index.html"
+          "https://storage.googleapis.com/gitlab-qa-allure-reports/#{allure_bucket_name(pipeline_name, sanity)}" \
+            "/master/#{pipeline_id}/index.html"
         end
 
         # Returns the GCP bucket name for different pipeline types

data/lib/dri/commands/fetch/runbooks.rb

@@ -14,7 +14,7 @@ module Dri
           @options = options
         end
 
-        def execute(folder: nil, input: $stdin, output: $stdout)
+        def execute(folder: nil, _input: $stdin, output: $stdout)
           return fetch_summary(output: output) unless folder
 
           fetch_runbook("#{folder}/index.md", output: output)

data/lib/dri/commands/fetch/testcases.rb

@@ -21,7 +21,7 @@ module Dri
           ]
         end
 
-        def execute(input: $stdin, output: $stdout) # rubocop:disable Metrics/AbcSize
+        def execute(_input: $stdin, output: $stdout) # rubocop:disable Metrics/AbcSize
           verify_config_exists
 
           if @options[:filter_pipelines]

data/lib/dri/commands/fetch/triaged.rb

@@ -16,7 +16,7 @@ module Dri
           @options = options
         end
 
-        def execute(input: $stdin, output: $stdout) # rubocop:disable Metrics/AbcSize
+        def execute(*) # rubocop:disable Metrics/AbcSize
           verify_config_exists
 
           title = add_color('Title', :magenta)

data/lib/dri/commands/fetch.rb

@@ -48,8 +48,6 @@ module Dri
       desc 'failures', 'Display failures opened on a given period.'
       method_option :help, aliases: '-h', type: :boolean,
                            desc: 'Display usage information'
-      method_option :urgent, type: :boolean,
-                             desc: 'Shows failures that quickly escalated'
       method_option :sort_by, type: :string,
                               desc: 'Shows failures in specified order'
       method_option :start_date, type: :string,
@@ -68,28 +66,6 @@ module Dri
         end
       end
 
-      desc 'quarantines', 'Display open quarantine MRs'
-      method_option :help, aliases: '-h', type: :boolean,
-                           desc: 'Display usage information'
-      def quarantines(*)
-        if options[:help]
-          invoke :help, ['quarantines']
-        else
-          require_relative 'fetch/quarantines'
-          Dri::Commands::Fetch::Quarantines.new(options, search: '[QUARANTINE]').execute
-        end
-      end
-
-      desc 'dequarantines', 'Display open dequarantine MRs'
-      method_option :help, aliases: '-h', type: :boolean,
-                           desc: 'Display usage information'
-      def dequarantines(*)
-        return invoke :help, %w[quarantines] if options[:help]
-
-        require_relative 'fetch/quarantines'
-        Dri::Commands::Fetch::Quarantines.new(options, search: '[DEQUARANTINE]').execute
-      end
-
       desc 'pipelines', 'Display status of pipelines'
       method_option :help, aliases: '-h', type: :boolean,
                            desc: 'Display pipelines usage information'

data/lib/dri/commands/incidents.rb

@@ -15,7 +15,7 @@ module Dri
         @options = options
       end
 
-      def execute(input: $stdin, output: $stdout) # rubocop:disable Metrics/AbcSize
+      def execute(_input: $stdin, output: $stdout) # rubocop:disable Metrics/AbcSize
         verify_config_exists
 
         incident = add_color('Incident', :bright_yellow)

data/lib/dri/commands/init.rb

@@ -14,7 +14,7 @@ module Dri
         puts pastel.yellow(font.write("DRI"))
       end
 
-      def execute(input: $stdin, output: $stdout) # rubocop:disable Metrics/AbcSize
+      def execute(_input: $stdin, output: $stdout) # rubocop:disable Metrics/AbcSize
         output.puts "🤖 Welcome to DRI 🤖\n"
 
         logger.info "🔎 Scanning for existing configurations...\n"

data/lib/dri/commands/profile.rb

@@ -11,7 +11,7 @@ module Dri
         @options = options
       end
 
-      def execute(input: $stdin, output: $stdout)
+      def execute(_input: $stdin, output: $stdout)
         if config.exist? && @options["edit"]
           editor.open(config.source_file)
           return
@@ -39,13 +39,13 @@ module Dri
 
       def pretty_print_profile
         <<~PROFILE
-          #{add_color('User:', :bright_cyan)} #{username} 
+          #{add_color('User:', :bright_cyan)} #{username}#{' '}
           #{add_color('Token:', :bright_cyan)} #{token}
           #{add_color('OpsToken:', :bright_cyan)} #{ops_token}
           #{add_color('Timezone:', :bright_cyan)} #{timezone}
           #{add_color('Emoji:', :bright_cyan)} #{emoji}
           #{add_color('Report Path:', :bright_cyan)} #{handover_report_path}
-          
+
         PROFILE
       end
     end

data/lib/dri/commands/publish/report.rb

@@ -24,7 +24,7 @@ module Dri
           @today_iso_format = Time.now.strftime('%Y-%m-%dT00:00:00Z')
         end
 
-        def execute(input: $stdin, output: $stdout) # rubocop:disable Metrics/AbcSize, Metrics/CyclomaticComplexity, Metrics/PerceivedComplexity, Metrics/MethodLength
+        def execute(_input: $stdin, output: $stdout) # rubocop:disable Metrics/AbcSize, Metrics/CyclomaticComplexity, Metrics/PerceivedComplexity, Metrics/MethodLength
           verify_config_exists
           report = Dri::Report.new(config)
 
@@ -195,9 +195,7 @@ module Dri
             FileUtils.mkdir_p("#{Dir.pwd}/handover_reports/.tmp")
             report_path = "handover_reports/.tmp/report-#{@date}.json"
 
-            File.open(report_path, 'w') do |out_file|
-              out_file.write(posted_note.to_h.to_json)
-            end
+            File.write(report_path, posted_note.to_h.to_json)
           end
 
           output.puts "Done! ✅\n"

data/lib/dri/commands/rm/emoji.rb

@@ -10,7 +10,7 @@ module Dri
           @options = options
         end
 
-        def execute(input: $stdin, output: $stdout) # rubocop:disable Metrics/AbcSize
+        def execute(_input: $stdin, output: $stdout) # rubocop:disable Metrics/AbcSize
           verify_config_exists
 
           remove = prompt.yes? "Are you sure you want to remove all #{emoji} award emojis from issues?"

data/lib/dri/commands/rm/profile.rb

@@ -11,7 +11,7 @@ module Dri
           @options = options
         end
 
-        def execute(input: $stdin, output: $stdout)
+        def execute(*)
           verify_config_exists
 
           remove = prompt.yes? "Are you sure you want to remove existing profile?"

data/lib/dri/commands/rm/reports.rb

@@ -11,7 +11,7 @@ module Dri
           @options = options
         end
 
-        def execute(input: $stdin, output: $stdout)
+        def execute(*)
           FileUtils.rm_rf(handover_report_path) if prompt.yes?("Remove everything in #{handover_report_path}?")
         end
       end

data/lib/dri/commands/view/fast_quarantine.rb

@@ -0,0 +1,27 @@
+# frozen_string_literal: true
+
+require_relative '../../command'
+
+module Dri
+  module Commands
+    class View
+      class FastQuarantine < Dri::Command
+        def initialize(options)
+          @options = options
+        end
+
+        def execute(*)
+          verify_config_exists
+
+          logger.info "Fetching fast quarantined tests..."
+
+          file_content = api_client.get_fast_quarantine_tests
+
+          file_content.each_line do |line|
+            puts "• #{line}"
+          end
+        end
+      end
+    end
+  end
+end

data/lib/dri/commands/view.rb

@@ -0,0 +1,23 @@
+# frozen_string_literal: true
+
+require 'thor'
+
+module Dri
+  module Commands
+    class View < Thor
+      namespace :view
+
+      desc 'fastquarantine', 'View fast-quarantine tests'
+      method_option :help, aliases: '-h', type: :boolean,
+                           desc: 'Display usage information'
+      def fastquarantine(*)
+        if options[:help]
+          invoke :help, ['fastquarantine']
+        else
+          require_relative 'view/fast_quarantine'
+          Dri::Commands::View::FastQuarantine.new(options).execute
+        end
+      end
+    end
+  end
+end

data/lib/dri/feature_flag_report.rb

@@ -26,7 +26,7 @@ module Dri
       feature_flag_data = [summary, changed_on, url]
 
       labels = feature_flag.labels
-      host_label = labels.select { |label| /^host::/.match(label) }.join('')
+      host_label = labels.select { |label| /^host::/.match(label) }.join
 
       case host_label
       when PRODUCTION

data/lib/dri/report.rb

@@ -63,66 +63,17 @@ module Dri
 
       failure_type = filter_failure_type_labels(labels)
       assigned_status = assigned?(failure["assignees"])
-      pipelines = filter_pipeline_labels(labels)
-
-      linked_pipelines = link_pipelines(project_id, iid, pipelines, description)
+      pipelines = format_pipelines(labels)
 
       actions_status = actions_status_template(failure_type, assigned_status, actions_opts)
       actions_fixes = actions_fixes_template(related_mrs)
 
-      @failures << [title, emojified_link, linked_pipelines, stack_trace, "#{actions_status}#{actions_fixes}"]
+      @failures << [title, emojified_link, pipelines, stack_trace, "#{actions_status}#{actions_fixes}"]
     end
 
     private
 
-    def link_pipelines(project_id, iid, pipelines, description) # rubocop:disable Metrics/CyclomaticComplexity, Metrics/PerceivedComplexity, Metrics/MethodLength
-      linked = []
-      label_pipeline_map = {
-        'gitlab.com' => '/quality/production',
-        'canary.gitlab.com' => '/quality/canary',
-        'canary.staging.gitlab.com' => '/quality/staging-canary',
-        'main' => '/gitlab-org/gitlab-qa-mirror',
-        'master' => '/gitlab-org/gitlab-qa-mirror',
-        'pre.gitlab.com' => '/quality/preprod',
-        'staging-ref' => '/quality/staging-ref',
-        'staging.gitlab.com' => '/quality/staging',
-        'release' => '/quality/release'
-      }
-
-      failure_notes = @api_client.fetch_failure_notes(project_id, iid)
-
-      return if pipelines.empty?
-
-      pipelines.each do |pipeline|
-        next unless label_pipeline_map.has_key?(pipeline)
-
-        pipeline_in_notes_found = false
-        pipeline_link = ''
-        pipeline_markdown = pipeline.gsub(/.gitlab.com/, '')
-
-        failure_notes.each do |note|
-          next unless note.body.include?(label_pipeline_map.fetch(pipeline))
-
-          pipeline_in_notes_found = true
-          pipeline_link = URI.extract(note.body, %w[https])
-        end
-
-        unless pipeline_in_notes_found
-          links_description = URI.extract(description, %w[https])
-          pipeline_link = links_description.select { |link| link.include? label_pipeline_map.fetch(pipeline) }
-        end
-
-        unless pipeline_link.empty?
-          pipeline_link_sanitized = pipeline_link.join.strip
-          pipeline_markdown = "[#{pipeline_markdown}](#{pipeline_link_sanitized})"
-        end
-
-        linked << pipeline_markdown
-      end
-      linked.join(', ')
-    end
-
-    def actions_status_template(failure_type, assigned_status, actions_opts) # rubocop:disable Metrics/CyclomaticComplexity, Metrics/PerceivedComplexity/MethodLength
+    def actions_status_template(failure_type, assigned_status, actions_opts) # rubocop:disable Metrics/CyclomaticComplexity, Metrics/PerceivedComplexity/
       quarantined = ''
       reproduced = ''
       transient = ''
@@ -184,6 +135,15 @@ module Dri
       pipelines
     end
 
+    def format_pipelines(labels)
+      labels.filter_map do |label|
+        next unless label.include?(FOUND)
+
+        sanitized_label = label.gsub(FOUND, ' ').strip
+        sanitized_label.gsub(/.gitlab.com/, '')
+      end.join(', ')
+    end
+
     def filter_failure_type_labels(labels)
       labels.each do |label|
         @type = label.gsub!(FAILURE, ' ').to_s if label.include? FAILURE

data/lib/dri/support/influxdb_tools.rb

@@ -0,0 +1,37 @@
+# frozen_string_literal: true
+
+module Dri
+  module Support
+    # Common tools for use with influxdb metrics setup
+    #
+    module InfluxdbTools
+      # @return [String] bucket for storing metrics from main runs
+      INFLUX_MAIN_TEST_METRICS_BUCKET = "e2e-test-stats-main"
+
+      private
+
+      # Query client
+      #
+      # @return [QueryApi]
+      def query_api
+        return nil unless influx_client
+
+        @query_api ||= influx_client.create_query_api
+      end
+
+      # InfluxDb client
+      #
+      # @return [InfluxDB2::Client]
+      def influx_client
+        return nil unless ENV["QA_INFLUXDB_URL"] && ENV["QA_INFLUXDB_TOKEN"]
+
+        @influx_client ||= InfluxDB2::Client.new(
+          ENV.fetch("QA_INFLUXDB_URL", nil),
+          ENV.fetch("QA_INFLUXDB_TOKEN", nil),
+          bucket: INFLUX_MAIN_TEST_METRICS_BUCKET,
+          org: "gitlab-qa"
+        )
+      end
+    end
+  end
+end

data/lib/dri/utils/constants.rb

@@ -11,13 +11,14 @@ module Dri
         FEATURE_FLAG_LOG_PROJECT_ID = 15208716
         INFRA_TEAM_PROD_PROJECT_ID  = 7444821
         RUNBOOKS_PROJECT_ID         = 41045213
+        FAST_QUARANTINE_PROJECT_ID  = 45427186
       end
 
       module Triage
         module Labels
           FOUND = 'found:'
           FAILURE = 'failure::'
-          FAILURE_NEW = "#{FAILURE}new"
+          FAILURE_NEW = "#{FAILURE}new".freeze
           QA = "QA"
           INCIDENT = 'Incident::'
           SERVICE = 'Service::'

data/lib/dri/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Dri
-  VERSION = "0.10.2"
+  VERSION = "0.11.0"
 end