gitlab_quality-test_tooling 1.11.0 → 1.17.0

data/lib/gitlab_quality/test_tooling/report/generate_test_session.rb

@@ -205,12 +205,9 @@ module GitlabQuality
 
         def generate_test_text(testcase, tests_with_same_testcase, passed)
           text = tests_with_same_testcase.map(&:name).uniq.join(', ')
-          encoded_text = ERB::Util.url_encode(text)
 
           if testcase && !passed
-            # Workaround for reducing system notes on testcase issues
-            # The first regex extracts the link to the issues list page from a link to a single issue show page by removing the issue id.
-            "[#{text}](#{testcase.match(%r{[\s\S]+/[^/\d]+})}?state=opened&search=#{encoded_text})"
+            "[#{text}](#{testcase})"
           else
             text
           end

data/lib/gitlab_quality/test_tooling/report/knapsack_report_issue.rb

@@ -0,0 +1,139 @@
+# frozen_string_literal: true
+
+module GitlabQuality
+  module TestTooling
+    module Report
+      # Uses the API to create GitLab issues for spec run time exceeding Knapsack expectation
+      #
+      # - Takes the expected and actual Knapsack JSON reports from the knapsack output
+      # - Takes a project where issues should be created
+      # - For every test file reported with unexpectedly long run time:
+      #   - Find issue by test file name, and if found:
+      #     - Reopen issue if it already exists, but is closed
+      #     - Update the issue with the new run time data
+      #   - If not found:
+      #     - Create a new issue with the run time data
+      class KnapsackReportIssue < ReportAsIssue
+        NEW_ISSUE_LABELS = Set.new(['test', 'type::maintenance', 'maintenance::performance', 'priority::3', 'severity::3', 'knapsack_report']).freeze
+        SEARCH_LABELS = %w[test maintenance::performance knapsack_report].freeze
+        JOB_TIMEOUT_EPIC_URL = 'https://gitlab.com/groups/gitlab-org/quality/engineering-productivity/-/epics/19'
+
+        def initialize(token:, input_files:, expected_report:, project: nil, dry_run: false)
+          super
+
+          @expected_report = expected_report
+        end
+
+        private
+
+        attr_reader :expected_report
+
+        def run!
+          puts "Reporting spec file exceeding Knapsack expectaton issues in project `#{project}` via the API at `#{Runtime::Env.gitlab_api_base}`."
+
+          search_and_create_issue
+        end
+
+        def search_and_create_issue
+          filtered_report = KnapsackReports::SpecRunTimeReport.new(
+            expected_report_path: expected_report_path,
+            actual_report_path: actual_report_path
+          ).filtered_report
+
+          puts "=> Reporting #{filtered_report.count} spec files exceeding Knapsack expectation."
+
+          filtered_report.each do |spec_with_run_time|
+            existing_issues = find_issues_for_test(spec_with_run_time, labels: SEARCH_LABELS)
+
+            if existing_issues.empty?
+              puts "Creating issue for #{spec_with_run_time.file}"
+              create_issue(spec_with_run_time)
+            else
+              update_issue(issue: existing_issues.last, spec_run_time: spec_with_run_time)
+            end
+          end
+        end
+
+        def expected_report_path
+          return if expected_report.nil? || !File.exist?(expected_report)
+
+          expected_report
+        end
+
+        def actual_report_path
+          return if files.nil? || !File.exist?(files.first)
+
+          files.first
+        end
+
+        def new_issue_labels(_spec_run_time)
+          NEW_ISSUE_LABELS
+        end
+
+        def new_issue_title(spec_run_time)
+          "Job timeout risk: #{spec_run_time.file} ran much longer than expected"
+        end
+
+        def new_issue_description(spec_run_time)
+          <<~MARKDOWN.chomp
+          /epic #{JOB_TIMEOUT_EPIC_URL}
+
+          ### Why was this issue created?
+
+          #{spec_run_time.file_link_markdown} was reported to have:
+
+          1. exceeded Knapsack's expected runtime by at least 50%, and
+          2. been identified as a notable pipeline bottleneck and a job timeout risk
+
+          ### Suggested steps for investigation
+
+          1. To reproduce in CI by running test files in the same order, you can follow the steps listed [here](https://docs.gitlab.com/ee/development/testing_guide/flaky_tests.html#recreate-job-failure-in-ci-by-forcing-the-job-to-run-the-same-set-of-test-files).
+          1. Identify if a specific test case is stalling the run time. Hint: You can search the job's log for `Starting example group #{spec_run_time.file}` and view the elapsed time after each test case in the proceeding lines starting with `[RSpecRunTime]`.
+          1. If the test file is large, consider refactoring it into multiple files to allow better test parallelization across runners.
+          1. If the run time cannot be fixed in time, consider quarantine the spec(s) to restore performance.
+
+          ### Run time details
+
+          #{run_time_detail(spec_run_time)}
+          MARKDOWN
+        end
+
+        def update_issue(issue:, spec_run_time:)
+          updated_description = <<~MARKDOWN.chomp
+          #{issue.description}
+
+          #{run_time_detail(spec_run_time)}
+          MARKDOWN
+
+          issue_attrs = {
+            description: updated_description
+          }
+
+          gitlab.edit_issue(iid: issue.iid, options: issue_attrs)
+          puts "  => Added a report in #{issue.web_url}!"
+        end
+
+        def run_time_detail(spec_run_time)
+          <<~MARKDOWN.chomp
+            - Reported from pipeline #{spec_run_time.ci_pipeline_url_markdown} created at `#{spec_run_time.ci_pipeline_created_at}`
+
+            | Field | Value |
+            | ------ | ------ |
+            | Job URL| #{spec_run_time.ci_job_link_markdown} |
+            | Job total RSpec suite run time | expected: `#{readable_duration(spec_run_time.expected_suite_duration)}`, actual: `#{readable_duration(spec_run_time.actual_suite_duration)}` |
+            | Spec file run time | expected: `#{readable_duration(spec_run_time.expected)}`, actual: `#{readable_duration(spec_run_time.actual)}` |
+            | Spec file weight | `#{spec_run_time.actual_percentage}%` of total suite run time |
+          MARKDOWN
+        end
+
+        def assert_input_files!(_files)
+          missing_expected_report_msg = "Missing a valid expected Knapsack report."
+          missing_actual_report_msg = "Missing a valid actual Knapsack report."
+
+          abort missing_expected_report_msg if expected_report_path.nil?
+          abort missing_actual_report_msg if actual_report_path.nil?
+        end
+      end
+    end
+  end
+end

data/lib/gitlab_quality/test_tooling/report/relate_failure_issue.rb

@@ -16,7 +16,7 @@ module GitlabQuality
       # - Find issue by title (with test description or test file), then further filter by stack trace, then pick the better-matching one
       # - Add the failed job to the issue description, and update labels
       class RelateFailureIssue < ReportAsIssue
-        include Concerns::FindSetDri
+        include TestTooling::Concerns::FindSetDri
         include Concerns::GroupAndCategoryLabels
         include Concerns::IssueReports
         include Amatch
@@ -64,8 +64,6 @@ module GitlabQuality
             puts "=> Reporting #{test_results.count} tests in #{test_results.path}"
             process_test_results(test_results)
           end
-
-          write_issues_log_file
         end
 
         def test_metric_collections
@@ -192,8 +190,8 @@ module GitlabQuality
         end
 
         def failure_issues(test)
-          find_issues_for_test(
-            test,
+          find_issues_by_hash(
+            test_hash(test),
             state: 'opened',
             labels: base_issue_labels + Set.new(%w[test]),
             not_labels: exclude_labels_for_search
@@ -270,7 +268,7 @@ module GitlabQuality
           if stacktrace_match
             stacktrace_match[:stacktrace].gsub(/^\s*#.*$/, '').gsub(/^[[:space:]]+/, '').strip
           else
-            puts "  => [DEBUG] Stacktrace doesn't match the regex (#{regex}):\n----------------\n#{stacktrace}\n----------------\n"
+            puts "  => [DEBUG] Stacktrace doesn't match the regex (#{regex})!"
           end
         end
 
@@ -332,7 +330,7 @@ module GitlabQuality
         def new_issue_assignee_id(test)
           return unless test.product_group?
 
-          dri = set_dri_via_group(test.product_group, test)
+          dri = set_dri_via_group(test.product_group, test.stage)
           puts "  => Assigning #{dri} as DRI for the issue."
 
           gitlab.find_user_id(username: dri)
@@ -349,7 +347,7 @@ module GitlabQuality
           state_event = issue.state == 'closed' ? 'reopen' : nil
 
           issue_attrs = {
-            description: add_report_to_issue_description(issue, test),
+            description: increment_reports(current_reports_content: issue.description, test: test),
             labels: up_to_date_labels(test: test, issue: issue)
           }
           issue_attrs[:state_event] = state_event if state_event
@@ -358,10 +356,6 @@ module GitlabQuality
           puts "  => Added a report in '#{issue.title}': #{issue.web_url}!"
         end
 
-        def new_issue_title(test)
-          "Failure in #{super}"
-        end
-
         def screenshot_section(test)
           return unless test.screenshot?
 

data/lib/gitlab_quality/test_tooling/report/report_as_issue.rb

@@ -19,7 +19,11 @@ module GitlabQuality
         def invoke!
           validate_input!
 
-          run!
+          issue_url = run!
+
+          write_issues_log_file
+
+          issue_url
         end
 
         private
@@ -56,7 +60,7 @@ module GitlabQuality
           | Description | `#{test.name}` |
           | Test level | #{test.level} |
           | Hash | `#{test_hash(test)}` |
-          | Duration | #{test.run_time} seconds |
+          | Reference duration | #{test.run_time} seconds |
           | Expected duration | < #{test.max_duration_for_test} seconds |
           #{"| Test case | #{test.testcase} |" if test.testcase}
           DESCRIPTION
@@ -138,8 +142,10 @@ module GitlabQuality
           labels
         end
 
-        def find_issues_by_hash(test_hash)
-          search_options = { search: test_hash }
+        def find_issues_by_hash(test_hash, labels: Set.new, not_labels: Set.new, state: nil)
+          search_options = { search: test_hash, labels: labels.to_a, not: { labels: not_labels.to_a } }
+          search_options[:state] = state if state
+          search_options[:in] = 'description'
           gitlab.find_issues(options: search_options)
         end
 
@@ -162,7 +168,12 @@ module GitlabQuality
         def issue_match_test?(issue, test)
           issue_title = issue.title.strip
           test_file_path_found = !test.file.to_s.empty? && issue_title.include?(partial_file_path(test.file))
-          issue_title.include?(test.name) || test_file_path_found
+
+          if test.name
+            issue_title.include?(test.name) || test_file_path_found
+          else
+            test_file_path_found
+          end
         end
 
         def pipeline_name_label

data/lib/gitlab_quality/test_tooling/report/slow_test_issue.rb

@@ -10,16 +10,21 @@ module GitlabQuality
       # - Find issue by title (with test description or test file)
       # - Add test metadata, duration to the issue with group and category labels
       class SlowTestIssue < ReportAsIssue
-        include Concerns::FindSetDri
         include Concerns::GroupAndCategoryLabels
+        include Concerns::IssueReports
 
-        NEW_ISSUE_LABELS = Set.new(['test', 'type::maintenance', 'maintenance::performance', 'priority::3', 'severity::3', 'rspec profiling', 'rspec:slow test']).freeze
-        SEARCH_LABELS = %w[test maintenance::performance].freeze
+        IDENTITY_LABELS       = ['test', 'rspec:slow test', 'rspec profiling', 'automation:bot-authored'].freeze
+        NEW_ISSUE_LABELS      = Set.new(['test', 'type::maintenance', 'maintenance::performance', 'priority::3', 'severity::3']).freeze
+        SEARCH_LABELS         = ['test'].freeze
+        FOUND_IN_MR_LABEL     = '~"found:in MR"'
+        FOUND_IN_MASTER_LABEL = '~"found:master"'
+        REPORT_SECTION_HEADER = '### Slowness reports'
+        REPORTS_DOCUMENTATION = <<~DOC
+          Slow tests were detected, please see the [test speed best practices guide](https://docs.gitlab.com/ee/development/testing_guide/best_practices.html#test-speed)
+          to improve them. More context available about this issue in the [top slow tests guide](https://docs.gitlab.com/ee/development/testing_guide/best_practices.html#top-slow-tests).
 
-        JOB_URL_REGEX = %r{(?<job_url>https://(?<host>[\w.]+)/(?<project_path>[\w\-./]+)/-/jobs/\d+)}
-        REPORT_ITEM_REGEX = /^1\. \d{4}-\d{2}-\d{2}: #{JOB_URL_REGEX} \((?<pipeline_url>.+)\)$/
-
-        MultipleIssuesFound = Class.new(StandardError)
+          Add `allowed_to_be_slow: true` to the RSpec test if this is a legit slow test and close the issue.
+        DOC
 
         private
 
@@ -29,101 +34,87 @@ module GitlabQuality
           TestResults::Builder.new(files).test_results_per_file do |test_results|
             puts "=> Reporting #{test_results.count} tests in #{test_results.path}"
 
-            test_results.each do |test|
-              create_slow_issue(test) if test.slow_test?
-            end
+            process_test_results(test_results)
           end
-
-          write_issues_log_file
         end
 
-        def new_issue_title(test)
-          "Slow test in #{super}"
-        end
+        def process_test_results(test_results)
+          test_results.each do |test|
+            next unless test.slow_test?
 
-        def new_issue_description(test)
-          super +
-            <<~DESCRIPTION.chomp
-              Slow tests were detected, please see the [test speed best practices guide](https://docs.gitlab.com/ee/development/testing_guide/best_practices.html#test-speed)
-              to improve them. More context available about this issue in the [top slow tests guide](https://docs.gitlab.com/ee/development/testing_guide/best_practices.html#top-slow-tests).
+            puts " => Reporting slowness for test '#{test.name}'..."
 
-              Add `allowed_to_be_slow: true` to the RSpec test if this is a legit slow test and close the issue.
+            issues = find_issues_by_hash(test_hash(test), state: 'opened', labels: SEARCH_LABELS)
+            issues << create_issue(test) if issues.empty?
 
-              #{reports_section(test)}
-            DESCRIPTION
+            update_reports(issues, test)
+            collect_issues(test, issues)
+          end
         end
 
-        def reports_section(test)
-          <<~REPORTS
-          ### Reports (1)
-
-          #{report_list_item(test)}
-          REPORTS
+        def test_is_applicable?(test)
+          test.slow_test?
         end
 
-        def report_list_item(test)
-          "1. #{Time.new.utc.strftime('%F')}: #{test.ci_job_url} (#{ENV.fetch('CI_PIPELINE_URL', 'pipeline url is missing')})"
+        def update_reports(issues, test)
+          issues.each do |issue|
+            puts "   => Adding the slow test to the existing issue: #{issue.web_url}"
+            add_report_to_issue(issue: issue, test: test, related_issues: (issues - [issue]))
+          end
         end
 
-        def slow_test_issues(test)
-          find_issues_for_test(
-            test,
-            state: 'opened',
-            labels: SEARCH_LABELS
-          )
+        def add_report_to_issue(issue:, test:, related_issues:)
+          reports_note = existing_reports_note(issue: issue)
+          note_body = [
+            report_body(reports_note: reports_note, test: test),
+            identity_labels_quick_action,
+            relate_issues_quick_actions(related_issues)
+          ].join("\n")
+
+          if reports_note
+            gitlab.edit_issue_note(
+              issue_iid: issue.iid,
+              note_id: reports_note.id,
+              note: note_body
+            )
+          else
+            gitlab.create_issue_note(iid: issue.iid, note: note_body)
+          end
         end
 
-        def create_slow_issue(test)
-          puts "   => Finding existing issues for slow test '#{test.name}' (run time: #{test.run_time} seconds)..."
+        def existing_reports_note(issue:)
+          gitlab.find_issue_notes(iid: issue.iid).find do |note|
+            note.body.start_with?(REPORT_SECTION_HEADER)
+          end
+        end
 
-          issues = slow_test_issues(test)
+        def report_body(reports_note:, test:)
+          increment_reports(
+            current_reports_content: reports_note&.body.to_s,
+            test: test,
+            reports_section_header: REPORT_SECTION_HEADER,
+            item_extra_content: found_label,
+            reports_extra_content: REPORTS_DOCUMENTATION
+          )
+        end
 
-          if issues.blank?
-            issues << create_issue(test)
+        def found_label
+          if ENV.key?('CI_MERGE_REQUEST_IID')
+            FOUND_IN_MR_LABEL
           else
-            issues.each do |issue|
-              puts "   => Existing issue link #{issue['web_url']}"
-
-              update_reports(issue, test)
-            end
+            FOUND_IN_MASTER_LABEL
           end
-
-          collect_issues(test, issues)
-        rescue MultipleIssuesFound => e
-          warn(e.message)
         end
 
-        def update_reports(issue, test)
-          # We reopen closed issues to not lose any history
-          state_event = issue.state == 'closed' ? 'reopen' : nil
-
-          issue_attrs = {
-            description: up_to_date_issue_description(issue.description, test)
-          }
-
-          issue_attrs[:state_event] = state_event if state_event
-
-          gitlab.edit_issue(iid: issue.iid, options: issue_attrs)
-          puts "  => Added a report in '#{issue.title}': #{issue.web_url}!"
+        def identity_labels_quick_action
+          labels_list = IDENTITY_LABELS.map { |label| %(~"#{label}") }.join(' ')
+          %(/label #{labels_list})
         end
 
-        def up_to_date_issue_description(issue_description, test)
-          new_issue_description =
-            if issue_description.include?('### Reports')
-              # We count the number of existing reports.
-              reports_count = issue_description
-                .scan(REPORT_ITEM_REGEX)
-                .size.to_i + 1
-              issue_description.sub(/^### Reports.*$/, "### Reports (#{reports_count})")
-            else # For issue with the legacy format, we add the Reports section
-              reports_count = issue_description
-                .scan(JOB_URL_REGEX)
-                .size.to_i + 1
-
-              "#{issue_description}\n\n### Reports (#{reports_count})"
-            end
-
-          "#{new_issue_description}\n#{report_list_item(test)}"
+        def relate_issues_quick_actions(issues)
+          issues.map do |issue|
+            "/relate #{issue.web_url}"
+          end.join("\n")
         end
       end
     end

data/lib/gitlab_quality/test_tooling/runtime/env.rb

@@ -25,7 +25,7 @@ module GitlabQuality
 
         ENV_VARIABLES.each do |env_name, method_name|
           define_method(method_name) do
-            env_var_value_if_defined(env_name) || (instance_variable_get("@#{method_name}") if instance_variable_defined?("@#{method_name}"))
+            env_var_value_if_defined(env_name) || (instance_variable_get(:"@#{method_name}") if instance_variable_defined?(:"@#{method_name}"))
           end
         end
 
@@ -33,6 +33,10 @@ module GitlabQuality
           env_var_value_if_defined('QA_LOG_LEVEL')&.upcase || 'INFO'
         end
 
+        def gitlab_bot_username
+          env_var_value_if_defined('GITLAB_BOT_USERNAME') || 'gitlab-bot'
+        end
+
         def log_path
           env_var_value_if_defined('QA_LOG_PATH') || host_artifacts_dir
         end

data/lib/gitlab_quality/test_tooling/slack/post_to_slack_dry.rb

@@ -0,0 +1,14 @@
+# frozen_string_literal: true
+
+module GitlabQuality
+  module TestTooling
+    module Slack
+      class PostToSlackDry < PostToSlack
+        def invoke!
+          puts "The following message would have posted to Slack:"
+          puts message
+        end
+      end
+    end
+  end
+end

data/lib/gitlab_quality/test_tooling/test_meta/processor/add_to_blocking_processor.rb

@@ -0,0 +1,143 @@
+# frozen_string_literal: true
+
+module GitlabQuality
+  module TestTooling
+    module TestMeta
+      module Processor
+        class AddToBlockingProcessor < MetaProcessor
+          BLOCKING_METADATA = ", :blocking%{suffix}"
+
+          class << self
+            # Execute the processor
+            #
+            # @param [Hash] spec the spec to update
+            # @param [TestMetaUpdater] context instance of TestMetaUpdater
+            def execute(spec, context) # rubocop:disable Metrics/AbcSize
+              @context = context
+              @existing_mrs = nil
+              @file_path = spec["file_path"]
+              testcase = spec["testcase"]
+              devops_stage = spec["stage"]
+              product_group = spec["product_group"]
+              @example_name = spec["name"]
+              @mr_title = format("%{prefix} %{example_name}", prefix: '[PROMOTE TO BLOCKING]', example_name: example_name).truncate(72, omission: '')
+
+              @file_contents = context.get_file_contents(file_path)
+
+              new_content, @changed_line_no = add_blocking_metadata
+
+              return unless proceed_with_merge_request?
+
+              branch = context.create_branch("blocking-promotion-#{SecureRandom.hex(4)}", example_name, context.ref)
+
+              context.commit_changes(branch, <<~COMMIT_MESSAGE, file_path, new_content)
+                Promote end-to-end test to blocking
+
+                #{"Promote to blocking: #{example_name}".truncate(72)}
+              COMMIT_MESSAGE
+
+              reviewer_id, assignee_handle = context.fetch_dri_id(product_group, devops_stage)
+
+              gitlab_bot_user_id = context.user_id_for_username(Runtime::Env.gitlab_bot_username)
+
+              merge_request = context.create_merge_request(mr_title, branch, gitlab_bot_user_id, [reviewer_id]) do
+                <<~MARKDOWN
+                  ## What does this MR do?
+
+                  Promotes the test [`#{example_name}`](https://gitlab.com/#{context.project}/-/blob/#{context.ref}/#{file_path}#L#{changed_line_no + 1})
+                  to the blocking bucket
+
+                  This test was identified in the reliable e2e test report: #{context.report_issue}
+
+                  [Testcase link](#{testcase})
+
+                  [Spec metrics link](#{context.single_spec_metrics_link(example_name)})
+
+                  /label ~"Quality" ~"QA" ~"type::maintenance"
+                  /label ~"devops::#{devops_stage}"
+                  #{context.label_from_product_group(product_group)}
+
+                  <div align="center">
+                  (This MR was automatically generated by [`gitlab_quality-test_tooling`](https://gitlab.com/gitlab-org/ruby/gems/gitlab_quality-test_tooling) at #{Time.now.utc})
+                  </div>
+                MARKDOWN
+              end
+
+              context.post_note_on_merge_request(<<~MARKDOWN, merge_request.iid)
+                @#{assignee_handle} Please review this MR, approve and assign it to a maintainer.
+
+                If you think this MR should not be merged, please close it and add a note of the reason to the blocking report: #{context.report_issue}
+              MARKDOWN
+
+              if merge_request
+                context.add_processed_record({ file_path => changed_line_no })
+                Runtime::Logger.info("  Created MR for promotion to blocking: #{merge_request.web_url}")
+              end
+
+              merge_request
+            end
+
+            # Performs post processing. Takes a list of MRs and posts them in a note on report_issue
+            #
+            # @param [Gitlab::ObjectifiedHash] merge_requests
+            def post_process(merge_requests)
+              web_urls = merge_requests.compact.map { |mr| "- #{mr.web_url}\n" }.join
+
+              return if web_urls.empty?
+
+              context.post_note_on_report_issue(<<~ISSUE_NOTE)
+                The following merge requests have been created to promote stable specs to blocking:
+
+                #{web_urls}
+              ISSUE_NOTE
+            end
+
+            private
+
+            attr_reader :context, :file_path, :file_contents, :example_name, :mr_title, :changed_line_no
+
+            # Checks if there is already an MR open
+            #
+            # @return [Boolean]
+            def proceed_with_merge_request? # rubocop:disable Metrics/AbcSize
+              if changed_line_no.negative?
+                Runtime::Logger.info("  No lines were changed in #{file_path}. Will not proceed with creating MR.")
+                return false
+              elsif context.record_processed?(file_path, changed_line_no)
+                Runtime::Logger.info("  Record already processed for #{file_path}:#{changed_line_no}. Will not proceed with creating MR.")
+                return false
+              elsif existing_mrs&.any?
+                Runtime::Logger.info("  An open MR already exists for '#{example_name}':  #{existing_mrs.first['web_url']}. Will not proceed with creating MR.")
+                return false
+              end
+
+              true
+            end
+
+            # Add blocking metadata to the file content and replace it
+            #
+            # @return [Array<String, Integer>] first value holds the new content, the second value holds the line number of the test
+            def add_blocking_metadata # rubocop:disable Metrics/AbcSize
+              matched_lines = context.find_example_match_lines(file_contents, example_name)
+
+              if matched_lines.any? { |line| line[0].include?(':blocking') }
+                puts "Example '#{example_name}' is already blocking"
+                return [file_contents, -1]
+              end
+
+              context.update_matched_line(matched_lines.last, file_contents.dup) do |line|
+                if line.sub(DESCRIPTION_REGEX, '').include?(',')
+                  line[line.index(',', end_of_description_index(line))] = format(BLOCKING_METADATA, suffix: ',')
+                else
+                  line[line.rindex(' ')] = format(BLOCKING_METADATA, suffix: ' ')
+                end
+
+                line
+              end
+            end
+          end
+        end
+      end
+    end
+  end
+end