jirametrics 2.31 → 3.1

data/lib/jirametrics/project_config.rb

@@ -55,22 +55,62 @@ class ProjectConfig
 
     anonymize_data if @anonymizer_needed
 
-    @file_configs.each do |file_config|
-      file_config.run
+    @file_configs.each(&:run)
+
+    # By this point the charts have walked the issue histories, so every deleted status has already
+    # been fabricated lazily. Just report them. (The mcp path has no chart phase, so it resolves them
+    # explicitly before reporting.)
+    report_fabricated_statuses
+  end
+
+  # Walk every issue's status history so that any status that no longer exists in Jira gets fabricated
+  # now. Fabrication is otherwise lazy (it happens the first time a status is touched, which for some
+  # code paths is mid-render), so this gives us a single deterministic point where we know they all
+  # exist before we report them.
+  def resolve_all_status_changes
+    issues.each do |issue|
+      issue.status_changes.each do |change|
+        issue.find_or_create_status id: change.value_id, name: change.value
+      end
     end
   end
 
+  # Print one consolidated warning covering every status we had to fabricate (a status that an issue
+  # passed through but that no longer exists in Jira). It fires often -- any workflow rework leaves
+  # these behind -- so we say the explanation once and hand over a single ready-to-paste block of
+  # mappings rather than one repetitive paragraph per status.
+  def report_fabricated_statuses
+    statuses = possible_statuses.fabricated_statuses
+    return if statuses.empty? || @fabricated_statuses_reported
+
+    @fabricated_statuses_reported = true
+
+    width = statuses.map { |status| "'#{status.name}:#{status.id}'".length }.max
+    mappings = statuses.map do |status|
+      "      #{"'#{status.name}:#{status.id}'".ljust(width)} => '#{status.category.name}',"
+    end.join("\n")
+
+    example_key = possible_statuses.example_issue_key
+    info_line =
+      if example_key
+        "\nRun `jirametrics info #{example_key}` on an affected issue to see how it moved through its workflow."
+      else
+        ''
+      end
+
+    file_system.warning(
+      'Some statuses in your issue history no longer exist in Jira (they were probably deleted). ' \
+      "We've guessed each one's category from its name. If a guess is right the mapping just confirms " \
+      'it and silences this warning; if it is wrong, change the category. Add these to your ' \
+      "config's status category mappings; with standard_project that's:\n" \
+      "    status_category_mappings: {\n#{mappings}\n    }\n#{info_line}"
+    )
+  end
+
   def load_settings
     # This is the weird exception that we don't ever want mocked out so we skip FileSystem entirely.
     settings = JSON.parse(File.read(File.join(__dir__, 'settings.json'), encoding: 'UTF-8'))
 
-    if settings['blocked_color']
-      file_system.deprecated message: 'blocked color should be set via css now', date: '2024-05-03'
-    end
-    if settings['stalled_color']
-      file_system.deprecated message: 'stalled color should be set via css now', date: '2024-05-03'
-    end
-
     settings['blocked_statuses'] = StatusCollection.new
     settings['stalled_statuses'] = StatusCollection.new
 
@@ -208,25 +248,45 @@ class ProjectConfig
     category, category_id = possible_statuses.parse_name_id category
 
     if status_id.nil?
-      guesses = find_ids_by_status_name_across_all_issues status
-      if guesses.empty?
-        file_system.warning "For status_category_mapping status: #{status.inspect}, category: #{category.inspect}\n" \
-          "Cannot guess status id for #{status.inspect} as no statuses found anywhere in the issues " \
-          "histories with that name. Since we can't find it, you probably don't need this mapping anymore so we're " \
-          "going to ignore it. If you really want it, then you'll need to specify a status id."
-        return
-      end
+      status_id = guess_status_id_for status, category
+      return if status_id.nil? # no id could be guessed; the mapping is ignored (already warned)
+    end
 
-      if guesses.size > 1
-        raise "Cannot guess status id as there are multiple ids for the name #{status.inspect}. Perhaps it's one " \
-          "of #{guesses.to_a.sort.inspect}. If you need this mapping then you must specify the status_id."
-      end
+    found_category = resolve_category category, category_id
+    add_possible_status(
+      Status.new(
+        name: status, id: status_id,
+        category_name: category, category_id: found_category.id, category_key: found_category.key
+      )
+    )
+  end
+
+  # When a status is named but not given an id, guess it from the ids seen under that name across all
+  # issue histories. Returns nil (after warning) when none is found, or raises when it's ambiguous.
+  def guess_status_id_for status, category
+    guesses = find_ids_by_status_name_across_all_issues status
+    if guesses.empty?
+      file_system.warning "For status_category_mapping status: #{status.inspect}, category: #{category.inspect}\n" \
+        "Cannot guess status id for #{status.inspect} as no statuses found anywhere in the issues " \
+        "histories with that name. Since we can't find it, you probably don't need this mapping anymore so we're " \
+        "going to ignore it. If you really want it, then you'll need to specify a status id."
+      return nil
+    end
 
-      status_id = guesses.first
-      file_system.log "status_category_mapping for #{status.inspect} has been mapped to id #{status_id}. " \
-        "If that's incorrect then specify the status_id."
+    if guesses.size > 1
+      raise "Cannot guess status id as there are multiple ids for the name #{status.inspect}. Perhaps it's one " \
+        "of #{guesses.to_a.sort.inspect}. If you need this mapping then you must specify the status_id."
     end
 
+    status_id = guesses.first
+    file_system.log "status_category_mapping for #{status.inspect} has been mapped to id #{status_id}. " \
+      "If that's incorrect then specify the status_id."
+    status_id
+  end
+
+  # Find the single status category with the given name, raising if there are none, several, or a
+  # supplied id that disagrees with the one we found.
+  def resolve_category category, category_id
     possible_categories = possible_statuses.find_all_categories_by_name category
     if possible_categories.empty?
       all = possible_statuses.find_all_categories.join(', ')
@@ -239,17 +299,11 @@ class ProjectConfig
     end
 
     found_category = possible_categories.first
-
     if category_id && category_id != found_category.id
       raise "ID is incorrect for status category #{category.inspect}. Did you mean #{found_category.id}?"
     end
 
-    add_possible_status(
-      Status.new(
-        name: status, id: status_id,
-        category_name: category, category_id: found_category.id, category_key: found_category.key
-      )
-    )
+    found_category
   end
 
   def add_possible_status status
@@ -283,9 +337,10 @@ class ProjectConfig
 
   def load_all_boards
     Dir.foreach(@target_path) do |file|
-      next unless file =~ /^#{get_file_prefix}_board_(\d+)_configuration\.json$/
+      match = file.match(/^#{get_file_prefix}_board_(?<board_id>\d+)_configuration\.json$/)
+      next unless match
 
-      board_id = $1.to_i
+      board_id = match[:board_id].to_i
       load_board board_id: board_id, filename: "#{@target_path}#{file}"
     end
   end
@@ -326,16 +381,24 @@ class ProjectConfig
       .each { |status| possible_statuses.historical_status_mappings[status.to_s] = status.category }
 
     possible_statuses
+  # This is an optional enrichment file. A malformed one surfaces as anything from a JSON::ParserError to
+  # a TypeError/NoMethodError out of Status.from_raw, so we deliberately catch broadly, warn, and carry on
+  # without it rather than fail the whole export. The exception itself goes to the log file only (via
+  # `more`) so we don't lose it, while the console stays uncluttered.
   rescue => e # rubocop:disable Style/RescueStandardError
-    file_system.warning "Unable to load status history due to #{e.message.inspect}. If this is because of a " \
-      'malformed file then it should be fixed on the next download.'
+    file_system.warning(
+      'Unable to load status history. If this is because of a malformed file then it should be ' \
+      'fixed on the next download.',
+      more: [e.message, *e.backtrace].join("\n")
+    )
   end
 
   def load_sprints
     file_system.foreach(@target_path) do |file|
-      next unless file =~ /^#{get_file_prefix}_board_(\d+)_sprints_\d+.json$/
+      match = file.match(/^#{get_file_prefix}_board_(?<board_id>\d+)_sprints_\d+.json$/)
+      next unless match
 
-      board_id = $1.to_i
+      board_id = match[:board_id].to_i
       file_path = File.join(@target_path, file)
       board = @all_boards[board_id]
       unless board
@@ -373,24 +436,25 @@ class ProjectConfig
     start = to_time(json['date_start'] || json['time_start']) # date_start is the current format. Time is the old.
     stop  = to_time(json['date_end'] || json['time_end'], end_of_day: true)
 
-    # If no_earlier_than was set then make sure it's applied here.
-    if download_config
-      download_config.run
-      no_earlier = download_config.no_earlier_than
-      if no_earlier
-        no_earlier = to_time(no_earlier.to_s)
-        start = no_earlier if start < no_earlier
-      end
-    end
-
-    @time_range = start..stop
-
+    @time_range = clamp_to_no_earlier_than(start)..stop
     @jira_url = json['jira_url']
   rescue Errno::ENOENT
     file_system.log "Can't load #{filename}. Have you done a download?", also_write_to_stderr: true
     raise
   end
 
+  # If the download was configured with a no_earlier_than, the data can't start before it.
+  def clamp_to_no_earlier_than start
+    return start unless download_config
+
+    download_config.run
+    no_earlier = download_config.no_earlier_than
+    return start unless no_earlier
+
+    no_earlier = to_time(no_earlier.to_s)
+    [start, no_earlier].max
+  end
+
   def load_users
     @users = []
     filename = File.join @target_path, "#{get_file_prefix}_users.json"
@@ -474,39 +538,46 @@ class ProjectConfig
 
       load_data if all_boards.empty?
 
-      timezone_offset = exporter.timezone_offset
-
       issues_path = File.join @target_path, "#{get_file_prefix}_issues"
-      if File.exist?(issues_path) && File.directory?(issues_path)
-        file_system.diagnostic "Loading issues from #{issues_path}"
-        issues = load_issues_from_issues_directory path: issues_path, timezone_offset: timezone_offset
-        file_system.diagnostic "Loaded #{issues.size} issues from disk"
-      else
+      # File.directory? is already false for a path that doesn't exist, so no need to also check File.exist?.
+      unless File.directory?(issues_path)
         file_system.log "Can't find directory #{issues_path}. Has a download been done?", also_write_to_stderr: true
         return IssueCollection.new
       end
 
-      # Attach related issues
-      file_system.diagnostic 'Starting attach phase'
-      issues_by_key = issues.to_h { |i| [i.key, i] }
-      issues.each do |i|
-        attach_subtasks issue: i, issues_by_key: issues_by_key
-        attach_parent issue: i, issues_by_key: issues_by_key
-        attach_linked_issues issue: i, issues_by_key: issues_by_key
-      end
-      file_system.diagnostic 'Attach phase complete'
-
-      # We'll have some issues that are in the list that weren't part of the initial query. Once we've
-      # attached them in the appropriate places, remove any that aren't part of that initial set.
-      issues.reject! { |i| !i.in_initial_query? } # rubocop:disable Style/InverseMethods
-      file_system.diagnostic "Retained #{issues.size} primary issues"
-      @issues = issues
+      @issues = build_issues_from_directory(issues_path)
       attach_github_prs
     end
 
     @issues
   end
 
+  def build_issues_from_directory issues_path
+    file_system.diagnostic "Loading issues from #{issues_path}"
+    issues = load_issues_from_issues_directory path: issues_path, timezone_offset: exporter.timezone_offset
+    file_system.diagnostic "Loaded #{issues.size} issues from disk"
+
+    attach_related_issues issues
+
+    # We'll have some issues that are in the list that weren't part of the initial query. Once we've
+    # attached them in the appropriate places, remove any that aren't part of that initial set.
+    issues.select!(&:in_initial_query?)
+    file_system.diagnostic "Retained #{issues.size} primary issues"
+    issues
+  end
+
+  # Wire up subtasks, parents and linked issues now that we have the whole set loaded.
+  def attach_related_issues issues
+    file_system.diagnostic 'Starting attach phase'
+    issues_by_key = issues.to_h { |issue| [issue.key, issue] }
+    issues.each do |issue|
+      attach_subtasks issue: issue, issues_by_key: issues_by_key
+      attach_parent issue: issue, issues_by_key: issues_by_key
+      attach_linked_issues issue: issue, issues_by_key: issues_by_key
+    end
+    file_system.diagnostic 'Attach phase complete'
+  end
+
   def attach_subtasks issue:, issues_by_key:
     issue.raw['fields']['subtasks']&.each do |subtask_element|
       subtask = issues_by_key[subtask_element['key']]
@@ -567,6 +638,10 @@ class ProjectConfig
   # It's ok if there are multiple files for the same issue. We load the newest one and map all the other
   # board ids appropriately.
   def group_filenames_and_board_ids path:
+    group_files_by_issue_key(path).values.to_h { |list| resolve_key_files(list, path) }
+  end
+
+  def group_files_by_issue_key path
     hash = {}
     file_system.foreach(path) do |filename|
       # Matches either FAKE-123.json or FAKE-123-456.json
@@ -574,32 +649,21 @@ class ProjectConfig
         (hash[key] ||= []) << [filename, board_id&.to_i || :unknown]
       end
     end
-
-    result = {}
-    hash.values.collect do |list|
-      if list.size == 1
-        filename, board_id = *list.first
-        result[filename] = board_id == :unknown ? board_id : [board_id]
-      else
-        max_time = nil
-        max_board_id = nil
-        max_filename = nil
-        all_board_ids = []
-
-        list.each do |filename, board_id|
-          mtime = File.mtime(File.join(path, filename))
-          if max_time.nil? || mtime > max_time
-            max_time = mtime
-            max_board_id = board_id
-            max_filename = filename
-          end
-          all_board_ids << board_id unless board_id == :unknown
-        end
-
-        result[max_filename] = all_board_ids
-      end
+    hash
+  end
+
+  # Given all the files that share an issue key, returns the [filename, board_ids] pair for the result.
+  # A lone file keeps its own board id; when the same issue was exported for several boards we keep the
+  # newest file and associate every known board id with it.
+  def resolve_key_files list, path
+    if list.size == 1
+      filename, board_id = *list.first
+      [filename, board_id == :unknown ? board_id : [board_id]]
+    else
+      newest_filename, = list.max_by { |filename, _| File.mtime(File.join(path, filename)) }
+      board_ids = list.filter_map { |_, board_id| board_id unless board_id == :unknown }
+      [newest_filename, board_ids]
     end
-    result
   end
 
   def anonymize
@@ -615,55 +679,64 @@ class ProjectConfig
   end
 
   def discard_changes_before status_becomes: nil, &block
-    cycletimes_touched = Set.new
-    if status_becomes
-      status_becomes = [status_becomes] unless status_becomes.is_a? Array
-
-      status_becomes.each { |status_name| validate_discard_status status_name }
+    block = discard_block_for(status_becomes) if status_becomes
+    apply_discard_block block
+  end
 
-      block = lambda do |issue|
-        trigger_statuses = status_becomes.collect do |status_name|
-          if status_name == :backlog
-            issue.board.backlog_statuses
-          else
-            possible_statuses.find_all_by_name status_name
-          end
-        end.flatten
+  # Build the block that, for an issue, returns the time it most recently entered one of the
+  # status_becomes statuses (or nil).
+  def discard_block_for status_becomes
+    status_becomes = [status_becomes] unless status_becomes.is_a? Array
+    status_becomes.each { |status_name| validate_discard_status status_name }
+    ->(issue) { last_matching_status_time issue, status_becomes }
+  end
 
-        next if trigger_statuses.empty?
+  def last_matching_status_time issue, status_becomes
+    trigger_status_ids = trigger_status_ids_for issue, status_becomes
+    return if trigger_status_ids.empty?
 
-        trigger_status_ids = trigger_statuses.collect(&:id)
+    time = nil
+    issue.status_changes.each do |change|
+      time = change.time if trigger_status_ids.include?(change.value_id) # && change.artificial? == false
+    end
+    time
+  end
 
-        time = nil
-        issue.status_changes.each do |change|
-          time = change.time if trigger_status_ids.include?(change.value_id) # && change.artificial? == false
-        end
-        time
+  def trigger_status_ids_for issue, status_becomes
+    status_becomes.collect do |status_name|
+      if status_name == :backlog
+        issue.board.backlog_statuses
+      else
+        possible_statuses.find_all_by_name status_name
       end
-    end
+    end.flatten.collect(&:id)
+  end
 
+  def apply_discard_block block
+    cycletimes_touched = Set.new
     file_system.diagnostic "discard_changes_before: processing #{issues.size} issues"
-    issues.each do |issue|
-      cutoff_time = block.call(issue)
-      next if cutoff_time.nil?
+    issues.each { |issue| discard_changes_for_issue issue, block, cycletimes_touched }
+    cycletimes_touched.each(&:flush_cache)
+  end
 
-      original_start_time = issue.started_stopped_times.first
-      next if original_start_time.nil?
+  def discard_changes_for_issue issue, block, cycletimes_touched
+    cutoff_time = block.call(issue)
+    return if cutoff_time.nil?
 
-      issue.discard_changes_before cutoff_time
-      cycletimes_touched << issue.board.cycletime
+    original_start_time = issue.started_stopped_times.first
+    return if original_start_time.nil?
 
-      next unless cutoff_time
-      next if original_start_time > cutoff_time # ie the cutoff would have made no difference.
+    issue.discard_changes_before cutoff_time
+    cycletimes_touched << issue.board.cycletime
 
-      (@discarded_changes_data ||= []) << {
-        cutoff_time: cutoff_time,
-        original_start_time: original_start_time,
-        issue: issue
-      }
-    end
+    return unless cutoff_time # a user-supplied block may return false rather than nil
+    return if original_start_time > cutoff_time # ie the cutoff would have made no difference.
 
-    cycletimes_touched.each { |c| c.flush_cache }
+    (@discarded_changes_data ||= []) << {
+      cutoff_time: cutoff_time,
+      original_start_time: original_start_time,
+      issue: issue
+    }
   end
 
   def stringify_keys value

data/lib/jirametrics/pull_request_cycle_time_histogram.rb

@@ -6,7 +6,6 @@ class PullRequestCycleTimeHistogram < TimeBasedHistogram
   def initialize block
     super()
 
-    @cycletime_unit = :days
     @x_axis_title = 'Cycle time in days'
 
     header_text 'PR Histogram'
@@ -25,15 +24,6 @@ class PullRequestCycleTimeHistogram < TimeBasedHistogram
     end
   end
 
-  def cycletime_unit unit
-    unless %i[minutes hours days].include?(unit)
-      raise ArgumentError, "cycletime_unit must be :minutes, :hours, or :days, got #{unit.inspect}"
-    end
-
-    @cycletime_unit = unit
-    @x_axis_title = "Cycle time in #{unit}"
-  end
-
   def all_items
     result = []
     issues.each do |issue|
@@ -49,16 +39,7 @@ class PullRequestCycleTimeHistogram < TimeBasedHistogram
   end
 
   def value_for_item item
-    divisor = { minutes: 60.0, hours: 3600.0, days: 86_400.0 }[@cycletime_unit]
-    ((item.closed_at - item.opened_at) / divisor).ceil
-  end
-
-  def label_cycletime value
-    case @cycletime_unit
-    when :minutes then label_minutes(value)
-    when :hours then label_hours(value)
-    when :days then label_days(value)
-    end
+    duration_in_unit item.opened_at, item.closed_at
   end
 
   def title_for_item count:, value:

data/lib/jirametrics/pull_request_cycle_time_scatterplot.rb

@@ -6,7 +6,6 @@ class PullRequestCycleTimeScatterplot < TimeBasedScatterplot
   def initialize block
     super()
 
-    @cycletime_unit = :days
     @y_axis_title = 'Cycle time in days'
 
     header_text 'Pull Request (PR) Scatterplot'
@@ -24,15 +23,6 @@ class PullRequestCycleTimeScatterplot < TimeBasedScatterplot
     end
   end
 
-  def cycletime_unit unit
-    unless %i[minutes hours days].include?(unit)
-      raise ArgumentError, "cycletime_unit must be :minutes, :hours, or :days, got #{unit.inspect}"
-    end
-
-    @cycletime_unit = unit
-    @y_axis_title = "Cycle time in #{unit}"
-  end
-
   def all_items
     result = []
     issues.each do |issue|
@@ -48,23 +38,7 @@ class PullRequestCycleTimeScatterplot < TimeBasedScatterplot
   end
 
   def y_value pull_request
-    if @cycletime_unit == :days
-      tz = timezone_offset || '+00:00'
-      opened = pull_request.opened_at.getlocal(tz).to_date
-      closed = pull_request.closed_at.getlocal(tz).to_date
-      (closed - opened).to_i + 1
-    else
-      divisor = { minutes: 60, hours: 3600 }[@cycletime_unit]
-      ((pull_request.closed_at - pull_request.opened_at) / divisor).round
-    end
-  end
-
-  def label_cycletime value
-    case @cycletime_unit
-    when :minutes then label_minutes(value)
-    when :hours then label_hours(value)
-    when :days then label_days(value)
-    end
+    duration_in_unit pull_request.opened_at, pull_request.closed_at
   end
 
   def title_value pull_request, rules: nil
@@ -76,13 +50,17 @@ class PullRequestCycleTimeScatterplot < TimeBasedScatterplot
   def lines_changed_text pull_request
     return '' unless pull_request.changed_files
 
+    summary = additions_deletions_summary pull_request
+    " | Lines changed: [#{summary}], Files changed: #{to_human_readable pull_request.changed_files}"
+  end
+
+  # The "+10 -4" bit: each side only appears when non-zero, joined by a space only when both do.
+  def additions_deletions_summary pull_request
     additions = pull_request.additions || 0
     deletions = pull_request.deletions || 0
-    text = +' | Lines changed: ['
-    text << "+#{to_human_readable additions}" unless additions.zero?
-    text << ' ' if additions != 0 && deletions != 0
-    text << "-#{to_human_readable deletions}" unless deletions.zero?
-    text << "], Files changed: #{to_human_readable pull_request.changed_files}"
-    text
+    parts = []
+    parts << "+#{to_human_readable additions}" unless additions.zero?
+    parts << "-#{to_human_readable deletions}" unless deletions.zero?
+    parts.join(' ')
   end
 end

data/lib/jirametrics/rules.rb

@@ -1,6 +1,7 @@
 # frozen_string_literal: true
 
 class Rules
+  # Boolean positional is the config-block DSL's public API (ignore false).
   def ignore value = true # rubocop:disable Style/OptionalBooleanParameter
     @ignore = value
   end

data/lib/jirametrics/self_or_issue_dispatcher.rb

@@ -1,15 +1,13 @@
 # frozen_string_literal: true
 
 module SelfOrIssueDispatcher
-  # rubocop:disable Style/ArgumentsForwarding
   def method_missing method_name, *args, &block
     raise "#{method_name} isn't a method on Issue or #{self.class}" unless ::Issue.method_defined? method_name.to_sym
 
-    ->(issue) do # rubocop:disable Style/Lambda
+    lambda do |issue|
       issue.__send__ method_name, *args, &block
     end
   end
-  # rubocop:enable Style/ArgumentsForwarding
 
   def respond_to_missing?(method_name, include_all = false)
     ::Issue.method_defined?(method_name.to_sym) || super