jirametrics 2.8 → 2.11

data/lib/jirametrics/chart_base.rb

@@ -27,9 +27,6 @@ class ChartBase
 
   def html_directory
     pathname = Pathname.new(File.realpath(__FILE__))
-    # basename = pathname.basename.to_s
-    # raise "Unexpected filename #{basename.inspect}" unless basename.match?(/^(.+)\.rb$/)
-
     "#{pathname.dirname}/html"
   end
 
@@ -133,6 +130,21 @@ class ChartBase
     result
   end
 
+  def working_days_annotation
+    holidays.each_with_index.collect do |range, index|
+      <<~TEXT
+        holiday#{index}: {
+          drawTime: 'beforeDraw',
+          type: 'box',
+          xMin: '#{range.begin}T00:00:00',
+          xMax: '#{range.end}T23:59:59',
+          backgroundColor: #{CssVariable.new('--non-working-days-color').to_json},
+          borderColor: #{CssVariable.new('--non-working-days-color').to_json}
+        },
+      TEXT
+    end.join
+  end
+
   # Return only the board columns for the current board.
   def current_board
     if @board_id.nil?
@@ -179,18 +191,33 @@ class ChartBase
     @description_text
   end
 
+  # Convert a number like 1234567 into the string "1,234,567"
   def format_integer number
     number.to_s.reverse.scan(/.{1,3}/).join(',').reverse
   end
 
-  def format_status name_or_id, board:, is_category: false
-    begin
-      statuses = board.possible_statuses.expand_statuses([name_or_id])
-    rescue StatusNotFoundError
-      return "<span style='color: red'>#{name_or_id}</span>"
+  # object will be either a Status or a ChangeItem
+  # if it's a ChangeItem then use_old_status will specify whether we're using the new or old
+  # Either way, is_category will format the category rather than the status
+  def format_status object, board:, is_category: false, use_old_status: false
+    status = nil
+    error_message = nil
+
+    case object
+    when ChangeItem
+      id = use_old_status ? object.old_value_id : object.value_id
+      status = board.possible_statuses.find_by_id(id)
+      if status.nil?
+        error_message = use_old_status ? object.old_value : object.value
+      end
+    when Status
+      status = object
+    else
+      raise "Unexpected type: #{object.class}"
     end
 
-    status = statuses.first
+    return "<span style='color: red'>#{error_message}</span>" if error_message
+
     color = status_category_color status
 
     visibility = ''
@@ -233,7 +260,10 @@ class ChartBase
 
   def color_block color, title: nil
     result = +''
-    result << "<div class='color_block' style='background: var(#{color});'"
+    result << "<div class='color_block' style='"
+    result << "background: #{CssVariable[color]};" if color
+    result << 'visibility: hidden;' unless color
+    result << "'"
     result << " title=#{title.inspect}" if title
     result << '></div>'
     result

data/lib/jirametrics/css_variable.rb

@@ -4,7 +4,7 @@ class CssVariable
   attr_reader :name
 
   def self.[](name)
-    if name.start_with? '--'
+    if name.is_a?(String) && name.start_with?('--')
       CssVariable.new name
     else
       name

data/lib/jirametrics/cycletime_config.rb

@@ -8,10 +8,14 @@ class CycleTimeConfig
 
   attr_reader :label, :parent_config
 
-  def initialize parent_config:, label:, block:, today: Date.today
+  def initialize parent_config:, label:, block:, file_system: nil, today: Date.today
     @parent_config = parent_config
     @label = label
     @today = today
+
+    # If we hit something deprecated and this is nil then we'll blow up. Although it's ugly, this
+    # may make it easier to find problems in the test code ;-)
+    @file_system = file_system
     instance_eval(&block) unless block.nil?
   end
 
@@ -35,17 +39,19 @@ class CycleTimeConfig
   end
 
   def started_time issue
-    deprecated date: '2024-10-16', message: 'Use started_stopped_times() instead'
+    @file_system.deprecated date: '2024-10-16', message: 'Use started_stopped_times() instead'
     started_stopped_times(issue).first
   end
 
   def stopped_time issue
-    deprecated date: '2024-10-16', message: 'Use started_stopped_times() instead'
+    @file_system.deprecated date: '2024-10-16', message: 'Use started_stopped_times() instead'
     started_stopped_times(issue).last
   end
 
   def fabricate_change_item time
-    deprecated date: '2024-12-16', message: 'This method should now return a ChangeItem not a Time', depth: 4
+    @file_system.deprecated(
+      date: '2024-12-16', message: "This method should now return a ChangeItem not a #{time.class}", depth: 4
+    )
     raw = {
       'field' => 'Fabricated change',
       'to' => '0',

data/lib/jirametrics/cycletime_histogram.rb

@@ -5,10 +5,14 @@ require 'jirametrics/groupable_issue_chart'
 class CycletimeHistogram < ChartBase
   include GroupableIssueChart
   attr_accessor :possible_statuses
+  attr_reader :show_stats
 
   def initialize block
     super()
 
+    percentiles [50, 85, 98]
+    @show_stats = true
+
     header_text 'Cycletime Histogram'
     description_text <<-HTML
       <p>
@@ -26,6 +30,15 @@ class CycletimeHistogram < ChartBase
     end
   end
 
+  def percentiles percs = nil
+    @percentiles = percs unless percs.nil?
+    @percentiles
+  end
+
+  def disable_stats
+    @show_stats = false
+  end
+
   def run
     stopped_issues = completed_issues_in_range include_unstarted: true
 
@@ -33,10 +46,18 @@ class CycletimeHistogram < ChartBase
     histogram_issues = stopped_issues.select { |issue| issue.board.cycletime.started_stopped_times(issue).first }
     rules_to_issues = group_issues histogram_issues
 
+    the_stats = {}
+
+    overall_stats = stats_for histogram_data: histogram_data_for(issues: histogram_issues), percentiles: @percentiles
+    the_stats[:all] = overall_stats
     data_sets = rules_to_issues.keys.collect do |rules|
+      the_issue_type = rules.label
+      the_histogram = histogram_data_for(issues: rules_to_issues[rules])
+      the_stats[the_issue_type] = stats_for histogram_data: the_histogram, percentiles: @percentiles if @show_stats
+
       data_set_for(
-        histogram_data: histogram_data_for(issues: rules_to_issues[rules]),
-        label: rules.label,
+        histogram_data: the_histogram,
+        label: the_issue_type,
         color: rules.color
       )
     end
@@ -55,6 +76,48 @@ class CycletimeHistogram < ChartBase
     count_hash
   end
 
+  def stats_for histogram_data:, percentiles:
+    return {} if histogram_data.empty?
+
+    total_values = histogram_data.values.sum
+
+    # Calculate the average
+    weighted_sum = histogram_data.reduce(0) { |sum, (value, frequency)| sum + (value * frequency) }
+    average = total_values.zero? ? 0 : weighted_sum.to_f / total_values
+
+    # Find the mode (or modes!) and the spread of the distribution
+    sorted_histogram = histogram_data.sort_by { |_value, frequency| frequency }
+    max_freq = sorted_histogram[-1][1]
+    mode = sorted_histogram.select { |_v, f| f == max_freq }
+
+    minmax = histogram_data.keys.minmax
+
+    # Calculate percentiles
+    sorted_values = histogram_data.keys.sort
+    cumulative_counts = {}
+    cumulative_sum = 0
+
+    sorted_values.each do |value|
+      cumulative_sum += histogram_data[value]
+      cumulative_counts[value] = cumulative_sum
+    end
+
+    percentile_results = {}
+    percentiles.each do |percentile|
+      rank = (percentile / 100.0) * total_values
+      percentile_value = sorted_values.find { |value| cumulative_counts[value] >= rank }
+      percentile_results[percentile] = percentile_value
+    end
+
+    {
+      average: average,
+      mode: mode.collect(&:first).sort,
+      min: minmax[0],
+      max: minmax[1],
+      percentiles: percentile_results
+    }
+  end
+
   def data_set_for histogram_data:, label:, color:
     keys = histogram_data.keys.sort
     {

data/lib/jirametrics/data_quality_report.rb

@@ -1,7 +1,7 @@
 # frozen_string_literal: true
 
 class DataQualityReport < ChartBase
-  attr_reader :original_issue_times # For testing purposes only
+  attr_reader :discarded_changes_data, :entries # Both for testing purposes only
   attr_accessor :board_id
 
   class Entry
@@ -19,10 +19,10 @@ class DataQualityReport < ChartBase
     end
   end
 
-  def initialize original_issue_times
+  def initialize discarded_changes_data
     super()
 
-    @original_issue_times = original_issue_times
+    @discarded_changes_data = discarded_changes_data
 
     header_text 'Data Quality Report'
     description_text <<-HTML
@@ -50,6 +50,7 @@ class DataQualityReport < ChartBase
       scan_for_issues_not_started_with_subtasks_that_have entry: entry
       scan_for_incomplete_subtasks_when_issue_done entry: entry
       scan_for_discarded_data entry: entry
+      scan_for_items_blocked_on_closed_tickets entry: entry
     end
 
     scan_for_issues_on_multiple_boards entries: @entries
@@ -73,6 +74,7 @@ class DataQualityReport < ChartBase
     result << render_problem_type(:issue_not_started_but_subtasks_have)
     result << render_problem_type(:incomplete_subtasks_when_issue_done)
     result << render_problem_type(:issue_on_multiple_boards)
+    result << render_problem_type(:items_blocked_on_closed_tickets)
     result << '</ul>'
 
     result
@@ -102,9 +104,13 @@ class DataQualityReport < ChartBase
 
   # Return a format that's easier to assert against
   def testable_entries
-    format = '%Y-%m-%d %H:%M:%S %z'
+    formatter = ->(time) { time&.strftime('%Y-%m-%d %H:%M:%S %z') || '' }
     @entries.collect do |entry|
-      [entry.started&.strftime(format) || '', entry.stopped&.strftime(format) || '', entry.issue]
+      [
+        formatter.call(entry.started),
+        formatter.call(entry.stopped),
+        entry.issue
+      ]
     end
   end
 
@@ -112,10 +118,6 @@ class DataQualityReport < ChartBase
     @entries.reject { |entry| entry.problems.empty? }
   end
 
-  def category_name_for status_id:, board:
-    board.possible_statuses.find_by_id(status_id)&.category&.name
-  end
-
   def initialize_entries
     @entries = @issues.filter_map do |issue|
       started, stopped = issue.board.cycletime.started_stopped_times(issue)
@@ -139,10 +141,8 @@ class DataQualityReport < ChartBase
   def scan_for_completed_issues_without_a_start_time entry:
     return unless entry.stopped && entry.started.nil?
 
-    status_names = entry.issue.changes.filter_map do |change|
-      next unless change.status?
-
-      format_status change.value, board: entry.issue.board
+    status_names = entry.issue.status_changes.filter_map do |change|
+      format_status change, board: entry.issue.board
     end
 
     entry.report(
@@ -157,14 +157,14 @@ class DataQualityReport < ChartBase
     changes_after_done = entry.issue.changes.select do |change|
       change.status? && change.time >= entry.stopped
     end
-    done_status = changes_after_done.shift.value
+    done_status = changes_after_done.shift
 
     return if changes_after_done.empty?
 
     board = entry.issue.board
     problem = "Completed on #{entry.stopped.to_date} with status #{format_status done_status, board: board}."
     changes_after_done.each do |change|
-      problem << " Changed to #{format_status change.value, board: board} on #{change.time.to_date}."
+      problem << " Changed to #{format_status change, board: board} on #{change.time.to_date}."
     end
     entry.report(
       problem_key: :status_changes_after_done,
@@ -186,9 +186,9 @@ class DataQualityReport < ChartBase
         # If it's a backlog status then ignore it. Not supposed to be visible.
         next if entry.issue.board.backlog_statuses.include?(board.possible_statuses.find_by_id(change.value_id))
 
-        detail = "Status #{format_status change.value, board: board} is not on the board"
-        if issue.board.possible_statuses.expand_statuses(change.value).empty?
-          detail = "Status #{format_status change.value, board: board} cannot be found at all. Was it deleted?"
+        detail = "Status #{format_status change, board: board} is not on the board"
+        if issue.board.possible_statuses.find_by_id(change.value_id).nil?
+          detail = "Status #{format_status change, board: board} cannot be found at all. Was it deleted?"
         end
 
         # If it's been moved back to backlog then it's on a different report. Ignore it here.
@@ -198,24 +198,24 @@ class DataQualityReport < ChartBase
       elsif change.old_value.nil?
         # Do nothing
       elsif index < last_index
-        new_category = category_name_for(status_id: change.value_id, board: board)
-        old_category = category_name_for(status_id: change.old_value_id, board: board)
+        new_category = board.possible_statuses.find_by_id(change.value_id).category.name
+        old_category = board.possible_statuses.find_by_id(change.old_value_id).category.name
 
         if new_category == old_category
           entry.report(
             problem_key: :backwords_through_statuses,
-            detail: "Moved from #{format_status change.old_value, board: board}" \
-              " to #{format_status change.value, board: board}" \
+            detail: "Moved from #{format_status change, use_old_status: true, board: board}" \
+              " to #{format_status change, board: board}" \
               " on #{change.time.to_date}"
           )
         else
           entry.report(
             problem_key: :backwards_through_status_categories,
-            detail: "Moved from #{format_status change.old_value, board: board}" \
-              " to #{format_status change.value, board: board}" \
-              " on #{change.time.to_date}, " \
-              " crossing from category #{format_status old_category, board: board, is_category: true}" \
-              " to #{format_status new_category, board: board, is_category: true}."
+            detail: "Moved from #{format_status change, use_old_status: true, board: board}" \
+              " to #{format_status change, board: board}" \
+              " on #{change.time.to_date}," \
+              " crossing from category #{format_status change, use_old_status: true, board: board, is_category: true}" \
+              " to #{format_status change, board: board, is_category: true}."
           )
         end
       end
@@ -224,16 +224,14 @@ class DataQualityReport < ChartBase
   end
 
   def scan_for_issues_not_created_in_a_backlog_status entry:, backlog_statuses:
-    return if backlog_statuses.empty?
-
     creation_change = entry.issue.changes.find { |issue| issue.status? }
 
     return if backlog_statuses.any? { |status| status.id == creation_change.value_id }
 
-    status_string = backlog_statuses.collect { |s| format_status s.name, board: entry.issue.board }.join(', ')
+    status_string = backlog_statuses.collect { |s| format_status s, board: entry.issue.board }.join(', ')
     entry.report(
       problem_key: :created_in_wrong_status,
-      detail: "Created in #{format_status creation_change.value, board: entry.issue.board}, " \
+      detail: "Created in #{format_status creation_change, board: entry.issue.board}, " \
         "which is not one of the backlog statuses for this board: #{status_string}"
     )
   end
@@ -266,6 +264,20 @@ class DataQualityReport < ChartBase
     )
   end
 
+  def scan_for_items_blocked_on_closed_tickets entry:
+    entry.issue.issue_links.each do |link|
+      this_active = !entry.stopped
+      other_active = !link.other_issue.board.cycletime.started_stopped_times(link.other_issue).last
+      next unless this_active && !other_active
+
+      entry.report(
+        problem_key: :items_blocked_on_closed_tickets,
+        detail: "#{entry.issue.key} thinks it's blocked by #{link.other_issue.key}, " \
+          "except #{link.other_issue.key} is closed."
+      )
+    end
+  end
+
   def subtask_label subtask
     "<img src='#{subtask.type_icon_url}' /> #{link_to_issue(subtask)} #{subtask.summary[..50].inspect}"
   end
@@ -314,10 +326,10 @@ class DataQualityReport < ChartBase
   end
 
   def scan_for_discarded_data entry:
-    hash = @original_issue_times[entry.issue]
+    hash = @discarded_changes_data&.find { |a| a[:issue] == entry.issue }
     return if hash.nil?
 
-    old_start_time = hash[:started_time]
+    old_start_time = hash[:original_start_time]
     cutoff_time = hash[:cutoff_time]
 
     old_start_date = old_start_time.to_date
@@ -352,7 +364,7 @@ class DataQualityReport < ChartBase
     <<-HTML
       #{label_issues problems.size} have had information discarded. This configuration is set
       to "reset the clock" if an item is moved back to the backlog after it's been started. This hides important
-      information and makes the data less accurate. <b>Moving items back to the backlog is strongly discouraged.</b>      HTML
+      information and makes the data less accurate. <b>Moving items back to the backlog is strongly discouraged.</b>
     HTML
   end
 
@@ -437,4 +449,11 @@ class DataQualityReport < ChartBase
       could result in more data points showing up on a chart then there really should be.
     HTML
   end
+
+  def render_items_blocked_on_closed_tickets problems
+    <<-HTML
+      For #{label_issues problems.size}, the issue is identified as being blocked by another issue. Yet,
+      that other issue is already completed so, by definition, it can't still be blocking.
+    HTML
+  end
 end

data/lib/jirametrics/downloader.rb

@@ -103,8 +103,6 @@ class Downloader
       json = @jira_gateway.call_url relative_url: '/rest/api/2/search' \
         "?jql=#{escaped_jql}&maxResults=#{max_results}&startAt=#{start_at}&expand=changelog&fields=*all"
 
-      exit_if_call_failed json
-
       json['issues'].each do |issue_json|
         issue_json['exporter'] = {
           'in_initial_query' => initial_query
@@ -139,15 +137,6 @@ class Downloader
     end
   end
 
-  def exit_if_call_failed json
-    # Sometimes Jira returns the singular form of errorMessage and sometimes the plural. Consistency FTW.
-    return unless json['error'] || json['errorMessages'] || json['errorMessage']
-
-    log "Download failed. See #{@file_system.logfile_name} for details.", both: true
-    log "  #{JSON.pretty_generate(json)}"
-    exit 1
-  end
-
   def download_statuses
     log '  Downloading all statuses', both: true
     json = @jira_gateway.call_url relative_url: '/rest/api/2/status'
@@ -188,8 +177,6 @@ class Downloader
     log "  Downloading board configuration for board #{board_id}", both: true
     json = @jira_gateway.call_url relative_url: "/rest/agile/1.0/board/#{board_id}/configuration"
 
-    exit_if_call_failed json
-
     @file_system.save_json(
       json: json,
       filename: File.join(@target_path, "#{file_prefix}_board_#{board_id}_configuration.json")
@@ -213,7 +200,6 @@ class Downloader
     while is_last == false
       json = @jira_gateway.call_url relative_url: "/rest/agile/1.0/board/#{board_id}/sprint?" \
         "maxResults=#{max_results}&startAt=#{start_at}"
-      exit_if_call_failed json
 
       @file_system.save_json(
         json: json,

data/lib/jirametrics/examples/standard_project.rb

@@ -28,8 +28,8 @@ class Exporter
         block = boards[board_id]
         if block == :default
           block = lambda do |_|
-            start_at first_time_in_status_category('In Progress')
-            stop_at still_in_status_category('Done')
+            start_at first_time_in_status_category(:indeterminate)
+            stop_at still_in_status_category(:done)
           end
         end
         board id: board_id do

data/lib/jirametrics/exporter.rb

@@ -2,18 +2,6 @@
 
 require 'fileutils'
 
-class Object
-  def deprecated message:, date:, depth: 2
-    text = +''
-    text << "Deprecated(#{date}): "
-    text << message
-    caller(1..depth).each do |line|
-      text << "\n-> Called from #{line}"
-    end
-    warn text
-  end
-end
-
 class Exporter
   attr_reader :project_configs
   attr_accessor :file_system
@@ -76,12 +64,8 @@ class Exporter
     selected = []
     each_project_config(name_filter: name_filter) do |project|
       project.evaluate_next_level
-      # next if project.aggregated_project?
 
       project.run load_only: true
-      project.board_configs.each do |board_config|
-        board_config.run
-      end
       project.issues.each do |issue|
         selected << [project, issue] if keys.include? issue.key
       end
@@ -91,9 +75,13 @@ class Exporter
       raise unless e.message.start_with? 'This is an aggregated project and issues should have been included'
     end
 
-    selected.each do |project, issue|
-      puts "\nProject #{project.name}"
-      puts issue.dump
+    if selected.empty?
+      file_system.log "No issues found to match #{keys.collect(&:inspect).join(', ')}"
+    else
+      selected.each do |project, issue|
+        file_system.log "\nProject #{project.name}", also_write_to_stderr: true
+        file_system.log issue.dump, also_write_to_stderr: true
+      end
     end
   end
 
@@ -128,7 +116,9 @@ class Exporter
 
   def jira_config filename = nil
     if filename
-      @jira_config = file_system.load_json(filename)
+      @jira_config = file_system.load_json(filename, fail_on_error: false)
+      raise "Unable to load Jira configuration file and cannot continue: #{filename.inspect}" if @jira_config.nil?
+
       @jira_config['url'] = $1 if @jira_config['url'] =~ /^(.+)\/+$/
     end
     @jira_config

data/lib/jirametrics/file_config.rb

@@ -66,15 +66,20 @@ class FileConfig
   # is that all empty values in the first column should be at the bottom.
   def sort_output all_lines
     all_lines.sort do |a, b|
+      result = nil
       if a[0] == b[0]
-        a[1..] <=> b[1..]
+        result = a[1..] <=> b[1..]
       elsif a[0].nil?
-        1
+        result = 1
       elsif b[0].nil?
-        -1
+        result = -1
       else
-        a[0] <=> b[0]
+        result = a[0] <=> b[0]
       end
+
+      # This will only happen if one of the objects isn't comparable. Seen in production.
+      result = -1 if result.nil?
+      result
     end
   end
 
@@ -85,6 +90,11 @@ class FileConfig
 
   def html_report &block
     assert_only_one_filetype_config_set
+    if block.nil?
+      project_config.file_system.warning 'No charts were specified for the report. This is almost certainly a mistake.'
+      block = ->(_) {}
+    end
+
     @html_report = HtmlReportConfig.new file_config: self, block: block
   end
 
@@ -120,4 +130,11 @@ class FileConfig
     @file_suffix = suffix unless suffix.nil?
     @file_suffix
   end
+
+  def children
+    result = []
+    result << @columns if @columns
+    result << @html_report if @html_report
+    result
+  end
 end

data/lib/jirametrics/file_system.rb

@@ -31,13 +31,22 @@ class FileSystem
     File.write(filename, content)
   end
 
-  def warning message
-    log "Warning: #{message}", also_write_to_stderr: true
+  def warning message, more: nil
+    log "Warning: #{message}", more: more, also_write_to_stderr: true
   end
 
-  def log message, also_write_to_stderr: false
+  def error message, more: nil
+    log "Error: #{message}", more: more, also_write_to_stderr: true
+  end
+
+  def log message, more: nil, also_write_to_stderr: false
+    message += " See #{logfile_name} for more details about this message." if more
+
     logfile.puts message
-    $stderr.puts message if also_write_to_stderr # rubocop:disable Style/StderrPuts
+    logfile.puts more if more
+    return unless also_write_to_stderr
+
+    $stderr.puts message # rubocop:disable Style/StderrPuts
   end
 
   # In some Jira instances, a sizeable portion of the JSON is made up of empty fields. I've seen
@@ -59,4 +68,14 @@ class FileSystem
   def file_exist? filename
     File.exist? filename
   end
+
+  def deprecated message:, date:, depth: 2
+    text = +''
+    text << "Deprecated(#{date}): "
+    text << message
+    caller(1..depth).each do |line|
+      text << "\n-> Called from #{line}"
+    end
+    log text, also_write_to_stderr: true
+  end
 end

data/lib/jirametrics/flow_efficiency_scatterplot.rb

@@ -27,7 +27,7 @@ class FlowEfficiencyScatterplot < ChartBase
           </mfrac>
         </math>
       </div>
-      <div style="background: yellow">Note that for this calculation to be accurate, we must be moving items into a
+      <div style="background: var(--warning-banner)">Note that for this calculation to be accurate, we must be moving items into a
         blocked or stalled state the moment we stop working on it, and most teams don't do that.
         So be aware that your team may have to change their behaviours if you want this chart to be useful.
       </div>

data/lib/jirametrics/groupable_issue_chart.rb

@@ -6,9 +6,7 @@ require 'jirametrics/grouping_rules'
 module GroupableIssueChart
   def init_configuration_block user_provided_block, &default_block
     instance_eval(&user_provided_block)
-    return if @group_by_block
-
-    instance_eval(&default_block)
+    instance_eval(&default_block) unless @group_by_block
   end
 
   def grouping_rules &block