jirametrics 2.0 → 2.11

data/lib/jirametrics/data_quality_report.rb

@@ -1,7 +1,7 @@
 # frozen_string_literal: true
 
 class DataQualityReport < ChartBase
-  attr_reader :original_issue_times # For testing purposes only
+  attr_reader :discarded_changes_data, :entries # Both for testing purposes only
   attr_accessor :board_id
 
   class Entry
@@ -19,10 +19,10 @@ class DataQualityReport < ChartBase
     end
   end
 
-  def initialize original_issue_times
+  def initialize discarded_changes_data
     super()
 
-    @original_issue_times = original_issue_times
+    @discarded_changes_data = discarded_changes_data
 
     header_text 'Data Quality Report'
     description_text <<-HTML
@@ -48,7 +48,9 @@ class DataQualityReport < ChartBase
       scan_for_issues_not_created_in_a_backlog_status entry: entry, backlog_statuses: backlog_statuses
       scan_for_stopped_before_started entry: entry
       scan_for_issues_not_started_with_subtasks_that_have entry: entry
+      scan_for_incomplete_subtasks_when_issue_done entry: entry
       scan_for_discarded_data entry: entry
+      scan_for_items_blocked_on_closed_tickets entry: entry
     end
 
     scan_for_issues_on_multiple_boards entries: @entries
@@ -56,7 +58,26 @@ class DataQualityReport < ChartBase
     entries_with_problems = entries_with_problems()
     return '' if entries_with_problems.empty?
 
-    wrap_and_render(binding, __FILE__)
+    caller_binding = binding
+    result = +''
+    result << render_top_text(caller_binding)
+
+    result << '<ul class="quality_report">'
+    result << render_problem_type(:discarded_changes)
+    result << render_problem_type(:completed_but_not_started)
+    result << render_problem_type(:status_changes_after_done)
+    result << render_problem_type(:backwards_through_status_categories)
+    result << render_problem_type(:backwords_through_statuses)
+    result << render_problem_type(:status_not_on_board)
+    result << render_problem_type(:created_in_wrong_status)
+    result << render_problem_type(:stopped_before_started)
+    result << render_problem_type(:issue_not_started_but_subtasks_have)
+    result << render_problem_type(:incomplete_subtasks_when_issue_done)
+    result << render_problem_type(:issue_on_multiple_boards)
+    result << render_problem_type(:items_blocked_on_closed_tickets)
+    result << '</ul>'
+
+    result
   end
 
   def problems_for key
@@ -69,24 +90,37 @@ class DataQualityReport < ChartBase
     result
   end
 
+  def render_problem_type problem_key
+    problems = problems_for problem_key
+    return '' if problems.empty?
+
+    <<-HTML
+      <li>
+        #{__send__ :"render_#{problem_key}", problems}
+        #{collapsible_issues_panel problems}
+      </li>
+    HTML
+  end
+
   # Return a format that's easier to assert against
   def testable_entries
-    @entries.collect { |entry| [entry.started.to_s, entry.stopped.to_s, entry.issue] }
+    formatter = ->(time) { time&.strftime('%Y-%m-%d %H:%M:%S %z') || '' }
+    @entries.collect do |entry|
+      [
+        formatter.call(entry.started),
+        formatter.call(entry.stopped),
+        entry.issue
+      ]
+    end
   end
 
   def entries_with_problems
     @entries.reject { |entry| entry.problems.empty? }
   end
 
-  def category_name_for status_name:, board:
-    board.possible_statuses.find { |status| status.name == status_name }&.category_name
-  end
-
   def initialize_entries
     @entries = @issues.filter_map do |issue|
-      cycletime = issue.board.cycletime
-      started = cycletime.started_time(issue)
-      stopped = cycletime.stopped_time(issue)
+      started, stopped = issue.board.cycletime.started_stopped_times(issue)
       next if stopped && stopped < time_range.begin
       next if started && started > time_range.end
 
@@ -107,10 +141,8 @@ class DataQualityReport < ChartBase
   def scan_for_completed_issues_without_a_start_time entry:
     return unless entry.stopped && entry.started.nil?
 
-    status_names = entry.issue.changes.filter_map do |change|
-      next unless change.status?
-
-      format_status change.value, board: entry.issue.board
+    status_names = entry.issue.status_changes.filter_map do |change|
+      format_status change, board: entry.issue.board
     end
 
     entry.report(
@@ -125,14 +157,14 @@ class DataQualityReport < ChartBase
     changes_after_done = entry.issue.changes.select do |change|
       change.status? && change.time >= entry.stopped
     end
-    done_status = changes_after_done.shift.value
+    done_status = changes_after_done.shift
 
     return if changes_after_done.empty?
 
     board = entry.issue.board
     problem = "Completed on #{entry.stopped.to_date} with status #{format_status done_status, board: board}."
     changes_after_done.each do |change|
-      problem << " Changed to #{format_status change.value, board: board} on #{change.time.to_date}."
+      problem << " Changed to #{format_status change, board: board} on #{change.time.to_date}."
     end
     entry.report(
       problem_key: :status_changes_after_done,
@@ -152,11 +184,11 @@ class DataQualityReport < ChartBase
       index = entry.issue.board.visible_columns.find_index { |column| column.status_ids.include? change.value_id }
       if index.nil?
         # If it's a backlog status then ignore it. Not supposed to be visible.
-        next if entry.issue.board.backlog_statuses.include? change.value_id
+        next if entry.issue.board.backlog_statuses.include?(board.possible_statuses.find_by_id(change.value_id))
 
-        detail = "Status #{format_status change.value, board: board} is not on the board"
-        if issue.board.possible_statuses.expand_statuses(change.value).empty?
-          detail = "Status #{format_status change.value, board: board} cannot be found at all. Was it deleted?"
+        detail = "Status #{format_status change, board: board} is not on the board"
+        if issue.board.possible_statuses.find_by_id(change.value_id).nil?
+          detail = "Status #{format_status change, board: board} cannot be found at all. Was it deleted?"
         end
 
         # If it's been moved back to backlog then it's on a different report. Ignore it here.
@@ -166,24 +198,24 @@ class DataQualityReport < ChartBase
       elsif change.old_value.nil?
         # Do nothing
       elsif index < last_index
-        new_category = category_name_for(status_name: change.value, board: board)
-        old_category = category_name_for(status_name: change.old_value, board: board)
+        new_category = board.possible_statuses.find_by_id(change.value_id).category.name
+        old_category = board.possible_statuses.find_by_id(change.old_value_id).category.name
 
         if new_category == old_category
           entry.report(
             problem_key: :backwords_through_statuses,
-            detail: "Moved from #{format_status change.old_value, board: board}" \
-              " to #{format_status change.value, board: board}" \
+            detail: "Moved from #{format_status change, use_old_status: true, board: board}" \
+              " to #{format_status change, board: board}" \
               " on #{change.time.to_date}"
           )
         else
           entry.report(
             problem_key: :backwards_through_status_categories,
-            detail: "Moved from #{format_status change.old_value, board: board}" \
-              " to #{format_status change.value, board: board}" \
-              " on #{change.time.to_date}, " \
-              " crossing from category #{format_status old_category, board: board, is_category: true}" \
-              " to #{format_status new_category, board: board, is_category: true}."
+            detail: "Moved from #{format_status change, use_old_status: true, board: board}" \
+              " to #{format_status change, board: board}" \
+              " on #{change.time.to_date}," \
+              " crossing from category #{format_status change, use_old_status: true, board: board, is_category: true}" \
+              " to #{format_status change, board: board, is_category: true}."
           )
         end
       end
@@ -192,16 +224,14 @@ class DataQualityReport < ChartBase
   end
 
   def scan_for_issues_not_created_in_a_backlog_status entry:, backlog_statuses:
-    return if backlog_statuses.empty?
-
     creation_change = entry.issue.changes.find { |issue| issue.status? }
 
     return if backlog_statuses.any? { |status| status.id == creation_change.value_id }
 
-    status_string = backlog_statuses.collect { |s| format_status s.name, board: entry.issue.board }.join(', ')
+    status_string = backlog_statuses.collect { |s| format_status s, board: entry.issue.board }.join(', ')
     entry.report(
       problem_key: :created_in_wrong_status,
-      detail: "Created in #{format_status creation_change.value, board: entry.issue.board}, " \
+      detail: "Created in #{format_status creation_change, board: entry.issue.board}, " \
         "which is not one of the backlog statuses for this board: #{status_string}"
     )
   end
@@ -220,14 +250,13 @@ class DataQualityReport < ChartBase
 
     started_subtasks = []
     entry.issue.subtasks.each do |subtask|
-      started_subtasks << subtask if subtask.board.cycletime.started_time(subtask)
+      started_subtasks << subtask if subtask.board.cycletime.started_stopped_times(subtask).first
     end
 
     return if started_subtasks.empty?
 
     subtask_labels = started_subtasks.collect do |subtask|
-      "Started subtask: #{link_to_issue(subtask)} (#{format_status subtask.status.name, board: entry.issue.board}) " \
-        "#{subtask.summary[..50].inspect}"
+      subtask_label(subtask)
     end
     entry.report(
       problem_key: :issue_not_started_but_subtasks_have,
@@ -235,6 +264,61 @@ class DataQualityReport < ChartBase
     )
   end
 
+  def scan_for_items_blocked_on_closed_tickets entry:
+    entry.issue.issue_links.each do |link|
+      this_active = !entry.stopped
+      other_active = !link.other_issue.board.cycletime.started_stopped_times(link.other_issue).last
+      next unless this_active && !other_active
+
+      entry.report(
+        problem_key: :items_blocked_on_closed_tickets,
+        detail: "#{entry.issue.key} thinks it's blocked by #{link.other_issue.key}, " \
+          "except #{link.other_issue.key} is closed."
+      )
+    end
+  end
+
+  def subtask_label subtask
+    "<img src='#{subtask.type_icon_url}' /> #{link_to_issue(subtask)} #{subtask.summary[..50].inspect}"
+  end
+
+  def time_as_english(from_time, to_time)
+    delta = (to_time - from_time).to_i
+    return "#{delta} seconds" if delta < 60
+
+    delta /= 60
+    return "#{delta} minutes" if delta < 60
+
+    delta /= 60
+    return "#{delta} hours" if delta < 24
+
+    delta /= 24
+    "#{delta} days"
+  end
+
+  def scan_for_incomplete_subtasks_when_issue_done entry:
+    return unless entry.stopped
+
+    subtask_labels = entry.issue.subtasks.filter_map do |subtask|
+      subtask_started, subtask_stopped = subtask.board.cycletime.started_stopped_times(subtask)
+
+      if !subtask_started && !subtask_stopped
+        "#{subtask_label subtask} (Not even started)"
+      elsif !subtask_stopped
+        "#{subtask_label subtask} (Still not done)"
+      elsif subtask_stopped > entry.stopped
+        "#{subtask_label subtask} (Closed #{time_as_english entry.stopped, subtask_stopped} later)"
+      end
+    end
+
+    return if subtask_labels.empty?
+
+    entry.report(
+      problem_key: :incomplete_subtasks_when_issue_done,
+      detail: subtask_labels.join('<br />')
+    )
+  end
+
   def label_issues number
     return '1 item' if number == 1
 
@@ -242,10 +326,10 @@ class DataQualityReport < ChartBase
   end
 
   def scan_for_discarded_data entry:
-    hash = @original_issue_times[entry.issue]
+    hash = @discarded_changes_data&.find { |a| a[:issue] == entry.issue }
     return if hash.nil?
 
-    old_start_time = hash[:started_time]
+    old_start_time = hash[:original_start_time]
     cutoff_time = hash[:cutoff_time]
 
     old_start_date = old_start_time.to_date
@@ -271,8 +355,105 @@ class DataQualityReport < ChartBase
       board_names = entry_list.collect { |entry| entry.issue.board.name.inspect }
       entry_list.first.report(
         problem_key: :issue_on_multiple_boards,
-        detail: "Found on boards: #{board_names.join(', ')}"
+        detail: "Found on boards: #{board_names.sort.join(', ')}"
       )
     end
   end
+
+  def render_discarded_changes problems
+    <<-HTML
+      #{label_issues problems.size} have had information discarded. This configuration is set
+      to "reset the clock" if an item is moved back to the backlog after it's been started. This hides important
+      information and makes the data less accurate. <b>Moving items back to the backlog is strongly discouraged.</b>
+    HTML
+  end
+
+  def render_completed_but_not_started problems
+    percentage_work_included = ((issues.size - problems.size).to_f / issues.size * 100).to_i
+    html = <<-HTML
+      #{label_issues problems.size} were discarded from all charts using cycletime (scatterplot, histogram, etc)
+      as we couldn't determine when they started.
+    HTML
+    if percentage_work_included < 85
+      html << <<-HTML
+        Consider whether looking at only #{percentage_work_included}% of the total data points is enough
+        to come to any reasonable conclusions. See <a href="https://unconsciousagile.com/2024/11/19/survivor-bias.html">
+        Survivor Bias</a>.
+      HTML
+    end
+    html
+  end
+
+  def render_status_changes_after_done problems
+    <<-HTML
+      #{label_issues problems.size} had a status change after being identified as done. We should question
+      whether they were really done at that point or if we stopped the clock too early.
+    HTML
+  end
+
+  def render_backwards_through_status_categories problems
+    <<-HTML
+      #{label_issues problems.size} moved backwards across the board, <b>crossing status categories</b>.
+      This will almost certainly have impacted timings as the end times are often taken at status category
+      boundaries. You should assume that any timing measurements for this item are wrong.
+    HTML
+  end
+
+  def render_backwords_through_statuses problems
+    <<-HTML
+      #{label_issues problems.size} moved backwards across the board. Depending where we have set the
+      start and end points, this may give us incorrect timing data. Note that these items did not cross
+      a status category and may not have affected metrics.
+    HTML
+  end
+
+  def render_status_not_on_board problems
+    <<-HTML
+      #{label_issues problems.size} were not visible on the board for some period of time. This may impact
+      timings as the work was likely to have been forgotten if it wasn't visible.
+    HTML
+  end
+
+  def render_created_in_wrong_status problems
+    <<-HTML
+      #{label_issues problems.size} were created in a status not designated as Backlog. This will impact
+      the measurement of start times and will therefore impact whether it's shown as in progress or not.
+    HTML
+  end
+
+  def render_stopped_before_started problems
+    <<-HTML
+      #{label_issues problems.size} were stopped before they were started and this will play havoc with
+      any cycletime or WIP calculations. The most common case for this is when an item gets closed and
+      then moved back into an in-progress status.
+    HTML
+  end
+
+  def render_issue_not_started_but_subtasks_have problems
+    <<-HTML
+      #{label_issues problems.size} still showing 'not started' while sub-tasks underneath them have
+      started. This is almost always a mistake; if we're working on subtasks, the top level item should
+      also have started.
+    HTML
+  end
+
+  def render_incomplete_subtasks_when_issue_done problems
+    <<-HTML
+      #{label_issues problems.size} issues were marked as done while subtasks were still not done.
+    HTML
+  end
+
+  def render_issue_on_multiple_boards problems
+    <<-HTML
+      For #{label_issues problems.size}, we have an issue that shows up on more than one board. This
+      could result in more data points showing up on a chart then there really should be.
+    HTML
+  end
+
+  def render_items_blocked_on_closed_tickets problems
+    <<-HTML
+      For #{label_issues problems.size}, the issue is identified as being blocked by another issue. Yet,
+      that other issue is already completed so, by definition, it can't still be blocking.
+    HTML
+  end
 end

data/lib/jirametrics/dependency_chart.rb

@@ -42,13 +42,9 @@ class DependencyChart < ChartBase
     HTML
 
     @rules_block = rules_block
-    @link_rules_block = ->(link_name, link_rules) {}
 
-    issue_rules do |issue, rules|
-      key = issue.key
-      key = "<S>#{key} </S> " if issue.status.category_name == 'Done'
-      rules.label = "<#{key} [#{issue.type}]<BR/>#{word_wrap issue.summary}>"
-    end
+    issue_rules(&default_issue_rules)
+    link_rules(&default_link_rules)
   end
 
   def run
@@ -84,7 +80,8 @@ class DependencyChart < ChartBase
     result << issue_link.other_issue.key.inspect
     result << '['
     result << 'label=' << (link_rules.label || issue_link.label).inspect
-    result << ',color=' << (link_rules.line_color || 'black').inspect
+    result << ',color=' << (link_rules.line_color || 'gray').inspect
+    result << ',fontcolor=' << (link_rules.line_color || 'gray').inspect
     result << ',dir=both' if link_rules.bidirectional_arrows?
     result << '];'
     result
@@ -101,12 +98,26 @@ class DependencyChart < ChartBase
     tooltip = "#{issue.key}: #{issue.summary}"
     result << ",tooltip=#{tooltip[0..80].inspect}"
     unless issue_rules.color == :none
-      result << %(,style=filled,fillcolor="#{issue_rules.color || color_for(type: issue.type, shade: :light)}")
+      result << %(,style=filled,fillcolor="#{issue_rules.color || color_for(type: issue.type)}")
     end
     result << ']'
     result
   end
 
+  # This used to pull colours from chart_base but the migration to CSS colours kept breaking
+  # this chart so we moved it here, until we're finished with the rest. TODO: Revisit whether
+  # this can also use customizable CSS colours
+  def color_for type:
+    @chart_colors = {
+      'Story' => '#90EE90',
+      'Task' => '#87CEFA',
+      'Bug' => '#ffdab9',
+      'Defect' => '#ffdab9',
+      'Epic' => '#fafad2',
+      'Spike' => '#DDA0DD' # light purple
+    }[type] ||= random_color
+  end
+
   def build_dot_graph
     issue_links = find_links
 
@@ -148,6 +159,7 @@ class DependencyChart < ChartBase
     dot_graph = []
     dot_graph << 'digraph mygraph {'
     dot_graph << 'rankdir=LR'
+    dot_graph << 'bgcolor="transparent"'
 
     # Sort the keys so they are proccessed in a deterministic order.
     visible_issues.values.sort_by(&:key_as_i).each do |issue|
@@ -171,24 +183,11 @@ class DependencyChart < ChartBase
       return stdout.read
     end
   rescue # rubocop:disable Style/RescueStandardError
-    message = "Unable to execute the command 'dot' which is part of graphviz. " \
-      'Ensure that graphviz is installed and that dot is in your path.'
-    puts message
+    message = 'Unable to generate the dependency chart because graphviz could not be found in the path.'
+    file_system.log message, also_write_to_stderr: true
     message
   end
 
-  def default_color_for_issue issue
-    {
-      'Story' => '#90EE90',
-      'Task' => '#87CEFA',
-      'Bug' => '#f08080',
-      'Defect' => '#f08080',
-      'Epic' => '#fafad2',
-      'Spike' => '#7fffd4',
-      'Sub-task' => '#dcdcdc'
-    }[issue.type]
-  end
-
   def shrink_svg svg
     scale = 0.8
     svg.sub(/width="([\d.]+)pt" height="([\d.]+)pt"/) do
@@ -215,4 +214,36 @@ class DependencyChart < ChartBase
       end
     end.join(separator)
   end
+
+  def default_issue_rules
+    chart = self
+    lambda do |issue, rules|
+      is_done = issue.done?
+
+      key = issue.key
+      key = "<S>#{key} </S> " if is_done
+      line2 = +'<BR/>'
+      if issue.artificial?
+        line2 << '(unknown state)' # Shouldn't happen if we've done a full download but is still possible.
+      elsif is_done
+        line2 << 'Done'
+      else
+        started_at = issue.board.cycletime.started_stopped_times(issue).first
+        if started_at.nil?
+          line2 << 'Not started'
+        else
+          line2 << "Age: #{issue.board.cycletime.age(issue, today: chart.date_range.end)} days"
+        end
+      end
+      rules.label = "<#{key} [#{issue.type}]#{line2}<BR/>#{word_wrap issue.summary}>"
+    end
+  end
+
+  def default_link_rules
+    lambda do |link, rules|
+      rules.ignore if link.origin.done? && link.other_issue.done?
+      rules.ignore if link.name == 'Cloners'
+      rules.merge_bidirectional keep: 'outward'
+    end
+  end
 end

data/lib/jirametrics/download_config.rb

@@ -19,4 +19,16 @@ class DownloadConfig
     @rolling_date_count = count unless count.nil?
     @rolling_date_count
   end
+
+  def no_earlier_than date = :not_set
+    @no_earlier_than = Date.parse(date) unless date == :not_set
+    @no_earlier_than
+  end
+
+  def start_date today:
+    date = today.to_date - @rolling_date_count if @rolling_date_count
+    date = [date, @no_earlier_than].max if date && @no_earlier_than
+    date = @no_earlier_than if date.nil? && @no_earlier_than
+    date
+  end
 end