jirametrics 3.1.1 → 3.3

data/lib/jirametrics/html_report_config.rb

@@ -106,9 +106,8 @@ class HtmlReportConfig < HtmlGenerator
     @file_config.project_config.exporter.timezone_offset
   end
 
-  def random_color
-    @palette_index = (@palette_index || -1) + 1
-    ChartBase::OKABE_ITO_PALETTE[@palette_index % ChartBase::OKABE_ITO_PALETTE.size]
+  def next_palette_color
+    color_palette.next_color
   end
 
   def html string, type: :body
@@ -137,6 +136,7 @@ class HtmlReportConfig < HtmlGenerator
     project_config = @file_config.project_config
 
     chart.file_system = file_system
+    chart.color_palette = color_palette
     chart.issues = issues
     chart.time_range = project_config.time_range
     chart.timezone_offset = timezone_offset

data/lib/jirametrics/percentile_validation.rb

@@ -0,0 +1,26 @@
+# frozen_string_literal: true
+
+# Percentile lists arrive from two different places in the config DSL: the chart level
+# "percentiles [50, 85]" setter and "rule.percentiles = [...]" inside a grouping_rules block.
+# Both are user supplied and both end up as JavaScript annotation ids in the rendered chart, so
+# both need the same guard rails. Including this module gives you a private validate_percentiles.
+module PercentileValidation
+  module_function
+
+  # Returns the cleaned up list. Raises ArgumentError, naming the offending value, for anything
+  # that isn't an Integer between 0 and 100.
+  def validate_percentiles list
+    list.each do |percentile|
+      raise ArgumentError, "percentile #{percentile} must be an integer" unless percentile.is_a? Integer
+
+      raise ArgumentError, "percentile #{percentile} must be between 0 and 100" unless percentile.between?(0, 100)
+    end
+    list.uniq.sort
+  end
+
+  # For the places where exactly one value is meaningful, such as a forecast that has to produce a
+  # single number of days. Returns the value; raises the same errors as the list form.
+  def validate_percentile value
+    validate_percentiles([value]).first
+  end
+end

data/lib/jirametrics/pull_request_cycle_time_scatterplot.rb

@@ -13,6 +13,7 @@ class PullRequestCycleTimeScatterplot < TimeBasedScatterplot
       <div class="p">
         This graph shows the cycle time for all closed pull requests (time from opened to closed).
       </div>
+      <%= trend_line_description %>
       #{describe_non_working_days}
     HTML
 

data/lib/jirametrics/settings.json

@@ -1,6 +1,7 @@
 {
   "stalled_threshold_days": 5,
   "stalled_statuses": [],
+  "stalled_ignored_fields": ["RemoteIssueLink"],
 
   "blocked_link_text": ["is blocked by"],
   "blocked_statuses": [],

data/lib/jirametrics/time_based_histogram.rb

@@ -1,10 +1,12 @@
 # frozen_string_literal: true
 
 require 'jirametrics/groupable_issue_chart'
+require 'jirametrics/percentile_validation'
 require 'jirametrics/time_based_chart'
 
 class TimeBasedHistogram < TimeBasedChart
   include GroupableIssueChart
+  include PercentileValidation
 
   attr_reader :show_stats
 
@@ -20,8 +22,11 @@ class TimeBasedHistogram < TimeBasedChart
     @x_axis_title = title
   end
 
-  def percentiles percs = nil
-    @percentiles = percs unless percs.nil?
+  # Which percentiles to show as columns in the statistics table. An empty list drops the columns
+  # entirely. Values are validated here rather than at use, because they feed the percentile
+  # arithmetic and the table headers, where a bad one produces a wrong chart instead of an error.
+  def percentiles list = nil
+    @percentiles = validate_percentiles(list) unless list.nil?
     @percentiles
   end
 
@@ -29,6 +34,25 @@ class TimeBasedHistogram < TimeBasedChart
     @show_stats = false
   end
 
+  # What a given percentile is actually good for. These used to be a hardcoded list describing
+  # the 50th, 85th and 98th, sitting underneath a table whose columns follow the configuration,
+  # so the two drifted apart the moment anyone changed the setting. Bands rather than exact
+  # values, because 90 deserves an answer just as much as 85 does.
+  def percentile_explanation percentile
+    case percentile
+    when 0..49
+      'below the median, so half or more of your work takes longer than this. Useful for ' \
+        'understanding your faster cases, but not a number to plan around.'
+    when 50
+      'also known as the <b>Median</b>. Useful to establish short feedback loops, to monitor ' \
+        "that it's not drifting to the right."
+    when 51..94
+      'useful to establish service level expectations, accounting for rare events.'
+    else
+      'useful to gauge worst case expectations.'
+    end
+  end
+
   def run
     histogram_items = all_items
     rules_to_items = group_issues histogram_items
@@ -98,19 +122,13 @@ class TimeBasedHistogram < TimeBasedChart
     sorted_by_frequency.select { |_value, frequency| frequency == max_frequency }.collect(&:first).sort
   end
 
-  def percentiles_for histogram_data, percentiles, total_values
-    sorted_values = histogram_data.keys.sort
-    cumulative_counts = {}
-    cumulative_sum = 0
-    sorted_values.each do |value|
-      cumulative_sum += histogram_data[value]
-      cumulative_counts[value] = cumulative_sum
-    end
-
-    percentiles.to_h do |percentile|
-      rank = (percentile / 100.0) * total_values
-      [percentile, sorted_values.find { |value| cumulative_counts[value] >= rank }]
-    end
+  # The data arrives as value => count. Expanded back to a flat list so that the one shared
+  # percentile implementation is used here too: this chart and the scatterplot must not report
+  # different answers for the same percentile of the same data. Chart sized data makes the
+  # expansion cheap.
+  def percentiles_for histogram_data, percentiles, _total_values
+    values = histogram_data.flat_map { |value, count| Array.new(count, value) }
+    percentiles.to_h { |percentile| [percentile, percentile_of(values, percentile)] }
   end
 
   def sort_items items

data/lib/jirametrics/time_based_scatterplot.rb

@@ -1,16 +1,30 @@
 # frozen_string_literal: true
 
 require 'jirametrics/groupable_issue_chart'
+require 'jirametrics/percentile_validation'
 require 'jirametrics/time_based_chart'
 
 class TimeBasedScatterplot < TimeBasedChart
   include GroupableIssueChart
 
+  # What the whole-data-set percentile lines call themselves when you hover them. They have no
+  # legend entry, so without this there is nothing identifying them at all. "items" rather than
+  # "data" or "everything" because it matches the description prose and because anything a
+  # grouping rule ignored has already been dropped by the time these lines are calculated.
+  OVERALL_LABEL = 'All items'
+  include PercentileValidation
+
+  # percentage_lines is internal, not part of the documented config DSL. It exists so that specs
+  # and the ERB can see the computed lines without reaching into instance variables. Its shape,
+  # including the :id strings that encode positional group indices, is free to change.
+  attr_reader :y_axis_cap_percentile, :percentage_lines
+
   def initialize
     super
 
     @percentage_lines = []
     @highest_y_value = 0
+    @percentiles = [85]
   end
 
   # On a scatterplot the cycle time is plotted up the y-axis.
@@ -18,11 +32,29 @@ class TimeBasedScatterplot < TimeBasedChart
     @y_axis_title = title
   end
 
+  def cap_y_axis percentile: 98
+    @y_axis_cap_percentile = percentile
+  end
+
+  # Percentile reference lines. The chart level value defines the lines drawn across the whole
+  # data set AND the default for each group; a group can override with rule.percentiles.
+  # An empty list switches the lines off.
+  def percentiles list = nil
+    @percentiles = validate_percentiles(list) unless list.nil?
+    @percentiles
+  end
+
   def run
     items = all_items
     data_sets = create_datasets items
-    overall_percent_line = calculate_percent_line(items)
-    @percentage_lines << [overall_percent_line, CssVariable['--cycletime-scatterplot-overall-trendline-color']]
+    overall_color = CssVariable['--cycletime-scatterplot-overall-trendline-color']
+
+    percentile_lines_for(items, @percentiles).each do |percentile, value|
+      @percentage_lines << {
+        percentile: percentile, value: value, color: overall_color,
+        id: "overall_#{percentile}", dataset_index: nil, label: OVERALL_LABEL
+      }
+    end
 
     if data_sets.empty?
       return "<h1 class='foldable'>#{@header_text}</h1>" \
@@ -33,15 +65,20 @@ class TimeBasedScatterplot < TimeBasedChart
   end
 
   def create_datasets items
+    @cap = compute_cap items
     data_sets = []
 
-    group_issues(items).each do |rules, items_by_type|
+    group_issues(items).each_with_index do |(rules, items_by_type), group_index|
       label = rules.label
       color = rules.color
-      percent_line = calculate_percent_line items_by_type
+      lines = percentile_lines_for items_by_type, (rules.percentiles || @percentiles)
       data = items_by_type.filter_map { |item| data_for_item(item, rules: rules) }
+
+      # Where this group's scatter set is about to land. The legend handler knows the clicked
+      # dataset by index, so that's what the annotation map is keyed by.
+      dataset_index = data_sets.size
       data_sets << {
-        label: "#{label} (85% at #{label_days(percent_line)})",
+        label: percentile_label(label, lines),
         data: data,
         fill: false,
         showLine: false,
@@ -50,25 +87,74 @@ class TimeBasedScatterplot < TimeBasedChart
 
       data_sets << trend_line_data_set(label: label, data: data, color: color)
 
-      @percentage_lines << [percent_line, color]
+      lines.each do |percentile, value|
+        @percentage_lines << {
+          percentile: percentile, value: value, color: color,
+          id: "group#{group_index}_#{percentile}", dataset_index: dataset_index, label: label
+        }
+      end
     end
     data_sets
   end
 
+  # "Story (85% at 81 days)" for one, comma separated for several, bare label for none.
+  def percentile_label label, lines
+    return label if lines.empty?
+
+    parts = lines.collect { |percentile, value| "#{percentile}% at #{label_days value}" }
+    "#{label} (#{parts.join ', '})"
+  end
+
   def show_trend_lines
     @show_trend_lines = true
   end
 
+  # The lines are always built but drawn hidden unless asked for, so this stays empty until they
+  # are actually switched on. Note the caller must be the ERB tag <%= trend_line_description %>:
+  # description_text is built during initialize, before the config block has called
+  # show_trend_lines, so interpolation would freeze "off" in permanently.
+  def trend_line_description
+    return '' unless @show_trend_lines
+
+    <<-HTML
+      <div class="p">
+        The dashed lines are trend lines, one per group in that group's colour. Each is a straight
+        line fitted through that group's dots, so the slope tells you whether cycle times have been
+        getting longer or shorter across the period shown. A line sloping up means work of that
+        kind has been taking progressively longer to finish.
+      </div>
+      <div class="p">
+        Read the slope as a description of this window rather than a prediction. A line is drawn
+        whenever a group has at least three dots and nothing checks how well it actually fits
+        them, so a scattered cloud with no real trend in it still gets a confident looking line.
+        It is a straight line, so it cannot show a trend that changed direction partway through,
+        and a handful of unusually long items will tilt it noticeably. If the slope surprises you,
+        look at the dots before you believe it.
+      </div>
+    HTML
+  end
+
+  # Dataset index to the annotation ids belonging to that dataset's group, so the legend handler
+  # can toggle all of a group's lines. Keyed by index rather than by label because two groups may
+  # legitimately share a label while differing in colour, and keying by label would then toggle
+  # both of them at once. Overall lines are deliberately absent; they are not owned by any group
+  # and stay visible when a group is switched off.
+  def legend_annotation_map
+    @percentage_lines.reject { |line| line[:dataset_index].nil? }
+      .group_by { |line| line[:dataset_index] }
+      .transform_values { |lines| lines.collect { |line| line[:id] } }
+  end
+
   def trend_line_data_set label:, data:, color:
     points = data.collect do |hash|
-      [Time.parse(hash[:x]).to_i, hash[:y]]
+      [Time.parse(hash[:x]).to_i, hash[:true_y] || hash[:y]]
     end
 
     # The trend calculation works with numbers only so convert Time to an int and back
     calculator = TrendLineCalculator.new(points)
     data_points = calculator.chart_datapoints(
       range: time_range.begin.to_i..time_range.end.to_i,
-      max_y: @highest_y_value
+      max_y: (@cap ? @cap[:cutoff] : @highest_y_value)
     )
     data_points.each do |point_hash|
       point_hash[:x] = chart_format Time.at(point_hash[:x])
@@ -97,20 +183,71 @@ class TimeBasedScatterplot < TimeBasedChart
     min = minimum_y_value
     return nil if min && y < min
 
-    @highest_y_value = y if @highest_y_value < y
+    over = @cap && y > @cap[:cutoff]
+    plotted_y = over ? @cap[:pin_row] : y
+    @highest_y_value = plotted_y if @highest_y_value < plotted_y
 
-    {
-      y: y,
+    point = {
+      y: plotted_y,
       x: chart_format(x_value(item)),
       title: [title_value(item, rules: rules)]
     }
+    if over
+      point[:over] = true
+      point[:true_y] = y
+    end
+    point
+  end
+
+  # Returns [[percentile, value], ...] for the requested percentiles, sorted ascending by
+  # percentile and dropping any that have no value because the item list is empty after
+  # filtering. Sorting happens here, not in the caller, because GroupingRules#percentiles is
+  # user-assigned with no ordering guarantee.
+  def percentile_lines_for items, percentiles
+    percentiles.sort.filter_map do |percentile|
+      value = percentile_value items, percentile
+      [percentile, value] unless value.nil?
+    end
+  end
+
+  def percentile_value items, percentile
+    percentile_of filtered_values(items), percentile
+  end
+
+  def compute_cap items
+    return nil unless @y_axis_cap_percentile
+
+    cutoff = percentile_value items, @y_axis_cap_percentile
+    return nil unless cutoff
+
+    values = filtered_values(items)
+    outlier_count = values.count { |value| value > cutoff }
+    return nil if outlier_count.zero?
+
+    pad = cutoff * 0.06        # breathing room so the top real dot does not touch the break
+    gutter_height = cutoff * 0.15
+    sep = cutoff + pad
+    {
+      cutoff: cutoff,
+      sep: sep,
+      pin_row: sep + (gutter_height * 0.55),
+      axis_max: (sep + gutter_height).ceil,
+      outlier_count: outlier_count,
+      label: cap_label(outlier_count: outlier_count, cutoff: cutoff)
+    }
+  end
+
+  private
+
+  def cap_label outlier_count:, cutoff:
+    item_word = outlier_count == 1 ? 'item' : 'items'
+    "#{outlier_count} #{item_word} above #{cutoff.round} days"
   end
 
-  def calculate_percent_line items
+  def filtered_values items
     min = minimum_y_value
-    times = items.collect { |item| y_value(item) }
-    times.reject! { |y| min && y < min }
-    index = times.size * 85 / 100
-    times.sort[index]
+    values = items.collect { |item| y_value(item) }
+    values.reject! { |value| min && value < min }
+    values
   end
 end

data/lib/jirametrics/trend_line_calculator.rb

@@ -3,9 +3,12 @@
 class TrendLineCalculator
   # Using math from https://math.stackexchange.com/questions/204020/what-is-the-equation-used-to-calculate-a-linear-trendline
 
+  # Three points minimum. Two will always fit a straight line perfectly, so the result looks
+  # authoritative while carrying no evidence of a trend at all; drawing it was misleading.
+  MINIMUM_POINTS = 3
+
   def initialize points
-    # We can't do trend calculations with less than two data points
-    @valid = points.size >= 2
+    @valid = points.size >= MINIMUM_POINTS
     return unless valid?
 
     sum_of_x = points.sum { |x, _y| x }

data/lib/jirametrics/wip_by_column_chart.rb

@@ -122,7 +122,7 @@ class WipByColumnChart < ChartBase
       cumulative = 0
       stat.wip_history.sort.find do |_wip, seconds|
         cumulative += seconds
-        cumulative / total >= 0.85
+        cumulative / total >= (85 / 100.0)
       end&.first
     end
   end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: jirametrics
 version: !ruby/object:Gem::Version
-  version: 3.1.1
+  version: '3.3'
 platform: ruby
 authors:
 - Mike Bowler
@@ -122,6 +122,7 @@ files:
 - lib/jirametrics/cfd_data_builder.rb
 - lib/jirametrics/change_item.rb
 - lib/jirametrics/chart_base.rb
+- lib/jirametrics/color_palette.rb
 - lib/jirametrics/columns_config.rb
 - lib/jirametrics/css_variable.rb
 - lib/jirametrics/cumulative_flow_diagram.rb
@@ -181,6 +182,7 @@ files:
 - lib/jirametrics/issue_printer.rb
 - lib/jirametrics/jira_gateway.rb
 - lib/jirametrics/mcp_server.rb
+- lib/jirametrics/percentile_validation.rb
 - lib/jirametrics/project_config.rb
 - lib/jirametrics/pull_request.rb
 - lib/jirametrics/pull_request_cycle_time_histogram.rb