jirametrics 2.6 → 2.7.1

data/lib/jirametrics/downloader.rb

@@ -41,8 +41,8 @@ class Downloader
     remove_old_files
     download_statuses
     find_board_ids.each do |id|
-      download_board_configuration board_id: id
-      download_issues board_id: id
+      board = download_board_configuration board_id: id
+      download_issues board: board
     end
 
     save_metadata
@@ -64,19 +64,19 @@ class Downloader
     ids
   end
 
-  def download_issues board_id:
-    log "  Downloading primary issues for board #{board_id}", both: true
+  def download_issues board:
+    log "  Downloading primary issues for board #{board.id}", both: true
     path = "#{@target_path}#{@download_config.project_config.file_prefix}_issues/"
     unless Dir.exist?(path)
       log "  Creating path #{path}"
       Dir.mkdir(path)
     end
 
-    filter_id = @board_id_to_filter_id[board_id]
+    filter_id = @board_id_to_filter_id[board.id]
     jql = make_jql(filter_id: filter_id)
-    jira_search_by_jql(jql: jql, initial_query: true, board_id: board_id, path: path)
+    jira_search_by_jql(jql: jql, initial_query: true, board: board, path: path)
 
-    log "  Downloading linked issues for board #{board_id}", both: true
+    log "  Downloading linked issues for board #{board.id}", both: true
     loop do
       @issue_keys_pending_download.reject! { |key| @issue_keys_downloaded_in_current_run.include? key }
       break if @issue_keys_pending_download.empty?
@@ -84,11 +84,11 @@ class Downloader
       keys_to_request = @issue_keys_pending_download[0..99]
       @issue_keys_pending_download.reject! { |key| keys_to_request.include? key }
       jql = "key in (#{keys_to_request.join(', ')})"
-      jira_search_by_jql(jql: jql, initial_query: false, board_id: board_id, path: path)
+      jira_search_by_jql(jql: jql, initial_query: false, board: board, path: path)
     end
   end
 
-  def jira_search_by_jql jql:, initial_query:, board_id:, path:
+  def jira_search_by_jql jql:, initial_query:, board:, path:
     intercept_jql = @download_config.project_config.settings['intercept_jql']
     jql = intercept_jql.call jql if intercept_jql
 
@@ -108,8 +108,8 @@ class Downloader
         issue_json['exporter'] = {
           'in_initial_query' => initial_query
         }
-        identify_other_issues_to_be_downloaded issue_json
-        file = "#{issue_json['key']}-#{board_id}.json"
+        identify_other_issues_to_be_downloaded raw_issue: issue_json, board: board
+        file = "#{issue_json['key']}-#{board.id}.json"
 
         @file_system.save_json(json: issue_json, filename: File.join(path, file))
       end
@@ -124,8 +124,8 @@ class Downloader
     end
   end
 
-  def identify_other_issues_to_be_downloaded raw_issue
-    issue = Issue.new raw: raw_issue, board: nil
+  def identify_other_issues_to_be_downloaded raw_issue:, board:
+    issue = Issue.new raw: raw_issue, board: board
     @issue_keys_downloaded_in_current_run << issue.key
 
     # Parent
@@ -171,6 +171,7 @@ class Downloader
     @board_id_to_filter_id[board_id] = json['filter']['id'].to_i
 
     download_sprints board_id: board_id if json['type'] == 'scrum'
+    Board.new raw: json
   end
 
   def download_sprints board_id:

data/lib/jirametrics/estimate_accuracy_chart.rb

@@ -83,8 +83,7 @@ class EstimateAccuracyChart < ChartBase
 
     issues.each do |issue|
       cycletime = issue.board.cycletime
-      start_time = cycletime.started_time(issue)
-      stop_time = cycletime.stopped_time(issue)
+      start_time, stop_time = cycletime.started_stopped_times(issue)
 
       next unless start_time
 

data/lib/jirametrics/examples/aggregated_project.rb

@@ -3,8 +3,6 @@
 # This file is really intended to give you ideas about how you might configure your own reports, not
 # as a complete setup that will work in every case.
 #
-# See https://github.com/mikebowler/jirametrics/wiki/Examples-folder for more details
-#
 # The point of an AGGREGATED report is that we're now looking at a higher level. We might use this in a
 # S2 meeting (Scrum of Scrums) to talk about the things that are happening across teams, not within a
 # single team. For that reason, we look at slightly different things that we would on a single team board.
@@ -33,7 +31,7 @@ class Exporter
           html '<h1>Boards included in this report</h1><ul>', type: :header
           board_lines = []
           included_projects.each do |project|
-            project.all_boards.values.each do |board|
+            project.all_boards.each_value do |board|
               board_lines << "<a href='#{project.file_prefix}.html'>#{board.name}</a> from project #{project.name}"
             end
           end

data/lib/jirametrics/examples/standard_project.rb

@@ -2,12 +2,11 @@
 
 # This file is really intended to give you ideas about how you might configure your own reports, not
 # as a complete setup that will work in every case.
-#
-# See https://github.com/mikebowler/jirametrics/wiki/Examples-folder for more
 class Exporter
   def standard_project name:, file_prefix:, ignore_issues: nil, starting_status: nil, boards: {},
       default_board: nil, anonymize: false, settings: {}, status_category_mappings: {},
-      rolling_date_count: 90, no_earlier_than: nil
+      rolling_date_count: 90, no_earlier_than: nil, ignore_types: %w[Sub-task Subtask Epic],
+      show_experimental_charts: false
 
     project name: name do
       puts name
@@ -38,11 +37,14 @@ class Exporter
         end
       end
 
+      issues.reject! do |issue|
+        ignore_types.include? issue.type
+      end
+
+      discard_changes_before status_becomes: (starting_status || :backlog) # rubocop:disable Style/RedundantParentheses
+
       file do
         file_suffix '.html'
-        issues.reject! do |issue|
-          %w[Sub-task Epic].include? issue.type
-        end
 
         issues.reject! { |issue| ignore_issues.include? issue.key } if ignore_issues
 
@@ -52,12 +54,10 @@ class Exporter
           html "<H1>#{name}</H1>", type: :header
           boards.each_key do |id|
             board = find_board id
-            html "<div><a href='#{board.url}'>#{id} #{board.name}</a></div>",
+            html "<div><a href='#{board.url}'>#{id} #{board.name}</a> (#{board.board_type})</div>",
                  type: :header
           end
 
-          discard_changes_before status_becomes: (starting_status || :backlog) # rubocop:disable Style/RedundantParentheses
-
           cycletime_scatterplot do
             show_trend_lines
           end
@@ -84,6 +84,7 @@ class Exporter
           daily_wip_by_age_chart
           daily_wip_by_blocked_stalled_chart
           daily_wip_by_parent_chart
+          flow_efficiency_scatterplot if show_experimental_charts
           expedited_chart
           sprint_burndown
           estimate_accuracy_chart

data/lib/jirametrics/expedited_chart.rb

@@ -109,8 +109,7 @@ class ExpeditedChart < ChartBase
 
   def make_expedite_lines_data_set issue:, expedite_data:
     cycletime = issue.board.cycletime
-    started_time = cycletime.started_time(issue)
-    stopped_time = cycletime.stopped_time(issue)
+    started_time, stopped_time = cycletime.started_stopped_times(issue)
 
     expedite_data << [started_time, :issue_started] if started_time
     expedite_data << [stopped_time, :issue_stopped] if stopped_time

data/lib/jirametrics/exporter.rb

@@ -72,6 +72,31 @@ class Exporter
     puts "Full output from downloader in #{file_system.logfile_name}"
   end
 
+  def info keys, name_filter:
+    selected = []
+    each_project_config(name_filter: name_filter) do |project|
+      project.evaluate_next_level
+      # next if project.aggregated_project?
+
+      project.run load_only: true
+      project.board_configs.each do |board_config|
+        board_config.run
+      end
+      project.issues.each do |issue|
+        selected << [project, issue] if keys.include? issue.key
+      end
+    rescue => e # rubocop:disable Style/RescueStandardError
+      # This happens when we're attempting to load an aggregated project because it hasn't been
+      # properly initialized. Since we don't care about aggregated projects, we just ignore it.
+      raise unless e.message.start_with? 'This is an aggregated project and issues should have been included'
+    end
+
+    selected.each do |project, issue|
+      puts "\nProject #{project.name}"
+      puts issue.dump
+    end
+  end
+
   def each_project_config name_filter:
     @project_configs.each do |project|
       yield project if project.name.nil? || File.fnmatch(name_filter, project.name)

data/lib/jirametrics/file_system.rb

@@ -29,7 +29,7 @@ class FileSystem
 
   def log message, also_write_to_stderr: false
     logfile.puts message
-    $stderr.puts message if also_write_to_stderr
+    $stderr.puts message if also_write_to_stderr # rubocop:disable Style/StderrPuts
   end
 
   # In some Jira instances, a sizeable portion of the JSON is made up of empty fields. I've seen

data/lib/jirametrics/flow_efficiency_scatterplot.rb

@@ -0,0 +1,111 @@
+# frozen_string_literal: true
+
+require 'jirametrics/groupable_issue_chart'
+
+class FlowEfficiencyScatterplot < ChartBase
+  include GroupableIssueChart
+
+  attr_accessor :possible_statuses
+
+  def initialize block
+    super()
+
+    header_text 'Flow Efficiency'
+    description_text <<-HTML
+      <div class="p">
+        This chart shows the active time against the the total time spent on a ticket.
+        <a href="https://improvingflow.com/2024/07/06/flow-efficiency.html">Flow  efficiency</a> is the ratio
+        between these two numbers.
+      </div>
+      <div class="p">
+        <math>
+          <mn>Flow efficiency (%)</mn>
+          <mo>=</mo>
+          <mfrac>
+            <mrow><mn>Time adding value</mn></mrow>
+            <mrow><mn>Total time</mn></mrow>
+          </mfrac>
+        </math>
+      </div>
+      <div style="background: yellow">Note that for this calculation to be accurate, we must be moving items into a
+        blocked or stalled state the moment we stop working on it, and most teams don't do that.
+        So be aware that your team may have to change their behaviours if you want this chart to be useful.
+      </div>
+    HTML
+
+    init_configuration_block block do
+      grouping_rules do |issue, rule|
+        active_time, total_time = issue.flow_efficiency_numbers end_time: time_range.end
+        flow_efficiency = active_time * 100.0 / total_time
+
+        if flow_efficiency > 99.0
+          rule.label = '~100%'
+          rule.color = 'green'
+        elsif flow_efficiency < 30.0
+          rule.label = '< 30%'
+          rule.color = 'orange'
+        else
+          rule.label = 'The rest'
+          rule.color = 'black'
+        end
+      end
+    end
+
+    @percentage_lines = []
+    @highest_cycletime = 0
+  end
+
+  def run
+    data_sets = group_issues(completed_issues_in_range include_unstarted: false).filter_map do |rules, issues|
+      create_dataset(issues: issues, label: rules.label, color: rules.color)
+    end
+
+    return "<h1>#{@header_text}</h1>No data matched the selected criteria. Nothing to show." if data_sets.empty?
+
+    wrap_and_render(binding, __FILE__)
+  end
+
+  def to_days seconds
+    seconds / 60 / 60 / 24
+  end
+
+  def create_dataset issues:, label:, color:
+    return nil if issues.empty?
+
+    data = issues.filter_map do |issue|
+      active_time, total_time = issue.flow_efficiency_numbers(
+        end_time: time_range.end, settings: settings
+      )
+
+      active_days = to_days(active_time)
+      total_days = to_days(total_time)
+      flow_efficiency = active_time * 100.0 / total_time
+
+      if flow_efficiency.nan?
+        # If this happens then something is probably misconfigured. We've seen it in production though
+        # so we have to handle it.
+        file_system.log(
+          "Issue(#{issue.key}) flow_efficiency: NaN, active_time: #{active_time}, total_time: #{total_time}"
+        )
+        flow_efficiency = 0.0
+      end
+
+      {
+        y: active_days,
+        x: total_days,
+        title: [
+          "#{issue.key} : #{issue.summary}, flow efficiency: #{flow_efficiency.to_i}%," \
+          " total: #{total_days.round(1)} days," \
+          " active: #{active_days.round(1)} days"
+        ]
+      }
+    end
+    {
+      label: label,
+      data: data,
+      fill: false,
+      showLine: false,
+      backgroundColor: color
+    }
+  end
+end

data/lib/jirametrics/html/flow_efficiency_scatterplot.erb

@@ -0,0 +1,85 @@
+<div class="chart">
+  <canvas id="<%= chart_id %>" width="<%= canvas_width %>" height="<%= canvas_height %>"></canvas>
+</div>
+<script>
+new Chart(document.getElementById('<%= chart_id %>').getContext('2d'), {
+  type: 'scatter',
+  data: {
+    datasets: <%= JSON.generate(data_sets) %>
+  },
+  options: {
+    title: {
+      display: true,
+      text: "Cycletime Scatterplot"
+    },
+    responsive: <%= canvas_responsive? %>, // If responsive is true then it fills the screen
+    scales: {
+      x: {
+        scaleLabel: {
+          display: true,
+          labelString: 'Days'
+        },
+        title: {
+          display: true,
+          text: 'Total time (days)'
+        },
+        grid: {
+          color: <%= CssVariable['--grid-line-color'].to_json %>
+        },
+
+      },
+      y: {
+        scaleLabel: {
+          display: true,
+          labelString: 'Percentage',
+          min: 0,
+          max: <%= @highest_cycletime %>
+        },
+        title: {
+          display: true,
+          text: 'Time adding value (days)'
+        },
+        grid: {
+          color: <%= CssVariable['--grid-line-color'].to_json %>
+        },
+      }
+    },
+    plugins: {
+      tooltip: {
+        callbacks: {
+          label: function(context) {
+            return context.dataset.data[context.dataIndex].title
+          }
+        }
+      },
+      autocolors: false,
+      legend: {
+        onClick: (evt, legendItem, legend) => {
+          // Find the datasetMeta that corresponds to the item clicked
+          var i = 0
+          while(legendItem.text != legend.chart.getDatasetMeta(i).label) {
+            i++;
+          }
+          nextVisibility = !!legend.chart.getDatasetMeta(i).hidden;
+
+          // Hide/show the 85% line for that dataset
+          legend.chart.options.plugins.annotation.annotations["line"+(i/2)].display = nextVisibility;
+
+          // Hide/show the trendline for this dataset, if they were enabled. The trendline is always
+          // there but not always visible.
+          legend.chart.setDatasetVisibility(i+1, <%= !!@show_trend_lines %> && nextVisibility);
+
+          // Still run the default behaviour
+          Chart.defaults.plugins.legend.onClick(evt, legendItem, legend);
+        },
+        labels: {
+          filter: function(item, chart) {
+            // Logic to remove a particular legend item goes here
+            return !item.text.includes('Trendline');
+          }
+        }
+      }
+    }
+  }
+});
+</script>

data/lib/jirametrics/html/index.css

@@ -99,9 +99,6 @@ table.standard {
     background-color: #eee;
   }
 }
-.quality_note_bullet {
-  color: red;
-}
 
 .chart {
   background-color: white;
@@ -119,6 +116,16 @@ div.color_block {
   border: 1px solid black;
 }
 
+ul.quality_report {
+  list-style-type: '⮕';
+  ::marker {
+    color: red;
+  }
+  li {
+    padding: 0.2em;
+  }
+}
+
 @media screen and (prefers-color-scheme: dark) {
   :root {
     --non-working-days-color: #2f2f2f;

data/lib/jirametrics/html_report_config.rb

@@ -33,6 +33,7 @@ class HtmlReportConfig
   define_chart name: 'cycletime_histogram', classname: 'CycletimeHistogram'
   define_chart name: 'estimate_accuracy_chart', classname: 'EstimateAccuracyChart'
   define_chart name: 'hierarchy_table', classname: 'HierarchyTable'
+  define_chart name: 'flow_efficiency_scatterplot', classname: 'FlowEfficiencyScatterplot'
 
   define_chart name: 'daily_wip_by_type', classname: 'DailyWipChart',
     deprecated_warning: 'This is the same as daily_wip_chart. Please use that one', deprecated_date: '2024-05-23'
@@ -145,7 +146,7 @@ class HtmlReportConfig
 
     @original_issue_times = {}
     issues_cutoff_times.each do |issue, cutoff_time|
-      started = issue.board.cycletime.started_time(issue)
+      started = issue.board.cycletime.started_stopped_times(issue).first
       if started && started <= cutoff_time
         # We only need to log this if data was discarded
         @original_issue_times[issue] = { cutoff_time: cutoff_time, started_time: started }

data/lib/jirametrics/issue.rb

@@ -13,6 +13,7 @@ class Issue
     @changes = []
     @board = board
 
+    raise "No board for issue #{key}" if board.nil?
     return unless @raw['changelog']
 
     load_history_into_changes
@@ -140,7 +141,7 @@ class Issue
     @board.project_config.file_system.log(
       "Warning: Status name #{name.inspect} for issue #{key} not found in" \
       " #{board.possible_statuses.collect(&:name).inspect}" \
-      "\n  See Q1 in the FAQ for more details: https://github.com/mikebowler/jirametrics/wiki/FAQ\n",
+      "\n  See https://jirametrics.org/faq/#q1\n",
       also_write_to_stderr: true
     )
     status = Status.new(name: name, category_name: 'In Progress')
@@ -259,7 +260,7 @@ class Issue
 
     blocked_link_texts = settings['blocked_link_text']
     stalled_threshold = settings['stalled_threshold_days']
-    flagged_means_blocked = !!settings['flagged_means_blocked']
+    flagged_means_blocked = !!settings['flagged_means_blocked'] # rubocop:disable Style/DoubleNegation
 
     blocking_issue_keys = []
 
@@ -373,12 +374,11 @@ class Issue
 
   # return [number of active seconds, total seconds] that this issue had up to the end_time.
   # It does not include data before issue start or after issue end
-  def flow_efficiency_numbers end_time:, settings: {}
-    issue_start = @board.cycletime.started_time(self)
+  def flow_efficiency_numbers end_time:, settings: @board.project_config.settings
+    issue_start, issue_stop = @board.cycletime.started_stopped_times(self)
     return [0.0, 0.0] if !issue_start || issue_start > end_time
 
     value_add_time = 0.0
-    issue_stop = @board.cycletime.stopped_time(self)
     end_time = issue_stop if issue_stop && issue_stop < end_time
 
     active_start = nil
@@ -542,6 +542,20 @@ class Issue
     comparison
   end
 
+  def discard_changes_before cutoff_time
+    rejected_any = false
+    @changes.reject! do |change|
+      reject = change.status? && change.time <= cutoff_time && change.artificial? == false
+      if reject
+        (@discarded_changes ||= []) << change
+        rejected_any = true
+      end
+      reject
+    end
+
+    (@discarded_change_times ||= []) << cutoff_time if rejected_any
+  end
+
   def dump
     result = +''
     result << "#{key} (#{type}): #{compact_text summary, 200}\n"
@@ -549,21 +563,59 @@ class Issue
     assignee = raw['fields']['assignee']
     result << "  [assignee] #{assignee['name'].inspect} <#{assignee['emailAddress']}>\n" unless assignee.nil?
 
-    raw['fields']['issuelinks'].each do |link|
+    raw['fields']['issuelinks']&.each do |link|
       result << "  [link] #{link['type']['outward']} #{link['outwardIssue']['key']}\n" if link['outwardIssue']
       result << "  [link] #{link['type']['inward']} #{link['inwardIssue']['key']}\n" if link['inwardIssue']
     end
-    changes.each do |change|
+    history = [] # time, type, detail
+
+    started_at, stopped_at = board.cycletime.started_stopped_times(self)
+    history << [started_at, nil, '↓↓↓↓ Started here ↓↓↓↓', true] if started_at
+    history << [stopped_at, nil, '↑↑↑↑ Finished here ↑↑↑↑', true] if stopped_at
+
+    @discarded_change_times&.each do |time|
+      history << [time, nil, '↑↑↑↑ Changes discarded ↑↑↑↑', true]
+    end
+
+    (changes + (@discarded_changes || [])).each do |change|
       value = change.value
       old_value = change.old_value
 
-      message = "  [change] #{change.time.strftime '%Y-%m-%d %H:%M:%S %z'} [#{change.field}] "
+      message = +''
       message << "#{compact_text(old_value).inspect} -> " unless old_value.nil? || old_value.empty?
       message << compact_text(value).inspect
-      message << " (#{change.author})"
-      message << ' <<artificial entry>>' if change.artificial?
-      result << message << "\n"
+      if change.artificial?
+        message << ' (Artificial entry)' if change.artificial?
+      else
+        message << " (Author: #{change.author})"
+      end
+      history << [change.time, change.field, message, change.artificial?]
+    end
+
+    result << "  History:\n"
+    type_width = history.collect { |_time, type, _detail, _artificial| type&.length || 0 }.max
+    history.sort! do |a, b|
+      if a[0] == b[0]
+        if a[1].nil?
+          1
+        elsif b[1].nil?
+          -1
+        else
+          a[1] <=> b[1]
+        end
+      else
+        a[0] <=> b[0]
+      end
     end
+    history.each do |time, type, detail, _artificial|
+      if type.nil?
+        type = '-' * type_width
+      else
+        type = (' ' * (type_width - type.length)) << type
+      end
+      result << "    #{time.strftime '%Y-%m-%d %H:%M:%S %z'} [#{type}] #{detail}\n"
+    end
+
     result
   end
 

data/lib/jirametrics/jira_gateway.rb

@@ -16,7 +16,7 @@ class JiraGateway
     result = call_command command
     JSON.parse result
   rescue => e # rubocop:disable Style/RescueStandardError
-    puts "Error #{e.inspect} when parsing result: #{result.inspect}"
+    raise "Error #{e.message.inspect} when parsing result: #{result.inspect}"
   end
 
   def call_command command