jirametrics 2.5 → 2.30

data/lib/jirametrics/examples/standard_project.rb

@@ -2,49 +2,52 @@
 
 # This file is really intended to give you ideas about how you might configure your own reports, not
 # as a complete setup that will work in every case.
-#
-# See https://github.com/mikebowler/jirametrics/wiki/Examples-folder for more
 class Exporter
   def standard_project name:, file_prefix:, ignore_issues: nil, starting_status: nil, boards: {},
       default_board: nil, anonymize: false, settings: {}, status_category_mappings: {},
-      rolling_date_count: 90, no_earlier_than: nil
-
+      rolling_date_count: 90, no_earlier_than: nil, ignore_types: %w[Sub-task Subtask Epic],
+      show_experimental_charts: false, github_repos: nil
+    exporter = self
     project name: name do
-      puts name
+      file_system.log name, also_write_to_stderr: true
+      file_prefix file_prefix
+
       self.anonymize if anonymize
+      self.settings.merge! stringify_keys(settings)
 
-      self.settings.merge! settings
+      boards.each_key do |board_id|
+        block = boards[board_id]
+        if block == :default
+          block = lambda do |_|
+            start_at first_time_in_status_category(:indeterminate)
+            stop_at still_in_status_category(:done)
+          end
+        end
+        board id: board_id do
+          cycletime(&block)
+        end
+      end
 
       status_category_mappings.each do |status, category|
         status_category_mapping status: status, category: category
       end
 
-      file_prefix file_prefix
       download do
         self.rolling_date_count(rolling_date_count) if rolling_date_count
         self.no_earlier_than(no_earlier_than) if no_earlier_than
+        github_repo *github_repos if github_repos
       end
 
-      boards.each_key do |board_id|
-        block = boards[board_id]
-        if block == :default
-          block = lambda do |_|
-            start_at first_time_in_status_category('In Progress')
-            stop_at still_in_status_category('Done')
-          end
-        end
-        board id: board_id do
-          cycletime(&block)
-        end
+      issues.reject! do |issue|
+        ignore_types.include? issue.type
       end
 
+      exporter.filter_issues issues, ignore_issues
+
+      discard_changes_before status_becomes: (starting_status || :backlog) # rubocop:disable Style/RedundantParentheses
+
       file do
         file_suffix '.html'
-        issues.reject! do |issue|
-          %w[Sub-task Epic].include? issue.type
-        end
-
-        issues.reject! { |issue| ignore_issues.include? issue.key } if ignore_issues
 
         html_report do
           board_id default_board if default_board
@@ -52,39 +55,42 @@ class Exporter
           html "<H1>#{name}</H1>", type: :header
           boards.each_key do |id|
             board = find_board id
-            html "<div><a href='#{board.url}'>#{id} #{board.name}</a></div>",
+            html "<div><a href='#{board.url}'>#{id} #{board.name}</a> (#{board.board_type})</div>",
                  type: :header
           end
-
-          discard_changes_before status_becomes: (starting_status || :backlog) # rubocop:disable Style/RedundantParentheses
-
+          daily_view
+          cumulative_flow_diagram
           cycletime_scatterplot do
             show_trend_lines
           end
           cycletime_histogram
 
           throughput_chart do
-            description_text '<h2>Number of items completed, grouped by issue type</h2>'
+            description_text <<~TEXT
+              <div>Throughput data is very useful for#{' '}
+                <a href="https://blog.mikebowler.ca/2024/06/02/probabilistic-forecasting/">probabilistic forecasting</a>,
+                to determine when we'll be done. Try it now with the
+                <a href="<%= throughput_forecaster_url %>" target="_blank" rel="noopener noreferrer">
+                Focused Objective throughput forecaster,</a> to see how long it would take to complete all of the
+                <%= @not_started_count %> items you currently have in your backlog.
+              </div>
+              <h2>Number of items completed, grouped by issue type</h2>'
+            TEXT
           end
-          throughput_chart do
-            header_text nil
+          throughput_by_completed_resolution_chart do
             description_text '<h2>Number of items completed, grouped by completion status and resolution</h2>'
-            grouping_rules do |issue, rules|
-              if issue.resolution
-                rules.label = "#{issue.status.name}:#{issue.resolution}"
-              else
-                rules.label = issue.status.name
-              end
-            end
           end
 
           aging_work_in_progress_chart
+          wip_by_column_chart do
+            show_recommendations
+          end
           aging_work_bar_chart
           aging_work_table
           daily_wip_by_age_chart
           daily_wip_by_blocked_stalled_chart
           daily_wip_by_parent_chart
-          expedited_chart
+          flow_efficiency_scatterplot if show_experimental_charts
           sprint_burndown
           estimate_accuracy_chart
           dependency_chart
@@ -92,4 +98,14 @@ class Exporter
       end
     end
   end
+
+  # Extracted as a separate method so it can be tested independently, without needing to invoke
+  # the full standard_project DSL setup.
+  def filter_issues issues, ignore_issues
+    return unless ignore_issues
+
+    issues.reject! do |issue|
+      ignore_issues.is_a?(Proc) ? ignore_issues.call(issue) : ignore_issues.include?(issue.key)
+    end
+  end
 end

data/lib/jirametrics/expedited_chart.rb

@@ -38,6 +38,8 @@ class ExpeditedChart < ChartBase
       </div>
       #{describe_non_working_days}
     HTML
+    @x_axis_title = 'Date'
+    @y_axis_title = 'Age in days'
 
     instance_eval(&block)
   end
@@ -48,7 +50,7 @@ class ExpeditedChart < ChartBase
     end
 
     if data_sets.empty?
-      '<h1>Expedited work</h1>There is no expedited work in this time period.'
+      '<h1 class="foldable">Expedited work</h1><div>There is no expedited work in this time period.</div>'
     else
       wrap_and_render(binding, __FILE__)
     end
@@ -63,7 +65,7 @@ class ExpeditedChart < ChartBase
       next unless change.priority?
 
       if expedited_priority_names.include? change.value
-        expedite_start = change.time
+        expedite_start = change.time.to_date
       elsif expedite_start
         start_date = expedite_start.to_date
         stop_date = change.time.to_date
@@ -72,7 +74,7 @@ class ExpeditedChart < ChartBase
            (start_date < date_range.begin && stop_date > date_range.end)
 
           result << [expedite_start, :expedite_start]
-          result << [change.time, :expedite_stop]
+          result << [change.time.to_date, :expedite_stop]
         end
         expedite_start = nil
       end
@@ -109,12 +111,11 @@ class ExpeditedChart < ChartBase
 
   def make_expedite_lines_data_set issue:, expedite_data:
     cycletime = issue.board.cycletime
-    started_time = cycletime.started_time(issue)
-    stopped_time = cycletime.stopped_time(issue)
+    started_date, stopped_date = cycletime.started_stopped_dates(issue)
 
-    expedite_data << [started_time, :issue_started] if started_time
-    expedite_data << [stopped_time, :issue_stopped] if stopped_time
-    expedite_data.sort_by! { |a| a[0] }
+    expedite_data << [started_date, :issue_started] if started_date
+    expedite_data << [stopped_date, :issue_stopped] if stopped_date
+    expedite_data.sort_by!(&:first)
 
     # If none of the data would be visible on the chart then skip it.
     return nil unless expedite_data.any? { |time, _action| time.to_date >= date_range.begin }
@@ -151,7 +152,7 @@ class ExpeditedChart < ChartBase
 
     unless expedite_data.empty?
       last_change_time = expedite_data[-1][0].to_date
-      if last_change_time && last_change_time <= date_range.end && stopped_time.nil?
+      if last_change_time && last_change_time <= date_range.end && stopped_date.nil?
         data << make_point(issue: issue, time: date_range.end, label: 'Still ongoing', expedited: expedited)
         dot_colors << '' # It won't be visible so it doesn't matter
         point_styles << 'dash'

data/lib/jirametrics/exporter.rb

@@ -2,25 +2,19 @@
 
 require 'fileutils'
 
-class Object
-  def deprecated message:, date:
-    text = +''
-    text << "Deprecated(#{date}): "
-    text << message
-    caller(1..2).each do |line|
-      text << "\n-> Called from #{line}"
-    end
-    warn text
-  end
-end
-
 class Exporter
   attr_reader :project_configs
   attr_accessor :file_system
 
   def self.configure &block
     logfile_name = 'jirametrics.log'
-    logfile = File.open logfile_name, 'w'
+    logfile = File.open(logfile_name, 'w')
+  rescue Errno::EACCES
+    # FileSystem can't be used here — it hasn't been created yet (it depends on this logfile).
+    warn "Error: Cannot write to #{File.expand_path(logfile_name)}. " \
+      'Please ensure the current directory is writable.'
+    exit 1
+  else
     file_system = FileSystem.new
     file_system.logfile = logfile
     file_system.logfile_name = logfile_name
@@ -52,6 +46,7 @@ class Exporter
 
   def download name_filter:
     @downloading = true
+    github_pr_cache = {}
     each_project_config(name_filter: name_filter) do |project|
       project.evaluate_next_level
       next if project.aggregated_project?
@@ -62,16 +57,54 @@ class Exporter
       end
 
       project.download_config.run
-      downloader = Downloader.new(
+      gateway = JiraGateway.new(
+        file_system: file_system, jira_config: project.jira_config, settings: project.settings
+      )
+      downloader = Downloader.create(
         download_config: project.download_config,
         file_system: file_system,
-        jira_gateway: JiraGateway.new(file_system: file_system)
+        jira_gateway: gateway,
+        github_pr_cache: github_pr_cache
       )
       downloader.run
     end
     puts "Full output from downloader in #{file_system.logfile_name}"
   end
 
+  def info key, name_filter:
+    selected = []
+    file_system.log_only = true
+    each_project_config(name_filter: name_filter) do |project|
+      project.evaluate_next_level
+
+      project.run load_only: true
+      project.issues.each do |issue|
+        selected << [project, issue] if key == issue.key
+        issue.subtasks.each do |subtask|
+          selected << [project, subtask] if key == subtask.key
+        end
+      end
+    rescue => e # rubocop:disable Style/RescueStandardError
+      # This happens when we're attempting to load an aggregated project because it hasn't been
+      # properly initialized. Since we don't care about aggregated projects, we just ignore it.
+      raise unless e.message.start_with? 'This is an aggregated project and issues should have been included'
+    end
+    file_system.log_only = false
+
+    if selected.empty?
+      file_system.log "No issues found to match #{key.inspect}"
+    else
+      selected.each do |project, issue|
+        file_system.log "\nProject #{project.name}", also_write_to_stderr: true
+        file_system.log issue.dump, also_write_to_stderr: true
+      end
+    end
+  end
+
+  def stitch stitch_file
+    Stitcher.new(file_system: file_system).run(stitch_file: stitch_file)
+  end
+
   def each_project_config name_filter:
     @project_configs.each do |project|
       yield project if project.name.nil? || File.fnmatch(name_filter, project.name)
@@ -103,7 +136,9 @@ class Exporter
 
   def jira_config filename = nil
     if filename
-      @jira_config = file_system.load_json(filename)
+      @jira_config = file_system.load_json(filename, fail_on_error: false)
+      raise "Unable to load Jira configuration file and cannot continue: #{filename.inspect}" if @jira_config.nil?
+
       @jira_config['url'] = $1 if @jira_config['url'] =~ /^(.+)\/+$/
     end
     @jira_config

data/lib/jirametrics/file_config.rb

@@ -13,7 +13,7 @@ class FileConfig
   end
 
   def run
-    @issues = project_config.issues.dup
+    @issues = project_config.issues
     instance_eval(&@block)
 
     if @columns
@@ -56,7 +56,7 @@ class FileConfig
   def output_filename
     segments = []
     segments << project_config.target_path
-    segments << project_config.file_prefix
+    segments << project_config.get_file_prefix
     segments << (@file_suffix || "-#{@today}.csv")
     segments.join
   end
@@ -65,8 +65,8 @@ class FileConfig
   # most common usecase - the Team Dashboard from FocusedObjective.com. The rule for that one
   # is that all empty values in the first column should be at the bottom.
   def sort_output all_lines
-    all_lines.sort do |a, b|
-      if a[0] == b[0]
+    all_lines.each_with_index.sort do |(a, a_idx), (b, b_idx)|
+      result = if a[0] == b[0]
         a[1..] <=> b[1..]
       elsif a[0].nil?
         1
@@ -75,7 +75,10 @@ class FileConfig
       else
         a[0] <=> b[0]
       end
-    end
+
+      # When objects aren't comparable, preserve original order for a stable sort.
+      result.nil? || result.zero? ? a_idx <=> b_idx : result
+    end.map(&:first)
   end
 
   def columns &block
@@ -85,6 +88,11 @@ class FileConfig
 
   def html_report &block
     assert_only_one_filetype_config_set
+    if block.nil?
+      project_config.file_system.warning 'No charts were specified for the report. This is almost certainly a mistake.'
+      block = ->(_) {}
+    end
+
     @html_report = HtmlReportConfig.new file_config: self, block: block
   end
 
@@ -103,7 +111,7 @@ class FileConfig
   def to_datetime object
     return nil if object.nil?
 
-    object = object.to_datetime
+    object = object.to_time.to_datetime
     object = object.new_offset(@timezone_offset) if @timezone_offset
     object
   end
@@ -120,4 +128,11 @@ class FileConfig
     @file_suffix = suffix unless suffix.nil?
     @file_suffix
   end
+
+  def children
+    result = []
+    result << @columns if @columns
+    result << @html_report if @html_report
+    result
+  end
 end

data/lib/jirametrics/file_system.rb

@@ -3,17 +3,29 @@
 require 'json'
 
 class FileSystem
-  attr_accessor :logfile, :logfile_name
+  attr_accessor :logfile, :logfile_name, :log_only
+
+  def initialize
+    # In almost all cases, this will be immediately replaced in the Exporter
+    # but if we fail before we get that far, this will at least let a useful
+    # error show up on the console.
+    @logfile = $stdout
+    @log_only = false
+  end
 
   # Effectively the same as File.read except it forces the encoding to UTF-8
-  def load filename
+  def load filename, supress_deprecation: false
+    if filename.end_with?('.json') && !supress_deprecation
+      deprecated(message: 'call load_json instead', date: '2024-11-13')
+    end
+
     File.read filename, encoding: 'UTF-8'
   end
 
   def load_json filename, fail_on_error: true
     return nil if fail_on_error == false && File.exist?(filename) == false
 
-    JSON.parse load(filename)
+    JSON.parse load(filename, supress_deprecation: true)
   end
 
   def save_json json:, filename:
@@ -27,8 +39,62 @@ class FileSystem
     File.write(filename, content)
   end
 
-  def log message
+  def mkdir path
+    FileUtils.mkdir_p path
+  end
+
+  def utime file:, time:
+    File.utime time, time, file
+  end
+
+  def warning message, more: nil
+    log "Warning: #{message}", more: more, also_write_to_stderr: true
+  end
+
+  def error message, more: nil
+    log "Error: #{message}", more: more, also_write_to_stderr: true
+  end
+
+  def log message, more: nil, also_write_to_stderr: false
+    message += " See #{logfile_name} for more details about this message." if more
+
+    logfile.puts message
+    logfile.puts more if more
+    return if log_only || !also_write_to_stderr
+
+    # Obscure edge-case where we're trying to log something before logging is even
+    # set up. Quick escape here so that we don't dump the error twice.
+    return if logfile == $stdout
+
+    $stderr.puts message # rubocop:disable Style/StderrPuts
+  end
+
+  def log_start message
     logfile.puts message
+    return if log_only || logfile == $stdout
+
+    $stderr.print message
+    $stderr.flush
+  end
+
+  def start_progress
+    return if log_only
+
+    $stderr.print '  '
+    $stderr.flush
+  end
+
+  def progress_dot
+    return if log_only
+
+    $stderr.print '.'
+    $stderr.flush
+  end
+
+  def end_progress
+    return if log_only
+
+    $stderr.puts '' # rubocop:disable Style/StderrPuts
   end
 
   # In some Jira instances, a sizeable portion of the JSON is made up of empty fields. I've seen
@@ -42,4 +108,30 @@ class FileSystem
     end
     node
   end
+
+  def foreach root, &block
+    Dir.foreach root, &block
+  end
+
+  def file_exist? filename
+    File.exist?(filename) && File.file?(filename)
+  end
+
+  def dir_exist? path
+    File.exist?(path) && File.directory?(path)
+  end
+
+  def unlink filename
+    File.unlink filename
+  end
+
+  def deprecated message:, date:, depth: 2
+    text = +''
+    text << "Deprecated(#{date}): "
+    text << message
+    caller(1..depth).each do |line|
+      text << "\n-> Called from #{line}"
+    end
+    log text, also_write_to_stderr: true
+  end
 end

data/lib/jirametrics/fix_version.rb

@@ -11,11 +11,24 @@ class FixVersion
     @raw['name']
   end
 
+  def description
+    @raw['description']
+  end
+
   def id
     @raw['id'].to_i
   end
 
+  def release_date
+    text = @raw['releaseDate']
+    text.nil? ? nil : Date.parse(text)
+  end
+
   def released?
     @raw['released']
   end
+
+  def archived?
+    @raw['archived']
+  end
 end

data/lib/jirametrics/flow_efficiency_scatterplot.rb

@@ -0,0 +1,115 @@
+# frozen_string_literal: true
+
+require 'jirametrics/groupable_issue_chart'
+
+class FlowEfficiencyScatterplot < ChartBase
+  include GroupableIssueChart
+
+  attr_accessor :possible_statuses
+
+  def initialize block
+    super()
+
+    header_text 'Flow Efficiency'
+    description_text <<-HTML
+      <div class="p">
+        This chart shows the active time against the the total time spent on a ticket.
+        <a href="https://improvingflow.com/2024/07/06/flow-efficiency.html">Flow  efficiency</a> is the ratio
+        between these two numbers.
+      </div>
+      <div class="p">
+        <math>
+          <mn>Flow efficiency (%)</mn>
+          <mo>=</mo>
+          <mfrac>
+            <mrow><mn>Time adding value</mn></mrow>
+            <mrow><mn>Total time</mn></mrow>
+          </mfrac>
+        </math>
+      </div>
+      <div style="background: var(--warning-banner)">Note that for this calculation to be accurate, we must be moving items into a
+        blocked or stalled state the moment we stop working on it, and most teams don't do that.
+        So be aware that your team may have to change their behaviours if you want this chart to be useful.
+      </div>
+    HTML
+    @x_axis_title = 'Total time (days)'
+    @y_axis_title = 'Time adding value (days)'
+
+    init_configuration_block block do
+      grouping_rules do |issue, rule|
+        active_time, total_time = issue.flow_efficiency_numbers end_time: time_range.end
+        flow_efficiency = active_time * 100.0 / total_time
+
+        if flow_efficiency > 99.0
+          rule.label = '~100%'
+          rule.color = 'green'
+        elsif flow_efficiency < 30.0
+          rule.label = '< 30%'
+          rule.color = 'orange'
+        else
+          rule.label = 'The rest'
+          rule.color = 'black'
+        end
+      end
+    end
+
+    @percentage_lines = []
+    @highest_cycletime = 0
+  end
+
+  def run
+    data_sets = group_issues(completed_issues_in_range include_unstarted: false).filter_map do |rules, issues|
+      create_dataset(issues: issues, label: rules.label, color: rules.color)
+    end
+
+    if data_sets.empty?
+      return "<h1 class='foldable'>#{@header_text}</h1><div>No data matched the selected criteria. Nothing to show.</div>"
+    end
+
+    wrap_and_render(binding, __FILE__)
+  end
+
+  def to_days seconds
+    seconds / 60 / 60 / 24
+  end
+
+  def create_dataset issues:, label:, color:
+    return nil if issues.empty?
+
+    data = issues.filter_map do |issue|
+      active_time, total_time = issue.flow_efficiency_numbers(
+        end_time: time_range.end, settings: settings
+      )
+
+      active_days = to_days(active_time)
+      total_days = to_days(total_time)
+      flow_efficiency = active_time * 100.0 / total_time
+
+      if flow_efficiency.nan?
+        # If this happens then something is probably misconfigured. We've seen it in production though
+        # so we have to handle it.
+        file_system.log(
+          "Issue(#{issue.key}) flow_efficiency: NaN, active_time: #{active_time}, total_time: #{total_time}"
+        )
+        flow_efficiency = 0.0
+      end
+
+      {
+        y: active_days,
+        x: total_days,
+        title: [
+          "#{issue.key} : #{issue.summary}, flow efficiency: #{flow_efficiency.to_i}%," \
+          " total: #{total_days.round(1)} days," \
+          " active: #{active_days.round(1)} days"
+        ]
+      }
+    end
+    {
+      label: label,
+      data: data,
+      fill: false,
+      showLine: false,
+      backgroundColor: color
+    }
+  end
+end