jirametrics 2.0 → 2.12.1

data/lib/jirametrics/examples/standard_project.rb

@@ -2,41 +2,49 @@
 
 # This file is really intended to give you ideas about how you might configure your own reports, not
 # as a complete setup that will work in every case.
-#
-# See https://github.com/mikebowler/jirametrics/wiki/Examples-folder for more
 class Exporter
   def standard_project name:, file_prefix:, ignore_issues: nil, starting_status: nil, boards: {},
-      default_board: nil, anonymize: false
+      default_board: nil, anonymize: false, settings: {}, status_category_mappings: {},
+      rolling_date_count: 90, no_earlier_than: nil, ignore_types: %w[Sub-task Subtask Epic],
+      show_experimental_charts: false
 
     project name: name do
       puts name
+      file_prefix file_prefix
+
       self.anonymize if anonymize
+      self.settings.merge! settings
+
+      status_category_mappings.each do |status, category|
+        status_category_mapping status: status, category: category
+      end
 
-      settings['blocked_link_text'] = ['is blocked by']
-      file_prefix file_prefix
       download do
-        rolling_date_count 90
+        self.rolling_date_count(rolling_date_count) if rolling_date_count
+        self.no_earlier_than(no_earlier_than) if no_earlier_than
       end
 
       boards.each_key do |board_id|
         block = boards[board_id]
         if block == :default
           block = lambda do |_|
-            start_at first_time_in_status_category('In Progress')
-            stop_at still_in_status_category('Done')
+            start_at first_time_in_status_category(:indeterminate)
+            stop_at still_in_status_category(:done)
           end
         end
         board id: board_id do
           cycletime(&block)
-          expedited_priority_names 'Critical', 'Highest', 'Immediate Gating'
         end
       end
 
+      issues.reject! do |issue|
+        ignore_types.include? issue.type
+      end
+
+      discard_changes_before status_becomes: (starting_status || :backlog) # rubocop:disable Style/RedundantParentheses
+
       file do
         file_suffix '.html'
-        issues.reject! do |issue|
-          %w[Sub-task Epic].include? issue.type
-        end
 
         issues.reject! { |issue| ignore_issues.include? issue.key } if ignore_issues
 
@@ -46,11 +54,11 @@ class Exporter
           html "<H1>#{name}</H1>", type: :header
           boards.each_key do |id|
             board = find_board id
-            html "<div><a href='#{board.url}'>#{id} #{board.name}</a></div>",
+            html "<div><a href='#{board.url}'>#{id} #{board.name}</a> (#{board.board_type})</div>",
                  type: :header
           end
 
-          discard_changes_before status_becomes: (starting_status || :backlog)
+          daily_view
 
           cycletime_scatterplot do
             show_trend_lines
@@ -77,42 +85,12 @@ class Exporter
           aging_work_table
           daily_wip_by_age_chart
           daily_wip_by_blocked_stalled_chart
-          daily_wip_chart do
-            header_text 'Daily WIP by Parent'
-            description_text <<-TEXT
-              How much work is in progress, grouped by the parent of the issue. This will give us an
-              indication of how focused we are on higher level objectives. If there are many parent
-              tickets in progress at the same time, either this team has their focus scattered or we
-              aren't doing a good job of
-              <a href="https://improvingflow.com/2024/02/21/slicing-epics.html">splitting those parent
-              tickets</a>. Neither of those is desirable.
-            TEXT
-            grouping_rules do |issue, rules|
-              rules.label = issue.parent&.key || 'No parent'
-              rules.color = 'white' if rules.label == 'No parent'
-            end
-          end
+          daily_wip_by_parent_chart
+          flow_efficiency_scatterplot if show_experimental_charts
           expedited_chart
           sprint_burndown
-          story_point_accuracy_chart
-
-          dependency_chart do
-            link_rules do |link, rules|
-              case link.name
-              when 'Cloners'
-                rules.ignore
-              when 'Dependency', 'Blocks', 'Parent/Child', 'Cause', 'Satisfy Requirement', 'Relates'
-                rules.merge_bidirectional keep: 'outward'
-                rules.merge_bidirectional keep: 'outward'
-              when 'Sync'
-                rules.use_bidirectional_arrows
-              else
-                # This is a link type that we don't recognized. Dump it to standard out to draw attention
-                # to it.
-                puts "name=#{link.name}, label=#{link.label}"
-              end
-            end
-          end
+          estimate_accuracy_chart
+          dependency_chart
         end
       end
     end

data/lib/jirametrics/expedited_chart.rb

@@ -3,11 +3,14 @@
 require 'jirametrics/chart_base'
 
 class ExpeditedChart < ChartBase
-  EXPEDITED_SEGMENT = Object.new.tap do |segment|
+  EXPEDITED_SEGMENT = ChartBase.new.tap do |segment|
     def segment.to_json *_args
+      expedited = CssVariable.new('--expedited-color').to_json
+      not_expedited = CssVariable.new('--expedited-chart-no-longer-expedited').to_json
+
       <<~SNIPPET
         {
-          borderColor: ctx => expedited(ctx, 'red') || notExpedited(ctx, 'gray'),
+          borderColor: ctx => expedited(ctx, #{expedited}) || notExpedited(ctx, #{not_expedited}),
           borderDash: ctx => notExpedited(ctx, [6, 6])
         }
       SNIPPET
@@ -17,25 +20,26 @@ class ExpeditedChart < ChartBase
   attr_accessor :issues, :cycletime, :possible_statuses, :date_range
   attr_reader :expedited_label
 
-  def initialize
+  def initialize block
     super()
 
     header_text 'Expedited work'
     description_text <<-HTML
-      <p>
+      <div class="p">
         This chart only shows issues that have been expedited at some point. We care about these as
         any form of expedited work will affect the entire system and will slow down non-expedited work.
         Refer to this article on
         <a href="https://improvingflow.com/2021/06/16/classes-of-service.html">classes of service</a>
         for a longer explanation on why we want to avoid expedited work.
-      </p>
-      <p>
-        The lines indicate time that this issue was expedited. When the line is red then the issue was
-        expedited at that time. When it's gray then it wasn't. Orange dots indicate the date the work
-        was started and green dots represent the completion date. Lastly, the vertical height of the
-        lines/dots indicates how long it's been since this issue was created.
-      </p>
+      </div>
+      <div class="p">
+        The colour of the line indicates time that this issue was #{color_block '--expedited-color'} expedited
+        or #{color_block '--expedited-chart-no-longer-expedited'} not expedited.
+      </div>
+      #{describe_non_working_days}
     HTML
+
+    instance_eval(&block)
   end
 
   def run
@@ -53,13 +57,13 @@ class ExpeditedChart < ChartBase
   def prepare_expedite_data issue
     expedite_start = nil
     result = []
-    expedited_priority_names = issue.board.expedited_priority_names
+    expedited_priority_names = issue.board.project_config.settings['expedited_priority_names']
 
     issue.changes.each do |change|
       next unless change.priority?
 
       if expedited_priority_names.include? change.value
-        expedite_start = change.time
+        expedite_start = change.time.to_date
       elsif expedite_start
         start_date = expedite_start.to_date
         stop_date = change.time.to_date
@@ -68,7 +72,7 @@ class ExpeditedChart < ChartBase
            (start_date < date_range.begin && stop_date > date_range.end)
 
           result << [expedite_start, :expedite_start]
-          result << [change.time, :expedite_stop]
+          result << [change.time.to_date, :expedite_stop]
         end
         expedite_start = nil
       end
@@ -105,12 +109,11 @@ class ExpeditedChart < ChartBase
 
   def make_expedite_lines_data_set issue:, expedite_data:
     cycletime = issue.board.cycletime
-    started_time = cycletime.started_time(issue)
-    stopped_time = cycletime.stopped_time(issue)
+    started_date, stopped_date = cycletime.started_stopped_dates(issue)
 
-    expedite_data << [started_time, :issue_started] if started_time
-    expedite_data << [stopped_time, :issue_stopped] if stopped_time
-    expedite_data.sort_by! { |a| a[0] }
+    expedite_data << [started_date, :issue_started] if started_date
+    expedite_data << [stopped_date, :issue_stopped] if stopped_date
+    expedite_data.sort_by!(&:first)
 
     # If none of the data would be visible on the chart then skip it.
     return nil unless expedite_data.any? { |time, _action| time.to_date >= date_range.begin }
@@ -124,20 +127,20 @@ class ExpeditedChart < ChartBase
       case action
       when :issue_started
         data << make_point(issue: issue, time: time, label: 'Started', expedited: expedited)
-        dot_colors << 'orange'
+        dot_colors << CssVariable['--expedited-chart-dot-issue-started-color']
         point_styles << 'rect'
       when :issue_stopped
         data << make_point(issue: issue, time: time, label: 'Completed', expedited: expedited)
-        dot_colors << 'green'
+        dot_colors << CssVariable['--expedited-chart-dot-issue-stopped-color']
         point_styles << 'rect'
       when :expedite_start
         data << make_point(issue: issue, time: time, label: 'Expedited', expedited: true)
-        dot_colors << 'red'
+        dot_colors << CssVariable['--expedited-chart-dot-expedite-started-color']
         point_styles << 'circle'
         expedited = true
       when :expedite_stop
         data << make_point(issue: issue, time: time, label: 'Not expedited', expedited: false)
-        dot_colors << 'gray'
+        dot_colors << CssVariable['--expedited-chart-dot-expedite-stopped-color']
         point_styles << 'circle'
         expedited = false
       else
@@ -147,9 +150,9 @@ class ExpeditedChart < ChartBase
 
     unless expedite_data.empty?
       last_change_time = expedite_data[-1][0].to_date
-      if last_change_time && last_change_time <= date_range.end && stopped_time.nil?
+      if last_change_time && last_change_time <= date_range.end && stopped_date.nil?
         data << make_point(issue: issue, time: date_range.end, label: 'Still ongoing', expedited: expedited)
-        dot_colors << 'blue' # It won't be visible so it doesn't matter
+        dot_colors << '' # It won't be visible so it doesn't matter
         point_styles << 'dash'
       end
     end

data/lib/jirametrics/exporter.rb

@@ -2,21 +2,19 @@
 
 require 'fileutils'
 
-class Object
-  def deprecated message:
-    text = +''
-    text << 'Deprecated:'
-    text << message
-    text << "\n-> Called from #{caller(1..1).first}"
-    warn text
-  end
-end
-
 class Exporter
-  attr_reader :project_configs, :file_system
+  attr_reader :project_configs
+  attr_accessor :file_system
 
   def self.configure &block
-    exporter = Exporter.new
+    logfile_name = 'jirametrics.log'
+    logfile = File.open logfile_name, 'w'
+    file_system = FileSystem.new
+    file_system.logfile = logfile
+    file_system.logfile_name = logfile_name
+
+    exporter = Exporter.new file_system: file_system
+
     exporter.instance_eval(&block)
     @@instance = exporter
   end
@@ -25,11 +23,12 @@ class Exporter
 
   def initialize file_system: FileSystem.new
     @project_configs = []
-    @timezone_offset = '+00:00'
     @target_path = '.'
     @holiday_dates = []
     @downloading = false
     @file_system = file_system
+
+    timezone_offset '+00:00'
   end
 
   def export name_filter:
@@ -41,25 +40,49 @@ class Exporter
 
   def download name_filter:
     @downloading = true
-    logfile_name = 'downloader.log'
-    File.open logfile_name, 'w' do |logfile|
-      file_system.logfile = logfile
-      file_system.logfile_name = logfile_name
-
-      each_project_config(name_filter: name_filter) do |project|
-        project.evaluate_next_level
-        next if project.aggregated_project?
-
-        project.download_config.run
-        downloader = Downloader.new(
-          download_config: project.download_config,
-          file_system: file_system,
-          jira_gateway: JiraGateway.new(file_system: file_system)
-        )
-        downloader.run
+    each_project_config(name_filter: name_filter) do |project|
+      project.evaluate_next_level
+      next if project.aggregated_project?
+
+      unless project.download_config
+        raise "Project #{project.name.inspect} is missing a download section in the config. " \
+          'That is required in order to download'
+      end
+
+      project.download_config.run
+      downloader = Downloader.new(
+        download_config: project.download_config,
+        file_system: file_system,
+        jira_gateway: JiraGateway.new(file_system: file_system)
+      )
+      downloader.run
+    end
+    puts "Full output from downloader in #{file_system.logfile_name}"
+  end
+
+  def info keys, name_filter:
+    selected = []
+    each_project_config(name_filter: name_filter) do |project|
+      project.evaluate_next_level
+
+      project.run load_only: true
+      project.issues.each do |issue|
+        selected << [project, issue] if keys.include? issue.key
+      end
+    rescue => e # rubocop:disable Style/RescueStandardError
+      # This happens when we're attempting to load an aggregated project because it hasn't been
+      # properly initialized. Since we don't care about aggregated projects, we just ignore it.
+      raise unless e.message.start_with? 'This is an aggregated project and issues should have been included'
+    end
+
+    if selected.empty?
+      file_system.log "No issues found to match #{keys.collect(&:inspect).join(', ')}"
+    else
+      selected.each do |project, issue|
+        file_system.log "\nProject #{project.name}", also_write_to_stderr: true
+        file_system.log issue.dump, also_write_to_stderr: true
       end
     end
-    puts "Full output from downloader in #{logfile_name}"
   end
 
   def each_project_config name_filter:
@@ -73,7 +96,6 @@ class Exporter
   end
 
   def project name: nil, &block
-    raise 'target_path was never set!' if @target_path.nil?
     raise 'jira_config not set' if @jira_config.nil?
 
     @project_configs << ProjectConfig.new(
@@ -93,7 +115,12 @@ class Exporter
   end
 
   def jira_config filename = nil
-    @jira_config = file_system.load_json(filename) unless filename.nil?
+    if filename
+      @jira_config = file_system.load_json(filename, fail_on_error: false)
+      raise "Unable to load Jira configuration file and cannot continue: #{filename.inspect}" if @jira_config.nil?
+
+      @jira_config['url'] = $1 if @jira_config['url'] =~ /^(.+)\/+$/
+    end
     @jira_config
   end
 

data/lib/jirametrics/file_config.rb

@@ -5,24 +5,22 @@ require 'csv'
 class FileConfig
   attr_reader :project_config, :issues
 
-  def initialize project_config:, block:
+  def initialize project_config:, block:, today: Date.today
     @project_config = project_config
     @block = block
     @columns = nil
+    @today = today
   end
 
   def run
-    @issues = project_config.issues.dup
+    @issues = project_config.issues
     instance_eval(&@block)
 
     if @columns
       all_lines = prepare_grid
 
-      File.open(output_filename, 'w') do |file|
-        all_lines.each do |output_line|
-          file.puts CSV.generate_line(output_line)
-        end
-      end
+      content = all_lines.collect { |line| CSV.generate_line line }.join
+      project_config.exporter.file_system.save_file content: content, filename: output_filename
     elsif @html_report
       @html_report.run
     else
@@ -58,8 +56,8 @@ class FileConfig
   def output_filename
     segments = []
     segments << project_config.target_path
-    segments << project_config.file_prefix
-    segments << (@file_suffix || "-#{Date.today}.csv")
+    segments << project_config.get_file_prefix
+    segments << (@file_suffix || "-#{@today}.csv")
     segments.join
   end
 
@@ -68,13 +66,20 @@ class FileConfig
   # is that all empty values in the first column should be at the bottom.
   def sort_output all_lines
     all_lines.sort do |a, b|
-      if a[0].nil?
-        1
+      result = nil
+      if a[0] == b[0]
+        result = a[1..] <=> b[1..]
+      elsif a[0].nil?
+        result = 1
       elsif b[0].nil?
-        -1
+        result = -1
       else
-        a[0] <=> b[0]
+        result = a[0] <=> b[0]
       end
+
+      # This will only happen if one of the objects isn't comparable. Seen in production.
+      result = -1 if result.nil?
+      result
     end
   end
 
@@ -85,6 +90,11 @@ class FileConfig
 
   def html_report &block
     assert_only_one_filetype_config_set
+    if block.nil?
+      project_config.file_system.warning 'No charts were specified for the report. This is almost certainly a mistake.'
+      block = ->(_) {}
+    end
+
     @html_report = HtmlReportConfig.new file_config: self, block: block
   end
 
@@ -103,7 +113,7 @@ class FileConfig
   def to_datetime object
     return nil if object.nil?
 
-    object = object.to_datetime
+    object = object.to_time.to_datetime
     object = object.new_offset(@timezone_offset) if @timezone_offset
     object
   end
@@ -112,8 +122,19 @@ class FileConfig
     object.to_s
   end
 
+  def to_integer object
+    object.to_i
+  end
+
   def file_suffix suffix = nil
     @file_suffix = suffix unless suffix.nil?
     @file_suffix
   end
+
+  def children
+    result = []
+    result << @columns if @columns
+    result << @html_report if @html_report
+    result
+  end
 end

data/lib/jirametrics/file_system.rb

@@ -5,21 +5,48 @@ require 'json'
 class FileSystem
   attr_accessor :logfile, :logfile_name
 
+  # Effectively the same as File.read except it forces the encoding to UTF-8
+  def load filename, supress_deprecation: false
+    if filename.end_with?('.json') && !supress_deprecation
+      deprecated(message: 'call load_json instead', date: '2024-11-13')
+    end
+
+    File.read filename, encoding: 'UTF-8'
+  end
+
   def load_json filename, fail_on_error: true
     return nil if fail_on_error == false && File.exist?(filename) == false
 
-    JSON.parse File.read(filename)
+    JSON.parse load(filename, supress_deprecation: true)
   end
 
   def save_json json:, filename:
+    save_file content: JSON.pretty_generate(compress json), filename: filename
+  end
+
+  def save_file content:, filename:
     file_path = File.dirname(filename)
     FileUtils.mkdir_p file_path unless File.exist?(file_path)
 
-    File.write(filename, JSON.pretty_generate(compress json))
+    File.write(filename, content)
+  end
+
+  def warning message, more: nil
+    log "Warning: #{message}", more: more, also_write_to_stderr: true
+  end
+
+  def error message, more: nil
+    log "Error: #{message}", more: more, also_write_to_stderr: true
   end
 
-  def log message
+  def log message, more: nil, also_write_to_stderr: false
+    message += " See #{logfile_name} for more details about this message." if more
+
     logfile.puts message
+    logfile.puts more if more
+    return unless also_write_to_stderr
+
+    $stderr.puts message # rubocop:disable Style/StderrPuts
   end
 
   # In some Jira instances, a sizeable portion of the JSON is made up of empty fields. I've seen
@@ -33,4 +60,22 @@ class FileSystem
     end
     node
   end
+
+  def foreach root, &block
+    Dir.foreach root, &block
+  end
+
+  def file_exist? filename
+    File.exist? filename
+  end
+
+  def deprecated message:, date:, depth: 2
+    text = +''
+    text << "Deprecated(#{date}): "
+    text << message
+    caller(1..depth).each do |line|
+      text << "\n-> Called from #{line}"
+    end
+    log text, also_write_to_stderr: true
+  end
 end

data/lib/jirametrics/flow_efficiency_scatterplot.rb

@@ -0,0 +1,111 @@
+# frozen_string_literal: true
+
+require 'jirametrics/groupable_issue_chart'
+
+class FlowEfficiencyScatterplot < ChartBase
+  include GroupableIssueChart
+
+  attr_accessor :possible_statuses
+
+  def initialize block
+    super()
+
+    header_text 'Flow Efficiency'
+    description_text <<-HTML
+      <div class="p">
+        This chart shows the active time against the the total time spent on a ticket.
+        <a href="https://improvingflow.com/2024/07/06/flow-efficiency.html">Flow  efficiency</a> is the ratio
+        between these two numbers.
+      </div>
+      <div class="p">
+        <math>
+          <mn>Flow efficiency (%)</mn>
+          <mo>=</mo>
+          <mfrac>
+            <mrow><mn>Time adding value</mn></mrow>
+            <mrow><mn>Total time</mn></mrow>
+          </mfrac>
+        </math>
+      </div>
+      <div style="background: var(--warning-banner)">Note that for this calculation to be accurate, we must be moving items into a
+        blocked or stalled state the moment we stop working on it, and most teams don't do that.
+        So be aware that your team may have to change their behaviours if you want this chart to be useful.
+      </div>
+    HTML
+
+    init_configuration_block block do
+      grouping_rules do |issue, rule|
+        active_time, total_time = issue.flow_efficiency_numbers end_time: time_range.end
+        flow_efficiency = active_time * 100.0 / total_time
+
+        if flow_efficiency > 99.0
+          rule.label = '~100%'
+          rule.color = 'green'
+        elsif flow_efficiency < 30.0
+          rule.label = '< 30%'
+          rule.color = 'orange'
+        else
+          rule.label = 'The rest'
+          rule.color = 'black'
+        end
+      end
+    end
+
+    @percentage_lines = []
+    @highest_cycletime = 0
+  end
+
+  def run
+    data_sets = group_issues(completed_issues_in_range include_unstarted: false).filter_map do |rules, issues|
+      create_dataset(issues: issues, label: rules.label, color: rules.color)
+    end
+
+    return "<h1>#{@header_text}</h1>No data matched the selected criteria. Nothing to show." if data_sets.empty?
+
+    wrap_and_render(binding, __FILE__)
+  end
+
+  def to_days seconds
+    seconds / 60 / 60 / 24
+  end
+
+  def create_dataset issues:, label:, color:
+    return nil if issues.empty?
+
+    data = issues.filter_map do |issue|
+      active_time, total_time = issue.flow_efficiency_numbers(
+        end_time: time_range.end, settings: settings
+      )
+
+      active_days = to_days(active_time)
+      total_days = to_days(total_time)
+      flow_efficiency = active_time * 100.0 / total_time
+
+      if flow_efficiency.nan?
+        # If this happens then something is probably misconfigured. We've seen it in production though
+        # so we have to handle it.
+        file_system.log(
+          "Issue(#{issue.key}) flow_efficiency: NaN, active_time: #{active_time}, total_time: #{total_time}"
+        )
+        flow_efficiency = 0.0
+      end
+
+      {
+        y: active_days,
+        x: total_days,
+        title: [
+          "#{issue.key} : #{issue.summary}, flow efficiency: #{flow_efficiency.to_i}%," \
+          " total: #{total_days.round(1)} days," \
+          " active: #{active_days.round(1)} days"
+        ]
+      }
+    end
+    {
+      label: label,
+      data: data,
+      fill: false,
+      showLine: false,
+      backgroundColor: color
+    }
+  end
+end