jirametrics 2.6 → 2.7

data/lib/jirametrics/exporter.rb

@@ -72,6 +72,31 @@ class Exporter
     puts "Full output from downloader in #{file_system.logfile_name}"
   end
 
+  def info keys, name_filter:
+    selected = []
+    each_project_config(name_filter: name_filter) do |project|
+      project.evaluate_next_level
+      # next if project.aggregated_project?
+
+      project.run load_only: true
+      project.board_configs.each do |board_config|
+        board_config.run
+      end
+      project.issues.each do |issue|
+        selected << [project, issue] if keys.include? issue.key
+      end
+    rescue => e # rubocop:disable Style/RescueStandardError
+      # This happens when we're attempting to load an aggregated project because it hasn't been
+      # properly initialized. Since we don't care about aggregated projects, we just ignore it.
+      raise unless e.message.start_with? 'This is an aggregated project and issues should have been included'
+    end
+
+    selected.each do |project, issue|
+      puts "\nProject #{project.name}"
+      puts issue.dump
+    end
+  end
+
   def each_project_config name_filter:
     @project_configs.each do |project|
       yield project if project.name.nil? || File.fnmatch(name_filter, project.name)

data/lib/jirametrics/file_system.rb

@@ -29,7 +29,7 @@ class FileSystem
 
   def log message, also_write_to_stderr: false
     logfile.puts message
-    $stderr.puts message if also_write_to_stderr
+    $stderr.puts message if also_write_to_stderr # rubocop:disable Style/StderrPuts
   end
 
   # In some Jira instances, a sizeable portion of the JSON is made up of empty fields. I've seen

data/lib/jirametrics/flow_efficiency_scatterplot.rb

@@ -0,0 +1,113 @@
+# frozen_string_literal: true
+
+require 'jirametrics/groupable_issue_chart'
+
+class FlowEfficiencyScatterplot < ChartBase
+  include GroupableIssueChart
+
+  attr_accessor :possible_statuses
+
+  def initialize block
+    super()
+
+    header_text 'Flow Efficiency'
+    description_text <<-HTML
+      <div class="p">
+        This chart shows the active time against the the total time spent on a ticket.
+        <a href="https://improvingflow.com/2024/07/06/flow-efficiency.html">Flow  efficiency</a> is the ratio
+        between these two numbers.
+      </div>
+      <div class="p">
+        <math>
+          <mn>Flow efficiency</mn>
+          <mo>=</mo>
+          <mfrac>
+            <mrow><mn>Time adding value</mn></mrow>
+            <mrow><mn>Total time</mn></mrow>
+          </mfrac>
+          <mo>x</mo>
+          <mn>100%</mn>
+        </math>
+      </div>
+      <div style="background: yellow">Note that for this calculation to be accurate, we must be moving items into a
+        blocked or stalled state the moment we stop working on it, and most teams don't do that.
+        So be aware that your team may have to change their behaviours if you want this chart to be useful.
+      </div>
+    HTML
+
+    init_configuration_block block do
+      grouping_rules do |issue, rule|
+        active_time, total_time = issue.flow_efficiency_numbers end_time: time_range.end
+        flow_efficiency = active_time * 100.0 / total_time
+
+        if flow_efficiency > 99.0
+          rule.label = '~100%'
+          rule.color = 'green'
+        elsif flow_efficiency < 30.0
+          rule.label = '< 30%'
+          rule.color = 'orange'
+        else
+          rule.label = 'The rest'
+          rule.color = 'black'
+        end
+      end
+    end
+
+    @percentage_lines = []
+    @highest_cycletime = 0
+  end
+
+  def run
+    data_sets = group_issues(completed_issues_in_range include_unstarted: false).filter_map do |rules, issues|
+      create_dataset(issues: issues, label: rules.label, color: rules.color)
+    end
+
+    return "<h1>#{@header_text}</h1>No data matched the selected criteria. Nothing to show." if data_sets.empty?
+
+    wrap_and_render(binding, __FILE__)
+  end
+
+  def to_days seconds
+    seconds / 60 / 60 / 24
+  end
+
+  def create_dataset issues:, label:, color:
+    return nil if issues.empty?
+
+    data = issues.filter_map do |issue|
+      active_time, total_time = issue.flow_efficiency_numbers(
+        end_time: time_range.end, settings: settings
+      )
+
+      active_days = to_days(active_time)
+      total_days = to_days(total_time)
+      flow_efficiency = active_time * 100.0 / total_time
+
+      if flow_efficiency.nan?
+        # If this happens then something is probably misconfigured. We've seen it in production though
+        # so we have to handle it.
+        file_system.log(
+          "Issue(#{issue.key}) flow_efficiency: NaN, active_time: #{active_time}, total_time: #{total_time}"
+        )
+        flow_efficiency = 0.0
+      end
+
+      {
+        y: active_days,
+        x: total_days,
+        title: [
+          "#{issue.key} : #{issue.summary}, flow efficiency: #{flow_efficiency.to_i}%," \
+          " total: #{total_days.round(1)} days," \
+          " active: #{active_days.round(1)} days"
+        ]
+      }
+    end
+    {
+      label: label,
+      data: data,
+      fill: false,
+      showLine: false,
+      backgroundColor: color
+    }
+  end
+end

data/lib/jirametrics/html/data_quality_report.erb

@@ -113,6 +113,18 @@
   end
 %>
 
+<% 
+  problems = problems_for :incomplete_subtasks_when_issue_done
+  unless problems.empty? 
+%>
+  <p>
+    <span class="quality_note_bullet">⮕</span> <%= label_issues problems.size %> issues were marked as done while subtasks were still not done.
+    <%= collapsible_issues_panel problems %>
+  </p>
+<% 
+  end
+%>
+
 <% 
   problems = problems_for :issue_on_multiple_boards
   unless problems.empty? 

data/lib/jirametrics/html/flow_efficiency_scatterplot.erb

@@ -0,0 +1,85 @@
+<div class="chart">
+  <canvas id="<%= chart_id %>" width="<%= canvas_width %>" height="<%= canvas_height %>"></canvas>
+</div>
+<script>
+new Chart(document.getElementById('<%= chart_id %>').getContext('2d'), {
+  type: 'scatter',
+  data: {
+    datasets: <%= JSON.generate(data_sets) %>
+  },
+  options: {
+    title: {
+      display: true,
+      text: "Cycletime Scatterplot"
+    },
+    responsive: <%= canvas_responsive? %>, // If responsive is true then it fills the screen
+    scales: {
+      x: {
+        scaleLabel: {
+          display: true,
+          labelString: 'Days'
+        },
+        title: {
+          display: true,
+          text: 'Total time (days)'
+        },
+        grid: {
+          color: <%= CssVariable['--grid-line-color'].to_json %>
+        },
+
+      },
+      y: {
+        scaleLabel: {
+          display: true,
+          labelString: 'Percentage',
+          min: 0,
+          max: <%= @highest_cycletime %>
+        },
+        title: {
+          display: true,
+          text: 'Time adding value (days)'
+        },
+        grid: {
+          color: <%= CssVariable['--grid-line-color'].to_json %>
+        },
+      }
+    },
+    plugins: {
+      tooltip: {
+        callbacks: {
+          label: function(context) {
+            return context.dataset.data[context.dataIndex].title
+          }
+        }
+      },
+      autocolors: false,
+      legend: {
+        onClick: (evt, legendItem, legend) => {
+          // Find the datasetMeta that corresponds to the item clicked
+          var i = 0
+          while(legendItem.text != legend.chart.getDatasetMeta(i).label) {
+            i++;
+          }
+          nextVisibility = !!legend.chart.getDatasetMeta(i).hidden;
+
+          // Hide/show the 85% line for that dataset
+          legend.chart.options.plugins.annotation.annotations["line"+(i/2)].display = nextVisibility;
+
+          // Hide/show the trendline for this dataset, if they were enabled. The trendline is always
+          // there but not always visible.
+          legend.chart.setDatasetVisibility(i+1, <%= !!@show_trend_lines %> && nextVisibility);
+
+          // Still run the default behaviour
+          Chart.defaults.plugins.legend.onClick(evt, legendItem, legend);
+        },
+        labels: {
+          filter: function(item, chart) {
+            // Logic to remove a particular legend item goes here
+            return !item.text.includes('Trendline');
+          }
+        }
+      }
+    }
+  }
+});
+</script>

data/lib/jirametrics/html_report_config.rb

@@ -33,6 +33,7 @@ class HtmlReportConfig
   define_chart name: 'cycletime_histogram', classname: 'CycletimeHistogram'
   define_chart name: 'estimate_accuracy_chart', classname: 'EstimateAccuracyChart'
   define_chart name: 'hierarchy_table', classname: 'HierarchyTable'
+  define_chart name: 'flow_efficiency_scatterplot', classname: 'FlowEfficiencyScatterplot'
 
   define_chart name: 'daily_wip_by_type', classname: 'DailyWipChart',
     deprecated_warning: 'This is the same as daily_wip_chart. Please use that one', deprecated_date: '2024-05-23'
@@ -145,7 +146,7 @@ class HtmlReportConfig
 
     @original_issue_times = {}
     issues_cutoff_times.each do |issue, cutoff_time|
-      started = issue.board.cycletime.started_time(issue)
+      started = issue.board.cycletime.started_stopped_times(issue).first
       if started && started <= cutoff_time
         # We only need to log this if data was discarded
         @original_issue_times[issue] = { cutoff_time: cutoff_time, started_time: started }

data/lib/jirametrics/issue.rb

@@ -13,6 +13,7 @@ class Issue
     @changes = []
     @board = board
 
+    raise "No board for issue #{key}" if board.nil?
     return unless @raw['changelog']
 
     load_history_into_changes
@@ -259,7 +260,7 @@ class Issue
 
     blocked_link_texts = settings['blocked_link_text']
     stalled_threshold = settings['stalled_threshold_days']
-    flagged_means_blocked = !!settings['flagged_means_blocked']
+    flagged_means_blocked = !!settings['flagged_means_blocked'] # rubocop:disable Style/DoubleNegation
 
     blocking_issue_keys = []
 
@@ -373,12 +374,11 @@ class Issue
 
   # return [number of active seconds, total seconds] that this issue had up to the end_time.
   # It does not include data before issue start or after issue end
-  def flow_efficiency_numbers end_time:, settings: {}
-    issue_start = @board.cycletime.started_time(self)
+  def flow_efficiency_numbers end_time:, settings: @board.project_config.settings
+    issue_start, issue_stop = @board.cycletime.started_stopped_times(self)
     return [0.0, 0.0] if !issue_start || issue_start > end_time
 
     value_add_time = 0.0
-    issue_stop = @board.cycletime.stopped_time(self)
     end_time = issue_stop if issue_stop && issue_stop < end_time
 
     active_start = nil
@@ -542,6 +542,20 @@ class Issue
     comparison
   end
 
+  def discard_changes_before cutoff_time
+    rejected_any = false
+    @changes.reject! do |change|
+      reject = change.status? && change.time <= cutoff_time && change.artificial? == false
+      if reject
+        (@discarded_changes ||= []) << change
+        rejected_any = true
+      end
+      reject
+    end
+
+    (@discarded_change_times ||= []) << cutoff_time if rejected_any
+  end
+
   def dump
     result = +''
     result << "#{key} (#{type}): #{compact_text summary, 200}\n"
@@ -549,21 +563,59 @@ class Issue
     assignee = raw['fields']['assignee']
     result << "  [assignee] #{assignee['name'].inspect} <#{assignee['emailAddress']}>\n" unless assignee.nil?
 
-    raw['fields']['issuelinks'].each do |link|
+    raw['fields']['issuelinks']&.each do |link|
       result << "  [link] #{link['type']['outward']} #{link['outwardIssue']['key']}\n" if link['outwardIssue']
       result << "  [link] #{link['type']['inward']} #{link['inwardIssue']['key']}\n" if link['inwardIssue']
     end
-    changes.each do |change|
+    history = [] # time, type, detail
+
+    started_at, stopped_at = board.cycletime.started_stopped_times(self)
+    history << [started_at, nil, '↓↓↓↓ Started here ↓↓↓↓', true] if started_at
+    history << [stopped_at, nil, '↑↑↑↑ Finished here ↑↑↑↑', true] if stopped_at
+
+    @discarded_change_times&.each do |time|
+      history << [time, nil, '↑↑↑↑ Changes discarded ↑↑↑↑', true]
+    end
+
+    (changes + (@discarded_changes || [])).each do |change|
       value = change.value
       old_value = change.old_value
 
-      message = "  [change] #{change.time.strftime '%Y-%m-%d %H:%M:%S %z'} [#{change.field}] "
+      message = +''
       message << "#{compact_text(old_value).inspect} -> " unless old_value.nil? || old_value.empty?
       message << compact_text(value).inspect
-      message << " (#{change.author})"
-      message << ' <<artificial entry>>' if change.artificial?
-      result << message << "\n"
+      if change.artificial?
+        message << ' (Artificial entry)' if change.artificial?
+      else
+        message << " (Author: #{change.author})"
+      end
+      history << [change.time, change.field, message, change.artificial?]
+    end
+
+    result << "  History:\n"
+    type_width = history.collect { |_time, type, _detail, _artificial| type&.length || 0 }.max
+    history.sort! do |a, b|
+      if a[0] == b[0]
+        if a[1].nil?
+          1
+        elsif b[1].nil?
+          -1
+        else
+          a[1] <=> b[1]
+        end
+      else
+        a[0] <=> b[0]
+      end
     end
+    history.each do |time, type, detail, _artificial|
+      if type.nil?
+        type = '-' * type_width
+      else
+        type = (' ' * (type_width - type.length)) << type
+      end
+      result << "    #{time.strftime '%Y-%m-%d %H:%M:%S %z'} [#{type}] #{detail}\n"
+    end
+
     result
   end
 

data/lib/jirametrics/jira_gateway.rb

@@ -16,7 +16,7 @@ class JiraGateway
     result = call_command command
     JSON.parse result
   rescue => e # rubocop:disable Style/RescueStandardError
-    puts "Error #{e.inspect} when parsing result: #{result.inspect}"
+    raise "Error #{e.message.inspect} when parsing result: #{result.inspect}"
   end
 
   def call_command command

data/lib/jirametrics/project_config.rb

@@ -24,22 +24,30 @@ class ProjectConfig
     @all_boards = {}
     @settings = load_settings
     @id = id
+    @has_loaded_data = false
   end
 
   def evaluate_next_level
     instance_eval(&@block) if @block
   end
 
-  def run
-    unless aggregated_project?
-      load_all_boards
-      @id = guess_project_id
-      load_status_category_mappings
-      load_project_metadata
-      load_sprints
-    end
+  def load_data
+    return if @has_loaded_data
+
+    @has_loaded_data = true
+    load_all_boards
+    @id = guess_project_id
+    load_status_category_mappings
+    load_project_metadata
+    load_sprints
+  end
+
+  def run load_only: false
+    load_data unless aggregated_project?
     anonymize_data if @anonymizer_needed
 
+    return if load_only
+
     @board_configs.each do |board_config|
       board_config.run
     end
@@ -117,7 +125,7 @@ class ProjectConfig
       board_id = $1.to_i
       load_board board_id: board_id, filename: "#{@target_path}#{file}"
     end
-    raise "No boards found for #{@file_prefix} in #{@target_path.inspect}" if @all_boards.empty?
+    raise "No boards found for #{@file_prefix.inspect} in #{@target_path.inspect}" if @all_boards.empty?
   end
 
   def load_board board_id:, filename:
@@ -232,7 +240,7 @@ class ProjectConfig
   end
 
   def load_project_metadata
-    filename = "#{@target_path}/#{file_prefix}_meta.json"
+    filename = File.join @target_path, "#{file_prefix}_meta.json"
     json = JSON.parse(file_system.load(filename))
 
     @data_version = json['version'] || 1
@@ -243,7 +251,7 @@ class ProjectConfig
 
     @jira_url = json['jira_url']
   rescue Errno::ENOENT
-    puts "== Can't load files from the target directory. Did you forget to download first? =="
+    file_system.log "Can't load #{filename}. Have you done a download?", also_write_to_stderr: true
     raise
   end
 
@@ -259,7 +267,7 @@ class ProjectConfig
     unless all_boards&.size == 1
       message = "If the board_id isn't set then we look for all board configurations in the target" \
         ' directory. '
-      if all_boards.nil? || all_boards.empty?
+      if all_boards.empty?
         message += ' In this case, we couldn\'t find any configuration files in the target directory.'
       else
         message += 'If there is only one, we use that. In this case we found configurations for' \
@@ -291,21 +299,21 @@ class ProjectConfig
   end
 
   def issues
-    raise "issues are being loaded before boards in project #{name.inspect}" if all_boards.nil? && !aggregated_project?
-
     unless @issues
-      if @aggregate_config
+      if aggregated_project?
         raise 'This is an aggregated project and issues should have been included with the include_issues_from ' \
           'declaration but none are here. Check your config.'
       end
+      load_data if all_boards.empty?
 
       timezone_offset = exporter.timezone_offset
 
-      issues_path = "#{@target_path}#{file_prefix}_issues/"
+      issues_path = File.join @target_path, "#{file_prefix}_issues"
       if File.exist?(issues_path) && File.directory?(issues_path)
         issues = load_issues_from_issues_directory path: issues_path, timezone_offset: timezone_offset
       else
-        puts "Can't find issues in #{path}. Has a download been done?"
+        file_system.log "Can't find directory #{issues_path}. Has a download been done?", also_write_to_stderr: true
+        return []
       end
 
       # Attach related issues
@@ -351,8 +359,8 @@ class ProjectConfig
     raise "No boards found for project #{name.inspect}" if all_boards.empty?
 
     if all_boards.size != 1
-      puts "Multiple boards are in use for project #{name.inspect}. " \
-        "Picked #{default_board.name.inspect} to attach issues to."
+      file_system.log "Multiple boards are in use for project #{name.inspect}. " \
+        "Picked #{default_board.name.inspect} to attach issues to.", also_write_to_stderr: true
     end
     default_board
   end

data/lib/jirametrics/self_or_issue_dispatcher.rb

@@ -1,6 +1,7 @@
 # frozen_string_literal: true
 
 module SelfOrIssueDispatcher
+  # rubocop:disable Style/ArgumentsForwarding
   def method_missing method_name, *args, &block
     raise "#{method_name} isn't a method on Issue or #{self.class}" unless ::Issue.method_defined? method_name.to_sym
 
@@ -8,6 +9,7 @@ module SelfOrIssueDispatcher
       issue.__send__ method_name, *args, &block
     end
   end
+  # rubocop:enable Style/ArgumentsForwarding
 
   def respond_to_missing?(method_name, include_all = false)
     ::Issue.method_defined?(method_name.to_sym) || super

data/lib/jirametrics/sprint_burndown.rb

@@ -18,7 +18,7 @@ class SprintBurndown < ChartBase
   attr_accessor :board_id
 
   def initialize
-    super()
+    super
 
     @summary_stats = {}
     header_text 'Sprint burndown'
@@ -126,7 +126,7 @@ class SprintBurndown < ChartBase
     currently_in_sprint = false
     change_data = []
 
-    issue_completed_time = issue.board.cycletime.stopped_time(issue)
+    issue_completed_time = issue.board.cycletime.started_stopped_times(issue).last
     completed_has_been_tracked = false
 
     issue.changes.each do |change|

data/lib/jirametrics/status.rb

@@ -38,7 +38,7 @@ class Status
   end
 
   def to_s
-    result = +"Status(name=#{@name.inspect}," \
+    result = "Status(name=#{@name.inspect}," \
       " id=#{@id.inspect}," \
       " category_name=#{@category_name.inspect}," \
       " category_id=#{@category_id.inspect}," \

data/lib/jirametrics/status_collection.rb

@@ -68,4 +68,8 @@ class StatusCollection
   def empty? = @list.empty?
   def clear = @list.clear
   def delete(object) = @list.delete(object)
+
+  def inspect
+    "StatusCollection(#{@list.collect(&:inspect).join(', ')})"
+  end
 end

data/lib/jirametrics/throughput_chart.rb

@@ -82,7 +82,7 @@ class ThroughputChart < ChartBase
   def throughput_dataset periods:, completed_issues:
     periods.collect do |period|
       closed_issues = completed_issues.filter_map do |issue|
-        stop_date = issue.board.cycletime.stopped_time(issue)&.to_date
+        stop_date = issue.board.cycletime.started_stopped_times(issue).last&.to_date
         [stop_date, issue] if stop_date && period.include?(stop_date)
       end
 

data/lib/jirametrics.rb

@@ -3,9 +3,13 @@
 require 'thor'
 
 class JiraMetrics < Thor
+  def self.exit_on_failure?
+    true
+  end
+
   option :config
   option :name
-  desc 'export only', "Export data into either reports or CSV's as per the configuration"
+  desc 'export', "Export data into either reports or CSV's as per the configuration"
   def export
     load_config options[:config]
     Exporter.instance.export(name_filter: options[:name] || '*')
@@ -13,7 +17,7 @@ class JiraMetrics < Thor
 
   option :config
   option :name
-  desc 'download only', 'Download data from Jira'
+  desc 'download', 'Download data from Jira'
   def download
     load_config options[:config]
     Exporter.instance.download(name_filter: options[:name] || '*')
@@ -21,7 +25,7 @@ class JiraMetrics < Thor
 
   option :config
   option :name
-  desc 'download and export', 'Same as running download, followed by export'
+  desc 'go', 'Same as running download, followed by export'
   def go
     load_config options[:config]
     Exporter.instance.download(name_filter: options[:name] || '*')
@@ -30,6 +34,13 @@ class JiraMetrics < Thor
     Exporter.instance.export(name_filter: options[:name] || '*')
   end
 
+  option :config
+  desc 'info', 'Dump information about one issue'
+  def info keys
+    load_config options[:config]
+    Exporter.instance.info(keys, name_filter: options[:name] || '*')
+  end
+
   private
 
   def load_config config_file
@@ -69,6 +80,7 @@ class JiraMetrics < Thor
     require 'jirametrics/daily_wip_by_parent_chart'
     require 'jirametrics/aging_work_in_progress_chart'
     require 'jirametrics/cycletime_scatterplot'
+    require 'jirametrics/flow_efficiency_scatterplot'
     require 'jirametrics/sprint_issue_change_data'
     require 'jirametrics/cycletime_histogram'
     require 'jirametrics/daily_wip_by_blocked_stalled_chart'
@@ -97,6 +109,4 @@ class JiraMetrics < Thor
     require 'jirametrics/board'
     load config_file
   end
-
-  # Dir.foreach('lib/jirametrics') {|file| puts "require 'jirametrics/#{$1}'" if file =~ /^(.+)\.rb$/}
 end