jirametrics 2.7 → 2.11

data/lib/jirametrics/downloader.rb

@@ -39,6 +39,7 @@ class Downloader
     # board_ids = @download_config.board_ids
 
     remove_old_files
+    update_status_history_file
     download_statuses
     find_board_ids.each do |id|
       board = download_board_configuration board_id: id
@@ -66,7 +67,7 @@ class Downloader
 
   def download_issues board:
     log "  Downloading primary issues for board #{board.id}", both: true
-    path = "#{@target_path}#{@download_config.project_config.file_prefix}_issues/"
+    path = File.join(@target_path, "#{file_prefix}_issues/")
     unless Dir.exist?(path)
       log "  Creating path #{path}"
       Dir.mkdir(path)
@@ -102,8 +103,6 @@ class Downloader
       json = @jira_gateway.call_url relative_url: '/rest/api/2/search' \
         "?jql=#{escaped_jql}&maxResults=#{max_results}&startAt=#{start_at}&expand=changelog&fields=*all"
 
-      exit_if_call_failed json
-
       json['issues'].each do |issue_json|
         issue_json['exporter'] = {
           'in_initial_query' => initial_query
@@ -138,45 +137,62 @@ class Downloader
     end
   end
 
-  def exit_if_call_failed json
-    # Sometimes Jira returns the singular form of errorMessage and sometimes the plural. Consistency FTW.
-    return unless json['error'] || json['errorMessages'] || json['errorMessage']
-
-    log "Download failed. See #{@file_system.logfile_name} for details.", both: true
-    log "  #{JSON.pretty_generate(json)}"
-    exit 1
-  end
-
   def download_statuses
     log '  Downloading all statuses', both: true
     json = @jira_gateway.call_url relative_url: '/rest/api/2/status'
 
     @file_system.save_json(
       json: json,
-      filename: "#{@target_path}#{@download_config.project_config.file_prefix}_statuses.json"
+      filename: File.join(@target_path, "#{file_prefix}_statuses.json")
     )
   end
 
+  def update_status_history_file
+    status_filename = File.join(@target_path, "#{file_prefix}_statuses.json")
+    return unless file_system.file_exist? status_filename
+
+    status_json = file_system.load_json(status_filename)
+
+    history_filename = File.join(@target_path, "#{file_prefix}_status_history.json")
+    history_json = file_system.load_json(history_filename) if file_system.file_exist? history_filename
+
+    if history_json
+      file_system.log '  Updating status history file', also_write_to_stderr: true
+    else
+      file_system.log '  Creating status history file', also_write_to_stderr: true
+      history_json = []
+    end
+
+    status_json.each do |status_item|
+      id = status_item['id']
+      history_item = history_json.find { |s| s['id'] == id }
+      history_json.delete(history_item) if history_item
+      history_json << status_item
+    end
+
+    file_system.save_json(filename: history_filename, json: history_json)
+  end
+
   def download_board_configuration board_id:
     log "  Downloading board configuration for board #{board_id}", both: true
     json = @jira_gateway.call_url relative_url: "/rest/agile/1.0/board/#{board_id}/configuration"
 
-    exit_if_call_failed json
-
-    file_prefix = @download_config.project_config.file_prefix
-    @file_system.save_json json: json, filename: "#{@target_path}#{file_prefix}_board_#{board_id}_configuration.json"
+    @file_system.save_json(
+      json: json,
+      filename: File.join(@target_path, "#{file_prefix}_board_#{board_id}_configuration.json")
+    )
 
     # We have a reported bug that blew up on this line. Moved it after the save so we can
     # actually look at the returned json.
     @board_id_to_filter_id[board_id] = json['filter']['id'].to_i
 
     download_sprints board_id: board_id if json['type'] == 'scrum'
-    Board.new raw: json
+    # TODO: Should be passing actual statuses, not empty list
+    Board.new raw: json, possible_statuses: StatusCollection.new
   end
 
   def download_sprints board_id:
     log "  Downloading sprints for board #{board_id}", both: true
-    file_prefix = @download_config.project_config.file_prefix
     max_results = 100
     start_at = 0
     is_last = false
@@ -184,11 +200,10 @@ class Downloader
     while is_last == false
       json = @jira_gateway.call_url relative_url: "/rest/agile/1.0/board/#{board_id}/sprint?" \
         "maxResults=#{max_results}&startAt=#{start_at}"
-      exit_if_call_failed json
 
       @file_system.save_json(
         json: json,
-        filename: "#{@target_path}#{file_prefix}_board_#{board_id}_sprints_#{start_at}.json"
+        filename: File.join(@target_path, "#{file_prefix}_board_#{board_id}_sprints_#{start_at}.json")
       )
       is_last = json['isLast']
       max_results = json['maxResults']
@@ -201,7 +216,7 @@ class Downloader
   end
 
   def metadata_pathname
-    "#{@target_path}#{@download_config.project_config.file_prefix}_meta.json"
+    File.join(@target_path, "#{file_prefix}_meta.json")
   end
 
   def load_metadata
@@ -244,17 +259,17 @@ class Downloader
   end
 
   def remove_old_files
-    file_prefix = @download_config.project_config.file_prefix
     Dir.foreach @target_path do |file|
       next unless file.match?(/^#{file_prefix}_\d+\.json$/)
+      next if file == "#{file_prefix}_status_history.json"
 
-      File.unlink "#{@target_path}#{file}"
+      File.unlink File.join(@target_path, file)
     end
 
     return if @cached_data_format_is_current
 
     # Also throw away all the previously downloaded issues.
-    path = File.join @target_path, "#{file_prefix}_issues"
+    path = File.join(@target_path, "#{file_prefix}_issues")
     return unless File.exist? path
 
     Dir.foreach path do |file|
@@ -292,4 +307,8 @@ class Downloader
 
     segments.join ' AND '
   end
+
+  def file_prefix
+    @download_config.project_config.get_file_prefix
+  end
 end

data/lib/jirametrics/examples/aggregated_project.rb

@@ -3,8 +3,6 @@
 # This file is really intended to give you ideas about how you might configure your own reports, not
 # as a complete setup that will work in every case.
 #
-# See https://github.com/mikebowler/jirametrics/wiki/Examples-folder for more details
-#
 # The point of an AGGREGATED report is that we're now looking at a higher level. We might use this in a
 # S2 meeting (Scrum of Scrums) to talk about the things that are happening across teams, not within a
 # single team. For that reason, we look at slightly different things that we would on a single team board.
@@ -13,6 +11,7 @@ class Exporter
   def aggregated_project name:, project_names:, settings: {}
     project name: name do
       puts name
+      file_prefix name
       self.settings.merge! settings
 
       aggregate do
@@ -21,8 +20,6 @@ class Exporter
         end
       end
 
-      file_prefix name
-
       file do
         file_suffix '.html'
         issues.reject! do |issue|
@@ -34,7 +31,7 @@ class Exporter
           board_lines = []
           included_projects.each do |project|
             project.all_boards.each_value do |board|
-              board_lines << "<a href='#{project.file_prefix}.html'>#{board.name}</a> from project #{project.name}"
+              board_lines << "<a href='#{project.get_file_prefix}.html'>#{board.name}</a> from project #{project.name}"
             end
           end
           board_lines.sort.each { |line| html "<li>#{line}</li>", type: :header }

data/lib/jirametrics/examples/standard_project.rb

@@ -2,8 +2,6 @@
 
 # This file is really intended to give you ideas about how you might configure your own reports, not
 # as a complete setup that will work in every case.
-#
-# See https://github.com/mikebowler/jirametrics/wiki/Examples-folder for more
 class Exporter
   def standard_project name:, file_prefix:, ignore_issues: nil, starting_status: nil, boards: {},
       default_board: nil, anonymize: false, settings: {}, status_category_mappings: {},
@@ -12,15 +10,15 @@ class Exporter
 
     project name: name do
       puts name
-      self.anonymize if anonymize
+      file_prefix file_prefix
 
+      self.anonymize if anonymize
       self.settings.merge! settings
 
       status_category_mappings.each do |status, category|
         status_category_mapping status: status, category: category
       end
 
-      file_prefix file_prefix
       download do
         self.rolling_date_count(rolling_date_count) if rolling_date_count
         self.no_earlier_than(no_earlier_than) if no_earlier_than
@@ -30,8 +28,8 @@ class Exporter
         block = boards[board_id]
         if block == :default
           block = lambda do |_|
-            start_at first_time_in_status_category('In Progress')
-            stop_at still_in_status_category('Done')
+            start_at first_time_in_status_category(:indeterminate)
+            stop_at still_in_status_category(:done)
           end
         end
         board id: board_id do

data/lib/jirametrics/expedited_chart.rb

@@ -63,7 +63,7 @@ class ExpeditedChart < ChartBase
       next unless change.priority?
 
       if expedited_priority_names.include? change.value
-        expedite_start = change.time
+        expedite_start = change.time.to_date
       elsif expedite_start
         start_date = expedite_start.to_date
         stop_date = change.time.to_date
@@ -72,7 +72,7 @@ class ExpeditedChart < ChartBase
            (start_date < date_range.begin && stop_date > date_range.end)
 
           result << [expedite_start, :expedite_start]
-          result << [change.time, :expedite_stop]
+          result << [change.time.to_date, :expedite_stop]
         end
         expedite_start = nil
       end
@@ -109,11 +109,11 @@ class ExpeditedChart < ChartBase
 
   def make_expedite_lines_data_set issue:, expedite_data:
     cycletime = issue.board.cycletime
-    started_time, stopped_time = cycletime.started_stopped_times(issue)
+    started_date, stopped_date = cycletime.started_stopped_dates(issue)
 
-    expedite_data << [started_time, :issue_started] if started_time
-    expedite_data << [stopped_time, :issue_stopped] if stopped_time
-    expedite_data.sort_by! { |a| a[0] }
+    expedite_data << [started_date, :issue_started] if started_date
+    expedite_data << [stopped_date, :issue_stopped] if stopped_date
+    expedite_data.sort_by!(&:first)
 
     # If none of the data would be visible on the chart then skip it.
     return nil unless expedite_data.any? { |time, _action| time.to_date >= date_range.begin }
@@ -150,7 +150,7 @@ class ExpeditedChart < ChartBase
 
     unless expedite_data.empty?
       last_change_time = expedite_data[-1][0].to_date
-      if last_change_time && last_change_time <= date_range.end && stopped_time.nil?
+      if last_change_time && last_change_time <= date_range.end && stopped_date.nil?
         data << make_point(issue: issue, time: date_range.end, label: 'Still ongoing', expedited: expedited)
         dot_colors << '' # It won't be visible so it doesn't matter
         point_styles << 'dash'

data/lib/jirametrics/exporter.rb

@@ -2,18 +2,6 @@
 
 require 'fileutils'
 
-class Object
-  def deprecated message:, date:
-    text = +''
-    text << "Deprecated(#{date}): "
-    text << message
-    caller(1..2).each do |line|
-      text << "\n-> Called from #{line}"
-    end
-    warn text
-  end
-end
-
 class Exporter
   attr_reader :project_configs
   attr_accessor :file_system
@@ -76,12 +64,8 @@ class Exporter
     selected = []
     each_project_config(name_filter: name_filter) do |project|
       project.evaluate_next_level
-      # next if project.aggregated_project?
 
       project.run load_only: true
-      project.board_configs.each do |board_config|
-        board_config.run
-      end
       project.issues.each do |issue|
         selected << [project, issue] if keys.include? issue.key
       end
@@ -91,9 +75,13 @@ class Exporter
       raise unless e.message.start_with? 'This is an aggregated project and issues should have been included'
     end
 
-    selected.each do |project, issue|
-      puts "\nProject #{project.name}"
-      puts issue.dump
+    if selected.empty?
+      file_system.log "No issues found to match #{keys.collect(&:inspect).join(', ')}"
+    else
+      selected.each do |project, issue|
+        file_system.log "\nProject #{project.name}", also_write_to_stderr: true
+        file_system.log issue.dump, also_write_to_stderr: true
+      end
     end
   end
 
@@ -128,7 +116,9 @@ class Exporter
 
   def jira_config filename = nil
     if filename
-      @jira_config = file_system.load_json(filename)
+      @jira_config = file_system.load_json(filename, fail_on_error: false)
+      raise "Unable to load Jira configuration file and cannot continue: #{filename.inspect}" if @jira_config.nil?
+
       @jira_config['url'] = $1 if @jira_config['url'] =~ /^(.+)\/+$/
     end
     @jira_config

data/lib/jirametrics/file_config.rb

@@ -56,7 +56,7 @@ class FileConfig
   def output_filename
     segments = []
     segments << project_config.target_path
-    segments << project_config.file_prefix
+    segments << project_config.get_file_prefix
     segments << (@file_suffix || "-#{@today}.csv")
     segments.join
   end
@@ -66,15 +66,20 @@ class FileConfig
   # is that all empty values in the first column should be at the bottom.
   def sort_output all_lines
     all_lines.sort do |a, b|
+      result = nil
       if a[0] == b[0]
-        a[1..] <=> b[1..]
+        result = a[1..] <=> b[1..]
       elsif a[0].nil?
-        1
+        result = 1
       elsif b[0].nil?
-        -1
+        result = -1
       else
-        a[0] <=> b[0]
+        result = a[0] <=> b[0]
       end
+
+      # This will only happen if one of the objects isn't comparable. Seen in production.
+      result = -1 if result.nil?
+      result
     end
   end
 
@@ -85,6 +90,11 @@ class FileConfig
 
   def html_report &block
     assert_only_one_filetype_config_set
+    if block.nil?
+      project_config.file_system.warning 'No charts were specified for the report. This is almost certainly a mistake.'
+      block = ->(_) {}
+    end
+
     @html_report = HtmlReportConfig.new file_config: self, block: block
   end
 
@@ -103,7 +113,7 @@ class FileConfig
   def to_datetime object
     return nil if object.nil?
 
-    object = object.to_datetime
+    object = object.to_time.to_datetime
     object = object.new_offset(@timezone_offset) if @timezone_offset
     object
   end
@@ -120,4 +130,11 @@ class FileConfig
     @file_suffix = suffix unless suffix.nil?
     @file_suffix
   end
+
+  def children
+    result = []
+    result << @columns if @columns
+    result << @html_report if @html_report
+    result
+  end
 end

data/lib/jirametrics/file_system.rb

@@ -6,14 +6,18 @@ class FileSystem
   attr_accessor :logfile, :logfile_name
 
   # Effectively the same as File.read except it forces the encoding to UTF-8
-  def load filename
+  def load filename, supress_deprecation: false
+    if filename.end_with?('.json') && !supress_deprecation
+      deprecated(message: 'call load_json instead', date: '2024-11-13')
+    end
+
     File.read filename, encoding: 'UTF-8'
   end
 
   def load_json filename, fail_on_error: true
     return nil if fail_on_error == false && File.exist?(filename) == false
 
-    JSON.parse load(filename)
+    JSON.parse load(filename, supress_deprecation: true)
   end
 
   def save_json json:, filename:
@@ -27,9 +31,22 @@ class FileSystem
     File.write(filename, content)
   end
 
-  def log message, also_write_to_stderr: false
+  def warning message, more: nil
+    log "Warning: #{message}", more: more, also_write_to_stderr: true
+  end
+
+  def error message, more: nil
+    log "Error: #{message}", more: more, also_write_to_stderr: true
+  end
+
+  def log message, more: nil, also_write_to_stderr: false
+    message += " See #{logfile_name} for more details about this message." if more
+
     logfile.puts message
-    $stderr.puts message if also_write_to_stderr # rubocop:disable Style/StderrPuts
+    logfile.puts more if more
+    return unless also_write_to_stderr
+
+    $stderr.puts message # rubocop:disable Style/StderrPuts
   end
 
   # In some Jira instances, a sizeable portion of the JSON is made up of empty fields. I've seen
@@ -43,4 +60,22 @@ class FileSystem
     end
     node
   end
+
+  def foreach root, &block
+    Dir.foreach root, &block
+  end
+
+  def file_exist? filename
+    File.exist? filename
+  end
+
+  def deprecated message:, date:, depth: 2
+    text = +''
+    text << "Deprecated(#{date}): "
+    text << message
+    caller(1..depth).each do |line|
+      text << "\n-> Called from #{line}"
+    end
+    log text, also_write_to_stderr: true
+  end
 end

data/lib/jirametrics/flow_efficiency_scatterplot.rb

@@ -19,17 +19,15 @@ class FlowEfficiencyScatterplot < ChartBase
       </div>
       <div class="p">
         <math>
-          <mn>Flow efficiency</mn>
+          <mn>Flow efficiency (%)</mn>
           <mo>=</mo>
           <mfrac>
             <mrow><mn>Time adding value</mn></mrow>
             <mrow><mn>Total time</mn></mrow>
           </mfrac>
-          <mo>x</mo>
-          <mn>100%</mn>
         </math>
       </div>
-      <div style="background: yellow">Note that for this calculation to be accurate, we must be moving items into a
+      <div style="background: var(--warning-banner)">Note that for this calculation to be accurate, we must be moving items into a
         blocked or stalled state the moment we stop working on it, and most teams don't do that.
         So be aware that your team may have to change their behaviours if you want this chart to be useful.
       </div>

data/lib/jirametrics/groupable_issue_chart.rb

@@ -6,9 +6,7 @@ require 'jirametrics/grouping_rules'
 module GroupableIssueChart
   def init_configuration_block user_provided_block, &default_block
     instance_eval(&user_provided_block)
-    return if @group_by_block
-
-    instance_eval(&default_block)
+    instance_eval(&default_block) unless @group_by_block
   end
 
   def grouping_rules &block

data/lib/jirametrics/html/aging_work_bar_chart.erb

@@ -38,22 +38,13 @@ new Chart(document.getElementById('<%= chart_id %>').getContext('2d'),
     plugins: {
       annotation: {
         annotations: {
-          <% holidays.each_with_index do |range, index| %>
-          holiday<%= index %>: {
-            drawTime: 'beforeDraw',
-            type: 'box',
-            xMin: '<%= range.begin %>T00:00:00',
-            xMax: '<%= range.end %>T23:59:59',
-            backgroundColor: <%= CssVariable.new('--non-working-days-color').to_json %>,
-            borderColor: <%= CssVariable.new('--non-working-days-color').to_json %>
-          },
-          <% end %>
+          <%= working_days_annotation %>
 
           <% if percentage_line_x %>
           line: {
             type: 'line',
-            xMin: '<%= percentage_line_x %>',
-            xMax: '<%= percentage_line_x %>',
+            scaleID: 'x',
+            value: '<%= percentage_line_x %>',
             borderColor: <%= CssVariable.new('--aging-work-bar-chart-percentage-line-color').to_json %>,
             borderWidth: 1,
             drawTime: 'afterDraw'

data/lib/jirametrics/html/aging_work_in_progress_chart.erb

@@ -6,7 +6,7 @@ new Chart(document.getElementById(<%= chart_id.inspect %>).getContext('2d'),
 {
   type: 'bar',
   data: {
-    labels: [<%= column_headings.collect(&:inspect).join(',') %>],
+    labels: [<%= @board_columns.collect { |c| c.name.inspect }.join(',') %>],
     datasets: <%= JSON.generate(data_sets) %>
   },
   options: {
@@ -22,8 +22,10 @@ new Chart(document.getElementById(<%= chart_id.inspect %>).getContext('2d'),
           labelString: 'Date Completed'
         },
         grid: {
-          color: <%= CssVariable['--grid-line-color'].to_json %>
+          color: <%= CssVariable['--grid-line-color'].to_json %>,
+          z: 1 // draw the grid lines on top of the bars
         },
+        stacked: true
       },
       y: {
         scaleLabel: {
@@ -35,8 +37,11 @@ new Chart(document.getElementById(<%= chart_id.inspect %>).getContext('2d'),
           text: 'Age in days'
         },
         grid: {
-          color: <%= CssVariable['--grid-line-color'].to_json %>
+          color: <%= CssVariable['--grid-line-color'].to_json %>,
+          z: 1 // draw the grid lines on top of the bars
         },
+        stacked: true,
+        max: <%= (@max_age * 1.1).to_i %>
       }
     },
     plugins: {
@@ -44,14 +49,26 @@ new Chart(document.getElementById(<%= chart_id.inspect %>).getContext('2d'),
         callbacks: {
           label: function(context) {
             if( typeof(context.dataset.data[context.dataIndex]) == "number" ) {
-              return "85% of the issues, leave this column in "+context.dataset.data[context.dataIndex]+" days";
+              let full_data = <%= @bar_data.inspect %>;
+              let columnIndex = context.dataIndex;
+              let rowIndex = context.datasetIndex - <%= @row_index_offset %>;
+              return context.dataset.label + " of completed work items left this column in " +full_data[rowIndex][columnIndex] + " days or less";
             }
             else {
-              return context.dataset.data[context.dataIndex].title
+              return context.dataset.data[context.dataIndex].title;
             }
           }
         }
+      },
+      legend: {
+        labels: {
+          filter: function(item, chart) {
+            // Logic to remove a particular legend item goes here
+            return !item.text.includes('%');
+          }
+        }
       }
+
     }
   }
 });

data/lib/jirametrics/html/aging_work_table.erb

@@ -1,11 +1,12 @@
 <table class='standard'>
   <thead>
     <tr>
-      <th>Age (days)</th>
-      <th>E</th>
-      <th>B</th>
+      <th title="Age in days">Age</th>
+      <th title="Expedited">E</th>
+      <th title="Blocked / Stalled">B/S</th>
       <th>Issue</th>
       <th>Status</th>
+      <th>Forecast</th>
       <th>Fix versions</th>
       <% if any_scrum_boards %>
         <th>Sprints</th>
@@ -40,7 +41,8 @@
             </div>
           <% end %>
         </td>
-        <td><%= format_status issue.status.name, board: issue.board %></td>
+        <td><%= format_status issue.status, board: issue.board %></td>
+        <td><%= dates_text(issue) %></td>
         <td><%= fix_versions_text(issue) %></td>
         <% if any_scrum_boards %>
           <td><%= sprints_text(issue) %></td>