jirametrics 2.0 → 2.12.1

data/lib/jirametrics/downloader.rb

@@ -6,11 +6,11 @@ require 'json'
 class Downloader
   CURRENT_METADATA_VERSION = 4
 
-  attr_accessor :metadata, :quiet_mode
+  attr_accessor :metadata
   attr_reader :file_system
 
   # For testing only
-  attr_reader :start_date_in_query
+  attr_reader :start_date_in_query, :board_id_to_filter_id
 
   def initialize download_config:, file_system:, jira_gateway:
     @metadata = {}
@@ -39,11 +39,13 @@ class Downloader
     # board_ids = @download_config.board_ids
 
     remove_old_files
+    update_status_history_file
     download_statuses
     find_board_ids.each do |id|
-      download_board_configuration board_id: id
-      download_issues board_id: id
+      board = download_board_configuration board_id: id
+      download_issues board: board
     end
+    download_users
 
     save_metadata
   end
@@ -54,8 +56,7 @@ class Downloader
   end
 
   def log text, both: false
-    @file_system.log text
-    puts text if both
+    @file_system.log text, also_write_to_stderr: both
   end
 
   def find_board_ids
@@ -65,19 +66,19 @@ class Downloader
     ids
   end
 
-  def download_issues board_id:
-    log "  Downloading primary issues for board #{board_id}", both: true
-    path = "#{@target_path}#{@download_config.project_config.file_prefix}_issues/"
+  def download_issues board:
+    log "  Downloading primary issues for board #{board.id}", both: true
+    path = File.join(@target_path, "#{file_prefix}_issues/")
     unless Dir.exist?(path)
       log "  Creating path #{path}"
       Dir.mkdir(path)
     end
 
-    filter_id = @board_id_to_filter_id[board_id]
+    filter_id = @board_id_to_filter_id[board.id]
     jql = make_jql(filter_id: filter_id)
-    jira_search_by_jql(jql: jql, initial_query: true, board_id: board_id, path: path)
+    jira_search_by_jql(jql: jql, initial_query: true, board: board, path: path)
 
-    log "  Downloading linked issues for board #{board_id}", both: true
+    log "  Downloading linked issues for board #{board.id}", both: true
     loop do
       @issue_keys_pending_download.reject! { |key| @issue_keys_downloaded_in_current_run.include? key }
       break if @issue_keys_pending_download.empty?
@@ -85,15 +86,15 @@ class Downloader
       keys_to_request = @issue_keys_pending_download[0..99]
       @issue_keys_pending_download.reject! { |key| keys_to_request.include? key }
       jql = "key in (#{keys_to_request.join(', ')})"
-      jira_search_by_jql(jql: jql, initial_query: false, board_id: board_id, path: path)
+      jira_search_by_jql(jql: jql, initial_query: false, board: board, path: path)
     end
   end
 
-  def jira_search_by_jql jql:, initial_query:, board_id:, path:
+  def jira_search_by_jql jql:, initial_query:, board:, path:
     intercept_jql = @download_config.project_config.settings['intercept_jql']
     jql = intercept_jql.call jql if intercept_jql
 
-    log "  #{jql}"
+    log "  JQL: #{jql}"
     escaped_jql = CGI.escape jql
 
     max_results = 100
@@ -103,14 +104,12 @@ class Downloader
       json = @jira_gateway.call_url relative_url: '/rest/api/2/search' \
         "?jql=#{escaped_jql}&maxResults=#{max_results}&startAt=#{start_at}&expand=changelog&fields=*all"
 
-      exit_if_call_failed json
-
       json['issues'].each do |issue_json|
         issue_json['exporter'] = {
           'in_initial_query' => initial_query
         }
-        identify_other_issues_to_be_downloaded issue_json
-        file = "#{issue_json['key']}-#{board_id}.json"
+        identify_other_issues_to_be_downloaded raw_issue: issue_json, board: board
+        file = "#{issue_json['key']}-#{board.id}.json"
 
         @file_system.save_json(json: issue_json, filename: File.join(path, file))
       end
@@ -125,8 +124,8 @@ class Downloader
     end
   end
 
-  def identify_other_issues_to_be_downloaded raw_issue
-    issue = Issue.new raw: raw_issue, board: nil
+  def identify_other_issues_to_be_downloaded raw_issue:, board:
+    issue = Issue.new raw: raw_issue, board: board
     @issue_keys_downloaded_in_current_run << issue.key
 
     # Parent
@@ -137,51 +136,74 @@ class Downloader
     issue.raw['fields']['subtasks']&.each do |raw_subtask|
       @issue_keys_pending_download << raw_subtask['key']
     end
-
-    # Links
-    # We shouldn't blindly follow links as some, like cloners, aren't valuable and are just wasting time/effort
-    # to download
-    # issue.raw['fields']['issuelinks'].each do |raw_link|
-    #   @issue_keys_pending_download << IssueLink(raw: raw_link).other_issue.key
-    # end
   end
 
-  def exit_if_call_failed json
-    # Sometimes Jira returns the singular form of errorMessage and sometimes the plural. Consistency FTW.
-    return unless json['errorMessages'] || json['errorMessage']
+  def download_statuses
+    log '  Downloading all statuses', both: true
+    json = @jira_gateway.call_url relative_url: '/rest/api/2/status'
 
-    log "Download failed. See #{@logfile_name} for details.", both: true
-    log "  #{JSON.pretty_generate(json)}"
-    exit 1
+    @file_system.save_json(
+      json: json,
+      filename: File.join(@target_path, "#{file_prefix}_statuses.json")
+    )
   end
 
-  def download_statuses
-    log '  Downloading all statuses', both: true
-    json = @jira_gateway.call_url relative_url: "/rest/api/2/status"
+  def download_users
+    log '  Downloading all users', both: true
+    json = @jira_gateway.call_url relative_url: '/rest/api/2/users'
 
     @file_system.save_json(
-      json: json, 
-      filename: "#{@target_path}#{@download_config.project_config.file_prefix}_statuses.json"
+      json: json,
+      filename: File.join(@target_path, "#{file_prefix}_users.json")
     )
   end
 
+  def update_status_history_file
+    status_filename = File.join(@target_path, "#{file_prefix}_statuses.json")
+    return unless file_system.file_exist? status_filename
+
+    status_json = file_system.load_json(status_filename)
+
+    history_filename = File.join(@target_path, "#{file_prefix}_status_history.json")
+    history_json = file_system.load_json(history_filename) if file_system.file_exist? history_filename
+
+    if history_json
+      file_system.log '  Updating status history file', also_write_to_stderr: true
+    else
+      file_system.log '  Creating status history file', also_write_to_stderr: true
+      history_json = []
+    end
+
+    status_json.each do |status_item|
+      id = status_item['id']
+      history_item = history_json.find { |s| s['id'] == id }
+      history_json.delete(history_item) if history_item
+      history_json << status_item
+    end
+
+    file_system.save_json(filename: history_filename, json: history_json)
+  end
+
   def download_board_configuration board_id:
     log "  Downloading board configuration for board #{board_id}", both: true
     json = @jira_gateway.call_url relative_url: "/rest/agile/1.0/board/#{board_id}/configuration"
-    exit_if_call_failed json
 
-    @board_id_to_filter_id[board_id] = json['filter']['id'].to_i
-    # @board_configuration = json if @download_config.board_ids.size == 1
+    @file_system.save_json(
+      json: json,
+      filename: File.join(@target_path, "#{file_prefix}_board_#{board_id}_configuration.json")
+    )
 
-    file_prefix = @download_config.project_config.file_prefix
-    @file_system.save_json json: json, filename: "#{@target_path}#{file_prefix}_board_#{board_id}_configuration.json"
+    # We have a reported bug that blew up on this line. Moved it after the save so we can
+    # actually look at the returned json.
+    @board_id_to_filter_id[board_id] = json['filter']['id'].to_i
 
     download_sprints board_id: board_id if json['type'] == 'scrum'
+    # TODO: Should be passing actual statuses, not empty list
+    Board.new raw: json, possible_statuses: StatusCollection.new
   end
 
   def download_sprints board_id:
     log "  Downloading sprints for board #{board_id}", both: true
-    file_prefix = @download_config.project_config.file_prefix
     max_results = 100
     start_at = 0
     is_last = false
@@ -189,20 +211,23 @@ class Downloader
     while is_last == false
       json = @jira_gateway.call_url relative_url: "/rest/agile/1.0/board/#{board_id}/sprint?" \
         "maxResults=#{max_results}&startAt=#{start_at}"
-      exit_if_call_failed json
 
       @file_system.save_json(
         json: json,
-        filename: "#{@target_path}#{file_prefix}_board_#{board_id}_sprints_#{start_at}.json"
+        filename: File.join(@target_path, "#{file_prefix}_board_#{board_id}_sprints_#{start_at}.json")
       )
       is_last = json['isLast']
       max_results = json['maxResults']
-      start_at += json['values'].size
+      if json['values']
+        start_at += json['values'].size
+      else
+        log "  No sprints found for board #{board_id}"
+      end
     end
   end
 
   def metadata_pathname
-    "#{@target_path}#{@download_config.project_config.file_prefix}_meta.json"
+    File.join(@target_path, "#{file_prefix}_meta.json")
   end
 
   def load_metadata
@@ -245,17 +270,17 @@ class Downloader
   end
 
   def remove_old_files
-    file_prefix = @download_config.project_config.file_prefix
     Dir.foreach @target_path do |file|
       next unless file.match?(/^#{file_prefix}_\d+\.json$/)
+      next if file == "#{file_prefix}_status_history.json"
 
-      File.unlink "#{@target_path}#{file}"
+      File.unlink File.join(@target_path, file)
     end
 
     return if @cached_data_format_is_current
 
     # Also throw away all the previously downloaded issues.
-    path = File.join @target_path, "#{file_prefix}_issues"
+    path = File.join(@target_path, "#{file_prefix}_issues")
     return unless File.exist? path
 
     Dir.foreach path do |file|
@@ -269,8 +294,10 @@ class Downloader
     segments = []
     segments << "filter=#{filter_id}"
 
-    unless @download_config.rolling_date_count.nil?
-      @download_date_range = (today.to_date - @download_config.rolling_date_count)..today.to_date
+    start_date = @download_config.start_date today: today
+
+    if start_date
+      @download_date_range = start_date..today.to_date
 
       # For an incremental download, we want to query from the end of the previous one, not from the
       # beginning of the full range.
@@ -283,13 +310,16 @@ class Downloader
 
       # Pick up any issues that had a status change in the range
       start_date_text = @start_date_in_query.strftime '%Y-%m-%d'
-      end_date_text = today.strftime '%Y-%m-%d'
       # find_in_range = %((status changed DURING ("#{start_date_text} 00:00","#{end_date_text} 23:59")))
-      find_in_range = %((updated >= "#{start_date_text} 00:00" AND updated <= "#{end_date_text} 23:59"))
+      find_in_range = %(updated >= "#{start_date_text} 00:00")
 
       segments << "(#{find_in_range} OR #{catch_all})"
     end
 
     segments.join ' AND '
   end
+
+  def file_prefix
+    @download_config.project_config.get_file_prefix
+  end
 end

data/lib/jirametrics/estimate_accuracy_chart.rb

@@ -0,0 +1,173 @@
+# frozen_string_literal: true
+
+class EstimateAccuracyChart < ChartBase
+  def initialize configuration_block
+    super()
+
+    header_text 'Estimate Accuracy'
+    description_text <<-HTML
+      <div class="p">
+        This chart graphs estimates against actual recorded cycle times. Since
+        estimates can change over time, we're graphing the estimate at the time that the story started.
+      </div>
+      <div class="p">
+        The #{color_block '--estimate-accuracy-chart-completed-fill-color'} completed dots indicate
+        cycletimes.
+        <% if @has_aging_data %>
+          The #{color_block '--estimate-accuracy-chart-active-fill-color'} aging dots
+          (click on the legend to turn them on) show the current
+          age of items, which will give you a hint as to where they might end up. If they're already
+          far to the right then you know you have a problem.
+        <% end %>
+      </div>
+    HTML
+
+    @y_axis_type = 'linear'
+    @y_axis_block = ->(issue, start_time) { estimate_at(issue: issue, start_time: start_time)&.to_f }
+    @y_axis_sort_order = nil
+
+    instance_eval(&configuration_block)
+  end
+
+  def run
+    if @y_axis_label.nil?
+      text = current_board.estimation_configuration.units == :story_points ? 'Story Points' : 'Days'
+      @y_axis_label = "Estimated #{text}"
+    end
+    data_sets = scan_issues
+
+    return '' if data_sets.empty?
+
+    wrap_and_render(binding, __FILE__)
+  end
+
+  def scan_issues
+    completed_hash, aging_hash = split_into_completed_and_aging issues: issues
+
+    estimation_units = current_board.estimation_configuration.units
+    @has_aging_data = !aging_hash.empty?
+
+    [
+      [completed_hash, 'Completed', 'completed', false],
+      [aging_hash, 'Still in progress', 'active', true]
+    ].filter_map do |hash, label, completed_or_active, starts_hidden|
+      fill_color = CssVariable["--estimate-accuracy-chart-#{completed_or_active}-fill-color"]
+      border_color = CssVariable["--estimate-accuracy-chart-#{completed_or_active}-border-color"]
+
+      # We sort so that the smaller circles are in front of the bigger circles.
+      data = hash.sort(&hash_sorter).collect do |key, values|
+        estimate, cycle_time = *key
+
+        title = [
+          "Estimate: #{estimate_label(estimate: estimate, estimation_units: estimation_units)}, " \
+            "Cycletime: #{label_days(cycle_time)}, " \
+            "#{values.size} issues"
+        ] + values.collect { |issue| "#{issue.key}: #{issue.summary}" }
+
+        {
+          'x' => cycle_time,
+          'y' => estimate,
+          'r' => values.size * 2,
+          'title' => title
+        }
+      end
+      next if data.empty?
+
+      {
+        'label' => label,
+        'data' => data,
+        'fill' => false,
+        'showLine' => false,
+        'backgroundColor' => fill_color,
+        'borderColor' => border_color,
+        'hidden' => starts_hidden
+      }
+    end
+  end
+
+  def estimate_label estimate:, estimation_units:
+    if @y_axis_type == 'linear'
+      if estimation_units == :story_points
+        estimate_label = "#{estimate}pts"
+      elsif estimation_units == :seconds
+        estimate_label = label_days estimate
+      end
+    end
+    estimate_label = estimate.to_s if estimate_label.nil?
+    estimate_label
+  end
+
+  def split_into_completed_and_aging issues:
+    aging_hash = {}
+    completed_hash = {}
+
+    issues.each do |issue|
+      cycletime = issue.board.cycletime
+      start_time, stop_time = cycletime.started_stopped_times(issue)
+
+      next unless start_time
+
+      hash = stop_time ? completed_hash : aging_hash
+
+      estimate = @y_axis_block.call issue, start_time
+      cycle_time = ((stop_time&.to_date || date_range.end) - start_time.to_date).to_i + 1
+
+      next if estimate.nil?
+
+      key = [estimate, cycle_time]
+      (hash[key] ||= []) << issue
+    end
+
+    [completed_hash, aging_hash]
+  end
+
+  def hash_sorter
+    lambda do |arg1, arg2|
+      estimate1 = arg1[0][0]
+      estimate2 = arg2[0][0]
+      sample_count1 = arg1.size
+      sample_count2 = arg2.size
+
+      if @y_axis_sort_order
+        index1 = @y_axis_sort_order.index estimate1
+        index2 = @y_axis_sort_order.index estimate2
+
+        if index1.nil?
+          comparison = 1
+        elsif index2.nil?
+          comparison = -1
+        else
+          comparison = index1 <=> index2
+        end
+        return comparison unless comparison.zero?
+      end
+
+      sample_count2 <=> sample_count1
+    end
+  end
+
+  def estimate_at issue:, start_time:, estimation_configuration: current_board.estimation_configuration
+    estimate = nil
+
+    issue.changes.each do |change|
+      return estimate if change.time >= start_time
+
+      if change.field == estimation_configuration.display_name || change.field == estimation_configuration.field_id
+        estimate = change.value
+        estimate = estimate.to_f / (24 * 60 * 60) if estimation_configuration.units == :seconds
+      end
+    end
+    estimate
+  end
+
+  def y_axis label:, sort_order: nil, &block
+    @y_axis_sort_order = sort_order
+    @y_axis_label = label
+    if sort_order
+      @y_axis_type = 'category'
+    else
+      @y_axis_type = 'linear'
+    end
+    @y_axis_block = block
+  end
+end

data/lib/jirametrics/estimation_configuration.rb

@@ -0,0 +1,25 @@
+# frozen_string_literal: true
+
+class EstimationConfiguration
+  attr_reader :units, :display_name, :field_id
+
+  def initialize raw:
+    @units = :story_points
+    @display_name = 'Story Points'
+
+    # If there wasn't an estimation section they rely on all defaults
+    return if raw.nil?
+
+    if raw['type'] == 'field'
+      @field_id = raw['field']['fieldId']
+      @display_name = raw['field']['displayName']
+      if @field_id == 'timeoriginalestimate'
+        @units = :seconds
+        @display_name = 'Original estimate'
+      end
+    elsif raw['type'] == 'issueCount'
+      @display_name = 'Issue Count'
+      @units = :issue_count
+    end
+  end
+end

data/lib/jirametrics/examples/aggregated_project.rb

@@ -3,24 +3,23 @@
 # This file is really intended to give you ideas about how you might configure your own reports, not
 # as a complete setup that will work in every case.
 #
-# See https://github.com/mikebowler/jirametrics/wiki/Examples-folder for more details
-#
 # The point of an AGGREGATED report is that we're now looking at a higher level. We might use this in a
 # S2 meeting (Scrum of Scrums) to talk about the things that are happening across teams, not within a
 # single team. For that reason, we look at slightly different things that we would on a single team board.
 
 class Exporter
-  def aggregated_project name:, project_names:
+  def aggregated_project name:, project_names:, settings: {}
     project name: name do
       puts name
+      file_prefix name
+      self.settings.merge! settings
+
       aggregate do
         project_names.each do |project_name|
           include_issues_from project_name
         end
       end
 
-      file_prefix name
-
       file do
         file_suffix '.html'
         issues.reject! do |issue|
@@ -28,32 +27,27 @@ class Exporter
         end
 
         html_report do
+          html '<h1>Boards included in this report</h1><ul>', type: :header
+          board_lines = []
+          included_projects.each do |project|
+            project.all_boards.each_value do |board|
+              board_lines << "<a href='#{project.get_file_prefix}.html'>#{board.name}</a> from project #{project.name}"
+            end
+          end
+          board_lines.sort.each { |line| html "<li>#{line}</li>", type: :header }
+          html '</ul>', type: :header
+
           cycletime_scatterplot do
             show_trend_lines
+            # For an aggregated report we group by board rather than by type
             grouping_rules do |issue, rules|
               rules.label = issue.board.name
             end
           end
           # aging_work_in_progress_chart
-          daily_wip_chart do
-            header_text 'Daily WIP by Parent'
-            description_text <<-TEXT
-              <p>How much work is in progress, grouped by the parent of the issue. This will give us an
-              indication of how focused we are on higher level objectives. If there are many parent
-              tickets in progress at the same time, either this team has their focus scattered or we
-              aren't doing a good job of
-              <a href="https://improvingflow.com/2024/02/21/slicing-epics.html">splitting those parent
-              tickets</a>. Neither of those is desirable.</p>
-              <p>If you're expecting all work items to have parents and there are a lot that don't,
-              that's also something to look at. Consider whether there is even value in aggregating
-              these projects if they don't share parent dependencies. Aggregation helps us when we're
-              looking at related work and if there aren't parent dependencies then the work may not
-              be related.</p>
-            TEXT
-            grouping_rules do |issue, rules|
-              rules.label = issue.parent&.key || 'No parent'
-              rules.color = 'white' if rules.label == 'No parent'
-            end
+          daily_wip_by_parent_chart do
+            # When aggregating, the chart tends to need more vertical space
+            canvas height: 400, width: 800
           end
           aging_work_table do
             # In an aggregated report, we likely only care about items that are old so exclude anything
@@ -67,25 +61,14 @@ class Exporter
 
             # By default, the issue doesn't show what board it's on and this is important for an
             # aggregated view
+            chart = self
             issue_rules do |issue, rules|
-              key = issue.key
-              key = "<S>#{key} </S> " if issue.status.category_name == 'Done'
-              rules.label = "<#{key} [#{issue.type}]<BR/>#{issue.board.name}<BR/>#{word_wrap issue.summary}>"
+              chart.default_issue_rules.call(issue, rules)
+              rules.label = rules.label.split('<BR/>').insert(1, "Board: #{issue.board.name}").join('<BR/>')
             end
 
             link_rules do |link, rules|
-              # By default, the dependency chart shows everything. Clean it up a bit.
-              case link.name
-              when 'Cloners'
-                # We don't want to see any clone links at all.
-                rules.ignore
-              when 'Blocks'
-                # For blocks, by default Jira will have links going both
-                # ways and we want them only going one way. Also make the
-                # link red.
-                rules.merge_bidirectional keep: 'outward'
-                rules.line_color = 'red'
-              end
+              chart.default_link_rules.call(link, rules)
 
               # Because this is the aggregated view, let's hide any link that doesn't cross boards.
               rules.ignore if link.origin.board == link.other_issue.board