jirametrics 2.0 → 2.11

data/lib/jirametrics/downloader.rb

@@ -6,11 +6,11 @@ require 'json'
 class Downloader
   CURRENT_METADATA_VERSION = 4
 
-  attr_accessor :metadata, :quiet_mode
+  attr_accessor :metadata
   attr_reader :file_system
 
   # For testing only
-  attr_reader :start_date_in_query
+  attr_reader :start_date_in_query, :board_id_to_filter_id
 
   def initialize download_config:, file_system:, jira_gateway:
     @metadata = {}
@@ -39,10 +39,11 @@ class Downloader
     # board_ids = @download_config.board_ids
 
     remove_old_files
+    update_status_history_file
     download_statuses
     find_board_ids.each do |id|
-      download_board_configuration board_id: id
-      download_issues board_id: id
+      board = download_board_configuration board_id: id
+      download_issues board: board
     end
 
     save_metadata
@@ -54,8 +55,7 @@ class Downloader
   end
 
   def log text, both: false
-    @file_system.log text
-    puts text if both
+    @file_system.log text, also_write_to_stderr: both
   end
 
   def find_board_ids
@@ -65,19 +65,19 @@ class Downloader
     ids
   end
 
-  def download_issues board_id:
-    log "  Downloading primary issues for board #{board_id}", both: true
-    path = "#{@target_path}#{@download_config.project_config.file_prefix}_issues/"
+  def download_issues board:
+    log "  Downloading primary issues for board #{board.id}", both: true
+    path = File.join(@target_path, "#{file_prefix}_issues/")
     unless Dir.exist?(path)
       log "  Creating path #{path}"
       Dir.mkdir(path)
     end
 
-    filter_id = @board_id_to_filter_id[board_id]
+    filter_id = @board_id_to_filter_id[board.id]
     jql = make_jql(filter_id: filter_id)
-    jira_search_by_jql(jql: jql, initial_query: true, board_id: board_id, path: path)
+    jira_search_by_jql(jql: jql, initial_query: true, board: board, path: path)
 
-    log "  Downloading linked issues for board #{board_id}", both: true
+    log "  Downloading linked issues for board #{board.id}", both: true
     loop do
       @issue_keys_pending_download.reject! { |key| @issue_keys_downloaded_in_current_run.include? key }
       break if @issue_keys_pending_download.empty?
@@ -85,15 +85,15 @@ class Downloader
       keys_to_request = @issue_keys_pending_download[0..99]
       @issue_keys_pending_download.reject! { |key| keys_to_request.include? key }
       jql = "key in (#{keys_to_request.join(', ')})"
-      jira_search_by_jql(jql: jql, initial_query: false, board_id: board_id, path: path)
+      jira_search_by_jql(jql: jql, initial_query: false, board: board, path: path)
     end
   end
 
-  def jira_search_by_jql jql:, initial_query:, board_id:, path:
+  def jira_search_by_jql jql:, initial_query:, board:, path:
     intercept_jql = @download_config.project_config.settings['intercept_jql']
     jql = intercept_jql.call jql if intercept_jql
 
-    log "  #{jql}"
+    log "  JQL: #{jql}"
     escaped_jql = CGI.escape jql
 
     max_results = 100
@@ -103,14 +103,12 @@ class Downloader
       json = @jira_gateway.call_url relative_url: '/rest/api/2/search' \
         "?jql=#{escaped_jql}&maxResults=#{max_results}&startAt=#{start_at}&expand=changelog&fields=*all"
 
-      exit_if_call_failed json
-
       json['issues'].each do |issue_json|
         issue_json['exporter'] = {
           'in_initial_query' => initial_query
         }
-        identify_other_issues_to_be_downloaded issue_json
-        file = "#{issue_json['key']}-#{board_id}.json"
+        identify_other_issues_to_be_downloaded raw_issue: issue_json, board: board
+        file = "#{issue_json['key']}-#{board.id}.json"
 
         @file_system.save_json(json: issue_json, filename: File.join(path, file))
       end
@@ -125,8 +123,8 @@ class Downloader
     end
   end
 
-  def identify_other_issues_to_be_downloaded raw_issue
-    issue = Issue.new raw: raw_issue, board: nil
+  def identify_other_issues_to_be_downloaded raw_issue:, board:
+    issue = Issue.new raw: raw_issue, board: board
     @issue_keys_downloaded_in_current_run << issue.key
 
     # Parent
@@ -137,51 +135,64 @@ class Downloader
     issue.raw['fields']['subtasks']&.each do |raw_subtask|
       @issue_keys_pending_download << raw_subtask['key']
     end
-
-    # Links
-    # We shouldn't blindly follow links as some, like cloners, aren't valuable and are just wasting time/effort
-    # to download
-    # issue.raw['fields']['issuelinks'].each do |raw_link|
-    #   @issue_keys_pending_download << IssueLink(raw: raw_link).other_issue.key
-    # end
-  end
-
-  def exit_if_call_failed json
-    # Sometimes Jira returns the singular form of errorMessage and sometimes the plural. Consistency FTW.
-    return unless json['errorMessages'] || json['errorMessage']
-
-    log "Download failed. See #{@logfile_name} for details.", both: true
-    log "  #{JSON.pretty_generate(json)}"
-    exit 1
   end
 
   def download_statuses
     log '  Downloading all statuses', both: true
-    json = @jira_gateway.call_url relative_url: "/rest/api/2/status"
+    json = @jira_gateway.call_url relative_url: '/rest/api/2/status'
 
     @file_system.save_json(
-      json: json, 
-      filename: "#{@target_path}#{@download_config.project_config.file_prefix}_statuses.json"
+      json: json,
+      filename: File.join(@target_path, "#{file_prefix}_statuses.json")
     )
   end
 
+  def update_status_history_file
+    status_filename = File.join(@target_path, "#{file_prefix}_statuses.json")
+    return unless file_system.file_exist? status_filename
+
+    status_json = file_system.load_json(status_filename)
+
+    history_filename = File.join(@target_path, "#{file_prefix}_status_history.json")
+    history_json = file_system.load_json(history_filename) if file_system.file_exist? history_filename
+
+    if history_json
+      file_system.log '  Updating status history file', also_write_to_stderr: true
+    else
+      file_system.log '  Creating status history file', also_write_to_stderr: true
+      history_json = []
+    end
+
+    status_json.each do |status_item|
+      id = status_item['id']
+      history_item = history_json.find { |s| s['id'] == id }
+      history_json.delete(history_item) if history_item
+      history_json << status_item
+    end
+
+    file_system.save_json(filename: history_filename, json: history_json)
+  end
+
   def download_board_configuration board_id:
     log "  Downloading board configuration for board #{board_id}", both: true
     json = @jira_gateway.call_url relative_url: "/rest/agile/1.0/board/#{board_id}/configuration"
-    exit_if_call_failed json
 
-    @board_id_to_filter_id[board_id] = json['filter']['id'].to_i
-    # @board_configuration = json if @download_config.board_ids.size == 1
+    @file_system.save_json(
+      json: json,
+      filename: File.join(@target_path, "#{file_prefix}_board_#{board_id}_configuration.json")
+    )
 
-    file_prefix = @download_config.project_config.file_prefix
-    @file_system.save_json json: json, filename: "#{@target_path}#{file_prefix}_board_#{board_id}_configuration.json"
+    # We have a reported bug that blew up on this line. Moved it after the save so we can
+    # actually look at the returned json.
+    @board_id_to_filter_id[board_id] = json['filter']['id'].to_i
 
     download_sprints board_id: board_id if json['type'] == 'scrum'
+    # TODO: Should be passing actual statuses, not empty list
+    Board.new raw: json, possible_statuses: StatusCollection.new
   end
 
   def download_sprints board_id:
     log "  Downloading sprints for board #{board_id}", both: true
-    file_prefix = @download_config.project_config.file_prefix
     max_results = 100
     start_at = 0
     is_last = false
@@ -189,20 +200,23 @@ class Downloader
     while is_last == false
       json = @jira_gateway.call_url relative_url: "/rest/agile/1.0/board/#{board_id}/sprint?" \
         "maxResults=#{max_results}&startAt=#{start_at}"
-      exit_if_call_failed json
 
       @file_system.save_json(
         json: json,
-        filename: "#{@target_path}#{file_prefix}_board_#{board_id}_sprints_#{start_at}.json"
+        filename: File.join(@target_path, "#{file_prefix}_board_#{board_id}_sprints_#{start_at}.json")
       )
       is_last = json['isLast']
       max_results = json['maxResults']
-      start_at += json['values'].size
+      if json['values']
+        start_at += json['values'].size
+      else
+        log "  No sprints found for board #{board_id}"
+      end
     end
   end
 
   def metadata_pathname
-    "#{@target_path}#{@download_config.project_config.file_prefix}_meta.json"
+    File.join(@target_path, "#{file_prefix}_meta.json")
   end
 
   def load_metadata
@@ -245,17 +259,17 @@ class Downloader
   end
 
   def remove_old_files
-    file_prefix = @download_config.project_config.file_prefix
     Dir.foreach @target_path do |file|
       next unless file.match?(/^#{file_prefix}_\d+\.json$/)
+      next if file == "#{file_prefix}_status_history.json"
 
-      File.unlink "#{@target_path}#{file}"
+      File.unlink File.join(@target_path, file)
     end
 
     return if @cached_data_format_is_current
 
     # Also throw away all the previously downloaded issues.
-    path = File.join @target_path, "#{file_prefix}_issues"
+    path = File.join(@target_path, "#{file_prefix}_issues")
     return unless File.exist? path
 
     Dir.foreach path do |file|
@@ -269,8 +283,10 @@ class Downloader
     segments = []
     segments << "filter=#{filter_id}"
 
-    unless @download_config.rolling_date_count.nil?
-      @download_date_range = (today.to_date - @download_config.rolling_date_count)..today.to_date
+    start_date = @download_config.start_date today: today
+
+    if start_date
+      @download_date_range = start_date..today.to_date
 
       # For an incremental download, we want to query from the end of the previous one, not from the
       # beginning of the full range.
@@ -283,13 +299,16 @@ class Downloader
 
       # Pick up any issues that had a status change in the range
       start_date_text = @start_date_in_query.strftime '%Y-%m-%d'
-      end_date_text = today.strftime '%Y-%m-%d'
       # find_in_range = %((status changed DURING ("#{start_date_text} 00:00","#{end_date_text} 23:59")))
-      find_in_range = %((updated >= "#{start_date_text} 00:00" AND updated <= "#{end_date_text} 23:59"))
+      find_in_range = %(updated >= "#{start_date_text} 00:00")
 
       segments << "(#{find_in_range} OR #{catch_all})"
     end
 
     segments.join ' AND '
   end
+
+  def file_prefix
+    @download_config.project_config.get_file_prefix
+  end
 end

data/lib/jirametrics/{story_point_accuracy_chart.rb → estimate_accuracy_chart.rb}

@@ -1,20 +1,25 @@
 # frozen_string_literal: true
 
-class StoryPointAccuracyChart < ChartBase
-  def initialize configuration_block = nil
+class EstimateAccuracyChart < ChartBase
+  def initialize configuration_block
     super()
 
     header_text 'Estimate Accuracy'
     description_text <<-HTML
-      <p>
+      <div class="p">
         This chart graphs estimates against actual recorded cycle times. Since
         estimates can change over time, we're graphing the estimate at the time that the story started.
-      </p>
-      <p>
-        The completed dots indicate cycletimes. The aging dots (if you turn them on) show the current
-        age of items, which will give you a hint as to where they might end up. If they're already
-        far to the right then you know you have a problem.
-      </p>
+      </div>
+      <div class="p">
+        The #{color_block '--estimate-accuracy-chart-completed-fill-color'} completed dots indicate
+        cycletimes.
+        <% if @has_aging_data %>
+          The #{color_block '--estimate-accuracy-chart-active-fill-color'} aging dots
+          (click on the legend to turn them on) show the current
+          age of items, which will give you a hint as to where they might end up. If they're already
+          far to the right then you know you have a problem.
+        <% end %>
+      </div>
     HTML
 
     @y_axis_label = 'Story Point Estimates'
@@ -22,7 +27,7 @@ class StoryPointAccuracyChart < ChartBase
     @y_axis_block = ->(issue, start_time) { story_points_at(issue: issue, start_time: start_time)&.to_f }
     @y_axis_sort_order = nil
 
-    instance_eval(&configuration_block) if configuration_block
+    instance_eval(&configuration_block)
   end
 
   def run
@@ -34,31 +39,17 @@ class StoryPointAccuracyChart < ChartBase
   end
 
   def scan_issues
-    aging_hash = {}
-    completed_hash = {}
-
-    issues.each do |issue|
-      cycletime = issue.board.cycletime
-      start_time = cycletime.started_time(issue)
-      stop_time = cycletime.stopped_time(issue)
-
-      next unless start_time
-
-      hash = stop_time ? completed_hash : aging_hash
-
-      estimate = @y_axis_block.call issue, start_time
-      cycle_time = ((stop_time&.to_date || date_range.end) - start_time.to_date).to_i + 1
+    completed_hash, aging_hash = split_into_completed_and_aging issues: issues
 
-      next if estimate.nil?
-
-      key = [estimate, cycle_time]
-      (hash[key] ||= []) << issue
-    end
+    @has_aging_data = !aging_hash.empty?
 
     [
-      [completed_hash, 'Completed', '#66FF99', 'green', false],
-      [aging_hash, 'Still in progress', '#FFCCCB', 'red', true]
-    ].filter_map do |hash, label, fill_color, border_color, starts_hidden|
+      [completed_hash, 'Completed', 'completed', false],
+      [aging_hash, 'Still in progress', 'active', true]
+    ].filter_map do |hash, label, completed_or_active, starts_hidden|
+      fill_color = CssVariable["--estimate-accuracy-chart-#{completed_or_active}-fill-color"]
+      border_color = CssVariable["--estimate-accuracy-chart-#{completed_or_active}-border-color"]
+
       # We sort so that the smaller circles are in front of the bigger circles.
       data = hash.sort(&hash_sorter).collect do |key, values|
         estimate, cycle_time = *key
@@ -83,7 +74,31 @@ class StoryPointAccuracyChart < ChartBase
         'borderColor' => border_color,
         'hidden' => starts_hidden
       }
-    end.compact
+    end
+  end
+
+  def split_into_completed_and_aging issues:
+    aging_hash = {}
+    completed_hash = {}
+
+    issues.each do |issue|
+      cycletime = issue.board.cycletime
+      start_time, stop_time = cycletime.started_stopped_times(issue)
+
+      next unless start_time
+
+      hash = stop_time ? completed_hash : aging_hash
+
+      estimate = @y_axis_block.call issue, start_time
+      cycle_time = ((stop_time&.to_date || date_range.end) - start_time.to_date).to_i + 1
+
+      next if estimate.nil?
+
+      key = [estimate, cycle_time]
+      (hash[key] ||= []) << issue
+    end
+
+    [completed_hash, aging_hash]
   end
 
   def hash_sorter

data/lib/jirametrics/examples/aggregated_project.rb

@@ -3,24 +3,23 @@
 # This file is really intended to give you ideas about how you might configure your own reports, not
 # as a complete setup that will work in every case.
 #
-# See https://github.com/mikebowler/jirametrics/wiki/Examples-folder for more details
-#
 # The point of an AGGREGATED report is that we're now looking at a higher level. We might use this in a
 # S2 meeting (Scrum of Scrums) to talk about the things that are happening across teams, not within a
 # single team. For that reason, we look at slightly different things that we would on a single team board.
 
 class Exporter
-  def aggregated_project name:, project_names:
+  def aggregated_project name:, project_names:, settings: {}
     project name: name do
       puts name
+      file_prefix name
+      self.settings.merge! settings
+
       aggregate do
         project_names.each do |project_name|
           include_issues_from project_name
         end
       end
 
-      file_prefix name
-
       file do
         file_suffix '.html'
         issues.reject! do |issue|
@@ -28,32 +27,27 @@ class Exporter
         end
 
         html_report do
+          html '<h1>Boards included in this report</h1><ul>', type: :header
+          board_lines = []
+          included_projects.each do |project|
+            project.all_boards.each_value do |board|
+              board_lines << "<a href='#{project.get_file_prefix}.html'>#{board.name}</a> from project #{project.name}"
+            end
+          end
+          board_lines.sort.each { |line| html "<li>#{line}</li>", type: :header }
+          html '</ul>', type: :header
+
           cycletime_scatterplot do
             show_trend_lines
+            # For an aggregated report we group by board rather than by type
             grouping_rules do |issue, rules|
               rules.label = issue.board.name
             end
           end
           # aging_work_in_progress_chart
-          daily_wip_chart do
-            header_text 'Daily WIP by Parent'
-            description_text <<-TEXT
-              <p>How much work is in progress, grouped by the parent of the issue. This will give us an
-              indication of how focused we are on higher level objectives. If there are many parent
-              tickets in progress at the same time, either this team has their focus scattered or we
-              aren't doing a good job of
-              <a href="https://improvingflow.com/2024/02/21/slicing-epics.html">splitting those parent
-              tickets</a>. Neither of those is desirable.</p>
-              <p>If you're expecting all work items to have parents and there are a lot that don't,
-              that's also something to look at. Consider whether there is even value in aggregating
-              these projects if they don't share parent dependencies. Aggregation helps us when we're
-              looking at related work and if there aren't parent dependencies then the work may not
-              be related.</p>
-            TEXT
-            grouping_rules do |issue, rules|
-              rules.label = issue.parent&.key || 'No parent'
-              rules.color = 'white' if rules.label == 'No parent'
-            end
+          daily_wip_by_parent_chart do
+            # When aggregating, the chart tends to need more vertical space
+            canvas height: 400, width: 800
           end
           aging_work_table do
             # In an aggregated report, we likely only care about items that are old so exclude anything
@@ -67,25 +61,14 @@ class Exporter
 
             # By default, the issue doesn't show what board it's on and this is important for an
             # aggregated view
+            chart = self
             issue_rules do |issue, rules|
-              key = issue.key
-              key = "<S>#{key} </S> " if issue.status.category_name == 'Done'
-              rules.label = "<#{key} [#{issue.type}]<BR/>#{issue.board.name}<BR/>#{word_wrap issue.summary}>"
+              chart.default_issue_rules.call(issue, rules)
+              rules.label = rules.label.split('<BR/>').insert(1, "Board: #{issue.board.name}").join('<BR/>')
             end
 
             link_rules do |link, rules|
-              # By default, the dependency chart shows everything. Clean it up a bit.
-              case link.name
-              when 'Cloners'
-                # We don't want to see any clone links at all.
-                rules.ignore
-              when 'Blocks'
-                # For blocks, by default Jira will have links going both
-                # ways and we want them only going one way. Also make the
-                # link red.
-                rules.merge_bidirectional keep: 'outward'
-                rules.line_color = 'red'
-              end
+              chart.default_link_rules.call(link, rules)
 
               # Because this is the aggregated view, let's hide any link that doesn't cross boards.
               rules.ignore if link.origin.board == link.other_issue.board

data/lib/jirametrics/examples/standard_project.rb

@@ -2,41 +2,49 @@
 
 # This file is really intended to give you ideas about how you might configure your own reports, not
 # as a complete setup that will work in every case.
-#
-# See https://github.com/mikebowler/jirametrics/wiki/Examples-folder for more
 class Exporter
   def standard_project name:, file_prefix:, ignore_issues: nil, starting_status: nil, boards: {},
-      default_board: nil, anonymize: false
+      default_board: nil, anonymize: false, settings: {}, status_category_mappings: {},
+      rolling_date_count: 90, no_earlier_than: nil, ignore_types: %w[Sub-task Subtask Epic],
+      show_experimental_charts: false
 
     project name: name do
       puts name
+      file_prefix file_prefix
+
       self.anonymize if anonymize
+      self.settings.merge! settings
+
+      status_category_mappings.each do |status, category|
+        status_category_mapping status: status, category: category
+      end
 
-      settings['blocked_link_text'] = ['is blocked by']
-      file_prefix file_prefix
       download do
-        rolling_date_count 90
+        self.rolling_date_count(rolling_date_count) if rolling_date_count
+        self.no_earlier_than(no_earlier_than) if no_earlier_than
       end
 
       boards.each_key do |board_id|
         block = boards[board_id]
         if block == :default
           block = lambda do |_|
-            start_at first_time_in_status_category('In Progress')
-            stop_at still_in_status_category('Done')
+            start_at first_time_in_status_category(:indeterminate)
+            stop_at still_in_status_category(:done)
           end
         end
         board id: board_id do
           cycletime(&block)
-          expedited_priority_names 'Critical', 'Highest', 'Immediate Gating'
         end
       end
 
+      issues.reject! do |issue|
+        ignore_types.include? issue.type
+      end
+
+      discard_changes_before status_becomes: (starting_status || :backlog) # rubocop:disable Style/RedundantParentheses
+
       file do
         file_suffix '.html'
-        issues.reject! do |issue|
-          %w[Sub-task Epic].include? issue.type
-        end
 
         issues.reject! { |issue| ignore_issues.include? issue.key } if ignore_issues
 
@@ -46,12 +54,10 @@ class Exporter
           html "<H1>#{name}</H1>", type: :header
           boards.each_key do |id|
             board = find_board id
-            html "<div><a href='#{board.url}'>#{id} #{board.name}</a></div>",
+            html "<div><a href='#{board.url}'>#{id} #{board.name}</a> (#{board.board_type})</div>",
                  type: :header
           end
 
-          discard_changes_before status_becomes: (starting_status || :backlog)
-
           cycletime_scatterplot do
             show_trend_lines
           end
@@ -77,42 +83,12 @@ class Exporter
           aging_work_table
           daily_wip_by_age_chart
           daily_wip_by_blocked_stalled_chart
-          daily_wip_chart do
-            header_text 'Daily WIP by Parent'
-            description_text <<-TEXT
-              How much work is in progress, grouped by the parent of the issue. This will give us an
-              indication of how focused we are on higher level objectives. If there are many parent
-              tickets in progress at the same time, either this team has their focus scattered or we
-              aren't doing a good job of
-              <a href="https://improvingflow.com/2024/02/21/slicing-epics.html">splitting those parent
-              tickets</a>. Neither of those is desirable.
-            TEXT
-            grouping_rules do |issue, rules|
-              rules.label = issue.parent&.key || 'No parent'
-              rules.color = 'white' if rules.label == 'No parent'
-            end
-          end
+          daily_wip_by_parent_chart
+          flow_efficiency_scatterplot if show_experimental_charts
           expedited_chart
           sprint_burndown
-          story_point_accuracy_chart
-
-          dependency_chart do
-            link_rules do |link, rules|
-              case link.name
-              when 'Cloners'
-                rules.ignore
-              when 'Dependency', 'Blocks', 'Parent/Child', 'Cause', 'Satisfy Requirement', 'Relates'
-                rules.merge_bidirectional keep: 'outward'
-                rules.merge_bidirectional keep: 'outward'
-              when 'Sync'
-                rules.use_bidirectional_arrows
-              else
-                # This is a link type that we don't recognized. Dump it to standard out to draw attention
-                # to it.
-                puts "name=#{link.name}, label=#{link.label}"
-              end
-            end
-          end
+          estimate_accuracy_chart
+          dependency_chart
         end
       end
     end