jirametrics 2.4 → 2.11

data/lib/jirametrics/downloader.rb

@@ -6,7 +6,7 @@ require 'json'
 class Downloader
   CURRENT_METADATA_VERSION = 4
 
-  attr_accessor :metadata, :quiet_mode
+  attr_accessor :metadata
   attr_reader :file_system
 
   # For testing only
@@ -39,10 +39,11 @@ class Downloader
     # board_ids = @download_config.board_ids
 
     remove_old_files
+    update_status_history_file
     download_statuses
     find_board_ids.each do |id|
-      download_board_configuration board_id: id
-      download_issues board_id: id
+      board = download_board_configuration board_id: id
+      download_issues board: board
     end
 
     save_metadata
@@ -54,8 +55,7 @@ class Downloader
   end
 
   def log text, both: false
-    @file_system.log text
-    puts text if both && !@quiet_mode
+    @file_system.log text, also_write_to_stderr: both
   end
 
   def find_board_ids
@@ -65,19 +65,19 @@ class Downloader
     ids
   end
 
-  def download_issues board_id:
-    log "  Downloading primary issues for board #{board_id}", both: true
-    path = "#{@target_path}#{@download_config.project_config.file_prefix}_issues/"
+  def download_issues board:
+    log "  Downloading primary issues for board #{board.id}", both: true
+    path = File.join(@target_path, "#{file_prefix}_issues/")
     unless Dir.exist?(path)
       log "  Creating path #{path}"
       Dir.mkdir(path)
     end
 
-    filter_id = @board_id_to_filter_id[board_id]
+    filter_id = @board_id_to_filter_id[board.id]
     jql = make_jql(filter_id: filter_id)
-    jira_search_by_jql(jql: jql, initial_query: true, board_id: board_id, path: path)
+    jira_search_by_jql(jql: jql, initial_query: true, board: board, path: path)
 
-    log "  Downloading linked issues for board #{board_id}", both: true
+    log "  Downloading linked issues for board #{board.id}", both: true
     loop do
       @issue_keys_pending_download.reject! { |key| @issue_keys_downloaded_in_current_run.include? key }
       break if @issue_keys_pending_download.empty?
@@ -85,11 +85,11 @@ class Downloader
       keys_to_request = @issue_keys_pending_download[0..99]
       @issue_keys_pending_download.reject! { |key| keys_to_request.include? key }
       jql = "key in (#{keys_to_request.join(', ')})"
-      jira_search_by_jql(jql: jql, initial_query: false, board_id: board_id, path: path)
+      jira_search_by_jql(jql: jql, initial_query: false, board: board, path: path)
     end
   end
 
-  def jira_search_by_jql jql:, initial_query:, board_id:, path:
+  def jira_search_by_jql jql:, initial_query:, board:, path:
     intercept_jql = @download_config.project_config.settings['intercept_jql']
     jql = intercept_jql.call jql if intercept_jql
 
@@ -103,14 +103,12 @@ class Downloader
       json = @jira_gateway.call_url relative_url: '/rest/api/2/search' \
         "?jql=#{escaped_jql}&maxResults=#{max_results}&startAt=#{start_at}&expand=changelog&fields=*all"
 
-      exit_if_call_failed json
-
       json['issues'].each do |issue_json|
         issue_json['exporter'] = {
           'in_initial_query' => initial_query
         }
-        identify_other_issues_to_be_downloaded issue_json
-        file = "#{issue_json['key']}-#{board_id}.json"
+        identify_other_issues_to_be_downloaded raw_issue: issue_json, board: board
+        file = "#{issue_json['key']}-#{board.id}.json"
 
         @file_system.save_json(json: issue_json, filename: File.join(path, file))
       end
@@ -125,8 +123,8 @@ class Downloader
     end
   end
 
-  def identify_other_issues_to_be_downloaded raw_issue
-    issue = Issue.new raw: raw_issue, board: nil
+  def identify_other_issues_to_be_downloaded raw_issue:, board:
+    issue = Issue.new raw: raw_issue, board: board
     @issue_keys_downloaded_in_current_run << issue.key
 
     # Parent
@@ -137,22 +135,6 @@ class Downloader
     issue.raw['fields']['subtasks']&.each do |raw_subtask|
       @issue_keys_pending_download << raw_subtask['key']
     end
-
-    # Links
-    # We shouldn't blindly follow links as some, like cloners, aren't valuable and are just wasting time/effort
-    # to download
-    # issue.raw['fields']['issuelinks'].each do |raw_link|
-    #   @issue_keys_pending_download << IssueLink(raw: raw_link).other_issue.key
-    # end
-  end
-
-  def exit_if_call_failed json
-    # Sometimes Jira returns the singular form of errorMessage and sometimes the plural. Consistency FTW.
-    return unless json['error'] || json['errorMessages'] || json['errorMessage']
-
-    log "Download failed. See #{@file_system.logfile_name} for details.", both: true
-    log "  #{JSON.pretty_generate(json)}"
-    exit 1
   end
 
   def download_statuses
@@ -161,29 +143,56 @@ class Downloader
 
     @file_system.save_json(
       json: json,
-      filename: "#{@target_path}#{@download_config.project_config.file_prefix}_statuses.json"
+      filename: File.join(@target_path, "#{file_prefix}_statuses.json")
     )
   end
 
+  def update_status_history_file
+    status_filename = File.join(@target_path, "#{file_prefix}_statuses.json")
+    return unless file_system.file_exist? status_filename
+
+    status_json = file_system.load_json(status_filename)
+
+    history_filename = File.join(@target_path, "#{file_prefix}_status_history.json")
+    history_json = file_system.load_json(history_filename) if file_system.file_exist? history_filename
+
+    if history_json
+      file_system.log '  Updating status history file', also_write_to_stderr: true
+    else
+      file_system.log '  Creating status history file', also_write_to_stderr: true
+      history_json = []
+    end
+
+    status_json.each do |status_item|
+      id = status_item['id']
+      history_item = history_json.find { |s| s['id'] == id }
+      history_json.delete(history_item) if history_item
+      history_json << status_item
+    end
+
+    file_system.save_json(filename: history_filename, json: history_json)
+  end
+
   def download_board_configuration board_id:
     log "  Downloading board configuration for board #{board_id}", both: true
     json = @jira_gateway.call_url relative_url: "/rest/agile/1.0/board/#{board_id}/configuration"
 
-    exit_if_call_failed json
-
-    file_prefix = @download_config.project_config.file_prefix
-    @file_system.save_json json: json, filename: "#{@target_path}#{file_prefix}_board_#{board_id}_configuration.json"
+    @file_system.save_json(
+      json: json,
+      filename: File.join(@target_path, "#{file_prefix}_board_#{board_id}_configuration.json")
+    )
 
     # We have a reported bug that blew up on this line. Moved it after the save so we can
     # actually look at the returned json.
     @board_id_to_filter_id[board_id] = json['filter']['id'].to_i
 
     download_sprints board_id: board_id if json['type'] == 'scrum'
+    # TODO: Should be passing actual statuses, not empty list
+    Board.new raw: json, possible_statuses: StatusCollection.new
   end
 
   def download_sprints board_id:
     log "  Downloading sprints for board #{board_id}", both: true
-    file_prefix = @download_config.project_config.file_prefix
     max_results = 100
     start_at = 0
     is_last = false
@@ -191,20 +200,23 @@ class Downloader
     while is_last == false
       json = @jira_gateway.call_url relative_url: "/rest/agile/1.0/board/#{board_id}/sprint?" \
         "maxResults=#{max_results}&startAt=#{start_at}"
-      exit_if_call_failed json
 
       @file_system.save_json(
         json: json,
-        filename: "#{@target_path}#{file_prefix}_board_#{board_id}_sprints_#{start_at}.json"
+        filename: File.join(@target_path, "#{file_prefix}_board_#{board_id}_sprints_#{start_at}.json")
       )
       is_last = json['isLast']
       max_results = json['maxResults']
-      start_at += json['values'].size
+      if json['values']
+        start_at += json['values'].size
+      else
+        log "  No sprints found for board #{board_id}"
+      end
     end
   end
 
   def metadata_pathname
-    "#{@target_path}#{@download_config.project_config.file_prefix}_meta.json"
+    File.join(@target_path, "#{file_prefix}_meta.json")
   end
 
   def load_metadata
@@ -247,17 +259,17 @@ class Downloader
   end
 
   def remove_old_files
-    file_prefix = @download_config.project_config.file_prefix
     Dir.foreach @target_path do |file|
       next unless file.match?(/^#{file_prefix}_\d+\.json$/)
+      next if file == "#{file_prefix}_status_history.json"
 
-      File.unlink "#{@target_path}#{file}"
+      File.unlink File.join(@target_path, file)
     end
 
     return if @cached_data_format_is_current
 
     # Also throw away all the previously downloaded issues.
-    path = File.join @target_path, "#{file_prefix}_issues"
+    path = File.join(@target_path, "#{file_prefix}_issues")
     return unless File.exist? path
 
     Dir.foreach path do |file|
@@ -271,8 +283,10 @@ class Downloader
     segments = []
     segments << "filter=#{filter_id}"
 
-    unless @download_config.rolling_date_count.nil?
-      @download_date_range = (today.to_date - @download_config.rolling_date_count)..today.to_date
+    start_date = @download_config.start_date today: today
+
+    if start_date
+      @download_date_range = start_date..today.to_date
 
       # For an incremental download, we want to query from the end of the previous one, not from the
       # beginning of the full range.
@@ -293,4 +307,8 @@ class Downloader
 
     segments.join ' AND '
   end
+
+  def file_prefix
+    @download_config.project_config.get_file_prefix
+  end
 end

data/lib/jirametrics/estimate_accuracy_chart.rb

@@ -83,8 +83,7 @@ class EstimateAccuracyChart < ChartBase
 
     issues.each do |issue|
       cycletime = issue.board.cycletime
-      start_time = cycletime.started_time(issue)
-      stop_time = cycletime.stopped_time(issue)
+      start_time, stop_time = cycletime.started_stopped_times(issue)
 
       next unless start_time
 

data/lib/jirametrics/examples/aggregated_project.rb

@@ -3,8 +3,6 @@
 # This file is really intended to give you ideas about how you might configure your own reports, not
 # as a complete setup that will work in every case.
 #
-# See https://github.com/mikebowler/jirametrics/wiki/Examples-folder for more details
-#
 # The point of an AGGREGATED report is that we're now looking at a higher level. We might use this in a
 # S2 meeting (Scrum of Scrums) to talk about the things that are happening across teams, not within a
 # single team. For that reason, we look at slightly different things that we would on a single team board.
@@ -13,6 +11,7 @@ class Exporter
   def aggregated_project name:, project_names:, settings: {}
     project name: name do
       puts name
+      file_prefix name
       self.settings.merge! settings
 
       aggregate do
@@ -21,8 +20,6 @@ class Exporter
         end
       end
 
-      file_prefix name
-
       file do
         file_suffix '.html'
         issues.reject! do |issue|
@@ -33,8 +30,8 @@ class Exporter
           html '<h1>Boards included in this report</h1><ul>', type: :header
           board_lines = []
           included_projects.each do |project|
-            project.all_boards.values.each do |board|
-              board_lines << "<a href='#{project.file_prefix}.html'>#{board.name}</a> from project #{project.name}"
+            project.all_boards.each_value do |board|
+              board_lines << "<a href='#{project.get_file_prefix}.html'>#{board.name}</a> from project #{project.name}"
             end
           end
           board_lines.sort.each { |line| html "<li>#{line}</li>", type: :header }
@@ -64,25 +61,14 @@ class Exporter
 
             # By default, the issue doesn't show what board it's on and this is important for an
             # aggregated view
+            chart = self
             issue_rules do |issue, rules|
-              key = issue.key
-              key = "<S>#{key} </S> " if issue.status.category_name == 'Done'
-              rules.label = "<#{key} [#{issue.type}]<BR/>#{issue.board.name}<BR/>#{word_wrap issue.summary}>"
+              chart.default_issue_rules.call(issue, rules)
+              rules.label = rules.label.split('<BR/>').insert(1, "Board: #{issue.board.name}").join('<BR/>')
             end
 
             link_rules do |link, rules|
-              # By default, the dependency chart shows everything. Clean it up a bit.
-              case link.name
-              when 'Cloners'
-                # We don't want to see any clone links at all.
-                rules.ignore
-              when 'Blocks'
-                # For blocks, by default Jira will have links going both
-                # ways and we want them only going one way. Also make the
-                # link red.
-                rules.merge_bidirectional keep: 'outward'
-                rules.line_color = 'red'
-              end
+              chart.default_link_rules.call(link, rules)
 
               # Because this is the aggregated view, let's hide any link that doesn't cross boards.
               rules.ignore if link.origin.board == link.other_issue.board

data/lib/jirametrics/examples/standard_project.rb

@@ -2,47 +2,49 @@
 
 # This file is really intended to give you ideas about how you might configure your own reports, not
 # as a complete setup that will work in every case.
-#
-# See https://github.com/mikebowler/jirametrics/wiki/Examples-folder for more
 class Exporter
   def standard_project name:, file_prefix:, ignore_issues: nil, starting_status: nil, boards: {},
-      default_board: nil, anonymize: false, settings: {}, status_category_mappings: {}
+      default_board: nil, anonymize: false, settings: {}, status_category_mappings: {},
+      rolling_date_count: 90, no_earlier_than: nil, ignore_types: %w[Sub-task Subtask Epic],
+      show_experimental_charts: false
 
     project name: name do
       puts name
-      self.anonymize if anonymize
+      file_prefix file_prefix
 
-      settings['blocked_link_text'] = ['is blocked by']
+      self.anonymize if anonymize
       self.settings.merge! settings
 
       status_category_mappings.each do |status, category|
         status_category_mapping status: status, category: category
       end
 
-      file_prefix file_prefix
       download do
-        rolling_date_count 90
+        self.rolling_date_count(rolling_date_count) if rolling_date_count
+        self.no_earlier_than(no_earlier_than) if no_earlier_than
       end
 
       boards.each_key do |board_id|
         block = boards[board_id]
         if block == :default
           block = lambda do |_|
-            start_at first_time_in_status_category('In Progress')
-            stop_at still_in_status_category('Done')
+            start_at first_time_in_status_category(:indeterminate)
+            stop_at still_in_status_category(:done)
           end
         end
         board id: board_id do
           cycletime(&block)
-          expedited_priority_names 'Critical', 'Highest', 'Immediate Gating'
         end
       end
 
+      issues.reject! do |issue|
+        ignore_types.include? issue.type
+      end
+
+      discard_changes_before status_becomes: (starting_status || :backlog) # rubocop:disable Style/RedundantParentheses
+
       file do
         file_suffix '.html'
-        issues.reject! do |issue|
-          %w[Sub-task Epic].include? issue.type
-        end
 
         issues.reject! { |issue| ignore_issues.include? issue.key } if ignore_issues
 
@@ -52,12 +54,10 @@ class Exporter
           html "<H1>#{name}</H1>", type: :header
           boards.each_key do |id|
             board = find_board id
-            html "<div><a href='#{board.url}'>#{id} #{board.name}</a></div>",
+            html "<div><a href='#{board.url}'>#{id} #{board.name}</a> (#{board.board_type})</div>",
                  type: :header
           end
 
-          discard_changes_before status_becomes: (starting_status || :backlog) # rubocop:disable Style/RedundantParentheses
-
           cycletime_scatterplot do
             show_trend_lines
           end
@@ -84,27 +84,11 @@ class Exporter
           daily_wip_by_age_chart
           daily_wip_by_blocked_stalled_chart
           daily_wip_by_parent_chart
+          flow_efficiency_scatterplot if show_experimental_charts
           expedited_chart
           sprint_burndown
           estimate_accuracy_chart
-
-          dependency_chart do
-            link_rules do |link, rules|
-              case link.name
-              when 'Cloners'
-                rules.ignore
-              when 'Dependency', 'Blocks', 'Parent/Child', 'Cause', 'Satisfy Requirement', 'Relates'
-                rules.merge_bidirectional keep: 'outward'
-                rules.merge_bidirectional keep: 'outward'
-              when 'Sync'
-                rules.use_bidirectional_arrows
-              else
-                # This is a link type that we don't recognize. Dump it to standard out to draw attention
-                # to it.
-                puts "name=#{link.name}, label=#{link.label}"
-              end
-            end
-          end
+          dependency_chart
         end
       end
     end

data/lib/jirametrics/expedited_chart.rb

@@ -57,13 +57,13 @@ class ExpeditedChart < ChartBase
   def prepare_expedite_data issue
     expedite_start = nil
     result = []
-    expedited_priority_names = issue.board.expedited_priority_names
+    expedited_priority_names = issue.board.project_config.settings['expedited_priority_names']
 
     issue.changes.each do |change|
       next unless change.priority?
 
       if expedited_priority_names.include? change.value
-        expedite_start = change.time
+        expedite_start = change.time.to_date
       elsif expedite_start
         start_date = expedite_start.to_date
         stop_date = change.time.to_date
@@ -72,7 +72,7 @@ class ExpeditedChart < ChartBase
            (start_date < date_range.begin && stop_date > date_range.end)
 
           result << [expedite_start, :expedite_start]
-          result << [change.time, :expedite_stop]
+          result << [change.time.to_date, :expedite_stop]
         end
         expedite_start = nil
       end
@@ -109,12 +109,11 @@ class ExpeditedChart < ChartBase
 
   def make_expedite_lines_data_set issue:, expedite_data:
     cycletime = issue.board.cycletime
-    started_time = cycletime.started_time(issue)
-    stopped_time = cycletime.stopped_time(issue)
+    started_date, stopped_date = cycletime.started_stopped_dates(issue)
 
-    expedite_data << [started_time, :issue_started] if started_time
-    expedite_data << [stopped_time, :issue_stopped] if stopped_time
-    expedite_data.sort_by! { |a| a[0] }
+    expedite_data << [started_date, :issue_started] if started_date
+    expedite_data << [stopped_date, :issue_stopped] if stopped_date
+    expedite_data.sort_by!(&:first)
 
     # If none of the data would be visible on the chart then skip it.
     return nil unless expedite_data.any? { |time, _action| time.to_date >= date_range.begin }
@@ -151,7 +150,7 @@ class ExpeditedChart < ChartBase
 
     unless expedite_data.empty?
       last_change_time = expedite_data[-1][0].to_date
-      if last_change_time && last_change_time <= date_range.end && stopped_time.nil?
+      if last_change_time && last_change_time <= date_range.end && stopped_date.nil?
         data << make_point(issue: issue, time: date_range.end, label: 'Still ongoing', expedited: expedited)
         dot_colors << '' # It won't be visible so it doesn't matter
         point_styles << 'dash'

data/lib/jirametrics/exporter.rb

@@ -2,16 +2,6 @@
 
 require 'fileutils'
 
-class Object
-  def deprecated message:, date:
-    text = +''
-    text << "Deprecated(#{date}): "
-    text << message
-    text << "\n-> Called from #{caller(1..1).first}"
-    warn text
-  end
-end
-
 class Exporter
   attr_reader :project_configs
   attr_accessor :file_system
@@ -70,6 +60,31 @@ class Exporter
     puts "Full output from downloader in #{file_system.logfile_name}"
   end
 
+  def info keys, name_filter:
+    selected = []
+    each_project_config(name_filter: name_filter) do |project|
+      project.evaluate_next_level
+
+      project.run load_only: true
+      project.issues.each do |issue|
+        selected << [project, issue] if keys.include? issue.key
+      end
+    rescue => e # rubocop:disable Style/RescueStandardError
+      # This happens when we're attempting to load an aggregated project because it hasn't been
+      # properly initialized. Since we don't care about aggregated projects, we just ignore it.
+      raise unless e.message.start_with? 'This is an aggregated project and issues should have been included'
+    end
+
+    if selected.empty?
+      file_system.log "No issues found to match #{keys.collect(&:inspect).join(', ')}"
+    else
+      selected.each do |project, issue|
+        file_system.log "\nProject #{project.name}", also_write_to_stderr: true
+        file_system.log issue.dump, also_write_to_stderr: true
+      end
+    end
+  end
+
   def each_project_config name_filter:
     @project_configs.each do |project|
       yield project if project.name.nil? || File.fnmatch(name_filter, project.name)
@@ -101,7 +116,9 @@ class Exporter
 
   def jira_config filename = nil
     if filename
-      @jira_config = file_system.load_json(filename)
+      @jira_config = file_system.load_json(filename, fail_on_error: false)
+      raise "Unable to load Jira configuration file and cannot continue: #{filename.inspect}" if @jira_config.nil?
+
       @jira_config['url'] = $1 if @jira_config['url'] =~ /^(.+)\/+$/
     end
     @jira_config

data/lib/jirametrics/file_config.rb

@@ -56,7 +56,7 @@ class FileConfig
   def output_filename
     segments = []
     segments << project_config.target_path
-    segments << project_config.file_prefix
+    segments << project_config.get_file_prefix
     segments << (@file_suffix || "-#{@today}.csv")
     segments.join
   end
@@ -66,15 +66,20 @@ class FileConfig
   # is that all empty values in the first column should be at the bottom.
   def sort_output all_lines
     all_lines.sort do |a, b|
+      result = nil
       if a[0] == b[0]
-        a[1..] <=> b[1..]
+        result = a[1..] <=> b[1..]
       elsif a[0].nil?
-        1
+        result = 1
       elsif b[0].nil?
-        -1
+        result = -1
       else
-        a[0] <=> b[0]
+        result = a[0] <=> b[0]
       end
+
+      # This will only happen if one of the objects isn't comparable. Seen in production.
+      result = -1 if result.nil?
+      result
     end
   end
 
@@ -85,6 +90,11 @@ class FileConfig
 
   def html_report &block
     assert_only_one_filetype_config_set
+    if block.nil?
+      project_config.file_system.warning 'No charts were specified for the report. This is almost certainly a mistake.'
+      block = ->(_) {}
+    end
+
     @html_report = HtmlReportConfig.new file_config: self, block: block
   end
 
@@ -103,7 +113,7 @@ class FileConfig
   def to_datetime object
     return nil if object.nil?
 
-    object = object.to_datetime
+    object = object.to_time.to_datetime
     object = object.new_offset(@timezone_offset) if @timezone_offset
     object
   end
@@ -120,4 +130,11 @@ class FileConfig
     @file_suffix = suffix unless suffix.nil?
     @file_suffix
   end
+
+  def children
+    result = []
+    result << @columns if @columns
+    result << @html_report if @html_report
+    result
+  end
 end

data/lib/jirametrics/file_system.rb

@@ -6,14 +6,18 @@ class FileSystem
   attr_accessor :logfile, :logfile_name
 
   # Effectively the same as File.read except it forces the encoding to UTF-8
-  def load filename
+  def load filename, supress_deprecation: false
+    if filename.end_with?('.json') && !supress_deprecation
+      deprecated(message: 'call load_json instead', date: '2024-11-13')
+    end
+
     File.read filename, encoding: 'UTF-8'
   end
 
   def load_json filename, fail_on_error: true
     return nil if fail_on_error == false && File.exist?(filename) == false
 
-    JSON.parse load(filename)
+    JSON.parse load(filename, supress_deprecation: true)
   end
 
   def save_json json:, filename:
@@ -27,8 +31,22 @@ class FileSystem
     File.write(filename, content)
   end
 
-  def log message
+  def warning message, more: nil
+    log "Warning: #{message}", more: more, also_write_to_stderr: true
+  end
+
+  def error message, more: nil
+    log "Error: #{message}", more: more, also_write_to_stderr: true
+  end
+
+  def log message, more: nil, also_write_to_stderr: false
+    message += " See #{logfile_name} for more details about this message." if more
+
     logfile.puts message
+    logfile.puts more if more
+    return unless also_write_to_stderr
+
+    $stderr.puts message # rubocop:disable Style/StderrPuts
   end
 
   # In some Jira instances, a sizeable portion of the JSON is made up of empty fields. I've seen
@@ -42,4 +60,22 @@ class FileSystem
     end
     node
   end
+
+  def foreach root, &block
+    Dir.foreach root, &block
+  end
+
+  def file_exist? filename
+    File.exist? filename
+  end
+
+  def deprecated message:, date:, depth: 2
+    text = +''
+    text << "Deprecated(#{date}): "
+    text << message
+    caller(1..depth).each do |line|
+      text << "\n-> Called from #{line}"
+    end
+    log text, also_write_to_stderr: true
+  end
 end