jirametrics 2.14 → 2.30

data/lib/jirametrics/downloader_for_data_center.rb

@@ -0,0 +1,95 @@
+# frozen_string_literal: true
+
+class DownloaderForDataCenter < Downloader
+  def jira_instance_type
+    'Jira DataCenter'
+  end
+
+  def download_issues board:
+    log "  Downloading primary issues for board #{board.id}", both: true
+    path = File.join(@target_path, "#{file_prefix}_issues/")
+    unless Dir.exist?(path)
+      log "  Creating path #{path}"
+      Dir.mkdir(path)
+    end
+
+    filter_id = board_id_to_filter_id[board.id]
+    jql = make_jql(filter_id: filter_id)
+    jira_search_by_jql(jql: jql, initial_query: true, board: board, path: path)
+
+    log "  Downloading linked issues for board #{board.id}", both: true
+    loop do
+      @issue_keys_pending_download.reject! { |key| @issue_keys_downloaded_in_current_run.include? key }
+      break if @issue_keys_pending_download.empty?
+
+      keys_to_request = @issue_keys_pending_download[0..99]
+      @issue_keys_pending_download.reject! { |key| keys_to_request.include? key }
+      jql = "key in (#{keys_to_request.join(', ')})"
+      jira_search_by_jql(jql: jql, initial_query: true, board: board, path: path)
+    end
+  end
+
+  def jira_search_by_jql jql:, initial_query:, board:, path:
+    intercept_jql = @download_config.project_config.settings['intercept_jql']
+    jql = intercept_jql.call jql if intercept_jql
+
+    log "  JQL: #{jql}"
+    escaped_jql = CGI.escape jql
+
+    max_results = 100
+    start_at = 0
+    total = 1
+    while start_at < total
+      json = @jira_gateway.call_url relative_url: '/rest/api/2/search' \
+        "?jql=#{escaped_jql}&maxResults=#{max_results}&startAt=#{start_at}&expand=changelog&fields=*all"
+
+      json['issues'].each do |issue_json|
+        issue_json['exporter'] = {
+          'in_initial_query' => initial_query
+        }
+        identify_other_issues_to_be_downloaded raw_issue: issue_json, board: board
+        file = "#{issue_json['key']}-#{board.id}.json"
+
+        @file_system.save_json(json: issue_json, filename: File.join(path, file))
+      end
+
+      total = json['total'].to_i
+      max_results = json['maxResults']
+
+      message = "    Downloaded #{start_at + 1}-#{[start_at + max_results, total].min} of #{total} issues to #{path} "
+      log message, both: true
+
+      start_at += json['issues'].size
+    end
+  end
+
+  def make_jql filter_id:, today: nil
+    today ||= today_in_project_timezone
+    segments = []
+    segments << "filter=#{filter_id}"
+
+    start_date = @download_config.start_date today: today
+
+    if start_date
+      @download_date_range = start_date..today.to_date
+
+      # For an incremental download, we want to query from the end of the previous one, not from the
+      # beginning of the full range.
+      @start_date_in_query = metadata['date_end'] || @download_date_range.begin
+      log "    Incremental download only. Pulling from #{@start_date_in_query}", both: true if metadata['date_end']
+
+      # Catch-all to pick up anything that's been around since before the range started but hasn't
+      # had an update during the range.
+      catch_all = '((status changed OR Sprint is not EMPTY) AND statusCategory != Done)'
+
+      # Pick up any issues that had a status change in the range
+      start_date_text = @start_date_in_query.strftime '%Y-%m-%d'
+      # find_in_range = %((status changed DURING ("#{start_date_text} 00:00","#{end_date_text} 23:59")))
+      find_in_range = %(updated >= "#{start_date_text} 00:00")
+
+      segments << "(#{find_in_range} OR #{catch_all})"
+    end
+
+    segments.join ' AND '
+  end
+end

data/lib/jirametrics/estimate_accuracy_chart.rb

@@ -5,7 +5,7 @@ class EstimateAccuracyChart < ChartBase
     super()
 
     header_text 'Estimate Accuracy'
-    description_text <<-HTML
+    description_text <<~HTML
       <div class="p">
         This chart graphs estimates against actual recorded cycle times. Since
         estimates can change over time, we're graphing the estimate at the time that the story started.
@@ -20,8 +20,18 @@ class EstimateAccuracyChart < ChartBase
           far to the right then you know you have a problem.
         <% end %>
       </div>
+      <% if @correlation_coefficient %>
+        <div class="p">
+          The completed items here have a correlation coefficient of <b><%= @correlation_coefficient.round(3) %></b>.
+          The closer it is to +1, the stronger the positive correlation. The closer it is to -1,
+          the stronger the negative collalation. Zero would mean no correlation at all.
+        </div>
+      <% end %>
     HTML
 
+    @x_axis_title = 'Cycletime (days)'
+    @y_axis_title = 'Estimate'
+
     @y_axis_type = 'linear'
     @y_axis_block = ->(issue, start_time) { estimate_at(issue: issue, start_time: start_time)&.to_f }
     @y_axis_sort_order = nil
@@ -30,9 +40,9 @@ class EstimateAccuracyChart < ChartBase
   end
 
   def run
-    if @y_axis_label.nil?
+    if @y_axis_title.nil?
       text = current_board.estimation_configuration.units == :story_points ? 'Story Points' : 'Days'
-      @y_axis_label = "Estimated #{text}"
+      @y_axis_title = "Estimated #{text}"
     end
     data_sets = scan_issues
 
@@ -43,7 +53,7 @@ class EstimateAccuracyChart < ChartBase
 
   def scan_issues
     completed_hash, aging_hash = split_into_completed_and_aging issues: issues
-
+    @correlation_coefficient = correlation_coefficient(completed_hash) unless completed_hash.empty?
     estimation_units = current_board.estimation_configuration.units
     @has_aging_data = !aging_hash.empty?
 
@@ -170,4 +180,32 @@ class EstimateAccuracyChart < ChartBase
     end
     @y_axis_block = block
   end
+
+  # Correlation coefficient is calculated using the Pearson Correlation Coefficient
+  # r = Σ((xi - x̄)(yi - ȳ)) / sqrt(Σ(xi - x̄)² · Σ(yi - ȳ)²)
+  def correlation_coefficient completed_hash
+    list1 = []
+    list2 = []
+    completed_hash.each do |(estimate, cycle_time), issues|
+      issues.size.times do
+        list1 << estimate
+        list2 << cycle_time
+      end
+    end
+
+    n = list1.size
+    return nil if n < 2
+
+    mean1 = list1.sum.to_f / n
+    mean2 = list2.sum.to_f / n
+
+    numerator = list1.zip(list2).sum { |x, y| (x - mean1) * (y - mean2) }
+    sum_sq1 = list1.sum { |x| (x - mean1)**2 }
+    sum_sq2 = list2.sum { |y| (y - mean2)**2 }
+
+    denominator = Math.sqrt(sum_sq1 * sum_sq2)
+    return nil if denominator.zero?
+
+    numerator / denominator
+  end
 end

data/lib/jirametrics/examples/aggregated_project.rb

@@ -10,9 +10,9 @@
 class Exporter
   def aggregated_project name:, project_names:, settings: {}
     project name: name do
-      puts name
+      file_system.log name
       file_prefix name
-      self.settings.merge! settings
+      self.settings.merge! stringify_keys(settings)
 
       aggregate do
         project_names.each do |project_name|

data/lib/jirametrics/examples/standard_project.rb

@@ -6,23 +6,14 @@ class Exporter
   def standard_project name:, file_prefix:, ignore_issues: nil, starting_status: nil, boards: {},
       default_board: nil, anonymize: false, settings: {}, status_category_mappings: {},
       rolling_date_count: 90, no_earlier_than: nil, ignore_types: %w[Sub-task Subtask Epic],
-      show_experimental_charts: false
-
+      show_experimental_charts: false, github_repos: nil
+    exporter = self
     project name: name do
-      puts name
+      file_system.log name, also_write_to_stderr: true
       file_prefix file_prefix
 
       self.anonymize if anonymize
-      self.settings.merge! settings
-
-      status_category_mappings.each do |status, category|
-        status_category_mapping status: status, category: category
-      end
-
-      download do
-        self.rolling_date_count(rolling_date_count) if rolling_date_count
-        self.no_earlier_than(no_earlier_than) if no_earlier_than
-      end
+      self.settings.merge! stringify_keys(settings)
 
       boards.each_key do |board_id|
         block = boards[board_id]
@@ -37,17 +28,27 @@ class Exporter
         end
       end
 
+      status_category_mappings.each do |status, category|
+        status_category_mapping status: status, category: category
+      end
+
+      download do
+        self.rolling_date_count(rolling_date_count) if rolling_date_count
+        self.no_earlier_than(no_earlier_than) if no_earlier_than
+        github_repo *github_repos if github_repos
+      end
+
       issues.reject! do |issue|
         ignore_types.include? issue.type
       end
 
+      exporter.filter_issues issues, ignore_issues
+
       discard_changes_before status_becomes: (starting_status || :backlog) # rubocop:disable Style/RedundantParentheses
 
       file do
         file_suffix '.html'
 
-        issues.reject! { |issue| ignore_issues.include? issue.key } if ignore_issues
-
         html_report do
           board_id default_board if default_board
 
@@ -57,37 +58,39 @@ class Exporter
             html "<div><a href='#{board.url}'>#{id} #{board.name}</a> (#{board.board_type})</div>",
                  type: :header
           end
-
           daily_view
-
+          cumulative_flow_diagram
           cycletime_scatterplot do
             show_trend_lines
           end
           cycletime_histogram
 
           throughput_chart do
-            description_text '<h2>Number of items completed, grouped by issue type</h2>'
+            description_text <<~TEXT
+              <div>Throughput data is very useful for#{' '}
+                <a href="https://blog.mikebowler.ca/2024/06/02/probabilistic-forecasting/">probabilistic forecasting</a>,
+                to determine when we'll be done. Try it now with the
+                <a href="<%= throughput_forecaster_url %>" target="_blank" rel="noopener noreferrer">
+                Focused Objective throughput forecaster,</a> to see how long it would take to complete all of the
+                <%= @not_started_count %> items you currently have in your backlog.
+              </div>
+              <h2>Number of items completed, grouped by issue type</h2>'
+            TEXT
           end
-          throughput_chart do
-            header_text nil
+          throughput_by_completed_resolution_chart do
             description_text '<h2>Number of items completed, grouped by completion status and resolution</h2>'
-            grouping_rules do |issue, rules|
-              if issue.resolution
-                rules.label = "#{issue.status.name}:#{issue.resolution}"
-              else
-                rules.label = issue.status.name
-              end
-            end
           end
 
           aging_work_in_progress_chart
+          wip_by_column_chart do
+            show_recommendations
+          end
           aging_work_bar_chart
           aging_work_table
           daily_wip_by_age_chart
           daily_wip_by_blocked_stalled_chart
           daily_wip_by_parent_chart
           flow_efficiency_scatterplot if show_experimental_charts
-          expedited_chart
           sprint_burndown
           estimate_accuracy_chart
           dependency_chart
@@ -95,4 +98,14 @@ class Exporter
       end
     end
   end
+
+  # Extracted as a separate method so it can be tested independently, without needing to invoke
+  # the full standard_project DSL setup.
+  def filter_issues issues, ignore_issues
+    return unless ignore_issues
+
+    issues.reject! do |issue|
+      ignore_issues.is_a?(Proc) ? ignore_issues.call(issue) : ignore_issues.include?(issue.key)
+    end
+  end
 end

data/lib/jirametrics/expedited_chart.rb

@@ -38,6 +38,8 @@ class ExpeditedChart < ChartBase
       </div>
       #{describe_non_working_days}
     HTML
+    @x_axis_title = 'Date'
+    @y_axis_title = 'Age in days'
 
     instance_eval(&block)
   end
@@ -48,7 +50,7 @@ class ExpeditedChart < ChartBase
     end
 
     if data_sets.empty?
-      '<h1>Expedited work</h1>There is no expedited work in this time period.'
+      '<h1 class="foldable">Expedited work</h1><div>There is no expedited work in this time period.</div>'
     else
       wrap_and_render(binding, __FILE__)
     end

data/lib/jirametrics/exporter.rb

@@ -8,7 +8,13 @@ class Exporter
 
   def self.configure &block
     logfile_name = 'jirametrics.log'
-    logfile = File.open logfile_name, 'w'
+    logfile = File.open(logfile_name, 'w')
+  rescue Errno::EACCES
+    # FileSystem can't be used here — it hasn't been created yet (it depends on this logfile).
+    warn "Error: Cannot write to #{File.expand_path(logfile_name)}. " \
+      'Please ensure the current directory is writable.'
+    exit 1
+  else
     file_system = FileSystem.new
     file_system.logfile = logfile
     file_system.logfile_name = logfile_name
@@ -40,6 +46,7 @@ class Exporter
 
   def download name_filter:
     @downloading = true
+    github_pr_cache = {}
     each_project_config(name_filter: name_filter) do |project|
       project.evaluate_next_level
       next if project.aggregated_project?
@@ -50,33 +57,42 @@ class Exporter
       end
 
       project.download_config.run
-      downloader = Downloader.new(
+      gateway = JiraGateway.new(
+        file_system: file_system, jira_config: project.jira_config, settings: project.settings
+      )
+      downloader = Downloader.create(
         download_config: project.download_config,
         file_system: file_system,
-        jira_gateway: JiraGateway.new(file_system: file_system)
+        jira_gateway: gateway,
+        github_pr_cache: github_pr_cache
       )
       downloader.run
     end
     puts "Full output from downloader in #{file_system.logfile_name}"
   end
 
-  def info keys, name_filter:
+  def info key, name_filter:
     selected = []
+    file_system.log_only = true
     each_project_config(name_filter: name_filter) do |project|
       project.evaluate_next_level
 
       project.run load_only: true
       project.issues.each do |issue|
-        selected << [project, issue] if keys.include? issue.key
+        selected << [project, issue] if key == issue.key
+        issue.subtasks.each do |subtask|
+          selected << [project, subtask] if key == subtask.key
+        end
       end
     rescue => e # rubocop:disable Style/RescueStandardError
       # This happens when we're attempting to load an aggregated project because it hasn't been
       # properly initialized. Since we don't care about aggregated projects, we just ignore it.
       raise unless e.message.start_with? 'This is an aggregated project and issues should have been included'
     end
+    file_system.log_only = false
 
     if selected.empty?
-      file_system.log "No issues found to match #{keys.collect(&:inspect).join(', ')}"
+      file_system.log "No issues found to match #{key.inspect}"
     else
       selected.each do |project, issue|
         file_system.log "\nProject #{project.name}", also_write_to_stderr: true
@@ -85,6 +101,10 @@ class Exporter
     end
   end
 
+  def stitch stitch_file
+    Stitcher.new(file_system: file_system).run(stitch_file: stitch_file)
+  end
+
   def each_project_config name_filter:
     @project_configs.each do |project|
       yield project if project.name.nil? || File.fnmatch(name_filter, project.name)

data/lib/jirametrics/file_config.rb

@@ -65,22 +65,20 @@ class FileConfig
   # most common usecase - the Team Dashboard from FocusedObjective.com. The rule for that one
   # is that all empty values in the first column should be at the bottom.
   def sort_output all_lines
-    all_lines.sort do |a, b|
-      result = nil
-      if a[0] == b[0]
-        result = a[1..] <=> b[1..]
+    all_lines.each_with_index.sort do |(a, a_idx), (b, b_idx)|
+      result = if a[0] == b[0]
+        a[1..] <=> b[1..]
       elsif a[0].nil?
-        result = 1
+        1
       elsif b[0].nil?
-        result = -1
+        -1
       else
-        result = a[0] <=> b[0]
+        a[0] <=> b[0]
       end
 
-      # This will only happen if one of the objects isn't comparable. Seen in production.
-      result = -1 if result.nil?
-      result
-    end
+      # When objects aren't comparable, preserve original order for a stable sort.
+      result.nil? || result.zero? ? a_idx <=> b_idx : result
+    end.map(&:first)
   end
 
   def columns &block

data/lib/jirametrics/file_system.rb

@@ -3,7 +3,15 @@
 require 'json'
 
 class FileSystem
-  attr_accessor :logfile, :logfile_name
+  attr_accessor :logfile, :logfile_name, :log_only
+
+  def initialize
+    # In almost all cases, this will be immediately replaced in the Exporter
+    # but if we fail before we get that far, this will at least let a useful
+    # error show up on the console.
+    @logfile = $stdout
+    @log_only = false
+  end
 
   # Effectively the same as File.read except it forces the encoding to UTF-8
   def load filename, supress_deprecation: false
@@ -31,6 +39,14 @@ class FileSystem
     File.write(filename, content)
   end
 
+  def mkdir path
+    FileUtils.mkdir_p path
+  end
+
+  def utime file:, time:
+    File.utime time, time, file
+  end
+
   def warning message, more: nil
     log "Warning: #{message}", more: more, also_write_to_stderr: true
   end
@@ -44,11 +60,43 @@ class FileSystem
 
     logfile.puts message
     logfile.puts more if more
-    return unless also_write_to_stderr
+    return if log_only || !also_write_to_stderr
+
+    # Obscure edge-case where we're trying to log something before logging is even
+    # set up. Quick escape here so that we don't dump the error twice.
+    return if logfile == $stdout
 
     $stderr.puts message # rubocop:disable Style/StderrPuts
   end
 
+  def log_start message
+    logfile.puts message
+    return if log_only || logfile == $stdout
+
+    $stderr.print message
+    $stderr.flush
+  end
+
+  def start_progress
+    return if log_only
+
+    $stderr.print '  '
+    $stderr.flush
+  end
+
+  def progress_dot
+    return if log_only
+
+    $stderr.print '.'
+    $stderr.flush
+  end
+
+  def end_progress
+    return if log_only
+
+    $stderr.puts '' # rubocop:disable Style/StderrPuts
+  end
+
   # In some Jira instances, a sizeable portion of the JSON is made up of empty fields. I've seen
   # cases where this simple compression will drop the filesize by half.
   def compress node
@@ -66,7 +114,15 @@ class FileSystem
   end
 
   def file_exist? filename
-    File.exist? filename
+    File.exist?(filename) && File.file?(filename)
+  end
+
+  def dir_exist? path
+    File.exist?(path) && File.directory?(path)
+  end
+
+  def unlink filename
+    File.unlink filename
   end
 
   def deprecated message:, date:, depth: 2

data/lib/jirametrics/fix_version.rb

@@ -11,11 +11,24 @@ class FixVersion
     @raw['name']
   end
 
+  def description
+    @raw['description']
+  end
+
   def id
     @raw['id'].to_i
   end
 
+  def release_date
+    text = @raw['releaseDate']
+    text.nil? ? nil : Date.parse(text)
+  end
+
   def released?
     @raw['released']
   end
+
+  def archived?
+    @raw['archived']
+  end
 end

data/lib/jirametrics/flow_efficiency_scatterplot.rb

@@ -32,6 +32,8 @@ class FlowEfficiencyScatterplot < ChartBase
         So be aware that your team may have to change their behaviours if you want this chart to be useful.
       </div>
     HTML
+    @x_axis_title = 'Total time (days)'
+    @y_axis_title = 'Time adding value (days)'
 
     init_configuration_block block do
       grouping_rules do |issue, rule|
@@ -60,7 +62,9 @@ class FlowEfficiencyScatterplot < ChartBase
       create_dataset(issues: issues, label: rules.label, color: rules.color)
     end
 
-    return "<h1>#{@header_text}</h1>No data matched the selected criteria. Nothing to show." if data_sets.empty?
+    if data_sets.empty?
+      return "<h1 class='foldable'>#{@header_text}</h1><div>No data matched the selected criteria. Nothing to show.</div>"
+    end
 
     wrap_and_render(binding, __FILE__)
   end