jirametrics 2.4 → 2.30

data/lib/jirametrics/downloader_for_cloud.rb

@@ -0,0 +1,287 @@
+# frozen_string_literal: true
+
+class DownloaderForCloud < Downloader
+  def jira_instance_type
+    'Jira Cloud'
+  end
+
+  def run
+    super
+    download_fix_versions
+  end
+
+  def download_board_configuration board_id:
+    board = super
+    location = board.raw['location']
+    @project_key ||= location['key'] if location&.[]('type') == 'project'
+    board
+  end
+
+  def download_fix_versions
+    return unless @project_key
+
+    log "  Downloading fix versions for project #{@project_key}", both: true
+    max_results = 50
+    start_at = 0
+    all_versions = []
+
+    loop do
+      json = @jira_gateway.call_url(
+        relative_url: "/rest/api/3/project/#{@project_key}/version?" \
+          "startAt=#{start_at}&maxResults=#{max_results}"
+      )
+
+      values = json['values'] || []
+      all_versions.concat(values)
+      break if json['isLast'] || values.empty?
+
+      start_at += values.size
+    end
+
+    @file_system.save_json(
+      json: all_versions,
+      filename: File.join(@target_path, "#{file_prefix}_fix_versions.json")
+    )
+  end
+
+  def search_for_issues jql:, board_id:, path:
+    log "  JQL: #{jql}"
+    escaped_jql = CGI.escape jql
+
+    hash = {}
+    max_results = 5_000 # The maximum allowed by Jira
+    next_page_token = nil
+    issue_count = 0
+
+    start_progress
+    loop do
+      relative_url = +''
+      relative_url << '/rest/api/3/search/jql'
+      relative_url << "?jql=#{escaped_jql}&maxResults=#{max_results}"
+      relative_url << "&nextPageToken=#{next_page_token}" if next_page_token
+      relative_url << '&fields=updated'
+
+      json = @jira_gateway.call_url relative_url: relative_url
+      next_page_token = json['nextPageToken']
+
+      json['issues'].each do |i|
+        key = i['key']
+        data = DownloadIssueData.new key: key
+        data.key = key
+        data.last_modified = Time.parse i['fields']['updated']
+        data.found_in_primary_query = true
+        data.cache_path = File.join(path, "#{key}-#{board_id}.json")
+        data.up_to_date = last_modified(filename: data.cache_path) == data.last_modified
+        hash[key] = data
+        issue_count += 1
+      end
+
+      progress_dot "    Found #{issue_count} issues"
+
+      break unless next_page_token
+    end
+    end_progress
+
+    hash
+  end
+
+  def bulk_fetch_issues issue_datas:, board:, in_initial_query:
+    # We used to use the expand option to pull in the changelog directly. Unfortunately
+    # that only returns the "recent" changes, not all of them. So now we get the issue
+    # without changes and then make a second call for that changes. Then we insert it
+    # into the raw issue as if it had been there all along.
+    log "  Downloading #{issue_datas.size} issues"
+    payload = {
+      'fields' => ['*all'],
+      'issueIdsOrKeys' => issue_datas.collect(&:key)
+    }
+    response = @jira_gateway.post_request(
+      relative_url: '/rest/api/3/issue/bulkfetch',
+      payload: JSON.generate(payload)
+    )
+
+    attach_changelog_to_issues issue_datas: issue_datas, issue_jsons: response['issues']
+
+    response['issues'].each do |issue_json|
+      issue_json['exporter'] = {
+        'in_initial_query' => in_initial_query
+      }
+      issue = Issue.new(raw: issue_json, board: board)
+      data = issue_datas.find { |d| d.key == issue.key }
+      unless data
+        log "  Skipping #{issue.key}: returned by Jira but key not in request (issue may have been moved)"
+        next
+      end
+      data.up_to_date = true
+      data.last_modified = issue.updated
+      data.issue = issue
+    end
+
+    # Mark any unmatched requests as up_to_date to prevent infinite re-fetching.
+    # This happens when Jira returns a different key (moved issue) leaving the original unmatched.
+    issue_datas.each do |data|
+      next if data.up_to_date
+
+      log "  Skipping #{data.key}: not returned by Jira (issue may have been deleted or moved)"
+      data.up_to_date = true
+    end
+
+    issue_datas
+  end
+
+  def attach_changelog_to_issues issue_datas:, issue_jsons:
+    max_results = 10_000 # The max jira accepts is 10K
+    payload = {
+      'issueIdsOrKeys' => issue_datas.collect(&:key),
+      'maxResults' => max_results
+    }
+    loop do
+      response = @jira_gateway.post_request(
+        relative_url: '/rest/api/3/changelog/bulkfetch',
+        payload: JSON.generate(payload)
+      )
+
+      response['issueChangeLogs'].each do |issue_change_log|
+        issue_id = issue_change_log['issueId']
+        json = issue_jsons.find { |json| json['id'] == issue_id }
+
+        unless json['changelog']
+          # If this is our first time in, there won't be a changelog section
+          json['changelog'] = {
+            'startAt' => 0,
+            'maxResults' => max_results,
+            'total' => 0,
+            'histories' => []
+          }
+        end
+
+        new_changes = issue_change_log['changeHistories']
+        json['changelog']['total'] += new_changes.size
+        json['changelog']['histories'] += new_changes
+      end
+
+      next_page_token = response['nextPageToken']
+      payload['nextPageToken'] = next_page_token
+      break if next_page_token.nil?
+    end
+  end
+
+  def download_issues board:
+    log "  Downloading primary issues for board #{board.id} from #{jira_instance_type}", both: true
+    path = File.join(@target_path, "#{file_prefix}_issues/")
+    unless @file_system.dir_exist?(path)
+      log "  Creating path #{path}"
+      @file_system.mkdir(path)
+    end
+
+    filter_id = @board_id_to_filter_id[board.id]
+    jql = make_jql(filter_id: filter_id)
+    intercept_jql = @download_config.project_config.settings['intercept_jql']
+    jql = intercept_jql.call jql if intercept_jql
+
+    issue_data_hash = search_for_issues jql: jql, board_id: board.id, path: path
+
+    checked_for_related = Set.new
+    in_related_phase = false
+
+    loop do
+      related_issue_keys = Set.new
+      stale = issue_data_hash.values.reject { |data| data.up_to_date }
+      unless stale.empty?
+        log_start '  Downloading more issues ' unless in_related_phase
+        stale.each_slice(100) do |slice|
+          slice = bulk_fetch_issues(issue_datas: slice, board: board, in_initial_query: true)
+          progress_dot
+          slice.each do |data|
+            next unless data.issue
+
+            @file_system.save_json(
+              json: data.issue.raw, filename: data.cache_path
+            )
+            # Set the timestamp on the file to match the updated one so that we don't have
+            # to parse the file just to find the timestamp
+            @file_system.utime time: data.issue.updated, file: data.cache_path
+
+            collect_related_issue_keys issue: data.issue, related_issue_keys: related_issue_keys
+            checked_for_related << data.key
+          end
+        end
+        end_progress unless in_related_phase
+      end
+
+      # Also scan up-to-date cached issues we haven't checked yet — they may reference
+      # related issues that are not in the primary query result.
+      issue_data_hash.each_value do |data|
+        next if checked_for_related.include?(data.key)
+        next unless @file_system.file_exist?(data.cache_path)
+
+        checked_for_related << data.key
+        raw = @file_system.load_json(data.cache_path)
+        collect_related_issue_keys issue: Issue.new(raw: raw, board: board), related_issue_keys: related_issue_keys
+      end
+
+      # Remove all the ones we already have
+      related_issue_keys.reject! { |key| issue_data_hash[key] }
+
+      related_issue_keys.each do |key|
+        data = DownloadIssueData.new key: key
+        data.found_in_primary_query = false
+        data.up_to_date = false
+        data.cache_path = File.join(path, "#{key}-#{board.id}.json")
+        issue_data_hash[key] = data
+      end
+      break if related_issue_keys.empty?
+
+      unless in_related_phase
+        in_related_phase = true
+        log "  Identifying related issues (parents, subtasks, links) for board #{board.id}", both: true
+        log_start '  Downloading more issues '
+      end
+    end
+
+    end_progress if in_related_phase
+
+    delete_issues_from_cache_that_are_not_in_server(
+      issue_data_hash: issue_data_hash, path: path
+    )
+  end
+
+  def delete_issues_from_cache_that_are_not_in_server issue_data_hash:, path:
+    # The gotcha with deleted issues is that they just stop being returned in queries
+    # and we have no way to know that they should be removed from our local cache.
+    # With the new approach, we ask for every issue that Jira knows about (within
+    # the parameters of the query) and then delete anything that's in our local cache
+    # but wasn't returned.
+    @file_system.foreach path do |file|
+      next if file.start_with? '.'
+      unless /^(?<key>\w+-\d+)-\d+\.json$/ =~ file
+        raise "Unexpected filename in #{path}: #{file}"
+      end
+      next if issue_data_hash[key] # Still in Jira
+
+      file_to_delete = File.join(path, file)
+      log "  Removing #{file_to_delete} from local cache"
+      file_system.unlink file_to_delete
+    end
+  end
+
+  def collect_related_issue_keys issue:, related_issue_keys:
+    parent_key = issue.parent_key(project_config: @download_config.project_config)
+    related_issue_keys << parent_key if parent_key
+
+    issue.raw['fields']['subtasks']&.each do |raw_subtask|
+      related_issue_keys << raw_subtask['key']
+    end
+
+    issue.raw['fields']['issuelinks']&.each do |link|
+      next if link['type']['name'] == 'Cloners'
+
+      linked = link['inwardIssue'] || link['outwardIssue']
+      related_issue_keys << linked['key'] if linked
+    end
+  end
+
+  def last_modified filename:
+    File.mtime(filename) if File.exist?(filename)
+  end
+end

data/lib/jirametrics/downloader_for_data_center.rb

@@ -0,0 +1,95 @@
+# frozen_string_literal: true
+
+class DownloaderForDataCenter < Downloader
+  def jira_instance_type
+    'Jira DataCenter'
+  end
+
+  def download_issues board:
+    log "  Downloading primary issues for board #{board.id}", both: true
+    path = File.join(@target_path, "#{file_prefix}_issues/")
+    unless Dir.exist?(path)
+      log "  Creating path #{path}"
+      Dir.mkdir(path)
+    end
+
+    filter_id = board_id_to_filter_id[board.id]
+    jql = make_jql(filter_id: filter_id)
+    jira_search_by_jql(jql: jql, initial_query: true, board: board, path: path)
+
+    log "  Downloading linked issues for board #{board.id}", both: true
+    loop do
+      @issue_keys_pending_download.reject! { |key| @issue_keys_downloaded_in_current_run.include? key }
+      break if @issue_keys_pending_download.empty?
+
+      keys_to_request = @issue_keys_pending_download[0..99]
+      @issue_keys_pending_download.reject! { |key| keys_to_request.include? key }
+      jql = "key in (#{keys_to_request.join(', ')})"
+      jira_search_by_jql(jql: jql, initial_query: true, board: board, path: path)
+    end
+  end
+
+  def jira_search_by_jql jql:, initial_query:, board:, path:
+    intercept_jql = @download_config.project_config.settings['intercept_jql']
+    jql = intercept_jql.call jql if intercept_jql
+
+    log "  JQL: #{jql}"
+    escaped_jql = CGI.escape jql
+
+    max_results = 100
+    start_at = 0
+    total = 1
+    while start_at < total
+      json = @jira_gateway.call_url relative_url: '/rest/api/2/search' \
+        "?jql=#{escaped_jql}&maxResults=#{max_results}&startAt=#{start_at}&expand=changelog&fields=*all"
+
+      json['issues'].each do |issue_json|
+        issue_json['exporter'] = {
+          'in_initial_query' => initial_query
+        }
+        identify_other_issues_to_be_downloaded raw_issue: issue_json, board: board
+        file = "#{issue_json['key']}-#{board.id}.json"
+
+        @file_system.save_json(json: issue_json, filename: File.join(path, file))
+      end
+
+      total = json['total'].to_i
+      max_results = json['maxResults']
+
+      message = "    Downloaded #{start_at + 1}-#{[start_at + max_results, total].min} of #{total} issues to #{path} "
+      log message, both: true
+
+      start_at += json['issues'].size
+    end
+  end
+
+  def make_jql filter_id:, today: nil
+    today ||= today_in_project_timezone
+    segments = []
+    segments << "filter=#{filter_id}"
+
+    start_date = @download_config.start_date today: today
+
+    if start_date
+      @download_date_range = start_date..today.to_date
+
+      # For an incremental download, we want to query from the end of the previous one, not from the
+      # beginning of the full range.
+      @start_date_in_query = metadata['date_end'] || @download_date_range.begin
+      log "    Incremental download only. Pulling from #{@start_date_in_query}", both: true if metadata['date_end']
+
+      # Catch-all to pick up anything that's been around since before the range started but hasn't
+      # had an update during the range.
+      catch_all = '((status changed OR Sprint is not EMPTY) AND statusCategory != Done)'
+
+      # Pick up any issues that had a status change in the range
+      start_date_text = @start_date_in_query.strftime '%Y-%m-%d'
+      # find_in_range = %((status changed DURING ("#{start_date_text} 00:00","#{end_date_text} 23:59")))
+      find_in_range = %(updated >= "#{start_date_text} 00:00")
+
+      segments << "(#{find_in_range} OR #{catch_all})"
+    end
+
+    segments.join ' AND '
+  end
+end

data/lib/jirametrics/estimate_accuracy_chart.rb

@@ -5,7 +5,7 @@ class EstimateAccuracyChart < ChartBase
     super()
 
     header_text 'Estimate Accuracy'
-    description_text <<-HTML
+    description_text <<~HTML
       <div class="p">
         This chart graphs estimates against actual recorded cycle times. Since
         estimates can change over time, we're graphing the estimate at the time that the story started.
@@ -20,17 +20,30 @@ class EstimateAccuracyChart < ChartBase
           far to the right then you know you have a problem.
         <% end %>
       </div>
+      <% if @correlation_coefficient %>
+        <div class="p">
+          The completed items here have a correlation coefficient of <b><%= @correlation_coefficient.round(3) %></b>.
+          The closer it is to +1, the stronger the positive correlation. The closer it is to -1,
+          the stronger the negative collalation. Zero would mean no correlation at all.
+        </div>
+      <% end %>
     HTML
 
-    @y_axis_label = 'Story Point Estimates'
+    @x_axis_title = 'Cycletime (days)'
+    @y_axis_title = 'Estimate'
+
     @y_axis_type = 'linear'
-    @y_axis_block = ->(issue, start_time) { story_points_at(issue: issue, start_time: start_time)&.to_f }
+    @y_axis_block = ->(issue, start_time) { estimate_at(issue: issue, start_time: start_time)&.to_f }
     @y_axis_sort_order = nil
 
     instance_eval(&configuration_block)
   end
 
   def run
+    if @y_axis_title.nil?
+      text = current_board.estimation_configuration.units == :story_points ? 'Story Points' : 'Days'
+      @y_axis_title = "Estimated #{text}"
+    end
     data_sets = scan_issues
 
     return '' if data_sets.empty?
@@ -40,7 +53,8 @@ class EstimateAccuracyChart < ChartBase
 
   def scan_issues
     completed_hash, aging_hash = split_into_completed_and_aging issues: issues
-
+    @correlation_coefficient = correlation_coefficient(completed_hash) unless completed_hash.empty?
+    estimation_units = current_board.estimation_configuration.units
     @has_aging_data = !aging_hash.empty?
 
     [
@@ -53,9 +67,13 @@ class EstimateAccuracyChart < ChartBase
       # We sort so that the smaller circles are in front of the bigger circles.
       data = hash.sort(&hash_sorter).collect do |key, values|
         estimate, cycle_time = *key
-        estimate_label = "#{estimate}#{'pts' if @y_axis_type == 'linear'}"
-        title = ["Estimate: #{estimate_label}, Cycletime: #{label_days(cycle_time)}, #{values.size} issues"] +
-          values.collect { |issue| "#{issue.key}: #{issue.summary}" }
+
+        title = [
+          "Estimate: #{estimate_label(estimate: estimate, estimation_units: estimation_units)}, " \
+            "Cycletime: #{label_days(cycle_time)}, " \
+            "#{values.size} issues"
+        ] + values.collect { |issue| "#{issue.key}: #{issue.summary}" }
+
         {
           'x' => cycle_time,
           'y' => estimate,
@@ -77,14 +95,25 @@ class EstimateAccuracyChart < ChartBase
     end
   end
 
+  def estimate_label estimate:, estimation_units:
+    if @y_axis_type == 'linear'
+      if estimation_units == :story_points
+        estimate_label = "#{estimate}pts"
+      elsif estimation_units == :seconds
+        estimate_label = label_days estimate
+      end
+    end
+    estimate_label = estimate.to_s if estimate_label.nil?
+    estimate_label
+  end
+
   def split_into_completed_and_aging issues:
     aging_hash = {}
     completed_hash = {}
 
     issues.each do |issue|
       cycletime = issue.board.cycletime
-      start_time = cycletime.started_time(issue)
-      stop_time = cycletime.stopped_time(issue)
+      start_time, stop_time = cycletime.started_stopped_times(issue)
 
       next unless start_time
 
@@ -127,14 +156,18 @@ class EstimateAccuracyChart < ChartBase
     end
   end
 
-  def story_points_at issue:, start_time:
-    story_points = nil
+  def estimate_at issue:, start_time:, estimation_configuration: current_board.estimation_configuration
+    estimate = nil
+
     issue.changes.each do |change|
-      return story_points if change.time >= start_time
+      return estimate if change.time >= start_time
 
-      story_points = change.value if change.story_points?
+      if change.field == estimation_configuration.display_name || change.field == estimation_configuration.field_id
+        estimate = change.value
+        estimate = estimate.to_f / (24 * 60 * 60) if estimation_configuration.units == :seconds
+      end
     end
-    story_points
+    estimate
   end
 
   def y_axis label:, sort_order: nil, &block
@@ -147,4 +180,32 @@ class EstimateAccuracyChart < ChartBase
     end
     @y_axis_block = block
   end
+
+  # Correlation coefficient is calculated using the Pearson Correlation Coefficient
+  # r = Σ((xi - x̄)(yi - ȳ)) / sqrt(Σ(xi - x̄)² · Σ(yi - ȳ)²)
+  def correlation_coefficient completed_hash
+    list1 = []
+    list2 = []
+    completed_hash.each do |(estimate, cycle_time), issues|
+      issues.size.times do
+        list1 << estimate
+        list2 << cycle_time
+      end
+    end
+
+    n = list1.size
+    return nil if n < 2
+
+    mean1 = list1.sum.to_f / n
+    mean2 = list2.sum.to_f / n
+
+    numerator = list1.zip(list2).sum { |x, y| (x - mean1) * (y - mean2) }
+    sum_sq1 = list1.sum { |x| (x - mean1)**2 }
+    sum_sq2 = list2.sum { |y| (y - mean2)**2 }
+
+    denominator = Math.sqrt(sum_sq1 * sum_sq2)
+    return nil if denominator.zero?
+
+    numerator / denominator
+  end
 end

data/lib/jirametrics/estimation_configuration.rb

@@ -0,0 +1,25 @@
+# frozen_string_literal: true
+
+class EstimationConfiguration
+  attr_reader :units, :display_name, :field_id
+
+  def initialize raw:
+    @units = :story_points
+    @display_name = 'Story Points'
+
+    # If there wasn't an estimation section they rely on all defaults
+    return if raw.nil?
+
+    if raw['type'] == 'field'
+      @field_id = raw['field']['fieldId']
+      @display_name = raw['field']['displayName']
+      if @field_id == 'timeoriginalestimate'
+        @units = :seconds
+        @display_name = 'Original estimate'
+      end
+    elsif raw['type'] == 'issueCount'
+      @display_name = 'Issue Count'
+      @units = :issue_count
+    end
+  end
+end

data/lib/jirametrics/examples/aggregated_project.rb

@@ -3,8 +3,6 @@
 # This file is really intended to give you ideas about how you might configure your own reports, not
 # as a complete setup that will work in every case.
 #
-# See https://github.com/mikebowler/jirametrics/wiki/Examples-folder for more details
-#
 # The point of an AGGREGATED report is that we're now looking at a higher level. We might use this in a
 # S2 meeting (Scrum of Scrums) to talk about the things that are happening across teams, not within a
 # single team. For that reason, we look at slightly different things that we would on a single team board.
@@ -12,8 +10,9 @@
 class Exporter
   def aggregated_project name:, project_names:, settings: {}
     project name: name do
-      puts name
-      self.settings.merge! settings
+      file_system.log name
+      file_prefix name
+      self.settings.merge! stringify_keys(settings)
 
       aggregate do
         project_names.each do |project_name|
@@ -21,8 +20,6 @@ class Exporter
         end
       end
 
-      file_prefix name
-
       file do
         file_suffix '.html'
         issues.reject! do |issue|
@@ -33,8 +30,8 @@ class Exporter
           html '<h1>Boards included in this report</h1><ul>', type: :header
           board_lines = []
           included_projects.each do |project|
-            project.all_boards.values.each do |board|
-              board_lines << "<a href='#{project.file_prefix}.html'>#{board.name}</a> from project #{project.name}"
+            project.all_boards.each_value do |board|
+              board_lines << "<a href='#{project.get_file_prefix}.html'>#{board.name}</a> from project #{project.name}"
             end
           end
           board_lines.sort.each { |line| html "<li>#{line}</li>", type: :header }
@@ -64,25 +61,14 @@ class Exporter
 
             # By default, the issue doesn't show what board it's on and this is important for an
             # aggregated view
+            chart = self
             issue_rules do |issue, rules|
-              key = issue.key
-              key = "<S>#{key} </S> " if issue.status.category_name == 'Done'
-              rules.label = "<#{key} [#{issue.type}]<BR/>#{issue.board.name}<BR/>#{word_wrap issue.summary}>"
+              chart.default_issue_rules.call(issue, rules)
+              rules.label = rules.label.split('<BR/>').insert(1, "Board: #{issue.board.name}").join('<BR/>')
             end
 
             link_rules do |link, rules|
-              # By default, the dependency chart shows everything. Clean it up a bit.
-              case link.name
-              when 'Cloners'
-                # We don't want to see any clone links at all.
-                rules.ignore
-              when 'Blocks'
-                # For blocks, by default Jira will have links going both
-                # ways and we want them only going one way. Also make the
-                # link red.
-                rules.merge_bidirectional keep: 'outward'
-                rules.line_color = 'red'
-              end
+              chart.default_link_rules.call(link, rules)
 
               # Because this is the aggregated view, let's hide any link that doesn't cross boards.
               rules.ignore if link.origin.board == link.other_issue.board