jirametrics 2.31 → 3.1

data/lib/jirametrics/downloader_for_cloud.rb

@@ -130,7 +130,7 @@ class DownloaderForCloud < Downloader
     issue_datas
   end
 
-  def attach_worklogs_to_issues issue_datas:, issue_jsons:, max_results: 100 # rubocop:disable Lint/UnusedMethodArgument
+  def attach_worklogs_to_issues issue_datas:, issue_jsons:, max_results: 100
     issue_jsons.each do |issue_json|
       worklog = issue_json['fields']['worklog']
       next unless worklog
@@ -140,27 +140,7 @@ class DownloaderForCloud < Downloader
       next if all_worklogs.size >= total
 
       key = issue_json['key']
-      start_at = all_worklogs.size
-
-      loop do
-        response = @jira_gateway.call_url(
-          relative_url: "/rest/api/3/issue/#{CGI.escape(key)}/worklog?startAt=#{start_at}&maxResults=#{max_results}"
-        )
-
-        worklogs = response['worklogs'] || []
-        all_worklogs.concat(worklogs)
-
-        total = response['total'].to_i
-        log "        #{key} worklogs: page startAt=#{start_at}, " \
-            "received=#{worklogs.size}, fetched=#{all_worklogs.size}/#{total}"
-        break if all_worklogs.size >= total
-        # Guard against Jira reporting a higher total than it will actually return — seen when
-        # worklogs are deleted or access-restricted after the initial fetch. Without this,
-        # start_at never advances and we loop forever requesting the same empty page.
-        break if worklogs.empty?
-
-        start_at += worklogs.size
-      end
+      fetch_remaining_worklogs key: key, all_worklogs: all_worklogs, max_results: max_results
 
       issue_json['fields']['worklog'] = {
         'startAt' => 0,
@@ -173,6 +153,30 @@ class DownloaderForCloud < Downloader
     end
   end
 
+  # Page through the worklogs Jira didn't include inline, appending each page to all_worklogs.
+  def fetch_remaining_worklogs key:, all_worklogs:, max_results:
+    start_at = all_worklogs.size
+    loop do
+      response = @jira_gateway.call_url(
+        relative_url: "/rest/api/3/issue/#{CGI.escape(key)}/worklog?startAt=#{start_at}&maxResults=#{max_results}"
+      )
+
+      worklogs = response['worklogs'] || []
+      all_worklogs.concat(worklogs)
+
+      total = response['total'].to_i
+      log "        #{key} worklogs: page startAt=#{start_at}, " \
+          "received=#{worklogs.size}, fetched=#{all_worklogs.size}/#{total}"
+      break if all_worklogs.size >= total
+      # Guard against Jira reporting a higher total than it will actually return - seen when
+      # worklogs are deleted or access-restricted after the initial fetch. Without this,
+      # start_at never advances and we loop forever requesting the same empty page.
+      break if worklogs.empty?
+
+      start_at += worklogs.size
+    end
+  end
+
   def attach_changelog_to_issues issue_datas:, issue_jsons:
     max_results = 10_000 # The max jira accepts is 10K
     payload = {
@@ -212,71 +216,59 @@ class DownloaderForCloud < Downloader
 
   def download_issues board:
     log "  Downloading primary issues for board #{board.id} from #{jira_instance_type}", both: true
+    path = ensure_issues_directory
+    issue_data_hash = search_for_issues jql: build_jql(board), board_id: board.id, path: path
+
+    download_all_issues board: board, path: path, issue_data_hash: issue_data_hash
+
+    delete_issues_from_cache_that_are_not_in_server(
+      issue_data_hash: issue_data_hash, path: path
+    )
+  end
+
+  def ensure_issues_directory
     path = File.join(@target_path, "#{file_prefix}_issues/")
     unless @file_system.dir_exist?(path)
       log "  Creating path #{path}"
       @file_system.mkdir(path)
     end
+    path
+  end
 
-    filter_id = @board_id_to_filter_id[board.id]
-    jql = make_jql(filter_id: filter_id)
+  def build_jql board
+    jql = make_jql(filter_id: @board_id_to_filter_id[board.id])
     intercept_jql = @download_config.project_config.settings['intercept_jql']
-    jql = intercept_jql.call jql if intercept_jql
-
-    issue_data_hash = search_for_issues jql: jql, board_id: board.id, path: path
+    intercept_jql ? intercept_jql.call(jql) : jql
+  end
 
+  # Downloads the primary issues, then repeatedly follows related issues (parents, subtasks, links)
+  # until no new ones turn up. `in_related_phase` is false for the first pass and true thereafter,
+  # which controls the once-only "Identifying related issues" announcement and the loop diagnostics.
+  def download_all_issues board:, path:, issue_data_hash:
     checked_for_related = Set.new
     in_related_phase = false
 
     loop do
       related_issue_keys = Set.new
-      stale = issue_data_hash.values.reject { |data| data.up_to_date }
+      stale = issue_data_hash.values.reject(&:up_to_date)
       if in_related_phase
         @file_system.diagnostic "Download loop: #{issue_data_hash.size} total known, " \
                                 "#{stale.size} stale, #{checked_for_related.size} link-scanned"
       end
-      unless stale.empty?
-        log_start '  Downloading more issues ' unless in_related_phase
-        stale.each_slice(100) do |slice|
-          slice = bulk_fetch_issues(issue_datas: slice, board: board, in_initial_query: !in_related_phase)
-          progress_dot
-          slice.each do |data|
-            next unless data.issue
-
-            @file_system.save_json(
-              json: data.issue.raw, filename: data.cache_path
-            )
-            # Set the timestamp on the file to match the updated one so that we don't have
-            # to parse the file just to find the timestamp
-            @file_system.utime time: data.issue.updated, file: data.cache_path
-
-            collect_or_log_related(
-              issue: data.issue, found_in_primary_query: data.found_in_primary_query,
-              related_issue_keys: related_issue_keys, issue_data_hash: issue_data_hash
-            )
-            checked_for_related << data.key
-          end
-        end
-        end_progress unless in_related_phase
-      end
+      download_stale_issues(
+        stale: stale, board: board, in_related_phase: in_related_phase,
+        checked_for_related: checked_for_related, related_issue_keys: related_issue_keys,
+        issue_data_hash: issue_data_hash
+      )
 
       scan_cached_issues_for_related(
         issue_data_hash: issue_data_hash, board: board,
         checked_for_related: checked_for_related, related_issue_keys: related_issue_keys
       )
-
-      # Remove all the ones we already have
-      related_issue_keys.reject! { |key| issue_data_hash[key] }
-
-      related_issue_keys.each do |key|
-        data = DownloadIssueData.new key: key
-        data.found_in_primary_query = false
-        data.up_to_date = false
-        data.cache_path = File.join(path, "#{key}-#{board.id}.json")
-        issue_data_hash[key] = data
-      end
+      register_related_issues(
+        related_issue_keys: related_issue_keys, issue_data_hash: issue_data_hash, path: path, board: board
+      )
       break if related_issue_keys.empty?
-
       next if in_related_phase
 
       in_related_phase = true
@@ -285,10 +277,52 @@ class DownloaderForCloud < Downloader
     end
 
     end_progress if in_related_phase
+  end
 
-    delete_issues_from_cache_that_are_not_in_server(
-      issue_data_hash: issue_data_hash, path: path
-    )
+  def download_stale_issues stale:, board:, in_related_phase:, checked_for_related:, related_issue_keys:,
+                            issue_data_hash:
+    return if stale.empty?
+
+    log_start '  Downloading more issues ' unless in_related_phase
+    stale.each_slice(100) do |slice|
+      slice = bulk_fetch_issues(issue_datas: slice, board: board, in_initial_query: !in_related_phase)
+      progress_dot
+      save_fetched_issues(
+        slice: slice, checked_for_related: checked_for_related,
+        related_issue_keys: related_issue_keys, issue_data_hash: issue_data_hash
+      )
+    end
+    end_progress unless in_related_phase
+  end
+
+  def save_fetched_issues slice:, checked_for_related:, related_issue_keys:, issue_data_hash:
+    slice.each do |data|
+      next unless data.issue
+
+      @file_system.save_json(json: data.issue.raw, filename: data.cache_path)
+      # Set the timestamp on the file to match the updated one so that we don't have
+      # to parse the file just to find the timestamp
+      @file_system.utime time: data.issue.updated, file: data.cache_path
+
+      collect_or_log_related(
+        issue: data.issue, found_in_primary_query: data.found_in_primary_query,
+        related_issue_keys: related_issue_keys, issue_data_hash: issue_data_hash
+      )
+      checked_for_related << data.key
+    end
+  end
+
+  def register_related_issues related_issue_keys:, issue_data_hash:, path:, board:
+    # Remove all the ones we already have
+    related_issue_keys.reject! { |key| issue_data_hash[key] }
+
+    related_issue_keys.each do |key|
+      data = DownloadIssueData.new key: key
+      data.found_in_primary_query = false
+      data.up_to_date = false
+      data.cache_path = File.join(path, "#{key}-#{board.id}.json")
+      issue_data_hash[key] = data
+    end
   end
 
   def delete_issues_from_cache_that_are_not_in_server issue_data_hash:, path:
@@ -310,7 +344,7 @@ class DownloaderForCloud < Downloader
     end
   end
 
-  # Scan up-to-date cached primary issues we haven't checked yet — they may reference related
+  # Scan up-to-date cached primary issues we haven't checked yet - they may reference related
   # issues that are not in the primary query result. We only follow links one hop out from the
   # primary issues, so related (non-primary) cached issues are not followed (just logged).
   def scan_cached_issues_for_related issue_data_hash:, board:, checked_for_related:, related_issue_keys:
@@ -348,23 +382,25 @@ class DownloaderForCloud < Downloader
     parent_key = issue.parent_key(project_config: @download_config.project_config)
     keys << parent_key if parent_key
 
-    issue.raw['fields']['subtasks']&.each do |raw_subtask|
-      keys << raw_subtask['key']
-    end
+    issue.raw['fields']['subtasks']&.each { |raw_subtask| keys << raw_subtask['key'] }
+    add_linked_issue_keys issue, keys
+
+    keys
+  end
 
+  # Add the keys of issues linked to this one, skipping clone links (a clone isn't a dependency).
+  def add_linked_issue_keys issue, keys
     issue.raw['fields']['issuelinks']&.each do |link|
       next if link['type']['name'] == 'Cloners'
 
       linked = link['inwardIssue'] || link['outwardIssue']
       keys << linked['key'] if linked
     end
-
-    keys
   end
 
   # We only follow links one hop out from the primary (board) issues. If a related issue
   # itself references further issues we haven't already downloaded, we deliberately don't
-  # follow them — but log it so we can diagnose later if an export fails because a
+  # follow them - but log it so we can diagnose later if an export fails because a
   # second-hop issue was missing. See GitHub #72.
   def log_unfollowed_related_keys issue:, issue_data_hash:
     onward = related_keys_for(issue).reject { |key| issue_data_hash[key] }

data/lib/jirametrics/estimate_accuracy_chart.rb

@@ -110,25 +110,25 @@ class EstimateAccuracyChart < ChartBase
   def split_into_completed_and_aging issues:
     aging_hash = {}
     completed_hash = {}
+    issues.each { |issue| bucket_issue_by_estimate issue, completed_hash: completed_hash, aging_hash: aging_hash }
+    [completed_hash, aging_hash]
+  end
 
-    issues.each do |issue|
-      cycletime = issue.board.cycletime
-      start_time, stop_time = cycletime.started_stopped_times(issue)
+  def bucket_issue_by_estimate issue, completed_hash:, aging_hash:
+    cycletime = issue.board.cycletime
+    start_time, stop_time = cycletime.started_stopped_times(issue)
 
-      next unless start_time
+    return unless start_time
 
-      hash = stop_time ? completed_hash : aging_hash
+    hash = stop_time ? completed_hash : aging_hash
 
-      estimate = @y_axis_block.call issue, start_time
-      cycle_time = ((stop_time&.to_date || date_range.end) - start_time.to_date).to_i + 1
+    estimate = @y_axis_block.call issue, start_time
+    cycle_time = ((stop_time&.to_date || date_range.end) - start_time.to_date).to_i + 1
 
-      next if estimate.nil?
+    return if estimate.nil?
 
-      key = [estimate, cycle_time]
-      (hash[key] ||= []) << issue
-    end
-
-    [completed_hash, aging_hash]
+    key = [estimate, cycle_time]
+    (hash[key] ||= []) << issue
   end
 
   def hash_sorter

data/lib/jirametrics/examples/standard_project.rb

@@ -35,7 +35,7 @@ class Exporter
       download do
         self.rolling_date_count(rolling_date_count) if rolling_date_count
         self.no_earlier_than(no_earlier_than) if no_earlier_than
-        github_repo *github_repos if github_repos
+        github_repo(*github_repos) if github_repos
       end
 
       issues.reject! do |issue|
@@ -44,7 +44,7 @@ class Exporter
 
       exporter.filter_issues issues, ignore_issues
 
-      discard_changes_before status_becomes: (starting_status || :backlog) # rubocop:disable Style/RedundantParentheses
+      discard_changes_before status_becomes: (starting_status || :backlog)
 
       file do
         file_suffix '.html'

data/lib/jirametrics/expedited_chart.rb

@@ -67,12 +67,7 @@ class ExpeditedChart < ChartBase
       if expedited_priority_names.include? change.value
         expedite_start = change.time.to_date
       elsif expedite_start
-        start_date = expedite_start.to_date
-        stop_date = change.time.to_date
-
-        if date_range.include?(start_date) || date_range.include?(stop_date) ||
-           (start_date < date_range.begin && stop_date > date_range.end)
-
+        if expedite_visible?(expedite_start, change.time.to_date)
           result << [expedite_start, :expedite_start]
           result << [change.time.to_date, :expedite_stop]
         end
@@ -85,6 +80,13 @@ class ExpeditedChart < ChartBase
     result
   end
 
+  # True when an expedite span overlaps the chart's date range: either endpoint falls inside it, or it
+  # starts before and ends after (spanning the whole range).
+  def expedite_visible? start_date, stop_date
+    date_range.include?(start_date) || date_range.include?(stop_date) ||
+      (start_date < date_range.begin && stop_date > date_range.end)
+  end
+
   def find_expedited_issues
     expedited_issues = @issues.reject do |issue|
       prepare_expedite_data(issue).empty?
@@ -110,8 +112,7 @@ class ExpeditedChart < ChartBase
   end
 
   def make_expedite_lines_data_set issue:, expedite_data:
-    cycletime = issue.board.cycletime
-    started_date, stopped_date = cycletime.started_stopped_dates(issue)
+    started_date, stopped_date = issue.board.cycletime.started_stopped_dates(issue)
 
     expedite_data << [started_date, :issue_started] if started_date
     expedite_data << [stopped_date, :issue_stopped] if stopped_date
@@ -126,37 +127,16 @@ class ExpeditedChart < ChartBase
     expedited = false
 
     expedite_data.each do |time, action|
-      case action
-      when :issue_started
-        data << make_point(issue: issue, time: time, label: 'Started', expedited: expedited)
-        dot_colors << CssVariable['--expedited-chart-dot-issue-started-color']
-        point_styles << 'rect'
-      when :issue_stopped
-        data << make_point(issue: issue, time: time, label: 'Completed', expedited: expedited)
-        dot_colors << CssVariable['--expedited-chart-dot-issue-stopped-color']
-        point_styles << 'rect'
-      when :expedite_start
-        data << make_point(issue: issue, time: time, label: 'Expedited', expedited: true)
-        dot_colors << CssVariable['--expedited-chart-dot-expedite-started-color']
-        point_styles << 'circle'
-        expedited = true
-      when :expedite_stop
-        data << make_point(issue: issue, time: time, label: 'Not expedited', expedited: false)
-        dot_colors << CssVariable['--expedited-chart-dot-expedite-stopped-color']
-        point_styles << 'circle'
-        expedited = false
-      else
-        raise "Unexpected action: #{action}"
-      end
+      point, color, style, expedited = expedite_point(issue, time, action, expedited)
+      data << point
+      dot_colors << color
+      point_styles << style
     end
 
-    unless expedite_data.empty?
-      last_change_time = expedite_data[-1][0].to_date
-      if last_change_time && last_change_time <= date_range.end && stopped_date.nil?
-        data << make_point(issue: issue, time: date_range.end, label: 'Still ongoing', expedited: expedited)
-        dot_colors << '' # It won't be visible so it doesn't matter
-        point_styles << 'dash'
-      end
+    if still_ongoing?(expedite_data, stopped_date)
+      data << make_point(issue: issue, time: date_range.end, label: 'Still ongoing', expedited: expedited)
+      dot_colors << '' # It won't be visible so it doesn't matter
+      point_styles << 'dash'
     end
 
     {
@@ -171,4 +151,34 @@ class ExpeditedChart < ChartBase
       segment: EXPEDITED_SEGMENT
     }
   end
+
+  # Returns [point, dot_color, point_style, expedited] for one timeline event. The returned expedited
+  # flag carries the (possibly changed) state forward to the next event.
+  def expedite_point issue, time, action, expedited
+    case action
+    when :issue_started
+      [make_point(issue: issue, time: time, label: 'Started', expedited: expedited),
+       CssVariable['--expedited-chart-dot-issue-started-color'], 'rect', expedited]
+    when :issue_stopped
+      [make_point(issue: issue, time: time, label: 'Completed', expedited: expedited),
+       CssVariable['--expedited-chart-dot-issue-stopped-color'], 'rect', expedited]
+    when :expedite_start
+      [make_point(issue: issue, time: time, label: 'Expedited', expedited: true),
+       CssVariable['--expedited-chart-dot-expedite-started-color'], 'circle', true]
+    when :expedite_stop
+      [make_point(issue: issue, time: time, label: 'Not expedited', expedited: false),
+       CssVariable['--expedited-chart-dot-expedite-stopped-color'], 'circle', false]
+    else
+      raise "Unexpected action: #{action}"
+    end
+  end
+
+  # The issue is still expedited/open at the end of the chart: it hasn't stopped and its last recorded
+  # change is on or before the end of the range.
+  def still_ongoing? expedite_data, stopped_date
+    return false if expedite_data.empty?
+
+    last_change_time = expedite_data[-1][0].to_date
+    last_change_time && last_change_time <= date_range.end && stopped_date.nil?
+  end
 end