jirametrics 2.14 → 2.30

data/lib/jirametrics/downloader.rb

@@ -3,8 +3,29 @@
 require 'cgi'
 require 'json'
 
+class DownloadIssueData
+  attr_accessor :key, :found_in_primary_query, :last_modified,
+    :up_to_date, :cache_path, :issue
+
+  def initialize(
+    key:,
+    found_in_primary_query: true,
+    last_modified: nil,
+    up_to_date: true,
+    cache_path: nil,
+    issue: nil
+  )
+    @key = key
+    @found_in_primary_query = found_in_primary_query
+    @last_modified = last_modified
+    @up_to_date = up_to_date
+    @cache_path = cache_path
+    @issue = issue
+  end
+end
+
 class Downloader
-  CURRENT_METADATA_VERSION = 4
+  CURRENT_METADATA_VERSION = 5
 
   attr_accessor :metadata
   attr_reader :file_system
@@ -12,12 +33,23 @@ class Downloader
   # For testing only
   attr_reader :start_date_in_query, :board_id_to_filter_id
 
-  def initialize download_config:, file_system:, jira_gateway:
+  def self.create download_config:, file_system:, jira_gateway:, github_pr_cache: {}
+    is_cloud = jira_gateway.settings['jira_cloud'] || jira_gateway.cloud?
+    (is_cloud ? DownloaderForCloud : DownloaderForDataCenter).new(
+      download_config: download_config,
+      file_system: file_system,
+      jira_gateway: jira_gateway,
+      github_pr_cache: github_pr_cache
+    )
+  end
+
+  def initialize download_config:, file_system:, jira_gateway:, github_pr_cache: {}
     @metadata = {}
     @download_config = download_config
     @target_path = @download_config.project_config.target_path
     @file_system = file_system
     @jira_gateway = jira_gateway
+    @github_pr_cache = github_pr_cache
     @board_id_to_filter_id = {}
 
     @issue_keys_downloaded_in_current_run = []
@@ -28,7 +60,6 @@ class Downloader
     log '', both: true
     log @download_config.project_config.name, both: true
 
-    init_gateway
     load_metadata
 
     if @metadata['no-download']
@@ -43,114 +74,41 @@ class Downloader
     download_statuses
     find_board_ids.each do |id|
       board = download_board_configuration board_id: id
+      board.project_config = @download_config.project_config
       download_issues board: board
     end
     download_users
 
     save_metadata
-  end
-
-  def init_gateway
-    @jira_gateway.load_jira_config(@download_config.project_config.jira_config)
-    @jira_gateway.ignore_ssl_errors = @download_config.project_config.settings['ignore_ssl_errors']
+    download_github_prs if @download_config.github_repos.any?
   end
 
   def log text, both: false
     @file_system.log text, also_write_to_stderr: both
   end
 
-  def find_board_ids
-    ids = @download_config.project_config.board_configs.collect(&:id)
-    raise 'Board ids must be specified' if ids.empty?
-
-    ids
+  def log_start text
+    @file_system.log_start text
   end
 
-  def download_issues board:
-    log "  Downloading primary issues for board #{board.id}", both: true
-    path = File.join(@target_path, "#{file_prefix}_issues/")
-    unless Dir.exist?(path)
-      log "  Creating path #{path}"
-      Dir.mkdir(path)
-    end
-
-    filter_id = @board_id_to_filter_id[board.id]
-    jql = make_jql(filter_id: filter_id)
-    jira_search_by_jql(jql: jql, initial_query: true, board: board, path: path)
-
-    log "  Downloading linked issues for board #{board.id}", both: true
-    loop do
-      @issue_keys_pending_download.reject! { |key| @issue_keys_downloaded_in_current_run.include? key }
-      break if @issue_keys_pending_download.empty?
-
-      keys_to_request = @issue_keys_pending_download[0..99]
-      @issue_keys_pending_download.reject! { |key| keys_to_request.include? key }
-      jql = "key in (#{keys_to_request.join(', ')})"
-      jira_search_by_jql(jql: jql, initial_query: false, board: board, path: path)
-    end
+  def start_progress
+    @file_system.start_progress
   end
 
-  def jira_search_by_jql jql:, initial_query:, board:, path:
-    intercept_jql = @download_config.project_config.settings['intercept_jql']
-    jql = intercept_jql.call jql if intercept_jql
-
-    log "  JQL: #{jql}"
-    escaped_jql = CGI.escape jql
-
-    if @jira_gateway.cloud?
-      max_results = 5_000 # The maximum allowed by Jira
-      next_page_token = nil
-      issue_count = 0
-
-      loop do
-        json = @jira_gateway.call_url relative_url: '/rest/api/3/search/jql' \
-          "?jql=#{escaped_jql}&maxResults=#{max_results}&" \
-          "nextPageToken=#{next_page_token}&expand=changelog&fields=*all"
-        next_page_token = json['nextPageToken']
-
-        json['issues'].each do |issue_json|
-          issue_json['exporter'] = {
-            'in_initial_query' => initial_query
-          }
-          identify_other_issues_to_be_downloaded raw_issue: issue_json, board: board
-          file = "#{issue_json['key']}-#{board.id}.json"
+  def progress_dot message = nil
+    @file_system.log message if message
+    @file_system.progress_dot
+  end
 
-          @file_system.save_json(json: issue_json, filename: File.join(path, file))
-          issue_count += 1
-        end
+  def end_progress
+    @file_system.end_progress
+  end
 
-        message = "    Downloaded #{issue_count} issues"
-        log message, both: true
+  def find_board_ids
+    ids = @download_config.project_config.board_configs.collect(&:id)
+    raise 'Board ids must be specified' if ids.empty?
 
-        break unless next_page_token
-      end
-    else
-      max_results = 100
-      start_at = 0
-      total = 1
-      while start_at < total
-        json = @jira_gateway.call_url relative_url: '/rest/api/2/search' \
-          "?jql=#{escaped_jql}&maxResults=#{max_results}&startAt=#{start_at}&expand=changelog&fields=*all"
-
-        json['issues'].each do |issue_json|
-          issue_json['exporter'] = {
-            'in_initial_query' => initial_query
-          }
-          identify_other_issues_to_be_downloaded raw_issue: issue_json, board: board
-          file = "#{issue_json['key']}-#{board.id}.json"
-
-          @file_system.save_json(json: issue_json, filename: File.join(path, file))
-        end
-
-        total = json['total'].to_i
-        max_results = json['maxResults']
-
-        message = "    Downloaded #{start_at + 1}-#{[start_at + max_results, total].min} of #{total} issues to #{path} "
-        log message, both: true
-
-        start_at += json['issues'].size
-      end
-    end
+    ids
   end
 
   def identify_other_issues_to_be_downloaded raw_issue:, board:
@@ -178,6 +136,8 @@ class Downloader
   end
 
   def download_users
+    return unless @jira_gateway.cloud?
+
     log '  Downloading all users', both: true
     json = @jira_gateway.call_url relative_url: '/rest/api/2/users'
 
@@ -226,11 +186,28 @@ class Downloader
     # actually look at the returned json.
     @board_id_to_filter_id[board_id] = json['filter']['id'].to_i
 
+    if json['type'] == 'simple'
+      features_json = download_features board_id: board_id
+      if BoardFeature.from_raw(features_json).any? { |f| f.name == 'jsw.agility.sprints' && f.enabled? }
+        download_sprints board_id: board_id
+      end
+    end
     download_sprints board_id: board_id if json['type'] == 'scrum'
     # TODO: Should be passing actual statuses, not empty list
     Board.new raw: json, possible_statuses: StatusCollection.new
   end
 
+  def download_features board_id:
+    log "  Downloading features for board #{board_id}", both: true
+    json = @jira_gateway.call_url relative_url: "/rest/agile/1.0/board/#{board_id}/features"
+
+    @file_system.save_json(
+      json: json,
+      filename: File.join(@target_path, "#{file_prefix}_board_#{board_id}_features.json")
+    )
+    json
+  end
+
   def download_sprints board_id:
     log "  Downloading sprints for board #{board_id}", both: true
     max_results = 100
@@ -272,19 +249,29 @@ class Downloader
         value = Date.parse(value) if value.is_a?(String) && value =~ /^\d{4}-\d{2}-\d{2}$/
         @metadata[key] = value
       end
+
     end
 
     # Even if this is the old format, we want to obey this one tag
     @metadata['no-download'] = hash['no-download'] if hash['no-download']
   end
 
+  def timezone_offset
+    @download_config.project_config.exporter.timezone_offset
+  end
+
+  def today_in_project_timezone
+    Time.now.getlocal(timezone_offset).to_date
+  end
+
   def save_metadata
     @metadata['version'] = CURRENT_METADATA_VERSION
+    @metadata['rolling_date_count'] = @download_config.rolling_date_count
     @metadata['date_start_from_last_query'] = @start_date_in_query if @start_date_in_query
 
     if @download_date_range.nil?
       log "Making up a date range in meta since one wasn't specified. You'll want to change that.", both: true
-      today = Date.today
+      today = today_in_project_timezone
       @download_date_range = (today - 7)..today
     end
 
@@ -319,7 +306,8 @@ class Downloader
     end
   end
 
-  def make_jql filter_id:, today: Date.today
+  def make_jql filter_id:, today: nil
+    today ||= today_in_project_timezone
     segments = []
     segments << "filter=#{filter_id}"
 
@@ -327,11 +315,7 @@ class Downloader
 
     if start_date
       @download_date_range = start_date..today.to_date
-
-      # For an incremental download, we want to query from the end of the previous one, not from the
-      # beginning of the full range.
-      @start_date_in_query = metadata['date_end'] || @download_date_range.begin
-      log "    Incremental download only. Pulling from #{@start_date_in_query}", both: true if metadata['date_end']
+      @start_date_in_query = @download_date_range.begin
 
       # Catch-all to pick up anything that's been around since before the range started but hasn't
       # had an update during the range.
@@ -348,6 +332,39 @@ class Downloader
     segments.join ' AND '
   end
 
+  def download_github_prs
+    project_keys = extract_project_keys_from_downloaded_issues
+    if project_keys.empty?
+      log '  No project keys found in downloaded issues, skipping GitHub PR download', both: true
+      return
+    end
+
+    prs = @download_config.github_repos.flat_map do |repo|
+      GithubGateway.new(
+        repo: repo,
+        project_keys: project_keys,
+        file_system: @file_system,
+        raw_pr_cache: @github_pr_cache
+      ).fetch_pull_requests(since: @download_date_range&.begin)
+    end
+
+    @file_system.save_json(
+      json: prs.map(&:raw),
+      filename: File.join(@target_path, "#{file_prefix}_github_prs.json")
+    )
+  end
+
+  def extract_project_keys_from_downloaded_issues
+    path = File.join(@target_path, "#{file_prefix}_issues")
+    return [] unless @file_system.dir_exist?(path)
+
+    keys = []
+    @file_system.foreach(path) do |filename|
+      keys << filename.split('-').first if filename.match?(/^[A-Z][A-Z_0-9]+-\d+-\d+\.json$/)
+    end
+    keys.uniq
+  end
+
   def file_prefix
     @download_config.project_config.get_file_prefix
   end

data/lib/jirametrics/downloader_for_cloud.rb

@@ -0,0 +1,287 @@
+# frozen_string_literal: true
+
+class DownloaderForCloud < Downloader
+  def jira_instance_type
+    'Jira Cloud'
+  end
+
+  def run
+    super
+    download_fix_versions
+  end
+
+  def download_board_configuration board_id:
+    board = super
+    location = board.raw['location']
+    @project_key ||= location['key'] if location&.[]('type') == 'project'
+    board
+  end
+
+  def download_fix_versions
+    return unless @project_key
+
+    log "  Downloading fix versions for project #{@project_key}", both: true
+    max_results = 50
+    start_at = 0
+    all_versions = []
+
+    loop do
+      json = @jira_gateway.call_url(
+        relative_url: "/rest/api/3/project/#{@project_key}/version?" \
+          "startAt=#{start_at}&maxResults=#{max_results}"
+      )
+
+      values = json['values'] || []
+      all_versions.concat(values)
+      break if json['isLast'] || values.empty?
+
+      start_at += values.size
+    end
+
+    @file_system.save_json(
+      json: all_versions,
+      filename: File.join(@target_path, "#{file_prefix}_fix_versions.json")
+    )
+  end
+
+  def search_for_issues jql:, board_id:, path:
+    log "  JQL: #{jql}"
+    escaped_jql = CGI.escape jql
+
+    hash = {}
+    max_results = 5_000 # The maximum allowed by Jira
+    next_page_token = nil
+    issue_count = 0
+
+    start_progress
+    loop do
+      relative_url = +''
+      relative_url << '/rest/api/3/search/jql'
+      relative_url << "?jql=#{escaped_jql}&maxResults=#{max_results}"
+      relative_url << "&nextPageToken=#{next_page_token}" if next_page_token
+      relative_url << '&fields=updated'
+
+      json = @jira_gateway.call_url relative_url: relative_url
+      next_page_token = json['nextPageToken']
+
+      json['issues'].each do |i|
+        key = i['key']
+        data = DownloadIssueData.new key: key
+        data.key = key
+        data.last_modified = Time.parse i['fields']['updated']
+        data.found_in_primary_query = true
+        data.cache_path = File.join(path, "#{key}-#{board_id}.json")
+        data.up_to_date = last_modified(filename: data.cache_path) == data.last_modified
+        hash[key] = data
+        issue_count += 1
+      end
+
+      progress_dot "    Found #{issue_count} issues"
+
+      break unless next_page_token
+    end
+    end_progress
+
+    hash
+  end
+
+  def bulk_fetch_issues issue_datas:, board:, in_initial_query:
+    # We used to use the expand option to pull in the changelog directly. Unfortunately
+    # that only returns the "recent" changes, not all of them. So now we get the issue
+    # without changes and then make a second call for that changes. Then we insert it
+    # into the raw issue as if it had been there all along.
+    log "  Downloading #{issue_datas.size} issues"
+    payload = {
+      'fields' => ['*all'],
+      'issueIdsOrKeys' => issue_datas.collect(&:key)
+    }
+    response = @jira_gateway.post_request(
+      relative_url: '/rest/api/3/issue/bulkfetch',
+      payload: JSON.generate(payload)
+    )
+
+    attach_changelog_to_issues issue_datas: issue_datas, issue_jsons: response['issues']
+
+    response['issues'].each do |issue_json|
+      issue_json['exporter'] = {
+        'in_initial_query' => in_initial_query
+      }
+      issue = Issue.new(raw: issue_json, board: board)
+      data = issue_datas.find { |d| d.key == issue.key }
+      unless data
+        log "  Skipping #{issue.key}: returned by Jira but key not in request (issue may have been moved)"
+        next
+      end
+      data.up_to_date = true
+      data.last_modified = issue.updated
+      data.issue = issue
+    end
+
+    # Mark any unmatched requests as up_to_date to prevent infinite re-fetching.
+    # This happens when Jira returns a different key (moved issue) leaving the original unmatched.
+    issue_datas.each do |data|
+      next if data.up_to_date
+
+      log "  Skipping #{data.key}: not returned by Jira (issue may have been deleted or moved)"
+      data.up_to_date = true
+    end
+
+    issue_datas
+  end
+
+  def attach_changelog_to_issues issue_datas:, issue_jsons:
+    max_results = 10_000 # The max jira accepts is 10K
+    payload = {
+      'issueIdsOrKeys' => issue_datas.collect(&:key),
+      'maxResults' => max_results
+    }
+    loop do
+      response = @jira_gateway.post_request(
+        relative_url: '/rest/api/3/changelog/bulkfetch',
+        payload: JSON.generate(payload)
+      )
+
+      response['issueChangeLogs'].each do |issue_change_log|
+        issue_id = issue_change_log['issueId']
+        json = issue_jsons.find { |json| json['id'] == issue_id }
+
+        unless json['changelog']
+          # If this is our first time in, there won't be a changelog section
+          json['changelog'] = {
+            'startAt' => 0,
+            'maxResults' => max_results,
+            'total' => 0,
+            'histories' => []
+          }
+        end
+
+        new_changes = issue_change_log['changeHistories']
+        json['changelog']['total'] += new_changes.size
+        json['changelog']['histories'] += new_changes
+      end
+
+      next_page_token = response['nextPageToken']
+      payload['nextPageToken'] = next_page_token
+      break if next_page_token.nil?
+    end
+  end
+
+  def download_issues board:
+    log "  Downloading primary issues for board #{board.id} from #{jira_instance_type}", both: true
+    path = File.join(@target_path, "#{file_prefix}_issues/")
+    unless @file_system.dir_exist?(path)
+      log "  Creating path #{path}"
+      @file_system.mkdir(path)
+    end
+
+    filter_id = @board_id_to_filter_id[board.id]
+    jql = make_jql(filter_id: filter_id)
+    intercept_jql = @download_config.project_config.settings['intercept_jql']
+    jql = intercept_jql.call jql if intercept_jql
+
+    issue_data_hash = search_for_issues jql: jql, board_id: board.id, path: path
+
+    checked_for_related = Set.new
+    in_related_phase = false
+
+    loop do
+      related_issue_keys = Set.new
+      stale = issue_data_hash.values.reject { |data| data.up_to_date }
+      unless stale.empty?
+        log_start '  Downloading more issues ' unless in_related_phase
+        stale.each_slice(100) do |slice|
+          slice = bulk_fetch_issues(issue_datas: slice, board: board, in_initial_query: true)
+          progress_dot
+          slice.each do |data|
+            next unless data.issue
+
+            @file_system.save_json(
+              json: data.issue.raw, filename: data.cache_path
+            )
+            # Set the timestamp on the file to match the updated one so that we don't have
+            # to parse the file just to find the timestamp
+            @file_system.utime time: data.issue.updated, file: data.cache_path
+
+            collect_related_issue_keys issue: data.issue, related_issue_keys: related_issue_keys
+            checked_for_related << data.key
+          end
+        end
+        end_progress unless in_related_phase
+      end
+
+      # Also scan up-to-date cached issues we haven't checked yet — they may reference
+      # related issues that are not in the primary query result.
+      issue_data_hash.each_value do |data|
+        next if checked_for_related.include?(data.key)
+        next unless @file_system.file_exist?(data.cache_path)
+
+        checked_for_related << data.key
+        raw = @file_system.load_json(data.cache_path)
+        collect_related_issue_keys issue: Issue.new(raw: raw, board: board), related_issue_keys: related_issue_keys
+      end
+
+      # Remove all the ones we already have
+      related_issue_keys.reject! { |key| issue_data_hash[key] }
+
+      related_issue_keys.each do |key|
+        data = DownloadIssueData.new key: key
+        data.found_in_primary_query = false
+        data.up_to_date = false
+        data.cache_path = File.join(path, "#{key}-#{board.id}.json")
+        issue_data_hash[key] = data
+      end
+      break if related_issue_keys.empty?
+
+      unless in_related_phase
+        in_related_phase = true
+        log "  Identifying related issues (parents, subtasks, links) for board #{board.id}", both: true
+        log_start '  Downloading more issues '
+      end
+    end
+
+    end_progress if in_related_phase
+
+    delete_issues_from_cache_that_are_not_in_server(
+      issue_data_hash: issue_data_hash, path: path
+    )
+  end
+
+  def delete_issues_from_cache_that_are_not_in_server issue_data_hash:, path:
+    # The gotcha with deleted issues is that they just stop being returned in queries
+    # and we have no way to know that they should be removed from our local cache.
+    # With the new approach, we ask for every issue that Jira knows about (within
+    # the parameters of the query) and then delete anything that's in our local cache
+    # but wasn't returned.
+    @file_system.foreach path do |file|
+      next if file.start_with? '.'
+      unless /^(?<key>\w+-\d+)-\d+\.json$/ =~ file
+        raise "Unexpected filename in #{path}: #{file}"
+      end
+      next if issue_data_hash[key] # Still in Jira
+
+      file_to_delete = File.join(path, file)
+      log "  Removing #{file_to_delete} from local cache"
+      file_system.unlink file_to_delete
+    end
+  end
+
+  def collect_related_issue_keys issue:, related_issue_keys:
+    parent_key = issue.parent_key(project_config: @download_config.project_config)
+    related_issue_keys << parent_key if parent_key
+
+    issue.raw['fields']['subtasks']&.each do |raw_subtask|
+      related_issue_keys << raw_subtask['key']
+    end
+
+    issue.raw['fields']['issuelinks']&.each do |link|
+      next if link['type']['name'] == 'Cloners'
+
+      linked = link['inwardIssue'] || link['outwardIssue']
+      related_issue_keys << linked['key'] if linked
+    end
+  end
+
+  def last_modified filename:
+    File.mtime(filename) if File.exist?(filename)
+  end
+end