jirametrics 2.5 → 2.30

data/lib/jirametrics/download_config.rb

@@ -20,15 +20,30 @@ class DownloadConfig
     @rolling_date_count
   end
 
-  def no_earlier_than date = nil
-    @no_earlier_than = Date.parse(date) unless date.nil?
+  def no_earlier_than date = :not_set
+    @no_earlier_than = Date.parse(date) unless date == :not_set
     @no_earlier_than
   end
 
+  def github_repos
+    @github_repos ||= []
+  end
+
+  def github_repo *repos
+    github_repos.concat(repos.map { |r| normalize_github_repo(r) })
+  end
+
   def start_date today:
     date = today.to_date - @rolling_date_count if @rolling_date_count
     date = [date, @no_earlier_than].max if date && @no_earlier_than
     date = @no_earlier_than if date.nil? && @no_earlier_than
     date
   end
+
+  private
+
+  def normalize_github_repo repo
+    match = repo.match(%r{github\.com/([^/]+/[^/]+?)/?$})
+    match ? match[1] : repo
+  end
 end

data/lib/jirametrics/downloader.rb

@@ -3,21 +3,53 @@
 require 'cgi'
 require 'json'
 
+class DownloadIssueData
+  attr_accessor :key, :found_in_primary_query, :last_modified,
+    :up_to_date, :cache_path, :issue
+
+  def initialize(
+    key:,
+    found_in_primary_query: true,
+    last_modified: nil,
+    up_to_date: true,
+    cache_path: nil,
+    issue: nil
+  )
+    @key = key
+    @found_in_primary_query = found_in_primary_query
+    @last_modified = last_modified
+    @up_to_date = up_to_date
+    @cache_path = cache_path
+    @issue = issue
+  end
+end
+
 class Downloader
-  CURRENT_METADATA_VERSION = 4
+  CURRENT_METADATA_VERSION = 5
 
-  attr_accessor :metadata, :quiet_mode
+  attr_accessor :metadata
   attr_reader :file_system
 
   # For testing only
   attr_reader :start_date_in_query, :board_id_to_filter_id
 
-  def initialize download_config:, file_system:, jira_gateway:
+  def self.create download_config:, file_system:, jira_gateway:, github_pr_cache: {}
+    is_cloud = jira_gateway.settings['jira_cloud'] || jira_gateway.cloud?
+    (is_cloud ? DownloaderForCloud : DownloaderForDataCenter).new(
+      download_config: download_config,
+      file_system: file_system,
+      jira_gateway: jira_gateway,
+      github_pr_cache: github_pr_cache
+    )
+  end
+
+  def initialize download_config:, file_system:, jira_gateway:, github_pr_cache: {}
     @metadata = {}
     @download_config = download_config
     @target_path = @download_config.project_config.target_path
     @file_system = file_system
     @jira_gateway = jira_gateway
+    @github_pr_cache = github_pr_cache
     @board_id_to_filter_id = {}
 
     @issue_keys_downloaded_in_current_run = []
@@ -28,7 +60,6 @@ class Downloader
     log '', both: true
     log @download_config.project_config.name, both: true
 
-    init_gateway
     load_metadata
 
     if @metadata['no-download']
@@ -39,94 +70,49 @@ class Downloader
     # board_ids = @download_config.board_ids
 
     remove_old_files
+    update_status_history_file
     download_statuses
     find_board_ids.each do |id|
-      download_board_configuration board_id: id
-      download_issues board_id: id
+      board = download_board_configuration board_id: id
+      board.project_config = @download_config.project_config
+      download_issues board: board
     end
+    download_users
 
     save_metadata
-  end
-
-  def init_gateway
-    @jira_gateway.load_jira_config(@download_config.project_config.jira_config)
-    @jira_gateway.ignore_ssl_errors = @download_config.project_config.settings['ignore_ssl_errors']
+    download_github_prs if @download_config.github_repos.any?
   end
 
   def log text, both: false
-    @file_system.log text
-    puts text if both && !@quiet_mode
+    @file_system.log text, also_write_to_stderr: both
   end
 
-  def find_board_ids
-    ids = @download_config.project_config.board_configs.collect(&:id)
-    raise 'Board ids must be specified' if ids.empty?
-
-    ids
+  def log_start text
+    @file_system.log_start text
   end
 
-  def download_issues board_id:
-    log "  Downloading primary issues for board #{board_id}", both: true
-    path = "#{@target_path}#{@download_config.project_config.file_prefix}_issues/"
-    unless Dir.exist?(path)
-      log "  Creating path #{path}"
-      Dir.mkdir(path)
-    end
-
-    filter_id = @board_id_to_filter_id[board_id]
-    jql = make_jql(filter_id: filter_id)
-    jira_search_by_jql(jql: jql, initial_query: true, board_id: board_id, path: path)
-
-    log "  Downloading linked issues for board #{board_id}", both: true
-    loop do
-      @issue_keys_pending_download.reject! { |key| @issue_keys_downloaded_in_current_run.include? key }
-      break if @issue_keys_pending_download.empty?
-
-      keys_to_request = @issue_keys_pending_download[0..99]
-      @issue_keys_pending_download.reject! { |key| keys_to_request.include? key }
-      jql = "key in (#{keys_to_request.join(', ')})"
-      jira_search_by_jql(jql: jql, initial_query: false, board_id: board_id, path: path)
-    end
+  def start_progress
+    @file_system.start_progress
   end
 
-  def jira_search_by_jql jql:, initial_query:, board_id:, path:
-    intercept_jql = @download_config.project_config.settings['intercept_jql']
-    jql = intercept_jql.call jql if intercept_jql
-
-    log "  JQL: #{jql}"
-    escaped_jql = CGI.escape jql
-
-    max_results = 100
-    start_at = 0
-    total = 1
-    while start_at < total
-      json = @jira_gateway.call_url relative_url: '/rest/api/2/search' \
-        "?jql=#{escaped_jql}&maxResults=#{max_results}&startAt=#{start_at}&expand=changelog&fields=*all"
-
-      exit_if_call_failed json
-
-      json['issues'].each do |issue_json|
-        issue_json['exporter'] = {
-          'in_initial_query' => initial_query
-        }
-        identify_other_issues_to_be_downloaded issue_json
-        file = "#{issue_json['key']}-#{board_id}.json"
-
-        @file_system.save_json(json: issue_json, filename: File.join(path, file))
-      end
+  def progress_dot message = nil
+    @file_system.log message if message
+    @file_system.progress_dot
+  end
 
-      total = json['total'].to_i
-      max_results = json['maxResults']
+  def end_progress
+    @file_system.end_progress
+  end
 
-      message = "    Downloaded #{start_at + 1}-#{[start_at + max_results, total].min} of #{total} issues to #{path} "
-      log message, both: true
+  def find_board_ids
+    ids = @download_config.project_config.board_configs.collect(&:id)
+    raise 'Board ids must be specified' if ids.empty?
 
-      start_at += json['issues'].size
-    end
+    ids
   end
 
-  def identify_other_issues_to_be_downloaded raw_issue
-    issue = Issue.new raw: raw_issue, board: nil
+  def identify_other_issues_to_be_downloaded raw_issue:, board:
+    issue = Issue.new raw: raw_issue, board: board
     @issue_keys_downloaded_in_current_run << issue.key
 
     # Parent
@@ -137,22 +123,6 @@ class Downloader
     issue.raw['fields']['subtasks']&.each do |raw_subtask|
       @issue_keys_pending_download << raw_subtask['key']
     end
-
-    # Links
-    # We shouldn't blindly follow links as some, like cloners, aren't valuable and are just wasting time/effort
-    # to download
-    # issue.raw['fields']['issuelinks'].each do |raw_link|
-    #   @issue_keys_pending_download << IssueLink(raw: raw_link).other_issue.key
-    # end
-  end
-
-  def exit_if_call_failed json
-    # Sometimes Jira returns the singular form of errorMessage and sometimes the plural. Consistency FTW.
-    return unless json['error'] || json['errorMessages'] || json['errorMessage']
-
-    log "Download failed. See #{@file_system.logfile_name} for details.", both: true
-    log "  #{JSON.pretty_generate(json)}"
-    exit 1
   end
 
   def download_statuses
@@ -161,29 +131,85 @@ class Downloader
 
     @file_system.save_json(
       json: json,
-      filename: "#{@target_path}#{@download_config.project_config.file_prefix}_statuses.json"
+      filename: File.join(@target_path, "#{file_prefix}_statuses.json")
     )
   end
 
+  def download_users
+    return unless @jira_gateway.cloud?
+
+    log '  Downloading all users', both: true
+    json = @jira_gateway.call_url relative_url: '/rest/api/2/users'
+
+    @file_system.save_json(
+      json: json,
+      filename: File.join(@target_path, "#{file_prefix}_users.json")
+    )
+  end
+
+  def update_status_history_file
+    status_filename = File.join(@target_path, "#{file_prefix}_statuses.json")
+    return unless file_system.file_exist? status_filename
+
+    status_json = file_system.load_json(status_filename)
+
+    history_filename = File.join(@target_path, "#{file_prefix}_status_history.json")
+    history_json = file_system.load_json(history_filename) if file_system.file_exist? history_filename
+
+    if history_json
+      file_system.log '  Updating status history file', also_write_to_stderr: true
+    else
+      file_system.log '  Creating status history file', also_write_to_stderr: true
+      history_json = []
+    end
+
+    status_json.each do |status_item|
+      id = status_item['id']
+      history_item = history_json.find { |s| s['id'] == id }
+      history_json.delete(history_item) if history_item
+      history_json << status_item
+    end
+
+    file_system.save_json(filename: history_filename, json: history_json)
+  end
+
   def download_board_configuration board_id:
     log "  Downloading board configuration for board #{board_id}", both: true
     json = @jira_gateway.call_url relative_url: "/rest/agile/1.0/board/#{board_id}/configuration"
 
-    exit_if_call_failed json
-
-    file_prefix = @download_config.project_config.file_prefix
-    @file_system.save_json json: json, filename: "#{@target_path}#{file_prefix}_board_#{board_id}_configuration.json"
+    @file_system.save_json(
+      json: json,
+      filename: File.join(@target_path, "#{file_prefix}_board_#{board_id}_configuration.json")
+    )
 
     # We have a reported bug that blew up on this line. Moved it after the save so we can
     # actually look at the returned json.
     @board_id_to_filter_id[board_id] = json['filter']['id'].to_i
 
+    if json['type'] == 'simple'
+      features_json = download_features board_id: board_id
+      if BoardFeature.from_raw(features_json).any? { |f| f.name == 'jsw.agility.sprints' && f.enabled? }
+        download_sprints board_id: board_id
+      end
+    end
     download_sprints board_id: board_id if json['type'] == 'scrum'
+    # TODO: Should be passing actual statuses, not empty list
+    Board.new raw: json, possible_statuses: StatusCollection.new
+  end
+
+  def download_features board_id:
+    log "  Downloading features for board #{board_id}", both: true
+    json = @jira_gateway.call_url relative_url: "/rest/agile/1.0/board/#{board_id}/features"
+
+    @file_system.save_json(
+      json: json,
+      filename: File.join(@target_path, "#{file_prefix}_board_#{board_id}_features.json")
+    )
+    json
   end
 
   def download_sprints board_id:
     log "  Downloading sprints for board #{board_id}", both: true
-    file_prefix = @download_config.project_config.file_prefix
     max_results = 100
     start_at = 0
     is_last = false
@@ -191,11 +217,10 @@ class Downloader
     while is_last == false
       json = @jira_gateway.call_url relative_url: "/rest/agile/1.0/board/#{board_id}/sprint?" \
         "maxResults=#{max_results}&startAt=#{start_at}"
-      exit_if_call_failed json
 
       @file_system.save_json(
         json: json,
-        filename: "#{@target_path}#{file_prefix}_board_#{board_id}_sprints_#{start_at}.json"
+        filename: File.join(@target_path, "#{file_prefix}_board_#{board_id}_sprints_#{start_at}.json")
       )
       is_last = json['isLast']
       max_results = json['maxResults']
@@ -208,7 +233,7 @@ class Downloader
   end
 
   def metadata_pathname
-    "#{@target_path}#{@download_config.project_config.file_prefix}_meta.json"
+    File.join(@target_path, "#{file_prefix}_meta.json")
   end
 
   def load_metadata
@@ -224,19 +249,29 @@ class Downloader
         value = Date.parse(value) if value.is_a?(String) && value =~ /^\d{4}-\d{2}-\d{2}$/
         @metadata[key] = value
       end
+
     end
 
     # Even if this is the old format, we want to obey this one tag
     @metadata['no-download'] = hash['no-download'] if hash['no-download']
   end
 
+  def timezone_offset
+    @download_config.project_config.exporter.timezone_offset
+  end
+
+  def today_in_project_timezone
+    Time.now.getlocal(timezone_offset).to_date
+  end
+
   def save_metadata
     @metadata['version'] = CURRENT_METADATA_VERSION
+    @metadata['rolling_date_count'] = @download_config.rolling_date_count
     @metadata['date_start_from_last_query'] = @start_date_in_query if @start_date_in_query
 
     if @download_date_range.nil?
       log "Making up a date range in meta since one wasn't specified. You'll want to change that.", both: true
-      today = Date.today
+      today = today_in_project_timezone
       @download_date_range = (today - 7)..today
     end
 
@@ -251,17 +286,17 @@ class Downloader
   end
 
   def remove_old_files
-    file_prefix = @download_config.project_config.file_prefix
     Dir.foreach @target_path do |file|
       next unless file.match?(/^#{file_prefix}_\d+\.json$/)
+      next if file == "#{file_prefix}_status_history.json"
 
-      File.unlink "#{@target_path}#{file}"
+      File.unlink File.join(@target_path, file)
     end
 
     return if @cached_data_format_is_current
 
     # Also throw away all the previously downloaded issues.
-    path = File.join @target_path, "#{file_prefix}_issues"
+    path = File.join(@target_path, "#{file_prefix}_issues")
     return unless File.exist? path
 
     Dir.foreach path do |file|
@@ -271,7 +306,8 @@ class Downloader
     end
   end
 
-  def make_jql filter_id:, today: Date.today
+  def make_jql filter_id:, today: nil
+    today ||= today_in_project_timezone
     segments = []
     segments << "filter=#{filter_id}"
 
@@ -279,11 +315,7 @@ class Downloader
 
     if start_date
       @download_date_range = start_date..today.to_date
-
-      # For an incremental download, we want to query from the end of the previous one, not from the
-      # beginning of the full range.
-      @start_date_in_query = metadata['date_end'] || @download_date_range.begin
-      log "    Incremental download only. Pulling from #{@start_date_in_query}", both: true if metadata['date_end']
+      @start_date_in_query = @download_date_range.begin
 
       # Catch-all to pick up anything that's been around since before the range started but hasn't
       # had an update during the range.
@@ -299,4 +331,41 @@ class Downloader
 
     segments.join ' AND '
   end
+
+  def download_github_prs
+    project_keys = extract_project_keys_from_downloaded_issues
+    if project_keys.empty?
+      log '  No project keys found in downloaded issues, skipping GitHub PR download', both: true
+      return
+    end
+
+    prs = @download_config.github_repos.flat_map do |repo|
+      GithubGateway.new(
+        repo: repo,
+        project_keys: project_keys,
+        file_system: @file_system,
+        raw_pr_cache: @github_pr_cache
+      ).fetch_pull_requests(since: @download_date_range&.begin)
+    end
+
+    @file_system.save_json(
+      json: prs.map(&:raw),
+      filename: File.join(@target_path, "#{file_prefix}_github_prs.json")
+    )
+  end
+
+  def extract_project_keys_from_downloaded_issues
+    path = File.join(@target_path, "#{file_prefix}_issues")
+    return [] unless @file_system.dir_exist?(path)
+
+    keys = []
+    @file_system.foreach(path) do |filename|
+      keys << filename.split('-').first if filename.match?(/^[A-Z][A-Z_0-9]+-\d+-\d+\.json$/)
+    end
+    keys.uniq
+  end
+
+  def file_prefix
+    @download_config.project_config.get_file_prefix
+  end
 end