jirametrics 2.17.1 → 2.18

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 4e9534fd6ca22944557cfe40c63ebf5c30ff111458ec52dd3e35617246315ce8
-  data.tar.gz: 2feb93d3ae826f133902664751f3cee4e54de3512daaaa882ae78fed47605e65
+  metadata.gz: b543d43a9d49fc8e9da3f0a484e37ca8001c2137357c903276ff0998094ab902
+  data.tar.gz: 21ab2447b75c14616cdabf685dd844bad51ef5f73babe1625b075f487fd97ae2
 SHA512:
-  metadata.gz: 84942ea82c68aa66325299e5db0e5f5a4fe4af95eb909d1fde004435b53898c60ee7dde2251fec0e075f4b2db7ba2f1eade384795370f3b90d7230926e79d2b9
-  data.tar.gz: 9015a7c5ef2b60726dca272a0b5b07de9f62d76e076038e4c8d45cbae13facf37f7e8f45dac82639c72615bbee549de756d3fb4792d21345fb4d3dc3fb17a32e
+  metadata.gz: e042949446b1b6e6419473d75d59116b165c9b2328f3db2a9231ae6a089720dc113b7a6d69a36291fcb26b614a881482fa3a487e339c55ca096cc8e51dc73a22
+  data.tar.gz: ffb695346193cf32062fac1622b7da8afe6263c1804986fe71764b391bc21e8df3522fe801047cf27ce09177dd87230deaa9b1a68003fa708b6ebc2a9f107c43

data/lib/jirametrics/atlassian_document_format.rb

@@ -13,9 +13,9 @@ class AtlassianDocumentFormat
       input
         .gsub(/{color:(#\w{6})}([^{]+){color}/, '<span style="color: \1">\2</span>') # Colours
         .gsub(/\[~accountid:([^\]]+)\]/) { expand_account_id $1 } # Tagged people
-        .gsub(/\[([^\|]+)\|(https?[^\]]+)\]/, '<a href="\2">\1</a>') # URLs
+        .gsub(/\[([^|]+)\|(https?[^\]]+)\]/, '<a href="\2">\1</a>') # URLs
         .gsub("\n", '<br />')
-    elsif input['content']
+    elsif input&.[]('content')
       input['content'].collect { |element| adf_node_to_html element }.join("\n")
     else
       # We have an actual ADF document with no content.

data/lib/jirametrics/downloader.rb

@@ -25,7 +25,7 @@ class DownloadIssueData
 end
 
 class Downloader
-  CURRENT_METADATA_VERSION = 4
+  CURRENT_METADATA_VERSION = 5
 
   attr_accessor :metadata
   attr_reader :file_system
@@ -284,92 +284,4 @@ class Downloader
   def file_prefix
     @download_config.project_config.get_file_prefix
   end
-
-  def download_issues board:
-    log "  Downloading primary issues for board #{board.id} from #{jira_instance_type}", both: true
-    path = File.join(@target_path, "#{file_prefix}_issues/")
-    unless @file_system.dir_exist?(path)
-      log "  Creating path #{path}"
-      @file_system.mkdir(path)
-    end
-
-    filter_id = @board_id_to_filter_id[board.id]
-    jql = make_jql(filter_id: filter_id)
-    intercept_jql = @download_config.project_config.settings['intercept_jql']
-    jql = intercept_jql.call jql if intercept_jql
-
-    issue_data_hash = search_for_issues jql: jql, board_id: board.id, path: path
-
-    loop do
-      related_issue_keys = Set.new
-      issue_data_hash
-        .values
-        .reject { |data| data.up_to_date }
-        .each_slice(100) do |slice|
-          slice = bulk_fetch_issues(
-            issue_datas: slice, board: board, in_initial_query: true
-          )
-          slice.each do |data|
-            @file_system.save_json(
-              json: data.issue.raw, filename: data.cache_path
-            )
-            # Set the timestamp on the file to match the updated one so that we don't have
-            # to parse the file just to find the timestamp
-            @file_system.utime time: data.issue.updated, file: data.cache_path
-
-            issue = data.issue
-            next unless issue
-
-            parent_key = issue.parent_key(project_config: @download_config.project_config)
-            related_issue_keys << parent_key if parent_key
-
-            # Sub-tasks
-            issue.raw['fields']['subtasks']&.each do |raw_subtask|
-              related_issue_keys << raw_subtask['key']
-            end
-          end
-        end
-
-      # Remove all the ones we already downloaded
-      related_issue_keys.reject! { |key| issue_data_hash[key] }
-
-      related_issue_keys.each do |key|
-        data = DownloadIssueData.new key: key
-        data.found_in_primary_query = false
-        data.up_to_date = false
-        data.cache_path = File.join(path, "#{key}-#{board.id}.json")
-        issue_data_hash[key] = data
-      end
-      break if related_issue_keys.empty?
-
-      log "  Downloading linked issues for board #{board.id}", both: true
-    end
-
-    delete_issues_from_cache_that_are_not_in_server(
-      issue_data_hash: issue_data_hash, path: path
-    )
-  end
-
-  def delete_issues_from_cache_that_are_not_in_server issue_data_hash:, path:
-    # The gotcha with deleted issues is that they just stop being returned in queries
-    # and we have no way to know that they should be removed from our local cache.
-    # With the new approach, we ask for every issue that Jira knows about (within
-    # the parameters of the query) and then delete anything that's in our local cache
-    # but wasn't returned.
-    @file_system.foreach path do |file|
-      next if file.start_with? '.'
-      unless /^(?<key>\w+-\d+)-\d+\.json$/ =~ file
-        raise "Unexpected filename in #{path}: #{file}"
-      end
-      next if issue_data_hash[key] # Still in Jira
-
-      file_to_delete = File.join(path, file)
-      log "  Removing #{file_to_delete} from local cache"
-      file_system.unlink file_to_delete
-    end
-  end
-
-  def last_modified filename:
-    File.mtime(filename) if File.exist?(filename)
-  end
 end

data/lib/jirametrics/downloader_for_cloud.rb

@@ -111,4 +111,92 @@ class DownloaderForCloud < Downloader
       break if next_page_token.nil?
     end
   end
+
+  def download_issues board:
+    log "  Downloading primary issues for board #{board.id} from #{jira_instance_type}", both: true
+    path = File.join(@target_path, "#{file_prefix}_issues/")
+    unless @file_system.dir_exist?(path)
+      log "  Creating path #{path}"
+      @file_system.mkdir(path)
+    end
+
+    filter_id = @board_id_to_filter_id[board.id]
+    jql = make_jql(filter_id: filter_id)
+    intercept_jql = @download_config.project_config.settings['intercept_jql']
+    jql = intercept_jql.call jql if intercept_jql
+
+    issue_data_hash = search_for_issues jql: jql, board_id: board.id, path: path
+
+    loop do
+      related_issue_keys = Set.new
+      issue_data_hash
+        .values
+        .reject { |data| data.up_to_date }
+        .each_slice(100) do |slice|
+          slice = bulk_fetch_issues(
+            issue_datas: slice, board: board, in_initial_query: true
+          )
+          slice.each do |data|
+            @file_system.save_json(
+              json: data.issue.raw, filename: data.cache_path
+            )
+            # Set the timestamp on the file to match the updated one so that we don't have
+            # to parse the file just to find the timestamp
+            @file_system.utime time: data.issue.updated, file: data.cache_path
+
+            issue = data.issue
+            next unless issue
+
+            parent_key = issue.parent_key(project_config: @download_config.project_config)
+            related_issue_keys << parent_key if parent_key
+
+            # Sub-tasks
+            issue.raw['fields']['subtasks']&.each do |raw_subtask|
+              related_issue_keys << raw_subtask['key']
+            end
+          end
+        end
+
+      # Remove all the ones we already downloaded
+      related_issue_keys.reject! { |key| issue_data_hash[key] }
+
+      related_issue_keys.each do |key|
+        data = DownloadIssueData.new key: key
+        data.found_in_primary_query = false
+        data.up_to_date = false
+        data.cache_path = File.join(path, "#{key}-#{board.id}.json")
+        issue_data_hash[key] = data
+      end
+      break if related_issue_keys.empty?
+
+      log "  Downloading linked issues for board #{board.id}", both: true
+    end
+
+    delete_issues_from_cache_that_are_not_in_server(
+      issue_data_hash: issue_data_hash, path: path
+    )
+  end
+
+  def delete_issues_from_cache_that_are_not_in_server issue_data_hash:, path:
+    # The gotcha with deleted issues is that they just stop being returned in queries
+    # and we have no way to know that they should be removed from our local cache.
+    # With the new approach, we ask for every issue that Jira knows about (within
+    # the parameters of the query) and then delete anything that's in our local cache
+    # but wasn't returned.
+    @file_system.foreach path do |file|
+      next if file.start_with? '.'
+      unless /^(?<key>\w+-\d+)-\d+\.json$/ =~ file
+        raise "Unexpected filename in #{path}: #{file}"
+      end
+      next if issue_data_hash[key] # Still in Jira
+
+      file_to_delete = File.join(path, file)
+      log "  Removing #{file_to_delete} from local cache"
+      file_system.unlink file_to_delete
+    end
+  end
+
+  def last_modified filename:
+    File.mtime(filename) if File.exist?(filename)
+  end
 end

data/lib/jirametrics/downloader_for_data_center.rb

@@ -5,64 +5,90 @@ class DownloaderForDataCenter < Downloader
     'Jira DataCenter'
   end
 
-  def search_for_issues jql:, board_id:, path:
+  def download_issues board:
+    log "  Downloading primary issues for board #{board.id}", both: true
+    path = File.join(@target_path, "#{file_prefix}_issues/")
+    unless Dir.exist?(path)
+      log "  Creating path #{path}"
+      Dir.mkdir(path)
+    end
+
+    filter_id = board_id_to_filter_id[board.id]
+    jql = make_jql(filter_id: filter_id)
+    jira_search_by_jql(jql: jql, initial_query: true, board: board, path: path)
+
+    log "  Downloading linked issues for board #{board.id}", both: true
+    loop do
+      @issue_keys_pending_download.reject! { |key| @issue_keys_downloaded_in_current_run.include? key }
+      break if @issue_keys_pending_download.empty?
+
+      keys_to_request = @issue_keys_pending_download[0..99]
+      @issue_keys_pending_download.reject! { |key| keys_to_request.include? key }
+      jql = "key in (#{keys_to_request.join(', ')})"
+      jira_search_by_jql(jql: jql, initial_query: false, board: board, path: path)
+    end
+  end
+
+  def jira_search_by_jql jql:, initial_query:, board:, path:
+    intercept_jql = @download_config.project_config.settings['intercept_jql']
+    jql = intercept_jql.call jql if intercept_jql
+
     log "  JQL: #{jql}"
     escaped_jql = CGI.escape jql
 
-    hash = {}
     max_results = 100
     start_at = 0
     total = 1
     while start_at < total
       json = @jira_gateway.call_url relative_url: '/rest/api/2/search' \
-        "?jql=#{escaped_jql}&maxResults=#{max_results}&startAt=#{start_at}&fields=updated"
-      json['issues'].each do |i|
-        key = i['key']
-        cache_path = File.join(path, "#{key}-#{board_id}.json")
-        last_modified = Time.parse(i['fields']['updated'])
-        data = DownloadIssueData.new(
-          key: key,
-          last_modified: last_modified,
-          found_in_primary_query: true,
-          cache_path: cache_path,
-          up_to_date: last_modified(filename: cache_path) == last_modified
-        )
-        hash[key] = data
+        "?jql=#{escaped_jql}&maxResults=#{max_results}&startAt=#{start_at}&expand=changelog&fields=*all"
+
+      json['issues'].each do |issue_json|
+        issue_json['exporter'] = {
+          'in_initial_query' => initial_query
+        }
+        identify_other_issues_to_be_downloaded raw_issue: issue_json, board: board
+        file = "#{issue_json['key']}-#{board.id}.json"
+
+        @file_system.save_json(json: issue_json, filename: File.join(path, file))
       end
+
       total = json['total'].to_i
       max_results = json['maxResults']
 
-      message = "    Found #{json['issues'].count} issues"
+      message = "    Downloaded #{start_at + 1}-#{[start_at + max_results, total].min} of #{total} issues to #{path} "
       log message, both: true
 
       start_at += json['issues'].size
     end
-    hash
   end
 
-  def bulk_fetch_issues issue_datas:, board:, in_initial_query:
-    log "  Downloading #{issue_datas.size} issues", both: true
-    payload = {
-      'expand' => [
-        'changelog'
-      ],
-      'fields' => ['*all'],
-      'issueIdsOrKeys' => issue_datas.collect(&:key)
-    }
-    response = @jira_gateway.post_request(
-      relative_url: '/rest/api/2/issue/bulkfetch',
-      payload: JSON.generate(payload)
-    )
-    response['issues'].each do |issue_json|
-      issue_json['exporter'] = {
-        'in_initial_query' => in_initial_query
-      }
-      issue = Issue.new(raw: issue_json, board: board)
-      data = issue_datas.find { |d| d.key == issue.key }
-      data.up_to_date = true
-      data.last_modified = issue.updated
-      data.issue = issue
+  def make_jql filter_id:, today: Date.today
+    segments = []
+    segments << "filter=#{filter_id}"
+
+    start_date = @download_config.start_date today: today
+
+    if start_date
+      @download_date_range = start_date..today.to_date
+
+      # For an incremental download, we want to query from the end of the previous one, not from the
+      # beginning of the full range.
+      @start_date_in_query = metadata['date_end'] || @download_date_range.begin
+      log "    Incremental download only. Pulling from #{@start_date_in_query}", both: true if metadata['date_end']
+
+      # Catch-all to pick up anything that's been around since before the range started but hasn't
+      # had an update during the range.
+      catch_all = '((status changed OR Sprint is not EMPTY) AND statusCategory != Done)'
+
+      # Pick up any issues that had a status change in the range
+      start_date_text = @start_date_in_query.strftime '%Y-%m-%d'
+      # find_in_range = %((status changed DURING ("#{start_date_text} 00:00","#{end_date_text} 23:59")))
+      find_in_range = %(updated >= "#{start_date_text} 00:00")
+
+      segments << "(#{find_in_range} OR #{catch_all})"
     end
-    issue_datas
+
+    segments.join ' AND '
   end
 end

data/lib/jirametrics/issue.rb

@@ -19,9 +19,10 @@ class Issue
 
     # There are cases where we create an Issue of fragments like linked issues and those won't have
     # changelogs.
-    return unless @raw['changelog']
+    load_history_into_changes if @raw['changelog']
 
-    load_history_into_changes
+    # As above with fragments, there may not be a fields section
+    return unless @raw['fields']
 
     # If this is an older pull of data then comments may not be there.
     load_comments_into_changes if @raw['fields']['comment']
@@ -152,7 +153,7 @@ class Issue
   # Are we currently in this status? If yes, then return the most recent status change.
   def currently_in_status *status_names
     change = most_recent_status_change
-    return false if change.nil?
+    return nil if change.nil?
 
     change if change.current_status_matches(*status_names)
   end
@@ -162,7 +163,7 @@ class Issue
     category_ids = find_status_category_ids_by_names category_names
 
     change = most_recent_status_change
-    return false if change.nil?
+    return nil if change.nil?
 
     status = find_or_create_status id: change.value_id, name: change.value
     change if status && category_ids.include?(status.category.id)
@@ -237,11 +238,11 @@ class Issue
   end
 
   def assigned_to
-    @raw['fields']&.[]('assignee')&.[]('displayName')
+    @raw['fields']['assignee']&.[]('displayName')
   end
 
   def assigned_to_icon_url
-    @raw['fields']&.[]('assignee')&.[]('avatarUrls')&.[]('16x16')
+    @raw['fields']['assignee']&.[]('avatarUrls')&.[]('16x16')
   end
 
   # Many test failures are simply unreadable because the default inspect on this class goes
@@ -763,6 +764,9 @@ class Issue
     first_status = nil
     first_status_id = nil
 
+    # There won't be a created timestamp in cases where this was a linked issue
+    return unless @raw['fields']['created']
+
     created_time = parse_time @raw['fields']['created']
     first_change = @changes.find { |change| change.field == field_name }
     if first_change.nil?

data/lib/jirametrics/jira_gateway.rb

@@ -18,11 +18,15 @@ class JiraGateway
 
   def post_request relative_url:, payload:
     command = make_curl_command url: "#{@jira_url}#{relative_url}", method: 'POST'
+    exec_and_parse_response command: command, stdin_data: payload
+  end
+
+  def exec_and_parse_response command:, stdin_data:
     log_entry = "  #{command.gsub(/\s+/, ' ')}"
     log_entry = sanitize_message log_entry
     @file_system.log log_entry
 
-    stdout, stderr, status = Open3.capture3(command, stdin_data: payload)
+    stdout, stderr, status = capture3(command, stdin_data: stdin_data)
     unless status.success?
       @file_system.log "Failed call with exit status #{status.exitstatus}!"
       @file_system.log "Returned (stdout): #{stdout.inspect}"
@@ -31,16 +35,20 @@ class JiraGateway
         "See #{@file_system.logfile_name} for details"
     end
 
-    @file_system.log "Returned (stderr): #{stderr}" unless stderr == ''
+    @file_system.log "Returned (stderr): #{stderr.inspect}" unless stderr == ''
     raise 'no response from curl on stdout' if stdout == ''
 
     parse_response(command: command, result: stdout)
   end
 
+  def capture3 command, stdin_data:
+    # In it's own method so we can mock it out in tests
+    Open3.capture3(command, stdin_data: stdin_data)
+  end
+
   def call_url relative_url:
     command = make_curl_command url: "#{@jira_url}#{relative_url}"
-    result = call_command command
-    parse_response(command: command, result: result)
+    exec_and_parse_response command: command, stdin_data: nil
   end
 
   def parse_response command:, result:
@@ -64,20 +72,6 @@ class JiraGateway
     message.gsub(token, '[API_TOKEN]')
   end
 
-  def call_command command
-    log_entry = "  #{command.gsub(/\s+/, ' ')}"
-    log_entry = sanitize_message log_entry
-    @file_system.log log_entry
-
-    result = `#{command}`
-    @file_system.log result unless $CHILD_STATUS.success?
-    return result if $CHILD_STATUS.success?
-
-    @file_system.log "Failed call with exit status #{$CHILD_STATUS.exitstatus}."
-    raise "Failed call with exit status #{$CHILD_STATUS.exitstatus}. " \
-      "See #{@file_system.logfile_name} for details"
-  end
-
   def load_jira_config jira_config
     @jira_url = jira_config['url']
     raise 'Must specify URL in config' if @jira_url.nil?

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: jirametrics
 version: !ruby/object:Gem::Version
-  version: 2.17.1
+  version: '2.18'
 platform: ruby
 authors:
 - Mike Bowler