jirametrics 2.16 → 2.17

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: ba8fee0ad6769e79ce646489af04aa0a6c5b551d5210b670aa96a4d10de45511
-  data.tar.gz: 96d90aeb54d71daf99bfed389bd44c7783d53ddc03ffa707067ff9141d2df867
+  metadata.gz: fe52b964cb227f3dac580d4648a325d61fed7c342a7dd7d7bcb62dddd571af61
+  data.tar.gz: 59eec793880fafb6c5e53b7daf87feb78b53b7624390adfa86ed6ca0be8125f6
 SHA512:
-  metadata.gz: 681b65b97aa5d4431e764d03aae7a1e1bf8eacadea9e2aea3bb694ba45be6e2edf1fe9c5ded299fad874e721eeda4ce1dc7d7a8c0d03a0bb18eb9174a05730da
-  data.tar.gz: 0c3550f70d8f164c1a16b4cf91e102c6a4eae403c376629fcdd4cf685f80a6a0d517151280a22e08665e64bca9679af25f25684ef22d4edd4bc7b2ac83e00bca
+  metadata.gz: a98b06c724a2fe57511382a952093d73a7dad414c493c5298c8d35c8e889117252a2a3a6d69eadd274ca6dd90ca03cf0231e23da7b18041752d60bbad9b2fc82
+  data.tar.gz: 90fc55a7e86358636797f00e6cd4fa4eb5268b3a993ed48ff3bc8aecd0e3ae7629d721a814f5e520b58feed72075858613d11faccdea1a0548ac4a7c19a31479

data/lib/jirametrics/daily_view.rb

@@ -181,7 +181,7 @@ class DailyView < ChartBase
     table = +''
     table << '<table>'
     history.each do |c|
-      time = c.time.strftime '%b %d, %I:%M%P'
+      time = c.time.strftime '%b %d, %Y @ %I:%M%P'
 
       table << '<tr>'
       table << "<td><span class='time' title='Timestamp: #{c.time}'>#{time}</span></td>"

data/lib/jirametrics/downloader.rb

@@ -3,6 +3,27 @@
 require 'cgi'
 require 'json'
 
+class DownloadIssueData
+  attr_accessor :key, :found_in_primary_query, :last_modified,
+    :up_to_date, :cache_path, :issue
+
+  def initialize(
+    key:,
+    found_in_primary_query: true,
+    last_modified: nil,
+    up_to_date: true,
+    cache_path: nil,
+    issue: nil
+  )
+    @key = key
+    @found_in_primary_query = found_in_primary_query
+    @last_modified = last_modified
+    @up_to_date = up_to_date
+    @cache_path = cache_path
+    @issue = issue
+  end
+end
+
 class Downloader
   CURRENT_METADATA_VERSION = 4
 
@@ -12,6 +33,15 @@ class Downloader
   # For testing only
   attr_reader :start_date_in_query, :board_id_to_filter_id
 
+  def self.create download_config:, file_system:, jira_gateway:
+    is_cloud = jira_gateway.settings['jira_cloud'] || jira_gateway.cloud?
+    (is_cloud ? DownloaderForCloud : DownloaderForDataCenter).new(
+      download_config: download_config,
+      file_system: file_system,
+      jira_gateway: jira_gateway
+    )
+  end
+
   def initialize download_config:, file_system:, jira_gateway:
     @metadata = {}
     @download_config = download_config
@@ -28,7 +58,6 @@ class Downloader
     log '', both: true
     log @download_config.project_config.name, both: true
 
-    init_gateway
     load_metadata
 
     if @metadata['no-download']
@@ -50,11 +79,6 @@ class Downloader
     save_metadata
   end
 
-  def init_gateway
-    @jira_gateway.load_jira_config(@download_config.project_config.jira_config)
-    @jira_gateway.ignore_ssl_errors = @download_config.project_config.settings['ignore_ssl_errors']
-  end
-
   def log text, both: false
     @file_system.log text, also_write_to_stderr: both
   end
@@ -66,93 +90,6 @@ class Downloader
     ids
   end
 
-  def download_issues board:
-    log "  Downloading primary issues for board #{board.id}", both: true
-    path = File.join(@target_path, "#{file_prefix}_issues/")
-    unless Dir.exist?(path)
-      log "  Creating path #{path}"
-      Dir.mkdir(path)
-    end
-
-    filter_id = @board_id_to_filter_id[board.id]
-    jql = make_jql(filter_id: filter_id)
-    jira_search_by_jql(jql: jql, initial_query: true, board: board, path: path)
-
-    log "  Downloading linked issues for board #{board.id}", both: true
-    loop do
-      @issue_keys_pending_download.reject! { |key| @issue_keys_downloaded_in_current_run.include? key }
-      break if @issue_keys_pending_download.empty?
-
-      keys_to_request = @issue_keys_pending_download[0..99]
-      @issue_keys_pending_download.reject! { |key| keys_to_request.include? key }
-      jql = "key in (#{keys_to_request.join(', ')})"
-      jira_search_by_jql(jql: jql, initial_query: false, board: board, path: path)
-    end
-  end
-
-  def jira_search_by_jql jql:, initial_query:, board:, path:
-    intercept_jql = @download_config.project_config.settings['intercept_jql']
-    jql = intercept_jql.call jql if intercept_jql
-
-    log "  JQL: #{jql}"
-    escaped_jql = CGI.escape jql
-
-    if @jira_gateway.cloud?
-      max_results = 5_000 # The maximum allowed by Jira
-      next_page_token = nil
-      issue_count = 0
-
-      loop do
-        json = @jira_gateway.call_url relative_url: '/rest/api/3/search/jql' \
-          "?jql=#{escaped_jql}&maxResults=#{max_results}&" \
-          "nextPageToken=#{next_page_token}&expand=changelog&fields=*all"
-        next_page_token = json['nextPageToken']
-
-        json['issues'].each do |issue_json|
-          issue_json['exporter'] = {
-            'in_initial_query' => initial_query
-          }
-          identify_other_issues_to_be_downloaded raw_issue: issue_json, board: board
-          file = "#{issue_json['key']}-#{board.id}.json"
-
-          @file_system.save_json(json: issue_json, filename: File.join(path, file))
-          issue_count += 1
-        end
-
-        message = "    Downloaded #{issue_count} issues"
-        log message, both: true
-
-        break unless next_page_token
-      end
-    else
-      max_results = 100
-      start_at = 0
-      total = 1
-      while start_at < total
-        json = @jira_gateway.call_url relative_url: '/rest/api/2/search' \
-          "?jql=#{escaped_jql}&maxResults=#{max_results}&startAt=#{start_at}&expand=changelog&fields=*all"
-
-        json['issues'].each do |issue_json|
-          issue_json['exporter'] = {
-            'in_initial_query' => initial_query
-          }
-          identify_other_issues_to_be_downloaded raw_issue: issue_json, board: board
-          file = "#{issue_json['key']}-#{board.id}.json"
-
-          @file_system.save_json(json: issue_json, filename: File.join(path, file))
-        end
-
-        total = json['total'].to_i
-        max_results = json['maxResults']
-
-        message = "    Downloaded #{start_at + 1}-#{[start_at + max_results, total].min} of #{total} issues to #{path} "
-        log message, both: true
-
-        start_at += json['issues'].size
-      end
-    end
-  end
-
   def identify_other_issues_to_be_downloaded raw_issue:, board:
     issue = Issue.new raw: raw_issue, board: board
     @issue_keys_downloaded_in_current_run << issue.key
@@ -327,11 +264,7 @@ class Downloader
 
     if start_date
       @download_date_range = start_date..today.to_date
-
-      # For an incremental download, we want to query from the end of the previous one, not from the
-      # beginning of the full range.
-      @start_date_in_query = metadata['date_end'] || @download_date_range.begin
-      log "    Incremental download only. Pulling from #{@start_date_in_query}", both: true if metadata['date_end']
+      @start_date_in_query = @download_date_range.begin
 
       # Catch-all to pick up anything that's been around since before the range started but hasn't
       # had an update during the range.
@@ -351,4 +284,118 @@ class Downloader
   def file_prefix
     @download_config.project_config.get_file_prefix
   end
+
+  def download_issues board:
+    log "  Downloading primary issues for board #{board.id} from #{jira_instance_type}", both: true
+    path = File.join(@target_path, "#{file_prefix}_issues/")
+    unless @file_system.dir_exist?(path)
+      log "  Creating path #{path}"
+      @file_system.mkdir(path)
+    end
+
+    filter_id = @board_id_to_filter_id[board.id]
+    jql = make_jql(filter_id: filter_id)
+    intercept_jql = @download_config.project_config.settings['intercept_jql']
+    jql = intercept_jql.call jql if intercept_jql
+
+    issue_data_hash = search_for_issues jql: jql, board_id: board.id, path: path
+
+    loop do
+      related_issue_keys = Set.new
+      issue_data_hash
+        .values
+        .reject { |data| data.up_to_date }
+        .each_slice(100) do |slice|
+          slice = bulk_fetch_issues(
+            issue_datas: slice, board: board, in_initial_query: true
+          )
+          slice.each do |data|
+            @file_system.save_json(
+              json: data.issue.raw, filename: data.cache_path
+            )
+            # Set the timestamp on the file to match the updated one so that we don't have
+            # to parse the file just to find the timestamp
+            @file_system.utime time: data.issue.updated, file: data.cache_path
+
+            issue = data.issue
+            next unless issue
+
+            parent_key = issue.parent_key(project_config: @download_config.project_config)
+            related_issue_keys << parent_key if parent_key
+
+            # Sub-tasks
+            issue.raw['fields']['subtasks']&.each do |raw_subtask|
+              related_issue_keys << raw_subtask['key']
+            end
+          end
+        end
+
+      # Remove all the ones we already downloaded
+      related_issue_keys.reject! { |key| issue_data_hash[key] }
+
+      related_issue_keys.each do |key|
+        data = DownloadIssueData.new key: key
+        data.found_in_primary_query = false
+        data.up_to_date = false
+        data.cache_path = File.join(path, "#{key}-#{board.id}.json")
+        issue_data_hash[key] = data
+      end
+      break if related_issue_keys.empty?
+
+      log "  Downloading linked issues for board #{board.id}", both: true
+    end
+
+    delete_issues_from_cache_that_are_not_in_server(
+      issue_data_hash: issue_data_hash, path: path
+    )
+  end
+
+  def bulk_fetch_issues issue_datas:, board:, in_initial_query:
+    log "  Downloading #{issue_datas.size} issues", both: true
+    payload = {
+      'expand' => [
+        'changelog'
+      ],
+      'fields' => ['*all'],
+      'issueIdsOrKeys' => issue_datas.collect(&:key)
+    }
+    response = @jira_gateway.post_request(
+      relative_url: issue_bulk_fetch_api,
+      payload: JSON.generate(payload)
+    )
+    response['issues'].each do |issue_json|
+      issue_json['exporter'] = {
+        'in_initial_query' => in_initial_query
+      }
+      issue = Issue.new(raw: issue_json, board: board)
+      data = issue_datas.find { |d| d.key == issue.key }
+      data.up_to_date = true
+      data.last_modified = issue.updated
+      data.issue = issue
+    end
+    issue_datas
+  end
+
+  def delete_issues_from_cache_that_are_not_in_server issue_data_hash:, path:
+    # The gotcha with deleted issues is that they just stop being returned in queries
+    # and we have no way to know that they should be removed from our local cache.
+    # With the new approach, we ask for every issue that Jira knows about (within
+    # the parameters of the query) and then delete anything that's in our local cache
+    # but wasn't returned.
+    @file_system.foreach path do |file|
+      next if file.start_with? '.'
+      unless /^(?<key>\w+-\d+)-\d+\.json$/ =~ file
+        raise "Unexpected filename in #{path}: #{file}"
+      end
+      next if issue_data_hash[key] # Still in Jira
+
+      file_to_delete = File.join(path, file)
+      log "  Removing #{file_to_delete} from local cache"
+      file_system.unlink file_to_delete
+    end
+  end
+
+  def last_modified filename:
+    File.mtime(filename) if File.exist?(filename)
+  end
 end

data/lib/jirametrics/downloader_for_cloud.rb

@@ -0,0 +1,50 @@
+# frozen_string_literal: true
+
+class DownloaderForCloud < Downloader
+  def jira_instance_type
+    'Jira Cloud'
+  end
+
+  def search_for_issues jql:, board_id:, path:
+    log "  JQL: #{jql}"
+    escaped_jql = CGI.escape jql
+
+    hash = {}
+    max_results = 5_000 # The maximum allowed by Jira
+    next_page_token = nil
+    issue_count = 0
+
+    loop do
+      relative_url = +''
+      relative_url << '/rest/api/3/search/jql'
+      relative_url << "?jql=#{escaped_jql}&maxResults=#{max_results}"
+      relative_url << "&nextPageToken=#{next_page_token}" if next_page_token
+      relative_url << '&fields=updated'
+
+      json = @jira_gateway.call_url relative_url: relative_url
+      next_page_token = json['nextPageToken']
+
+      json['issues'].each do |i|
+        key = i['key']
+        data = DownloadIssueData.new key: key
+        data.key = key
+        data.last_modified = Time.parse i['fields']['updated']
+        data.found_in_primary_query = true
+        data.cache_path = File.join(path, "#{key}-#{board_id}.json")
+        data.up_to_date = last_modified(filename: data.cache_path) == data.last_modified
+        hash[key] = data
+        issue_count += 1
+      end
+
+      message = "    Found #{issue_count} issues"
+      log message, both: true
+
+      break unless next_page_token
+    end
+    hash
+  end
+
+  def issue_bulk_fetch_api
+    '/rest/api/3/issue/bulkfetch'
+  end
+end

data/lib/jirametrics/downloader_for_data_center.rb

@@ -0,0 +1,46 @@
+# frozen_string_literal: true
+
+class DownloaderForDataCenter < Downloader
+  def jira_instance_type
+    'Jira DataCenter'
+  end
+
+  def search_for_issues jql:, board_id:, path:
+    log "  JQL: #{jql}"
+    escaped_jql = CGI.escape jql
+
+    hash = {}
+    max_results = 100
+    start_at = 0
+    total = 1
+    while start_at < total
+      json = @jira_gateway.call_url relative_url: '/rest/api/2/search' \
+        "?jql=#{escaped_jql}&maxResults=#{max_results}&startAt=#{start_at}&fields=updated"
+      json['issues'].each do |i|
+        key = i['key']
+        cache_path = File.join(path, "#{key}-#{board_id}.json")
+        last_modified = Time.parse(i['fields']['updated'])
+        data = DownloadIssueData.new(
+          key: key,
+          last_modified: last_modified,
+          found_in_primary_query: true,
+          cache_path: cache_path,
+          up_to_date: last_modified(filename: cache_path) == last_modified
+        )
+        hash[key] = data
+      end
+      total = json['total'].to_i
+      max_results = json['maxResults']
+
+      message = "    Found #{json['issues'].count} issues"
+      log message, both: true
+
+      start_at += json['issues'].size
+    end
+    hash
+  end
+
+  def issue_bulk_fetch_api
+    '/rest/api/2/issue/bulkfetch'
+  end
+end

data/lib/jirametrics/exporter.rb

@@ -50,10 +50,15 @@ class Exporter
       end
 
       project.download_config.run
-      downloader = Downloader.new(
+    # load_jira_config(download_config.project_config.jira_config)
+    # @ignore_ssl_errors = download_config.project_config.settings['ignore_ssl_errors']
+      gateway = JiraGateway.new(
+        file_system: file_system, jira_config: project.jira_config, settings: project.settings
+      )
+      downloader = Downloader.create(
         download_config: project.download_config,
         file_system: file_system,
-        jira_gateway: JiraGateway.new(file_system: file_system)
+        jira_gateway: gateway
       )
       downloader.run
     end

data/lib/jirametrics/file_system.rb

@@ -5,6 +5,13 @@ require 'json'
 class FileSystem
   attr_accessor :logfile, :logfile_name
 
+  def initialize
+    # In almost all cases, this will be immediately replaced in the Exporter
+    # but if we fail before we get that far, this will at least let a useful
+    # error show up on the console.
+    @logfile = $stdout
+  end
+
   # Effectively the same as File.read except it forces the encoding to UTF-8
   def load filename, supress_deprecation: false
     if filename.end_with?('.json') && !supress_deprecation
@@ -31,6 +38,14 @@ class FileSystem
     File.write(filename, content)
   end
 
+  def mkdir path
+    FileUtils.mkdir_p path
+  end
+
+  def utime file:, time:
+    File.utime time, time, file
+  end
+
   def warning message, more: nil
     log "Warning: #{message}", more: more, also_write_to_stderr: true
   end
@@ -66,7 +81,15 @@ class FileSystem
   end
 
   def file_exist? filename
-    File.exist? filename
+    File.exist?(filename) && File.file?(filename)
+  end
+
+  def dir_exist? path
+    File.exist?(path) && File.directory?(path)
+  end
+
+  def unlink filename
+    File.unlink filename
   end
 
   def deprecated message:, date:, depth: 2

data/lib/jirametrics/jira_gateway.rb

@@ -3,17 +3,39 @@
 require 'cgi'
 require 'json'
 require 'English'
+require 'open3'
 
 class JiraGateway
-  attr_accessor :ignore_ssl_errors, :jira_url
+  attr_accessor :ignore_ssl_errors
+  attr_reader :jira_url, :settings
 
-  def initialize file_system:
+  def initialize file_system:, jira_config:, settings:
     @file_system = file_system
+    load_jira_config(jira_config)
+    @settings = settings
+    @ignore_ssl_errors = settings['ignore_ssl_errors']
+  end
+
+  def post_request relative_url:, payload:
+    command = make_curl_command url: "#{@jira_url}#{relative_url}", method: 'POST'
+    stdout, stderr, status = Open3.capture3(command, stdin_data: payload)
+    @file_system.log "Error: #{stderr}" unless stderr == ''
+    raise 'no response' if stdout == ''
+    return parse_response(command: command, result: stdout) if status.success?
+
+    @file_system.log result
+    @file_system.log "Failed call with exit status #{status.exitstatus}."
+    raise "Failed call with exit status #{status.exitstatus}. " \
+      "See #{@file_system.logfile_name} for details"
   end
 
   def call_url relative_url:
     command = make_curl_command url: "#{@jira_url}#{relative_url}"
     result = call_command command
+    parse_response(command: command, result: result)
+  end
+
+  def parse_response command:, result:
     begin
       json = JSON.parse(result)
     rescue # rubocop:disable Style/RescueStandardError
@@ -65,7 +87,7 @@ class JiraGateway
     @cookies = (jira_config['cookies'] || []).collect { |key, value| "#{key}=#{value}" }.join(';')
   end
 
-  def make_curl_command url:
+  def make_curl_command url:, method: 'GET'
     command = +''
     command << 'curl'
     command << ' -L' # follow redirects
@@ -74,8 +96,13 @@ class JiraGateway
     command << " --cookie #{@cookies.inspect}" unless @cookies.empty?
     command << " --user #{@jira_email}:#{@jira_api_token}" if @jira_api_token
     command << " -H \"Authorization: Bearer #{@jira_personal_access_token}\"" if @jira_personal_access_token
-    command << ' --request GET'
+    command << " --request #{method}"
+    if method == 'POST'
+      command << ' --data @-'
+      command << ' --header "Content-Type: application/json"'
+    end
     command << ' --header "Accept: application/json"'
+    command << ' --show-error --fail' # Better diagnostics when the server returns an error
     command << " --url \"#{url}\""
     command
   end

metadata

@@ -1,13 +1,13 @@
 --- !ruby/object:Gem::Specification
 name: jirametrics
 version: !ruby/object:Gem::Version
-  version: '2.16'
+  version: '2.17'
 platform: ruby
 authors:
 - Mike Bowler
 bindir: bin
 cert_chain: []
-date: 2025-09-10 00:00:00.000000000 Z
+date: 2025-09-20 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: random-word
@@ -87,6 +87,8 @@ files:
 - lib/jirametrics/dependency_chart.rb
 - lib/jirametrics/download_config.rb
 - lib/jirametrics/downloader.rb
+- lib/jirametrics/downloader_for_cloud.rb
+- lib/jirametrics/downloader_for_data_center.rb
 - lib/jirametrics/estimate_accuracy_chart.rb
 - lib/jirametrics/estimation_configuration.rb
 - lib/jirametrics/examples/aggregated_project.rb