jirametrics 2.13 → 2.20.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: f219c386b574f433dbaf649b9b7cccc42114401b45ff703a7c8f228c52d815c4
-  data.tar.gz: 68af46ec7256e3b4004c4fd8016d52b17b0fa2bb717e9265b164dc5fa59fca56
+  metadata.gz: cbe1101b082615d38939850c0adc688aedefe02a74537503bcd390cdf11d0d4e
+  data.tar.gz: eeffbda7c7ba8280273e0d749ede1ed3c1caa33d06a1f50a2c47b2331035d5af
 SHA512:
-  metadata.gz: 71d289169562b3e4029549ab0db5ef1c4d25bc933d860f6f2a834f6caa39bbb5de93db2932996b7a5d75b4b6fb16f021ac5666cad3d57092dea6b4db4f8b7e97
-  data.tar.gz: 425c213d81982d99f9d933d7ad8dbeb4f87f92ec97188d763b51edcb2f7b85a1072da80c0a5df8566a6325223e9b35d98b180e18fd67d1e50cb2d44b1b0e3872
+  metadata.gz: b73533c90e457c2c5f7a7f2d1759ccd44d218ebe984052fcb581d8ab88225abf43b5612923217b4fa7467ed11e82ba5b4fe2cc656a324f269b71c2497bf67659
+  data.tar.gz: 57cbc54fe6c739d0c85f68cc0efcfbc6005975af0b40174ed9ee35790a83dac9b5e524601b770dffc6a3fefbd10f0d577d19b840560b3acfb63b0b542728d5fc

data/lib/jirametrics/anonymizer.rb

@@ -2,11 +2,12 @@
 
 require 'random-word'
 
-class Anonymizer
+class Anonymizer < ChartBase
   # needed for testing
   attr_reader :project_config, :issues
 
   def initialize project_config:, date_adjustment: -200
+    super()
     @project_config = project_config
     @issues = @project_config.issues
     @all_boards = @project_config.all_boards
@@ -130,18 +131,19 @@ class Anonymizer
     end
   end
 
-  def shift_all_dates
-    @file_system.log "Shifting all dates by #{@date_adjustment} days"
+  def shift_all_dates date_adjustment: @date_adjustment
+    adjustment_in_seconds = 60 * 60 * 24 * date_adjustment
+    @file_system.log "Shifting all dates by #{label_days date_adjustment}"
     @issues.each do |issue|
       issue.changes.each do |change|
-        change.time = change.time + @date_adjustment
+        change.time = change.time + adjustment_in_seconds
       end
 
-      issue.raw['fields']['updated'] = (issue.updated + @date_adjustment).to_s
+      issue.raw['fields']['updated'] = (issue.updated + adjustment_in_seconds).to_s
     end
 
     range = @project_config.time_range
-    @project_config.time_range = (range.begin + @date_adjustment)..(range.end + @date_adjustment)
+    @project_config.time_range = (range.begin + date_adjustment)..(range.end + date_adjustment)
   end
 
   def random_name

data/lib/jirametrics/atlassian_document_format.rb

@@ -5,6 +5,7 @@ class AtlassianDocumentFormat
 
   def initialize users:, timezone_offset:
     @users = users
+    @timezone_offset = timezone_offset
   end
 
   def to_html input
@@ -12,16 +13,19 @@ class AtlassianDocumentFormat
       input
         .gsub(/{color:(#\w{6})}([^{]+){color}/, '<span style="color: \1">\2</span>') # Colours
         .gsub(/\[~accountid:([^\]]+)\]/) { expand_account_id $1 } # Tagged people
-        .gsub(/\[([^\|]+)\|(https?[^\]]+)\]/, '<a href="\2">\1</a>') # URLs
+        .gsub(/\[([^|]+)\|(https?[^\]]+)\]/, '<a href="\2">\1</a>') # URLs
         .gsub("\n", '<br />')
-    else
+    elsif input&.[]('content')
       input['content'].collect { |element| adf_node_to_html element }.join("\n")
+    else
+      # We have an actual ADF document with no content.
+      ''
     end
   end
 
   # ADF is Atlassian Document Format
   # https://developer.atlassian.com/cloud/jira/platform/apis/document/structure/
-  def adf_node_to_html node
+  def adf_node_to_html node # rubocop:disable Metrics/CyclomaticComplexity
     closing_tag = nil
     node_attrs = node['attrs']
 
@@ -153,4 +157,4 @@ class AtlassianDocumentFormat
     text = "@#{user.display_name}" if user
     "<span class='account_id'>#{text}</span>"
   end
-end
+end

data/lib/jirametrics/board_config.rb

@@ -11,6 +11,7 @@ class BoardConfig
 
   def run
     @board = @project_config.all_boards[id]
+    raise "Can't find board #{id.inspect} in #{@project_config.all_boards.keys.inspect}" unless @board
 
     instance_eval(&@block)
     raise "Must specify a cycletime for board #{@id}" if @board.cycletime.nil?
@@ -23,7 +24,8 @@ class BoardConfig
     end
 
     @board.cycletime = CycleTimeConfig.new(
-      parent_config: self, label: label, block: block, file_system: project_config.file_system
+      parent_config: self, label: label, block: block, file_system: project_config.file_system,
+      settings: project_config.settings
     )
   end
 

data/lib/jirametrics/change_item.rb

@@ -1,8 +1,8 @@
 # frozen_string_literal: true
 
 class ChangeItem
-  attr_reader :field, :value_id, :old_value_id, :raw, :time, :author_raw
-  attr_accessor :value, :old_value
+  attr_reader :field, :value_id, :old_value_id, :raw, :author_raw
+  attr_accessor :value, :old_value, :time
 
   def initialize raw:, author_raw:, time:, artificial: false
     @raw = raw

data/lib/jirametrics/chart_base.rb

@@ -2,7 +2,8 @@
 
 class ChartBase
   attr_accessor :timezone_offset, :board_id, :all_boards, :date_range,
-    :time_range, :data_quality, :holiday_dates, :settings, :issues, :file_system, :users
+    :time_range, :data_quality, :holiday_dates, :settings, :issues, :file_system,
+    :atlassian_document_format
   attr_writer :aggregated_project
   attr_reader :canvas_width, :canvas_height
 
@@ -44,7 +45,7 @@ class ChartBase
 
   def render_top_text caller_binding
     result = +''
-    result << "<h1>#{@header_text}</h1>" if @header_text
+    result << "<h1 class='foldable'>#{@header_text}</h1>" if @header_text
     result << ERB.new(@description_text).result(caller_binding) if @description_text
     result
   end
@@ -66,6 +67,8 @@ class ChartBase
   end
 
   def label_days days
+    return 'unknown' if days.nil?
+
     "#{days} day#{'s' unless days == 1}"
   end
 

data/lib/jirametrics/cycletime_config.rb

@@ -6,12 +6,15 @@ require 'date'
 class CycleTimeConfig
   include SelfOrIssueDispatcher
 
-  attr_reader :label, :parent_config
+  attr_reader :label, :parent_config, :settings, :file_system
+
+  def initialize parent_config:, label:, block:, settings:, file_system: nil, today: Date.today
 
-  def initialize parent_config:, label:, block:, file_system: nil, today: Date.today
     @parent_config = parent_config
     @label = label
     @today = today
+    @settings = settings
+    @cache_cycletime_calculations = settings['cache_cycletime_calculations']
 
     # If we hit something deprecated and this is nil then we'll blow up. Although it's ugly, this
     # may make it easier to find problems in the test code ;-)
@@ -63,6 +66,10 @@ class CycleTimeConfig
   end
 
   def started_stopped_changes issue
+    cache_key = "#{issue.key}:#{issue.board.id}"
+    last_result = (@cache ||= {})[cache_key]
+    return *last_result if last_result && @cache_cycletime_calculations
+
     started = @start_at.call(issue)
     stopped = @stop_at.call(issue)
 
@@ -80,7 +87,15 @@ class CycleTimeConfig
     # for the start and not have it conflict.
     started = nil if started&.time == stopped&.time
 
-    [started, stopped]
+    result = [started, stopped]
+    if last_result && result != last_result
+      @file_system.error(
+        "Calculation mismatch; this could break caching. #{issue.inspect} new=#{result.inspect}, " \
+          "previous=#{last_result.inspect}"
+      )
+    end
+    @cache[cache_key] = result
+    result
   end
 
   def started_stopped_times issue
@@ -88,6 +103,10 @@ class CycleTimeConfig
     [started&.time, stopped&.time]
   end
 
+  def flush_cache
+    @cache = nil
+  end
+
   def started_stopped_dates issue
     started_time, stopped_time = started_stopped_times(issue)
     [started_time&.to_date, stopped_time&.to_date]

data/lib/jirametrics/cycletime_histogram.rb

@@ -62,7 +62,9 @@ class CycletimeHistogram < ChartBase
       )
     end
 
-    return "<h1>#{@header_text}</h1>No data matched the selected criteria. Nothing to show." if data_sets.empty?
+    if data_sets.empty?
+      return "<h1 class='foldable'>#{@header_text}</h1><div>No data matched the selected criteria. Nothing to show.</div>"
+    end
 
     wrap_and_render(binding, __FILE__)
   end

data/lib/jirametrics/daily_view.rb

@@ -23,7 +23,7 @@ class DailyView < ChartBase
   def run
     aging_issues = select_aging_issues
 
-    return "<h1>#{@header_text}</h1>There are no items currently in progress" if aging_issues.empty?
+    return "<h1 class='foldable'>#{@header_text}</h1>There are no items currently in progress" if aging_issues.empty?
 
     result = +''
     result << render_top_text(binding)
@@ -33,10 +33,6 @@ class DailyView < ChartBase
     result
   end
 
-  def atlassian_document_format
-    @atlassian_document_format ||= AtlassianDocumentFormat.new(users: users, timezone_offset: timezone_offset)
-  end
-
   def select_aging_issues
     aging_issues = issues.select do |issue|
       started_at, stopped_at = issue.board.cycletime.started_stopped_times(issue)
@@ -82,7 +78,7 @@ class DailyView < ChartBase
     blocked_stalled = issue.blocked_stalled_by_date(
       date_range: today..today, chart_end_time: time_range.end, settings: settings
     )[today]
-    return [] unless blocked_stalled
+    return [] if blocked_stalled.active?
 
     lines = []
     if blocked_stalled.blocked?
@@ -102,15 +98,18 @@ class DailyView < ChartBase
     lines
   end
 
-  def make_issue_label issue
-    "<img src='#{issue.type_icon_url}' title='#{issue.type}' class='icon' /> " \
-      "<b><a href='#{issue.url}'>#{issue.key}</a></b> &nbsp;<i>#{issue.summary}</i>"
+  def make_issue_label issue:, done:
+    label = "<img src='#{issue.type_icon_url}' title='#{issue.type}' class='icon' /> "
+    label << '<s>' if done
+    label << "<b><a href='#{issue.url}'>#{issue.key}</a></b> &nbsp;<i>#{issue.summary}</i>"
+    label << '</s>' if done
+    label
   end
 
-  def make_title_line issue
+  def make_title_line issue:, done:
     title_line = +''
     title_line << color_block('--expedited-color', title: 'Expedited') if issue.expedited?
-    title_line << make_issue_label(issue)
+    title_line << make_issue_label(issue: issue, done: done)
     title_line
   end
 
@@ -119,20 +118,25 @@ class DailyView < ChartBase
     parent_key = issue.parent_key
     if parent_key
       parent = issues.find_by_key key: parent_key, include_hidden: true
-      text = parent ? make_issue_label(parent) : parent_key
+      text = parent ? make_issue_label(issue: parent, done: parent.done?) : parent_key
       lines << ["Parent: #{text}"]
     end
     lines
   end
 
-  def make_stats_lines issue
+  def make_stats_lines issue:, done:
     line = []
 
     line << "<img src='#{issue.priority_url}' class='icon' /> <b>#{issue.priority_name}</b>"
 
-    age = issue.board.cycletime.age(issue, today: date_range.end)
-    line << "Age: <b>#{age ? label_days(age) : '(Not Started)'}</b>"
+    if done
+      cycletime = issue.board.cycletime.cycletime(issue)
 
+      line << "Cycletime: <b>#{label_days cycletime}</b>"
+    else
+      age = issue.board.cycletime.age(issue, today: date_range.end)
+      line << "Age: <b>#{age ? label_days(age) : '(Not Started)'}</b>"
+    end
     line << "Status: <b>#{format_status issue.status, board: issue.board}</b>"
 
     column = issue.board.visible_columns.find { |c| c.status_ids.include?(issue.status.id) }
@@ -158,13 +162,14 @@ class DailyView < ChartBase
 
   def make_child_lines issue
     lines = []
-    subtasks = issue.subtasks.reject { |i| i.done? }
+    subtasks = issue.subtasks
+
+    return lines if subtasks.empty?
+
+    lines << '<section><div class="foldable">Child issues</div>'
+    lines += subtasks
+    lines << '</section>'
 
-    unless subtasks.empty?
-      icon_urls = subtasks.collect(&:type_icon_url).uniq.collect { |url| "<img src='#{url}' class='icon' />" }
-      lines << (icon_urls << 'Incomplete child issues')
-      lines += subtasks
-    end
     lines
   end
 
@@ -172,16 +177,11 @@ class DailyView < ChartBase
     history = issue.changes.reverse
     lines = []
 
-    id = next_id
-    lines << [
-      "<a href=\"javascript:toggle_visibility('open#{id}', 'close#{id}', 'table#{id}');\">" \
-      "<span id='open#{id}'>▶ Issue History</span>" \
-      "<span id='close#{id}' style='display: none'>▼ Issue History</span></a>"
-    ]
+    lines << '<section><div class="foldable startFolded">Issue history</div>'
     table = +''
-    table << "<table id='table#{id}' style='display: none'>"
+    table << '<table>'
     history.each do |c|
-      time = c.time.strftime '%b %d, %I:%M%P'
+      time = c.time.strftime '%b %d, %Y @ %I:%M%P'
 
       table << '<tr>'
       table << "<td><span class='time' title='Timestamp: #{c.time}'>#{time}</span></td>"
@@ -192,23 +192,24 @@ class DailyView < ChartBase
     end
     table << '</table>'
     lines << [table]
+    lines << '</section>'
     lines
   end
 
   def history_text change:, board:
+    convertor = ->(value, _id) { value.inspect }
+    convertor = ->(_value, id) { format_status(board.possible_statuses.find_by_id(id), board: board) } if change.status?
+
     if change.comment? || change.description?
       atlassian_document_format.to_html(change.value)
-    elsif change.status?
-      convertor = ->(id) { format_status(board.possible_statuses.find_by_id(id), board: board) }
-      to = convertor.call(change.value_id)
+    elsif %w[status priority assignee duedate issuetype].include?(change.field)
+      to = convertor.call(change.value, change.value_id)
       if change.old_value
-        from = convertor.call(change.old_value_id)
+        from = convertor.call(change.old_value, change.old_value_id)
         "Changed from #{from} to #{to}"
       else
         "Set to #{to}"
       end
-    elsif %w[priority assignee duedate issuetype].include?(change.field)
-      "Changed from \"#{change.old_value}\" to \"#{change.value}\""
     elsif change.flagged?
       change.value == '' ? 'Off' : 'On'
     else
@@ -242,15 +243,19 @@ class DailyView < ChartBase
   end
 
   def assemble_issue_lines issue, child:
+    done = issue.done?
+
     lines = []
-    lines << [make_title_line(issue)]
+    lines << [make_title_line(issue: issue, done: done)]
     lines += make_parent_lines(issue) unless child
-    lines += make_stats_lines(issue)
-    lines += make_description_lines(issue)
-    lines += make_sprints_lines(issue)
-    lines += make_blocked_stalled_lines(issue)
-    lines += make_child_lines(issue)
-    lines += make_history_lines(issue)
+    lines += make_stats_lines(issue: issue, done: done)
+    unless done
+      lines += make_description_lines(issue)
+      lines += make_sprints_lines(issue)
+      lines += make_blocked_stalled_lines(issue)
+      lines += make_child_lines(issue)
+      lines += make_history_lines(issue)
+    end
     lines
   end
 
@@ -261,6 +266,8 @@ class DailyView < ChartBase
     assemble_issue_lines(issue, child: child).each do |row|
       if row.is_a? Issue
         result << render_issue(row, child: true)
+      elsif row.is_a?(String)
+        result << row
       else
         result << '<div class="heading">'
         row.each do |chunk|

data/lib/jirametrics/data_quality_report.rb

@@ -410,14 +410,17 @@ class DataQualityReport < ChartBase
   def render_status_not_on_board problems
     <<-HTML
       #{label_issues problems.size} were not visible on the board for some period of time. This may impact
-      timings as the work was likely to have been forgotten if it wasn't visible.
+      timings as the work was likely to have been forgotten if it wasn't visible. What does "not visible"
+      mean in this context? The issue was in a status that is not mapped to any visible column on the board.
+      Look in "unmapped statuses" on your board.
     HTML
   end
 
   def render_created_in_wrong_status problems
     <<-HTML
-      #{label_issues problems.size} were created in a status not designated as Backlog. This will impact
-      the measurement of start times and will therefore impact whether it's shown as in progress or not.
+      #{label_issues problems.size} were created in a status that is not considered to be some varient
+      of To Do. Most likely this means that the issue was created from one of the columns on the board,
+      rather than in the backlog. Why Jira allows this is still a mystery.
     HTML
   end
 

data/lib/jirametrics/dependency_chart.rb

@@ -51,7 +51,10 @@ class DependencyChart < ChartBase
     instance_eval(&@rules_block) if @rules_block
 
     dot_graph = build_dot_graph
-    return "<h1>#{@header_text}</h1>No data matched the selected criteria. Nothing to show." if dot_graph.nil?
+    if dot_graph.nil?
+      return "<h1 class='foldable'>#{@header_text}</h1>" \
+        '<div>No data matched the selected criteria. Nothing to show.</div>'
+    end
 
     svg = execute_graphviz(dot_graph.join("\n"))
     "<h1>#{@header_text}</h1><div>#{@description_text}</div>#{shrink_svg svg}"

data/lib/jirametrics/downloader.rb

@@ -3,8 +3,29 @@
 require 'cgi'
 require 'json'
 
+class DownloadIssueData
+  attr_accessor :key, :found_in_primary_query, :last_modified,
+    :up_to_date, :cache_path, :issue
+
+  def initialize(
+    key:,
+    found_in_primary_query: true,
+    last_modified: nil,
+    up_to_date: true,
+    cache_path: nil,
+    issue: nil
+  )
+    @key = key
+    @found_in_primary_query = found_in_primary_query
+    @last_modified = last_modified
+    @up_to_date = up_to_date
+    @cache_path = cache_path
+    @issue = issue
+  end
+end
+
 class Downloader
-  CURRENT_METADATA_VERSION = 4
+  CURRENT_METADATA_VERSION = 5
 
   attr_accessor :metadata
   attr_reader :file_system
@@ -12,6 +33,15 @@ class Downloader
   # For testing only
   attr_reader :start_date_in_query, :board_id_to_filter_id
 
+  def self.create download_config:, file_system:, jira_gateway:
+    is_cloud = jira_gateway.settings['jira_cloud'] || jira_gateway.cloud?
+    (is_cloud ? DownloaderForCloud : DownloaderForDataCenter).new(
+      download_config: download_config,
+      file_system: file_system,
+      jira_gateway: jira_gateway
+    )
+  end
+
   def initialize download_config:, file_system:, jira_gateway:
     @metadata = {}
     @download_config = download_config
@@ -28,7 +58,6 @@ class Downloader
     log '', both: true
     log @download_config.project_config.name, both: true
 
-    init_gateway
     load_metadata
 
     if @metadata['no-download']
@@ -50,11 +79,6 @@ class Downloader
     save_metadata
   end
 
-  def init_gateway
-    @jira_gateway.load_jira_config(@download_config.project_config.jira_config)
-    @jira_gateway.ignore_ssl_errors = @download_config.project_config.settings['ignore_ssl_errors']
-  end
-
   def log text, both: false
     @file_system.log text, also_write_to_stderr: both
   end
@@ -66,93 +90,6 @@ class Downloader
     ids
   end
 
-  def download_issues board:
-    log "  Downloading primary issues for board #{board.id}", both: true
-    path = File.join(@target_path, "#{file_prefix}_issues/")
-    unless Dir.exist?(path)
-      log "  Creating path #{path}"
-      Dir.mkdir(path)
-    end
-
-    filter_id = @board_id_to_filter_id[board.id]
-    jql = make_jql(filter_id: filter_id)
-    jira_search_by_jql(jql: jql, initial_query: true, board: board, path: path)
-
-    log "  Downloading linked issues for board #{board.id}", both: true
-    loop do
-      @issue_keys_pending_download.reject! { |key| @issue_keys_downloaded_in_current_run.include? key }
-      break if @issue_keys_pending_download.empty?
-
-      keys_to_request = @issue_keys_pending_download[0..99]
-      @issue_keys_pending_download.reject! { |key| keys_to_request.include? key }
-      jql = "key in (#{keys_to_request.join(', ')})"
-      jira_search_by_jql(jql: jql, initial_query: false, board: board, path: path)
-    end
-  end
-
-  def jira_search_by_jql jql:, initial_query:, board:, path:
-    intercept_jql = @download_config.project_config.settings['intercept_jql']
-    jql = intercept_jql.call jql if intercept_jql
-
-    log "  JQL: #{jql}"
-    escaped_jql = CGI.escape jql
-
-    if @jira_gateway.cloud?
-      max_results = 5_000 # The maximum allowed by Jira
-      next_page_token = nil
-      issue_count = 0
-
-      loop do
-        json = @jira_gateway.call_url relative_url: '/rest/api/3/search/jql' \
-          "?jql=#{escaped_jql}&maxResults=#{max_results}&" \
-          "nextPageToken=#{next_page_token}&expand=changelog&fields=*all"
-        next_page_token = json['nextPageToken']
-
-        json['issues'].each do |issue_json|
-          issue_json['exporter'] = {
-            'in_initial_query' => initial_query
-          }
-          identify_other_issues_to_be_downloaded raw_issue: issue_json, board: board
-          file = "#{issue_json['key']}-#{board.id}.json"
-
-          @file_system.save_json(json: issue_json, filename: File.join(path, file))
-          issue_count += 1
-        end
-
-        message = "    Downloaded #{issue_count} issues"
-        log message, both: true
-
-        break unless next_page_token
-      end
-    else
-      max_results = 100
-      start_at = 0
-      total = 1
-      while start_at < total
-        json = @jira_gateway.call_url relative_url: '/rest/api/2/search' \
-          "?jql=#{escaped_jql}&maxResults=#{max_results}&startAt=#{start_at}&expand=changelog&fields=*all"
-
-        json['issues'].each do |issue_json|
-          issue_json['exporter'] = {
-            'in_initial_query' => initial_query
-          }
-          identify_other_issues_to_be_downloaded raw_issue: issue_json, board: board
-          file = "#{issue_json['key']}-#{board.id}.json"
-
-          @file_system.save_json(json: issue_json, filename: File.join(path, file))
-        end
-
-        total = json['total'].to_i
-        max_results = json['maxResults']
-
-        message = "    Downloaded #{start_at + 1}-#{[start_at + max_results, total].min} of #{total} issues to #{path} "
-        log message, both: true
-
-        start_at += json['issues'].size
-      end
-    end
-  end
-
   def identify_other_issues_to_be_downloaded raw_issue:, board:
     issue = Issue.new raw: raw_issue, board: board
     @issue_keys_downloaded_in_current_run << issue.key
@@ -178,6 +115,8 @@ class Downloader
   end
 
   def download_users
+    return unless @jira_gateway.cloud?
+
     log '  Downloading all users', both: true
     json = @jira_gateway.call_url relative_url: '/rest/api/2/users'
 
@@ -327,11 +266,7 @@ class Downloader
 
     if start_date
       @download_date_range = start_date..today.to_date
-
-      # For an incremental download, we want to query from the end of the previous one, not from the
-      # beginning of the full range.
-      @start_date_in_query = metadata['date_end'] || @download_date_range.begin
-      log "    Incremental download only. Pulling from #{@start_date_in_query}", both: true if metadata['date_end']
+      @start_date_in_query = @download_date_range.begin
 
       # Catch-all to pick up anything that's been around since before the range started but hasn't
       # had an update during the range.