jirametrics 2.4 → 2.11

data/lib/jirametrics/html_report_config.rb

@@ -5,16 +5,15 @@ require 'jirametrics/self_or_issue_dispatcher'
 
 class HtmlReportConfig
   include SelfOrIssueDispatcher
-  include DiscardChangesBefore
 
-  attr_reader :file_config, :sections
+  attr_reader :file_config, :sections, :charts
 
   def self.define_chart name:, classname:, deprecated_warning: nil, deprecated_date: nil
     lines = []
     lines << "def #{name} &block"
     lines << '  block = ->(_) {} unless block'
     if deprecated_warning
-      lines << "  deprecated date: #{deprecated_date.inspect}, message: #{deprecated_warning.inspect}"
+      lines << "  file_system.deprecated date: #{deprecated_date.inspect}, message: #{deprecated_warning.inspect}"
     end
     lines << "  execute_chart #{classname}.new(block)"
     lines << 'end'
@@ -33,6 +32,7 @@ class HtmlReportConfig
   define_chart name: 'cycletime_histogram', classname: 'CycletimeHistogram'
   define_chart name: 'estimate_accuracy_chart', classname: 'EstimateAccuracyChart'
   define_chart name: 'hierarchy_table', classname: 'HierarchyTable'
+  define_chart name: 'flow_efficiency_scatterplot', classname: 'FlowEfficiencyScatterplot'
 
   define_chart name: 'daily_wip_by_type', classname: 'DailyWipChart',
     deprecated_warning: 'This is the same as daily_wip_chart. Please use that one', deprecated_date: '2024-05-23'
@@ -42,14 +42,15 @@ class HtmlReportConfig
   def initialize file_config:, block:
     @file_config = file_config
     @block = block
-    @sections = []
+    @sections = [] # Where we store the chunks of text that will be assembled into the HTML
+    @charts = [] # Where we store all the charts we executed so we can assert against them.
   end
 
   def cycletime label = nil, &block
     @file_config.project_config.all_boards.each_value do |board|
       raise 'Multiple cycletimes not supported' if board.cycletime
 
-      board.cycletime = CycleTimeConfig.new(parent_config: self, label: label, block: block)
+      board.cycletime = CycleTimeConfig.new(parent_config: self, label: label, block: block, file_system: file_system)
     end
   end
 
@@ -63,9 +64,11 @@ class HtmlReportConfig
 
     # The quality report has to be generated last because otherwise cycletime won't have been
     # set. Then we have to rotate it to the first position so it's at the top of the report.
-    execute_chart DataQualityReport.new(@original_issue_times || {})
+    execute_chart DataQualityReport.new(file_config.project_config.discarded_changes_data)
     @sections.rotate!(-1)
 
+    html create_footer
+
     html_directory = "#{Pathname.new(File.realpath(__FILE__)).dirname}/html"
     css = load_css html_directory: html_directory
     erb = ERB.new file_system.load(File.join(html_directory, 'index.erb'))
@@ -98,9 +101,8 @@ class HtmlReportConfig
     base_css
   end
 
-  def board_id id = nil
-    @board_id = id unless id.nil?
-    @board_id
+  def board_id id
+    @board_id = id
   end
 
   def timezone_offset
@@ -140,19 +142,6 @@ class HtmlReportConfig
     end
   end
 
-  def discard_changes_before_hook issues_cutoff_times
-    # raise 'Cycletime must be defined before using discard_changes_before' unless @cycletime
-
-    @original_issue_times = {}
-    issues_cutoff_times.each do |issue, cutoff_time|
-      started = issue.board.cycletime.started_time(issue)
-      if started && started <= cutoff_time
-        # We only need to log this if data was discarded
-        @original_issue_times[issue] = { cutoff_time: cutoff_time, started_time: started }
-      end
-    end
-  end
-
   def dependency_chart &block
     execute_chart DependencyChart.new block
   end
@@ -172,7 +161,7 @@ class HtmlReportConfig
     chart.settings = settings
 
     chart.all_boards = project_config.all_boards
-    chart.board_id = find_board_id if chart.respond_to? :board_id=
+    chart.board_id = find_board_id
     chart.holiday_dates = project_config.exporter.holiday_dates
 
     time_range = @file_config.project_config.time_range
@@ -181,6 +170,7 @@ class HtmlReportConfig
 
     after_init_block&.call chart
 
+    @charts << chart
     html chart.run
   end
 
@@ -201,4 +191,24 @@ class HtmlReportConfig
   def boards
     @file_config.project_config.board_configs.collect(&:id).collect { |id| find_board id }
   end
+
+  def create_footer now: DateTime.now
+    now = now.new_offset(timezone_offset)
+    version = Gem.loaded_specs['jirametrics']&.version || 'Next'
+
+    <<~HTML
+      <section id="footer">
+        Report generated on <b>#{now.strftime('%Y-%b-%d')}</b> at <b>#{now.strftime('%I:%M:%S%P %Z')}</b>
+        with <a href="https://jirametrics.org">JiraMetrics</a> <b>v#{version}</b>
+      </section>
+    HTML
+  end
+
+  def discard_changes_before status_becomes: nil, &block
+    file_system.deprecated(
+      date: '2025-01-09',
+      message: 'discard_changes_before is now only supported at the project level'
+    )
+    file_config.project_config.discard_changes_before status_becomes: status_becomes, &block
+  end
 end

data/lib/jirametrics/issue.rb

@@ -13,6 +13,12 @@ class Issue
     @changes = []
     @board = board
 
+    # We only check for this here because if a board isn't passed in then things will fail much
+    # later and be hard to find. Let's find out early.
+    raise "No board for issue #{key}" if board.nil?
+
+    # There are cases where we create an Issue of fragments like linked issues and those won't have
+    # changelogs.
     return unless @raw['changelog']
 
     load_history_into_changes
@@ -43,14 +49,26 @@ class Issue
 
   def summary = @raw['fields']['summary']
 
-  def status = Status.new(raw: @raw['fields']['status'])
-
   def labels = @raw['fields']['labels'] || []
 
   def author = @raw['fields']['creator']&.[]('displayName') || ''
 
   def resolution = @raw['fields']['resolution']&.[]('name')
 
+  def status
+    @status = Status.from_raw(@raw['fields']['status']) unless @status
+    @status
+  end
+
+  def status= status
+    @status = status
+  end
+
+  def due_date
+    text = @raw['fields']['duedate']
+    text.nil? ? nil : Date.parse(text)
+  end
+
   def url
     # Strangely, the URL isn't anywhere in the returned data so we have to fabricate it.
     "#{@board.server_url_prefix}/browse/#{key}"
@@ -65,35 +83,43 @@ class Issue
   end
 
   def first_time_in_status *status_names
-    @changes.find { |change| change.current_status_matches(*status_names) }&.time
+    @changes.find { |change| change.current_status_matches(*status_names) }
   end
 
   def first_time_not_in_status *status_names
-    @changes.find { |change| change.status? && status_names.include?(change.value) == false }&.time
+    @changes.find { |change| change.status? && status_names.include?(change.value) == false }
   end
 
   def first_time_in_or_right_of_column column_name
     first_time_in_status(*board.status_ids_in_or_right_of_column(column_name))
   end
 
+  def first_time_label_added *labels
+    @changes.each do |change|
+      next unless change.labels?
+
+      change_labels = change.value.split
+      return change if change_labels.any? { |l| labels.include?(l) }
+    end
+    nil
+  end
+
   def still_in_or_right_of_column column_name
     still_in_status(*board.status_ids_in_or_right_of_column(column_name))
   end
 
   def still_in
-    time = nil
-    @changes.each do |change|
-      next unless change.status?
-
+    result = nil
+    status_changes.each do |change|
       current_status_matched = yield change
 
-      if current_status_matched && time.nil?
-        time = change.time
-      elsif !current_status_matched && time
-        time = nil
+      if current_status_matched && result.nil?
+        result = change
+      elsif !current_status_matched && result
+        result = nil
       end
     end
-    time
+    result
   end
   private :still_in
 
@@ -106,50 +132,75 @@ class Issue
 
   # If it ever entered one of these categories and it's still there then what was the last time it entered
   def still_in_status_category *category_names
+    category_ids = find_status_category_ids_by_names category_names
+
     still_in do |change|
-      status = find_status_by_name change.value
-      category_names.include?(status.category_name) || category_names.include?(status.category_id)
+      status = find_or_create_status id: change.value_id, name: change.value
+      category_ids.include? status.category.id
     end
   end
 
   def most_recent_status_change
-    changes.reverse.find { |change| change.status? }
+    # Any issue that we loaded from its own file will always have a status as we artificially insert a status
+    # change to represent creation. Issues that were just fragments referenced from a main issue (ie a linked issue)
+    # may not have any status changes as we have no idea when it was created. This will be nil in that case
+    status_changes.last
   end
 
-  # Are we currently in this status? If yes, then return the time of the most recent status change.
+  # Are we currently in this status? If yes, then return the most recent status change.
   def currently_in_status *status_names
     change = most_recent_status_change
     return false if change.nil?
 
-    change.time if change.current_status_matches(*status_names)
+    change if change.current_status_matches(*status_names)
   end
 
-  # Are we currently in this status category? If yes, then return the time of the most recent status change.
+  # Are we currently in this status category? If yes, then return the most recent status change.
   def currently_in_status_category *category_names
+    category_ids = find_status_category_ids_by_names category_names
+
     change = most_recent_status_change
     return false if change.nil?
 
-    status = find_status_by_name change.value
-    change.time if status && category_names.include?(status.category_name)
+    status = find_or_create_status id: change.value_id, name: change.value
+    change if status && category_ids.include?(status.category.id)
   end
 
-  def find_status_by_name name
-    status = board.possible_statuses.find_by_name(name)
-    return status if status
+  def find_or_create_status id:, name:
+    status = board.possible_statuses.find_by_id(id)
 
-    raise "Status name #{name.inspect} for issue #{key} not found in #{board.possible_statuses.collect(&:name).inspect}"
+    unless status
+      # Have to pull this list before the call to fabricate or else the warning will incorrectly
+      # list this status as one it actually found
+      found_statuses = board.possible_statuses.to_s
+
+      status = board.possible_statuses.fabricate_status_for id: id, name: name
+
+      message = +'The history for issue '
+      message << key
+      message << ' references the status ('
+      message << "#{name.inspect}:#{id.inspect}"
+      message << ') that can\'t be found. We are guessing that this belongs to the '
+      message << status.category.to_s
+      message << ' status category but that may be wrong. See https://jirametrics.org/faq/#q1 for more '
+      message << 'details on defining statuses.'
+      board.project_config.file_system.warning message, more: "The statuses we did find are: #{found_statuses}"
+    end
+
+    status
   end
 
   def first_status_change_after_created
-    @changes.find { |change| change.status? && change.artificial? == false }&.time
+    status_changes.find { |change| change.artificial? == false }
   end
 
   def first_time_in_status_category *category_names
-    @changes.each do |change|
-      next unless change.status?
+    category_ids = find_status_category_ids_by_names category_names
 
-      category = find_status_by_name(change.value).category_name
-      return change.time if category_names.include? category
+    status_changes.each do |change|
+      to_status = find_or_create_status(id: change.value_id, name: change.value)
+      id = to_status.category.id
+      return change if category_ids.include? id
     end
     nil
   end
@@ -168,11 +219,11 @@ class Issue
   end
 
   def first_resolution
-    @changes.find { |change| change.resolution? }&.time
+    @changes.find { |change| change.resolution? }
   end
 
   def last_resolution
-    @changes.reverse.find { |change| change.resolution? }&.time
+    @changes.reverse.find { |change| change.resolution? }
   end
 
   def assigned_to
@@ -251,6 +302,7 @@ class Issue
 
     blocked_link_texts = settings['blocked_link_text']
     stalled_threshold = settings['stalled_threshold_days']
+    flagged_means_blocked = !!settings['flagged_means_blocked'] # rubocop:disable Style/DoubleNegation
 
     blocking_issue_keys = []
 
@@ -273,7 +325,7 @@ class Issue
         blocking_stalled_changes: result
       )
 
-      if change.flagged?
+      if change.flagged? && flagged_means_blocked
         flag = change.value
         flag = nil if change.value == ''
       elsif change.status?
@@ -362,6 +414,45 @@ class Issue
     inserted_stalled
   end
 
+  # return [number of active seconds, total seconds] that this issue had up to the end_time.
+  # It does not include data before issue start or after issue end
+  def flow_efficiency_numbers end_time:, settings: @board.project_config.settings
+    issue_start, issue_stop = @board.cycletime.started_stopped_times(self)
+    return [0.0, 0.0] if !issue_start || issue_start > end_time
+
+    value_add_time = 0.0
+    end_time = issue_stop if issue_stop && issue_stop < end_time
+
+    active_start = nil
+    blocked_stalled_changes(end_time: end_time, settings: settings).each_with_index do |change, index|
+      break if change.time > end_time
+
+      if index.zero?
+        active_start = change.time if change.active?
+        next
+      end
+
+      # Already active and we just got another active.
+      next if active_start && change.active?
+
+      if change.active?
+        active_start = change.time
+      elsif active_start && change.time >= issue_start
+        # Not active now but we have been. Record the active time.
+        change_delta = change.time - [issue_start, active_start].max
+        value_add_time += change_delta
+        active_start = nil
+      end
+    end
+
+    if active_start
+      change_delta = end_time - [issue_start, active_start].max
+      value_add_time += change_delta if change_delta.positive?
+    end
+
+    [value_add_time, end_time - issue_start]
+  end
+
   def all_subtask_activity_times
     subtask_activity_times = []
     @subtasks.each do |subtask|
@@ -371,7 +462,9 @@ class Issue
   end
 
   def expedited?
-    names = @board&.expedited_priority_names
+    return false unless @board&.project_config
+
+    names = @board.project_config.settings['expedited_priority_names']
     return false unless names
 
     current_priority = raw['fields']['priority']&.[]('name')
@@ -380,7 +473,9 @@ class Issue
 
   def expedited_on_date? date
     expedited_start = nil
-    expedited_names = @board&.expedited_priority_names
+    return false unless @board&.project_config
+
+    expedited_names = @board.project_config.settings['expedited_priority_names']
 
     changes.each do |change|
       next unless change.priority?
@@ -433,6 +528,10 @@ class Issue
     @fix_versions
   end
 
+  def looks_like_issue_key? key
+    !!(key.is_a?(String) && key =~ /^[^-]+-\d+$/)
+  end
+
   def parent_key project_config: @board.project_config
     # Although Atlassian is trying to standardize on one way to determine the parent, today it's a mess.
     # We try a variety of ways to get the parent and hopefully one of them will work. See this link:
@@ -454,8 +553,13 @@ class Issue
 
       custom_field_names&.each do |field_name|
         parent = fields[field_name]
-        # A break would be more appropriate than a return but the runtime caused an error when we do that
-        return parent if parent
+        next if parent.nil?
+        break if looks_like_issue_key? parent
+
+        project_config.file_system.log(
+          "Custom field #{field_name.inspect} should point to a parent id but found #{parent.inspect}"
+        )
+        parent = nil
       end
     end
 
@@ -480,6 +584,20 @@ class Issue
     comparison
   end
 
+  def discard_changes_before cutoff_time
+    rejected_any = false
+    @changes.reject! do |change|
+      reject = change.status? && change.time <= cutoff_time && change.artificial? == false
+      if reject
+        (@discarded_changes ||= []) << change
+        rejected_any = true
+      end
+      reject
+    end
+
+    (@discarded_change_times ||= []) << cutoff_time if rejected_any
+  end
+
   def dump
     result = +''
     result << "#{key} (#{type}): #{compact_text summary, 200}\n"
@@ -487,24 +605,82 @@ class Issue
     assignee = raw['fields']['assignee']
     result << "  [assignee] #{assignee['name'].inspect} <#{assignee['emailAddress']}>\n" unless assignee.nil?
 
-    raw['fields']['issuelinks'].each do |link|
+    raw['fields']['issuelinks']&.each do |link|
       result << "  [link] #{link['type']['outward']} #{link['outwardIssue']['key']}\n" if link['outwardIssue']
       result << "  [link] #{link['type']['inward']} #{link['inwardIssue']['key']}\n" if link['inwardIssue']
     end
-    changes.each do |change|
-      value = change.value
-      old_value = change.old_value
+    history = [] # time, type, detail
+
+    started_at, stopped_at = board.cycletime.started_stopped_times(self)
+    history << [started_at, nil, '↓↓↓↓ Started here ↓↓↓↓', true] if started_at
+    history << [stopped_at, nil, '↑↑↑↑ Finished here ↑↑↑↑', true] if stopped_at
 
-      message = "  [change] #{change.time.strftime '%Y-%m-%d %H:%M:%S %z'} [#{change.field}] "
-      message << "#{compact_text(old_value).inspect} -> " unless old_value.nil? || old_value.empty?
-      message << compact_text(value).inspect
-      message << " (#{change.author})"
-      message << ' <<artificial entry>>' if change.artificial?
-      result << message << "\n"
+    @discarded_change_times&.each do |time|
+      history << [time, nil, '↑↑↑↑ Changes discarded ↑↑↑↑', true]
+    end
+
+    (changes + (@discarded_changes || [])).each do |change|
+      if change.status?
+        value = "#{change.value.inspect}:#{change.value_id.inspect}"
+        old_value = change.old_value ? "#{change.old_value.inspect}:#{change.old_value_id.inspect}" : nil
+      else
+        value = compact_text(change.value).inspect
+        old_value = change.old_value ? compact_text(change.old_value).inspect : nil
+      end
+
+      message = +''
+      message << "#{old_value} -> " unless old_value.nil? || old_value.empty?
+      message << value
+      if change.artificial?
+        message << ' (Artificial entry)' if change.artificial?
+      else
+        message << " (Author: #{change.author})"
+      end
+      history << [change.time, change.field, message, change.artificial?]
     end
+
+    result << "  History:\n"
+    type_width = history.collect { |_time, type, _detail, _artificial| type&.length || 0 }.max
+    history.sort! do |a, b|
+      if a[0] == b[0]
+        if a[1].nil?
+          1
+        elsif b[1].nil?
+          -1
+        else
+          a[1] <=> b[1]
+        end
+      else
+        a[0] <=> b[0]
+      end
+    end
+    history.each do |time, type, detail, _artificial|
+      if type.nil?
+        type = '-' * type_width
+      else
+        type = (' ' * (type_width - type.length)) << type
+      end
+      result << "    #{time.strftime '%Y-%m-%d %H:%M:%S %z'} [#{type}] #{detail}\n"
+    end
+
     result
   end
 
+  def done?
+    if artificial? || board.cycletime.nil?
+      # This was probably loaded as a linked issue, which means we don't know what board it really
+      # belonged to. The best we can do is look at the status category. This case should be rare but
+      # it can happen.
+      status.category.name == 'Done'
+    else
+      board.cycletime.done? self
+    end
+  end
+
+  def status_changes
+    @changes.select { |change| change.status? }
+  end
+
   private
 
   def assemble_author raw
@@ -580,4 +756,13 @@ class Issue
       'toString' => first_status
     }
   end
+
+  def find_status_category_ids_by_names category_names
+    category_names.filter_map do |name|
+      list = board.possible_statuses.find_all_categories_by_name name
+      raise "No status categories found for name: #{name}" if list.empty?
+
+      list
+    end.flatten.collect(&:id)
+  end
 end

data/lib/jirametrics/jira_gateway.rb

@@ -14,9 +14,15 @@ class JiraGateway
   def call_url relative_url:
     command = make_curl_command url: "#{@jira_url}#{relative_url}"
     result = call_command command
-    JSON.parse result
-  rescue => e # rubocop:disable Style/RescueStandardError
-    puts "Error #{e.inspect} when parsing result: #{result.inspect}"
+    begin
+      json = JSON.parse(result)
+    rescue # rubocop:disable Style/RescueStandardError
+      raise "Error when parsing result: #{result.inspect}"
+    end
+
+    raise "Download failed with: #{JSON.pretty_generate(json)}" unless json_successful?(json)
+
+    json
   end
 
   def call_command command
@@ -61,4 +67,11 @@ class JiraGateway
     command << " --url \"#{url}\""
     command
   end
+
+  def json_successful? json
+    return false if json.is_a?(Hash) && (json['error'] || json['errorMessages'] || json['errorMessage'])
+    return false if json.is_a?(Array) && json.first == 'errorMessage'
+
+    true
+  end
 end