jirametrics 2.8 → 2.9.1pre1

data/lib/jirametrics/html_report_config.rb

@@ -5,16 +5,15 @@ require 'jirametrics/self_or_issue_dispatcher'
 
 class HtmlReportConfig
   include SelfOrIssueDispatcher
-  include DiscardChangesBefore
 
-  attr_reader :file_config, :sections
+  attr_reader :file_config, :sections, :charts
 
   def self.define_chart name:, classname:, deprecated_warning: nil, deprecated_date: nil
     lines = []
     lines << "def #{name} &block"
     lines << '  block = ->(_) {} unless block'
     if deprecated_warning
-      lines << "  deprecated date: #{deprecated_date.inspect}, message: #{deprecated_warning.inspect}"
+      lines << "  file_system.deprecated date: #{deprecated_date.inspect}, message: #{deprecated_warning.inspect}"
     end
     lines << "  execute_chart #{classname}.new(block)"
     lines << 'end'
@@ -43,14 +42,15 @@ class HtmlReportConfig
   def initialize file_config:, block:
     @file_config = file_config
     @block = block
-    @sections = []
+    @sections = [] # Where we store the chunks of text that will be assembled into the HTML
+    @charts = [] # Where we store all the charts we executed so we can assert against them.
   end
 
   def cycletime label = nil, &block
     @file_config.project_config.all_boards.each_value do |board|
       raise 'Multiple cycletimes not supported' if board.cycletime
 
-      board.cycletime = CycleTimeConfig.new(parent_config: self, label: label, block: block)
+      board.cycletime = CycleTimeConfig.new(parent_config: self, label: label, block: block, file_system: file_system)
     end
   end
 
@@ -64,7 +64,7 @@ class HtmlReportConfig
 
     # The quality report has to be generated last because otherwise cycletime won't have been
     # set. Then we have to rotate it to the first position so it's at the top of the report.
-    execute_chart DataQualityReport.new(@original_issue_times || {})
+    execute_chart DataQualityReport.new(file_config.project_config.discarded_changes_data)
     @sections.rotate!(-1)
 
     html create_footer
@@ -101,9 +101,8 @@ class HtmlReportConfig
     base_css
   end
 
-  def board_id id = nil
-    @board_id = id unless id.nil?
-    @board_id
+  def board_id id
+    @board_id = id
   end
 
   def timezone_offset
@@ -143,19 +142,6 @@ class HtmlReportConfig
     end
   end
 
-  def discard_changes_before_hook issues_cutoff_times
-    # raise 'Cycletime must be defined before using discard_changes_before' unless @cycletime
-
-    @original_issue_times = {}
-    issues_cutoff_times.each do |issue, cutoff_time|
-      started = issue.board.cycletime.started_stopped_times(issue).first
-      if started && started <= cutoff_time
-        # We only need to log this if data was discarded
-        @original_issue_times[issue] = { cutoff_time: cutoff_time, started_time: started }
-      end
-    end
-  end
-
   def dependency_chart &block
     execute_chart DependencyChart.new block
   end
@@ -175,7 +161,7 @@ class HtmlReportConfig
     chart.settings = settings
 
     chart.all_boards = project_config.all_boards
-    chart.board_id = find_board_id if chart.respond_to? :board_id=
+    chart.board_id = find_board_id
     chart.holiday_dates = project_config.exporter.holiday_dates
 
     time_range = @file_config.project_config.time_range
@@ -184,6 +170,7 @@ class HtmlReportConfig
 
     after_init_block&.call chart
 
+    @charts << chart
     html chart.run
   end
 
@@ -216,4 +203,12 @@ class HtmlReportConfig
       </section>
     HTML
   end
+
+  def discard_changes_before status_becomes: nil, &block
+    file_system.deprecated(
+      date: '2025-01-09',
+      message: 'discard_changes_before is now only supported at the project level'
+    )
+    file_config.project_config.discard_changes_before status_becomes: status_becomes, &block
+  end
 end

data/lib/jirametrics/issue.rb

@@ -13,7 +13,12 @@ class Issue
     @changes = []
     @board = board
 
+    # We only check for this here because if a board isn't passed in then things will fail much
+    # later and be hard to find. Let's find out early.
     raise "No board for issue #{key}" if board.nil?
+
+    # There are cases where we create an Issue of fragments like linked issues and those won't have
+    # changelogs.
     return unless @raw['changelog']
 
     load_history_into_changes
@@ -93,9 +98,7 @@ class Issue
 
   def still_in
     result = nil
-    @changes.each do |change|
-      next unless change.status?
-
+    status_changes.each do |change|
       current_status_matched = yield change
 
       if current_status_matched && result.nil?
@@ -117,63 +120,71 @@ class Issue
 
   # If it ever entered one of these categories and it's still there then what was the last time it entered
   def still_in_status_category *category_names
+    category_ids = find_status_category_ids_by_names category_names
+
     still_in do |change|
-      status = find_status_by_id change.value_id, name: change.value
-      category_names.include?(status.category.name) || category_names.include?(status.category.id)
+      status = find_or_create_status id: change.value_id, name: change.value
+      category_ids.include? status.category.id
     end
   end
 
   def most_recent_status_change
+    # We artificially insert a status change to represent creation so by definition there will always be at least one.
     changes.reverse.find { |change| change.status? }
   end
 
   # Are we currently in this status? If yes, then return the most recent status change.
   def currently_in_status *status_names
     change = most_recent_status_change
-    return false if change.nil?
 
     change if change.current_status_matches(*status_names)
   end
 
   # Are we currently in this status category? If yes, then return the most recent status change.
   def currently_in_status_category *category_names
+    category_ids = find_status_category_ids_by_names category_names
+
     change = most_recent_status_change
-    return false if change.nil?
 
-    status = find_status_by_id change.value_id, name: change.value
-    change if status && category_names.include?(status.category.name)
+    status = find_or_create_status id: change.value_id, name: change.value
+    change if status && category_ids.include?(status.category.id)
   end
 
-  def find_status_by_id id, name: nil
+  def find_or_create_status id:, name:
     status = board.possible_statuses.find_by_id(id)
-    return status if status
 
-    status = board.possible_statuses.fabricate_status_for id: id, name: name
+    unless status
+      # Have to pull this list before the call to fabricate or else the warning will incorrectly
+      # list this status as one it actually found
+      found_statuses = board.possible_statuses.to_s
+
+      status = board.possible_statuses.fabricate_status_for id: id, name: name
 
-    message = +'The history for issue '
-    message << key
-    message << ' references a status ('
-    message << name.inspect << ':' if name
-    message << id.to_s
-    message << ') that can\'t be found in ['
-    message << board.possible_statuses.collect(&:to_s).join(', ')
-    message << "]. We are guessing that this belongs to the #{status.category} status category "
-    message << 'and that may be wrong. See https://jirametrics.org/faq/#q1 for more details'
-    board.project_config.file_system.warning message
+      message = +'The history for issue '
+      message << key
+      message << ' references the status ('
+      message << "#{name.inspect}:#{id.inspect}"
+      message << ') that can\'t be found. We are guessing that this belongs to the '
+      message << status.category.to_s
+      message << ' status category but that may be wrong. See https://jirametrics.org/faq/#q1 for more '
+      message << 'details on defining statuses.'
+      board.project_config.file_system.warning message, more: "The statuses we did find are: #{found_statuses}"
+    end
 
     status
   end
 
   def first_status_change_after_created
-    @changes.find { |change| change.status? && change.artificial? == false }
+    status_changes.find { |change| change.artificial? == false }
   end
 
   def first_time_in_status_category *category_names
-    @changes.each do |change|
-      next unless change.status?
+    category_ids = find_status_category_ids_by_names category_names
 
-      category = find_status_by_id(change.value_id).category.name
-      return change if category_names.include? category
+    status_changes.each do |change|
+      to_status = find_or_create_status(id: change.value_id, name: change.value)
+      id = to_status.category.id
+      return change if category_ids.include? id
     end
     nil
   end
@@ -645,6 +656,10 @@ class Issue
     end
   end
 
+  def status_changes
+    @changes.select { |change| change.status? }
+  end
+
   private
 
   def assemble_author raw
@@ -720,4 +735,13 @@ class Issue
       'toString' => first_status
     }
   end
+
+  def find_status_category_ids_by_names category_names
+    category_names.filter_map do |name|
+      list = board.possible_statuses.find_all_categories_by_name name
+      raise "No status categories found for name: #{name}" if list.empty?
+
+      list
+    end.flatten.collect(&:id)
+  end
 end

data/lib/jirametrics/project_config.rb

@@ -4,11 +4,9 @@ require 'time'
 require 'jirametrics/status_collection'
 
 class ProjectConfig
-  include DiscardChangesBefore
-
   attr_reader :target_path, :jira_config, :all_boards, :possible_statuses,
     :download_config, :file_configs, :exporter, :data_version, :name, :board_configs,
-    :settings, :aggregate_config
+    :settings, :aggregate_config, :discarded_changes_data
   attr_accessor :time_range, :jira_url, :id
 
   def initialize exporter:, jira_config:, block:, target_path: '.', name: '', id: nil
@@ -32,14 +30,13 @@ class ProjectConfig
   end
 
   def data_downloaded?
-    File.exist? File.join(@target_path, "#{get_file_prefix}_meta.json")
+    file_system.file_exist? File.join(@target_path, "#{get_file_prefix}_meta.json")
   end
 
   def load_data
     return if @has_loaded_data
 
     @has_loaded_data = true
-    load_all_boards
     @id = guess_project_id
     load_project_metadata
     load_sprints
@@ -53,16 +50,13 @@ class ProjectConfig
 
     return if load_only
 
-    @board_configs.each do |board_config|
-      board_config.run
-    end
     @file_configs.each do |file_config|
       file_config.run
     end
   end
 
   def load_settings
-    # This is the wierd exception that we don't ever want mocked out so we skip FileSystem entirely.
+    # This is the weird exception that we don't ever want mocked out so we skip FileSystem entirely.
     JSON.parse(File.read(File.join(__dir__, 'settings.json'), encoding: 'UTF-8'))
   end
 
@@ -112,6 +106,7 @@ class ProjectConfig
 
   def board id:, &block
     config = BoardConfig.new(id: id, block: block, project_config: self)
+    config.run if data_downloaded?
     @board_configs << config
   end
 
@@ -128,6 +123,8 @@ class ProjectConfig
     # is set but before anything inside the project block is run. If only we had made file_prefix an attribute
     # on project, we wouldn't have this ugliness. 🤷‍♂️
     load_status_category_mappings
+    load_status_history
+    load_all_boards
 
     @file_prefix
   end
@@ -138,29 +135,28 @@ class ProjectConfig
     @file_prefix
   end
 
-  def status_guesses
-    statuses = {}
+  # Walk across all the issues and find any status with that name. Return a list of ids that match.
+  def find_ids_by_status_name_across_all_issues name
+    ids = Set.new
+
     issues.each do |issue|
       issue.changes.each do |change|
         next unless change.status?
 
-        (statuses[change.value] ||= Set.new) << change.value_id
-        (statuses[change.old_value] ||= Set.new) << change.old_value_id if change.old_value
+        ids << change.value_id.to_i if change.value == name
+        ids << change.old_value_id.to_i if change.old_value == name
       end
     end
-    statuses
+    ids.to_a
   end
 
   def status_category_mapping status:, category:
-    # Status might just be a name like "Review" or it might be a name:id pair like "Review:4"
-    parse_status = ->(line) { line =~ /^(.+):(\d+)$/ ? [$1, $2.to_i] : [line, nil] }
-
-    status, status_id = parse_status.call status
-    category, category_id = parse_status.call category
+    status, status_id = possible_statuses.parse_name_id status
+    category, category_id = possible_statuses.parse_name_id category
 
     if status_id.nil?
-      guesses = status_guesses[status]
-      if guesses.nil?
+      guesses = find_ids_by_status_name_across_all_issues status
+      if guesses.empty?
         file_system.warning "For status_category_mapping status: #{status.inspect}, category: #{category.inspect}\n" \
           "Cannot guess status id for #{status.inspect} as no statuses found anywhere in the issues " \
           "histories with that name. Since we can't find it, you probably don't need this mapping anymore so we're " \
@@ -174,21 +170,31 @@ class ProjectConfig
       end
 
       status_id = guesses.first
-      # TODO: This should be a deprecation instead but we can't currently assert against those
       file_system.log "status_category_mapping for #{status.inspect} has been mapped to id #{status_id}. " \
         "If that's incorrect then specify the status_id."
     end
 
-    found_category = possible_statuses.find_category_by_name category
-    found_category_id = found_category&.id # possible_statuses.find_category_id_by_name category
-    if category_id && category_id != found_category_id
-      raise "ID is incorrect for status category #{category.inspect}. Did you mean #{found_category_id}?"
+    possible_categories = possible_statuses.find_all_categories_by_name category
+    if possible_categories.empty?
+      all = possible_statuses.find_all_categories.join(', ')
+      raise "No status categories found for name #{category.inspect} in [#{all}]. " \
+        'Either fix the name or add an ID.'
+    elsif possible_categories.size > 1
+      # Theoretically impossible and yet we've seen wierder things out of Jira so we're prepared.
+      raise "More than one status category found with the name #{category.inspect} in " \
+        "[#{possible_categories.join(', ')}]. Either fix the name or add an ID"
+    end
+
+    found_category = possible_categories.first
+
+    if category_id && category_id != found_category.id
+      raise "ID is incorrect for status category #{category.inspect}. Did you mean #{found_category.id}?"
     end
 
     add_possible_status(
       Status.new(
         name: status, id: status_id,
-        category_name: category, category_id: found_category_id, category_key: found_category.key
+        category_name: category, category_id: found_category.id, category_key: found_category.key
       )
     )
   end
@@ -211,7 +217,14 @@ class ProjectConfig
     end
 
     # We're registering one we already knew about. This may happen if someone specified a status_category_mapping
-    # for something that was already returned from jira. This isn't  problem so just ignore it.
+    # for something that was already returned from jira.
+    #
+    # You may be looking at this code and thinking of changing it to spit out a warning since obviously
+    # the user has made a mistake. Unfortunately, they may not have made any mistake. Due to inconsistency with the
+    # status API, it's possible for two different people to make a request to the same API at the same time and get
+    # back a different set of statuses. So that means that some people might need more status/categories mappings than
+    # other people for exactly the same instance. See this article for more on that API:
+    # https://agiletechnicalexcellence.com/2024/04/12/jira-api-statuses.html
     existing_status
   end
 
@@ -222,7 +235,6 @@ class ProjectConfig
       board_id = $1.to_i
       load_board board_id: board_id, filename: "#{@target_path}#{file}"
     end
-    raise "No boards found for #{get_file_prefix.inspect} in #{@target_path.inspect}" if @all_boards.empty?
   end
 
   def load_board board_id:, filename:
@@ -235,9 +247,7 @@ class ProjectConfig
 
   def load_status_category_mappings
     filename = File.join @target_path, "#{get_file_prefix}_statuses.json"
-
-    # We may not always have this file. Load it if we can.
-    return unless File.exist? filename
+    return unless file_system.file_exist? filename
 
     file_system
       .load_json(filename)
@@ -245,6 +255,22 @@ class ProjectConfig
       .each { |status| add_possible_status status }
   end
 
+  def load_status_history
+    filename = File.join @target_path, "#{get_file_prefix}_status_history.json"
+    return unless file_system.file_exist? filename
+
+    file_system.log '  Loading historical statuses', also_write_to_stderr: true
+    file_system
+      .load_json(filename)
+      .map { |snippet| Status.from_raw(snippet) }
+      .each { |status| possible_statuses.historical_status_mappings[status.to_s] = status.category }
+
+    possible_statuses
+  rescue => e # rubocop:disable Style/RescueStandardError
+    file_system.warning "Unable to load status history due to #{e.message.inspect}. If this is because of a " \
+      'malformed file then it should be fixed on the next download.'
+  end
+
   def load_sprints
     file_system.foreach(@target_path) do |file|
       next unless file =~ /^#{get_file_prefix}_board_(\d+)_sprints_\d+.json$/
@@ -478,21 +504,52 @@ class ProjectConfig
     Anonymizer.new(project_config: self).run
   end
 
-  def discard_changes_before_hook issues_cutoff_times
-    issues_cutoff_times.each do |issue, cutoff_time|
-      days = (cutoff_time.to_date - issue.changes.first.time.to_date).to_i + 1
-      message = "#{issue.key}(#{issue.type}) discarding #{days} "
-      if days == 1
-        message << "day of data on #{cutoff_time.to_date}"
-      else
-        message << "days of data from #{issue.changes.first.time.to_date} to #{cutoff_time.to_date}"
+  def file_system
+    @exporter.file_system
+  end
+
+  def discard_changes_before status_becomes: nil, &block
+    if status_becomes
+      status_becomes = [status_becomes] unless status_becomes.is_a? Array
+
+      block = lambda do |issue|
+        trigger_statuses = status_becomes.collect do |status_name|
+          if status_name == :backlog
+            issue.board.backlog_statuses
+          else
+            possible_statuses.find_all_by_name status_name
+          end
+        end.flatten
+
+        next if trigger_statuses.empty?
+
+        trigger_status_ids = trigger_statuses.collect(&:id)
+
+        time = nil
+        issue.status_changes.each do |change|
+          time = change.time if trigger_status_ids.include?(change.value_id) # && change.artificial? == false
+        end
+        time
       end
-      exporter.file_system.log message
     end
-    exporter.file_system.log "Discarded data from #{issues_cutoff_times.count} issues out of a total #{issues.size}"
-  end
 
-  def file_system
-    @exporter.file_system
+    issues.each do |issue|
+      cutoff_time = block.call(issue)
+      next if cutoff_time.nil?
+
+      original_start_time = issue.board.cycletime.started_stopped_times(issue).first
+      next if original_start_time.nil?
+
+      issue.discard_changes_before cutoff_time
+
+      next unless cutoff_time
+      next if original_start_time > cutoff_time # ie the cutoff would have made no difference.
+
+      (@discarded_changes_data ||= []) << {
+        cutoff_time: cutoff_time,
+        original_start_time: original_start_time,
+        issue: issue
+      }
+    end
   end
 end

data/lib/jirametrics/status.rb

@@ -3,7 +3,6 @@
 require 'jirametrics/value_equality'
 
 class Status
-  include ValueEquality
   attr_reader :id, :project_id, :category
   attr_accessor :name
 
@@ -20,6 +19,17 @@ class Status
       "#{name.inspect}:#{id.inspect}"
     end
 
+    def <=> other
+      id <=> other.id
+    end
+
+    def == other
+      id == other.id
+    end
+
+    def eql?(other) = id.eql?(other.id)
+    def hash = id.hash
+
     def new? = (@key == 'new')
     def indeterminate? = (@key == 'indeterminate')
     def done? = (@key == 'done')
@@ -74,9 +84,21 @@ class Status
   end
 
   def == other
+    return false unless other.is_a? Status
+
     @id == other.id && @name == other.name && @category.id == other.category.id && @category.name == other.category.name
   end
 
+  def eql?(other)
+    self == other
+  end
+
+  def <=> other
+    result = @name.casecmp(other.name)
+    result = @id <=> other.id if result.zero?
+    result
+  end
+
   def inspect
     result = []
     result << "Status(name: #{@name.inspect}"