jirametrics 2.8 → 2.9

data/lib/jirametrics/issue.rb

@@ -13,7 +13,12 @@ class Issue
     @changes = []
     @board = board
 
+    # We only check for this here because if a board isn't passed in then things will fail much
+    # later and be hard to find. Let's find out early.
     raise "No board for issue #{key}" if board.nil?
+
+    # There are cases where we create an Issue of fragments like linked issues and those won't have
+    # changelogs.
     return unless @raw['changelog']
 
     load_history_into_changes
@@ -93,9 +98,7 @@ class Issue
 
   def still_in
     result = nil
-    @changes.each do |change|
-      next unless change.status?
-
+    status_changes.each do |change|
       current_status_matched = yield change
 
       if current_status_matched && result.nil?
@@ -117,63 +120,71 @@ class Issue
 
   # If it ever entered one of these categories and it's still there then what was the last time it entered
   def still_in_status_category *category_names
+    category_ids = find_status_category_ids_by_names category_names
+
     still_in do |change|
-      status = find_status_by_id change.value_id, name: change.value
-      category_names.include?(status.category.name) || category_names.include?(status.category.id)
+      status = find_or_create_status id: change.value_id, name: change.value
+      category_ids.include? status.category.id
     end
   end
 
   def most_recent_status_change
+    # We artificially insert a status change to represent creation so by definition there will always be at least one.
     changes.reverse.find { |change| change.status? }
   end
 
   # Are we currently in this status? If yes, then return the most recent status change.
   def currently_in_status *status_names
     change = most_recent_status_change
-    return false if change.nil?
 
     change if change.current_status_matches(*status_names)
   end
 
   # Are we currently in this status category? If yes, then return the most recent status change.
   def currently_in_status_category *category_names
+    category_ids = find_status_category_ids_by_names category_names
+
     change = most_recent_status_change
-    return false if change.nil?
 
-    status = find_status_by_id change.value_id, name: change.value
-    change if status && category_names.include?(status.category.name)
+    status = find_or_create_status id: change.value_id, name: change.value
+    change if status && category_ids.include?(status.category.id)
   end
 
-  def find_status_by_id id, name: nil
+  def find_or_create_status id:, name:
     status = board.possible_statuses.find_by_id(id)
-    return status if status
 
-    status = board.possible_statuses.fabricate_status_for id: id, name: name
+    unless status
+      # Have to pull this list before the call to fabricate or else the warning will incorrectly
+      # list this status as one it actually found
+      found_statuses = board.possible_statuses.to_s
+
+      status = board.possible_statuses.fabricate_status_for id: id, name: name
 
-    message = +'The history for issue '
-    message << key
-    message << ' references a status ('
-    message << name.inspect << ':' if name
-    message << id.to_s
-    message << ') that can\'t be found in ['
-    message << board.possible_statuses.collect(&:to_s).join(', ')
-    message << "]. We are guessing that this belongs to the #{status.category} status category "
-    message << 'and that may be wrong. See https://jirametrics.org/faq/#q1 for more details'
-    board.project_config.file_system.warning message
+      message = +'The history for issue '
+      message << key
+      message << ' references the status ('
+      message << "#{name.inspect}:#{id.inspect}"
+      message << ') that can\'t be found. We are guessing that this belongs to the '
+      message << status.category.to_s
+      message << ' status category but that may be wrong. See https://jirametrics.org/faq/#q1 for more '
+      message << 'details on defining statuses.'
+      board.project_config.file_system.warning message, more: "The statuses we did find are: #{found_statuses}"
+    end
 
     status
   end
 
   def first_status_change_after_created
-    @changes.find { |change| change.status? && change.artificial? == false }
+    status_changes.find { |change| change.artificial? == false }
   end
 
   def first_time_in_status_category *category_names
-    @changes.each do |change|
-      next unless change.status?
+    category_ids = find_status_category_ids_by_names category_names
 
-      category = find_status_by_id(change.value_id).category.name
-      return change if category_names.include? category
+    status_changes.each do |change|
+      to_status = find_or_create_status(id: change.value_id, name: change.value)
+      id = to_status.category.id
+      return change if category_ids.include? id
     end
     nil
   end
@@ -645,6 +656,10 @@ class Issue
     end
   end
 
+  def status_changes
+    @changes.select { |change| change.status? }
+  end
+
   private
 
   def assemble_author raw
@@ -720,4 +735,13 @@ class Issue
       'toString' => first_status
     }
   end
+
+  def find_status_category_ids_by_names category_names
+    category_names.filter_map do |name|
+      list = board.possible_statuses.find_all_categories_by_name name
+      raise "No status categories found for name: #{name}" if list.empty?
+
+      list
+    end.flatten.collect(&:id)
+  end
 end

data/lib/jirametrics/project_config.rb

@@ -4,11 +4,9 @@ require 'time'
 require 'jirametrics/status_collection'
 
 class ProjectConfig
-  include DiscardChangesBefore
-
   attr_reader :target_path, :jira_config, :all_boards, :possible_statuses,
     :download_config, :file_configs, :exporter, :data_version, :name, :board_configs,
-    :settings, :aggregate_config
+    :settings, :aggregate_config, :discarded_changes_data
   attr_accessor :time_range, :jira_url, :id
 
   def initialize exporter:, jira_config:, block:, target_path: '.', name: '', id: nil
@@ -32,14 +30,13 @@ class ProjectConfig
   end
 
   def data_downloaded?
-    File.exist? File.join(@target_path, "#{get_file_prefix}_meta.json")
+    file_system.file_exist? File.join(@target_path, "#{get_file_prefix}_meta.json")
   end
 
   def load_data
     return if @has_loaded_data
 
     @has_loaded_data = true
-    load_all_boards
     @id = guess_project_id
     load_project_metadata
     load_sprints
@@ -53,16 +50,13 @@ class ProjectConfig
 
     return if load_only
 
-    @board_configs.each do |board_config|
-      board_config.run
-    end
     @file_configs.each do |file_config|
       file_config.run
     end
   end
 
   def load_settings
-    # This is the wierd exception that we don't ever want mocked out so we skip FileSystem entirely.
+    # This is the weird exception that we don't ever want mocked out so we skip FileSystem entirely.
     JSON.parse(File.read(File.join(__dir__, 'settings.json'), encoding: 'UTF-8'))
   end
 
@@ -112,6 +106,7 @@ class ProjectConfig
 
   def board id:, &block
     config = BoardConfig.new(id: id, block: block, project_config: self)
+    config.run
     @board_configs << config
   end
 
@@ -128,6 +123,8 @@ class ProjectConfig
     # is set but before anything inside the project block is run. If only we had made file_prefix an attribute
     # on project, we wouldn't have this ugliness. 🤷‍♂️
     load_status_category_mappings
+    load_status_history
+    load_all_boards
 
     @file_prefix
   end
@@ -138,29 +135,28 @@ class ProjectConfig
     @file_prefix
   end
 
-  def status_guesses
-    statuses = {}
+  # Walk across all the issues and find any status with that name. Return a list of ids that match.
+  def find_ids_by_status_name_across_all_issues name
+    ids = Set.new
+
     issues.each do |issue|
       issue.changes.each do |change|
         next unless change.status?
 
-        (statuses[change.value] ||= Set.new) << change.value_id
-        (statuses[change.old_value] ||= Set.new) << change.old_value_id if change.old_value
+        ids << change.value_id.to_i if change.value == name
+        ids << change.old_value_id.to_i if change.old_value == name
       end
     end
-    statuses
+    ids.to_a
   end
 
   def status_category_mapping status:, category:
-    # Status might just be a name like "Review" or it might be a name:id pair like "Review:4"
-    parse_status = ->(line) { line =~ /^(.+):(\d+)$/ ? [$1, $2.to_i] : [line, nil] }
-
-    status, status_id = parse_status.call status
-    category, category_id = parse_status.call category
+    status, status_id = possible_statuses.parse_name_id status
+    category, category_id = possible_statuses.parse_name_id category
 
     if status_id.nil?
-      guesses = status_guesses[status]
-      if guesses.nil?
+      guesses = find_ids_by_status_name_across_all_issues status
+      if guesses.empty?
         file_system.warning "For status_category_mapping status: #{status.inspect}, category: #{category.inspect}\n" \
           "Cannot guess status id for #{status.inspect} as no statuses found anywhere in the issues " \
           "histories with that name. Since we can't find it, you probably don't need this mapping anymore so we're " \
@@ -174,21 +170,31 @@ class ProjectConfig
       end
 
       status_id = guesses.first
-      # TODO: This should be a deprecation instead but we can't currently assert against those
       file_system.log "status_category_mapping for #{status.inspect} has been mapped to id #{status_id}. " \
         "If that's incorrect then specify the status_id."
     end
 
-    found_category = possible_statuses.find_category_by_name category
-    found_category_id = found_category&.id # possible_statuses.find_category_id_by_name category
-    if category_id && category_id != found_category_id
-      raise "ID is incorrect for status category #{category.inspect}. Did you mean #{found_category_id}?"
+    possible_categories = possible_statuses.find_all_categories_by_name category
+    if possible_categories.empty?
+      all = possible_statuses.find_all_categories.join(', ')
+      raise "No status categories found for name #{category.inspect} in [#{all}]. " \
+        'Either fix the name or add an ID.'
+    elsif possible_categories.size > 1
+      # Theoretically impossible and yet we've seen wierder things out of Jira so we're prepared.
+      raise "More than one status category found with the name #{category.inspect} in " \
+        "[#{possible_categories.join(', ')}]. Either fix the name or add an ID"
+    end
+
+    found_category = possible_categories.first
+
+    if category_id && category_id != found_category.id
+      raise "ID is incorrect for status category #{category.inspect}. Did you mean #{found_category.id}?"
     end
 
     add_possible_status(
       Status.new(
         name: status, id: status_id,
-        category_name: category, category_id: found_category_id, category_key: found_category.key
+        category_name: category, category_id: found_category.id, category_key: found_category.key
       )
     )
   end
@@ -211,7 +217,14 @@ class ProjectConfig
     end
 
     # We're registering one we already knew about. This may happen if someone specified a status_category_mapping
-    # for something that was already returned from jira. This isn't  problem so just ignore it.
+    # for something that was already returned from jira.
+    #
+    # You may be looking at this code and thinking of changing it to spit out a warning since obviously
+    # the user has made a mistake. Unfortunately, they may not have made any mistake. Due to inconsistency with the
+    # status API, it's possible for two different people to make a request to the same API at the same time and get
+    # back a different set of statuses. So that means that some people might need more status/categories mappings than
+    # other people for exactly the same instance. See this article for more on that API:
+    # https://agiletechnicalexcellence.com/2024/04/12/jira-api-statuses.html
     existing_status
   end
 
@@ -222,7 +235,6 @@ class ProjectConfig
       board_id = $1.to_i
       load_board board_id: board_id, filename: "#{@target_path}#{file}"
     end
-    raise "No boards found for #{get_file_prefix.inspect} in #{@target_path.inspect}" if @all_boards.empty?
   end
 
   def load_board board_id:, filename:
@@ -235,9 +247,7 @@ class ProjectConfig
 
   def load_status_category_mappings
     filename = File.join @target_path, "#{get_file_prefix}_statuses.json"
-
-    # We may not always have this file. Load it if we can.
-    return unless File.exist? filename
+    return unless file_system.file_exist? filename
 
     file_system
       .load_json(filename)
@@ -245,6 +255,22 @@ class ProjectConfig
       .each { |status| add_possible_status status }
   end
 
+  def load_status_history
+    filename = File.join @target_path, "#{get_file_prefix}_status_history.json"
+    return unless file_system.file_exist? filename
+
+    file_system.log '  Loading historical statuses', also_write_to_stderr: true
+    file_system
+      .load_json(filename)
+      .map { |snippet| Status.from_raw(snippet) }
+      .each { |status| possible_statuses.historical_status_mappings[status.to_s] = status.category }
+
+    possible_statuses
+  rescue => e # rubocop:disable Style/RescueStandardError
+    file_system.warning "Unable to load status history due to #{e.message.inspect}. If this is because of a " \
+      'malformed file then it should be fixed on the next download.'
+  end
+
   def load_sprints
     file_system.foreach(@target_path) do |file|
       next unless file =~ /^#{get_file_prefix}_board_(\d+)_sprints_\d+.json$/
@@ -478,21 +504,52 @@ class ProjectConfig
     Anonymizer.new(project_config: self).run
   end
 
-  def discard_changes_before_hook issues_cutoff_times
-    issues_cutoff_times.each do |issue, cutoff_time|
-      days = (cutoff_time.to_date - issue.changes.first.time.to_date).to_i + 1
-      message = "#{issue.key}(#{issue.type}) discarding #{days} "
-      if days == 1
-        message << "day of data on #{cutoff_time.to_date}"
-      else
-        message << "days of data from #{issue.changes.first.time.to_date} to #{cutoff_time.to_date}"
+  def file_system
+    @exporter.file_system
+  end
+
+  def discard_changes_before status_becomes: nil, &block
+    if status_becomes
+      status_becomes = [status_becomes] unless status_becomes.is_a? Array
+
+      block = lambda do |issue|
+        trigger_statuses = status_becomes.collect do |status_name|
+          if status_name == :backlog
+            issue.board.backlog_statuses
+          else
+            possible_statuses.find_all_by_name status_name
+          end
+        end.flatten
+
+        next if trigger_statuses.empty?
+
+        trigger_status_ids = trigger_statuses.collect(&:id)
+
+        time = nil
+        issue.status_changes.each do |change|
+          time = change.time if trigger_status_ids.include?(change.value_id) # && change.artificial? == false
+        end
+        time
       end
-      exporter.file_system.log message
     end
-    exporter.file_system.log "Discarded data from #{issues_cutoff_times.count} issues out of a total #{issues.size}"
-  end
 
-  def file_system
-    @exporter.file_system
+    issues.each do |issue|
+      cutoff_time = block.call(issue)
+      next if cutoff_time.nil?
+
+      original_start_time = issue.board.cycletime.started_stopped_times(issue).first
+      next if original_start_time.nil?
+
+      issue.discard_changes_before cutoff_time
+
+      next unless cutoff_time
+      next if original_start_time > cutoff_time # ie the cutoff would have made no difference.
+
+      (@discarded_changes_data ||= []) << {
+        cutoff_time: cutoff_time,
+        original_start_time: original_start_time,
+        issue: issue
+      }
+    end
   end
 end

data/lib/jirametrics/status.rb

@@ -3,7 +3,6 @@
 require 'jirametrics/value_equality'
 
 class Status
-  include ValueEquality
   attr_reader :id, :project_id, :category
   attr_accessor :name
 
@@ -20,6 +19,17 @@ class Status
       "#{name.inspect}:#{id.inspect}"
     end
 
+    def <=> other
+      id <=> other.id
+    end
+
+    def == other
+      id == other.id
+    end
+
+    def eql?(other) = id.eql?(other.id)
+    def hash = id.hash
+
     def new? = (@key == 'new')
     def indeterminate? = (@key == 'indeterminate')
     def done? = (@key == 'done')
@@ -74,9 +84,21 @@ class Status
   end
 
   def == other
+    return false unless other.is_a? Status
+
     @id == other.id && @name == other.name && @category.id == other.category.id && @category.name == other.category.name
   end
 
+  def eql?(other)
+    self == other
+  end
+
+  def <=> other
+    result = @name.casecmp(other.name)
+    result = @id <=> other.id if result.zero?
+    result
+  end
+
   def inspect
     result = []
     result << "Status(name: #{@name.inspect}"

data/lib/jirametrics/status_collection.rb

@@ -4,105 +4,106 @@ class StatusNotFoundError < StandardError
 end
 
 class StatusCollection
+  attr_reader :historical_status_mappings
+
   def initialize
     @list = []
+    @historical_status_mappings = {} # 'name:id' => category
   end
 
-  def filter_status_names category_name:, including: nil, excluding: nil
-    including = expand_statuses including
-    excluding = expand_statuses excluding
+  # Return the status matching this id or nil if it can't be found.
+  def find_by_id id
+    @list.find { |status| status.id == id }
+  end
 
-    @list.filter_map do |status|
-      keep = status.category.name == category_name ||
-        including.any? { |s| s.name == status.name }
-      keep = false if excluding.any? { |s| s.name == status.name }
+  def find_all_by_name identifier
+    name, id = parse_name_id identifier
 
-      status.name if keep
-    end
-  end
+    if id
+      status = find_by_id id
+      return [] if status.nil?
 
-  def expand_statuses names_or_ids
-    result = []
-    return result if names_or_ids.nil?
-
-    names_or_ids = [names_or_ids] unless names_or_ids.is_a? Array
-
-    names_or_ids.each do |name_or_id|
-      status = @list.find { |s| s.name == name_or_id || s.id == name_or_id }
-      if status.nil?
-        if block_given?
-          yield name_or_id
-          next
-        else
-          all_status_names = @list.collect { |s| "#{s.name.inspect}:#{s.id.inspect}" }.uniq.sort.join(', ')
-          raise StatusNotFoundError, "Status not found: \"#{name_or_id}\". Possible statuses are: #{all_status_names}"
-        end
+      if name && status.name != name
+        raise "Specified status ID of #{id} does not match specified name #{name.inspect}. " \
+          "You might have meant one of these: #{self}."
       end
-
-      result << status
+      [status]
+    else
+      @list.select { |status| status.name == name }
     end
-    result
   end
 
-  def todo including: nil, excluding: nil
-    filter_status_names category_name: 'To Do', including: including, excluding: excluding
+  def find_all_categories
+    @list
+      .collect(&:category)
+      .uniq
+      .sort_by(&:id)
   end
 
-  def in_progress including: nil, excluding: nil
-    filter_status_names category_name: 'In Progress', including: including, excluding: excluding
+  def parse_name_id name
+    # Names could arrive in one of the following formats: "Done:3", "3", "Done"
+    if name =~ /^(.*):(\d+)$/
+      [$1, $2.to_i]
+    elsif name.match?(/^\d+$/)
+      [nil, name.to_i]
+    else
+      [name, nil]
+    end
   end
 
-  def done including: nil, excluding: nil
-    filter_status_names category_name: 'Done', including: including, excluding: excluding
-  end
+  def find_all_categories_by_name identifier
+    key = nil
+    id = nil
 
-  # Return the status matching this id or nil if it can't be found.
-  def find_by_id id
-    @list.find { |status| status.id == id }
+    if identifier.is_a? Symbol
+      key = identifier.to_s
+    else
+      name, id = parse_name_id identifier
+    end
+
+    find_all_categories.select { |c| c.id == id || c.name == name || c.key == key }
   end
 
-  def find_all_by_name name
-    @list.select { |status| status.name == name }
+  def collect(&block) = @list.collect(&block)
+  def find(&block) = @list.find(&block)
+  def each(&block) = @list.each(&block)
+  def select(&block) = @list.select(&block)
+  def <<(arg) = @list << arg
+  def empty? = @list.empty?
+  def clear = @list.clear
+  def delete(object) = @list.delete(object)
+
+  def to_s
+    "[#{@list.sort.join(', ')}]"
   end
 
-  def find_category_by_name name
-    category = @list.find { |status| status.category.name == name }&.category
-    unless category
-      set = Set.new
-      @list.each do |status|
-        set << status.category.to_s
-      end
-      raise "Unable to find status category #{name.inspect} in [#{set.to_a.sort.join(', ')}]"
-    end
-    category
+  def inspect
+    "StatusCollection#{self}"
   end
 
-  # This is used to create a status that was found in the history but has since been deleted.
   def fabricate_status_for id:, name:
-    first_in_progress_status = @list.find { |s| s.category.indeterminate? }
-    raise "Can't find even one in-progress status in [#{set.to_a.sort.join(', ')}]" unless first_in_progress_status
+    category = @historical_status_mappings["#{name.inspect}:#{id.inspect}"]
+    category = in_progress_category if category.nil?
 
     status = Status.new(
       name: name,
       id: id,
-      category_name: first_in_progress_status.category.name,
-      category_id: first_in_progress_status.category.id,
-      category_key: first_in_progress_status.category.key
+      category_name: category.name,
+      category_id: category.id,
+      category_key: category.key,
+      artificial: true
     )
-    self << status
+    @list << status
     status
   end
 
-  def collect(&block) = @list.collect(&block)
-  def find(&block) = @list.find(&block)
-  def each(&block) = @list.each(&block)
-  def select(&block) = @list.select(&block)
-  def <<(arg) = @list << arg
-  def empty? = @list.empty?
-  def clear = @list.clear
-  def delete(object) = @list.delete(object)
+  private
 
-  def inspect
-    "StatusCollection(#{@list.join(', ')})"
+  # Return the in-progress category or raise an error if we can't find one.
+  def in_progress_category
+    first_in_progress_status = find { |s| s.category.indeterminate? }
+    raise "Can't find even one in-progress status in #{self}" unless first_in_progress_status
+
+    first_in_progress_status.category
   end
 end

data/lib/jirametrics/value_equality.rb

@@ -9,9 +9,9 @@ module ValueEquality
       names = object.instance_variables
       if object.respond_to? :value_equality_ignored_variables
         ignored_variables = object.value_equality_ignored_variables
-        names.reject! { |n| ignored_variables.include? n }
+        names.reject! { |n| ignored_variables.include? n.to_sym }
       end
-      names.map { |variable| instance_variable_get variable }
+      names.map { |variable| object.instance_variable_get variable }
     end
 
     code.call(self) == code.call(other)

data/lib/jirametrics.rb

@@ -68,7 +68,6 @@ class JiraMetrics < Thor
     require 'jirametrics/grouping_rules'
     require 'jirametrics/daily_wip_chart'
     require 'jirametrics/groupable_issue_chart'
-    require 'jirametrics/discard_changes_before'
     require 'jirametrics/css_variable'
 
     require 'jirametrics/aggregate_config'