jirametrics 2.8 → 2.10

data/lib/jirametrics/project_config.rb

@@ -4,11 +4,9 @@ require 'time'
 require 'jirametrics/status_collection'
 
 class ProjectConfig
-  include DiscardChangesBefore
-
   attr_reader :target_path, :jira_config, :all_boards, :possible_statuses,
     :download_config, :file_configs, :exporter, :data_version, :name, :board_configs,
-    :settings, :aggregate_config
+    :settings, :aggregate_config, :discarded_changes_data
   attr_accessor :time_range, :jira_url, :id
 
   def initialize exporter:, jira_config:, block:, target_path: '.', name: '', id: nil
@@ -32,14 +30,13 @@ class ProjectConfig
   end
 
   def data_downloaded?
-    File.exist? File.join(@target_path, "#{get_file_prefix}_meta.json")
+    file_system.file_exist? File.join(@target_path, "#{get_file_prefix}_meta.json")
   end
 
   def load_data
     return if @has_loaded_data
 
     @has_loaded_data = true
-    load_all_boards
     @id = guess_project_id
     load_project_metadata
     load_sprints
@@ -53,16 +50,13 @@ class ProjectConfig
 
     return if load_only
 
-    @board_configs.each do |board_config|
-      board_config.run
-    end
     @file_configs.each do |file_config|
       file_config.run
     end
   end
 
   def load_settings
-    # This is the wierd exception that we don't ever want mocked out so we skip FileSystem entirely.
+    # This is the weird exception that we don't ever want mocked out so we skip FileSystem entirely.
     JSON.parse(File.read(File.join(__dir__, 'settings.json'), encoding: 'UTF-8'))
   end
 
@@ -112,6 +106,7 @@ class ProjectConfig
 
   def board id:, &block
     config = BoardConfig.new(id: id, block: block, project_config: self)
+    config.run if data_downloaded?
     @board_configs << config
   end
 
@@ -128,6 +123,8 @@ class ProjectConfig
     # is set but before anything inside the project block is run. If only we had made file_prefix an attribute
     # on project, we wouldn't have this ugliness. 🤷‍♂️
     load_status_category_mappings
+    load_status_history
+    load_all_boards
 
     @file_prefix
   end
@@ -138,29 +135,28 @@ class ProjectConfig
     @file_prefix
   end
 
-  def status_guesses
-    statuses = {}
+  # Walk across all the issues and find any status with that name. Return a list of ids that match.
+  def find_ids_by_status_name_across_all_issues name
+    ids = Set.new
+
     issues.each do |issue|
       issue.changes.each do |change|
         next unless change.status?
 
-        (statuses[change.value] ||= Set.new) << change.value_id
-        (statuses[change.old_value] ||= Set.new) << change.old_value_id if change.old_value
+        ids << change.value_id.to_i if change.value == name
+        ids << change.old_value_id.to_i if change.old_value == name
       end
     end
-    statuses
+    ids.to_a
   end
 
   def status_category_mapping status:, category:
-    # Status might just be a name like "Review" or it might be a name:id pair like "Review:4"
-    parse_status = ->(line) { line =~ /^(.+):(\d+)$/ ? [$1, $2.to_i] : [line, nil] }
-
-    status, status_id = parse_status.call status
-    category, category_id = parse_status.call category
+    status, status_id = possible_statuses.parse_name_id status
+    category, category_id = possible_statuses.parse_name_id category
 
     if status_id.nil?
-      guesses = status_guesses[status]
-      if guesses.nil?
+      guesses = find_ids_by_status_name_across_all_issues status
+      if guesses.empty?
         file_system.warning "For status_category_mapping status: #{status.inspect}, category: #{category.inspect}\n" \
           "Cannot guess status id for #{status.inspect} as no statuses found anywhere in the issues " \
           "histories with that name. Since we can't find it, you probably don't need this mapping anymore so we're " \
@@ -174,21 +170,31 @@ class ProjectConfig
       end
 
       status_id = guesses.first
-      # TODO: This should be a deprecation instead but we can't currently assert against those
       file_system.log "status_category_mapping for #{status.inspect} has been mapped to id #{status_id}. " \
         "If that's incorrect then specify the status_id."
     end
 
-    found_category = possible_statuses.find_category_by_name category
-    found_category_id = found_category&.id # possible_statuses.find_category_id_by_name category
-    if category_id && category_id != found_category_id
-      raise "ID is incorrect for status category #{category.inspect}. Did you mean #{found_category_id}?"
+    possible_categories = possible_statuses.find_all_categories_by_name category
+    if possible_categories.empty?
+      all = possible_statuses.find_all_categories.join(', ')
+      raise "No status categories found for name #{category.inspect} in [#{all}]. " \
+        'Either fix the name or add an ID.'
+    elsif possible_categories.size > 1
+      # Theoretically impossible and yet we've seen wierder things out of Jira so we're prepared.
+      raise "More than one status category found with the name #{category.inspect} in " \
+        "[#{possible_categories.join(', ')}]. Either fix the name or add an ID"
+    end
+
+    found_category = possible_categories.first
+
+    if category_id && category_id != found_category.id
+      raise "ID is incorrect for status category #{category.inspect}. Did you mean #{found_category.id}?"
     end
 
     add_possible_status(
       Status.new(
         name: status, id: status_id,
-        category_name: category, category_id: found_category_id, category_key: found_category.key
+        category_name: category, category_id: found_category.id, category_key: found_category.key
       )
     )
   end
@@ -211,7 +217,14 @@ class ProjectConfig
     end
 
     # We're registering one we already knew about. This may happen if someone specified a status_category_mapping
-    # for something that was already returned from jira. This isn't  problem so just ignore it.
+    # for something that was already returned from jira.
+    #
+    # You may be looking at this code and thinking of changing it to spit out a warning since obviously
+    # the user has made a mistake. Unfortunately, they may not have made any mistake. Due to inconsistency with the
+    # status API, it's possible for two different people to make a request to the same API at the same time and get
+    # back a different set of statuses. So that means that some people might need more status/categories mappings than
+    # other people for exactly the same instance. See this article for more on that API:
+    # https://agiletechnicalexcellence.com/2024/04/12/jira-api-statuses.html
     existing_status
   end
 
@@ -222,7 +235,6 @@ class ProjectConfig
       board_id = $1.to_i
       load_board board_id: board_id, filename: "#{@target_path}#{file}"
     end
-    raise "No boards found for #{get_file_prefix.inspect} in #{@target_path.inspect}" if @all_boards.empty?
   end
 
   def load_board board_id:, filename:
@@ -235,9 +247,7 @@ class ProjectConfig
 
   def load_status_category_mappings
     filename = File.join @target_path, "#{get_file_prefix}_statuses.json"
-
-    # We may not always have this file. Load it if we can.
-    return unless File.exist? filename
+    return unless file_system.file_exist? filename
 
     file_system
       .load_json(filename)
@@ -245,6 +255,22 @@ class ProjectConfig
       .each { |status| add_possible_status status }
   end
 
+  def load_status_history
+    filename = File.join @target_path, "#{get_file_prefix}_status_history.json"
+    return unless file_system.file_exist? filename
+
+    file_system.log '  Loading historical statuses', also_write_to_stderr: true
+    file_system
+      .load_json(filename)
+      .map { |snippet| Status.from_raw(snippet) }
+      .each { |status| possible_statuses.historical_status_mappings[status.to_s] = status.category }
+
+    possible_statuses
+  rescue => e # rubocop:disable Style/RescueStandardError
+    file_system.warning "Unable to load status history due to #{e.message.inspect}. If this is because of a " \
+      'malformed file then it should be fixed on the next download.'
+  end
+
   def load_sprints
     file_system.foreach(@target_path) do |file|
       next unless file =~ /^#{get_file_prefix}_board_(\d+)_sprints_\d+.json$/
@@ -478,21 +504,52 @@ class ProjectConfig
     Anonymizer.new(project_config: self).run
   end
 
-  def discard_changes_before_hook issues_cutoff_times
-    issues_cutoff_times.each do |issue, cutoff_time|
-      days = (cutoff_time.to_date - issue.changes.first.time.to_date).to_i + 1
-      message = "#{issue.key}(#{issue.type}) discarding #{days} "
-      if days == 1
-        message << "day of data on #{cutoff_time.to_date}"
-      else
-        message << "days of data from #{issue.changes.first.time.to_date} to #{cutoff_time.to_date}"
+  def file_system
+    @exporter.file_system
+  end
+
+  def discard_changes_before status_becomes: nil, &block
+    if status_becomes
+      status_becomes = [status_becomes] unless status_becomes.is_a? Array
+
+      block = lambda do |issue|
+        trigger_statuses = status_becomes.collect do |status_name|
+          if status_name == :backlog
+            issue.board.backlog_statuses
+          else
+            possible_statuses.find_all_by_name status_name
+          end
+        end.flatten
+
+        next if trigger_statuses.empty?
+
+        trigger_status_ids = trigger_statuses.collect(&:id)
+
+        time = nil
+        issue.status_changes.each do |change|
+          time = change.time if trigger_status_ids.include?(change.value_id) # && change.artificial? == false
+        end
+        time
       end
-      exporter.file_system.log message
     end
-    exporter.file_system.log "Discarded data from #{issues_cutoff_times.count} issues out of a total #{issues.size}"
-  end
 
-  def file_system
-    @exporter.file_system
+    issues.each do |issue|
+      cutoff_time = block.call(issue)
+      next if cutoff_time.nil?
+
+      original_start_time = issue.board.cycletime.started_stopped_times(issue).first
+      next if original_start_time.nil?
+
+      issue.discard_changes_before cutoff_time
+
+      next unless cutoff_time
+      next if original_start_time > cutoff_time # ie the cutoff would have made no difference.
+
+      (@discarded_changes_data ||= []) << {
+        cutoff_time: cutoff_time,
+        original_start_time: original_start_time,
+        issue: issue
+      }
+    end
   end
 end

data/lib/jirametrics/status.rb

@@ -3,7 +3,6 @@
 require 'jirametrics/value_equality'
 
 class Status
-  include ValueEquality
   attr_reader :id, :project_id, :category
   attr_accessor :name
 
@@ -20,6 +19,17 @@ class Status
       "#{name.inspect}:#{id.inspect}"
     end
 
+    def <=> other
+      id <=> other.id
+    end
+
+    def == other
+      id == other.id
+    end
+
+    def eql?(other) = id.eql?(other.id)
+    def hash = id.hash
+
     def new? = (@key == 'new')
     def indeterminate? = (@key == 'indeterminate')
     def done? = (@key == 'done')
@@ -28,12 +38,6 @@ class Status
   def self.from_raw raw
     category_config = raw['statusCategory']
 
-    legal_keys = %w[new indeterminate done]
-    unless legal_keys.include? category_config['key']
-      puts "Category key #{category_config['key'].inspect} should be one of #{legal_keys.inspect}. Found:\n" \
-        "#{category_config}"
-    end
-
     Status.new(
       name: raw['name'],
       id: raw['id'].to_i,
@@ -74,9 +78,21 @@ class Status
   end
 
   def == other
+    return false unless other.is_a? Status
+
     @id == other.id && @name == other.name && @category.id == other.category.id && @category.name == other.category.name
   end
 
+  def eql?(other)
+    self == other
+  end
+
+  def <=> other
+    result = @name.casecmp(other.name)
+    result = @id <=> other.id if result.zero?
+    result
+  end
+
   def inspect
     result = []
     result << "Status(name: #{@name.inspect}"

data/lib/jirametrics/status_collection.rb

@@ -4,105 +4,106 @@ class StatusNotFoundError < StandardError
 end
 
 class StatusCollection
+  attr_reader :historical_status_mappings
+
   def initialize
     @list = []
+    @historical_status_mappings = {} # 'name:id' => category
   end
 
-  def filter_status_names category_name:, including: nil, excluding: nil
-    including = expand_statuses including
-    excluding = expand_statuses excluding
+  # Return the status matching this id or nil if it can't be found.
+  def find_by_id id
+    @list.find { |status| status.id == id }
+  end
 
-    @list.filter_map do |status|
-      keep = status.category.name == category_name ||
-        including.any? { |s| s.name == status.name }
-      keep = false if excluding.any? { |s| s.name == status.name }
+  def find_all_by_name identifier
+    name, id = parse_name_id identifier
 
-      status.name if keep
-    end
-  end
+    if id
+      status = find_by_id id
+      return [] if status.nil?
 
-  def expand_statuses names_or_ids
-    result = []
-    return result if names_or_ids.nil?
-
-    names_or_ids = [names_or_ids] unless names_or_ids.is_a? Array
-
-    names_or_ids.each do |name_or_id|
-      status = @list.find { |s| s.name == name_or_id || s.id == name_or_id }
-      if status.nil?
-        if block_given?
-          yield name_or_id
-          next
-        else
-          all_status_names = @list.collect { |s| "#{s.name.inspect}:#{s.id.inspect}" }.uniq.sort.join(', ')
-          raise StatusNotFoundError, "Status not found: \"#{name_or_id}\". Possible statuses are: #{all_status_names}"
-        end
+      if name && status.name != name
+        raise "Specified status ID of #{id} does not match specified name #{name.inspect}. " \
+          "You might have meant one of these: #{self}."
       end
-
-      result << status
+      [status]
+    else
+      @list.select { |status| status.name == name }
     end
-    result
   end
 
-  def todo including: nil, excluding: nil
-    filter_status_names category_name: 'To Do', including: including, excluding: excluding
+  def find_all_categories
+    @list
+      .collect(&:category)
+      .uniq
+      .sort_by(&:id)
   end
 
-  def in_progress including: nil, excluding: nil
-    filter_status_names category_name: 'In Progress', including: including, excluding: excluding
+  def parse_name_id name
+    # Names could arrive in one of the following formats: "Done:3", "3", "Done"
+    if name =~ /^(.*):(\d+)$/
+      [$1, $2.to_i]
+    elsif name.match?(/^\d+$/)
+      [nil, name.to_i]
+    else
+      [name, nil]
+    end
   end
 
-  def done including: nil, excluding: nil
-    filter_status_names category_name: 'Done', including: including, excluding: excluding
-  end
+  def find_all_categories_by_name identifier
+    key = nil
+    id = nil
 
-  # Return the status matching this id or nil if it can't be found.
-  def find_by_id id
-    @list.find { |status| status.id == id }
+    if identifier.is_a? Symbol
+      key = identifier.to_s
+    else
+      name, id = parse_name_id identifier
+    end
+
+    find_all_categories.select { |c| c.id == id || c.name == name || c.key == key }
   end
 
-  def find_all_by_name name
-    @list.select { |status| status.name == name }
+  def collect(&block) = @list.collect(&block)
+  def find(&block) = @list.find(&block)
+  def each(&block) = @list.each(&block)
+  def select(&block) = @list.select(&block)
+  def <<(arg) = @list << arg
+  def empty? = @list.empty?
+  def clear = @list.clear
+  def delete(object) = @list.delete(object)
+
+  def to_s
+    "[#{@list.sort.join(', ')}]"
   end
 
-  def find_category_by_name name
-    category = @list.find { |status| status.category.name == name }&.category
-    unless category
-      set = Set.new
-      @list.each do |status|
-        set << status.category.to_s
-      end
-      raise "Unable to find status category #{name.inspect} in [#{set.to_a.sort.join(', ')}]"
-    end
-    category
+  def inspect
+    "StatusCollection#{self}"
   end
 
-  # This is used to create a status that was found in the history but has since been deleted.
   def fabricate_status_for id:, name:
-    first_in_progress_status = @list.find { |s| s.category.indeterminate? }
-    raise "Can't find even one in-progress status in [#{set.to_a.sort.join(', ')}]" unless first_in_progress_status
+    category = @historical_status_mappings["#{name.inspect}:#{id.inspect}"]
+    category = in_progress_category if category.nil?
 
     status = Status.new(
       name: name,
       id: id,
-      category_name: first_in_progress_status.category.name,
-      category_id: first_in_progress_status.category.id,
-      category_key: first_in_progress_status.category.key
+      category_name: category.name,
+      category_id: category.id,
+      category_key: category.key,
+      artificial: true
     )
-    self << status
+    @list << status
     status
   end
 
-  def collect(&block) = @list.collect(&block)
-  def find(&block) = @list.find(&block)
-  def each(&block) = @list.each(&block)
-  def select(&block) = @list.select(&block)
-  def <<(arg) = @list << arg
-  def empty? = @list.empty?
-  def clear = @list.clear
-  def delete(object) = @list.delete(object)
+  private
 
-  def inspect
-    "StatusCollection(#{@list.join(', ')})"
+  # Return the in-progress category or raise an error if we can't find one.
+  def in_progress_category
+    first_in_progress_status = find { |s| s.category.indeterminate? }
+    raise "Can't find even one in-progress status in #{self}" unless first_in_progress_status
+
+    first_in_progress_status.category
   end
 end

data/lib/jirametrics/value_equality.rb

@@ -9,9 +9,9 @@ module ValueEquality
       names = object.instance_variables
       if object.respond_to? :value_equality_ignored_variables
         ignored_variables = object.value_equality_ignored_variables
-        names.reject! { |n| ignored_variables.include? n }
+        names.reject! { |n| ignored_variables.include? n.to_sym }
       end
-      names.map { |variable| instance_variable_get variable }
+      names.map { |variable| object.instance_variable_get variable }
     end
 
     code.call(self) == code.call(other)

data/lib/jirametrics.rb

@@ -68,7 +68,6 @@ class JiraMetrics < Thor
     require 'jirametrics/grouping_rules'
     require 'jirametrics/daily_wip_chart'
     require 'jirametrics/groupable_issue_chart'
-    require 'jirametrics/discard_changes_before'
     require 'jirametrics/css_variable'
 
     require 'jirametrics/aggregate_config'

metadata

@@ -1,14 +1,13 @@
 --- !ruby/object:Gem::Specification
 name: jirametrics
 version: !ruby/object:Gem::Version
-  version: '2.8'
+  version: '2.10'
 platform: ruby
 authors:
 - Mike Bowler
-autorequire:
 bindir: bin
 cert_chain: []
-date: 2024-12-30 00:00:00.000000000 Z
+date: 2025-02-06 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: random-word
@@ -52,8 +51,7 @@ dependencies:
     - - "~>"
       - !ruby/object:Gem::Version
         version: 1.2.2
-description: Tool to extract metrics from Jira and export to either a report or to
-  CSV files
+description: Extract metrics from Jira and export to either a report or to CSV files
 email: mbowler@gargoylesoftware.com
 executables:
 - jirametrics
@@ -84,7 +82,6 @@ files:
 - lib/jirametrics/daily_wip_chart.rb
 - lib/jirametrics/data_quality_report.rb
 - lib/jirametrics/dependency_chart.rb
-- lib/jirametrics/discard_changes_before.rb
 - lib/jirametrics/download_config.rb
 - lib/jirametrics/downloader.rb
 - lib/jirametrics/estimate_accuracy_chart.rb
@@ -139,7 +136,6 @@ metadata:
   bug_tracker_uri: https://github.com/mikebowler/jirametrics/issues
   changelog_uri: https://jirametrics.org/changes
   documentation_uri: https://jirametrics.org
-post_install_message:
 rdoc_options: []
 require_paths:
 - lib
@@ -154,8 +150,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.5.21
-signing_key:
+rubygems_version: 3.6.2
 specification_version: 4
 summary: Extract Jira metrics
 test_files: []

data/lib/jirametrics/discard_changes_before.rb

@@ -1,37 +0,0 @@
-# frozen_string_literal: true
-
-module DiscardChangesBefore
-  def discard_changes_before status_becomes: nil, &block
-    if status_becomes
-      status_becomes = [status_becomes] unless status_becomes.is_a? Array
-
-      block = lambda do |issue|
-        trigger_statuses = status_becomes.collect do |status_name|
-          if status_name == :backlog
-            issue.board.backlog_statuses.collect(&:name)
-          else
-            status_name
-          end
-        end.flatten
-
-        time = nil
-        issue.changes.each do |change|
-          time = change.time if change.status? && trigger_statuses.include?(change.value) && change.artificial? == false
-        end
-        time
-      end
-    end
-
-    issues_cutoff_times = []
-    issues.each do |issue|
-      cutoff_time = block.call(issue)
-      issues_cutoff_times << [issue, cutoff_time] if cutoff_time
-    end
-
-    discard_changes_before_hook issues_cutoff_times
-
-    issues_cutoff_times.each do |issue, cutoff_time|
-      issue.discard_changes_before cutoff_time
-    end
-  end
-end