jirametrics 2.8 → 2.11

data/lib/jirametrics/issue.rb

@@ -13,7 +13,12 @@ class Issue
     @changes = []
     @board = board
 
+    # We only check for this here because if a board isn't passed in then things will fail much
+    # later and be hard to find. Let's find out early.
     raise "No board for issue #{key}" if board.nil?
+
+    # There are cases where we create an Issue of fragments like linked issues and those won't have
+    # changelogs.
     return unless @raw['changelog']
 
     load_history_into_changes
@@ -44,14 +49,26 @@ class Issue
 
   def summary = @raw['fields']['summary']
 
-  def status = Status.from_raw(@raw['fields']['status'])
-
   def labels = @raw['fields']['labels'] || []
 
   def author = @raw['fields']['creator']&.[]('displayName') || ''
 
   def resolution = @raw['fields']['resolution']&.[]('name')
 
+  def status
+    @status = Status.from_raw(@raw['fields']['status']) unless @status
+    @status
+  end
+
+  def status= status
+    @status = status
+  end
+
+  def due_date
+    text = @raw['fields']['duedate']
+    text.nil? ? nil : Date.parse(text)
+  end
+
   def url
     # Strangely, the URL isn't anywhere in the returned data so we have to fabricate it.
     "#{@board.server_url_prefix}/browse/#{key}"
@@ -93,9 +110,7 @@ class Issue
 
   def still_in
     result = nil
-    @changes.each do |change|
-      next unless change.status?
-
+    status_changes.each do |change|
       current_status_matched = yield change
 
       if current_status_matched && result.nil?
@@ -117,14 +132,19 @@ class Issue
 
   # If it ever entered one of these categories and it's still there then what was the last time it entered
   def still_in_status_category *category_names
+    category_ids = find_status_category_ids_by_names category_names
+
     still_in do |change|
-      status = find_status_by_id change.value_id, name: change.value
-      category_names.include?(status.category.name) || category_names.include?(status.category.id)
+      status = find_or_create_status id: change.value_id, name: change.value
+      category_ids.include? status.category.id
     end
   end
 
   def most_recent_status_change
-    changes.reverse.find { |change| change.status? }
+    # Any issue that we loaded from its own file will always have a status as we artificially insert a status
+    # change to represent creation. Issues that were just fragments referenced from a main issue (ie a linked issue)
+    # may not have any status changes as we have no idea when it was created. This will be nil in that case
+    status_changes.last
   end
 
   # Are we currently in this status? If yes, then return the most recent status change.
@@ -137,43 +157,50 @@ class Issue
 
   # Are we currently in this status category? If yes, then return the most recent status change.
   def currently_in_status_category *category_names
+    category_ids = find_status_category_ids_by_names category_names
+
     change = most_recent_status_change
     return false if change.nil?
 
-    status = find_status_by_id change.value_id, name: change.value
-    change if status && category_names.include?(status.category.name)
+    status = find_or_create_status id: change.value_id, name: change.value
+    change if status && category_ids.include?(status.category.id)
   end
 
-  def find_status_by_id id, name: nil
+  def find_or_create_status id:, name:
     status = board.possible_statuses.find_by_id(id)
-    return status if status
 
-    status = board.possible_statuses.fabricate_status_for id: id, name: name
+    unless status
+      # Have to pull this list before the call to fabricate or else the warning will incorrectly
+      # list this status as one it actually found
+      found_statuses = board.possible_statuses.to_s
+
+      status = board.possible_statuses.fabricate_status_for id: id, name: name
 
-    message = +'The history for issue '
-    message << key
-    message << ' references a status ('
-    message << name.inspect << ':' if name
-    message << id.to_s
-    message << ') that can\'t be found in ['
-    message << board.possible_statuses.collect(&:to_s).join(', ')
-    message << "]. We are guessing that this belongs to the #{status.category} status category "
-    message << 'and that may be wrong. See https://jirametrics.org/faq/#q1 for more details'
-    board.project_config.file_system.warning message
+      message = +'The history for issue '
+      message << key
+      message << ' references the status ('
+      message << "#{name.inspect}:#{id.inspect}"
+      message << ') that can\'t be found. We are guessing that this belongs to the '
+      message << status.category.to_s
+      message << ' status category but that may be wrong. See https://jirametrics.org/faq/#q1 for more '
+      message << 'details on defining statuses.'
+      board.project_config.file_system.warning message, more: "The statuses we did find are: #{found_statuses}"
+    end
 
     status
   end
 
   def first_status_change_after_created
-    @changes.find { |change| change.status? && change.artificial? == false }
+    status_changes.find { |change| change.artificial? == false }
   end
 
   def first_time_in_status_category *category_names
-    @changes.each do |change|
-      next unless change.status?
+    category_ids = find_status_category_ids_by_names category_names
 
-      category = find_status_by_id(change.value_id).category.name
-      return change if category_names.include? category
+    status_changes.each do |change|
+      to_status = find_or_create_status(id: change.value_id, name: change.value)
+      id = to_status.category.id
+      return change if category_ids.include? id
     end
     nil
   end
@@ -593,12 +620,17 @@ class Issue
     end
 
     (changes + (@discarded_changes || [])).each do |change|
-      value = change.value
-      old_value = change.old_value
+      if change.status?
+        value = "#{change.value.inspect}:#{change.value_id.inspect}"
+        old_value = change.old_value ? "#{change.old_value.inspect}:#{change.old_value_id.inspect}" : nil
+      else
+        value = compact_text(change.value).inspect
+        old_value = change.old_value ? compact_text(change.old_value).inspect : nil
+      end
 
       message = +''
-      message << "#{compact_text(old_value).inspect} -> " unless old_value.nil? || old_value.empty?
-      message << compact_text(value).inspect
+      message << "#{old_value} -> " unless old_value.nil? || old_value.empty?
+      message << value
       if change.artificial?
         message << ' (Artificial entry)' if change.artificial?
       else
@@ -645,6 +677,10 @@ class Issue
     end
   end
 
+  def status_changes
+    @changes.select { |change| change.status? }
+  end
+
   private
 
   def assemble_author raw
@@ -720,4 +756,13 @@ class Issue
       'toString' => first_status
     }
   end
+
+  def find_status_category_ids_by_names category_names
+    category_names.filter_map do |name|
+      list = board.possible_statuses.find_all_categories_by_name name
+      raise "No status categories found for name: #{name}" if list.empty?
+
+      list
+    end.flatten.collect(&:id)
+  end
 end

data/lib/jirametrics/jira_gateway.rb

@@ -14,9 +14,15 @@ class JiraGateway
   def call_url relative_url:
     command = make_curl_command url: "#{@jira_url}#{relative_url}"
     result = call_command command
-    JSON.parse result
-  rescue => e # rubocop:disable Style/RescueStandardError
-    raise "Error #{e.message.inspect} when parsing result: #{result.inspect}"
+    begin
+      json = JSON.parse(result)
+    rescue # rubocop:disable Style/RescueStandardError
+      raise "Error when parsing result: #{result.inspect}"
+    end
+
+    raise "Download failed with: #{JSON.pretty_generate(json)}" unless json_successful?(json)
+
+    json
   end
 
   def call_command command
@@ -61,4 +67,11 @@ class JiraGateway
     command << " --url \"#{url}\""
     command
   end
+
+  def json_successful? json
+    return false if json.is_a?(Hash) && (json['error'] || json['errorMessages'] || json['errorMessage'])
+    return false if json.is_a?(Array) && json.first == 'errorMessage'
+
+    true
+  end
 end

data/lib/jirametrics/project_config.rb

@@ -4,11 +4,9 @@ require 'time'
 require 'jirametrics/status_collection'
 
 class ProjectConfig
-  include DiscardChangesBefore
-
   attr_reader :target_path, :jira_config, :all_boards, :possible_statuses,
     :download_config, :file_configs, :exporter, :data_version, :name, :board_configs,
-    :settings, :aggregate_config
+    :settings, :aggregate_config, :discarded_changes_data
   attr_accessor :time_range, :jira_url, :id
 
   def initialize exporter:, jira_config:, block:, target_path: '.', name: '', id: nil
@@ -32,14 +30,13 @@ class ProjectConfig
   end
 
   def data_downloaded?
-    File.exist? File.join(@target_path, "#{get_file_prefix}_meta.json")
+    file_system.file_exist? File.join(@target_path, "#{get_file_prefix}_meta.json")
   end
 
   def load_data
     return if @has_loaded_data
 
     @has_loaded_data = true
-    load_all_boards
     @id = guess_project_id
     load_project_metadata
     load_sprints
@@ -53,16 +50,13 @@ class ProjectConfig
 
     return if load_only
 
-    @board_configs.each do |board_config|
-      board_config.run
-    end
     @file_configs.each do |file_config|
       file_config.run
     end
   end
 
   def load_settings
-    # This is the wierd exception that we don't ever want mocked out so we skip FileSystem entirely.
+    # This is the weird exception that we don't ever want mocked out so we skip FileSystem entirely.
     JSON.parse(File.read(File.join(__dir__, 'settings.json'), encoding: 'UTF-8'))
   end
 
@@ -112,6 +106,7 @@ class ProjectConfig
 
   def board id:, &block
     config = BoardConfig.new(id: id, block: block, project_config: self)
+    config.run if data_downloaded?
     @board_configs << config
   end
 
@@ -128,6 +123,8 @@ class ProjectConfig
     # is set but before anything inside the project block is run. If only we had made file_prefix an attribute
     # on project, we wouldn't have this ugliness. 🤷‍♂️
     load_status_category_mappings
+    load_status_history
+    load_all_boards
 
     @file_prefix
   end
@@ -138,29 +135,28 @@ class ProjectConfig
     @file_prefix
   end
 
-  def status_guesses
-    statuses = {}
+  # Walk across all the issues and find any status with that name. Return a list of ids that match.
+  def find_ids_by_status_name_across_all_issues name
+    ids = Set.new
+
     issues.each do |issue|
       issue.changes.each do |change|
         next unless change.status?
 
-        (statuses[change.value] ||= Set.new) << change.value_id
-        (statuses[change.old_value] ||= Set.new) << change.old_value_id if change.old_value
+        ids << change.value_id.to_i if change.value == name
+        ids << change.old_value_id.to_i if change.old_value == name
       end
     end
-    statuses
+    ids.to_a
   end
 
   def status_category_mapping status:, category:
-    # Status might just be a name like "Review" or it might be a name:id pair like "Review:4"
-    parse_status = ->(line) { line =~ /^(.+):(\d+)$/ ? [$1, $2.to_i] : [line, nil] }
-
-    status, status_id = parse_status.call status
-    category, category_id = parse_status.call category
+    status, status_id = possible_statuses.parse_name_id status
+    category, category_id = possible_statuses.parse_name_id category
 
     if status_id.nil?
-      guesses = status_guesses[status]
-      if guesses.nil?
+      guesses = find_ids_by_status_name_across_all_issues status
+      if guesses.empty?
         file_system.warning "For status_category_mapping status: #{status.inspect}, category: #{category.inspect}\n" \
           "Cannot guess status id for #{status.inspect} as no statuses found anywhere in the issues " \
           "histories with that name. Since we can't find it, you probably don't need this mapping anymore so we're " \
@@ -174,21 +170,31 @@ class ProjectConfig
       end
 
       status_id = guesses.first
-      # TODO: This should be a deprecation instead but we can't currently assert against those
       file_system.log "status_category_mapping for #{status.inspect} has been mapped to id #{status_id}. " \
         "If that's incorrect then specify the status_id."
     end
 
-    found_category = possible_statuses.find_category_by_name category
-    found_category_id = found_category&.id # possible_statuses.find_category_id_by_name category
-    if category_id && category_id != found_category_id
-      raise "ID is incorrect for status category #{category.inspect}. Did you mean #{found_category_id}?"
+    possible_categories = possible_statuses.find_all_categories_by_name category
+    if possible_categories.empty?
+      all = possible_statuses.find_all_categories.join(', ')
+      raise "No status categories found for name #{category.inspect} in [#{all}]. " \
+        'Either fix the name or add an ID.'
+    elsif possible_categories.size > 1
+      # Theoretically impossible and yet we've seen wierder things out of Jira so we're prepared.
+      raise "More than one status category found with the name #{category.inspect} in " \
+        "[#{possible_categories.join(', ')}]. Either fix the name or add an ID"
+    end
+
+    found_category = possible_categories.first
+
+    if category_id && category_id != found_category.id
+      raise "ID is incorrect for status category #{category.inspect}. Did you mean #{found_category.id}?"
     end
 
     add_possible_status(
       Status.new(
         name: status, id: status_id,
-        category_name: category, category_id: found_category_id, category_key: found_category.key
+        category_name: category, category_id: found_category.id, category_key: found_category.key
       )
     )
   end
@@ -211,7 +217,14 @@ class ProjectConfig
     end
 
     # We're registering one we already knew about. This may happen if someone specified a status_category_mapping
-    # for something that was already returned from jira. This isn't  problem so just ignore it.
+    # for something that was already returned from jira.
+    #
+    # You may be looking at this code and thinking of changing it to spit out a warning since obviously
+    # the user has made a mistake. Unfortunately, they may not have made any mistake. Due to inconsistency with the
+    # status API, it's possible for two different people to make a request to the same API at the same time and get
+    # back a different set of statuses. So that means that some people might need more status/categories mappings than
+    # other people for exactly the same instance. See this article for more on that API:
+    # https://agiletechnicalexcellence.com/2024/04/12/jira-api-statuses.html
     existing_status
   end
 
@@ -222,7 +235,6 @@ class ProjectConfig
       board_id = $1.to_i
       load_board board_id: board_id, filename: "#{@target_path}#{file}"
     end
-    raise "No boards found for #{get_file_prefix.inspect} in #{@target_path.inspect}" if @all_boards.empty?
   end
 
   def load_board board_id:, filename:
@@ -235,9 +247,7 @@ class ProjectConfig
 
   def load_status_category_mappings
     filename = File.join @target_path, "#{get_file_prefix}_statuses.json"
-
-    # We may not always have this file. Load it if we can.
-    return unless File.exist? filename
+    return unless file_system.file_exist? filename
 
     file_system
       .load_json(filename)
@@ -245,6 +255,22 @@ class ProjectConfig
       .each { |status| add_possible_status status }
   end
 
+  def load_status_history
+    filename = File.join @target_path, "#{get_file_prefix}_status_history.json"
+    return unless file_system.file_exist? filename
+
+    file_system.log '  Loading historical statuses', also_write_to_stderr: true
+    file_system
+      .load_json(filename)
+      .map { |snippet| Status.from_raw(snippet) }
+      .each { |status| possible_statuses.historical_status_mappings[status.to_s] = status.category }
+
+    possible_statuses
+  rescue => e # rubocop:disable Style/RescueStandardError
+    file_system.warning "Unable to load status history due to #{e.message.inspect}. If this is because of a " \
+      'malformed file then it should be fixed on the next download.'
+  end
+
   def load_sprints
     file_system.foreach(@target_path) do |file|
       next unless file =~ /^#{get_file_prefix}_board_(\d+)_sprints_\d+.json$/
@@ -478,21 +504,52 @@ class ProjectConfig
     Anonymizer.new(project_config: self).run
   end
 
-  def discard_changes_before_hook issues_cutoff_times
-    issues_cutoff_times.each do |issue, cutoff_time|
-      days = (cutoff_time.to_date - issue.changes.first.time.to_date).to_i + 1
-      message = "#{issue.key}(#{issue.type}) discarding #{days} "
-      if days == 1
-        message << "day of data on #{cutoff_time.to_date}"
-      else
-        message << "days of data from #{issue.changes.first.time.to_date} to #{cutoff_time.to_date}"
+  def file_system
+    @exporter.file_system
+  end
+
+  def discard_changes_before status_becomes: nil, &block
+    if status_becomes
+      status_becomes = [status_becomes] unless status_becomes.is_a? Array
+
+      block = lambda do |issue|
+        trigger_statuses = status_becomes.collect do |status_name|
+          if status_name == :backlog
+            issue.board.backlog_statuses
+          else
+            possible_statuses.find_all_by_name status_name
+          end
+        end.flatten
+
+        next if trigger_statuses.empty?
+
+        trigger_status_ids = trigger_statuses.collect(&:id)
+
+        time = nil
+        issue.status_changes.each do |change|
+          time = change.time if trigger_status_ids.include?(change.value_id) # && change.artificial? == false
+        end
+        time
       end
-      exporter.file_system.log message
     end
-    exporter.file_system.log "Discarded data from #{issues_cutoff_times.count} issues out of a total #{issues.size}"
-  end
 
-  def file_system
-    @exporter.file_system
+    issues.each do |issue|
+      cutoff_time = block.call(issue)
+      next if cutoff_time.nil?
+
+      original_start_time = issue.board.cycletime.started_stopped_times(issue).first
+      next if original_start_time.nil?
+
+      issue.discard_changes_before cutoff_time
+
+      next unless cutoff_time
+      next if original_start_time > cutoff_time # ie the cutoff would have made no difference.
+
+      (@discarded_changes_data ||= []) << {
+        cutoff_time: cutoff_time,
+        original_start_time: original_start_time,
+        issue: issue
+      }
+    end
   end
 end

data/lib/jirametrics/status.rb

@@ -3,7 +3,6 @@
 require 'jirametrics/value_equality'
 
 class Status
-  include ValueEquality
   attr_reader :id, :project_id, :category
   attr_accessor :name
 
@@ -20,19 +19,27 @@ class Status
       "#{name.inspect}:#{id.inspect}"
     end
 
+    def <=> other
+      id <=> other.id
+    end
+
+    def == other
+      id == other.id
+    end
+
+    def eql?(other) = id.eql?(other.id)
+    def hash = id.hash
+
     def new? = (@key == 'new')
     def indeterminate? = (@key == 'indeterminate')
     def done? = (@key == 'done')
   end
 
   def self.from_raw raw
-    category_config = raw['statusCategory']
+    raise "raw cannot be nil" if raw.nil?
 
-    legal_keys = %w[new indeterminate done]
-    unless legal_keys.include? category_config['key']
-      puts "Category key #{category_config['key'].inspect} should be one of #{legal_keys.inspect}. Found:\n" \
-        "#{category_config}"
-    end
+    category_config = raw['statusCategory']
+    raise "statusCategory can't be nil in #{category_config.inspect}" if category_config.nil?
 
     Status.new(
       name: raw['name'],
@@ -74,9 +81,21 @@ class Status
   end
 
   def == other
+    return false unless other.is_a? Status
+
     @id == other.id && @name == other.name && @category.id == other.category.id && @category.name == other.category.name
   end
 
+  def eql?(other)
+    self == other
+  end
+
+  def <=> other
+    result = @name.casecmp(other.name)
+    result = @id <=> other.id if result.zero?
+    result
+  end
+
   def inspect
     result = []
     result << "Status(name: #{@name.inspect}"