jirametrics 2.4 → 2.11

data/lib/jirametrics/project_config.rb

@@ -4,11 +4,9 @@ require 'time'
 require 'jirametrics/status_collection'
 
 class ProjectConfig
-  include DiscardChangesBefore
-
   attr_reader :target_path, :jira_config, :all_boards, :possible_statuses,
     :download_config, :file_configs, :exporter, :data_version, :name, :board_configs,
-    :settings, :aggregate_config
+    :settings, :aggregate_config, :discarded_changes_data
   attr_accessor :time_range, :jira_url, :id
 
   def initialize exporter:, jira_config:, block:, target_path: '.', name: '', id: nil
@@ -24,32 +22,42 @@ class ProjectConfig
     @all_boards = {}
     @settings = load_settings
     @id = id
+    @has_loaded_data = false
   end
 
   def evaluate_next_level
     instance_eval(&@block) if @block
   end
 
-  def run
-    unless aggregated_project?
-      load_all_boards
-      @id = guess_project_id
-      load_status_category_mappings
-      load_project_metadata
-      load_sprints
-    end
+  def data_downloaded?
+    file_system.file_exist? File.join(@target_path, "#{get_file_prefix}_meta.json")
+  end
+
+  def load_data
+    return if @has_loaded_data
+
+    @has_loaded_data = true
+    @id = guess_project_id
+    load_project_metadata
+    load_sprints
+  end
+
+  def run load_only: false
+    return if @exporter.downloading?
+
+    load_data unless aggregated_project?
     anonymize_data if @anonymizer_needed
 
-    @board_configs.each do |board_config|
-      board_config.run
-    end
+    return if load_only
+
     @file_configs.each do |file_config|
       file_config.run
     end
   end
 
   def load_settings
-    JSON.parse(file_system.load(File.join(__dir__, 'settings.json')))
+    # This is the weird exception that we don't ever want mocked out so we skip FileSystem entirely.
+    JSON.parse(File.read(File.join(__dir__, 'settings.json'), encoding: 'UTF-8'))
   end
 
   def guess_project_id
@@ -98,90 +106,188 @@ class ProjectConfig
 
   def board id:, &block
     config = BoardConfig.new(id: id, block: block, project_config: self)
+    config.run if data_downloaded?
     @board_configs << config
   end
 
-  def file_prefix prefix = nil
-    @file_prefix = prefix unless prefix.nil?
+  def file_prefix prefix
+    # The file_prefix has to be set before almost everything else. It really should have been an attribute
+    # on the project declaration itself. Hindsight is 20/20.
+    if @file_prefix
+      raise "file_prefix should only be set once. Was #{@file_prefix.inspect} and now changed to #{prefix.inspect}."
+    end
+
+    @file_prefix = prefix
+
+    # Yes, this is a wierd place to be initializing this. Unfortunately, it has to happen after the file_prefix
+    # is set but before anything inside the project block is run. If only we had made file_prefix an attribute
+    # on project, we wouldn't have this ugliness. 🤷‍♂️
+    load_status_category_mappings
+    load_status_history
+    load_all_boards
+
     @file_prefix
   end
 
-  def status_category_mapping status:, category:
-    add_possible_status Status.new(name: status, category_name: category)
+  def get_file_prefix # rubocop:disable Naming/AccessorMethodName
+    raise 'file_prefix has not been set yet. Move it to the top of the project declaration.' unless @file_prefix
+
+    @file_prefix
   end
 
-  def load_all_boards
-    Dir.foreach(@target_path) do |file|
-      next unless file =~ /^#{@file_prefix}_board_(\d+)_configuration\.json$/
+  # Walk across all the issues and find any status with that name. Return a list of ids that match.
+  def find_ids_by_status_name_across_all_issues name
+    ids = Set.new
 
-      board_id = $1.to_i
-      load_board board_id: board_id, filename: "#{@target_path}#{file}"
+    issues.each do |issue|
+      issue.changes.each do |change|
+        next unless change.status?
+
+        ids << change.value_id.to_i if change.value == name
+        ids << change.old_value_id.to_i if change.old_value == name
+      end
     end
-    raise "No boards found for #{@file_prefix} in #{@target_path.inspect}" if @all_boards.empty?
+    ids.to_a
   end
 
-  def load_board board_id:, filename:
-    board = Board.new(
-      raw: JSON.parse(file_system.load(filename)), possible_statuses: @possible_statuses
+  def status_category_mapping status:, category:
+    status, status_id = possible_statuses.parse_name_id status
+    category, category_id = possible_statuses.parse_name_id category
+
+    if status_id.nil?
+      guesses = find_ids_by_status_name_across_all_issues status
+      if guesses.empty?
+        file_system.warning "For status_category_mapping status: #{status.inspect}, category: #{category.inspect}\n" \
+          "Cannot guess status id for #{status.inspect} as no statuses found anywhere in the issues " \
+          "histories with that name. Since we can't find it, you probably don't need this mapping anymore so we're " \
+          "going to ignore it. If you really want it, then you'll need to specify a status id."
+        return
+      end
+
+      if guesses.size > 1
+        raise "Cannot guess status id as there are multiple ids for the name #{status.inspect}. Perhaps it's one " \
+          "of #{guesses.to_a.sort.inspect}. If you need this mapping then you must specify the status_id."
+      end
+
+      status_id = guesses.first
+      file_system.log "status_category_mapping for #{status.inspect} has been mapped to id #{status_id}. " \
+        "If that's incorrect then specify the status_id."
+    end
+
+    possible_categories = possible_statuses.find_all_categories_by_name category
+    if possible_categories.empty?
+      all = possible_statuses.find_all_categories.join(', ')
+      raise "No status categories found for name #{category.inspect} in [#{all}]. " \
+        'Either fix the name or add an ID.'
+    elsif possible_categories.size > 1
+      # Theoretically impossible and yet we've seen wierder things out of Jira so we're prepared.
+      raise "More than one status category found with the name #{category.inspect} in " \
+        "[#{possible_categories.join(', ')}]. Either fix the name or add an ID"
+    end
+
+    found_category = possible_categories.first
+
+    if category_id && category_id != found_category.id
+      raise "ID is incorrect for status category #{category.inspect}. Did you mean #{found_category.id}?"
+    end
+
+    add_possible_status(
+      Status.new(
+        name: status, id: status_id,
+        category_name: category, category_id: found_category.id, category_key: found_category.key
+      )
     )
-    board.project_config = self
-    @all_boards[board_id] = board
   end
 
-  def raise_with_message_about_missing_category_information all_issues = @issues
-    message = +''
-    message << "Could not determine categories for some of the statuses used in this data set.\n" \
-      "Use the 'status_category_mapping' declaration in your config to manually add one.\n" \
-      'The mappings we do know about are below:'
+  def add_possible_status status
+    existing_status = @possible_statuses.find_by_id status.id
 
-    @possible_statuses.each do |status|
-      message << "\n  status: #{status.name.inspect}, category: #{status.category_name.inspect}"
+    if existing_status && existing_status.name != status.name
+      raise "Attempting to redefine the name for status #{status.id} from " \
+        "#{existing_status.name.inspect} to #{status.name.inspect}"
     end
 
-    message << "\n\nThe ones we're missing are the following:"
+    # If it isn't there, add it and go.
+    return @possible_statuses << status unless existing_status
 
-    find_statuses_with_no_category_information(all_issues).each do |status_name|
-      message << "\n  status: #{status_name.inspect}, category: <unknown>"
+    unless status == existing_status
+      raise "Redefining status category for status #{status}. " \
+        "original: #{existing_status.category}, " \
+        "new: #{status.category}"
     end
 
-    raise message
+    # We're registering one we already knew about. This may happen if someone specified a status_category_mapping
+    # for something that was already returned from jira.
+    #
+    # You may be looking at this code and thinking of changing it to spit out a warning since obviously
+    # the user has made a mistake. Unfortunately, they may not have made any mistake. Due to inconsistency with the
+    # status API, it's possible for two different people to make a request to the same API at the same time and get
+    # back a different set of statuses. So that means that some people might need more status/categories mappings than
+    # other people for exactly the same instance. See this article for more on that API:
+    # https://agiletechnicalexcellence.com/2024/04/12/jira-api-statuses.html
+    existing_status
   end
 
-  def find_statuses_with_no_category_information all_issues
-    missing_statuses = []
-    all_issues.each do |issue|
-      issue.changes.each do |change|
-        next unless change.status?
+  def load_all_boards
+    Dir.foreach(@target_path) do |file|
+      next unless file =~ /^#{get_file_prefix}_board_(\d+)_configuration\.json$/
 
-        missing_statuses << change.value unless find_status(name: change.value)
-      end
+      board_id = $1.to_i
+      load_board board_id: board_id, filename: "#{@target_path}#{file}"
     end
-    missing_statuses.uniq
+  end
+
+  def load_board board_id:, filename:
+    board = Board.new(
+      raw: file_system.load_json(filename), possible_statuses: @possible_statuses
+    )
+    board.project_config = self
+    @all_boards[board_id] = board
   end
 
   def load_status_category_mappings
-    filename = "#{@target_path}/#{file_prefix}_statuses.json"
-    # We may not always have this file. Load it if we can.
-    return unless File.exist? filename
-
-    statuses = JSON.parse(file_system.load(filename))
-      .map { |snippet| Status.new(raw: snippet) }
-    statuses
-      .find_all { |status| status.global? }
-      .each { |status| add_possible_status status }
-    statuses
-      .find_all { |status| status.project_scoped? }
+    filename = File.join @target_path, "#{get_file_prefix}_statuses.json"
+    return unless file_system.file_exist? filename
+
+    file_system
+      .load_json(filename)
+      .map { |snippet| Status.from_raw(snippet) }
       .each { |status| add_possible_status status }
   end
 
+  def load_status_history
+    filename = File.join @target_path, "#{get_file_prefix}_status_history.json"
+    return unless file_system.file_exist? filename
+
+    file_system.log '  Loading historical statuses', also_write_to_stderr: true
+    file_system
+      .load_json(filename)
+      .map { |snippet| Status.from_raw(snippet) }
+      .each { |status| possible_statuses.historical_status_mappings[status.to_s] = status.category }
+
+    possible_statuses
+  rescue => e # rubocop:disable Style/RescueStandardError
+    file_system.warning "Unable to load status history due to #{e.message.inspect}. If this is because of a " \
+      'malformed file then it should be fixed on the next download.'
+  end
+
   def load_sprints
-    Dir.foreach(@target_path) do |file|
-      next unless file =~ /#{file_prefix}_board_(\d+)_sprints_\d+/
+    file_system.foreach(@target_path) do |file|
+      next unless file =~ /^#{get_file_prefix}_board_(\d+)_sprints_\d+.json$/
+
+      file_path = File.join(@target_path, file)
+      board = @all_boards[$1.to_i]
+      unless board
+        @exporter.file_system.log(
+          'Found sprint data but can\'t find a matching board in config. ' \
+            "File: #{file_path}, Boards: #{@all_boards.keys.sort}"
+        )
+        next
+      end
 
-      board_id = $1.to_i
       timezone_offset = exporter.timezone_offset
-      JSON.parse(file_system.load("#{target_path}#{file}"))['values'].each do |json|
-        @all_boards[board_id].sprints << Sprint.new(raw: json, timezone_offset: timezone_offset)
+      file_system.load_json(file_path)['values']&.each do |json|
+        board.sprints << Sprint.new(raw: json, timezone_offset: timezone_offset)
       end
     end
 
@@ -190,60 +296,30 @@ class ProjectConfig
     end
   end
 
-  def add_possible_status status
-    existing_status = find_status(name: status.name)
-
-    if status.project_scoped?
-      # If the project specific status doesn't change anything then we don't care whether it's
-      # our project or not.
-      return if existing_status && existing_status.category_name == status.category_name
+  def load_project_metadata
+    filename = File.join @target_path, "#{get_file_prefix}_meta.json"
+    json = file_system.load_json(filename)
 
-      raise_ambiguous_project_id if @id.nil?
+    @data_version = json['version'] || 1
 
-      # Not our project, ignore it.
-      return unless status.project_id == @id
+    start = to_time(json['date_start'] || json['time_start']) # date_start is the current format. Time is the old.
+    stop  = to_time(json['date_end'] || json['time_end'], end_of_day: true)
 
-      # Replace the old one with this
-      @possible_statuses.delete(existing_status)
-      @possible_statuses << status
-      return
+    # If no_earlier_than was set then make sure it's applied here.
+    if download_config
+      download_config.run
+      no_earlier = download_config.no_earlier_than
+      if no_earlier
+        no_earlier = to_time(no_earlier.to_s)
+        start = no_earlier if start < no_earlier
+      end
     end
 
-    # If it isn't there, add it and go.
-    return @possible_statuses << status unless existing_status
-
-    # We're registering the same one twice. Shouldn't be possible with the new status API but it
-    # did happen with the project specific one.
-    return if status.category_name == existing_status.category_name
-
-    # If we got this far then someone has called status_category_mapping and is attempting to
-    # change the category.
-    raise "Redefining status category #{status} with #{existing_status}. Was one set in the config?"
-  end
-
-  def raise_ambiguous_project_id
-    raise 'Ambiguous project id: There is a project specific status that could affect out calculations. ' \
-      'We are unable to automatically detect the id of the project so you will have to set it manually ' \
-      'in the configuration like: "project id: 5"'
-  end
-
-  def find_status name:
-    @possible_statuses.find_by_name name
-  end
-
-  def load_project_metadata
-    filename = "#{@target_path}/#{file_prefix}_meta.json"
-    json = JSON.parse(file_system.load(filename))
-
-    @data_version = json['version'] || 1
-
-    start = json['date_start'] || json['time_start'] # date_start is the current format. Time is the old.
-    stop  = json['date_end'] || json['time_end']
-    @time_range = to_time(start)..to_time(stop, end_of_day: true)
+    @time_range = start..stop
 
     @jira_url = json['jira_url']
   rescue Errno::ENOENT
-    puts "== Can't load files from the target directory. Did you forget to download first? =="
+    file_system.log "Can't load #{filename}. Have you done a download?", also_write_to_stderr: true
     raise
   end
 
@@ -259,7 +335,7 @@ class ProjectConfig
     unless all_boards&.size == 1
       message = "If the board_id isn't set then we look for all board configurations in the target" \
         ' directory. '
-      if all_boards.nil? || all_boards.empty?
+      if all_boards.empty?
         message += ' In this case, we couldn\'t find any configuration files in the target directory.'
       else
         message += 'If there is only one, we use that. In this case we found configurations for' \
@@ -291,21 +367,25 @@ class ProjectConfig
   end
 
   def issues
-    raise "issues are being loaded before boards in project #{name.inspect}" if all_boards.nil? && !aggregated_project?
-
     unless @issues
-      if @aggregate_config
+      if aggregated_project?
         raise 'This is an aggregated project and issues should have been included with the include_issues_from ' \
           'declaration but none are here. Check your config.'
       end
 
+      return @issues = [] if @exporter.downloading?
+      raise 'No data found. Must do a download before an export' unless data_downloaded?
+
+      load_data if all_boards.empty?
+
       timezone_offset = exporter.timezone_offset
 
-      issues_path = "#{@target_path}#{file_prefix}_issues/"
+      issues_path = File.join @target_path, "#{get_file_prefix}_issues"
       if File.exist?(issues_path) && File.directory?(issues_path)
         issues = load_issues_from_issues_directory path: issues_path, timezone_offset: timezone_offset
       else
-        puts "Can't find issues in #{path}. Has a download been done?"
+        file_system.log "Can't find directory #{issues_path}. Has a download been done?", also_write_to_stderr: true
+        return []
       end
 
       # Attach related issues
@@ -351,8 +431,8 @@ class ProjectConfig
     raise "No boards found for project #{name.inspect}" if all_boards.empty?
 
     if all_boards.size != 1
-      puts "Multiple boards are in use for project #{name.inspect}. " \
-        "Picked #{default_board.name.inspect} to attach issues to."
+      file_system.log "Multiple boards are in use for project #{name.inspect}. " \
+        "Picked #{default_board.name.inspect} to attach issues to.", also_write_to_stderr: true
     end
     default_board
   end
@@ -362,7 +442,7 @@ class ProjectConfig
     default_board = nil
 
     group_filenames_and_board_ids(path: path).each do |filename, board_ids|
-      content = file_system.load(File.join(path, filename))
+      content = file_system.load_json(File.join(path, filename))
       if board_ids == :unknown
         boards = [(default_board ||= find_default_board)]
       else
@@ -370,7 +450,7 @@ class ProjectConfig
       end
 
       boards.each do |board|
-        issues << Issue.new(raw: JSON.parse(content), timezone_offset: timezone_offset, board: board)
+        issues << Issue.new(raw: content, timezone_offset: timezone_offset, board: board)
       end
     end
 
@@ -424,21 +504,52 @@ class ProjectConfig
     Anonymizer.new(project_config: self).run
   end
 
-  def discard_changes_before_hook issues_cutoff_times
-    issues_cutoff_times.each do |issue, cutoff_time|
-      days = (cutoff_time.to_date - issue.changes.first.time.to_date).to_i + 1
-      message = "#{issue.key}(#{issue.type}) discarding #{days} "
-      if days == 1
-        message << "day of data on #{cutoff_time.to_date}"
-      else
-        message << "days of data from #{issue.changes.first.time.to_date} to #{cutoff_time.to_date}"
+  def file_system
+    @exporter.file_system
+  end
+
+  def discard_changes_before status_becomes: nil, &block
+    if status_becomes
+      status_becomes = [status_becomes] unless status_becomes.is_a? Array
+
+      block = lambda do |issue|
+        trigger_statuses = status_becomes.collect do |status_name|
+          if status_name == :backlog
+            issue.board.backlog_statuses
+          else
+            possible_statuses.find_all_by_name status_name
+          end
+        end.flatten
+
+        next if trigger_statuses.empty?
+
+        trigger_status_ids = trigger_statuses.collect(&:id)
+
+        time = nil
+        issue.status_changes.each do |change|
+          time = change.time if trigger_status_ids.include?(change.value_id) # && change.artificial? == false
+        end
+        time
       end
-      exporter.file_system.log message
     end
-    exporter.file_system.log "Discarded data from #{issues_cutoff_times.count} issues out of a total #{issues.size}"
-  end
 
-  def file_system
-    @exporter.file_system
+    issues.each do |issue|
+      cutoff_time = block.call(issue)
+      next if cutoff_time.nil?
+
+      original_start_time = issue.board.cycletime.started_stopped_times(issue).first
+      next if original_start_time.nil?
+
+      issue.discard_changes_before cutoff_time
+
+      next unless cutoff_time
+      next if original_start_time > cutoff_time # ie the cutoff would have made no difference.
+
+      (@discarded_changes_data ||= []) << {
+        cutoff_time: cutoff_time,
+        original_start_time: original_start_time,
+        issue: issue
+      }
+    end
   end
 end

data/lib/jirametrics/rules.rb

@@ -1,8 +1,8 @@
 # frozen_string_literal: true
 
 class Rules
-  def ignore
-    @ignore = true
+  def ignore value = true # rubocop:disable Style/OptionalBooleanParameter
+    @ignore = value
   end
 
   def ignored?

data/lib/jirametrics/self_or_issue_dispatcher.rb

@@ -1,6 +1,7 @@
 # frozen_string_literal: true
 
 module SelfOrIssueDispatcher
+  # rubocop:disable Style/ArgumentsForwarding
   def method_missing method_name, *args, &block
     raise "#{method_name} isn't a method on Issue or #{self.class}" unless ::Issue.method_defined? method_name.to_sym
 
@@ -8,6 +9,7 @@ module SelfOrIssueDispatcher
       issue.__send__ method_name, *args, &block
     end
   end
+  # rubocop:enable Style/ArgumentsForwarding
 
   def respond_to_missing?(method_name, include_all = false)
     ::Issue.method_defined?(method_name.to_sym) || super

data/lib/jirametrics/settings.json

@@ -2,6 +2,9 @@
   "stalled_threshold_days": 5,
   "stalled_statuses": [],
 
-  "blocked_link_text": [],
-  "blocked_statuses": []
+  "blocked_link_text": ["is blocked by"],
+  "blocked_statuses": [],
+  "flagged_means_blocked": true,
+
+  "expedited_priority_names": ["Critical", "Highest"]
 }

data/lib/jirametrics/sprint_burndown.rb

@@ -18,7 +18,7 @@ class SprintBurndown < ChartBase
   attr_accessor :board_id
 
   def initialize
-    super()
+    super
 
     @summary_stats = {}
     header_text 'Sprint burndown'
@@ -126,7 +126,7 @@ class SprintBurndown < ChartBase
     currently_in_sprint = false
     change_data = []
 
-    issue_completed_time = issue.board.cycletime.stopped_time(issue)
+    issue_completed_time = issue.board.cycletime.started_stopped_times(issue).last
     completed_has_been_tracked = false
 
     issue.changes.each do |change|
@@ -172,7 +172,7 @@ class SprintBurndown < ChartBase
     change_item.raw['to'].split(/\s*,\s*/).any? { |id| id.to_i == sprint.id }
   end
 
-  def data_set_by_story_points sprint:, change_data_for_sprint:
+  def data_set_by_story_points sprint:, change_data_for_sprint: # rubocop:disable Metrics/CyclomaticComplexity
     summary_stats = SprintSummaryStats.new
     summary_stats.completed = 0.0