jirametrics 2.7.1 → 2.8

data/lib/jirametrics/issue.rb

@@ -44,7 +44,7 @@ class Issue
 
   def summary = @raw['fields']['summary']
 
-  def status = Status.new(raw: @raw['fields']['status'])
+  def status = Status.from_raw(@raw['fields']['status'])
 
   def labels = @raw['fields']['labels'] || []
 
@@ -66,35 +66,45 @@ class Issue
   end
 
   def first_time_in_status *status_names
-    @changes.find { |change| change.current_status_matches(*status_names) }&.time
+    @changes.find { |change| change.current_status_matches(*status_names) }
   end
 
   def first_time_not_in_status *status_names
-    @changes.find { |change| change.status? && status_names.include?(change.value) == false }&.time
+    @changes.find { |change| change.status? && status_names.include?(change.value) == false }
   end
 
   def first_time_in_or_right_of_column column_name
     first_time_in_status(*board.status_ids_in_or_right_of_column(column_name))
   end
 
+  def first_time_label_added *labels
+    @changes.each do |change|
+      next unless change.labels?
+
+      change_labels = change.value.split
+      return change if change_labels.any? { |l| labels.include?(l) }
+    end
+    nil
+  end
+
   def still_in_or_right_of_column column_name
     still_in_status(*board.status_ids_in_or_right_of_column(column_name))
   end
 
   def still_in
-    time = nil
+    result = nil
     @changes.each do |change|
       next unless change.status?
 
       current_status_matched = yield change
 
-      if current_status_matched && time.nil?
-        time = change.time
-      elsif !current_status_matched && time
-        time = nil
+      if current_status_matched && result.nil?
+        result = change
+      elsif !current_status_matched && result
+        result = nil
       end
     end
-    time
+    result
   end
   private :still_in
 
@@ -108,8 +118,8 @@ class Issue
   # If it ever entered one of these categories and it's still there then what was the last time it entered
   def still_in_status_category *category_names
     still_in do |change|
-      status = find_status_by_name change.value
-      category_names.include?(status.category_name) || category_names.include?(status.category_id)
+      status = find_status_by_id change.value_id, name: change.value
+      category_names.include?(status.category.name) || category_names.include?(status.category.id)
     end
   end
 
@@ -117,48 +127,53 @@ class Issue
     changes.reverse.find { |change| change.status? }
   end
 
-  # Are we currently in this status? If yes, then return the time of the most recent status change.
+  # Are we currently in this status? If yes, then return the most recent status change.
   def currently_in_status *status_names
     change = most_recent_status_change
     return false if change.nil?
 
-    change.time if change.current_status_matches(*status_names)
+    change if change.current_status_matches(*status_names)
   end
 
-  # Are we currently in this status category? If yes, then return the time of the most recent status change.
+  # Are we currently in this status category? If yes, then return the most recent status change.
   def currently_in_status_category *category_names
     change = most_recent_status_change
     return false if change.nil?
 
-    status = find_status_by_name change.value
-    change.time if status && category_names.include?(status.category_name)
+    status = find_status_by_id change.value_id, name: change.value
+    change if status && category_names.include?(status.category.name)
   end
 
-  def find_status_by_name name
-    status = board.possible_statuses.find_by_name(name)
+  def find_status_by_id id, name: nil
+    status = board.possible_statuses.find_by_id(id)
     return status if status
 
-    @board.project_config.file_system.log(
-      "Warning: Status name #{name.inspect} for issue #{key} not found in" \
-      " #{board.possible_statuses.collect(&:name).inspect}" \
-      "\n  See https://jirametrics.org/faq/#q1\n",
-      also_write_to_stderr: true
-    )
-    status = Status.new(name: name, category_name: 'In Progress')
-    board.possible_statuses << status
+    status = board.possible_statuses.fabricate_status_for id: id, name: name
+
+    message = +'The history for issue '
+    message << key
+    message << ' references a status ('
+    message << name.inspect << ':' if name
+    message << id.to_s
+    message << ') that can\'t be found in ['
+    message << board.possible_statuses.collect(&:to_s).join(', ')
+    message << "]. We are guessing that this belongs to the #{status.category} status category "
+    message << 'and that may be wrong. See https://jirametrics.org/faq/#q1 for more details'
+    board.project_config.file_system.warning message
+
     status
   end
 
   def first_status_change_after_created
-    @changes.find { |change| change.status? && change.artificial? == false }&.time
+    @changes.find { |change| change.status? && change.artificial? == false }
   end
 
   def first_time_in_status_category *category_names
     @changes.each do |change|
       next unless change.status?
 
-      category = find_status_by_name(change.value).category_name
-      return change.time if category_names.include? category
+      category = find_status_by_id(change.value_id).category.name
+      return change if category_names.include? category
     end
     nil
   end
@@ -177,11 +192,11 @@ class Issue
   end
 
   def first_resolution
-    @changes.find { |change| change.resolution? }&.time
+    @changes.find { |change| change.resolution? }
   end
 
   def last_resolution
-    @changes.reverse.find { |change| change.resolution? }&.time
+    @changes.reverse.find { |change| change.resolution? }
   end
 
   def assigned_to
@@ -624,7 +639,7 @@ class Issue
       # This was probably loaded as a linked issue, which means we don't know what board it really
       # belonged to. The best we can do is look at the status category. This case should be rare but
       # it can happen.
-      status.category_name == 'Done'
+      status.category.name == 'Done'
     else
       board.cycletime.done? self
     end

data/lib/jirametrics/project_config.rb

@@ -31,18 +31,23 @@ class ProjectConfig
     instance_eval(&@block) if @block
   end
 
+  def data_downloaded?
+    File.exist? File.join(@target_path, "#{get_file_prefix}_meta.json")
+  end
+
   def load_data
     return if @has_loaded_data
 
     @has_loaded_data = true
     load_all_boards
     @id = guess_project_id
-    load_status_category_mappings
     load_project_metadata
     load_sprints
   end
 
   def run load_only: false
+    return if @exporter.downloading?
+
     load_data unless aggregated_project?
     anonymize_data if @anonymizer_needed
 
@@ -57,7 +62,8 @@ class ProjectConfig
   end
 
   def load_settings
-    JSON.parse(file_system.load(File.join(__dir__, 'settings.json')))
+    # This is the wierd exception that we don't ever want mocked out so we skip FileSystem entirely.
+    JSON.parse(File.read(File.join(__dir__, 'settings.json'), encoding: 'UTF-8'))
   end
 
   def guess_project_id
@@ -109,87 +115,153 @@ class ProjectConfig
     @board_configs << config
   end
 
-  def file_prefix prefix = nil
-    @file_prefix = prefix unless prefix.nil?
+  def file_prefix prefix
+    # The file_prefix has to be set before almost everything else. It really should have been an attribute
+    # on the project declaration itself. Hindsight is 20/20.
+    if @file_prefix
+      raise "file_prefix should only be set once. Was #{@file_prefix.inspect} and now changed to #{prefix.inspect}."
+    end
+
+    @file_prefix = prefix
+
+    # Yes, this is a wierd place to be initializing this. Unfortunately, it has to happen after the file_prefix
+    # is set but before anything inside the project block is run. If only we had made file_prefix an attribute
+    # on project, we wouldn't have this ugliness. 🤷‍♂️
+    load_status_category_mappings
+
     @file_prefix
   end
 
-  def status_category_mapping status:, category:
-    add_possible_status Status.new(name: status, category_name: category)
+  def get_file_prefix # rubocop:disable Naming/AccessorMethodName
+    raise 'file_prefix has not been set yet. Move it to the top of the project declaration.' unless @file_prefix
+
+    @file_prefix
   end
 
-  def load_all_boards
-    Dir.foreach(@target_path) do |file|
-      next unless file =~ /^#{@file_prefix}_board_(\d+)_configuration\.json$/
+  def status_guesses
+    statuses = {}
+    issues.each do |issue|
+      issue.changes.each do |change|
+        next unless change.status?
 
-      board_id = $1.to_i
-      load_board board_id: board_id, filename: "#{@target_path}#{file}"
+        (statuses[change.value] ||= Set.new) << change.value_id
+        (statuses[change.old_value] ||= Set.new) << change.old_value_id if change.old_value
+      end
     end
-    raise "No boards found for #{@file_prefix.inspect} in #{@target_path.inspect}" if @all_boards.empty?
+    statuses
   end
 
-  def load_board board_id:, filename:
-    board = Board.new(
-      raw: JSON.parse(file_system.load(filename)), possible_statuses: @possible_statuses
+  def status_category_mapping status:, category:
+    # Status might just be a name like "Review" or it might be a name:id pair like "Review:4"
+    parse_status = ->(line) { line =~ /^(.+):(\d+)$/ ? [$1, $2.to_i] : [line, nil] }
+
+    status, status_id = parse_status.call status
+    category, category_id = parse_status.call category
+
+    if status_id.nil?
+      guesses = status_guesses[status]
+      if guesses.nil?
+        file_system.warning "For status_category_mapping status: #{status.inspect}, category: #{category.inspect}\n" \
+          "Cannot guess status id for #{status.inspect} as no statuses found anywhere in the issues " \
+          "histories with that name. Since we can't find it, you probably don't need this mapping anymore so we're " \
+          "going to ignore it. If you really want it, then you'll need to specify a status id."
+        return
+      end
+
+      if guesses.size > 1
+        raise "Cannot guess status id as there are multiple ids for the name #{status.inspect}. Perhaps it's one " \
+          "of #{guesses.to_a.sort.inspect}. If you need this mapping then you must specify the status_id."
+      end
+
+      status_id = guesses.first
+      # TODO: This should be a deprecation instead but we can't currently assert against those
+      file_system.log "status_category_mapping for #{status.inspect} has been mapped to id #{status_id}. " \
+        "If that's incorrect then specify the status_id."
+    end
+
+    found_category = possible_statuses.find_category_by_name category
+    found_category_id = found_category&.id # possible_statuses.find_category_id_by_name category
+    if category_id && category_id != found_category_id
+      raise "ID is incorrect for status category #{category.inspect}. Did you mean #{found_category_id}?"
+    end
+
+    add_possible_status(
+      Status.new(
+        name: status, id: status_id,
+        category_name: category, category_id: found_category_id, category_key: found_category.key
+      )
     )
-    board.project_config = self
-    @all_boards[board_id] = board
   end
 
-  def raise_with_message_about_missing_category_information all_issues = @issues
-    message = +''
-    message << "Could not determine categories for some of the statuses used in this data set.\n" \
-      "Use the 'status_category_mapping' declaration in your config to manually add one.\n" \
-      'The mappings we do know about are below:'
+  def add_possible_status status
+    existing_status = @possible_statuses.find_by_id status.id
 
-    @possible_statuses.each do |status|
-      message << "\n  status: #{status.name.inspect}, category: #{status.category_name.inspect}"
+    if existing_status && existing_status.name != status.name
+      raise "Attempting to redefine the name for status #{status.id} from " \
+        "#{existing_status.name.inspect} to #{status.name.inspect}"
     end
 
-    message << "\n\nThe ones we're missing are the following:"
+    # If it isn't there, add it and go.
+    return @possible_statuses << status unless existing_status
 
-    find_statuses_with_no_category_information(all_issues).each do |status_name|
-      message << "\n  status: #{status_name.inspect}, category: <unknown>"
+    unless status == existing_status
+      raise "Redefining status category for status #{status}. " \
+        "original: #{existing_status.category}, " \
+        "new: #{status.category}"
     end
 
-    raise message
+    # We're registering one we already knew about. This may happen if someone specified a status_category_mapping
+    # for something that was already returned from jira. This isn't  problem so just ignore it.
+    existing_status
   end
 
-  def find_statuses_with_no_category_information all_issues
-    missing_statuses = []
-    all_issues.each do |issue|
-      issue.changes.each do |change|
-        next unless change.status?
+  def load_all_boards
+    Dir.foreach(@target_path) do |file|
+      next unless file =~ /^#{get_file_prefix}_board_(\d+)_configuration\.json$/
 
-        missing_statuses << change.value unless find_status(name: change.value)
-      end
+      board_id = $1.to_i
+      load_board board_id: board_id, filename: "#{@target_path}#{file}"
     end
-    missing_statuses.uniq
+    raise "No boards found for #{get_file_prefix.inspect} in #{@target_path.inspect}" if @all_boards.empty?
+  end
+
+  def load_board board_id:, filename:
+    board = Board.new(
+      raw: file_system.load_json(filename), possible_statuses: @possible_statuses
+    )
+    board.project_config = self
+    @all_boards[board_id] = board
   end
 
   def load_status_category_mappings
-    filename = "#{@target_path}/#{file_prefix}_statuses.json"
+    filename = File.join @target_path, "#{get_file_prefix}_statuses.json"
+
     # We may not always have this file. Load it if we can.
     return unless File.exist? filename
 
-    statuses = JSON.parse(file_system.load(filename))
-      .map { |snippet| Status.new(raw: snippet) }
-    statuses
-      .find_all { |status| status.global? }
-      .each { |status| add_possible_status status }
-    statuses
-      .find_all { |status| status.project_scoped? }
+    file_system
+      .load_json(filename)
+      .map { |snippet| Status.from_raw(snippet) }
       .each { |status| add_possible_status status }
   end
 
   def load_sprints
-    Dir.foreach(@target_path) do |file|
-      next unless file =~ /#{file_prefix}_board_(\d+)_sprints_\d+/
+    file_system.foreach(@target_path) do |file|
+      next unless file =~ /^#{get_file_prefix}_board_(\d+)_sprints_\d+.json$/
+
+      file_path = File.join(@target_path, file)
+      board = @all_boards[$1.to_i]
+      unless board
+        @exporter.file_system.log(
+          'Found sprint data but can\'t find a matching board in config. ' \
+            "File: #{file_path}, Boards: #{@all_boards.keys.sort}"
+        )
+        next
+      end
 
-      board_id = $1.to_i
       timezone_offset = exporter.timezone_offset
-      JSON.parse(file_system.load("#{target_path}#{file}"))['values']&.each do |json|
-        @all_boards[board_id].sprints << Sprint.new(raw: json, timezone_offset: timezone_offset)
+      file_system.load_json(file_path)['values']&.each do |json|
+        board.sprints << Sprint.new(raw: json, timezone_offset: timezone_offset)
       end
     end
 
@@ -198,56 +270,26 @@ class ProjectConfig
     end
   end
 
-  def add_possible_status status
-    existing_status = find_status(name: status.name)
-
-    if status.project_scoped?
-      # If the project specific status doesn't change anything then we don't care whether it's
-      # our project or not.
-      return if existing_status && existing_status.category_name == status.category_name
+  def load_project_metadata
+    filename = File.join @target_path, "#{get_file_prefix}_meta.json"
+    json = file_system.load_json(filename)
 
-      raise_ambiguous_project_id if @id.nil?
+    @data_version = json['version'] || 1
 
-      # Not our project, ignore it.
-      return unless status.project_id == @id
+    start = to_time(json['date_start'] || json['time_start']) # date_start is the current format. Time is the old.
+    stop  = to_time(json['date_end'] || json['time_end'], end_of_day: true)
 
-      # Replace the old one with this
-      @possible_statuses.delete(existing_status)
-      @possible_statuses << status
-      return
+    # If no_earlier_than was set then make sure it's applied here.
+    if download_config
+      download_config.run
+      no_earlier = download_config.no_earlier_than
+      if no_earlier
+        no_earlier = to_time(no_earlier.to_s)
+        start = no_earlier if start < no_earlier
+      end
     end
 
-    # If it isn't there, add it and go.
-    return @possible_statuses << status unless existing_status
-
-    # We're registering the same one twice. Shouldn't be possible with the new status API but it
-    # did happen with the project specific one.
-    return if status.category_name == existing_status.category_name
-
-    # If we got this far then someone has called status_category_mapping and is attempting to
-    # change the category.
-    raise "Redefining status category #{status} with #{existing_status}. Was one set in the config?"
-  end
-
-  def raise_ambiguous_project_id
-    raise 'Ambiguous project id: There is a project specific status that could affect out calculations. ' \
-      'We are unable to automatically detect the id of the project so you will have to set it manually ' \
-      'in the configuration like: "project id: 5"'
-  end
-
-  def find_status name:
-    @possible_statuses.find_by_name name
-  end
-
-  def load_project_metadata
-    filename = File.join @target_path, "#{file_prefix}_meta.json"
-    json = JSON.parse(file_system.load(filename))
-
-    @data_version = json['version'] || 1
-
-    start = json['date_start'] || json['time_start'] # date_start is the current format. Time is the old.
-    stop  = json['date_end'] || json['time_end']
-    @time_range = to_time(start)..to_time(stop, end_of_day: true)
+    @time_range = start..stop
 
     @jira_url = json['jira_url']
   rescue Errno::ENOENT
@@ -304,11 +346,15 @@ class ProjectConfig
         raise 'This is an aggregated project and issues should have been included with the include_issues_from ' \
           'declaration but none are here. Check your config.'
       end
+
+      return @issues = [] if @exporter.downloading?
+      raise 'No data found. Must do a download before an export' unless data_downloaded?
+
       load_data if all_boards.empty?
 
       timezone_offset = exporter.timezone_offset
 
-      issues_path = File.join @target_path, "#{file_prefix}_issues"
+      issues_path = File.join @target_path, "#{get_file_prefix}_issues"
       if File.exist?(issues_path) && File.directory?(issues_path)
         issues = load_issues_from_issues_directory path: issues_path, timezone_offset: timezone_offset
       else
@@ -370,7 +416,7 @@ class ProjectConfig
     default_board = nil
 
     group_filenames_and_board_ids(path: path).each do |filename, board_ids|
-      content = file_system.load(File.join(path, filename))
+      content = file_system.load_json(File.join(path, filename))
       if board_ids == :unknown
         boards = [(default_board ||= find_default_board)]
       else
@@ -378,7 +424,7 @@ class ProjectConfig
       end
 
       boards.each do |board|
-        issues << Issue.new(raw: JSON.parse(content), timezone_offset: timezone_offset, board: board)
+        issues << Issue.new(raw: content, timezone_offset: timezone_offset, board: board)
       end
     end
 

data/lib/jirametrics/sprint_burndown.rb

@@ -172,7 +172,7 @@ class SprintBurndown < ChartBase
     change_item.raw['to'].split(/\s*,\s*/).any? { |id| id.to_i == sprint.id }
   end
 
-  def data_set_by_story_points sprint:, change_data_for_sprint:
+  def data_set_by_story_points sprint:, change_data_for_sprint: # rubocop:disable Metrics/CyclomaticComplexity
     summary_stats = SprintSummaryStats.new
     summary_stats.completed = 0.0
 

data/lib/jirametrics/status.rb

@@ -4,29 +4,57 @@ require 'jirametrics/value_equality'
 
 class Status
   include ValueEquality
-  attr_reader :id, :category_name, :category_id, :project_id
+  attr_reader :id, :project_id, :category
   attr_accessor :name
 
-  def initialize name: nil, id: nil, category_name: nil, category_id: nil, project_id: nil, raw: nil
-    @name = name
-    @id = id
-    @category_name = category_name
-    @category_id = category_id
-    @project_id = project_id
+  class Category
+    attr_reader :id, :name, :key
+
+    def initialize id:, name:, key:
+      @id = id
+      @name = name
+      @key = key
+    end
 
-    return unless raw
+    def to_s
+      "#{name.inspect}:#{id.inspect}"
+    end
 
-    @raw = raw
-    @name = raw['name']
-    @id = raw['id'].to_i
+    def new? = (@key == 'new')
+    def indeterminate? = (@key == 'indeterminate')
+    def done? = (@key == 'done')
+  end
 
+  def self.from_raw raw
     category_config = raw['statusCategory']
-    @category_name = category_config['name']
-    @category_id = category_config['id'].to_i
 
-    # If this is a NextGen project then this status may be project specific. When this field is
-    # nil then the status is global.
-    @project_id = raw['scope']&.[]('project')&.[]('id')
+    legal_keys = %w[new indeterminate done]
+    unless legal_keys.include? category_config['key']
+      puts "Category key #{category_config['key'].inspect} should be one of #{legal_keys.inspect}. Found:\n" \
+        "#{category_config}"
+    end
+
+    Status.new(
+      name: raw['name'],
+      id: raw['id'].to_i,
+      category_name: category_config['name'],
+      category_id: category_config['id'].to_i,
+      category_key: category_config['key'],
+      project_id: raw['scope']&.[]('project')&.[]('id'),
+      artificial: false
+    )
+  end
+
+  def initialize name:, id:, category_name:, category_id:, category_key:, project_id: nil, artificial: true
+    # These checks are needed because nils used to be possible and now they aren't.
+    raise 'id cannot be nil' if id.nil?
+    raise 'category_id cannot be nil' if category_id.nil?
+
+    @name = name
+    @id = id
+    @category = Category.new id: category_id, name: category_name, key: category_key
+    @project_id = project_id
+    @artificial = artificial
   end
 
   def project_scoped?
@@ -38,18 +66,26 @@ class Status
   end
 
   def to_s
-    result = "Status(name=#{@name.inspect}," \
-      " id=#{@id.inspect}," \
-      " category_name=#{@category_name.inspect}," \
-      " category_id=#{@category_id.inspect}," \
-      " project_id=#{@project_id}"
-    result << ' artificial' if artificial?
-    result << ')'
-    result
+    "#{name.inspect}:#{id.inspect}"
   end
 
   def artificial?
-    @raw.nil?
+    @artificial
+  end
+
+  def == other
+    @id == other.id && @name == other.name && @category.id == other.category.id && @category.name == other.category.name
+  end
+
+  def inspect
+    result = []
+    result << "Status(name: #{@name.inspect}"
+    result << "id: #{@id.inspect}"
+    result << "project_id: #{@project_id}" if @project_id
+    category = self.category
+    result << "category: {name:#{category.name.inspect}, id: #{category.id.inspect}, key: #{category.key.inspect}}"
+    result << 'artificial' if artificial?
+    result.join(', ') << ')'
   end
 
   def value_equality_ignored_variables