jirametrics 1.0.0

data/lib/jirametrics/downloader.rb

@@ -0,0 +1,337 @@
+# frozen_string_literal: true
+
+require 'cgi'
+require 'json'
+require 'english'
+
+class Downloader
+  CURRENT_METADATA_VERSION = 4
+
+  attr_accessor :metadata, :quiet_mode, :logfile, :logfile_name
+
+  # For testing only
+  attr_reader :start_date_in_query
+
+  def initialize download_config:, json_file_loader: JsonFileLoader.new
+    @metadata = {}
+    @download_config = download_config
+    @target_path = @download_config.project_config.target_path
+    @json_file_loader = json_file_loader
+    @board_id_to_filter_id = {}
+
+    @issue_keys_downloaded_in_current_run = []
+    @issue_keys_pending_download = []
+  end
+
+  def run
+    log '', both: true
+    log @download_config.project_config.name, both: true
+
+    load_jira_config(@download_config.project_config.jira_config)
+    load_metadata
+
+    if @metadata['no-download']
+      log '  Skipping download. Found no-download in meta file', both: true
+      return
+    end
+
+    # board_ids = @download_config.board_ids
+
+    remove_old_files
+    download_statuses
+    find_board_ids.each do |id|
+      download_board_configuration board_id: id
+      download_issues board_id: id
+    end
+
+    save_metadata
+  end
+
+  def log text, both: false
+    @logfile&.puts text
+    puts text if both
+  end
+
+  def find_board_ids
+    ids = @download_config.project_config.board_configs.collect(&:id)
+    if ids.empty?
+      deprecated message: 'board_ids in the download block have been deprecated. See https://github.com/mikebowler/jira-export/wiki/Deprecated'
+      ids = @download_config.board_ids
+    end
+    raise 'Board ids must be specified' if ids.empty?
+
+    ids
+  end
+
+  def load_jira_config jira_config
+    @jira_url = jira_config['url']
+    @jira_email = jira_config['email']
+    @jira_api_token = jira_config['api_token']
+    @jira_personal_access_token = jira_config['personal_access_token']
+
+    raise 'When specifying an api-token, you must also specify email' if @jira_api_token && !@jira_email
+
+    if @jira_api_token && @jira_personal_access_token
+      raise "You can't specify both an api-token and a personal-access-token. They don't work together."
+    end
+
+    @cookies = (jira_config['cookies'] || []).collect { |key, value| "#{key}=#{value}" }.join(';')
+  end
+
+  def call_command command
+    log "  #{command.gsub(/\s+/, ' ')}"
+    result = `#{command}`
+    log result unless $CHILD_STATUS.success?
+    return result if $CHILD_STATUS.success?
+
+    log "Failed call with exit status #{$CHILD_STATUS.exitstatus}. See #{@logfile_name} for details", both: true
+    exit $CHILD_STATUS.exitstatus
+  end
+
+  def make_curl_command url:
+    command = 'curl'
+    command += ' -s'
+    command += ' -k' if @download_config.project_config.settings['ignore_ssl_errors']
+    command += " --cookie #{@cookies.inspect}" unless @cookies.empty?
+    command += " --user #{@jira_email}:#{@jira_api_token}" if @jira_api_token
+    command += " -H \"Authorization: Bearer #{@jira_personal_access_token}\"" if @jira_personal_access_token
+    command += ' --request GET'
+    command += ' --header "Accept: application/json"'
+    command += " --url \"#{url}\""
+    command
+  end
+
+  def download_issues board_id:
+    log "  Downloading primary issues for board #{board_id}", both: true
+    path = "#{@target_path}#{@download_config.project_config.file_prefix}_issues/"
+    unless Dir.exist?(path)
+      log "  Creating path #{path}"
+      Dir.mkdir(path)
+    end
+
+    filter_id = @board_id_to_filter_id[board_id]
+    jql = make_jql(filter_id: filter_id)
+    jira_search_by_jql(jql: jql, initial_query: true, board_id: board_id, path: path)
+
+    log "  Downloading linked issues for board #{board_id}", both: true
+    loop do
+      @issue_keys_pending_download.reject! { |key| @issue_keys_downloaded_in_current_run.include? key }
+      break if @issue_keys_pending_download.empty?
+
+      keys_to_request = @issue_keys_pending_download[0..99]
+      @issue_keys_pending_download.reject! { |key| keys_to_request.include? key }
+      jql = "key in (#{keys_to_request.join(', ')})"
+      jira_search_by_jql(jql: jql, initial_query: false, board_id: board_id, path: path)
+    end
+  end
+
+  def jira_search_by_jql jql:, initial_query:, board_id:, path:
+    intercept_jql = @download_config.project_config.settings['intercept_jql']
+    jql = intercept_jql.call jql if intercept_jql
+
+    log "  #{jql}"
+    escaped_jql = CGI.escape jql
+
+    max_results = 100
+    start_at = 0
+    total = 1
+    while start_at < total
+      command = make_curl_command url: "#{@jira_url}/rest/api/2/search" \
+        "?jql=#{escaped_jql}&maxResults=#{max_results}&startAt=#{start_at}&expand=changelog&fields=*all"
+
+      json = JSON.parse call_command(command)
+      exit_if_call_failed json
+
+      json['issues'].each do |issue_json|
+        issue_json['exporter'] = {
+          'in_initial_query' => initial_query
+        }
+        identify_other_issues_to_be_downloaded issue_json
+        file = "#{issue_json['key']}-#{board_id}.json"
+        write_json(issue_json, File.join(path, file))
+      end
+
+      total = json['total'].to_i
+      max_results = json['maxResults']
+
+      message = "    Downloaded #{start_at + 1}-#{[start_at + max_results, total].min} of #{total} issues to #{path} "
+      log message, both: true
+
+      start_at += json['issues'].size
+    end
+  end
+
+  def identify_other_issues_to_be_downloaded raw_issue
+    issue = Issue.new raw: raw_issue, board: nil
+    @issue_keys_downloaded_in_current_run << issue.key
+
+    # Parent
+    parent_key = issue.parent_key(project_config: @download_config.project_config)
+    @issue_keys_pending_download << parent_key if parent_key
+
+    # Sub-tasks
+    issue.raw['fields']['subtasks'].each do |raw_subtask|
+      @issue_keys_pending_download << raw_subtask['key']
+    end
+
+    # Links
+    # We shouldn't blindly follow links as some, like cloners, aren't valuable and are just wasting time/effort
+    # to download
+    # issue.raw['fields']['issuelinks'].each do |raw_link|
+    #   @issue_keys_pending_download << IssueLink(raw: raw_link).other_issue.key
+    # end
+  end
+
+  def exit_if_call_failed json
+    # Sometimes Jira returns the singular form of errorMessage and sometimes the plural. Consistency FTW.
+    return unless json['errorMessages'] || json['errorMessage']
+
+    log "Download failed. See #{@logfile_name} for details.", both: true
+    log "  #{JSON.pretty_generate(json)}"
+    exit 1
+  end
+
+  def download_statuses
+    log '  Downloading all statuses', both: true
+    command = make_curl_command url: "\"#{@jira_url}/rest/api/2/status\""
+    json = JSON.parse call_command(command)
+
+    write_json json, "#{@target_path}#{@download_config.project_config.file_prefix}_statuses.json"
+  end
+
+  def download_board_configuration board_id:
+    log "  Downloading board configuration for board #{board_id}", both: true
+    command = make_curl_command url: "#{@jira_url}/rest/agile/1.0/board/#{board_id}/configuration"
+
+    json = JSON.parse call_command(command)
+    exit_if_call_failed json
+
+    @board_id_to_filter_id[board_id] = json['filter']['id'].to_i
+    # @board_configuration = json if @download_config.board_ids.size == 1
+
+    file_prefix = @download_config.project_config.file_prefix
+    write_json json, "#{@target_path}#{file_prefix}_board_#{board_id}_configuration.json"
+
+    download_sprints board_id: board_id if json['type'] == 'scrum'
+  end
+
+  def download_sprints board_id:
+    log "  Downloading sprints for board #{board_id}", both: true
+    file_prefix = @download_config.project_config.file_prefix
+    max_results = 100
+    start_at = 0
+    is_last = false
+
+    while is_last == false
+      command = make_curl_command url: "#{@jira_url}/rest/agile/1.0/board/#{board_id}/sprint?" \
+        "maxResults=#{max_results}&startAt=#{start_at}"
+      json = JSON.parse call_command(command)
+      exit_if_call_failed json
+
+      write_json json, "#{@target_path}#{file_prefix}_board_#{board_id}_sprints_#{start_at}.json"
+      is_last = json['isLast']
+      max_results = json['maxResults']
+      start_at += json['values'].size
+    end
+  end
+
+  def write_json json, filename
+    file_path = File.dirname(filename)
+    FileUtils.mkdir_p file_path unless File.exist?(file_path)
+
+    File.write(filename, JSON.pretty_generate(json))
+  end
+
+  def metadata_pathname
+    "#{@target_path}#{@download_config.project_config.file_prefix}_meta.json"
+  end
+
+  def load_metadata
+    # If we've never done a download before then this file won't be there. That's ok.
+    return unless File.exist? metadata_pathname
+
+    hash = JSON.parse(File.read metadata_pathname)
+
+    # Only use the saved metadata if the version number is the same one that we're currently using.
+    # If the cached data is in an older format then we're going to throw most of it away.
+    @cached_data_format_is_current = (hash['version'] || 0) == CURRENT_METADATA_VERSION
+    if @cached_data_format_is_current
+      hash.each do |key, value|
+        value = Date.parse(value) if value.is_a?(String) && value =~ /^\d{4}-\d{2}-\d{2}$/
+        @metadata[key] = value
+      end
+    end
+
+    # Even if this is the old format, we want to obey this one tag
+    @metadata['no-download'] = hash['no-download'] if hash['no-download']
+  end
+
+  def save_metadata
+    @metadata['version'] = CURRENT_METADATA_VERSION
+    @metadata['date_start_from_last_query'] = @start_date_in_query if @start_date_in_query
+
+    if @download_date_range.nil?
+      log "Making up a date range in meta since one wasn't specified. You'll want to change that.", both: true
+      today = Date.today
+      @download_date_range = (today - 7)..today
+    end
+
+    @metadata['earliest_date_start'] = @download_date_range.begin if @metadata['earliest_date_start'].nil?
+
+    @metadata['date_start'] = @download_date_range.begin
+    @metadata['date_end'] = @download_date_range.end
+
+    @metadata['jira_url'] = @jira_url
+
+    write_json @metadata, metadata_pathname
+  end
+
+  def remove_old_files
+    file_prefix = @download_config.project_config.file_prefix
+    Dir.foreach @target_path do |file|
+      next unless file =~ /^#{file_prefix}_\d+\.json$/
+
+      File.unlink "#{@target_path}#{file}"
+    end
+
+    return if @cached_data_format_is_current
+
+    # Also throw away all the previously downloaded issues.
+    path = File.join @target_path, "#{file_prefix}_issues"
+    return unless File.exist? path
+
+    Dir.foreach path do |file|
+      next unless file =~ /\.json$/
+
+      File.unlink File.join(path, file)
+    end
+  end
+
+  def make_jql filter_id:, today: Date.today
+    segments = []
+    segments << "filter=#{filter_id}"
+
+    unless @download_config.rolling_date_count.nil?
+      @download_date_range = (today.to_date - @download_config.rolling_date_count)..today.to_date
+
+      # For an incremental download, we want to query from the end of the previous one, not from the
+      # beginning of the full range.
+      @start_date_in_query = metadata['date_end'] || @download_date_range.begin
+      log "    Incremental download only. Pulling from #{@start_date_in_query}", both: true if metadata['date_end']
+
+      # Catch-all to pick up anything that's been around since before the range started but hasn't
+      # had an update during the range.
+      catch_all = '((status changed OR Sprint is not EMPTY) AND statusCategory != Done)'
+
+      # Pick up any issues that had a status change in the range
+      start_date_text = @start_date_in_query.strftime '%Y-%m-%d'
+      end_date_text = today.strftime '%Y-%m-%d'
+      # find_in_range = %((status changed DURING ("#{start_date_text} 00:00","#{end_date_text} 23:59")))
+      find_in_range = %((updated >= "#{start_date_text} 00:00" AND updated <= "#{end_date_text} 23:59"))
+
+      segments << "(#{find_in_range} OR #{catch_all})"
+    end
+
+    segments.join ' AND '
+  end
+end

data/lib/jirametrics/examples/aggregated_project.rb

@@ -0,0 +1,36 @@
+# frozen_string_literal: true
+
+class Exporter
+  def aggregated_project name:, project_names:
+    project name: name do
+      puts name
+      aggregate do
+        project_names.each do |project_name|
+          include_issues_from project_name
+        end
+      end
+
+      file_prefix name
+
+      file do
+        file_suffix '.html'
+        issues.reject! do |issue|
+          %w[Sub-task Epic].include? issue.type
+        end
+
+        html_report do
+          cycletime_scatterplot do
+            show_trend_lines
+            grouping_rules do |issue, rules|
+              rules.label = issue.board.name
+            end
+          end
+          aging_work_in_progress_chart
+          aging_work_table do
+            age_cutoff 21
+          end
+        end
+      end
+    end
+  end
+end

data/lib/jirametrics/examples/standard_project.rb

@@ -0,0 +1,111 @@
+# frozen_string_literal: true
+
+class Exporter
+  def standard_project name:, file_prefix:, ignore_issues: nil, starting_status: nil, boards: {}, default_board: nil
+    project name: name do
+      puts name
+
+      settings['blocked_link_text'] = ['is blocked by']
+      file_prefix file_prefix
+      download do
+        rolling_date_count 90
+      end
+
+      boards.each_key do |board_id|
+        block = boards[board_id]
+        if block == :default
+          block = lambda do |_|
+            start_at first_time_in_status_category('In Progress')
+            stop_at still_in_status_category('Done')
+          end
+        end
+        board id: board_id do
+          cycletime(&block)
+          expedited_priority_names 'Critical', 'Highest', 'Immediate Gating'
+        end
+      end
+
+      file do
+        file_suffix '.html'
+        issues.reject! do |issue|
+          %w[Sub-task Epic].include? issue.type
+        end
+
+        issues.reject! { |issue| ignore_issues.include? issue.key } if ignore_issues
+
+        html_report do
+          board_id default_board if default_board
+
+          html "<H1>#{file_prefix}</H1>", type: :header
+          boards.each_key do |id|
+            board = find_board id
+            html "<div><a href='#{board.url}'>#{id} #{board.name}</a></div>",
+                 type: :header
+          end
+
+          discard_changes_before status_becomes: (starting_status || :backlog)
+
+          cycletime_scatterplot do
+            show_trend_lines
+          end
+          cycletime_scatterplot do # Epics
+            header_text 'Parents only'
+            filter_issues { |i| i.parent }
+          end
+          cycletime_histogram
+          cycletime_histogram do
+            grouping_rules do |issue, rules|
+              rules.label = issue.board.cycletime.stopped_time(issue).to_date.strftime('%b %Y')
+            end
+          end
+
+          throughput_chart do
+            description_text '<h2>Number of items completed, grouped by issue type</h2>'
+          end
+          throughput_chart do
+            header_text nil
+            description_text '<h2>Number of items completed, grouped by completion status and resolution</h2>'
+            grouping_rules do |issue, rules|
+              if issue.resolution
+                rules.label = "#{issue.status.name}:#{issue.resolution}"
+              else
+                rules.label = issue.status.name
+              end
+            end
+          end
+
+          aging_work_in_progress_chart
+          aging_work_bar_chart
+          aging_work_table
+          daily_wip_by_age_chart
+          daily_wip_by_blocked_stalled_chart
+          expedited_chart
+          sprint_burndown
+          story_point_accuracy_chart
+          # story_point_accuracy_chart do
+          #   header_text nil
+          #   description_text nil
+          #   y_axis(sort_order: %w[Story Task Defect], label: 'TShirt Sizes') { |issue, _started_time| issue.type }
+          # end
+
+          dependency_chart do
+            link_rules do |link, rules|
+              case link.name
+              when 'Cloners'
+                rules.ignore
+              when 'Dependency', 'Blocks', 'Parent/Child', 'Cause', 'Satisfy Requirement', 'Relates'
+                rules.merge_bidirectional keep: 'outward'
+                rules.merge_bidirectional keep: 'outward'
+              when 'Sync'
+                rules.use_bidirectional_arrows
+                #   rules.line_color = 'red'
+              else
+                puts "name=#{link.name}, label=#{link.label}"
+              end
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/jirametrics/expedited_chart.rb

@@ -0,0 +1,169 @@
+# frozen_string_literal: true
+
+require 'jirametrics/chart_base'
+
+class ExpeditedChart < ChartBase
+  EXPEDITED_SEGMENT = Object.new.tap do |segment|
+    def segment.to_json *_args
+      <<~SNIPPET
+        {
+          borderColor: ctx => expedited(ctx, 'red') || notExpedited(ctx, 'gray'),
+          borderDash: ctx => notExpedited(ctx, [6, 6])
+        }
+      SNIPPET
+    end
+  end
+
+  attr_accessor :issues, :cycletime, :possible_statuses, :date_range
+  attr_reader :expedited_label
+
+  def initialize
+    super()
+
+    header_text 'Expedited work'
+    description_text <<-HTML
+      <p>
+        This chart only shows issues that have been expedited at some point. We care about these as
+        any form of expedited work will affect the entire system and will slow down non-expedited work.
+        Refer to this article on
+        <a href="https://improvingflow.com/2021/06/16/classes-of-service.html">classes of service</a>
+        for a longer explanation on why we want to avoid expedited work.
+      </p>
+      <p>
+        The lines indicate time that this issue was expedited. When the line is red then the issue was
+        expedited at that time. When it's gray then it wasn't. Orange dots indicate the date the work
+        was started and green dots represent the completion date. Lastly, the vertical height of the
+        lines/dots indicates how long it's been since this issue was created.
+      </p>
+    HTML
+  end
+
+  def run
+    data_sets = find_expedited_issues.collect do |issue|
+      make_expedite_lines_data_set(issue: issue, expedite_data: prepare_expedite_data(issue))
+    end.compact
+
+    if data_sets.empty?
+      '<h1>Expedited work</h1>There is no expedited work in this time period.'
+    else
+      wrap_and_render(binding, __FILE__)
+    end
+  end
+
+  def prepare_expedite_data issue
+    expedite_start = nil
+    result = []
+    expedited_priority_names = issue.board.expedited_priority_names
+
+    issue.changes.each do |change|
+      next unless change.priority?
+
+      if expedited_priority_names.include? change.value
+        expedite_start = change.time
+      elsif expedite_start
+        start_date = expedite_start.to_date
+        stop_date = change.time.to_date
+
+        if date_range.include?(start_date) || date_range.include?(stop_date) ||
+           (start_date < date_range.begin && stop_date > date_range.end)
+
+          result << [expedite_start, :expedite_start]
+          result << [change.time, :expedite_stop]
+        end
+        expedite_start = nil
+      end
+    end
+
+    # If expedite_start is still set then we never ended.
+    result << [expedite_start, :expedite_start] if expedite_start
+    result
+  end
+
+  def find_expedited_issues
+    expedited_issues = @issues.reject do |issue|
+      prepare_expedite_data(issue).empty?
+    end
+
+    expedited_issues.sort { |a, b| a.key_as_i <=> b.key_as_i }
+  end
+
+  def later_date date1, date2
+    return date1 if date2.nil?
+    return date2 if date1.nil?
+
+    [date1, date2].max
+  end
+
+  def make_point issue:, time:, label:, expedited:
+    {
+      y: (time.to_date - issue.created.to_date).to_i + 1,
+      x: time.to_date.to_s,
+      title: ["#{issue.key} #{label} : #{issue.summary}"],
+      expedited: (expedited ? 1 : 0)
+    }
+  end
+
+  def make_expedite_lines_data_set issue:, expedite_data:
+    cycletime = issue.board.cycletime
+    started_time = cycletime.started_time(issue)
+    stopped_time = cycletime.stopped_time(issue)
+
+    expedite_data << [started_time, :issue_started] if started_time
+    expedite_data << [stopped_time, :issue_stopped] if stopped_time
+    expedite_data.sort! { |a, b| a[0] <=> b[0] }
+
+    # If none of the data would be visible on the chart then skip it.
+    return nil unless expedite_data.any? { |time, _action| time.to_date >= date_range.begin }
+
+    data = []
+    dot_colors = []
+    point_styles = []
+    expedited = false
+
+    expedite_data.each do |time, action|
+      case action
+      when :issue_started
+        data << make_point(issue: issue, time: time, label: 'Started', expedited: expedited)
+        dot_colors << 'orange'
+        point_styles << 'rect'
+      when :issue_stopped
+        data << make_point(issue: issue, time: time, label: 'Completed', expedited: expedited)
+        dot_colors << 'green'
+        point_styles << 'rect'
+      when :expedite_start
+        data << make_point(issue: issue, time: time, label: 'Expedited', expedited: true)
+        dot_colors << 'red'
+        point_styles << 'circle'
+        expedited = true
+      when :expedite_stop
+        data << make_point(issue: issue, time: time, label: 'Not expedited', expedited: false)
+        dot_colors << 'gray'
+        point_styles << 'circle'
+        expedited = false
+      else
+        raise "Unexpected action: #{action}"
+      end
+    end
+
+    unless expedite_data.empty?
+      last_change_time = expedite_data[-1][0].to_date
+      if last_change_time && last_change_time <= date_range.end && stopped_time.nil?
+        data << make_point(issue: issue, time: date_range.end, label: 'Still ongoing', expedited: expedited)
+        dot_colors << 'blue' # It won't be visible so it doesn't matter
+        point_styles << 'dash'
+      end
+    end
+
+    {
+      type: 'line',
+      label: issue.key,
+      data: data,
+      fill: false,
+      showLine: true,
+      backgroundColor: dot_colors,
+      pointBorderColor: 'black',
+      pointStyle: point_styles,
+      segment: EXPEDITED_SEGMENT
+    }
+  end
+end