jirametrics 2.22 → 2.24

data/lib/jirametrics/download_config.rb

@@ -25,10 +25,25 @@ class DownloadConfig
     @no_earlier_than
   end
 
+  def github_repos
+    @github_repos ||= []
+  end
+
+  def github_repo *repos
+    github_repos.concat(repos.map { |r| normalize_github_repo(r) })
+  end
+
   def start_date today:
     date = today.to_date - @rolling_date_count if @rolling_date_count
     date = [date, @no_earlier_than].max if date && @no_earlier_than
     date = @no_earlier_than if date.nil? && @no_earlier_than
     date
   end
+
+  private
+
+  def normalize_github_repo repo
+    match = repo.match(%r{github\.com/([^/]+/[^/]+?)/?$})
+    match ? match[1] : repo
+  end
 end

data/lib/jirametrics/downloader.rb

@@ -33,21 +33,23 @@ class Downloader
   # For testing only
   attr_reader :start_date_in_query, :board_id_to_filter_id
 
-  def self.create download_config:, file_system:, jira_gateway:
+  def self.create download_config:, file_system:, jira_gateway:, github_pr_cache: {}
     is_cloud = jira_gateway.settings['jira_cloud'] || jira_gateway.cloud?
     (is_cloud ? DownloaderForCloud : DownloaderForDataCenter).new(
       download_config: download_config,
       file_system: file_system,
-      jira_gateway: jira_gateway
+      jira_gateway: jira_gateway,
+      github_pr_cache: github_pr_cache
     )
   end
 
-  def initialize download_config:, file_system:, jira_gateway:
+  def initialize download_config:, file_system:, jira_gateway:, github_pr_cache: {}
     @metadata = {}
     @download_config = download_config
     @target_path = @download_config.project_config.target_path
     @file_system = file_system
     @jira_gateway = jira_gateway
+    @github_pr_cache = github_pr_cache
     @board_id_to_filter_id = {}
 
     @issue_keys_downloaded_in_current_run = []
@@ -77,6 +79,7 @@ class Downloader
     download_users
 
     save_metadata
+    download_github_prs if @download_config.github_repos.any?
   end
 
   def log text, both: false
@@ -165,11 +168,28 @@ class Downloader
     # actually look at the returned json.
     @board_id_to_filter_id[board_id] = json['filter']['id'].to_i
 
+    if json['type'] == 'simple'
+      features_json = download_features board_id: board_id
+      if BoardFeature.from_raw(features_json).any? { |f| f.name == 'jsw.agility.sprints' && f.enabled? }
+        download_sprints board_id: board_id
+      end
+    end
     download_sprints board_id: board_id if json['type'] == 'scrum'
     # TODO: Should be passing actual statuses, not empty list
     Board.new raw: json, possible_statuses: StatusCollection.new
   end
 
+  def download_features board_id:
+    log "  Downloading features for board #{board_id}", both: true
+    json = @jira_gateway.call_url relative_url: "/rest/agile/1.0/board/#{board_id}/features"
+
+    @file_system.save_json(
+      json: json,
+      filename: File.join(@target_path, "#{file_prefix}_board_#{board_id}_features.json")
+    )
+    json
+  end
+
   def download_sprints board_id:
     log "  Downloading sprints for board #{board_id}", both: true
     max_results = 100
@@ -211,19 +231,36 @@ class Downloader
         value = Date.parse(value) if value.is_a?(String) && value =~ /^\d{4}-\d{2}-\d{2}$/
         @metadata[key] = value
       end
+
+      # If rolling_date_count has changed, we may be missing data outside the previous range,
+      # so force a full re-download.
+      if @metadata['rolling_date_count'] != @download_config.rolling_date_count
+        log '  rolling_date_count has changed. Forcing a full download.', both: true
+        @cached_data_format_is_current = false
+        @metadata = {}
+      end
     end
 
     # Even if this is the old format, we want to obey this one tag
     @metadata['no-download'] = hash['no-download'] if hash['no-download']
   end
 
+  def timezone_offset
+    @download_config.project_config.exporter.timezone_offset
+  end
+
+  def today_in_project_timezone
+    Time.now.getlocal(timezone_offset).to_date
+  end
+
   def save_metadata
     @metadata['version'] = CURRENT_METADATA_VERSION
+    @metadata['rolling_date_count'] = @download_config.rolling_date_count
     @metadata['date_start_from_last_query'] = @start_date_in_query if @start_date_in_query
 
     if @download_date_range.nil?
       log "Making up a date range in meta since one wasn't specified. You'll want to change that.", both: true
-      today = Date.today
+      today = today_in_project_timezone
       @download_date_range = (today - 7)..today
     end
 
@@ -258,7 +295,8 @@ class Downloader
     end
   end
 
-  def make_jql filter_id:, today: Date.today
+  def make_jql filter_id:, today: nil
+    today ||= today_in_project_timezone
     segments = []
     segments << "filter=#{filter_id}"
 
@@ -283,6 +321,39 @@ class Downloader
     segments.join ' AND '
   end
 
+  def download_github_prs
+    project_keys = extract_project_keys_from_downloaded_issues
+    if project_keys.empty?
+      log '  No project keys found in downloaded issues, skipping GitHub PR download', both: true
+      return
+    end
+
+    prs = @download_config.github_repos.flat_map do |repo|
+      GithubGateway.new(
+        repo: repo,
+        project_keys: project_keys,
+        file_system: @file_system,
+        raw_pr_cache: @github_pr_cache
+      ).fetch_pull_requests(since: @download_date_range&.begin)
+    end
+
+    @file_system.save_json(
+      json: prs.map(&:raw),
+      filename: File.join(@target_path, "#{file_prefix}_github_prs.json")
+    )
+  end
+
+  def extract_project_keys_from_downloaded_issues
+    path = File.join(@target_path, "#{file_prefix}_issues")
+    return [] unless @file_system.dir_exist?(path)
+
+    keys = []
+    @file_system.foreach(path) do |filename|
+      keys << filename.split('-').first if filename.match?(/^[A-Z][A-Z_0-9]+-\d+-\d+\.json$/)
+    end
+    keys.uniq
+  end
+
   def file_prefix
     @download_config.project_config.get_file_prefix
   end

data/lib/jirametrics/downloader_for_cloud.rb

@@ -5,6 +5,45 @@ class DownloaderForCloud < Downloader
     'Jira Cloud'
   end
 
+  def run
+    super
+    download_fix_versions
+  end
+
+  def download_board_configuration board_id:
+    board = super
+    location = board.raw['location']
+    @project_key ||= location['key'] if location&.[]('type') == 'project'
+    board
+  end
+
+  def download_fix_versions
+    return unless @project_key
+
+    log "  Downloading fix versions for project #{@project_key}", both: true
+    max_results = 50
+    start_at = 0
+    all_versions = []
+
+    loop do
+      json = @jira_gateway.call_url(
+        relative_url: "/rest/api/3/project/#{@project_key}/version?" \
+          "startAt=#{start_at}&maxResults=#{max_results}"
+      )
+
+      values = json['values'] || []
+      all_versions.concat(values)
+      break if json['isLast'] || values.empty?
+
+      start_at += values.size
+    end
+
+    @file_system.save_json(
+      json: all_versions,
+      filename: File.join(@target_path, "#{file_prefix}_fix_versions.json")
+    )
+  end
+
   def search_for_issues jql:, board_id:, path:
     log "  JQL: #{jql}"
     escaped_jql = CGI.escape jql

data/lib/jirametrics/downloader_for_data_center.rb

@@ -63,7 +63,8 @@ class DownloaderForDataCenter < Downloader
     end
   end
 
-  def make_jql filter_id:, today: Date.today
+  def make_jql filter_id:, today: nil
+    today ||= today_in_project_timezone
     segments = []
     segments << "filter=#{filter_id}"
 

data/lib/jirametrics/estimate_accuracy_chart.rb

@@ -5,7 +5,7 @@ class EstimateAccuracyChart < ChartBase
     super()
 
     header_text 'Estimate Accuracy'
-    description_text <<-HTML
+    description_text <<~HTML
       <div class="p">
         This chart graphs estimates against actual recorded cycle times. Since
         estimates can change over time, we're graphing the estimate at the time that the story started.
@@ -20,8 +20,18 @@ class EstimateAccuracyChart < ChartBase
           far to the right then you know you have a problem.
         <% end %>
       </div>
+      <% if @correlation_coefficient %>
+        <div class="p">
+          The completed items here have a correlation coefficient of <b><%= @correlation_coefficient.round(3) %></b>.
+          The closer it is to +1, the stronger the positive correlation. The closer it is to -1,
+          the stronger the negative collalation. Zero would mean no correlation at all.
+        </div>
+      <% end %>
     HTML
 
+    @x_axis_title = 'Cycletime (days)'
+    @y_axis_title = 'Estimate'
+
     @y_axis_type = 'linear'
     @y_axis_block = ->(issue, start_time) { estimate_at(issue: issue, start_time: start_time)&.to_f }
     @y_axis_sort_order = nil
@@ -30,9 +40,9 @@ class EstimateAccuracyChart < ChartBase
   end
 
   def run
-    if @y_axis_label.nil?
+    if @y_axis_title.nil?
       text = current_board.estimation_configuration.units == :story_points ? 'Story Points' : 'Days'
-      @y_axis_label = "Estimated #{text}"
+      @y_axis_title = "Estimated #{text}"
     end
     data_sets = scan_issues
 
@@ -43,7 +53,7 @@ class EstimateAccuracyChart < ChartBase
 
   def scan_issues
     completed_hash, aging_hash = split_into_completed_and_aging issues: issues
-
+    @correlation_coefficient = correlation_coefficient(completed_hash) unless completed_hash.empty?
     estimation_units = current_board.estimation_configuration.units
     @has_aging_data = !aging_hash.empty?
 
@@ -170,4 +180,32 @@ class EstimateAccuracyChart < ChartBase
     end
     @y_axis_block = block
   end
+
+  # Correlation coefficient is calculated using the Pearson Correlation Coefficient
+  # r = Σ((xi - x̄)(yi - ȳ)) / sqrt(Σ(xi - x̄)² · Σ(yi - ȳ)²)
+  def correlation_coefficient completed_hash
+    list1 = []
+    list2 = []
+    completed_hash.each do |(estimate, cycle_time), issues|
+      issues.size.times do
+        list1 << estimate
+        list2 << cycle_time
+      end
+    end
+
+    n = list1.size
+    return nil if n < 2
+
+    mean1 = list1.sum.to_f / n
+    mean2 = list2.sum.to_f / n
+
+    numerator = list1.zip(list2).sum { |x, y| (x - mean1) * (y - mean2) }
+    sum_sq1 = list1.sum { |x| (x - mean1)**2 }
+    sum_sq2 = list2.sum { |y| (y - mean2)**2 }
+
+    denominator = Math.sqrt(sum_sq1 * sum_sq2)
+    return nil if denominator.zero?
+
+    numerator / denominator
+  end
 end

data/lib/jirametrics/examples/aggregated_project.rb

@@ -12,7 +12,7 @@ class Exporter
     project name: name do
       puts name
       file_prefix name
-      self.settings.merge! settings
+      self.settings.merge! stringify_keys(settings)
 
       aggregate do
         project_names.each do |project_name|

data/lib/jirametrics/examples/standard_project.rb

@@ -6,14 +6,14 @@ class Exporter
   def standard_project name:, file_prefix:, ignore_issues: nil, starting_status: nil, boards: {},
       default_board: nil, anonymize: false, settings: {}, status_category_mappings: {},
       rolling_date_count: 90, no_earlier_than: nil, ignore_types: %w[Sub-task Subtask Epic],
-      show_experimental_charts: false
-
+      show_experimental_charts: false, github_repos: nil
+    exporter = self
     project name: name do
       puts name
       file_prefix file_prefix
 
       self.anonymize if anonymize
-      self.settings.merge! settings
+      self.settings.merge! stringify_keys(settings)
 
       boards.each_key do |board_id|
         block = boards[board_id]
@@ -35,19 +35,20 @@ class Exporter
       download do
         self.rolling_date_count(rolling_date_count) if rolling_date_count
         self.no_earlier_than(no_earlier_than) if no_earlier_than
+        github_repo github_repos if github_repos
       end
 
       issues.reject! do |issue|
         ignore_types.include? issue.type
       end
 
+      exporter.filter_issues issues, ignore_issues
+
       discard_changes_before status_becomes: (starting_status || :backlog) # rubocop:disable Style/RedundantParentheses
 
       file do
         file_suffix '.html'
 
-        issues.reject! { |issue| ignore_issues.include? issue.key } if ignore_issues
-
         html_report do
           board_id default_board if default_board
 
@@ -72,10 +73,11 @@ class Exporter
             header_text nil
             description_text '<h2>Number of items completed, grouped by completion status and resolution</h2>'
             grouping_rules do |issue, rules|
-              if issue.resolution
-                rules.label = "#{issue.status.name}:#{issue.resolution}"
+              status, resolution = issue.status_resolution_at_done
+              if resolution
+                rules.label = "#{status.name}:#{resolution}"
               else
-                rules.label = issue.status.name
+                rules.label = status.name
               end
             end
           end
@@ -87,7 +89,6 @@ class Exporter
           daily_wip_by_blocked_stalled_chart
           daily_wip_by_parent_chart
           flow_efficiency_scatterplot if show_experimental_charts
-          expedited_chart
           sprint_burndown
           estimate_accuracy_chart
           dependency_chart
@@ -95,4 +96,14 @@ class Exporter
       end
     end
   end
+
+  # Extracted as a separate method so it can be tested independently, without needing to invoke
+  # the full standard_project DSL setup.
+  def filter_issues issues, ignore_issues
+    return unless ignore_issues
+
+    issues.reject! do |issue|
+      ignore_issues.is_a?(Proc) ? ignore_issues.call(issue) : ignore_issues.include?(issue.key)
+    end
+  end
 end

data/lib/jirametrics/expedited_chart.rb

@@ -38,6 +38,8 @@ class ExpeditedChart < ChartBase
       </div>
       #{describe_non_working_days}
     HTML
+    @x_axis_title = 'Date'
+    @y_axis_title = 'Age in days'
 
     instance_eval(&block)
   end

data/lib/jirametrics/exporter.rb

@@ -40,6 +40,7 @@ class Exporter
 
   def download name_filter:
     @downloading = true
+    github_pr_cache = {}
     each_project_config(name_filter: name_filter) do |project|
       project.evaluate_next_level
       next if project.aggregated_project?
@@ -56,7 +57,8 @@ class Exporter
       downloader = Downloader.create(
         download_config: project.download_config,
         file_system: file_system,
-        jira_gateway: gateway
+        jira_gateway: gateway,
+        github_pr_cache: github_pr_cache
       )
       downloader.run
     end

data/lib/jirametrics/file_system.rb

@@ -61,6 +61,10 @@ class FileSystem
     logfile.puts more if more
     return unless also_write_to_stderr
 
+    # Obscure edge-case where we're trying to log something before logging is even
+    # set up. Quick escape here so that we don't dump the error twice.
+    return if logfile == $stdout
+
     $stderr.puts message # rubocop:disable Style/StderrPuts
   end
 

data/lib/jirametrics/flow_efficiency_scatterplot.rb

@@ -32,6 +32,8 @@ class FlowEfficiencyScatterplot < ChartBase
         So be aware that your team may have to change their behaviours if you want this chart to be useful.
       </div>
     HTML
+    @x_axis_title = 'Total time (days)'
+    @y_axis_title = 'Time adding value (days)'
 
     init_configuration_block block do
       grouping_rules do |issue, rule|

data/lib/jirametrics/github_gateway.rb

@@ -0,0 +1,106 @@
+# frozen_string_literal: true
+
+require 'open3'
+require 'json'
+
+class GithubGateway
+  attr_reader :repo
+
+  def initialize repo:, project_keys:, file_system:, raw_pr_cache: {}
+    @repo = repo
+    @project_keys = project_keys
+    @file_system = file_system
+    @raw_pr_cache = raw_pr_cache
+    @issue_key_pattern = build_issue_key_pattern
+  end
+
+  def fetch_pull_requests since: nil
+    raw_prs = @raw_pr_cache[[@repo, since]] ||= fetch_raw_pull_requests(since: since)
+    raw_prs.filter_map { |pr| build_pr_data(pr) }
+  end
+
+  def fetch_raw_pull_requests since: nil
+    # Note: 'commits' is intentionally excluded — including it triggers GitHub's GraphQL node
+    # limit (authors sub-connection × PRs × commits exceeds 500,000 nodes). Branch name,
+    # title, and body are sufficient for issue key extraction in the vast majority of cases.
+    json_fields = %w[number title body headRefName createdAt closedAt mergedAt
+                     url state reviews additions deletions changedFiles].join(',')
+    args = ['pr', 'list', '--state', 'all', '--limit', '5000', '--json', json_fields]
+    args += ['--repo', @repo]
+    args += ['--search', "updated:>=#{since}"] if since
+
+    @file_system.log "  Downloading pull requests from #{@repo}", also_write_to_stderr: true
+    run_command(args)
+  end
+
+  def build_pr_data raw_pr
+    issue_keys = extract_issue_keys(raw_pr)
+    return nil if issue_keys.empty?
+
+    PullRequest.new(raw: {
+      'number'     => raw_pr['number'],
+      'repo'       => @repo,
+      'url'        => raw_pr['url'],
+      'title'      => raw_pr['title'],
+      'branch'     => raw_pr['headRefName'],
+      'opened_at'  => raw_pr['createdAt'],
+      'closed_at'  => raw_pr['closedAt'],
+      'merged_at'  => raw_pr['mergedAt'],
+      'state'      => raw_pr['state'],
+      'issue_keys'    => issue_keys,
+      'reviews'       => extract_reviews(raw_pr['reviews'] || []),
+      'additions'     => raw_pr['additions'],
+      'deletions'     => raw_pr['deletions'],
+      'changed_files' => raw_pr['changedFiles']
+    })
+  end
+
+  def extract_issue_keys raw_pr
+    return [] if @issue_key_pattern.nil?
+
+    sources = [
+      raw_pr['headRefName'],
+      raw_pr['title'],
+      raw_pr['body']
+    ]
+
+    sources.compact
+           .flat_map { |s| s.scan(@issue_key_pattern) }
+           .uniq
+  end
+
+  def extract_reviews raw_reviews
+    raw_reviews
+      .select { |r| %w[APPROVED CHANGES_REQUESTED].include?(r['state']) }
+      .map do |r|
+        {
+          'author'       => r.dig('author', 'login'),
+          'submitted_at' => r['submittedAt'],
+          'state'        => r['state']
+        }
+      end
+  end
+
+  private
+
+  def build_issue_key_pattern
+    return nil if @project_keys.empty?
+
+    keys_pattern = @project_keys.map { |k| Regexp.escape(k) }.join('|')
+    Regexp.new("\\b(?:#{keys_pattern})-\\d+\\b")
+  end
+
+  def run_command args
+    stdout, stderr, status = Open3.capture3('gh', *args)
+
+    # This extra check seems to only matter on Windows. On the mac, auth failures don't pass status.success?
+    if stderr.include?('SAML enforcement')
+      raise "GitHub CLI is not authorized to access #{@repo}. " \
+            "Run: gh auth refresh -h github.com -s read:org"
+    end
+
+    raise "GitHub CLI command failed for #{@repo}: #{stderr}" unless status.success?
+
+    JSON.parse(stdout)
+  end
+end

data/lib/jirametrics/groupable_issue_chart.rb

@@ -16,6 +16,7 @@ module GroupableIssueChart
   def group_issues completed_issues
     result = {}
     ignored_issues = []
+    @issue_hints = {}
     completed_issues.each do |issue|
       rules = GroupingRules.new
       @group_by_block.call(issue, rules)
@@ -24,6 +25,7 @@ module GroupableIssueChart
         next
       end
 
+      @issue_hints[issue] = rules.issue_hint
       (result[rules] ||= []) << issue
     end
 

data/lib/jirametrics/grouping_rules.rb

@@ -1,7 +1,7 @@
 # frozen_string_literal: true
 
 class GroupingRules < Rules
-  attr_accessor :label
+  attr_accessor :label, :issue_hint
   attr_reader :color
 
   def eql? other
@@ -13,7 +13,25 @@ class GroupingRules < Rules
   end
 
   def color= color
-    color = CssVariable[color] unless color.is_a?(CssVariable)
-    @color = color
+    if color.is_a?(Array)
+      raise ArgumentError, 'Color pair must have exactly two elements: [light_color, dark_color]' unless color.size == 2
+      raise ArgumentError, 'Color pair elements must be strings' unless color.all?(String)
+
+      if color.any? { |c| c.start_with?('--') }
+        raise ArgumentError,
+          'CSS variable references are not supported as color pair elements; use a literal color value instead'
+      end
+
+      light, dark = color
+      @color = RawJavascript.new(
+        "(document.documentElement.dataset.theme === 'dark' || " \
+        '(!document.documentElement.dataset.theme && ' \
+        "window.matchMedia('(prefers-color-scheme: dark)').matches)) " \
+        "? #{dark.to_json} : #{light.to_json}"
+      )
+    else
+      color = CssVariable[color] unless color.is_a?(CssVariable)
+      @color = color
+    end
   end
 end

data/lib/jirametrics/html/aging_work_bar_chart.erb

@@ -16,11 +16,9 @@ new Chart(document.getElementById('<%= chart_id %>').getContext('2d'),
       x: {
         type: 'time',
         min: '<%= @date_range.begin.to_s %>',
-        max: '<%= (@date_range.end ).to_s %>',
+        max: '<%= (@date_range.end + 1).to_s %>',
         stacked: false,
-        title: {
-          display: false
-        },
+        <%= render_axis_title :x %>
         grid: {
           color: <%= CssVariable['--grid-line-color'].to_json %>
         },
@@ -31,6 +29,7 @@ new Chart(document.getElementById('<%= chart_id %>').getContext('2d'),
         ticks: {
           display: true
         },
+        <%= render_axis_title :y %>
         grid: {
           color: <%= CssVariable['--grid-line-color'].to_json %>
         },

data/lib/jirametrics/html/aging_work_table.erb

@@ -41,6 +41,9 @@
                 <%= link_to_issue parent, style: "color: #{color}" %>
               </span>
               <i><%= parent.summary.strip.inspect %></i>
+              <% if parent == issue && (text = not_visible_text(issue)) %>
+                <br /><%= text %>
+              <% end %>
             </div>
           <% end %>
         </td>

data/lib/jirametrics/html/daily_wip_chart.erb

@@ -21,7 +21,10 @@ new Chart(document.getElementById('<%= chart_id %>').getContext('2d'),
         time: {
           unit: 'day'
         },
+        min: "<%= date_range.begin.to_s %>",
+        max: "<%= (date_range.end + 1).to_s %>",
         stacked: true,
+        <%= render_axis_title :x %>
         grid: {
           color: <%= CssVariable['--grid-line-color'].to_json %>
         },
@@ -32,10 +35,7 @@ new Chart(document.getElementById('<%= chart_id %>').getContext('2d'),
           display: true,
           labelString: 'WIP'
         },
-        title: {
-          display: true,
-          text: 'Count of items'
-        },
+        <%= render_axis_title :y %>
         grid: {
           color: <%= CssVariable['--grid-line-color'].to_json %>
         },
@@ -52,6 +52,7 @@ new Chart(document.getElementById('<%= chart_id %>').getContext('2d'),
       annotation: {
         annotations: {
           <%= working_days_annotation %>
+          <%= date_annotation %>
         }
       },
       legend: {