snitcher_reports 0.2.0

data/lib/snitcher/sources/jira/client.rb

@@ -0,0 +1,346 @@
+# frozen_string_literal: true
+
+require 'base64'
+require 'faraday'
+require 'faraday/retry'
+require 'json'
+
+module Snitcher
+  module Sources
+    class Jira < Source
+      # Jira Cloud REST API v3 transport.
+      #
+      # Extracted from Jira so the HTTP surface can be tested on its own and so the
+      # source stays focused on interpreting the data. Mirrors the Faraday setup used
+      # by Linear (linear.rb:73-83) and ClickUp (clickup.rb:145-154), with two additions:
+      # `raise_error` and a retry stack ordered to work alongside it.
+      #
+      # Without `raise_error` a 401 from an expired token returns a body that just does
+      # not match the expected shape, and the source reports "no work" instead of an
+      # outage — see Orchestrator#fetch, which records raising sources and marks the
+      # report incomplete. The same reasoning is why an unparsed body raises here
+      # instead of degrading to an empty Hash: a proxy interstitial or maintenance page
+      # served with a 200 is an outage, not an empty result set.
+      #
+      # What each caller does with those exceptions:
+      # - #search: nothing catches them, so a failure fails the source.
+      # - #worklogs: Jira#fetch_worklogs warns past a Faraday::Error or a rejected issue
+      #   key and drops that one ticket's worklogs. A wrong-shape body is not caught —
+      #   an interstitial answers every request, not one ticket, so it fails the source.
+      # - #changelogs: partial failures warn and cost the tickets they affect (see the
+      #   method); a run where no changelog at all could be read raises.
+      #
+      # Middleware order matters: Faraday nests handlers in registration order, so
+      # `raise_error` has to be registered *before* `:retry` to end up outside it.
+      # Registered after, it converts a 429/5xx into a Faraday::Error before retry can
+      # inspect the status, and those exceptions are not in faraday-retry's default
+      # list — `retry_statuses` would then never fire. `methods` is widened past
+      # faraday-retry's idempotent-only default because /search/jql is a POST read.
+      #
+      # Cloud only. Data Center (/rest/api/2 + Bearer PAT) is not supported.
+      # @api private
+      class Client
+        API_PATH = '/rest/api/3'
+        PAGE_SIZE = 100
+        MAX_PAGES = 10 # Safety limit to prevent infinite loops
+        WORKLOG_PAGE_SIZE = 100
+        # Issues per POST /issue/bulkfetch call. The endpoint documents a higher ceiling
+        # for requests that name every field explicitly; `expand` requests do not
+        # qualify, so this stays at 100 rather than being raised to match the docs.
+        BULK_FETCH_SIZE = 100
+        RETRY_STATUSES = [429, 500, 502, 503, 504].freeze
+        # /search/jql and /issue/bulkfetch are POST reads, so POST joins the retryable
+        # verbs. Jira exposes no POST write on the paths this client touches.
+        RETRY_METHODS = %i[get head options put delete post].freeze
+        RETRY_MAX = 3
+        RETRY_INTERVAL = 0.5
+        # Jira sends Retry-After on a 429 and its points quota is hourly, so the header
+        # can ask for minutes. faraday-retry defaults max_interval to Float::MAX and
+        # sleeps the header value verbatim; three retries of that inside the sequential
+        # per-ticket loop makes a throttled report indistinguishable from a hung one.
+        # Above this bound faraday-retry skips the retry, so the call fails fast and the
+        # source reaches Orchestrator#failed_sources instead of blocking.
+        RETRY_MAX_INTERVAL = 5.0
+        # Jitter, as a fraction of the interval: three retries at a flat interval from
+        # every ticket in a loop is a thundering herd against the per-second burst limit.
+        RETRY_RANDOMNESS = 0.5
+
+        # @param retry_interval [Float] seconds between retries. Lower it to keep a test
+        #   suite from sleeping through the backoff; nothing in production passes it.
+        def initialize(base_url:, username:, api_token:, retry_interval: RETRY_INTERVAL)
+          @base_url = base_url
+          @username = username
+          @api_token = api_token
+          @retry_interval = retry_interval
+        end
+
+        # POST /rest/api/3/search/jql
+        #
+        # The legacy /rest/api/3/search endpoint is withdrawn. Its replacement paginates
+        # by opaque token and returns no `total`, so the loop ends on a nil token.
+        #
+        # @param jql [String] the query
+        # @param fields [Array<String>] field ids to return
+        # @param max_pages [Integer] page cap; 1 makes the call a single-page sample
+        # @return [Array<Hash>] issues, each with 'key' at the top level and 'fields' nested
+        def search(jql:, fields:, max_pages: MAX_PAGES)
+          issues = []
+          token = nil
+          pages = 0
+
+          loop do
+            body = { jql: jql, fields: fields, maxResults: PAGE_SIZE }
+            body[:nextPageToken] = token if token
+
+            payload = post("#{API_PATH}/search/jql", body)
+            issues.concat(payload['issues'] || [])
+
+            token = payload['nextPageToken']
+            pages += 1
+            break if token.nil? || pages >= max_pages
+          end
+
+          issues
+        end
+
+        # GET /rest/api/3/issue/{key}/worklog
+        # @return [Array<Hash>] worklog entries with 'author' (Hash) and 'timeSpentSeconds'
+        def worklogs(issue_key)
+          entries = []
+          start_at = 0
+          pages = 0
+
+          loop do
+            payload = get("#{API_PATH}/issue/#{issue_path_segment(issue_key)}/worklog",
+                          startAt: start_at, maxResults: WORKLOG_PAGE_SIZE)
+            batch = payload['worklogs'] || []
+            entries.concat(batch)
+
+            start_at += batch.size
+            pages += 1
+            break if batch.empty? || start_at >= payload['total'].to_i || pages >= MAX_PAGES
+          end
+
+          entries
+        end
+
+        # POST /rest/api/3/issue/bulkfetch — changelogs for many issues at once.
+        #
+        # Replaces one #changelog call per issue with ceil(n / BULK_FETCH_SIZE) calls.
+        # `expand: changelog` embeds the histories in each issue, and the embedded form
+        # is capped by the server, so an issue whose histories look truncated is re-read
+        # through the paginated #changelog.
+        #
+        # Failure is deliberately granular, because a batch call otherwise turns one
+        # error into every ticket in the run reporting no hours:
+        # - a key the path guard rejects costs only itself, not its slice;
+        # - a slice that fails falls back to one #changelog call per issue in it — the
+        #   endpoint the batching replaced, which answers when bulkfetch does not;
+        # - an issue that fails on its own costs that ticket, and warns;
+        # - `issueErrors` in an otherwise-successful response warns per issue.
+        # Slices that already succeeded are kept in every case. Only a run that produced
+        # no changelog at all re-raises, so a changelog outage reaches Orchestrator#fetch
+        # and the report marks itself incomplete instead of quietly reporting zeroes.
+        #
+        # /changelog/bulkfetch would carry the pagination itself and filter by field, but
+        # it 404s on some instances — see the Jira ARCHITECTURE NOTES.
+        #
+        # @param issue_keys [Array<String>]
+        # @return [Hash{String => Array<Hash>}] issue key => status-change histories
+        def changelogs(issue_keys)
+          keys = usable_issue_keys(issue_keys)
+          return {} if keys.empty?
+
+          errors = []
+          result = keys.each_slice(BULK_FETCH_SIZE).with_object({}) do |batch, acc|
+            acc.merge!(slice_changelogs(batch, errors))
+          end
+          raise errors.first if result.empty? && errors.any?
+
+          result
+        end
+
+        # GET /rest/api/3/issue/{key}/changelog
+        #
+        # The paginated endpoint, not the `?expand=changelog` embedding: the embedded
+        # form is truncated at the server's page size with no way to ask for the rest,
+        # which silently shortens the time calculated for a long-lived ticket.
+        #
+        # @return [Array<Hash>] changelog histories, each with 'created' and 'items'
+        def changelog(issue_key)
+          entries = []
+          start_at = 0
+          pages = 0
+
+          loop do
+            payload = get("#{API_PATH}/issue/#{issue_path_segment(issue_key)}/changelog",
+                          startAt: start_at, maxResults: PAGE_SIZE)
+            batch = payload['values'] || []
+            entries.concat(batch)
+
+            start_at += batch.size
+            pages += 1
+            break if batch.empty? || payload['isLast'] || start_at >= payload['total'].to_i || pages >= MAX_PAGES
+          end
+
+          entries
+        end
+
+        # GET /rest/api/3/project/search
+        # @return [Array<Hash>] projects with 'key' and 'name'
+        def projects
+          values = []
+          start_at = 0
+          pages = 0
+
+          loop do
+            payload = get("#{API_PATH}/project/search", startAt: start_at, maxResults: PAGE_SIZE)
+            batch = payload['values'] || []
+            values.concat(batch)
+
+            start_at += batch.size
+            pages += 1
+            break if batch.empty? || payload['isLast'] || pages >= MAX_PAGES
+          end
+
+          values
+        end
+
+        # GET /rest/api/3/status — every status on the instance, with its category.
+        # Not paginated. Used to build the status-id → statusCategory map.
+        # @return [Array<Hash>] statuses with 'id', 'name' and nested 'statusCategory'
+        def statuses
+          path = "#{API_PATH}/status"
+          expect_type(connection.get(path).body, Array, path)
+        end
+
+        private
+
+        # Keys are validated one at a time: the guard rejecting one key must not cost the
+        # other 99 in its slice, and the warning names the key so it can be chased.
+        def usable_issue_keys(issue_keys)
+          issue_keys.filter_map do |key|
+            issue_path_segment(key)
+          rescue Validators::ValidationError => e
+            warn "[jira] Warning: skipping unusable issue key #{key.inspect} (#{e.message})"
+            nil
+          end
+        end
+
+        def slice_changelogs(batch, errors)
+          payload = post("#{API_PATH}/issue/bulkfetch",
+                         issueIdsOrKeys: batch, fields: %w[status], expand: %w[changelog])
+          warn_issue_errors(payload)
+          collect_histories(payload['issues'] || [], errors)
+        rescue Faraday::Error, Error => e
+          errors << e
+          warn "[jira] Warning: changelog bulkfetch failed for #{batch.first}..#{batch.last} " \
+               "(#{e.message}); falling back to one request per issue"
+          per_issue_changelogs(batch, errors)
+        end
+
+        def collect_histories(issues, errors)
+          issues.each_with_object({}) do |issue, acc|
+            key = issue['key']
+            next unless key
+
+            acc[key] = histories_for(issue)
+          rescue Faraday::Error, Error => e
+            errors << e
+            warn "[jira] Warning: changelog fetch failed for #{key} (#{e.message})"
+          end
+        end
+
+        def per_issue_changelogs(batch, errors)
+          batch.each_with_object({}) do |key, acc|
+            acc[key] = status_histories(changelog(key))
+          rescue Faraday::Error, Error => e
+            errors << e
+            warn "[jira] Warning: changelog fetch failed for #{key} (#{e.message})"
+          end
+        end
+
+        # bulkfetch reports per-issue problems (deleted issue, no permission) alongside
+        # the issues it could return, so without this the one failure mode that costs
+        # tickets in an HTTP 200 would warn about nothing.
+        def warn_issue_errors(payload)
+          Array(payload['issueErrors']).each do |issue_error|
+            messages = Array(issue_error['errorMessages']).join('; ')
+            detail = messages.empty? ? issue_error.inspect : messages
+            warn "[jira] Warning: bulkfetch could not return an issue (#{detail})"
+          end
+        end
+
+        # Histories embedded by `expand: changelog`, re-read through the paginated
+        # endpoint unless the response states they are all there. Two things about that
+        # comparison:
+        # - `total` counts every field change, not just status ones, so a ticket with a
+        #   long description history is re-read for status changes it may not have. The
+        #   embedded form offers no way to ask for the rest and `expand` cannot filter by
+        #   field, so the re-read is the only way to know; MAX_PAGES bounds its cost.
+        # - a response carrying no `total` says nothing about truncation, which is not the
+        #   same as saying there is nothing more to fetch, so it re-reads too.
+        def histories_for(issue)
+          embedded = issue['changelog'] || {}
+          total = embedded['total']
+          histories = embedded['histories'] || []
+          return status_histories(histories) if total.is_a?(Integer) && histories.size >= total
+
+          status_histories(changelog(issue['key']))
+        end
+
+        # Only status changes are ever read from a changelog (see
+        # Jira#calculate_time_from_changelog), and a batch holds 100 tickets' histories
+        # at once — description diffs included — so the rest is dropped on arrival.
+        def status_histories(histories)
+          histories.select do |history|
+            Array(history['items']).any? { |item| item['field'] == 'status' }
+          end
+        end
+
+        def get(path, params = {})
+          expect_type(connection.get(path, params).body, Hash, path)
+        end
+
+        def post(path, payload)
+          expect_type(connection.post(path, payload).body, Hash, path)
+        end
+
+        # `f.response :json` only parses JSON content types. A 200 carrying HTML — a
+        # proxy interstitial, a maintenance page — leaves a String here, and treating
+        # that as an empty payload would report "no work" for an outage.
+        def expect_type(body, type, path)
+          return body if body.is_a?(type)
+
+          raise Error, "Unexpected response body from #{path}: expected #{type}, got #{body.class}"
+        end
+
+        # Issue keys come from Jira itself, but they are interpolated into a URL path,
+        # so they get the same guard ClickUp applies to team_id. Jira keys (SCRUM-123)
+        # satisfy Validators::ALPHANUMERIC_ID_REGEX.
+        def issue_path_segment(issue_key)
+          Validators.validate_id!(issue_key, name: 'issue_key')
+        end
+
+        def connection
+          @connection ||= Faraday.new(url: @base_url) do |f|
+            f.request :json
+            f.response :raise_error
+            f.request :retry, max: RETRY_MAX, interval: @retry_interval,
+                              max_interval: RETRY_MAX_INTERVAL, interval_randomness: RETRY_RANDOMNESS,
+                              retry_statuses: RETRY_STATUSES, methods: RETRY_METHODS
+            f.response :json
+            f.headers['Authorization'] = basic_auth_header
+            f.headers['Accept'] = 'application/json'
+            f.options.timeout = 15
+            f.options.open_timeout = 5
+          end
+        end
+
+        def basic_auth_header
+          "Basic #{Base64.strict_encode64("#{@username}:#{@api_token}")}"
+        end
+      end
+    end
+  end
+end

data/lib/snitcher/sources/jira/status_categories.rb

@@ -0,0 +1,60 @@
+# frozen_string_literal: true
+
+require 'faraday'
+
+module Snitcher
+  module Sources
+    class Jira < Source
+      # Resolves a Jira status ID to its language-independent statusCategory key.
+      #
+      # The changelog does not carry statusCategory. A status change gives only
+      # from/fromString/to/toString, where `to` is the numeric status ID and `toString`
+      # its localised name ("In Arbeit", "W realizacji"). Resolving the ID through the
+      # instance's status catalogue keeps time calculation independent of language —
+      # the design principle stated in the ARCHITECTURE NOTES on Jira.
+      #
+      # The catalogue is fetched once per source instance and memoised. A failed fetch
+      # degrades to an empty map, which pushes extract_status_category onto its
+      # name-based fallback rather than failing the whole report.
+      # @api private
+      class StatusCategories
+        # Jira's "No Category" statusCategory. It carries no information about whether
+        # the status means started, finished or neither, but it is a truthy String, so
+        # returning it would short-circuit the `||` in Jira#extract_status_category and
+        # hide the configured status names — in exactly the case they are needed.
+        UNCATEGORISED_KEY = 'undefined'
+
+        def initialize(client)
+          @client = client
+        end
+
+        # @param status_id [String, Integer, nil] the changelog item's `to` value
+        # @return [String, nil] 'new', 'indeterminate', 'done', or nil when unknown
+        def category_for(status_id)
+          return nil if status_id.nil?
+
+          key = map[status_id.to_s]
+          key unless key == UNCATEGORISED_KEY
+        end
+
+        private
+
+        def map
+          @map ||= build_map
+        end
+
+        def build_map
+          @client.statuses.each_with_object({}) do |status, acc|
+            id = status['id']
+            key = status.dig('statusCategory', 'key')
+            acc[id.to_s] = key if id && key
+          end
+        rescue Faraday::Error, Snitcher::Error => e
+          warn "[jira] Warning: could not fetch status categories (#{e.message}); " \
+               'falling back to status-name mapping'
+          {}
+        end
+      end
+    end
+  end
+end