pgbus 0.19.4 → 0.20.1

data/lib/pgbus/web/data_source.rb

@@ -13,6 +13,9 @@ module Pgbus
       include ProcessList
       include DashboardSignals
       include DeployTaskList
+      include WorkflowList
+      include WorkflowActions
+      include WorkflowInsights
 
       # pgbus_failed_events rows outside the EventBus handler queues; binds
       # handler_queue_names_literal as $1.
@@ -318,7 +321,7 @@ module Pgbus
       def discard_failed_event(id)
         event = failed_event(id)
         if event
-          release_lock_for_payload(event["payload"])
+          release_locks_held_by_failed_events("id = $1", [id.to_i])
           archive_failed_message(event)
         end
 
@@ -917,7 +920,8 @@ module Pgbus
         # memory — otherwise the message is archived but the lock row stays
         # behind, blocking later publishes with the same key. Mirrors
         # discard_event.
-        release_lock_for_payload(detail[:message])
+        release_lock_held_by(detail[:message], queue_name: lock_queue_for_table(queue_name), msg_id: detail[:msg_id],
+                                               enqueued_at: detail[:enqueued_at])
         @client.archive_message(queue_name, msg_id.to_i, prefixed: false)
         clear_event_failure(queue_name, msg_id)
         true
@@ -1551,6 +1555,13 @@ module Pgbus
       end
 
       # --- Lock cleanup helpers ---
+      #
+      # Discarding a message releases only the uniqueness lock that message
+      # holds (UniquenessKey.release_held_by!): its job may have handed the
+      # lock to a retry that is still waiting (issue #564), or a successor may
+      # have taken the key, and neither is the discarded message's to free.
+      # The Locks page's own deletes stay key-only: there the key is the
+      # operator's target.
 
       # Extract uniqueness key from a queue message and release its lock.
       def release_lock_for_message(queue_name, msg_id)
@@ -1561,39 +1572,139 @@ module Pgbus
         )
         return unless row
 
-        release_lock_for_payload(row["message"])
+        holder_queue, holder_msg_id, sent_at = lock_holder(queue_name, row["msg_id"], row["headers"], row["enqueued_at"])
+        release_lock_held_by(row["message"], queue_name: holder_queue, msg_id: holder_msg_id, enqueued_at: sent_at)
       rescue StandardError => e
         Pgbus.logger.debug { "[Pgbus::Web] Error releasing lock for message #{msg_id}: #{e.message}" }
       end
 
-      # Extract uniqueness key from a JSON payload string and release its lock.
-      def release_lock_for_payload(payload_str)
+      # [queue, msg_id, sent_at] of the message whose lock a discard may
+      # release. A live message is its own holder. A dead-lettered copy's is
+      # the message it was moved from, which its dead-letter header names:
+      # the copy's own enqueued_at is the move to the DLQ, later than a lock
+      # its job may have handed to a retry. A copy whose header does not name
+      # the original (dead-lettered by an earlier release, which freed its
+      # lock then) has no send time to go by, so only a bound lock could
+      # match, and none is bound to a DLQ copy.
+      def lock_holder(queue_name, msg_id, headers, enqueued_at)
+        own = [lock_queue_for_table(queue_name), msg_id.to_i]
+        return [*own, enqueued_at] unless queue_name.to_s.end_with?(Pgbus::DEAD_LETTER_SUFFIX)
+
+        block = DeadLetterHeader.parse(headers)
+        source_msg_id = block && Integer(block["source_msg_id"], exception: false)
+        return [*own, nil] unless source_msg_id && block["source_queue"]
+
+        [lock_queue_for_table(block["source_queue"]), source_msg_id, block["source_enqueued_at"]]
+      end
+
+      # Release the lock held by the message whose JSON payload this is.
+      def release_lock_held_by(payload_str, queue_name:, msg_id:, enqueued_at:)
         key = extract_uniqueness_key_from_payload_str(payload_str)
-        UniquenessKey.release!(key) if key
+        return unless key
+
+        UniquenessKey.release_held_by!(key, queue_name: queue_name, msg_id: msg_id.to_i, enqueued_at: enqueued_at)
       end
 
-      # Extract uniqueness keys from a collection of formatted messages and
-      # release all associated locks in a single query.
+      # Release the locks a collection of formatted messages hold, in a single
+      # query.
       def release_locks_for_messages(messages)
-        keys = messages.filter_map { |m| extract_uniqueness_key_from_payload_str(m[:message]) }
-        UniquenessKey.where(lock_key: keys).delete_all if keys.any?
+        holders = messages.filter_map do |m|
+          key = extract_uniqueness_key_from_payload_str(m[:message])
+          [key, *lock_holder(m[:queue_name], m[:msg_id], m[:headers], m[:enqueued_at])] if key
+        end
+        UniquenessKey.release_all_held_by!(holders) if holders.any?
       rescue StandardError => e
         Pgbus.logger.debug { "[Pgbus::Web] Error releasing locks for messages: #{e.message}" }
       end
 
-      # Collect uniqueness keys from all failed events and release their locks.
       def release_locks_for_failed_events
+        release_locks_held_by_failed_events(NOT_HANDLER_QUEUE, [handler_queue_names_literal])
+      end
+
+      # Release the locks the messages of the failed jobs matching +condition+
+      # hold. A failed job's message is still in its queue, waiting for its
+      # next read, and its enqueued_at is the send time: failed_at is not, a
+      # lock handed to a retry before the failure would count as the job's.
+      # A message that is gone has no send time, so only a bound lock goes.
+      def release_locks_held_by_failed_events(condition, binds)
         rows = connection.select_all(
-          "SELECT payload FROM pgbus_failed_events WHERE #{NOT_HANDLER_QUEUE}", "Pgbus Collect Failed Keys",
-          [handler_queue_names_literal]
+          "SELECT queue_name, msg_id, payload FROM pgbus_failed_events WHERE #{condition}",
+          "Pgbus Collect Failed Keys", binds
         )
 
-        keys = rows.to_a.filter_map { |row| extract_uniqueness_key_from_payload_str(row["payload"]) }
-        UniquenessKey.where(lock_key: keys).delete_all if keys.any?
+        # queue_name is the logical name the executor saw: the message is looked
+        # up under it, and the lock is released under its table spelling.
+        # A key proc may return a number; the queue lookup reads it as text.
+        holders = rows.to_a.filter_map do |row|
+          key = extract_uniqueness_key_from_payload_str(row["payload"])&.to_s
+          [key, row["queue_name"], row["msg_id"].to_i] if key && row["msg_id"]
+        end
+        holders = holders.group_by { |_key, queue, _msg_id| queue }.flat_map do |queue, entries|
+          failed_queue_lock_holders(queue, entries)
+        end
+        UniquenessKey.release_all_held_by!(holders) if holders.any?
       rescue StandardError => e
         Pgbus.logger.debug { "[Pgbus::Web] Error releasing locks for failed events: #{e.message}" }
       end
 
+      # The release tuples for one queue's failed jobs. A queue that cannot
+      # be resolved (a stored name no longer valid as a table name) is skipped
+      # with a warning, so it does not stop the other queues' release.
+      def failed_queue_lock_holders(queue, entries)
+        sent = queued_send_times(queue, entries)
+        lock_queue = Uniqueness.lock_queue_name(queue)
+        entries.map { |key, _queue, msg_id| [key, lock_queue, msg_id, sent[[msg_id, key]]] }
+      rescue ActiveRecord::StatementInvalid
+        raise
+      rescue StandardError => e
+        Pgbus.logger.warn { "[Pgbus::Web] Skipped releasing locks for failed jobs on #{queue.inspect}: #{e.message}" }
+        []
+      end
+
+      # { [msg_id, uniqueness key] => enqueued_at } for the messages of
+      # +entries+ ([key, queue, msg_id]) still in the logical queue. Every
+      # priority sub-queue is searched: a failed event records the logical
+      # queue only, and msg_ids repeat across sub-queues, so the key must
+      # match too, and the earliest send wins.
+      def queued_send_times(queue, entries)
+        msg_ids = "{#{entries.map { |_key, _queue, msg_id| msg_id }.uniq.join(",")}}"
+        @client.physical_queue_names(queue).each_with_object({}) do |physical, found|
+          queued_rows(physical, msg_ids).each do |row|
+            pair = [row["msg_id"].to_i, row["lock_key"]]
+            found[pair] = [found[pair], row["enqueued_at"]].compact.min
+          end
+        end
+      end
+
+      # Each read runs in its own savepoint: a sub-queue the strategy names
+      # but that does not exist raises, and inside a caller's transaction
+      # that would abort the discard's own statements after it.
+      def queued_rows(physical, msg_ids)
+        connection.transaction(requires_new: true) do
+          connection.select_all(
+            "SELECT msg_id, enqueued_at, message->>'#{Uniqueness::METADATA_KEY}' AS lock_key " \
+            "FROM pgmq.q_#{sanitize_name(physical)} WHERE msg_id = ANY($1::bigint[])",
+            "Pgbus Failed Message Sent At", [msg_ids]
+          ).to_a
+        end
+      rescue ActiveRecord::StatementInvalid => e
+        raise unless defined?(PG::UndefinedTable) && e.cause.is_a?(PG::UndefinedTable)
+
+        []
+      end
+
+      # The queue a uniqueness lock records for a message in this physical
+      # queue: the table name without its prefix and a priority sub-queue's
+      # _pN, which is how Uniqueness.lock_queue_name spells the queue the lock
+      # was bound for. A dead-letter queue keeps its suffix, so its copy's
+      # msg_id (another sequence) never matches a lock bound to a live message.
+      def lock_queue_for_table(physical_name)
+        name = physical_name.to_s.delete_prefix("#{Pgbus.configuration.queue_prefix}_")
+        return name if name.end_with?(Pgbus::DEAD_LETTER_SUFFIX) || Pgbus.configuration.priority_levels.to_i <= 1
+
+        name.sub(/_p\d+\z/, "")
+      end
+
       # Single unwrap point for PGMQ message / failed_event payload strings.
       # Accepts a raw JSON string or an already-parsed Hash and returns the
       # uniqueness metadata key, or nil when the payload is blank, unparseable,

data/lib/pgbus/web/workflow_action_plan.rb

@@ -0,0 +1,62 @@
+# frozen_string_literal: true
+
+module Pgbus
+  module Web
+    # What a Workflows dashboard action will do (issue #542), so its button
+    # can say so before anyone presses it. Which actions exist at all is
+    # Workflow::States.available_actions; this only names their effect.
+    module WorkflowActionPlan
+      # The status the runs of a needs-attention group are in.
+      GROUP_STATUS = { "stalled" => "paused", "paused_error" => "paused", "paused_manual" => "paused",
+                       "overdue" => "ready", "waiting_overdue" => "waiting" }.freeze
+
+      module_function
+
+      def group_status(reason) = GROUP_STATUS.fetch(reason.to_s)
+
+      # [action, step_name] for resume! of a paused run, as OutsideVerbs.resume
+      # decides it from the run's latest step row:
+      #   :retry    the step failed: it runs again as a new attempt
+      #   :continue the step completed or was skipped: the next step runs
+      #   :finish   …and it was the last: the run finishes
+      #   :resend   the step was scheduled when the run paused: it is sent again
+      #   :rewait   the step was waiting: it waits again
+      #   :carry_on the step was still running: the run goes on performing it
+      # [nil, nil] when the run is not paused, or its class is gone.
+      def resume_plan(record, latest, klass)
+        return [nil, nil] unless record.status == "paused" && latest
+
+        case latest.status
+        when "failed" then [:retry, latest.step_name]
+        when "scheduled" then [:resend, latest.step_name]
+        when "waiting" then [:rewait, latest.step_name]
+        when "in_progress" then [:carry_on, latest.step_name]
+        when "completed", "skipped" then following_plan(latest, klass)
+        else [nil, nil]
+        end
+      end
+
+      def following_plan(latest, klass)
+        return [nil, nil] unless klass
+        # resume! raises StepNotDefined for a step the class no longer
+        # declares (and no removed_step covers): no plan to promise.
+        return [nil, nil] unless klass.workflow_steps.entries.any? { |entry| entry.name == latest.step_name }
+
+        following = klass.workflow_steps.next_after(latest.step_name)
+        following ? [:continue, following.name] : [:finish, nil]
+      end
+
+      # The signal names a waiting step takes, for the Send signal select:
+      # [name] for a wait_for: name or SignalMatcher, [] for a custom matcher
+      # (the form then takes free text) or a step that is not waiting on one.
+      def signal_names(latest, klass)
+        return [] unless latest&.status == "waiting" && klass
+        return [] if latest.has_attribute?(:batch_id) && latest.batch_id
+
+        matcher = klass.workflow_steps.fetch(latest.step_name)&.wait_for
+        name = matcher.respond_to?(:signal_name) ? matcher.signal_name : nil
+        name ? [name.to_s] : []
+      end
+    end
+  end
+end

data/lib/pgbus/web/workflow_insights_queries.rb

@@ -0,0 +1,250 @@
+# frozen_string_literal: true
+
+module Pgbus
+  module Web
+    # The SQL behind the workflow insights (issue #549): one query per block
+    # of the page, each bounded by the window and served by an index #540
+    # created. The step stats read idx_pgbus_workflow_steps_insights
+    # (workflow_class, step_name, finished_at); the run counts read
+    # idx_pgbus_workflows_class_status (workflow_class, status, created_at),
+    # with every status listed so the created_at range applies under each;
+    # the ended runs read idx_pgbus_workflows_status (status, updated_at): a
+    # run ends in the UPDATE that sets its finished_at/cancelled_at, so
+    # updated_at >= the ended-at bound keeps every row the window wants.
+    #
+    # Status lists are the States constants, inlined; every value that comes
+    # from outside (a class name, a time, a bucket length) is a bind.
+    module WorkflowInsightsQueries
+      ALL = Workflow::States::WORKFLOW
+      ONGOING = Workflow::States::ONGOING
+      # Upper bounds (ms) of the end-to-end duration histogram: < 1 s, 10 s,
+      # 1 min, 10 min, 1 h, 6 h, 1 d, 7 d, and the open last bin.
+      DURATION_BOUNDS_MS = [1_000, 10_000, 60_000, 600_000, 3_600_000, 21_600_000, 86_400_000, 604_800_000].freeze
+      REVISIONS = 5
+
+      module_function
+
+      # Every workflow_class with a row, by a loose index scan (one probe per
+      # class) instead of a DISTINCT over every run.
+      def class_names
+        sql = <<~SQL
+          WITH RECURSIVE classes AS (
+            (SELECT workflow_class FROM #{runs} ORDER BY workflow_class LIMIT 1)
+            UNION ALL
+            SELECT (SELECT w.workflow_class FROM #{runs} w WHERE w.workflow_class > c.workflow_class
+                    ORDER BY w.workflow_class LIMIT 1)
+            FROM classes c WHERE c.workflow_class IS NOT NULL
+          )
+          SELECT workflow_class FROM classes WHERE workflow_class IS NOT NULL
+        SQL
+        select(sql, "Classes").rows.map(&:first)
+      end
+
+      def class_known?(workflow_class)
+        select("SELECT 1 FROM #{runs} WHERE workflow_class = $1 LIMIT 1", "Class Known", [workflow_class]).rows.any?
+      end
+
+      # The ongoing runs by (class, status, current step).
+      def ongoing(workflow_class: nil)
+        sql = <<~SQL
+          SELECT workflow_class, status, current_step, COUNT(*)
+          FROM #{runs}
+          WHERE status IN (#{list(ONGOING)})#{" AND workflow_class = $1" if workflow_class}
+          GROUP BY workflow_class, status, current_step
+        SQL
+        select(sql, "Ongoing", [workflow_class].compact).cast_values.map do |klass, status, step, runs|
+          { workflow_class: klass, status: status, step: step, runs: runs.to_i }
+        end
+      end
+
+      # Per class: the runs that finished or were cancelled between +since+
+      # and +until_time+, and the end-to-end percentiles (created → finished)
+      # of the finished.
+      def ended(since:, until_time:, workflow_class: nil)
+        sql = <<~SQL
+          SELECT workflow_class, COUNT(*) FILTER (WHERE status = 'finished'), COUNT(*) FILTER (WHERE status = 'cancelled'),
+                 COUNT(e2e_ms), #{percentiles("e2e_ms", 0.5, 0.95, 0.99)}
+          FROM (#{ended_runs(until_bind: "$2", class_bind: ("$3" if workflow_class))}) ended
+          GROUP BY workflow_class
+        SQL
+        select(sql, "Ended", [since, until_time, workflow_class].compact).cast_values.map do |klass, finished, cancelled, samples, *pcts|
+          { workflow_class: klass, finished: finished.to_i, cancelled: cancelled.to_i,
+            e2e_ms: percentile_hash(%i[p50 p95 p99], pcts, samples) }
+        end
+      end
+
+      # { bin => runs } of one class's finished runs over DURATION_BOUNDS_MS
+      # (bin 0 is under the first bound, the last bin is past the last one).
+      def duration_histogram(workflow_class, since:, until_time:)
+        sql = <<~SQL
+          SELECT width_bucket(e2e_ms::float8, ARRAY[#{DURATION_BOUNDS_MS.join(", ")}]::float8[]), COUNT(*)
+          FROM (#{ended_runs(until_bind: "$2", class_bind: "$3")}) ended
+          WHERE status = 'finished'
+          GROUP BY 1
+        SQL
+        select(sql, "Durations", [since, until_time, workflow_class]).rows.to_h { |bin, runs| [bin.to_i, runs.to_i] }
+      end
+
+      # { bucket start => [started, finished, cancelled] }. Buckets are
+      # multiples of +bucket+ seconds on the wall clock of +zone+ (an IANA
+      # name), so a day bucket starts at that zone's midnight on both sides
+      # of a DST change; a bucket start is that wall clock's epoch seconds
+      # (WorkflowInsightsReport.wall_epoch).
+      def throughput(workflow_class, since:, until_time:, bucket:, zone: "UTC")
+        sql = <<~SQL
+          SELECT bucket, SUM(started), SUM(finished), SUM(cancelled) FROM (
+            SELECT #{wall_bucket("created_at")} AS bucket, 1 AS started, 0 AS finished, 0 AS cancelled
+            FROM #{runs}
+            WHERE workflow_class = $2 AND status IN (#{list(ALL)}) AND created_at BETWEEN $1 AND $5
+            UNION ALL
+            SELECT #{wall_bucket("ended_at")}, 0, (status = 'finished')::int, (status = 'cancelled')::int
+            FROM (#{ended_runs(until_bind: "$5", class_bind: "$2")}) ended
+          ) buckets
+          GROUP BY bucket
+        SQL
+        select(sql, "Throughput", [since, workflow_class, bucket, zone, until_time]).rows.to_h do |start, started, finished, cancelled|
+          [start.to_i * bucket, [started.to_i, finished.to_i, cancelled.to_i]]
+        end
+      end
+
+      # The bucket number of a UTC timestamp column on the wall clock of the
+      # zone bound as $4, in buckets of $3 seconds.
+      def wall_bucket(column)
+        "FLOOR(EXTRACT(EPOCH FROM (#{column} AT TIME ZONE 'UTC' AT TIME ZONE $4)) / $3)::bigint"
+      end
+
+      # Per step: the attempts that finished between +since+ and +until_time+.
+      def step_stats(workflow_class, since:, until_time:)
+        sql = <<~SQL
+          SELECT step_name, #{stat_columns}, MIN(created_at)
+          FROM (#{attempt_rows}) attempts
+          GROUP BY step_name
+        SQL
+        select(sql, "Steps", [workflow_class, since, until_time]).cast_values.to_h do |name, *stats, first_seen_at|
+          [name, attempt_stats(stats).merge(first_seen_at: first_seen_at)]
+        end
+      end
+
+      # Per step, over the runs started between +since+ and +until_time+: how many have a row
+      # for it (reached) and its mean position in their runs (the step order
+      # of a class that no longer loads). Partitioned by the run's bigint id:
+      # sorting 50k workflow_id strings under a locale collation costs ~4x.
+      def funnel_reached(workflow_class, since:, until_time:)
+        sql = <<~SQL
+          SELECT step_name, COUNT(*), AVG(position) FROM (
+            SELECT s.step_name, RANK() OVER (PARTITION BY w.id ORDER BY MIN(s.id)) AS position
+            FROM #{runs} w JOIN #{steps} s ON s.workflow_id = w.workflow_id
+            WHERE w.workflow_class = $1 AND w.status IN (#{list(ALL)}) AND w.created_at BETWEEN $2 AND $3
+            GROUP BY w.id, s.step_name
+          ) reached
+          GROUP BY step_name
+        SQL
+        select(sql, "Funnel", [workflow_class, since, until_time]).rows.to_h do |name, reached, position|
+          [name, { reached: reached.to_i, position: position.to_f }]
+        end
+      end
+
+      # Where each run started between +since+ and +until_time+ is now:
+      # { [step, status] => runs }.
+      def funnel_stopped(workflow_class, since:, until_time:)
+        sql = <<~SQL
+          SELECT current_step, status, COUNT(*)
+          FROM #{runs}
+          WHERE workflow_class = $1 AND status IN (#{list(ALL)}) AND created_at BETWEEN $2 AND $3
+          GROUP BY current_step, status
+        SQL
+        select(sql, "Stopped", [workflow_class, since, until_time]).rows.to_h { |step, status, runs| [[step, status], runs.to_i] }
+      end
+
+      # The step stats of the REVISIONS app revisions that finished an
+      # attempt last, per (revision, step) and per revision (step nil).
+      def by_revision(workflow_class, since:, until_time:)
+        sql = <<~SQL
+          WITH revisions AS (
+            SELECT app_revision, MIN(finished_at) AS first_seen_at, MAX(finished_at) AS last_seen_at
+            FROM #{steps}
+            WHERE workflow_class = $1 AND finished_at BETWEEN $2 AND $3 AND app_revision IS NOT NULL
+            GROUP BY app_revision
+            ORDER BY last_seen_at DESC
+            LIMIT #{REVISIONS}
+          )
+          SELECT attempts.app_revision, revisions.first_seen_at, revisions.last_seen_at, attempts.step_name, #{stat_columns}
+          FROM (#{attempt_rows}) attempts
+          JOIN revisions ON revisions.app_revision = attempts.app_revision
+          GROUP BY GROUPING SETS ((attempts.app_revision, revisions.first_seen_at, revisions.last_seen_at, attempts.step_name),
+                                  (attempts.app_revision, revisions.first_seen_at, revisions.last_seen_at))
+          ORDER BY revisions.last_seen_at DESC, attempts.step_name NULLS FIRST
+        SQL
+        select(sql, "Revisions", [workflow_class, since, until_time]).cast_values.map do |revision, first_seen, last_seen, name, *stats|
+          { revision: revision, first_seen_at: first_seen, last_seen_at: last_seen, step: name, stats: attempt_stats(stats) }
+        end
+      end
+
+      # The finished and cancelled runs that ended between $1 and +until_bind+
+      # (of the class at +class_bind+): never one dated after now (a skewed
+      # clock), which the charts would not show either.
+      def ended_runs(until_bind:, class_bind: nil)
+        <<~SQL
+          SELECT workflow_class, status, COALESCE(finished_at, cancelled_at) AS ended_at,
+                 CASE WHEN status = 'finished' THEN GREATEST(EXTRACT(EPOCH FROM (finished_at - created_at)) * 1000, 0) END AS e2e_ms
+          FROM #{runs}
+          WHERE status IN ('finished', 'cancelled') AND updated_at >= $1
+            AND COALESCE(finished_at, cancelled_at) BETWEEN $1 AND #{until_bind}#{" AND workflow_class = #{class_bind}" if class_bind}
+        SQL
+      end
+
+      # The attempts of class $1 that finished between $2 and $3 (never one
+      # dated after now). run_ms: completed or
+      # failed attempts that started; pickup_ms (scheduled → started, never
+      # negative): every attempt that started.
+      def attempt_rows
+        <<~SQL
+          SELECT step_name, status, outcome, error_class, app_revision, created_at,
+                 CASE WHEN status IN ('completed', 'failed') AND started_at IS NOT NULL
+                      THEN GREATEST(EXTRACT(EPOCH FROM (finished_at - started_at)) * 1000, 0) END AS run_ms,
+                 CASE WHEN started_at IS NOT NULL
+                      THEN GREATEST(EXTRACT(EPOCH FROM (started_at - scheduled_at)) * 1000, 0) END AS pickup_ms
+          FROM #{steps}
+          WHERE workflow_class = $1 AND finished_at BETWEEN $2 AND $3
+        SQL
+      end
+
+      STAT_KEYS = %i[attempts completed failed reattempted skipped cancelled].freeze
+
+      def stat_columns
+        <<~SQL.squish
+          COUNT(*), COUNT(*) FILTER (WHERE status = 'completed'), COUNT(*) FILTER (WHERE error_class IS NOT NULL),
+          COUNT(*) FILTER (WHERE outcome = 'reattempted'), COUNT(*) FILTER (WHERE status = 'skipped'),
+          COUNT(*) FILTER (WHERE status = 'cancelled'),
+          COUNT(run_ms), #{percentiles("run_ms", 0.5, 0.95, 0.99)},
+          COUNT(pickup_ms), #{percentiles("pickup_ms", 0.5, 0.95)}
+        SQL
+      end
+
+      def attempt_stats(values)
+        counts = STAT_KEYS.zip(values.first(STAT_KEYS.size).map(&:to_i)).to_h
+        run_samples, run50, run95, run99, pickup_samples, pickup50, pickup95 = values.drop(STAT_KEYS.size)
+        counts.merge(run_ms: percentile_hash(%i[p50 p95 p99], [run50, run95, run99], run_samples),
+                     pickup_ms: percentile_hash(%i[p50 p95], [pickup50, pickup95], pickup_samples))
+      end
+
+      def percentiles(column, *fractions)
+        fractions.map { |f| "percentile_cont(#{f}) WITHIN GROUP (ORDER BY #{column})" }.join(", ")
+      end
+
+      def percentile_hash(keys, values, samples)
+        keys.zip(values.map { |v| v&.to_f&.round(1) }).to_h.merge(samples: samples.to_i)
+      end
+
+      def list(statuses) = statuses.map { |status| connection.quote(status) }.join(", ")
+
+      def runs = WorkflowEntry.quoted_table_name
+
+      def steps = WorkflowStep.quoted_table_name
+
+      def connection = WorkflowEntry.connection
+
+      def select(sql, name, binds = []) = connection.exec_query(sql, "WorkflowInsights #{name}", binds)
+    end
+  end
+end