@worker-manager/metrics 1.0.0

package/dist/HistoryStore.d.ts

@@ -0,0 +1,37 @@
+import type { MetricsClient } from './connection';
+import { type MetricsKeys } from './keys';
+export interface Retention {
+    /** Days of minute-level detail. Doubles as the recorder's catch-up window. */
+    minutes: number;
+    /** Days of hourly rollup. */
+    hours: number;
+    /** Days of daily totals, which is what the shipped charts read. */
+    days: number;
+}
+export declare class HistoryStore {
+    private readonly redis;
+    private readonly keys;
+    readonly retention: Retention;
+    constructor(opts: {
+        redis: MetricsClient;
+        keys: MetricsKeys;
+        retention: Retention;
+    });
+    upsertMinute(queue: string, metric: string, minute: number, value: number): Promise<void>;
+    /**
+     * Raw HMGET of the totals hash: `null` means the day was never recorded, a present
+     * string (including `'0'`) means it was. Distinguishing "missing" from "stored zero"
+     * matters for callers deciding between empty history vs. a zero-backfilled series.
+     */
+    readDailyTotalsRaw(queue: string, metric: string, days: string[]): Promise<(string | null)[]>;
+    readDayMinutes(queue: string, metric: string, day: string): Promise<Record<string, number>>;
+    /**
+     * Hourly buckets for one day, keyed by absolute hour index.
+     *
+     * Falls back to folding the minute hash when the hourly rollup is absent, which covers
+     * days recorded before the rollup existed. Once those minute hashes age out, the days
+     * they cover are already outside the minute window anyway.
+     */
+    readDayHours(queue: string, metric: string, day: string): Promise<Record<string, number>>;
+    private readNumericHash;
+}

package/dist/HistoryStore.js

@@ -0,0 +1,140 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.HistoryStore = void 0;
+const keys_1 = require("./keys");
+/**
+ * Atomic, idempotent upsert of one minute bucket into all three resolutions at once.
+ *
+ * KEYS[1] queue minute hash     ARGV[1] minute field   ARGV[5] minute-tier ttl seconds
+ * KEYS[2] queue hour hash       ARGV[2] hour field     ARGV[6] hour-tier ttl seconds
+ * KEYS[3] queue totals hash     ARGV[3] day field      ARGV[7] day-tier ttl seconds
+ * KEYS[4] global minute hash    ARGV[4] value          ARGV[8] oldest day to keep
+ * KEYS[5] global hour hash
+ * KEYS[6] global totals hash
+ *
+ * The minute hash is the ledger the whole thing is built on: `delta` is the difference
+ * against the value already stored there, so re-snapshotting an overlapping window (a
+ * restart, or a second recorder process) applies a delta of zero and the coarser tiers
+ * never double-count. Rolling up on write rather than compacting later keeps that property:
+ * one script, one delta, every resolution consistent, nothing to schedule or resume.
+ *
+ * Retention is per tier. Day-scoped keys fall off on their own, since their TTL is only
+ * refreshed while that day is being written, so each dies its tier's retention after the
+ * day it holds. The totals hashes are written every day, so their TTL keeps rolling forward
+ * and they would otherwise grow a field per day forever; they are trimmed here instead.
+ * Day fields are ISO `YYYY-MM-DD`, which sorts lexicographically, so a plain string compare
+ * against the cutoff is enough. Trimming only runs on the first write of a new day, which
+ * is about once a day per queue.
+ */
+const UPSERT_MINUTE = `
+local function trim(key, cutoff)
+  local fields = redis.call('HKEYS', key)
+  local stale = {}
+  for i = 1, #fields do
+    if fields[i] < cutoff then
+      stale[#stale + 1] = fields[i]
+      if #stale == 256 then
+        redis.call('HDEL', key, unpack(stale))
+        stale = {}
+      end
+    end
+  end
+  if #stale > 0 then
+    redis.call('HDEL', key, unpack(stale))
+  end
+end
+
+local old = tonumber(redis.call('HGET', KEYS[1], ARGV[1]) or '0')
+local val = tonumber(ARGV[4])
+if val == old then
+  return 0
+end
+local delta = val - old
+local newDay = redis.call('HEXISTS', KEYS[3], ARGV[3]) == 0
+redis.call('HSET', KEYS[1], ARGV[1], val)
+redis.call('HINCRBY', KEYS[2], ARGV[2], delta)
+redis.call('HINCRBY', KEYS[3], ARGV[3], delta)
+redis.call('HINCRBY', KEYS[4], ARGV[1], delta)
+redis.call('HINCRBY', KEYS[5], ARGV[2], delta)
+redis.call('HINCRBY', KEYS[6], ARGV[3], delta)
+redis.call('EXPIRE', KEYS[1], ARGV[5])
+redis.call('EXPIRE', KEYS[2], ARGV[6])
+redis.call('EXPIRE', KEYS[3], ARGV[7])
+redis.call('EXPIRE', KEYS[4], ARGV[5])
+redis.call('EXPIRE', KEYS[5], ARGV[6])
+redis.call('EXPIRE', KEYS[6], ARGV[7])
+if newDay then
+  trim(KEYS[3], ARGV[8])
+  trim(KEYS[6], ARGV[8])
+end
+return delta
+`;
+const SECONDS_PER_DAY = 86400;
+function ttl(days) {
+    return String(Math.max(1, Math.floor(days * SECONDS_PER_DAY)));
+}
+class HistoryStore {
+    constructor(opts) {
+        this.redis = opts.redis;
+        this.keys = opts.keys;
+        this.retention = {
+            minutes: Math.max(1, Math.floor(opts.retention.minutes)),
+            hours: Math.max(1, Math.floor(opts.retention.hours)),
+            days: Math.max(1, Math.floor(opts.retention.days)),
+        };
+    }
+    async upsertMinute(queue, metric, minute, value) {
+        const day = (0, keys_1.minuteToDay)(minute);
+        await this.redis.eval(UPSERT_MINUTE, 6, this.keys.day(queue, metric, day), this.keys.hour(queue, metric, day), this.keys.totals(queue, metric), this.keys.day(keys_1.GLOBAL_QUEUE, metric, day), this.keys.hour(keys_1.GLOBAL_QUEUE, metric, day), this.keys.totals(keys_1.GLOBAL_QUEUE, metric), String(minute), String((0, keys_1.minuteToHour)(minute)), day, String(value), ttl(this.retention.minutes), ttl(this.retention.hours), ttl(this.retention.days), (0, keys_1.shiftDay)(day, -this.retention.days));
+    }
+    /**
+     * Raw HMGET of the totals hash: `null` means the day was never recorded, a present
+     * string (including `'0'`) means it was. Distinguishing "missing" from "stored zero"
+     * matters for callers deciding between empty history vs. a zero-backfilled series.
+     */
+    async readDailyTotalsRaw(queue, metric, days) {
+        if (days.length === 0) {
+            return [];
+        }
+        // Pass `days` as a single array (ioredis flattens one level internally) instead of
+        // spreading it: spreading blows the JS argument-count limit for large ranges
+        // ("Maximum call stack size exceeded"). The cast works around ioredis's types only
+        // declaring the spread overload; kept as a method call to preserve `this` binding.
+        const redis = this.redis;
+        return redis.hmget(this.keys.totals(queue, metric), days);
+    }
+    async readDayMinutes(queue, metric, day) {
+        return this.readNumericHash(this.keys.day(queue, metric, day));
+    }
+    /**
+     * Hourly buckets for one day, keyed by absolute hour index.
+     *
+     * Falls back to folding the minute hash when the hourly rollup is absent, which covers
+     * days recorded before the rollup existed. Once those minute hashes age out, the days
+     * they cover are already outside the minute window anyway.
+     */
+    async readDayHours(queue, metric, day) {
+        var _a;
+        const hours = await this.readNumericHash(this.keys.hour(queue, metric, day));
+        if (Object.keys(hours).length > 0) {
+            return hours;
+        }
+        const minutes = await this.readDayMinutes(queue, metric, day);
+        const folded = {};
+        for (const field of Object.keys(minutes)) {
+            const hour = String((0, keys_1.minuteToHour)(Number(field)));
+            folded[hour] = ((_a = folded[hour]) !== null && _a !== void 0 ? _a : 0) + minutes[field];
+        }
+        return folded;
+    }
+    async readNumericHash(key) {
+        const raw = await this.redis.hgetall(key);
+        const out = {};
+        for (const field of Object.keys(raw)) {
+            out[field] = Number(raw[field]) || 0;
+        }
+        return out;
+    }
+}
+exports.HistoryStore = HistoryStore;
+//# sourceMappingURL=HistoryStore.js.map

package/dist/HistoryStore.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"HistoryStore.js","sourceRoot":"","sources":["../src/HistoryStore.ts"],"names":[],"mappings":";;;AACA,iCAA6F;AAW7F;;;;;;;;;;;;;;;;;;;;;;;GAuBG;AACH,MAAM,aAAa,GAAG;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;CA0CrB,CAAC;AAEF,MAAM,eAAe,GAAG,KAAK,CAAC;AAE9B,SAAS,GAAG,CAAC,IAAY;IACvB,OAAO,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,IAAI,CAAC,KAAK,CAAC,IAAI,GAAG,eAAe,CAAC,CAAC,CAAC,CAAC;AACjE,CAAC;AAED,MAAa,YAAY;IAKvB,YAAY,IAAuE;QACjF,IAAI,CAAC,KAAK,GAAG,IAAI,CAAC,KAAK,CAAC;QACxB,IAAI,CAAC,IAAI,GAAG,IAAI,CAAC,IAAI,CAAC;QACtB,IAAI,CAAC,SAAS,GAAG;YACf,OAAO,EAAE,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,SAAS,CAAC,OAAO,CAAC,CAAC;YACxD,KAAK,EAAE,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,SAAS,CAAC,KAAK,CAAC,CAAC;YACpD,IAAI,EAAE,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,CAAC;SACnD,CAAC;IACJ,CAAC;IAED,KAAK,CAAC,YAAY,CAAC,KAAa,EAAE,MAAc,EAAE,MAAc,EAAE,KAAa;QAC7E,MAAM,GAAG,GAAG,IAAA,kBAAW,EAAC,MAAM,CAAC,CAAC;QAChC,MAAM,IAAI,CAAC,KAAK,CAAC,IAAI,CACnB,aAAa,EACb,CAAC,EACD,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,KAAK,EAAE,MAAM,EAAE,GAAG,CAAC,EACjC,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,KAAK,EAAE,MAAM,EAAE,GAAG,CAAC,EAClC,IAAI,CAAC,IAAI,CAAC,MAAM,CAAC,KAAK,EAAE,MAAM,CAAC,EAC/B,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,mBAAY,EAAE,MAAM,EAAE,GAAG,CAAC,EACxC,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,mBAAY,EAAE,MAAM,EAAE,GAAG,CAAC,EACzC,IAAI,CAAC,IAAI,CAAC,MAAM,CAAC,mBAAY,EAAE,MAAM,CAAC,EACtC,MAAM,CAAC,MAAM,CAAC,EACd,MAAM,CAAC,IAAA,mBAAY,EAAC,MAAM,CAAC,CAAC,EAC5B,GAAG,EACH,MAAM,CAAC,KAAK,CAAC,EACb,GAAG,CAAC,IAAI,CAAC,SAAS,CAAC,OAAO,CAAC,EAC3B,GAAG,CAAC,IAAI,CAAC,SAAS,CAAC,KAAK,CAAC,EACzB,GAAG,CAAC,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,EACxB,IAAA,eAAQ,EAAC,GAAG,EAAE,CAAC,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,CACpC,CAAC;IACJ,CAAC;IAED;;;;OAIG;IACH,KAAK,CAAC,kBAAkB,CACtB,KAAa,EACb,MAAc,EACd,IAAc;QAEd,IAAI,IAAI,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YACtB,OAAO,EAAE,CAAC;QACZ,CAAC;QACD,mFAAmF;QACnF,6EAA6E;QAC7E,mFAAmF;QACnF,mFAAmF;QACnF,MAAM,KAAK,GAAG,IAAI,CAAC,KAElB,CAAC;QACF,OAAO,KAAK,CAAC,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,MAAM,CAAC,KAAK,EAAE,MAAM,CAAC,EAAE,IAAI,CAAC,CAAC;IAC5D,CAAC;IAED,KAAK,CAAC,cAAc,CAClB,KAAa,EACb,MAAc,EACd,GAAW;QAEX,OAAO,IAAI,CAAC,eAAe,CAAC,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,KAAK,EAAE,MAAM,EAAE,GAAG,CAAC,CAAC,CAAC;IACjE,CAAC;IAED;;;;;;OAMG;IACH,KAAK,CAAC,YAAY,CAAC,KAAa,EAAE,MAAc,EAAE,GAAW;;QAC3D,MAAM,KAAK,GAAG,MAAM,IAAI,CAAC,eAAe,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,KAAK,EAAE,MAAM,EAAE,GAAG,CAAC,CAAC,CAAC;QAC7E,IAAI,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YAClC,OAAO,KAAK,CAAC;QACf,CAAC;QACD,MAAM,OAAO,GAAG,MAAM,IAAI,CAAC,cAAc,CAAC,KAAK,EAAE,MAAM,EAAE,GAAG,CAAC,CAAC;QAC9D,MAAM,MAAM,GAA2B,EAAE,CAAC;QAC1C,KAAK,MAAM,KAAK,IAAI,MAAM,CAAC,IAAI,CAAC,OAAO,CAAC,EAAE,CAAC;YACzC,MAAM,IAAI,GAAG,MAAM,CAAC,IAAA,mBAAY,EAAC,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC;YACjD,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC,MAAA,MAAM,CAAC,IAAI,CAAC,mCAAI,CAAC,CAAC,GAAG,OAAO,CAAC,KAAK,CAAC,CAAC;QACtD,CAAC;QACD,OAAO,MAAM,CAAC;IAChB,CAAC;IAEO,KAAK,CAAC,eAAe,CAAC,GAAW;QACvC,MAAM,GAAG,GAAG,MAAM,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC;QAC1C,MAAM,GAAG,GAA2B,EAAE,CAAC;QACvC,KAAK,MAAM,KAAK,IAAI,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC;YACrC,GAAG,CAAC,KAAK,CAAC,GAAG,MAAM,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC,IAAI,CAAC,CAAC;QACvC,CAAC;QACD,OAAO,GAAG,CAAC;IACb,CAAC;CACF;AAjGD,oCAiGC"}

package/dist/LatencySampler.d.ts

@@ -0,0 +1,88 @@
+import type { BaseAdapter } from '@worker-manager/api/baseAdapter';
+import type { MetricsClient } from './connection';
+import type { MetricsKeys } from './keys';
+import type { LatencyStore } from './LatencyStore';
+export interface LatencySamplerOptions {
+    redis: MetricsClient;
+    keys: MetricsKeys;
+    store: LatencyStore;
+    /** Recorder tick, used to size the lease and to bound a cold start. */
+    tickMs: number;
+    /** Above this, the tick subsamples uniformly rather than fetching every job. */
+    maxSamplesPerTick?: number;
+    /**
+     * How far back from now a scan stops. Defaults to SAFETY_MARGIN_MS. Injectable so tests
+     * can set it to 0 and sample jobs that just finished, rather than sleeping past the
+     * margin in every case.
+     */
+    safetyMarginMs?: number;
+    /**
+     * Called with anything `sample()` swallows. The default is silent, which keeps a broken
+     * latency scan from taking the counter snapshot down with it, but also makes a collector
+     * that is failing every tick look exactly like an idle board. Supply this to tell the two
+     * apart. Errors thrown by the hook itself are ignored, since a throwing reporter would
+     * undo the containment it was added to observe.
+     */
+    onError?: (error: unknown, queueName: string) => void;
+}
+/**
+ * Copies job durations out of BullMQ's finished sets on the recorder's tick.
+ *
+ * BullMQ's moveToFinished does `ZADD targetSet, timestamp, jobId` and writes the same value
+ * to finishedOn, so the completed and failed sets are sorted sets scored by finish time.
+ * Scanning past a stored watermark therefore returns exactly the jobs finished since the
+ * last tick, with no gaps and no bias. The only loss is a queue whose removeOnComplete
+ * trims faster than the tick runs.
+ */
+export declare class LatencySampler {
+    private readonly redis;
+    private readonly keys;
+    private readonly store;
+    private readonly tickMs;
+    private readonly maxSamples;
+    private readonly safetyMarginMs;
+    private readonly onError?;
+    private readonly id;
+    private readonly redisBacked;
+    constructor(opts: LatencySamplerOptions);
+    static supports(adapter: BaseAdapter): boolean;
+    /**
+     * One queue, one tick. Swallows its own errors: the counter snapshot is the more
+     * important metric and must not fail as collateral damage from a latency scan. Pass
+     * `onError` to see what was swallowed; without it a collector failing every tick is
+     * indistinguishable from a queue with nothing to sample.
+     */
+    sample(adapter: BaseAdapter): Promise<void>;
+    /** `getQueueKey` answers for a PostgreSQL-backed queue too, with keys no Redis holds. */
+    private isRedisBacked;
+    /**
+     * Increments are not idempotent the way the counter upsert is, so two recorders scanning
+     * the same range would double every histogram. Only the lease holder scans.
+     *
+     * The TTL is a crash ceiling, not the normal lifetime: a process that dies mid-scan must
+     * not lock the queue out forever. The normal path releases in a finally, because a lease
+     * outliving its scan would make the next tick no-op and halve the sampling rate.
+     */
+    private acquireLease;
+    /** Compare and delete, so a lease that already expired and was retaken is left alone. */
+    private releaseLease;
+    /**
+     * Bounded rather than eternal: an unexpiring watermark would leave one key behind per
+     * queue forever once that queue is purged or decommissioned, which is exactly the
+     * unbounded-storage failure this package exists to avoid. The day retention is already the
+     * horizon everything else here is bounded by. Losing the watermark just means the next
+     * tick cold starts, which is already a supported path.
+     */
+    private watermarkTtlSeconds;
+    private sampleDurations;
+    private flush;
+    /**
+     * The backlog is not all in one place. `wait` holds it while the queue is running, but
+     * pausing RENAMEs that list to `paused` and routes new jobs there, and anything added with
+     * a priority goes to the `prioritized` sorted set instead, which can leave `wait`
+     * permanently empty. Reading only `wait` reports a healthy zero for a queue that is badly
+     * backed up, which is the opposite of what this gauge is for, so all three are consulted
+     * and the worst age wins.
+     */
+    private sampleQueueAge;
+}

package/dist/LatencySampler.js

@@ -0,0 +1,285 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.LatencySampler = void 0;
+const histogram_1 = require("./histogram");
+const MS_PER_HOUR = 3600000;
+const SECONDS_PER_DAY = 86400;
+const DEFAULT_MAX_SAMPLES = 5000;
+/**
+ * ZADD into the finished set and this scan are not atomic with respect to each other, so a
+ * job finishing microseconds before the scan can be absent from it. Advancing the watermark
+ * to the highest score seen would skip that job forever. Scanning only up to a slightly
+ * stale bound, and advancing the watermark to that bound rather than to what was observed,
+ * means anything finishing inside the margin is picked up by the next tick instead.
+ * Costs a few seconds of freshness in data that is bucketed hourly.
+ *
+ * The score is `Date.now()` read in the worker process and passed into BullMQ's Lua, not a
+ * Redis-side clock, so the margin only holds while the worker and this recorder agree on
+ * the time: the effective margin is `margin - skew`. A worker running far enough ahead of
+ * the recorder can still land jobs below an already-advanced watermark and lose them.
+ */
+const SAFETY_MARGIN_MS = 5000;
+/**
+ * Copies job durations out of BullMQ's finished sets on the recorder's tick.
+ *
+ * BullMQ's moveToFinished does `ZADD targetSet, timestamp, jobId` and writes the same value
+ * to finishedOn, so the completed and failed sets are sorted sets scored by finish time.
+ * Scanning past a stored watermark therefore returns exactly the jobs finished since the
+ * last tick, with no gaps and no bias. The only loss is a queue whose removeOnComplete
+ * trims faster than the tick runs.
+ */
+class LatencySampler {
+    constructor(opts) {
+        var _a, _b;
+        this.id = `${process.pid}-${Math.random().toString(36).slice(2)}`;
+        this.redisBacked = new Map();
+        this.redis = opts.redis;
+        this.keys = opts.keys;
+        this.store = opts.store;
+        this.tickMs = opts.tickMs;
+        this.maxSamples = (_a = opts.maxSamplesPerTick) !== null && _a !== void 0 ? _a : DEFAULT_MAX_SAMPLES;
+        this.safetyMarginMs = (_b = opts.safetyMarginMs) !== null && _b !== void 0 ? _b : SAFETY_MARGIN_MS;
+        this.onError = opts.onError;
+    }
+    static supports(adapter) {
+        return typeof adapter.getQueueKey === 'function';
+    }
+    /**
+     * One queue, one tick. Swallows its own errors: the counter snapshot is the more
+     * important metric and must not fail as collateral damage from a latency scan. Pass
+     * `onError` to see what was swallowed; without it a collector failing every tick is
+     * indistinguishable from a queue with nothing to sample.
+     */
+    async sample(adapter) {
+        var _a;
+        if (!LatencySampler.supports(adapter)) {
+            return;
+        }
+        const name = adapter.getName();
+        try {
+            if (!(await this.isRedisBacked(adapter, name))) {
+                return;
+            }
+            if (!(await this.acquireLease(name))) {
+                return;
+            }
+            try {
+                await this.sampleDurations(adapter, name);
+                await this.sampleQueueAge(adapter, name);
+            }
+            finally {
+                await this.releaseLease(name);
+            }
+        }
+        catch (error) {
+            // Intentionally swallowed, see the method comment.
+            try {
+                (_a = this.onError) === null || _a === void 0 ? void 0 : _a.call(this, error, name);
+            }
+            catch {
+                // A reporter that throws must not resurrect the failure this catch contains.
+            }
+        }
+    }
+    /** `getQueueKey` answers for a PostgreSQL-backed queue too, with keys no Redis holds. */
+    async isRedisBacked(adapter, name) {
+        const known = this.redisBacked.get(name);
+        if (known !== undefined) {
+            return known;
+        }
+        const backed = (await adapter.getRedisInfo()) !== null;
+        this.redisBacked.set(name, backed);
+        return backed;
+    }
+    /**
+     * Increments are not idempotent the way the counter upsert is, so two recorders scanning
+     * the same range would double every histogram. Only the lease holder scans.
+     *
+     * The TTL is a crash ceiling, not the normal lifetime: a process that dies mid-scan must
+     * not lock the queue out forever. The normal path releases in a finally, because a lease
+     * outliving its scan would make the next tick no-op and halve the sampling rate.
+     */
+    async acquireLease(name) {
+        const held = await this.redis.set(this.keys.lease(name), this.id, 'PX', this.tickMs * 2, 'NX');
+        return held === 'OK';
+    }
+    /** Compare and delete, so a lease that already expired and was retaken is left alone. */
+    async releaseLease(name) {
+        await this.redis.eval(`if redis.call('GET', KEYS[1]) == ARGV[1] then return redis.call('DEL', KEYS[1]) end
+       return 0`, 1, this.keys.lease(name), this.id);
+    }
+    /**
+     * Bounded rather than eternal: an unexpiring watermark would leave one key behind per
+     * queue forever once that queue is purged or decommissioned, which is exactly the
+     * unbounded-storage failure this package exists to avoid. The day retention is already the
+     * horizon everything else here is bounded by. Losing the watermark just means the next
+     * tick cold starts, which is already a supported path.
+     */
+    watermarkTtlSeconds() {
+        return Math.max(1, Math.floor(this.store.retention.days * SECONDS_PER_DAY));
+    }
+    async sampleDurations(adapter, name) {
+        var _a;
+        const watermarkRaw = await this.redis.get(this.keys.watermark(name));
+        // Cold start covers one tick ending at the safety bound rather than backfilling, since a
+        // first run against a large completed set would be a surprise fetch storm. Ending at the
+        // bound rather than at now is what keeps a tick shorter than the margin from producing a
+        // window that is empty on every tick, leaving the watermark stuck forever.
+        const watermark = watermarkRaw
+            ? Number(watermarkRaw)
+            : Date.now() - this.tickMs - this.safetyMarginMs;
+        const upperBound = Date.now() - this.safetyMarginMs;
+        if (upperBound <= watermark) {
+            return; // ticks closer together than the margin; next tick covers this range
+        }
+        const ids = [];
+        for (const set of ['completed', 'failed']) {
+            const found = await this.redis.zrangebyscore(adapter.getQueueKey(set), `(${watermark}`, upperBound);
+            ids.push(...found);
+        }
+        if (ids.length === 0) {
+            await this.redis.set(this.keys.watermark(name), String(upperBound), 'EX', this.watermarkTtlSeconds());
+            return;
+        }
+        // The id list is one cheap round trip; the HMGETs are the real cost. Above the cap take
+        // a uniform subset rather than the first N, which would bias towards the tick's start.
+        const selected = ids.length > this.maxSamples ? sampleUniformly(ids, this.maxSamples) : ids;
+        // Counts are scaled back up by this ratio, so a subsampled hour reads as an estimate
+        // with the same shape rather than a dip. The fact that it was subsampled is currently
+        // invisible to clients: marking it would mean persisting a flag alongside the packed
+        // vector, which is a storage-format change. Known follow-up.
+        const ratio = ids.length / selected.length;
+        const pipeline = this.redis.pipeline();
+        for (const id of selected) {
+            pipeline.hmget(adapter.getQueueKey(String(id)), 'timestamp', 'processedOn', 'finishedOn', 
+            // BullMQ 5 counts attempts in `atm`; `attemptsMade` is the pre-v5 name and is read
+            // as a fallback exactly the way Job.fromJSON does.
+            'atm', 'attemptsMade');
+        }
+        const rows = await pipeline.exec();
+        const runByHour = new Map();
+        const waitByHour = new Map();
+        for (const row of rows !== null && rows !== void 0 ? rows : []) {
+            const values = row === null || row === void 0 ? void 0 : row[1];
+            if (!values) {
+                continue;
+            }
+            const [timestamp, processedOn, finishedOn, atm, attemptsMade] = values;
+            if (!processedOn || !finishedOn) {
+                continue;
+            }
+            const finished = Number(finishedOn);
+            const processed = Number(processedOn);
+            const hour = Math.floor(finished / MS_PER_HOUR);
+            observe(runByHour, hour, finished - processed, ratio);
+            // A retried job's timestamp is its creation, but processedOn is the latest attempt,
+            // so wait would absorb every prior attempt and backoff. Run time is unaffected.
+            const attempts = Number((_a = atm !== null && atm !== void 0 ? atm : attemptsMade) !== null && _a !== void 0 ? _a : 0);
+            if (timestamp && attempts <= 1) {
+                // Every one of these is a Date.now() taken in some client process, not a Redis-side
+                // clock: timestamp in the producer, processedOn in the worker. Skew between the two
+                // can make the difference negative, hence the clamp.
+                observe(waitByHour, hour, Math.max(0, processed - Number(timestamp)), ratio);
+            }
+        }
+        await this.flush(name, 'runtime', runByHour);
+        await this.flush(name, 'waittime', waitByHour);
+        // The bound, not the highest score observed. See SAFETY_MARGIN_MS.
+        await this.redis.set(this.keys.watermark(name), String(upperBound), 'EX', this.watermarkTtlSeconds());
+    }
+    async flush(name, metric, byHour) {
+        for (const [hour, vector] of byHour) {
+            // Scaled counts are fractional. Left unrounded, join(',') would write seventeen
+            // significant digits per bucket and blow up the packed value the storage design
+            // depends on staying small.
+            await this.store.addSamples(name, metric, hour, vector.map((v) => Math.round(v)));
+        }
+    }
+    /**
+     * The backlog is not all in one place. `wait` holds it while the queue is running, but
+     * pausing RENAMEs that list to `paused` and routes new jobs there, and anything added with
+     * a priority goes to the `prioritized` sorted set instead, which can leave `wait`
+     * permanently empty. Reading only `wait` reports a healthy zero for a queue that is badly
+     * backed up, which is the opposite of what this gauge is for, so all three are consulted
+     * and the worst age wins.
+     */
+    async sampleQueueAge(adapter, name) {
+        var _a;
+        const hour = Math.floor(Date.now() / MS_PER_HOUR);
+        const candidates = await this.redis
+            .pipeline()
+            // BullMQ LPUSHes to the wait list and workers RPOPLPUSH from it, so the tail is oldest.
+            .lrange(adapter.getQueueKey('wait'), -1, -1)
+            .lrange(adapter.getQueueKey('paused'), -1, -1)
+            // The prioritized set is scored by priority, not by time, so neither end is guaranteed
+            // to hold the oldest job. Both ends are an approximation, and a cheap one: the true
+            // oldest would mean fetching the whole set every tick.
+            // String indices: ioredis v6 types `zrange`'s stop arg as string-only (Redis coerces
+            // either way), so numeric literals no longer typecheck. '0'/'-1' are the first/last members.
+            .zrange(adapter.getQueueKey('prioritized'), '0', '0')
+            .zrange(adapter.getQueueKey('prioritized'), '-1', '-1')
+            .exec();
+        // A pipeline reports failures per command: a Redis error arrives in the entry's error
+        // slot next to a null result, rather than as a throw. Reading only the result slot turns
+        // a failed read into an empty backlog and records an age of 0, which is the most
+        // reassuring number this gauge can produce, at the moment it is least entitled to. A gap
+        // in the series reads as "not measured" and is recoverable; a zero is a wrong answer that
+        // looks like a healthy one, so a tick that could not read cleanly records nothing.
+        if (!candidates || candidates.some(([error]) => error)) {
+            return;
+        }
+        const ids = new Set();
+        for (const entry of candidates) {
+            for (const id of (_a = entry[1]) !== null && _a !== void 0 ? _a : []) {
+                ids.add(String(id));
+            }
+        }
+        if (ids.size === 0) {
+            await this.store.recordQueueAge(name, hour, 0);
+            return;
+        }
+        const stamps = this.redis.pipeline();
+        for (const id of ids) {
+            stamps.hget(adapter.getQueueKey(id), 'timestamp');
+        }
+        const rows = await stamps.exec();
+        // Same reasoning: a failed timestamp read does not zero the gauge, it understates it,
+        // which is the same wrong-but-reassuring answer in a quieter form.
+        if (!rows || rows.some(([error]) => error)) {
+            return;
+        }
+        const now = Date.now();
+        let oldest = 0;
+        for (const row of rows) {
+            const raw = row[1];
+            if (!raw) {
+                continue;
+            }
+            const enqueuedAt = Number(raw);
+            if (!Number.isFinite(enqueuedAt)) {
+                continue;
+            }
+            oldest = Math.max(oldest, now - enqueuedAt);
+        }
+        await this.store.recordQueueAge(name, hour, Math.max(0, oldest));
+    }
+}
+exports.LatencySampler = LatencySampler;
+function observe(byHour, hour, durationMs, ratio) {
+    let vector = byHour.get(hour);
+    if (!vector) {
+        vector = (0, histogram_1.emptyVector)();
+        byHour.set(hour, vector);
+    }
+    vector[(0, histogram_1.bucketIndex)(durationMs)] += ratio;
+}
+/** Evenly spaced pick across the list, which preserves the distribution's shape. */
+function sampleUniformly(ids, target) {
+    const stride = ids.length / target;
+    const out = [];
+    for (let i = 0; i < target; i++) {
+        out.push(ids[Math.floor(i * stride)]);
+    }
+    return out;
+}
+//# sourceMappingURL=LatencySampler.js.map

package/dist/LatencySampler.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"LatencySampler.js","sourceRoot":"","sources":["../src/LatencySampler.ts"],"names":[],"mappings":";;;AAEA,2CAAuD;AAIvD,MAAM,WAAW,GAAG,OAAO,CAAC;AAC5B,MAAM,eAAe,GAAG,KAAK,CAAC;AAC9B,MAAM,mBAAmB,GAAG,IAAI,CAAC;AACjC;;;;;;;;;;;;GAYG;AACH,MAAM,gBAAgB,GAAG,IAAI,CAAC;AA8B9B;;;;;;;;GAQG;AACH,MAAa,cAAc;IAWzB,YAAY,IAA2B;;QAHtB,OAAE,GAAG,GAAG,OAAO,CAAC,GAAG,IAAI,IAAI,CAAC,MAAM,EAAE,CAAC,QAAQ,CAAC,EAAE,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE,CAAC;QAC7D,gBAAW,GAAG,IAAI,GAAG,EAAmB,CAAC;QAGxD,IAAI,CAAC,KAAK,GAAG,IAAI,CAAC,KAAK,CAAC;QACxB,IAAI,CAAC,IAAI,GAAG,IAAI,CAAC,IAAI,CAAC;QACtB,IAAI,CAAC,KAAK,GAAG,IAAI,CAAC,KAAK,CAAC;QACxB,IAAI,CAAC,MAAM,GAAG,IAAI,CAAC,MAAM,CAAC;QAC1B,IAAI,CAAC,UAAU,GAAG,MAAA,IAAI,CAAC,iBAAiB,mCAAI,mBAAmB,CAAC;QAChE,IAAI,CAAC,cAAc,GAAG,MAAA,IAAI,CAAC,cAAc,mCAAI,gBAAgB,CAAC;QAC9D,IAAI,CAAC,OAAO,GAAG,IAAI,CAAC,OAAO,CAAC;IAC9B,CAAC;IAED,MAAM,CAAC,QAAQ,CAAC,OAAoB;QAClC,OAAO,OAAQ,OAAoC,CAAC,WAAW,KAAK,UAAU,CAAC;IACjF,CAAC;IAED;;;;;OAKG;IACH,KAAK,CAAC,MAAM,CAAC,OAAoB;;QAC/B,IAAI,CAAC,cAAc,CAAC,QAAQ,CAAC,OAAO,CAAC,EAAE,CAAC;YACtC,OAAO;QACT,CAAC;QACD,MAAM,IAAI,GAAG,OAAO,CAAC,OAAO,EAAE,CAAC;QAC/B,IAAI,CAAC;YACH,IAAI,CAAC,CAAC,MAAM,IAAI,CAAC,aAAa,CAAC,OAAO,EAAE,IAAI,CAAC,CAAC,EAAE,CAAC;gBAC/C,OAAO;YACT,CAAC;YACD,IAAI,CAAC,CAAC,MAAM,IAAI,CAAC,YAAY,CAAC,IAAI,CAAC,CAAC,EAAE,CAAC;gBACrC,OAAO;YACT,CAAC;YACD,IAAI,CAAC;gBACH,MAAM,IAAI,CAAC,eAAe,CAAC,OAA0B,EAAE,IAAI,CAAC,CAAC;gBAC7D,MAAM,IAAI,CAAC,cAAc,CAAC,OAA0B,EAAE,IAAI,CAAC,CAAC;YAC9D,CAAC;oBAAS,CAAC;gBACT,MAAM,IAAI,CAAC,YAAY,CAAC,IAAI,CAAC,CAAC;YAChC,CAAC;QACH,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,mDAAmD;YACnD,IAAI,CAAC;gBACH,MAAA,IAAI,CAAC,OAAO,qDAAG,KAAK,EAAE,IAAI,CAAC,CAAC;YAC9B,CAAC;YAAC,MAAM,CAAC;gBACP,6EAA6E;YAC/E,CAAC;QACH,CAAC;IACH,CAAC;IAED,yFAAyF;IACjF,KAAK,CAAC,aAAa,CAAC,OAAoB,EAAE,IAAY;QAC5D,MAAM,KAAK,GAAG,IAAI,CAAC,WAAW,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC;QACzC,IAAI,KAAK,KAAK,SAAS,EAAE,CAAC;YACxB,OAAO,KAAK,CAAC;QACf,CAAC;QACD,MAAM,MAAM,GAAG,CAAC,MAAM,OAAO,CAAC,YAAY,EAAE,CAAC,KAAK,IAAI,CAAC;QACvD,IAAI,CAAC,WAAW,CAAC,GAAG,CAAC,IAAI,EAAE,MAAM,CAAC,CAAC;QACnC,OAAO,MAAM,CAAC;IAChB,CAAC;IAED;;;;;;;OAOG;IACK,KAAK,CAAC,YAAY,CAAC,IAAY;QACrC,MAAM,IAAI,GAAG,MAAM,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,IAAI,CAAC,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,EAAE,IAAI,CAAC,EAAE,EAAE,IAAI,EAAE,IAAI,CAAC,MAAM,GAAG,CAAC,EAAE,IAAI,CAAC,CAAC;QAC/F,OAAO,IAAI,KAAK,IAAI,CAAC;IACvB,CAAC;IAED,yFAAyF;IACjF,KAAK,CAAC,YAAY,CAAC,IAAY;QACrC,MAAM,IAAI,CAAC,KAAK,CAAC,IAAI,CACnB;gBACU,EACV,CAAC,EACD,IAAI,CAAC,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,EACrB,IAAI,CAAC,EAAE,CACR,CAAC;IACJ,CAAC;IAED;;;;;;OAMG;IACK,mBAAmB;QACzB,OAAO,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,KAAK,CAAC,SAAS,CAAC,IAAI,GAAG,eAAe,CAAC,CAAC,CAAC;IAC9E,CAAC;IAEO,KAAK,CAAC,eAAe,CAAC,OAAwB,EAAE,IAAY;;QAClE,MAAM,YAAY,GAAG,MAAM,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,IAAI,CAAC,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,CAAC,CAAC;QACrE,yFAAyF;QACzF,yFAAyF;QACzF,yFAAyF;QACzF,2EAA2E;QAC3E,MAAM,SAAS,GAAG,YAAY;YAC5B,CAAC,CAAC,MAAM,CAAC,YAAY,CAAC;YACtB,CAAC,CAAC,IAAI,CAAC,GAAG,EAAE,GAAG,IAAI,CAAC,MAAM,GAAG,IAAI,CAAC,cAAc,CAAC;QAEnD,MAAM,UAAU,GAAG,IAAI,CAAC,GAAG,EAAE,GAAG,IAAI,CAAC,cAAc,CAAC;QACpD,IAAI,UAAU,IAAI,SAAS,EAAE,CAAC;YAC5B,OAAO,CAAC,qEAAqE;QAC/E,CAAC;QAED,MAAM,GAAG,GAAa,EAAE,CAAC;QACzB,KAAK,MAAM,GAAG,IAAI,CAAC,WAAW,EAAE,QAAQ,CAAC,EAAE,CAAC;YAC1C,MAAM,KAAK,GAAG,MAAM,IAAI,CAAC,KAAK,CAAC,aAAa,CAC1C,OAAO,CAAC,WAAW,CAAC,GAAG,CAAC,EACxB,IAAI,SAAS,EAAE,EACf,UAAU,CACX,CAAC;YACF,GAAG,CAAC,IAAI,CAAC,GAAG,KAAK,CAAC,CAAC;QACrB,CAAC;QACD,IAAI,GAAG,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YACrB,MAAM,IAAI,CAAC,KAAK,CAAC,GAAG,CAClB,IAAI,CAAC,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,EACzB,MAAM,CAAC,UAAU,CAAC,EAClB,IAAI,EACJ,IAAI,CAAC,mBAAmB,EAAE,CAC3B,CAAC;YACF,OAAO;QACT,CAAC;QAED,wFAAwF;QACxF,uFAAuF;QACvF,MAAM,QAAQ,GAAG,GAAG,CAAC,MAAM,GAAG,IAAI,CAAC,UAAU,CAAC,CAAC,CAAC,eAAe,CAAC,GAAG,EAAE,IAAI,CAAC,UAAU,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC;QAC5F,qFAAqF;QACrF,sFAAsF;QACtF,qFAAqF;QACrF,6DAA6D;QAC7D,MAAM,KAAK,GAAG,GAAG,CAAC,MAAM,GAAG,QAAQ,CAAC,MAAM,CAAC;QAE3C,MAAM,QAAQ,GAAG,IAAI,CAAC,KAAK,CAAC,QAAQ,EAAE,CAAC;QACvC,KAAK,MAAM,EAAE,IAAI,QAAQ,EAAE,CAAC;YAC1B,QAAQ,CAAC,KAAK,CACZ,OAAO,CAAC,WAAW,CAAC,MAAM,CAAC,EAAE,CAAC,CAAC,EAC/B,WAAW,EACX,aAAa,EACb,YAAY;YACZ,mFAAmF;YACnF,mDAAmD;YACnD,KAAK,EACL,cAAc,CACf,CAAC;QACJ,CAAC;QACD,MAAM,IAAI,GAAG,MAAM,QAAQ,CAAC,IAAI,EAAE,CAAC;QAEnC,MAAM,SAAS,GAAG,IAAI,GAAG,EAAoB,CAAC;QAC9C,MAAM,UAAU,GAAG,IAAI,GAAG,EAAoB,CAAC;QAE/C,KAAK,MAAM,GAAG,IAAI,IAAI,aAAJ,IAAI,cAAJ,IAAI,GAAI,EAAE,EAAE,CAAC;YAC7B,MAAM,MAAM,GAAG,GAAG,aAAH,GAAG,uBAAH,GAAG,CAAG,CAAC,CAAkC,CAAC;YACzD,IAAI,CAAC,MAAM,EAAE,CAAC;gBACZ,SAAS;YACX,CAAC;YACD,MAAM,CAAC,SAAS,EAAE,WAAW,EAAE,UAAU,EAAE,GAAG,EAAE,YAAY,CAAC,GAAG,MAAM,CAAC;YACvE,IAAI,CAAC,WAAW,IAAI,CAAC,UAAU,EAAE,CAAC;gBAChC,SAAS;YACX,CAAC;YACD,MAAM,QAAQ,GAAG,MAAM,CAAC,UAAU,CAAC,CAAC;YACpC,MAAM,SAAS,GAAG,MAAM,CAAC,WAAW,CAAC,CAAC;YACtC,MAAM,IAAI,GAAG,IAAI,CAAC,KAAK,CAAC,QAAQ,GAAG,WAAW,CAAC,CAAC;YAEhD,OAAO,CAAC,SAAS,EAAE,IAAI,EAAE,QAAQ,GAAG,SAAS,EAAE,KAAK,CAAC,CAAC;YAEtD,oFAAoF;YACpF,gFAAgF;YAChF,MAAM,QAAQ,GAAG,MAAM,CAAC,MAAA,GAAG,aAAH,GAAG,cAAH,GAAG,GAAI,YAAY,mCAAI,CAAC,CAAC,CAAC;YAClD,IAAI,SAAS,IAAI,QAAQ,IAAI,CAAC,EAAE,CAAC;gBAC/B,oFAAoF;gBACpF,oFAAoF;gBACpF,qDAAqD;gBACrD,OAAO,CAAC,UAAU,EAAE,IAAI,EAAE,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,SAAS,GAAG,MAAM,CAAC,SAAS,CAAC,CAAC,EAAE,KAAK,CAAC,CAAC;YAC/E,CAAC;QACH,CAAC;QAED,MAAM,IAAI,CAAC,KAAK,CAAC,IAAI,EAAE,SAAS,EAAE,SAAS,CAAC,CAAC;QAC7C,MAAM,IAAI,CAAC,KAAK,CAAC,IAAI,EAAE,UAAU,EAAE,UAAU,CAAC,CAAC;QAC/C,mEAAmE;QACnE,MAAM,IAAI,CAAC,KAAK,CAAC,GAAG,CAClB,IAAI,CAAC,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,EACzB,MAAM,CAAC,UAAU,CAAC,EAClB,IAAI,EACJ,IAAI,CAAC,mBAAmB,EAAE,CAC3B,CAAC;IACJ,CAAC;IAEO,KAAK,CAAC,KAAK,CACjB,IAAY,EACZ,MAAqB,EACrB,MAA6B;QAE7B,KAAK,MAAM,CAAC,IAAI,EAAE,MAAM,CAAC,IAAI,MAAM,EAAE,CAAC;YACpC,gFAAgF;YAChF,gFAAgF;YAChF,4BAA4B;YAC5B,MAAM,IAAI,CAAC,KAAK,CAAC,UAAU,CACzB,IAAI,EACJ,MAAM,EACN,IAAI,EACJ,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,CACjC,CAAC;QACJ,CAAC;IACH,CAAC;IAED;;;;;;;OAOG;IACK,KAAK,CAAC,cAAc,CAAC,OAAwB,EAAE,IAAY;;QACjE,MAAM,IAAI,GAAG,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,GAAG,EAAE,GAAG,WAAW,CAAC,CAAC;QAClD,MAAM,UAAU,GAAG,MAAM,IAAI,CAAC,KAAK;aAChC,QAAQ,EAAE;YACX,wFAAwF;aACvF,MAAM,CAAC,OAAO,CAAC,WAAW,CAAC,MAAM,CAAC,EAAE,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC;aAC3C,MAAM,CAAC,OAAO,CAAC,WAAW,CAAC,QAAQ,CAAC,EAAE,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC;YAC9C,uFAAuF;YACvF,oFAAoF;YACpF,uDAAuD;YACvD,qFAAqF;YACrF,6FAA6F;aAC5F,MAAM,CAAC,OAAO,CAAC,WAAW,CAAC,aAAa,CAAC,EAAE,GAAG,EAAE,GAAG,CAAC;aACpD,MAAM,CAAC,OAAO,CAAC,WAAW,CAAC,aAAa,CAAC,EAAE,IAAI,EAAE,IAAI,CAAC;aACtD,IAAI,EAAE,CAAC;QAEV,sFAAsF;QACtF,yFAAyF;QACzF,iFAAiF;QACjF,yFAAyF;QACzF,0FAA0F;QAC1F,mFAAmF;QACnF,IAAI,CAAC,UAAU,IAAI,UAAU,CAAC,IAAI,CAAC,CAAC,CAAC,KAAK,CAAC,EAAE,EAAE,CAAC,KAAK,CAAC,EAAE,CAAC;YACvD,OAAO;QACT,CAAC;QAED,MAAM,GAAG,GAAG,IAAI,GAAG,EAAU,CAAC;QAC9B,KAAK,MAAM,KAAK,IAAI,UAAU,EAAE,CAAC;YAC/B,KAAK,MAAM,EAAE,IAAI,MAAC,KAAK,CAAC,CAAC,CAAqB,mCAAI,EAAE,EAAE,CAAC;gBACrD,GAAG,CAAC,GAAG,CAAC,MAAM,CAAC,EAAE,CAAC,CAAC,CAAC;YACtB,CAAC;QACH,CAAC;QACD,IAAI,GAAG,CAAC,IAAI,KAAK,CAAC,EAAE,CAAC;YACnB,MAAM,IAAI,CAAC,KAAK,CAAC,cAAc,CAAC,IAAI,EAAE,IAAI,EAAE,CAAC,CAAC,CAAC;YAC/C,OAAO;QACT,CAAC;QAED,MAAM,MAAM,GAAG,IAAI,CAAC,KAAK,CAAC,QAAQ,EAAE,CAAC;QACrC,KAAK,MAAM,EAAE,IAAI,GAAG,EAAE,CAAC;YACrB,MAAM,CAAC,IAAI,CAAC,OAAO,CAAC,WAAW,CAAC,EAAE,CAAC,EAAE,WAAW,CAAC,CAAC;QACpD,CAAC;QACD,MAAM,IAAI,GAAG,MAAM,MAAM,CAAC,IAAI,EAAE,CAAC;QACjC,sFAAsF;QACtF,mEAAmE;QACnE,IAAI,CAAC,IAAI,IAAI,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC,KAAK,CAAC,EAAE,EAAE,CAAC,KAAK,CAAC,EAAE,CAAC;YAC3C,OAAO;QACT,CAAC;QAED,MAAM,GAAG,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;QACvB,IAAI,MAAM,GAAG,CAAC,CAAC;QACf,KAAK,MAAM,GAAG,IAAI,IAAI,EAAE,CAAC;YACvB,MAAM,GAAG,GAAG,GAAG,CAAC,CAAC,CAA8B,CAAC;YAChD,IAAI,CAAC,GAAG,EAAE,CAAC;gBACT,SAAS;YACX,CAAC;YACD,MAAM,UAAU,GAAG,MAAM,CAAC,GAAG,CAAC,CAAC;YAC/B,IAAI,CAAC,MAAM,CAAC,QAAQ,CAAC,UAAU,CAAC,EAAE,CAAC;gBACjC,SAAS;YACX,CAAC;YACD,MAAM,GAAG,IAAI,CAAC,GAAG,CAAC,MAAM,EAAE,GAAG,GAAG,UAAU,CAAC,CAAC;QAC9C,CAAC;QACD,MAAM,IAAI,CAAC,KAAK,CAAC,cAAc,CAAC,IAAI,EAAE,IAAI,EAAE,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,MAAM,CAAC,CAAC,CAAC;IACnE,CAAC;CACF;AApSD,wCAoSC;AAED,SAAS,OAAO,CACd,MAA6B,EAC7B,IAAY,EACZ,UAAkB,EAClB,KAAa;IAEb,IAAI,MAAM,GAAG,MAAM,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC;IAC9B,IAAI,CAAC,MAAM,EAAE,CAAC;QACZ,MAAM,GAAG,IAAA,uBAAW,GAAE,CAAC;QACvB,MAAM,CAAC,GAAG,CAAC,IAAI,EAAE,MAAM,CAAC,CAAC;IAC3B,CAAC;IACD,MAAM,CAAC,IAAA,uBAAW,EAAC,UAAU,CAAC,CAAC,IAAI,KAAK,CAAC;AAC3C,CAAC;AAED,oFAAoF;AACpF,SAAS,eAAe,CAAC,GAAa,EAAE,MAAc;IACpD,MAAM,MAAM,GAAG,GAAG,CAAC,MAAM,GAAG,MAAM,CAAC;IACnC,MAAM,GAAG,GAAa,EAAE,CAAC;IACzB,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;QAChC,GAAG,CAAC,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,GAAG,MAAM,CAAC,CAAC,CAAC,CAAC;IACxC,CAAC;IACD,OAAO,GAAG,CAAC;AACb,CAAC"}

package/dist/LatencyStore.d.ts

@@ -0,0 +1,21 @@
+import type { MetricsClient } from './connection';
+import type { Retention } from './HistoryStore';
+import { type MetricsKeys } from './keys';
+export type LatencyMetric = 'runtime' | 'waittime';
+export declare const QUEUE_AGE_METRIC = "queueage";
+export declare class LatencyStore {
+    private readonly redis;
+    private readonly keys;
+    readonly retention: Retention;
+    constructor(opts: {
+        redis: MetricsClient;
+        keys: MetricsKeys;
+        retention: Retention;
+    });
+    addSamples(queue: string, metric: LatencyMetric, hour: number, vector: number[]): Promise<void>;
+    recordQueueAge(queue: string, hour: number, ms: number): Promise<void>;
+    readRange(queue: string, metric: LatencyMetric, granularity: 'hour' | 'day', days: string[]): Promise<Record<string, number[]>>;
+    readQueueAge(queue: string, granularity: 'hour' | 'day', days: string[]): Promise<Record<string, number>>;
+    /** Epoch ms for an absolute hour index, for building response timestamps. */
+    static hourToMs(hour: number): number;
+}