bunqueue 2.8.60 → 2.9.2

package/dist/application/postgres-queue-manager/snapshot.js

@@ -0,0 +1,236 @@
+import { processingShardIndex, shardIndex } from '../../shared/hash';
+import { LRUMap } from '../../shared/lru';
+import { countSnapshotJobs, countSnapshotPriorities, findSnapshotJob, listSnapshotDlq, listSnapshotJobs, snapshotJobIds, snapshotQueueNames, } from './snapshotViews';
+/** Eventually-consistent read model; PostgreSQL remains authoritative for every transition. */
+export class PostgresQueueSnapshot {
+    maxCompletedJobs;
+    maxJobResults;
+    jobs = new Map();
+    results;
+    queues = new Map();
+    knownQueues = new Set();
+    completedOrder = new Map();
+    constructor(maxCompletedJobs, maxJobResults) {
+        this.maxCompletedJobs = maxCompletedJobs;
+        this.maxJobResults = maxJobResults;
+        this.results = new LRUMap(maxJobResults);
+    }
+    hydrate(rows) {
+        this.jobs.clear();
+        this.completedOrder.clear();
+        this.knownQueues.clear();
+        for (const row of rows)
+            this.put(row);
+    }
+    replaceQueue(queue, rows, state, completions = []) {
+        for (const [id, row] of this.jobs) {
+            if (row.job.queue === queue)
+                this.remove(id);
+        }
+        this.removeQueueResults(queue);
+        for (const row of rows)
+            this.put(row);
+        for (const completion of completions)
+            this.putCompletion(completion);
+        this.queues.set(queue, state);
+        this.knownQueues.add(queue);
+    }
+    removeQueue(queue) {
+        const removed = [];
+        for (const [id, row] of this.jobs) {
+            if (row.job.queue !== queue)
+                continue;
+            this.remove(id);
+            removed.push(id);
+        }
+        this.removeQueueResults(queue);
+        this.queues.delete(queue);
+        this.knownQueues.delete(queue);
+        return removed;
+    }
+    hydrateResults(entries) {
+        this.results.clear();
+        for (const entry of entries)
+            this.putCompletion(entry);
+    }
+    put(row) {
+        this.knownQueues.add(row.job.queue);
+        this.completedOrder.delete(row.job.id);
+        this.jobs.set(row.job.id, row);
+        this.results.delete(row.job.id);
+        if (row.state === 'completed') {
+            this.putResult(row.job.id, row.job.queue, row.result);
+            this.completedOrder.set(row.job.id, true);
+            this.trimCompletedJobs();
+        }
+    }
+    claim(claimed) {
+        const current = this.jobs.get(claimed.job.id);
+        this.put({
+            job: claimed.job,
+            state: 'active',
+            result: current?.result,
+            error: null,
+            dlqEntry: null,
+            dlqRetryState: current?.dlqRetryState ?? null,
+            token: claimed.token,
+            leaseOwner: claimed.owner,
+            leaseBrokerId: claimed.brokerId,
+            leaseUntil: claimed.leaseUntil,
+            leaseRenewals: 0,
+            version: (current?.version ?? 0) + 1,
+        });
+    }
+    remove(id) {
+        this.jobs.delete(id);
+        this.completedOrder.delete(id);
+        this.results.delete(id);
+    }
+    reconcile(id, row, completion) {
+        if (row) {
+            this.put(row);
+            return;
+        }
+        this.remove(id);
+        if (completion)
+            this.putCompletion(completion);
+    }
+    setQueueState(state) {
+        this.queues.set(state.queue, state);
+        this.knownQueues.add(state.queue);
+    }
+    apply(event) {
+        if (event.removed || event.type === 'removed') {
+            this.remove(event.jobId);
+            if (event.type === 'completed' &&
+                event.state === 'completed' &&
+                Object.hasOwn(event, 'result')) {
+                this.putResult(event.jobId, event.queue, event.result);
+            }
+            return;
+        }
+        if (Object.hasOwn(event, 'result'))
+            this.putResult(event.jobId, event.queue, event.result);
+        if (!event.job || !event.state)
+            return;
+        const current = this.jobs.get(event.jobId);
+        this.put({
+            job: event.job,
+            state: event.state,
+            result: Object.hasOwn(event, 'result') ? event.result : current?.result,
+            error: event.error ?? current?.error ?? null,
+            dlqEntry: Object.hasOwn(event, 'dlqEntry')
+                ? (event.dlqEntry ?? null)
+                : (current?.dlqEntry ?? null),
+            dlqRetryState: Object.hasOwn(event, 'dlqRetryState')
+                ? (event.dlqRetryState ?? null)
+                : (current?.dlqRetryState ?? null),
+            token: event.state === 'active' ? (current?.token ?? null) : null,
+            leaseOwner: event.state === 'active' ? (current?.leaseOwner ?? null) : null,
+            leaseBrokerId: event.state === 'active' ? (current?.leaseBrokerId ?? null) : null,
+            leaseUntil: event.state === 'active' ? (current?.leaseUntil ?? null) : null,
+            leaseRenewals: event.state === 'active' ? (current?.leaseRenewals ?? 0) : 0,
+            version: (current?.version ?? 0) + 1,
+        });
+    }
+    get(id) {
+        return this.jobs.get(id) ?? null;
+    }
+    getResult(id) {
+        return this.results.get(id)?.result;
+    }
+    hasResult(id) {
+        return this.results.has(id);
+    }
+    locations() {
+        const locations = new Map();
+        for (const [id, row] of this.jobs) {
+            if (row.state === 'active') {
+                locations.set(id, { type: 'processing', shardIdx: processingShardIndex(id) });
+            }
+            else if (row.state === 'completed') {
+                locations.set(id, { type: 'completed', queueName: row.job.queue });
+            }
+            else if (row.state === 'failed') {
+                locations.set(id, { type: 'dlq', queueName: row.job.queue });
+            }
+            else {
+                locations.set(id, {
+                    type: 'queue',
+                    shardIdx: shardIndex(row.job.queue),
+                    queueName: row.job.queue,
+                });
+            }
+        }
+        return locations;
+    }
+    completedIds() {
+        return new Set([...this.jobs.values()].flatMap((row) => (row.state === 'completed' ? [row.job.id] : [])));
+    }
+    completionIds() {
+        return new Set([...this.completedIds(), ...this.results.keys()]);
+    }
+    putCompletion(entry) {
+        this.putResult(entry.jobId, entry.queue, entry.result);
+    }
+    putResult(id, queue, result) {
+        if (this.maxJobResults === 0)
+            return;
+        this.results.set(id, { queue, result });
+    }
+    removeQueueResults(queue) {
+        for (const [id, entry] of this.results) {
+            if (entry.queue === queue)
+                this.results.delete(id);
+        }
+    }
+    trimCompletedJobs() {
+        while (this.completedOrder.size > this.maxCompletedJobs) {
+            const oldest = this.completedOrder.keys().next().value;
+            if (oldest === undefined)
+                return;
+            this.completedOrder.delete(oldest);
+            this.jobs.delete(oldest);
+        }
+    }
+    getByCustomId(customId) {
+        return findSnapshotJob(this.jobs, (job) => job.customId === customId);
+    }
+    getByUniqueKey(queue, key) {
+        return findSnapshotJob(this.jobs, (job) => job.queue === queue && job.uniqueKey === key);
+    }
+    ids(queue, states) {
+        return snapshotJobIds(this.jobs, queue, states);
+    }
+    removeStates(queue, states) {
+        let count = 0;
+        for (const [id, row] of this.jobs) {
+            if (row.job.queue !== queue || !states.includes(row.state))
+                continue;
+            this.remove(id);
+            count++;
+        }
+        return count;
+    }
+    queueState(queue) {
+        return this.queues.get(queue) ?? null;
+    }
+    queueNames() {
+        return snapshotQueueNames(this.jobs, this.queues, this.knownQueues);
+    }
+    list(queue, options = {}) {
+        return listSnapshotJobs(this.jobs, this.queues, queue, options);
+    }
+    counts(queue) {
+        return countSnapshotJobs(this.jobs, queue);
+    }
+    priorities(queue) {
+        return countSnapshotPriorities(this.jobs, queue);
+    }
+    dlq(queue, filter) {
+        return listSnapshotDlq(this.jobs, queue, filter);
+    }
+    size() {
+        return this.jobs.size;
+    }
+}

package/dist/application/postgres-queue-manager/snapshotViews.d.ts

@@ -0,0 +1,15 @@
+import type { DlqEntry, DlqFilter } from '../../domain/types/dlq';
+import type { Job, JobId } from '../../domain/types/job';
+import type { PostgresCounts, PostgresJobState, PostgresQueueState, PostgresStoredJob } from '../../infrastructure/persistence/postgres';
+export declare function findSnapshotJob(jobs: ReadonlyMap<JobId, PostgresStoredJob>, predicate: (job: Job) => boolean): Job | null;
+export declare function snapshotJobIds(jobs: ReadonlyMap<JobId, PostgresStoredJob>, queue: string, states?: readonly PostgresJobState[]): JobId[];
+export declare function snapshotQueueNames(jobs: ReadonlyMap<JobId, PostgresStoredJob>, queues: ReadonlyMap<string, PostgresQueueState>, knownQueues?: ReadonlySet<string>): string[];
+export declare function listSnapshotJobs(jobs: ReadonlyMap<JobId, PostgresStoredJob>, queues: ReadonlyMap<string, PostgresQueueState>, queue: string, options: {
+    state?: string | string[];
+    start?: number;
+    end?: number;
+    asc?: boolean;
+}): Job[];
+export declare function countSnapshotJobs(jobs: ReadonlyMap<JobId, PostgresStoredJob>, queue?: string): PostgresCounts;
+export declare function countSnapshotPriorities(jobs: ReadonlyMap<JobId, PostgresStoredJob>, queue: string): Record<number, number>;
+export declare function listSnapshotDlq(jobs: ReadonlyMap<JobId, PostgresStoredJob>, queue: string, filter?: DlqFilter): DlqEntry[];

package/dist/application/postgres-queue-manager/snapshotViews.js

@@ -0,0 +1,100 @@
+const EMPTY_COUNTS = {
+    waiting: 0,
+    prioritized: 0,
+    delayed: 0,
+    active: 0,
+    completed: 0,
+    failed: 0,
+    waitingChildren: 0,
+};
+function stateMatches(state, requested, paused) {
+    if (!requested)
+        return true;
+    if (requested.includes('paused') && paused) {
+        if (state === 'waiting' || state === 'prioritized')
+            return true;
+    }
+    if (paused && (state === 'waiting' || state === 'prioritized'))
+        return false;
+    return requested.includes(state);
+}
+function compareIds(left, right) {
+    return left < right ? -1 : left > right ? 1 : 0;
+}
+export function findSnapshotJob(jobs, predicate) {
+    for (const row of jobs.values())
+        if (predicate(row.job))
+            return row.job;
+    return null;
+}
+export function snapshotJobIds(jobs, queue, states) {
+    return [...jobs.values()].flatMap((row) => row.job.queue === queue && (!states || states.includes(row.state)) ? [row.job.id] : []);
+}
+export function snapshotQueueNames(jobs, queues, knownQueues = new Set()) {
+    const names = new Set([...knownQueues, ...queues.keys()]);
+    for (const row of jobs.values())
+        names.add(row.job.queue);
+    return [...names].sort();
+}
+export function listSnapshotJobs(jobs, queues, queue, options) {
+    const requested = options.state === undefined || (Array.isArray(options.state) && options.state.length === 0)
+        ? null
+        : Array.isArray(options.state)
+            ? options.state
+            : [options.state];
+    const paused = queues.get(queue)?.paused ?? false;
+    const rows = [...jobs.values()].filter((row) => row.job.queue === queue && stateMatches(row.state, requested, paused));
+    const asc = options.asc ?? true;
+    rows.sort((left, right) => {
+        const created = left.job.createdAt - right.job.createdAt;
+        if (created !== 0)
+            return asc ? created : -created;
+        return asc
+            ? compareIds(String(left.job.id), String(right.job.id))
+            : compareIds(String(right.job.id), String(left.job.id));
+    });
+    const start = Math.max(0, options.start ?? 0);
+    const end = Math.max(start, options.end ?? 100);
+    return rows.slice(start, end).map((row) => row.job);
+}
+export function countSnapshotJobs(jobs, queue) {
+    const counts = { ...EMPTY_COUNTS };
+    for (const row of jobs.values()) {
+        if (queue && row.job.queue !== queue)
+            continue;
+        if (row.state === 'waiting-children')
+            counts.waitingChildren++;
+        else
+            counts[row.state]++;
+    }
+    return counts;
+}
+export function countSnapshotPriorities(jobs, queue) {
+    const counts = {};
+    for (const row of jobs.values()) {
+        if (row.job.queue !== queue || !['waiting', 'prioritized', 'delayed'].includes(row.state)) {
+            continue;
+        }
+        counts[row.job.priority] = (counts[row.job.priority] ?? 0) + 1;
+    }
+    return counts;
+}
+export function listSnapshotDlq(jobs, queue, filter) {
+    const now = Date.now();
+    let entries = [...jobs.values()].flatMap((row) => row.job.queue === queue && row.state === 'failed' && row.dlqEntry ? [row.dlqEntry] : []);
+    if (filter?.reason)
+        entries = entries.filter((entry) => entry.reason === filter.reason);
+    const olderThan = filter?.olderThan;
+    const newerThan = filter?.newerThan;
+    if (olderThan)
+        entries = entries.filter((entry) => entry.enteredAt < olderThan);
+    if (newerThan)
+        entries = entries.filter((entry) => entry.enteredAt > newerThan);
+    if (filter?.expired !== undefined) {
+        entries = entries.filter((entry) => (entry.expiresAt !== null && entry.expiresAt <= now) === filter.expired);
+    }
+    const offset = Math.max(0, filter?.offset ?? 0);
+    return entries
+        .sort((left, right) => right.enteredAt - left.enteredAt || compareIds(String(right.job.id), String(left.job.id)))
+        .slice(offset, offset + Math.max(0, filter?.limit ?? entries.length));
+}

package/dist/application/postgres-queue-manager/startupEventBuffer.d.ts

@@ -0,0 +1,12 @@
+import type { PostgresDeliveredStoreEvent } from '../../infrastructure/persistence/postgres';
+/** Bounded capture used to reconcile events with an in-flight authoritative snapshot. */
+export declare class PostgresStartupEventBuffer {
+    private readonly limit;
+    private readonly events;
+    private didOverflow;
+    constructor(limit?: number);
+    get overflowed(): boolean;
+    capture(event: PostgresDeliveredStoreEvent): void;
+    reset(): void;
+    take(): readonly PostgresDeliveredStoreEvent[] | null;
+}

package/dist/application/postgres-queue-manager/startupEventBuffer.js

@@ -0,0 +1,36 @@
+const DEFAULT_STARTUP_EVENT_LIMIT = 256;
+/** Bounded capture used to reconcile events with an in-flight authoritative snapshot. */
+export class PostgresStartupEventBuffer {
+    limit;
+    events = [];
+    didOverflow = false;
+    constructor(limit = DEFAULT_STARTUP_EVENT_LIMIT) {
+        this.limit = limit;
+        if (!Number.isSafeInteger(limit) || limit < 1) {
+            throw new RangeError('PostgreSQL startup event buffer limit must be a positive integer');
+        }
+    }
+    get overflowed() {
+        return this.didOverflow;
+    }
+    capture(event) {
+        if (this.didOverflow)
+            return;
+        if (this.events.length === this.limit) {
+            this.didOverflow = true;
+            this.events.length = 0;
+            return;
+        }
+        this.events.push(event);
+    }
+    reset() {
+        this.events.length = 0;
+        this.didOverflow = false;
+    }
+    take() {
+        if (this.didOverflow)
+            return null;
+        const captured = this.events.splice(0);
+        return captured;
+    }
+}

package/dist/application/postgres-queue-manager/state.d.ts

@@ -0,0 +1,47 @@
+import type { JobId } from '../../domain/types/job';
+import { PostgresQueueStore, type PostgresStoredJob } from '../../infrastructure/persistence/postgres';
+import { QueueManager } from '../queueManager';
+import type { PostgresQueueManagerConfig } from './config';
+import { PostgresQueueSnapshot } from './snapshot';
+import { PostgresOperationGate } from './operationGate';
+export declare abstract class PostgresQueueManagerState extends QueueManager {
+    protected readonly postgresStore: PostgresQueueStore;
+    protected readonly postgresSnapshot: PostgresQueueSnapshot;
+    protected readonly activeTokens: Map<JobId, string>;
+    protected readonly postgresReady: Promise<void>;
+    private readonly refreshes;
+    private readonly dirtyQueues;
+    private readonly queueEventVersions;
+    private snapshotEventBuffer;
+    private terminalMetricBaselineCommitSeq;
+    private readonly unsubscribeEvent;
+    private readonly unsubscribeInvalidation;
+    private readonly writes;
+    protected readonly operations: PostgresOperationGate;
+    private readonly projectionRefreshes;
+    private shutdownPromise;
+    private shutdownComplete;
+    private baseStopped;
+    private stopQueueRefreshes;
+    constructor(config: PostgresQueueManagerConfig);
+    waitUntilReady(): Promise<void>;
+    flushPostgresWrites(): Promise<void>;
+    shutdownPostgres(): Promise<void>;
+    shutdown(): void;
+    private performPostgresShutdown;
+    protected tokenFor(id: JobId, token?: string): string | null;
+    protected forgetToken(id: JobId): void;
+    protected runPostgresOperation<T>(operation: () => Promise<T>): Promise<T>;
+    protected refreshJob(id: JobId, queue?: string): Promise<void>;
+    protected fetchPostgresJob(id: JobId): Promise<PostgresStoredJob | null>;
+    protected refreshJobs(ids: readonly JobId[], queue?: string): Promise<void>;
+    protected refreshQueue(queue: string): Promise<boolean>;
+    protected refreshQueueAfterCommit(queue: string): Promise<void>;
+    protected enqueueWrite(operation: () => Promise<void>): void;
+    private initializePostgres;
+    private scheduleQueueRefresh;
+    private refreshDirtyQueue;
+    private onStoreEvent;
+    protected applyPostgresClaim(claim: Parameters<PostgresQueueSnapshot['claim']>[0]): void;
+    private stopBaseRuntime;
+}