bunqueue 2.8.60 → 2.9.0

package/dist/infrastructure/persistence/postgres/completionOutcome.js

@@ -0,0 +1,112 @@
+import { buildRepeatSuccessor } from '../../../application/repeatJobs';
+import { createJob, generateJobId, MAX_TIMELINE_ENTRIES, } from '../../../domain/types/job';
+import { admitPostgresJob } from './admission';
+import { decodePostgresJob, encodePostgresValue } from './codec';
+import { databaseNow, recordPostgresEvent } from './context';
+import { planPostgresDependencyCompletion } from './dependencyPromotion';
+import { clearPostgresParentFlowFailures } from './flowFailures';
+import { runPostgresTransactionWithRetry } from './transactionRetry';
+import { tryLockPostgresRepeatQueues } from './queueLifecycle';
+async function lockActiveJob(tx, ctx, id) {
+    const rows = await tx `
+    SELECT * FROM bunqueue_jobs
+    WHERE namespace = ${ctx.config.namespace} AND id = ${String(id)}
+    FOR UPDATE
+  `;
+    return rows[0] ?? null;
+}
+function notApplied(row) {
+    return {
+        applied: false,
+        alreadyFinalized: row?.state === 'completed' || row?.state === 'failed',
+        job: row ? decodePostgresJob(row).job : null,
+        state: row?.state ?? null,
+    };
+}
+async function deleteLiveJob(tx, ctx, id) {
+    await tx `
+    DELETE FROM bunqueue_job_logs
+    WHERE namespace = ${ctx.config.namespace} AND job_id = ${String(id)}
+  `;
+    await tx `
+    DELETE FROM bunqueue_dependencies
+    WHERE namespace = ${ctx.config.namespace} AND job_id = ${String(id)}
+  `;
+    await tx `
+    DELETE FROM bunqueue_jobs
+    WHERE namespace = ${ctx.config.namespace} AND id = ${String(id)}
+  `;
+}
+export async function completePostgresJob(ctx, id, token, result, removeOnComplete = false) {
+    return await runPostgresTransactionWithRetry(ctx, async (tx) => {
+        const dependencyCompletion = await planPostgresDependencyCompletion(tx, ctx, [id]);
+        const row = await lockActiveJob(tx, ctx, id);
+        const now = await databaseNow(tx);
+        if (row?.state !== 'active' ||
+            row.lease_token !== token ||
+            Number(row.lease_until ?? 0) <= now) {
+            return notApplied(row);
+        }
+        const job = decodePostgresJob(row).job;
+        job.completedAt = now;
+        job.progress = 100;
+        if (job.timeline.length < MAX_TIMELINE_ENTRIES) {
+            job.timeline.push({ state: 'completed', timestamp: now });
+        }
+        const resultPayload = encodePostgresValue(result);
+        await clearPostgresParentFlowFailures(tx, ctx, id);
+        await tx `
+      INSERT INTO bunqueue_completions (namespace, job_id, queue, completed_at, result)
+      VALUES (${ctx.config.namespace}, ${String(id)}, ${job.queue}, ${now}, ${resultPayload})
+      ON CONFLICT (namespace, job_id) DO UPDATE SET
+        queue = excluded.queue, completed_at = excluded.completed_at, result = excluded.result
+    `;
+        const removed = job.removeOnComplete || removeOnComplete;
+        if (removed) {
+            await deleteLiveJob(tx, ctx, id);
+        }
+        else {
+            await tx `
+        UPDATE bunqueue_jobs
+        SET payload = ${encodePostgresValue(job)}, state = 'completed', completed_at = ${now},
+            result = ${resultPayload}, lease_owner = NULL, lease_token = NULL,
+            lease_broker_id = NULL, lease_broker_session_id = NULL,
+            lease_until = NULL, dlq_retry_state = NULL,
+            version = version + 1
+        WHERE namespace = ${ctx.config.namespace} AND id = ${String(id)}
+      `;
+        }
+        await recordPostgresEvent(tx, ctx, {
+            queue: job.queue,
+            type: 'completed',
+            jobId: id,
+            occurredAt: now,
+            job,
+            state: 'completed',
+            result,
+            removed,
+            dlqEntry: null,
+            dlqRetryState: null,
+        });
+        await dependencyCompletion.promote(now);
+        const shouldRepeat = job.repeat !== null &&
+            (job.repeat.limit === undefined || job.repeat.count < job.repeat.limit);
+        const successorInput = shouldRepeat ? buildRepeatSuccessor(job, now) : null;
+        if (successorInput) {
+            if (!(await tryLockPostgresRepeatQueues(tx, ctx, [job.queue]))) {
+                throw new Error(`Queue ${job.queue} is being obliterated; retry completion`);
+            }
+            const successor = createJob(generateJobId(), job.queue, successorInput, now);
+            const admitted = await admitPostgresJob(tx, ctx, successor, {
+                linkParent: false,
+                queueLifecycleLocksHeld: true,
+            });
+            await tx `
+        INSERT INTO bunqueue_repeat_links (namespace, original_id, successor_id)
+        VALUES (${ctx.config.namespace}, ${String(job.id)}, ${String(admitted.job.id)})
+        ON CONFLICT (namespace, original_id) DO UPDATE SET successor_id = excluded.successor_id
+      `;
+        }
+        return { applied: true, alreadyFinalized: false, job, state: 'completed' };
+    });
+}

package/dist/infrastructure/persistence/postgres/completionQueries.d.ts

@@ -0,0 +1,10 @@
+import { type JobId } from '../../../domain/types/job';
+import type { PostgresContext } from './context';
+import type { PostgresReadSql } from './context';
+import type { PostgresCompletionResult } from './types';
+export declare function getPostgresCompletionResult(ctx: PostgresContext, id: JobId): Promise<{
+    found: boolean;
+    result: unknown;
+}>;
+export declare function loadPostgresCompletionResults(ctx: PostgresContext, queue?: string, sql?: PostgresReadSql): Promise<PostgresCompletionResult[]>;
+export declare function getPostgresChildrenValues(ctx: PostgresContext, parentId: JobId): Promise<Record<string, unknown>>;

package/dist/infrastructure/persistence/postgres/completionQueries.js

@@ -0,0 +1,87 @@
+import { jobId } from '../../../domain/types/job';
+import { decodePostgresValue } from './codec';
+const LIVE_CONSUMER_STATES = [
+    'waiting',
+    'prioritized',
+    'delayed',
+    'waiting-children',
+    'active',
+];
+export async function getPostgresCompletionResult(ctx, id) {
+    const rows = await ctx.sql `
+    SELECT result FROM bunqueue_completions
+    WHERE namespace = ${ctx.config.namespace} AND job_id = ${String(id)}
+  `;
+    if (!rows[0])
+        return { found: false, result: undefined };
+    return {
+        found: true,
+        result: decodePostgresValue(rows[0].result, null, `postgresCompletion:${String(id)}`),
+    };
+}
+async function loadCompletionRows(ctx, queue, sql = ctx.sql) {
+    const queueFilter = queue === undefined ? null : queue;
+    return await sql `
+    WITH pinned AS MATERIALIZED (
+      SELECT DISTINCT dependency.dependency_id AS job_id
+      FROM bunqueue_dependencies AS dependency
+      JOIN bunqueue_jobs AS consumer
+        ON consumer.namespace = dependency.namespace AND consumer.id = dependency.job_id
+      WHERE dependency.namespace = ${ctx.config.namespace}
+        AND consumer.state = ANY(${ctx.sql.array([...LIVE_CONSUMER_STATES], 'TEXT')})
+    ), recent AS MATERIALIZED (
+      SELECT completion.job_id, completion.queue, completion.result,
+             FALSE AS pinned, completion.completed_at
+      FROM bunqueue_completions AS completion
+      WHERE completion.namespace = ${ctx.config.namespace}
+        AND (${queueFilter}::text IS NULL OR completion.queue = ${queueFilter})
+        AND NOT EXISTS (
+          SELECT 1 FROM pinned WHERE pinned.job_id = completion.job_id
+        )
+      ORDER BY completion.completed_at DESC, completion.job_id DESC
+      LIMIT ${ctx.config.maxJobResults}
+    )
+    SELECT completion.job_id, completion.queue, completion.result, TRUE AS pinned
+    FROM bunqueue_completions AS completion
+    JOIN pinned ON pinned.job_id = completion.job_id
+    WHERE completion.namespace = ${ctx.config.namespace}
+      AND (${queueFilter}::text IS NULL OR completion.queue = ${queueFilter})
+    UNION ALL
+    SELECT recent.job_id, recent.queue, recent.result, recent.pinned
+    FROM recent
+    ORDER BY job_id
+  `;
+}
+export async function loadPostgresCompletionResults(ctx, queue, sql = ctx.sql) {
+    const rows = await loadCompletionRows(ctx, queue, sql);
+    return rows.map((row) => ({
+        jobId: jobId(row.job_id),
+        queue: row.queue,
+        result: decodePostgresValue(row.result, null, `postgresCompletion:${row.job_id}`),
+        pinned: row.pinned,
+    }));
+}
+export async function getPostgresChildrenValues(ctx, parentId) {
+    const parents = await ctx.sql `
+    SELECT payload FROM bunqueue_jobs
+    WHERE namespace = ${ctx.config.namespace} AND id = ${String(parentId)}
+  `;
+    if (!parents[0])
+        return {};
+    const parent = decodePostgresValue(parents[0].payload, null, `postgresChildrenParent:${String(parentId)}`);
+    const childIds = [
+        ...new Set([...(parent?.childrenIds ?? []), ...(parent?.dependsOn ?? [])].map(String)),
+    ];
+    if (childIds.length === 0)
+        return {};
+    const rows = await ctx.sql `
+    SELECT job_id, queue, result FROM bunqueue_completions
+    WHERE namespace = ${ctx.config.namespace}
+      AND job_id = ANY(${ctx.sql.array(childIds, 'TEXT')})
+    ORDER BY job_id
+  `;
+    return Object.fromEntries(rows.map((row) => [
+        `${row.queue}:${row.job_id}`,
+        decodePostgresValue(row.result, null, `postgresChildResult:${row.job_id}`),
+    ]));
+}

package/dist/infrastructure/persistence/postgres/connection.d.ts

@@ -0,0 +1,3 @@
+import { SQL } from 'bun';
+import type { ResolvedPostgresStorageConfig } from './types';
+export declare function createPostgresConnection(config: ResolvedPostgresStorageConfig): SQL;

package/dist/infrastructure/persistence/postgres/connection.js

@@ -0,0 +1,15 @@
+import { SQL } from 'bun';
+export function createPostgresConnection(config) {
+    return new SQL(config.url, {
+        max: config.poolSize,
+        connectionTimeout: 10,
+        idleTimeout: 30,
+        maxLifetime: 3600,
+        connection: {
+            application_name: `bunqueue:${config.brokerId}`.slice(0, 63),
+            statement_timeout: config.statementTimeoutMs,
+            lock_timeout: config.lockTimeoutMs,
+            idle_in_transaction_session_timeout: config.idleTransactionTimeoutMs,
+        },
+    });
+}

package/dist/infrastructure/persistence/postgres/context.d.ts

@@ -0,0 +1,28 @@
+import type { SQL, TransactionSQL } from 'bun';
+import type { DlqEntry, DlqRetryState } from '../../../domain/types/dlq';
+import type { Job, JobId } from '../../../domain/types/job';
+import type { PostgresJobState, ResolvedPostgresStorageConfig } from './types';
+export type PostgresReadSql = SQL | TransactionSQL;
+export interface PostgresContext {
+    readonly sql: SQL;
+    readonly config: ResolvedPostgresStorageConfig;
+    readonly postCommitMaintenance?: (subsystem: string, operation: () => Promise<unknown>) => Promise<void>;
+}
+/** Run work that follows an already committed transition without changing its outcome. */
+export declare function runPostgresPostCommitMaintenance(ctx: PostgresContext, subsystem: string, operation: () => Promise<unknown>): Promise<void>;
+export declare function databaseNowSql(): string;
+export declare function databaseNow(tx: SQL | TransactionSQL): Promise<number>;
+export interface RecordPostgresEventInput {
+    readonly queue: string;
+    readonly type: string;
+    readonly jobId: JobId;
+    readonly occurredAt: number;
+    readonly job?: Job;
+    readonly state?: PostgresJobState;
+    readonly result?: unknown;
+    readonly error?: string;
+    readonly removed?: boolean;
+    readonly dlqEntry?: DlqEntry | null;
+    readonly dlqRetryState?: DlqRetryState | null;
+}
+export declare function recordPostgresEvent(tx: TransactionSQL, ctx: PostgresContext, input: RecordPostgresEventInput): Promise<number>;

package/dist/infrastructure/persistence/postgres/context.js

@@ -0,0 +1,74 @@
+import { eventPayload } from './codec';
+import { recordPostgresEventPruneWatermarks, summarizePostgresPrunedEvents, } from './eventPruneWatermarks';
+import { prunePostgresQueueEvents, tryLockPostgresEventRetention } from './eventRetention';
+import { POSTGRES_EVENT_CHANNEL } from './schema';
+import { recordPostgresMetricAdditions } from './metricWrites';
+/** Run work that follows an already committed transition without changing its outcome. */
+export async function runPostgresPostCommitMaintenance(ctx, subsystem, operation) {
+    if (ctx.postCommitMaintenance) {
+        await ctx.postCommitMaintenance(subsystem, operation);
+        return;
+    }
+    await operation();
+}
+export function databaseNowSql() {
+    return 'floor(extract(epoch from clock_timestamp()) * 1000)::bigint';
+}
+export async function databaseNow(tx) {
+    const [row] = await tx `
+    SELECT floor(extract(epoch from clock_timestamp()) * 1000)::bigint AS now
+  `;
+    return Number(row.now);
+}
+export async function recordPostgresEvent(tx, ctx, input) {
+    const payload = eventPayload({
+        ...(input.job && { job: input.job }),
+        ...(input.state && { state: input.state }),
+        ...(Object.hasOwn(input, 'result') && { result: input.result }),
+        ...(input.error !== undefined && { error: input.error }),
+        ...(input.removed !== undefined && { removed: input.removed }),
+        ...(Object.hasOwn(input, 'dlqEntry') && { dlqEntry: input.dlqEntry }),
+        ...(Object.hasOwn(input, 'dlqRetryState') && { dlqRetryState: input.dlqRetryState }),
+    });
+    const [row] = await tx `
+    INSERT INTO bunqueue_events
+      (namespace, queue, event_type, job_id, occurred_at, payload)
+    VALUES
+      (${ctx.config.namespace}, ${input.queue}, ${input.type}, ${String(input.jobId)},
+       ${input.occurredAt}, ${payload})
+    RETURNING id
+  `;
+    const eventId = Number(row.id);
+    const eventLimit = ctx.config.maxQueueEvents;
+    const retentionLockAcquired = await tryLockPostgresEventRetention(tx, ctx, input.queue);
+    const pruned = retentionLockAcquired
+        ? await prunePostgresQueueEvents(tx, ctx, input.queue, eventLimit)
+        : [];
+    const prune = summarizePostgresPrunedEvents(pruned);
+    const { prunedThrough } = prune;
+    if (prunedThrough > 0) {
+        await recordPostgresEventPruneWatermarks(tx, ctx, [
+            { queue: input.queue, sourceEventId: eventId, ...prune },
+        ]);
+    }
+    const metricType = input.type === 'completed'
+        ? 'completed'
+        : input.type === 'failed' && input.state === 'failed'
+            ? 'failed'
+            : null;
+    if (metricType) {
+        await recordPostgresMetricAdditions(tx, ctx, metricType, [{ queue: input.queue, count: 1 }], input.occurredAt);
+    }
+    await tx.notify(POSTGRES_EVENT_CHANNEL, JSON.stringify({
+        namespace: ctx.config.namespace,
+        queue: input.queue,
+        brokerId: ctx.config.brokerId,
+        eventId,
+        eventStartId: eventId,
+        eventEndId: eventId,
+        invalidateQueue: input.type.startsWith('queue-'),
+        ...(prunedThrough > 0 && { prunedThrough }),
+        ...(!retentionLockAcquired && { retentionRequested: true }),
+    }));
+    return eventId;
+}

package/dist/infrastructure/persistence/postgres/control.d.ts

@@ -0,0 +1,23 @@
+import { type DlqConfig } from '../../../domain/types/dlq';
+import { type StallConfig } from '../../../domain/types/stall';
+import { type PostgresContext, type PostgresReadSql } from './context';
+import type { PostgresQueueState } from './types';
+export interface PostgresQueueStateRow {
+    queue: string;
+    paused: boolean;
+    rate_limit: number | null;
+    rate_duration_ms: number | string | bigint | null;
+    rate_window_started_at: number | string | bigint | null;
+    rate_expires_at: number | string | bigint | null;
+    rate_count: number;
+    concurrency_limit: number | null;
+    stall_config: Uint8Array | null;
+    dlq_config: Uint8Array | null;
+}
+export declare function decodePostgresQueueState(queue: string, row?: PostgresQueueStateRow): PostgresQueueState;
+export declare function getPostgresQueueState(ctx: PostgresContext, queue: string, sql?: PostgresReadSql): Promise<PostgresQueueState>;
+export declare function setPostgresPaused(ctx: PostgresContext, queue: string, paused: boolean): Promise<void>;
+export declare function setPostgresRateLimit(ctx: PostgresContext, queue: string, limit: number | null, durationMs: number | null, ttlMs?: number | null): Promise<void>;
+export declare function setPostgresConcurrency(ctx: PostgresContext, queue: string, limit: number | null): Promise<void>;
+export declare function setPostgresStallConfig(ctx: PostgresContext, queue: string, config: StallConfig): Promise<void>;
+export declare function setPostgresDlqConfig(ctx: PostgresContext, queue: string, config: DlqConfig): Promise<void>;

package/dist/infrastructure/persistence/postgres/control.js

@@ -0,0 +1,105 @@
+import { DEFAULT_DLQ_CONFIG } from '../../../domain/types/dlq';
+import { jobId } from '../../../domain/types/job';
+import { DEFAULT_STALL_CONFIG } from '../../../domain/types/stall';
+import { decodePostgresValue, encodePostgresValue } from './codec';
+import { enforcePostgresDlqLimit } from './dlqLifecycle';
+import { databaseNow, recordPostgresEvent, } from './context';
+import { normalizePostgresRateLimit } from './rateLimit';
+function optionalNumber(value) {
+    return value === null || value === undefined ? null : Number(value);
+}
+export function decodePostgresQueueState(queue, row) {
+    return {
+        queue,
+        paused: row?.paused ?? false,
+        rateLimit: row?.rate_limit ?? null,
+        rateDurationMs: optionalNumber(row?.rate_duration_ms),
+        rateWindowStartedAt: optionalNumber(row?.rate_window_started_at),
+        rateExpiresAt: optionalNumber(row?.rate_expires_at),
+        rateCount: row?.rate_count ?? 0,
+        concurrencyLimit: row?.concurrency_limit ?? null,
+        stallConfig: decodePostgresValue(row?.stall_config ?? null, DEFAULT_STALL_CONFIG, 'stallConfig'),
+        dlqConfig: decodePostgresValue(row?.dlq_config ?? null, DEFAULT_DLQ_CONFIG, 'dlqConfig'),
+    };
+}
+async function recordQueueInvalidation(tx, ctx, queue, type, occurredAt) {
+    await recordPostgresEvent(tx, ctx, {
+        queue,
+        type,
+        jobId: jobId(`__queue__:${queue}`),
+        occurredAt: occurredAt ?? (await databaseNow(tx)),
+    });
+}
+export async function getPostgresQueueState(ctx, queue, sql = ctx.sql) {
+    const rows = await sql `
+    SELECT queue, paused, rate_limit, rate_duration_ms, rate_window_started_at, rate_expires_at,
+           rate_count, concurrency_limit, stall_config, dlq_config
+    FROM bunqueue_queue_state
+    WHERE namespace = ${ctx.config.namespace} AND queue = ${queue}
+  `;
+    return decodePostgresQueueState(queue, rows[0]);
+}
+export async function setPostgresPaused(ctx, queue, paused) {
+    await ctx.sql.begin(async (tx) => {
+        await tx `
+      INSERT INTO bunqueue_queue_state (namespace, queue, paused)
+      VALUES (${ctx.config.namespace}, ${queue}, ${paused})
+      ON CONFLICT (namespace, queue) DO UPDATE SET paused = excluded.paused
+    `;
+        await recordQueueInvalidation(tx, ctx, queue, paused ? 'queue-paused' : 'queue-resumed');
+    });
+}
+export async function setPostgresRateLimit(ctx, queue, limit, durationMs, ttlMs) {
+    const normalized = normalizePostgresRateLimit(durationMs, ttlMs);
+    await ctx.sql.begin(async (tx) => {
+        const now = await databaseNow(tx);
+        const rateExpiresAt = limit !== null && normalized.ttlMs !== null ? now + normalized.ttlMs : null;
+        await tx `
+      INSERT INTO bunqueue_queue_state
+        (namespace, queue, rate_limit, rate_duration_ms, rate_window_started_at,
+         rate_expires_at, rate_count)
+      VALUES
+        (${ctx.config.namespace}, ${queue}, ${limit}, ${normalized.durationMs}, ${now},
+         ${rateExpiresAt}, 0)
+      ON CONFLICT (namespace, queue) DO UPDATE SET
+        rate_limit = excluded.rate_limit,
+        rate_duration_ms = excluded.rate_duration_ms,
+        rate_window_started_at = excluded.rate_window_started_at,
+        rate_expires_at = excluded.rate_expires_at,
+        rate_count = 0
+    `;
+        await recordQueueInvalidation(tx, ctx, queue, 'queue-rate-limit-changed', now);
+    });
+}
+export async function setPostgresConcurrency(ctx, queue, limit) {
+    await ctx.sql.begin(async (tx) => {
+        await tx `
+      INSERT INTO bunqueue_queue_state (namespace, queue, concurrency_limit)
+      VALUES (${ctx.config.namespace}, ${queue}, ${limit})
+      ON CONFLICT (namespace, queue) DO UPDATE SET concurrency_limit = excluded.concurrency_limit
+    `;
+        await recordQueueInvalidation(tx, ctx, queue, 'queue-concurrency-changed');
+    });
+}
+export async function setPostgresStallConfig(ctx, queue, config) {
+    await ctx.sql.begin(async (tx) => {
+        await tx `
+      INSERT INTO bunqueue_queue_state (namespace, queue, stall_config)
+      VALUES (${ctx.config.namespace}, ${queue}, ${encodePostgresValue(config)})
+      ON CONFLICT (namespace, queue) DO UPDATE SET stall_config = excluded.stall_config
+    `;
+        await recordQueueInvalidation(tx, ctx, queue, 'queue-stall-config-changed');
+    });
+}
+export async function setPostgresDlqConfig(ctx, queue, config) {
+    await ctx.sql.begin(async (tx) => {
+        const now = await databaseNow(tx);
+        await tx `
+      INSERT INTO bunqueue_queue_state (namespace, queue, dlq_config)
+      VALUES (${ctx.config.namespace}, ${queue}, ${encodePostgresValue(config)})
+      ON CONFLICT (namespace, queue) DO UPDATE SET dlq_config = excluded.dlq_config
+    `;
+        await recordQueueInvalidation(tx, ctx, queue, 'queue-dlq-config-changed', now);
+    });
+    await enforcePostgresDlqLimit(ctx, queue, config.maxEntries);
+}

package/dist/infrastructure/persistence/postgres/crons.d.ts

@@ -0,0 +1,18 @@
+import { type CronJob, type CronJobInput } from '../../../domain/types/cron';
+import { type PostgresContext, type PostgresReadSql } from './context';
+export interface PostgresCronRow {
+    name: string;
+    payload: Uint8Array;
+    next_run: number | string | bigint;
+    executions: number;
+    max_limit: number | null;
+}
+export declare function decodePostgresCron(row: PostgresCronRow): CronJob;
+export declare function upsertPostgresCron(ctx: PostgresContext, input: CronJobInput): Promise<CronJob>;
+export declare function getPostgresCron(ctx: PostgresContext, name: string): Promise<CronJob | undefined>;
+export declare function listPostgresCrons(ctx: PostgresContext, sql?: PostgresReadSql): Promise<CronJob[]>;
+export declare function removePostgresCron(ctx: PostgresContext, name: string): Promise<boolean>;
+/** Recalculate missed rows only on the oldest live broker for this namespace. */
+export declare function reconcilePostgresCronsOnStartup(ctx: PostgresContext): Promise<number>;
+/** Advance and enqueue each due cron once under PostgreSQL row locks. */
+export declare function processPostgresCrons(ctx: PostgresContext, limit?: number): Promise<number>;