bunqueue 2.8.54 → 2.8.57

package/dist/application/operations/push.js

@@ -1,92 +1,35 @@
 /**
  * Push Operations
- * Job push and batch push logic
+ * Single-job push logic and stable batch re-export
  */
 import { createJob } from '../../domain/types/job';
 import { shardIndex } from '../../shared/hash';
 import { latencyTracker } from '../latencyTracker';
 import { throughputTracker } from '../throughputTracker';
 import { handleCustomId } from './customId';
-import { insertJobToShard } from './pushInsert';
+import { acceptParentedJob, validateParentLinkInputs } from './parentLink';
+import { isParentLinkInput } from './parentLinkInput';
+import { acceptStandardJob, throwPushErrors } from './pushAdmission';
 import { withPushWriteLocks } from './pushLocks';
-/**
- * Handle unique key deduplication
- * Returns existing job ID if duplicate found and should skip, or allows insert
- */
-function handleDeduplication(job, input, queue, shard, ctx) {
-    if (!job.uniqueKey) {
-        return { skip: false };
-    }
-    const q = shard.getQueue(queue);
-    const existingEntry = shard.getUniqueKeyEntry(queue, job.uniqueKey);
-    if (!existingEntry) {
-        shard.registerUniqueKeyWithTtl(queue, job.uniqueKey, job.id, input.dedup?.ttl);
-        return { skip: false };
-    }
-    const dedupOpts = input.dedup;
-    // Replace strategy: remove old, insert new
-    if (dedupOpts?.replace) {
-        const existingJob = q.find(existingEntry.jobId);
-        if (existingJob) {
-            q.remove(existingEntry.jobId);
-            shard.decrementQueued(existingEntry.jobId);
-            ctx.jobIndex.delete(existingEntry.jobId);
-        }
-        shard.releaseUniqueKey(queue, job.uniqueKey);
-        shard.registerUniqueKeyWithTtl(queue, job.uniqueKey, job.id, dedupOpts?.ttl);
-        return { skip: false };
-    }
-    // Extend strategy: reset TTL, return existing
-    if (dedupOpts?.extend && dedupOpts?.ttl) {
-        shard.extendUniqueKeyTtl(queue, job.uniqueKey, dedupOpts.ttl);
-        if (input.customId)
-            ctx.customIdMap.delete(input.customId);
-        const existingJob = q.find(existingEntry.jobId);
-        if (existingJob) {
-            ctx.dashboardEmit?.('job:deduplicated', {
-                queue,
-                jobId: String(existingEntry.jobId),
-                strategy: 'extend',
-            });
-            return { skip: true, existingId: existingEntry.jobId };
-        }
-        throw new Error('Duplicate unique_key (extended TTL)');
-    }
-    // Default: return existing job (BullMQ-style)
-    // Block if job is waiting (in priority queue) OR active (uniqueKey held until ack,
-    // jobIndex entry type 'processing' means the worker is still running it).
-    if (input.customId)
-        ctx.customIdMap.delete(input.customId);
-    const existingJob = q.find(existingEntry.jobId);
-    if (existingJob || ctx.jobIndex.has(existingEntry.jobId)) {
-        ctx.broadcast({
-            eventType: "duplicated" /* EventType.Duplicated */,
-            queue,
-            jobId: existingEntry.jobId,
-            timestamp: Date.now(),
-        });
-        ctx.dashboardEmit?.('job:deduplicated', {
-            queue,
-            jobId: String(existingEntry.jobId),
-            strategy: 'default',
-        });
-        return { skip: true, existingId: existingEntry.jobId };
-    }
-    // Job is completed/failed (not in jobIndex) - allow new insert
-    shard.registerUniqueKeyWithTtl(queue, job.uniqueKey, job.id, input.dedup?.ttl);
-    return { skip: false };
-}
+import { releaseDependencyCompletionPins } from '../dependencyCompletions';
+import { handleDeduplication, replaceActiveDedupJob, replacePendingDedupJob, } from './pushDeduplication';
+import { validateRepeatJobInput } from '../repeatJobs';
+export { pushJobBatch } from './pushBatch';
 /**
  * Push a single job to queue
  * NOTE: customId check happens INSIDE lock to prevent race conditions
  */
 export async function pushJob(queue, input, ctx) {
+    validateRepeatJobInput(input);
     const startNs = Bun.nanoseconds();
     const idx = shardIndex(queue);
     const now = Date.now();
+    const releasedDependencies = [];
     let result;
     await withPushWriteLocks(queue, [input], ctx, (lockedShardIndexes) => {
         const shard = ctx.shards[idx];
+        validateParentLinkInputs([input], ctx);
+        const linkParent = isParentLinkInput(input);
         // Check custom ID idempotency INSIDE lock to prevent race conditions
         const customIdResult = handleCustomId(input, ctx, lockedShardIndexes);
         if (customIdResult.skip) {
@@ -115,17 +58,62 @@ export async function pushJob(queue, input, ctx) {
             };
             return;
         }
-        // Insert to shard
-        insertJobToShard(job, { queue, shard, shardIdx: idx }, ctx);
+        const target = { queue, shard, shardIdx: idx };
+        let storageHandled = false;
+        if (linkParent) {
+            releasedDependencies.push(...acceptParentedJob({
+                job,
+                input,
+                target,
+                dedup: dedupResult,
+                customId: customIdResult,
+                ctx,
+            }));
+        }
+        else if (dedupResult.replacement) {
+            releasedDependencies.push(...replacePendingDedupJob(dedupResult.replacement, job, input, target, {
+                ctx,
+                customId: customIdResult,
+            }));
+        }
+        else if (dedupResult.activeOwnerId) {
+            replaceActiveDedupJob(dedupResult.activeOwnerId, job, input, target, {
+                ctx,
+                customId: customIdResult,
+            });
+        }
+        else {
+            storageHandled = acceptStandardJob(job, input, target, customIdResult, ctx).storageHandled;
+        }
         shard.notify(queue);
-        result = { job, persisted: true };
+        result = {
+            job,
+            persisted: true,
+            storageHandled: Boolean(storageHandled || linkParent || dedupResult.replacement || dedupResult.activeOwnerId),
+        };
     });
+    let cleanupError;
+    try {
+        releaseDependencyCompletionPins(releasedDependencies, ctx);
+    }
+    catch (error) {
+        cleanupError = error;
+    }
     if (!result) {
+        if (cleanupError)
+            throw cleanupError;
         console.error('[Push] Push failed unexpectedly', { queue, input });
         throw new Error('Push failed');
     }
+    let persistenceError;
     if (result.persisted) {
-        ctx.storage?.insertJob(result.job, input.durable);
+        try {
+            if (!result.storageHandled)
+                ctx.storage?.insertJob(result.job, input.durable);
+        }
+        catch (error) {
+            persistenceError = error;
+        }
         ctx.totalPushed.value++;
         throughputTracker.pushRate.increment();
         ctx.broadcast({
@@ -136,84 +124,6 @@ export async function pushJob(queue, input, ctx) {
         });
     }
     latencyTracker.push.observe((Bun.nanoseconds() - startNs) / 1e6);
+    throwPushErrors([cleanupError, persistenceError], 'Push failed while finalizing an accepted job');
     return result.job;
 }
-/**
- * Push multiple jobs to queue
- * NOTE: customId check happens INSIDE lock (safer for concurrent batch inserts)
- */
-export async function pushJobBatch(queue, inputs, ctx) {
-    const startNs = Bun.nanoseconds();
-    const now = Date.now();
-    const idx = shardIndex(queue);
-    const resultIds = [];
-    const jobsToInsert = [];
-    // Jobs flagged durable must bypass the 10ms write buffer (immediate fsync
-    // path), exactly like a single durable push — otherwise addBulk silently
-    // downgrades the documented "durable" guarantee.
-    const durableJobs = [];
-    await withPushWriteLocks(queue, inputs, ctx, (lockedShardIndexes) => {
-        const shard = ctx.shards[idx];
-        for (const input of inputs) {
-            // Check custom ID idempotency
-            const customIdResult = handleCustomId(input, ctx, lockedShardIndexes);
-            if (customIdResult.skip) {
-                // Idempotent re-add (queued, active, or waiting-children) — no insert.
-                resultIds.push('existingJob' in customIdResult
-                    ? customIdResult.existingJob.id
-                    : customIdResult.existingId);
-                continue;
-            }
-            const job = createJob(customIdResult.id, queue, input, now);
-            // Check deduplication
-            const dedupResult = handleDeduplication(job, input, queue, shard, ctx);
-            if (dedupResult.skip) {
-                resultIds.push(dedupResult.existingId);
-                continue;
-            }
-            // Insert to shard
-            insertJobToShard(job, { queue, shard, shardIdx: idx }, ctx);
-            jobsToInsert.push(job);
-            if (input.durable)
-                durableJobs.push(job);
-            resultIds.push(job.id);
-        }
-        if (jobsToInsert.length > 0) {
-            shard.notifyBatch(queue, jobsToInsert.length);
-        }
-    });
-    if (jobsToInsert.length > 0) {
-        if (durableJobs.length === 0) {
-            ctx.storage?.insertJobsBatch(jobsToInsert);
-        }
-        else {
-            // Durable jobs bypass the write buffer (immediate disk write); the rest
-            // still go through the batched buffer for throughput.
-            const durableSet = new Set(durableJobs);
-            const buffered = jobsToInsert.filter((j) => !durableSet.has(j));
-            if (buffered.length > 0)
-                ctx.storage?.insertJobsBatch(buffered);
-            ctx.storage?.insertJobsBatch(durableJobs, true);
-        }
-        ctx.totalPushed.value += BigInt(jobsToInsert.length);
-        throughputTracker.pushRate.increment(jobsToInsert.length);
-        for (const job of jobsToInsert) {
-            ctx.broadcast({
-                eventType: 'pushed',
-                queue: job.queue,
-                jobId: job.id,
-                timestamp: now,
-            });
-        }
-    }
-    if (jobsToInsert.length > 0 && inputs.length > 1) {
-        ctx.dashboardEmit?.('batch:pushed', {
-            queue,
-            total: inputs.length,
-            inserted: jobsToInsert.length,
-            duplicates: inputs.length - jobsToInsert.length,
-        });
-    }
-    latencyTracker.push.observe((Bun.nanoseconds() - startNs) / 1e6);
-    return resultIds;
-}

package/dist/application/operations/pushAdmission.d.ts

@@ -0,0 +1,32 @@
+import type { Job, JobInput } from '../../domain/types/job';
+import type { Shard } from '../../domain/queue/shard';
+import type { DurableAdmissionMetadata } from '../../infrastructure/persistence/sqlite';
+import { type CustomIdResult } from './customId';
+import type { PushContext } from './pushContext';
+import { type PreparedJobInsertion } from './pushInsert';
+export type AcceptedCustomId = Exclude<CustomIdResult, {
+    skip: true;
+}>;
+export interface PushTarget {
+    queue: string;
+    shard: Shard;
+    shardIdx: number;
+}
+interface PublishPreparedJobOptions {
+    job: Job;
+    input: JobInput;
+    target: PushTarget;
+    customId: AcceptedCustomId;
+    prepared: PreparedJobInsertion;
+    ctx: PushContext;
+    registerDedupKey?: boolean;
+}
+export declare function admissionMetadata(customId: AcceptedCustomId, prepared: PreparedJobInsertion): DurableAdmissionMetadata | undefined;
+/** Publish only non-throwing RAM ownership after the storage admission commits. */
+export declare function publishPreparedJob(options: PublishPreparedJobOptions): void;
+/** Persist first whenever admission can fail, then expose the job in RAM. */
+export declare function acceptStandardJob(job: Job, input: JobInput, target: PushTarget, customId: AcceptedCustomId, ctx: PushContext): {
+    storageHandled: boolean;
+};
+export declare function throwPushErrors(errors: unknown[], message: string): void;
+export {};

package/dist/application/operations/pushAdmission.js

@@ -0,0 +1,38 @@
+import { commitCustomIdAdmission } from './customId';
+import { insertPreparedJobToShard, prepareJobInsertion, } from './pushInsert';
+export function admissionMetadata(customId, prepared) {
+    const retireGenerationId = customId.retirement ? customId.id : undefined;
+    if (!retireGenerationId && prepared.completionPins.length === 0)
+        return undefined;
+    return {
+        retireGenerationId,
+        completionPins: prepared.completionPins.length > 0 ? prepared.completionPins : undefined,
+    };
+}
+/** Publish only non-throwing RAM ownership after the storage admission commits. */
+export function publishPreparedJob(options) {
+    const { job, input, target, customId, prepared, ctx, registerDedupKey = true } = options;
+    commitCustomIdAdmission(input, customId, ctx);
+    if (registerDedupKey && job.uniqueKey) {
+        target.shard.registerUniqueKeyWithTtl(target.queue, job.uniqueKey, job.id, job.deduplicationTtl ?? undefined);
+    }
+    insertPreparedJobToShard(job, target, ctx, prepared);
+}
+/** Persist first whenever admission can fail, then expose the job in RAM. */
+export function acceptStandardJob(job, input, target, customId, ctx) {
+    const prepared = prepareJobInsertion(job, ctx);
+    const admission = admissionMetadata(customId, prepared);
+    const mustPersistBeforePublish = Boolean(ctx.storage && (input.durable || admission));
+    if (mustPersistBeforePublish) {
+        ctx.storage?.insertJob(job, input.durable, admission);
+    }
+    publishPreparedJob({ job, input, target, customId, prepared, ctx });
+    return { storageHandled: mustPersistBeforePublish };
+}
+export function throwPushErrors(errors, message) {
+    const present = errors.filter((error) => error !== undefined);
+    if (present.length === 1)
+        throw present[0];
+    if (present.length > 1)
+        throw new AggregateError(present, message);
+}

package/dist/application/operations/pushBatch.d.ts

@@ -0,0 +1,4 @@
+import { type JobId, type JobInput } from '../../domain/types/job';
+import type { PushContext } from './pushContext';
+/** Push an ordered batch while preserving its accepted-prefix contract. */
+export declare function pushJobBatch(queue: string, inputs: JobInput[], ctx: PushContext): Promise<JobId[]>;

package/dist/application/operations/pushBatch.js

@@ -0,0 +1,136 @@
+import { createJob, normalizeJobPayload, } from '../../domain/types/job';
+import { shardIndex } from '../../shared/hash';
+import { releaseDependencyCompletionPins } from '../dependencyCompletions';
+import { latencyTracker } from '../latencyTracker';
+import { validateRepeatJobInput } from '../repeatJobs';
+import { throughputTracker } from '../throughputTracker';
+import { handleCustomId } from './customId';
+import { acceptParentedJob, validateParentLinkInputs } from './parentLink';
+import { isParentLinkInput } from './parentLinkInput';
+import { acceptStandardJob, throwPushErrors } from './pushAdmission';
+import { handleDeduplication, replaceActiveDedupJob, replacePendingDedupJob, } from './pushDeduplication';
+import { withPushWriteLocks } from './pushLocks';
+/** Push an ordered batch while preserving its accepted-prefix contract. */
+export async function pushJobBatch(queue, inputs, ctx) {
+    for (const input of inputs) {
+        validateRepeatJobInput(input);
+        normalizeJobPayload(input);
+    }
+    const startNs = Bun.nanoseconds();
+    const now = Date.now();
+    const idx = shardIndex(queue);
+    const resultIds = [];
+    const releasedDependencies = [];
+    const acceptedJobs = [];
+    const jobsToInsert = [];
+    let batchError;
+    try {
+        await withPushWriteLocks(queue, inputs, ctx, (lockedShardIndexes) => {
+            const shard = ctx.shards[idx];
+            validateParentLinkInputs(inputs, ctx);
+            for (const input of inputs) {
+                const linkParent = isParentLinkInput(input);
+                const customIdResult = handleCustomId(input, ctx, lockedShardIndexes);
+                if (customIdResult.skip) {
+                    resultIds.push('existingJob' in customIdResult
+                        ? customIdResult.existingJob.id
+                        : customIdResult.existingId);
+                    continue;
+                }
+                const job = createJob(customIdResult.id, queue, input, now);
+                const dedupResult = handleDeduplication(job, input, queue, shard, ctx);
+                if (dedupResult.skip) {
+                    resultIds.push(dedupResult.existingId);
+                    continue;
+                }
+                const target = { queue, shard, shardIdx: idx };
+                let storageHandled = false;
+                if (linkParent) {
+                    releasedDependencies.push(...acceptParentedJob({
+                        job,
+                        input,
+                        target,
+                        dedup: dedupResult,
+                        customId: customIdResult,
+                        ctx,
+                    }));
+                    const supersededIndex = jobsToInsert.findIndex(({ job: pending }) => pending.id === dedupResult.replacement?.job.id);
+                    if (supersededIndex !== -1)
+                        jobsToInsert.splice(supersededIndex, 1);
+                    storageHandled = true;
+                }
+                else if (dedupResult.replacement) {
+                    releasedDependencies.push(...replacePendingDedupJob(dedupResult.replacement, job, input, target, {
+                        ctx,
+                        customId: customIdResult,
+                    }));
+                    const supersededIndex = jobsToInsert.findIndex(({ job: pending }) => pending.id === dedupResult.replacement?.job.id);
+                    if (supersededIndex !== -1)
+                        jobsToInsert.splice(supersededIndex, 1);
+                    storageHandled = true;
+                }
+                else if (dedupResult.activeOwnerId) {
+                    replaceActiveDedupJob(dedupResult.activeOwnerId, job, input, target, {
+                        ctx,
+                        customId: customIdResult,
+                    });
+                    storageHandled = true;
+                }
+                else {
+                    storageHandled = acceptStandardJob(job, input, target, customIdResult, ctx).storageHandled;
+                }
+                jobsToInsert.push({ job, durable: Boolean(input.durable), storageHandled });
+                acceptedJobs.push(job);
+                resultIds.push(job.id);
+            }
+        });
+    }
+    catch (error) {
+        batchError = error;
+    }
+    let cleanupError;
+    try {
+        releaseDependencyCompletionPins(releasedDependencies, ctx);
+    }
+    catch (error) {
+        cleanupError = error;
+    }
+    let persistenceError;
+    if (acceptedJobs.length > 0) {
+        const pendingPersistence = jobsToInsert.filter(({ storageHandled }) => !storageHandled);
+        const durableJobs = pendingPersistence.filter(({ durable }) => durable).map(({ job }) => job);
+        const bufferedJobs = pendingPersistence.filter(({ durable }) => !durable).map(({ job }) => job);
+        try {
+            if (bufferedJobs.length > 0)
+                ctx.storage?.insertJobsBatch(bufferedJobs);
+            if (durableJobs.length > 0)
+                ctx.storage?.insertJobsBatch(durableJobs, true);
+        }
+        catch (error) {
+            persistenceError = error;
+        }
+        ctx.totalPushed.value += BigInt(acceptedJobs.length);
+        throughputTracker.pushRate.increment(acceptedJobs.length);
+        for (const job of acceptedJobs) {
+            ctx.broadcast({
+                eventType: 'pushed',
+                queue: job.queue,
+                jobId: job.id,
+                timestamp: now,
+            });
+        }
+    }
+    if (jobsToInsert.length > 0)
+        ctx.shards[idx].notifyBatch(queue, jobsToInsert.length);
+    if (!batchError && !persistenceError && acceptedJobs.length > 0 && inputs.length > 1) {
+        ctx.dashboardEmit?.('batch:pushed', {
+            queue,
+            total: inputs.length,
+            inserted: acceptedJobs.length,
+            duplicates: inputs.length - acceptedJobs.length,
+        });
+    }
+    latencyTracker.push.observe((Bun.nanoseconds() - startNs) / 1e6);
+    throwPushErrors([batchError, cleanupError, persistenceError], 'Batch push failed while finalizing its accepted prefix');
+    return resultIds;
+}

package/dist/application/operations/pushContext.d.ts

@@ -0,0 +1,39 @@
+import type { Shard } from '../../domain/queue/shard';
+import type { Job, JobId } from '../../domain/types/job';
+import type { JobLocation } from '../../domain/types/queue';
+import { type EventType } from '../../domain/types/queue';
+import type { SqliteStorage } from '../../infrastructure/persistence/sqlite';
+import type { RWLock } from '../../shared/lock';
+import type { MapLike, SetLike } from '../../shared/lru';
+import type { DependencyCompletionTracker } from '../dependencyCompletions';
+import type { DependencyResultTracker } from '../dependencyResultTracker';
+import type { RetiredTimeoutGeneration } from '../types/background';
+/** Shared mutable state required by queue and flow insertion operations. */
+export interface PushContext {
+    storage: SqliteStorage | null;
+    shards: Shard[];
+    shardLocks: RWLock[];
+    customIdLock: RWLock;
+    completedJobs: SetLike<JobId>;
+    completedJobsData: MapLike<JobId, Job>;
+    depCompletions?: DependencyCompletionTracker;
+    maxDependencyCompletions: number;
+    timedOutJobs?: MapLike<JobId, RetiredTimeoutGeneration>;
+    retiredTimeoutLeaseTokens?: MapLike<string, RetiredTimeoutGeneration>;
+    retiredCronLeaseTokens?: MapLike<JobId, string>;
+    jobResults: MapLike<JobId, unknown>;
+    dependencyResults: DependencyResultTracker;
+    customIdMap: MapLike<string, JobId>;
+    jobIndex: Map<JobId, JobLocation>;
+    totalPushed: {
+        value: bigint;
+    };
+    broadcast: (event: {
+        eventType: EventType;
+        queue: string;
+        jobId: JobId;
+        timestamp: number;
+    }) => void;
+    dashboardEmit?: (event: string, data: Record<string, unknown>) => void;
+    registerQueueName?: (queue: string) => void;
+}

package/dist/application/operations/pushContext.js

@@ -0,0 +1 @@
+export {};

package/dist/application/operations/pushDeduplication.d.ts

@@ -0,0 +1,42 @@
+import type { Shard } from '../../domain/queue/shard';
+import type { Job, JobId, JobInput } from '../../domain/types/job';
+import type { DurableAdmissionMetadata } from '../../infrastructure/persistence/sqlite';
+import { type AcceptedCustomId } from './pushAdmission';
+import type { PushContext } from './push';
+type PendingLocation = 'queue' | 'waiting-deps' | 'waiting-children';
+export interface DedupReplacement {
+    job: Job;
+    location: PendingLocation;
+}
+export interface DedupCommitOptions {
+    readonly ctx: PushContext;
+    readonly customId: AcceptedCustomId;
+    readonly persist?: (admission?: DurableAdmissionMetadata) => void;
+}
+export type DedupResult = {
+    skip: true;
+    existingId: JobId;
+} | {
+    skip: false;
+    replacement?: DedupReplacement;
+    activeOwnerId?: JobId;
+};
+/** Inspect dedup state while deferring destructive replacement mutations. */
+export declare function handleDeduplication(job: Job, input: JobInput, queue: string, shard: Shard, ctx: PushContext): DedupResult;
+/** Commit the RAM half of a replacement after persistence has succeeded. */
+export declare function commitDedupReplacement(replacement: DedupReplacement, newJob: Job, queue: string, shard: Shard, ctx: PushContext): JobId[];
+/** Defensive cleanup for callers that may have pre-published custom-ID ownership. */
+export declare function rollbackIncomingCustomId(job: Job, ctx: PushContext): void;
+/** Persist, detach, and insert a replacement without exposing a half-transition. */
+export declare function replacePendingDedupJob(replacement: DedupReplacement, newJob: Job, input: JobInput, target: {
+    queue: string;
+    shard: Shard;
+    shardIdx: number;
+}, options: DedupCommitOptions): JobId[];
+/** Persist and publish a successor while leaving its active predecessor live. */
+export declare function replaceActiveDedupJob(activeOwnerId: JobId, newJob: Job, input: JobInput, target: {
+    queue: string;
+    shard: Shard;
+    shardIdx: number;
+}, options: DedupCommitOptions): void;
+export {};