bunqueue 2.8.54 → 2.8.57

package/dist/application/operations/customId.d.ts

@@ -3,7 +3,8 @@ import { type Job, type JobId, type JobInput } from '../../domain/types/job';
 import type { JobLocation } from '../../domain/types/queue';
 import type { SqliteStorage } from '../../infrastructure/persistence/sqlite';
 import type { MapLike, SetLike } from '../../shared/lru';
-import { type DependencyCompletionTracker } from '../dependencyCompletions';
+import type { DependencyCompletionTracker } from '../dependencyCompletions';
+import type { RetiredTimeoutGeneration } from '../types/background';
 export interface CustomIdContext {
     storage: SqliteStorage | null;
     shards: Shard[];
@@ -11,7 +12,9 @@ export interface CustomIdContext {
     completedJobsData: MapLike<JobId, Job>;
     depCompletions?: DependencyCompletionTracker;
     maxDependencyCompletions: number;
-    timedOutJobs?: SetLike<JobId>;
+    timedOutJobs?: MapLike<JobId, RetiredTimeoutGeneration>;
+    retiredTimeoutLeaseTokens?: MapLike<string, RetiredTimeoutGeneration>;
+    retiredCronLeaseTokens?: MapLike<JobId, string>;
     jobResults: MapLike<JobId, unknown>;
     customIdMap: MapLike<string, JobId>;
     jobIndex: Map<JobId, JobLocation>;
@@ -25,10 +28,20 @@ export type CustomIdResult = {
 } | {
     skip: false;
     id: JobId;
+    retirement?: CustomIdRetirement;
 };
+export interface CustomIdRetirement {
+    readonly completed: boolean;
+    readonly dlqQueue?: string;
+    readonly dependencyCompletion: boolean;
+}
 /**
- * Enforce custom-ID idempotency while the caller holds the target shard lock.
- * Live generations are returned unchanged; terminal generations are retired
- * before the deterministic ID is admitted again.
+ * Inspect custom-ID idempotency while the caller holds the target shard lock.
+ * Terminal retirement is deliberately deferred until persistence commits, so
+ * a rejected durable generation cannot destroy the previous one.
  */
 export declare function handleCustomId(input: JobInput, ctx: CustomIdContext, lockedShardIndexes: ReadonlySet<number>): CustomIdResult;
+/** Publish custom-ID ownership after the matching storage admission succeeds. */
+export declare function commitCustomIdAdmission(input: JobInput, decision: Exclude<CustomIdResult, {
+    skip: true;
+}>, ctx: CustomIdContext): void;

package/dist/application/operations/customId.js

@@ -1,10 +1,9 @@
 import { generateJobId, jobId } from '../../domain/types/job';
 import { shardIndex } from '../../shared/hash';
-import { releaseDependencyCompletionPins, } from '../dependencyCompletions';
 /**
- * Enforce custom-ID idempotency while the caller holds the target shard lock.
- * Live generations are returned unchanged; terminal generations are retired
- * before the deterministic ID is admitted again.
+ * Inspect custom-ID idempotency while the caller holds the target shard lock.
+ * Terminal retirement is deliberately deferred until persistence commits, so
+ * a rejected durable generation cannot destroy the previous one.
  */
 export function handleCustomId(input, ctx, lockedShardIndexes) {
     if (!input.customId) {
@@ -31,39 +30,51 @@ export function handleCustomId(input, ctx, lockedShardIndexes) {
             return { skip: true, existingId: id };
         }
     }
-    if (ctx.completedJobs.has(id)) {
-        ctx.completedJobs.delete(id);
-        ctx.completedJobsData.delete(id);
-        ctx.jobResults.delete(id);
-        ctx.jobIndex.delete(id);
-        ctx.storage?.deleteJob(id);
-    }
+    const completed = ctx.completedJobs.has(id);
     const terminalLocation = ctx.jobIndex.get(id);
+    let dlqQueue;
     if (terminalLocation?.type === 'dlq') {
         const terminalShardIdx = shardIndex(terminalLocation.queueName);
         if (!lockedShardIndexes.has(terminalShardIdx)) {
             throw new Error('Terminal custom ID shard must be locked before reuse');
         }
-        const terminalShard = ctx.shards[terminalShardIdx];
-        terminalShard.removeFromDlq(terminalLocation.queueName, id);
-        ctx.jobIndex.delete(id);
-        // deleteDlqEntry removes every persisted generation for this deterministic
-        // id. The fresh jobs row is inserted only after this retirement completes.
-        ctx.storage?.deleteDlqEntry(id);
+        dlqQueue = terminalLocation.queueName;
     }
+    let dependencyCompletion = false;
     if (ctx.depCompletions?.has(id)) {
         const hasUnresolvedConsumers = ctx.shards.some((shard) => (shard.getJobsWaitingFor(id)?.size ?? 0) > 0);
         if (hasUnresolvedConsumers) {
             throw new Error(`Custom ID ${String(id)} still has unresolved dependency consumers`);
         }
-        releaseDependencyCompletionPins([id], ctx);
-        ctx.storage?.deleteDependencyCompletion(id);
-        ctx.depCompletions.delete(id);
+        dependencyCompletion = true;
+    }
+    const retirement = completed || dlqQueue || dependencyCompletion
+        ? { completed, dlqQueue, dependencyCompletion }
+        : undefined;
+    return { skip: false, id, retirement };
+}
+/** Publish custom-ID ownership after the matching storage admission succeeds. */
+export function commitCustomIdAdmission(input, decision, ctx) {
+    const { id, retirement } = decision;
+    if (retirement?.completed) {
+        ctx.completedJobs.delete(id);
+        ctx.completedJobsData.delete(id);
+        ctx.jobResults.delete(id);
+        ctx.jobIndex.delete(id);
+    }
+    if (retirement?.dlqQueue) {
+        const terminalShard = ctx.shards[shardIndex(retirement.dlqQueue)];
+        terminalShard.removeFromDlq(retirement.dlqQueue, id);
+        ctx.jobIndex.delete(id);
     }
-    // A durable jobs row may outlive its in-memory tracking after an interrupted
-    // cleanup. The storage insert upserts that orphan without adding a DELETE to
-    // every custom-ID push.
+    if (retirement?.dependencyCompletion)
+        ctx.depCompletions?.delete(id);
+    // A recycled deterministic ID starts with no timeout or retired-lease state.
+    const timedOutGeneration = ctx.timedOutJobs?.get(id);
     ctx.timedOutJobs?.delete(id);
-    ctx.customIdMap.set(input.customId, id);
-    return { skip: false, id };
+    if (timedOutGeneration?.token)
+        ctx.retiredTimeoutLeaseTokens?.delete(timedOutGeneration.token);
+    ctx.retiredCronLeaseTokens?.delete(id);
+    if (input.customId)
+        ctx.customIdMap.set(input.customId, id);
 }

package/dist/application/operations/flowValidation.js

@@ -1,3 +1,4 @@
+import { normalizeJobPayload } from '../../domain/types/job';
 import { validateFlowTopology } from './flowTopologyValidation';
 const MAX_FLOW_JOBS = 10_000;
 const MAX_JOB_DATA_BYTES = 10 * 1024 * 1024;
@@ -99,11 +100,13 @@ function validateData(data) {
         throw new Error('Job data too large (max 10MB)');
     }
     const value = data;
-    if (typeof value.name !== 'string' || value.name.length === 0 || value.name.length > 256) {
-        throw new Error('flow job data.name must be a non-empty string of at most 256 characters');
-    }
     return { encodedLength: encoded.length, value };
 }
+function validateName(name) {
+    if (name.length === 0 || name.length > 256) {
+        throw new Error('flow job name must be a non-empty string of at most 256 characters');
+    }
+}
 function validateIdList(value, name) {
     if (value !== undefined &&
         (!Array.isArray(value) ||
@@ -154,7 +157,9 @@ export function validateAtomicFlowBatch(batch) {
         }
         validateIdList(job.input.dependsOn, 'dependsOn');
         validateIdList(job.input.childrenIds, 'childrenIds');
-        const validatedData = validateData(job.input.data);
+        const payload = normalizeJobPayload(job.input);
+        validateName(payload.name);
+        const validatedData = validateData(payload.data);
         totalDataBytes += validatedData.encodedLength;
         if (totalDataBytes > MAX_FLOW_DATA_BYTES) {
             throw new Error('flow data is too large (max 64MB per atomic batch)');

package/dist/application/operations/jobManagement.js

@@ -7,6 +7,38 @@ import { webhookLog } from '../../shared/logger';
 import { withWriteLock } from '../../shared/lock';
 import { releaseDependencyCompletionPins, } from '../dependencyCompletions';
 export { discardJob, moveJobToDelayed } from './jobMoveOperations';
+const FLOW_METADATA_KEYS = [
+    '__parentId',
+    '__parentQueue',
+    '__childrenIds',
+    '__flowParentId',
+    '__flowParentIds',
+];
+function dataRecord(value) {
+    return typeof value === 'object' && value !== null && !Array.isArray(value)
+        ? value
+        : null;
+}
+/** Merge caller data without allowing it to erase or forge broker-owned flow metadata. */
+function updatedJobData(job, userData) {
+    const current = dataRecord(job.data);
+    const metadata = current
+        ? Object.fromEntries(FLOW_METADATA_KEYS.filter((key) => Object.hasOwn(current, key)).map((key) => [
+            key,
+            current[key],
+        ]))
+        : {};
+    if (Object.keys(metadata).length === 0)
+        return userData;
+    const next = dataRecord(userData);
+    if (!next)
+        throw new Error('flow job data must be an object');
+    for (const key of Object.keys(next)) {
+        if (key.startsWith('__'))
+            throw new Error(`flow job data key is reserved: ${key}`);
+    }
+    return { ...next, ...metadata };
+}
 /** Cancel a job (remove from queue) */
 export async function cancelJob(jobId, ctx) {
     const location = ctx.jobIndex.get(jobId);
@@ -120,10 +152,13 @@ export async function updateJobData(jobId, data, ctx) {
     if (location?.type === 'queue') {
         return withWriteLock(ctx.shardLocks[location.shardIdx], () => {
             const shard = ctx.shards[location.shardIdx];
-            const job = shard.getQueue(location.queueName).find(jobId) ?? shard.waitingDeps.get(jobId);
+            const job = shard.getQueue(location.queueName).find(jobId) ??
+                shard.waitingDeps.get(jobId) ??
+                shard.waitingChildren.get(jobId);
             if (job) {
-                job.data = data;
-                ctx.storage?.updateJobData(jobId, data);
+                const updated = updatedJobData(job, data);
+                ctx.storage?.updateJobData(jobId, updated);
+                job.data = updated;
                 return true;
             }
             return false;
@@ -134,8 +169,9 @@ export async function updateJobData(jobId, data, ctx) {
         return withWriteLock(ctx.processingLocks[procIdx], () => {
             const job = ctx.processingShards[procIdx].get(jobId);
             if (job) {
-                job.data = data;
-                ctx.storage?.updateJobData(jobId, data);
+                const updated = updatedJobData(job, data);
+                ctx.storage?.updateJobData(jobId, updated);
+                job.data = updated;
                 return true;
             }
             return false;
@@ -163,8 +199,9 @@ async function updateRepeatSuccessor(originalId, data, ctx) {
             .getQueue(successorLoc.queueName)
             .find(successorId);
         if (job) {
-            job.data = data;
-            ctx.storage?.updateJobData(successorId, data);
+            const updated = updatedJobData(job, data);
+            ctx.storage?.updateJobData(successorId, updated);
+            job.data = updated;
             return true;
         }
         return false;

package/dist/application/operations/parentLink.d.ts

@@ -0,0 +1,36 @@
+import { type Job, type JobId, type JobInput } from '../../domain/types/job';
+import type { Shard } from '../../domain/queue/shard';
+import { type DedupResult } from './pushDeduplication';
+import { type AcceptedCustomId } from './pushAdmission';
+import type { PushContext } from './pushContext';
+type ParentMembership = 'queue' | 'waiting-deps' | 'waiting-children';
+export interface ParentLinkPlan {
+    readonly child: Job;
+    readonly parent: Job;
+    readonly linkedChild: Job;
+    readonly linkedParent: Job;
+    readonly parentMembership: ParentMembership;
+    readonly childFinished: boolean;
+}
+/** Validate every parent before a batch mutates its accepted prefix. */
+export declare function validateParentLinkInputs(inputs: readonly JobInput[], ctx: PushContext): void;
+export declare function prepareParentLink(child: Job, ctx: PushContext, childFinished?: boolean, parentIdOverride?: JobId, now?: number): ParentLinkPlan;
+/** Publish a persisted link while the parent and child queue shards are locked. */
+export declare function commitParentLink(plan: ParentLinkPlan, ctx: PushContext): void;
+export declare function parentLinkState(plan: ParentLinkPlan): 'waiting-children' | 'waiting' | 'prioritized' | 'delayed';
+/** Persist and publish a new child and its existing parent as one locked operation. */
+export declare function acceptParentedJob(options: {
+    job: Job;
+    input: JobInput;
+    target: {
+        queue: string;
+        shard: Shard;
+        shardIdx: number;
+    };
+    dedup: Exclude<DedupResult, {
+        skip: true;
+    }>;
+    customId: AcceptedCustomId;
+    ctx: PushContext;
+}): JobId[];
+export {};

package/dist/application/operations/parentLink.js

@@ -0,0 +1,153 @@
+import { MAX_TIMELINE_ENTRIES } from '../../domain/types/job';
+import { shardIndex } from '../../shared/hash';
+import { replaceActiveDedupJob, replacePendingDedupJob, } from './pushDeduplication';
+import { admissionMetadata, publishPreparedJob } from './pushAdmission';
+import { prepareJobInsertion } from './pushInsert';
+import { parentLinkQueue } from './parentLinkInput';
+function recordData(value) {
+    return value && typeof value === 'object' && !Array.isArray(value)
+        ? value
+        : {};
+}
+function locateParent(parentId, ctx) {
+    const location = ctx.jobIndex.get(parentId);
+    if (!location)
+        throw new Error(`Parent job not found: ${String(parentId)}`);
+    if (location.type !== 'queue') {
+        throw new Error(`Parent job ${String(parentId)} is not linkable`);
+    }
+    const shard = ctx.shards[location.shardIdx];
+    const queued = shard.getQueue(location.queueName).find(parentId);
+    if (queued)
+        return { job: queued, membership: 'queue' };
+    const waiting = shard.waitingDeps.get(parentId);
+    if (waiting)
+        return { job: waiting, membership: 'waiting-deps' };
+    const waitingChildren = shard.waitingChildren.get(parentId);
+    if (waitingChildren)
+        return { job: waitingChildren, membership: 'waiting-children' };
+    throw new Error(`Parent job ${String(parentId)} changed state while linking`);
+}
+/** Validate every parent before a batch mutates its accepted prefix. */
+export function validateParentLinkInputs(inputs, ctx) {
+    for (const input of inputs) {
+        const expectedQueue = parentLinkQueue(input);
+        if (!input.parentId || !expectedQueue)
+            continue;
+        if (input.customId && String(input.parentId) === input.customId) {
+            throw new Error('A job cannot be its own parent');
+        }
+        const { job: parent } = locateParent(input.parentId, ctx);
+        if (parent.queue !== expectedQueue) {
+            throw new Error(`Parent job ${String(input.parentId)} belongs to queue ${parent.queue}, not ${expectedQueue}`);
+        }
+    }
+}
+export function prepareParentLink(child, ctx, childFinished = false, parentIdOverride, now = Date.now()) {
+    const parentId = parentIdOverride ?? child.parentId;
+    if (!parentId)
+        throw new Error(`Job ${String(child.id)} has no parent`);
+    if (child.id === parentId)
+        throw new Error('A job cannot be its own parent');
+    const { job: parent, membership } = locateParent(parentId, ctx);
+    const childrenIds = parent.childrenIds.includes(child.id)
+        ? [...parent.childrenIds]
+        : [...parent.childrenIds, child.id];
+    const dependsOn = parent.dependsOn.includes(child.id)
+        ? [...parent.dependsOn]
+        : [...parent.dependsOn, child.id];
+    const childData = {
+        ...recordData(child.data),
+        __parentId: String(parent.id),
+        __parentQueue: parent.queue,
+    };
+    const parentData = { ...recordData(parent.data), __childrenIds: childrenIds.map(String) };
+    const timeline = [...parent.timeline];
+    if (!childFinished &&
+        timeline.at(-1)?.state !== 'waiting-children' &&
+        timeline.length < MAX_TIMELINE_ENTRIES) {
+        timeline.push({ state: 'waiting-children', timestamp: now });
+    }
+    return {
+        child,
+        parent,
+        linkedChild: { ...child, parentId: parent.id, data: childData },
+        linkedParent: { ...parent, childrenIds, dependsOn, data: parentData, timeline },
+        parentMembership: membership,
+        childFinished,
+    };
+}
+/** Publish a persisted link while the parent and child queue shards are locked. */
+export function commitParentLink(plan, ctx) {
+    const { child, parent, linkedChild, linkedParent } = plan;
+    child.parentId = linkedChild.parentId;
+    child.data = linkedChild.data;
+    parent.childrenIds = linkedParent.childrenIds;
+    parent.dependsOn = linkedParent.dependsOn;
+    parent.data = linkedParent.data;
+    parent.timeline = linkedParent.timeline;
+    if (!plan.childFinished) {
+        const parentShard = ctx.shards[shardIndex(parent.queue)];
+        if (plan.parentMembership === 'queue') {
+            if (parentShard.getQueue(parent.queue).remove(parent.id)) {
+                parentShard.decrementQueued(parent.id);
+                parentShard.removeFromTemporalIndex(parent.id);
+            }
+            parentShard.waitingDeps.set(parent.id, parent);
+        }
+        else if (plan.parentMembership === 'waiting-children') {
+            parentShard.waitingChildren.delete(parent.id);
+            parentShard.waitingDeps.set(parent.id, parent);
+        }
+        parentShard.registerDependencies(parent.id, linkedParent.dependsOn);
+    }
+    ctx.dependencyResults.registerConsumer(parent.id, linkedParent.dependsOn);
+    for (const dependencyId of linkedParent.dependsOn) {
+        if (ctx.jobResults.has(dependencyId)) {
+            ctx.dependencyResults.retain(dependencyId, ctx.jobResults.get(dependencyId));
+        }
+    }
+}
+export function parentLinkState(plan) {
+    if (!plan.childFinished)
+        return 'waiting-children';
+    if (plan.parent.runAt > Date.now())
+        return 'delayed';
+    return plan.parent.priority > 0 ? 'prioritized' : 'waiting';
+}
+function persistLink(plan, ctx, mode, admission) {
+    ctx.storage?.commitFlowLink(plan.linkedChild, plan.linkedParent, parentLinkState(plan), mode, admission);
+}
+/** Persist and publish a new child and its existing parent as one locked operation. */
+export function acceptParentedJob(options) {
+    const { job, input, target, dedup, customId, ctx } = options;
+    const plan = prepareParentLink(job, ctx);
+    let released = [];
+    if (dedup.replacement) {
+        const replacement = dedup.replacement;
+        released = replacePendingDedupJob(replacement, job, input, target, {
+            ctx,
+            customId,
+            persist: (admission) => {
+                persistLink(plan, ctx, { type: 'replace', previousJobId: replacement.job.id }, admission);
+            },
+        });
+    }
+    else if (dedup.activeOwnerId) {
+        const activeOwnerId = dedup.activeOwnerId;
+        replaceActiveDedupJob(activeOwnerId, job, input, target, {
+            ctx,
+            customId,
+            persist: (admission) => {
+                persistLink(plan, ctx, { type: 'transfer-active', previousJobId: activeOwnerId }, admission);
+            },
+        });
+    }
+    else {
+        const prepared = prepareJobInsertion(job, ctx);
+        persistLink(plan, ctx, { type: 'insert' }, admissionMetadata(customId, prepared));
+        publishPreparedJob({ job, input, target, customId, prepared, ctx });
+    }
+    commitParentLink(plan, ctx);
+    return released;
+}

package/dist/application/operations/parentLinkInput.d.ts

@@ -0,0 +1,8 @@
+import type { JobInput } from '../../domain/types/job';
+/**
+ * Queue.add parent options carry the parent queue in reserved metadata. A bare
+ * parentId remains the legacy flow forward-reference contract and is linked
+ * later when the parent declares the edge.
+ */
+export declare function parentLinkQueue(input: JobInput): string | null;
+export declare function isParentLinkInput(input: JobInput): boolean;

package/dist/application/operations/parentLinkInput.js

@@ -0,0 +1,23 @@
+function inputData(input) {
+    return input.data && typeof input.data === 'object' && !Array.isArray(input.data)
+        ? input.data
+        : null;
+}
+/**
+ * Queue.add parent options carry the parent queue in reserved metadata. A bare
+ * parentId remains the legacy flow forward-reference contract and is linked
+ * later when the parent declares the edge.
+ */
+export function parentLinkQueue(input) {
+    if (!input.parentId)
+        return null;
+    const data = inputData(input);
+    if (!data || String(data.__parentId) !== String(input.parentId))
+        return null;
+    return typeof data.__parentQueue === 'string' && data.__parentQueue.length > 0
+        ? data.__parentQueue
+        : null;
+}
+export function isParentLinkInput(input) {
+    return parentLinkQueue(input) !== null;
+}

package/dist/application/operations/push.d.ts

@@ -1,52 +1,13 @@
 /**
  * Push Operations
- * Job push and batch push logic
+ * Single-job push logic and stable batch re-export
  */
-import { type Job, type JobId, type JobInput } from '../../domain/types/job';
-import { type JobLocation, EventType } from '../../domain/types/queue';
-import type { Shard } from '../../domain/queue/shard';
-import type { SqliteStorage } from '../../infrastructure/persistence/sqlite';
-import type { RWLock } from '../../shared/lock';
-import type { SetLike, MapLike } from '../../shared/lru';
-import type { DependencyResultTracker } from '../dependencyResultTracker';
-import type { DependencyCompletionTracker } from '../dependencyCompletions';
-/** Push operation context */
-export interface PushContext {
-    storage: SqliteStorage | null;
-    shards: Shard[];
-    shardLocks: RWLock[];
-    /** Serializes deterministic IDs before any shard lock is acquired. */
-    customIdLock: RWLock;
-    completedJobs: SetLike<JobId>;
-    completedJobsData: MapLike<JobId, Job>;
-    /** Bare completion ids for removeOnComplete jobs so dependents start ready */
-    depCompletions?: DependencyCompletionTracker;
-    maxDependencyCompletions: number;
-    /** Timeout markers — cleared on custom-id reuse so a recycled id starts clean */
-    timedOutJobs?: SetLike<JobId>;
-    jobResults: MapLike<JobId, unknown>;
-    dependencyResults: DependencyResultTracker;
-    customIdMap: MapLike<string, JobId>;
-    jobIndex: Map<JobId, JobLocation>;
-    totalPushed: {
-        value: bigint;
-    };
-    broadcast: (event: {
-        eventType: EventType;
-        queue: string;
-        jobId: JobId;
-        timestamp: number;
-    }) => void;
-    dashboardEmit?: (event: string, data: Record<string, unknown>) => void;
-    registerQueueName?: (queue: string) => void;
-}
+import { type Job, type JobInput } from '../../domain/types/job';
+import type { PushContext } from './pushContext';
+export type { PushContext } from './pushContext';
+export { pushJobBatch } from './pushBatch';
 /**
  * Push a single job to queue
  * NOTE: customId check happens INSIDE lock to prevent race conditions
  */
 export declare function pushJob(queue: string, input: JobInput, ctx: PushContext): Promise<Job>;
-/**
- * Push multiple jobs to queue
- * NOTE: customId check happens INSIDE lock (safer for concurrent batch inserts)
- */
-export declare function pushJobBatch(queue: string, inputs: JobInput[], ctx: PushContext): Promise<JobId[]>;