bunqueue 2.8.49 → 2.8.50

package/dist/application/operations/flowValidation.js

@@ -0,0 +1,166 @@
+import { validateFlowTopology } from './flowTopologyValidation';
+const MAX_FLOW_JOBS = 10_000;
+const MAX_JOB_DATA_BYTES = 10 * 1024 * 1024;
+const MAX_FLOW_DATA_BYTES = 64 * 1024 * 1024;
+const QUEUE_RE = /^[a-zA-Z0-9_\-.:]+$/;
+const BOOLEAN_OPTIONS = [
+    'lifo',
+    'removeOnComplete',
+    'removeOnFail',
+    'durable',
+    'failParentOnFailure',
+    'removeDependencyOnFailure',
+    'continueParentOnFailure',
+    'ignoreDependencyOnFailure',
+];
+function numeric(value, name, min, max, integer = false) {
+    if (value === undefined || value === null)
+        return;
+    if (typeof value !== 'number' || !Number.isFinite(value)) {
+        throw new Error(`${name} must be a finite number`);
+    }
+    if (integer && !Number.isInteger(value))
+        throw new Error(`${name} must be an integer`);
+    if (value < min || value > max) {
+        throw new Error(`${name} must be between ${min} and ${max}`);
+    }
+}
+function validateOptions(input) {
+    numeric(input.priority, 'priority', -1_000_000, 1_000_000, true);
+    numeric(input.delay, 'delay', 0, 365 * 24 * 60 * 60 * 1_000);
+    numeric(input.timeout, 'timeout', 0, 24 * 60 * 60 * 1_000);
+    numeric(input.ttl, 'ttl', 0, 365 * 24 * 60 * 60 * 1_000);
+    numeric(input.maxAttempts, 'attempts', 1, 1_000, true);
+    numeric(input.stallTimeout, 'stallTimeout', 0, 24 * 60 * 60 * 1_000);
+    numeric(input.stackTraceLimit, 'stackTraceLimit', 0, 10_000, true);
+    numeric(input.keepLogs, 'keepLogs', 0, 1_000_000, true);
+    numeric(input.sizeLimit, 'sizeLimit', 0, MAX_JOB_DATA_BYTES, true);
+    numeric(input.timestamp, 'timestamp', 0, Number.MAX_SAFE_INTEGER);
+    if (typeof input.backoff === 'object' && input.backoff !== null) {
+        if (input.backoff.type !== 'fixed' && input.backoff.type !== 'exponential') {
+            throw new Error("backoff.type must be 'fixed' or 'exponential'");
+        }
+        numeric(input.backoff.delay, 'backoff.delay', 0, 24 * 60 * 60 * 1_000);
+    }
+    else {
+        numeric(input.backoff, 'backoff', 0, 24 * 60 * 60 * 1_000);
+    }
+    for (const name of BOOLEAN_OPTIONS) {
+        const value = input[name];
+        if (value !== undefined && typeof value !== 'boolean') {
+            throw new Error(`${name} must be a boolean`);
+        }
+    }
+    if (input.groupId !== undefined &&
+        (typeof input.groupId !== 'string' || input.groupId.length > 256)) {
+        throw new Error('groupId must be a string of at most 256 characters');
+    }
+    if (input.tags !== undefined &&
+        (!Array.isArray(input.tags) ||
+            input.tags.some((tag) => typeof tag !== 'string' || tag.length > 256))) {
+        throw new Error('tags must be an array of strings of at most 256 characters');
+    }
+    if (input.repeat !== undefined) {
+        throw new Error('repeat is not supported inside an atomic flow');
+    }
+    if (input.uniqueKey !== undefined || input.dedup !== undefined) {
+        throw new Error('deduplication is not supported inside an atomic flow');
+    }
+    if (input.debounceId !== undefined || input.debounceTtl !== undefined) {
+        throw new Error('debounce is not supported inside an atomic flow');
+    }
+    const policies = [
+        input.failParentOnFailure,
+        input.removeDependencyOnFailure,
+        input.continueParentOnFailure,
+        input.ignoreDependencyOnFailure,
+    ].filter((enabled) => enabled === true);
+    if (policies.length > 1) {
+        throw new Error('flow failure policies are mutually exclusive');
+    }
+    if (policies.length === 1 && !input.parentId) {
+        throw new Error('a flow failure policy requires a parent');
+    }
+}
+function validateData(data) {
+    if (typeof data !== 'object' || data === null || Array.isArray(data)) {
+        throw new Error('flow job data must be an object');
+    }
+    let encoded;
+    try {
+        encoded = JSON.stringify(data);
+    }
+    catch {
+        throw new Error('job data must be JSON serializable');
+    }
+    if (encoded === undefined)
+        throw new Error('job data must be JSON serializable');
+    if (encoded.length > MAX_JOB_DATA_BYTES) {
+        throw new Error('Job data too large (max 10MB)');
+    }
+    const value = data;
+    if (typeof value.name !== 'string' || value.name.length === 0 || value.name.length > 256) {
+        throw new Error('flow job data.name must be a non-empty string of at most 256 characters');
+    }
+    return { encodedLength: encoded.length, value };
+}
+function validateIdList(value, name) {
+    if (value !== undefined &&
+        (!Array.isArray(value) ||
+            value.some((id) => typeof id !== 'string' || id.length === 0 || id.length > 1_024))) {
+        throw new Error(`${name} must be an array of valid string job ids`);
+    }
+}
+/** Validate the whole graph before any lock, storage write, or queue mutation. */
+export function validateAtomicFlowBatch(batch) {
+    if (!batch || typeof batch !== 'object' || !Array.isArray(batch.jobs)) {
+        throw new Error('flow jobs must be an array');
+    }
+    if (batch.jobs.length > MAX_FLOW_JOBS) {
+        throw new Error(`flow exceeds the ${MAX_FLOW_JOBS} job limit`);
+    }
+    const ids = new Set();
+    const dataById = new Map();
+    let totalDataBytes = 0;
+    for (const [index, job] of batch.jobs.entries()) {
+        if (!job || typeof job !== 'object')
+            throw new Error(`flow jobs[${index}] must be an object`);
+        if (typeof job.id !== 'string')
+            throw new Error('flow job id must be a string');
+        const id = job.id;
+        if (!id || id.length > 1_024)
+            throw new Error('flow job id is invalid');
+        if (id.includes(':'))
+            throw new Error('flow job id cannot contain a colon');
+        if (ids.has(id))
+            throw new Error(`duplicate flow job id: ${id}`);
+        ids.add(id);
+        if (typeof job.queue !== 'string' ||
+            !job.queue ||
+            job.queue.length > 256 ||
+            !QUEUE_RE.test(job.queue)) {
+            throw new Error(`invalid flow queue: ${job.queue}`);
+        }
+        if (!job.input || typeof job.input !== 'object') {
+            throw new Error(`flow job input is invalid: ${id}`);
+        }
+        if (job.input.customId !== undefined &&
+            (typeof job.input.customId !== 'string' || job.input.customId !== id)) {
+            throw new Error(`custom id does not match planned id: ${id}`);
+        }
+        if (job.input.parentId !== undefined &&
+            (typeof job.input.parentId !== 'string' || job.input.parentId.length === 0)) {
+            throw new Error(`flow parent id is invalid: ${id}`);
+        }
+        validateIdList(job.input.dependsOn, 'dependsOn');
+        validateIdList(job.input.childrenIds, 'childrenIds');
+        const validatedData = validateData(job.input.data);
+        totalDataBytes += validatedData.encodedLength;
+        if (totalDataBytes > MAX_FLOW_DATA_BYTES) {
+            throw new Error('flow data is too large (max 64MB per atomic batch)');
+        }
+        dataById.set(id, validatedData.value);
+        validateOptions(job.input);
+    }
+    validateFlowTopology(batch, dataById);
+}

package/dist/application/operations/push.d.ts

@@ -14,6 +14,8 @@ export interface PushContext {
     storage: SqliteStorage | null;
     shards: Shard[];
     shardLocks: RWLock[];
+    /** Serializes deterministic IDs before any shard lock is acquired. */
+    customIdLock: RWLock;
     completedJobs: SetLike<JobId>;
     completedJobsData: MapLike<JobId, Job>;
     /** Bare completion ids for removeOnComplete jobs so dependents start ready */
@@ -34,6 +36,7 @@ export interface PushContext {
         timestamp: number;
     }) => void;
     dashboardEmit?: (event: string, data: Record<string, unknown>) => void;
+    registerQueueName?: (queue: string) => void;
 }
 /**
  * Push a single job to queue

package/dist/application/operations/push.js

@@ -116,7 +116,7 @@ export async function pushJob(queue, input, ctx) {
             return;
         }
         // Insert to shard
-        insertJobToShard(job, queue, shard, idx, ctx);
+        insertJobToShard(job, { queue, shard, shardIdx: idx }, ctx);
         shard.notify(queue);
         result = { job, persisted: true };
     });
@@ -172,7 +172,7 @@ export async function pushJobBatch(queue, inputs, ctx) {
                 continue;
             }
             // Insert to shard
-            insertJobToShard(job, queue, shard, idx, ctx);
+            insertJobToShard(job, { queue, shard, shardIdx: idx }, ctx);
             jobsToInsert.push(job);
             if (input.durable)
                 durableJobs.push(job);

package/dist/application/operations/pushInsert.d.ts

@@ -12,5 +12,11 @@ export interface PushInsertContext {
     jobIndex: Map<JobId, JobLocation>;
     dashboardEmit?: (event: string, data: Record<string, unknown>) => void;
 }
+/** Resolve the first externally visible state without mutating queue structures. */
+export declare function initialJobState(job: Job, ctx: Pick<PushInsertContext, 'completedJobs' | 'depCompletions'>, now?: number): 'waiting-children' | 'delayed' | 'prioritized' | 'waiting';
 /** Insert a new job into the runnable heap or dependency wait set. */
-export declare function insertJobToShard(job: Job, queue: string, shard: Shard, shardIdx: number, ctx: PushInsertContext): void;
+export declare function insertJobToShard(job: Job, target: {
+    queue: string;
+    shard: Shard;
+    shardIdx: number;
+}, ctx: PushInsertContext, recordTimeline?: boolean): void;

package/dist/application/operations/pushInsert.js

@@ -1,13 +1,23 @@
-/** Insert a new job into the runnable heap or dependency wait set. */
-export function insertJobToShard(job, queue, shard, shardIdx, ctx) {
-    const hasDeps = job.dependsOn.length > 0;
-    const needsWaiting = hasDeps &&
+/** Resolve the first externally visible state without mutating queue structures. */
+export function initialJobState(job, ctx, now = Date.now()) {
+    const needsWaiting = job.dependsOn.length > 0 &&
         !job.dependsOn.every((depId) => ctx.completedJobs.has(depId) || (ctx.depCompletions?.has(depId) ?? false));
+    if (needsWaiting)
+        return 'waiting-children';
+    if (job.runAt > now)
+        return 'delayed';
+    return job.priority > 0 ? 'prioritized' : 'waiting';
+}
+/** Insert a new job into the runnable heap or dependency wait set. */
+export function insertJobToShard(job, target, ctx, recordTimeline = true) {
+    const { queue, shard, shardIdx } = target;
     const now = Date.now();
-    if (needsWaiting) {
+    const state = initialJobState(job, ctx, now);
+    if (state === 'waiting-children') {
         shard.waitingDeps.set(job.id, job);
         shard.registerDependencies(job.id, job.dependsOn);
-        job.timeline.push({ state: 'waiting-children', timestamp: now });
+        if (recordTimeline)
+            job.timeline.push({ state, timestamp: now });
         ctx.dashboardEmit?.('job:waiting-children', {
             jobId: String(job.id),
             queue,
@@ -18,8 +28,8 @@ export function insertJobToShard(job, queue, shard, shardIdx, ctx) {
         shard.getQueue(queue).push(job);
         const isDelayed = job.runAt > now;
         shard.incrementQueued(job.id, isDelayed, job.createdAt, queue, job.runAt);
-        const state = isDelayed ? 'delayed' : job.priority > 0 ? 'prioritized' : 'waiting';
-        job.timeline.push({ state, timestamp: now });
+        if (recordTimeline)
+            job.timeline.push({ state, timestamp: now });
     }
     ctx.jobIndex.set(job.id, { type: 'queue', shardIdx, queueName: queue });
     ctx.dependencyResults.registerConsumer(job.id, job.dependsOn);

package/dist/application/operations/pushLocks.d.ts

@@ -3,6 +3,7 @@ import type { JobLocation } from '../../domain/types/queue';
 import type { RWLock } from '../../shared/lock';
 interface PushLockContext {
     shardLocks: RWLock[];
+    customIdLock: RWLock;
     jobIndex: Map<JobId, JobLocation>;
 }
 /**

package/dist/application/operations/pushLocks.js

@@ -22,9 +22,12 @@ function requiredShardIndexes(queue, inputs, jobIndex) {
  */
 export async function withPushWriteLocks(queue, inputs, ctx, fn) {
     while (true) {
-        const indexes = requiredShardIndexes(queue, inputs, ctx.jobIndex);
+        const customIdGuard = inputs.some((input) => input.customId)
+            ? await ctx.customIdLock.acquireWrite()
+            : null;
         const guards = [];
         try {
+            const indexes = requiredShardIndexes(queue, inputs, ctx.jobIndex);
             for (const index of indexes) {
                 guards.push(await ctx.shardLocks[index].acquireWrite());
             }
@@ -38,6 +41,7 @@ export async function withPushWriteLocks(queue, inputs, ctx, fn) {
             for (let i = guards.length - 1; i >= 0; i--) {
                 guards[i].release();
             }
+            customIdGuard?.release();
         }
     }
 }

package/dist/application/operations/queryOperations.js

@@ -361,9 +361,10 @@ function querySqliteByLogicalState(storage, queue, sqlFilteredStates, opts) {
     if (opts.excludedIds.size === 0) {
         return storage.queryJobsByLogicalStates(queue, sqlFilteredStates, opts);
     }
-    // waitingDeps/waitingChildren rows remain persisted as waiting/delayed. Fetch
-    // enough rows to account for every possible exclusion, then paginate the
-    // logical result so parked jobs cannot create short or shifted pages.
+    // Legacy waitingDeps/waitingChildren rows may be persisted as waiting/delayed,
+    // while current inserts retain waiting-children. Fetch enough rows to account
+    // for every possible exclusion, then paginate the logical result so parked
+    // jobs cannot create short or shifted pages.
     const pageEnd = opts.offset + opts.limit;
     const jobs = storage.queryJobsByLogicalStates(queue, sqlFilteredStates, {
         limit: pageEnd + opts.excludedIds.size,
@@ -419,9 +420,9 @@ export function getJobs(queue, shardIdx, options, ctx) {
             s !== 'waiting-children' &&
             s !== 'paused' &&
             !(isPaused && (s === 'waiting' || s === 'prioritized')));
-        // In-memory parked state is authoritative over the persisted row. Initial
-        // dependency jobs are stored as waiting/delayed, while parents moved after
-        // a pull can still be stored as active.
+        // In-memory parked state is authoritative over the persisted row. Current
+        // inserts retain waiting-children; legacy rows may still be waiting/delayed,
+        // while parents moved after a pull can still be stored as active.
         const parkedJobs = states.includes('waiting-children') || sqlFilteredStates.length > 0
             ? collectWaitingChildrenJobs(shard, queue)
             : [];

package/dist/application/queueManager.d.ts

@@ -3,6 +3,7 @@
  * Core orchestrator for all queue operations
  */
 import type { Job, JobId, JobInput, JobLock, LockToken } from '../domain/types/job';
+import type { AtomicFlowBatchInput, AtomicFlowBatchResult } from '../domain/types/flow';
 import type { JobLocation, JobEvent } from '../domain/types/queue';
 import type { CronJob, CronJobInput } from '../domain/types/cron';
 import type { JobLogEntry, CreateWorkerOptions } from '../domain/types/worker';
@@ -24,6 +25,7 @@ export declare class QueueManager {
     private readonly storage;
     private readonly shards;
     private readonly shardLocks;
+    private readonly customIdLock;
     private readonly processingShards;
     private readonly processingLocks;
     private readonly jobIndex;
@@ -67,6 +69,7 @@ export declare class QueueManager {
     private handleRepeat;
     push(queue: string, input: JobInput): Promise<Job>;
     pushBatch(queue: string, inputs: JobInput[]): Promise<JobId[]>;
+    pushFlow(batch: AtomicFlowBatchInput): Promise<AtomicFlowBatchResult>;
     pull(queue: string, timeoutMs?: number, signal?: AbortSignal): Promise<Job | null>;
     pullWithLock(queue: string, owner: string, timeoutMs?: number, lockTtl?: number, signal?: AbortSignal): Promise<{
         job: Job | null;
@@ -278,6 +281,8 @@ export declare class QueueManager {
     getDepCompletions(): SetLike<JobId>;
     getShards(): Shard[];
     private onJobCompleted;
+    /** Release failure metadata owned by a parent that reached terminal failure. */
+    private onJobFailed;
     /** Check if completing this job completes an entire flow */
     private checkFlowCompleted;
     /**