@tstdl/base 0.93.77 → 0.93.78

package/queue/postgres/queue.js

@@ -4,67 +4,106 @@ var __decorate = (this && this.__decorate) || function (decorators, target, key,
     else for (var i = decorators.length - 1; i >= 0; i--) if (d = decorators[i]) r = (c < 3 ? d(r) : c > 3 ? d(target, key, r) : d(target, key)) || r;
     return c > 3 && r && Object.defineProperty(target, key, r), r;
 };
-import { and, asc, eq, inArray, isNull as isSqlNull, lt, lte, or, sql } from 'drizzle-orm';
+import { and, asc, eq, gt, inArray, lt, lte, or, sql, isNull as sqlIsNull } from 'drizzle-orm';
 import { merge } from 'rxjs';
 import { CancellationSignal } from '../../cancellation/index.js';
+import { CircuitBreaker, CircuitBreakerState } from '../../circuit-breaker/index.js';
+import { Enumerable } from '../../enumerable/enumerable.js';
 import { inject, injectArgument, provide, Singleton } from '../../injector/index.js';
 import { MessageBus } from '../../message-bus/index.js';
-import { interval, RANDOM_UUID_V4, TRANSACTION_TIMESTAMP } from '../../orm/index.js';
+import { coalesce, interval, RANDOM_UUID_V4, TRANSACTION_TIMESTAMP } from '../../orm/index.js';
 import { DatabaseConfig, injectRepository } from '../../orm/server/index.js';
+import { toArray } from '../../utils/array/array.js';
+import { currentTimestamp } from '../../utils/date-time.js';
+import { serializeError } from '../../utils/format-error.js';
 import { cancelableTimeout } from '../../utils/timing.js';
-import { isDefined, isString, isUndefined } from '../../utils/type-guards.js';
+import { isDefined, isNotNull, isNull, isString, isUndefined } from '../../utils/type-guards.js';
 import { millisecondsPerSecond } from '../../utils/units.js';
-import { defaultQueueConfig, Queue, UniqueTagStrategy } from '../queue.js';
-import { PostgresJob } from './job.model.js';
+import { defaultQueueConfig, Queue, TaskState, UniqueTagStrategy } from '../queue.js';
 import { PostgresQueueModuleConfig } from './module.js';
-import { job } from './schemas.js';
+import { task as taskTable } from './schemas.js';
+import { PostgresTask } from './task.model.js';
 let PostgresQueue = class PostgresQueue extends Queue {
-    #repository = injectRepository(PostgresJob);
+    #repository = injectRepository(PostgresTask);
     #config = injectArgument(this);
     #queueName = isString(this.#config) ? this.#config : this.#config.name;
     #messageBus = inject((MessageBus), `PostgresQueue:${this.#queueName}`);
-    #keepOldUpdate = { id: sql `${job.id}` };
+    #circuitBreaker = inject(CircuitBreaker, {
+        key: this.#queueName,
+        threshold: (isString(this.#config) ? undefined : this.#config.circuitBreakerThreshold) ?? defaultQueueConfig.circuitBreakerThreshold,
+        resetTimeout: (isString(this.#config) ? undefined : this.#config.circuitBreakerResetTimeout) ?? defaultQueueConfig.circuitBreakerResetTimeout,
+    });
     processTimeout = (isString(this.#config) ? undefined : this.#config.processTimeout) ?? defaultQueueConfig.processTimeout;
+    executionTimeout = (isString(this.#config) ? undefined : this.#config.executionTimeout) ?? defaultQueueConfig.executionTimeout;
     maxTries = (isString(this.#config) ? undefined : this.#config.maxTries) ?? defaultQueueConfig.maxTries;
+    retryDelayMinimum = (isString(this.#config) ? undefined : this.#config.retryDelayMinimum) ?? defaultQueueConfig.retryDelayMinimum;
+    retryDelayMaximum = (isString(this.#config) ? undefined : this.#config.retryDelayMaximum) ?? defaultQueueConfig.retryDelayMaximum;
+    retryDelayGrowth = (isString(this.#config) ? undefined : this.#config.retryDelayGrowth) ?? defaultQueueConfig.retryDelayGrowth;
+    retentionPeriod = (isString(this.#config) ? undefined : this.#config.retentionPeriod) ?? defaultQueueConfig.retentionPeriod;
+    globalConcurrency = (isString(this.#config) ? undefined : this.#config.globalConcurrency) ?? defaultQueueConfig.globalConcurrency;
+    // -- Updates --
+    #keepOldUpdate = { id: sql `${taskTable.id}` };
     #takeNewUpdate = {
         id: RANDOM_UUID_V4,
         queue: this.#queueName,
         priority: sql `excluded.priority`,
         tag: sql `excluded.tag`,
+        status: TaskState.Pending,
+        lease: null,
         tries: 0,
+        progress: 0,
         enqueueTimestamp: TRANSACTION_TIMESTAMP,
-        lastDequeueTimestamp: null,
+        scheduleTimestamp: sql `excluded.schedule_timestamp`,
+        startTimestamp: null,
+        expirationTimestamp: sql `excluded.expiration_timestamp`,
+        lockExpirationTimestamp: null,
         data: sql `excluded.data`,
-    };
-    #dequeueQuery = and(eq(job.queue, this.#queueName), lt(job.tries, this.maxTries), or(isSqlNull(job.lastDequeueTimestamp), lte(sql `${job.lastDequeueTimestamp} + ${interval(this.processTimeout, 'milliseconds')}`, TRANSACTION_TIMESTAMP)));
-    #dequeueUpdate = {
-        tries: sql `${job.tries} + 1`,
-        lastDequeueTimestamp: TRANSACTION_TIMESTAMP,
+        state: null,
+        error: null,
+        result: null,
     };
     async enqueue(data, options) {
-        const jobs = await this.enqueueMany([{ data, tag: options?.tag, priority: options?.priority }], { uniqueTag: options?.uniqueTag, returnJobs: true });
-        return jobs[0];
+        const tasks = await this.enqueueMany([{ data, ...options }], { uniqueTag: options?.uniqueTag, returnTasks: true, transaction: options?.transaction });
+        return tasks[0];
     }
     async enqueueMany(items, options) {
+        if (items.length == 0) {
+            return (options?.returnTasks == true) ? [] : undefined;
+        }
         const newEntities = items.map((item) => ({
             queue: this.#queueName,
+            status: TaskState.Pending,
+            lease: null,
             priority: item.priority ?? 1000,
             tag: item.tag ?? null,
+            parentId: item.parentId ?? null,
             tries: 0,
+            progress: 0,
             enqueueTimestamp: TRANSACTION_TIMESTAMP,
-            lastDequeueTimestamp: null,
+            scheduleTimestamp: item.scheduleTimestamp ?? TRANSACTION_TIMESTAMP,
+            startTimestamp: null,
+            expirationTimestamp: item.expirationTimestamp ?? null,
+            lockExpirationTimestamp: null,
+            completeTimestamp: null,
             data: item.data,
+            state: null,
+            result: null,
+            error: null,
         }));
         const update = (options?.uniqueTag == UniqueTagStrategy.TakeNew)
             ? this.#takeNewUpdate
             : (options?.uniqueTag == UniqueTagStrategy.KeepOld)
                 ? this.#keepOldUpdate
                 : undefined;
-        const jobs = isUndefined(update)
-            ? await this.#repository.insertMany(newEntities)
-            : await this.#repository.upsertMany(['queue', 'tag'], newEntities, update);
+        const repository = this.#repository.withOptionalTransaction(options?.transaction);
+        const tasks = isUndefined(update)
+            ? await repository.insertMany(newEntities)
+            : await repository.upsertMany(['queue', 'tag'], newEntities, update);
         this.#messageBus.publishAndForget();
-        return jobs;
+        if (options?.returnTasks == true) {
+            return tasks;
+        }
+        return undefined;
     }
     async has(id) {
         return await this.#repository.hasByQuery({ queue: this.#queueName, id });
@@ -81,60 +120,539 @@ let PostgresQueue = class PostgresQueue extends Queue {
     async getByTags(tags) {
         return await this.#repository.loadManyByQuery({ queue: this.#queueName, tag: { $in: tags } });
     }
-    async cancel(id) {
-        await this.#repository.hardDeleteByQuery({ queue: this.#queueName, id });
+    async getTree(idOrIds, transaction) {
+        const ids = toArray(idOrIds);
+        return await this.getTreeByQuery({ id: { $in: ids } }, transaction);
     }
-    async cancelMany(ids) {
-        await this.#repository.hardDeleteManyByQuery({ queue: this.#queueName, id: { $in: ids } });
+    async getTreeByQuery(query, transaction) {
+        return await this.#repository.useTransaction(transaction, async (tx) => {
+            const repositoryWithTransaction = this.#repository.withTransaction(tx);
+            const tasks = await repositoryWithTransaction.loadManyByQuery(query);
+            if (tasks.length == 0) {
+                return [];
+            }
+            let currentLevelIds = tasks.map((task) => task.id);
+            let depth = 0;
+            while (true) {
+                if (depth++ > 100) {
+                    throw new Error('Possible cyclic task parent-child relationship detected in queue "' + this.#queueName + '"');
+                }
+                const childTasks = await repositoryWithTransaction.loadManyByQuery({ parentId: { $in: currentLevelIds } });
+                if (childTasks.length == 0) {
+                    break;
+                }
+                currentLevelIds = childTasks.map((task) => task.id);
+                tasks.push(...childTasks);
+            }
+            return tasks;
+        });
+    }
+    async cancel(id, transaction) {
+        await this.cancelMany([id], transaction);
+    }
+    async cancelMany(ids, transaction) {
+        await this.#repository.useTransaction(transaction, async (tx) => {
+            const tree = await this.getTree(ids, tx);
+            const treeIds = tree.map((task) => task.id);
+            if (treeIds.length == 0) {
+                return;
+            }
+            await this.#repository.withTransaction(tx).updateMany(treeIds, {
+                status: TaskState.Cancelled,
+                lease: null,
+                completeTimestamp: TRANSACTION_TIMESTAMP,
+            });
+            const uniqueParents = new Set();
+            for (const t of tree) {
+                const task = t;
+                if (isNotNull(task.parentId)) {
+                    uniqueParents.add(task.parentId);
+                }
+            }
+            for (const parentId of uniqueParents) {
+                await this.#triggerParentFanIn(parentId, tx);
+            }
+        });
     }
     async cancelByTag(tag) {
-        await this.#repository.hardDeleteByQuery({ queue: this.#queueName, tag });
+        await this.cancelManyByTag([tag]);
     }
-    async cancelByTags(tags) {
-        await this.#repository.hardDeleteManyByQuery({ queue: this.#queueName, tag: { $in: tags } });
+    async cancelManyByTag(tags) {
+        await this.#repository.transaction(async (tx) => {
+            const task = await this.getByTags(tags);
+            const ids = task.map((t) => t.id);
+            await this.cancelMany(ids, tx);
+        });
+    }
+    async clear() {
+        await this.#repository.hardDeleteManyByQuery({ queue: this.#queueName });
     }
     async dequeue() {
-        const jobs = await this.dequeueMany(1);
-        if (jobs.length == 0) {
+        const tasks = await this.dequeueMany(1);
+        if (tasks.length == 0) {
             return undefined;
         }
-        return jobs[0];
+        return tasks[0];
     }
     async dequeueMany(count) {
-        /*
-         * Materialization required for LIMIT clause
-         * https://stackoverflow.com/questions/73966670/select-for-update-subquery-not-respecting-limit-clause-under-load
-         * https://dba.stackexchange.com/questions/69471/postgres-update-limit-1
-         */
-        const selection = this.#repository.session.$with('selection').as((qb) => qb
-            .select({ id: job.id })
-            .from(job)
-            .where(and(this.#dequeueQuery, sql `pg_sleep(0) IS NOT NULL` // workaround to force materialization until drizzle implements https://github.com/drizzle-team/drizzle-orm/issues/2318
-        ))
-            .orderBy(asc(job.priority), asc(job.enqueueTimestamp), asc(job.lastDequeueTimestamp), asc(job.tries))
-            .limit(count)
-            .for('update', { skipLocked: true }));
-        const rows = await this.#repository.session
-            .with(selection)
-            .update(job)
-            .set(this.#dequeueUpdate)
-            .where(inArray(job.id, this.#repository.session.select().from(selection)))
-            .returning();
-        return await this.#repository.mapManyToEntity(rows);
-    }
-    async acknowledge(job) {
-        await this.cancel(job.id);
-    }
-    async acknowledgeMany(jobs) {
-        const ids = jobs.map((job) => job.id);
-        await this.cancelMany(ids);
+        return await this.#repository.transaction(async (tx) => {
+            let effectiveCount = count;
+            // 1. Check Circuit Breaker
+            const result = await this.#circuitBreaker.check();
+            if (!result.allowed) {
+                return [];
+            }
+            if (result.state == CircuitBreakerState.HalfOpen) {
+                // If we are probing (HalfOpen), we must ensure no other tasks are running locally.
+                // NOTE: PostgresCircuitBreakerService.check() handles the "one probe per timeout" via the DB state transition.
+                // However, we still might want to limit this batch to 1 task if we are the probe.
+                if (result.isProbe != true) {
+                    const runningCount = await this.#repository.withTransaction(tx).countByQuery({ queue: this.#queueName, status: TaskState.Running });
+                    if (runningCount > 0) {
+                        return [];
+                    }
+                }
+                effectiveCount = 1;
+            }
+            // 2. Check Global Concurrency
+            if (isNotNull(this.globalConcurrency)) {
+                const runningCount = await this.#repository.withTransaction(tx).countByQuery({ queue: this.#queueName, status: TaskState.Running });
+                if (runningCount >= this.globalConcurrency) {
+                    return [];
+                }
+            }
+            /*
+             * Materialization required for LIMIT clause
+             * https://stackoverflow.com/questions/73966670/select-for-update-subquery-not-respecting-limit-clause-under-load
+             * https://dba.stackexchange.com/questions/69471/postgres-update-limit-1
+             */
+            const selection = this.#repository.session.$with('selection').as((qb) => qb
+                .select({ id: taskTable.id })
+                .from(taskTable)
+                .where(and(eq(taskTable.queue, this.#queueName), lte(taskTable.scheduleTimestamp, TRANSACTION_TIMESTAMP), or(sqlIsNull(taskTable.expirationTimestamp), lt(TRANSACTION_TIMESTAMP, taskTable.expirationTimestamp)), or(eq(taskTable.status, TaskState.Pending), and(eq(taskTable.status, TaskState.Running), lt(taskTable.lockExpirationTimestamp, TRANSACTION_TIMESTAMP), // Zombie detection (only non-exhausted ones)
+            lt(taskTable.tries, this.maxTries))), sql `pg_sleep(0) IS NOT NULL` // Materialization hack until drizzle implements https://github.com/drizzle-team/drizzle-orm/issues/2318
+            ))
+                .orderBy(asc(taskTable.priority), asc(taskTable.scheduleTimestamp), asc(taskTable.tries))
+                .limit(effectiveCount)
+                .for('update', { skipLocked: true }));
+            const rows = await this.#repository.session
+                .with(selection)
+                .update(taskTable)
+                .set({
+                status: TaskState.Running,
+                lease: RANDOM_UUID_V4,
+                lockExpirationTimestamp: sql `${TRANSACTION_TIMESTAMP} + ${interval(this.processTimeout, 'milliseconds')}`,
+                startTimestamp: TRANSACTION_TIMESTAMP,
+                // If it was PENDING, it's the first try (tries=0) -> tries=1.
+                // If it was RUNNING (Zombie), previous try failed -> increment tries.
+                tries: sql `${taskTable.tries} + 1`,
+            })
+                .where(inArray(taskTable.id, this.#repository.session.select().from(selection)))
+                .returning();
+            return await this.#repository.mapManyToEntity(rows);
+        });
+    }
+    async reschedule(id, timestamp, transaction) {
+        await this.rescheduleMany([id], timestamp, transaction);
+    }
+    async rescheduleMany(ids, timestamp, transaction) {
+        await this.#repository.withOptionalTransaction(transaction).updateMany(ids, {
+            status: TaskState.Pending,
+            lease: null,
+            scheduleTimestamp: timestamp,
+            lockExpirationTimestamp: null,
+            tries: sql `CASE
+        WHEN ${taskTable.status} = ${TaskState.Running} THEN GREATEST(0, ${taskTable.tries} - 1)
+        ELSE ${taskTable.tries}
+      END`,
+        });
+    }
+    async rescheduleByTag(tag, timestamp, transaction) {
+        await this.rescheduleManyByTag([tag], timestamp, transaction);
+    }
+    async rescheduleManyByTag(tags, timestamp, transaction) {
+        await this.#repository.withOptionalTransaction(transaction).updateManyByQuery({
+            queue: this.#queueName,
+            tag: { $in: tags },
+        }, {
+            status: TaskState.Pending,
+            lease: null,
+            scheduleTimestamp: timestamp,
+            lockExpirationTimestamp: null,
+            tries: sql `CASE
+        WHEN ${taskTable.status} = ${TaskState.Running} THEN GREATEST(0, ${taskTable.tries} - 1)
+        ELSE ${taskTable.tries}
+      END`,
+        });
+    }
+    async touch(task, options) {
+        if (isNull(task.lease)) {
+            return undefined;
+        }
+        return await this.#repository.useTransaction(options?.transaction, async (tx) => {
+            const update = {
+                lockExpirationTimestamp: sql `${TRANSACTION_TIMESTAMP} + ${interval(this.processTimeout, 'milliseconds')}`,
+            };
+            if (isDefined(options?.progress)) {
+                update.progress = options.progress;
+            }
+            if (isDefined(options?.state)) {
+                update.state = options.state;
+            }
+            // Attempt to update lease.
+            // FAILS (returns undefined) if lease has changed (timeout/cancel/stolen)
+            // OR if hard execution timeout has passed.
+            const result = await this.#repository.withTransaction(tx).tryUpdateByQuery({
+                queue: this.#queueName,
+                id: task.id,
+                status: TaskState.Running,
+                lease: task.lease,
+                startTimestamp: { $gt: sql `${TRANSACTION_TIMESTAMP} - ${interval(this.executionTimeout, 'milliseconds')}` },
+            }, update);
+            return result;
+        });
+    }
+    async touchMany(tasks, progresses, states, transaction) {
+        if (tasks.length == 0) {
+            return [];
+        }
+        const rows = tasks.map((t, i) => {
+            const progress = progresses?.[i] ?? null;
+            const state = states?.[i] ?? null;
+            return sql `(${t.id}::uuid, ${t.lease}::uuid, ${progress}::numeric, ${state}::jsonb)`;
+        });
+        const updates = this.#repository.session.$with('updates').as((qb) => qb
+            .select({
+            updateId: sql `(id)::uuid`.as('update_id'),
+            updateLease: sql `(lease)::uuid`.as('update_lease'),
+            updateProgress: sql `(progress)::numeric`.as('update_progress'),
+            updateState: sql `(state)::jsonb`.as('update_state'),
+        })
+            .from(sql `(VALUES ${sql.join(rows, sql `, `)}) AS t(id, lease, progress, state)`));
+        const updated = this.#repository.session.$with('updated').as(() => this.#repository.session
+            .update(taskTable)
+            .set({
+            lockExpirationTimestamp: sql `${TRANSACTION_TIMESTAMP} + ${interval(this.processTimeout, 'milliseconds')}`,
+            progress: coalesce(updates.updateProgress, taskTable.progress),
+            state: coalesce(updates.updateState, taskTable['state']),
+        })
+            .from(updates)
+            .where(and(eq(taskTable.id, updates.updateId), eq(taskTable.queue, this.#queueName), eq(taskTable.lease, updates.updateLease), eq(taskTable.status, TaskState.Running), gt(taskTable.startTimestamp, sql `${TRANSACTION_TIMESTAMP} - ${interval(this.executionTimeout, 'milliseconds')}`)))
+            .returning({ id: taskTable.id }));
+        const result = await this.#repository.withOptionalTransaction(transaction).session
+            .with(updates, updated)
+            .select({ id: updated.id })
+            .from(updated)
+            .execute();
+        return result.map((r) => r.id);
+    }
+    async acknowledge(task, result, transaction, { skipFanIn = false } = {}) {
+        await this.#repository.useTransaction(transaction, async (tx) => {
+            const repository = this.#repository.withTransaction(tx);
+            // 1. Fan-Out Check: Does this task have active children?
+            // If yes, this is a Parent task that has spawned children.
+            // It should enter 'Waiting' instead of 'Completed'.
+            const activeChildrenCount = await repository.countByQuery({
+                parentId: task.id,
+                status: { $nin: [TaskState.Completed, TaskState.Cancelled, TaskState.Dead] },
+            });
+            let updatedTask;
+            if (activeChildrenCount > 0) {
+                updatedTask = await repository.tryUpdateByQuery({
+                    id: task.id,
+                    lease: task.lease,
+                }, {
+                    status: TaskState.Waiting,
+                    lease: null,
+                    lockExpirationTimestamp: null,
+                    state: result,
+                });
+            }
+            else {
+                updatedTask = await repository.tryUpdateByQuery({
+                    id: task.id,
+                    lease: task.lease,
+                }, {
+                    status: TaskState.Completed,
+                    lease: null,
+                    result: result,
+                    progress: 1,
+                    completeTimestamp: TRANSACTION_TIMESTAMP,
+                    lockExpirationTimestamp: null,
+                });
+            }
+            if (isUndefined(updatedTask)) {
+                return;
+            }
+            // 2. Reset Circuit Breaker (Success detected)
+            await this.#circuitBreaker.recordSuccess();
+            // 3. Fan-In: Wake up parent if all siblings are done
+            if (!skipFanIn && isNotNull(task.parentId)) {
+                await this.#triggerParentFanIn(task.parentId, tx);
+            }
+        });
+    }
+    async acknowledgeMany(tasks, results, transaction) {
+        if (tasks.length == 0) {
+            return;
+        }
+        await this.#repository.useTransaction(transaction, async (tx) => {
+            // 1. Bulk Check for Active Children
+            const taskIds = tasks.map((task) => task.id);
+            const parentsWithChildrenRows = await this.#repository.withTransaction(tx).session
+                .selectDistinct({ id: taskTable.parentId }) // Distinct is cleaner
+                .from(taskTable)
+                .where(and(inArray(taskTable.parentId, taskIds), inArray(taskTable.status, [TaskState.Pending, TaskState.Running, TaskState.Waiting])))
+                .execute();
+            const allParentIds = parentsWithChildrenRows.map((row) => row.id);
+            const distinctParentIds = new Set(allParentIds);
+            // 2. Separate tasks
+            const tasksToWait = [];
+            const tasksToComplete = [];
+            const resultsToComplete = [];
+            const resultsToWait = [];
+            for (let i = 0; i < tasks.length; i++) {
+                const task = tasks[i];
+                if (distinctParentIds.has(task.id)) {
+                    tasksToWait.push(task);
+                    if (isDefined(results)) {
+                        resultsToWait.push(results[i]);
+                    }
+                }
+                else {
+                    tasksToComplete.push(task);
+                    if (isDefined(results)) {
+                        resultsToComplete.push(results[i]);
+                    }
+                }
+            }
+            // 3. Update 'Waiting' tasks
+            if (tasksToWait.length > 0) {
+                const rows = tasksToWait.map((task, i) => {
+                    const result = isDefined(results) ? resultsToWait[i] : null;
+                    return sql `(${task.id}::uuid, ${task.lease}::uuid, ${result}::jsonb)`;
+                });
+                const updates = this.#repository.session.$with('updates').as((qb) => qb
+                    .select({
+                    updateId: sql `(id)::uuid`.as('update_id'),
+                    updateLease: sql `(lease)::uuid`.as('update_lease'),
+                    updateResult: sql `(result)::jsonb`.as('update_result'),
+                })
+                    .from(sql `(VALUES ${sql.join(rows, sql `, `)}) AS t(id, lease, result)`));
+                const updated = this.#repository.session.$with('updated').as(() => this.#repository.session
+                    .update(taskTable)
+                    .set({
+                    status: TaskState.Waiting,
+                    lease: null,
+                    lockExpirationTimestamp: null,
+                    result: updates.updateResult,
+                })
+                    .from(updates)
+                    .where(and(eq(taskTable.id, updates.updateId), eq(taskTable.lease, updates.updateLease), eq(taskTable.queue, this.#queueName)))
+                    .returning({ id: taskTable.id }));
+                await this.#repository.withTransaction(tx).session
+                    .with(updates, updated)
+                    .select({ id: updated.id })
+                    .from(updated)
+                    .execute();
+            }
+            // 4. Update 'Completed' tasks
+            if (tasksToComplete.length > 0) {
+                const rows = tasksToComplete.map((t, i) => {
+                    const result = isDefined(results) ? resultsToComplete[i] : null;
+                    return sql `(${t.id}::uuid, ${t.lease}::uuid, ${result}::jsonb)`;
+                });
+                const updates = this.#repository.session.$with('updates').as((qb) => qb
+                    .select({
+                    updateId: sql `(id)::uuid`.as('update_id'),
+                    updateLease: sql `(lease)::uuid`.as('update_lease'),
+                    updateResult: sql `(result)::jsonb`.as('update_result'),
+                })
+                    .from(sql `(VALUES ${sql.join(rows, sql `, `)}) AS t(id, lease, result)`));
+                const updated = this.#repository.session.$with('updated').as(() => this.#repository.session
+                    .update(taskTable)
+                    .set({
+                    status: TaskState.Completed,
+                    lease: null,
+                    result: updates.updateResult,
+                    progress: 1,
+                    completeTimestamp: TRANSACTION_TIMESTAMP,
+                    lockExpirationTimestamp: null,
+                })
+                    .from(updates)
+                    .where(and(eq(taskTable.id, updates.updateId), eq(taskTable.lease, updates.updateLease), eq(taskTable.queue, this.#queueName)))
+                    .returning({ id: taskTable.id }));
+                await this.#repository.withTransaction(tx).session
+                    .with(updates, updated)
+                    .select({ id: updated.id })
+                    .from(updated)
+                    .execute();
+                // Reset circuit breaker
+                await this.#circuitBreaker.recordSuccess();
+            }
+            // 5. Fan-In
+            const parentIds = Enumerable.from(tasks)
+                .filter((task) => isNotNull(task.parentId))
+                .map((task) => task.parentId)
+                .distinct();
+            for (const parentId of parentIds) {
+                await this.#triggerParentFanIn(parentId, tx);
+            }
+        });
+    }
+    async fail(task, error, fatal = false, transaction, { skipFanIn = false } = {}) {
+        const isRetryable = !fatal && (task.tries < this.maxTries);
+        const nextStatus = isRetryable ? TaskState.Pending : TaskState.Dead;
+        const delay = isRetryable
+            ? Math.min(this.retryDelayMaximum, this.retryDelayMinimum * (this.retryDelayGrowth ** task.tries))
+            : 0;
+        const nextSchedule = currentTimestamp() + delay;
+        await this.#repository.useTransaction(transaction, async (tx) => {
+            const updatedTask = await this.#repository.withTransaction(tx).tryUpdateByQuery({
+                queue: this.#queueName,
+                id: task.id,
+                lease: task.lease,
+                tries: task.tries,
+            }, {
+                status: nextStatus,
+                lease: null,
+                error: serializeError(error),
+                lockExpirationTimestamp: null,
+                scheduleTimestamp: nextSchedule,
+                startTimestamp: null,
+                completeTimestamp: (nextStatus == TaskState.Dead) ? TRANSACTION_TIMESTAMP : null,
+            });
+            if (isUndefined(updatedTask)) {
+                return;
+            }
+            await this.#circuitBreaker.recordFailure();
+            if (!skipFanIn && (nextStatus == TaskState.Dead) && isNotNull(task.parentId)) {
+                await this.#triggerParentFanIn(task.parentId, tx);
+            }
+        });
+    }
+    async #triggerParentFanIn(parentId, transaction) {
+        await this.#repository.useTransaction(transaction, async (tx) => {
+            // 1. Lock Parent
+            const [parent] = await this.#repository.withTransaction(tx).session
+                .select({ id: taskTable.id, status: taskTable.status })
+                .from(taskTable)
+                .where(eq(taskTable.id, parentId))
+                .for('update')
+                .execute();
+            if (parent?.status != TaskState.Waiting) {
+                return;
+            }
+            // 2. Check Children
+            const hasActiveChildren = await this.#repository.withTransaction(tx).hasByQuery({
+                parentId,
+                status: { $in: [TaskState.Pending, TaskState.Running, TaskState.Waiting] },
+            });
+            if (!hasActiveChildren) {
+                // 3. Update Parent
+                await this.#repository.withTransaction(tx).updateByQuery({ id: parentId }, {
+                    status: TaskState.Pending,
+                    scheduleTimestamp: TRANSACTION_TIMESTAMP,
+                });
+            }
+        });
+    }
+    async failMany(tasks, errors, transaction) {
+        if (tasks.length == 0) {
+            return;
+        }
+        await this.#repository.useTransaction(transaction, async (tx) => {
+            await Promise.all(tasks.map(async (task, index) => await this.fail(task, errors[index], false, tx, { skipFanIn: true })));
+            const parentIds = Enumerable.from(tasks)
+                .filter((task) => isNotNull(task.parentId))
+                .map((task) => task.parentId)
+                .distinct();
+            for (const parentId of parentIds) {
+                await this.#triggerParentFanIn(parentId, tx);
+            }
+        });
+    }
+    async prune() {
+        // 1. Clean up old finished tasks
+        await this.#repository.deleteManyByQuery({
+            queue: this.#queueName,
+            status: { $in: [TaskState.Completed, TaskState.Cancelled, TaskState.Dead] },
+            completeTimestamp: { $lt: sql `${TRANSACTION_TIMESTAMP} - ${interval(this.retentionPeriod, 'milliseconds')}` },
+        });
+        await this.#repository.transaction(async (tx) => {
+            // 2. Fail tasks that sat in Pending state past their expiration
+            const expiredTasks = await this.#repository.updateManyByQuery({
+                queue: this.#queueName,
+                status: TaskState.Pending,
+                expirationTimestamp: { $lt: TRANSACTION_TIMESTAMP },
+            }, {
+                status: TaskState.Dead,
+                lease: null,
+                error: { message: 'Queue Timeout: Task expired before processing' },
+                completeTimestamp: TRANSACTION_TIMESTAMP,
+            });
+            // 3. Fail "Rotting Zombies" (Tasks that crashed workers and exceeded max retries)
+            // These were excluded from dequeueMany to prevent infinite loops, so we capture them here.
+            const zombieTasks = await this.#repository.updateManyByQuery({
+                queue: this.#queueName,
+                status: TaskState.Running,
+                lockExpirationTimestamp: { $lt: TRANSACTION_TIMESTAMP },
+                tries: { $gte: this.maxTries },
+            }, {
+                status: TaskState.Dead,
+                lease: null,
+                error: { message: 'Zombie Task: Exceeded max retries after repeated crashes' },
+                completeTimestamp: TRANSACTION_TIMESTAMP,
+                lockExpirationTimestamp: null,
+            });
+            // 4. Fail tasks that exceeded hard execution timeout
+            const hardTimedoutTasks = await this.#repository.updateManyByQuery({
+                queue: this.#queueName,
+                status: TaskState.Running,
+                startTimestamp: { $lt: sql `${TRANSACTION_TIMESTAMP} - ${interval(this.executionTimeout, 'milliseconds')}` },
+            }, {
+                status: TaskState.Dead,
+                lease: null,
+                error: { message: `Hard Execution Timeout: Task ran longer than ${this.executionTimeout}ms` },
+                completeTimestamp: TRANSACTION_TIMESTAMP,
+                lockExpirationTimestamp: null,
+            });
+            const distinctParentTaskIds = Enumerable
+                .from(expiredTasks)
+                .concat(zombieTasks, hardTimedoutTasks)
+                .filter((task) => isNotNull(task.parentId))
+                .map((task) => task.parentId)
+                .distinct();
+            // 3. Trigger Fan-In for all affected parents
+            for (const parentId of distinctParentTaskIds) {
+                await this.#triggerParentFanIn(parentId, tx);
+            }
+        });
+    }
+    async restart(id, transaction) {
+        await this.#repository.withOptionalTransaction(transaction).updateByQuery({
+            id,
+            $or: [
+                { status: { $ne: TaskState.Running } },
+                { lockExpirationTimestamp: { $lt: TRANSACTION_TIMESTAMP } },
+            ],
+        }, {
+            status: TaskState.Pending,
+            lease: null,
+            error: null,
+            scheduleTimestamp: TRANSACTION_TIMESTAMP,
+            tries: 0,
+            progress: 0,
+            result: null,
+            completeTimestamp: null,
+        });
     }
     async *getConsumer(cancellationSignal) {
         const continue$ = merge(this.#messageBus.allMessages$, cancellationSignal);
         while (cancellationSignal.isUnset) {
-            const job = await this.dequeue();
-            if (isDefined(job)) {
-                yield job;
+            const task = await this.dequeue();
+            if (isDefined(task)) {
+                yield task;
                 continue;
             }
             await cancelableTimeout(5 * millisecondsPerSecond, continue$);
@@ -143,9 +661,9 @@ let PostgresQueue = class PostgresQueue extends Queue {
     async *getBatchConsumer(size, cancellationSignal) {
         const continue$ = merge(this.#messageBus.allMessages$, cancellationSignal);
         while (cancellationSignal.isUnset) {
-            const jobs = await this.dequeueMany(size);
-            if (jobs.length > 0) {
-                yield jobs;
+            const tasks = await this.dequeueMany(size);
+            if (tasks.length > 0) {
+                yield tasks;
                 continue;
             }
             await cancelableTimeout(5 * millisecondsPerSecond, continue$);