@tstdl/base 0.93.157 → 0.93.158

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@tstdl/base",
-  "version": "0.93.157",
+  "version": "0.93.158",
   "author": "Patrick Hein",
   "publishConfig": {
     "access": "public"

package/task-queue/postgres/task-queue.d.ts

@@ -110,6 +110,7 @@ export declare class PostgresTaskQueue<Definitions extends TaskDefinitionMap = T
     failMany(tasks: Task<Definitions>[], errors: unknown[], options?: {
         transaction?: Transaction;
     }): Promise<void>;
+    private getFailureUpdate;
     private resolveDependencies;
     private resolveDependenciesMany;
     maintenance(options?: {
@@ -118,9 +119,7 @@ export declare class PostgresTaskQueue<Definitions extends TaskDefinitionMap = T
     private performArchival;
     private performArchivePurge;
     private processExpirations;
-    private processZombieRetries;
-    private processZombieExhaustions;
-    private processHardTimeouts;
+    private recoverStaleTasks;
     private processPriorityAging;
     restart(id: string, options?: {
         resetState?: boolean;

package/task-queue/postgres/task-queue.js

@@ -56,7 +56,7 @@ var __disposeResources = (this && this.__disposeResources) || (function (Suppres
     var e = new Error(message);
     return e.name = "SuppressedError", e.error = error, e.suppressed = suppressed, e;
 });
-import { aliasedTable, and, asc, count, eq, gt, gte, inArray, lt, lte, notExists, notInArray, or, sql, count as sqlCount, isNotNull as sqlIsNotNull, isNull as sqlIsNull } from 'drizzle-orm';
+import { aliasedTable, and, asc, count, eq, gt, gte, inArray, lt, lte, not, notExists, notInArray, or, sql, count as sqlCount, isNotNull as sqlIsNotNull, isNull as sqlIsNull } from 'drizzle-orm';
 import { filter, merge, throttleTime } from 'rxjs';
 import { CancellationSignal } from '../../cancellation/index.js';
 import { CircuitBreaker, CircuitBreakerState } from '../../circuit-breaker/index.js';
@@ -837,23 +837,17 @@ let PostgresTaskQueue = class PostgresTaskQueue extends TaskQueue {
     }
     async fail(task, caughtError, options) {
         const error = ensureTaskError(caughtError);
-        const isRetryable = (options?.fatal != true) && (task.tries < this.maxTries);
-        const nextStatus = isRetryable ? TaskStatus.Retrying : TaskStatus.Dead;
-        const delay = isRetryable
-            ? Math.min(this.retryDelayMaximum, this.retryDelayMinimum * (this.retryDelayGrowth ** (task.tries - 1)))
-            : 0;
-        const nextSchedule = sql `${TRANSACTION_TIMESTAMP} + ${interval(delay, 'milliseconds')}`;
+        const update = this.getFailureUpdate(task.tries, error, options);
         await this.#repository.useTransaction(options?.transaction, async (tx) => {
             const [updatedRow] = await tx.pgTransaction
                 .update(taskTable)
                 .set({
-                status: nextStatus,
+                status: update.status,
                 token: null,
-                error: serializeError(error),
+                error: update.error,
                 visibilityDeadline: null,
-                scheduleTimestamp: isRetryable ? nextSchedule : taskTable.scheduleTimestamp,
-                startTimestamp: null,
-                completeTimestamp: (nextStatus == TaskStatus.Dead) ? TRANSACTION_TIMESTAMP : null,
+                scheduleTimestamp: (update.status == TaskStatus.Retrying) ? update.scheduleTimestamp : taskTable.scheduleTimestamp,
+                completeTimestamp: update.completeTimestamp,
             })
                 .where(and(eq(taskTable.id, task.id), isNull(task.token) ? sqlIsNull(taskTable.token) : eq(taskTable.token, task.token), eq(taskTable.tries, task.tries)))
                 .returning();
@@ -861,7 +855,7 @@ let PostgresTaskQueue = class PostgresTaskQueue extends TaskQueue {
                 return;
             }
             await this.#circuitBreaker.recordFailure();
-            await this.resolveDependencies(task.id, nextStatus, { namespace: task.namespace, transaction: tx });
+            await this.resolveDependencies(task.id, update.status, { namespace: task.namespace, transaction: tx });
         });
     }
     async failMany(tasks, errors, options) {
@@ -871,14 +865,8 @@ let PostgresTaskQueue = class PostgresTaskQueue extends TaskQueue {
         await this.#repository.useTransaction(options?.transaction, async (tx) => {
             const rows = tasks.map((task, index) => {
                 const error = ensureTaskError(errors[index]);
-                const isRetryable = (task.tries < this.maxTries);
-                const nextStatus = isRetryable ? TaskStatus.Retrying : TaskStatus.Dead;
-                const delay = isRetryable
-                    ? Math.min(this.retryDelayMaximum, this.retryDelayMinimum * (this.retryDelayGrowth ** (task.tries - 1)))
-                    : 0;
-                const nextSchedule = sql `(${TRANSACTION_TIMESTAMP} + ${interval(delay, 'milliseconds')})`;
-                const completeTimestamp = (nextStatus == TaskStatus.Dead) ? TRANSACTION_TIMESTAMP : null;
-                return sql `(${task.id}::uuid, ${task.token}::uuid, ${task.tries}::int, ${nextStatus}::${taskStatus}, ${serializeError(error)}::jsonb, ${nextSchedule}::timestamptz, ${completeTimestamp}::timestamptz)`;
+                const update = this.getFailureUpdate(task.tries, error);
+                return sql `(${task.id}::uuid, ${task.token}::uuid, ${task.tries}::int, ${update.status}::${taskStatus}, ${update.error}::jsonb, ${update.scheduleTimestamp}::timestamptz, ${update.completeTimestamp}::timestamptz)`;
             });
             const updates = tx.pgTransaction.$with('updates').as((qb) => qb
                 .select({
@@ -899,7 +887,6 @@ let PostgresTaskQueue = class PostgresTaskQueue extends TaskQueue {
                 error: sql `${updates.updateError}`,
                 visibilityDeadline: null,
                 scheduleTimestamp: caseWhen(eq(updates.updateStatus, TaskStatus.Retrying), updates.updateSchedule).else(taskTable.scheduleTimestamp),
-                startTimestamp: null,
                 completeTimestamp: sql `${updates.updateComplete}`,
             })
                 .from(updates)
@@ -915,6 +902,16 @@ let PostgresTaskQueue = class PostgresTaskQueue extends TaskQueue {
             }
         });
     }
+    getFailureUpdate(tries, error, options) {
+        const isRetryable = (options?.fatal != true) && (tries < this.maxTries);
+        const status = isRetryable ? TaskStatus.Retrying : TaskStatus.Dead;
+        const delay = isRetryable
+            ? Math.min(this.retryDelayMaximum, this.retryDelayMinimum * (this.retryDelayGrowth ** (tries - 1)))
+            : 0;
+        const scheduleTimestamp = sql `(${TRANSACTION_TIMESTAMP} + ${interval(delay, 'milliseconds')})`;
+        const completeTimestamp = (status == TaskStatus.Dead) ? TRANSACTION_TIMESTAMP : null;
+        return { status, error: serializeError(error), scheduleTimestamp, completeTimestamp };
+    }
     async resolveDependencies(id, status, options) {
         await this.resolveDependenciesMany([{ id, status, namespace: options?.namespace }], options);
     }
@@ -1063,10 +1060,8 @@ let PostgresTaskQueue = class PostgresTaskQueue extends TaskQueue {
         }
     }
     async maintenance(options) {
+        await this.recoverStaleTasks(options);
         await this.processExpirations(options);
-        await this.processZombieRetries(options);
-        await this.processZombieExhaustions(options);
-        await this.processHardTimeouts(options);
         await this.processPriorityAging(options);
         await this.performArchival(options);
         await this.performArchivePurge(options);
@@ -1152,91 +1147,35 @@ let PostgresTaskQueue = class PostgresTaskQueue extends TaskQueue {
             }
         }
     }
-    async processZombieRetries(options) {
+    async recoverStaleTasks(options) {
         const session = options?.transaction?.pgTransaction ?? this.#repository.session;
-        const zombieRetrySelection = session.$with('zombie_retry_selection').as((qb) => qb
-            .select({ id: taskTable.id })
-            .from(taskTable)
-            .where(and(eq(taskTable.namespace, this.#namespace), eq(taskTable.status, TaskStatus.Running), lt(taskTable.visibilityDeadline, TRANSACTION_TIMESTAMP), lt(taskTable.tries, this.maxTries)))
-            .limit(1000)
-            .for('update', { skipLocked: true }));
-        while (true) {
-            const zombieRetryRows = await session
-                .with(zombieRetrySelection)
-                .update(taskTable)
-                .set({
-                status: TaskStatus.Retrying,
-                token: null,
-                visibilityDeadline: null,
-                startTimestamp: null,
-                scheduleTimestamp: sql `${TRANSACTION_TIMESTAMP} + ${interval(least(this.retryDelayMaximum, sql `${this.retryDelayMinimum} * ${power(this.retryDelayGrowth, sql `${taskTable.tries} - 1`)}`), 'milliseconds')}`,
-                error: jsonbBuildObject({ code: 'VisibilityTimeout', message: 'Worker Lost', lastError: taskTable.error }),
-            })
-                .where(inArray(taskTable.id, session.select().from(zombieRetrySelection)))
-                .returning({ id: taskTable.id });
-            if (zombieRetryRows.length < 1000) {
-                break;
-            }
-        }
-    }
-    async processZombieExhaustions(options) {
-        const zombieExhaustionSelection = this.#repository.session.$with('selection').as((qb) => qb
-            .select({ id: taskTable.id })
-            .from(taskTable)
-            .where(and(eq(taskTable.namespace, this.#namespace), eq(taskTable.status, TaskStatus.Running), lt(taskTable.visibilityDeadline, TRANSACTION_TIMESTAMP), gte(taskTable.tries, this.maxTries)))
-            .limit(1000)
-            .for('update', { skipLocked: true }));
-        while (true) {
-            const updatedCount = await this.#repository.useTransaction(options?.transaction, async (tx) => {
-                const exhaustionRows = await tx.pgTransaction
-                    .with(zombieExhaustionSelection)
-                    .update(taskTable)
-                    .set({
-                    status: TaskStatus.Orphaned,
-                    token: null,
-                    visibilityDeadline: null,
-                    completeTimestamp: TRANSACTION_TIMESTAMP,
-                    error: jsonbBuildObject({ code: 'ZombieExhausted', message: 'Exceeded max retries after repeated crashes', lastError: taskTable.error }),
-                })
-                    .where(inArray(taskTable.id, tx.pgTransaction.select().from(zombieExhaustionSelection)))
-                    .returning({ id: taskTable.id });
-                if (exhaustionRows.length > 0) {
-                    await this.resolveDependenciesMany(exhaustionRows.map((r) => ({ id: r.id, status: TaskStatus.Orphaned, namespace: this.#namespace })), { transaction: tx });
-                }
-                return exhaustionRows.length;
-            });
-            if (updatedCount < 1000) {
-                break;
-            }
-        }
-    }
-    async processHardTimeouts(options) {
-        const timeoutSelection = this.#repository.session.$with('selection').as((qb) => qb
+        const selection = session.$with('selection').as((qb) => qb
             .select({ id: taskTable.id })
             .from(taskTable)
-            .where(and(eq(taskTable.namespace, this.#namespace), eq(taskTable.status, TaskStatus.Running), lt(taskTable.startTimestamp, sql `${TRANSACTION_TIMESTAMP} - ${interval(this.maxExecutionTime, 'milliseconds')}`)))
+            .where(and(eq(taskTable.namespace, this.#namespace), eq(taskTable.status, TaskStatus.Running), or(lt(taskTable.visibilityDeadline, TRANSACTION_TIMESTAMP), lt(taskTable.startTimestamp, sql `${TRANSACTION_TIMESTAMP} - ${interval(this.maxExecutionTime, 'milliseconds')}`))))
             .limit(1000)
             .for('update', { skipLocked: true }));
         while (true) {
-            const updatedCount = await this.#repository.useTransaction(options?.transaction, async (tx) => {
-                const timeoutRows = await tx.pgTransaction
-                    .with(timeoutSelection)
+            const rows = await this.#repository.useTransaction(options?.transaction, async (tx) => {
+                const updatedRows = await tx.pgTransaction
+                    .with(selection)
                     .update(taskTable)
                     .set({
-                    status: TaskStatus.TimedOut,
+                    status: caseWhen(lt(taskTable.startTimestamp, sql `${TRANSACTION_TIMESTAMP} - ${interval(this.maxExecutionTime, 'milliseconds')}`), enumValue(TaskStatus, taskStatus, TaskStatus.TimedOut)).else(caseWhen(lt(taskTable.tries, this.maxTries), enumValue(TaskStatus, taskStatus, TaskStatus.Retrying)).else(enumValue(TaskStatus, taskStatus, TaskStatus.Orphaned))),
                     token: null,
                     visibilityDeadline: null,
-                    completeTimestamp: TRANSACTION_TIMESTAMP,
-                    error: jsonbBuildObject({ code: 'MaxTimeExceeded', message: sql `'Hard Execution Timeout: Task ran longer than ' || ${this.maxExecutionTime} || 'ms'`, lastError: taskTable.error }),
+                    completeTimestamp: caseWhen(or(lt(taskTable.startTimestamp, sql `${TRANSACTION_TIMESTAMP} - ${interval(this.maxExecutionTime, 'milliseconds')}`), gte(taskTable.tries, this.maxTries)), TRANSACTION_TIMESTAMP).else(null),
+                    scheduleTimestamp: caseWhen(and(lt(taskTable.visibilityDeadline, TRANSACTION_TIMESTAMP), lt(taskTable.tries, this.maxTries), not(lt(taskTable.startTimestamp, sql `${TRANSACTION_TIMESTAMP} - ${interval(this.maxExecutionTime, 'milliseconds')}`))), sql `${TRANSACTION_TIMESTAMP} + ${interval(least(this.retryDelayMaximum, sql `${this.retryDelayMinimum} * ${power(this.retryDelayGrowth, sql `${taskTable.tries} - 1`)}`), 'milliseconds')}`).else(taskTable.scheduleTimestamp),
+                    error: caseWhen(lt(taskTable.startTimestamp, sql `${TRANSACTION_TIMESTAMP} - ${interval(this.maxExecutionTime, 'milliseconds')}`), jsonbBuildObject({ code: 'MaxTimeExceeded', message: sql `'Hard Execution Timeout: Task ran longer than ' || ${this.maxExecutionTime} || 'ms'`, lastError: taskTable.error })).else(caseWhen(lt(taskTable.tries, this.maxTries), jsonbBuildObject({ code: 'VisibilityTimeout', message: 'Worker Lost', lastError: taskTable.error })).else(jsonbBuildObject({ code: 'ZombieExhausted', message: 'Exceeded max retries after repeated crashes', lastError: taskTable.error }))),
                 })
-                    .where(inArray(taskTable.id, tx.pgTransaction.select().from(timeoutSelection)))
-                    .returning({ id: taskTable.id });
-                if (timeoutRows.length > 0) {
-                    await this.resolveDependenciesMany(timeoutRows.map((r) => ({ id: r.id, status: TaskStatus.TimedOut, namespace: this.#namespace })), { transaction: tx });
+                    .where(inArray(taskTable.id, tx.pgTransaction.select().from(selection)))
+                    .returning({ id: taskTable.id, status: taskTable.status });
+                if (updatedRows.length > 0) {
+                    await this.resolveDependenciesMany(updatedRows.map((r) => ({ id: r.id, status: r.status, namespace: this.#namespace })), { transaction: tx });
                 }
-                return timeoutRows.length;
+                return updatedRows;
             });
-            if (updatedCount < 1000) {
+            if (rows.length < 1000) {
                 break;
             }
         }
@@ -1345,12 +1284,7 @@ let PostgresTaskQueue = class PostgresTaskQueue extends TaskQueue {
     async highFrequencyMaintenanceLoop() {
         while (this.#cancellationSignal.isUnset) {
             try {
-                const results = await Promise.allSettled([
-                    this.processZombieRetries(),
-                    this.processZombieExhaustions(),
-                    this.processHardTimeouts(),
-                ]);
-                this.logPromiseAllSettledErrors(results, 'high frequency maintenance loop');
+                await this.recoverStaleTasks();
             }
             catch (error) {
                 this.#logger.error('Error during high frequency maintenance loop', error);

package/task-queue/tests/optimization-edge-cases.test.js

@@ -117,7 +117,7 @@ describe('Task Queue Optimization Edge Cases', () => {
             tries: 2,
         })
             .where(and(eq(taskTable.id, task.id), eq(taskTable.namespace, namespace)));
-        await queue.processZombieRetries();
+        await queue.recoverStaleTasks();
         const updated = await queue.getTask(task.id);
         expect(updated.status).toBe(TaskStatus.Retrying);
         const delay = updated.scheduleTimestamp - Date.now();

package/task-queue/tests/queue.test.js

@@ -185,6 +185,42 @@ describe('PostgresQueue (Distributed Task Orchestration)', () => {
             expect(updated?.tries).toBe(1);
             expect(updated?.error).toBeDefined();
         });
+        it('should NOT clear startTimestamp when transitioning to terminal or retry states', async () => {
+            // Re-create queue with high circuit breaker threshold for this test
+            const q = injector.resolve(TaskQueueProvider).get(`start-timestamp-${crypto.randomUUID()}`, {
+                circuitBreakerThreshold: 10,
+            });
+            const task = await q.enqueue('foo', { foo: 'bar' });
+            // 1. Dequeue to start it (sets startTimestamp)
+            const dequeued = await q.dequeue();
+            expect(dequeued?.startTimestamp).not.toBeNull();
+            const startTimestamp = dequeued.startTimestamp;
+            // 2. Fail it (transitions to Retrying)
+            await q.fail(dequeued, new Error('temp failure'));
+            const retryingTask = await q.getTask(task.id);
+            expect(retryingTask?.status).toBe(TaskStatus.Retrying);
+            expect(retryingTask?.startTimestamp).toBe(startTimestamp);
+            // 3. Dequeue again (updates startTimestamp)
+            await q.reschedule(task.id, currentTimestamp());
+            const dequeued2 = await q.dequeue();
+            expect(dequeued2?.startTimestamp).not.toBe(startTimestamp);
+            const startTimestamp2 = dequeued2.startTimestamp;
+            // 4. Fail fatally (transitions to Dead)
+            await q.fail(dequeued2, new Error('fatal failure'), { fatal: true });
+            const deadTask = await q.getTask(task.id);
+            expect(deadTask?.status).toBe(TaskStatus.Dead);
+            expect(deadTask?.startTimestamp).toBe(startTimestamp2);
+            // 5. Success (transitions to Completed)
+            const task2 = await q.enqueue('foo', { foo: 'bar2' });
+            const dequeued3 = await q.dequeue();
+            expect(dequeued3).toBeDefined();
+            const startTimestamp3 = dequeued3.startTimestamp;
+            await q.complete(dequeued3);
+            const completedTask = await q.getTask(task2.id);
+            expect(completedTask?.status).toBe(TaskStatus.Completed);
+            expect(completedTask?.startTimestamp).toBe(startTimestamp3);
+            await q.clear();
+        });
     });
     describe('Hierarchy and Cross-Namespace', () => {
         it('should correctly increment parent unresolved dependencies when a child is spawned in a different namespace', async () => {
@@ -336,9 +372,11 @@ describe('PostgresQueue (Distributed Task Orchestration)', () => {
         });
     });
     describe('Timeouts and Maintenance (Pruning)', () => {
-        it('should recover "Zombie" tasks (crashed workers)', async () => {
+        it('should recover "Zombie" tasks (crashed workers) and preserve startTimestamp', async () => {
             const task = await queue.enqueue('foo', { foo: 'zombie' });
-            await queue.dequeue(); // Task is now Running with a token
+            const dequeued = await queue.dequeue(); // Task is now Running with a token
+            expect(dequeued?.startTimestamp).not.toBeNull();
+            const startTimestamp = dequeued.startTimestamp;
             // processTimeout is 50ms. Wait for it to expire.
             await timeout(100);
             await queue.maintenance();
@@ -346,6 +384,7 @@ describe('PostgresQueue (Distributed Task Orchestration)', () => {
             expect(recovered?.status).toBe(TaskStatus.Retrying);
             expect(recovered?.tries).toBe(1);
             expect(recovered?.token).toBeNull();
+            expect(recovered?.startTimestamp).toBe(startTimestamp);
         });
         it('should fail tasks that exceed Hard Execution Timeout via prune', async () => {
             // Re-configure queue with very short execution timeout