@tstdl/base 0.93.156 → 0.93.158
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/document-management/server/services/document-workflow.service.js +1 -1
- package/orm/server/repository.js +1 -1
- package/orm/server/transactional.d.ts +2 -0
- package/orm/server/transactional.js +4 -0
- package/orm/sqls/sqls.js +11 -2
- package/package.json +1 -1
- package/task-queue/postgres/task-queue.d.ts +3 -3
- package/task-queue/postgres/task-queue.js +49 -105
- package/task-queue/tests/optimization-edge-cases.test.js +1 -1
- package/task-queue/tests/queue.test.js +41 -2
|
@@ -42,7 +42,7 @@ let DocumentWorkflowService = DocumentWorkflowService_1 = class DocumentWorkflow
|
|
|
42
42
|
documentService = inject(forwardRef(() => DocumentService));
|
|
43
43
|
repository = injectRepository(DocumentWorkflow);
|
|
44
44
|
[afterResolve](_, { cancellationSignal }) {
|
|
45
|
-
if (this.
|
|
45
|
+
if (this.isFork) {
|
|
46
46
|
return;
|
|
47
47
|
}
|
|
48
48
|
this.#taskQueue.process({ concurrency: 5, cancellationSignal }, async (taskContext) => await this.processWorkflowJob(taskContext));
|
package/orm/server/repository.js
CHANGED
|
@@ -11,6 +11,7 @@ export type TransactionInitOptions = TransactionConfig & {
|
|
|
11
11
|
};
|
|
12
12
|
export type TransactionHandler<R> = (transaction: Transaction) => Promise<R>;
|
|
13
13
|
type TransactionalContext<ContextData = unknown> = {
|
|
14
|
+
isFork: boolean;
|
|
14
15
|
session: Database | PgTransaction;
|
|
15
16
|
instances: WeakMap<Type, WeakMap<Database | PgTransaction, any>>;
|
|
16
17
|
data: ContextData;
|
|
@@ -26,6 +27,7 @@ export declare abstract class Transactional<ContextData = unknown> {
|
|
|
26
27
|
protected transactionalContextData: ContextData | undefined;
|
|
27
28
|
readonly session: Database | PgTransaction;
|
|
28
29
|
readonly isInTransaction: boolean;
|
|
30
|
+
readonly isFork: boolean;
|
|
29
31
|
constructor();
|
|
30
32
|
/**
|
|
31
33
|
* Starts a new database transaction.
|
|
@@ -42,6 +42,7 @@ export class Transactional {
|
|
|
42
42
|
transactionalContextData = this.#context.data;
|
|
43
43
|
session = this.#context.session ?? inject(Database);
|
|
44
44
|
isInTransaction = this.session instanceof DrizzlePgTransaction;
|
|
45
|
+
isFork = (this.#context.isFork ?? false) || this.isInTransaction;
|
|
45
46
|
constructor() {
|
|
46
47
|
this.#classConstructor = new.target;
|
|
47
48
|
}
|
|
@@ -77,6 +78,7 @@ export class Transactional {
|
|
|
77
78
|
return this;
|
|
78
79
|
}
|
|
79
80
|
const context = {
|
|
81
|
+
isFork: true,
|
|
80
82
|
session,
|
|
81
83
|
instances: this.#instances,
|
|
82
84
|
data: this.getTransactionalContextData(),
|
|
@@ -119,6 +121,7 @@ export class Transactional {
|
|
|
119
121
|
return await this.transaction(handler);
|
|
120
122
|
}
|
|
121
123
|
const context = {
|
|
124
|
+
isFork: true,
|
|
122
125
|
session: existingTransaction.pgTransaction,
|
|
123
126
|
instances: this.#instances,
|
|
124
127
|
data: this.getTransactionalContextData(),
|
|
@@ -139,6 +142,7 @@ export class Transactional {
|
|
|
139
142
|
async transaction(handler, config) {
|
|
140
143
|
const transaction = await this.startTransaction(config);
|
|
141
144
|
const context = {
|
|
145
|
+
isFork: true,
|
|
142
146
|
session: transaction.pgTransaction,
|
|
143
147
|
instances: this.#instances,
|
|
144
148
|
data: this.getTransactionalContextData(),
|
package/orm/sqls/sqls.js
CHANGED
|
@@ -8,7 +8,7 @@ import { and, Column, eq, isSQLWrapper, sql, isNotNull as sqlIsNotNull, isNull a
|
|
|
8
8
|
import { match, P } from 'ts-pattern';
|
|
9
9
|
import { distinct, toArray } from '../../utils/array/array.js';
|
|
10
10
|
import { objectEntries, objectValues } from '../../utils/object/object.js';
|
|
11
|
-
import { assertDefined, isArray, isBoolean, isDefined, isInstanceOf, isNotNull, isNull, isNullOrUndefined, isNumber,
|
|
11
|
+
import { assertDefined, isArray, isBoolean, isDefined, isInstanceOf, isLiteralObject, isNotNull, isNull, isNullOrUndefined, isNumber, isString } from '../../utils/type-guards.js';
|
|
12
12
|
import { getEnumName } from '../enums.js';
|
|
13
13
|
import { caseWhen } from './case-when.js';
|
|
14
14
|
const isJsonbSymbol = Symbol('isJsonb');
|
|
@@ -454,8 +454,17 @@ export function buildJsonb(value) {
|
|
|
454
454
|
const elements = value.map((inner) => buildJsonb(inner));
|
|
455
455
|
return markAsJsonb(sql `jsonb_build_array(${sql.join(elements, sql `, `)})`);
|
|
456
456
|
}
|
|
457
|
-
if (
|
|
457
|
+
if (isLiteralObject(value)) {
|
|
458
458
|
return jsonbBuildObject(value);
|
|
459
459
|
}
|
|
460
|
+
if (isString(value)) {
|
|
461
|
+
return markAsJsonb(sql `to_jsonb(${value}::text)`);
|
|
462
|
+
}
|
|
463
|
+
if (isNumber(value)) {
|
|
464
|
+
return markAsJsonb(sql `to_jsonb(${value}::numeric)`);
|
|
465
|
+
}
|
|
466
|
+
if (isBoolean(value)) {
|
|
467
|
+
return markAsJsonb(sql `to_jsonb(${value}::boolean)`);
|
|
468
|
+
}
|
|
460
469
|
return markAsJsonb(sql `to_jsonb(${value})`);
|
|
461
470
|
}
|
package/package.json
CHANGED
|
@@ -110,6 +110,7 @@ export declare class PostgresTaskQueue<Definitions extends TaskDefinitionMap = T
|
|
|
110
110
|
failMany(tasks: Task<Definitions>[], errors: unknown[], options?: {
|
|
111
111
|
transaction?: Transaction;
|
|
112
112
|
}): Promise<void>;
|
|
113
|
+
private getFailureUpdate;
|
|
113
114
|
private resolveDependencies;
|
|
114
115
|
private resolveDependenciesMany;
|
|
115
116
|
maintenance(options?: {
|
|
@@ -118,9 +119,7 @@ export declare class PostgresTaskQueue<Definitions extends TaskDefinitionMap = T
|
|
|
118
119
|
private performArchival;
|
|
119
120
|
private performArchivePurge;
|
|
120
121
|
private processExpirations;
|
|
121
|
-
private
|
|
122
|
-
private processZombieExhaustions;
|
|
123
|
-
private processHardTimeouts;
|
|
122
|
+
private recoverStaleTasks;
|
|
124
123
|
private processPriorityAging;
|
|
125
124
|
restart(id: string, options?: {
|
|
126
125
|
resetState?: boolean;
|
|
@@ -135,6 +134,7 @@ export declare class PostgresTaskQueue<Definitions extends TaskDefinitionMap = T
|
|
|
135
134
|
private lowFrequencyMaintenanceLoop;
|
|
136
135
|
private mediumFrequencyMaintenanceLoop;
|
|
137
136
|
private highFrequencyMaintenanceLoop;
|
|
137
|
+
private logPromiseAllSettledErrors;
|
|
138
138
|
getBatchConsumer<Type extends TaskTypes<Definitions>>(size: number, cancellationSignal: CancellationSignal, options?: {
|
|
139
139
|
forceDequeue?: boolean;
|
|
140
140
|
types?: Type[];
|
|
@@ -56,7 +56,7 @@ var __disposeResources = (this && this.__disposeResources) || (function (Suppres
|
|
|
56
56
|
var e = new Error(message);
|
|
57
57
|
return e.name = "SuppressedError", e.error = error, e.suppressed = suppressed, e;
|
|
58
58
|
});
|
|
59
|
-
import { aliasedTable, and, asc, count, eq, gt, gte, inArray, lt, lte, notExists, notInArray, or, sql, count as sqlCount, isNotNull as sqlIsNotNull, isNull as sqlIsNull } from 'drizzle-orm';
|
|
59
|
+
import { aliasedTable, and, asc, count, eq, gt, gte, inArray, lt, lte, not, notExists, notInArray, or, sql, count as sqlCount, isNotNull as sqlIsNotNull, isNull as sqlIsNull } from 'drizzle-orm';
|
|
60
60
|
import { filter, merge, throttleTime } from 'rxjs';
|
|
61
61
|
import { CancellationSignal } from '../../cancellation/index.js';
|
|
62
62
|
import { CircuitBreaker, CircuitBreakerState } from '../../circuit-breaker/index.js';
|
|
@@ -137,7 +137,7 @@ let PostgresTaskQueue = class PostgresTaskQueue extends TaskQueue {
|
|
|
137
137
|
result: null,
|
|
138
138
|
};
|
|
139
139
|
[afterResolve]() {
|
|
140
|
-
if (!this.
|
|
140
|
+
if (!this.isFork) {
|
|
141
141
|
this.maintenanceLoop();
|
|
142
142
|
}
|
|
143
143
|
}
|
|
@@ -837,23 +837,17 @@ let PostgresTaskQueue = class PostgresTaskQueue extends TaskQueue {
|
|
|
837
837
|
}
|
|
838
838
|
async fail(task, caughtError, options) {
|
|
839
839
|
const error = ensureTaskError(caughtError);
|
|
840
|
-
const
|
|
841
|
-
const nextStatus = isRetryable ? TaskStatus.Retrying : TaskStatus.Dead;
|
|
842
|
-
const delay = isRetryable
|
|
843
|
-
? Math.min(this.retryDelayMaximum, this.retryDelayMinimum * (this.retryDelayGrowth ** (task.tries - 1)))
|
|
844
|
-
: 0;
|
|
845
|
-
const nextSchedule = sql `${TRANSACTION_TIMESTAMP} + ${interval(delay, 'milliseconds')}`;
|
|
840
|
+
const update = this.getFailureUpdate(task.tries, error, options);
|
|
846
841
|
await this.#repository.useTransaction(options?.transaction, async (tx) => {
|
|
847
842
|
const [updatedRow] = await tx.pgTransaction
|
|
848
843
|
.update(taskTable)
|
|
849
844
|
.set({
|
|
850
|
-
status:
|
|
845
|
+
status: update.status,
|
|
851
846
|
token: null,
|
|
852
|
-
error:
|
|
847
|
+
error: update.error,
|
|
853
848
|
visibilityDeadline: null,
|
|
854
|
-
scheduleTimestamp:
|
|
855
|
-
|
|
856
|
-
completeTimestamp: (nextStatus == TaskStatus.Dead) ? TRANSACTION_TIMESTAMP : null,
|
|
849
|
+
scheduleTimestamp: (update.status == TaskStatus.Retrying) ? update.scheduleTimestamp : taskTable.scheduleTimestamp,
|
|
850
|
+
completeTimestamp: update.completeTimestamp,
|
|
857
851
|
})
|
|
858
852
|
.where(and(eq(taskTable.id, task.id), isNull(task.token) ? sqlIsNull(taskTable.token) : eq(taskTable.token, task.token), eq(taskTable.tries, task.tries)))
|
|
859
853
|
.returning();
|
|
@@ -861,7 +855,7 @@ let PostgresTaskQueue = class PostgresTaskQueue extends TaskQueue {
|
|
|
861
855
|
return;
|
|
862
856
|
}
|
|
863
857
|
await this.#circuitBreaker.recordFailure();
|
|
864
|
-
await this.resolveDependencies(task.id,
|
|
858
|
+
await this.resolveDependencies(task.id, update.status, { namespace: task.namespace, transaction: tx });
|
|
865
859
|
});
|
|
866
860
|
}
|
|
867
861
|
async failMany(tasks, errors, options) {
|
|
@@ -871,14 +865,8 @@ let PostgresTaskQueue = class PostgresTaskQueue extends TaskQueue {
|
|
|
871
865
|
await this.#repository.useTransaction(options?.transaction, async (tx) => {
|
|
872
866
|
const rows = tasks.map((task, index) => {
|
|
873
867
|
const error = ensureTaskError(errors[index]);
|
|
874
|
-
const
|
|
875
|
-
|
|
876
|
-
const delay = isRetryable
|
|
877
|
-
? Math.min(this.retryDelayMaximum, this.retryDelayMinimum * (this.retryDelayGrowth ** (task.tries - 1)))
|
|
878
|
-
: 0;
|
|
879
|
-
const nextSchedule = sql `(${TRANSACTION_TIMESTAMP} + ${interval(delay, 'milliseconds')})`;
|
|
880
|
-
const completeTimestamp = (nextStatus == TaskStatus.Dead) ? TRANSACTION_TIMESTAMP : null;
|
|
881
|
-
return sql `(${task.id}::uuid, ${task.token}::uuid, ${task.tries}::int, ${nextStatus}::${taskStatus}, ${serializeError(error)}::jsonb, ${nextSchedule}::timestamptz, ${completeTimestamp}::timestamptz)`;
|
|
868
|
+
const update = this.getFailureUpdate(task.tries, error);
|
|
869
|
+
return sql `(${task.id}::uuid, ${task.token}::uuid, ${task.tries}::int, ${update.status}::${taskStatus}, ${update.error}::jsonb, ${update.scheduleTimestamp}::timestamptz, ${update.completeTimestamp}::timestamptz)`;
|
|
882
870
|
});
|
|
883
871
|
const updates = tx.pgTransaction.$with('updates').as((qb) => qb
|
|
884
872
|
.select({
|
|
@@ -899,7 +887,6 @@ let PostgresTaskQueue = class PostgresTaskQueue extends TaskQueue {
|
|
|
899
887
|
error: sql `${updates.updateError}`,
|
|
900
888
|
visibilityDeadline: null,
|
|
901
889
|
scheduleTimestamp: caseWhen(eq(updates.updateStatus, TaskStatus.Retrying), updates.updateSchedule).else(taskTable.scheduleTimestamp),
|
|
902
|
-
startTimestamp: null,
|
|
903
890
|
completeTimestamp: sql `${updates.updateComplete}`,
|
|
904
891
|
})
|
|
905
892
|
.from(updates)
|
|
@@ -915,6 +902,16 @@ let PostgresTaskQueue = class PostgresTaskQueue extends TaskQueue {
|
|
|
915
902
|
}
|
|
916
903
|
});
|
|
917
904
|
}
|
|
905
|
+
getFailureUpdate(tries, error, options) {
|
|
906
|
+
const isRetryable = (options?.fatal != true) && (tries < this.maxTries);
|
|
907
|
+
const status = isRetryable ? TaskStatus.Retrying : TaskStatus.Dead;
|
|
908
|
+
const delay = isRetryable
|
|
909
|
+
? Math.min(this.retryDelayMaximum, this.retryDelayMinimum * (this.retryDelayGrowth ** (tries - 1)))
|
|
910
|
+
: 0;
|
|
911
|
+
const scheduleTimestamp = sql `(${TRANSACTION_TIMESTAMP} + ${interval(delay, 'milliseconds')})`;
|
|
912
|
+
const completeTimestamp = (status == TaskStatus.Dead) ? TRANSACTION_TIMESTAMP : null;
|
|
913
|
+
return { status, error: serializeError(error), scheduleTimestamp, completeTimestamp };
|
|
914
|
+
}
|
|
918
915
|
async resolveDependencies(id, status, options) {
|
|
919
916
|
await this.resolveDependenciesMany([{ id, status, namespace: options?.namespace }], options);
|
|
920
917
|
}
|
|
@@ -1063,10 +1060,8 @@ let PostgresTaskQueue = class PostgresTaskQueue extends TaskQueue {
|
|
|
1063
1060
|
}
|
|
1064
1061
|
}
|
|
1065
1062
|
async maintenance(options) {
|
|
1063
|
+
await this.recoverStaleTasks(options);
|
|
1066
1064
|
await this.processExpirations(options);
|
|
1067
|
-
await this.processZombieRetries(options);
|
|
1068
|
-
await this.processZombieExhaustions(options);
|
|
1069
|
-
await this.processHardTimeouts(options);
|
|
1070
1065
|
await this.processPriorityAging(options);
|
|
1071
1066
|
await this.performArchival(options);
|
|
1072
1067
|
await this.performArchivePurge(options);
|
|
@@ -1152,91 +1147,35 @@ let PostgresTaskQueue = class PostgresTaskQueue extends TaskQueue {
|
|
|
1152
1147
|
}
|
|
1153
1148
|
}
|
|
1154
1149
|
}
|
|
1155
|
-
async
|
|
1150
|
+
async recoverStaleTasks(options) {
|
|
1156
1151
|
const session = options?.transaction?.pgTransaction ?? this.#repository.session;
|
|
1157
|
-
const
|
|
1158
|
-
.select({ id: taskTable.id })
|
|
1159
|
-
.from(taskTable)
|
|
1160
|
-
.where(and(eq(taskTable.namespace, this.#namespace), eq(taskTable.status, TaskStatus.Running), lt(taskTable.visibilityDeadline, TRANSACTION_TIMESTAMP), lt(taskTable.tries, this.maxTries)))
|
|
1161
|
-
.limit(1000)
|
|
1162
|
-
.for('update', { skipLocked: true }));
|
|
1163
|
-
while (true) {
|
|
1164
|
-
const zombieRetryRows = await session
|
|
1165
|
-
.with(zombieRetrySelection)
|
|
1166
|
-
.update(taskTable)
|
|
1167
|
-
.set({
|
|
1168
|
-
status: TaskStatus.Retrying,
|
|
1169
|
-
token: null,
|
|
1170
|
-
visibilityDeadline: null,
|
|
1171
|
-
startTimestamp: null,
|
|
1172
|
-
scheduleTimestamp: sql `${TRANSACTION_TIMESTAMP} + ${interval(least(this.retryDelayMaximum, sql `${this.retryDelayMinimum} * ${power(this.retryDelayGrowth, sql `${taskTable.tries} - 1`)}`), 'milliseconds')}`,
|
|
1173
|
-
error: jsonbBuildObject({ code: 'VisibilityTimeout', message: 'Worker Lost', lastError: taskTable.error }),
|
|
1174
|
-
})
|
|
1175
|
-
.where(inArray(taskTable.id, session.select().from(zombieRetrySelection)))
|
|
1176
|
-
.returning({ id: taskTable.id });
|
|
1177
|
-
if (zombieRetryRows.length < 1000) {
|
|
1178
|
-
break;
|
|
1179
|
-
}
|
|
1180
|
-
}
|
|
1181
|
-
}
|
|
1182
|
-
async processZombieExhaustions(options) {
|
|
1183
|
-
const zombieExhaustionSelection = this.#repository.session.$with('selection').as((qb) => qb
|
|
1184
|
-
.select({ id: taskTable.id })
|
|
1185
|
-
.from(taskTable)
|
|
1186
|
-
.where(and(eq(taskTable.namespace, this.#namespace), eq(taskTable.status, TaskStatus.Running), lt(taskTable.visibilityDeadline, TRANSACTION_TIMESTAMP), gte(taskTable.tries, this.maxTries)))
|
|
1187
|
-
.limit(1000)
|
|
1188
|
-
.for('update', { skipLocked: true }));
|
|
1189
|
-
while (true) {
|
|
1190
|
-
const updatedCount = await this.#repository.useTransaction(options?.transaction, async (tx) => {
|
|
1191
|
-
const exhaustionRows = await tx.pgTransaction
|
|
1192
|
-
.with(zombieExhaustionSelection)
|
|
1193
|
-
.update(taskTable)
|
|
1194
|
-
.set({
|
|
1195
|
-
status: TaskStatus.Orphaned,
|
|
1196
|
-
token: null,
|
|
1197
|
-
visibilityDeadline: null,
|
|
1198
|
-
completeTimestamp: TRANSACTION_TIMESTAMP,
|
|
1199
|
-
error: jsonbBuildObject({ code: 'ZombieExhausted', message: 'Exceeded max retries after repeated crashes', lastError: taskTable.error }),
|
|
1200
|
-
})
|
|
1201
|
-
.where(inArray(taskTable.id, tx.pgTransaction.select().from(zombieExhaustionSelection)))
|
|
1202
|
-
.returning({ id: taskTable.id });
|
|
1203
|
-
if (exhaustionRows.length > 0) {
|
|
1204
|
-
await this.resolveDependenciesMany(exhaustionRows.map((r) => ({ id: r.id, status: TaskStatus.Orphaned, namespace: this.#namespace })), { transaction: tx });
|
|
1205
|
-
}
|
|
1206
|
-
return exhaustionRows.length;
|
|
1207
|
-
});
|
|
1208
|
-
if (updatedCount < 1000) {
|
|
1209
|
-
break;
|
|
1210
|
-
}
|
|
1211
|
-
}
|
|
1212
|
-
}
|
|
1213
|
-
async processHardTimeouts(options) {
|
|
1214
|
-
const timeoutSelection = this.#repository.session.$with('selection').as((qb) => qb
|
|
1152
|
+
const selection = session.$with('selection').as((qb) => qb
|
|
1215
1153
|
.select({ id: taskTable.id })
|
|
1216
1154
|
.from(taskTable)
|
|
1217
|
-
.where(and(eq(taskTable.namespace, this.#namespace), eq(taskTable.status, TaskStatus.Running), lt(taskTable.startTimestamp, sql `${TRANSACTION_TIMESTAMP} - ${interval(this.maxExecutionTime, 'milliseconds')}`)))
|
|
1155
|
+
.where(and(eq(taskTable.namespace, this.#namespace), eq(taskTable.status, TaskStatus.Running), or(lt(taskTable.visibilityDeadline, TRANSACTION_TIMESTAMP), lt(taskTable.startTimestamp, sql `${TRANSACTION_TIMESTAMP} - ${interval(this.maxExecutionTime, 'milliseconds')}`))))
|
|
1218
1156
|
.limit(1000)
|
|
1219
1157
|
.for('update', { skipLocked: true }));
|
|
1220
1158
|
while (true) {
|
|
1221
|
-
const
|
|
1222
|
-
const
|
|
1223
|
-
.with(
|
|
1159
|
+
const rows = await this.#repository.useTransaction(options?.transaction, async (tx) => {
|
|
1160
|
+
const updatedRows = await tx.pgTransaction
|
|
1161
|
+
.with(selection)
|
|
1224
1162
|
.update(taskTable)
|
|
1225
1163
|
.set({
|
|
1226
|
-
status: TaskStatus.TimedOut,
|
|
1164
|
+
status: caseWhen(lt(taskTable.startTimestamp, sql `${TRANSACTION_TIMESTAMP} - ${interval(this.maxExecutionTime, 'milliseconds')}`), enumValue(TaskStatus, taskStatus, TaskStatus.TimedOut)).else(caseWhen(lt(taskTable.tries, this.maxTries), enumValue(TaskStatus, taskStatus, TaskStatus.Retrying)).else(enumValue(TaskStatus, taskStatus, TaskStatus.Orphaned))),
|
|
1227
1165
|
token: null,
|
|
1228
1166
|
visibilityDeadline: null,
|
|
1229
|
-
completeTimestamp: TRANSACTION_TIMESTAMP,
|
|
1230
|
-
|
|
1167
|
+
completeTimestamp: caseWhen(or(lt(taskTable.startTimestamp, sql `${TRANSACTION_TIMESTAMP} - ${interval(this.maxExecutionTime, 'milliseconds')}`), gte(taskTable.tries, this.maxTries)), TRANSACTION_TIMESTAMP).else(null),
|
|
1168
|
+
scheduleTimestamp: caseWhen(and(lt(taskTable.visibilityDeadline, TRANSACTION_TIMESTAMP), lt(taskTable.tries, this.maxTries), not(lt(taskTable.startTimestamp, sql `${TRANSACTION_TIMESTAMP} - ${interval(this.maxExecutionTime, 'milliseconds')}`))), sql `${TRANSACTION_TIMESTAMP} + ${interval(least(this.retryDelayMaximum, sql `${this.retryDelayMinimum} * ${power(this.retryDelayGrowth, sql `${taskTable.tries} - 1`)}`), 'milliseconds')}`).else(taskTable.scheduleTimestamp),
|
|
1169
|
+
error: caseWhen(lt(taskTable.startTimestamp, sql `${TRANSACTION_TIMESTAMP} - ${interval(this.maxExecutionTime, 'milliseconds')}`), jsonbBuildObject({ code: 'MaxTimeExceeded', message: sql `'Hard Execution Timeout: Task ran longer than ' || ${this.maxExecutionTime} || 'ms'`, lastError: taskTable.error })).else(caseWhen(lt(taskTable.tries, this.maxTries), jsonbBuildObject({ code: 'VisibilityTimeout', message: 'Worker Lost', lastError: taskTable.error })).else(jsonbBuildObject({ code: 'ZombieExhausted', message: 'Exceeded max retries after repeated crashes', lastError: taskTable.error }))),
|
|
1231
1170
|
})
|
|
1232
|
-
.where(inArray(taskTable.id, tx.pgTransaction.select().from(
|
|
1233
|
-
.returning({ id: taskTable.id });
|
|
1234
|
-
if (
|
|
1235
|
-
await this.resolveDependenciesMany(
|
|
1171
|
+
.where(inArray(taskTable.id, tx.pgTransaction.select().from(selection)))
|
|
1172
|
+
.returning({ id: taskTable.id, status: taskTable.status });
|
|
1173
|
+
if (updatedRows.length > 0) {
|
|
1174
|
+
await this.resolveDependenciesMany(updatedRows.map((r) => ({ id: r.id, status: r.status, namespace: this.#namespace })), { transaction: tx });
|
|
1236
1175
|
}
|
|
1237
|
-
return
|
|
1176
|
+
return updatedRows;
|
|
1238
1177
|
});
|
|
1239
|
-
if (
|
|
1178
|
+
if (rows.length < 1000) {
|
|
1240
1179
|
break;
|
|
1241
1180
|
}
|
|
1242
1181
|
}
|
|
@@ -1311,10 +1250,11 @@ let PostgresTaskQueue = class PostgresTaskQueue extends TaskQueue {
|
|
|
1311
1250
|
async lowFrequencyMaintenanceLoop() {
|
|
1312
1251
|
while (this.#cancellationSignal.isUnset) {
|
|
1313
1252
|
try {
|
|
1314
|
-
await Promise.allSettled([
|
|
1253
|
+
const results = await Promise.allSettled([
|
|
1315
1254
|
this.performArchival(),
|
|
1316
1255
|
this.performArchivePurge(),
|
|
1317
1256
|
]);
|
|
1257
|
+
this.logPromiseAllSettledErrors(results, 'low frequency maintenance loop');
|
|
1318
1258
|
}
|
|
1319
1259
|
catch (error) {
|
|
1320
1260
|
this.#logger.error('Error during low frequency maintenance loop', error);
|
|
@@ -1327,10 +1267,11 @@ let PostgresTaskQueue = class PostgresTaskQueue extends TaskQueue {
|
|
|
1327
1267
|
async mediumFrequencyMaintenanceLoop() {
|
|
1328
1268
|
while (this.#cancellationSignal.isUnset) {
|
|
1329
1269
|
try {
|
|
1330
|
-
await Promise.allSettled([
|
|
1270
|
+
const results = await Promise.allSettled([
|
|
1331
1271
|
this.processExpirations(),
|
|
1332
1272
|
this.processPriorityAging(),
|
|
1333
1273
|
]);
|
|
1274
|
+
this.logPromiseAllSettledErrors(results, 'medium frequency maintenance loop');
|
|
1334
1275
|
}
|
|
1335
1276
|
catch (error) {
|
|
1336
1277
|
this.#logger.error('Error during medium frequency maintenance loop', error);
|
|
@@ -1343,11 +1284,7 @@ let PostgresTaskQueue = class PostgresTaskQueue extends TaskQueue {
|
|
|
1343
1284
|
async highFrequencyMaintenanceLoop() {
|
|
1344
1285
|
while (this.#cancellationSignal.isUnset) {
|
|
1345
1286
|
try {
|
|
1346
|
-
await
|
|
1347
|
-
this.processZombieRetries(),
|
|
1348
|
-
this.processZombieExhaustions(),
|
|
1349
|
-
this.processHardTimeouts(),
|
|
1350
|
-
]);
|
|
1287
|
+
await this.recoverStaleTasks();
|
|
1351
1288
|
}
|
|
1352
1289
|
catch (error) {
|
|
1353
1290
|
this.#logger.error('Error during high frequency maintenance loop', error);
|
|
@@ -1357,6 +1294,13 @@ let PostgresTaskQueue = class PostgresTaskQueue extends TaskQueue {
|
|
|
1357
1294
|
}
|
|
1358
1295
|
}
|
|
1359
1296
|
}
|
|
1297
|
+
logPromiseAllSettledErrors(results, context) {
|
|
1298
|
+
for (const result of results) {
|
|
1299
|
+
if (result.status == 'rejected') {
|
|
1300
|
+
this.#logger.error(`Error during ${context}`, result.reason);
|
|
1301
|
+
}
|
|
1302
|
+
}
|
|
1303
|
+
}
|
|
1360
1304
|
async *getBatchConsumer(size, cancellationSignal, options) {
|
|
1361
1305
|
const continue$ = merge(this.#messageBus.allMessages$).pipe(filter((namespace) => namespace == this.#namespace));
|
|
1362
1306
|
const mergedContinue$ = merge(continue$, cancellationSignal);
|
|
@@ -117,7 +117,7 @@ describe('Task Queue Optimization Edge Cases', () => {
|
|
|
117
117
|
tries: 2,
|
|
118
118
|
})
|
|
119
119
|
.where(and(eq(taskTable.id, task.id), eq(taskTable.namespace, namespace)));
|
|
120
|
-
await queue.
|
|
120
|
+
await queue.recoverStaleTasks();
|
|
121
121
|
const updated = await queue.getTask(task.id);
|
|
122
122
|
expect(updated.status).toBe(TaskStatus.Retrying);
|
|
123
123
|
const delay = updated.scheduleTimestamp - Date.now();
|
|
@@ -185,6 +185,42 @@ describe('PostgresQueue (Distributed Task Orchestration)', () => {
|
|
|
185
185
|
expect(updated?.tries).toBe(1);
|
|
186
186
|
expect(updated?.error).toBeDefined();
|
|
187
187
|
});
|
|
188
|
+
it('should NOT clear startTimestamp when transitioning to terminal or retry states', async () => {
|
|
189
|
+
// Re-create queue with high circuit breaker threshold for this test
|
|
190
|
+
const q = injector.resolve(TaskQueueProvider).get(`start-timestamp-${crypto.randomUUID()}`, {
|
|
191
|
+
circuitBreakerThreshold: 10,
|
|
192
|
+
});
|
|
193
|
+
const task = await q.enqueue('foo', { foo: 'bar' });
|
|
194
|
+
// 1. Dequeue to start it (sets startTimestamp)
|
|
195
|
+
const dequeued = await q.dequeue();
|
|
196
|
+
expect(dequeued?.startTimestamp).not.toBeNull();
|
|
197
|
+
const startTimestamp = dequeued.startTimestamp;
|
|
198
|
+
// 2. Fail it (transitions to Retrying)
|
|
199
|
+
await q.fail(dequeued, new Error('temp failure'));
|
|
200
|
+
const retryingTask = await q.getTask(task.id);
|
|
201
|
+
expect(retryingTask?.status).toBe(TaskStatus.Retrying);
|
|
202
|
+
expect(retryingTask?.startTimestamp).toBe(startTimestamp);
|
|
203
|
+
// 3. Dequeue again (updates startTimestamp)
|
|
204
|
+
await q.reschedule(task.id, currentTimestamp());
|
|
205
|
+
const dequeued2 = await q.dequeue();
|
|
206
|
+
expect(dequeued2?.startTimestamp).not.toBe(startTimestamp);
|
|
207
|
+
const startTimestamp2 = dequeued2.startTimestamp;
|
|
208
|
+
// 4. Fail fatally (transitions to Dead)
|
|
209
|
+
await q.fail(dequeued2, new Error('fatal failure'), { fatal: true });
|
|
210
|
+
const deadTask = await q.getTask(task.id);
|
|
211
|
+
expect(deadTask?.status).toBe(TaskStatus.Dead);
|
|
212
|
+
expect(deadTask?.startTimestamp).toBe(startTimestamp2);
|
|
213
|
+
// 5. Success (transitions to Completed)
|
|
214
|
+
const task2 = await q.enqueue('foo', { foo: 'bar2' });
|
|
215
|
+
const dequeued3 = await q.dequeue();
|
|
216
|
+
expect(dequeued3).toBeDefined();
|
|
217
|
+
const startTimestamp3 = dequeued3.startTimestamp;
|
|
218
|
+
await q.complete(dequeued3);
|
|
219
|
+
const completedTask = await q.getTask(task2.id);
|
|
220
|
+
expect(completedTask?.status).toBe(TaskStatus.Completed);
|
|
221
|
+
expect(completedTask?.startTimestamp).toBe(startTimestamp3);
|
|
222
|
+
await q.clear();
|
|
223
|
+
});
|
|
188
224
|
});
|
|
189
225
|
describe('Hierarchy and Cross-Namespace', () => {
|
|
190
226
|
it('should correctly increment parent unresolved dependencies when a child is spawned in a different namespace', async () => {
|
|
@@ -336,9 +372,11 @@ describe('PostgresQueue (Distributed Task Orchestration)', () => {
|
|
|
336
372
|
});
|
|
337
373
|
});
|
|
338
374
|
describe('Timeouts and Maintenance (Pruning)', () => {
|
|
339
|
-
it('should recover "Zombie" tasks (crashed workers)', async () => {
|
|
375
|
+
it('should recover "Zombie" tasks (crashed workers) and preserve startTimestamp', async () => {
|
|
340
376
|
const task = await queue.enqueue('foo', { foo: 'zombie' });
|
|
341
|
-
await queue.dequeue(); // Task is now Running with a token
|
|
377
|
+
const dequeued = await queue.dequeue(); // Task is now Running with a token
|
|
378
|
+
expect(dequeued?.startTimestamp).not.toBeNull();
|
|
379
|
+
const startTimestamp = dequeued.startTimestamp;
|
|
342
380
|
// processTimeout is 50ms. Wait for it to expire.
|
|
343
381
|
await timeout(100);
|
|
344
382
|
await queue.maintenance();
|
|
@@ -346,6 +384,7 @@ describe('PostgresQueue (Distributed Task Orchestration)', () => {
|
|
|
346
384
|
expect(recovered?.status).toBe(TaskStatus.Retrying);
|
|
347
385
|
expect(recovered?.tries).toBe(1);
|
|
348
386
|
expect(recovered?.token).toBeNull();
|
|
387
|
+
expect(recovered?.startTimestamp).toBe(startTimestamp);
|
|
349
388
|
});
|
|
350
389
|
it('should fail tasks that exceed Hard Execution Timeout via prune', async () => {
|
|
351
390
|
// Re-configure queue with very short execution timeout
|