@tstdl/base 0.93.141 → 0.93.142
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/authentication/client/authentication.service.d.ts +1 -0
- package/authentication/client/authentication.service.js +3 -2
- package/circuit-breaker/circuit-breaker.d.ts +6 -4
- package/circuit-breaker/postgres/circuit-breaker.d.ts +1 -0
- package/circuit-breaker/postgres/circuit-breaker.js +8 -5
- package/circuit-breaker/tests/circuit-breaker.test.js +20 -0
- package/examples/document-management/main.js +2 -2
- package/notification/tests/notification-api.test.js +5 -1
- package/notification/tests/notification-flow.test.js +9 -6
- package/orm/decorators.d.ts +17 -4
- package/orm/decorators.js +9 -0
- package/orm/server/bootstrap.d.ts +11 -0
- package/orm/server/bootstrap.js +31 -0
- package/orm/server/drizzle/schema-converter.d.ts +3 -1
- package/orm/server/drizzle/schema-converter.js +71 -29
- package/orm/server/extension.d.ts +14 -0
- package/orm/server/extension.js +27 -0
- package/orm/server/index.d.ts +2 -0
- package/orm/server/index.js +2 -0
- package/orm/server/migration.d.ts +2 -3
- package/orm/server/migration.js +7 -21
- package/orm/server/repository.d.ts +1 -0
- package/orm/server/repository.js +19 -9
- package/orm/server/transaction.d.ts +1 -0
- package/orm/server/transaction.js +3 -0
- package/orm/tests/database-extension.test.js +63 -0
- package/orm/tests/database-migration.test.js +7 -6
- package/orm/tests/repository-compound-primary-key.test.d.ts +2 -0
- package/orm/tests/repository-compound-primary-key.test.js +234 -0
- package/orm/tests/schema-generation.test.d.ts +1 -0
- package/orm/tests/schema-generation.test.js +52 -5
- package/package.json +4 -4
- package/task-queue/README.md +0 -1
- package/task-queue/postgres/drizzle/0000_great_gwen_stacy.sql +84 -0
- package/task-queue/postgres/drizzle/meta/0000_snapshot.json +151 -68
- package/task-queue/postgres/drizzle/meta/_journal.json +2 -2
- package/task-queue/postgres/module.js +2 -1
- package/task-queue/postgres/schemas.d.ts +6 -0
- package/task-queue/postgres/task-queue.d.ts +18 -5
- package/task-queue/postgres/task-queue.js +593 -372
- package/task-queue/postgres/task.model.d.ts +9 -5
- package/task-queue/postgres/task.model.js +26 -26
- package/task-queue/task-context.d.ts +10 -5
- package/task-queue/task-context.js +5 -3
- package/task-queue/task-queue.d.ts +339 -35
- package/task-queue/task-queue.js +135 -31
- package/task-queue/tests/coverage-branch.test.js +45 -57
- package/task-queue/tests/coverage-enhancement.test.js +123 -117
- package/task-queue/tests/{extensive-dependencies.test.js → dag.test.js} +61 -32
- package/task-queue/tests/dependencies.test.js +139 -21
- package/task-queue/tests/enqueue-batch.test.js +125 -0
- package/task-queue/tests/fan-out-spawning.test.js +43 -2
- package/task-queue/tests/idempotent-replacement.test.js +54 -1
- package/task-queue/tests/missing-idempotent-tasks.test.js +9 -8
- package/task-queue/tests/queue.test.js +261 -25
- package/task-queue/tests/shutdown.test.js +41 -0
- package/task-queue/tests/transactions.test.d.ts +1 -0
- package/task-queue/tests/transactions.test.js +47 -0
- package/task-queue/tests/worker.test.js +46 -13
- package/task-queue/tests/zombie-parent.test.js +1 -1
- package/task-queue/tests/zombie-recovery.test.js +3 -3
- package/testing/integration-setup.js +5 -3
- package/utils/timing.d.ts +2 -2
- package/task-queue/postgres/drizzle/0000_wakeful_sunspot.sql +0 -82
- package/task-queue/tests/cascading-cancellations.test.js +0 -38
- package/task-queue/tests/complex.test.js +0 -122
- package/task-queue/tests/dag-dependencies.test.js +0 -41
- /package/{task-queue/tests/cascading-cancellations.test.d.ts → orm/tests/database-extension.test.d.ts} +0 -0
- /package/task-queue/tests/{complex.test.d.ts → dag.test.d.ts} +0 -0
- /package/task-queue/tests/{dag-dependencies.test.d.ts → enqueue-batch.test.d.ts} +0 -0
- /package/task-queue/tests/{extensive-dependencies.test.d.ts → shutdown.test.d.ts} +0 -0
|
@@ -56,33 +56,34 @@ var __disposeResources = (this && this.__disposeResources) || (function (Suppres
|
|
|
56
56
|
var e = new Error(message);
|
|
57
57
|
return e.name = "SuppressedError", e.error = error, e.suppressed = suppressed, e;
|
|
58
58
|
});
|
|
59
|
-
import { aliasedTable, and, asc, count, eq, gt, gte, inArray, lt, lte, notExists, or, sql, count as sqlCount, isNull as sqlIsNull } from 'drizzle-orm';
|
|
60
|
-
import { merge, throttleTime } from 'rxjs';
|
|
59
|
+
import { aliasedTable, and, asc, count, eq, gt, gte, inArray, lt, lte, notExists, notInArray, or, sql, count as sqlCount, isNotNull as sqlIsNotNull, isNull as sqlIsNull } from 'drizzle-orm';
|
|
60
|
+
import { filter, merge, throttleTime } from 'rxjs';
|
|
61
61
|
import { CancellationSignal } from '../../cancellation/index.js';
|
|
62
62
|
import { CircuitBreaker, CircuitBreakerState } from '../../circuit-breaker/index.js';
|
|
63
63
|
import { serializeError, TimeoutError } from '../../errors/index.js';
|
|
64
64
|
import { afterResolve, inject, provide, Singleton } from '../../injector/index.js';
|
|
65
65
|
import { Logger } from '../../logger/index.js';
|
|
66
66
|
import { MessageBus } from '../../message-bus/index.js';
|
|
67
|
-
import { arrayOverlaps, coalesce, getEntityIds, interval, RANDOM_UUID_V4, TRANSACTION_TIMESTAMP } from '../../orm/index.js';
|
|
68
|
-
import { DatabaseConfig, injectRepository } from '../../orm/server/index.js';
|
|
67
|
+
import { arrayOverlaps, caseWhen, coalesce, enumValue, getEntityIds, greatest, interval, jsonbBuildObject, RANDOM_UUID_V4, TRANSACTION_TIMESTAMP } from '../../orm/index.js';
|
|
68
|
+
import { Database, DatabaseConfig, injectRepository } from '../../orm/server/index.js';
|
|
69
69
|
import { RateLimiter } from '../../rate-limit/index.js';
|
|
70
|
-
import {
|
|
70
|
+
import { distinct, toArray } from '../../utils/array/array.js';
|
|
71
71
|
import { currentTimestamp } from '../../utils/date-time.js';
|
|
72
72
|
import { Timer } from '../../utils/timer.js';
|
|
73
73
|
import { cancelableTimeout } from '../../utils/timing.js';
|
|
74
|
-
import { isDefined, isNotNull,
|
|
75
|
-
import { millisecondsPerSecond } from '../../utils/units.js';
|
|
74
|
+
import { isDefined, isNotNull, isNull, isNumber, isString, isUndefined } from '../../utils/type-guards.js';
|
|
75
|
+
import { millisecondsPerMinute, millisecondsPerSecond } from '../../utils/units.js';
|
|
76
76
|
import { defaultQueueConfig, TaskDependencyType, TaskQueue, TaskStatus } from '../task-queue.js';
|
|
77
77
|
import { PostgresTaskQueueModuleConfig } from './module.js';
|
|
78
|
-
import { taskArchive as taskArchiveTable, taskDependency as taskDependencyTable, taskStatus, task as taskTable } from './schemas.js';
|
|
79
|
-
import { PostgresTask, PostgresTaskArchive } from './task.model.js';
|
|
78
|
+
import { taskArchive as taskArchiveTable, taskDependency as taskDependencyTable, taskDependencyType, taskStatus, task as taskTable } from './schemas.js';
|
|
79
|
+
import { finalizedStatuses, PostgresTask, PostgresTaskArchive, queueableOrWaitableStatuses, queueableStatuses, terminalStatuses } from './task.model.js';
|
|
80
80
|
let PostgresTaskQueue = class PostgresTaskQueue extends TaskQueue {
|
|
81
|
+
#database = inject(Database);
|
|
81
82
|
#repository = injectRepository(PostgresTask);
|
|
82
83
|
#archiveRepository = injectRepository(PostgresTaskArchive);
|
|
83
84
|
#config = this.config;
|
|
84
85
|
#namespace = isString(this.#config) ? this.#config : this.#config.namespace;
|
|
85
|
-
#messageBus = inject((MessageBus),
|
|
86
|
+
#messageBus = inject((MessageBus), 'PostgresTaskQueue');
|
|
86
87
|
#logger = inject(Logger, `PostgresTaskQueue:${this.#namespace}`);
|
|
87
88
|
#cancellationSignal = inject(CancellationSignal);
|
|
88
89
|
#rateLimiter = inject(RateLimiter, {
|
|
@@ -109,18 +110,18 @@ let PostgresTaskQueue = class PostgresTaskQueue extends TaskQueue {
|
|
|
109
110
|
rateLimit = this.#config.rateLimit ?? defaultQueueConfig.rateLimit;
|
|
110
111
|
rateInterval = this.#config.rateInterval ?? defaultQueueConfig.rateInterval;
|
|
111
112
|
idempotencyWindow = this.#config.idempotencyWindow ?? defaultQueueConfig.idempotencyWindow;
|
|
112
|
-
globalConcurrency = this.#config.globalConcurrency ?? defaultQueueConfig.globalConcurrency;
|
|
113
113
|
#takeNewUpdate = {
|
|
114
114
|
namespace: this.#namespace,
|
|
115
115
|
type: sql `excluded.type`,
|
|
116
|
-
status: sql `
|
|
116
|
+
status: caseWhen(gt(sql `excluded.unresolved_schedule_dependencies`, 0), enumValue(TaskStatus, taskStatus, TaskStatus.Waiting)).else(enumValue(TaskStatus, taskStatus, TaskStatus.Pending)),
|
|
117
117
|
token: null,
|
|
118
118
|
priority: sql `excluded.priority`,
|
|
119
119
|
idempotencyKey: sql `excluded.idempotency_key`,
|
|
120
120
|
traceId: sql `excluded.trace_id`,
|
|
121
|
+
parentId: sql `excluded.parent_id`,
|
|
121
122
|
tags: sql `excluded.tags`,
|
|
122
|
-
unresolvedScheduleDependencies:
|
|
123
|
-
unresolvedCompleteDependencies:
|
|
123
|
+
unresolvedScheduleDependencies: taskTable.unresolvedScheduleDependencies,
|
|
124
|
+
unresolvedCompleteDependencies: taskTable.unresolvedCompleteDependencies,
|
|
124
125
|
failFast: sql `excluded.fail_fast`,
|
|
125
126
|
tries: 0,
|
|
126
127
|
creationTimestamp: TRANSACTION_TIMESTAMP,
|
|
@@ -138,7 +139,7 @@ let PostgresTaskQueue = class PostgresTaskQueue extends TaskQueue {
|
|
|
138
139
|
};
|
|
139
140
|
[afterResolve]() {
|
|
140
141
|
if (!this.isInTransaction) {
|
|
141
|
-
|
|
142
|
+
this.maintenanceLoop();
|
|
142
143
|
}
|
|
143
144
|
}
|
|
144
145
|
async enqueue(type, data, options) {
|
|
@@ -151,119 +152,167 @@ let PostgresTaskQueue = class PostgresTaskQueue extends TaskQueue {
|
|
|
151
152
|
if (items.length == 0) {
|
|
152
153
|
return (options?.returnTasks == true) ? [] : undefined;
|
|
153
154
|
}
|
|
154
|
-
const
|
|
155
|
-
|
|
156
|
-
|
|
157
|
-
|
|
155
|
+
const itemsWithDistinctDependencies = items.map((item) => ({
|
|
156
|
+
...item,
|
|
157
|
+
scheduleAfter: isDefined(item.scheduleAfter) ? Array.from(new Map(item.scheduleAfter.map((s) => [isNumber(s) || isString(s) ? s : JSON.stringify(s), s])).values()) : undefined,
|
|
158
|
+
completeAfter: isDefined(item.completeAfter) ? Array.from(new Map(item.completeAfter.map((s) => [isNumber(s) || isString(s) ? s : JSON.stringify(s), s])).values()) : undefined,
|
|
159
|
+
}));
|
|
160
|
+
const entitiesWithIndex = itemsWithDistinctDependencies.map((item, index) => {
|
|
158
161
|
return {
|
|
159
|
-
|
|
160
|
-
|
|
161
|
-
|
|
162
|
-
|
|
163
|
-
|
|
164
|
-
|
|
165
|
-
|
|
166
|
-
|
|
167
|
-
|
|
168
|
-
|
|
169
|
-
|
|
170
|
-
|
|
171
|
-
|
|
172
|
-
|
|
173
|
-
|
|
174
|
-
|
|
175
|
-
|
|
176
|
-
|
|
177
|
-
|
|
178
|
-
|
|
179
|
-
|
|
180
|
-
|
|
181
|
-
|
|
182
|
-
|
|
183
|
-
|
|
162
|
+
index,
|
|
163
|
+
entity: {
|
|
164
|
+
namespace: this.#namespace,
|
|
165
|
+
type: item.type,
|
|
166
|
+
status: TaskStatus.Pending,
|
|
167
|
+
token: null,
|
|
168
|
+
priority: item.priority ?? 1000,
|
|
169
|
+
idempotencyKey: item.idempotencyKey ?? null,
|
|
170
|
+
traceId: null,
|
|
171
|
+
tags: item.tags ?? [],
|
|
172
|
+
unresolvedScheduleDependencies: 0,
|
|
173
|
+
unresolvedCompleteDependencies: 0,
|
|
174
|
+
failFast: item.failFast ?? false,
|
|
175
|
+
parentId: item.parentId ?? null,
|
|
176
|
+
tries: 0,
|
|
177
|
+
progress: 0,
|
|
178
|
+
creationTimestamp: TRANSACTION_TIMESTAMP,
|
|
179
|
+
priorityAgeTimestamp: TRANSACTION_TIMESTAMP,
|
|
180
|
+
scheduleTimestamp: item.scheduleTimestamp ?? TRANSACTION_TIMESTAMP,
|
|
181
|
+
startTimestamp: null,
|
|
182
|
+
timeToLive: item.timeToLive ?? sql `${TRANSACTION_TIMESTAMP} + ${interval(this.defaultTimeToLive, 'milliseconds')}`,
|
|
183
|
+
visibilityDeadline: null,
|
|
184
|
+
completeTimestamp: null,
|
|
185
|
+
data: item.data,
|
|
186
|
+
state: null,
|
|
187
|
+
result: null,
|
|
188
|
+
error: null,
|
|
189
|
+
},
|
|
184
190
|
};
|
|
185
191
|
});
|
|
186
|
-
const itemsWithIdempotency =
|
|
187
|
-
const itemsWithoutIdempotency =
|
|
188
|
-
const hasDependencies =
|
|
189
|
-
const mustUseTransaction = (
|
|
192
|
+
const itemsWithIdempotency = entitiesWithIndex.filter((e) => isNotNull(e.entity.idempotencyKey));
|
|
193
|
+
const itemsWithoutIdempotency = entitiesWithIndex.filter((e) => isNull(e.entity.idempotencyKey));
|
|
194
|
+
const hasDependencies = itemsWithDistinctDependencies.some((item) => ((item.scheduleAfter?.length ?? 0) > 0) || ((item.completeAfter?.length ?? 0) > 0) || (isDefined(item.parentId) && (item.waitForCompletion ?? true)));
|
|
195
|
+
const mustUseTransaction = (entitiesWithIndex.length > 1) || hasDependencies;
|
|
190
196
|
const newTransaction = __addDisposableResource(env_1, (mustUseTransaction && isUndefined(options?.transaction)) ? await this.#repository.startTransaction() : undefined, true);
|
|
191
197
|
const transaction = newTransaction ?? options?.transaction;
|
|
192
198
|
const session = transaction?.pgTransaction ?? this.#repository.session;
|
|
193
|
-
const tasks =
|
|
199
|
+
const tasks = new Array(itemsWithDistinctDependencies.length);
|
|
200
|
+
let insertedRows = [];
|
|
201
|
+
let upsertedRows = [];
|
|
194
202
|
if (itemsWithoutIdempotency.length > 0) {
|
|
195
|
-
|
|
196
|
-
|
|
203
|
+
insertedRows = await session.insert(taskTable).values(itemsWithoutIdempotency.map((i) => i.entity)).returning();
|
|
204
|
+
for (const [i, row] of insertedRows.entries()) {
|
|
205
|
+
tasks[itemsWithoutIdempotency[i].index] = row;
|
|
206
|
+
}
|
|
197
207
|
}
|
|
198
208
|
if (itemsWithIdempotency.length > 0) {
|
|
199
209
|
const windowCutoff = sql `${TRANSACTION_TIMESTAMP} - ${interval(this.idempotencyWindow, 'milliseconds')}`;
|
|
200
210
|
const setWhere = (options?.replace == true) ? undefined : sql `${taskTable.creationTimestamp} < ${windowCutoff}`;
|
|
201
|
-
|
|
211
|
+
upsertedRows = await session
|
|
202
212
|
.insert(taskTable)
|
|
203
|
-
.values(itemsWithIdempotency)
|
|
213
|
+
.values(itemsWithIdempotency.map((i) => i.entity))
|
|
204
214
|
.onConflictDoUpdate({
|
|
205
215
|
target: [taskTable.namespace, taskTable.idempotencyKey],
|
|
206
216
|
set: this.#takeNewUpdate,
|
|
207
217
|
setWhere,
|
|
208
218
|
})
|
|
209
219
|
.returning();
|
|
210
|
-
|
|
211
|
-
if (
|
|
212
|
-
|
|
213
|
-
|
|
214
|
-
|
|
215
|
-
|
|
220
|
+
const upsertedIds = upsertedRows.map((t) => t.id);
|
|
221
|
+
if (upsertedIds.length > 0) {
|
|
222
|
+
await session
|
|
223
|
+
.delete(taskDependencyTable)
|
|
224
|
+
.where(and(inArray(taskDependencyTable.taskId, upsertedIds), inArray(taskDependencyTable.type, [TaskDependencyType.Schedule, TaskDependencyType.Complete])));
|
|
225
|
+
await session
|
|
226
|
+
.update(taskTable)
|
|
227
|
+
.set({ unresolvedScheduleDependencies: 0 })
|
|
228
|
+
.where(inArray(taskTable.id, upsertedIds));
|
|
216
229
|
}
|
|
217
|
-
if ((
|
|
218
|
-
const
|
|
219
|
-
const
|
|
220
|
-
|
|
230
|
+
if ((options?.returnTasks == true) || hasDependencies) {
|
|
231
|
+
const upsertedKeysMap = new Map(upsertedRows.map((t) => [t.idempotencyKey, t]));
|
|
232
|
+
const missingKeysWithIndex = [];
|
|
233
|
+
for (const item of itemsWithIdempotency) {
|
|
234
|
+
const key = item.entity.idempotencyKey;
|
|
235
|
+
const task = upsertedKeysMap.get(key);
|
|
236
|
+
if (isDefined(task)) {
|
|
237
|
+
tasks[item.index] = task;
|
|
238
|
+
}
|
|
239
|
+
else {
|
|
240
|
+
missingKeysWithIndex.push({ index: item.index, key });
|
|
241
|
+
}
|
|
242
|
+
}
|
|
243
|
+
if (missingKeysWithIndex.length > 0) {
|
|
221
244
|
const existingRows = await session
|
|
222
245
|
.select()
|
|
223
246
|
.from(taskTable)
|
|
224
|
-
.where(and(eq(taskTable.namespace, this.#namespace), inArray(taskTable.idempotencyKey,
|
|
225
|
-
|
|
247
|
+
.where(and(eq(taskTable.namespace, this.#namespace), inArray(taskTable.idempotencyKey, missingKeysWithIndex.map((m) => m.key))));
|
|
248
|
+
const existingRowsMap = new Map(existingRows.map((t) => [t.idempotencyKey, t]));
|
|
249
|
+
for (const missing of missingKeysWithIndex) {
|
|
250
|
+
tasks[missing.index] = existingRowsMap.get(missing.key);
|
|
251
|
+
}
|
|
226
252
|
}
|
|
227
253
|
}
|
|
228
254
|
}
|
|
229
255
|
if (hasDependencies) {
|
|
230
256
|
const dependencies = [];
|
|
231
|
-
const
|
|
232
|
-
for (const [index, item] of
|
|
257
|
+
const processedTaskIds = new Set([...insertedRows.map((r) => r.id), ...upsertedRows.map((r) => r.id)]);
|
|
258
|
+
for (const [index, item] of itemsWithDistinctDependencies.entries()) {
|
|
233
259
|
const task = tasks[index];
|
|
260
|
+
if (!processedTaskIds.has(task.id)) {
|
|
261
|
+
continue;
|
|
262
|
+
}
|
|
234
263
|
if (isDefined(item.parentId) && (item.waitForCompletion ?? true)) {
|
|
235
264
|
dependencies.push({
|
|
236
|
-
namespace: this.#namespace,
|
|
237
265
|
taskId: item.parentId,
|
|
238
266
|
dependencyTaskId: task.id,
|
|
239
|
-
type: TaskDependencyType.
|
|
267
|
+
type: TaskDependencyType.Child,
|
|
240
268
|
requiredStatuses: [TaskStatus.Completed],
|
|
241
269
|
});
|
|
242
|
-
parentIncrements.set(item.parentId, (parentIncrements.get(item.parentId) ?? 0) + 1);
|
|
243
270
|
}
|
|
244
271
|
if (isDefined(item.scheduleAfter)) {
|
|
245
272
|
for (const dependency of item.scheduleAfter) {
|
|
246
|
-
|
|
247
|
-
|
|
273
|
+
let dependencyTaskId;
|
|
274
|
+
let requiredStatuses;
|
|
275
|
+
if (isNumber(dependency)) {
|
|
276
|
+
dependencyTaskId = tasks[dependency].id;
|
|
277
|
+
requiredStatuses = [TaskStatus.Completed];
|
|
278
|
+
}
|
|
279
|
+
else if (isString(dependency)) {
|
|
280
|
+
dependencyTaskId = dependency;
|
|
281
|
+
requiredStatuses = [TaskStatus.Completed];
|
|
282
|
+
}
|
|
283
|
+
else {
|
|
284
|
+
dependencyTaskId = isDefined(dependency.index) ? tasks[dependency.index].id : dependency.id;
|
|
285
|
+
requiredStatuses = dependency.requiredStatuses ?? [TaskStatus.Completed];
|
|
286
|
+
}
|
|
248
287
|
dependencies.push({
|
|
249
|
-
namespace: this.#namespace,
|
|
250
288
|
taskId: task.id,
|
|
251
289
|
dependencyTaskId,
|
|
252
290
|
type: TaskDependencyType.Schedule,
|
|
253
|
-
requiredStatuses,
|
|
291
|
+
requiredStatuses: requiredStatuses,
|
|
254
292
|
});
|
|
255
293
|
}
|
|
256
294
|
}
|
|
257
295
|
if (isDefined(item.completeAfter)) {
|
|
258
296
|
for (const dependency of item.completeAfter) {
|
|
259
|
-
|
|
260
|
-
|
|
297
|
+
let dependencyTaskId;
|
|
298
|
+
let requiredStatuses;
|
|
299
|
+
if (isNumber(dependency)) {
|
|
300
|
+
dependencyTaskId = tasks[dependency].id;
|
|
301
|
+
requiredStatuses = [TaskStatus.Completed];
|
|
302
|
+
}
|
|
303
|
+
else if (isString(dependency)) {
|
|
304
|
+
dependencyTaskId = dependency;
|
|
305
|
+
requiredStatuses = [TaskStatus.Completed];
|
|
306
|
+
}
|
|
307
|
+
else {
|
|
308
|
+
dependencyTaskId = isDefined(dependency.index) ? tasks[dependency.index].id : dependency.id;
|
|
309
|
+
requiredStatuses = dependency.requiredStatuses ?? [TaskStatus.Completed];
|
|
310
|
+
}
|
|
261
311
|
dependencies.push({
|
|
262
|
-
namespace: this.#namespace,
|
|
263
312
|
taskId: task.id,
|
|
264
313
|
dependencyTaskId,
|
|
265
314
|
type: TaskDependencyType.Complete,
|
|
266
|
-
requiredStatuses,
|
|
315
|
+
requiredStatuses: requiredStatuses,
|
|
267
316
|
});
|
|
268
317
|
}
|
|
269
318
|
}
|
|
@@ -274,27 +323,35 @@ let PostgresTaskQueue = class PostgresTaskQueue extends TaskQueue {
|
|
|
274
323
|
.values(dependencies)
|
|
275
324
|
.onConflictDoNothing()
|
|
276
325
|
.returning();
|
|
277
|
-
|
|
278
|
-
|
|
279
|
-
|
|
280
|
-
|
|
281
|
-
|
|
282
|
-
|
|
283
|
-
|
|
284
|
-
|
|
285
|
-
|
|
286
|
-
|
|
287
|
-
|
|
288
|
-
|
|
289
|
-
}
|
|
326
|
+
const distinctDependencyIds = distinct(dependencies.map((d) => d.dependencyTaskId));
|
|
327
|
+
if (distinctDependencyIds.length > 0) {
|
|
328
|
+
await this.incrementCounters(inserted, { transaction });
|
|
329
|
+
const dependencyStatuses = await session
|
|
330
|
+
.select({ id: taskTable.id, status: taskTable.status })
|
|
331
|
+
.from(taskTable)
|
|
332
|
+
.where(inArray(taskTable.id, distinctDependencyIds))
|
|
333
|
+
.unionAll(session
|
|
334
|
+
.select({ id: taskArchiveTable.id, status: sql `${enumValue(TaskStatus, taskStatus, TaskStatus.Completed)}` })
|
|
335
|
+
.from(taskArchiveTable)
|
|
336
|
+
.where(inArray(taskArchiveTable.id, distinctDependencyIds)));
|
|
337
|
+
if (dependencyStatuses.length > 0) {
|
|
338
|
+
await this.resolveDependenciesMany(dependencyStatuses.map((s) => ({ id: s.id, status: s.status })), { transaction: transaction });
|
|
290
339
|
}
|
|
291
340
|
}
|
|
292
341
|
}
|
|
293
342
|
}
|
|
294
|
-
|
|
295
|
-
|
|
343
|
+
if (isDefined(newTransaction) && !newTransaction.isDone) {
|
|
344
|
+
await newTransaction.commit();
|
|
345
|
+
}
|
|
346
|
+
this.notify();
|
|
296
347
|
if (options?.returnTasks == true) {
|
|
297
|
-
const
|
|
348
|
+
const finalTasks = await session
|
|
349
|
+
.select()
|
|
350
|
+
.from(taskTable)
|
|
351
|
+
.where(inArray(taskTable.id, tasks.map((t) => t.id)));
|
|
352
|
+
const finalTasksMap = new Map(finalTasks.map((t) => [t.id, t]));
|
|
353
|
+
const orderedFinalTasks = tasks.map((t) => finalTasksMap.get(t.id));
|
|
354
|
+
const mapped = await this.#repository.mapManyToEntity(orderedFinalTasks);
|
|
298
355
|
return mapped;
|
|
299
356
|
}
|
|
300
357
|
return undefined;
|
|
@@ -309,12 +366,53 @@ let PostgresTaskQueue = class PostgresTaskQueue extends TaskQueue {
|
|
|
309
366
|
await result_1;
|
|
310
367
|
}
|
|
311
368
|
}
|
|
369
|
+
async incrementCounters(dependencies, options) {
|
|
370
|
+
if (dependencies.length == 0) {
|
|
371
|
+
return;
|
|
372
|
+
}
|
|
373
|
+
const session = options?.transaction?.pgTransaction ?? this.#repository.session;
|
|
374
|
+
const decrementsToApply = new Map();
|
|
375
|
+
for (const dep of dependencies) {
|
|
376
|
+
const current = decrementsToApply.get(dep.taskId) ?? { schedule: 0, complete: 0 };
|
|
377
|
+
if (dep.type == TaskDependencyType.Schedule) {
|
|
378
|
+
current.schedule++;
|
|
379
|
+
}
|
|
380
|
+
else if (dep.type == TaskDependencyType.Complete || dep.type == TaskDependencyType.Child) {
|
|
381
|
+
current.complete++;
|
|
382
|
+
}
|
|
383
|
+
decrementsToApply.set(dep.taskId, current);
|
|
384
|
+
}
|
|
385
|
+
const values = [...decrementsToApply]
|
|
386
|
+
.toSorted(([idA], [idB]) => idA.localeCompare(idB))
|
|
387
|
+
.map(([taskId, d]) => sql `(${taskId}::uuid, ${d.schedule}::int, ${d.complete}::int)`);
|
|
388
|
+
const updates = session.$with('updates').as((qb) => qb
|
|
389
|
+
.select({
|
|
390
|
+
taskId: sql `(id)::uuid`.as('task_id'),
|
|
391
|
+
scheduleIncrement: sql `(schedule)::int`.as('schedule_increment'),
|
|
392
|
+
completeIncrement: sql `(complete)::int`.as('complete_increment'),
|
|
393
|
+
})
|
|
394
|
+
.from(sql `(VALUES ${sql.join(values, sql `, `)}) AS t(id, schedule, complete)`));
|
|
395
|
+
const updatedRows = await session
|
|
396
|
+
.with(updates)
|
|
397
|
+
.update(taskTable)
|
|
398
|
+
.set({
|
|
399
|
+
unresolvedScheduleDependencies: sql `${taskTable.unresolvedScheduleDependencies} + ${updates.scheduleIncrement}`,
|
|
400
|
+
unresolvedCompleteDependencies: sql `${taskTable.unresolvedCompleteDependencies} + ${updates.completeIncrement}`,
|
|
401
|
+
status: caseWhen(and(eq(taskTable.status, TaskStatus.Pending), gt(sql `${taskTable.unresolvedScheduleDependencies} + ${updates.scheduleIncrement}`, 0)), enumValue(TaskStatus, taskStatus, TaskStatus.Waiting)).else(taskTable.status),
|
|
402
|
+
})
|
|
403
|
+
.from(updates)
|
|
404
|
+
.where(eq(taskTable.id, updates.taskId))
|
|
405
|
+
.returning({ id: taskTable.id, status: taskTable.status, namespace: taskTable.namespace });
|
|
406
|
+
for (const row of updatedRows) {
|
|
407
|
+
this.notify(row.namespace);
|
|
408
|
+
}
|
|
409
|
+
}
|
|
312
410
|
async has(id, options) {
|
|
313
411
|
const session = options?.transaction?.pgTransaction ?? this.#repository.session;
|
|
314
412
|
const [result] = await session
|
|
315
413
|
.select({ id: taskTable.id })
|
|
316
414
|
.from(taskTable)
|
|
317
|
-
.where(
|
|
415
|
+
.where(eq(taskTable.id, id))
|
|
318
416
|
.limit(1);
|
|
319
417
|
return isDefined(result);
|
|
320
418
|
}
|
|
@@ -330,19 +428,27 @@ let PostgresTaskQueue = class PostgresTaskQueue extends TaskQueue {
|
|
|
330
428
|
.where(and(eq(taskTable.namespace, this.#namespace), arrayOverlaps(taskTable.tags, tagArray)));
|
|
331
429
|
return result?.count ?? 0;
|
|
332
430
|
}
|
|
431
|
+
async count(options) {
|
|
432
|
+
const session = options?.transaction?.pgTransaction ?? this.#repository.session;
|
|
433
|
+
const [result] = await session
|
|
434
|
+
.select({ count: count() })
|
|
435
|
+
.from(taskTable)
|
|
436
|
+
.where(and(eq(taskTable.namespace, this.#namespace), isDefined(options?.status) ? eq(taskTable.status, options.status) : undefined));
|
|
437
|
+
return result?.count ?? 0;
|
|
438
|
+
}
|
|
333
439
|
async getTask(id, options) {
|
|
334
440
|
const session = options?.transaction?.pgTransaction ?? this.#repository.session;
|
|
335
441
|
const [activeRow] = await session
|
|
336
442
|
.select()
|
|
337
443
|
.from(taskTable)
|
|
338
|
-
.where(
|
|
444
|
+
.where(eq(taskTable.id, id));
|
|
339
445
|
if (isDefined(activeRow)) {
|
|
340
446
|
return await this.#repository.mapToEntity(activeRow);
|
|
341
447
|
}
|
|
342
448
|
const [archiveRow] = await session
|
|
343
449
|
.select()
|
|
344
450
|
.from(taskArchiveTable)
|
|
345
|
-
.where(
|
|
451
|
+
.where(eq(taskArchiveTable.id, id));
|
|
346
452
|
if (isDefined(archiveRow)) {
|
|
347
453
|
return await this.#archiveRepository.mapToEntity(archiveRow);
|
|
348
454
|
}
|
|
@@ -381,7 +487,7 @@ let PostgresTaskQueue = class PostgresTaskQueue extends TaskQueue {
|
|
|
381
487
|
FROM ${taskTable}
|
|
382
488
|
WHERE ${inArray(taskTable.id, rootIds)}
|
|
383
489
|
|
|
384
|
-
UNION
|
|
490
|
+
UNION ALL
|
|
385
491
|
|
|
386
492
|
SELECT child.*, parent.depth + 1
|
|
387
493
|
FROM ${taskTable} child
|
|
@@ -400,10 +506,10 @@ let PostgresTaskQueue = class PostgresTaskQueue extends TaskQueue {
|
|
|
400
506
|
const timeout = options?.timeout ?? Infinity;
|
|
401
507
|
const interval = options?.interval ?? 1000;
|
|
402
508
|
const cancellationSignal = this.#cancellationSignal.optionallyInherit(options?.cancellationSignal);
|
|
403
|
-
const
|
|
509
|
+
const waitStatuses = options?.statuses ?? finalizedStatuses;
|
|
510
|
+
const messageBus$ = this.#messageBus.allMessages$.pipe(filter((namespace) => namespace == this.#namespace), throttleTime(500, undefined, { leading: true, trailing: true }));
|
|
404
511
|
const continue$ = merge(messageBus$, cancellationSignal);
|
|
405
512
|
const timer = Timer.startNew();
|
|
406
|
-
const finalizedStatuses = [TaskStatus.Completed, TaskStatus.Cancelled, TaskStatus.Dead];
|
|
407
513
|
while (true) {
|
|
408
514
|
if (cancellationSignal.isSet) {
|
|
409
515
|
return { cancelled: true };
|
|
@@ -411,11 +517,12 @@ let PostgresTaskQueue = class PostgresTaskQueue extends TaskQueue {
|
|
|
411
517
|
if (timer.milliseconds > timeout) {
|
|
412
518
|
throw new TimeoutError('Timeout while waiting for tasks to complete');
|
|
413
519
|
}
|
|
414
|
-
const
|
|
415
|
-
|
|
416
|
-
|
|
417
|
-
|
|
418
|
-
|
|
520
|
+
const [remaining] = await this.#repository.session
|
|
521
|
+
.select({ id: taskTable.id })
|
|
522
|
+
.from(taskTable)
|
|
523
|
+
.where(and(inArray(taskTable.id, ids), notInArray(taskTable.status, waitStatuses)))
|
|
524
|
+
.limit(1);
|
|
525
|
+
if (isUndefined(remaining)) {
|
|
419
526
|
return { cancelled: false };
|
|
420
527
|
}
|
|
421
528
|
const remainingTimeout = timeout - timer.milliseconds;
|
|
@@ -433,16 +540,17 @@ let PostgresTaskQueue = class PostgresTaskQueue extends TaskQueue {
|
|
|
433
540
|
if (treeIds.length == 0) {
|
|
434
541
|
return;
|
|
435
542
|
}
|
|
436
|
-
await tx.pgTransaction
|
|
543
|
+
const cancelledRows = await tx.pgTransaction
|
|
437
544
|
.update(taskTable)
|
|
438
545
|
.set({
|
|
439
546
|
status: TaskStatus.Cancelled,
|
|
440
547
|
token: null,
|
|
441
548
|
completeTimestamp: TRANSACTION_TIMESTAMP,
|
|
442
549
|
})
|
|
443
|
-
.where(and(
|
|
444
|
-
|
|
445
|
-
|
|
550
|
+
.where(and(inArray(taskTable.id, treeIds), notInArray(taskTable.status, terminalStatuses)))
|
|
551
|
+
.returning({ id: taskTable.id, namespace: taskTable.namespace });
|
|
552
|
+
if (cancelledRows.length > 0) {
|
|
553
|
+
await this.resolveDependenciesMany(cancelledRows.map((row) => ({ id: row.id, status: TaskStatus.Cancelled, namespace: row.namespace })), { transaction: tx });
|
|
446
554
|
}
|
|
447
555
|
});
|
|
448
556
|
}
|
|
@@ -455,6 +563,32 @@ let PostgresTaskQueue = class PostgresTaskQueue extends TaskQueue {
|
|
|
455
563
|
}
|
|
456
564
|
async clear(options) {
|
|
457
565
|
await this.#repository.useTransaction(options?.transaction, async (tx) => {
|
|
566
|
+
const nonFinalizedTasks = await tx.pgTransaction
|
|
567
|
+
.select({ id: taskTable.id, namespace: taskTable.namespace })
|
|
568
|
+
.from(taskTable)
|
|
569
|
+
.where(and(eq(taskTable.namespace, this.#namespace), notInArray(taskTable.status, finalizedStatuses)))
|
|
570
|
+
.for('update');
|
|
571
|
+
if (nonFinalizedTasks.length > 0) {
|
|
572
|
+
const ids = nonFinalizedTasks.map((t) => t.id);
|
|
573
|
+
await tx.pgTransaction
|
|
574
|
+
.update(taskTable)
|
|
575
|
+
.set({
|
|
576
|
+
status: TaskStatus.Cancelled,
|
|
577
|
+
token: null,
|
|
578
|
+
completeTimestamp: TRANSACTION_TIMESTAMP,
|
|
579
|
+
})
|
|
580
|
+
.where(inArray(taskTable.id, ids));
|
|
581
|
+
await this.resolveDependenciesMany(nonFinalizedTasks.map((t) => ({ id: t.id, status: TaskStatus.Cancelled, namespace: t.namespace })), { transaction: tx });
|
|
582
|
+
}
|
|
583
|
+
// Break internal parent-child links to allow deleting tasks within the same namespace.
|
|
584
|
+
// Cross-namespace links will still cause a foreign key violation as requested.
|
|
585
|
+
await tx.pgTransaction
|
|
586
|
+
.update(taskTable)
|
|
587
|
+
.set({ parentId: null })
|
|
588
|
+
.where(and(eq(taskTable.namespace, this.#namespace), sqlIsNotNull(taskTable.parentId), inArray(taskTable.parentId, tx.pgTransaction
|
|
589
|
+
.select({ id: taskTable.id })
|
|
590
|
+
.from(taskTable)
|
|
591
|
+
.where(eq(taskTable.namespace, this.#namespace)))));
|
|
458
592
|
await tx.pgTransaction
|
|
459
593
|
.delete(taskTable)
|
|
460
594
|
.where(eq(taskTable.namespace, this.#namespace));
|
|
@@ -501,20 +635,6 @@ let PostgresTaskQueue = class PostgresTaskQueue extends TaskQueue {
|
|
|
501
635
|
}
|
|
502
636
|
effectiveCount = 1;
|
|
503
637
|
}
|
|
504
|
-
// 2. Check Global Concurrency
|
|
505
|
-
if (!forceDequeue && isNotNull(this.globalConcurrency)) {
|
|
506
|
-
// WARN: This is a check-then-act race condition.
|
|
507
|
-
// A distributed lock or an atomic update strategy is needed for strict enforcement.
|
|
508
|
-
const [runningResult] = await tx.pgTransaction
|
|
509
|
-
.select({ count: sqlCount() })
|
|
510
|
-
.from(taskTable)
|
|
511
|
-
.where(and(eq(taskTable.namespace, this.#namespace), eq(taskTable.status, TaskStatus.Running)));
|
|
512
|
-
const runningCount = runningResult?.count ?? 0;
|
|
513
|
-
if (runningCount >= this.globalConcurrency) {
|
|
514
|
-
return [];
|
|
515
|
-
}
|
|
516
|
-
effectiveCount = Math.min(effectiveCount, this.globalConcurrency - runningCount);
|
|
517
|
-
}
|
|
518
638
|
/*
|
|
519
639
|
* Materialization required for LIMIT clause
|
|
520
640
|
* https://stackoverflow.com/questions/73966670/select-for-update-subquery-not-respecting-limit-clause-under-load
|
|
@@ -523,7 +643,7 @@ let PostgresTaskQueue = class PostgresTaskQueue extends TaskQueue {
|
|
|
523
643
|
const selection = tx.pgTransaction.$with('selection').as((qb) => qb
|
|
524
644
|
.select({ id: taskTable.id })
|
|
525
645
|
.from(taskTable)
|
|
526
|
-
.where(and(eq(taskTable.namespace, this.#namespace), lte(taskTable.scheduleTimestamp, TRANSACTION_TIMESTAMP),
|
|
646
|
+
.where(and(eq(taskTable.namespace, this.#namespace), lte(taskTable.scheduleTimestamp, TRANSACTION_TIMESTAMP), inArray(taskTable.status, queueableStatuses), or(sqlIsNull(taskTable.timeToLive), lt(TRANSACTION_TIMESTAMP, taskTable.timeToLive)), isDefined(options?.types) ? inArray(taskTable.type, options.types) : undefined, sql `pg_sleep(0) IS NOT NULL` // Materialization hack until drizzle implements https://github.com/drizzle-team/drizzle-orm/issues/2318
|
|
527
647
|
))
|
|
528
648
|
.orderBy(asc(taskTable.priority), asc(taskTable.scheduleTimestamp))
|
|
529
649
|
.limit(effectiveCount)
|
|
@@ -559,32 +679,26 @@ let PostgresTaskQueue = class PostgresTaskQueue extends TaskQueue {
|
|
|
559
679
|
await session
|
|
560
680
|
.update(taskTable)
|
|
561
681
|
.set({
|
|
562
|
-
status: TaskStatus.Pending,
|
|
682
|
+
status: caseWhen(gt(taskTable.unresolvedScheduleDependencies, 0), enumValue(TaskStatus, taskStatus, TaskStatus.Waiting)).else(enumValue(TaskStatus, taskStatus, TaskStatus.Pending)),
|
|
563
683
|
token: null,
|
|
564
684
|
scheduleTimestamp: timestamp,
|
|
565
685
|
visibilityDeadline: null,
|
|
566
|
-
tries: sql `
|
|
567
|
-
WHEN ${taskTable.status} = ${TaskStatus.Running} THEN GREATEST(0, ${taskTable.tries} - 1)
|
|
568
|
-
ELSE ${taskTable.tries}
|
|
569
|
-
END`,
|
|
686
|
+
tries: caseWhen(eq(taskTable.status, enumValue(TaskStatus, taskStatus, TaskStatus.Running)), greatest(0, sql `${taskTable.tries} - 1`)).else(taskTable.tries),
|
|
570
687
|
})
|
|
571
|
-
.where(and(
|
|
688
|
+
.where(and(inArray(taskTable.id, ids), notInArray(taskTable.status, terminalStatuses)));
|
|
572
689
|
}
|
|
573
690
|
async rescheduleManyByTags(tags, timestamp, options) {
|
|
574
691
|
const session = options?.transaction?.pgTransaction ?? this.#repository.session;
|
|
575
692
|
await session
|
|
576
693
|
.update(taskTable)
|
|
577
694
|
.set({
|
|
578
|
-
status: TaskStatus.Pending,
|
|
695
|
+
status: caseWhen(gt(taskTable.unresolvedScheduleDependencies, 0), enumValue(TaskStatus, taskStatus, TaskStatus.Waiting)).else(enumValue(TaskStatus, taskStatus, TaskStatus.Pending)),
|
|
579
696
|
token: null,
|
|
580
697
|
scheduleTimestamp: timestamp,
|
|
581
698
|
visibilityDeadline: null,
|
|
582
|
-
tries: sql `
|
|
583
|
-
WHEN ${taskTable.status} = ${TaskStatus.Running} THEN GREATEST(0, ${taskTable.tries} - 1)
|
|
584
|
-
ELSE ${taskTable.tries}
|
|
585
|
-
END`,
|
|
699
|
+
tries: caseWhen(eq(taskTable.status, enumValue(TaskStatus, taskStatus, TaskStatus.Running)), greatest(0, sql `${taskTable.tries} - 1`)).else(taskTable.tries),
|
|
586
700
|
})
|
|
587
|
-
.where(and(eq(taskTable.namespace, this.#namespace), arrayOverlaps(taskTable.tags, toArray(tags))));
|
|
701
|
+
.where(and(eq(taskTable.namespace, this.#namespace), arrayOverlaps(taskTable.tags, toArray(tags)), notInArray(taskTable.status, terminalStatuses)));
|
|
588
702
|
}
|
|
589
703
|
async touch(task, options) {
|
|
590
704
|
if (isNull(task.token)) {
|
|
@@ -603,7 +717,7 @@ let PostgresTaskQueue = class PostgresTaskQueue extends TaskQueue {
|
|
|
603
717
|
const [updatedRow] = await tx.pgTransaction
|
|
604
718
|
.update(taskTable)
|
|
605
719
|
.set(update)
|
|
606
|
-
.where(and(eq(taskTable.
|
|
720
|
+
.where(and(eq(taskTable.id, task.id), eq(taskTable.status, TaskStatus.Running), isNull(task.token) ? sqlIsNull(taskTable.token) : eq(taskTable.token, task.token), gt(taskTable.startTimestamp, sql `${TRANSACTION_TIMESTAMP} - ${interval(this.maxExecutionTime, 'milliseconds')}`)))
|
|
607
721
|
.returning();
|
|
608
722
|
if (isDefined(updatedRow)) {
|
|
609
723
|
return await this.#repository.mapToEntity(updatedRow);
|
|
@@ -611,9 +725,18 @@ let PostgresTaskQueue = class PostgresTaskQueue extends TaskQueue {
|
|
|
611
725
|
const [existingRow] = await tx.pgTransaction
|
|
612
726
|
.select({ startTimestamp: taskTable.startTimestamp })
|
|
613
727
|
.from(taskTable)
|
|
614
|
-
.where(and(eq(taskTable.
|
|
728
|
+
.where(and(eq(taskTable.id, task.id), eq(taskTable.status, TaskStatus.Running), isNull(task.token) ? sqlIsNull(taskTable.token) : eq(taskTable.token, task.token)));
|
|
615
729
|
if (isDefined(existingRow) && isNotNull(existingRow.startTimestamp) && (currentTimestamp() - existingRow.startTimestamp) > this.maxExecutionTime) {
|
|
616
|
-
await
|
|
730
|
+
await tx.pgTransaction
|
|
731
|
+
.update(taskTable)
|
|
732
|
+
.set({
|
|
733
|
+
status: TaskStatus.TimedOut,
|
|
734
|
+
completeTimestamp: TRANSACTION_TIMESTAMP,
|
|
735
|
+
error: { code: 'MaxTimeExceeded', message: 'Hard Execution Timeout' },
|
|
736
|
+
})
|
|
737
|
+
.where(eq(taskTable.id, task.id));
|
|
738
|
+
await this.resolveDependenciesMany([{ id: task.id, status: TaskStatus.TimedOut, namespace: task.namespace }], { transaction: tx });
|
|
739
|
+
this.notify();
|
|
617
740
|
}
|
|
618
741
|
return undefined;
|
|
619
742
|
});
|
|
@@ -644,7 +767,7 @@ let PostgresTaskQueue = class PostgresTaskQueue extends TaskQueue {
|
|
|
644
767
|
state: coalesce(updates.updateState, taskTable.state),
|
|
645
768
|
})
|
|
646
769
|
.from(updates)
|
|
647
|
-
.where(and(eq(taskTable.id, updates.updateId),
|
|
770
|
+
.where(and(eq(taskTable.id, updates.updateId), sql `${taskTable.token} IS NOT DISTINCT FROM ${updates.updateToken}`, eq(taskTable.status, TaskStatus.Running), gt(taskTable.startTimestamp, sql `${TRANSACTION_TIMESTAMP} - ${interval(this.maxExecutionTime, 'milliseconds')}`)))
|
|
648
771
|
.returning({ id: taskTable.id }));
|
|
649
772
|
const result = await session
|
|
650
773
|
.with(updates, updated)
|
|
@@ -657,7 +780,7 @@ let PostgresTaskQueue = class PostgresTaskQueue extends TaskQueue {
|
|
|
657
780
|
const [freshTask] = await tx.pgTransaction
|
|
658
781
|
.select({ unresolvedCompleteDependencies: taskTable.unresolvedCompleteDependencies })
|
|
659
782
|
.from(taskTable)
|
|
660
|
-
.where(
|
|
783
|
+
.where(eq(taskTable.id, task.id))
|
|
661
784
|
.for('update');
|
|
662
785
|
if (isUndefined(freshTask)) {
|
|
663
786
|
return;
|
|
@@ -673,7 +796,7 @@ let PostgresTaskQueue = class PostgresTaskQueue extends TaskQueue {
|
|
|
673
796
|
completeTimestamp: (nextStatus == TaskStatus.Completed) ? TRANSACTION_TIMESTAMP : null,
|
|
674
797
|
visibilityDeadline: null,
|
|
675
798
|
})
|
|
676
|
-
.where(and(eq(taskTable.
|
|
799
|
+
.where(and(eq(taskTable.id, task.id), isNull(task.token) ? sqlIsNull(taskTable.token) : eq(taskTable.token, task.token)))
|
|
677
800
|
.returning({ id: taskTable.id });
|
|
678
801
|
if (isUndefined(updatedTask)) {
|
|
679
802
|
return;
|
|
@@ -681,7 +804,7 @@ let PostgresTaskQueue = class PostgresTaskQueue extends TaskQueue {
|
|
|
681
804
|
if (nextStatus == TaskStatus.Completed) {
|
|
682
805
|
await this.#circuitBreaker.recordSuccess();
|
|
683
806
|
}
|
|
684
|
-
await this.resolveDependencies(task.id, nextStatus, { transaction: tx });
|
|
807
|
+
await this.resolveDependencies(task.id, nextStatus, { namespace: task.namespace, transaction: tx });
|
|
685
808
|
});
|
|
686
809
|
}
|
|
687
810
|
async completeMany(tasks, options) {
|
|
@@ -689,43 +812,30 @@ let PostgresTaskQueue = class PostgresTaskQueue extends TaskQueue {
|
|
|
689
812
|
return;
|
|
690
813
|
}
|
|
691
814
|
await this.#repository.useTransaction(options?.transaction, async (tx) => {
|
|
692
|
-
const taskIds = getEntityIds(tasks);
|
|
693
|
-
const freshTasks = await tx.pgTransaction
|
|
694
|
-
.select({ id: taskTable.id, unresolvedCompleteDependencies: taskTable.unresolvedCompleteDependencies })
|
|
695
|
-
.from(taskTable)
|
|
696
|
-
.where(and(eq(taskTable.namespace, this.#namespace), inArray(taskTable.id, taskIds)))
|
|
697
|
-
.for('update');
|
|
698
|
-
const unresolvedDependencyCountMap = new Map(freshTasks.map((task) => [task.id, task.unresolvedCompleteDependencies]));
|
|
699
815
|
const rows = tasks.map((t, i) => {
|
|
700
816
|
const result = options?.results?.[i] ?? null;
|
|
701
|
-
|
|
702
|
-
const nextStatus = (unresolvedCount > 0) ? TaskStatus.WaitingChildren : TaskStatus.Completed;
|
|
703
|
-
const progress = (nextStatus == TaskStatus.Completed) ? 1 : t.progress;
|
|
704
|
-
const completeTimestamp = (nextStatus == TaskStatus.Completed) ? TRANSACTION_TIMESTAMP : null;
|
|
705
|
-
return sql `(${t.id}::uuid, ${t.token}::uuid, ${nextStatus}::text, ${result}::jsonb, ${progress}::numeric, ${completeTimestamp}::timestamptz)`;
|
|
817
|
+
return sql `(${t.id}::uuid, ${t.token}::uuid, ${result}::jsonb, ${t.progress}::numeric)`;
|
|
706
818
|
});
|
|
707
819
|
const updates = tx.pgTransaction.$with('updates').as((qb) => qb
|
|
708
820
|
.select({
|
|
709
821
|
updateId: sql `(id)::uuid`.as('update_id'),
|
|
710
822
|
updateToken: sql `(token)::uuid`.as('update_token'),
|
|
711
|
-
updateStatus: sql `(status)::text`.as('update_status'),
|
|
712
823
|
updateResult: sql `(result)::jsonb`.as('update_result'),
|
|
713
824
|
updateProgress: sql `(progress)::numeric`.as('update_progress'),
|
|
714
|
-
updateComplete: sql `(complete_timestamp)::timestamptz`.as('update_complete'),
|
|
715
825
|
})
|
|
716
|
-
.from(sql `(VALUES ${sql.join(rows, sql `, `)}) AS t(id, token,
|
|
826
|
+
.from(sql `(VALUES ${sql.join(rows, sql `, `)}) AS t(id, token, result, progress)`));
|
|
717
827
|
const updated = tx.pgTransaction.$with('updated').as(() => tx.pgTransaction
|
|
718
828
|
.update(taskTable)
|
|
719
829
|
.set({
|
|
720
|
-
status:
|
|
830
|
+
status: caseWhen(gt(taskTable.unresolvedCompleteDependencies, 0), enumValue(TaskStatus, taskStatus, TaskStatus.WaitingChildren)).else(enumValue(TaskStatus, taskStatus, TaskStatus.Completed)),
|
|
721
831
|
token: null,
|
|
722
832
|
result: updates.updateResult,
|
|
723
|
-
progress:
|
|
724
|
-
completeTimestamp:
|
|
833
|
+
progress: caseWhen(gt(taskTable.unresolvedCompleteDependencies, 0), updates.updateProgress).else(1),
|
|
834
|
+
completeTimestamp: caseWhen(gt(taskTable.unresolvedCompleteDependencies, 0), taskTable.completeTimestamp).else(TRANSACTION_TIMESTAMP),
|
|
725
835
|
visibilityDeadline: null,
|
|
726
836
|
})
|
|
727
837
|
.from(updates)
|
|
728
|
-
.where(and(eq(taskTable.id, updates.updateId),
|
|
838
|
+
.where(and(eq(taskTable.id, updates.updateId), sql `${taskTable.token} IS NOT DISTINCT FROM ${updates.updateToken}`))
|
|
729
839
|
.returning({ id: taskTable.id, status: taskTable.status }));
|
|
730
840
|
const updatedRows = await tx.pgTransaction
|
|
731
841
|
.with(updates, updated)
|
|
@@ -735,15 +845,13 @@ let PostgresTaskQueue = class PostgresTaskQueue extends TaskQueue {
|
|
|
735
845
|
if (updatedRows.some((r) => r.status == TaskStatus.Completed)) {
|
|
736
846
|
await this.#circuitBreaker.recordSuccess();
|
|
737
847
|
}
|
|
738
|
-
|
|
739
|
-
await this.resolveDependencies(row.id, row.status, { transaction: tx });
|
|
740
|
-
}
|
|
848
|
+
await this.resolveDependenciesMany(updatedRows.map((r) => ({ id: r.id, status: r.status })), { transaction: tx });
|
|
741
849
|
}
|
|
742
850
|
});
|
|
743
851
|
}
|
|
744
852
|
async fail(task, error, options) {
|
|
745
853
|
const isRetryable = (options?.fatal != true) && (task.tries < this.maxTries);
|
|
746
|
-
const nextStatus = isRetryable ? TaskStatus.
|
|
854
|
+
const nextStatus = isRetryable ? TaskStatus.Retrying : TaskStatus.Dead;
|
|
747
855
|
const delay = isRetryable
|
|
748
856
|
? Math.min(this.retryDelayMaximum, this.retryDelayMinimum * (this.retryDelayGrowth ** task.tries))
|
|
749
857
|
: 0;
|
|
@@ -760,13 +868,13 @@ let PostgresTaskQueue = class PostgresTaskQueue extends TaskQueue {
|
|
|
760
868
|
startTimestamp: null,
|
|
761
869
|
completeTimestamp: (nextStatus == TaskStatus.Dead) ? TRANSACTION_TIMESTAMP : null,
|
|
762
870
|
})
|
|
763
|
-
.where(and(eq(taskTable.
|
|
871
|
+
.where(and(eq(taskTable.id, task.id), isNull(task.token) ? sqlIsNull(taskTable.token) : eq(taskTable.token, task.token), eq(taskTable.tries, task.tries)))
|
|
764
872
|
.returning();
|
|
765
873
|
if (isUndefined(updatedRow)) {
|
|
766
874
|
return;
|
|
767
875
|
}
|
|
768
876
|
await this.#circuitBreaker.recordFailure();
|
|
769
|
-
await this.resolveDependencies(task.id, nextStatus, { transaction: tx });
|
|
877
|
+
await this.resolveDependencies(task.id, nextStatus, { namespace: task.namespace, transaction: tx });
|
|
770
878
|
});
|
|
771
879
|
}
|
|
772
880
|
async failMany(tasks, errors, options) {
|
|
@@ -777,7 +885,7 @@ let PostgresTaskQueue = class PostgresTaskQueue extends TaskQueue {
|
|
|
777
885
|
const rows = tasks.map((task, index) => {
|
|
778
886
|
const error = errors[index];
|
|
779
887
|
const isRetryable = (task.tries < this.maxTries);
|
|
780
|
-
const nextStatus = isRetryable ? TaskStatus.
|
|
888
|
+
const nextStatus = isRetryable ? TaskStatus.Retrying : TaskStatus.Dead;
|
|
781
889
|
const delay = isRetryable
|
|
782
890
|
? Math.min(this.retryDelayMaximum, this.retryDelayMinimum * (this.retryDelayGrowth ** task.tries))
|
|
783
891
|
: 0;
|
|
@@ -808,148 +916,199 @@ let PostgresTaskQueue = class PostgresTaskQueue extends TaskQueue {
|
|
|
808
916
|
completeTimestamp: sql `${updates.updateComplete}`,
|
|
809
917
|
})
|
|
810
918
|
.from(updates)
|
|
811
|
-
.where(and(eq(taskTable.
|
|
919
|
+
.where(and(eq(taskTable.id, updates.updateId), sql `${taskTable.token} IS NOT DISTINCT FROM ${updates.updateToken}`, eq(taskTable.tries, updates.updateTries)))
|
|
812
920
|
.returning({ id: taskTable.id, status: taskTable.status }));
|
|
813
921
|
const result = await tx.pgTransaction
|
|
814
922
|
.with(updates, updated)
|
|
815
923
|
.select({ id: updated.id, status: updated.status })
|
|
816
924
|
.from(updated);
|
|
817
925
|
if (result.length > 0) {
|
|
818
|
-
await
|
|
819
|
-
|
|
820
|
-
await this.resolveDependencies(row.id, row.status, { transaction: tx });
|
|
821
|
-
}
|
|
926
|
+
await this.#circuitBreaker.recordFailures(result.length);
|
|
927
|
+
await this.resolveDependenciesMany(result.map((r) => ({ id: r.id, status: r.status })), { transaction: tx });
|
|
822
928
|
}
|
|
823
929
|
});
|
|
824
930
|
}
|
|
825
931
|
async resolveDependencies(id, status, options) {
|
|
826
|
-
this
|
|
932
|
+
await this.resolveDependenciesMany([{ id, status, namespace: options?.namespace }], options);
|
|
933
|
+
}
|
|
934
|
+
async resolveDependenciesMany(tasks, options) {
|
|
935
|
+
this.#logger.debug(`Resolving dependencies for ${tasks.length} tasks`);
|
|
936
|
+
if (tasks.length == 0) {
|
|
937
|
+
return;
|
|
938
|
+
}
|
|
939
|
+
const taskStatusMap = new Map(tasks.map((t) => [t.id, t.status]));
|
|
940
|
+
const notifiedNamespaces = new Set();
|
|
827
941
|
await this.#repository.useTransaction(options?.transaction, async (tx) => {
|
|
828
|
-
|
|
942
|
+
const taskIds = tasks.map((t) => t.id);
|
|
829
943
|
const dependents = await tx.pgTransaction
|
|
830
944
|
.select({
|
|
831
945
|
taskId: taskDependencyTable.taskId,
|
|
946
|
+
dependencyTaskId: taskDependencyTable.dependencyTaskId,
|
|
832
947
|
type: taskDependencyTable.type,
|
|
833
948
|
requiredStatuses: taskDependencyTable.requiredStatuses,
|
|
834
949
|
})
|
|
835
950
|
.from(taskDependencyTable)
|
|
836
|
-
.where(
|
|
951
|
+
.where(inArray(taskDependencyTable.dependencyTaskId, taskIds));
|
|
837
952
|
if (dependents.length == 0) {
|
|
838
953
|
return;
|
|
839
954
|
}
|
|
955
|
+
const resolvedEdges = [];
|
|
956
|
+
const failFastTaskIds = new Set();
|
|
840
957
|
for (const dep of dependents) {
|
|
958
|
+
const status = taskStatusMap.get(dep.dependencyTaskId);
|
|
841
959
|
const isMatched = dep.requiredStatuses.includes(status);
|
|
842
|
-
const isTerminal = (status
|
|
960
|
+
const isTerminal = terminalStatuses.includes(status);
|
|
843
961
|
if (isMatched || isTerminal) {
|
|
844
|
-
|
|
845
|
-
|
|
846
|
-
.
|
|
847
|
-
.where(and(eq(taskDependencyTable.namespace, this.#namespace), eq(taskDependencyTable.taskId, dep.taskId), eq(taskDependencyTable.dependencyTaskId, id), eq(taskDependencyTable.type, dep.type)))
|
|
848
|
-
.returning();
|
|
849
|
-
if (deletedEdge.length > 0) {
|
|
850
|
-
const [dependentTask] = await tx.pgTransaction
|
|
851
|
-
.select({ failFast: taskTable.failFast, status: taskTable.status })
|
|
852
|
-
.from(taskTable)
|
|
853
|
-
.where(and(eq(taskTable.namespace, this.#namespace), eq(taskTable.id, dep.taskId)));
|
|
854
|
-
if (isDefined(dependentTask)) {
|
|
855
|
-
if (!isMatched && dependentTask.failFast && dependentTask.status != TaskStatus.Dead && dependentTask.status != TaskStatus.Cancelled) {
|
|
856
|
-
// Dependency failed and failFast is enabled: fail the dependent task immediately
|
|
857
|
-
await this.transitionTask(dep.taskId, TaskStatus.Dead, { error: { code: 'DependencyFailed', message: `Dependency ${id} failed (status: ${status}). Task failed fast because it required one of: ${dep.requiredStatuses.join(', ')}` }, transaction: tx });
|
|
858
|
-
}
|
|
859
|
-
else {
|
|
860
|
-
// Dependency resolved (matched, or failed but no failFast): decrement counter
|
|
861
|
-
const [updatedTask] = await tx.pgTransaction
|
|
862
|
-
.update(taskTable)
|
|
863
|
-
.set((dep.type == TaskDependencyType.Schedule)
|
|
864
|
-
? { unresolvedScheduleDependencies: sql `${taskTable.unresolvedScheduleDependencies} - 1` }
|
|
865
|
-
: { unresolvedCompleteDependencies: sql `${taskTable.unresolvedCompleteDependencies} - 1` })
|
|
866
|
-
.where(and(eq(taskTable.namespace, this.#namespace), eq(taskTable.id, dep.taskId), gt((dep.type == TaskDependencyType.Schedule) ? taskTable.unresolvedScheduleDependencies : taskTable.unresolvedCompleteDependencies, 0)))
|
|
867
|
-
.returning();
|
|
868
|
-
if (isDefined(updatedTask)) {
|
|
869
|
-
// Check for transitions
|
|
870
|
-
if (updatedTask.unresolvedScheduleDependencies == 0 || updatedTask.unresolvedCompleteDependencies == 0) {
|
|
871
|
-
await this.evaluateTaskStatus(updatedTask.id, { transaction: tx });
|
|
872
|
-
}
|
|
873
|
-
}
|
|
874
|
-
}
|
|
875
|
-
}
|
|
962
|
+
resolvedEdges.push(dep);
|
|
963
|
+
if (!isMatched) {
|
|
964
|
+
failFastTaskIds.add(dep.taskId);
|
|
876
965
|
}
|
|
877
966
|
}
|
|
878
967
|
}
|
|
879
|
-
|
|
880
|
-
}
|
|
881
|
-
async evaluateTaskStatus(id, options) {
|
|
882
|
-
await this.#repository.useTransaction(options?.transaction, async (tx) => {
|
|
883
|
-
const [rawTask] = await tx.pgTransaction
|
|
884
|
-
.select()
|
|
885
|
-
.from(taskTable)
|
|
886
|
-
.where(and(eq(taskTable.namespace, this.#namespace), eq(taskTable.id, id)))
|
|
887
|
-
.for('update');
|
|
888
|
-
if (isUndefined(rawTask)) {
|
|
968
|
+
if (resolvedEdges.length == 0) {
|
|
889
969
|
return;
|
|
890
970
|
}
|
|
891
|
-
const
|
|
892
|
-
|
|
893
|
-
|
|
894
|
-
|
|
895
|
-
if (task.unresolvedScheduleDependencies == 0) {
|
|
896
|
-
if (task.unresolvedCompleteDependencies == 0) {
|
|
897
|
-
// If it hasn't started yet (startTimestamp is null), it should be Pending.
|
|
898
|
-
// If it has already run (startTimestamp is set), it should be Completed.
|
|
899
|
-
const nextStatus = isNull(task.startTimestamp) ? TaskStatus.Pending : TaskStatus.Completed;
|
|
900
|
-
if (task.status != nextStatus) {
|
|
901
|
-
await this.transitionTask(id, nextStatus, { transaction: tx });
|
|
902
|
-
}
|
|
971
|
+
const sortedResolvedEdges = resolvedEdges.toSorted((a, b) => {
|
|
972
|
+
const idCompare = a.taskId.localeCompare(b.taskId);
|
|
973
|
+
if (idCompare != 0) {
|
|
974
|
+
return idCompare;
|
|
903
975
|
}
|
|
904
|
-
|
|
905
|
-
|
|
976
|
+
const depIdCompare = a.dependencyTaskId.localeCompare(b.dependencyTaskId);
|
|
977
|
+
if (depIdCompare != 0) {
|
|
978
|
+
return depIdCompare;
|
|
906
979
|
}
|
|
907
|
-
|
|
908
|
-
|
|
909
|
-
|
|
980
|
+
return a.type.localeCompare(b.type);
|
|
981
|
+
});
|
|
982
|
+
const edgeValues = sortedResolvedEdges.map((e) => sql `(${e.taskId}::uuid, ${e.dependencyTaskId}::uuid, ${e.type}::text)`);
|
|
983
|
+
await tx.pgTransaction.execute(sql `
|
|
984
|
+
DELETE FROM ${taskDependencyTable}
|
|
985
|
+
WHERE (task_id, dependency_task_id, type) IN (
|
|
986
|
+
SELECT t.task_id, t.dependency_task_id, t.type::${taskDependencyType}
|
|
987
|
+
FROM (VALUES ${sql.join(edgeValues, sql `, `)}) AS t(task_id, dependency_task_id, type)
|
|
988
|
+
)
|
|
989
|
+
`);
|
|
990
|
+
const terminalTasks = [];
|
|
991
|
+
const skippedTaskIds = new Set();
|
|
992
|
+
if (failFastTaskIds.size > 0) {
|
|
993
|
+
const sortedFailFastIds = [...failFastTaskIds].toSorted();
|
|
994
|
+
const dependentTasks = await tx.pgTransaction
|
|
995
|
+
.select({ id: taskTable.id, namespace: taskTable.namespace, failFast: taskTable.failFast, status: taskTable.status })
|
|
996
|
+
.from(taskTable)
|
|
997
|
+
.where(inArray(taskTable.id, sortedFailFastIds))
|
|
998
|
+
.orderBy(asc(taskTable.id))
|
|
999
|
+
.for('update');
|
|
1000
|
+
const tasksToSkip = [];
|
|
1001
|
+
for (const task of dependentTasks) {
|
|
1002
|
+
if (task.failFast && !terminalStatuses.includes(task.status)) {
|
|
1003
|
+
tasksToSkip.push(task.id);
|
|
1004
|
+
skippedTaskIds.add(task.id);
|
|
1005
|
+
}
|
|
1006
|
+
}
|
|
1007
|
+
if (tasksToSkip.length > 0) {
|
|
1008
|
+
const skippedRows = await tx.pgTransaction
|
|
1009
|
+
.update(taskTable)
|
|
1010
|
+
.set({
|
|
1011
|
+
status: TaskStatus.Skipped,
|
|
1012
|
+
token: null,
|
|
1013
|
+
error: jsonbBuildObject({ code: 'DependencyFailed', message: 'One or more dependencies failed and failFast is enabled' }),
|
|
1014
|
+
completeTimestamp: TRANSACTION_TIMESTAMP,
|
|
1015
|
+
})
|
|
1016
|
+
.where(inArray(taskTable.id, tasksToSkip))
|
|
1017
|
+
.returning({ id: taskTable.id, status: taskTable.status, namespace: taskTable.namespace });
|
|
1018
|
+
terminalTasks.push(...skippedRows.map((r) => ({ id: r.id, status: r.status, namespace: r.namespace })));
|
|
1019
|
+
for (const row of skippedRows) {
|
|
1020
|
+
notifiedNamespaces.add(row.namespace);
|
|
1021
|
+
}
|
|
910
1022
|
}
|
|
911
1023
|
}
|
|
912
|
-
|
|
913
|
-
|
|
914
|
-
|
|
915
|
-
|
|
916
|
-
|
|
917
|
-
|
|
918
|
-
|
|
919
|
-
|
|
920
|
-
|
|
921
|
-
|
|
922
|
-
|
|
1024
|
+
// 4. Group all resolved edges into decrements (for any dependent NOT skipped above)
|
|
1025
|
+
const decrementsToApply = new Map();
|
|
1026
|
+
for (const dep of sortedResolvedEdges) {
|
|
1027
|
+
if (skippedTaskIds.has(dep.taskId)) {
|
|
1028
|
+
continue;
|
|
1029
|
+
}
|
|
1030
|
+
const current = decrementsToApply.get(dep.taskId) ?? { schedule: 0, complete: 0 };
|
|
1031
|
+
if (dep.type == TaskDependencyType.Schedule) {
|
|
1032
|
+
current.schedule++;
|
|
1033
|
+
}
|
|
1034
|
+
else if (dep.type == TaskDependencyType.Complete || dep.type == TaskDependencyType.Child) {
|
|
1035
|
+
current.complete++;
|
|
1036
|
+
}
|
|
1037
|
+
decrementsToApply.set(dep.taskId, current);
|
|
923
1038
|
}
|
|
924
|
-
if (
|
|
925
|
-
|
|
926
|
-
|
|
1039
|
+
if (decrementsToApply.size > 0) {
|
|
1040
|
+
const decrementValues = [...decrementsToApply]
|
|
1041
|
+
.toSorted(([idA], [idB]) => idA.localeCompare(idB))
|
|
1042
|
+
.map(([taskId, d]) => sql `(${taskId}::uuid, ${d.schedule}::int, ${d.complete}::int)`);
|
|
1043
|
+
const updates = tx.pgTransaction.$with('updates').as((qb) => qb
|
|
1044
|
+
.select({
|
|
1045
|
+
taskId: sql `(id)::uuid`.as('task_id'),
|
|
1046
|
+
scheduleDecrement: sql `(schedule)::int`.as('schedule_decrement'),
|
|
1047
|
+
completeDecrement: sql `(complete)::int`.as('complete_decrement'),
|
|
1048
|
+
})
|
|
1049
|
+
.from(sql `(VALUES ${sql.join(decrementValues, sql `, `)}) AS t(id, schedule, complete)`));
|
|
1050
|
+
const updatedRows = await tx.pgTransaction
|
|
1051
|
+
.with(updates)
|
|
1052
|
+
.update(taskTable)
|
|
1053
|
+
.set({
|
|
1054
|
+
unresolvedScheduleDependencies: greatest(0, sql `${taskTable.unresolvedScheduleDependencies} - ${updates.scheduleDecrement}`),
|
|
1055
|
+
unresolvedCompleteDependencies: greatest(0, sql `${taskTable.unresolvedCompleteDependencies} - ${updates.completeDecrement}`),
|
|
1056
|
+
status: caseWhen(and(eq(taskTable.status, TaskStatus.Waiting), eq(greatest(0, sql `${taskTable.unresolvedScheduleDependencies} - ${updates.scheduleDecrement}`), 0)), enumValue(TaskStatus, taskStatus, TaskStatus.Pending)).else(caseWhen(and(eq(taskTable.status, TaskStatus.WaitingChildren), eq(greatest(0, sql `${taskTable.unresolvedCompleteDependencies} - ${updates.completeDecrement}`), 0)), enumValue(TaskStatus, taskStatus, TaskStatus.Completed)).else(taskTable.status)),
|
|
1057
|
+
progress: caseWhen(and(eq(taskTable.status, TaskStatus.WaitingChildren), eq(greatest(0, sql `${taskTable.unresolvedCompleteDependencies} - ${updates.completeDecrement}`), 0)), 1).else(taskTable.progress),
|
|
1058
|
+
completeTimestamp: caseWhen(and(eq(taskTable.status, TaskStatus.WaitingChildren), eq(greatest(0, sql `${taskTable.unresolvedCompleteDependencies} - ${updates.completeDecrement}`), 0)), TRANSACTION_TIMESTAMP).else(taskTable.completeTimestamp),
|
|
1059
|
+
token: caseWhen(and(eq(taskTable.status, TaskStatus.WaitingChildren), eq(greatest(0, sql `${taskTable.unresolvedCompleteDependencies} - ${updates.completeDecrement}`), 0)), null).else(taskTable.token),
|
|
1060
|
+
})
|
|
1061
|
+
.from(updates)
|
|
1062
|
+
.where(eq(taskTable.id, updates.taskId))
|
|
1063
|
+
.returning({
|
|
1064
|
+
id: taskTable.id,
|
|
1065
|
+
namespace: taskTable.namespace,
|
|
1066
|
+
status: taskTable.status,
|
|
1067
|
+
});
|
|
1068
|
+
for (const row of updatedRows) {
|
|
1069
|
+
if (finalizedStatuses.includes(row.status)) {
|
|
1070
|
+
terminalTasks.push({ id: row.id, status: row.status, namespace: row.namespace });
|
|
1071
|
+
}
|
|
1072
|
+
notifiedNamespaces.add(row.namespace);
|
|
1073
|
+
}
|
|
927
1074
|
}
|
|
928
|
-
|
|
929
|
-
.
|
|
930
|
-
.
|
|
931
|
-
|
|
932
|
-
|
|
933
|
-
|
|
934
|
-
await this.resolveDependencies(id, status, { transaction: tx });
|
|
935
|
-
this.#messageBus.publishAndForget();
|
|
1075
|
+
if (terminalTasks.length > 0) {
|
|
1076
|
+
const terminalTaskIds = terminalTasks.map((t) => t.id);
|
|
1077
|
+
await tx.pgTransaction
|
|
1078
|
+
.delete(taskDependencyTable)
|
|
1079
|
+
.where(inArray(taskDependencyTable.taskId, terminalTaskIds));
|
|
1080
|
+
await this.resolveDependenciesMany(terminalTasks, { transaction: tx });
|
|
936
1081
|
}
|
|
937
1082
|
});
|
|
1083
|
+
for (const namespace of notifiedNamespaces) {
|
|
1084
|
+
this.notify(namespace);
|
|
1085
|
+
}
|
|
938
1086
|
}
|
|
939
1087
|
async maintenance(options) {
|
|
940
|
-
|
|
941
|
-
|
|
942
|
-
|
|
1088
|
+
await Promise.allSettled([
|
|
1089
|
+
this.processExpirations(options),
|
|
1090
|
+
this.processZombieRetries(options),
|
|
1091
|
+
this.processZombieExhaustions(options),
|
|
1092
|
+
this.processHardTimeouts(options),
|
|
1093
|
+
this.processPriorityAging(options),
|
|
1094
|
+
]);
|
|
1095
|
+
await this.performArchival(options);
|
|
1096
|
+
await this.performArchivePurge(options);
|
|
1097
|
+
}
|
|
1098
|
+
async performArchival(options) {
|
|
943
1099
|
while (true) {
|
|
944
|
-
const archivedCount = await repository.useTransaction(options?.transaction, async (tx) => {
|
|
1100
|
+
const archivedCount = await this.#repository.useTransaction(options?.transaction, async (tx) => {
|
|
945
1101
|
const childTaskTable = aliasedTable(taskTable, 'childTask');
|
|
946
1102
|
const rowsToArchive = await tx.pgTransaction
|
|
947
1103
|
.select()
|
|
948
1104
|
.from(taskTable)
|
|
949
|
-
.where(and(eq(taskTable.namespace, this.#namespace), inArray(taskTable.status,
|
|
1105
|
+
.where(and(eq(taskTable.namespace, this.#namespace), inArray(taskTable.status, finalizedStatuses), lte(taskTable.completeTimestamp, sql `${TRANSACTION_TIMESTAMP} - ${interval(this.retention, 'milliseconds')}`), notExists(tx.pgTransaction
|
|
950
1106
|
.select({ id: childTaskTable.id })
|
|
951
1107
|
.from(childTaskTable)
|
|
952
|
-
.where(
|
|
1108
|
+
.where(eq(childTaskTable.parentId, taskTable.id))), notExists(tx.pgTransaction
|
|
1109
|
+
.select({ taskId: taskDependencyTable.taskId })
|
|
1110
|
+
.from(taskDependencyTable)
|
|
1111
|
+
.where(eq(taskDependencyTable.dependencyTaskId, taskTable.id)))))
|
|
953
1112
|
.limit(1000)
|
|
954
1113
|
.for('update', { skipLocked: true });
|
|
955
1114
|
if (rowsToArchive.length > 0) {
|
|
@@ -963,144 +1122,163 @@ let PostgresTaskQueue = class PostgresTaskQueue extends TaskQueue {
|
|
|
963
1122
|
break;
|
|
964
1123
|
}
|
|
965
1124
|
}
|
|
966
|
-
|
|
1125
|
+
}
|
|
1126
|
+
async performArchivePurge(options) {
|
|
1127
|
+
const session = options?.transaction?.pgTransaction ?? this.#database;
|
|
1128
|
+
const selection = session.$with('archive_purge_selection').as((qb) => qb
|
|
1129
|
+
.select({ id: taskArchiveTable.id })
|
|
1130
|
+
.from(taskArchiveTable)
|
|
1131
|
+
.where(and(eq(taskArchiveTable.namespace, this.#namespace), lte(taskArchiveTable.completeTimestamp, sql `${TRANSACTION_TIMESTAMP} - ${interval(this.archiveRetention, 'milliseconds')}`)))
|
|
1132
|
+
.limit(1000));
|
|
967
1133
|
while (true) {
|
|
968
|
-
const
|
|
969
|
-
|
|
970
|
-
|
|
971
|
-
|
|
972
|
-
|
|
973
|
-
|
|
974
|
-
const result = await tx.pgTransaction
|
|
975
|
-
.with(selection)
|
|
976
|
-
.delete(taskArchiveTable)
|
|
977
|
-
.where(inArray(taskArchiveTable.id, tx.pgTransaction.select().from(selection)))
|
|
978
|
-
.returning({ id: taskArchiveTable.id });
|
|
979
|
-
return result.length;
|
|
980
|
-
});
|
|
981
|
-
if (deletedArchiveCount < 1000) {
|
|
1134
|
+
const result = await session
|
|
1135
|
+
.with(selection)
|
|
1136
|
+
.delete(taskArchiveTable)
|
|
1137
|
+
.where(inArray(taskArchiveTable.id, session.select().from(selection)))
|
|
1138
|
+
.returning({ id: taskArchiveTable.id });
|
|
1139
|
+
if (result.length < 1000) {
|
|
982
1140
|
break;
|
|
983
1141
|
}
|
|
984
1142
|
}
|
|
985
|
-
|
|
1143
|
+
}
|
|
1144
|
+
async processExpirations(options) {
|
|
1145
|
+
const expiredSelection = this.#database.$with('expired_selection').as((qb) => qb
|
|
1146
|
+
.select({ id: taskTable.id })
|
|
1147
|
+
.from(taskTable)
|
|
1148
|
+
.where(and(eq(taskTable.namespace, this.#namespace), inArray(taskTable.status, queueableOrWaitableStatuses), lt(taskTable.timeToLive, TRANSACTION_TIMESTAMP)))
|
|
1149
|
+
.limit(1000)
|
|
1150
|
+
.for('update', { skipLocked: true }));
|
|
986
1151
|
while (true) {
|
|
987
|
-
const
|
|
988
|
-
let totalUpdated = 0;
|
|
989
|
-
// 3.1 Handle Pending Expiration
|
|
990
|
-
const expiredSelection = tx.pgTransaction.$with('expired_selection').as((qb) => qb
|
|
991
|
-
.select({ id: taskTable.id })
|
|
992
|
-
.from(taskTable)
|
|
993
|
-
.where(and(eq(taskTable.namespace, this.#namespace), eq(taskTable.status, TaskStatus.Pending), lt(taskTable.timeToLive, TRANSACTION_TIMESTAMP)))
|
|
994
|
-
.limit(1000)
|
|
995
|
-
.for('update', { skipLocked: true }));
|
|
1152
|
+
const updatedCount = await this.#repository.useTransaction(options?.transaction, async (tx) => {
|
|
996
1153
|
const expiredRows = await tx.pgTransaction
|
|
997
1154
|
.with(expiredSelection)
|
|
998
1155
|
.update(taskTable)
|
|
999
1156
|
.set({
|
|
1000
|
-
status: TaskStatus.
|
|
1157
|
+
status: TaskStatus.Expired,
|
|
1001
1158
|
token: null,
|
|
1002
1159
|
error: { code: 'Expired', message: 'Task expired before processing' },
|
|
1003
1160
|
completeTimestamp: TRANSACTION_TIMESTAMP,
|
|
1004
1161
|
})
|
|
1005
1162
|
.where(inArray(taskTable.id, tx.pgTransaction.select().from(expiredSelection)))
|
|
1006
1163
|
.returning({ id: taskTable.id });
|
|
1007
|
-
totalUpdated += expiredRows.length;
|
|
1008
1164
|
if (expiredRows.length > 0) {
|
|
1009
|
-
|
|
1010
|
-
await this.resolveDependencies(row.id, TaskStatus.Dead, { transaction: tx });
|
|
1011
|
-
}
|
|
1165
|
+
await this.resolveDependenciesMany(expiredRows.map((r) => ({ id: r.id, status: TaskStatus.Expired, namespace: this.#namespace })), { transaction: tx });
|
|
1012
1166
|
}
|
|
1013
|
-
|
|
1014
|
-
|
|
1015
|
-
|
|
1016
|
-
|
|
1017
|
-
|
|
1018
|
-
|
|
1019
|
-
|
|
1020
|
-
|
|
1021
|
-
|
|
1022
|
-
|
|
1023
|
-
|
|
1024
|
-
|
|
1025
|
-
|
|
1026
|
-
|
|
1027
|
-
|
|
1028
|
-
|
|
1029
|
-
|
|
1030
|
-
|
|
1031
|
-
|
|
1032
|
-
|
|
1033
|
-
|
|
1034
|
-
|
|
1035
|
-
|
|
1036
|
-
|
|
1037
|
-
|
|
1038
|
-
|
|
1039
|
-
|
|
1040
|
-
|
|
1167
|
+
return expiredRows.length;
|
|
1168
|
+
});
|
|
1169
|
+
if (updatedCount < 1000) {
|
|
1170
|
+
break;
|
|
1171
|
+
}
|
|
1172
|
+
}
|
|
1173
|
+
}
|
|
1174
|
+
async processZombieRetries(options) {
|
|
1175
|
+
const session = options?.transaction?.pgTransaction ?? this.#database;
|
|
1176
|
+
const zombieRetrySelection = session.$with('zombie_retry_selection').as((qb) => qb
|
|
1177
|
+
.select({ id: taskTable.id })
|
|
1178
|
+
.from(taskTable)
|
|
1179
|
+
.where(and(eq(taskTable.namespace, this.#namespace), eq(taskTable.status, TaskStatus.Running), lt(taskTable.visibilityDeadline, TRANSACTION_TIMESTAMP), lt(taskTable.tries, this.maxTries)))
|
|
1180
|
+
.limit(1000)
|
|
1181
|
+
.for('update', { skipLocked: true }));
|
|
1182
|
+
while (true) {
|
|
1183
|
+
const zombieRetryRows = await session
|
|
1184
|
+
.with(zombieRetrySelection)
|
|
1185
|
+
.update(taskTable)
|
|
1186
|
+
.set({
|
|
1187
|
+
status: TaskStatus.Retrying,
|
|
1188
|
+
token: null,
|
|
1189
|
+
visibilityDeadline: null,
|
|
1190
|
+
startTimestamp: null,
|
|
1191
|
+
scheduleTimestamp: sql `${TRANSACTION_TIMESTAMP} + ${interval(this.retryDelayMinimum, 'milliseconds')}`, // Simple backoff for zombies
|
|
1192
|
+
error: jsonbBuildObject({ code: 'VisibilityTimeout', message: 'Worker Lost', lastError: taskTable.error }),
|
|
1193
|
+
})
|
|
1194
|
+
.where(inArray(taskTable.id, session.select().from(zombieRetrySelection)))
|
|
1195
|
+
.returning({ id: taskTable.id });
|
|
1196
|
+
if (zombieRetryRows.length < 1000) {
|
|
1197
|
+
break;
|
|
1198
|
+
}
|
|
1199
|
+
}
|
|
1200
|
+
}
|
|
1201
|
+
async processZombieExhaustions(options) {
|
|
1202
|
+
const zombieExhaustionSelection = this.#database.$with('zombie_exhaustion_selection').as((qb) => qb
|
|
1203
|
+
.select({ id: taskTable.id })
|
|
1204
|
+
.from(taskTable)
|
|
1205
|
+
.where(and(eq(taskTable.namespace, this.#namespace), eq(taskTable.status, TaskStatus.Running), lt(taskTable.visibilityDeadline, TRANSACTION_TIMESTAMP), gte(taskTable.tries, this.maxTries)))
|
|
1206
|
+
.limit(1000)
|
|
1207
|
+
.for('update', { skipLocked: true }));
|
|
1208
|
+
while (true) {
|
|
1209
|
+
const updatedCount = await this.#repository.useTransaction(options?.transaction, async (tx) => {
|
|
1041
1210
|
const exhaustionRows = await tx.pgTransaction
|
|
1042
1211
|
.with(zombieExhaustionSelection)
|
|
1043
1212
|
.update(taskTable)
|
|
1044
1213
|
.set({
|
|
1045
|
-
status: TaskStatus.
|
|
1214
|
+
status: TaskStatus.Orphaned,
|
|
1046
1215
|
token: null,
|
|
1047
1216
|
visibilityDeadline: null,
|
|
1048
1217
|
completeTimestamp: TRANSACTION_TIMESTAMP,
|
|
1049
|
-
error:
|
|
1218
|
+
error: jsonbBuildObject({ code: 'ZombieExhausted', message: 'Exceeded max retries after repeated crashes', lastError: taskTable.error }),
|
|
1050
1219
|
})
|
|
1051
1220
|
.where(inArray(taskTable.id, tx.pgTransaction.select().from(zombieExhaustionSelection)))
|
|
1052
1221
|
.returning({ id: taskTable.id });
|
|
1053
|
-
totalUpdated += exhaustionRows.length;
|
|
1054
1222
|
if (exhaustionRows.length > 0) {
|
|
1055
|
-
|
|
1056
|
-
await this.resolveDependencies(row.id, TaskStatus.Dead, { transaction: tx });
|
|
1057
|
-
}
|
|
1223
|
+
await this.resolveDependenciesMany(exhaustionRows.map((r) => ({ id: r.id, status: TaskStatus.Orphaned, namespace: this.#namespace })), { transaction: tx });
|
|
1058
1224
|
}
|
|
1059
|
-
|
|
1060
|
-
|
|
1061
|
-
|
|
1062
|
-
|
|
1063
|
-
|
|
1064
|
-
|
|
1065
|
-
|
|
1225
|
+
return exhaustionRows.length;
|
|
1226
|
+
});
|
|
1227
|
+
if (updatedCount < 1000) {
|
|
1228
|
+
break;
|
|
1229
|
+
}
|
|
1230
|
+
}
|
|
1231
|
+
}
|
|
1232
|
+
async processHardTimeouts(options) {
|
|
1233
|
+
const timeoutSelection = this.#database.$with('timeout_selection').as((qb) => qb
|
|
1234
|
+
.select({ id: taskTable.id })
|
|
1235
|
+
.from(taskTable)
|
|
1236
|
+
.where(and(eq(taskTable.namespace, this.#namespace), eq(taskTable.status, TaskStatus.Running), lt(taskTable.startTimestamp, sql `${TRANSACTION_TIMESTAMP} - ${interval(this.maxExecutionTime, 'milliseconds')}`)))
|
|
1237
|
+
.limit(1000)
|
|
1238
|
+
.for('update', { skipLocked: true }));
|
|
1239
|
+
while (true) {
|
|
1240
|
+
const updatedCount = await this.#repository.useTransaction(options?.transaction, async (tx) => {
|
|
1066
1241
|
const timeoutRows = await tx.pgTransaction
|
|
1067
1242
|
.with(timeoutSelection)
|
|
1068
1243
|
.update(taskTable)
|
|
1069
1244
|
.set({
|
|
1070
|
-
status: TaskStatus.
|
|
1245
|
+
status: TaskStatus.TimedOut,
|
|
1071
1246
|
token: null,
|
|
1072
1247
|
visibilityDeadline: null,
|
|
1073
1248
|
completeTimestamp: TRANSACTION_TIMESTAMP,
|
|
1074
|
-
error:
|
|
1249
|
+
error: jsonbBuildObject({ code: 'MaxTimeExceeded', message: sql `'Hard Execution Timeout: Task ran longer than ' || ${this.maxExecutionTime} || 'ms'`, lastError: taskTable.error }),
|
|
1075
1250
|
})
|
|
1076
1251
|
.where(inArray(taskTable.id, tx.pgTransaction.select().from(timeoutSelection)))
|
|
1077
1252
|
.returning({ id: taskTable.id });
|
|
1078
|
-
totalUpdated += timeoutRows.length;
|
|
1079
1253
|
if (timeoutRows.length > 0) {
|
|
1080
|
-
|
|
1081
|
-
await this.resolveDependencies(row.id, TaskStatus.Dead, { transaction: tx });
|
|
1082
|
-
}
|
|
1254
|
+
await this.resolveDependenciesMany(timeoutRows.map((r) => ({ id: r.id, status: TaskStatus.TimedOut, namespace: this.#namespace })), { transaction: tx });
|
|
1083
1255
|
}
|
|
1084
|
-
|
|
1085
|
-
const agingSelection = tx.pgTransaction.$with('aging_selection').as((qb) => qb
|
|
1086
|
-
.select({ id: taskTable.id })
|
|
1087
|
-
.from(taskTable)
|
|
1088
|
-
.where(and(eq(taskTable.namespace, this.#namespace), eq(taskTable.status, TaskStatus.Pending), lt(taskTable.priorityAgeTimestamp, sql `${TRANSACTION_TIMESTAMP} - ${interval(this.priorityAgingInterval, 'milliseconds')}`)))
|
|
1089
|
-
.limit(1000)
|
|
1090
|
-
.for('update', { skipLocked: true }));
|
|
1091
|
-
const agingRows = await tx.pgTransaction
|
|
1092
|
-
.with(agingSelection)
|
|
1093
|
-
.update(taskTable)
|
|
1094
|
-
.set({
|
|
1095
|
-
priority: sql `${taskTable.priority} - ${this.priorityAgingStep}`,
|
|
1096
|
-
priorityAgeTimestamp: TRANSACTION_TIMESTAMP,
|
|
1097
|
-
})
|
|
1098
|
-
.where(inArray(taskTable.id, tx.pgTransaction.select().from(agingSelection)))
|
|
1099
|
-
.returning({ id: taskTable.id });
|
|
1100
|
-
totalUpdated += agingRows.length;
|
|
1101
|
-
return totalUpdated;
|
|
1256
|
+
return timeoutRows.length;
|
|
1102
1257
|
});
|
|
1103
|
-
if (
|
|
1258
|
+
if (updatedCount < 1000) {
|
|
1259
|
+
break;
|
|
1260
|
+
}
|
|
1261
|
+
}
|
|
1262
|
+
}
|
|
1263
|
+
async processPriorityAging(options) {
|
|
1264
|
+
const session = options?.transaction?.pgTransaction ?? this.#database;
|
|
1265
|
+
const agingSelection = session.$with('aging_selection').as((qb) => qb
|
|
1266
|
+
.select({ id: taskTable.id })
|
|
1267
|
+
.from(taskTable)
|
|
1268
|
+
.where(and(eq(taskTable.namespace, this.#namespace), inArray(taskTable.status, queueableStatuses), lt(taskTable.priorityAgeTimestamp, sql `${TRANSACTION_TIMESTAMP} - ${interval(this.priorityAgingInterval, 'milliseconds')}`)))
|
|
1269
|
+
.limit(1000)
|
|
1270
|
+
.for('update', { skipLocked: true }));
|
|
1271
|
+
while (true) {
|
|
1272
|
+
const agingRows = await session
|
|
1273
|
+
.with(agingSelection)
|
|
1274
|
+
.update(taskTable)
|
|
1275
|
+
.set({
|
|
1276
|
+
priority: sql `${taskTable.priority} - ${this.priorityAgingStep}`,
|
|
1277
|
+
priorityAgeTimestamp: TRANSACTION_TIMESTAMP,
|
|
1278
|
+
})
|
|
1279
|
+
.where(inArray(taskTable.id, session.select().from(agingSelection)))
|
|
1280
|
+
.returning({ id: taskTable.id });
|
|
1281
|
+
if (agingRows.length < 1000) {
|
|
1104
1282
|
break;
|
|
1105
1283
|
}
|
|
1106
1284
|
}
|
|
@@ -1122,44 +1300,87 @@ let PostgresTaskQueue = class PostgresTaskQueue extends TaskQueue {
|
|
|
1122
1300
|
priorityAgeTimestamp: TRANSACTION_TIMESTAMP,
|
|
1123
1301
|
state: (options?.resetState == true) ? null : undefined,
|
|
1124
1302
|
})
|
|
1125
|
-
.where(and(eq(taskTable.
|
|
1303
|
+
.where(and(eq(taskTable.id, id), or(inArray(taskTable.status, queueableStatuses), inArray(taskTable.status, finalizedStatuses), lt(taskTable.visibilityDeadline, TRANSACTION_TIMESTAMP))));
|
|
1126
1304
|
}
|
|
1127
|
-
notify() {
|
|
1128
|
-
this.#messageBus.publishAndForget();
|
|
1305
|
+
notify(namespace = this.#namespace) {
|
|
1306
|
+
this.#messageBus.publishAndForget(namespace);
|
|
1129
1307
|
}
|
|
1130
1308
|
async *getConsumer(cancellationSignal, options) {
|
|
1131
|
-
const continue$ = merge(this.#messageBus.allMessages
|
|
1309
|
+
const continue$ = merge(this.#messageBus.allMessages$).pipe(filter((namespace) => namespace == this.#namespace));
|
|
1310
|
+
const mergedContinue$ = merge(continue$, cancellationSignal);
|
|
1132
1311
|
while (cancellationSignal.isUnset) {
|
|
1133
1312
|
const task = await this.dequeue(options);
|
|
1134
1313
|
if (isDefined(task)) {
|
|
1135
1314
|
yield task;
|
|
1136
1315
|
continue;
|
|
1137
1316
|
}
|
|
1138
|
-
await cancelableTimeout(5 * millisecondsPerSecond,
|
|
1317
|
+
await cancelableTimeout(5 * millisecondsPerSecond, mergedContinue$);
|
|
1318
|
+
}
|
|
1319
|
+
}
|
|
1320
|
+
maintenanceLoop() {
|
|
1321
|
+
void this.lowFrequencyMaintenanceLoop();
|
|
1322
|
+
void this.mediumFrequencyMaintenanceLoop();
|
|
1323
|
+
void this.highFrequencyMaintenanceLoop();
|
|
1324
|
+
}
|
|
1325
|
+
async lowFrequencyMaintenanceLoop() {
|
|
1326
|
+
while (this.#cancellationSignal.isUnset) {
|
|
1327
|
+
try {
|
|
1328
|
+
await Promise.allSettled([
|
|
1329
|
+
this.performArchival(),
|
|
1330
|
+
this.performArchivePurge(),
|
|
1331
|
+
]);
|
|
1332
|
+
}
|
|
1333
|
+
catch (error) {
|
|
1334
|
+
this.#logger.error('Error during low frequency maintenance loop', error);
|
|
1335
|
+
}
|
|
1336
|
+
finally {
|
|
1337
|
+
await cancelableTimeout(5 * millisecondsPerMinute, this.#cancellationSignal);
|
|
1338
|
+
}
|
|
1339
|
+
}
|
|
1340
|
+
}
|
|
1341
|
+
async mediumFrequencyMaintenanceLoop() {
|
|
1342
|
+
while (this.#cancellationSignal.isUnset) {
|
|
1343
|
+
try {
|
|
1344
|
+
await Promise.allSettled([
|
|
1345
|
+
this.processExpirations(),
|
|
1346
|
+
this.processPriorityAging(),
|
|
1347
|
+
]);
|
|
1348
|
+
}
|
|
1349
|
+
catch (error) {
|
|
1350
|
+
this.#logger.error('Error during medium frequency maintenance loop', error);
|
|
1351
|
+
}
|
|
1352
|
+
finally {
|
|
1353
|
+
await cancelableTimeout(millisecondsPerMinute, this.#cancellationSignal);
|
|
1354
|
+
}
|
|
1139
1355
|
}
|
|
1140
1356
|
}
|
|
1141
|
-
async
|
|
1357
|
+
async highFrequencyMaintenanceLoop() {
|
|
1142
1358
|
while (this.#cancellationSignal.isUnset) {
|
|
1143
1359
|
try {
|
|
1144
|
-
await
|
|
1360
|
+
await Promise.allSettled([
|
|
1361
|
+
this.processZombieRetries(),
|
|
1362
|
+
this.processZombieExhaustions(),
|
|
1363
|
+
this.processHardTimeouts(),
|
|
1364
|
+
]);
|
|
1145
1365
|
}
|
|
1146
1366
|
catch (error) {
|
|
1147
|
-
this.#logger.error('Error during maintenance loop', error);
|
|
1367
|
+
this.#logger.error('Error during high frequency maintenance loop', error);
|
|
1148
1368
|
}
|
|
1149
1369
|
finally {
|
|
1150
|
-
await cancelableTimeout(
|
|
1370
|
+
await cancelableTimeout(10 * millisecondsPerSecond, this.#cancellationSignal);
|
|
1151
1371
|
}
|
|
1152
1372
|
}
|
|
1153
1373
|
}
|
|
1154
1374
|
async *getBatchConsumer(size, cancellationSignal, options) {
|
|
1155
|
-
const continue$ = merge(this.#messageBus.allMessages
|
|
1375
|
+
const continue$ = merge(this.#messageBus.allMessages$).pipe(filter((namespace) => namespace == this.#namespace));
|
|
1376
|
+
const mergedContinue$ = merge(continue$, cancellationSignal);
|
|
1156
1377
|
while (cancellationSignal.isUnset) {
|
|
1157
1378
|
const tasks = await this.dequeueMany(size, options);
|
|
1158
1379
|
if (tasks.length > 0) {
|
|
1159
1380
|
yield tasks;
|
|
1160
1381
|
continue;
|
|
1161
1382
|
}
|
|
1162
|
-
await cancelableTimeout(5 * millisecondsPerSecond,
|
|
1383
|
+
await cancelableTimeout(5 * millisecondsPerSecond, mergedContinue$);
|
|
1163
1384
|
}
|
|
1164
1385
|
}
|
|
1165
1386
|
};
|