@tstdl/base 0.93.141 → 0.93.143
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/authentication/client/authentication.service.d.ts +1 -0
- package/authentication/client/authentication.service.js +3 -2
- package/circuit-breaker/circuit-breaker.d.ts +6 -4
- package/circuit-breaker/postgres/circuit-breaker.d.ts +1 -0
- package/circuit-breaker/postgres/circuit-breaker.js +8 -5
- package/circuit-breaker/tests/circuit-breaker.test.js +20 -0
- package/examples/document-management/main.js +2 -2
- package/notification/tests/notification-api.test.js +5 -1
- package/notification/tests/notification-flow.test.js +9 -6
- package/orm/decorators.d.ts +17 -4
- package/orm/decorators.js +9 -0
- package/orm/server/bootstrap.d.ts +11 -0
- package/orm/server/bootstrap.js +31 -0
- package/orm/server/drizzle/schema-converter.d.ts +3 -1
- package/orm/server/drizzle/schema-converter.js +71 -29
- package/orm/server/extension.d.ts +14 -0
- package/orm/server/extension.js +27 -0
- package/orm/server/index.d.ts +2 -0
- package/orm/server/index.js +2 -0
- package/orm/server/migration.d.ts +2 -3
- package/orm/server/migration.js +7 -21
- package/orm/server/repository.d.ts +1 -0
- package/orm/server/repository.js +19 -9
- package/orm/server/transaction.d.ts +1 -0
- package/orm/server/transaction.js +3 -0
- package/orm/tests/database-extension.test.js +63 -0
- package/orm/tests/database-migration.test.js +7 -6
- package/orm/tests/repository-compound-primary-key.test.d.ts +2 -0
- package/orm/tests/repository-compound-primary-key.test.js +234 -0
- package/orm/tests/schema-generation.test.d.ts +1 -0
- package/orm/tests/schema-generation.test.js +52 -5
- package/package.json +4 -4
- package/task-queue/README.md +0 -1
- package/task-queue/postgres/drizzle/0000_faithful_daimon_hellstrom.sql +84 -0
- package/task-queue/postgres/drizzle/meta/0000_snapshot.json +155 -72
- package/task-queue/postgres/drizzle/meta/_journal.json +2 -2
- package/task-queue/postgres/module.js +2 -1
- package/task-queue/postgres/schemas.d.ts +6 -0
- package/task-queue/postgres/task-queue.d.ts +18 -5
- package/task-queue/postgres/task-queue.js +577 -357
- package/task-queue/postgres/task.model.d.ts +10 -6
- package/task-queue/postgres/task.model.js +28 -28
- package/task-queue/task-context.d.ts +10 -5
- package/task-queue/task-context.js +5 -3
- package/task-queue/task-queue.d.ts +342 -38
- package/task-queue/task-queue.js +135 -31
- package/task-queue/tests/coverage-branch.test.js +46 -58
- package/task-queue/tests/coverage-enhancement.test.js +123 -117
- package/task-queue/tests/{extensive-dependencies.test.js → dag.test.js} +66 -37
- package/task-queue/tests/dependencies.test.js +143 -25
- package/task-queue/tests/enqueue-batch.test.js +125 -0
- package/task-queue/tests/fan-out-spawning.test.js +46 -5
- package/task-queue/tests/idempotent-replacement.test.js +54 -1
- package/task-queue/tests/missing-idempotent-tasks.test.js +9 -8
- package/task-queue/tests/queue.test.js +261 -25
- package/task-queue/tests/shutdown.test.js +41 -0
- package/task-queue/tests/transactions.test.d.ts +1 -0
- package/task-queue/tests/transactions.test.js +47 -0
- package/task-queue/tests/worker.test.js +46 -13
- package/task-queue/tests/zombie-parent.test.js +4 -4
- package/task-queue/tests/zombie-recovery.test.js +3 -3
- package/testing/integration-setup.js +5 -3
- package/utils/timing.d.ts +2 -2
- package/task-queue/postgres/drizzle/0000_wakeful_sunspot.sql +0 -82
- package/task-queue/tests/cascading-cancellations.test.js +0 -38
- package/task-queue/tests/complex.test.js +0 -122
- package/task-queue/tests/dag-dependencies.test.js +0 -41
- /package/{task-queue/tests/cascading-cancellations.test.d.ts → orm/tests/database-extension.test.d.ts} +0 -0
- /package/task-queue/tests/{complex.test.d.ts → dag.test.d.ts} +0 -0
- /package/task-queue/tests/{dag-dependencies.test.d.ts → enqueue-batch.test.d.ts} +0 -0
- /package/task-queue/tests/{extensive-dependencies.test.d.ts → shutdown.test.d.ts} +0 -0
|
@@ -56,33 +56,34 @@ var __disposeResources = (this && this.__disposeResources) || (function (Suppres
|
|
|
56
56
|
var e = new Error(message);
|
|
57
57
|
return e.name = "SuppressedError", e.error = error, e.suppressed = suppressed, e;
|
|
58
58
|
});
|
|
59
|
-
import { aliasedTable, and, asc, count, eq, gt, gte, inArray, lt, lte, notExists, or, sql, count as sqlCount, isNull as sqlIsNull } from 'drizzle-orm';
|
|
60
|
-
import { merge, throttleTime } from 'rxjs';
|
|
59
|
+
import { aliasedTable, and, asc, count, eq, gt, gte, inArray, lt, lte, notExists, notInArray, or, sql, count as sqlCount, isNotNull as sqlIsNotNull, isNull as sqlIsNull } from 'drizzle-orm';
|
|
60
|
+
import { filter, merge, throttleTime } from 'rxjs';
|
|
61
61
|
import { CancellationSignal } from '../../cancellation/index.js';
|
|
62
62
|
import { CircuitBreaker, CircuitBreakerState } from '../../circuit-breaker/index.js';
|
|
63
63
|
import { serializeError, TimeoutError } from '../../errors/index.js';
|
|
64
64
|
import { afterResolve, inject, provide, Singleton } from '../../injector/index.js';
|
|
65
65
|
import { Logger } from '../../logger/index.js';
|
|
66
66
|
import { MessageBus } from '../../message-bus/index.js';
|
|
67
|
-
import { arrayOverlaps, coalesce, getEntityIds, interval, RANDOM_UUID_V4, TRANSACTION_TIMESTAMP } from '../../orm/index.js';
|
|
68
|
-
import { DatabaseConfig, injectRepository } from '../../orm/server/index.js';
|
|
67
|
+
import { arrayOverlaps, caseWhen, coalesce, enumValue, getEntityIds, greatest, interval, jsonbBuildObject, RANDOM_UUID_V4, TRANSACTION_TIMESTAMP } from '../../orm/index.js';
|
|
68
|
+
import { Database, DatabaseConfig, injectRepository } from '../../orm/server/index.js';
|
|
69
69
|
import { RateLimiter } from '../../rate-limit/index.js';
|
|
70
|
-
import {
|
|
70
|
+
import { distinct, toArray } from '../../utils/array/array.js';
|
|
71
71
|
import { currentTimestamp } from '../../utils/date-time.js';
|
|
72
72
|
import { Timer } from '../../utils/timer.js';
|
|
73
73
|
import { cancelableTimeout } from '../../utils/timing.js';
|
|
74
|
-
import { isDefined, isNotNull,
|
|
75
|
-
import { millisecondsPerSecond } from '../../utils/units.js';
|
|
74
|
+
import { isArray, isDefined, isNotNull, isNull, isNumber, isString, isUndefined } from '../../utils/type-guards.js';
|
|
75
|
+
import { millisecondsPerMinute, millisecondsPerSecond } from '../../utils/units.js';
|
|
76
76
|
import { defaultQueueConfig, TaskDependencyType, TaskQueue, TaskStatus } from '../task-queue.js';
|
|
77
77
|
import { PostgresTaskQueueModuleConfig } from './module.js';
|
|
78
|
-
import { taskArchive as taskArchiveTable, taskDependency as taskDependencyTable, taskStatus, task as taskTable } from './schemas.js';
|
|
79
|
-
import { PostgresTask, PostgresTaskArchive } from './task.model.js';
|
|
78
|
+
import { taskArchive as taskArchiveTable, taskDependency as taskDependencyTable, taskDependencyType, taskStatus, task as taskTable } from './schemas.js';
|
|
79
|
+
import { finalizedStatuses, PostgresTask, PostgresTaskArchive, queueableOrWaitableStatuses, queueableStatuses, terminalStatuses } from './task.model.js';
|
|
80
80
|
let PostgresTaskQueue = class PostgresTaskQueue extends TaskQueue {
|
|
81
|
+
#database = inject(Database);
|
|
81
82
|
#repository = injectRepository(PostgresTask);
|
|
82
83
|
#archiveRepository = injectRepository(PostgresTaskArchive);
|
|
83
84
|
#config = this.config;
|
|
84
85
|
#namespace = isString(this.#config) ? this.#config : this.#config.namespace;
|
|
85
|
-
#messageBus = inject((MessageBus),
|
|
86
|
+
#messageBus = inject((MessageBus), 'PostgresTaskQueue');
|
|
86
87
|
#logger = inject(Logger, `PostgresTaskQueue:${this.#namespace}`);
|
|
87
88
|
#cancellationSignal = inject(CancellationSignal);
|
|
88
89
|
#rateLimiter = inject(RateLimiter, {
|
|
@@ -109,19 +110,19 @@ let PostgresTaskQueue = class PostgresTaskQueue extends TaskQueue {
|
|
|
109
110
|
rateLimit = this.#config.rateLimit ?? defaultQueueConfig.rateLimit;
|
|
110
111
|
rateInterval = this.#config.rateInterval ?? defaultQueueConfig.rateInterval;
|
|
111
112
|
idempotencyWindow = this.#config.idempotencyWindow ?? defaultQueueConfig.idempotencyWindow;
|
|
112
|
-
globalConcurrency = this.#config.globalConcurrency ?? defaultQueueConfig.globalConcurrency;
|
|
113
113
|
#takeNewUpdate = {
|
|
114
114
|
namespace: this.#namespace,
|
|
115
115
|
type: sql `excluded.type`,
|
|
116
|
-
status: sql `
|
|
116
|
+
status: caseWhen(gt(sql `excluded.unresolved_schedule_dependencies`, 0), enumValue(TaskStatus, taskStatus, TaskStatus.Waiting)).else(enumValue(TaskStatus, taskStatus, TaskStatus.Pending)),
|
|
117
117
|
token: null,
|
|
118
118
|
priority: sql `excluded.priority`,
|
|
119
119
|
idempotencyKey: sql `excluded.idempotency_key`,
|
|
120
120
|
traceId: sql `excluded.trace_id`,
|
|
121
|
+
parentId: sql `excluded.parent_id`,
|
|
121
122
|
tags: sql `excluded.tags`,
|
|
122
|
-
unresolvedScheduleDependencies:
|
|
123
|
-
unresolvedCompleteDependencies:
|
|
124
|
-
|
|
123
|
+
unresolvedScheduleDependencies: taskTable.unresolvedScheduleDependencies,
|
|
124
|
+
unresolvedCompleteDependencies: taskTable.unresolvedCompleteDependencies,
|
|
125
|
+
abortOnDependencyFailure: sql `excluded.abort_on_dependency_failure`,
|
|
125
126
|
tries: 0,
|
|
126
127
|
creationTimestamp: TRANSACTION_TIMESTAMP,
|
|
127
128
|
priorityAgeTimestamp: TRANSACTION_TIMESTAMP,
|
|
@@ -138,7 +139,7 @@ let PostgresTaskQueue = class PostgresTaskQueue extends TaskQueue {
|
|
|
138
139
|
};
|
|
139
140
|
[afterResolve]() {
|
|
140
141
|
if (!this.isInTransaction) {
|
|
141
|
-
|
|
142
|
+
this.maintenanceLoop();
|
|
142
143
|
}
|
|
143
144
|
}
|
|
144
145
|
async enqueue(type, data, options) {
|
|
@@ -151,22 +152,25 @@ let PostgresTaskQueue = class PostgresTaskQueue extends TaskQueue {
|
|
|
151
152
|
if (items.length == 0) {
|
|
152
153
|
return (options?.returnTasks == true) ? [] : undefined;
|
|
153
154
|
}
|
|
154
|
-
const
|
|
155
|
-
|
|
156
|
-
|
|
157
|
-
|
|
158
|
-
|
|
155
|
+
const itemsWithDistinctDependencies = items.map((item) => ({
|
|
156
|
+
...item,
|
|
157
|
+
scheduleAfter: isDefined(item.scheduleAfter) ? Array.from(new Map(item.scheduleAfter.map((s) => [isNumber(s) || isString(s) ? s : JSON.stringify(s), s])).values()) : undefined,
|
|
158
|
+
completeAfter: isDefined(item.completeAfter) ? Array.from(new Map(item.completeAfter.map((s) => [isNumber(s) || isString(s) ? s : JSON.stringify(s), s])).values()) : undefined,
|
|
159
|
+
}));
|
|
160
|
+
const entitiesWithIndex = itemsWithDistinctDependencies.map((item, index) => ({
|
|
161
|
+
index,
|
|
162
|
+
entity: {
|
|
159
163
|
namespace: this.#namespace,
|
|
160
164
|
type: item.type,
|
|
161
|
-
status:
|
|
165
|
+
status: TaskStatus.Pending,
|
|
162
166
|
token: null,
|
|
163
167
|
priority: item.priority ?? 1000,
|
|
164
168
|
idempotencyKey: item.idempotencyKey ?? null,
|
|
165
169
|
traceId: null,
|
|
166
170
|
tags: item.tags ?? [],
|
|
167
|
-
unresolvedScheduleDependencies:
|
|
168
|
-
unresolvedCompleteDependencies:
|
|
169
|
-
|
|
171
|
+
unresolvedScheduleDependencies: 0,
|
|
172
|
+
unresolvedCompleteDependencies: 0,
|
|
173
|
+
abortOnDependencyFailure: item.abortOnDependencyFailure ?? false,
|
|
170
174
|
parentId: item.parentId ?? null,
|
|
171
175
|
tries: 0,
|
|
172
176
|
progress: 0,
|
|
@@ -181,89 +185,134 @@ let PostgresTaskQueue = class PostgresTaskQueue extends TaskQueue {
|
|
|
181
185
|
state: null,
|
|
182
186
|
result: null,
|
|
183
187
|
error: null,
|
|
184
|
-
}
|
|
185
|
-
});
|
|
186
|
-
const itemsWithIdempotency =
|
|
187
|
-
const itemsWithoutIdempotency =
|
|
188
|
-
const hasDependencies =
|
|
189
|
-
const mustUseTransaction = (
|
|
188
|
+
},
|
|
189
|
+
}));
|
|
190
|
+
const itemsWithIdempotency = entitiesWithIndex.filter((e) => isNotNull(e.entity.idempotencyKey));
|
|
191
|
+
const itemsWithoutIdempotency = entitiesWithIndex.filter((e) => isNull(e.entity.idempotencyKey));
|
|
192
|
+
const hasDependencies = itemsWithDistinctDependencies.some((item) => ((item.scheduleAfter?.length ?? 0) > 0) || ((item.completeAfter?.length ?? 0) > 0) || (isDefined(item.parentId) && (item.blockParent != false) && !(isArray(item.blockParent) && (item.blockParent.length == 0))));
|
|
193
|
+
const mustUseTransaction = (entitiesWithIndex.length > 1) || hasDependencies;
|
|
190
194
|
const newTransaction = __addDisposableResource(env_1, (mustUseTransaction && isUndefined(options?.transaction)) ? await this.#repository.startTransaction() : undefined, true);
|
|
191
195
|
const transaction = newTransaction ?? options?.transaction;
|
|
192
196
|
const session = transaction?.pgTransaction ?? this.#repository.session;
|
|
193
|
-
const tasks =
|
|
197
|
+
const tasks = new Array(itemsWithDistinctDependencies.length);
|
|
198
|
+
let insertedRows = [];
|
|
199
|
+
let upsertedRows = [];
|
|
194
200
|
if (itemsWithoutIdempotency.length > 0) {
|
|
195
|
-
|
|
196
|
-
|
|
201
|
+
insertedRows = await session.insert(taskTable).values(itemsWithoutIdempotency.map((i) => i.entity)).returning();
|
|
202
|
+
for (const [i, row] of insertedRows.entries()) {
|
|
203
|
+
tasks[itemsWithoutIdempotency[i].index] = row;
|
|
204
|
+
}
|
|
197
205
|
}
|
|
198
206
|
if (itemsWithIdempotency.length > 0) {
|
|
199
207
|
const windowCutoff = sql `${TRANSACTION_TIMESTAMP} - ${interval(this.idempotencyWindow, 'milliseconds')}`;
|
|
200
208
|
const setWhere = (options?.replace == true) ? undefined : sql `${taskTable.creationTimestamp} < ${windowCutoff}`;
|
|
201
|
-
|
|
209
|
+
upsertedRows = await session
|
|
202
210
|
.insert(taskTable)
|
|
203
|
-
.values(itemsWithIdempotency)
|
|
211
|
+
.values(itemsWithIdempotency.map((i) => i.entity))
|
|
204
212
|
.onConflictDoUpdate({
|
|
205
213
|
target: [taskTable.namespace, taskTable.idempotencyKey],
|
|
206
214
|
set: this.#takeNewUpdate,
|
|
207
215
|
setWhere,
|
|
208
216
|
})
|
|
209
217
|
.returning();
|
|
210
|
-
|
|
211
|
-
if (
|
|
212
|
-
|
|
213
|
-
|
|
214
|
-
|
|
215
|
-
|
|
218
|
+
const upsertedIds = upsertedRows.map((t) => t.id);
|
|
219
|
+
if (upsertedIds.length > 0) {
|
|
220
|
+
await session
|
|
221
|
+
.delete(taskDependencyTable)
|
|
222
|
+
.where(and(inArray(taskDependencyTable.taskId, upsertedIds), inArray(taskDependencyTable.type, [TaskDependencyType.Schedule, TaskDependencyType.Complete])));
|
|
223
|
+
await session
|
|
224
|
+
.update(taskTable)
|
|
225
|
+
.set({ unresolvedScheduleDependencies: 0 })
|
|
226
|
+
.where(inArray(taskTable.id, upsertedIds));
|
|
216
227
|
}
|
|
217
|
-
if ((
|
|
218
|
-
const
|
|
219
|
-
const
|
|
220
|
-
|
|
228
|
+
if ((options?.returnTasks == true) || hasDependencies) {
|
|
229
|
+
const upsertedKeysMap = new Map(upsertedRows.map((t) => [t.idempotencyKey, t]));
|
|
230
|
+
const missingKeysWithIndex = [];
|
|
231
|
+
for (const item of itemsWithIdempotency) {
|
|
232
|
+
const key = item.entity.idempotencyKey;
|
|
233
|
+
const task = upsertedKeysMap.get(key);
|
|
234
|
+
if (isDefined(task)) {
|
|
235
|
+
tasks[item.index] = task;
|
|
236
|
+
}
|
|
237
|
+
else {
|
|
238
|
+
missingKeysWithIndex.push({ index: item.index, key });
|
|
239
|
+
}
|
|
240
|
+
}
|
|
241
|
+
if (missingKeysWithIndex.length > 0) {
|
|
221
242
|
const existingRows = await session
|
|
222
243
|
.select()
|
|
223
244
|
.from(taskTable)
|
|
224
|
-
.where(and(eq(taskTable.namespace, this.#namespace), inArray(taskTable.idempotencyKey,
|
|
225
|
-
|
|
245
|
+
.where(and(eq(taskTable.namespace, this.#namespace), inArray(taskTable.idempotencyKey, missingKeysWithIndex.map((m) => m.key))));
|
|
246
|
+
const existingRowsMap = new Map(existingRows.map((t) => [t.idempotencyKey, t]));
|
|
247
|
+
for (const missing of missingKeysWithIndex) {
|
|
248
|
+
tasks[missing.index] = existingRowsMap.get(missing.key);
|
|
249
|
+
}
|
|
226
250
|
}
|
|
227
251
|
}
|
|
228
252
|
}
|
|
229
253
|
if (hasDependencies) {
|
|
230
254
|
const dependencies = [];
|
|
231
|
-
const
|
|
232
|
-
for (const [index, item] of
|
|
255
|
+
const processedTaskIds = new Set([...insertedRows.map((r) => r.id), ...upsertedRows.map((r) => r.id)]);
|
|
256
|
+
for (const [index, item] of itemsWithDistinctDependencies.entries()) {
|
|
233
257
|
const task = tasks[index];
|
|
234
|
-
if (
|
|
258
|
+
if (!processedTaskIds.has(task.id)) {
|
|
259
|
+
continue;
|
|
260
|
+
}
|
|
261
|
+
if (isDefined(item.parentId) && (item.blockParent != false) && !(isArray(item.blockParent) && (item.blockParent.length == 0))) {
|
|
235
262
|
dependencies.push({
|
|
236
|
-
namespace: this.#namespace,
|
|
237
263
|
taskId: item.parentId,
|
|
238
264
|
dependencyTaskId: task.id,
|
|
239
|
-
type: TaskDependencyType.
|
|
240
|
-
requiredStatuses:
|
|
265
|
+
type: TaskDependencyType.Child,
|
|
266
|
+
requiredStatuses: isArray(item.blockParent)
|
|
267
|
+
? item.blockParent
|
|
268
|
+
: [TaskStatus.Completed],
|
|
241
269
|
});
|
|
242
|
-
parentIncrements.set(item.parentId, (parentIncrements.get(item.parentId) ?? 0) + 1);
|
|
243
270
|
}
|
|
244
271
|
if (isDefined(item.scheduleAfter)) {
|
|
245
272
|
for (const dependency of item.scheduleAfter) {
|
|
246
|
-
|
|
247
|
-
|
|
273
|
+
let dependencyTaskId;
|
|
274
|
+
let requiredStatuses;
|
|
275
|
+
if (isNumber(dependency)) {
|
|
276
|
+
dependencyTaskId = tasks[dependency].id;
|
|
277
|
+
requiredStatuses = [TaskStatus.Completed];
|
|
278
|
+
}
|
|
279
|
+
else if (isString(dependency)) {
|
|
280
|
+
dependencyTaskId = dependency;
|
|
281
|
+
requiredStatuses = [TaskStatus.Completed];
|
|
282
|
+
}
|
|
283
|
+
else {
|
|
284
|
+
dependencyTaskId = isDefined(dependency.index) ? tasks[dependency.index].id : dependency.id;
|
|
285
|
+
requiredStatuses = dependency.requiredStatuses ?? [TaskStatus.Completed];
|
|
286
|
+
}
|
|
248
287
|
dependencies.push({
|
|
249
|
-
namespace: this.#namespace,
|
|
250
288
|
taskId: task.id,
|
|
251
289
|
dependencyTaskId,
|
|
252
290
|
type: TaskDependencyType.Schedule,
|
|
253
|
-
requiredStatuses,
|
|
291
|
+
requiredStatuses: requiredStatuses,
|
|
254
292
|
});
|
|
255
293
|
}
|
|
256
294
|
}
|
|
257
295
|
if (isDefined(item.completeAfter)) {
|
|
258
296
|
for (const dependency of item.completeAfter) {
|
|
259
|
-
|
|
260
|
-
|
|
297
|
+
let dependencyTaskId;
|
|
298
|
+
let requiredStatuses;
|
|
299
|
+
if (isNumber(dependency)) {
|
|
300
|
+
dependencyTaskId = tasks[dependency].id;
|
|
301
|
+
requiredStatuses = [TaskStatus.Completed];
|
|
302
|
+
}
|
|
303
|
+
else if (isString(dependency)) {
|
|
304
|
+
dependencyTaskId = dependency;
|
|
305
|
+
requiredStatuses = [TaskStatus.Completed];
|
|
306
|
+
}
|
|
307
|
+
else {
|
|
308
|
+
dependencyTaskId = isDefined(dependency.index) ? tasks[dependency.index].id : dependency.id;
|
|
309
|
+
requiredStatuses = dependency.requiredStatuses ?? [TaskStatus.Completed];
|
|
310
|
+
}
|
|
261
311
|
dependencies.push({
|
|
262
|
-
namespace: this.#namespace,
|
|
263
312
|
taskId: task.id,
|
|
264
313
|
dependencyTaskId,
|
|
265
314
|
type: TaskDependencyType.Complete,
|
|
266
|
-
requiredStatuses,
|
|
315
|
+
requiredStatuses: requiredStatuses,
|
|
267
316
|
});
|
|
268
317
|
}
|
|
269
318
|
}
|
|
@@ -274,27 +323,35 @@ let PostgresTaskQueue = class PostgresTaskQueue extends TaskQueue {
|
|
|
274
323
|
.values(dependencies)
|
|
275
324
|
.onConflictDoNothing()
|
|
276
325
|
.returning();
|
|
277
|
-
|
|
278
|
-
|
|
279
|
-
|
|
280
|
-
|
|
281
|
-
|
|
282
|
-
|
|
283
|
-
|
|
284
|
-
|
|
285
|
-
|
|
286
|
-
|
|
287
|
-
|
|
288
|
-
|
|
289
|
-
}
|
|
326
|
+
const distinctDependencyIds = distinct(dependencies.map((d) => d.dependencyTaskId));
|
|
327
|
+
if (distinctDependencyIds.length > 0) {
|
|
328
|
+
await this.incrementCounters(inserted, { transaction });
|
|
329
|
+
const dependencyStatuses = await session
|
|
330
|
+
.select({ id: taskTable.id, status: taskTable.status })
|
|
331
|
+
.from(taskTable)
|
|
332
|
+
.where(inArray(taskTable.id, distinctDependencyIds))
|
|
333
|
+
.unionAll(session
|
|
334
|
+
.select({ id: taskArchiveTable.id, status: sql `${enumValue(TaskStatus, taskStatus, TaskStatus.Completed)}` })
|
|
335
|
+
.from(taskArchiveTable)
|
|
336
|
+
.where(inArray(taskArchiveTable.id, distinctDependencyIds)));
|
|
337
|
+
if (dependencyStatuses.length > 0) {
|
|
338
|
+
await this.resolveDependenciesMany(dependencyStatuses.map((s) => ({ id: s.id, status: s.status })), { transaction });
|
|
290
339
|
}
|
|
291
340
|
}
|
|
292
341
|
}
|
|
293
342
|
}
|
|
294
|
-
|
|
295
|
-
|
|
343
|
+
if (isDefined(newTransaction) && !newTransaction.isDone) {
|
|
344
|
+
await newTransaction.commit();
|
|
345
|
+
}
|
|
346
|
+
this.notify();
|
|
296
347
|
if (options?.returnTasks == true) {
|
|
297
|
-
const
|
|
348
|
+
const finalTasks = await session
|
|
349
|
+
.select()
|
|
350
|
+
.from(taskTable)
|
|
351
|
+
.where(inArray(taskTable.id, tasks.map((t) => t.id)));
|
|
352
|
+
const finalTasksMap = new Map(finalTasks.map((t) => [t.id, t]));
|
|
353
|
+
const orderedFinalTasks = tasks.map((t) => finalTasksMap.get(t.id));
|
|
354
|
+
const mapped = await this.#repository.mapManyToEntity(orderedFinalTasks);
|
|
298
355
|
return mapped;
|
|
299
356
|
}
|
|
300
357
|
return undefined;
|
|
@@ -309,12 +366,53 @@ let PostgresTaskQueue = class PostgresTaskQueue extends TaskQueue {
|
|
|
309
366
|
await result_1;
|
|
310
367
|
}
|
|
311
368
|
}
|
|
369
|
+
async incrementCounters(dependencies, options) {
|
|
370
|
+
if (dependencies.length == 0) {
|
|
371
|
+
return;
|
|
372
|
+
}
|
|
373
|
+
const session = options?.transaction?.pgTransaction ?? this.#repository.session;
|
|
374
|
+
const decrementsToApply = new Map();
|
|
375
|
+
for (const dep of dependencies) {
|
|
376
|
+
const current = decrementsToApply.get(dep.taskId) ?? { schedule: 0, complete: 0 };
|
|
377
|
+
if (dep.type == TaskDependencyType.Schedule) {
|
|
378
|
+
current.schedule++;
|
|
379
|
+
}
|
|
380
|
+
else if (dep.type == TaskDependencyType.Complete || dep.type == TaskDependencyType.Child) {
|
|
381
|
+
current.complete++;
|
|
382
|
+
}
|
|
383
|
+
decrementsToApply.set(dep.taskId, current);
|
|
384
|
+
}
|
|
385
|
+
const values = [...decrementsToApply]
|
|
386
|
+
.toSorted(([idA], [idB]) => idA.localeCompare(idB))
|
|
387
|
+
.map(([taskId, d]) => sql `(${taskId}::uuid, ${d.schedule}::int, ${d.complete}::int)`);
|
|
388
|
+
const updates = session.$with('updates').as((qb) => qb
|
|
389
|
+
.select({
|
|
390
|
+
taskId: sql `(id)::uuid`.as('task_id'),
|
|
391
|
+
scheduleIncrement: sql `(schedule)::int`.as('schedule_increment'),
|
|
392
|
+
completeIncrement: sql `(complete)::int`.as('complete_increment'),
|
|
393
|
+
})
|
|
394
|
+
.from(sql `(VALUES ${sql.join(values, sql `, `)}) AS t(id, schedule, complete)`));
|
|
395
|
+
const updatedRows = await session
|
|
396
|
+
.with(updates)
|
|
397
|
+
.update(taskTable)
|
|
398
|
+
.set({
|
|
399
|
+
unresolvedScheduleDependencies: sql `${taskTable.unresolvedScheduleDependencies} + ${updates.scheduleIncrement}`,
|
|
400
|
+
unresolvedCompleteDependencies: sql `${taskTable.unresolvedCompleteDependencies} + ${updates.completeIncrement}`,
|
|
401
|
+
status: caseWhen(and(eq(taskTable.status, TaskStatus.Pending), gt(sql `${taskTable.unresolvedScheduleDependencies} + ${updates.scheduleIncrement}`, 0)), enumValue(TaskStatus, taskStatus, TaskStatus.Waiting)).else(taskTable.status),
|
|
402
|
+
})
|
|
403
|
+
.from(updates)
|
|
404
|
+
.where(eq(taskTable.id, updates.taskId))
|
|
405
|
+
.returning({ id: taskTable.id, status: taskTable.status, namespace: taskTable.namespace });
|
|
406
|
+
for (const row of updatedRows) {
|
|
407
|
+
this.notify(row.namespace);
|
|
408
|
+
}
|
|
409
|
+
}
|
|
312
410
|
async has(id, options) {
|
|
313
411
|
const session = options?.transaction?.pgTransaction ?? this.#repository.session;
|
|
314
412
|
const [result] = await session
|
|
315
413
|
.select({ id: taskTable.id })
|
|
316
414
|
.from(taskTable)
|
|
317
|
-
.where(
|
|
415
|
+
.where(eq(taskTable.id, id))
|
|
318
416
|
.limit(1);
|
|
319
417
|
return isDefined(result);
|
|
320
418
|
}
|
|
@@ -330,19 +428,27 @@ let PostgresTaskQueue = class PostgresTaskQueue extends TaskQueue {
|
|
|
330
428
|
.where(and(eq(taskTable.namespace, this.#namespace), arrayOverlaps(taskTable.tags, tagArray)));
|
|
331
429
|
return result?.count ?? 0;
|
|
332
430
|
}
|
|
431
|
+
async count(options) {
|
|
432
|
+
const session = options?.transaction?.pgTransaction ?? this.#repository.session;
|
|
433
|
+
const [result] = await session
|
|
434
|
+
.select({ count: count() })
|
|
435
|
+
.from(taskTable)
|
|
436
|
+
.where(and(eq(taskTable.namespace, this.#namespace), isDefined(options?.status) ? eq(taskTable.status, options.status) : undefined));
|
|
437
|
+
return result?.count ?? 0;
|
|
438
|
+
}
|
|
333
439
|
async getTask(id, options) {
|
|
334
440
|
const session = options?.transaction?.pgTransaction ?? this.#repository.session;
|
|
335
441
|
const [activeRow] = await session
|
|
336
442
|
.select()
|
|
337
443
|
.from(taskTable)
|
|
338
|
-
.where(
|
|
444
|
+
.where(eq(taskTable.id, id));
|
|
339
445
|
if (isDefined(activeRow)) {
|
|
340
446
|
return await this.#repository.mapToEntity(activeRow);
|
|
341
447
|
}
|
|
342
448
|
const [archiveRow] = await session
|
|
343
449
|
.select()
|
|
344
450
|
.from(taskArchiveTable)
|
|
345
|
-
.where(
|
|
451
|
+
.where(eq(taskArchiveTable.id, id));
|
|
346
452
|
if (isDefined(archiveRow)) {
|
|
347
453
|
return await this.#archiveRepository.mapToEntity(archiveRow);
|
|
348
454
|
}
|
|
@@ -381,7 +487,7 @@ let PostgresTaskQueue = class PostgresTaskQueue extends TaskQueue {
|
|
|
381
487
|
FROM ${taskTable}
|
|
382
488
|
WHERE ${inArray(taskTable.id, rootIds)}
|
|
383
489
|
|
|
384
|
-
UNION
|
|
490
|
+
UNION ALL
|
|
385
491
|
|
|
386
492
|
SELECT child.*, parent.depth + 1
|
|
387
493
|
FROM ${taskTable} child
|
|
@@ -400,10 +506,10 @@ let PostgresTaskQueue = class PostgresTaskQueue extends TaskQueue {
|
|
|
400
506
|
const timeout = options?.timeout ?? Infinity;
|
|
401
507
|
const interval = options?.interval ?? 1000;
|
|
402
508
|
const cancellationSignal = this.#cancellationSignal.optionallyInherit(options?.cancellationSignal);
|
|
403
|
-
const
|
|
509
|
+
const waitStatuses = options?.statuses ?? finalizedStatuses;
|
|
510
|
+
const messageBus$ = this.#messageBus.allMessages$.pipe(filter((namespace) => namespace == this.#namespace), throttleTime(500, undefined, { leading: true, trailing: true }));
|
|
404
511
|
const continue$ = merge(messageBus$, cancellationSignal);
|
|
405
512
|
const timer = Timer.startNew();
|
|
406
|
-
const finalizedStatuses = [TaskStatus.Completed, TaskStatus.Cancelled, TaskStatus.Dead];
|
|
407
513
|
while (true) {
|
|
408
514
|
if (cancellationSignal.isSet) {
|
|
409
515
|
return { cancelled: true };
|
|
@@ -411,11 +517,12 @@ let PostgresTaskQueue = class PostgresTaskQueue extends TaskQueue {
|
|
|
411
517
|
if (timer.milliseconds > timeout) {
|
|
412
518
|
throw new TimeoutError('Timeout while waiting for tasks to complete');
|
|
413
519
|
}
|
|
414
|
-
const
|
|
415
|
-
|
|
416
|
-
|
|
417
|
-
|
|
418
|
-
|
|
520
|
+
const [remaining] = await this.#repository.session
|
|
521
|
+
.select({ id: taskTable.id })
|
|
522
|
+
.from(taskTable)
|
|
523
|
+
.where(and(inArray(taskTable.id, ids), notInArray(taskTable.status, waitStatuses)))
|
|
524
|
+
.limit(1);
|
|
525
|
+
if (isUndefined(remaining)) {
|
|
419
526
|
return { cancelled: false };
|
|
420
527
|
}
|
|
421
528
|
const remainingTimeout = timeout - timer.milliseconds;
|
|
@@ -433,16 +540,17 @@ let PostgresTaskQueue = class PostgresTaskQueue extends TaskQueue {
|
|
|
433
540
|
if (treeIds.length == 0) {
|
|
434
541
|
return;
|
|
435
542
|
}
|
|
436
|
-
await tx.pgTransaction
|
|
543
|
+
const cancelledRows = await tx.pgTransaction
|
|
437
544
|
.update(taskTable)
|
|
438
545
|
.set({
|
|
439
546
|
status: TaskStatus.Cancelled,
|
|
440
547
|
token: null,
|
|
441
548
|
completeTimestamp: TRANSACTION_TIMESTAMP,
|
|
442
549
|
})
|
|
443
|
-
.where(and(
|
|
444
|
-
|
|
445
|
-
|
|
550
|
+
.where(and(inArray(taskTable.id, treeIds), notInArray(taskTable.status, terminalStatuses)))
|
|
551
|
+
.returning({ id: taskTable.id, namespace: taskTable.namespace });
|
|
552
|
+
if (cancelledRows.length > 0) {
|
|
553
|
+
await this.resolveDependenciesMany(cancelledRows.map((row) => ({ id: row.id, status: TaskStatus.Cancelled, namespace: row.namespace })), { transaction: tx });
|
|
446
554
|
}
|
|
447
555
|
});
|
|
448
556
|
}
|
|
@@ -455,6 +563,32 @@ let PostgresTaskQueue = class PostgresTaskQueue extends TaskQueue {
|
|
|
455
563
|
}
|
|
456
564
|
async clear(options) {
|
|
457
565
|
await this.#repository.useTransaction(options?.transaction, async (tx) => {
|
|
566
|
+
const nonFinalizedTasks = await tx.pgTransaction
|
|
567
|
+
.select({ id: taskTable.id, namespace: taskTable.namespace })
|
|
568
|
+
.from(taskTable)
|
|
569
|
+
.where(and(eq(taskTable.namespace, this.#namespace), notInArray(taskTable.status, finalizedStatuses)))
|
|
570
|
+
.for('update');
|
|
571
|
+
if (nonFinalizedTasks.length > 0) {
|
|
572
|
+
const ids = nonFinalizedTasks.map((t) => t.id);
|
|
573
|
+
await tx.pgTransaction
|
|
574
|
+
.update(taskTable)
|
|
575
|
+
.set({
|
|
576
|
+
status: TaskStatus.Cancelled,
|
|
577
|
+
token: null,
|
|
578
|
+
completeTimestamp: TRANSACTION_TIMESTAMP,
|
|
579
|
+
})
|
|
580
|
+
.where(inArray(taskTable.id, ids));
|
|
581
|
+
await this.resolveDependenciesMany(nonFinalizedTasks.map((t) => ({ id: t.id, status: TaskStatus.Cancelled, namespace: t.namespace })), { transaction: tx });
|
|
582
|
+
}
|
|
583
|
+
// Break internal parent-child links to allow deleting tasks within the same namespace.
|
|
584
|
+
// Cross-namespace links will still cause a foreign key violation as requested.
|
|
585
|
+
await tx.pgTransaction
|
|
586
|
+
.update(taskTable)
|
|
587
|
+
.set({ parentId: null })
|
|
588
|
+
.where(and(eq(taskTable.namespace, this.#namespace), sqlIsNotNull(taskTable.parentId), inArray(taskTable.parentId, tx.pgTransaction
|
|
589
|
+
.select({ id: taskTable.id })
|
|
590
|
+
.from(taskTable)
|
|
591
|
+
.where(eq(taskTable.namespace, this.#namespace)))));
|
|
458
592
|
await tx.pgTransaction
|
|
459
593
|
.delete(taskTable)
|
|
460
594
|
.where(eq(taskTable.namespace, this.#namespace));
|
|
@@ -501,20 +635,6 @@ let PostgresTaskQueue = class PostgresTaskQueue extends TaskQueue {
|
|
|
501
635
|
}
|
|
502
636
|
effectiveCount = 1;
|
|
503
637
|
}
|
|
504
|
-
// 2. Check Global Concurrency
|
|
505
|
-
if (!forceDequeue && isNotNull(this.globalConcurrency)) {
|
|
506
|
-
// WARN: This is a check-then-act race condition.
|
|
507
|
-
// A distributed lock or an atomic update strategy is needed for strict enforcement.
|
|
508
|
-
const [runningResult] = await tx.pgTransaction
|
|
509
|
-
.select({ count: sqlCount() })
|
|
510
|
-
.from(taskTable)
|
|
511
|
-
.where(and(eq(taskTable.namespace, this.#namespace), eq(taskTable.status, TaskStatus.Running)));
|
|
512
|
-
const runningCount = runningResult?.count ?? 0;
|
|
513
|
-
if (runningCount >= this.globalConcurrency) {
|
|
514
|
-
return [];
|
|
515
|
-
}
|
|
516
|
-
effectiveCount = Math.min(effectiveCount, this.globalConcurrency - runningCount);
|
|
517
|
-
}
|
|
518
638
|
/*
|
|
519
639
|
* Materialization required for LIMIT clause
|
|
520
640
|
* https://stackoverflow.com/questions/73966670/select-for-update-subquery-not-respecting-limit-clause-under-load
|
|
@@ -523,7 +643,7 @@ let PostgresTaskQueue = class PostgresTaskQueue extends TaskQueue {
|
|
|
523
643
|
const selection = tx.pgTransaction.$with('selection').as((qb) => qb
|
|
524
644
|
.select({ id: taskTable.id })
|
|
525
645
|
.from(taskTable)
|
|
526
|
-
.where(and(eq(taskTable.namespace, this.#namespace), lte(taskTable.scheduleTimestamp, TRANSACTION_TIMESTAMP),
|
|
646
|
+
.where(and(eq(taskTable.namespace, this.#namespace), lte(taskTable.scheduleTimestamp, TRANSACTION_TIMESTAMP), inArray(taskTable.status, queueableStatuses), or(sqlIsNull(taskTable.timeToLive), lt(TRANSACTION_TIMESTAMP, taskTable.timeToLive)), isDefined(options?.types) ? inArray(taskTable.type, options.types) : undefined, sql `pg_sleep(0) IS NOT NULL` // Materialization hack until drizzle implements https://github.com/drizzle-team/drizzle-orm/issues/2318
|
|
527
647
|
))
|
|
528
648
|
.orderBy(asc(taskTable.priority), asc(taskTable.scheduleTimestamp))
|
|
529
649
|
.limit(effectiveCount)
|
|
@@ -559,32 +679,26 @@ let PostgresTaskQueue = class PostgresTaskQueue extends TaskQueue {
|
|
|
559
679
|
await session
|
|
560
680
|
.update(taskTable)
|
|
561
681
|
.set({
|
|
562
|
-
status: TaskStatus.Pending,
|
|
682
|
+
status: caseWhen(gt(taskTable.unresolvedScheduleDependencies, 0), enumValue(TaskStatus, taskStatus, TaskStatus.Waiting)).else(enumValue(TaskStatus, taskStatus, TaskStatus.Pending)),
|
|
563
683
|
token: null,
|
|
564
684
|
scheduleTimestamp: timestamp,
|
|
565
685
|
visibilityDeadline: null,
|
|
566
|
-
tries: sql `
|
|
567
|
-
WHEN ${taskTable.status} = ${TaskStatus.Running} THEN GREATEST(0, ${taskTable.tries} - 1)
|
|
568
|
-
ELSE ${taskTable.tries}
|
|
569
|
-
END`,
|
|
686
|
+
tries: caseWhen(eq(taskTable.status, enumValue(TaskStatus, taskStatus, TaskStatus.Running)), greatest(0, sql `${taskTable.tries} - 1`)).else(taskTable.tries),
|
|
570
687
|
})
|
|
571
|
-
.where(and(
|
|
688
|
+
.where(and(inArray(taskTable.id, ids), notInArray(taskTable.status, terminalStatuses)));
|
|
572
689
|
}
|
|
573
690
|
async rescheduleManyByTags(tags, timestamp, options) {
|
|
574
691
|
const session = options?.transaction?.pgTransaction ?? this.#repository.session;
|
|
575
692
|
await session
|
|
576
693
|
.update(taskTable)
|
|
577
694
|
.set({
|
|
578
|
-
status: TaskStatus.Pending,
|
|
695
|
+
status: caseWhen(gt(taskTable.unresolvedScheduleDependencies, 0), enumValue(TaskStatus, taskStatus, TaskStatus.Waiting)).else(enumValue(TaskStatus, taskStatus, TaskStatus.Pending)),
|
|
579
696
|
token: null,
|
|
580
697
|
scheduleTimestamp: timestamp,
|
|
581
698
|
visibilityDeadline: null,
|
|
582
|
-
tries: sql `
|
|
583
|
-
WHEN ${taskTable.status} = ${TaskStatus.Running} THEN GREATEST(0, ${taskTable.tries} - 1)
|
|
584
|
-
ELSE ${taskTable.tries}
|
|
585
|
-
END`,
|
|
699
|
+
tries: caseWhen(eq(taskTable.status, enumValue(TaskStatus, taskStatus, TaskStatus.Running)), greatest(0, sql `${taskTable.tries} - 1`)).else(taskTable.tries),
|
|
586
700
|
})
|
|
587
|
-
.where(and(eq(taskTable.namespace, this.#namespace), arrayOverlaps(taskTable.tags, toArray(tags))));
|
|
701
|
+
.where(and(eq(taskTable.namespace, this.#namespace), arrayOverlaps(taskTable.tags, toArray(tags)), notInArray(taskTable.status, terminalStatuses)));
|
|
588
702
|
}
|
|
589
703
|
async touch(task, options) {
|
|
590
704
|
if (isNull(task.token)) {
|
|
@@ -603,7 +717,7 @@ let PostgresTaskQueue = class PostgresTaskQueue extends TaskQueue {
|
|
|
603
717
|
const [updatedRow] = await tx.pgTransaction
|
|
604
718
|
.update(taskTable)
|
|
605
719
|
.set(update)
|
|
606
|
-
.where(and(eq(taskTable.
|
|
720
|
+
.where(and(eq(taskTable.id, task.id), eq(taskTable.status, TaskStatus.Running), isNull(task.token) ? sqlIsNull(taskTable.token) : eq(taskTable.token, task.token), gt(taskTable.startTimestamp, sql `${TRANSACTION_TIMESTAMP} - ${interval(this.maxExecutionTime, 'milliseconds')}`)))
|
|
607
721
|
.returning();
|
|
608
722
|
if (isDefined(updatedRow)) {
|
|
609
723
|
return await this.#repository.mapToEntity(updatedRow);
|
|
@@ -611,9 +725,18 @@ let PostgresTaskQueue = class PostgresTaskQueue extends TaskQueue {
|
|
|
611
725
|
const [existingRow] = await tx.pgTransaction
|
|
612
726
|
.select({ startTimestamp: taskTable.startTimestamp })
|
|
613
727
|
.from(taskTable)
|
|
614
|
-
.where(and(eq(taskTable.
|
|
728
|
+
.where(and(eq(taskTable.id, task.id), eq(taskTable.status, TaskStatus.Running), isNull(task.token) ? sqlIsNull(taskTable.token) : eq(taskTable.token, task.token)));
|
|
615
729
|
if (isDefined(existingRow) && isNotNull(existingRow.startTimestamp) && (currentTimestamp() - existingRow.startTimestamp) > this.maxExecutionTime) {
|
|
616
|
-
await
|
|
730
|
+
await tx.pgTransaction
|
|
731
|
+
.update(taskTable)
|
|
732
|
+
.set({
|
|
733
|
+
status: TaskStatus.TimedOut,
|
|
734
|
+
completeTimestamp: TRANSACTION_TIMESTAMP,
|
|
735
|
+
error: { code: 'MaxTimeExceeded', message: 'Hard Execution Timeout' },
|
|
736
|
+
})
|
|
737
|
+
.where(eq(taskTable.id, task.id));
|
|
738
|
+
await this.resolveDependenciesMany([{ id: task.id, status: TaskStatus.TimedOut, namespace: task.namespace }], { transaction: tx });
|
|
739
|
+
this.notify();
|
|
617
740
|
}
|
|
618
741
|
return undefined;
|
|
619
742
|
});
|
|
@@ -644,7 +767,7 @@ let PostgresTaskQueue = class PostgresTaskQueue extends TaskQueue {
|
|
|
644
767
|
state: coalesce(updates.updateState, taskTable.state),
|
|
645
768
|
})
|
|
646
769
|
.from(updates)
|
|
647
|
-
.where(and(eq(taskTable.id, updates.updateId),
|
|
770
|
+
.where(and(eq(taskTable.id, updates.updateId), sql `${taskTable.token} IS NOT DISTINCT FROM ${updates.updateToken}`, eq(taskTable.status, TaskStatus.Running), gt(taskTable.startTimestamp, sql `${TRANSACTION_TIMESTAMP} - ${interval(this.maxExecutionTime, 'milliseconds')}`)))
|
|
648
771
|
.returning({ id: taskTable.id }));
|
|
649
772
|
const result = await session
|
|
650
773
|
.with(updates, updated)
|
|
@@ -657,7 +780,7 @@ let PostgresTaskQueue = class PostgresTaskQueue extends TaskQueue {
|
|
|
657
780
|
const [freshTask] = await tx.pgTransaction
|
|
658
781
|
.select({ unresolvedCompleteDependencies: taskTable.unresolvedCompleteDependencies })
|
|
659
782
|
.from(taskTable)
|
|
660
|
-
.where(
|
|
783
|
+
.where(eq(taskTable.id, task.id))
|
|
661
784
|
.for('update');
|
|
662
785
|
if (isUndefined(freshTask)) {
|
|
663
786
|
return;
|
|
@@ -673,7 +796,7 @@ let PostgresTaskQueue = class PostgresTaskQueue extends TaskQueue {
|
|
|
673
796
|
completeTimestamp: (nextStatus == TaskStatus.Completed) ? TRANSACTION_TIMESTAMP : null,
|
|
674
797
|
visibilityDeadline: null,
|
|
675
798
|
})
|
|
676
|
-
.where(and(eq(taskTable.
|
|
799
|
+
.where(and(eq(taskTable.id, task.id), isNull(task.token) ? sqlIsNull(taskTable.token) : eq(taskTable.token, task.token)))
|
|
677
800
|
.returning({ id: taskTable.id });
|
|
678
801
|
if (isUndefined(updatedTask)) {
|
|
679
802
|
return;
|
|
@@ -681,7 +804,7 @@ let PostgresTaskQueue = class PostgresTaskQueue extends TaskQueue {
|
|
|
681
804
|
if (nextStatus == TaskStatus.Completed) {
|
|
682
805
|
await this.#circuitBreaker.recordSuccess();
|
|
683
806
|
}
|
|
684
|
-
await this.resolveDependencies(task.id, nextStatus, { transaction: tx });
|
|
807
|
+
await this.resolveDependencies(task.id, nextStatus, { namespace: task.namespace, transaction: tx });
|
|
685
808
|
});
|
|
686
809
|
}
|
|
687
810
|
async completeMany(tasks, options) {
|
|
@@ -689,43 +812,30 @@ let PostgresTaskQueue = class PostgresTaskQueue extends TaskQueue {
|
|
|
689
812
|
return;
|
|
690
813
|
}
|
|
691
814
|
await this.#repository.useTransaction(options?.transaction, async (tx) => {
|
|
692
|
-
const taskIds = getEntityIds(tasks);
|
|
693
|
-
const freshTasks = await tx.pgTransaction
|
|
694
|
-
.select({ id: taskTable.id, unresolvedCompleteDependencies: taskTable.unresolvedCompleteDependencies })
|
|
695
|
-
.from(taskTable)
|
|
696
|
-
.where(and(eq(taskTable.namespace, this.#namespace), inArray(taskTable.id, taskIds)))
|
|
697
|
-
.for('update');
|
|
698
|
-
const unresolvedDependencyCountMap = new Map(freshTasks.map((task) => [task.id, task.unresolvedCompleteDependencies]));
|
|
699
815
|
const rows = tasks.map((t, i) => {
|
|
700
816
|
const result = options?.results?.[i] ?? null;
|
|
701
|
-
|
|
702
|
-
const nextStatus = (unresolvedCount > 0) ? TaskStatus.WaitingChildren : TaskStatus.Completed;
|
|
703
|
-
const progress = (nextStatus == TaskStatus.Completed) ? 1 : t.progress;
|
|
704
|
-
const completeTimestamp = (nextStatus == TaskStatus.Completed) ? TRANSACTION_TIMESTAMP : null;
|
|
705
|
-
return sql `(${t.id}::uuid, ${t.token}::uuid, ${nextStatus}::text, ${result}::jsonb, ${progress}::numeric, ${completeTimestamp}::timestamptz)`;
|
|
817
|
+
return sql `(${t.id}::uuid, ${t.token}::uuid, ${result}::jsonb, ${t.progress}::numeric)`;
|
|
706
818
|
});
|
|
707
819
|
const updates = tx.pgTransaction.$with('updates').as((qb) => qb
|
|
708
820
|
.select({
|
|
709
821
|
updateId: sql `(id)::uuid`.as('update_id'),
|
|
710
822
|
updateToken: sql `(token)::uuid`.as('update_token'),
|
|
711
|
-
updateStatus: sql `(status)::text`.as('update_status'),
|
|
712
823
|
updateResult: sql `(result)::jsonb`.as('update_result'),
|
|
713
824
|
updateProgress: sql `(progress)::numeric`.as('update_progress'),
|
|
714
|
-
updateComplete: sql `(complete_timestamp)::timestamptz`.as('update_complete'),
|
|
715
825
|
})
|
|
716
|
-
.from(sql `(VALUES ${sql.join(rows, sql `, `)}) AS t(id, token,
|
|
826
|
+
.from(sql `(VALUES ${sql.join(rows, sql `, `)}) AS t(id, token, result, progress)`));
|
|
717
827
|
const updated = tx.pgTransaction.$with('updated').as(() => tx.pgTransaction
|
|
718
828
|
.update(taskTable)
|
|
719
829
|
.set({
|
|
720
|
-
status:
|
|
830
|
+
status: caseWhen(gt(taskTable.unresolvedCompleteDependencies, 0), enumValue(TaskStatus, taskStatus, TaskStatus.WaitingChildren)).else(enumValue(TaskStatus, taskStatus, TaskStatus.Completed)),
|
|
721
831
|
token: null,
|
|
722
832
|
result: updates.updateResult,
|
|
723
|
-
progress:
|
|
724
|
-
completeTimestamp:
|
|
833
|
+
progress: caseWhen(gt(taskTable.unresolvedCompleteDependencies, 0), updates.updateProgress).else(1),
|
|
834
|
+
completeTimestamp: caseWhen(gt(taskTable.unresolvedCompleteDependencies, 0), taskTable.completeTimestamp).else(TRANSACTION_TIMESTAMP),
|
|
725
835
|
visibilityDeadline: null,
|
|
726
836
|
})
|
|
727
837
|
.from(updates)
|
|
728
|
-
.where(and(eq(taskTable.id, updates.updateId),
|
|
838
|
+
.where(and(eq(taskTable.id, updates.updateId), sql `${taskTable.token} IS NOT DISTINCT FROM ${updates.updateToken}`))
|
|
729
839
|
.returning({ id: taskTable.id, status: taskTable.status }));
|
|
730
840
|
const updatedRows = await tx.pgTransaction
|
|
731
841
|
.with(updates, updated)
|
|
@@ -735,15 +845,13 @@ let PostgresTaskQueue = class PostgresTaskQueue extends TaskQueue {
|
|
|
735
845
|
if (updatedRows.some((r) => r.status == TaskStatus.Completed)) {
|
|
736
846
|
await this.#circuitBreaker.recordSuccess();
|
|
737
847
|
}
|
|
738
|
-
|
|
739
|
-
await this.resolveDependencies(row.id, row.status, { transaction: tx });
|
|
740
|
-
}
|
|
848
|
+
await this.resolveDependenciesMany(updatedRows.map((r) => ({ id: r.id, status: r.status })), { transaction: tx });
|
|
741
849
|
}
|
|
742
850
|
});
|
|
743
851
|
}
|
|
744
852
|
async fail(task, error, options) {
|
|
745
853
|
const isRetryable = (options?.fatal != true) && (task.tries < this.maxTries);
|
|
746
|
-
const nextStatus = isRetryable ? TaskStatus.
|
|
854
|
+
const nextStatus = isRetryable ? TaskStatus.Retrying : TaskStatus.Dead;
|
|
747
855
|
const delay = isRetryable
|
|
748
856
|
? Math.min(this.retryDelayMaximum, this.retryDelayMinimum * (this.retryDelayGrowth ** task.tries))
|
|
749
857
|
: 0;
|
|
@@ -760,13 +868,13 @@ let PostgresTaskQueue = class PostgresTaskQueue extends TaskQueue {
|
|
|
760
868
|
startTimestamp: null,
|
|
761
869
|
completeTimestamp: (nextStatus == TaskStatus.Dead) ? TRANSACTION_TIMESTAMP : null,
|
|
762
870
|
})
|
|
763
|
-
.where(and(eq(taskTable.
|
|
871
|
+
.where(and(eq(taskTable.id, task.id), isNull(task.token) ? sqlIsNull(taskTable.token) : eq(taskTable.token, task.token), eq(taskTable.tries, task.tries)))
|
|
764
872
|
.returning();
|
|
765
873
|
if (isUndefined(updatedRow)) {
|
|
766
874
|
return;
|
|
767
875
|
}
|
|
768
876
|
await this.#circuitBreaker.recordFailure();
|
|
769
|
-
await this.resolveDependencies(task.id, nextStatus, { transaction: tx });
|
|
877
|
+
await this.resolveDependencies(task.id, nextStatus, { namespace: task.namespace, transaction: tx });
|
|
770
878
|
});
|
|
771
879
|
}
|
|
772
880
|
async failMany(tasks, errors, options) {
|
|
@@ -777,7 +885,7 @@ let PostgresTaskQueue = class PostgresTaskQueue extends TaskQueue {
|
|
|
777
885
|
const rows = tasks.map((task, index) => {
|
|
778
886
|
const error = errors[index];
|
|
779
887
|
const isRetryable = (task.tries < this.maxTries);
|
|
780
|
-
const nextStatus = isRetryable ? TaskStatus.
|
|
888
|
+
const nextStatus = isRetryable ? TaskStatus.Retrying : TaskStatus.Dead;
|
|
781
889
|
const delay = isRetryable
|
|
782
890
|
? Math.min(this.retryDelayMaximum, this.retryDelayMinimum * (this.retryDelayGrowth ** task.tries))
|
|
783
891
|
: 0;
|
|
@@ -808,148 +916,198 @@ let PostgresTaskQueue = class PostgresTaskQueue extends TaskQueue {
|
|
|
808
916
|
completeTimestamp: sql `${updates.updateComplete}`,
|
|
809
917
|
})
|
|
810
918
|
.from(updates)
|
|
811
|
-
.where(and(eq(taskTable.
|
|
919
|
+
.where(and(eq(taskTable.id, updates.updateId), sql `${taskTable.token} IS NOT DISTINCT FROM ${updates.updateToken}`, eq(taskTable.tries, updates.updateTries)))
|
|
812
920
|
.returning({ id: taskTable.id, status: taskTable.status }));
|
|
813
921
|
const result = await tx.pgTransaction
|
|
814
922
|
.with(updates, updated)
|
|
815
923
|
.select({ id: updated.id, status: updated.status })
|
|
816
924
|
.from(updated);
|
|
817
925
|
if (result.length > 0) {
|
|
818
|
-
await
|
|
819
|
-
|
|
820
|
-
await this.resolveDependencies(row.id, row.status, { transaction: tx });
|
|
821
|
-
}
|
|
926
|
+
await this.#circuitBreaker.recordFailures(result.length);
|
|
927
|
+
await this.resolveDependenciesMany(result.map((r) => ({ id: r.id, status: r.status })), { transaction: tx });
|
|
822
928
|
}
|
|
823
929
|
});
|
|
824
930
|
}
|
|
825
931
|
async resolveDependencies(id, status, options) {
|
|
826
|
-
this
|
|
932
|
+
await this.resolveDependenciesMany([{ id, status, namespace: options?.namespace }], options);
|
|
933
|
+
}
|
|
934
|
+
async resolveDependenciesMany(tasks, options) {
|
|
935
|
+
if (tasks.length == 0) {
|
|
936
|
+
return;
|
|
937
|
+
}
|
|
938
|
+
const taskStatusMap = new Map(tasks.map((t) => [t.id, t.status]));
|
|
939
|
+
const notifiedNamespaces = new Set();
|
|
827
940
|
await this.#repository.useTransaction(options?.transaction, async (tx) => {
|
|
828
|
-
|
|
941
|
+
const taskIds = tasks.map((t) => t.id);
|
|
829
942
|
const dependents = await tx.pgTransaction
|
|
830
943
|
.select({
|
|
831
944
|
taskId: taskDependencyTable.taskId,
|
|
945
|
+
dependencyTaskId: taskDependencyTable.dependencyTaskId,
|
|
832
946
|
type: taskDependencyTable.type,
|
|
833
947
|
requiredStatuses: taskDependencyTable.requiredStatuses,
|
|
834
948
|
})
|
|
835
949
|
.from(taskDependencyTable)
|
|
836
|
-
.where(
|
|
950
|
+
.where(inArray(taskDependencyTable.dependencyTaskId, taskIds));
|
|
837
951
|
if (dependents.length == 0) {
|
|
838
952
|
return;
|
|
839
953
|
}
|
|
954
|
+
const resolvedEdges = [];
|
|
955
|
+
const abortOnDependencyFailureTaskIds = new Set();
|
|
840
956
|
for (const dep of dependents) {
|
|
957
|
+
const status = taskStatusMap.get(dep.dependencyTaskId);
|
|
841
958
|
const isMatched = dep.requiredStatuses.includes(status);
|
|
842
|
-
const isTerminal = (status
|
|
959
|
+
const isTerminal = terminalStatuses.includes(status);
|
|
843
960
|
if (isMatched || isTerminal) {
|
|
844
|
-
|
|
845
|
-
|
|
846
|
-
.
|
|
847
|
-
.where(and(eq(taskDependencyTable.namespace, this.#namespace), eq(taskDependencyTable.taskId, dep.taskId), eq(taskDependencyTable.dependencyTaskId, id), eq(taskDependencyTable.type, dep.type)))
|
|
848
|
-
.returning();
|
|
849
|
-
if (deletedEdge.length > 0) {
|
|
850
|
-
const [dependentTask] = await tx.pgTransaction
|
|
851
|
-
.select({ failFast: taskTable.failFast, status: taskTable.status })
|
|
852
|
-
.from(taskTable)
|
|
853
|
-
.where(and(eq(taskTable.namespace, this.#namespace), eq(taskTable.id, dep.taskId)));
|
|
854
|
-
if (isDefined(dependentTask)) {
|
|
855
|
-
if (!isMatched && dependentTask.failFast && dependentTask.status != TaskStatus.Dead && dependentTask.status != TaskStatus.Cancelled) {
|
|
856
|
-
// Dependency failed and failFast is enabled: fail the dependent task immediately
|
|
857
|
-
await this.transitionTask(dep.taskId, TaskStatus.Dead, { error: { code: 'DependencyFailed', message: `Dependency ${id} failed (status: ${status}). Task failed fast because it required one of: ${dep.requiredStatuses.join(', ')}` }, transaction: tx });
|
|
858
|
-
}
|
|
859
|
-
else {
|
|
860
|
-
// Dependency resolved (matched, or failed but no failFast): decrement counter
|
|
861
|
-
const [updatedTask] = await tx.pgTransaction
|
|
862
|
-
.update(taskTable)
|
|
863
|
-
.set((dep.type == TaskDependencyType.Schedule)
|
|
864
|
-
? { unresolvedScheduleDependencies: sql `${taskTable.unresolvedScheduleDependencies} - 1` }
|
|
865
|
-
: { unresolvedCompleteDependencies: sql `${taskTable.unresolvedCompleteDependencies} - 1` })
|
|
866
|
-
.where(and(eq(taskTable.namespace, this.#namespace), eq(taskTable.id, dep.taskId), gt((dep.type == TaskDependencyType.Schedule) ? taskTable.unresolvedScheduleDependencies : taskTable.unresolvedCompleteDependencies, 0)))
|
|
867
|
-
.returning();
|
|
868
|
-
if (isDefined(updatedTask)) {
|
|
869
|
-
// Check for transitions
|
|
870
|
-
if (updatedTask.unresolvedScheduleDependencies == 0 || updatedTask.unresolvedCompleteDependencies == 0) {
|
|
871
|
-
await this.evaluateTaskStatus(updatedTask.id, { transaction: tx });
|
|
872
|
-
}
|
|
873
|
-
}
|
|
874
|
-
}
|
|
875
|
-
}
|
|
961
|
+
resolvedEdges.push(dep);
|
|
962
|
+
if (!isMatched) {
|
|
963
|
+
abortOnDependencyFailureTaskIds.add(dep.taskId);
|
|
876
964
|
}
|
|
877
965
|
}
|
|
878
966
|
}
|
|
879
|
-
|
|
880
|
-
}
|
|
881
|
-
async evaluateTaskStatus(id, options) {
|
|
882
|
-
await this.#repository.useTransaction(options?.transaction, async (tx) => {
|
|
883
|
-
const [rawTask] = await tx.pgTransaction
|
|
884
|
-
.select()
|
|
885
|
-
.from(taskTable)
|
|
886
|
-
.where(and(eq(taskTable.namespace, this.#namespace), eq(taskTable.id, id)))
|
|
887
|
-
.for('update');
|
|
888
|
-
if (isUndefined(rawTask)) {
|
|
889
|
-
return;
|
|
890
|
-
}
|
|
891
|
-
const task = await this.#repository.mapToEntity(rawTask);
|
|
892
|
-
if (task.status != TaskStatus.Waiting && task.status != TaskStatus.WaitingChildren && task.status != TaskStatus.Pending) {
|
|
967
|
+
if (resolvedEdges.length == 0) {
|
|
893
968
|
return;
|
|
894
969
|
}
|
|
895
|
-
|
|
896
|
-
|
|
897
|
-
|
|
898
|
-
|
|
899
|
-
const nextStatus = isNull(task.startTimestamp) ? TaskStatus.Pending : TaskStatus.Completed;
|
|
900
|
-
if (task.status != nextStatus) {
|
|
901
|
-
await this.transitionTask(id, nextStatus, { transaction: tx });
|
|
902
|
-
}
|
|
970
|
+
const sortedResolvedEdges = resolvedEdges.toSorted((a, b) => {
|
|
971
|
+
const idCompare = a.taskId.localeCompare(b.taskId);
|
|
972
|
+
if (idCompare != 0) {
|
|
973
|
+
return idCompare;
|
|
903
974
|
}
|
|
904
|
-
|
|
905
|
-
|
|
975
|
+
const depIdCompare = a.dependencyTaskId.localeCompare(b.dependencyTaskId);
|
|
976
|
+
if (depIdCompare != 0) {
|
|
977
|
+
return depIdCompare;
|
|
906
978
|
}
|
|
907
|
-
|
|
908
|
-
|
|
909
|
-
|
|
979
|
+
return a.type.localeCompare(b.type);
|
|
980
|
+
});
|
|
981
|
+
const edgeValues = sortedResolvedEdges.map((e) => sql `(${e.taskId}::uuid, ${e.dependencyTaskId}::uuid, ${e.type}::text)`);
|
|
982
|
+
await tx.pgTransaction.execute(sql `
|
|
983
|
+
DELETE FROM ${taskDependencyTable}
|
|
984
|
+
WHERE (task_id, dependency_task_id, type) IN (
|
|
985
|
+
SELECT t.task_id, t.dependency_task_id, t.type::${taskDependencyType}
|
|
986
|
+
FROM (VALUES ${sql.join(edgeValues, sql `, `)}) AS t(task_id, dependency_task_id, type)
|
|
987
|
+
)
|
|
988
|
+
`);
|
|
989
|
+
const terminalTasks = [];
|
|
990
|
+
const skippedTaskIds = new Set();
|
|
991
|
+
if (abortOnDependencyFailureTaskIds.size > 0) {
|
|
992
|
+
const sortedAbortIds = [...abortOnDependencyFailureTaskIds].toSorted();
|
|
993
|
+
const dependentTasks = await tx.pgTransaction
|
|
994
|
+
.select({ id: taskTable.id, namespace: taskTable.namespace, abortOnDependencyFailure: taskTable.abortOnDependencyFailure, status: taskTable.status })
|
|
995
|
+
.from(taskTable)
|
|
996
|
+
.where(inArray(taskTable.id, sortedAbortIds))
|
|
997
|
+
.orderBy(asc(taskTable.id))
|
|
998
|
+
.for('update');
|
|
999
|
+
const tasksToSkip = [];
|
|
1000
|
+
for (const task of dependentTasks) {
|
|
1001
|
+
if (task.abortOnDependencyFailure && !terminalStatuses.includes(task.status)) {
|
|
1002
|
+
tasksToSkip.push(task.id);
|
|
1003
|
+
skippedTaskIds.add(task.id);
|
|
1004
|
+
}
|
|
1005
|
+
}
|
|
1006
|
+
if (tasksToSkip.length > 0) {
|
|
1007
|
+
const skippedRows = await tx.pgTransaction
|
|
1008
|
+
.update(taskTable)
|
|
1009
|
+
.set({
|
|
1010
|
+
status: TaskStatus.Skipped,
|
|
1011
|
+
token: null,
|
|
1012
|
+
error: jsonbBuildObject({ code: 'DependencyFailed', message: 'One or more dependencies failed and abortOnDependencyFailure is enabled' }),
|
|
1013
|
+
completeTimestamp: TRANSACTION_TIMESTAMP,
|
|
1014
|
+
})
|
|
1015
|
+
.where(inArray(taskTable.id, tasksToSkip))
|
|
1016
|
+
.returning({ id: taskTable.id, status: taskTable.status, namespace: taskTable.namespace });
|
|
1017
|
+
terminalTasks.push(...skippedRows.map((r) => ({ id: r.id, status: r.status, namespace: r.namespace })));
|
|
1018
|
+
for (const row of skippedRows) {
|
|
1019
|
+
notifiedNamespaces.add(row.namespace);
|
|
1020
|
+
}
|
|
910
1021
|
}
|
|
911
1022
|
}
|
|
912
|
-
|
|
913
|
-
|
|
914
|
-
|
|
915
|
-
|
|
916
|
-
|
|
917
|
-
|
|
918
|
-
|
|
919
|
-
|
|
920
|
-
|
|
921
|
-
|
|
922
|
-
|
|
1023
|
+
// 4. Group all resolved edges into decrements (for any dependent NOT skipped above)
|
|
1024
|
+
const decrementsToApply = new Map();
|
|
1025
|
+
for (const dep of sortedResolvedEdges) {
|
|
1026
|
+
if (skippedTaskIds.has(dep.taskId)) {
|
|
1027
|
+
continue;
|
|
1028
|
+
}
|
|
1029
|
+
const current = decrementsToApply.get(dep.taskId) ?? { schedule: 0, complete: 0 };
|
|
1030
|
+
if (dep.type == TaskDependencyType.Schedule) {
|
|
1031
|
+
current.schedule++;
|
|
1032
|
+
}
|
|
1033
|
+
else if (dep.type == TaskDependencyType.Complete || dep.type == TaskDependencyType.Child) {
|
|
1034
|
+
current.complete++;
|
|
1035
|
+
}
|
|
1036
|
+
decrementsToApply.set(dep.taskId, current);
|
|
923
1037
|
}
|
|
924
|
-
if (
|
|
925
|
-
|
|
926
|
-
|
|
1038
|
+
if (decrementsToApply.size > 0) {
|
|
1039
|
+
const decrementValues = [...decrementsToApply]
|
|
1040
|
+
.toSorted(([idA], [idB]) => idA.localeCompare(idB))
|
|
1041
|
+
.map(([taskId, d]) => sql `(${taskId}::uuid, ${d.schedule}::int, ${d.complete}::int)`);
|
|
1042
|
+
const updates = tx.pgTransaction.$with('updates').as((qb) => qb
|
|
1043
|
+
.select({
|
|
1044
|
+
taskId: sql `(id)::uuid`.as('task_id'),
|
|
1045
|
+
scheduleDecrement: sql `(schedule)::int`.as('schedule_decrement'),
|
|
1046
|
+
completeDecrement: sql `(complete)::int`.as('complete_decrement'),
|
|
1047
|
+
})
|
|
1048
|
+
.from(sql `(VALUES ${sql.join(decrementValues, sql `, `)}) AS t(id, schedule, complete)`));
|
|
1049
|
+
const updatedRows = await tx.pgTransaction
|
|
1050
|
+
.with(updates)
|
|
1051
|
+
.update(taskTable)
|
|
1052
|
+
.set({
|
|
1053
|
+
unresolvedScheduleDependencies: greatest(0, sql `${taskTable.unresolvedScheduleDependencies} - ${updates.scheduleDecrement}`),
|
|
1054
|
+
unresolvedCompleteDependencies: greatest(0, sql `${taskTable.unresolvedCompleteDependencies} - ${updates.completeDecrement}`),
|
|
1055
|
+
status: caseWhen(and(eq(taskTable.status, TaskStatus.Waiting), eq(greatest(0, sql `${taskTable.unresolvedScheduleDependencies} - ${updates.scheduleDecrement}`), 0)), enumValue(TaskStatus, taskStatus, TaskStatus.Pending)).else(caseWhen(and(eq(taskTable.status, TaskStatus.WaitingChildren), eq(greatest(0, sql `${taskTable.unresolvedCompleteDependencies} - ${updates.completeDecrement}`), 0)), enumValue(TaskStatus, taskStatus, TaskStatus.Completed)).else(taskTable.status)),
|
|
1056
|
+
progress: caseWhen(and(eq(taskTable.status, TaskStatus.WaitingChildren), eq(greatest(0, sql `${taskTable.unresolvedCompleteDependencies} - ${updates.completeDecrement}`), 0)), 1).else(taskTable.progress),
|
|
1057
|
+
completeTimestamp: caseWhen(and(eq(taskTable.status, TaskStatus.WaitingChildren), eq(greatest(0, sql `${taskTable.unresolvedCompleteDependencies} - ${updates.completeDecrement}`), 0)), TRANSACTION_TIMESTAMP).else(taskTable.completeTimestamp),
|
|
1058
|
+
token: caseWhen(and(eq(taskTable.status, TaskStatus.WaitingChildren), eq(greatest(0, sql `${taskTable.unresolvedCompleteDependencies} - ${updates.completeDecrement}`), 0)), null).else(taskTable.token),
|
|
1059
|
+
})
|
|
1060
|
+
.from(updates)
|
|
1061
|
+
.where(eq(taskTable.id, updates.taskId))
|
|
1062
|
+
.returning({
|
|
1063
|
+
id: taskTable.id,
|
|
1064
|
+
namespace: taskTable.namespace,
|
|
1065
|
+
status: taskTable.status,
|
|
1066
|
+
});
|
|
1067
|
+
for (const row of updatedRows) {
|
|
1068
|
+
if (finalizedStatuses.includes(row.status)) {
|
|
1069
|
+
terminalTasks.push({ id: row.id, status: row.status, namespace: row.namespace });
|
|
1070
|
+
}
|
|
1071
|
+
notifiedNamespaces.add(row.namespace);
|
|
1072
|
+
}
|
|
927
1073
|
}
|
|
928
|
-
|
|
929
|
-
.
|
|
930
|
-
.
|
|
931
|
-
|
|
932
|
-
|
|
933
|
-
|
|
934
|
-
await this.resolveDependencies(id, status, { transaction: tx });
|
|
935
|
-
this.#messageBus.publishAndForget();
|
|
1074
|
+
if (terminalTasks.length > 0) {
|
|
1075
|
+
const terminalTaskIds = terminalTasks.map((t) => t.id);
|
|
1076
|
+
await tx.pgTransaction
|
|
1077
|
+
.delete(taskDependencyTable)
|
|
1078
|
+
.where(inArray(taskDependencyTable.taskId, terminalTaskIds));
|
|
1079
|
+
await this.resolveDependenciesMany(terminalTasks, { transaction: tx });
|
|
936
1080
|
}
|
|
937
1081
|
});
|
|
1082
|
+
for (const namespace of notifiedNamespaces) {
|
|
1083
|
+
this.notify(namespace);
|
|
1084
|
+
}
|
|
938
1085
|
}
|
|
939
1086
|
async maintenance(options) {
|
|
940
|
-
|
|
941
|
-
|
|
942
|
-
|
|
1087
|
+
await Promise.allSettled([
|
|
1088
|
+
this.processExpirations(options),
|
|
1089
|
+
this.processZombieRetries(options),
|
|
1090
|
+
this.processZombieExhaustions(options),
|
|
1091
|
+
this.processHardTimeouts(options),
|
|
1092
|
+
this.processPriorityAging(options),
|
|
1093
|
+
]);
|
|
1094
|
+
await this.performArchival(options);
|
|
1095
|
+
await this.performArchivePurge(options);
|
|
1096
|
+
}
|
|
1097
|
+
async performArchival(options) {
|
|
943
1098
|
while (true) {
|
|
944
|
-
const archivedCount = await repository.useTransaction(options?.transaction, async (tx) => {
|
|
1099
|
+
const archivedCount = await this.#repository.useTransaction(options?.transaction, async (tx) => {
|
|
945
1100
|
const childTaskTable = aliasedTable(taskTable, 'childTask');
|
|
946
1101
|
const rowsToArchive = await tx.pgTransaction
|
|
947
1102
|
.select()
|
|
948
1103
|
.from(taskTable)
|
|
949
|
-
.where(and(eq(taskTable.namespace, this.#namespace), inArray(taskTable.status,
|
|
1104
|
+
.where(and(eq(taskTable.namespace, this.#namespace), inArray(taskTable.status, finalizedStatuses), lte(taskTable.completeTimestamp, sql `${TRANSACTION_TIMESTAMP} - ${interval(this.retention, 'milliseconds')}`), notExists(tx.pgTransaction
|
|
950
1105
|
.select({ id: childTaskTable.id })
|
|
951
1106
|
.from(childTaskTable)
|
|
952
|
-
.where(
|
|
1107
|
+
.where(eq(childTaskTable.parentId, taskTable.id))), notExists(tx.pgTransaction
|
|
1108
|
+
.select({ taskId: taskDependencyTable.taskId })
|
|
1109
|
+
.from(taskDependencyTable)
|
|
1110
|
+
.where(eq(taskDependencyTable.dependencyTaskId, taskTable.id)))))
|
|
953
1111
|
.limit(1000)
|
|
954
1112
|
.for('update', { skipLocked: true });
|
|
955
1113
|
if (rowsToArchive.length > 0) {
|
|
@@ -963,144 +1121,163 @@ let PostgresTaskQueue = class PostgresTaskQueue extends TaskQueue {
|
|
|
963
1121
|
break;
|
|
964
1122
|
}
|
|
965
1123
|
}
|
|
966
|
-
|
|
1124
|
+
}
|
|
1125
|
+
async performArchivePurge(options) {
|
|
1126
|
+
const session = options?.transaction?.pgTransaction ?? this.#database;
|
|
1127
|
+
const selection = session.$with('archive_purge_selection').as((qb) => qb
|
|
1128
|
+
.select({ id: taskArchiveTable.id })
|
|
1129
|
+
.from(taskArchiveTable)
|
|
1130
|
+
.where(and(eq(taskArchiveTable.namespace, this.#namespace), lte(taskArchiveTable.completeTimestamp, sql `${TRANSACTION_TIMESTAMP} - ${interval(this.archiveRetention, 'milliseconds')}`)))
|
|
1131
|
+
.limit(1000));
|
|
967
1132
|
while (true) {
|
|
968
|
-
const
|
|
969
|
-
|
|
970
|
-
|
|
971
|
-
|
|
972
|
-
|
|
973
|
-
|
|
974
|
-
const result = await tx.pgTransaction
|
|
975
|
-
.with(selection)
|
|
976
|
-
.delete(taskArchiveTable)
|
|
977
|
-
.where(inArray(taskArchiveTable.id, tx.pgTransaction.select().from(selection)))
|
|
978
|
-
.returning({ id: taskArchiveTable.id });
|
|
979
|
-
return result.length;
|
|
980
|
-
});
|
|
981
|
-
if (deletedArchiveCount < 1000) {
|
|
1133
|
+
const result = await session
|
|
1134
|
+
.with(selection)
|
|
1135
|
+
.delete(taskArchiveTable)
|
|
1136
|
+
.where(inArray(taskArchiveTable.id, session.select().from(selection)))
|
|
1137
|
+
.returning({ id: taskArchiveTable.id });
|
|
1138
|
+
if (result.length < 1000) {
|
|
982
1139
|
break;
|
|
983
1140
|
}
|
|
984
1141
|
}
|
|
985
|
-
|
|
1142
|
+
}
|
|
1143
|
+
async processExpirations(options) {
|
|
1144
|
+
const expiredSelection = this.#database.$with('expired_selection').as((qb) => qb
|
|
1145
|
+
.select({ id: taskTable.id })
|
|
1146
|
+
.from(taskTable)
|
|
1147
|
+
.where(and(eq(taskTable.namespace, this.#namespace), inArray(taskTable.status, queueableOrWaitableStatuses), lt(taskTable.timeToLive, TRANSACTION_TIMESTAMP)))
|
|
1148
|
+
.limit(1000)
|
|
1149
|
+
.for('update', { skipLocked: true }));
|
|
986
1150
|
while (true) {
|
|
987
|
-
const
|
|
988
|
-
let totalUpdated = 0;
|
|
989
|
-
// 3.1 Handle Pending Expiration
|
|
990
|
-
const expiredSelection = tx.pgTransaction.$with('expired_selection').as((qb) => qb
|
|
991
|
-
.select({ id: taskTable.id })
|
|
992
|
-
.from(taskTable)
|
|
993
|
-
.where(and(eq(taskTable.namespace, this.#namespace), eq(taskTable.status, TaskStatus.Pending), lt(taskTable.timeToLive, TRANSACTION_TIMESTAMP)))
|
|
994
|
-
.limit(1000)
|
|
995
|
-
.for('update', { skipLocked: true }));
|
|
1151
|
+
const updatedCount = await this.#repository.useTransaction(options?.transaction, async (tx) => {
|
|
996
1152
|
const expiredRows = await tx.pgTransaction
|
|
997
1153
|
.with(expiredSelection)
|
|
998
1154
|
.update(taskTable)
|
|
999
1155
|
.set({
|
|
1000
|
-
status: TaskStatus.
|
|
1156
|
+
status: TaskStatus.Expired,
|
|
1001
1157
|
token: null,
|
|
1002
1158
|
error: { code: 'Expired', message: 'Task expired before processing' },
|
|
1003
1159
|
completeTimestamp: TRANSACTION_TIMESTAMP,
|
|
1004
1160
|
})
|
|
1005
1161
|
.where(inArray(taskTable.id, tx.pgTransaction.select().from(expiredSelection)))
|
|
1006
1162
|
.returning({ id: taskTable.id });
|
|
1007
|
-
totalUpdated += expiredRows.length;
|
|
1008
1163
|
if (expiredRows.length > 0) {
|
|
1009
|
-
|
|
1010
|
-
await this.resolveDependencies(row.id, TaskStatus.Dead, { transaction: tx });
|
|
1011
|
-
}
|
|
1164
|
+
await this.resolveDependenciesMany(expiredRows.map((r) => ({ id: r.id, status: TaskStatus.Expired, namespace: this.#namespace })), { transaction: tx });
|
|
1012
1165
|
}
|
|
1013
|
-
|
|
1014
|
-
|
|
1015
|
-
|
|
1016
|
-
|
|
1017
|
-
|
|
1018
|
-
|
|
1019
|
-
|
|
1020
|
-
|
|
1021
|
-
|
|
1022
|
-
|
|
1023
|
-
|
|
1024
|
-
|
|
1025
|
-
|
|
1026
|
-
|
|
1027
|
-
|
|
1028
|
-
|
|
1029
|
-
|
|
1030
|
-
|
|
1031
|
-
|
|
1032
|
-
|
|
1033
|
-
|
|
1034
|
-
|
|
1035
|
-
|
|
1036
|
-
|
|
1037
|
-
|
|
1038
|
-
|
|
1039
|
-
|
|
1040
|
-
|
|
1166
|
+
return expiredRows.length;
|
|
1167
|
+
});
|
|
1168
|
+
if (updatedCount < 1000) {
|
|
1169
|
+
break;
|
|
1170
|
+
}
|
|
1171
|
+
}
|
|
1172
|
+
}
|
|
1173
|
+
async processZombieRetries(options) {
|
|
1174
|
+
const session = options?.transaction?.pgTransaction ?? this.#database;
|
|
1175
|
+
const zombieRetrySelection = session.$with('zombie_retry_selection').as((qb) => qb
|
|
1176
|
+
.select({ id: taskTable.id })
|
|
1177
|
+
.from(taskTable)
|
|
1178
|
+
.where(and(eq(taskTable.namespace, this.#namespace), eq(taskTable.status, TaskStatus.Running), lt(taskTable.visibilityDeadline, TRANSACTION_TIMESTAMP), lt(taskTable.tries, this.maxTries)))
|
|
1179
|
+
.limit(1000)
|
|
1180
|
+
.for('update', { skipLocked: true }));
|
|
1181
|
+
while (true) {
|
|
1182
|
+
const zombieRetryRows = await session
|
|
1183
|
+
.with(zombieRetrySelection)
|
|
1184
|
+
.update(taskTable)
|
|
1185
|
+
.set({
|
|
1186
|
+
status: TaskStatus.Retrying,
|
|
1187
|
+
token: null,
|
|
1188
|
+
visibilityDeadline: null,
|
|
1189
|
+
startTimestamp: null,
|
|
1190
|
+
scheduleTimestamp: sql `${TRANSACTION_TIMESTAMP} + ${interval(this.retryDelayMinimum, 'milliseconds')}`, // Simple backoff for zombies
|
|
1191
|
+
error: jsonbBuildObject({ code: 'VisibilityTimeout', message: 'Worker Lost', lastError: taskTable.error }),
|
|
1192
|
+
})
|
|
1193
|
+
.where(inArray(taskTable.id, session.select().from(zombieRetrySelection)))
|
|
1194
|
+
.returning({ id: taskTable.id });
|
|
1195
|
+
if (zombieRetryRows.length < 1000) {
|
|
1196
|
+
break;
|
|
1197
|
+
}
|
|
1198
|
+
}
|
|
1199
|
+
}
|
|
1200
|
+
async processZombieExhaustions(options) {
|
|
1201
|
+
const zombieExhaustionSelection = this.#database.$with('zombie_exhaustion_selection').as((qb) => qb
|
|
1202
|
+
.select({ id: taskTable.id })
|
|
1203
|
+
.from(taskTable)
|
|
1204
|
+
.where(and(eq(taskTable.namespace, this.#namespace), eq(taskTable.status, TaskStatus.Running), lt(taskTable.visibilityDeadline, TRANSACTION_TIMESTAMP), gte(taskTable.tries, this.maxTries)))
|
|
1205
|
+
.limit(1000)
|
|
1206
|
+
.for('update', { skipLocked: true }));
|
|
1207
|
+
while (true) {
|
|
1208
|
+
const updatedCount = await this.#repository.useTransaction(options?.transaction, async (tx) => {
|
|
1041
1209
|
const exhaustionRows = await tx.pgTransaction
|
|
1042
1210
|
.with(zombieExhaustionSelection)
|
|
1043
1211
|
.update(taskTable)
|
|
1044
1212
|
.set({
|
|
1045
|
-
status: TaskStatus.
|
|
1213
|
+
status: TaskStatus.Orphaned,
|
|
1046
1214
|
token: null,
|
|
1047
1215
|
visibilityDeadline: null,
|
|
1048
1216
|
completeTimestamp: TRANSACTION_TIMESTAMP,
|
|
1049
|
-
error:
|
|
1217
|
+
error: jsonbBuildObject({ code: 'ZombieExhausted', message: 'Exceeded max retries after repeated crashes', lastError: taskTable.error }),
|
|
1050
1218
|
})
|
|
1051
1219
|
.where(inArray(taskTable.id, tx.pgTransaction.select().from(zombieExhaustionSelection)))
|
|
1052
1220
|
.returning({ id: taskTable.id });
|
|
1053
|
-
totalUpdated += exhaustionRows.length;
|
|
1054
1221
|
if (exhaustionRows.length > 0) {
|
|
1055
|
-
|
|
1056
|
-
await this.resolveDependencies(row.id, TaskStatus.Dead, { transaction: tx });
|
|
1057
|
-
}
|
|
1222
|
+
await this.resolveDependenciesMany(exhaustionRows.map((r) => ({ id: r.id, status: TaskStatus.Orphaned, namespace: this.#namespace })), { transaction: tx });
|
|
1058
1223
|
}
|
|
1059
|
-
|
|
1060
|
-
|
|
1061
|
-
|
|
1062
|
-
|
|
1063
|
-
|
|
1064
|
-
|
|
1065
|
-
|
|
1224
|
+
return exhaustionRows.length;
|
|
1225
|
+
});
|
|
1226
|
+
if (updatedCount < 1000) {
|
|
1227
|
+
break;
|
|
1228
|
+
}
|
|
1229
|
+
}
|
|
1230
|
+
}
|
|
1231
|
+
async processHardTimeouts(options) {
|
|
1232
|
+
const timeoutSelection = this.#database.$with('timeout_selection').as((qb) => qb
|
|
1233
|
+
.select({ id: taskTable.id })
|
|
1234
|
+
.from(taskTable)
|
|
1235
|
+
.where(and(eq(taskTable.namespace, this.#namespace), eq(taskTable.status, TaskStatus.Running), lt(taskTable.startTimestamp, sql `${TRANSACTION_TIMESTAMP} - ${interval(this.maxExecutionTime, 'milliseconds')}`)))
|
|
1236
|
+
.limit(1000)
|
|
1237
|
+
.for('update', { skipLocked: true }));
|
|
1238
|
+
while (true) {
|
|
1239
|
+
const updatedCount = await this.#repository.useTransaction(options?.transaction, async (tx) => {
|
|
1066
1240
|
const timeoutRows = await tx.pgTransaction
|
|
1067
1241
|
.with(timeoutSelection)
|
|
1068
1242
|
.update(taskTable)
|
|
1069
1243
|
.set({
|
|
1070
|
-
status: TaskStatus.
|
|
1244
|
+
status: TaskStatus.TimedOut,
|
|
1071
1245
|
token: null,
|
|
1072
1246
|
visibilityDeadline: null,
|
|
1073
1247
|
completeTimestamp: TRANSACTION_TIMESTAMP,
|
|
1074
|
-
error:
|
|
1248
|
+
error: jsonbBuildObject({ code: 'MaxTimeExceeded', message: sql `'Hard Execution Timeout: Task ran longer than ' || ${this.maxExecutionTime} || 'ms'`, lastError: taskTable.error }),
|
|
1075
1249
|
})
|
|
1076
1250
|
.where(inArray(taskTable.id, tx.pgTransaction.select().from(timeoutSelection)))
|
|
1077
1251
|
.returning({ id: taskTable.id });
|
|
1078
|
-
totalUpdated += timeoutRows.length;
|
|
1079
1252
|
if (timeoutRows.length > 0) {
|
|
1080
|
-
|
|
1081
|
-
await this.resolveDependencies(row.id, TaskStatus.Dead, { transaction: tx });
|
|
1082
|
-
}
|
|
1253
|
+
await this.resolveDependenciesMany(timeoutRows.map((r) => ({ id: r.id, status: TaskStatus.TimedOut, namespace: this.#namespace })), { transaction: tx });
|
|
1083
1254
|
}
|
|
1084
|
-
|
|
1085
|
-
const agingSelection = tx.pgTransaction.$with('aging_selection').as((qb) => qb
|
|
1086
|
-
.select({ id: taskTable.id })
|
|
1087
|
-
.from(taskTable)
|
|
1088
|
-
.where(and(eq(taskTable.namespace, this.#namespace), eq(taskTable.status, TaskStatus.Pending), lt(taskTable.priorityAgeTimestamp, sql `${TRANSACTION_TIMESTAMP} - ${interval(this.priorityAgingInterval, 'milliseconds')}`)))
|
|
1089
|
-
.limit(1000)
|
|
1090
|
-
.for('update', { skipLocked: true }));
|
|
1091
|
-
const agingRows = await tx.pgTransaction
|
|
1092
|
-
.with(agingSelection)
|
|
1093
|
-
.update(taskTable)
|
|
1094
|
-
.set({
|
|
1095
|
-
priority: sql `${taskTable.priority} - ${this.priorityAgingStep}`,
|
|
1096
|
-
priorityAgeTimestamp: TRANSACTION_TIMESTAMP,
|
|
1097
|
-
})
|
|
1098
|
-
.where(inArray(taskTable.id, tx.pgTransaction.select().from(agingSelection)))
|
|
1099
|
-
.returning({ id: taskTable.id });
|
|
1100
|
-
totalUpdated += agingRows.length;
|
|
1101
|
-
return totalUpdated;
|
|
1255
|
+
return timeoutRows.length;
|
|
1102
1256
|
});
|
|
1103
|
-
if (
|
|
1257
|
+
if (updatedCount < 1000) {
|
|
1258
|
+
break;
|
|
1259
|
+
}
|
|
1260
|
+
}
|
|
1261
|
+
}
|
|
1262
|
+
async processPriorityAging(options) {
|
|
1263
|
+
const session = options?.transaction?.pgTransaction ?? this.#database;
|
|
1264
|
+
const agingSelection = session.$with('aging_selection').as((qb) => qb
|
|
1265
|
+
.select({ id: taskTable.id })
|
|
1266
|
+
.from(taskTable)
|
|
1267
|
+
.where(and(eq(taskTable.namespace, this.#namespace), inArray(taskTable.status, queueableStatuses), lt(taskTable.priorityAgeTimestamp, sql `${TRANSACTION_TIMESTAMP} - ${interval(this.priorityAgingInterval, 'milliseconds')}`)))
|
|
1268
|
+
.limit(1000)
|
|
1269
|
+
.for('update', { skipLocked: true }));
|
|
1270
|
+
while (true) {
|
|
1271
|
+
const agingRows = await session
|
|
1272
|
+
.with(agingSelection)
|
|
1273
|
+
.update(taskTable)
|
|
1274
|
+
.set({
|
|
1275
|
+
priority: sql `${taskTable.priority} - ${this.priorityAgingStep}`,
|
|
1276
|
+
priorityAgeTimestamp: TRANSACTION_TIMESTAMP,
|
|
1277
|
+
})
|
|
1278
|
+
.where(inArray(taskTable.id, session.select().from(agingSelection)))
|
|
1279
|
+
.returning({ id: taskTable.id });
|
|
1280
|
+
if (agingRows.length < 1000) {
|
|
1104
1281
|
break;
|
|
1105
1282
|
}
|
|
1106
1283
|
}
|
|
@@ -1122,44 +1299,87 @@ let PostgresTaskQueue = class PostgresTaskQueue extends TaskQueue {
|
|
|
1122
1299
|
priorityAgeTimestamp: TRANSACTION_TIMESTAMP,
|
|
1123
1300
|
state: (options?.resetState == true) ? null : undefined,
|
|
1124
1301
|
})
|
|
1125
|
-
.where(and(eq(taskTable.
|
|
1302
|
+
.where(and(eq(taskTable.id, id), or(inArray(taskTable.status, queueableStatuses), inArray(taskTable.status, finalizedStatuses), lt(taskTable.visibilityDeadline, TRANSACTION_TIMESTAMP))));
|
|
1126
1303
|
}
|
|
1127
|
-
notify() {
|
|
1128
|
-
this.#messageBus.publishAndForget();
|
|
1304
|
+
notify(namespace = this.#namespace) {
|
|
1305
|
+
this.#messageBus.publishAndForget(namespace);
|
|
1129
1306
|
}
|
|
1130
1307
|
async *getConsumer(cancellationSignal, options) {
|
|
1131
|
-
const continue$ = merge(this.#messageBus.allMessages
|
|
1308
|
+
const continue$ = merge(this.#messageBus.allMessages$).pipe(filter((namespace) => namespace == this.#namespace));
|
|
1309
|
+
const mergedContinue$ = merge(continue$, cancellationSignal);
|
|
1132
1310
|
while (cancellationSignal.isUnset) {
|
|
1133
1311
|
const task = await this.dequeue(options);
|
|
1134
1312
|
if (isDefined(task)) {
|
|
1135
1313
|
yield task;
|
|
1136
1314
|
continue;
|
|
1137
1315
|
}
|
|
1138
|
-
await cancelableTimeout(5 * millisecondsPerSecond,
|
|
1316
|
+
await cancelableTimeout(5 * millisecondsPerSecond, mergedContinue$);
|
|
1317
|
+
}
|
|
1318
|
+
}
|
|
1319
|
+
maintenanceLoop() {
|
|
1320
|
+
void this.lowFrequencyMaintenanceLoop();
|
|
1321
|
+
void this.mediumFrequencyMaintenanceLoop();
|
|
1322
|
+
void this.highFrequencyMaintenanceLoop();
|
|
1323
|
+
}
|
|
1324
|
+
async lowFrequencyMaintenanceLoop() {
|
|
1325
|
+
while (this.#cancellationSignal.isUnset) {
|
|
1326
|
+
try {
|
|
1327
|
+
await Promise.allSettled([
|
|
1328
|
+
this.performArchival(),
|
|
1329
|
+
this.performArchivePurge(),
|
|
1330
|
+
]);
|
|
1331
|
+
}
|
|
1332
|
+
catch (error) {
|
|
1333
|
+
this.#logger.error('Error during low frequency maintenance loop', error);
|
|
1334
|
+
}
|
|
1335
|
+
finally {
|
|
1336
|
+
await cancelableTimeout(5 * millisecondsPerMinute, this.#cancellationSignal);
|
|
1337
|
+
}
|
|
1338
|
+
}
|
|
1339
|
+
}
|
|
1340
|
+
async mediumFrequencyMaintenanceLoop() {
|
|
1341
|
+
while (this.#cancellationSignal.isUnset) {
|
|
1342
|
+
try {
|
|
1343
|
+
await Promise.allSettled([
|
|
1344
|
+
this.processExpirations(),
|
|
1345
|
+
this.processPriorityAging(),
|
|
1346
|
+
]);
|
|
1347
|
+
}
|
|
1348
|
+
catch (error) {
|
|
1349
|
+
this.#logger.error('Error during medium frequency maintenance loop', error);
|
|
1350
|
+
}
|
|
1351
|
+
finally {
|
|
1352
|
+
await cancelableTimeout(millisecondsPerMinute, this.#cancellationSignal);
|
|
1353
|
+
}
|
|
1139
1354
|
}
|
|
1140
1355
|
}
|
|
1141
|
-
async
|
|
1356
|
+
async highFrequencyMaintenanceLoop() {
|
|
1142
1357
|
while (this.#cancellationSignal.isUnset) {
|
|
1143
1358
|
try {
|
|
1144
|
-
await
|
|
1359
|
+
await Promise.allSettled([
|
|
1360
|
+
this.processZombieRetries(),
|
|
1361
|
+
this.processZombieExhaustions(),
|
|
1362
|
+
this.processHardTimeouts(),
|
|
1363
|
+
]);
|
|
1145
1364
|
}
|
|
1146
1365
|
catch (error) {
|
|
1147
|
-
this.#logger.error('Error during maintenance loop', error);
|
|
1366
|
+
this.#logger.error('Error during high frequency maintenance loop', error);
|
|
1148
1367
|
}
|
|
1149
1368
|
finally {
|
|
1150
|
-
await cancelableTimeout(
|
|
1369
|
+
await cancelableTimeout(10 * millisecondsPerSecond, this.#cancellationSignal);
|
|
1151
1370
|
}
|
|
1152
1371
|
}
|
|
1153
1372
|
}
|
|
1154
1373
|
async *getBatchConsumer(size, cancellationSignal, options) {
|
|
1155
|
-
const continue$ = merge(this.#messageBus.allMessages
|
|
1374
|
+
const continue$ = merge(this.#messageBus.allMessages$).pipe(filter((namespace) => namespace == this.#namespace));
|
|
1375
|
+
const mergedContinue$ = merge(continue$, cancellationSignal);
|
|
1156
1376
|
while (cancellationSignal.isUnset) {
|
|
1157
1377
|
const tasks = await this.dequeueMany(size, options);
|
|
1158
1378
|
if (tasks.length > 0) {
|
|
1159
1379
|
yield tasks;
|
|
1160
1380
|
continue;
|
|
1161
1381
|
}
|
|
1162
|
-
await cancelableTimeout(5 * millisecondsPerSecond,
|
|
1382
|
+
await cancelableTimeout(5 * millisecondsPerSecond, mergedContinue$);
|
|
1163
1383
|
}
|
|
1164
1384
|
}
|
|
1165
1385
|
};
|