@tstdl/base 0.93.140 → 0.93.142
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/application/application.d.ts +1 -1
- package/application/application.js +1 -1
- package/application/providers.d.ts +20 -2
- package/application/providers.js +34 -7
- package/audit/module.d.ts +5 -0
- package/audit/module.js +9 -1
- package/authentication/client/authentication.service.d.ts +1 -0
- package/authentication/client/authentication.service.js +3 -2
- package/authentication/server/module.d.ts +5 -0
- package/authentication/server/module.js +9 -1
- package/authentication/tests/authentication.api-controller.test.js +1 -1
- package/authentication/tests/authentication.api-request-token.provider.test.js +1 -1
- package/authentication/tests/authentication.client-service.test.js +1 -1
- package/circuit-breaker/circuit-breaker.d.ts +6 -4
- package/circuit-breaker/postgres/circuit-breaker.d.ts +1 -0
- package/circuit-breaker/postgres/circuit-breaker.js +8 -5
- package/circuit-breaker/postgres/module.d.ts +1 -0
- package/circuit-breaker/postgres/module.js +5 -1
- package/circuit-breaker/tests/circuit-breaker.test.js +20 -0
- package/document-management/server/configure.js +5 -1
- package/document-management/server/module.d.ts +1 -1
- package/document-management/server/module.js +1 -1
- package/document-management/server/services/document-management-ancillary.service.js +1 -1
- package/document-management/tests/ai-config-hierarchy.test.js +0 -5
- package/document-management/tests/document-management-ai-overrides.test.js +0 -1
- package/document-management/tests/document-validation-ai-overrides.test.js +0 -1
- package/examples/document-management/main.d.ts +1 -0
- package/examples/document-management/main.js +14 -11
- package/key-value-store/postgres/module.d.ts +1 -0
- package/key-value-store/postgres/module.js +5 -1
- package/lock/postgres/module.d.ts +1 -0
- package/lock/postgres/module.js +5 -1
- package/mail/module.d.ts +5 -1
- package/mail/module.js +11 -6
- package/module/modules/web-server.module.js +2 -3
- package/notification/server/module.d.ts +1 -0
- package/notification/server/module.js +5 -1
- package/notification/tests/notification-api.test.js +5 -1
- package/notification/tests/notification-flow.test.js +8 -5
- package/orm/decorators.d.ts +22 -5
- package/orm/decorators.js +10 -1
- package/orm/server/bootstrap.d.ts +11 -0
- package/orm/server/bootstrap.js +31 -0
- package/orm/server/drizzle/schema-converter.d.ts +3 -1
- package/orm/server/drizzle/schema-converter.js +85 -56
- package/orm/server/encryption.d.ts +0 -1
- package/orm/server/encryption.js +1 -4
- package/orm/server/extension.d.ts +14 -0
- package/orm/server/extension.js +27 -0
- package/orm/server/index.d.ts +3 -6
- package/orm/server/index.js +3 -6
- package/orm/server/migration.d.ts +18 -0
- package/orm/server/migration.js +58 -0
- package/orm/server/repository.d.ts +2 -1
- package/orm/server/repository.js +19 -9
- package/orm/server/transaction.d.ts +6 -10
- package/orm/server/transaction.js +25 -26
- package/orm/server/transactional.js +3 -3
- package/orm/tests/database-extension.test.js +63 -0
- package/orm/tests/database-migration.test.js +83 -0
- package/orm/tests/encryption.test.js +3 -4
- package/orm/tests/repository-compound-primary-key.test.d.ts +2 -0
- package/orm/tests/repository-compound-primary-key.test.js +234 -0
- package/orm/tests/schema-generation.test.d.ts +1 -0
- package/orm/tests/schema-generation.test.js +52 -5
- package/orm/utils.d.ts +17 -2
- package/orm/utils.js +49 -1
- package/package.json +5 -4
- package/rate-limit/postgres/module.d.ts +1 -0
- package/rate-limit/postgres/module.js +5 -1
- package/reflection/decorator-data.js +11 -12
- package/task-queue/README.md +2 -10
- package/task-queue/postgres/drizzle/0000_great_gwen_stacy.sql +84 -0
- package/task-queue/postgres/drizzle/meta/0000_snapshot.json +250 -89
- package/task-queue/postgres/drizzle/meta/_journal.json +2 -2
- package/task-queue/postgres/module.d.ts +1 -0
- package/task-queue/postgres/module.js +6 -1
- package/task-queue/postgres/schemas.d.ts +15 -6
- package/task-queue/postgres/schemas.js +4 -3
- package/task-queue/postgres/task-queue.d.ts +18 -15
- package/task-queue/postgres/task-queue.js +797 -499
- package/task-queue/postgres/task.model.d.ts +20 -9
- package/task-queue/postgres/task.model.js +65 -39
- package/task-queue/task-context.d.ts +12 -7
- package/task-queue/task-context.js +8 -6
- package/task-queue/task-queue.d.ts +364 -43
- package/task-queue/task-queue.js +153 -41
- package/task-queue/tests/coverage-branch.test.d.ts +1 -0
- package/task-queue/tests/coverage-branch.test.js +395 -0
- package/task-queue/tests/coverage-enhancement.test.d.ts +1 -0
- package/task-queue/tests/coverage-enhancement.test.js +150 -0
- package/task-queue/tests/dag.test.d.ts +1 -0
- package/task-queue/tests/dag.test.js +188 -0
- package/task-queue/tests/dependencies.test.js +165 -47
- package/task-queue/tests/enqueue-batch.test.d.ts +1 -0
- package/task-queue/tests/enqueue-batch.test.js +125 -0
- package/task-queue/tests/fan-out-spawning.test.d.ts +1 -0
- package/task-queue/tests/fan-out-spawning.test.js +94 -0
- package/task-queue/tests/idempotent-replacement.test.d.ts +1 -0
- package/task-queue/tests/idempotent-replacement.test.js +114 -0
- package/task-queue/tests/missing-idempotent-tasks.test.d.ts +1 -0
- package/task-queue/tests/missing-idempotent-tasks.test.js +39 -0
- package/task-queue/tests/queue.test.js +294 -49
- package/task-queue/tests/shutdown.test.d.ts +1 -0
- package/task-queue/tests/shutdown.test.js +41 -0
- package/task-queue/tests/transactions.test.d.ts +1 -0
- package/task-queue/tests/transactions.test.js +47 -0
- package/task-queue/tests/worker.test.js +63 -15
- package/task-queue/tests/zombie-parent.test.d.ts +1 -0
- package/task-queue/tests/zombie-parent.test.js +45 -0
- package/task-queue/tests/zombie-recovery.test.d.ts +1 -0
- package/task-queue/tests/zombie-recovery.test.js +51 -0
- package/test5.js +5 -5
- package/testing/integration-setup.d.ts +4 -4
- package/testing/integration-setup.js +56 -29
- package/text/localization.service.js +2 -2
- package/utils/file-reader.js +1 -2
- package/utils/timing.d.ts +2 -2
- package/task-queue/postgres/drizzle/0000_simple_invisible_woman.sql +0 -74
- package/task-queue/tests/complex.test.js +0 -306
- package/task-queue/tests/extensive-dependencies.test.js +0 -234
- /package/{task-queue/tests/complex.test.d.ts → orm/tests/database-extension.test.d.ts} +0 -0
- /package/{task-queue/tests/extensive-dependencies.test.d.ts → orm/tests/database-migration.test.d.ts} +0 -0
|
@@ -56,35 +56,34 @@ var __disposeResources = (this && this.__disposeResources) || (function (Suppres
|
|
|
56
56
|
var e = new Error(message);
|
|
57
57
|
return e.name = "SuppressedError", e.error = error, e.suppressed = suppressed, e;
|
|
58
58
|
});
|
|
59
|
-
import { and, asc, count, eq, gt, gte, inArray, lt, lte, notInArray, or, sql, isNull as sqlIsNull } from 'drizzle-orm';
|
|
60
|
-
import { merge, throttleTime } from 'rxjs';
|
|
59
|
+
import { aliasedTable, and, asc, count, eq, gt, gte, inArray, lt, lte, notExists, notInArray, or, sql, count as sqlCount, isNotNull as sqlIsNotNull, isNull as sqlIsNull } from 'drizzle-orm';
|
|
60
|
+
import { filter, merge, throttleTime } from 'rxjs';
|
|
61
61
|
import { CancellationSignal } from '../../cancellation/index.js';
|
|
62
62
|
import { CircuitBreaker, CircuitBreakerState } from '../../circuit-breaker/index.js';
|
|
63
63
|
import { serializeError, TimeoutError } from '../../errors/index.js';
|
|
64
64
|
import { afterResolve, inject, provide, Singleton } from '../../injector/index.js';
|
|
65
65
|
import { Logger } from '../../logger/index.js';
|
|
66
66
|
import { MessageBus } from '../../message-bus/index.js';
|
|
67
|
-
import { arrayOverlaps, coalesce, interval, RANDOM_UUID_V4, TRANSACTION_TIMESTAMP
|
|
68
|
-
import { DatabaseConfig, injectRepository } from '../../orm/server/index.js';
|
|
67
|
+
import { arrayOverlaps, caseWhen, coalesce, enumValue, getEntityIds, greatest, interval, jsonbBuildObject, RANDOM_UUID_V4, TRANSACTION_TIMESTAMP } from '../../orm/index.js';
|
|
68
|
+
import { Database, DatabaseConfig, injectRepository } from '../../orm/server/index.js';
|
|
69
69
|
import { RateLimiter } from '../../rate-limit/index.js';
|
|
70
|
-
import {
|
|
71
|
-
import { digest } from '../../utils/cryptography.js';
|
|
70
|
+
import { distinct, toArray } from '../../utils/array/array.js';
|
|
72
71
|
import { currentTimestamp } from '../../utils/date-time.js';
|
|
73
72
|
import { Timer } from '../../utils/timer.js';
|
|
74
73
|
import { cancelableTimeout } from '../../utils/timing.js';
|
|
75
|
-
import { isDefined, isNotNull, isNull, isString, isUndefined } from '../../utils/type-guards.js';
|
|
76
|
-
import { millisecondsPerSecond } from '../../utils/units.js';
|
|
77
|
-
import { defaultQueueConfig,
|
|
74
|
+
import { isDefined, isNotNull, isNull, isNumber, isString, isUndefined } from '../../utils/type-guards.js';
|
|
75
|
+
import { millisecondsPerMinute, millisecondsPerSecond } from '../../utils/units.js';
|
|
76
|
+
import { defaultQueueConfig, TaskDependencyType, TaskQueue, TaskStatus } from '../task-queue.js';
|
|
78
77
|
import { PostgresTaskQueueModuleConfig } from './module.js';
|
|
79
|
-
import { taskArchive as taskArchiveTable, taskStatus, task as taskTable } from './schemas.js';
|
|
80
|
-
import { PostgresTask, PostgresTaskArchive } from './task.model.js';
|
|
78
|
+
import { taskArchive as taskArchiveTable, taskDependency as taskDependencyTable, taskDependencyType, taskStatus, task as taskTable } from './schemas.js';
|
|
79
|
+
import { finalizedStatuses, PostgresTask, PostgresTaskArchive, queueableOrWaitableStatuses, queueableStatuses, terminalStatuses } from './task.model.js';
|
|
81
80
|
let PostgresTaskQueue = class PostgresTaskQueue extends TaskQueue {
|
|
82
|
-
#
|
|
81
|
+
#database = inject(Database);
|
|
83
82
|
#repository = injectRepository(PostgresTask);
|
|
84
83
|
#archiveRepository = injectRepository(PostgresTaskArchive);
|
|
85
84
|
#config = this.config;
|
|
86
85
|
#namespace = isString(this.#config) ? this.#config : this.#config.namespace;
|
|
87
|
-
#messageBus = inject((MessageBus),
|
|
86
|
+
#messageBus = inject((MessageBus), 'PostgresTaskQueue');
|
|
88
87
|
#logger = inject(Logger, `PostgresTaskQueue:${this.#namespace}`);
|
|
89
88
|
#cancellationSignal = inject(CancellationSignal);
|
|
90
89
|
#rateLimiter = inject(RateLimiter, {
|
|
@@ -111,22 +110,19 @@ let PostgresTaskQueue = class PostgresTaskQueue extends TaskQueue {
|
|
|
111
110
|
rateLimit = this.#config.rateLimit ?? defaultQueueConfig.rateLimit;
|
|
112
111
|
rateInterval = this.#config.rateInterval ?? defaultQueueConfig.rateInterval;
|
|
113
112
|
idempotencyWindow = this.#config.idempotencyWindow ?? defaultQueueConfig.idempotencyWindow;
|
|
114
|
-
globalConcurrency = this.#config.globalConcurrency ?? defaultQueueConfig.globalConcurrency;
|
|
115
113
|
#takeNewUpdate = {
|
|
116
|
-
id: RANDOM_UUID_V4,
|
|
117
114
|
namespace: this.#namespace,
|
|
118
115
|
type: sql `excluded.type`,
|
|
119
|
-
status: TaskStatus.Pending,
|
|
116
|
+
status: caseWhen(gt(sql `excluded.unresolved_schedule_dependencies`, 0), enumValue(TaskStatus, taskStatus, TaskStatus.Waiting)).else(enumValue(TaskStatus, taskStatus, TaskStatus.Pending)),
|
|
120
117
|
token: null,
|
|
121
118
|
priority: sql `excluded.priority`,
|
|
122
119
|
idempotencyKey: sql `excluded.idempotency_key`,
|
|
123
120
|
traceId: sql `excluded.trace_id`,
|
|
121
|
+
parentId: sql `excluded.parent_id`,
|
|
124
122
|
tags: sql `excluded.tags`,
|
|
125
|
-
|
|
126
|
-
|
|
123
|
+
unresolvedScheduleDependencies: taskTable.unresolvedScheduleDependencies,
|
|
124
|
+
unresolvedCompleteDependencies: taskTable.unresolvedCompleteDependencies,
|
|
127
125
|
failFast: sql `excluded.fail_fast`,
|
|
128
|
-
dependencyJoinMode: sql `excluded.dependency_join_mode`,
|
|
129
|
-
dependencyTriggerStatuses: sql `excluded.dependency_trigger_statuses`,
|
|
130
126
|
tries: 0,
|
|
131
127
|
creationTimestamp: TRANSACTION_TIMESTAMP,
|
|
132
128
|
priorityAgeTimestamp: TRANSACTION_TIMESTAMP,
|
|
@@ -143,8 +139,7 @@ let PostgresTaskQueue = class PostgresTaskQueue extends TaskQueue {
|
|
|
143
139
|
};
|
|
144
140
|
[afterResolve]() {
|
|
145
141
|
if (!this.isInTransaction) {
|
|
146
|
-
this.
|
|
147
|
-
void this.maintenanceLoop();
|
|
142
|
+
this.maintenanceLoop();
|
|
148
143
|
}
|
|
149
144
|
}
|
|
150
145
|
async enqueue(type, data, options) {
|
|
@@ -157,76 +152,207 @@ let PostgresTaskQueue = class PostgresTaskQueue extends TaskQueue {
|
|
|
157
152
|
if (items.length == 0) {
|
|
158
153
|
return (options?.returnTasks == true) ? [] : undefined;
|
|
159
154
|
}
|
|
160
|
-
const
|
|
161
|
-
|
|
155
|
+
const itemsWithDistinctDependencies = items.map((item) => ({
|
|
156
|
+
...item,
|
|
157
|
+
scheduleAfter: isDefined(item.scheduleAfter) ? Array.from(new Map(item.scheduleAfter.map((s) => [isNumber(s) || isString(s) ? s : JSON.stringify(s), s])).values()) : undefined,
|
|
158
|
+
completeAfter: isDefined(item.completeAfter) ? Array.from(new Map(item.completeAfter.map((s) => [isNumber(s) || isString(s) ? s : JSON.stringify(s), s])).values()) : undefined,
|
|
159
|
+
}));
|
|
160
|
+
const entitiesWithIndex = itemsWithDistinctDependencies.map((item, index) => {
|
|
162
161
|
return {
|
|
163
|
-
|
|
164
|
-
|
|
165
|
-
|
|
166
|
-
|
|
167
|
-
|
|
168
|
-
|
|
169
|
-
|
|
170
|
-
|
|
171
|
-
|
|
172
|
-
|
|
173
|
-
|
|
174
|
-
|
|
175
|
-
|
|
176
|
-
|
|
177
|
-
|
|
178
|
-
|
|
179
|
-
|
|
180
|
-
|
|
181
|
-
|
|
182
|
-
|
|
183
|
-
|
|
184
|
-
|
|
185
|
-
|
|
186
|
-
|
|
187
|
-
|
|
188
|
-
|
|
189
|
-
|
|
162
|
+
index,
|
|
163
|
+
entity: {
|
|
164
|
+
namespace: this.#namespace,
|
|
165
|
+
type: item.type,
|
|
166
|
+
status: TaskStatus.Pending,
|
|
167
|
+
token: null,
|
|
168
|
+
priority: item.priority ?? 1000,
|
|
169
|
+
idempotencyKey: item.idempotencyKey ?? null,
|
|
170
|
+
traceId: null,
|
|
171
|
+
tags: item.tags ?? [],
|
|
172
|
+
unresolvedScheduleDependencies: 0,
|
|
173
|
+
unresolvedCompleteDependencies: 0,
|
|
174
|
+
failFast: item.failFast ?? false,
|
|
175
|
+
parentId: item.parentId ?? null,
|
|
176
|
+
tries: 0,
|
|
177
|
+
progress: 0,
|
|
178
|
+
creationTimestamp: TRANSACTION_TIMESTAMP,
|
|
179
|
+
priorityAgeTimestamp: TRANSACTION_TIMESTAMP,
|
|
180
|
+
scheduleTimestamp: item.scheduleTimestamp ?? TRANSACTION_TIMESTAMP,
|
|
181
|
+
startTimestamp: null,
|
|
182
|
+
timeToLive: item.timeToLive ?? sql `${TRANSACTION_TIMESTAMP} + ${interval(this.defaultTimeToLive, 'milliseconds')}`,
|
|
183
|
+
visibilityDeadline: null,
|
|
184
|
+
completeTimestamp: null,
|
|
185
|
+
data: item.data,
|
|
186
|
+
state: null,
|
|
187
|
+
result: null,
|
|
188
|
+
error: null,
|
|
189
|
+
},
|
|
190
190
|
};
|
|
191
191
|
});
|
|
192
|
-
const itemsWithIdempotency =
|
|
193
|
-
const itemsWithoutIdempotency =
|
|
194
|
-
const hasDependencies =
|
|
195
|
-
const mustUseTransaction = (
|
|
192
|
+
const itemsWithIdempotency = entitiesWithIndex.filter((e) => isNotNull(e.entity.idempotencyKey));
|
|
193
|
+
const itemsWithoutIdempotency = entitiesWithIndex.filter((e) => isNull(e.entity.idempotencyKey));
|
|
194
|
+
const hasDependencies = itemsWithDistinctDependencies.some((item) => ((item.scheduleAfter?.length ?? 0) > 0) || ((item.completeAfter?.length ?? 0) > 0) || (isDefined(item.parentId) && (item.waitForCompletion ?? true)));
|
|
195
|
+
const mustUseTransaction = (entitiesWithIndex.length > 1) || hasDependencies;
|
|
196
196
|
const newTransaction = __addDisposableResource(env_1, (mustUseTransaction && isUndefined(options?.transaction)) ? await this.#repository.startTransaction() : undefined, true);
|
|
197
197
|
const transaction = newTransaction ?? options?.transaction;
|
|
198
|
-
const
|
|
198
|
+
const session = transaction?.pgTransaction ?? this.#repository.session;
|
|
199
|
+
const tasks = new Array(itemsWithDistinctDependencies.length);
|
|
200
|
+
let insertedRows = [];
|
|
201
|
+
let upsertedRows = [];
|
|
199
202
|
if (itemsWithoutIdempotency.length > 0) {
|
|
200
|
-
|
|
201
|
-
|
|
203
|
+
insertedRows = await session.insert(taskTable).values(itemsWithoutIdempotency.map((i) => i.entity)).returning();
|
|
204
|
+
for (const [i, row] of insertedRows.entries()) {
|
|
205
|
+
tasks[itemsWithoutIdempotency[i].index] = row;
|
|
206
|
+
}
|
|
202
207
|
}
|
|
203
208
|
if (itemsWithIdempotency.length > 0) {
|
|
204
|
-
const repository = this.#repository.withOptionalTransaction(transaction);
|
|
205
209
|
const windowCutoff = sql `${TRANSACTION_TIMESTAMP} - ${interval(this.idempotencyWindow, 'milliseconds')}`;
|
|
206
210
|
const setWhere = (options?.replace == true) ? undefined : sql `${taskTable.creationTimestamp} < ${windowCutoff}`;
|
|
207
|
-
|
|
208
|
-
|
|
209
|
-
|
|
210
|
-
|
|
211
|
-
|
|
212
|
-
|
|
213
|
-
|
|
214
|
-
|
|
215
|
-
|
|
211
|
+
upsertedRows = await session
|
|
212
|
+
.insert(taskTable)
|
|
213
|
+
.values(itemsWithIdempotency.map((i) => i.entity))
|
|
214
|
+
.onConflictDoUpdate({
|
|
215
|
+
target: [taskTable.namespace, taskTable.idempotencyKey],
|
|
216
|
+
set: this.#takeNewUpdate,
|
|
217
|
+
setWhere,
|
|
218
|
+
})
|
|
219
|
+
.returning();
|
|
220
|
+
const upsertedIds = upsertedRows.map((t) => t.id);
|
|
221
|
+
if (upsertedIds.length > 0) {
|
|
222
|
+
await session
|
|
223
|
+
.delete(taskDependencyTable)
|
|
224
|
+
.where(and(inArray(taskDependencyTable.taskId, upsertedIds), inArray(taskDependencyTable.type, [TaskDependencyType.Schedule, TaskDependencyType.Complete])));
|
|
225
|
+
await session
|
|
226
|
+
.update(taskTable)
|
|
227
|
+
.set({ unresolvedScheduleDependencies: 0 })
|
|
228
|
+
.where(inArray(taskTable.id, upsertedIds));
|
|
229
|
+
}
|
|
230
|
+
if ((options?.returnTasks == true) || hasDependencies) {
|
|
231
|
+
const upsertedKeysMap = new Map(upsertedRows.map((t) => [t.idempotencyKey, t]));
|
|
232
|
+
const missingKeysWithIndex = [];
|
|
233
|
+
for (const item of itemsWithIdempotency) {
|
|
234
|
+
const key = item.entity.idempotencyKey;
|
|
235
|
+
const task = upsertedKeysMap.get(key);
|
|
236
|
+
if (isDefined(task)) {
|
|
237
|
+
tasks[item.index] = task;
|
|
238
|
+
}
|
|
239
|
+
else {
|
|
240
|
+
missingKeysWithIndex.push({ index: item.index, key });
|
|
241
|
+
}
|
|
242
|
+
}
|
|
243
|
+
if (missingKeysWithIndex.length > 0) {
|
|
244
|
+
const existingRows = await session
|
|
245
|
+
.select()
|
|
246
|
+
.from(taskTable)
|
|
247
|
+
.where(and(eq(taskTable.namespace, this.#namespace), inArray(taskTable.idempotencyKey, missingKeysWithIndex.map((m) => m.key))));
|
|
248
|
+
const existingRowsMap = new Map(existingRows.map((t) => [t.idempotencyKey, t]));
|
|
249
|
+
for (const missing of missingKeysWithIndex) {
|
|
250
|
+
tasks[missing.index] = existingRowsMap.get(missing.key);
|
|
251
|
+
}
|
|
252
|
+
}
|
|
253
|
+
}
|
|
254
|
+
}
|
|
255
|
+
if (hasDependencies) {
|
|
256
|
+
const dependencies = [];
|
|
257
|
+
const processedTaskIds = new Set([...insertedRows.map((r) => r.id), ...upsertedRows.map((r) => r.id)]);
|
|
258
|
+
for (const [index, item] of itemsWithDistinctDependencies.entries()) {
|
|
259
|
+
const task = tasks[index];
|
|
260
|
+
if (!processedTaskIds.has(task.id)) {
|
|
261
|
+
continue;
|
|
262
|
+
}
|
|
263
|
+
if (isDefined(item.parentId) && (item.waitForCompletion ?? true)) {
|
|
264
|
+
dependencies.push({
|
|
265
|
+
taskId: item.parentId,
|
|
266
|
+
dependencyTaskId: task.id,
|
|
267
|
+
type: TaskDependencyType.Child,
|
|
268
|
+
requiredStatuses: [TaskStatus.Completed],
|
|
216
269
|
});
|
|
217
|
-
|
|
270
|
+
}
|
|
271
|
+
if (isDefined(item.scheduleAfter)) {
|
|
272
|
+
for (const dependency of item.scheduleAfter) {
|
|
273
|
+
let dependencyTaskId;
|
|
274
|
+
let requiredStatuses;
|
|
275
|
+
if (isNumber(dependency)) {
|
|
276
|
+
dependencyTaskId = tasks[dependency].id;
|
|
277
|
+
requiredStatuses = [TaskStatus.Completed];
|
|
278
|
+
}
|
|
279
|
+
else if (isString(dependency)) {
|
|
280
|
+
dependencyTaskId = dependency;
|
|
281
|
+
requiredStatuses = [TaskStatus.Completed];
|
|
282
|
+
}
|
|
283
|
+
else {
|
|
284
|
+
dependencyTaskId = isDefined(dependency.index) ? tasks[dependency.index].id : dependency.id;
|
|
285
|
+
requiredStatuses = dependency.requiredStatuses ?? [TaskStatus.Completed];
|
|
286
|
+
}
|
|
287
|
+
dependencies.push({
|
|
288
|
+
taskId: task.id,
|
|
289
|
+
dependencyTaskId,
|
|
290
|
+
type: TaskDependencyType.Schedule,
|
|
291
|
+
requiredStatuses: requiredStatuses,
|
|
292
|
+
});
|
|
293
|
+
}
|
|
294
|
+
}
|
|
295
|
+
if (isDefined(item.completeAfter)) {
|
|
296
|
+
for (const dependency of item.completeAfter) {
|
|
297
|
+
let dependencyTaskId;
|
|
298
|
+
let requiredStatuses;
|
|
299
|
+
if (isNumber(dependency)) {
|
|
300
|
+
dependencyTaskId = tasks[dependency].id;
|
|
301
|
+
requiredStatuses = [TaskStatus.Completed];
|
|
302
|
+
}
|
|
303
|
+
else if (isString(dependency)) {
|
|
304
|
+
dependencyTaskId = dependency;
|
|
305
|
+
requiredStatuses = [TaskStatus.Completed];
|
|
306
|
+
}
|
|
307
|
+
else {
|
|
308
|
+
dependencyTaskId = isDefined(dependency.index) ? tasks[dependency.index].id : dependency.id;
|
|
309
|
+
requiredStatuses = dependency.requiredStatuses ?? [TaskStatus.Completed];
|
|
310
|
+
}
|
|
311
|
+
dependencies.push({
|
|
312
|
+
taskId: task.id,
|
|
313
|
+
dependencyTaskId,
|
|
314
|
+
type: TaskDependencyType.Complete,
|
|
315
|
+
requiredStatuses: requiredStatuses,
|
|
316
|
+
});
|
|
317
|
+
}
|
|
318
|
+
}
|
|
319
|
+
}
|
|
320
|
+
if (dependencies.length > 0) {
|
|
321
|
+
const inserted = await session
|
|
322
|
+
.insert(taskDependencyTable)
|
|
323
|
+
.values(dependencies)
|
|
324
|
+
.onConflictDoNothing()
|
|
325
|
+
.returning();
|
|
326
|
+
const distinctDependencyIds = distinct(dependencies.map((d) => d.dependencyTaskId));
|
|
327
|
+
if (distinctDependencyIds.length > 0) {
|
|
328
|
+
await this.incrementCounters(inserted, { transaction });
|
|
329
|
+
const dependencyStatuses = await session
|
|
330
|
+
.select({ id: taskTable.id, status: taskTable.status })
|
|
331
|
+
.from(taskTable)
|
|
332
|
+
.where(inArray(taskTable.id, distinctDependencyIds))
|
|
333
|
+
.unionAll(session
|
|
334
|
+
.select({ id: taskArchiveTable.id, status: sql `${enumValue(TaskStatus, taskStatus, TaskStatus.Completed)}` })
|
|
335
|
+
.from(taskArchiveTable)
|
|
336
|
+
.where(inArray(taskArchiveTable.id, distinctDependencyIds)));
|
|
337
|
+
if (dependencyStatuses.length > 0) {
|
|
338
|
+
await this.resolveDependenciesMany(dependencyStatuses.map((s) => ({ id: s.id, status: s.status })), { transaction: transaction });
|
|
339
|
+
}
|
|
218
340
|
}
|
|
219
341
|
}
|
|
220
342
|
}
|
|
221
|
-
|
|
222
|
-
|
|
223
|
-
const anyWaiting = tasks.some((t) => t.status == TaskStatus.Waiting);
|
|
224
|
-
if (anyWaiting) {
|
|
225
|
-
const tagsToTrigger = distinct(tasks.filter((t) => t.status == TaskStatus.Waiting).flatMap((t) => [...t.completeAfterTags, ...t.scheduleAfterTags]));
|
|
226
|
-
await this.triggerTagFanIn(tagsToTrigger, { transaction });
|
|
343
|
+
if (isDefined(newTransaction) && !newTransaction.isDone) {
|
|
344
|
+
await newTransaction.commit();
|
|
227
345
|
}
|
|
346
|
+
this.notify();
|
|
228
347
|
if (options?.returnTasks == true) {
|
|
229
|
-
|
|
348
|
+
const finalTasks = await session
|
|
349
|
+
.select()
|
|
350
|
+
.from(taskTable)
|
|
351
|
+
.where(inArray(taskTable.id, tasks.map((t) => t.id)));
|
|
352
|
+
const finalTasksMap = new Map(finalTasks.map((t) => [t.id, t]));
|
|
353
|
+
const orderedFinalTasks = tasks.map((t) => finalTasksMap.get(t.id));
|
|
354
|
+
const mapped = await this.#repository.mapManyToEntity(orderedFinalTasks);
|
|
355
|
+
return mapped;
|
|
230
356
|
}
|
|
231
357
|
return undefined;
|
|
232
358
|
}
|
|
@@ -240,39 +366,105 @@ let PostgresTaskQueue = class PostgresTaskQueue extends TaskQueue {
|
|
|
240
366
|
await result_1;
|
|
241
367
|
}
|
|
242
368
|
}
|
|
369
|
+
async incrementCounters(dependencies, options) {
|
|
370
|
+
if (dependencies.length == 0) {
|
|
371
|
+
return;
|
|
372
|
+
}
|
|
373
|
+
const session = options?.transaction?.pgTransaction ?? this.#repository.session;
|
|
374
|
+
const decrementsToApply = new Map();
|
|
375
|
+
for (const dep of dependencies) {
|
|
376
|
+
const current = decrementsToApply.get(dep.taskId) ?? { schedule: 0, complete: 0 };
|
|
377
|
+
if (dep.type == TaskDependencyType.Schedule) {
|
|
378
|
+
current.schedule++;
|
|
379
|
+
}
|
|
380
|
+
else if (dep.type == TaskDependencyType.Complete || dep.type == TaskDependencyType.Child) {
|
|
381
|
+
current.complete++;
|
|
382
|
+
}
|
|
383
|
+
decrementsToApply.set(dep.taskId, current);
|
|
384
|
+
}
|
|
385
|
+
const values = [...decrementsToApply]
|
|
386
|
+
.toSorted(([idA], [idB]) => idA.localeCompare(idB))
|
|
387
|
+
.map(([taskId, d]) => sql `(${taskId}::uuid, ${d.schedule}::int, ${d.complete}::int)`);
|
|
388
|
+
const updates = session.$with('updates').as((qb) => qb
|
|
389
|
+
.select({
|
|
390
|
+
taskId: sql `(id)::uuid`.as('task_id'),
|
|
391
|
+
scheduleIncrement: sql `(schedule)::int`.as('schedule_increment'),
|
|
392
|
+
completeIncrement: sql `(complete)::int`.as('complete_increment'),
|
|
393
|
+
})
|
|
394
|
+
.from(sql `(VALUES ${sql.join(values, sql `, `)}) AS t(id, schedule, complete)`));
|
|
395
|
+
const updatedRows = await session
|
|
396
|
+
.with(updates)
|
|
397
|
+
.update(taskTable)
|
|
398
|
+
.set({
|
|
399
|
+
unresolvedScheduleDependencies: sql `${taskTable.unresolvedScheduleDependencies} + ${updates.scheduleIncrement}`,
|
|
400
|
+
unresolvedCompleteDependencies: sql `${taskTable.unresolvedCompleteDependencies} + ${updates.completeIncrement}`,
|
|
401
|
+
status: caseWhen(and(eq(taskTable.status, TaskStatus.Pending), gt(sql `${taskTable.unresolvedScheduleDependencies} + ${updates.scheduleIncrement}`, 0)), enumValue(TaskStatus, taskStatus, TaskStatus.Waiting)).else(taskTable.status),
|
|
402
|
+
})
|
|
403
|
+
.from(updates)
|
|
404
|
+
.where(eq(taskTable.id, updates.taskId))
|
|
405
|
+
.returning({ id: taskTable.id, status: taskTable.status, namespace: taskTable.namespace });
|
|
406
|
+
for (const row of updatedRows) {
|
|
407
|
+
this.notify(row.namespace);
|
|
408
|
+
}
|
|
409
|
+
}
|
|
243
410
|
async has(id, options) {
|
|
244
|
-
|
|
411
|
+
const session = options?.transaction?.pgTransaction ?? this.#repository.session;
|
|
412
|
+
const [result] = await session
|
|
413
|
+
.select({ id: taskTable.id })
|
|
414
|
+
.from(taskTable)
|
|
415
|
+
.where(eq(taskTable.id, id))
|
|
416
|
+
.limit(1);
|
|
417
|
+
return isDefined(result);
|
|
245
418
|
}
|
|
246
419
|
async countByTags(tags, options) {
|
|
247
420
|
const tagArray = toArray(tags);
|
|
248
421
|
if (tagArray.length == 0) {
|
|
249
422
|
return 0;
|
|
250
423
|
}
|
|
251
|
-
const
|
|
424
|
+
const session = options?.transaction?.pgTransaction ?? this.#repository.session;
|
|
425
|
+
const [result] = await session
|
|
426
|
+
.select({ count: count() })
|
|
427
|
+
.from(taskTable)
|
|
428
|
+
.where(and(eq(taskTable.namespace, this.#namespace), arrayOverlaps(taskTable.tags, tagArray)));
|
|
429
|
+
return result?.count ?? 0;
|
|
430
|
+
}
|
|
431
|
+
async count(options) {
|
|
432
|
+
const session = options?.transaction?.pgTransaction ?? this.#repository.session;
|
|
433
|
+
const [result] = await session
|
|
252
434
|
.select({ count: count() })
|
|
253
435
|
.from(taskTable)
|
|
254
|
-
.where(and(eq(taskTable.namespace, this.#namespace),
|
|
255
|
-
.execute();
|
|
436
|
+
.where(and(eq(taskTable.namespace, this.#namespace), isDefined(options?.status) ? eq(taskTable.status, options.status) : undefined));
|
|
256
437
|
return result?.count ?? 0;
|
|
257
438
|
}
|
|
258
439
|
async getTask(id, options) {
|
|
259
|
-
const
|
|
260
|
-
|
|
261
|
-
|
|
440
|
+
const session = options?.transaction?.pgTransaction ?? this.#repository.session;
|
|
441
|
+
const [activeRow] = await session
|
|
442
|
+
.select()
|
|
443
|
+
.from(taskTable)
|
|
444
|
+
.where(eq(taskTable.id, id));
|
|
445
|
+
if (isDefined(activeRow)) {
|
|
446
|
+
return await this.#repository.mapToEntity(activeRow);
|
|
447
|
+
}
|
|
448
|
+
const [archiveRow] = await session
|
|
449
|
+
.select()
|
|
450
|
+
.from(taskArchiveTable)
|
|
451
|
+
.where(eq(taskArchiveTable.id, id));
|
|
452
|
+
if (isDefined(archiveRow)) {
|
|
453
|
+
return await this.#archiveRepository.mapToEntity(archiveRow);
|
|
262
454
|
}
|
|
263
|
-
return
|
|
455
|
+
return undefined;
|
|
264
456
|
}
|
|
265
457
|
async getManyByTags(tags, options) {
|
|
266
458
|
const tagArray = toArray(tags);
|
|
267
459
|
if (tagArray.length == 0) {
|
|
268
460
|
return [];
|
|
269
461
|
}
|
|
270
|
-
const
|
|
462
|
+
const session = options?.transaction?.pgTransaction ?? this.#repository.session;
|
|
463
|
+
const rows = await session
|
|
271
464
|
.select()
|
|
272
465
|
.from(taskTable)
|
|
273
|
-
.where(and(eq(taskTable.namespace, this.#namespace), arrayOverlaps(taskTable.tags, tagArray)))
|
|
274
|
-
|
|
275
|
-
return await this.#repository.withOptionalTransaction(options?.transaction).mapManyToEntity(rows);
|
|
466
|
+
.where(and(eq(taskTable.namespace, this.#namespace), arrayOverlaps(taskTable.tags, tagArray)));
|
|
467
|
+
return await this.#repository.mapManyToEntity(rows);
|
|
276
468
|
}
|
|
277
469
|
async getTree(rootId, options) {
|
|
278
470
|
const ids = toArray(rootId);
|
|
@@ -280,19 +472,22 @@ let PostgresTaskQueue = class PostgresTaskQueue extends TaskQueue {
|
|
|
280
472
|
}
|
|
281
473
|
async getTreeByQuery(query, options) {
|
|
282
474
|
return await this.#repository.useTransaction(options?.transaction, async (tx) => {
|
|
283
|
-
const
|
|
284
|
-
const roots = await
|
|
475
|
+
const querySql = this.#repository.convertQuery(query);
|
|
476
|
+
const roots = await tx.pgTransaction
|
|
477
|
+
.select({ id: taskTable.id })
|
|
478
|
+
.from(taskTable)
|
|
479
|
+
.where(querySql);
|
|
285
480
|
if (roots.length == 0) {
|
|
286
481
|
return [];
|
|
287
482
|
}
|
|
288
483
|
const rootIds = roots.map((t) => t.id);
|
|
289
|
-
const
|
|
484
|
+
const { rows } = await tx.pgTransaction.execute(sql `
|
|
290
485
|
WITH RECURSIVE task_tree AS (
|
|
291
486
|
SELECT *, 0 as depth
|
|
292
487
|
FROM ${taskTable}
|
|
293
488
|
WHERE ${inArray(taskTable.id, rootIds)}
|
|
294
489
|
|
|
295
|
-
UNION
|
|
490
|
+
UNION ALL
|
|
296
491
|
|
|
297
492
|
SELECT child.*, parent.depth + 1
|
|
298
493
|
FROM ${taskTable} child
|
|
@@ -301,8 +496,7 @@ let PostgresTaskQueue = class PostgresTaskQueue extends TaskQueue {
|
|
|
301
496
|
)
|
|
302
497
|
SELECT * FROM task_tree
|
|
303
498
|
`);
|
|
304
|
-
|
|
305
|
-
return await repositoryWithTransaction.mapManyToEntity(rows);
|
|
499
|
+
return await this.#repository.mapManyToEntity(rows);
|
|
306
500
|
});
|
|
307
501
|
}
|
|
308
502
|
async waitForTasks(ids, options) {
|
|
@@ -312,9 +506,10 @@ let PostgresTaskQueue = class PostgresTaskQueue extends TaskQueue {
|
|
|
312
506
|
const timeout = options?.timeout ?? Infinity;
|
|
313
507
|
const interval = options?.interval ?? 1000;
|
|
314
508
|
const cancellationSignal = this.#cancellationSignal.optionallyInherit(options?.cancellationSignal);
|
|
315
|
-
const
|
|
509
|
+
const waitStatuses = options?.statuses ?? finalizedStatuses;
|
|
510
|
+
const messageBus$ = this.#messageBus.allMessages$.pipe(filter((namespace) => namespace == this.#namespace), throttleTime(500, undefined, { leading: true, trailing: true }));
|
|
511
|
+
const continue$ = merge(messageBus$, cancellationSignal);
|
|
316
512
|
const timer = Timer.startNew();
|
|
317
|
-
const finalizedStatuses = [TaskStatus.Completed, TaskStatus.Cancelled, TaskStatus.Dead];
|
|
318
513
|
while (true) {
|
|
319
514
|
if (cancellationSignal.isSet) {
|
|
320
515
|
return { cancelled: true };
|
|
@@ -322,11 +517,12 @@ let PostgresTaskQueue = class PostgresTaskQueue extends TaskQueue {
|
|
|
322
517
|
if (timer.milliseconds > timeout) {
|
|
323
518
|
throw new TimeoutError('Timeout while waiting for tasks to complete');
|
|
324
519
|
}
|
|
325
|
-
const
|
|
326
|
-
|
|
327
|
-
|
|
328
|
-
|
|
329
|
-
|
|
520
|
+
const [remaining] = await this.#repository.session
|
|
521
|
+
.select({ id: taskTable.id })
|
|
522
|
+
.from(taskTable)
|
|
523
|
+
.where(and(inArray(taskTable.id, ids), notInArray(taskTable.status, waitStatuses)))
|
|
524
|
+
.limit(1);
|
|
525
|
+
if (isUndefined(remaining)) {
|
|
330
526
|
return { cancelled: false };
|
|
331
527
|
}
|
|
332
528
|
const remainingTimeout = timeout - timer.milliseconds;
|
|
@@ -344,13 +540,18 @@ let PostgresTaskQueue = class PostgresTaskQueue extends TaskQueue {
|
|
|
344
540
|
if (treeIds.length == 0) {
|
|
345
541
|
return;
|
|
346
542
|
}
|
|
347
|
-
await
|
|
543
|
+
const cancelledRows = await tx.pgTransaction
|
|
544
|
+
.update(taskTable)
|
|
545
|
+
.set({
|
|
348
546
|
status: TaskStatus.Cancelled,
|
|
349
547
|
token: null,
|
|
350
548
|
completeTimestamp: TRANSACTION_TIMESTAMP,
|
|
351
|
-
})
|
|
352
|
-
|
|
353
|
-
|
|
549
|
+
})
|
|
550
|
+
.where(and(inArray(taskTable.id, treeIds), notInArray(taskTable.status, terminalStatuses)))
|
|
551
|
+
.returning({ id: taskTable.id, namespace: taskTable.namespace });
|
|
552
|
+
if (cancelledRows.length > 0) {
|
|
553
|
+
await this.resolveDependenciesMany(cancelledRows.map((row) => ({ id: row.id, status: TaskStatus.Cancelled, namespace: row.namespace })), { transaction: tx });
|
|
554
|
+
}
|
|
354
555
|
});
|
|
355
556
|
}
|
|
356
557
|
async cancelManyByTags(tags, options) {
|
|
@@ -362,17 +563,35 @@ let PostgresTaskQueue = class PostgresTaskQueue extends TaskQueue {
|
|
|
362
563
|
}
|
|
363
564
|
async clear(options) {
|
|
364
565
|
await this.#repository.useTransaction(options?.transaction, async (tx) => {
|
|
365
|
-
const
|
|
366
|
-
|
|
367
|
-
.select({ id: taskTable.id })
|
|
566
|
+
const nonFinalizedTasks = await tx.pgTransaction
|
|
567
|
+
.select({ id: taskTable.id, namespace: taskTable.namespace })
|
|
368
568
|
.from(taskTable)
|
|
369
|
-
.where(eq(taskTable.namespace, this.#namespace))
|
|
370
|
-
|
|
569
|
+
.where(and(eq(taskTable.namespace, this.#namespace), notInArray(taskTable.status, finalizedStatuses)))
|
|
570
|
+
.for('update');
|
|
571
|
+
if (nonFinalizedTasks.length > 0) {
|
|
572
|
+
const ids = nonFinalizedTasks.map((t) => t.id);
|
|
573
|
+
await tx.pgTransaction
|
|
574
|
+
.update(taskTable)
|
|
575
|
+
.set({
|
|
576
|
+
status: TaskStatus.Cancelled,
|
|
577
|
+
token: null,
|
|
578
|
+
completeTimestamp: TRANSACTION_TIMESTAMP,
|
|
579
|
+
})
|
|
580
|
+
.where(inArray(taskTable.id, ids));
|
|
581
|
+
await this.resolveDependenciesMany(nonFinalizedTasks.map((t) => ({ id: t.id, status: TaskStatus.Cancelled, namespace: t.namespace })), { transaction: tx });
|
|
582
|
+
}
|
|
583
|
+
// Break internal parent-child links to allow deleting tasks within the same namespace.
|
|
584
|
+
// Cross-namespace links will still cause a foreign key violation as requested.
|
|
585
|
+
await tx.pgTransaction
|
|
371
586
|
.update(taskTable)
|
|
372
587
|
.set({ parentId: null })
|
|
373
|
-
.where(inArray(taskTable.parentId,
|
|
374
|
-
.
|
|
375
|
-
|
|
588
|
+
.where(and(eq(taskTable.namespace, this.#namespace), sqlIsNotNull(taskTable.parentId), inArray(taskTable.parentId, tx.pgTransaction
|
|
589
|
+
.select({ id: taskTable.id })
|
|
590
|
+
.from(taskTable)
|
|
591
|
+
.where(eq(taskTable.namespace, this.#namespace)))));
|
|
592
|
+
await tx.pgTransaction
|
|
593
|
+
.delete(taskTable)
|
|
594
|
+
.where(eq(taskTable.namespace, this.#namespace));
|
|
376
595
|
});
|
|
377
596
|
}
|
|
378
597
|
async dequeue(options) {
|
|
@@ -386,52 +605,50 @@ let PostgresTaskQueue = class PostgresTaskQueue extends TaskQueue {
|
|
|
386
605
|
// 1. Flow Control
|
|
387
606
|
const forceDequeue = options?.forceDequeue ?? false;
|
|
388
607
|
let circuitBreakerResult;
|
|
608
|
+
let rateLimitAcquired = 0;
|
|
389
609
|
if (!forceDequeue) {
|
|
390
610
|
circuitBreakerResult = await this.#circuitBreaker.check();
|
|
391
611
|
if (!circuitBreakerResult.allowed) {
|
|
392
612
|
return [];
|
|
393
613
|
}
|
|
394
|
-
|
|
395
|
-
|
|
614
|
+
for (let i = count; i > 0; i--) {
|
|
615
|
+
if (await this.#rateLimiter.tryAcquire(this.#namespace, i)) {
|
|
616
|
+
rateLimitAcquired = i;
|
|
617
|
+
break;
|
|
618
|
+
}
|
|
619
|
+
}
|
|
620
|
+
if (rateLimitAcquired == 0) {
|
|
396
621
|
return [];
|
|
397
622
|
}
|
|
398
623
|
}
|
|
399
624
|
const tasks = await this.#repository.useTransaction(options?.transaction, async (tx) => {
|
|
400
|
-
|
|
401
|
-
let effectiveCount = count;
|
|
625
|
+
let effectiveCount = forceDequeue ? count : rateLimitAcquired;
|
|
402
626
|
if (!forceDequeue && circuitBreakerResult?.state == CircuitBreakerState.HalfOpen) {
|
|
403
627
|
if (circuitBreakerResult.isProbe != true) {
|
|
404
|
-
const
|
|
405
|
-
|
|
628
|
+
const [runningResult] = await tx.pgTransaction
|
|
629
|
+
.select({ count: sqlCount() })
|
|
630
|
+
.from(taskTable)
|
|
631
|
+
.where(and(eq(taskTable.namespace, this.#namespace), eq(taskTable.status, TaskStatus.Running)));
|
|
632
|
+
if ((runningResult?.count ?? 0) > 0) {
|
|
406
633
|
return [];
|
|
407
634
|
}
|
|
408
635
|
}
|
|
409
636
|
effectiveCount = 1;
|
|
410
637
|
}
|
|
411
|
-
// 2. Check Global Concurrency
|
|
412
|
-
if (!forceDequeue && isNotNull(this.globalConcurrency)) {
|
|
413
|
-
// WARN: This is a check-then-act race condition.
|
|
414
|
-
// A distributed lock or an atomic update strategy is needed for strict enforcement.
|
|
415
|
-
const runningCount = await repository.countByQuery({ namespace: this.#namespace, status: TaskStatus.Running });
|
|
416
|
-
if (runningCount >= this.globalConcurrency) {
|
|
417
|
-
return [];
|
|
418
|
-
}
|
|
419
|
-
effectiveCount = Math.min(effectiveCount, this.globalConcurrency - runningCount);
|
|
420
|
-
}
|
|
421
638
|
/*
|
|
422
639
|
* Materialization required for LIMIT clause
|
|
423
640
|
* https://stackoverflow.com/questions/73966670/select-for-update-subquery-not-respecting-limit-clause-under-load
|
|
424
641
|
* https://dba.stackexchange.com/questions/69471/postgres-update-limit-1
|
|
425
642
|
*/
|
|
426
|
-
const selection =
|
|
643
|
+
const selection = tx.pgTransaction.$with('selection').as((qb) => qb
|
|
427
644
|
.select({ id: taskTable.id })
|
|
428
645
|
.from(taskTable)
|
|
429
|
-
.where(and(eq(taskTable.namespace, this.#namespace), lte(taskTable.scheduleTimestamp, TRANSACTION_TIMESTAMP),
|
|
646
|
+
.where(and(eq(taskTable.namespace, this.#namespace), lte(taskTable.scheduleTimestamp, TRANSACTION_TIMESTAMP), inArray(taskTable.status, queueableStatuses), or(sqlIsNull(taskTable.timeToLive), lt(TRANSACTION_TIMESTAMP, taskTable.timeToLive)), isDefined(options?.types) ? inArray(taskTable.type, options.types) : undefined, sql `pg_sleep(0) IS NOT NULL` // Materialization hack until drizzle implements https://github.com/drizzle-team/drizzle-orm/issues/2318
|
|
430
647
|
))
|
|
431
648
|
.orderBy(asc(taskTable.priority), asc(taskTable.scheduleTimestamp))
|
|
432
649
|
.limit(effectiveCount)
|
|
433
650
|
.for('update', { skipLocked: true }));
|
|
434
|
-
const rows = await
|
|
651
|
+
const rows = await tx.pgTransaction
|
|
435
652
|
.with(selection)
|
|
436
653
|
.update(taskTable)
|
|
437
654
|
.set({
|
|
@@ -443,13 +660,13 @@ let PostgresTaskQueue = class PostgresTaskQueue extends TaskQueue {
|
|
|
443
660
|
// If it was RUNNING (Zombie), previous try failed -> increment tries.
|
|
444
661
|
tries: sql `${taskTable.tries} + 1`,
|
|
445
662
|
})
|
|
446
|
-
.where(inArray(taskTable.id,
|
|
663
|
+
.where(inArray(taskTable.id, tx.pgTransaction.select().from(selection)))
|
|
447
664
|
.returning();
|
|
448
665
|
return await this.#repository.mapManyToEntity(rows);
|
|
449
666
|
});
|
|
450
667
|
// 3. Compensation
|
|
451
|
-
if (!forceDequeue && (tasks.length <
|
|
452
|
-
const shortfall =
|
|
668
|
+
if (!forceDequeue && (tasks.length < rateLimitAcquired)) {
|
|
669
|
+
const shortfall = rateLimitAcquired - tasks.length;
|
|
453
670
|
await this.#rateLimiter.refund(this.#namespace, shortfall);
|
|
454
671
|
}
|
|
455
672
|
return tasks;
|
|
@@ -458,35 +675,36 @@ let PostgresTaskQueue = class PostgresTaskQueue extends TaskQueue {
|
|
|
458
675
|
await this.rescheduleMany([id], timestamp, options);
|
|
459
676
|
}
|
|
460
677
|
async rescheduleMany(ids, timestamp, options) {
|
|
461
|
-
|
|
462
|
-
|
|
678
|
+
const session = options?.transaction?.pgTransaction ?? this.#repository.session;
|
|
679
|
+
await session
|
|
680
|
+
.update(taskTable)
|
|
681
|
+
.set({
|
|
682
|
+
status: caseWhen(gt(taskTable.unresolvedScheduleDependencies, 0), enumValue(TaskStatus, taskStatus, TaskStatus.Waiting)).else(enumValue(TaskStatus, taskStatus, TaskStatus.Pending)),
|
|
463
683
|
token: null,
|
|
464
684
|
scheduleTimestamp: timestamp,
|
|
465
685
|
visibilityDeadline: null,
|
|
466
|
-
tries: sql `
|
|
467
|
-
|
|
468
|
-
|
|
469
|
-
END`,
|
|
470
|
-
});
|
|
686
|
+
tries: caseWhen(eq(taskTable.status, enumValue(TaskStatus, taskStatus, TaskStatus.Running)), greatest(0, sql `${taskTable.tries} - 1`)).else(taskTable.tries),
|
|
687
|
+
})
|
|
688
|
+
.where(and(inArray(taskTable.id, ids), notInArray(taskTable.status, terminalStatuses)));
|
|
471
689
|
}
|
|
472
690
|
async rescheduleManyByTags(tags, timestamp, options) {
|
|
473
|
-
|
|
474
|
-
|
|
691
|
+
const session = options?.transaction?.pgTransaction ?? this.#repository.session;
|
|
692
|
+
await session
|
|
693
|
+
.update(taskTable)
|
|
694
|
+
.set({
|
|
695
|
+
status: caseWhen(gt(taskTable.unresolvedScheduleDependencies, 0), enumValue(TaskStatus, taskStatus, TaskStatus.Waiting)).else(enumValue(TaskStatus, taskStatus, TaskStatus.Pending)),
|
|
475
696
|
token: null,
|
|
476
697
|
scheduleTimestamp: timestamp,
|
|
477
698
|
visibilityDeadline: null,
|
|
478
|
-
tries: sql `
|
|
479
|
-
|
|
480
|
-
|
|
481
|
-
END`,
|
|
482
|
-
});
|
|
699
|
+
tries: caseWhen(eq(taskTable.status, enumValue(TaskStatus, taskStatus, TaskStatus.Running)), greatest(0, sql `${taskTable.tries} - 1`)).else(taskTable.tries),
|
|
700
|
+
})
|
|
701
|
+
.where(and(eq(taskTable.namespace, this.#namespace), arrayOverlaps(taskTable.tags, toArray(tags)), notInArray(taskTable.status, terminalStatuses)));
|
|
483
702
|
}
|
|
484
703
|
async touch(task, options) {
|
|
485
704
|
if (isNull(task.token)) {
|
|
486
705
|
return undefined;
|
|
487
706
|
}
|
|
488
707
|
return await this.#repository.useTransaction(options?.transaction, async (tx) => {
|
|
489
|
-
const repository = this.#repository.withTransaction(tx);
|
|
490
708
|
const update = {
|
|
491
709
|
visibilityDeadline: sql `${TRANSACTION_TIMESTAMP} + ${interval(this.visibilityTimeout, 'milliseconds')}`,
|
|
492
710
|
};
|
|
@@ -496,34 +714,44 @@ let PostgresTaskQueue = class PostgresTaskQueue extends TaskQueue {
|
|
|
496
714
|
if (isDefined(options?.state)) {
|
|
497
715
|
update.state = options.state;
|
|
498
716
|
}
|
|
499
|
-
const
|
|
500
|
-
|
|
501
|
-
|
|
502
|
-
status
|
|
503
|
-
|
|
504
|
-
|
|
505
|
-
|
|
506
|
-
|
|
507
|
-
|
|
508
|
-
|
|
509
|
-
|
|
510
|
-
|
|
511
|
-
|
|
717
|
+
const [updatedRow] = await tx.pgTransaction
|
|
718
|
+
.update(taskTable)
|
|
719
|
+
.set(update)
|
|
720
|
+
.where(and(eq(taskTable.id, task.id), eq(taskTable.status, TaskStatus.Running), isNull(task.token) ? sqlIsNull(taskTable.token) : eq(taskTable.token, task.token), gt(taskTable.startTimestamp, sql `${TRANSACTION_TIMESTAMP} - ${interval(this.maxExecutionTime, 'milliseconds')}`)))
|
|
721
|
+
.returning();
|
|
722
|
+
if (isDefined(updatedRow)) {
|
|
723
|
+
return await this.#repository.mapToEntity(updatedRow);
|
|
724
|
+
}
|
|
725
|
+
const [existingRow] = await tx.pgTransaction
|
|
726
|
+
.select({ startTimestamp: taskTable.startTimestamp })
|
|
727
|
+
.from(taskTable)
|
|
728
|
+
.where(and(eq(taskTable.id, task.id), eq(taskTable.status, TaskStatus.Running), isNull(task.token) ? sqlIsNull(taskTable.token) : eq(taskTable.token, task.token)));
|
|
729
|
+
if (isDefined(existingRow) && isNotNull(existingRow.startTimestamp) && (currentTimestamp() - existingRow.startTimestamp) > this.maxExecutionTime) {
|
|
730
|
+
await tx.pgTransaction
|
|
731
|
+
.update(taskTable)
|
|
732
|
+
.set({
|
|
733
|
+
status: TaskStatus.TimedOut,
|
|
734
|
+
completeTimestamp: TRANSACTION_TIMESTAMP,
|
|
735
|
+
error: { code: 'MaxTimeExceeded', message: 'Hard Execution Timeout' },
|
|
736
|
+
})
|
|
737
|
+
.where(eq(taskTable.id, task.id));
|
|
738
|
+
await this.resolveDependenciesMany([{ id: task.id, status: TaskStatus.TimedOut, namespace: task.namespace }], { transaction: tx });
|
|
739
|
+
this.notify();
|
|
512
740
|
}
|
|
513
|
-
return
|
|
741
|
+
return undefined;
|
|
514
742
|
});
|
|
515
743
|
}
|
|
516
744
|
async touchMany(tasks, options) {
|
|
517
745
|
if (tasks.length == 0) {
|
|
518
746
|
return [];
|
|
519
747
|
}
|
|
520
|
-
const
|
|
748
|
+
const session = options?.transaction?.pgTransaction ?? this.#repository.session;
|
|
521
749
|
const rows = tasks.map((t, i) => {
|
|
522
750
|
const progress = options?.progresses?.[i] ?? null;
|
|
523
751
|
const state = options?.states?.[i] ?? null;
|
|
524
752
|
return sql `(${t.id}::uuid, ${t.token}::uuid, ${progress}::numeric, ${state}::jsonb)`;
|
|
525
753
|
});
|
|
526
|
-
const updates =
|
|
754
|
+
const updates = session.$with('updates').as((qb) => qb
|
|
527
755
|
.select({
|
|
528
756
|
updateId: sql `(id)::uuid`.as('update_id'),
|
|
529
757
|
updateToken: sql `(token)::uuid`.as('update_token'),
|
|
@@ -531,7 +759,7 @@ let PostgresTaskQueue = class PostgresTaskQueue extends TaskQueue {
|
|
|
531
759
|
updateState: sql `(state)::jsonb`.as('update_state'),
|
|
532
760
|
})
|
|
533
761
|
.from(sql `(VALUES ${sql.join(rows, sql `, `)}) AS t(id, token, progress, state)`));
|
|
534
|
-
const updated =
|
|
762
|
+
const updated = session.$with('updated').as(() => session
|
|
535
763
|
.update(taskTable)
|
|
536
764
|
.set({
|
|
537
765
|
visibilityDeadline: sql `${TRANSACTION_TIMESTAMP} + ${interval(this.visibilityTimeout, 'milliseconds')}`,
|
|
@@ -539,34 +767,44 @@ let PostgresTaskQueue = class PostgresTaskQueue extends TaskQueue {
|
|
|
539
767
|
state: coalesce(updates.updateState, taskTable.state),
|
|
540
768
|
})
|
|
541
769
|
.from(updates)
|
|
542
|
-
.where(and(eq(taskTable.id, updates.updateId),
|
|
770
|
+
.where(and(eq(taskTable.id, updates.updateId), sql `${taskTable.token} IS NOT DISTINCT FROM ${updates.updateToken}`, eq(taskTable.status, TaskStatus.Running), gt(taskTable.startTimestamp, sql `${TRANSACTION_TIMESTAMP} - ${interval(this.maxExecutionTime, 'milliseconds')}`)))
|
|
543
771
|
.returning({ id: taskTable.id }));
|
|
544
|
-
const result = await
|
|
772
|
+
const result = await session
|
|
545
773
|
.with(updates, updated)
|
|
546
774
|
.select({ id: updated.id })
|
|
547
|
-
.from(updated)
|
|
548
|
-
.execute();
|
|
775
|
+
.from(updated);
|
|
549
776
|
return result.map((r) => r.id);
|
|
550
777
|
}
|
|
551
778
|
async complete(task, options) {
|
|
552
779
|
await this.#repository.useTransaction(options?.transaction, async (tx) => {
|
|
553
|
-
const
|
|
554
|
-
|
|
555
|
-
|
|
556
|
-
|
|
557
|
-
|
|
558
|
-
|
|
780
|
+
const [freshTask] = await tx.pgTransaction
|
|
781
|
+
.select({ unresolvedCompleteDependencies: taskTable.unresolvedCompleteDependencies })
|
|
782
|
+
.from(taskTable)
|
|
783
|
+
.where(eq(taskTable.id, task.id))
|
|
784
|
+
.for('update');
|
|
785
|
+
if (isUndefined(freshTask)) {
|
|
786
|
+
return;
|
|
787
|
+
}
|
|
788
|
+
const hasActiveChildren = freshTask.unresolvedCompleteDependencies > 0;
|
|
789
|
+
const nextStatus = hasActiveChildren ? TaskStatus.WaitingChildren : TaskStatus.Completed;
|
|
790
|
+
const [updatedTask] = await tx.pgTransaction.update(taskTable)
|
|
791
|
+
.set({
|
|
792
|
+
status: nextStatus,
|
|
559
793
|
token: null,
|
|
560
794
|
result: options?.result,
|
|
561
|
-
progress: 1,
|
|
562
|
-
completeTimestamp: TRANSACTION_TIMESTAMP,
|
|
795
|
+
progress: hasActiveChildren ? task.progress : 1,
|
|
796
|
+
completeTimestamp: (nextStatus == TaskStatus.Completed) ? TRANSACTION_TIMESTAMP : null,
|
|
563
797
|
visibilityDeadline: null,
|
|
564
|
-
})
|
|
798
|
+
})
|
|
799
|
+
.where(and(eq(taskTable.id, task.id), isNull(task.token) ? sqlIsNull(taskTable.token) : eq(taskTable.token, task.token)))
|
|
800
|
+
.returning({ id: taskTable.id });
|
|
565
801
|
if (isUndefined(updatedTask)) {
|
|
566
802
|
return;
|
|
567
803
|
}
|
|
568
|
-
|
|
569
|
-
|
|
804
|
+
if (nextStatus == TaskStatus.Completed) {
|
|
805
|
+
await this.#circuitBreaker.recordSuccess();
|
|
806
|
+
}
|
|
807
|
+
await this.resolveDependencies(task.id, nextStatus, { namespace: task.namespace, transaction: tx });
|
|
570
808
|
});
|
|
571
809
|
}
|
|
572
810
|
async completeMany(tasks, options) {
|
|
@@ -574,57 +812,54 @@ let PostgresTaskQueue = class PostgresTaskQueue extends TaskQueue {
|
|
|
574
812
|
return;
|
|
575
813
|
}
|
|
576
814
|
await this.#repository.useTransaction(options?.transaction, async (tx) => {
|
|
577
|
-
const repository = this.#repository.withTransaction(tx);
|
|
578
815
|
const rows = tasks.map((t, i) => {
|
|
579
816
|
const result = options?.results?.[i] ?? null;
|
|
580
|
-
return sql `(${t.id}::uuid, ${t.token}::uuid, ${result}::jsonb)`;
|
|
817
|
+
return sql `(${t.id}::uuid, ${t.token}::uuid, ${result}::jsonb, ${t.progress}::numeric)`;
|
|
581
818
|
});
|
|
582
|
-
const updates =
|
|
819
|
+
const updates = tx.pgTransaction.$with('updates').as((qb) => qb
|
|
583
820
|
.select({
|
|
584
821
|
updateId: sql `(id)::uuid`.as('update_id'),
|
|
585
822
|
updateToken: sql `(token)::uuid`.as('update_token'),
|
|
586
823
|
updateResult: sql `(result)::jsonb`.as('update_result'),
|
|
824
|
+
updateProgress: sql `(progress)::numeric`.as('update_progress'),
|
|
587
825
|
})
|
|
588
|
-
.from(sql `(VALUES ${sql.join(rows, sql `, `)}) AS t(id, token, result)`));
|
|
589
|
-
const updated =
|
|
826
|
+
.from(sql `(VALUES ${sql.join(rows, sql `, `)}) AS t(id, token, result, progress)`));
|
|
827
|
+
const updated = tx.pgTransaction.$with('updated').as(() => tx.pgTransaction
|
|
590
828
|
.update(taskTable)
|
|
591
829
|
.set({
|
|
592
|
-
status: TaskStatus.Completed,
|
|
830
|
+
status: caseWhen(gt(taskTable.unresolvedCompleteDependencies, 0), enumValue(TaskStatus, taskStatus, TaskStatus.WaitingChildren)).else(enumValue(TaskStatus, taskStatus, TaskStatus.Completed)),
|
|
593
831
|
token: null,
|
|
594
832
|
result: updates.updateResult,
|
|
595
|
-
progress: 1,
|
|
596
|
-
completeTimestamp: TRANSACTION_TIMESTAMP,
|
|
833
|
+
progress: caseWhen(gt(taskTable.unresolvedCompleteDependencies, 0), updates.updateProgress).else(1),
|
|
834
|
+
completeTimestamp: caseWhen(gt(taskTable.unresolvedCompleteDependencies, 0), taskTable.completeTimestamp).else(TRANSACTION_TIMESTAMP),
|
|
597
835
|
visibilityDeadline: null,
|
|
598
836
|
})
|
|
599
837
|
.from(updates)
|
|
600
|
-
.where(and(eq(taskTable.id, updates.updateId),
|
|
601
|
-
.returning({ id: taskTable.id,
|
|
602
|
-
const updatedRows = await
|
|
838
|
+
.where(and(eq(taskTable.id, updates.updateId), sql `${taskTable.token} IS NOT DISTINCT FROM ${updates.updateToken}`))
|
|
839
|
+
.returning({ id: taskTable.id, status: taskTable.status }));
|
|
840
|
+
const updatedRows = await tx.pgTransaction
|
|
603
841
|
.with(updates, updated)
|
|
604
|
-
.select({ id: updated.id,
|
|
605
|
-
.from(updated)
|
|
606
|
-
.execute();
|
|
842
|
+
.select({ id: updated.id, status: updated.status })
|
|
843
|
+
.from(updated);
|
|
607
844
|
if (updatedRows.length > 0) {
|
|
608
|
-
|
|
609
|
-
|
|
610
|
-
|
|
845
|
+
if (updatedRows.some((r) => r.status == TaskStatus.Completed)) {
|
|
846
|
+
await this.#circuitBreaker.recordSuccess();
|
|
847
|
+
}
|
|
848
|
+
await this.resolveDependenciesMany(updatedRows.map((r) => ({ id: r.id, status: r.status })), { transaction: tx });
|
|
611
849
|
}
|
|
612
850
|
});
|
|
613
851
|
}
|
|
614
852
|
async fail(task, error, options) {
|
|
615
853
|
const isRetryable = (options?.fatal != true) && (task.tries < this.maxTries);
|
|
616
|
-
const nextStatus = isRetryable ? TaskStatus.
|
|
854
|
+
const nextStatus = isRetryable ? TaskStatus.Retrying : TaskStatus.Dead;
|
|
617
855
|
const delay = isRetryable
|
|
618
856
|
? Math.min(this.retryDelayMaximum, this.retryDelayMinimum * (this.retryDelayGrowth ** task.tries))
|
|
619
857
|
: 0;
|
|
620
858
|
const nextSchedule = currentTimestamp() + delay;
|
|
621
859
|
await this.#repository.useTransaction(options?.transaction, async (tx) => {
|
|
622
|
-
const
|
|
623
|
-
|
|
624
|
-
|
|
625
|
-
token: task.token,
|
|
626
|
-
tries: task.tries,
|
|
627
|
-
}, {
|
|
860
|
+
const [updatedRow] = await tx.pgTransaction
|
|
861
|
+
.update(taskTable)
|
|
862
|
+
.set({
|
|
628
863
|
status: nextStatus,
|
|
629
864
|
token: null,
|
|
630
865
|
error: serializeError(error),
|
|
@@ -632,28 +867,14 @@ let PostgresTaskQueue = class PostgresTaskQueue extends TaskQueue {
|
|
|
632
867
|
scheduleTimestamp: nextSchedule,
|
|
633
868
|
startTimestamp: null,
|
|
634
869
|
completeTimestamp: (nextStatus == TaskStatus.Dead) ? TRANSACTION_TIMESTAMP : null,
|
|
635
|
-
})
|
|
636
|
-
|
|
870
|
+
})
|
|
871
|
+
.where(and(eq(taskTable.id, task.id), isNull(task.token) ? sqlIsNull(taskTable.token) : eq(taskTable.token, task.token), eq(taskTable.tries, task.tries)))
|
|
872
|
+
.returning();
|
|
873
|
+
if (isUndefined(updatedRow)) {
|
|
637
874
|
return;
|
|
638
875
|
}
|
|
639
876
|
await this.#circuitBreaker.recordFailure();
|
|
640
|
-
|
|
641
|
-
await this.triggerTagFanIn(task.tags, { transaction: tx });
|
|
642
|
-
}
|
|
643
|
-
});
|
|
644
|
-
}
|
|
645
|
-
async triggerTagFanIn(tags, options) {
|
|
646
|
-
if (tags.length == 0) {
|
|
647
|
-
return;
|
|
648
|
-
}
|
|
649
|
-
const distinctSortedTags = distinct(tags).toSorted();
|
|
650
|
-
const hash = await digest('SHA-256', distinctSortedTags.join(',')).toHex();
|
|
651
|
-
const idempotencyKey = `Sys:FanIn:${hash}`;
|
|
652
|
-
await this.#internalThis.enqueue('[SystemWorker]:FanIn', { targetTags: distinctSortedTags }, {
|
|
653
|
-
priority: 0,
|
|
654
|
-
idempotencyKey,
|
|
655
|
-
replace: true,
|
|
656
|
-
transaction: options?.transaction,
|
|
877
|
+
await this.resolveDependencies(task.id, nextStatus, { namespace: task.namespace, transaction: tx });
|
|
657
878
|
});
|
|
658
879
|
}
|
|
659
880
|
async failMany(tasks, errors, options) {
|
|
@@ -661,11 +882,10 @@ let PostgresTaskQueue = class PostgresTaskQueue extends TaskQueue {
|
|
|
661
882
|
return;
|
|
662
883
|
}
|
|
663
884
|
await this.#repository.useTransaction(options?.transaction, async (tx) => {
|
|
664
|
-
const repository = this.#repository.withTransaction(tx);
|
|
665
885
|
const rows = tasks.map((task, index) => {
|
|
666
886
|
const error = errors[index];
|
|
667
887
|
const isRetryable = (task.tries < this.maxTries);
|
|
668
|
-
const nextStatus = isRetryable ? TaskStatus.
|
|
888
|
+
const nextStatus = isRetryable ? TaskStatus.Retrying : TaskStatus.Dead;
|
|
669
889
|
const delay = isRetryable
|
|
670
890
|
? Math.min(this.retryDelayMaximum, this.retryDelayMinimum * (this.retryDelayGrowth ** task.tries))
|
|
671
891
|
: 0;
|
|
@@ -673,7 +893,7 @@ let PostgresTaskQueue = class PostgresTaskQueue extends TaskQueue {
|
|
|
673
893
|
const completeTimestamp = (nextStatus == TaskStatus.Dead) ? new Date() : null;
|
|
674
894
|
return sql `(${task.id}::uuid, ${task.token}::uuid, ${task.tries}::int, ${nextStatus}::text, ${serializeError(error)}::jsonb, ${nextSchedule}::timestamptz, ${completeTimestamp}::timestamptz)`;
|
|
675
895
|
});
|
|
676
|
-
const updates =
|
|
896
|
+
const updates = tx.pgTransaction.$with('updates').as((qb) => qb
|
|
677
897
|
.select({
|
|
678
898
|
updateId: sql `(id)::uuid`.as('update_id'),
|
|
679
899
|
updateToken: sql `(token)::uuid`.as('update_token'),
|
|
@@ -684,7 +904,7 @@ let PostgresTaskQueue = class PostgresTaskQueue extends TaskQueue {
|
|
|
684
904
|
updateComplete: sql `(complete_timestamp)::timestamptz`.as('update_complete'),
|
|
685
905
|
})
|
|
686
906
|
.from(sql `(VALUES ${sql.join(rows, sql `, `)}) AS t(id, token, tries, status, error, schedule_timestamp, complete_timestamp)`));
|
|
687
|
-
const updated =
|
|
907
|
+
const updated = tx.pgTransaction.$with('updated').as(() => tx.pgTransaction
|
|
688
908
|
.update(taskTable)
|
|
689
909
|
.set({
|
|
690
910
|
status: sql `${updates.updateStatus}::${taskStatus}`,
|
|
@@ -696,304 +916,369 @@ let PostgresTaskQueue = class PostgresTaskQueue extends TaskQueue {
|
|
|
696
916
|
completeTimestamp: sql `${updates.updateComplete}`,
|
|
697
917
|
})
|
|
698
918
|
.from(updates)
|
|
699
|
-
.where(and(eq(taskTable.
|
|
700
|
-
.returning({ id: taskTable.id, status: taskTable.status
|
|
701
|
-
const result = await
|
|
919
|
+
.where(and(eq(taskTable.id, updates.updateId), sql `${taskTable.token} IS NOT DISTINCT FROM ${updates.updateToken}`, eq(taskTable.tries, updates.updateTries)))
|
|
920
|
+
.returning({ id: taskTable.id, status: taskTable.status }));
|
|
921
|
+
const result = await tx.pgTransaction
|
|
702
922
|
.with(updates, updated)
|
|
703
|
-
.select()
|
|
704
|
-
.from(updated)
|
|
705
|
-
.execute();
|
|
923
|
+
.select({ id: updated.id, status: updated.status })
|
|
924
|
+
.from(updated);
|
|
706
925
|
if (result.length > 0) {
|
|
707
|
-
await
|
|
708
|
-
|
|
709
|
-
.filter((r) => r.status == TaskStatus.Dead)
|
|
710
|
-
.flatMap((r) => r.tags);
|
|
711
|
-
await this.triggerTagFanIn(deadTags, { transaction: tx });
|
|
926
|
+
await this.#circuitBreaker.recordFailures(result.length);
|
|
927
|
+
await this.resolveDependenciesMany(result.map((r) => ({ id: r.id, status: r.status })), { transaction: tx });
|
|
712
928
|
}
|
|
713
929
|
});
|
|
714
930
|
}
|
|
715
|
-
async resolveDependencies(
|
|
716
|
-
await this
|
|
717
|
-
|
|
718
|
-
|
|
719
|
-
|
|
720
|
-
|
|
721
|
-
|
|
722
|
-
|
|
723
|
-
|
|
724
|
-
|
|
725
|
-
|
|
726
|
-
|
|
727
|
-
|
|
728
|
-
}
|
|
729
|
-
const allRequiredTags = distinct(entities.flatMap((t) => [...t.completeAfterTags, ...t.scheduleAfterTags]));
|
|
730
|
-
const statsRows = await repository.session
|
|
931
|
+
async resolveDependencies(id, status, options) {
|
|
932
|
+
await this.resolveDependenciesMany([{ id, status, namespace: options?.namespace }], options);
|
|
933
|
+
}
|
|
934
|
+
async resolveDependenciesMany(tasks, options) {
|
|
935
|
+
this.#logger.debug(`Resolving dependencies for ${tasks.length} tasks`);
|
|
936
|
+
if (tasks.length == 0) {
|
|
937
|
+
return;
|
|
938
|
+
}
|
|
939
|
+
const taskStatusMap = new Map(tasks.map((t) => [t.id, t.status]));
|
|
940
|
+
const notifiedNamespaces = new Set();
|
|
941
|
+
await this.#repository.useTransaction(options?.transaction, async (tx) => {
|
|
942
|
+
const taskIds = tasks.map((t) => t.id);
|
|
943
|
+
const dependents = await tx.pgTransaction
|
|
731
944
|
.select({
|
|
732
|
-
|
|
733
|
-
|
|
734
|
-
|
|
945
|
+
taskId: taskDependencyTable.taskId,
|
|
946
|
+
dependencyTaskId: taskDependencyTable.dependencyTaskId,
|
|
947
|
+
type: taskDependencyTable.type,
|
|
948
|
+
requiredStatuses: taskDependencyTable.requiredStatuses,
|
|
735
949
|
})
|
|
736
|
-
.from(
|
|
737
|
-
.where(
|
|
738
|
-
|
|
739
|
-
|
|
740
|
-
for (const row of statsRows) {
|
|
741
|
-
if (!tagStats.has(row.tag)) {
|
|
742
|
-
tagStats.set(row.tag, new Map());
|
|
743
|
-
}
|
|
744
|
-
tagStats.get(row.tag).set(row.status, row.count);
|
|
950
|
+
.from(taskDependencyTable)
|
|
951
|
+
.where(inArray(taskDependencyTable.dependencyTaskId, taskIds));
|
|
952
|
+
if (dependents.length == 0) {
|
|
953
|
+
return;
|
|
745
954
|
}
|
|
746
|
-
const
|
|
747
|
-
|
|
748
|
-
|
|
749
|
-
|
|
750
|
-
|
|
751
|
-
|
|
752
|
-
|
|
753
|
-
|
|
754
|
-
|
|
755
|
-
|
|
756
|
-
return sum;
|
|
757
|
-
};
|
|
758
|
-
const idsToFail = [];
|
|
759
|
-
const idsToSchedule = [];
|
|
760
|
-
const idsToComplete = [];
|
|
761
|
-
const fanInTags = [];
|
|
762
|
-
for (const waiter of entities) {
|
|
763
|
-
const requiredTags = distinct([...waiter.completeAfterTags, ...waiter.scheduleAfterTags]);
|
|
764
|
-
let isReady = false;
|
|
765
|
-
let shouldFail = false;
|
|
766
|
-
// 1. Check Fail Fast
|
|
767
|
-
if (waiter.failFast) {
|
|
768
|
-
if (getCount(requiredTags, [TaskStatus.Dead, TaskStatus.Cancelled]) > 0) {
|
|
769
|
-
shouldFail = true;
|
|
955
|
+
const resolvedEdges = [];
|
|
956
|
+
const failFastTaskIds = new Set();
|
|
957
|
+
for (const dep of dependents) {
|
|
958
|
+
const status = taskStatusMap.get(dep.dependencyTaskId);
|
|
959
|
+
const isMatched = dep.requiredStatuses.includes(status);
|
|
960
|
+
const isTerminal = terminalStatuses.includes(status);
|
|
961
|
+
if (isMatched || isTerminal) {
|
|
962
|
+
resolvedEdges.push(dep);
|
|
963
|
+
if (!isMatched) {
|
|
964
|
+
failFastTaskIds.add(dep.taskId);
|
|
770
965
|
}
|
|
771
966
|
}
|
|
772
|
-
|
|
773
|
-
|
|
774
|
-
|
|
775
|
-
|
|
776
|
-
|
|
777
|
-
|
|
778
|
-
|
|
779
|
-
|
|
780
|
-
isReady = requiredTags.every((tag) => {
|
|
781
|
-
const stats = tagStats.get(tag);
|
|
782
|
-
if (isUndefined(stats)) {
|
|
783
|
-
return false;
|
|
784
|
-
}
|
|
785
|
-
const activeCount = (stats.get(TaskStatus.Pending) ?? 0)
|
|
786
|
-
+ (stats.get(TaskStatus.Running) ?? 0)
|
|
787
|
-
+ (stats.get(TaskStatus.Waiting) ?? 0);
|
|
788
|
-
return activeCount == 0;
|
|
789
|
-
});
|
|
790
|
-
}
|
|
967
|
+
}
|
|
968
|
+
if (resolvedEdges.length == 0) {
|
|
969
|
+
return;
|
|
970
|
+
}
|
|
971
|
+
const sortedResolvedEdges = resolvedEdges.toSorted((a, b) => {
|
|
972
|
+
const idCompare = a.taskId.localeCompare(b.taskId);
|
|
973
|
+
if (idCompare != 0) {
|
|
974
|
+
return idCompare;
|
|
791
975
|
}
|
|
792
|
-
|
|
793
|
-
if (
|
|
794
|
-
|
|
976
|
+
const depIdCompare = a.dependencyTaskId.localeCompare(b.dependencyTaskId);
|
|
977
|
+
if (depIdCompare != 0) {
|
|
978
|
+
return depIdCompare;
|
|
795
979
|
}
|
|
796
|
-
|
|
797
|
-
|
|
798
|
-
|
|
799
|
-
|
|
980
|
+
return a.type.localeCompare(b.type);
|
|
981
|
+
});
|
|
982
|
+
const edgeValues = sortedResolvedEdges.map((e) => sql `(${e.taskId}::uuid, ${e.dependencyTaskId}::uuid, ${e.type}::text)`);
|
|
983
|
+
await tx.pgTransaction.execute(sql `
|
|
984
|
+
DELETE FROM ${taskDependencyTable}
|
|
985
|
+
WHERE (task_id, dependency_task_id, type) IN (
|
|
986
|
+
SELECT t.task_id, t.dependency_task_id, t.type::${taskDependencyType}
|
|
987
|
+
FROM (VALUES ${sql.join(edgeValues, sql `, `)}) AS t(task_id, dependency_task_id, type)
|
|
988
|
+
)
|
|
989
|
+
`);
|
|
990
|
+
const terminalTasks = [];
|
|
991
|
+
const skippedTaskIds = new Set();
|
|
992
|
+
if (failFastTaskIds.size > 0) {
|
|
993
|
+
const sortedFailFastIds = [...failFastTaskIds].toSorted();
|
|
994
|
+
const dependentTasks = await tx.pgTransaction
|
|
995
|
+
.select({ id: taskTable.id, namespace: taskTable.namespace, failFast: taskTable.failFast, status: taskTable.status })
|
|
996
|
+
.from(taskTable)
|
|
997
|
+
.where(inArray(taskTable.id, sortedFailFastIds))
|
|
998
|
+
.orderBy(asc(taskTable.id))
|
|
999
|
+
.for('update');
|
|
1000
|
+
const tasksToSkip = [];
|
|
1001
|
+
for (const task of dependentTasks) {
|
|
1002
|
+
if (task.failFast && !terminalStatuses.includes(task.status)) {
|
|
1003
|
+
tasksToSkip.push(task.id);
|
|
1004
|
+
skippedTaskIds.add(task.id);
|
|
800
1005
|
}
|
|
801
|
-
|
|
802
|
-
|
|
803
|
-
|
|
1006
|
+
}
|
|
1007
|
+
if (tasksToSkip.length > 0) {
|
|
1008
|
+
const skippedRows = await tx.pgTransaction
|
|
1009
|
+
.update(taskTable)
|
|
1010
|
+
.set({
|
|
1011
|
+
status: TaskStatus.Skipped,
|
|
1012
|
+
token: null,
|
|
1013
|
+
error: jsonbBuildObject({ code: 'DependencyFailed', message: 'One or more dependencies failed and failFast is enabled' }),
|
|
1014
|
+
completeTimestamp: TRANSACTION_TIMESTAMP,
|
|
1015
|
+
})
|
|
1016
|
+
.where(inArray(taskTable.id, tasksToSkip))
|
|
1017
|
+
.returning({ id: taskTable.id, status: taskTable.status, namespace: taskTable.namespace });
|
|
1018
|
+
terminalTasks.push(...skippedRows.map((r) => ({ id: r.id, status: r.status, namespace: r.namespace })));
|
|
1019
|
+
for (const row of skippedRows) {
|
|
1020
|
+
notifiedNamespaces.add(row.namespace);
|
|
804
1021
|
}
|
|
805
1022
|
}
|
|
806
1023
|
}
|
|
807
|
-
|
|
808
|
-
|
|
809
|
-
|
|
810
|
-
|
|
811
|
-
|
|
812
|
-
}
|
|
813
|
-
const
|
|
814
|
-
|
|
1024
|
+
// 4. Group all resolved edges into decrements (for any dependent NOT skipped above)
|
|
1025
|
+
const decrementsToApply = new Map();
|
|
1026
|
+
for (const dep of sortedResolvedEdges) {
|
|
1027
|
+
if (skippedTaskIds.has(dep.taskId)) {
|
|
1028
|
+
continue;
|
|
1029
|
+
}
|
|
1030
|
+
const current = decrementsToApply.get(dep.taskId) ?? { schedule: 0, complete: 0 };
|
|
1031
|
+
if (dep.type == TaskDependencyType.Schedule) {
|
|
1032
|
+
current.schedule++;
|
|
1033
|
+
}
|
|
1034
|
+
else if (dep.type == TaskDependencyType.Complete || dep.type == TaskDependencyType.Child) {
|
|
1035
|
+
current.complete++;
|
|
1036
|
+
}
|
|
1037
|
+
decrementsToApply.set(dep.taskId, current);
|
|
815
1038
|
}
|
|
816
|
-
if (
|
|
817
|
-
|
|
818
|
-
|
|
819
|
-
|
|
1039
|
+
if (decrementsToApply.size > 0) {
|
|
1040
|
+
const decrementValues = [...decrementsToApply]
|
|
1041
|
+
.toSorted(([idA], [idB]) => idA.localeCompare(idB))
|
|
1042
|
+
.map(([taskId, d]) => sql `(${taskId}::uuid, ${d.schedule}::int, ${d.complete}::int)`);
|
|
1043
|
+
const updates = tx.pgTransaction.$with('updates').as((qb) => qb
|
|
1044
|
+
.select({
|
|
1045
|
+
taskId: sql `(id)::uuid`.as('task_id'),
|
|
1046
|
+
scheduleDecrement: sql `(schedule)::int`.as('schedule_decrement'),
|
|
1047
|
+
completeDecrement: sql `(complete)::int`.as('complete_decrement'),
|
|
1048
|
+
})
|
|
1049
|
+
.from(sql `(VALUES ${sql.join(decrementValues, sql `, `)}) AS t(id, schedule, complete)`));
|
|
1050
|
+
const updatedRows = await tx.pgTransaction
|
|
1051
|
+
.with(updates)
|
|
1052
|
+
.update(taskTable)
|
|
1053
|
+
.set({
|
|
1054
|
+
unresolvedScheduleDependencies: greatest(0, sql `${taskTable.unresolvedScheduleDependencies} - ${updates.scheduleDecrement}`),
|
|
1055
|
+
unresolvedCompleteDependencies: greatest(0, sql `${taskTable.unresolvedCompleteDependencies} - ${updates.completeDecrement}`),
|
|
1056
|
+
status: caseWhen(and(eq(taskTable.status, TaskStatus.Waiting), eq(greatest(0, sql `${taskTable.unresolvedScheduleDependencies} - ${updates.scheduleDecrement}`), 0)), enumValue(TaskStatus, taskStatus, TaskStatus.Pending)).else(caseWhen(and(eq(taskTable.status, TaskStatus.WaitingChildren), eq(greatest(0, sql `${taskTable.unresolvedCompleteDependencies} - ${updates.completeDecrement}`), 0)), enumValue(TaskStatus, taskStatus, TaskStatus.Completed)).else(taskTable.status)),
|
|
1057
|
+
progress: caseWhen(and(eq(taskTable.status, TaskStatus.WaitingChildren), eq(greatest(0, sql `${taskTable.unresolvedCompleteDependencies} - ${updates.completeDecrement}`), 0)), 1).else(taskTable.progress),
|
|
1058
|
+
completeTimestamp: caseWhen(and(eq(taskTable.status, TaskStatus.WaitingChildren), eq(greatest(0, sql `${taskTable.unresolvedCompleteDependencies} - ${updates.completeDecrement}`), 0)), TRANSACTION_TIMESTAMP).else(taskTable.completeTimestamp),
|
|
1059
|
+
token: caseWhen(and(eq(taskTable.status, TaskStatus.WaitingChildren), eq(greatest(0, sql `${taskTable.unresolvedCompleteDependencies} - ${updates.completeDecrement}`), 0)), null).else(taskTable.token),
|
|
1060
|
+
})
|
|
1061
|
+
.from(updates)
|
|
1062
|
+
.where(eq(taskTable.id, updates.taskId))
|
|
1063
|
+
.returning({
|
|
1064
|
+
id: taskTable.id,
|
|
1065
|
+
namespace: taskTable.namespace,
|
|
1066
|
+
status: taskTable.status,
|
|
820
1067
|
});
|
|
1068
|
+
for (const row of updatedRows) {
|
|
1069
|
+
if (finalizedStatuses.includes(row.status)) {
|
|
1070
|
+
terminalTasks.push({ id: row.id, status: row.status, namespace: row.namespace });
|
|
1071
|
+
}
|
|
1072
|
+
notifiedNamespaces.add(row.namespace);
|
|
1073
|
+
}
|
|
821
1074
|
}
|
|
822
|
-
if (
|
|
823
|
-
|
|
824
|
-
|
|
825
|
-
|
|
826
|
-
|
|
827
|
-
|
|
828
|
-
});
|
|
829
|
-
await this.triggerTagFanIn(fanInTags, { transaction: tx });
|
|
1075
|
+
if (terminalTasks.length > 0) {
|
|
1076
|
+
const terminalTaskIds = terminalTasks.map((t) => t.id);
|
|
1077
|
+
await tx.pgTransaction
|
|
1078
|
+
.delete(taskDependencyTable)
|
|
1079
|
+
.where(inArray(taskDependencyTable.taskId, terminalTaskIds));
|
|
1080
|
+
await this.resolveDependenciesMany(terminalTasks, { transaction: tx });
|
|
830
1081
|
}
|
|
831
1082
|
});
|
|
1083
|
+
for (const namespace of notifiedNamespaces) {
|
|
1084
|
+
this.notify(namespace);
|
|
1085
|
+
}
|
|
832
1086
|
}
|
|
833
1087
|
async maintenance(options) {
|
|
834
|
-
|
|
835
|
-
|
|
836
|
-
|
|
1088
|
+
await Promise.allSettled([
|
|
1089
|
+
this.processExpirations(options),
|
|
1090
|
+
this.processZombieRetries(options),
|
|
1091
|
+
this.processZombieExhaustions(options),
|
|
1092
|
+
this.processHardTimeouts(options),
|
|
1093
|
+
this.processPriorityAging(options),
|
|
1094
|
+
]);
|
|
1095
|
+
await this.performArchival(options);
|
|
1096
|
+
await this.performArchivePurge(options);
|
|
1097
|
+
}
|
|
1098
|
+
async performArchival(options) {
|
|
837
1099
|
while (true) {
|
|
838
|
-
const archivedCount = await repository.useTransaction(options?.transaction, async (tx) => {
|
|
839
|
-
const
|
|
840
|
-
const
|
|
841
|
-
|
|
842
|
-
|
|
843
|
-
|
|
844
|
-
|
|
845
|
-
|
|
846
|
-
|
|
847
|
-
|
|
848
|
-
|
|
849
|
-
|
|
850
|
-
|
|
851
|
-
|
|
852
|
-
|
|
853
|
-
|
|
854
|
-
|
|
855
|
-
await
|
|
856
|
-
await repositoryWithTx.hardDeleteMany(tasksToArchive.map((t) => t.id));
|
|
1100
|
+
const archivedCount = await this.#repository.useTransaction(options?.transaction, async (tx) => {
|
|
1101
|
+
const childTaskTable = aliasedTable(taskTable, 'childTask');
|
|
1102
|
+
const rowsToArchive = await tx.pgTransaction
|
|
1103
|
+
.select()
|
|
1104
|
+
.from(taskTable)
|
|
1105
|
+
.where(and(eq(taskTable.namespace, this.#namespace), inArray(taskTable.status, finalizedStatuses), lte(taskTable.completeTimestamp, sql `${TRANSACTION_TIMESTAMP} - ${interval(this.retention, 'milliseconds')}`), notExists(tx.pgTransaction
|
|
1106
|
+
.select({ id: childTaskTable.id })
|
|
1107
|
+
.from(childTaskTable)
|
|
1108
|
+
.where(eq(childTaskTable.parentId, taskTable.id))), notExists(tx.pgTransaction
|
|
1109
|
+
.select({ taskId: taskDependencyTable.taskId })
|
|
1110
|
+
.from(taskDependencyTable)
|
|
1111
|
+
.where(eq(taskDependencyTable.dependencyTaskId, taskTable.id)))))
|
|
1112
|
+
.limit(1000)
|
|
1113
|
+
.for('update', { skipLocked: true });
|
|
1114
|
+
if (rowsToArchive.length > 0) {
|
|
1115
|
+
const rowsToArchiveIds = getEntityIds(rowsToArchive);
|
|
1116
|
+
await tx.pgTransaction.insert(taskArchiveTable).values(rowsToArchive);
|
|
1117
|
+
await tx.pgTransaction.delete(taskTable).where(inArray(taskTable.id, rowsToArchiveIds));
|
|
857
1118
|
}
|
|
858
|
-
return
|
|
1119
|
+
return rowsToArchive.length;
|
|
859
1120
|
});
|
|
860
1121
|
if (archivedCount < 1000) {
|
|
861
1122
|
break;
|
|
862
1123
|
}
|
|
863
1124
|
}
|
|
864
|
-
|
|
1125
|
+
}
|
|
1126
|
+
async performArchivePurge(options) {
|
|
1127
|
+
const session = options?.transaction?.pgTransaction ?? this.#database;
|
|
1128
|
+
const selection = session.$with('archive_purge_selection').as((qb) => qb
|
|
1129
|
+
.select({ id: taskArchiveTable.id })
|
|
1130
|
+
.from(taskArchiveTable)
|
|
1131
|
+
.where(and(eq(taskArchiveTable.namespace, this.#namespace), lte(taskArchiveTable.completeTimestamp, sql `${TRANSACTION_TIMESTAMP} - ${interval(this.archiveRetention, 'milliseconds')}`)))
|
|
1132
|
+
.limit(1000));
|
|
865
1133
|
while (true) {
|
|
866
|
-
const
|
|
867
|
-
|
|
868
|
-
|
|
869
|
-
|
|
870
|
-
|
|
871
|
-
|
|
872
|
-
.limit(1000));
|
|
873
|
-
const result = await repositoryWithTx.session
|
|
874
|
-
.with(selection)
|
|
875
|
-
.delete(taskArchiveTable)
|
|
876
|
-
.where(inArray(taskArchiveTable.id, repositoryWithTx.session.select().from(selection)))
|
|
877
|
-
.returning({ id: taskArchiveTable.id });
|
|
878
|
-
return result.length;
|
|
879
|
-
});
|
|
880
|
-
if (deletedArchiveCount < 1000) {
|
|
1134
|
+
const result = await session
|
|
1135
|
+
.with(selection)
|
|
1136
|
+
.delete(taskArchiveTable)
|
|
1137
|
+
.where(inArray(taskArchiveTable.id, session.select().from(selection)))
|
|
1138
|
+
.returning({ id: taskArchiveTable.id });
|
|
1139
|
+
if (result.length < 1000) {
|
|
881
1140
|
break;
|
|
882
1141
|
}
|
|
883
1142
|
}
|
|
884
|
-
|
|
1143
|
+
}
|
|
1144
|
+
async processExpirations(options) {
|
|
1145
|
+
const expiredSelection = this.#database.$with('expired_selection').as((qb) => qb
|
|
1146
|
+
.select({ id: taskTable.id })
|
|
1147
|
+
.from(taskTable)
|
|
1148
|
+
.where(and(eq(taskTable.namespace, this.#namespace), inArray(taskTable.status, queueableOrWaitableStatuses), lt(taskTable.timeToLive, TRANSACTION_TIMESTAMP)))
|
|
1149
|
+
.limit(1000)
|
|
1150
|
+
.for('update', { skipLocked: true }));
|
|
885
1151
|
while (true) {
|
|
886
|
-
const
|
|
887
|
-
const
|
|
888
|
-
let totalUpdated = 0;
|
|
889
|
-
// 3.1 Handle Pending Expiration
|
|
890
|
-
const expiredSelection = repositoryWithTx.session.$with('expired_selection').as((qb) => qb
|
|
891
|
-
.select({ id: taskTable.id })
|
|
892
|
-
.from(taskTable)
|
|
893
|
-
.where(and(eq(taskTable.namespace, this.#namespace), eq(taskTable.status, TaskStatus.Pending), lt(taskTable.timeToLive, TRANSACTION_TIMESTAMP)))
|
|
894
|
-
.limit(1000)
|
|
895
|
-
.for('update', { skipLocked: true }));
|
|
896
|
-
const expiredRows = await repositoryWithTx.session
|
|
1152
|
+
const updatedCount = await this.#repository.useTransaction(options?.transaction, async (tx) => {
|
|
1153
|
+
const expiredRows = await tx.pgTransaction
|
|
897
1154
|
.with(expiredSelection)
|
|
898
1155
|
.update(taskTable)
|
|
899
1156
|
.set({
|
|
900
|
-
status: TaskStatus.
|
|
1157
|
+
status: TaskStatus.Expired,
|
|
901
1158
|
token: null,
|
|
902
1159
|
error: { code: 'Expired', message: 'Task expired before processing' },
|
|
903
1160
|
completeTimestamp: TRANSACTION_TIMESTAMP,
|
|
904
1161
|
})
|
|
905
|
-
.where(inArray(taskTable.id,
|
|
906
|
-
.returning({
|
|
907
|
-
totalUpdated += expiredRows.length;
|
|
1162
|
+
.where(inArray(taskTable.id, tx.pgTransaction.select().from(expiredSelection)))
|
|
1163
|
+
.returning({ id: taskTable.id });
|
|
908
1164
|
if (expiredRows.length > 0) {
|
|
909
|
-
await this.
|
|
1165
|
+
await this.resolveDependenciesMany(expiredRows.map((r) => ({ id: r.id, status: TaskStatus.Expired, namespace: this.#namespace })), { transaction: tx });
|
|
910
1166
|
}
|
|
911
|
-
|
|
912
|
-
|
|
913
|
-
|
|
914
|
-
|
|
915
|
-
|
|
916
|
-
|
|
917
|
-
|
|
918
|
-
|
|
919
|
-
|
|
920
|
-
|
|
921
|
-
|
|
922
|
-
|
|
923
|
-
|
|
924
|
-
|
|
925
|
-
|
|
926
|
-
|
|
927
|
-
|
|
928
|
-
|
|
929
|
-
|
|
930
|
-
|
|
931
|
-
|
|
932
|
-
|
|
933
|
-
|
|
934
|
-
|
|
935
|
-
|
|
936
|
-
|
|
937
|
-
|
|
938
|
-
|
|
1167
|
+
return expiredRows.length;
|
|
1168
|
+
});
|
|
1169
|
+
if (updatedCount < 1000) {
|
|
1170
|
+
break;
|
|
1171
|
+
}
|
|
1172
|
+
}
|
|
1173
|
+
}
|
|
1174
|
+
async processZombieRetries(options) {
|
|
1175
|
+
const session = options?.transaction?.pgTransaction ?? this.#database;
|
|
1176
|
+
const zombieRetrySelection = session.$with('zombie_retry_selection').as((qb) => qb
|
|
1177
|
+
.select({ id: taskTable.id })
|
|
1178
|
+
.from(taskTable)
|
|
1179
|
+
.where(and(eq(taskTable.namespace, this.#namespace), eq(taskTable.status, TaskStatus.Running), lt(taskTable.visibilityDeadline, TRANSACTION_TIMESTAMP), lt(taskTable.tries, this.maxTries)))
|
|
1180
|
+
.limit(1000)
|
|
1181
|
+
.for('update', { skipLocked: true }));
|
|
1182
|
+
while (true) {
|
|
1183
|
+
const zombieRetryRows = await session
|
|
1184
|
+
.with(zombieRetrySelection)
|
|
1185
|
+
.update(taskTable)
|
|
1186
|
+
.set({
|
|
1187
|
+
status: TaskStatus.Retrying,
|
|
1188
|
+
token: null,
|
|
1189
|
+
visibilityDeadline: null,
|
|
1190
|
+
startTimestamp: null,
|
|
1191
|
+
scheduleTimestamp: sql `${TRANSACTION_TIMESTAMP} + ${interval(this.retryDelayMinimum, 'milliseconds')}`, // Simple backoff for zombies
|
|
1192
|
+
error: jsonbBuildObject({ code: 'VisibilityTimeout', message: 'Worker Lost', lastError: taskTable.error }),
|
|
1193
|
+
})
|
|
1194
|
+
.where(inArray(taskTable.id, session.select().from(zombieRetrySelection)))
|
|
1195
|
+
.returning({ id: taskTable.id });
|
|
1196
|
+
if (zombieRetryRows.length < 1000) {
|
|
1197
|
+
break;
|
|
1198
|
+
}
|
|
1199
|
+
}
|
|
1200
|
+
}
|
|
1201
|
+
async processZombieExhaustions(options) {
|
|
1202
|
+
const zombieExhaustionSelection = this.#database.$with('zombie_exhaustion_selection').as((qb) => qb
|
|
1203
|
+
.select({ id: taskTable.id })
|
|
1204
|
+
.from(taskTable)
|
|
1205
|
+
.where(and(eq(taskTable.namespace, this.#namespace), eq(taskTable.status, TaskStatus.Running), lt(taskTable.visibilityDeadline, TRANSACTION_TIMESTAMP), gte(taskTable.tries, this.maxTries)))
|
|
1206
|
+
.limit(1000)
|
|
1207
|
+
.for('update', { skipLocked: true }));
|
|
1208
|
+
while (true) {
|
|
1209
|
+
const updatedCount = await this.#repository.useTransaction(options?.transaction, async (tx) => {
|
|
1210
|
+
const exhaustionRows = await tx.pgTransaction
|
|
939
1211
|
.with(zombieExhaustionSelection)
|
|
940
1212
|
.update(taskTable)
|
|
941
1213
|
.set({
|
|
942
|
-
status: TaskStatus.
|
|
1214
|
+
status: TaskStatus.Orphaned,
|
|
943
1215
|
token: null,
|
|
944
1216
|
visibilityDeadline: null,
|
|
945
1217
|
completeTimestamp: TRANSACTION_TIMESTAMP,
|
|
946
|
-
error:
|
|
1218
|
+
error: jsonbBuildObject({ code: 'ZombieExhausted', message: 'Exceeded max retries after repeated crashes', lastError: taskTable.error }),
|
|
947
1219
|
})
|
|
948
|
-
.where(inArray(taskTable.id,
|
|
949
|
-
.returning({
|
|
950
|
-
totalUpdated += exhaustionRows.length;
|
|
1220
|
+
.where(inArray(taskTable.id, tx.pgTransaction.select().from(zombieExhaustionSelection)))
|
|
1221
|
+
.returning({ id: taskTable.id });
|
|
951
1222
|
if (exhaustionRows.length > 0) {
|
|
952
|
-
await this.
|
|
1223
|
+
await this.resolveDependenciesMany(exhaustionRows.map((r) => ({ id: r.id, status: TaskStatus.Orphaned, namespace: this.#namespace })), { transaction: tx });
|
|
953
1224
|
}
|
|
954
|
-
|
|
955
|
-
|
|
956
|
-
|
|
957
|
-
|
|
958
|
-
|
|
959
|
-
|
|
960
|
-
|
|
961
|
-
|
|
1225
|
+
return exhaustionRows.length;
|
|
1226
|
+
});
|
|
1227
|
+
if (updatedCount < 1000) {
|
|
1228
|
+
break;
|
|
1229
|
+
}
|
|
1230
|
+
}
|
|
1231
|
+
}
|
|
1232
|
+
async processHardTimeouts(options) {
|
|
1233
|
+
const timeoutSelection = this.#database.$with('timeout_selection').as((qb) => qb
|
|
1234
|
+
.select({ id: taskTable.id })
|
|
1235
|
+
.from(taskTable)
|
|
1236
|
+
.where(and(eq(taskTable.namespace, this.#namespace), eq(taskTable.status, TaskStatus.Running), lt(taskTable.startTimestamp, sql `${TRANSACTION_TIMESTAMP} - ${interval(this.maxExecutionTime, 'milliseconds')}`)))
|
|
1237
|
+
.limit(1000)
|
|
1238
|
+
.for('update', { skipLocked: true }));
|
|
1239
|
+
while (true) {
|
|
1240
|
+
const updatedCount = await this.#repository.useTransaction(options?.transaction, async (tx) => {
|
|
1241
|
+
const timeoutRows = await tx.pgTransaction
|
|
962
1242
|
.with(timeoutSelection)
|
|
963
1243
|
.update(taskTable)
|
|
964
1244
|
.set({
|
|
965
|
-
status: TaskStatus.
|
|
1245
|
+
status: TaskStatus.TimedOut,
|
|
966
1246
|
token: null,
|
|
967
1247
|
visibilityDeadline: null,
|
|
968
1248
|
completeTimestamp: TRANSACTION_TIMESTAMP,
|
|
969
|
-
error:
|
|
1249
|
+
error: jsonbBuildObject({ code: 'MaxTimeExceeded', message: sql `'Hard Execution Timeout: Task ran longer than ' || ${this.maxExecutionTime} || 'ms'`, lastError: taskTable.error }),
|
|
970
1250
|
})
|
|
971
|
-
.where(inArray(taskTable.id,
|
|
972
|
-
.returning({
|
|
973
|
-
totalUpdated += timeoutRows.length;
|
|
1251
|
+
.where(inArray(taskTable.id, tx.pgTransaction.select().from(timeoutSelection)))
|
|
1252
|
+
.returning({ id: taskTable.id });
|
|
974
1253
|
if (timeoutRows.length > 0) {
|
|
975
|
-
await this.
|
|
1254
|
+
await this.resolveDependenciesMany(timeoutRows.map((r) => ({ id: r.id, status: TaskStatus.TimedOut, namespace: this.#namespace })), { transaction: tx });
|
|
976
1255
|
}
|
|
977
|
-
|
|
978
|
-
const agingSelection = repositoryWithTx.session.$with('aging_selection').as((qb) => qb
|
|
979
|
-
.select({ id: taskTable.id })
|
|
980
|
-
.from(taskTable)
|
|
981
|
-
.where(and(eq(taskTable.namespace, this.#namespace), eq(taskTable.status, TaskStatus.Pending), lt(taskTable.priorityAgeTimestamp, sql `${TRANSACTION_TIMESTAMP} - ${interval(this.priorityAgingInterval, 'milliseconds')}`)))
|
|
982
|
-
.limit(1000)
|
|
983
|
-
.for('update', { skipLocked: true }));
|
|
984
|
-
const agingRows = await repositoryWithTx.session
|
|
985
|
-
.with(agingSelection)
|
|
986
|
-
.update(taskTable)
|
|
987
|
-
.set({
|
|
988
|
-
priority: sql `${taskTable.priority} - ${this.priorityAgingStep}`,
|
|
989
|
-
priorityAgeTimestamp: TRANSACTION_TIMESTAMP,
|
|
990
|
-
})
|
|
991
|
-
.where(inArray(taskTable.id, repositoryWithTx.session.select().from(agingSelection)))
|
|
992
|
-
.returning({ id: taskTable.id });
|
|
993
|
-
totalUpdated += agingRows.length;
|
|
994
|
-
return totalUpdated;
|
|
1256
|
+
return timeoutRows.length;
|
|
995
1257
|
});
|
|
996
|
-
if (
|
|
1258
|
+
if (updatedCount < 1000) {
|
|
1259
|
+
break;
|
|
1260
|
+
}
|
|
1261
|
+
}
|
|
1262
|
+
}
|
|
1263
|
+
async processPriorityAging(options) {
|
|
1264
|
+
const session = options?.transaction?.pgTransaction ?? this.#database;
|
|
1265
|
+
const agingSelection = session.$with('aging_selection').as((qb) => qb
|
|
1266
|
+
.select({ id: taskTable.id })
|
|
1267
|
+
.from(taskTable)
|
|
1268
|
+
.where(and(eq(taskTable.namespace, this.#namespace), inArray(taskTable.status, queueableStatuses), lt(taskTable.priorityAgeTimestamp, sql `${TRANSACTION_TIMESTAMP} - ${interval(this.priorityAgingInterval, 'milliseconds')}`)))
|
|
1269
|
+
.limit(1000)
|
|
1270
|
+
.for('update', { skipLocked: true }));
|
|
1271
|
+
while (true) {
|
|
1272
|
+
const agingRows = await session
|
|
1273
|
+
.with(agingSelection)
|
|
1274
|
+
.update(taskTable)
|
|
1275
|
+
.set({
|
|
1276
|
+
priority: sql `${taskTable.priority} - ${this.priorityAgingStep}`,
|
|
1277
|
+
priorityAgeTimestamp: TRANSACTION_TIMESTAMP,
|
|
1278
|
+
})
|
|
1279
|
+
.where(inArray(taskTable.id, session.select().from(agingSelection)))
|
|
1280
|
+
.returning({ id: taskTable.id });
|
|
1281
|
+
if (agingRows.length < 1000) {
|
|
997
1282
|
break;
|
|
998
1283
|
}
|
|
999
1284
|
}
|
|
@@ -1009,80 +1294,93 @@ let PostgresTaskQueue = class PostgresTaskQueue extends TaskQueue {
|
|
|
1009
1294
|
result: null,
|
|
1010
1295
|
scheduleTimestamp: TRANSACTION_TIMESTAMP,
|
|
1011
1296
|
completeTimestamp: null,
|
|
1297
|
+
startTimestamp: null,
|
|
1012
1298
|
tries: 0,
|
|
1013
1299
|
progress: 0,
|
|
1014
1300
|
priorityAgeTimestamp: TRANSACTION_TIMESTAMP,
|
|
1015
1301
|
state: (options?.resetState == true) ? null : undefined,
|
|
1016
1302
|
})
|
|
1017
|
-
.where(and(eq(taskTable.
|
|
1018
|
-
.execute();
|
|
1303
|
+
.where(and(eq(taskTable.id, id), or(inArray(taskTable.status, queueableStatuses), inArray(taskTable.status, finalizedStatuses), lt(taskTable.visibilityDeadline, TRANSACTION_TIMESTAMP))));
|
|
1019
1304
|
}
|
|
1020
|
-
notify() {
|
|
1021
|
-
this.#messageBus.publishAndForget();
|
|
1305
|
+
notify(namespace = this.#namespace) {
|
|
1306
|
+
this.#messageBus.publishAndForget(namespace);
|
|
1022
1307
|
}
|
|
1023
1308
|
async *getConsumer(cancellationSignal, options) {
|
|
1024
|
-
const continue$ = merge(this.#messageBus.allMessages
|
|
1309
|
+
const continue$ = merge(this.#messageBus.allMessages$).pipe(filter((namespace) => namespace == this.#namespace));
|
|
1310
|
+
const mergedContinue$ = merge(continue$, cancellationSignal);
|
|
1025
1311
|
while (cancellationSignal.isUnset) {
|
|
1026
1312
|
const task = await this.dequeue(options);
|
|
1027
1313
|
if (isDefined(task)) {
|
|
1028
1314
|
yield task;
|
|
1029
1315
|
continue;
|
|
1030
1316
|
}
|
|
1031
|
-
await cancelableTimeout(5 * millisecondsPerSecond,
|
|
1317
|
+
await cancelableTimeout(5 * millisecondsPerSecond, mergedContinue$);
|
|
1032
1318
|
}
|
|
1033
1319
|
}
|
|
1034
|
-
|
|
1320
|
+
maintenanceLoop() {
|
|
1321
|
+
void this.lowFrequencyMaintenanceLoop();
|
|
1322
|
+
void this.mediumFrequencyMaintenanceLoop();
|
|
1323
|
+
void this.highFrequencyMaintenanceLoop();
|
|
1324
|
+
}
|
|
1325
|
+
async lowFrequencyMaintenanceLoop() {
|
|
1035
1326
|
while (this.#cancellationSignal.isUnset) {
|
|
1036
1327
|
try {
|
|
1037
|
-
await
|
|
1328
|
+
await Promise.allSettled([
|
|
1329
|
+
this.performArchival(),
|
|
1330
|
+
this.performArchivePurge(),
|
|
1331
|
+
]);
|
|
1038
1332
|
}
|
|
1039
1333
|
catch (error) {
|
|
1040
|
-
this.#logger.error('Error during maintenance loop', error);
|
|
1334
|
+
this.#logger.error('Error during low frequency maintenance loop', error);
|
|
1041
1335
|
}
|
|
1042
1336
|
finally {
|
|
1043
|
-
await cancelableTimeout(
|
|
1337
|
+
await cancelableTimeout(5 * millisecondsPerMinute, this.#cancellationSignal);
|
|
1044
1338
|
}
|
|
1045
1339
|
}
|
|
1046
1340
|
}
|
|
1047
|
-
|
|
1048
|
-
|
|
1049
|
-
* @param options
|
|
1050
|
-
* @returns
|
|
1051
|
-
*/
|
|
1052
|
-
async processPendingFanIn(options) {
|
|
1053
|
-
let processedCount = 0;
|
|
1054
|
-
while (true) {
|
|
1055
|
-
const task = await this.#internalThis.dequeue({ types: ['[SystemWorker]:FanIn'], forceDequeue: true, transaction: options?.transaction });
|
|
1056
|
-
if (isUndefined(task)) {
|
|
1057
|
-
break;
|
|
1058
|
-
}
|
|
1341
|
+
async mediumFrequencyMaintenanceLoop() {
|
|
1342
|
+
while (this.#cancellationSignal.isUnset) {
|
|
1059
1343
|
try {
|
|
1060
|
-
await
|
|
1061
|
-
|
|
1062
|
-
|
|
1344
|
+
await Promise.allSettled([
|
|
1345
|
+
this.processExpirations(),
|
|
1346
|
+
this.processPriorityAging(),
|
|
1347
|
+
]);
|
|
1063
1348
|
}
|
|
1064
1349
|
catch (error) {
|
|
1065
|
-
|
|
1066
|
-
|
|
1350
|
+
this.#logger.error('Error during medium frequency maintenance loop', error);
|
|
1351
|
+
}
|
|
1352
|
+
finally {
|
|
1353
|
+
await cancelableTimeout(millisecondsPerMinute, this.#cancellationSignal);
|
|
1067
1354
|
}
|
|
1068
1355
|
}
|
|
1069
|
-
return processedCount;
|
|
1070
1356
|
}
|
|
1071
|
-
|
|
1072
|
-
|
|
1073
|
-
|
|
1074
|
-
|
|
1075
|
-
|
|
1357
|
+
async highFrequencyMaintenanceLoop() {
|
|
1358
|
+
while (this.#cancellationSignal.isUnset) {
|
|
1359
|
+
try {
|
|
1360
|
+
await Promise.allSettled([
|
|
1361
|
+
this.processZombieRetries(),
|
|
1362
|
+
this.processZombieExhaustions(),
|
|
1363
|
+
this.processHardTimeouts(),
|
|
1364
|
+
]);
|
|
1365
|
+
}
|
|
1366
|
+
catch (error) {
|
|
1367
|
+
this.#logger.error('Error during high frequency maintenance loop', error);
|
|
1368
|
+
}
|
|
1369
|
+
finally {
|
|
1370
|
+
await cancelableTimeout(10 * millisecondsPerSecond, this.#cancellationSignal);
|
|
1371
|
+
}
|
|
1372
|
+
}
|
|
1076
1373
|
}
|
|
1077
1374
|
async *getBatchConsumer(size, cancellationSignal, options) {
|
|
1078
|
-
const continue$ = merge(this.#messageBus.allMessages
|
|
1375
|
+
const continue$ = merge(this.#messageBus.allMessages$).pipe(filter((namespace) => namespace == this.#namespace));
|
|
1376
|
+
const mergedContinue$ = merge(continue$, cancellationSignal);
|
|
1079
1377
|
while (cancellationSignal.isUnset) {
|
|
1080
1378
|
const tasks = await this.dequeueMany(size, options);
|
|
1081
1379
|
if (tasks.length > 0) {
|
|
1082
1380
|
yield tasks;
|
|
1083
1381
|
continue;
|
|
1084
1382
|
}
|
|
1085
|
-
await cancelableTimeout(5 * millisecondsPerSecond,
|
|
1383
|
+
await cancelableTimeout(5 * millisecondsPerSecond, mergedContinue$);
|
|
1086
1384
|
}
|
|
1087
1385
|
}
|
|
1088
1386
|
};
|