@tstdl/base 0.93.140 → 0.93.142

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (123) hide show
  1. package/application/application.d.ts +1 -1
  2. package/application/application.js +1 -1
  3. package/application/providers.d.ts +20 -2
  4. package/application/providers.js +34 -7
  5. package/audit/module.d.ts +5 -0
  6. package/audit/module.js +9 -1
  7. package/authentication/client/authentication.service.d.ts +1 -0
  8. package/authentication/client/authentication.service.js +3 -2
  9. package/authentication/server/module.d.ts +5 -0
  10. package/authentication/server/module.js +9 -1
  11. package/authentication/tests/authentication.api-controller.test.js +1 -1
  12. package/authentication/tests/authentication.api-request-token.provider.test.js +1 -1
  13. package/authentication/tests/authentication.client-service.test.js +1 -1
  14. package/circuit-breaker/circuit-breaker.d.ts +6 -4
  15. package/circuit-breaker/postgres/circuit-breaker.d.ts +1 -0
  16. package/circuit-breaker/postgres/circuit-breaker.js +8 -5
  17. package/circuit-breaker/postgres/module.d.ts +1 -0
  18. package/circuit-breaker/postgres/module.js +5 -1
  19. package/circuit-breaker/tests/circuit-breaker.test.js +20 -0
  20. package/document-management/server/configure.js +5 -1
  21. package/document-management/server/module.d.ts +1 -1
  22. package/document-management/server/module.js +1 -1
  23. package/document-management/server/services/document-management-ancillary.service.js +1 -1
  24. package/document-management/tests/ai-config-hierarchy.test.js +0 -5
  25. package/document-management/tests/document-management-ai-overrides.test.js +0 -1
  26. package/document-management/tests/document-validation-ai-overrides.test.js +0 -1
  27. package/examples/document-management/main.d.ts +1 -0
  28. package/examples/document-management/main.js +14 -11
  29. package/key-value-store/postgres/module.d.ts +1 -0
  30. package/key-value-store/postgres/module.js +5 -1
  31. package/lock/postgres/module.d.ts +1 -0
  32. package/lock/postgres/module.js +5 -1
  33. package/mail/module.d.ts +5 -1
  34. package/mail/module.js +11 -6
  35. package/module/modules/web-server.module.js +2 -3
  36. package/notification/server/module.d.ts +1 -0
  37. package/notification/server/module.js +5 -1
  38. package/notification/tests/notification-api.test.js +5 -1
  39. package/notification/tests/notification-flow.test.js +8 -5
  40. package/orm/decorators.d.ts +22 -5
  41. package/orm/decorators.js +10 -1
  42. package/orm/server/bootstrap.d.ts +11 -0
  43. package/orm/server/bootstrap.js +31 -0
  44. package/orm/server/drizzle/schema-converter.d.ts +3 -1
  45. package/orm/server/drizzle/schema-converter.js +85 -56
  46. package/orm/server/encryption.d.ts +0 -1
  47. package/orm/server/encryption.js +1 -4
  48. package/orm/server/extension.d.ts +14 -0
  49. package/orm/server/extension.js +27 -0
  50. package/orm/server/index.d.ts +3 -6
  51. package/orm/server/index.js +3 -6
  52. package/orm/server/migration.d.ts +18 -0
  53. package/orm/server/migration.js +58 -0
  54. package/orm/server/repository.d.ts +2 -1
  55. package/orm/server/repository.js +19 -9
  56. package/orm/server/transaction.d.ts +6 -10
  57. package/orm/server/transaction.js +25 -26
  58. package/orm/server/transactional.js +3 -3
  59. package/orm/tests/database-extension.test.js +63 -0
  60. package/orm/tests/database-migration.test.js +83 -0
  61. package/orm/tests/encryption.test.js +3 -4
  62. package/orm/tests/repository-compound-primary-key.test.d.ts +2 -0
  63. package/orm/tests/repository-compound-primary-key.test.js +234 -0
  64. package/orm/tests/schema-generation.test.d.ts +1 -0
  65. package/orm/tests/schema-generation.test.js +52 -5
  66. package/orm/utils.d.ts +17 -2
  67. package/orm/utils.js +49 -1
  68. package/package.json +5 -4
  69. package/rate-limit/postgres/module.d.ts +1 -0
  70. package/rate-limit/postgres/module.js +5 -1
  71. package/reflection/decorator-data.js +11 -12
  72. package/task-queue/README.md +2 -10
  73. package/task-queue/postgres/drizzle/0000_great_gwen_stacy.sql +84 -0
  74. package/task-queue/postgres/drizzle/meta/0000_snapshot.json +250 -89
  75. package/task-queue/postgres/drizzle/meta/_journal.json +2 -2
  76. package/task-queue/postgres/module.d.ts +1 -0
  77. package/task-queue/postgres/module.js +6 -1
  78. package/task-queue/postgres/schemas.d.ts +15 -6
  79. package/task-queue/postgres/schemas.js +4 -3
  80. package/task-queue/postgres/task-queue.d.ts +18 -15
  81. package/task-queue/postgres/task-queue.js +797 -499
  82. package/task-queue/postgres/task.model.d.ts +20 -9
  83. package/task-queue/postgres/task.model.js +65 -39
  84. package/task-queue/task-context.d.ts +12 -7
  85. package/task-queue/task-context.js +8 -6
  86. package/task-queue/task-queue.d.ts +364 -43
  87. package/task-queue/task-queue.js +153 -41
  88. package/task-queue/tests/coverage-branch.test.d.ts +1 -0
  89. package/task-queue/tests/coverage-branch.test.js +395 -0
  90. package/task-queue/tests/coverage-enhancement.test.d.ts +1 -0
  91. package/task-queue/tests/coverage-enhancement.test.js +150 -0
  92. package/task-queue/tests/dag.test.d.ts +1 -0
  93. package/task-queue/tests/dag.test.js +188 -0
  94. package/task-queue/tests/dependencies.test.js +165 -47
  95. package/task-queue/tests/enqueue-batch.test.d.ts +1 -0
  96. package/task-queue/tests/enqueue-batch.test.js +125 -0
  97. package/task-queue/tests/fan-out-spawning.test.d.ts +1 -0
  98. package/task-queue/tests/fan-out-spawning.test.js +94 -0
  99. package/task-queue/tests/idempotent-replacement.test.d.ts +1 -0
  100. package/task-queue/tests/idempotent-replacement.test.js +114 -0
  101. package/task-queue/tests/missing-idempotent-tasks.test.d.ts +1 -0
  102. package/task-queue/tests/missing-idempotent-tasks.test.js +39 -0
  103. package/task-queue/tests/queue.test.js +294 -49
  104. package/task-queue/tests/shutdown.test.d.ts +1 -0
  105. package/task-queue/tests/shutdown.test.js +41 -0
  106. package/task-queue/tests/transactions.test.d.ts +1 -0
  107. package/task-queue/tests/transactions.test.js +47 -0
  108. package/task-queue/tests/worker.test.js +63 -15
  109. package/task-queue/tests/zombie-parent.test.d.ts +1 -0
  110. package/task-queue/tests/zombie-parent.test.js +45 -0
  111. package/task-queue/tests/zombie-recovery.test.d.ts +1 -0
  112. package/task-queue/tests/zombie-recovery.test.js +51 -0
  113. package/test5.js +5 -5
  114. package/testing/integration-setup.d.ts +4 -4
  115. package/testing/integration-setup.js +56 -29
  116. package/text/localization.service.js +2 -2
  117. package/utils/file-reader.js +1 -2
  118. package/utils/timing.d.ts +2 -2
  119. package/task-queue/postgres/drizzle/0000_simple_invisible_woman.sql +0 -74
  120. package/task-queue/tests/complex.test.js +0 -306
  121. package/task-queue/tests/extensive-dependencies.test.js +0 -234
  122. /package/{task-queue/tests/complex.test.d.ts → orm/tests/database-extension.test.d.ts} +0 -0
  123. /package/{task-queue/tests/extensive-dependencies.test.d.ts → orm/tests/database-migration.test.d.ts} +0 -0
@@ -56,35 +56,34 @@ var __disposeResources = (this && this.__disposeResources) || (function (Suppres
56
56
  var e = new Error(message);
57
57
  return e.name = "SuppressedError", e.error = error, e.suppressed = suppressed, e;
58
58
  });
59
- import { and, asc, count, eq, gt, gte, inArray, lt, lte, notInArray, or, sql, isNull as sqlIsNull } from 'drizzle-orm';
60
- import { merge, throttleTime } from 'rxjs';
59
+ import { aliasedTable, and, asc, count, eq, gt, gte, inArray, lt, lte, notExists, notInArray, or, sql, count as sqlCount, isNotNull as sqlIsNotNull, isNull as sqlIsNull } from 'drizzle-orm';
60
+ import { filter, merge, throttleTime } from 'rxjs';
61
61
  import { CancellationSignal } from '../../cancellation/index.js';
62
62
  import { CircuitBreaker, CircuitBreakerState } from '../../circuit-breaker/index.js';
63
63
  import { serializeError, TimeoutError } from '../../errors/index.js';
64
64
  import { afterResolve, inject, provide, Singleton } from '../../injector/index.js';
65
65
  import { Logger } from '../../logger/index.js';
66
66
  import { MessageBus } from '../../message-bus/index.js';
67
- import { arrayOverlaps, coalesce, interval, RANDOM_UUID_V4, TRANSACTION_TIMESTAMP, unnest } from '../../orm/index.js';
68
- import { DatabaseConfig, injectRepository } from '../../orm/server/index.js';
67
+ import { arrayOverlaps, caseWhen, coalesce, enumValue, getEntityIds, greatest, interval, jsonbBuildObject, RANDOM_UUID_V4, TRANSACTION_TIMESTAMP } from '../../orm/index.js';
68
+ import { Database, DatabaseConfig, injectRepository } from '../../orm/server/index.js';
69
69
  import { RateLimiter } from '../../rate-limit/index.js';
70
- import { createArray, distinct, toArray } from '../../utils/array/array.js';
71
- import { digest } from '../../utils/cryptography.js';
70
+ import { distinct, toArray } from '../../utils/array/array.js';
72
71
  import { currentTimestamp } from '../../utils/date-time.js';
73
72
  import { Timer } from '../../utils/timer.js';
74
73
  import { cancelableTimeout } from '../../utils/timing.js';
75
- import { isDefined, isNotNull, isNull, isString, isUndefined } from '../../utils/type-guards.js';
76
- import { millisecondsPerSecond } from '../../utils/units.js';
77
- import { defaultQueueConfig, DependencyJoinMode, TaskProcessResult, TaskQueue, TaskStatus } from '../task-queue.js';
74
+ import { isDefined, isNotNull, isNull, isNumber, isString, isUndefined } from '../../utils/type-guards.js';
75
+ import { millisecondsPerMinute, millisecondsPerSecond } from '../../utils/units.js';
76
+ import { defaultQueueConfig, TaskDependencyType, TaskQueue, TaskStatus } from '../task-queue.js';
78
77
  import { PostgresTaskQueueModuleConfig } from './module.js';
79
- import { taskArchive as taskArchiveTable, taskStatus, task as taskTable } from './schemas.js';
80
- import { PostgresTask, PostgresTaskArchive } from './task.model.js';
78
+ import { taskArchive as taskArchiveTable, taskDependency as taskDependencyTable, taskDependencyType, taskStatus, task as taskTable } from './schemas.js';
79
+ import { finalizedStatuses, PostgresTask, PostgresTaskArchive, queueableOrWaitableStatuses, queueableStatuses, terminalStatuses } from './task.model.js';
81
80
  let PostgresTaskQueue = class PostgresTaskQueue extends TaskQueue {
82
- #internalThis = this;
81
+ #database = inject(Database);
83
82
  #repository = injectRepository(PostgresTask);
84
83
  #archiveRepository = injectRepository(PostgresTaskArchive);
85
84
  #config = this.config;
86
85
  #namespace = isString(this.#config) ? this.#config : this.#config.namespace;
87
- #messageBus = inject((MessageBus), `PostgresTaskQueue:${this.#namespace}`);
86
+ #messageBus = inject((MessageBus), 'PostgresTaskQueue');
88
87
  #logger = inject(Logger, `PostgresTaskQueue:${this.#namespace}`);
89
88
  #cancellationSignal = inject(CancellationSignal);
90
89
  #rateLimiter = inject(RateLimiter, {
@@ -111,22 +110,19 @@ let PostgresTaskQueue = class PostgresTaskQueue extends TaskQueue {
111
110
  rateLimit = this.#config.rateLimit ?? defaultQueueConfig.rateLimit;
112
111
  rateInterval = this.#config.rateInterval ?? defaultQueueConfig.rateInterval;
113
112
  idempotencyWindow = this.#config.idempotencyWindow ?? defaultQueueConfig.idempotencyWindow;
114
- globalConcurrency = this.#config.globalConcurrency ?? defaultQueueConfig.globalConcurrency;
115
113
  #takeNewUpdate = {
116
- id: RANDOM_UUID_V4,
117
114
  namespace: this.#namespace,
118
115
  type: sql `excluded.type`,
119
- status: TaskStatus.Pending,
116
+ status: caseWhen(gt(sql `excluded.unresolved_schedule_dependencies`, 0), enumValue(TaskStatus, taskStatus, TaskStatus.Waiting)).else(enumValue(TaskStatus, taskStatus, TaskStatus.Pending)),
120
117
  token: null,
121
118
  priority: sql `excluded.priority`,
122
119
  idempotencyKey: sql `excluded.idempotency_key`,
123
120
  traceId: sql `excluded.trace_id`,
121
+ parentId: sql `excluded.parent_id`,
124
122
  tags: sql `excluded.tags`,
125
- completeAfterTags: sql `excluded.complete_after_tags`,
126
- scheduleAfterTags: sql `excluded.schedule_after_tags`,
123
+ unresolvedScheduleDependencies: taskTable.unresolvedScheduleDependencies,
124
+ unresolvedCompleteDependencies: taskTable.unresolvedCompleteDependencies,
127
125
  failFast: sql `excluded.fail_fast`,
128
- dependencyJoinMode: sql `excluded.dependency_join_mode`,
129
- dependencyTriggerStatuses: sql `excluded.dependency_trigger_statuses`,
130
126
  tries: 0,
131
127
  creationTimestamp: TRANSACTION_TIMESTAMP,
132
128
  priorityAgeTimestamp: TRANSACTION_TIMESTAMP,
@@ -143,8 +139,7 @@ let PostgresTaskQueue = class PostgresTaskQueue extends TaskQueue {
143
139
  };
144
140
  [afterResolve]() {
145
141
  if (!this.isInTransaction) {
146
- this.startSystemWorker();
147
- void this.maintenanceLoop();
142
+ this.maintenanceLoop();
148
143
  }
149
144
  }
150
145
  async enqueue(type, data, options) {
@@ -157,76 +152,207 @@ let PostgresTaskQueue = class PostgresTaskQueue extends TaskQueue {
157
152
  if (items.length == 0) {
158
153
  return (options?.returnTasks == true) ? [] : undefined;
159
154
  }
160
- const newEntities = items.map((item) => {
161
- const hasDependencies = (item.completeAfterTags?.length ?? 0) > 0 || (item.scheduleAfterTags?.length ?? 0) > 0;
155
+ const itemsWithDistinctDependencies = items.map((item) => ({
156
+ ...item,
157
+ scheduleAfter: isDefined(item.scheduleAfter) ? Array.from(new Map(item.scheduleAfter.map((s) => [isNumber(s) || isString(s) ? s : JSON.stringify(s), s])).values()) : undefined,
158
+ completeAfter: isDefined(item.completeAfter) ? Array.from(new Map(item.completeAfter.map((s) => [isNumber(s) || isString(s) ? s : JSON.stringify(s), s])).values()) : undefined,
159
+ }));
160
+ const entitiesWithIndex = itemsWithDistinctDependencies.map((item, index) => {
162
161
  return {
163
- namespace: this.#namespace,
164
- type: item.type,
165
- status: hasDependencies ? TaskStatus.Waiting : TaskStatus.Pending,
166
- token: null,
167
- priority: item.priority ?? 1000,
168
- idempotencyKey: item.idempotencyKey ?? null,
169
- traceId: null,
170
- tags: item.tags ?? [],
171
- completeAfterTags: item.completeAfterTags ?? [],
172
- scheduleAfterTags: item.scheduleAfterTags ?? [],
173
- failFast: item.failFast ?? false,
174
- dependencyJoinMode: item.dependencyJoinMode ?? DependencyJoinMode.And,
175
- dependencyTriggerStatuses: item.dependencyTriggerStatuses ?? [TaskStatus.Completed],
176
- parentId: item.parentId ?? null,
177
- tries: 0,
178
- progress: 0,
179
- creationTimestamp: TRANSACTION_TIMESTAMP,
180
- priorityAgeTimestamp: TRANSACTION_TIMESTAMP,
181
- scheduleTimestamp: item.scheduleTimestamp ?? TRANSACTION_TIMESTAMP,
182
- startTimestamp: null,
183
- timeToLive: item.timeToLive ?? sql `${TRANSACTION_TIMESTAMP} + ${interval(this.defaultTimeToLive, 'milliseconds')}`,
184
- visibilityDeadline: null,
185
- completeTimestamp: null,
186
- data: item.data,
187
- state: null,
188
- result: null,
189
- error: null,
162
+ index,
163
+ entity: {
164
+ namespace: this.#namespace,
165
+ type: item.type,
166
+ status: TaskStatus.Pending,
167
+ token: null,
168
+ priority: item.priority ?? 1000,
169
+ idempotencyKey: item.idempotencyKey ?? null,
170
+ traceId: null,
171
+ tags: item.tags ?? [],
172
+ unresolvedScheduleDependencies: 0,
173
+ unresolvedCompleteDependencies: 0,
174
+ failFast: item.failFast ?? false,
175
+ parentId: item.parentId ?? null,
176
+ tries: 0,
177
+ progress: 0,
178
+ creationTimestamp: TRANSACTION_TIMESTAMP,
179
+ priorityAgeTimestamp: TRANSACTION_TIMESTAMP,
180
+ scheduleTimestamp: item.scheduleTimestamp ?? TRANSACTION_TIMESTAMP,
181
+ startTimestamp: null,
182
+ timeToLive: item.timeToLive ?? sql `${TRANSACTION_TIMESTAMP} + ${interval(this.defaultTimeToLive, 'milliseconds')}`,
183
+ visibilityDeadline: null,
184
+ completeTimestamp: null,
185
+ data: item.data,
186
+ state: null,
187
+ result: null,
188
+ error: null,
189
+ },
190
190
  };
191
191
  });
192
- const itemsWithIdempotency = newEntities.filter((e) => isNotNull(e.idempotencyKey));
193
- const itemsWithoutIdempotency = newEntities.filter((e) => isNull(e.idempotencyKey));
194
- const hasDependencies = items.some((item) => ((item.completeAfterTags?.length ?? 0) > 0) || ((item.scheduleAfterTags?.length ?? 0) > 0));
195
- const mustUseTransaction = ((itemsWithoutIdempotency.length + itemsWithIdempotency.length) > 1) || hasDependencies;
192
+ const itemsWithIdempotency = entitiesWithIndex.filter((e) => isNotNull(e.entity.idempotencyKey));
193
+ const itemsWithoutIdempotency = entitiesWithIndex.filter((e) => isNull(e.entity.idempotencyKey));
194
+ const hasDependencies = itemsWithDistinctDependencies.some((item) => ((item.scheduleAfter?.length ?? 0) > 0) || ((item.completeAfter?.length ?? 0) > 0) || (isDefined(item.parentId) && (item.waitForCompletion ?? true)));
195
+ const mustUseTransaction = (entitiesWithIndex.length > 1) || hasDependencies;
196
196
  const newTransaction = __addDisposableResource(env_1, (mustUseTransaction && isUndefined(options?.transaction)) ? await this.#repository.startTransaction() : undefined, true);
197
197
  const transaction = newTransaction ?? options?.transaction;
198
- const tasks = [];
198
+ const session = transaction?.pgTransaction ?? this.#repository.session;
199
+ const tasks = new Array(itemsWithDistinctDependencies.length);
200
+ let insertedRows = [];
201
+ let upsertedRows = [];
199
202
  if (itemsWithoutIdempotency.length > 0) {
200
- const insertedTasks = await this.#repository.withOptionalTransaction(transaction).insertMany(itemsWithoutIdempotency);
201
- tasks.push(...insertedTasks);
203
+ insertedRows = await session.insert(taskTable).values(itemsWithoutIdempotency.map((i) => i.entity)).returning();
204
+ for (const [i, row] of insertedRows.entries()) {
205
+ tasks[itemsWithoutIdempotency[i].index] = row;
206
+ }
202
207
  }
203
208
  if (itemsWithIdempotency.length > 0) {
204
- const repository = this.#repository.withOptionalTransaction(transaction);
205
209
  const windowCutoff = sql `${TRANSACTION_TIMESTAMP} - ${interval(this.idempotencyWindow, 'milliseconds')}`;
206
210
  const setWhere = (options?.replace == true) ? undefined : sql `${taskTable.creationTimestamp} < ${windowCutoff}`;
207
- const upsertedTasks = await repository.upsertMany(['namespace', 'idempotencyKey'], itemsWithIdempotency, this.#takeNewUpdate, { set: setWhere });
208
- tasks.push(...upsertedTasks);
209
- if (options?.returnTasks == true && upsertedTasks.length < itemsWithIdempotency.length) {
210
- const upsertedKeys = new Set(upsertedTasks.map((t) => t.idempotencyKey));
211
- const missingKeys = itemsWithIdempotency.map((i) => i.idempotencyKey).filter((k) => !upsertedKeys.has(k));
212
- if (missingKeys.length > 0) {
213
- const existingTasks = await repository.loadManyByQuery({
214
- namespace: this.#namespace,
215
- idempotencyKey: { $in: missingKeys },
211
+ upsertedRows = await session
212
+ .insert(taskTable)
213
+ .values(itemsWithIdempotency.map((i) => i.entity))
214
+ .onConflictDoUpdate({
215
+ target: [taskTable.namespace, taskTable.idempotencyKey],
216
+ set: this.#takeNewUpdate,
217
+ setWhere,
218
+ })
219
+ .returning();
220
+ const upsertedIds = upsertedRows.map((t) => t.id);
221
+ if (upsertedIds.length > 0) {
222
+ await session
223
+ .delete(taskDependencyTable)
224
+ .where(and(inArray(taskDependencyTable.taskId, upsertedIds), inArray(taskDependencyTable.type, [TaskDependencyType.Schedule, TaskDependencyType.Complete])));
225
+ await session
226
+ .update(taskTable)
227
+ .set({ unresolvedScheduleDependencies: 0 })
228
+ .where(inArray(taskTable.id, upsertedIds));
229
+ }
230
+ if ((options?.returnTasks == true) || hasDependencies) {
231
+ const upsertedKeysMap = new Map(upsertedRows.map((t) => [t.idempotencyKey, t]));
232
+ const missingKeysWithIndex = [];
233
+ for (const item of itemsWithIdempotency) {
234
+ const key = item.entity.idempotencyKey;
235
+ const task = upsertedKeysMap.get(key);
236
+ if (isDefined(task)) {
237
+ tasks[item.index] = task;
238
+ }
239
+ else {
240
+ missingKeysWithIndex.push({ index: item.index, key });
241
+ }
242
+ }
243
+ if (missingKeysWithIndex.length > 0) {
244
+ const existingRows = await session
245
+ .select()
246
+ .from(taskTable)
247
+ .where(and(eq(taskTable.namespace, this.#namespace), inArray(taskTable.idempotencyKey, missingKeysWithIndex.map((m) => m.key))));
248
+ const existingRowsMap = new Map(existingRows.map((t) => [t.idempotencyKey, t]));
249
+ for (const missing of missingKeysWithIndex) {
250
+ tasks[missing.index] = existingRowsMap.get(missing.key);
251
+ }
252
+ }
253
+ }
254
+ }
255
+ if (hasDependencies) {
256
+ const dependencies = [];
257
+ const processedTaskIds = new Set([...insertedRows.map((r) => r.id), ...upsertedRows.map((r) => r.id)]);
258
+ for (const [index, item] of itemsWithDistinctDependencies.entries()) {
259
+ const task = tasks[index];
260
+ if (!processedTaskIds.has(task.id)) {
261
+ continue;
262
+ }
263
+ if (isDefined(item.parentId) && (item.waitForCompletion ?? true)) {
264
+ dependencies.push({
265
+ taskId: item.parentId,
266
+ dependencyTaskId: task.id,
267
+ type: TaskDependencyType.Child,
268
+ requiredStatuses: [TaskStatus.Completed],
216
269
  });
217
- tasks.push(...existingTasks);
270
+ }
271
+ if (isDefined(item.scheduleAfter)) {
272
+ for (const dependency of item.scheduleAfter) {
273
+ let dependencyTaskId;
274
+ let requiredStatuses;
275
+ if (isNumber(dependency)) {
276
+ dependencyTaskId = tasks[dependency].id;
277
+ requiredStatuses = [TaskStatus.Completed];
278
+ }
279
+ else if (isString(dependency)) {
280
+ dependencyTaskId = dependency;
281
+ requiredStatuses = [TaskStatus.Completed];
282
+ }
283
+ else {
284
+ dependencyTaskId = isDefined(dependency.index) ? tasks[dependency.index].id : dependency.id;
285
+ requiredStatuses = dependency.requiredStatuses ?? [TaskStatus.Completed];
286
+ }
287
+ dependencies.push({
288
+ taskId: task.id,
289
+ dependencyTaskId,
290
+ type: TaskDependencyType.Schedule,
291
+ requiredStatuses: requiredStatuses,
292
+ });
293
+ }
294
+ }
295
+ if (isDefined(item.completeAfter)) {
296
+ for (const dependency of item.completeAfter) {
297
+ let dependencyTaskId;
298
+ let requiredStatuses;
299
+ if (isNumber(dependency)) {
300
+ dependencyTaskId = tasks[dependency].id;
301
+ requiredStatuses = [TaskStatus.Completed];
302
+ }
303
+ else if (isString(dependency)) {
304
+ dependencyTaskId = dependency;
305
+ requiredStatuses = [TaskStatus.Completed];
306
+ }
307
+ else {
308
+ dependencyTaskId = isDefined(dependency.index) ? tasks[dependency.index].id : dependency.id;
309
+ requiredStatuses = dependency.requiredStatuses ?? [TaskStatus.Completed];
310
+ }
311
+ dependencies.push({
312
+ taskId: task.id,
313
+ dependencyTaskId,
314
+ type: TaskDependencyType.Complete,
315
+ requiredStatuses: requiredStatuses,
316
+ });
317
+ }
318
+ }
319
+ }
320
+ if (dependencies.length > 0) {
321
+ const inserted = await session
322
+ .insert(taskDependencyTable)
323
+ .values(dependencies)
324
+ .onConflictDoNothing()
325
+ .returning();
326
+ const distinctDependencyIds = distinct(dependencies.map((d) => d.dependencyTaskId));
327
+ if (distinctDependencyIds.length > 0) {
328
+ await this.incrementCounters(inserted, { transaction });
329
+ const dependencyStatuses = await session
330
+ .select({ id: taskTable.id, status: taskTable.status })
331
+ .from(taskTable)
332
+ .where(inArray(taskTable.id, distinctDependencyIds))
333
+ .unionAll(session
334
+ .select({ id: taskArchiveTable.id, status: sql `${enumValue(TaskStatus, taskStatus, TaskStatus.Completed)}` })
335
+ .from(taskArchiveTable)
336
+ .where(inArray(taskArchiveTable.id, distinctDependencyIds)));
337
+ if (dependencyStatuses.length > 0) {
338
+ await this.resolveDependenciesMany(dependencyStatuses.map((s) => ({ id: s.id, status: s.status })), { transaction: transaction });
339
+ }
218
340
  }
219
341
  }
220
342
  }
221
- await newTransaction?.commit();
222
- this.#messageBus.publishAndForget();
223
- const anyWaiting = tasks.some((t) => t.status == TaskStatus.Waiting);
224
- if (anyWaiting) {
225
- const tagsToTrigger = distinct(tasks.filter((t) => t.status == TaskStatus.Waiting).flatMap((t) => [...t.completeAfterTags, ...t.scheduleAfterTags]));
226
- await this.triggerTagFanIn(tagsToTrigger, { transaction });
343
+ if (isDefined(newTransaction) && !newTransaction.isDone) {
344
+ await newTransaction.commit();
227
345
  }
346
+ this.notify();
228
347
  if (options?.returnTasks == true) {
229
- return tasks;
348
+ const finalTasks = await session
349
+ .select()
350
+ .from(taskTable)
351
+ .where(inArray(taskTable.id, tasks.map((t) => t.id)));
352
+ const finalTasksMap = new Map(finalTasks.map((t) => [t.id, t]));
353
+ const orderedFinalTasks = tasks.map((t) => finalTasksMap.get(t.id));
354
+ const mapped = await this.#repository.mapManyToEntity(orderedFinalTasks);
355
+ return mapped;
230
356
  }
231
357
  return undefined;
232
358
  }
@@ -240,39 +366,105 @@ let PostgresTaskQueue = class PostgresTaskQueue extends TaskQueue {
240
366
  await result_1;
241
367
  }
242
368
  }
369
+ async incrementCounters(dependencies, options) {
370
+ if (dependencies.length == 0) {
371
+ return;
372
+ }
373
+ const session = options?.transaction?.pgTransaction ?? this.#repository.session;
374
+ const decrementsToApply = new Map();
375
+ for (const dep of dependencies) {
376
+ const current = decrementsToApply.get(dep.taskId) ?? { schedule: 0, complete: 0 };
377
+ if (dep.type == TaskDependencyType.Schedule) {
378
+ current.schedule++;
379
+ }
380
+ else if (dep.type == TaskDependencyType.Complete || dep.type == TaskDependencyType.Child) {
381
+ current.complete++;
382
+ }
383
+ decrementsToApply.set(dep.taskId, current);
384
+ }
385
+ const values = [...decrementsToApply]
386
+ .toSorted(([idA], [idB]) => idA.localeCompare(idB))
387
+ .map(([taskId, d]) => sql `(${taskId}::uuid, ${d.schedule}::int, ${d.complete}::int)`);
388
+ const updates = session.$with('updates').as((qb) => qb
389
+ .select({
390
+ taskId: sql `(id)::uuid`.as('task_id'),
391
+ scheduleIncrement: sql `(schedule)::int`.as('schedule_increment'),
392
+ completeIncrement: sql `(complete)::int`.as('complete_increment'),
393
+ })
394
+ .from(sql `(VALUES ${sql.join(values, sql `, `)}) AS t(id, schedule, complete)`));
395
+ const updatedRows = await session
396
+ .with(updates)
397
+ .update(taskTable)
398
+ .set({
399
+ unresolvedScheduleDependencies: sql `${taskTable.unresolvedScheduleDependencies} + ${updates.scheduleIncrement}`,
400
+ unresolvedCompleteDependencies: sql `${taskTable.unresolvedCompleteDependencies} + ${updates.completeIncrement}`,
401
+ status: caseWhen(and(eq(taskTable.status, TaskStatus.Pending), gt(sql `${taskTable.unresolvedScheduleDependencies} + ${updates.scheduleIncrement}`, 0)), enumValue(TaskStatus, taskStatus, TaskStatus.Waiting)).else(taskTable.status),
402
+ })
403
+ .from(updates)
404
+ .where(eq(taskTable.id, updates.taskId))
405
+ .returning({ id: taskTable.id, status: taskTable.status, namespace: taskTable.namespace });
406
+ for (const row of updatedRows) {
407
+ this.notify(row.namespace);
408
+ }
409
+ }
243
410
  async has(id, options) {
244
- return await this.#repository.withOptionalTransaction(options?.transaction).hasByQuery({ namespace: this.#namespace, id });
411
+ const session = options?.transaction?.pgTransaction ?? this.#repository.session;
412
+ const [result] = await session
413
+ .select({ id: taskTable.id })
414
+ .from(taskTable)
415
+ .where(eq(taskTable.id, id))
416
+ .limit(1);
417
+ return isDefined(result);
245
418
  }
246
419
  async countByTags(tags, options) {
247
420
  const tagArray = toArray(tags);
248
421
  if (tagArray.length == 0) {
249
422
  return 0;
250
423
  }
251
- const [result] = await this.#repository.withOptionalTransaction(options?.transaction).session
424
+ const session = options?.transaction?.pgTransaction ?? this.#repository.session;
425
+ const [result] = await session
426
+ .select({ count: count() })
427
+ .from(taskTable)
428
+ .where(and(eq(taskTable.namespace, this.#namespace), arrayOverlaps(taskTable.tags, tagArray)));
429
+ return result?.count ?? 0;
430
+ }
431
+ async count(options) {
432
+ const session = options?.transaction?.pgTransaction ?? this.#repository.session;
433
+ const [result] = await session
252
434
  .select({ count: count() })
253
435
  .from(taskTable)
254
- .where(and(eq(taskTable.namespace, this.#namespace), arrayOverlaps(taskTable.tags, tagArray)))
255
- .execute();
436
+ .where(and(eq(taskTable.namespace, this.#namespace), isDefined(options?.status) ? eq(taskTable.status, options.status) : undefined));
256
437
  return result?.count ?? 0;
257
438
  }
258
439
  async getTask(id, options) {
259
- const active = await this.#repository.withOptionalTransaction(options?.transaction).tryLoadByQuery({ namespace: this.#namespace, id });
260
- if (isDefined(active)) {
261
- return active;
440
+ const session = options?.transaction?.pgTransaction ?? this.#repository.session;
441
+ const [activeRow] = await session
442
+ .select()
443
+ .from(taskTable)
444
+ .where(eq(taskTable.id, id));
445
+ if (isDefined(activeRow)) {
446
+ return await this.#repository.mapToEntity(activeRow);
447
+ }
448
+ const [archiveRow] = await session
449
+ .select()
450
+ .from(taskArchiveTable)
451
+ .where(eq(taskArchiveTable.id, id));
452
+ if (isDefined(archiveRow)) {
453
+ return await this.#archiveRepository.mapToEntity(archiveRow);
262
454
  }
263
- return await this.#archiveRepository.withOptionalTransaction(options?.transaction).tryLoadByQuery({ namespace: this.#namespace, id });
455
+ return undefined;
264
456
  }
265
457
  async getManyByTags(tags, options) {
266
458
  const tagArray = toArray(tags);
267
459
  if (tagArray.length == 0) {
268
460
  return [];
269
461
  }
270
- const rows = await this.#repository.withOptionalTransaction(options?.transaction).session
462
+ const session = options?.transaction?.pgTransaction ?? this.#repository.session;
463
+ const rows = await session
271
464
  .select()
272
465
  .from(taskTable)
273
- .where(and(eq(taskTable.namespace, this.#namespace), arrayOverlaps(taskTable.tags, tagArray)))
274
- .execute();
275
- return await this.#repository.withOptionalTransaction(options?.transaction).mapManyToEntity(rows);
466
+ .where(and(eq(taskTable.namespace, this.#namespace), arrayOverlaps(taskTable.tags, tagArray)));
467
+ return await this.#repository.mapManyToEntity(rows);
276
468
  }
277
469
  async getTree(rootId, options) {
278
470
  const ids = toArray(rootId);
@@ -280,19 +472,22 @@ let PostgresTaskQueue = class PostgresTaskQueue extends TaskQueue {
280
472
  }
281
473
  async getTreeByQuery(query, options) {
282
474
  return await this.#repository.useTransaction(options?.transaction, async (tx) => {
283
- const repositoryWithTransaction = this.#repository.withTransaction(tx);
284
- const roots = await repositoryWithTransaction.loadManyByQuery(query);
475
+ const querySql = this.#repository.convertQuery(query);
476
+ const roots = await tx.pgTransaction
477
+ .select({ id: taskTable.id })
478
+ .from(taskTable)
479
+ .where(querySql);
285
480
  if (roots.length == 0) {
286
481
  return [];
287
482
  }
288
483
  const rootIds = roots.map((t) => t.id);
289
- const rawResult = await repositoryWithTransaction.session.execute(sql `
484
+ const { rows } = await tx.pgTransaction.execute(sql `
290
485
  WITH RECURSIVE task_tree AS (
291
486
  SELECT *, 0 as depth
292
487
  FROM ${taskTable}
293
488
  WHERE ${inArray(taskTable.id, rootIds)}
294
489
 
295
- UNION
490
+ UNION ALL
296
491
 
297
492
  SELECT child.*, parent.depth + 1
298
493
  FROM ${taskTable} child
@@ -301,8 +496,7 @@ let PostgresTaskQueue = class PostgresTaskQueue extends TaskQueue {
301
496
  )
302
497
  SELECT * FROM task_tree
303
498
  `);
304
- const rows = rawResult.rows;
305
- return await repositoryWithTransaction.mapManyToEntity(rows);
499
+ return await this.#repository.mapManyToEntity(rows);
306
500
  });
307
501
  }
308
502
  async waitForTasks(ids, options) {
@@ -312,9 +506,10 @@ let PostgresTaskQueue = class PostgresTaskQueue extends TaskQueue {
312
506
  const timeout = options?.timeout ?? Infinity;
313
507
  const interval = options?.interval ?? 1000;
314
508
  const cancellationSignal = this.#cancellationSignal.optionallyInherit(options?.cancellationSignal);
315
- const continue$ = merge(this.#messageBus.allMessages$.pipe(throttleTime(50, undefined, { leading: true, trailing: true })), cancellationSignal);
509
+ const waitStatuses = options?.statuses ?? finalizedStatuses;
510
+ const messageBus$ = this.#messageBus.allMessages$.pipe(filter((namespace) => namespace == this.#namespace), throttleTime(500, undefined, { leading: true, trailing: true }));
511
+ const continue$ = merge(messageBus$, cancellationSignal);
316
512
  const timer = Timer.startNew();
317
- const finalizedStatuses = [TaskStatus.Completed, TaskStatus.Cancelled, TaskStatus.Dead];
318
513
  while (true) {
319
514
  if (cancellationSignal.isSet) {
320
515
  return { cancelled: true };
@@ -322,11 +517,12 @@ let PostgresTaskQueue = class PostgresTaskQueue extends TaskQueue {
322
517
  if (timer.milliseconds > timeout) {
323
518
  throw new TimeoutError('Timeout while waiting for tasks to complete');
324
519
  }
325
- const hasNonFinalized = await this.#repository.hasByQuery({
326
- id: { $in: ids },
327
- status: { $nin: finalizedStatuses },
328
- });
329
- if (!hasNonFinalized) {
520
+ const [remaining] = await this.#repository.session
521
+ .select({ id: taskTable.id })
522
+ .from(taskTable)
523
+ .where(and(inArray(taskTable.id, ids), notInArray(taskTable.status, waitStatuses)))
524
+ .limit(1);
525
+ if (isUndefined(remaining)) {
330
526
  return { cancelled: false };
331
527
  }
332
528
  const remainingTimeout = timeout - timer.milliseconds;
@@ -344,13 +540,18 @@ let PostgresTaskQueue = class PostgresTaskQueue extends TaskQueue {
344
540
  if (treeIds.length == 0) {
345
541
  return;
346
542
  }
347
- await this.#repository.withTransaction(tx).updateMany(treeIds, {
543
+ const cancelledRows = await tx.pgTransaction
544
+ .update(taskTable)
545
+ .set({
348
546
  status: TaskStatus.Cancelled,
349
547
  token: null,
350
548
  completeTimestamp: TRANSACTION_TIMESTAMP,
351
- });
352
- const tags = tree.flatMap((t) => t.tags);
353
- await this.triggerTagFanIn(tags, { transaction: tx });
549
+ })
550
+ .where(and(inArray(taskTable.id, treeIds), notInArray(taskTable.status, terminalStatuses)))
551
+ .returning({ id: taskTable.id, namespace: taskTable.namespace });
552
+ if (cancelledRows.length > 0) {
553
+ await this.resolveDependenciesMany(cancelledRows.map((row) => ({ id: row.id, status: TaskStatus.Cancelled, namespace: row.namespace })), { transaction: tx });
554
+ }
354
555
  });
355
556
  }
356
557
  async cancelManyByTags(tags, options) {
@@ -362,17 +563,35 @@ let PostgresTaskQueue = class PostgresTaskQueue extends TaskQueue {
362
563
  }
363
564
  async clear(options) {
364
565
  await this.#repository.useTransaction(options?.transaction, async (tx) => {
365
- const repository = this.#repository.withTransaction(tx);
366
- const parentIds = repository.session
367
- .select({ id: taskTable.id })
566
+ const nonFinalizedTasks = await tx.pgTransaction
567
+ .select({ id: taskTable.id, namespace: taskTable.namespace })
368
568
  .from(taskTable)
369
- .where(eq(taskTable.namespace, this.#namespace));
370
- await repository.session
569
+ .where(and(eq(taskTable.namespace, this.#namespace), notInArray(taskTable.status, finalizedStatuses)))
570
+ .for('update');
571
+ if (nonFinalizedTasks.length > 0) {
572
+ const ids = nonFinalizedTasks.map((t) => t.id);
573
+ await tx.pgTransaction
574
+ .update(taskTable)
575
+ .set({
576
+ status: TaskStatus.Cancelled,
577
+ token: null,
578
+ completeTimestamp: TRANSACTION_TIMESTAMP,
579
+ })
580
+ .where(inArray(taskTable.id, ids));
581
+ await this.resolveDependenciesMany(nonFinalizedTasks.map((t) => ({ id: t.id, status: TaskStatus.Cancelled, namespace: t.namespace })), { transaction: tx });
582
+ }
583
+ // Break internal parent-child links to allow deleting tasks within the same namespace.
584
+ // Cross-namespace links will still cause a foreign key violation as requested.
585
+ await tx.pgTransaction
371
586
  .update(taskTable)
372
587
  .set({ parentId: null })
373
- .where(inArray(taskTable.parentId, parentIds))
374
- .execute();
375
- await repository.hardDeleteManyByQuery({ namespace: this.#namespace });
588
+ .where(and(eq(taskTable.namespace, this.#namespace), sqlIsNotNull(taskTable.parentId), inArray(taskTable.parentId, tx.pgTransaction
589
+ .select({ id: taskTable.id })
590
+ .from(taskTable)
591
+ .where(eq(taskTable.namespace, this.#namespace)))));
592
+ await tx.pgTransaction
593
+ .delete(taskTable)
594
+ .where(eq(taskTable.namespace, this.#namespace));
376
595
  });
377
596
  }
378
597
  async dequeue(options) {
@@ -386,52 +605,50 @@ let PostgresTaskQueue = class PostgresTaskQueue extends TaskQueue {
386
605
  // 1. Flow Control
387
606
  const forceDequeue = options?.forceDequeue ?? false;
388
607
  let circuitBreakerResult;
608
+ let rateLimitAcquired = 0;
389
609
  if (!forceDequeue) {
390
610
  circuitBreakerResult = await this.#circuitBreaker.check();
391
611
  if (!circuitBreakerResult.allowed) {
392
612
  return [];
393
613
  }
394
- const rateLimitAllowed = await this.#rateLimiter.tryAcquire(this.#namespace, count);
395
- if (!rateLimitAllowed) {
614
+ for (let i = count; i > 0; i--) {
615
+ if (await this.#rateLimiter.tryAcquire(this.#namespace, i)) {
616
+ rateLimitAcquired = i;
617
+ break;
618
+ }
619
+ }
620
+ if (rateLimitAcquired == 0) {
396
621
  return [];
397
622
  }
398
623
  }
399
624
  const tasks = await this.#repository.useTransaction(options?.transaction, async (tx) => {
400
- const repository = this.#repository.withTransaction(tx);
401
- let effectiveCount = count;
625
+ let effectiveCount = forceDequeue ? count : rateLimitAcquired;
402
626
  if (!forceDequeue && circuitBreakerResult?.state == CircuitBreakerState.HalfOpen) {
403
627
  if (circuitBreakerResult.isProbe != true) {
404
- const runningCount = await repository.countByQuery({ namespace: this.#namespace, status: TaskStatus.Running });
405
- if (runningCount > 0) {
628
+ const [runningResult] = await tx.pgTransaction
629
+ .select({ count: sqlCount() })
630
+ .from(taskTable)
631
+ .where(and(eq(taskTable.namespace, this.#namespace), eq(taskTable.status, TaskStatus.Running)));
632
+ if ((runningResult?.count ?? 0) > 0) {
406
633
  return [];
407
634
  }
408
635
  }
409
636
  effectiveCount = 1;
410
637
  }
411
- // 2. Check Global Concurrency
412
- if (!forceDequeue && isNotNull(this.globalConcurrency)) {
413
- // WARN: This is a check-then-act race condition.
414
- // A distributed lock or an atomic update strategy is needed for strict enforcement.
415
- const runningCount = await repository.countByQuery({ namespace: this.#namespace, status: TaskStatus.Running });
416
- if (runningCount >= this.globalConcurrency) {
417
- return [];
418
- }
419
- effectiveCount = Math.min(effectiveCount, this.globalConcurrency - runningCount);
420
- }
421
638
  /*
422
639
  * Materialization required for LIMIT clause
423
640
  * https://stackoverflow.com/questions/73966670/select-for-update-subquery-not-respecting-limit-clause-under-load
424
641
  * https://dba.stackexchange.com/questions/69471/postgres-update-limit-1
425
642
  */
426
- const selection = repository.session.$with('selection').as((qb) => qb
643
+ const selection = tx.pgTransaction.$with('selection').as((qb) => qb
427
644
  .select({ id: taskTable.id })
428
645
  .from(taskTable)
429
- .where(and(eq(taskTable.namespace, this.#namespace), lte(taskTable.scheduleTimestamp, TRANSACTION_TIMESTAMP), eq(taskTable.status, TaskStatus.Pending), or(sqlIsNull(taskTable.timeToLive), lt(TRANSACTION_TIMESTAMP, taskTable.timeToLive)), isDefined(options?.types) ? inArray(taskTable.type, options.types) : undefined, sql `pg_sleep(0) IS NOT NULL` // Materialization hack until drizzle implements https://github.com/drizzle-team/drizzle-orm/issues/2318
646
+ .where(and(eq(taskTable.namespace, this.#namespace), lte(taskTable.scheduleTimestamp, TRANSACTION_TIMESTAMP), inArray(taskTable.status, queueableStatuses), or(sqlIsNull(taskTable.timeToLive), lt(TRANSACTION_TIMESTAMP, taskTable.timeToLive)), isDefined(options?.types) ? inArray(taskTable.type, options.types) : undefined, sql `pg_sleep(0) IS NOT NULL` // Materialization hack until drizzle implements https://github.com/drizzle-team/drizzle-orm/issues/2318
430
647
  ))
431
648
  .orderBy(asc(taskTable.priority), asc(taskTable.scheduleTimestamp))
432
649
  .limit(effectiveCount)
433
650
  .for('update', { skipLocked: true }));
434
- const rows = await repository.session
651
+ const rows = await tx.pgTransaction
435
652
  .with(selection)
436
653
  .update(taskTable)
437
654
  .set({
@@ -443,13 +660,13 @@ let PostgresTaskQueue = class PostgresTaskQueue extends TaskQueue {
443
660
  // If it was RUNNING (Zombie), previous try failed -> increment tries.
444
661
  tries: sql `${taskTable.tries} + 1`,
445
662
  })
446
- .where(inArray(taskTable.id, repository.session.select().from(selection)))
663
+ .where(inArray(taskTable.id, tx.pgTransaction.select().from(selection)))
447
664
  .returning();
448
665
  return await this.#repository.mapManyToEntity(rows);
449
666
  });
450
667
  // 3. Compensation
451
- if (!forceDequeue && (tasks.length < count)) {
452
- const shortfall = count - tasks.length;
668
+ if (!forceDequeue && (tasks.length < rateLimitAcquired)) {
669
+ const shortfall = rateLimitAcquired - tasks.length;
453
670
  await this.#rateLimiter.refund(this.#namespace, shortfall);
454
671
  }
455
672
  return tasks;
@@ -458,35 +675,36 @@ let PostgresTaskQueue = class PostgresTaskQueue extends TaskQueue {
458
675
  await this.rescheduleMany([id], timestamp, options);
459
676
  }
460
677
  async rescheduleMany(ids, timestamp, options) {
461
- await this.#repository.withOptionalTransaction(options?.transaction).updateMany(ids, {
462
- status: TaskStatus.Pending,
678
+ const session = options?.transaction?.pgTransaction ?? this.#repository.session;
679
+ await session
680
+ .update(taskTable)
681
+ .set({
682
+ status: caseWhen(gt(taskTable.unresolvedScheduleDependencies, 0), enumValue(TaskStatus, taskStatus, TaskStatus.Waiting)).else(enumValue(TaskStatus, taskStatus, TaskStatus.Pending)),
463
683
  token: null,
464
684
  scheduleTimestamp: timestamp,
465
685
  visibilityDeadline: null,
466
- tries: sql `CASE
467
- WHEN ${taskTable.status} = ${TaskStatus.Running} THEN GREATEST(0, ${taskTable.tries} - 1)
468
- ELSE ${taskTable.tries}
469
- END`,
470
- });
686
+ tries: caseWhen(eq(taskTable.status, enumValue(TaskStatus, taskStatus, TaskStatus.Running)), greatest(0, sql `${taskTable.tries} - 1`)).else(taskTable.tries),
687
+ })
688
+ .where(and(inArray(taskTable.id, ids), notInArray(taskTable.status, terminalStatuses)));
471
689
  }
472
690
  async rescheduleManyByTags(tags, timestamp, options) {
473
- await this.#repository.withOptionalTransaction(options?.transaction).updateManyByQuery(and(eq(taskTable.namespace, this.#namespace), arrayOverlaps(taskTable.tags, toArray(tags))), {
474
- status: TaskStatus.Pending,
691
+ const session = options?.transaction?.pgTransaction ?? this.#repository.session;
692
+ await session
693
+ .update(taskTable)
694
+ .set({
695
+ status: caseWhen(gt(taskTable.unresolvedScheduleDependencies, 0), enumValue(TaskStatus, taskStatus, TaskStatus.Waiting)).else(enumValue(TaskStatus, taskStatus, TaskStatus.Pending)),
475
696
  token: null,
476
697
  scheduleTimestamp: timestamp,
477
698
  visibilityDeadline: null,
478
- tries: sql `CASE
479
- WHEN ${taskTable.status} = ${TaskStatus.Running} THEN GREATEST(0, ${taskTable.tries} - 1)
480
- ELSE ${taskTable.tries}
481
- END`,
482
- });
699
+ tries: caseWhen(eq(taskTable.status, enumValue(TaskStatus, taskStatus, TaskStatus.Running)), greatest(0, sql `${taskTable.tries} - 1`)).else(taskTable.tries),
700
+ })
701
+ .where(and(eq(taskTable.namespace, this.#namespace), arrayOverlaps(taskTable.tags, toArray(tags)), notInArray(taskTable.status, terminalStatuses)));
483
702
  }
484
703
  async touch(task, options) {
485
704
  if (isNull(task.token)) {
486
705
  return undefined;
487
706
  }
488
707
  return await this.#repository.useTransaction(options?.transaction, async (tx) => {
489
- const repository = this.#repository.withTransaction(tx);
490
708
  const update = {
491
709
  visibilityDeadline: sql `${TRANSACTION_TIMESTAMP} + ${interval(this.visibilityTimeout, 'milliseconds')}`,
492
710
  };
@@ -496,34 +714,44 @@ let PostgresTaskQueue = class PostgresTaskQueue extends TaskQueue {
496
714
  if (isDefined(options?.state)) {
497
715
  update.state = options.state;
498
716
  }
499
- const result = await repository.tryUpdateByQuery({
500
- namespace: this.#namespace,
501
- id: task.id,
502
- status: TaskStatus.Running,
503
- token: task.token,
504
- startTimestamp: { $gt: sql `${TRANSACTION_TIMESTAMP} - ${interval(this.maxExecutionTime, 'milliseconds')}` },
505
- }, update);
506
- // TODO: reduce required DB roundtrips
507
- if (isUndefined(result)) {
508
- const existing = await repository.tryLoadByQuery({ id: task.id, namespace: this.#namespace, status: TaskStatus.Running, token: task.token });
509
- if (isDefined(existing) && isNotNull(existing.startTimestamp) && (currentTimestamp() - existing.startTimestamp) > this.maxExecutionTime) {
510
- await this.fail(task, { message: 'Hard Execution Timeout' }, { fatal: true, transaction: tx });
511
- }
717
+ const [updatedRow] = await tx.pgTransaction
718
+ .update(taskTable)
719
+ .set(update)
720
+ .where(and(eq(taskTable.id, task.id), eq(taskTable.status, TaskStatus.Running), isNull(task.token) ? sqlIsNull(taskTable.token) : eq(taskTable.token, task.token), gt(taskTable.startTimestamp, sql `${TRANSACTION_TIMESTAMP} - ${interval(this.maxExecutionTime, 'milliseconds')}`)))
721
+ .returning();
722
+ if (isDefined(updatedRow)) {
723
+ return await this.#repository.mapToEntity(updatedRow);
724
+ }
725
+ const [existingRow] = await tx.pgTransaction
726
+ .select({ startTimestamp: taskTable.startTimestamp })
727
+ .from(taskTable)
728
+ .where(and(eq(taskTable.id, task.id), eq(taskTable.status, TaskStatus.Running), isNull(task.token) ? sqlIsNull(taskTable.token) : eq(taskTable.token, task.token)));
729
+ if (isDefined(existingRow) && isNotNull(existingRow.startTimestamp) && (currentTimestamp() - existingRow.startTimestamp) > this.maxExecutionTime) {
730
+ await tx.pgTransaction
731
+ .update(taskTable)
732
+ .set({
733
+ status: TaskStatus.TimedOut,
734
+ completeTimestamp: TRANSACTION_TIMESTAMP,
735
+ error: { code: 'MaxTimeExceeded', message: 'Hard Execution Timeout' },
736
+ })
737
+ .where(eq(taskTable.id, task.id));
738
+ await this.resolveDependenciesMany([{ id: task.id, status: TaskStatus.TimedOut, namespace: task.namespace }], { transaction: tx });
739
+ this.notify();
512
740
  }
513
- return result;
741
+ return undefined;
514
742
  });
515
743
  }
516
744
  async touchMany(tasks, options) {
517
745
  if (tasks.length == 0) {
518
746
  return [];
519
747
  }
520
- const repository = this.#repository.withOptionalTransaction(options?.transaction);
748
+ const session = options?.transaction?.pgTransaction ?? this.#repository.session;
521
749
  const rows = tasks.map((t, i) => {
522
750
  const progress = options?.progresses?.[i] ?? null;
523
751
  const state = options?.states?.[i] ?? null;
524
752
  return sql `(${t.id}::uuid, ${t.token}::uuid, ${progress}::numeric, ${state}::jsonb)`;
525
753
  });
526
- const updates = repository.session.$with('updates').as((qb) => qb
754
+ const updates = session.$with('updates').as((qb) => qb
527
755
  .select({
528
756
  updateId: sql `(id)::uuid`.as('update_id'),
529
757
  updateToken: sql `(token)::uuid`.as('update_token'),
@@ -531,7 +759,7 @@ let PostgresTaskQueue = class PostgresTaskQueue extends TaskQueue {
531
759
  updateState: sql `(state)::jsonb`.as('update_state'),
532
760
  })
533
761
  .from(sql `(VALUES ${sql.join(rows, sql `, `)}) AS t(id, token, progress, state)`));
534
- const updated = repository.session.$with('updated').as(() => repository.session
762
+ const updated = session.$with('updated').as(() => session
535
763
  .update(taskTable)
536
764
  .set({
537
765
  visibilityDeadline: sql `${TRANSACTION_TIMESTAMP} + ${interval(this.visibilityTimeout, 'milliseconds')}`,
@@ -539,34 +767,44 @@ let PostgresTaskQueue = class PostgresTaskQueue extends TaskQueue {
539
767
  state: coalesce(updates.updateState, taskTable.state),
540
768
  })
541
769
  .from(updates)
542
- .where(and(eq(taskTable.id, updates.updateId), eq(taskTable.namespace, this.#namespace), eq(taskTable.token, updates.updateToken), eq(taskTable.status, TaskStatus.Running), gt(taskTable.startTimestamp, sql `${TRANSACTION_TIMESTAMP} - ${interval(this.maxExecutionTime, 'milliseconds')}`)))
770
+ .where(and(eq(taskTable.id, updates.updateId), sql `${taskTable.token} IS NOT DISTINCT FROM ${updates.updateToken}`, eq(taskTable.status, TaskStatus.Running), gt(taskTable.startTimestamp, sql `${TRANSACTION_TIMESTAMP} - ${interval(this.maxExecutionTime, 'milliseconds')}`)))
543
771
  .returning({ id: taskTable.id }));
544
- const result = await repository.session
772
+ const result = await session
545
773
  .with(updates, updated)
546
774
  .select({ id: updated.id })
547
- .from(updated)
548
- .execute();
775
+ .from(updated);
549
776
  return result.map((r) => r.id);
550
777
  }
551
778
  async complete(task, options) {
552
779
  await this.#repository.useTransaction(options?.transaction, async (tx) => {
553
- const repository = this.#repository.withTransaction(tx);
554
- const updatedTask = await repository.tryUpdateByQuery({
555
- id: task.id,
556
- token: task.token,
557
- }, {
558
- status: TaskStatus.Completed,
780
+ const [freshTask] = await tx.pgTransaction
781
+ .select({ unresolvedCompleteDependencies: taskTable.unresolvedCompleteDependencies })
782
+ .from(taskTable)
783
+ .where(eq(taskTable.id, task.id))
784
+ .for('update');
785
+ if (isUndefined(freshTask)) {
786
+ return;
787
+ }
788
+ const hasActiveChildren = freshTask.unresolvedCompleteDependencies > 0;
789
+ const nextStatus = hasActiveChildren ? TaskStatus.WaitingChildren : TaskStatus.Completed;
790
+ const [updatedTask] = await tx.pgTransaction.update(taskTable)
791
+ .set({
792
+ status: nextStatus,
559
793
  token: null,
560
794
  result: options?.result,
561
- progress: 1,
562
- completeTimestamp: TRANSACTION_TIMESTAMP,
795
+ progress: hasActiveChildren ? task.progress : 1,
796
+ completeTimestamp: (nextStatus == TaskStatus.Completed) ? TRANSACTION_TIMESTAMP : null,
563
797
  visibilityDeadline: null,
564
- });
798
+ })
799
+ .where(and(eq(taskTable.id, task.id), isNull(task.token) ? sqlIsNull(taskTable.token) : eq(taskTable.token, task.token)))
800
+ .returning({ id: taskTable.id });
565
801
  if (isUndefined(updatedTask)) {
566
802
  return;
567
803
  }
568
- await this.#circuitBreaker.recordSuccess();
569
- await this.triggerTagFanIn(task.tags, { transaction: tx });
804
+ if (nextStatus == TaskStatus.Completed) {
805
+ await this.#circuitBreaker.recordSuccess();
806
+ }
807
+ await this.resolveDependencies(task.id, nextStatus, { namespace: task.namespace, transaction: tx });
570
808
  });
571
809
  }
572
810
  async completeMany(tasks, options) {
@@ -574,57 +812,54 @@ let PostgresTaskQueue = class PostgresTaskQueue extends TaskQueue {
574
812
  return;
575
813
  }
576
814
  await this.#repository.useTransaction(options?.transaction, async (tx) => {
577
- const repository = this.#repository.withTransaction(tx);
578
815
  const rows = tasks.map((t, i) => {
579
816
  const result = options?.results?.[i] ?? null;
580
- return sql `(${t.id}::uuid, ${t.token}::uuid, ${result}::jsonb)`;
817
+ return sql `(${t.id}::uuid, ${t.token}::uuid, ${result}::jsonb, ${t.progress}::numeric)`;
581
818
  });
582
- const updates = repository.session.$with('updates').as((qb) => qb
819
+ const updates = tx.pgTransaction.$with('updates').as((qb) => qb
583
820
  .select({
584
821
  updateId: sql `(id)::uuid`.as('update_id'),
585
822
  updateToken: sql `(token)::uuid`.as('update_token'),
586
823
  updateResult: sql `(result)::jsonb`.as('update_result'),
824
+ updateProgress: sql `(progress)::numeric`.as('update_progress'),
587
825
  })
588
- .from(sql `(VALUES ${sql.join(rows, sql `, `)}) AS t(id, token, result)`));
589
- const updated = repository.session.$with('updated').as(() => repository.session
826
+ .from(sql `(VALUES ${sql.join(rows, sql `, `)}) AS t(id, token, result, progress)`));
827
+ const updated = tx.pgTransaction.$with('updated').as(() => tx.pgTransaction
590
828
  .update(taskTable)
591
829
  .set({
592
- status: TaskStatus.Completed,
830
+ status: caseWhen(gt(taskTable.unresolvedCompleteDependencies, 0), enumValue(TaskStatus, taskStatus, TaskStatus.WaitingChildren)).else(enumValue(TaskStatus, taskStatus, TaskStatus.Completed)),
593
831
  token: null,
594
832
  result: updates.updateResult,
595
- progress: 1,
596
- completeTimestamp: TRANSACTION_TIMESTAMP,
833
+ progress: caseWhen(gt(taskTable.unresolvedCompleteDependencies, 0), updates.updateProgress).else(1),
834
+ completeTimestamp: caseWhen(gt(taskTable.unresolvedCompleteDependencies, 0), taskTable.completeTimestamp).else(TRANSACTION_TIMESTAMP),
597
835
  visibilityDeadline: null,
598
836
  })
599
837
  .from(updates)
600
- .where(and(eq(taskTable.id, updates.updateId), eq(taskTable.token, updates.updateToken), eq(taskTable.namespace, this.#namespace)))
601
- .returning({ id: taskTable.id, tags: taskTable.tags }));
602
- const updatedRows = await repository.session
838
+ .where(and(eq(taskTable.id, updates.updateId), sql `${taskTable.token} IS NOT DISTINCT FROM ${updates.updateToken}`))
839
+ .returning({ id: taskTable.id, status: taskTable.status }));
840
+ const updatedRows = await tx.pgTransaction
603
841
  .with(updates, updated)
604
- .select({ id: updated.id, tags: updated.tags })
605
- .from(updated)
606
- .execute();
842
+ .select({ id: updated.id, status: updated.status })
843
+ .from(updated);
607
844
  if (updatedRows.length > 0) {
608
- await this.#circuitBreaker.recordSuccess();
609
- const tags = updatedRows.flatMap((t) => t.tags);
610
- await this.triggerTagFanIn(tags, { transaction: tx });
845
+ if (updatedRows.some((r) => r.status == TaskStatus.Completed)) {
846
+ await this.#circuitBreaker.recordSuccess();
847
+ }
848
+ await this.resolveDependenciesMany(updatedRows.map((r) => ({ id: r.id, status: r.status })), { transaction: tx });
611
849
  }
612
850
  });
613
851
  }
614
852
  async fail(task, error, options) {
615
853
  const isRetryable = (options?.fatal != true) && (task.tries < this.maxTries);
616
- const nextStatus = isRetryable ? TaskStatus.Pending : TaskStatus.Dead;
854
+ const nextStatus = isRetryable ? TaskStatus.Retrying : TaskStatus.Dead;
617
855
  const delay = isRetryable
618
856
  ? Math.min(this.retryDelayMaximum, this.retryDelayMinimum * (this.retryDelayGrowth ** task.tries))
619
857
  : 0;
620
858
  const nextSchedule = currentTimestamp() + delay;
621
859
  await this.#repository.useTransaction(options?.transaction, async (tx) => {
622
- const updatedTask = await this.#repository.withTransaction(tx).tryUpdateByQuery({
623
- namespace: this.#namespace,
624
- id: task.id,
625
- token: task.token,
626
- tries: task.tries,
627
- }, {
860
+ const [updatedRow] = await tx.pgTransaction
861
+ .update(taskTable)
862
+ .set({
628
863
  status: nextStatus,
629
864
  token: null,
630
865
  error: serializeError(error),
@@ -632,28 +867,14 @@ let PostgresTaskQueue = class PostgresTaskQueue extends TaskQueue {
632
867
  scheduleTimestamp: nextSchedule,
633
868
  startTimestamp: null,
634
869
  completeTimestamp: (nextStatus == TaskStatus.Dead) ? TRANSACTION_TIMESTAMP : null,
635
- });
636
- if (isUndefined(updatedTask)) {
870
+ })
871
+ .where(and(eq(taskTable.id, task.id), isNull(task.token) ? sqlIsNull(taskTable.token) : eq(taskTable.token, task.token), eq(taskTable.tries, task.tries)))
872
+ .returning();
873
+ if (isUndefined(updatedRow)) {
637
874
  return;
638
875
  }
639
876
  await this.#circuitBreaker.recordFailure();
640
- if (nextStatus == TaskStatus.Dead) {
641
- await this.triggerTagFanIn(task.tags, { transaction: tx });
642
- }
643
- });
644
- }
645
- async triggerTagFanIn(tags, options) {
646
- if (tags.length == 0) {
647
- return;
648
- }
649
- const distinctSortedTags = distinct(tags).toSorted();
650
- const hash = await digest('SHA-256', distinctSortedTags.join(',')).toHex();
651
- const idempotencyKey = `Sys:FanIn:${hash}`;
652
- await this.#internalThis.enqueue('[SystemWorker]:FanIn', { targetTags: distinctSortedTags }, {
653
- priority: 0,
654
- idempotencyKey,
655
- replace: true,
656
- transaction: options?.transaction,
877
+ await this.resolveDependencies(task.id, nextStatus, { namespace: task.namespace, transaction: tx });
657
878
  });
658
879
  }
659
880
  async failMany(tasks, errors, options) {
@@ -661,11 +882,10 @@ let PostgresTaskQueue = class PostgresTaskQueue extends TaskQueue {
661
882
  return;
662
883
  }
663
884
  await this.#repository.useTransaction(options?.transaction, async (tx) => {
664
- const repository = this.#repository.withTransaction(tx);
665
885
  const rows = tasks.map((task, index) => {
666
886
  const error = errors[index];
667
887
  const isRetryable = (task.tries < this.maxTries);
668
- const nextStatus = isRetryable ? TaskStatus.Pending : TaskStatus.Dead;
888
+ const nextStatus = isRetryable ? TaskStatus.Retrying : TaskStatus.Dead;
669
889
  const delay = isRetryable
670
890
  ? Math.min(this.retryDelayMaximum, this.retryDelayMinimum * (this.retryDelayGrowth ** task.tries))
671
891
  : 0;
@@ -673,7 +893,7 @@ let PostgresTaskQueue = class PostgresTaskQueue extends TaskQueue {
673
893
  const completeTimestamp = (nextStatus == TaskStatus.Dead) ? new Date() : null;
674
894
  return sql `(${task.id}::uuid, ${task.token}::uuid, ${task.tries}::int, ${nextStatus}::text, ${serializeError(error)}::jsonb, ${nextSchedule}::timestamptz, ${completeTimestamp}::timestamptz)`;
675
895
  });
676
- const updates = repository.session.$with('updates').as((qb) => qb
896
+ const updates = tx.pgTransaction.$with('updates').as((qb) => qb
677
897
  .select({
678
898
  updateId: sql `(id)::uuid`.as('update_id'),
679
899
  updateToken: sql `(token)::uuid`.as('update_token'),
@@ -684,7 +904,7 @@ let PostgresTaskQueue = class PostgresTaskQueue extends TaskQueue {
684
904
  updateComplete: sql `(complete_timestamp)::timestamptz`.as('update_complete'),
685
905
  })
686
906
  .from(sql `(VALUES ${sql.join(rows, sql `, `)}) AS t(id, token, tries, status, error, schedule_timestamp, complete_timestamp)`));
687
- const updated = repository.session.$with('updated').as(() => repository.session
907
+ const updated = tx.pgTransaction.$with('updated').as(() => tx.pgTransaction
688
908
  .update(taskTable)
689
909
  .set({
690
910
  status: sql `${updates.updateStatus}::${taskStatus}`,
@@ -696,304 +916,369 @@ let PostgresTaskQueue = class PostgresTaskQueue extends TaskQueue {
696
916
  completeTimestamp: sql `${updates.updateComplete}`,
697
917
  })
698
918
  .from(updates)
699
- .where(and(eq(taskTable.namespace, this.#namespace), eq(taskTable.id, updates.updateId), eq(taskTable.token, updates.updateToken), eq(taskTable.tries, updates.updateTries)))
700
- .returning({ id: taskTable.id, status: taskTable.status, tags: taskTable.tags }));
701
- const result = await repository.session
919
+ .where(and(eq(taskTable.id, updates.updateId), sql `${taskTable.token} IS NOT DISTINCT FROM ${updates.updateToken}`, eq(taskTable.tries, updates.updateTries)))
920
+ .returning({ id: taskTable.id, status: taskTable.status }));
921
+ const result = await tx.pgTransaction
702
922
  .with(updates, updated)
703
- .select()
704
- .from(updated)
705
- .execute();
923
+ .select({ id: updated.id, status: updated.status })
924
+ .from(updated);
706
925
  if (result.length > 0) {
707
- await Promise.all(createArray(result.length, async () => await this.#circuitBreaker.recordFailure()));
708
- const deadTags = result
709
- .filter((r) => r.status == TaskStatus.Dead)
710
- .flatMap((r) => r.tags);
711
- await this.triggerTagFanIn(deadTags, { transaction: tx });
926
+ await this.#circuitBreaker.recordFailures(result.length);
927
+ await this.resolveDependenciesMany(result.map((r) => ({ id: r.id, status: r.status })), { transaction: tx });
712
928
  }
713
929
  });
714
930
  }
715
- async resolveDependencies(targetTags) {
716
- await this.#repository.transaction(async (tx) => {
717
- const repository = this.#repository.withTransaction(tx);
718
- // Identify Dependents
719
- const waiters = await repository.session
720
- .select()
721
- .from(taskTable)
722
- .where(and(eq(taskTable.namespace, this.#namespace), eq(taskTable.status, TaskStatus.Waiting), or(arrayOverlaps(taskTable.completeAfterTags, targetTags), arrayOverlaps(taskTable.scheduleAfterTags, targetTags))))
723
- .for('update', { skipLocked: true })
724
- .execute();
725
- const entities = await repository.mapManyToEntity(waiters);
726
- if (entities.length == 0) {
727
- return;
728
- }
729
- const allRequiredTags = distinct(entities.flatMap((t) => [...t.completeAfterTags, ...t.scheduleAfterTags]));
730
- const statsRows = await repository.session
931
+ async resolveDependencies(id, status, options) {
932
+ await this.resolveDependenciesMany([{ id, status, namespace: options?.namespace }], options);
933
+ }
934
+ async resolveDependenciesMany(tasks, options) {
935
+ this.#logger.debug(`Resolving dependencies for ${tasks.length} tasks`);
936
+ if (tasks.length == 0) {
937
+ return;
938
+ }
939
+ const taskStatusMap = new Map(tasks.map((t) => [t.id, t.status]));
940
+ const notifiedNamespaces = new Set();
941
+ await this.#repository.useTransaction(options?.transaction, async (tx) => {
942
+ const taskIds = tasks.map((t) => t.id);
943
+ const dependents = await tx.pgTransaction
731
944
  .select({
732
- tag: unnest(taskTable.tags).as('tag'),
733
- status: taskTable.status,
734
- count: count().as('count'),
945
+ taskId: taskDependencyTable.taskId,
946
+ dependencyTaskId: taskDependencyTable.dependencyTaskId,
947
+ type: taskDependencyTable.type,
948
+ requiredStatuses: taskDependencyTable.requiredStatuses,
735
949
  })
736
- .from(taskTable)
737
- .where(and(eq(taskTable.namespace, this.#namespace), arrayOverlaps(taskTable.tags, allRequiredTags)))
738
- .groupBy(unnest(taskTable.tags), taskTable.status);
739
- const tagStats = new Map();
740
- for (const row of statsRows) {
741
- if (!tagStats.has(row.tag)) {
742
- tagStats.set(row.tag, new Map());
743
- }
744
- tagStats.get(row.tag).set(row.status, row.count);
950
+ .from(taskDependencyTable)
951
+ .where(inArray(taskDependencyTable.dependencyTaskId, taskIds));
952
+ if (dependents.length == 0) {
953
+ return;
745
954
  }
746
- const getCount = (tags, Statuses) => {
747
- let sum = 0;
748
- for (const tag of tags) {
749
- const stats = tagStats.get(tag);
750
- if (isDefined(stats)) {
751
- for (const status of Statuses) {
752
- sum += stats.get(status) ?? 0;
753
- }
754
- }
755
- }
756
- return sum;
757
- };
758
- const idsToFail = [];
759
- const idsToSchedule = [];
760
- const idsToComplete = [];
761
- const fanInTags = [];
762
- for (const waiter of entities) {
763
- const requiredTags = distinct([...waiter.completeAfterTags, ...waiter.scheduleAfterTags]);
764
- let isReady = false;
765
- let shouldFail = false;
766
- // 1. Check Fail Fast
767
- if (waiter.failFast) {
768
- if (getCount(requiredTags, [TaskStatus.Dead, TaskStatus.Cancelled]) > 0) {
769
- shouldFail = true;
955
+ const resolvedEdges = [];
956
+ const failFastTaskIds = new Set();
957
+ for (const dep of dependents) {
958
+ const status = taskStatusMap.get(dep.dependencyTaskId);
959
+ const isMatched = dep.requiredStatuses.includes(status);
960
+ const isTerminal = terminalStatuses.includes(status);
961
+ if (isMatched || isTerminal) {
962
+ resolvedEdges.push(dep);
963
+ if (!isMatched) {
964
+ failFastTaskIds.add(dep.taskId);
770
965
  }
771
966
  }
772
- // 2. Check Trigger Conditions
773
- if (!shouldFail) {
774
- if (waiter.dependencyJoinMode == DependencyJoinMode.Or) {
775
- if (getCount(requiredTags, waiter.dependencyTriggerStatuses) > 0) {
776
- isReady = true;
777
- }
778
- }
779
- else if (waiter.dependencyJoinMode == DependencyJoinMode.And) {
780
- isReady = requiredTags.every((tag) => {
781
- const stats = tagStats.get(tag);
782
- if (isUndefined(stats)) {
783
- return false;
784
- }
785
- const activeCount = (stats.get(TaskStatus.Pending) ?? 0)
786
- + (stats.get(TaskStatus.Running) ?? 0)
787
- + (stats.get(TaskStatus.Waiting) ?? 0);
788
- return activeCount == 0;
789
- });
790
- }
967
+ }
968
+ if (resolvedEdges.length == 0) {
969
+ return;
970
+ }
971
+ const sortedResolvedEdges = resolvedEdges.toSorted((a, b) => {
972
+ const idCompare = a.taskId.localeCompare(b.taskId);
973
+ if (idCompare != 0) {
974
+ return idCompare;
791
975
  }
792
- // 3. Transition
793
- if (shouldFail) {
794
- idsToFail.push(waiter.id);
976
+ const depIdCompare = a.dependencyTaskId.localeCompare(b.dependencyTaskId);
977
+ if (depIdCompare != 0) {
978
+ return depIdCompare;
795
979
  }
796
- else if (isReady) {
797
- if (waiter.scheduleAfterTags.length > 0) {
798
- idsToSchedule.push(waiter.id);
799
- fanInTags.push(...waiter.tags);
980
+ return a.type.localeCompare(b.type);
981
+ });
982
+ const edgeValues = sortedResolvedEdges.map((e) => sql `(${e.taskId}::uuid, ${e.dependencyTaskId}::uuid, ${e.type}::text)`);
983
+ await tx.pgTransaction.execute(sql `
984
+ DELETE FROM ${taskDependencyTable}
985
+ WHERE (task_id, dependency_task_id, type) IN (
986
+ SELECT t.task_id, t.dependency_task_id, t.type::${taskDependencyType}
987
+ FROM (VALUES ${sql.join(edgeValues, sql `, `)}) AS t(task_id, dependency_task_id, type)
988
+ )
989
+ `);
990
+ const terminalTasks = [];
991
+ const skippedTaskIds = new Set();
992
+ if (failFastTaskIds.size > 0) {
993
+ const sortedFailFastIds = [...failFastTaskIds].toSorted();
994
+ const dependentTasks = await tx.pgTransaction
995
+ .select({ id: taskTable.id, namespace: taskTable.namespace, failFast: taskTable.failFast, status: taskTable.status })
996
+ .from(taskTable)
997
+ .where(inArray(taskTable.id, sortedFailFastIds))
998
+ .orderBy(asc(taskTable.id))
999
+ .for('update');
1000
+ const tasksToSkip = [];
1001
+ for (const task of dependentTasks) {
1002
+ if (task.failFast && !terminalStatuses.includes(task.status)) {
1003
+ tasksToSkip.push(task.id);
1004
+ skippedTaskIds.add(task.id);
800
1005
  }
801
- else if (waiter.completeAfterTags.length > 0) {
802
- idsToComplete.push(waiter.id);
803
- fanInTags.push(...waiter.tags);
1006
+ }
1007
+ if (tasksToSkip.length > 0) {
1008
+ const skippedRows = await tx.pgTransaction
1009
+ .update(taskTable)
1010
+ .set({
1011
+ status: TaskStatus.Skipped,
1012
+ token: null,
1013
+ error: jsonbBuildObject({ code: 'DependencyFailed', message: 'One or more dependencies failed and failFast is enabled' }),
1014
+ completeTimestamp: TRANSACTION_TIMESTAMP,
1015
+ })
1016
+ .where(inArray(taskTable.id, tasksToSkip))
1017
+ .returning({ id: taskTable.id, status: taskTable.status, namespace: taskTable.namespace });
1018
+ terminalTasks.push(...skippedRows.map((r) => ({ id: r.id, status: r.status, namespace: r.namespace })));
1019
+ for (const row of skippedRows) {
1020
+ notifiedNamespaces.add(row.namespace);
804
1021
  }
805
1022
  }
806
1023
  }
807
- if (idsToFail.length > 0) {
808
- const rows = await repository.updateMany(idsToFail, {
809
- status: TaskStatus.Dead,
810
- error: { code: 'DependencyFailed' },
811
- completeTimestamp: TRANSACTION_TIMESTAMP,
812
- });
813
- const tags = rows.flatMap((r) => r.tags);
814
- await this.triggerTagFanIn(tags, { transaction: tx });
1024
+ // 4. Group all resolved edges into decrements (for any dependent NOT skipped above)
1025
+ const decrementsToApply = new Map();
1026
+ for (const dep of sortedResolvedEdges) {
1027
+ if (skippedTaskIds.has(dep.taskId)) {
1028
+ continue;
1029
+ }
1030
+ const current = decrementsToApply.get(dep.taskId) ?? { schedule: 0, complete: 0 };
1031
+ if (dep.type == TaskDependencyType.Schedule) {
1032
+ current.schedule++;
1033
+ }
1034
+ else if (dep.type == TaskDependencyType.Complete || dep.type == TaskDependencyType.Child) {
1035
+ current.complete++;
1036
+ }
1037
+ decrementsToApply.set(dep.taskId, current);
815
1038
  }
816
- if (idsToSchedule.length > 0) {
817
- await repository.updateMany(idsToSchedule, {
818
- status: TaskStatus.Pending,
819
- scheduleTimestamp: TRANSACTION_TIMESTAMP,
1039
+ if (decrementsToApply.size > 0) {
1040
+ const decrementValues = [...decrementsToApply]
1041
+ .toSorted(([idA], [idB]) => idA.localeCompare(idB))
1042
+ .map(([taskId, d]) => sql `(${taskId}::uuid, ${d.schedule}::int, ${d.complete}::int)`);
1043
+ const updates = tx.pgTransaction.$with('updates').as((qb) => qb
1044
+ .select({
1045
+ taskId: sql `(id)::uuid`.as('task_id'),
1046
+ scheduleDecrement: sql `(schedule)::int`.as('schedule_decrement'),
1047
+ completeDecrement: sql `(complete)::int`.as('complete_decrement'),
1048
+ })
1049
+ .from(sql `(VALUES ${sql.join(decrementValues, sql `, `)}) AS t(id, schedule, complete)`));
1050
+ const updatedRows = await tx.pgTransaction
1051
+ .with(updates)
1052
+ .update(taskTable)
1053
+ .set({
1054
+ unresolvedScheduleDependencies: greatest(0, sql `${taskTable.unresolvedScheduleDependencies} - ${updates.scheduleDecrement}`),
1055
+ unresolvedCompleteDependencies: greatest(0, sql `${taskTable.unresolvedCompleteDependencies} - ${updates.completeDecrement}`),
1056
+ status: caseWhen(and(eq(taskTable.status, TaskStatus.Waiting), eq(greatest(0, sql `${taskTable.unresolvedScheduleDependencies} - ${updates.scheduleDecrement}`), 0)), enumValue(TaskStatus, taskStatus, TaskStatus.Pending)).else(caseWhen(and(eq(taskTable.status, TaskStatus.WaitingChildren), eq(greatest(0, sql `${taskTable.unresolvedCompleteDependencies} - ${updates.completeDecrement}`), 0)), enumValue(TaskStatus, taskStatus, TaskStatus.Completed)).else(taskTable.status)),
1057
+ progress: caseWhen(and(eq(taskTable.status, TaskStatus.WaitingChildren), eq(greatest(0, sql `${taskTable.unresolvedCompleteDependencies} - ${updates.completeDecrement}`), 0)), 1).else(taskTable.progress),
1058
+ completeTimestamp: caseWhen(and(eq(taskTable.status, TaskStatus.WaitingChildren), eq(greatest(0, sql `${taskTable.unresolvedCompleteDependencies} - ${updates.completeDecrement}`), 0)), TRANSACTION_TIMESTAMP).else(taskTable.completeTimestamp),
1059
+ token: caseWhen(and(eq(taskTable.status, TaskStatus.WaitingChildren), eq(greatest(0, sql `${taskTable.unresolvedCompleteDependencies} - ${updates.completeDecrement}`), 0)), null).else(taskTable.token),
1060
+ })
1061
+ .from(updates)
1062
+ .where(eq(taskTable.id, updates.taskId))
1063
+ .returning({
1064
+ id: taskTable.id,
1065
+ namespace: taskTable.namespace,
1066
+ status: taskTable.status,
820
1067
  });
1068
+ for (const row of updatedRows) {
1069
+ if (finalizedStatuses.includes(row.status)) {
1070
+ terminalTasks.push({ id: row.id, status: row.status, namespace: row.namespace });
1071
+ }
1072
+ notifiedNamespaces.add(row.namespace);
1073
+ }
821
1074
  }
822
- if (idsToComplete.length > 0) {
823
- await repository.updateMany(idsToComplete, {
824
- status: TaskStatus.Completed,
825
- completeTimestamp: TRANSACTION_TIMESTAMP,
826
- progress: 1,
827
- token: null,
828
- });
829
- await this.triggerTagFanIn(fanInTags, { transaction: tx });
1075
+ if (terminalTasks.length > 0) {
1076
+ const terminalTaskIds = terminalTasks.map((t) => t.id);
1077
+ await tx.pgTransaction
1078
+ .delete(taskDependencyTable)
1079
+ .where(inArray(taskDependencyTable.taskId, terminalTaskIds));
1080
+ await this.resolveDependenciesMany(terminalTasks, { transaction: tx });
830
1081
  }
831
1082
  });
1083
+ for (const namespace of notifiedNamespaces) {
1084
+ this.notify(namespace);
1085
+ }
832
1086
  }
833
1087
  async maintenance(options) {
834
- const repository = this.#repository;
835
- const archiveRepository = this.#archiveRepository;
836
- // 1. Archival: Move old terminal tasks to archive
1088
+ await Promise.allSettled([
1089
+ this.processExpirations(options),
1090
+ this.processZombieRetries(options),
1091
+ this.processZombieExhaustions(options),
1092
+ this.processHardTimeouts(options),
1093
+ this.processPriorityAging(options),
1094
+ ]);
1095
+ await this.performArchival(options);
1096
+ await this.performArchivePurge(options);
1097
+ }
1098
+ async performArchival(options) {
837
1099
  while (true) {
838
- const archivedCount = await repository.useTransaction(options?.transaction, async (tx) => {
839
- const repositoryWithTx = repository.withTransaction(tx);
840
- const archiveRepositoryWithTx = archiveRepository.withTransaction(tx);
841
- const tasksToArchive = await repositoryWithTx.loadManyByQuery({
842
- $and: [
843
- {
844
- namespace: this.#namespace,
845
- status: { $in: [TaskStatus.Completed, TaskStatus.Dead, TaskStatus.Cancelled] },
846
- completeTimestamp: { $lt: sql `${TRANSACTION_TIMESTAMP} - ${interval(this.retention, 'milliseconds')}` },
847
- },
848
- notInArray(taskTable.id, repositoryWithTx.session
849
- .select({ parentId: taskTable.parentId })
850
- .from(taskTable)
851
- .where(and(eq(taskTable.namespace, this.#namespace), sql `${taskTable.parentId} IS NOT NULL`))),
852
- ],
853
- }, { limit: 1000, for: { mode: 'update', skipLocked: true } });
854
- if (tasksToArchive.length > 0) {
855
- await archiveRepositoryWithTx.insertMany(tasksToArchive);
856
- await repositoryWithTx.hardDeleteMany(tasksToArchive.map((t) => t.id));
1100
+ const archivedCount = await this.#repository.useTransaction(options?.transaction, async (tx) => {
1101
+ const childTaskTable = aliasedTable(taskTable, 'childTask');
1102
+ const rowsToArchive = await tx.pgTransaction
1103
+ .select()
1104
+ .from(taskTable)
1105
+ .where(and(eq(taskTable.namespace, this.#namespace), inArray(taskTable.status, finalizedStatuses), lte(taskTable.completeTimestamp, sql `${TRANSACTION_TIMESTAMP} - ${interval(this.retention, 'milliseconds')}`), notExists(tx.pgTransaction
1106
+ .select({ id: childTaskTable.id })
1107
+ .from(childTaskTable)
1108
+ .where(eq(childTaskTable.parentId, taskTable.id))), notExists(tx.pgTransaction
1109
+ .select({ taskId: taskDependencyTable.taskId })
1110
+ .from(taskDependencyTable)
1111
+ .where(eq(taskDependencyTable.dependencyTaskId, taskTable.id)))))
1112
+ .limit(1000)
1113
+ .for('update', { skipLocked: true });
1114
+ if (rowsToArchive.length > 0) {
1115
+ const rowsToArchiveIds = getEntityIds(rowsToArchive);
1116
+ await tx.pgTransaction.insert(taskArchiveTable).values(rowsToArchive);
1117
+ await tx.pgTransaction.delete(taskTable).where(inArray(taskTable.id, rowsToArchiveIds));
857
1118
  }
858
- return tasksToArchive.length;
1119
+ return rowsToArchive.length;
859
1120
  });
860
1121
  if (archivedCount < 1000) {
861
1122
  break;
862
1123
  }
863
1124
  }
864
- // 2. Purge Archive: Remove very old archived tasks
1125
+ }
1126
+ async performArchivePurge(options) {
1127
+ const session = options?.transaction?.pgTransaction ?? this.#database;
1128
+ const selection = session.$with('archive_purge_selection').as((qb) => qb
1129
+ .select({ id: taskArchiveTable.id })
1130
+ .from(taskArchiveTable)
1131
+ .where(and(eq(taskArchiveTable.namespace, this.#namespace), lte(taskArchiveTable.completeTimestamp, sql `${TRANSACTION_TIMESTAMP} - ${interval(this.archiveRetention, 'milliseconds')}`)))
1132
+ .limit(1000));
865
1133
  while (true) {
866
- const deletedArchiveCount = await archiveRepository.useTransaction(options?.transaction, async (tx) => {
867
- const repositoryWithTx = archiveRepository.withTransaction(tx);
868
- const selection = repositoryWithTx.session.$with('archive_purge_selection').as((qb) => qb
869
- .select({ id: taskArchiveTable.id })
870
- .from(taskArchiveTable)
871
- .where(and(eq(taskArchiveTable.namespace, this.#namespace), lt(taskArchiveTable.completeTimestamp, sql `${TRANSACTION_TIMESTAMP} - ${interval(this.archiveRetention, 'milliseconds')}`)))
872
- .limit(1000));
873
- const result = await repositoryWithTx.session
874
- .with(selection)
875
- .delete(taskArchiveTable)
876
- .where(inArray(taskArchiveTable.id, repositoryWithTx.session.select().from(selection)))
877
- .returning({ id: taskArchiveTable.id });
878
- return result.length;
879
- });
880
- if (deletedArchiveCount < 1000) {
1134
+ const result = await session
1135
+ .with(selection)
1136
+ .delete(taskArchiveTable)
1137
+ .where(inArray(taskArchiveTable.id, session.select().from(selection)))
1138
+ .returning({ id: taskArchiveTable.id });
1139
+ if (result.length < 1000) {
881
1140
  break;
882
1141
  }
883
1142
  }
884
- // 3. Maintenance Loop
1143
+ }
1144
+ async processExpirations(options) {
1145
+ const expiredSelection = this.#database.$with('expired_selection').as((qb) => qb
1146
+ .select({ id: taskTable.id })
1147
+ .from(taskTable)
1148
+ .where(and(eq(taskTable.namespace, this.#namespace), inArray(taskTable.status, queueableOrWaitableStatuses), lt(taskTable.timeToLive, TRANSACTION_TIMESTAMP)))
1149
+ .limit(1000)
1150
+ .for('update', { skipLocked: true }));
885
1151
  while (true) {
886
- const maintenanceCount = await repository.useTransaction(options?.transaction, async (tx) => {
887
- const repositoryWithTx = repository.withTransaction(tx);
888
- let totalUpdated = 0;
889
- // 3.1 Handle Pending Expiration
890
- const expiredSelection = repositoryWithTx.session.$with('expired_selection').as((qb) => qb
891
- .select({ id: taskTable.id })
892
- .from(taskTable)
893
- .where(and(eq(taskTable.namespace, this.#namespace), eq(taskTable.status, TaskStatus.Pending), lt(taskTable.timeToLive, TRANSACTION_TIMESTAMP)))
894
- .limit(1000)
895
- .for('update', { skipLocked: true }));
896
- const expiredRows = await repositoryWithTx.session
1152
+ const updatedCount = await this.#repository.useTransaction(options?.transaction, async (tx) => {
1153
+ const expiredRows = await tx.pgTransaction
897
1154
  .with(expiredSelection)
898
1155
  .update(taskTable)
899
1156
  .set({
900
- status: TaskStatus.Dead,
1157
+ status: TaskStatus.Expired,
901
1158
  token: null,
902
1159
  error: { code: 'Expired', message: 'Task expired before processing' },
903
1160
  completeTimestamp: TRANSACTION_TIMESTAMP,
904
1161
  })
905
- .where(inArray(taskTable.id, repositoryWithTx.session.select().from(expiredSelection)))
906
- .returning({ tags: taskTable.tags });
907
- totalUpdated += expiredRows.length;
1162
+ .where(inArray(taskTable.id, tx.pgTransaction.select().from(expiredSelection)))
1163
+ .returning({ id: taskTable.id });
908
1164
  if (expiredRows.length > 0) {
909
- await this.triggerTagFanIn(expiredRows.flatMap((r) => r.tags), { transaction: tx });
1165
+ await this.resolveDependenciesMany(expiredRows.map((r) => ({ id: r.id, status: TaskStatus.Expired, namespace: this.#namespace })), { transaction: tx });
910
1166
  }
911
- // 3.2 Handle Zombie Tasks (Retry)
912
- const zombieRetrySelection = repositoryWithTx.session.$with('zombie_retry_selection').as((qb) => qb
913
- .select({ id: taskTable.id })
914
- .from(taskTable)
915
- .where(and(eq(taskTable.namespace, this.#namespace), eq(taskTable.status, TaskStatus.Running), lt(taskTable.visibilityDeadline, TRANSACTION_TIMESTAMP), lt(taskTable.tries, this.maxTries)))
916
- .limit(1000)
917
- .for('update', { skipLocked: true }));
918
- const zombieRetryRows = await repositoryWithTx.session
919
- .with(zombieRetrySelection)
920
- .update(taskTable)
921
- .set({
922
- status: TaskStatus.Pending,
923
- token: null,
924
- visibilityDeadline: null,
925
- scheduleTimestamp: sql `${TRANSACTION_TIMESTAMP} + ${interval(this.retryDelayMinimum, 'milliseconds')}`, // Simple backoff for zombies
926
- error: sql `jsonb_build_object('code', 'VisibilityTimeout', 'message', 'Worker Lost', 'last_error', ${taskTable.error})`,
927
- })
928
- .where(inArray(taskTable.id, repositoryWithTx.session.select().from(zombieRetrySelection)))
929
- .returning({ id: taskTable.id });
930
- totalUpdated += zombieRetryRows.length;
931
- // 3.3 Handle Zombie Tasks (Exhaustion)
932
- const zombieExhaustionSelection = repositoryWithTx.session.$with('zombie_exhaustion_selection').as((qb) => qb
933
- .select({ id: taskTable.id })
934
- .from(taskTable)
935
- .where(and(eq(taskTable.namespace, this.#namespace), eq(taskTable.status, TaskStatus.Running), lt(taskTable.visibilityDeadline, TRANSACTION_TIMESTAMP), gte(taskTable.tries, this.maxTries)))
936
- .limit(1000)
937
- .for('update', { skipLocked: true }));
938
- const exhaustionRows = await repositoryWithTx.session
1167
+ return expiredRows.length;
1168
+ });
1169
+ if (updatedCount < 1000) {
1170
+ break;
1171
+ }
1172
+ }
1173
+ }
1174
+ async processZombieRetries(options) {
1175
+ const session = options?.transaction?.pgTransaction ?? this.#database;
1176
+ const zombieRetrySelection = session.$with('zombie_retry_selection').as((qb) => qb
1177
+ .select({ id: taskTable.id })
1178
+ .from(taskTable)
1179
+ .where(and(eq(taskTable.namespace, this.#namespace), eq(taskTable.status, TaskStatus.Running), lt(taskTable.visibilityDeadline, TRANSACTION_TIMESTAMP), lt(taskTable.tries, this.maxTries)))
1180
+ .limit(1000)
1181
+ .for('update', { skipLocked: true }));
1182
+ while (true) {
1183
+ const zombieRetryRows = await session
1184
+ .with(zombieRetrySelection)
1185
+ .update(taskTable)
1186
+ .set({
1187
+ status: TaskStatus.Retrying,
1188
+ token: null,
1189
+ visibilityDeadline: null,
1190
+ startTimestamp: null,
1191
+ scheduleTimestamp: sql `${TRANSACTION_TIMESTAMP} + ${interval(this.retryDelayMinimum, 'milliseconds')}`, // Simple backoff for zombies
1192
+ error: jsonbBuildObject({ code: 'VisibilityTimeout', message: 'Worker Lost', lastError: taskTable.error }),
1193
+ })
1194
+ .where(inArray(taskTable.id, session.select().from(zombieRetrySelection)))
1195
+ .returning({ id: taskTable.id });
1196
+ if (zombieRetryRows.length < 1000) {
1197
+ break;
1198
+ }
1199
+ }
1200
+ }
1201
+ async processZombieExhaustions(options) {
1202
+ const zombieExhaustionSelection = this.#database.$with('zombie_exhaustion_selection').as((qb) => qb
1203
+ .select({ id: taskTable.id })
1204
+ .from(taskTable)
1205
+ .where(and(eq(taskTable.namespace, this.#namespace), eq(taskTable.status, TaskStatus.Running), lt(taskTable.visibilityDeadline, TRANSACTION_TIMESTAMP), gte(taskTable.tries, this.maxTries)))
1206
+ .limit(1000)
1207
+ .for('update', { skipLocked: true }));
1208
+ while (true) {
1209
+ const updatedCount = await this.#repository.useTransaction(options?.transaction, async (tx) => {
1210
+ const exhaustionRows = await tx.pgTransaction
939
1211
  .with(zombieExhaustionSelection)
940
1212
  .update(taskTable)
941
1213
  .set({
942
- status: TaskStatus.Dead,
1214
+ status: TaskStatus.Orphaned,
943
1215
  token: null,
944
1216
  visibilityDeadline: null,
945
1217
  completeTimestamp: TRANSACTION_TIMESTAMP,
946
- error: sql `jsonb_build_object('code', 'ZombieExhausted', 'message', 'Exceeded max retries after repeated crashes', 'last_error', ${taskTable.error})`,
1218
+ error: jsonbBuildObject({ code: 'ZombieExhausted', message: 'Exceeded max retries after repeated crashes', lastError: taskTable.error }),
947
1219
  })
948
- .where(inArray(taskTable.id, repositoryWithTx.session.select().from(zombieExhaustionSelection)))
949
- .returning({ tags: taskTable.tags });
950
- totalUpdated += exhaustionRows.length;
1220
+ .where(inArray(taskTable.id, tx.pgTransaction.select().from(zombieExhaustionSelection)))
1221
+ .returning({ id: taskTable.id });
951
1222
  if (exhaustionRows.length > 0) {
952
- await this.triggerTagFanIn(exhaustionRows.flatMap((r) => r.tags), { transaction: tx });
1223
+ await this.resolveDependenciesMany(exhaustionRows.map((r) => ({ id: r.id, status: TaskStatus.Orphaned, namespace: this.#namespace })), { transaction: tx });
953
1224
  }
954
- // 3.4 Handle Hard Timeout
955
- const timeoutSelection = repositoryWithTx.session.$with('timeout_selection').as((qb) => qb
956
- .select({ id: taskTable.id })
957
- .from(taskTable)
958
- .where(and(eq(taskTable.namespace, this.#namespace), eq(taskTable.status, TaskStatus.Running), lt(taskTable.startTimestamp, sql `${TRANSACTION_TIMESTAMP} - ${interval(this.maxExecutionTime, 'milliseconds')}`)))
959
- .limit(1000)
960
- .for('update', { skipLocked: true }));
961
- const timeoutRows = await repositoryWithTx.session
1225
+ return exhaustionRows.length;
1226
+ });
1227
+ if (updatedCount < 1000) {
1228
+ break;
1229
+ }
1230
+ }
1231
+ }
1232
+ async processHardTimeouts(options) {
1233
+ const timeoutSelection = this.#database.$with('timeout_selection').as((qb) => qb
1234
+ .select({ id: taskTable.id })
1235
+ .from(taskTable)
1236
+ .where(and(eq(taskTable.namespace, this.#namespace), eq(taskTable.status, TaskStatus.Running), lt(taskTable.startTimestamp, sql `${TRANSACTION_TIMESTAMP} - ${interval(this.maxExecutionTime, 'milliseconds')}`)))
1237
+ .limit(1000)
1238
+ .for('update', { skipLocked: true }));
1239
+ while (true) {
1240
+ const updatedCount = await this.#repository.useTransaction(options?.transaction, async (tx) => {
1241
+ const timeoutRows = await tx.pgTransaction
962
1242
  .with(timeoutSelection)
963
1243
  .update(taskTable)
964
1244
  .set({
965
- status: TaskStatus.Dead,
1245
+ status: TaskStatus.TimedOut,
966
1246
  token: null,
967
1247
  visibilityDeadline: null,
968
1248
  completeTimestamp: TRANSACTION_TIMESTAMP,
969
- error: sql `jsonb_build_object('code', 'MaxTimeExceeded', 'message', 'Hard Execution Timeout: Task ran longer than ' || ${this.maxExecutionTime} || 'ms', 'last_error', ${taskTable.error})`,
1249
+ error: jsonbBuildObject({ code: 'MaxTimeExceeded', message: sql `'Hard Execution Timeout: Task ran longer than ' || ${this.maxExecutionTime} || 'ms'`, lastError: taskTable.error }),
970
1250
  })
971
- .where(inArray(taskTable.id, repositoryWithTx.session.select().from(timeoutSelection)))
972
- .returning({ tags: taskTable.tags });
973
- totalUpdated += timeoutRows.length;
1251
+ .where(inArray(taskTable.id, tx.pgTransaction.select().from(timeoutSelection)))
1252
+ .returning({ id: taskTable.id });
974
1253
  if (timeoutRows.length > 0) {
975
- await this.triggerTagFanIn(timeoutRows.flatMap((r) => r.tags), { transaction: tx });
1254
+ await this.resolveDependenciesMany(timeoutRows.map((r) => ({ id: r.id, status: TaskStatus.TimedOut, namespace: this.#namespace })), { transaction: tx });
976
1255
  }
977
- // 3.5 Promote Priority (Aging)
978
- const agingSelection = repositoryWithTx.session.$with('aging_selection').as((qb) => qb
979
- .select({ id: taskTable.id })
980
- .from(taskTable)
981
- .where(and(eq(taskTable.namespace, this.#namespace), eq(taskTable.status, TaskStatus.Pending), lt(taskTable.priorityAgeTimestamp, sql `${TRANSACTION_TIMESTAMP} - ${interval(this.priorityAgingInterval, 'milliseconds')}`)))
982
- .limit(1000)
983
- .for('update', { skipLocked: true }));
984
- const agingRows = await repositoryWithTx.session
985
- .with(agingSelection)
986
- .update(taskTable)
987
- .set({
988
- priority: sql `${taskTable.priority} - ${this.priorityAgingStep}`,
989
- priorityAgeTimestamp: TRANSACTION_TIMESTAMP,
990
- })
991
- .where(inArray(taskTable.id, repositoryWithTx.session.select().from(agingSelection)))
992
- .returning({ id: taskTable.id });
993
- totalUpdated += agingRows.length;
994
- return totalUpdated;
1256
+ return timeoutRows.length;
995
1257
  });
996
- if (maintenanceCount == 0) {
1258
+ if (updatedCount < 1000) {
1259
+ break;
1260
+ }
1261
+ }
1262
+ }
1263
+ async processPriorityAging(options) {
1264
+ const session = options?.transaction?.pgTransaction ?? this.#database;
1265
+ const agingSelection = session.$with('aging_selection').as((qb) => qb
1266
+ .select({ id: taskTable.id })
1267
+ .from(taskTable)
1268
+ .where(and(eq(taskTable.namespace, this.#namespace), inArray(taskTable.status, queueableStatuses), lt(taskTable.priorityAgeTimestamp, sql `${TRANSACTION_TIMESTAMP} - ${interval(this.priorityAgingInterval, 'milliseconds')}`)))
1269
+ .limit(1000)
1270
+ .for('update', { skipLocked: true }));
1271
+ while (true) {
1272
+ const agingRows = await session
1273
+ .with(agingSelection)
1274
+ .update(taskTable)
1275
+ .set({
1276
+ priority: sql `${taskTable.priority} - ${this.priorityAgingStep}`,
1277
+ priorityAgeTimestamp: TRANSACTION_TIMESTAMP,
1278
+ })
1279
+ .where(inArray(taskTable.id, session.select().from(agingSelection)))
1280
+ .returning({ id: taskTable.id });
1281
+ if (agingRows.length < 1000) {
997
1282
  break;
998
1283
  }
999
1284
  }
@@ -1009,80 +1294,93 @@ let PostgresTaskQueue = class PostgresTaskQueue extends TaskQueue {
1009
1294
  result: null,
1010
1295
  scheduleTimestamp: TRANSACTION_TIMESTAMP,
1011
1296
  completeTimestamp: null,
1297
+ startTimestamp: null,
1012
1298
  tries: 0,
1013
1299
  progress: 0,
1014
1300
  priorityAgeTimestamp: TRANSACTION_TIMESTAMP,
1015
1301
  state: (options?.resetState == true) ? null : undefined,
1016
1302
  })
1017
- .where(and(eq(taskTable.namespace, this.#namespace), eq(taskTable.id, id), or(eq(taskTable.status, TaskStatus.Pending), eq(taskTable.status, TaskStatus.Completed), eq(taskTable.status, TaskStatus.Cancelled), eq(taskTable.status, TaskStatus.Dead), lt(taskTable.visibilityDeadline, TRANSACTION_TIMESTAMP))))
1018
- .execute();
1303
+ .where(and(eq(taskTable.id, id), or(inArray(taskTable.status, queueableStatuses), inArray(taskTable.status, finalizedStatuses), lt(taskTable.visibilityDeadline, TRANSACTION_TIMESTAMP))));
1019
1304
  }
1020
- notify() {
1021
- this.#messageBus.publishAndForget();
1305
+ notify(namespace = this.#namespace) {
1306
+ this.#messageBus.publishAndForget(namespace);
1022
1307
  }
1023
1308
  async *getConsumer(cancellationSignal, options) {
1024
- const continue$ = merge(this.#messageBus.allMessages$, cancellationSignal);
1309
+ const continue$ = merge(this.#messageBus.allMessages$).pipe(filter((namespace) => namespace == this.#namespace));
1310
+ const mergedContinue$ = merge(continue$, cancellationSignal);
1025
1311
  while (cancellationSignal.isUnset) {
1026
1312
  const task = await this.dequeue(options);
1027
1313
  if (isDefined(task)) {
1028
1314
  yield task;
1029
1315
  continue;
1030
1316
  }
1031
- await cancelableTimeout(5 * millisecondsPerSecond, continue$);
1317
+ await cancelableTimeout(5 * millisecondsPerSecond, mergedContinue$);
1032
1318
  }
1033
1319
  }
1034
- async maintenanceLoop() {
1320
+ maintenanceLoop() {
1321
+ void this.lowFrequencyMaintenanceLoop();
1322
+ void this.mediumFrequencyMaintenanceLoop();
1323
+ void this.highFrequencyMaintenanceLoop();
1324
+ }
1325
+ async lowFrequencyMaintenanceLoop() {
1035
1326
  while (this.#cancellationSignal.isUnset) {
1036
1327
  try {
1037
- await this.maintenance();
1328
+ await Promise.allSettled([
1329
+ this.performArchival(),
1330
+ this.performArchivePurge(),
1331
+ ]);
1038
1332
  }
1039
1333
  catch (error) {
1040
- this.#logger.error('Error during maintenance loop', error);
1334
+ this.#logger.error('Error during low frequency maintenance loop', error);
1041
1335
  }
1042
1336
  finally {
1043
- await cancelableTimeout(30 * millisecondsPerSecond, this.#cancellationSignal);
1337
+ await cancelableTimeout(5 * millisecondsPerMinute, this.#cancellationSignal);
1044
1338
  }
1045
1339
  }
1046
1340
  }
1047
- /**
1048
- * @internal for internal unit test use only
1049
- * @param options
1050
- * @returns
1051
- */
1052
- async processPendingFanIn(options) {
1053
- let processedCount = 0;
1054
- while (true) {
1055
- const task = await this.#internalThis.dequeue({ types: ['[SystemWorker]:FanIn'], forceDequeue: true, transaction: options?.transaction });
1056
- if (isUndefined(task)) {
1057
- break;
1058
- }
1341
+ async mediumFrequencyMaintenanceLoop() {
1342
+ while (this.#cancellationSignal.isUnset) {
1059
1343
  try {
1060
- await this.resolveDependencies(task.data.targetTags);
1061
- await this.#internalThis.complete(task, { transaction: options?.transaction });
1062
- processedCount++;
1344
+ await Promise.allSettled([
1345
+ this.processExpirations(),
1346
+ this.processPriorityAging(),
1347
+ ]);
1063
1348
  }
1064
1349
  catch (error) {
1065
- await this.#internalThis.fail(task, error, { transaction: options?.transaction });
1066
- throw error;
1350
+ this.#logger.error('Error during medium frequency maintenance loop', error);
1351
+ }
1352
+ finally {
1353
+ await cancelableTimeout(millisecondsPerMinute, this.#cancellationSignal);
1067
1354
  }
1068
1355
  }
1069
- return processedCount;
1070
1356
  }
1071
- startSystemWorker() {
1072
- this.#internalThis.process({ concurrency: 1, cancellationSignal: this.#cancellationSignal, forceDequeue: true, types: ['[SystemWorker]:FanIn'] }, async (context) => {
1073
- await this.resolveDependencies(context.data.targetTags);
1074
- return TaskProcessResult.Complete();
1075
- });
1357
+ async highFrequencyMaintenanceLoop() {
1358
+ while (this.#cancellationSignal.isUnset) {
1359
+ try {
1360
+ await Promise.allSettled([
1361
+ this.processZombieRetries(),
1362
+ this.processZombieExhaustions(),
1363
+ this.processHardTimeouts(),
1364
+ ]);
1365
+ }
1366
+ catch (error) {
1367
+ this.#logger.error('Error during high frequency maintenance loop', error);
1368
+ }
1369
+ finally {
1370
+ await cancelableTimeout(10 * millisecondsPerSecond, this.#cancellationSignal);
1371
+ }
1372
+ }
1076
1373
  }
1077
1374
  async *getBatchConsumer(size, cancellationSignal, options) {
1078
- const continue$ = merge(this.#messageBus.allMessages$, cancellationSignal);
1375
+ const continue$ = merge(this.#messageBus.allMessages$).pipe(filter((namespace) => namespace == this.#namespace));
1376
+ const mergedContinue$ = merge(continue$, cancellationSignal);
1079
1377
  while (cancellationSignal.isUnset) {
1080
1378
  const tasks = await this.dequeueMany(size, options);
1081
1379
  if (tasks.length > 0) {
1082
1380
  yield tasks;
1083
1381
  continue;
1084
1382
  }
1085
- await cancelableTimeout(5 * millisecondsPerSecond, continue$);
1383
+ await cancelableTimeout(5 * millisecondsPerSecond, mergedContinue$);
1086
1384
  }
1087
1385
  }
1088
1386
  };