@tstdl/base 0.93.140 → 0.93.142
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/application/application.d.ts +1 -1
- package/application/application.js +1 -1
- package/application/providers.d.ts +20 -2
- package/application/providers.js +34 -7
- package/audit/module.d.ts +5 -0
- package/audit/module.js +9 -1
- package/authentication/client/authentication.service.d.ts +1 -0
- package/authentication/client/authentication.service.js +3 -2
- package/authentication/server/module.d.ts +5 -0
- package/authentication/server/module.js +9 -1
- package/authentication/tests/authentication.api-controller.test.js +1 -1
- package/authentication/tests/authentication.api-request-token.provider.test.js +1 -1
- package/authentication/tests/authentication.client-service.test.js +1 -1
- package/circuit-breaker/circuit-breaker.d.ts +6 -4
- package/circuit-breaker/postgres/circuit-breaker.d.ts +1 -0
- package/circuit-breaker/postgres/circuit-breaker.js +8 -5
- package/circuit-breaker/postgres/module.d.ts +1 -0
- package/circuit-breaker/postgres/module.js +5 -1
- package/circuit-breaker/tests/circuit-breaker.test.js +20 -0
- package/document-management/server/configure.js +5 -1
- package/document-management/server/module.d.ts +1 -1
- package/document-management/server/module.js +1 -1
- package/document-management/server/services/document-management-ancillary.service.js +1 -1
- package/document-management/tests/ai-config-hierarchy.test.js +0 -5
- package/document-management/tests/document-management-ai-overrides.test.js +0 -1
- package/document-management/tests/document-validation-ai-overrides.test.js +0 -1
- package/examples/document-management/main.d.ts +1 -0
- package/examples/document-management/main.js +14 -11
- package/key-value-store/postgres/module.d.ts +1 -0
- package/key-value-store/postgres/module.js +5 -1
- package/lock/postgres/module.d.ts +1 -0
- package/lock/postgres/module.js +5 -1
- package/mail/module.d.ts +5 -1
- package/mail/module.js +11 -6
- package/module/modules/web-server.module.js +2 -3
- package/notification/server/module.d.ts +1 -0
- package/notification/server/module.js +5 -1
- package/notification/tests/notification-api.test.js +5 -1
- package/notification/tests/notification-flow.test.js +8 -5
- package/orm/decorators.d.ts +22 -5
- package/orm/decorators.js +10 -1
- package/orm/server/bootstrap.d.ts +11 -0
- package/orm/server/bootstrap.js +31 -0
- package/orm/server/drizzle/schema-converter.d.ts +3 -1
- package/orm/server/drizzle/schema-converter.js +85 -56
- package/orm/server/encryption.d.ts +0 -1
- package/orm/server/encryption.js +1 -4
- package/orm/server/extension.d.ts +14 -0
- package/orm/server/extension.js +27 -0
- package/orm/server/index.d.ts +3 -6
- package/orm/server/index.js +3 -6
- package/orm/server/migration.d.ts +18 -0
- package/orm/server/migration.js +58 -0
- package/orm/server/repository.d.ts +2 -1
- package/orm/server/repository.js +19 -9
- package/orm/server/transaction.d.ts +6 -10
- package/orm/server/transaction.js +25 -26
- package/orm/server/transactional.js +3 -3
- package/orm/tests/database-extension.test.js +63 -0
- package/orm/tests/database-migration.test.js +83 -0
- package/orm/tests/encryption.test.js +3 -4
- package/orm/tests/repository-compound-primary-key.test.d.ts +2 -0
- package/orm/tests/repository-compound-primary-key.test.js +234 -0
- package/orm/tests/schema-generation.test.d.ts +1 -0
- package/orm/tests/schema-generation.test.js +52 -5
- package/orm/utils.d.ts +17 -2
- package/orm/utils.js +49 -1
- package/package.json +5 -4
- package/rate-limit/postgres/module.d.ts +1 -0
- package/rate-limit/postgres/module.js +5 -1
- package/reflection/decorator-data.js +11 -12
- package/task-queue/README.md +2 -10
- package/task-queue/postgres/drizzle/0000_great_gwen_stacy.sql +84 -0
- package/task-queue/postgres/drizzle/meta/0000_snapshot.json +250 -89
- package/task-queue/postgres/drizzle/meta/_journal.json +2 -2
- package/task-queue/postgres/module.d.ts +1 -0
- package/task-queue/postgres/module.js +6 -1
- package/task-queue/postgres/schemas.d.ts +15 -6
- package/task-queue/postgres/schemas.js +4 -3
- package/task-queue/postgres/task-queue.d.ts +18 -15
- package/task-queue/postgres/task-queue.js +797 -499
- package/task-queue/postgres/task.model.d.ts +20 -9
- package/task-queue/postgres/task.model.js +65 -39
- package/task-queue/task-context.d.ts +12 -7
- package/task-queue/task-context.js +8 -6
- package/task-queue/task-queue.d.ts +364 -43
- package/task-queue/task-queue.js +153 -41
- package/task-queue/tests/coverage-branch.test.d.ts +1 -0
- package/task-queue/tests/coverage-branch.test.js +395 -0
- package/task-queue/tests/coverage-enhancement.test.d.ts +1 -0
- package/task-queue/tests/coverage-enhancement.test.js +150 -0
- package/task-queue/tests/dag.test.d.ts +1 -0
- package/task-queue/tests/dag.test.js +188 -0
- package/task-queue/tests/dependencies.test.js +165 -47
- package/task-queue/tests/enqueue-batch.test.d.ts +1 -0
- package/task-queue/tests/enqueue-batch.test.js +125 -0
- package/task-queue/tests/fan-out-spawning.test.d.ts +1 -0
- package/task-queue/tests/fan-out-spawning.test.js +94 -0
- package/task-queue/tests/idempotent-replacement.test.d.ts +1 -0
- package/task-queue/tests/idempotent-replacement.test.js +114 -0
- package/task-queue/tests/missing-idempotent-tasks.test.d.ts +1 -0
- package/task-queue/tests/missing-idempotent-tasks.test.js +39 -0
- package/task-queue/tests/queue.test.js +294 -49
- package/task-queue/tests/shutdown.test.d.ts +1 -0
- package/task-queue/tests/shutdown.test.js +41 -0
- package/task-queue/tests/transactions.test.d.ts +1 -0
- package/task-queue/tests/transactions.test.js +47 -0
- package/task-queue/tests/worker.test.js +63 -15
- package/task-queue/tests/zombie-parent.test.d.ts +1 -0
- package/task-queue/tests/zombie-parent.test.js +45 -0
- package/task-queue/tests/zombie-recovery.test.d.ts +1 -0
- package/task-queue/tests/zombie-recovery.test.js +51 -0
- package/test5.js +5 -5
- package/testing/integration-setup.d.ts +4 -4
- package/testing/integration-setup.js +56 -29
- package/text/localization.service.js +2 -2
- package/utils/file-reader.js +1 -2
- package/utils/timing.d.ts +2 -2
- package/task-queue/postgres/drizzle/0000_simple_invisible_woman.sql +0 -74
- package/task-queue/tests/complex.test.js +0 -306
- package/task-queue/tests/extensive-dependencies.test.js +0 -234
- /package/{task-queue/tests/complex.test.d.ts → orm/tests/database-extension.test.d.ts} +0 -0
- /package/{task-queue/tests/extensive-dependencies.test.d.ts → orm/tests/database-migration.test.d.ts} +0 -0
|
@@ -1,306 +0,0 @@
|
|
|
1
|
-
import { afterAll, afterEach, beforeAll, beforeEach, describe, expect, it } from 'vitest';
|
|
2
|
-
import { DependencyJoinMode, TaskQueueProvider, TaskStatus } from '../../task-queue/index.js';
|
|
3
|
-
import { setupIntegrationTest } from '../../testing/index.js';
|
|
4
|
-
import { currentTimestamp } from '../../utils/date-time.js';
|
|
5
|
-
import { timeout } from '../../utils/timing.js';
|
|
6
|
-
describe('Complex Queue Scenarios', () => {
|
|
7
|
-
let injector;
|
|
8
|
-
let queue;
|
|
9
|
-
beforeAll(async () => {
|
|
10
|
-
({ injector } = await setupIntegrationTest({ modules: { taskQueue: true } }));
|
|
11
|
-
});
|
|
12
|
-
beforeEach(() => {
|
|
13
|
-
const queueProvider = injector.resolve(TaskQueueProvider);
|
|
14
|
-
const queueName = `complex-queue-${Date.now()}-${Math.random()}`;
|
|
15
|
-
// Configure with specific settings for testing logic
|
|
16
|
-
queue = queueProvider.get(queueName, {
|
|
17
|
-
visibilityTimeout: 1000,
|
|
18
|
-
priorityAgingInterval: 50, // Fast aging
|
|
19
|
-
priorityAgingStep: 10,
|
|
20
|
-
rateLimit: 5,
|
|
21
|
-
rateInterval: 200,
|
|
22
|
-
retryDelayMinimum: 50,
|
|
23
|
-
retryDelayGrowth: 2,
|
|
24
|
-
retention: 50, // Fast retention for archive test
|
|
25
|
-
});
|
|
26
|
-
});
|
|
27
|
-
afterEach(async () => {
|
|
28
|
-
await queue.clear();
|
|
29
|
-
const queueProvider = injector.resolve(TaskQueueProvider);
|
|
30
|
-
await queueProvider.get('other-queue').clear();
|
|
31
|
-
});
|
|
32
|
-
afterAll(async () => {
|
|
33
|
-
await injector?.dispose();
|
|
34
|
-
});
|
|
35
|
-
async function waitForStatus(id, status) {
|
|
36
|
-
for (let i = 0; i < 50; i++) {
|
|
37
|
-
const task = await queue.getTask(id);
|
|
38
|
-
if (task?.status === status)
|
|
39
|
-
return;
|
|
40
|
-
await queue.processPendingFanIn();
|
|
41
|
-
await timeout(10);
|
|
42
|
-
}
|
|
43
|
-
}
|
|
44
|
-
describe('Complex Dependencies', () => {
|
|
45
|
-
it('should handle Diamond Dependency (A -> B, A -> C, (B&C) -> D)', async () => {
|
|
46
|
-
// D waits for B and C
|
|
47
|
-
const taskD = await queue.enqueue('D', {}, { scheduleAfterTags: ['tag-b', 'tag-c'] });
|
|
48
|
-
// B and C wait for A
|
|
49
|
-
const taskB = await queue.enqueue('B', {}, { tags: ['tag-b'], scheduleAfterTags: ['tag-a'] });
|
|
50
|
-
const taskC = await queue.enqueue('C', {}, { tags: ['tag-c'], scheduleAfterTags: ['tag-a'] });
|
|
51
|
-
// A runs first
|
|
52
|
-
const taskA = await queue.enqueue('A', {}, { tags: ['tag-a'] });
|
|
53
|
-
expect(taskD.status).toBe(TaskStatus.Waiting);
|
|
54
|
-
expect(taskB.status).toBe(TaskStatus.Waiting);
|
|
55
|
-
expect(taskC.status).toBe(TaskStatus.Waiting);
|
|
56
|
-
expect(taskA.status).toBe(TaskStatus.Pending);
|
|
57
|
-
// Process A
|
|
58
|
-
const dA = await queue.dequeue({ types: ['A'] });
|
|
59
|
-
await queue.complete(dA);
|
|
60
|
-
await waitForStatus(taskB.id, TaskStatus.Pending);
|
|
61
|
-
await waitForStatus(taskC.id, TaskStatus.Pending);
|
|
62
|
-
// Process B
|
|
63
|
-
const dB = await queue.dequeue({ types: ['B'] });
|
|
64
|
-
await queue.complete(dB);
|
|
65
|
-
await timeout(20);
|
|
66
|
-
await queue.processPendingFanIn();
|
|
67
|
-
// D still waiting (needs C)
|
|
68
|
-
const uD2 = await queue.getTask(taskD.id);
|
|
69
|
-
expect(uD2?.status).toBe(TaskStatus.Waiting);
|
|
70
|
-
// Process C
|
|
71
|
-
const dC = await queue.dequeue({ types: ['C'] });
|
|
72
|
-
await queue.complete(dC);
|
|
73
|
-
await waitForStatus(taskD.id, TaskStatus.Pending);
|
|
74
|
-
// D should be Pending
|
|
75
|
-
const uD3 = await queue.getTask(taskD.id);
|
|
76
|
-
expect(uD3?.status).toBe(TaskStatus.Pending);
|
|
77
|
-
});
|
|
78
|
-
it('should handle Deep Chain (A -> B -> C -> D)', async () => {
|
|
79
|
-
const D = await queue.enqueue('D', {}, { scheduleAfterTags: ['C'] });
|
|
80
|
-
const C = await queue.enqueue('C', {}, { tags: ['C'], scheduleAfterTags: ['B'] });
|
|
81
|
-
const B = await queue.enqueue('B', {}, { tags: ['B'], scheduleAfterTags: ['A'] });
|
|
82
|
-
const A = await queue.enqueue('A', {}, { tags: ['A'] });
|
|
83
|
-
// Run A
|
|
84
|
-
await queue.complete((await queue.dequeue({ types: ['A'] })));
|
|
85
|
-
await waitForStatus(B.id, TaskStatus.Pending);
|
|
86
|
-
// Run B
|
|
87
|
-
await queue.complete((await queue.dequeue({ types: ['B'] })));
|
|
88
|
-
await waitForStatus(C.id, TaskStatus.Pending);
|
|
89
|
-
// Run C
|
|
90
|
-
await queue.complete((await queue.dequeue({ types: ['C'] })));
|
|
91
|
-
await waitForStatus(D.id, TaskStatus.Pending);
|
|
92
|
-
expect((await queue.getTask(D.id))?.status).toBe(TaskStatus.Pending);
|
|
93
|
-
});
|
|
94
|
-
it('should propagate cancellation down the dependency tree', async () => {
|
|
95
|
-
// Use parentId for explicit tree structure which `cancel` supports
|
|
96
|
-
const root = await queue.enqueue('root', {});
|
|
97
|
-
const child = await queue.enqueue('child', {}, { parentId: root.id });
|
|
98
|
-
const grandChild = await queue.enqueue('grand', {}, { parentId: child.id });
|
|
99
|
-
await queue.cancel(root.id);
|
|
100
|
-
expect((await queue.getTask(root.id))?.status).toBe(TaskStatus.Cancelled);
|
|
101
|
-
expect((await queue.getTask(child.id))?.status).toBe(TaskStatus.Cancelled);
|
|
102
|
-
expect((await queue.getTask(grandChild.id))?.status).toBe(TaskStatus.Cancelled);
|
|
103
|
-
});
|
|
104
|
-
});
|
|
105
|
-
describe('Scheduling & Priorities', () => {
|
|
106
|
-
it('should promote priority of old pending tasks (Aging)', async () => {
|
|
107
|
-
const t1 = await queue.enqueue('low', {}, { priority: 2000 });
|
|
108
|
-
// Wait for aging interval (50ms)
|
|
109
|
-
await timeout(60);
|
|
110
|
-
await queue.maintenance();
|
|
111
|
-
const updated = await queue.getTask(t1.id);
|
|
112
|
-
// Default step is 10. 2000 - 10 = 1990
|
|
113
|
-
expect(updated?.priority).toBe(1990);
|
|
114
|
-
});
|
|
115
|
-
it('should respect exponential backoff on failure', async () => {
|
|
116
|
-
const task = await queue.enqueue('fail-test', {});
|
|
117
|
-
// Try 1
|
|
118
|
-
const d1 = await queue.dequeue();
|
|
119
|
-
await queue.fail(d1, new Error('fail 1'));
|
|
120
|
-
const u1 = await queue.getTask(task.id);
|
|
121
|
-
expect(u1?.tries).toBe(1);
|
|
122
|
-
const delay1 = u1.scheduleTimestamp - currentTimestamp();
|
|
123
|
-
expect(delay1).toBeGreaterThan(20); // Approx check
|
|
124
|
-
// Force reschedule to now
|
|
125
|
-
await queue.reschedule(task.id, currentTimestamp());
|
|
126
|
-
// Try 2
|
|
127
|
-
const d2 = await queue.dequeue();
|
|
128
|
-
await queue.fail(d2, new Error('fail 2'));
|
|
129
|
-
const u2 = await queue.getTask(task.id);
|
|
130
|
-
expect(u2?.tries).toBe(2);
|
|
131
|
-
const now = currentTimestamp();
|
|
132
|
-
expect(u2.scheduleTimestamp > now + 50).toBe(true);
|
|
133
|
-
});
|
|
134
|
-
});
|
|
135
|
-
describe('Rate Limiting & Concurrency', () => {
|
|
136
|
-
it('should limit burst dequeue rate', async () => {
|
|
137
|
-
// Rate limit 5, interval 100ms
|
|
138
|
-
await queue.enqueueMany(Array.from({ length: 10 }, (_, i) => ({ type: 'burst', data: { i } })));
|
|
139
|
-
// Request burstCapacity (5)
|
|
140
|
-
const batch1 = await queue.dequeueMany(5);
|
|
141
|
-
expect(batch1.length).toBe(5);
|
|
142
|
-
// Try immediately again
|
|
143
|
-
const batch2 = await queue.dequeueMany(1);
|
|
144
|
-
expect(batch2.length).toBe(0); // Rate limited
|
|
145
|
-
// Wait for refill
|
|
146
|
-
await timeout(300);
|
|
147
|
-
const batch3 = await queue.dequeueMany(5);
|
|
148
|
-
expect(batch3.length).toBe(5); // Refilled
|
|
149
|
-
});
|
|
150
|
-
it('should respect global concurrency limit', async () => {
|
|
151
|
-
const queueProvider = injector.resolve(TaskQueueProvider);
|
|
152
|
-
const limitQueue = queueProvider.get(`limit-${Date.now()}`, { globalConcurrency: 2 });
|
|
153
|
-
await limitQueue.enqueueMany([
|
|
154
|
-
{ type: 'c', data: {} },
|
|
155
|
-
{ type: 'c', data: {} },
|
|
156
|
-
{ type: 'c', data: {} },
|
|
157
|
-
]);
|
|
158
|
-
const t1 = await limitQueue.dequeue();
|
|
159
|
-
const t2 = await limitQueue.dequeue();
|
|
160
|
-
const t3 = await limitQueue.dequeue();
|
|
161
|
-
expect(t1).toBeDefined();
|
|
162
|
-
expect(t2).toBeDefined();
|
|
163
|
-
expect(t3).toBeUndefined(); // Max concurrency reached
|
|
164
|
-
await limitQueue.complete(t1);
|
|
165
|
-
const t3_retry = await limitQueue.dequeue();
|
|
166
|
-
expect(t3_retry).toBeDefined();
|
|
167
|
-
await limitQueue.clear();
|
|
168
|
-
});
|
|
169
|
-
});
|
|
170
|
-
describe('Archival & Maintenance', () => {
|
|
171
|
-
it('should archive completed tasks after retention period', async () => {
|
|
172
|
-
const qProvider = injector.resolve(TaskQueueProvider);
|
|
173
|
-
const archiveQueue = qProvider.get(`archive-${Date.now()}-${Math.random()}`, { retention: 100 });
|
|
174
|
-
const task = await archiveQueue.enqueue('archive-me', {});
|
|
175
|
-
const d = await archiveQueue.dequeue();
|
|
176
|
-
await archiveQueue.complete(d);
|
|
177
|
-
// Verify it is in main table
|
|
178
|
-
const before = await archiveQueue.getTask(task.id);
|
|
179
|
-
expect(before).toBeDefined();
|
|
180
|
-
expect(before?.status).toBe(TaskStatus.Completed);
|
|
181
|
-
expect(before.completeTimestamp > 0).toBe(true);
|
|
182
|
-
// Wait for retention (50ms).
|
|
183
|
-
await timeout(60);
|
|
184
|
-
await archiveQueue.maintenance();
|
|
185
|
-
// Should move from main table to archive
|
|
186
|
-
const loaded = await archiveQueue.getTask(task.id);
|
|
187
|
-
expect(loaded).toBeDefined();
|
|
188
|
-
expect(loaded?.status).toBe(TaskStatus.Completed);
|
|
189
|
-
await archiveQueue.clear();
|
|
190
|
-
});
|
|
191
|
-
it('should prune expired pending tasks', async () => {
|
|
192
|
-
// Time to live: 50ms
|
|
193
|
-
const task = await queue.enqueue('expire-me', {}, { timeToLive: currentTimestamp() + 50 });
|
|
194
|
-
await timeout(60);
|
|
195
|
-
await queue.maintenance();
|
|
196
|
-
const updated = await queue.getTask(task.id);
|
|
197
|
-
expect(updated?.status).toBe(TaskStatus.Dead);
|
|
198
|
-
expect((updated?.error)['code']).toBe('Expired');
|
|
199
|
-
});
|
|
200
|
-
it('should retrieve task from archive', async () => {
|
|
201
|
-
// Manually insert into archive? We can't access archiveRepository directly.
|
|
202
|
-
// But we can use maintenance to force it.
|
|
203
|
-
// Wait, if the previous test failed, this might also fail.
|
|
204
|
-
// But let's try with a VERY clear scenario.
|
|
205
|
-
const task = await queue.enqueue('archive-retrieval-test', {});
|
|
206
|
-
const d = await queue.dequeue();
|
|
207
|
-
await queue.complete(d);
|
|
208
|
-
// Force move
|
|
209
|
-
await timeout(60);
|
|
210
|
-
await queue.maintenance();
|
|
211
|
-
// Verify retrieval
|
|
212
|
-
const fromArchive = await queue.getTask(task.id);
|
|
213
|
-
expect(fromArchive).toBeDefined();
|
|
214
|
-
expect(fromArchive?.status).toBe(TaskStatus.Completed);
|
|
215
|
-
});
|
|
216
|
-
it('should defer archival of parent tasks until children are archived', async () => {
|
|
217
|
-
const qProvider = injector.resolve(TaskQueueProvider);
|
|
218
|
-
const treeQueue = qProvider.get(`archive-tree-${Date.now()}`, { retention: 50 });
|
|
219
|
-
const parent = await treeQueue.enqueue('parent', {});
|
|
220
|
-
const child = await treeQueue.enqueue('child', {}, { parentId: parent.id });
|
|
221
|
-
const d1 = await treeQueue.dequeue();
|
|
222
|
-
const d2 = await treeQueue.dequeue();
|
|
223
|
-
await treeQueue.complete(d1);
|
|
224
|
-
await treeQueue.complete(d2);
|
|
225
|
-
// Wait for retention
|
|
226
|
-
await timeout(60);
|
|
227
|
-
// First maintenance: should archive child, but parent stays because child is still in main table (until it's deleted in the same tx maybe? No, loadMany happens before delete)
|
|
228
|
-
await treeQueue.maintenance();
|
|
229
|
-
const parentStillActive = await treeQueue.getTask(parent.id);
|
|
230
|
-
const childInArchive = await treeQueue.getTask(child.id);
|
|
231
|
-
// Verify child moved
|
|
232
|
-
expect(childInArchive).toBeDefined();
|
|
233
|
-
// Parent MUST still be in main table because child was present when archival query ran
|
|
234
|
-
expect(parentStillActive).toBeDefined();
|
|
235
|
-
// Second maintenance: should now archive parent because child is gone from main table
|
|
236
|
-
await treeQueue.maintenance();
|
|
237
|
-
const finalParent = await treeQueue.getTask(parent.id);
|
|
238
|
-
expect(finalParent).toBeDefined(); // Still defined (getTask checks both main and archive)
|
|
239
|
-
// We need to check if it's REALLY archived.
|
|
240
|
-
// We can use the #repository directly if we were in the class, but here we can check if it moved by checking against the main table?
|
|
241
|
-
// Actually getTask is implemented as:
|
|
242
|
-
/*
|
|
243
|
-
const active = await this.#repository.tryLoadByQuery({ namespace: this.#namespace, id });
|
|
244
|
-
if (isDefined(active)) return active;
|
|
245
|
-
return await this.#archiveRepository.tryLoadByQuery({ namespace: this.#namespace, id });
|
|
246
|
-
*/
|
|
247
|
-
await treeQueue.clear();
|
|
248
|
-
});
|
|
249
|
-
});
|
|
250
|
-
describe('Batch & Bulk Operations', () => {
|
|
251
|
-
it('should enqueue and dequeue large batches', async () => {
|
|
252
|
-
const items = Array.from({ length: 50 }, (_, i) => ({ type: 'bulk', data: { i } }));
|
|
253
|
-
const queueProvider = injector.resolve(TaskQueueProvider);
|
|
254
|
-
const bulkQueue = queueProvider.get(`bulk-${Date.now()}`, { rateLimit: 100 });
|
|
255
|
-
await bulkQueue.enqueueMany(items);
|
|
256
|
-
const count = await bulkQueue.countByTags([]); // All -> 0 because empty intersection
|
|
257
|
-
expect(count).toBe(0);
|
|
258
|
-
const batch = await bulkQueue.dequeueMany(50);
|
|
259
|
-
expect(batch.length).toBe(50);
|
|
260
|
-
await bulkQueue.completeMany(batch);
|
|
261
|
-
await bulkQueue.clear();
|
|
262
|
-
});
|
|
263
|
-
it('should handle idempotency window', async () => {
|
|
264
|
-
const key = 'idem-test';
|
|
265
|
-
const t1 = await queue.enqueue('t', { v: 1 }, { idempotencyKey: key });
|
|
266
|
-
const t2 = await queue.enqueue('t', { v: 2 }, { idempotencyKey: key });
|
|
267
|
-
expect(t1.id).toBe(t2.id); // Deduplicated
|
|
268
|
-
const t3 = await queue.enqueue('t', { v: 3 }, { idempotencyKey: key, replace: true });
|
|
269
|
-
expect(t3.id).not.toBe(t1.id); // Replaced
|
|
270
|
-
expect(t3.data['v']).toBe(3);
|
|
271
|
-
});
|
|
272
|
-
});
|
|
273
|
-
describe('Edge Cases', () => {
|
|
274
|
-
it('should not fan-in if dependency fails and failFast=false', async () => {
|
|
275
|
-
const dependent = await queue.enqueue('dep', {}, { scheduleAfterTags: ['fail-tag'], failFast: false });
|
|
276
|
-
const prereq = await queue.enqueue('pre', {}, { tags: ['fail-tag'] });
|
|
277
|
-
const d = await queue.dequeue({ types: ['pre'] });
|
|
278
|
-
await queue.fail(d, new Error('fail'));
|
|
279
|
-
await queue.processPendingFanIn();
|
|
280
|
-
// Retry check because fan-in might be processed by background worker asynchronously
|
|
281
|
-
let u = await queue.getTask(dependent.id);
|
|
282
|
-
for (let i = 0; i < 5; i++) {
|
|
283
|
-
if (u?.status == TaskStatus.Waiting)
|
|
284
|
-
break;
|
|
285
|
-
await timeout(10);
|
|
286
|
-
u = await queue.getTask(dependent.id);
|
|
287
|
-
}
|
|
288
|
-
expect(u?.status).toBe(TaskStatus.Waiting); // Should still be waiting because dependency didn't Complete
|
|
289
|
-
});
|
|
290
|
-
it('should handle mixed AND/OR dependencies', async () => {
|
|
291
|
-
const dep = await queue.enqueue('dep', {}, {
|
|
292
|
-
scheduleAfterTags: ['A', 'B'],
|
|
293
|
-
dependencyJoinMode: DependencyJoinMode.Or,
|
|
294
|
-
});
|
|
295
|
-
const A = await queue.enqueue('A', {}, { tags: ['A'] });
|
|
296
|
-
await queue.complete((await queue.dequeue({ types: ['A'] })));
|
|
297
|
-
await waitForStatus(dep.id, TaskStatus.Pending);
|
|
298
|
-
expect((await queue.getTask(dep.id))?.status).toBe(TaskStatus.Pending);
|
|
299
|
-
});
|
|
300
|
-
it('should not reschedule if task is not running', async () => {
|
|
301
|
-
const task = await queue.enqueue('t', {});
|
|
302
|
-
const res = await queue.touch(task);
|
|
303
|
-
expect(res).toBeUndefined();
|
|
304
|
-
});
|
|
305
|
-
});
|
|
306
|
-
});
|
|
@@ -1,234 +0,0 @@
|
|
|
1
|
-
import { afterAll, afterEach, beforeAll, beforeEach, describe, expect, it } from 'vitest';
|
|
2
|
-
import { DependencyJoinMode, TaskQueueProvider, TaskStatus } from '../../task-queue/index.js';
|
|
3
|
-
import { setupIntegrationTest } from '../../testing/index.js';
|
|
4
|
-
import { timeout } from '../../utils/timing.js';
|
|
5
|
-
describe('Extensive Task Queue Dependency Tests', () => {
|
|
6
|
-
let injector;
|
|
7
|
-
let queue;
|
|
8
|
-
beforeAll(async () => {
|
|
9
|
-
({ injector } = await setupIntegrationTest({ modules: { taskQueue: true } }));
|
|
10
|
-
});
|
|
11
|
-
beforeEach(() => {
|
|
12
|
-
const queueProvider = injector.resolve(TaskQueueProvider);
|
|
13
|
-
const queueName = `extensive-dep-queue-${Date.now()}-${Math.random()}`;
|
|
14
|
-
queue = queueProvider.get(queueName, {
|
|
15
|
-
visibilityTimeout: 1000,
|
|
16
|
-
});
|
|
17
|
-
});
|
|
18
|
-
afterEach(async () => {
|
|
19
|
-
await queue.clear();
|
|
20
|
-
});
|
|
21
|
-
afterAll(async () => {
|
|
22
|
-
await injector?.dispose();
|
|
23
|
-
});
|
|
24
|
-
async function waitForStatus(id, status) {
|
|
25
|
-
for (let i = 0; i < 100; i++) {
|
|
26
|
-
const task = await queue.getTask(id);
|
|
27
|
-
if (task?.status === status)
|
|
28
|
-
return;
|
|
29
|
-
await queue.processPendingFanIn();
|
|
30
|
-
await timeout(50);
|
|
31
|
-
}
|
|
32
|
-
const finalTask = await queue.getTask(id);
|
|
33
|
-
throw new Error(`Task ${id} did not reach status ${status}. Current status: ${finalTask?.status}`);
|
|
34
|
-
}
|
|
35
|
-
async function completeTask(type) {
|
|
36
|
-
const dequeued = await queue.dequeue({ types: [type] });
|
|
37
|
-
if (!dequeued)
|
|
38
|
-
throw new Error(`Could not dequeue task of type ${type}`);
|
|
39
|
-
await queue.complete(dequeued);
|
|
40
|
-
await queue.processPendingFanIn();
|
|
41
|
-
}
|
|
42
|
-
it('should handle complex mixed chain: A -> (B -> D, C -> E) -> F', async () => {
|
|
43
|
-
// F depends on D and E
|
|
44
|
-
const taskF = await queue.enqueue('F', {}, { scheduleAfterTags: ['mc-d', 'mc-e'] });
|
|
45
|
-
// D depends on B, E depends on C
|
|
46
|
-
const taskD = await queue.enqueue('D', {}, { tags: ['mc-d'], scheduleAfterTags: ['mc-b'] });
|
|
47
|
-
const taskE = await queue.enqueue('E', {}, { tags: ['mc-e'], scheduleAfterTags: ['mc-c'] });
|
|
48
|
-
// B and C depend on A
|
|
49
|
-
const taskB = await queue.enqueue('B', {}, { tags: ['mc-b'], scheduleAfterTags: ['mc-a'] });
|
|
50
|
-
const taskC = await queue.enqueue('C', {}, { tags: ['mc-c'], scheduleAfterTags: ['mc-a'] });
|
|
51
|
-
// A is the root
|
|
52
|
-
const taskA = await queue.enqueue('A', {}, { tags: ['mc-a'] });
|
|
53
|
-
expect(taskA.status).toBe(TaskStatus.Pending);
|
|
54
|
-
expect(taskB.status).toBe(TaskStatus.Waiting);
|
|
55
|
-
expect(taskC.status).toBe(TaskStatus.Waiting);
|
|
56
|
-
expect(taskD.status).toBe(TaskStatus.Waiting);
|
|
57
|
-
expect(taskE.status).toBe(TaskStatus.Waiting);
|
|
58
|
-
expect(taskF.status).toBe(TaskStatus.Waiting);
|
|
59
|
-
// 1. Complete A
|
|
60
|
-
await completeTask('A');
|
|
61
|
-
await waitForStatus(taskB.id, TaskStatus.Pending);
|
|
62
|
-
await waitForStatus(taskC.id, TaskStatus.Pending);
|
|
63
|
-
// 2. Complete B
|
|
64
|
-
await completeTask('B');
|
|
65
|
-
await waitForStatus(taskD.id, TaskStatus.Pending);
|
|
66
|
-
// 3. Complete C
|
|
67
|
-
await completeTask('C');
|
|
68
|
-
await waitForStatus(taskE.id, TaskStatus.Pending);
|
|
69
|
-
// F should still be waiting
|
|
70
|
-
const uF = await queue.getTask(taskF.id);
|
|
71
|
-
expect(uF?.status).toBe(TaskStatus.Waiting);
|
|
72
|
-
// 4. Complete D
|
|
73
|
-
await completeTask('D');
|
|
74
|
-
// F still waiting for E
|
|
75
|
-
expect((await queue.getTask(taskF.id))?.status).toBe(TaskStatus.Waiting);
|
|
76
|
-
// 5. Complete E
|
|
77
|
-
await completeTask('E');
|
|
78
|
-
await waitForStatus(taskF.id, TaskStatus.Pending);
|
|
79
|
-
});
|
|
80
|
-
it('should handle requested pattern: A -> B & C -> D', async () => {
|
|
81
|
-
const taskD = await queue.enqueue('D', {}, { scheduleAfterTags: ['B', 'C'] });
|
|
82
|
-
const taskB = await queue.enqueue('B', {}, { tags: ['B'], scheduleAfterTags: ['A'] });
|
|
83
|
-
const taskC = await queue.enqueue('C', {}, { tags: ['C'], scheduleAfterTags: ['A'] });
|
|
84
|
-
const taskA = await queue.enqueue('A', {}, { tags: ['A'] });
|
|
85
|
-
// Initial check
|
|
86
|
-
expect(taskA.status).toBe(TaskStatus.Pending);
|
|
87
|
-
expect(taskB.status).toBe(TaskStatus.Waiting);
|
|
88
|
-
expect(taskC.status).toBe(TaskStatus.Waiting);
|
|
89
|
-
expect(taskD.status).toBe(TaskStatus.Waiting);
|
|
90
|
-
// Complete A
|
|
91
|
-
await completeTask('A');
|
|
92
|
-
await waitForStatus(taskB.id, TaskStatus.Pending);
|
|
93
|
-
await waitForStatus(taskC.id, TaskStatus.Pending);
|
|
94
|
-
// B and C can be dequeued now (in any order)
|
|
95
|
-
const dB = await queue.dequeue({ types: ['B'] });
|
|
96
|
-
const dC = await queue.dequeue({ types: ['C'] });
|
|
97
|
-
expect(dB).toBeDefined();
|
|
98
|
-
expect(dC).toBeDefined();
|
|
99
|
-
// D still waiting
|
|
100
|
-
expect((await queue.getTask(taskD.id))?.status).toBe(TaskStatus.Waiting);
|
|
101
|
-
// Complete B
|
|
102
|
-
await queue.complete(dB);
|
|
103
|
-
await queue.processPendingFanIn();
|
|
104
|
-
expect((await queue.getTask(taskD.id))?.status).toBe(TaskStatus.Waiting);
|
|
105
|
-
// Complete C
|
|
106
|
-
await queue.complete(dC);
|
|
107
|
-
await waitForStatus(taskD.id, TaskStatus.Pending);
|
|
108
|
-
});
|
|
109
|
-
it('should strictly adhere to order: A -> B -> C', async () => {
|
|
110
|
-
const taskC = await queue.enqueue('C', {}, { tags: ['C'], scheduleAfterTags: ['B'] });
|
|
111
|
-
const taskB = await queue.enqueue('B', {}, { tags: ['B'], scheduleAfterTags: ['A'] });
|
|
112
|
-
const taskA = await queue.enqueue('A', {}, { tags: ['A'] });
|
|
113
|
-
// Try to dequeue B and C - should fail
|
|
114
|
-
const dC = await queue.dequeue({ types: ['C'] });
|
|
115
|
-
const dB = await queue.dequeue({ types: ['B'] });
|
|
116
|
-
expect(dC).toBeUndefined();
|
|
117
|
-
expect(dB).toBeUndefined();
|
|
118
|
-
// Complete A
|
|
119
|
-
const dA = await queue.dequeue({ types: ['A'] });
|
|
120
|
-
expect(dA?.id).toBe(taskA.id);
|
|
121
|
-
await queue.complete(dA);
|
|
122
|
-
await queue.processPendingFanIn();
|
|
123
|
-
// Now B should be available, but C still not
|
|
124
|
-
const dC_2 = await queue.dequeue({ types: ['C'] });
|
|
125
|
-
expect(dC_2).toBeUndefined();
|
|
126
|
-
const dB_2 = await queue.dequeue({ types: ['B'] });
|
|
127
|
-
expect(dB_2?.id).toBe(taskB.id);
|
|
128
|
-
await queue.complete(dB_2);
|
|
129
|
-
await queue.processPendingFanIn();
|
|
130
|
-
// Now C should be available
|
|
131
|
-
const dC_3 = await queue.dequeue({ types: ['C'] });
|
|
132
|
-
expect(dC_3?.id).toBe(taskC.id);
|
|
133
|
-
await queue.complete(dC_3);
|
|
134
|
-
});
|
|
135
|
-
it('should handle large fan-in: (T1, T2, T3, T4, T5) -> Result', async () => {
|
|
136
|
-
const tags = ['t1', 't2', 't3', 't4', 't5'];
|
|
137
|
-
const taskResult = await queue.enqueue('Result', {}, { scheduleAfterTags: tags });
|
|
138
|
-
for (const tag of tags) {
|
|
139
|
-
await queue.enqueue(`Task-${tag}`, {}, { tags: [tag] });
|
|
140
|
-
}
|
|
141
|
-
// Complete all but one
|
|
142
|
-
for (let i = 0; i < 4; i++) {
|
|
143
|
-
await completeTask(`Task-${tags[i]}`);
|
|
144
|
-
expect((await queue.getTask(taskResult.id))?.status).toBe(TaskStatus.Waiting);
|
|
145
|
-
}
|
|
146
|
-
// Complete the last one
|
|
147
|
-
await completeTask(`Task-${tags[4]}`);
|
|
148
|
-
await waitForStatus(taskResult.id, TaskStatus.Pending);
|
|
149
|
-
});
|
|
150
|
-
it('should handle large fan-in with OR: (T1, T2, T3, T4, T5) -> Result', async () => {
|
|
151
|
-
const tags = ['o1', 'o2', 'o3', 'o4', 'o5'];
|
|
152
|
-
const taskResult = await queue.enqueue('Result', {}, {
|
|
153
|
-
scheduleAfterTags: tags,
|
|
154
|
-
dependencyJoinMode: DependencyJoinMode.Or
|
|
155
|
-
});
|
|
156
|
-
for (const tag of tags) {
|
|
157
|
-
await queue.enqueue(`Task-${tag}`, {}, { tags: [tag] });
|
|
158
|
-
}
|
|
159
|
-
// Complete one
|
|
160
|
-
await completeTask(`Task-${tags[2]}`);
|
|
161
|
-
await waitForStatus(taskResult.id, TaskStatus.Pending);
|
|
162
|
-
});
|
|
163
|
-
it('should handle Diamond of Diamonds: A -> (B1, B2) -> C -> (D1, D2) -> E', async () => {
|
|
164
|
-
const taskE = await queue.enqueue('E', {}, { scheduleAfterTags: ['tag-d1', 'tag-d2'] });
|
|
165
|
-
const taskD1 = await queue.enqueue('D1', {}, { tags: ['tag-d1'], scheduleAfterTags: ['tag-c'] });
|
|
166
|
-
const taskD2 = await queue.enqueue('D2', {}, { tags: ['tag-d2'], scheduleAfterTags: ['tag-c'] });
|
|
167
|
-
const taskC = await queue.enqueue('C', {}, { tags: ['tag-c'], scheduleAfterTags: ['tag-b1', 'tag-b2'] });
|
|
168
|
-
const taskB1 = await queue.enqueue('B1', {}, { tags: ['tag-b1'], scheduleAfterTags: ['tag-a'] });
|
|
169
|
-
const taskB2 = await queue.enqueue('B2', {}, { tags: ['tag-b2'], scheduleAfterTags: ['tag-a'] });
|
|
170
|
-
const taskA = await queue.enqueue('A', {}, { tags: ['tag-a'] });
|
|
171
|
-
// Step by step completion
|
|
172
|
-
await completeTask('A');
|
|
173
|
-
await waitForStatus(taskB1.id, TaskStatus.Pending);
|
|
174
|
-
await waitForStatus(taskB2.id, TaskStatus.Pending);
|
|
175
|
-
await completeTask('B1');
|
|
176
|
-
expect((await queue.getTask(taskC.id))?.status).toBe(TaskStatus.Waiting);
|
|
177
|
-
await completeTask('B2');
|
|
178
|
-
await waitForStatus(taskC.id, TaskStatus.Pending);
|
|
179
|
-
await completeTask('C');
|
|
180
|
-
await waitForStatus(taskD1.id, TaskStatus.Pending);
|
|
181
|
-
await waitForStatus(taskD2.id, TaskStatus.Pending);
|
|
182
|
-
await completeTask('D1');
|
|
183
|
-
expect((await queue.getTask(taskE.id))?.status).toBe(TaskStatus.Waiting);
|
|
184
|
-
await completeTask('D2');
|
|
185
|
-
await waitForStatus(taskE.id, TaskStatus.Pending);
|
|
186
|
-
});
|
|
187
|
-
it('should fail-fast entire branch if one dependency fails fatal', async () => {
|
|
188
|
-
// A -> B -> C
|
|
189
|
-
// -> D -> E
|
|
190
|
-
// (C & E) -> F
|
|
191
|
-
const taskF = await queue.enqueue('F', {}, { scheduleAfterTags: ['C', 'E'], failFast: true });
|
|
192
|
-
const taskC = await queue.enqueue('C', {}, { tags: ['C'], scheduleAfterTags: ['B'], failFast: true });
|
|
193
|
-
const taskE = await queue.enqueue('E', {}, { tags: ['E'], scheduleAfterTags: ['D'], failFast: true });
|
|
194
|
-
const taskB = await queue.enqueue('B', {}, { tags: ['B'], scheduleAfterTags: ['A'], failFast: true });
|
|
195
|
-
const taskD = await queue.enqueue('D', {}, { tags: ['D'], scheduleAfterTags: ['A'], failFast: true });
|
|
196
|
-
const taskA = await queue.enqueue('A', {}, { tags: ['A'] });
|
|
197
|
-
await completeTask('A');
|
|
198
|
-
await waitForStatus(taskB.id, TaskStatus.Pending);
|
|
199
|
-
await waitForStatus(taskD.id, TaskStatus.Pending);
|
|
200
|
-
// Fail B fatally
|
|
201
|
-
const dB = await queue.dequeue({ types: ['B'] });
|
|
202
|
-
await queue.fail(dB, new Error('fatal B'), { fatal: true });
|
|
203
|
-
await queue.processPendingFanIn();
|
|
204
|
-
// B failed fatally -> C should die -> F should die
|
|
205
|
-
await waitForStatus(taskC.id, TaskStatus.Dead);
|
|
206
|
-
await waitForStatus(taskF.id, TaskStatus.Dead);
|
|
207
|
-
// D and E should be unaffected (except E is still waiting for D)
|
|
208
|
-
const uD = await queue.getTask(taskD.id);
|
|
209
|
-
expect(uD?.status).toBe(TaskStatus.Pending);
|
|
210
|
-
});
|
|
211
|
-
it('should handle many-to-many dependencies', async () => {
|
|
212
|
-
// {A, B} -> {C, D} -> {E, F}
|
|
213
|
-
// Each of C, D depends on BOTH A and B.
|
|
214
|
-
// Each of E, F depends on BOTH C and D.
|
|
215
|
-
const taskE = await queue.enqueue('E', {}, { scheduleAfterTags: ['C', 'D'] });
|
|
216
|
-
const taskF = await queue.enqueue('F', {}, { scheduleAfterTags: ['C', 'D'] });
|
|
217
|
-
const taskC = await queue.enqueue('C', {}, { tags: ['C'], scheduleAfterTags: ['A', 'B'] });
|
|
218
|
-
const taskD = await queue.enqueue('D', {}, { tags: ['D'], scheduleAfterTags: ['A', 'B'] });
|
|
219
|
-
const taskA = await queue.enqueue('A', {}, { tags: ['A'] });
|
|
220
|
-
const taskB = await queue.enqueue('B', {}, { tags: ['B'] });
|
|
221
|
-
await completeTask('A');
|
|
222
|
-
expect((await queue.getTask(taskC.id))?.status).toBe(TaskStatus.Waiting);
|
|
223
|
-
expect((await queue.getTask(taskD.id))?.status).toBe(TaskStatus.Waiting);
|
|
224
|
-
await completeTask('B');
|
|
225
|
-
await waitForStatus(taskC.id, TaskStatus.Pending);
|
|
226
|
-
await waitForStatus(taskD.id, TaskStatus.Pending);
|
|
227
|
-
await completeTask('C');
|
|
228
|
-
expect((await queue.getTask(taskE.id))?.status).toBe(TaskStatus.Waiting);
|
|
229
|
-
expect((await queue.getTask(taskF.id))?.status).toBe(TaskStatus.Waiting);
|
|
230
|
-
await completeTask('D');
|
|
231
|
-
await waitForStatus(taskE.id, TaskStatus.Pending);
|
|
232
|
-
await waitForStatus(taskF.id, TaskStatus.Pending);
|
|
233
|
-
});
|
|
234
|
-
});
|
|
File without changes
|
|
File without changes
|