@tstdl/base 0.93.127 → 0.93.128
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/api/client/client.js +45 -9
- package/api/client/tests/api-client.test.d.ts +1 -0
- package/api/client/tests/api-client.test.js +194 -0
- package/api/types.d.ts +34 -2
- package/api/types.js +9 -2
- package/authentication/client/authentication.service.js +30 -11
- package/authentication/client/http-client.middleware.js +10 -3
- package/authentication/server/authentication.service.d.ts +12 -0
- package/authentication/server/authentication.service.js +14 -2
- package/authentication/tests/authentication.client-error-handling.test.js +23 -66
- package/authentication/tests/authentication.client-service-refresh.test.js +14 -14
- package/cancellation/token.d.ts +6 -0
- package/cancellation/token.js +8 -0
- package/http/client/adapters/undici.adapter.js +0 -2
- package/http/client/http-client-request.d.ts +2 -0
- package/http/client/http-client-request.js +4 -0
- package/http/client/http-client-response.d.ts +1 -1
- package/http/client/http-client-response.js +3 -2
- package/http/utils.d.ts +6 -0
- package/http/utils.js +71 -0
- package/injector/injector.js +2 -0
- package/mail/drizzle/0000_numerous_the_watchers.sql +8 -0
- package/mail/drizzle/meta/0000_snapshot.json +1 -32
- package/mail/drizzle/meta/_journal.json +2 -9
- package/object-storage/s3/tests/s3.object-storage.integration.test.js +22 -53
- package/orm/tests/repository-expiration.test.js +3 -3
- package/package.json +1 -1
- package/rate-limit/tests/postgres-rate-limiter.test.js +9 -7
- package/task-queue/tests/complex.test.js +22 -22
- package/task-queue/tests/dependencies.test.js +15 -13
- package/task-queue/tests/queue.test.js +13 -13
- package/task-queue/tests/worker.test.js +12 -12
- package/testing/integration-setup.d.ts +2 -0
- package/testing/integration-setup.js +13 -7
- package/utils/backoff.d.ts +27 -3
- package/utils/backoff.js +31 -9
- package/utils/index.d.ts +1 -0
- package/utils/index.js +1 -0
- package/utils/retry-with-backoff.d.ts +22 -0
- package/utils/retry-with-backoff.js +64 -0
- package/utils/tests/backoff.test.d.ts +1 -0
- package/utils/tests/backoff.test.js +41 -0
- package/utils/tests/retry-with-backoff.test.d.ts +1 -0
- package/utils/tests/retry-with-backoff.test.js +49 -0
- package/mail/drizzle/0000_previous_malcolm_colcord.sql +0 -13
- package/mail/drizzle/0001_flimsy_bloodscream.sql +0 -5
- package/mail/drizzle/meta/0001_snapshot.json +0 -69
|
@@ -51,7 +51,7 @@ describe('ORM Repository Expiration', () => {
|
|
|
51
51
|
], TtlEntity.prototype, "name", void 0);
|
|
52
52
|
TtlEntity = __decorate([
|
|
53
53
|
Table('ttl_entities', { schema }),
|
|
54
|
-
TimeToLive(
|
|
54
|
+
TimeToLive(100, 'hard') // 100ms TTL
|
|
55
55
|
], TtlEntity);
|
|
56
56
|
beforeAll(async () => {
|
|
57
57
|
injector = new Injector('Test');
|
|
@@ -95,8 +95,8 @@ describe('ORM Repository Expiration', () => {
|
|
|
95
95
|
await runInInjectionContext(injector, async () => {
|
|
96
96
|
const repository = injectRepository(TtlEntity);
|
|
97
97
|
const e1 = await repository.insert(Object.assign(new TtlEntity(), { name: 'Valid' }));
|
|
98
|
-
// Wait
|
|
99
|
-
await new Promise((resolve) => setTimeout(resolve,
|
|
98
|
+
// Wait 150ms for expiration
|
|
99
|
+
await new Promise((resolve) => setTimeout(resolve, 150));
|
|
100
100
|
await repository.processExpirations();
|
|
101
101
|
const all = await repository.loadAll({ withDeleted: true });
|
|
102
102
|
expect(all).toHaveLength(0);
|
package/package.json
CHANGED
|
@@ -11,7 +11,7 @@ describe('PostgresRateLimiter Integration Tests', () => {
|
|
|
11
11
|
const rateLimiterProvider = injector.resolve(RateLimiterProvider);
|
|
12
12
|
rateLimiter = rateLimiterProvider.get(limiterName, {
|
|
13
13
|
burstCapacity: 10,
|
|
14
|
-
refillInterval:
|
|
14
|
+
refillInterval: 500, // 10 tokens per 500ms -> 1 token per 50ms
|
|
15
15
|
});
|
|
16
16
|
});
|
|
17
17
|
afterAll(async () => {
|
|
@@ -30,9 +30,9 @@ describe('PostgresRateLimiter Integration Tests', () => {
|
|
|
30
30
|
const resource = 'res-2';
|
|
31
31
|
await rateLimiter.tryAcquire(resource, 10);
|
|
32
32
|
expect(await rateLimiter.tryAcquire(resource, 1)).toBe(false);
|
|
33
|
-
// Wait for
|
|
34
|
-
await timeout(
|
|
35
|
-
expect(await rateLimiter.tryAcquire(resource,
|
|
33
|
+
// Wait for 1 token (50ms) + buffer
|
|
34
|
+
await timeout(75);
|
|
35
|
+
expect(await rateLimiter.tryAcquire(resource, 1)).toBe(true);
|
|
36
36
|
expect(await rateLimiter.tryAcquire(resource, 1)).toBe(false);
|
|
37
37
|
});
|
|
38
38
|
it('should refund tokens', async () => {
|
|
@@ -55,7 +55,9 @@ describe('PostgresRateLimiter Integration Tests', () => {
|
|
|
55
55
|
await rateLimiter.tryAcquire(resource, 0);
|
|
56
56
|
const results = await Promise.all(Array.from({ length: 20 }).map(() => rateLimiter.tryAcquire(resource, 1)));
|
|
57
57
|
const successCount = results.filter(Boolean).length;
|
|
58
|
-
expect
|
|
58
|
+
// We expect 10, but allow up to 12 if tokens refilled during the Promise.all
|
|
59
|
+
expect(successCount).toBeGreaterThanOrEqual(10);
|
|
60
|
+
expect(successCount).toBeLessThanOrEqual(12);
|
|
59
61
|
}, 15000);
|
|
60
62
|
it('should always allow zero or negative cost', async () => {
|
|
61
63
|
const resource = 'res-zero';
|
|
@@ -72,8 +74,8 @@ describe('PostgresRateLimiter Integration Tests', () => {
|
|
|
72
74
|
// Drain
|
|
73
75
|
await rateLimiter.tryAcquire(resource, 10);
|
|
74
76
|
expect(await rateLimiter.tryAcquire(resource, 1)).toBe(false);
|
|
75
|
-
// Wait for full refill (
|
|
76
|
-
await timeout(
|
|
77
|
+
// Wait for full refill (500ms) + extra
|
|
78
|
+
await timeout(600);
|
|
77
79
|
// Should only have 10 tokens
|
|
78
80
|
expect(await rateLimiter.tryAcquire(resource, 10)).toBe(true);
|
|
79
81
|
// Should be empty again immediately
|
|
@@ -15,13 +15,13 @@ describe('Complex Queue Scenarios', () => {
|
|
|
15
15
|
// Configure with specific settings for testing logic
|
|
16
16
|
queue = queueProvider.get(queueName, {
|
|
17
17
|
visibilityTimeout: 1000,
|
|
18
|
-
priorityAgingInterval:
|
|
18
|
+
priorityAgingInterval: 50, // Fast aging
|
|
19
19
|
priorityAgingStep: 10,
|
|
20
20
|
rateLimit: 5,
|
|
21
|
-
rateInterval:
|
|
22
|
-
retryDelayMinimum:
|
|
21
|
+
rateInterval: 50,
|
|
22
|
+
retryDelayMinimum: 50,
|
|
23
23
|
retryDelayGrowth: 2,
|
|
24
|
-
retention:
|
|
24
|
+
retention: 50, // Fast retention for archive test
|
|
25
25
|
});
|
|
26
26
|
});
|
|
27
27
|
afterEach(async () => {
|
|
@@ -33,12 +33,12 @@ describe('Complex Queue Scenarios', () => {
|
|
|
33
33
|
await injector?.dispose();
|
|
34
34
|
});
|
|
35
35
|
async function waitForStatus(id, status) {
|
|
36
|
-
for (let i = 0; i <
|
|
36
|
+
for (let i = 0; i < 50; i++) {
|
|
37
37
|
const task = await queue.getTask(id);
|
|
38
38
|
if (task?.status === status)
|
|
39
39
|
return;
|
|
40
40
|
await queue.processPendingFanIn();
|
|
41
|
-
await timeout(
|
|
41
|
+
await timeout(10);
|
|
42
42
|
}
|
|
43
43
|
}
|
|
44
44
|
describe('Complex Dependencies', () => {
|
|
@@ -62,7 +62,7 @@ describe('Complex Queue Scenarios', () => {
|
|
|
62
62
|
// Process B
|
|
63
63
|
const dB = await queue.dequeue({ types: ['B'] });
|
|
64
64
|
await queue.complete(dB);
|
|
65
|
-
await timeout(
|
|
65
|
+
await timeout(20);
|
|
66
66
|
await queue.processPendingFanIn();
|
|
67
67
|
// D still waiting (needs C)
|
|
68
68
|
const uD2 = await queue.getTask(taskD.id);
|
|
@@ -105,8 +105,8 @@ describe('Complex Queue Scenarios', () => {
|
|
|
105
105
|
describe('Scheduling & Priorities', () => {
|
|
106
106
|
it('should promote priority of old pending tasks (Aging)', async () => {
|
|
107
107
|
const t1 = await queue.enqueue('low', {}, { priority: 2000 });
|
|
108
|
-
// Wait for aging interval (
|
|
109
|
-
await timeout(
|
|
108
|
+
// Wait for aging interval (50ms)
|
|
109
|
+
await timeout(60);
|
|
110
110
|
await queue.maintenance();
|
|
111
111
|
const updated = await queue.getTask(t1.id);
|
|
112
112
|
// Default step is 10. 2000 - 10 = 1990
|
|
@@ -120,7 +120,7 @@ describe('Complex Queue Scenarios', () => {
|
|
|
120
120
|
const u1 = await queue.getTask(task.id);
|
|
121
121
|
expect(u1?.tries).toBe(1);
|
|
122
122
|
const delay1 = u1.scheduleTimestamp - currentTimestamp();
|
|
123
|
-
expect(delay1).toBeGreaterThan(
|
|
123
|
+
expect(delay1).toBeGreaterThan(20); // Approx check
|
|
124
124
|
// Force reschedule to now
|
|
125
125
|
await queue.reschedule(task.id, currentTimestamp());
|
|
126
126
|
// Try 2
|
|
@@ -129,12 +129,12 @@ describe('Complex Queue Scenarios', () => {
|
|
|
129
129
|
const u2 = await queue.getTask(task.id);
|
|
130
130
|
expect(u2?.tries).toBe(2);
|
|
131
131
|
const now = currentTimestamp();
|
|
132
|
-
expect(u2.scheduleTimestamp > now +
|
|
132
|
+
expect(u2.scheduleTimestamp > now + 50).toBe(true);
|
|
133
133
|
});
|
|
134
134
|
});
|
|
135
135
|
describe('Rate Limiting & Concurrency', () => {
|
|
136
136
|
it('should limit burst dequeue rate', async () => {
|
|
137
|
-
// Rate limit 5, interval
|
|
137
|
+
// Rate limit 5, interval 100ms
|
|
138
138
|
await queue.enqueueMany(Array.from({ length: 10 }, (_, i) => ({ type: 'burst', data: { i } })));
|
|
139
139
|
// Request burstCapacity (5)
|
|
140
140
|
const batch1 = await queue.dequeueMany(5);
|
|
@@ -143,7 +143,7 @@ describe('Complex Queue Scenarios', () => {
|
|
|
143
143
|
const batch2 = await queue.dequeueMany(1);
|
|
144
144
|
expect(batch2.length).toBe(0); // Rate limited
|
|
145
145
|
// Wait for refill
|
|
146
|
-
await timeout(
|
|
146
|
+
await timeout(60);
|
|
147
147
|
const batch3 = await queue.dequeueMany(5);
|
|
148
148
|
expect(batch3.length).toBe(5); // Refilled
|
|
149
149
|
});
|
|
@@ -179,8 +179,8 @@ describe('Complex Queue Scenarios', () => {
|
|
|
179
179
|
expect(before).toBeDefined();
|
|
180
180
|
expect(before?.status).toBe(TaskStatus.Completed);
|
|
181
181
|
expect(before.completeTimestamp > 0).toBe(true);
|
|
182
|
-
// Wait for retention (
|
|
183
|
-
await timeout(
|
|
182
|
+
// Wait for retention (50ms).
|
|
183
|
+
await timeout(60);
|
|
184
184
|
await archiveQueue.maintenance();
|
|
185
185
|
// Should move from main table to archive
|
|
186
186
|
const loaded = await archiveQueue.getTask(task.id);
|
|
@@ -189,9 +189,9 @@ describe('Complex Queue Scenarios', () => {
|
|
|
189
189
|
await archiveQueue.clear();
|
|
190
190
|
});
|
|
191
191
|
it('should prune expired pending tasks', async () => {
|
|
192
|
-
// Time to live:
|
|
193
|
-
const task = await queue.enqueue('expire-me', {}, { timeToLive: currentTimestamp() +
|
|
194
|
-
await timeout(
|
|
192
|
+
// Time to live: 50ms
|
|
193
|
+
const task = await queue.enqueue('expire-me', {}, { timeToLive: currentTimestamp() + 50 });
|
|
194
|
+
await timeout(60);
|
|
195
195
|
await queue.maintenance();
|
|
196
196
|
const updated = await queue.getTask(task.id);
|
|
197
197
|
expect(updated?.status).toBe(TaskStatus.Dead);
|
|
@@ -206,7 +206,7 @@ describe('Complex Queue Scenarios', () => {
|
|
|
206
206
|
const d = await queue.dequeue();
|
|
207
207
|
await queue.complete(d);
|
|
208
208
|
// Force move
|
|
209
|
-
await timeout(
|
|
209
|
+
await timeout(60);
|
|
210
210
|
await queue.maintenance();
|
|
211
211
|
// Verify retrieval
|
|
212
212
|
const fromArchive = await queue.getTask(task.id);
|
|
@@ -215,7 +215,7 @@ describe('Complex Queue Scenarios', () => {
|
|
|
215
215
|
});
|
|
216
216
|
it('should defer archival of parent tasks until children are archived', async () => {
|
|
217
217
|
const qProvider = injector.resolve(TaskQueueProvider);
|
|
218
|
-
const treeQueue = qProvider.get(`archive-tree-${Date.now()}`, { retention:
|
|
218
|
+
const treeQueue = qProvider.get(`archive-tree-${Date.now()}`, { retention: 50 });
|
|
219
219
|
const parent = await treeQueue.enqueue('parent', {});
|
|
220
220
|
const child = await treeQueue.enqueue('child', {}, { parentId: parent.id });
|
|
221
221
|
const d1 = await treeQueue.dequeue();
|
|
@@ -223,7 +223,7 @@ describe('Complex Queue Scenarios', () => {
|
|
|
223
223
|
await treeQueue.complete(d1);
|
|
224
224
|
await treeQueue.complete(d2);
|
|
225
225
|
// Wait for retention
|
|
226
|
-
await timeout(
|
|
226
|
+
await timeout(60);
|
|
227
227
|
// First maintenance: should archive child, but parent stays because child is still in main table (until it's deleted in the same tx maybe? No, loadMany happens before delete)
|
|
228
228
|
await treeQueue.maintenance();
|
|
229
229
|
const parentStillActive = await treeQueue.getTask(parent.id);
|
|
@@ -282,7 +282,7 @@ describe('Complex Queue Scenarios', () => {
|
|
|
282
282
|
for (let i = 0; i < 5; i++) {
|
|
283
283
|
if (u?.status == TaskStatus.Waiting)
|
|
284
284
|
break;
|
|
285
|
-
await timeout(
|
|
285
|
+
await timeout(10);
|
|
286
286
|
u = await queue.getTask(dependent.id);
|
|
287
287
|
}
|
|
288
288
|
expect(u?.status).toBe(TaskStatus.Waiting); // Should still be waiting because dependency didn't Complete
|
|
@@ -22,6 +22,16 @@ describe('Queue Dependencies & Tree Tests', () => {
|
|
|
22
22
|
afterAll(async () => {
|
|
23
23
|
await injector?.dispose();
|
|
24
24
|
});
|
|
25
|
+
async function waitForStatus(id, status) {
|
|
26
|
+
for (let i = 0; i < 20; i++) {
|
|
27
|
+
const task = await queue.getTask(id);
|
|
28
|
+
if (task?.status == status) {
|
|
29
|
+
return;
|
|
30
|
+
}
|
|
31
|
+
await queue.processPendingFanIn();
|
|
32
|
+
await timeout(50);
|
|
33
|
+
}
|
|
34
|
+
}
|
|
25
35
|
describe('Dependencies (Fan-In)', () => {
|
|
26
36
|
it('should schedule a task only after dependency completes (completeAfterTags)', async () => {
|
|
27
37
|
// 1. Create a dependent task (Waiting)
|
|
@@ -35,7 +45,7 @@ describe('Queue Dependencies & Tree Tests', () => {
|
|
|
35
45
|
const dequeued = await queue.dequeue({ types: ['prereq'] });
|
|
36
46
|
expect(dequeued?.id).toBe(prereq.id);
|
|
37
47
|
await queue.complete(dequeued);
|
|
38
|
-
await
|
|
48
|
+
await waitForStatus(dependent.id, TaskStatus.Completed);
|
|
39
49
|
const updatedDependent = await queue.getTask(dependent.id);
|
|
40
50
|
expect(updatedDependent?.status).toBe(TaskStatus.Completed);
|
|
41
51
|
});
|
|
@@ -50,7 +60,7 @@ describe('Queue Dependencies & Tree Tests', () => {
|
|
|
50
60
|
// 3. Complete prereq
|
|
51
61
|
const dequeued = await queue.dequeue({ types: ['prereq'] });
|
|
52
62
|
await queue.complete(dequeued);
|
|
53
|
-
await
|
|
63
|
+
await waitForStatus(dependent.id, TaskStatus.Pending);
|
|
54
64
|
// 5. Dependent should be Pending (ready to run)
|
|
55
65
|
const updatedDependent = await queue.getTask(dependent.id);
|
|
56
66
|
expect(updatedDependent?.status).toBe(TaskStatus.Pending);
|
|
@@ -67,16 +77,8 @@ describe('Queue Dependencies & Tree Tests', () => {
|
|
|
67
77
|
const dequeued = await queue.dequeue({ types: ['prereq'] });
|
|
68
78
|
// Fail fatally
|
|
69
79
|
await queue.fail(dequeued, new Error('boom'), { fatal: true });
|
|
70
|
-
|
|
71
|
-
await queue.
|
|
72
|
-
let updatedDependent;
|
|
73
|
-
for (let i = 0; i < 20; i++) {
|
|
74
|
-
await timeout(100);
|
|
75
|
-
updatedDependent = await queue.getTask(dependent.id);
|
|
76
|
-
if (updatedDependent?.status === TaskStatus.Dead)
|
|
77
|
-
break;
|
|
78
|
-
await queue.processPendingFanIn(); // Retry processing if it didn't catch it yet
|
|
79
|
-
}
|
|
80
|
+
await waitForStatus(dependent.id, TaskStatus.Dead);
|
|
81
|
+
const updatedDependent = await queue.getTask(dependent.id);
|
|
80
82
|
expect(updatedDependent?.status).toBe(TaskStatus.Dead);
|
|
81
83
|
expect(updatedDependent?.error?.code).toBe('DependencyFailed');
|
|
82
84
|
});
|
|
@@ -90,7 +92,7 @@ describe('Queue Dependencies & Tree Tests', () => {
|
|
|
90
92
|
await queue.enqueue('t1', {}, { tags: ['tag-1'] });
|
|
91
93
|
const d1 = await queue.dequeue({ types: ['t1'] });
|
|
92
94
|
await queue.complete(d1);
|
|
93
|
-
await
|
|
95
|
+
await waitForStatus(dependent.id, TaskStatus.Pending);
|
|
94
96
|
const updated = await queue.getTask(dependent.id);
|
|
95
97
|
expect(updated?.status).toBe(TaskStatus.Pending);
|
|
96
98
|
});
|
|
@@ -134,11 +134,11 @@ describe('PostgresQueue (Distributed Task Orchestration)', () => {
|
|
|
134
134
|
const queueProvider = injector.resolve(TaskQueueProvider);
|
|
135
135
|
const queueName = `pg-test-queue-${Date.now()}-${Math.random()}`;
|
|
136
136
|
queue = queueProvider.get(queueName, {
|
|
137
|
-
visibilityTimeout:
|
|
137
|
+
visibilityTimeout: 50, // Short timeout for testing
|
|
138
138
|
retryDelayMinimum: 50,
|
|
139
139
|
retryDelayGrowth: 1,
|
|
140
140
|
circuitBreakerThreshold: 2,
|
|
141
|
-
circuitBreakerResetTimeout:
|
|
141
|
+
circuitBreakerResetTimeout: 50,
|
|
142
142
|
});
|
|
143
143
|
});
|
|
144
144
|
afterEach(async () => {
|
|
@@ -188,11 +188,11 @@ describe('PostgresQueue (Distributed Task Orchestration)', () => {
|
|
|
188
188
|
expect((t3?.data)['foo']).toBe('low');
|
|
189
189
|
});
|
|
190
190
|
it('should not dequeue a task scheduled in the future', async () => {
|
|
191
|
-
const future = currentTimestamp() +
|
|
191
|
+
const future = currentTimestamp() + 100;
|
|
192
192
|
await queue.enqueue('foo', { foo: 'future' }, { scheduleTimestamp: future });
|
|
193
193
|
const task = await queue.dequeue();
|
|
194
194
|
expect(task).toBeUndefined();
|
|
195
|
-
await timeout(
|
|
195
|
+
await timeout(150);
|
|
196
196
|
const taskLater = await queue.dequeue();
|
|
197
197
|
expect(taskLater).toBeDefined();
|
|
198
198
|
});
|
|
@@ -237,8 +237,8 @@ describe('PostgresQueue (Distributed Task Orchestration)', () => {
|
|
|
237
237
|
]);
|
|
238
238
|
await queue.fail((await queue.dequeue()), 'err');
|
|
239
239
|
await queue.fail((await queue.dequeue()), 'err');
|
|
240
|
-
// Breaker is Open. Wait for reset timeout (
|
|
241
|
-
await timeout(
|
|
240
|
+
// Breaker is Open. Wait for reset timeout (50ms)
|
|
241
|
+
await timeout(75);
|
|
242
242
|
const probe = await queue.dequeue();
|
|
243
243
|
expect(probe).toBeDefined();
|
|
244
244
|
const secondAttempt = await queue.dequeue();
|
|
@@ -249,8 +249,8 @@ describe('PostgresQueue (Distributed Task Orchestration)', () => {
|
|
|
249
249
|
it('should recover "Zombie" tasks (crashed workers)', async () => {
|
|
250
250
|
const task = await queue.enqueue('foo', { foo: 'zombie' });
|
|
251
251
|
await queue.dequeue(); // Task is now Running with a token
|
|
252
|
-
// processTimeout is
|
|
253
|
-
await timeout(
|
|
252
|
+
// processTimeout is 50ms. Wait for it to expire.
|
|
253
|
+
await timeout(100);
|
|
254
254
|
await queue.maintenance();
|
|
255
255
|
const recovered = await queue.getTask(task.id);
|
|
256
256
|
expect(recovered?.status).toBe(TaskStatus.Pending);
|
|
@@ -260,10 +260,10 @@ describe('PostgresQueue (Distributed Task Orchestration)', () => {
|
|
|
260
260
|
it('should fail tasks that exceed Hard Execution Timeout via prune', async () => {
|
|
261
261
|
// Re-configure queue with very short execution timeout
|
|
262
262
|
const queueProvider = injector.resolve(TaskQueueProvider);
|
|
263
|
-
const shortQueue = queueProvider.get(`prune-test-${Date.now()}`, { maxExecutionTime:
|
|
263
|
+
const shortQueue = queueProvider.get(`prune-test-${Date.now()}`, { maxExecutionTime: 50 });
|
|
264
264
|
const task = await shortQueue.enqueue('foo', { foo: 'long-running' });
|
|
265
265
|
await shortQueue.dequeue();
|
|
266
|
-
await timeout(
|
|
266
|
+
await timeout(75);
|
|
267
267
|
await shortQueue.maintenance();
|
|
268
268
|
const updated = await shortQueue.getTask(task.id);
|
|
269
269
|
expect(updated?.status).toBe(TaskStatus.Dead);
|
|
@@ -274,7 +274,7 @@ describe('PostgresQueue (Distributed Task Orchestration)', () => {
|
|
|
274
274
|
const task = await queue.enqueue('foo', { foo: 'work' });
|
|
275
275
|
const dequeued = await queue.dequeue();
|
|
276
276
|
const initialLock = dequeued.visibilityDeadline;
|
|
277
|
-
await timeout(
|
|
277
|
+
await timeout(20);
|
|
278
278
|
const touched = await queue.touch(dequeued);
|
|
279
279
|
expect(touched?.visibilityDeadline > initialLock).toBe(true);
|
|
280
280
|
});
|
|
@@ -282,8 +282,8 @@ describe('PostgresQueue (Distributed Task Orchestration)', () => {
|
|
|
282
282
|
await queue.enqueue('foo', { foo: 'work' });
|
|
283
283
|
const dequeued = await queue.dequeue();
|
|
284
284
|
expect(dequeued).toBeDefined();
|
|
285
|
-
// processTimeout is
|
|
286
|
-
await timeout(
|
|
285
|
+
// processTimeout is 50ms. Wait for it to expire.
|
|
286
|
+
await timeout(100);
|
|
287
287
|
await queue.maintenance();
|
|
288
288
|
await queue.dequeue(); // Stolen by another worker (tries=2)
|
|
289
289
|
// Original worker tries to touch
|
|
@@ -14,7 +14,7 @@ describe('Worker & Base Class Tests', () => {
|
|
|
14
14
|
const queueProvider = injector.resolve(TaskQueueProvider);
|
|
15
15
|
const queueName = `worker-queue-${Date.now()}-${Math.random()}`;
|
|
16
16
|
queue = queueProvider.get(queueName, {
|
|
17
|
-
visibilityTimeout:
|
|
17
|
+
visibilityTimeout: 200, // Short visibility for testing lease loss
|
|
18
18
|
});
|
|
19
19
|
token = new CancellationToken();
|
|
20
20
|
});
|
|
@@ -36,10 +36,10 @@ describe('Worker & Base Class Tests', () => {
|
|
|
36
36
|
return TaskProcessResult.Complete();
|
|
37
37
|
});
|
|
38
38
|
// Wait until 2 tasks are processed
|
|
39
|
-
for (let i = 0; i <
|
|
39
|
+
for (let i = 0; i < 50; i++) {
|
|
40
40
|
if (processed.length === 2)
|
|
41
41
|
break;
|
|
42
|
-
await timeout(
|
|
42
|
+
await timeout(20);
|
|
43
43
|
}
|
|
44
44
|
token.set(); // Stop worker
|
|
45
45
|
expect(processed).toContain(1);
|
|
@@ -55,7 +55,7 @@ describe('Worker & Base Class Tests', () => {
|
|
|
55
55
|
queue.process({ cancellationSignal: token }, async () => {
|
|
56
56
|
throw new Error('worker error');
|
|
57
57
|
});
|
|
58
|
-
await timeout(
|
|
58
|
+
await timeout(50);
|
|
59
59
|
token.set();
|
|
60
60
|
const updated = await queue.getTask(task.id);
|
|
61
61
|
expect(updated?.status).toBe(TaskStatus.Pending); // Should retry
|
|
@@ -66,12 +66,12 @@ describe('Worker & Base Class Tests', () => {
|
|
|
66
66
|
const task = await queue.enqueue('long', {});
|
|
67
67
|
let executed = false;
|
|
68
68
|
queue.process({ cancellationSignal: token }, async (_context) => {
|
|
69
|
-
// Simulate long work > visibilityTimeout (
|
|
70
|
-
await timeout(
|
|
69
|
+
// Simulate long work > visibilityTimeout (200ms)
|
|
70
|
+
await timeout(300);
|
|
71
71
|
executed = true;
|
|
72
72
|
return TaskProcessResult.Complete();
|
|
73
73
|
});
|
|
74
|
-
await timeout(
|
|
74
|
+
await timeout(500);
|
|
75
75
|
token.set();
|
|
76
76
|
expect(executed).toBe(true);
|
|
77
77
|
const updated = await queue.getTask(task.id);
|
|
@@ -91,10 +91,10 @@ describe('Worker & Base Class Tests', () => {
|
|
|
91
91
|
}
|
|
92
92
|
return TaskProcessResult.Complete();
|
|
93
93
|
});
|
|
94
|
-
for (let i = 0; i <
|
|
94
|
+
for (let i = 0; i < 50; i++) {
|
|
95
95
|
if (processed.size === 2)
|
|
96
96
|
break;
|
|
97
|
-
await timeout(
|
|
97
|
+
await timeout(20);
|
|
98
98
|
}
|
|
99
99
|
token.set();
|
|
100
100
|
const uFail = await queue.getTask(tFail.id);
|
|
@@ -129,10 +129,10 @@ describe('Worker & Base Class Tests', () => {
|
|
|
129
129
|
executed = true;
|
|
130
130
|
return TaskProcessResult.Complete();
|
|
131
131
|
});
|
|
132
|
-
for (let i = 0; i <
|
|
132
|
+
for (let i = 0; i < 50; i++) {
|
|
133
133
|
if (executed)
|
|
134
134
|
break;
|
|
135
|
-
await timeout(
|
|
135
|
+
await timeout(20);
|
|
136
136
|
}
|
|
137
137
|
token.set();
|
|
138
138
|
expect(executed).toBe(true);
|
|
@@ -159,7 +159,7 @@ describe('Worker & Base Class Tests', () => {
|
|
|
159
159
|
if (finalAttemptValues.length === 2)
|
|
160
160
|
break;
|
|
161
161
|
testQueue.notify();
|
|
162
|
-
await timeout(
|
|
162
|
+
await timeout(20);
|
|
163
163
|
}
|
|
164
164
|
token.set();
|
|
165
165
|
expect(finalAttemptValues).toEqual([false, true]);
|
|
@@ -2,6 +2,7 @@ import type { PoolConfig } from 'pg';
|
|
|
2
2
|
import { type AuthenticationAncillaryService } from '../authentication/server/index.js';
|
|
3
3
|
import { Injector } from '../injector/index.js';
|
|
4
4
|
import { LogLevel } from '../logger/index.js';
|
|
5
|
+
import { type S3ObjectStorageProviderConfig } from '../object-storage/s3/index.js';
|
|
5
6
|
import { Database } from '../orm/server/index.js';
|
|
6
7
|
import type { Type } from '../types/index.js';
|
|
7
8
|
export type IntegrationTestOptions = {
|
|
@@ -13,6 +14,7 @@ export type IntegrationTestOptions = {
|
|
|
13
14
|
baseUrl?: string;
|
|
14
15
|
port?: number;
|
|
15
16
|
};
|
|
17
|
+
s3?: Partial<S3ObjectStorageProviderConfig>;
|
|
16
18
|
logLevels?: Record<string, LogLevel>;
|
|
17
19
|
modules?: {
|
|
18
20
|
api?: boolean;
|
|
@@ -98,7 +98,11 @@ export async function setupIntegrationTest(options = {}) {
|
|
|
98
98
|
}
|
|
99
99
|
if (options.modules?.authentication) {
|
|
100
100
|
configureAuthenticationServer({
|
|
101
|
-
serviceOptions: {
|
|
101
|
+
serviceOptions: {
|
|
102
|
+
secret: 'test-secret',
|
|
103
|
+
hashIterations: 10,
|
|
104
|
+
signingSecretsDerivationIterations: 10,
|
|
105
|
+
},
|
|
102
106
|
authenticationAncillaryService: options.authenticationAncillaryService,
|
|
103
107
|
injector,
|
|
104
108
|
});
|
|
@@ -121,13 +125,15 @@ export async function setupIntegrationTest(options = {}) {
|
|
|
121
125
|
await runInInjectionContext(injector, migrateDocumentManagementSchema);
|
|
122
126
|
}
|
|
123
127
|
if (options.modules?.objectStorage) {
|
|
128
|
+
const bucketPerModule = options.s3?.bucketPerModule ?? configParser.boolean('S3_BUCKET_PER_MODULE', true);
|
|
124
129
|
configureS3ObjectStorage({
|
|
125
|
-
endpoint: configParser.string('S3_ENDPOINT', 'http://127.0.0.1:9000'),
|
|
126
|
-
accessKey: configParser.string('S3_ACCESS_KEY', 'tstdl-dev'),
|
|
127
|
-
secretKey: configParser.string('S3_SECRET_KEY', 'tstdl-dev'),
|
|
128
|
-
bucket: configParser.string('S3_BUCKET', 'test-bucket'),
|
|
129
|
-
|
|
130
|
-
|
|
130
|
+
endpoint: options.s3?.endpoint ?? configParser.string('S3_ENDPOINT', 'http://127.0.0.1:9000'),
|
|
131
|
+
accessKey: options.s3?.accessKey ?? configParser.string('S3_ACCESS_KEY', 'tstdl-dev'),
|
|
132
|
+
secretKey: options.s3?.secretKey ?? configParser.string('S3_SECRET_KEY', 'tstdl-dev'),
|
|
133
|
+
bucket: bucketPerModule ? undefined : (options.s3?.bucket ?? configParser.string('S3_BUCKET', 'test-bucket')),
|
|
134
|
+
bucketPerModule,
|
|
135
|
+
region: options.s3?.region ?? configParser.string('S3_REGION', 'us-east-1'),
|
|
136
|
+
forcePathStyle: options.s3?.forcePathStyle ?? configParser.boolean('S3_FORCE_PATH_STYLE', true),
|
|
131
137
|
injector,
|
|
132
138
|
});
|
|
133
139
|
}
|
package/utils/backoff.d.ts
CHANGED
|
@@ -1,5 +1,6 @@
|
|
|
1
1
|
import type { CancellationSignal } from '../cancellation/token.js';
|
|
2
2
|
export type BackoffStrategy = 'linear' | 'exponential';
|
|
3
|
+
export type JitterStrategy = 'full' | 'proportional';
|
|
3
4
|
/**
|
|
4
5
|
* Configuration for the backoff behavior.
|
|
5
6
|
*/
|
|
@@ -11,6 +12,13 @@ export type BackoffOptions = {
|
|
|
11
12
|
* @default 'exponential'
|
|
12
13
|
*/
|
|
13
14
|
strategy?: BackoffStrategy;
|
|
15
|
+
/**
|
|
16
|
+
* The strategy to use for applying jitter.
|
|
17
|
+
* - `full`: Random value between 0 and the calculated delay.
|
|
18
|
+
* - `proportional`: Random value within `± jitter * delay`.
|
|
19
|
+
* @default 'proportional'
|
|
20
|
+
*/
|
|
21
|
+
jitterStrategy?: JitterStrategy;
|
|
14
22
|
/**
|
|
15
23
|
* The initial delay in milliseconds. Must be non-negative.
|
|
16
24
|
* @default 1000
|
|
@@ -32,7 +40,8 @@ export type BackoffOptions = {
|
|
|
32
40
|
/**
|
|
33
41
|
* A factor to randomize the delay, e.g., 0.1 for 10% jitter.
|
|
34
42
|
* This helps prevent the "thundering herd" problem in distributed systems.
|
|
35
|
-
*
|
|
43
|
+
* For `proportional` jitter, the actual delay will be `delay ± delay * jitter`.
|
|
44
|
+
* For `full` jitter, this value is ignored.
|
|
36
45
|
* Must be a value between 0 and 1.
|
|
37
46
|
* @default 0.15
|
|
38
47
|
*/
|
|
@@ -59,14 +68,20 @@ export type BackoffLoopController = {
|
|
|
59
68
|
/**
|
|
60
69
|
* Schedules a backoff delay before the next iteration of the loop.
|
|
61
70
|
* If this is not called, the backoff delay is reset for the next attempt.
|
|
71
|
+
* @param delay An optional explicit delay in milliseconds to use for the next backoff.
|
|
72
|
+
* If not provided, the backoff strategy is used.
|
|
62
73
|
*/
|
|
63
|
-
backoff: () => void;
|
|
74
|
+
backoff: (delay?: number) => void;
|
|
64
75
|
/**
|
|
65
76
|
* Immediately breaks out of the loop.
|
|
66
77
|
*/
|
|
67
78
|
break: () => void;
|
|
68
79
|
};
|
|
69
|
-
export type
|
|
80
|
+
export type BackoffStatus = {
|
|
81
|
+
attempt: number;
|
|
82
|
+
currentDelay: number;
|
|
83
|
+
};
|
|
84
|
+
export type BackoffLoopFunction = (controller: BackoffLoopController, cancellationSignal: CancellationSignal, status: BackoffStatus) => unknown;
|
|
70
85
|
/**
|
|
71
86
|
* A function yielded by `backoffGenerator` to control the next iteration.
|
|
72
87
|
*/
|
|
@@ -82,6 +97,7 @@ export type BackoffGeneratorCallback = (options?: {
|
|
|
82
97
|
*/
|
|
83
98
|
export declare const DEFAULT_BACKOFF_OPTIONS: {
|
|
84
99
|
readonly strategy: "exponential";
|
|
100
|
+
readonly jitterStrategy: "proportional";
|
|
85
101
|
readonly initialDelay: 1000;
|
|
86
102
|
readonly increase: 2;
|
|
87
103
|
readonly maximumDelay: 30000;
|
|
@@ -93,6 +109,14 @@ export declare const DEFAULT_BACKOFF_OPTIONS: {
|
|
|
93
109
|
export declare class BackoffHelper {
|
|
94
110
|
private readonly options;
|
|
95
111
|
private currentDelay;
|
|
112
|
+
private attempt;
|
|
113
|
+
/**
|
|
114
|
+
* The current status of the backoff.
|
|
115
|
+
*/
|
|
116
|
+
get status(): {
|
|
117
|
+
attempt: number;
|
|
118
|
+
currentDelay: number;
|
|
119
|
+
};
|
|
96
120
|
/**
|
|
97
121
|
* Creates a new BackoffHelper.
|
|
98
122
|
* @param options Partial backoff options, which will be merged with sane defaults.
|