@tstdl/base 0.93.127 → 0.93.129
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/api/client/client.js +45 -9
- package/api/client/tests/api-client.test.d.ts +1 -0
- package/api/client/tests/api-client.test.js +194 -0
- package/api/types.d.ts +35 -3
- package/authentication/client/authentication.service.js +30 -11
- package/authentication/client/http-client.middleware.js +10 -3
- package/authentication/server/authentication.service.d.ts +12 -0
- package/authentication/server/authentication.service.js +14 -2
- package/authentication/tests/authentication.client-error-handling.test.js +23 -66
- package/authentication/tests/authentication.client-service-refresh.test.js +14 -14
- package/cancellation/token.d.ts +6 -0
- package/cancellation/token.js +8 -0
- package/document-management/server/services/document-file.service.js +10 -9
- package/document-management/server/services/document-management-ancillary.service.d.ts +12 -1
- package/document-management/server/services/document-management-ancillary.service.js +9 -0
- package/file/server/temporary-file.d.ts +2 -1
- package/file/server/temporary-file.js +5 -1
- package/http/client/adapters/undici.adapter.js +0 -2
- package/http/client/http-client-request.d.ts +2 -0
- package/http/client/http-client-request.js +4 -0
- package/http/client/http-client-response.d.ts +1 -1
- package/http/client/http-client-response.js +3 -2
- package/http/utils.d.ts +6 -0
- package/http/utils.js +71 -0
- package/injector/injector.js +2 -0
- package/mail/drizzle/0000_numerous_the_watchers.sql +8 -0
- package/mail/drizzle/meta/0000_snapshot.json +1 -32
- package/mail/drizzle/meta/_journal.json +2 -9
- package/object-storage/s3/s3.object-storage.js +6 -6
- package/object-storage/s3/tests/s3.object-storage.integration.test.js +22 -53
- package/orm/tests/repository-expiration.test.js +3 -3
- package/package.json +1 -1
- package/pdf/utils.d.ts +24 -3
- package/pdf/utils.js +89 -30
- package/process/spawn.d.ts +1 -1
- package/rate-limit/tests/postgres-rate-limiter.test.js +9 -7
- package/renderer/typst.d.ts +5 -0
- package/renderer/typst.js +9 -5
- package/task-queue/tests/complex.test.js +22 -22
- package/task-queue/tests/dependencies.test.js +15 -13
- package/task-queue/tests/queue.test.js +13 -13
- package/task-queue/tests/worker.test.js +12 -12
- package/testing/integration-setup.d.ts +2 -0
- package/testing/integration-setup.js +13 -7
- package/utils/backoff.d.ts +27 -3
- package/utils/backoff.js +31 -9
- package/utils/index.d.ts +1 -0
- package/utils/index.js +1 -0
- package/utils/retry-with-backoff.d.ts +22 -0
- package/utils/retry-with-backoff.js +64 -0
- package/utils/tests/backoff.test.d.ts +1 -0
- package/utils/tests/backoff.test.js +41 -0
- package/utils/tests/retry-with-backoff.test.d.ts +1 -0
- package/utils/tests/retry-with-backoff.test.js +49 -0
- package/mail/drizzle/0000_previous_malcolm_colcord.sql +0 -13
- package/mail/drizzle/0001_flimsy_bloodscream.sql +0 -5
- package/mail/drizzle/meta/0001_snapshot.json +0 -69
|
@@ -22,6 +22,16 @@ describe('Queue Dependencies & Tree Tests', () => {
|
|
|
22
22
|
afterAll(async () => {
|
|
23
23
|
await injector?.dispose();
|
|
24
24
|
});
|
|
25
|
+
async function waitForStatus(id, status) {
|
|
26
|
+
for (let i = 0; i < 20; i++) {
|
|
27
|
+
const task = await queue.getTask(id);
|
|
28
|
+
if (task?.status == status) {
|
|
29
|
+
return;
|
|
30
|
+
}
|
|
31
|
+
await queue.processPendingFanIn();
|
|
32
|
+
await timeout(50);
|
|
33
|
+
}
|
|
34
|
+
}
|
|
25
35
|
describe('Dependencies (Fan-In)', () => {
|
|
26
36
|
it('should schedule a task only after dependency completes (completeAfterTags)', async () => {
|
|
27
37
|
// 1. Create a dependent task (Waiting)
|
|
@@ -35,7 +45,7 @@ describe('Queue Dependencies & Tree Tests', () => {
|
|
|
35
45
|
const dequeued = await queue.dequeue({ types: ['prereq'] });
|
|
36
46
|
expect(dequeued?.id).toBe(prereq.id);
|
|
37
47
|
await queue.complete(dequeued);
|
|
38
|
-
await
|
|
48
|
+
await waitForStatus(dependent.id, TaskStatus.Completed);
|
|
39
49
|
const updatedDependent = await queue.getTask(dependent.id);
|
|
40
50
|
expect(updatedDependent?.status).toBe(TaskStatus.Completed);
|
|
41
51
|
});
|
|
@@ -50,7 +60,7 @@ describe('Queue Dependencies & Tree Tests', () => {
|
|
|
50
60
|
// 3. Complete prereq
|
|
51
61
|
const dequeued = await queue.dequeue({ types: ['prereq'] });
|
|
52
62
|
await queue.complete(dequeued);
|
|
53
|
-
await
|
|
63
|
+
await waitForStatus(dependent.id, TaskStatus.Pending);
|
|
54
64
|
// 5. Dependent should be Pending (ready to run)
|
|
55
65
|
const updatedDependent = await queue.getTask(dependent.id);
|
|
56
66
|
expect(updatedDependent?.status).toBe(TaskStatus.Pending);
|
|
@@ -67,16 +77,8 @@ describe('Queue Dependencies & Tree Tests', () => {
|
|
|
67
77
|
const dequeued = await queue.dequeue({ types: ['prereq'] });
|
|
68
78
|
// Fail fatally
|
|
69
79
|
await queue.fail(dequeued, new Error('boom'), { fatal: true });
|
|
70
|
-
|
|
71
|
-
await queue.
|
|
72
|
-
let updatedDependent;
|
|
73
|
-
for (let i = 0; i < 20; i++) {
|
|
74
|
-
await timeout(100);
|
|
75
|
-
updatedDependent = await queue.getTask(dependent.id);
|
|
76
|
-
if (updatedDependent?.status === TaskStatus.Dead)
|
|
77
|
-
break;
|
|
78
|
-
await queue.processPendingFanIn(); // Retry processing if it didn't catch it yet
|
|
79
|
-
}
|
|
80
|
+
await waitForStatus(dependent.id, TaskStatus.Dead);
|
|
81
|
+
const updatedDependent = await queue.getTask(dependent.id);
|
|
80
82
|
expect(updatedDependent?.status).toBe(TaskStatus.Dead);
|
|
81
83
|
expect(updatedDependent?.error?.code).toBe('DependencyFailed');
|
|
82
84
|
});
|
|
@@ -90,7 +92,7 @@ describe('Queue Dependencies & Tree Tests', () => {
|
|
|
90
92
|
await queue.enqueue('t1', {}, { tags: ['tag-1'] });
|
|
91
93
|
const d1 = await queue.dequeue({ types: ['t1'] });
|
|
92
94
|
await queue.complete(d1);
|
|
93
|
-
await
|
|
95
|
+
await waitForStatus(dependent.id, TaskStatus.Pending);
|
|
94
96
|
const updated = await queue.getTask(dependent.id);
|
|
95
97
|
expect(updated?.status).toBe(TaskStatus.Pending);
|
|
96
98
|
});
|
|
@@ -134,11 +134,11 @@ describe('PostgresQueue (Distributed Task Orchestration)', () => {
|
|
|
134
134
|
const queueProvider = injector.resolve(TaskQueueProvider);
|
|
135
135
|
const queueName = `pg-test-queue-${Date.now()}-${Math.random()}`;
|
|
136
136
|
queue = queueProvider.get(queueName, {
|
|
137
|
-
visibilityTimeout:
|
|
137
|
+
visibilityTimeout: 50, // Short timeout for testing
|
|
138
138
|
retryDelayMinimum: 50,
|
|
139
139
|
retryDelayGrowth: 1,
|
|
140
140
|
circuitBreakerThreshold: 2,
|
|
141
|
-
circuitBreakerResetTimeout:
|
|
141
|
+
circuitBreakerResetTimeout: 50,
|
|
142
142
|
});
|
|
143
143
|
});
|
|
144
144
|
afterEach(async () => {
|
|
@@ -188,11 +188,11 @@ describe('PostgresQueue (Distributed Task Orchestration)', () => {
|
|
|
188
188
|
expect((t3?.data)['foo']).toBe('low');
|
|
189
189
|
});
|
|
190
190
|
it('should not dequeue a task scheduled in the future', async () => {
|
|
191
|
-
const future = currentTimestamp() +
|
|
191
|
+
const future = currentTimestamp() + 100;
|
|
192
192
|
await queue.enqueue('foo', { foo: 'future' }, { scheduleTimestamp: future });
|
|
193
193
|
const task = await queue.dequeue();
|
|
194
194
|
expect(task).toBeUndefined();
|
|
195
|
-
await timeout(
|
|
195
|
+
await timeout(150);
|
|
196
196
|
const taskLater = await queue.dequeue();
|
|
197
197
|
expect(taskLater).toBeDefined();
|
|
198
198
|
});
|
|
@@ -237,8 +237,8 @@ describe('PostgresQueue (Distributed Task Orchestration)', () => {
|
|
|
237
237
|
]);
|
|
238
238
|
await queue.fail((await queue.dequeue()), 'err');
|
|
239
239
|
await queue.fail((await queue.dequeue()), 'err');
|
|
240
|
-
// Breaker is Open. Wait for reset timeout (
|
|
241
|
-
await timeout(
|
|
240
|
+
// Breaker is Open. Wait for reset timeout (50ms)
|
|
241
|
+
await timeout(75);
|
|
242
242
|
const probe = await queue.dequeue();
|
|
243
243
|
expect(probe).toBeDefined();
|
|
244
244
|
const secondAttempt = await queue.dequeue();
|
|
@@ -249,8 +249,8 @@ describe('PostgresQueue (Distributed Task Orchestration)', () => {
|
|
|
249
249
|
it('should recover "Zombie" tasks (crashed workers)', async () => {
|
|
250
250
|
const task = await queue.enqueue('foo', { foo: 'zombie' });
|
|
251
251
|
await queue.dequeue(); // Task is now Running with a token
|
|
252
|
-
// processTimeout is
|
|
253
|
-
await timeout(
|
|
252
|
+
// processTimeout is 50ms. Wait for it to expire.
|
|
253
|
+
await timeout(100);
|
|
254
254
|
await queue.maintenance();
|
|
255
255
|
const recovered = await queue.getTask(task.id);
|
|
256
256
|
expect(recovered?.status).toBe(TaskStatus.Pending);
|
|
@@ -260,10 +260,10 @@ describe('PostgresQueue (Distributed Task Orchestration)', () => {
|
|
|
260
260
|
it('should fail tasks that exceed Hard Execution Timeout via prune', async () => {
|
|
261
261
|
// Re-configure queue with very short execution timeout
|
|
262
262
|
const queueProvider = injector.resolve(TaskQueueProvider);
|
|
263
|
-
const shortQueue = queueProvider.get(`prune-test-${Date.now()}`, { maxExecutionTime:
|
|
263
|
+
const shortQueue = queueProvider.get(`prune-test-${Date.now()}`, { maxExecutionTime: 50 });
|
|
264
264
|
const task = await shortQueue.enqueue('foo', { foo: 'long-running' });
|
|
265
265
|
await shortQueue.dequeue();
|
|
266
|
-
await timeout(
|
|
266
|
+
await timeout(75);
|
|
267
267
|
await shortQueue.maintenance();
|
|
268
268
|
const updated = await shortQueue.getTask(task.id);
|
|
269
269
|
expect(updated?.status).toBe(TaskStatus.Dead);
|
|
@@ -274,7 +274,7 @@ describe('PostgresQueue (Distributed Task Orchestration)', () => {
|
|
|
274
274
|
const task = await queue.enqueue('foo', { foo: 'work' });
|
|
275
275
|
const dequeued = await queue.dequeue();
|
|
276
276
|
const initialLock = dequeued.visibilityDeadline;
|
|
277
|
-
await timeout(
|
|
277
|
+
await timeout(20);
|
|
278
278
|
const touched = await queue.touch(dequeued);
|
|
279
279
|
expect(touched?.visibilityDeadline > initialLock).toBe(true);
|
|
280
280
|
});
|
|
@@ -282,8 +282,8 @@ describe('PostgresQueue (Distributed Task Orchestration)', () => {
|
|
|
282
282
|
await queue.enqueue('foo', { foo: 'work' });
|
|
283
283
|
const dequeued = await queue.dequeue();
|
|
284
284
|
expect(dequeued).toBeDefined();
|
|
285
|
-
// processTimeout is
|
|
286
|
-
await timeout(
|
|
285
|
+
// processTimeout is 50ms. Wait for it to expire.
|
|
286
|
+
await timeout(100);
|
|
287
287
|
await queue.maintenance();
|
|
288
288
|
await queue.dequeue(); // Stolen by another worker (tries=2)
|
|
289
289
|
// Original worker tries to touch
|
|
@@ -14,7 +14,7 @@ describe('Worker & Base Class Tests', () => {
|
|
|
14
14
|
const queueProvider = injector.resolve(TaskQueueProvider);
|
|
15
15
|
const queueName = `worker-queue-${Date.now()}-${Math.random()}`;
|
|
16
16
|
queue = queueProvider.get(queueName, {
|
|
17
|
-
visibilityTimeout:
|
|
17
|
+
visibilityTimeout: 200, // Short visibility for testing lease loss
|
|
18
18
|
});
|
|
19
19
|
token = new CancellationToken();
|
|
20
20
|
});
|
|
@@ -36,10 +36,10 @@ describe('Worker & Base Class Tests', () => {
|
|
|
36
36
|
return TaskProcessResult.Complete();
|
|
37
37
|
});
|
|
38
38
|
// Wait until 2 tasks are processed
|
|
39
|
-
for (let i = 0; i <
|
|
39
|
+
for (let i = 0; i < 50; i++) {
|
|
40
40
|
if (processed.length === 2)
|
|
41
41
|
break;
|
|
42
|
-
await timeout(
|
|
42
|
+
await timeout(20);
|
|
43
43
|
}
|
|
44
44
|
token.set(); // Stop worker
|
|
45
45
|
expect(processed).toContain(1);
|
|
@@ -55,7 +55,7 @@ describe('Worker & Base Class Tests', () => {
|
|
|
55
55
|
queue.process({ cancellationSignal: token }, async () => {
|
|
56
56
|
throw new Error('worker error');
|
|
57
57
|
});
|
|
58
|
-
await timeout(
|
|
58
|
+
await timeout(50);
|
|
59
59
|
token.set();
|
|
60
60
|
const updated = await queue.getTask(task.id);
|
|
61
61
|
expect(updated?.status).toBe(TaskStatus.Pending); // Should retry
|
|
@@ -66,12 +66,12 @@ describe('Worker & Base Class Tests', () => {
|
|
|
66
66
|
const task = await queue.enqueue('long', {});
|
|
67
67
|
let executed = false;
|
|
68
68
|
queue.process({ cancellationSignal: token }, async (_context) => {
|
|
69
|
-
// Simulate long work > visibilityTimeout (
|
|
70
|
-
await timeout(
|
|
69
|
+
// Simulate long work > visibilityTimeout (200ms)
|
|
70
|
+
await timeout(300);
|
|
71
71
|
executed = true;
|
|
72
72
|
return TaskProcessResult.Complete();
|
|
73
73
|
});
|
|
74
|
-
await timeout(
|
|
74
|
+
await timeout(500);
|
|
75
75
|
token.set();
|
|
76
76
|
expect(executed).toBe(true);
|
|
77
77
|
const updated = await queue.getTask(task.id);
|
|
@@ -91,10 +91,10 @@ describe('Worker & Base Class Tests', () => {
|
|
|
91
91
|
}
|
|
92
92
|
return TaskProcessResult.Complete();
|
|
93
93
|
});
|
|
94
|
-
for (let i = 0; i <
|
|
94
|
+
for (let i = 0; i < 50; i++) {
|
|
95
95
|
if (processed.size === 2)
|
|
96
96
|
break;
|
|
97
|
-
await timeout(
|
|
97
|
+
await timeout(20);
|
|
98
98
|
}
|
|
99
99
|
token.set();
|
|
100
100
|
const uFail = await queue.getTask(tFail.id);
|
|
@@ -129,10 +129,10 @@ describe('Worker & Base Class Tests', () => {
|
|
|
129
129
|
executed = true;
|
|
130
130
|
return TaskProcessResult.Complete();
|
|
131
131
|
});
|
|
132
|
-
for (let i = 0; i <
|
|
132
|
+
for (let i = 0; i < 50; i++) {
|
|
133
133
|
if (executed)
|
|
134
134
|
break;
|
|
135
|
-
await timeout(
|
|
135
|
+
await timeout(20);
|
|
136
136
|
}
|
|
137
137
|
token.set();
|
|
138
138
|
expect(executed).toBe(true);
|
|
@@ -159,7 +159,7 @@ describe('Worker & Base Class Tests', () => {
|
|
|
159
159
|
if (finalAttemptValues.length === 2)
|
|
160
160
|
break;
|
|
161
161
|
testQueue.notify();
|
|
162
|
-
await timeout(
|
|
162
|
+
await timeout(20);
|
|
163
163
|
}
|
|
164
164
|
token.set();
|
|
165
165
|
expect(finalAttemptValues).toEqual([false, true]);
|
|
@@ -2,6 +2,7 @@ import type { PoolConfig } from 'pg';
|
|
|
2
2
|
import { type AuthenticationAncillaryService } from '../authentication/server/index.js';
|
|
3
3
|
import { Injector } from '../injector/index.js';
|
|
4
4
|
import { LogLevel } from '../logger/index.js';
|
|
5
|
+
import { type S3ObjectStorageProviderConfig } from '../object-storage/s3/index.js';
|
|
5
6
|
import { Database } from '../orm/server/index.js';
|
|
6
7
|
import type { Type } from '../types/index.js';
|
|
7
8
|
export type IntegrationTestOptions = {
|
|
@@ -13,6 +14,7 @@ export type IntegrationTestOptions = {
|
|
|
13
14
|
baseUrl?: string;
|
|
14
15
|
port?: number;
|
|
15
16
|
};
|
|
17
|
+
s3?: Partial<S3ObjectStorageProviderConfig>;
|
|
16
18
|
logLevels?: Record<string, LogLevel>;
|
|
17
19
|
modules?: {
|
|
18
20
|
api?: boolean;
|
|
@@ -98,7 +98,11 @@ export async function setupIntegrationTest(options = {}) {
|
|
|
98
98
|
}
|
|
99
99
|
if (options.modules?.authentication) {
|
|
100
100
|
configureAuthenticationServer({
|
|
101
|
-
serviceOptions: {
|
|
101
|
+
serviceOptions: {
|
|
102
|
+
secret: 'test-secret',
|
|
103
|
+
hashIterations: 10,
|
|
104
|
+
signingSecretsDerivationIterations: 10,
|
|
105
|
+
},
|
|
102
106
|
authenticationAncillaryService: options.authenticationAncillaryService,
|
|
103
107
|
injector,
|
|
104
108
|
});
|
|
@@ -121,13 +125,15 @@ export async function setupIntegrationTest(options = {}) {
|
|
|
121
125
|
await runInInjectionContext(injector, migrateDocumentManagementSchema);
|
|
122
126
|
}
|
|
123
127
|
if (options.modules?.objectStorage) {
|
|
128
|
+
const bucketPerModule = options.s3?.bucketPerModule ?? configParser.boolean('S3_BUCKET_PER_MODULE', true);
|
|
124
129
|
configureS3ObjectStorage({
|
|
125
|
-
endpoint: configParser.string('S3_ENDPOINT', 'http://127.0.0.1:9000'),
|
|
126
|
-
accessKey: configParser.string('S3_ACCESS_KEY', 'tstdl-dev'),
|
|
127
|
-
secretKey: configParser.string('S3_SECRET_KEY', 'tstdl-dev'),
|
|
128
|
-
bucket: configParser.string('S3_BUCKET', 'test-bucket'),
|
|
129
|
-
|
|
130
|
-
|
|
130
|
+
endpoint: options.s3?.endpoint ?? configParser.string('S3_ENDPOINT', 'http://127.0.0.1:9000'),
|
|
131
|
+
accessKey: options.s3?.accessKey ?? configParser.string('S3_ACCESS_KEY', 'tstdl-dev'),
|
|
132
|
+
secretKey: options.s3?.secretKey ?? configParser.string('S3_SECRET_KEY', 'tstdl-dev'),
|
|
133
|
+
bucket: bucketPerModule ? undefined : (options.s3?.bucket ?? configParser.string('S3_BUCKET', 'test-bucket')),
|
|
134
|
+
bucketPerModule,
|
|
135
|
+
region: options.s3?.region ?? configParser.string('S3_REGION', 'us-east-1'),
|
|
136
|
+
forcePathStyle: options.s3?.forcePathStyle ?? configParser.boolean('S3_FORCE_PATH_STYLE', true),
|
|
131
137
|
injector,
|
|
132
138
|
});
|
|
133
139
|
}
|
package/utils/backoff.d.ts
CHANGED
|
@@ -1,5 +1,6 @@
|
|
|
1
1
|
import type { CancellationSignal } from '../cancellation/token.js';
|
|
2
2
|
export type BackoffStrategy = 'linear' | 'exponential';
|
|
3
|
+
export type JitterStrategy = 'full' | 'proportional';
|
|
3
4
|
/**
|
|
4
5
|
* Configuration for the backoff behavior.
|
|
5
6
|
*/
|
|
@@ -11,6 +12,13 @@ export type BackoffOptions = {
|
|
|
11
12
|
* @default 'exponential'
|
|
12
13
|
*/
|
|
13
14
|
strategy?: BackoffStrategy;
|
|
15
|
+
/**
|
|
16
|
+
* The strategy to use for applying jitter.
|
|
17
|
+
* - `full`: Random value between 0 and the calculated delay.
|
|
18
|
+
* - `proportional`: Random value within `± jitter * delay`.
|
|
19
|
+
* @default 'proportional'
|
|
20
|
+
*/
|
|
21
|
+
jitterStrategy?: JitterStrategy;
|
|
14
22
|
/**
|
|
15
23
|
* The initial delay in milliseconds. Must be non-negative.
|
|
16
24
|
* @default 1000
|
|
@@ -32,7 +40,8 @@ export type BackoffOptions = {
|
|
|
32
40
|
/**
|
|
33
41
|
* A factor to randomize the delay, e.g., 0.1 for 10% jitter.
|
|
34
42
|
* This helps prevent the "thundering herd" problem in distributed systems.
|
|
35
|
-
*
|
|
43
|
+
* For `proportional` jitter, the actual delay will be `delay ± delay * jitter`.
|
|
44
|
+
* For `full` jitter, this value is ignored.
|
|
36
45
|
* Must be a value between 0 and 1.
|
|
37
46
|
* @default 0.15
|
|
38
47
|
*/
|
|
@@ -59,14 +68,20 @@ export type BackoffLoopController = {
|
|
|
59
68
|
/**
|
|
60
69
|
* Schedules a backoff delay before the next iteration of the loop.
|
|
61
70
|
* If this is not called, the backoff delay is reset for the next attempt.
|
|
71
|
+
* @param delay An optional explicit delay in milliseconds to use for the next backoff.
|
|
72
|
+
* If not provided, the backoff strategy is used.
|
|
62
73
|
*/
|
|
63
|
-
backoff: () => void;
|
|
74
|
+
backoff: (delay?: number) => void;
|
|
64
75
|
/**
|
|
65
76
|
* Immediately breaks out of the loop.
|
|
66
77
|
*/
|
|
67
78
|
break: () => void;
|
|
68
79
|
};
|
|
69
|
-
export type
|
|
80
|
+
export type BackoffStatus = {
|
|
81
|
+
attempt: number;
|
|
82
|
+
currentDelay: number;
|
|
83
|
+
};
|
|
84
|
+
export type BackoffLoopFunction = (controller: BackoffLoopController, cancellationSignal: CancellationSignal, status: BackoffStatus) => unknown;
|
|
70
85
|
/**
|
|
71
86
|
* A function yielded by `backoffGenerator` to control the next iteration.
|
|
72
87
|
*/
|
|
@@ -82,6 +97,7 @@ export type BackoffGeneratorCallback = (options?: {
|
|
|
82
97
|
*/
|
|
83
98
|
export declare const DEFAULT_BACKOFF_OPTIONS: {
|
|
84
99
|
readonly strategy: "exponential";
|
|
100
|
+
readonly jitterStrategy: "proportional";
|
|
85
101
|
readonly initialDelay: 1000;
|
|
86
102
|
readonly increase: 2;
|
|
87
103
|
readonly maximumDelay: 30000;
|
|
@@ -93,6 +109,14 @@ export declare const DEFAULT_BACKOFF_OPTIONS: {
|
|
|
93
109
|
export declare class BackoffHelper {
|
|
94
110
|
private readonly options;
|
|
95
111
|
private currentDelay;
|
|
112
|
+
private attempt;
|
|
113
|
+
/**
|
|
114
|
+
* The current status of the backoff.
|
|
115
|
+
*/
|
|
116
|
+
get status(): {
|
|
117
|
+
attempt: number;
|
|
118
|
+
currentDelay: number;
|
|
119
|
+
};
|
|
96
120
|
/**
|
|
97
121
|
* Creates a new BackoffHelper.
|
|
98
122
|
* @param options Partial backoff options, which will be merged with sane defaults.
|
package/utils/backoff.js
CHANGED
|
@@ -8,6 +8,7 @@ import { isDefined } from './type-guards.js';
|
|
|
8
8
|
*/
|
|
9
9
|
export const DEFAULT_BACKOFF_OPTIONS = {
|
|
10
10
|
strategy: 'exponential',
|
|
11
|
+
jitterStrategy: 'proportional',
|
|
11
12
|
initialDelay: 1000,
|
|
12
13
|
increase: 2,
|
|
13
14
|
maximumDelay: 30_000, // 30 seconds
|
|
@@ -19,6 +20,16 @@ export const DEFAULT_BACKOFF_OPTIONS = {
|
|
|
19
20
|
export class BackoffHelper {
|
|
20
21
|
options;
|
|
21
22
|
currentDelay;
|
|
23
|
+
attempt;
|
|
24
|
+
/**
|
|
25
|
+
* The current status of the backoff.
|
|
26
|
+
*/
|
|
27
|
+
get status() {
|
|
28
|
+
return {
|
|
29
|
+
attempt: this.attempt,
|
|
30
|
+
currentDelay: this.currentDelay,
|
|
31
|
+
};
|
|
32
|
+
}
|
|
22
33
|
/**
|
|
23
34
|
* Creates a new BackoffHelper.
|
|
24
35
|
* @param options Partial backoff options, which will be merged with sane defaults.
|
|
@@ -39,12 +50,14 @@ export class BackoffHelper {
|
|
|
39
50
|
console.warn('Using an exponential backoff with an increase factor <= 1 is not recommended as the delay will not grow.');
|
|
40
51
|
}
|
|
41
52
|
this.currentDelay = this.options.initialDelay;
|
|
53
|
+
this.attempt = 0;
|
|
42
54
|
}
|
|
43
55
|
/**
|
|
44
56
|
* Resets the current delay to the initial delay.
|
|
45
57
|
*/
|
|
46
58
|
reset() {
|
|
47
59
|
this.currentDelay = this.options.initialDelay;
|
|
60
|
+
this.attempt = 0;
|
|
48
61
|
}
|
|
49
62
|
/**
|
|
50
63
|
* Calculates and returns the next backoff delay based on the configured strategy.
|
|
@@ -52,13 +65,18 @@ export class BackoffHelper {
|
|
|
52
65
|
* @returns The next delay in milliseconds.
|
|
53
66
|
*/
|
|
54
67
|
getNextDelay() {
|
|
55
|
-
const
|
|
68
|
+
const baseDelay = this.currentDelay;
|
|
69
|
+
const newDelay = getNewDelay(this.options.strategy, baseDelay, this.options.increase, this.options.maximumDelay);
|
|
56
70
|
this.currentDelay = newDelay;
|
|
71
|
+
this.attempt++;
|
|
72
|
+
if (this.options.jitterStrategy == 'full') {
|
|
73
|
+
return randomFloat(0, baseDelay);
|
|
74
|
+
}
|
|
57
75
|
if (this.options.jitter > 0) {
|
|
58
|
-
const jitterAmount =
|
|
59
|
-
return Math.max(0,
|
|
76
|
+
const jitterAmount = baseDelay * randomFloat(-this.options.jitter, this.options.jitter); // ±jitter
|
|
77
|
+
return Math.max(0, baseDelay + jitterAmount);
|
|
60
78
|
}
|
|
61
|
-
return
|
|
79
|
+
return baseDelay;
|
|
62
80
|
}
|
|
63
81
|
}
|
|
64
82
|
/**
|
|
@@ -72,9 +90,9 @@ export class BackoffHelper {
|
|
|
72
90
|
*/
|
|
73
91
|
export async function autoBackoffLoop(loopFunction, options = {}) {
|
|
74
92
|
const { errorHandler, ...loopOptions } = options;
|
|
75
|
-
await backoffLoop(async (controller, signal) => {
|
|
93
|
+
await backoffLoop(async (controller, signal, status) => {
|
|
76
94
|
try {
|
|
77
|
-
await loopFunction(controller, signal);
|
|
95
|
+
await loopFunction(controller, signal, status);
|
|
78
96
|
}
|
|
79
97
|
catch (error) {
|
|
80
98
|
errorHandler?.(error);
|
|
@@ -95,22 +113,26 @@ export async function backoffLoop(loopFunction, options = {}) {
|
|
|
95
113
|
const backoffHelper = new BackoffHelper(backoffOptions);
|
|
96
114
|
const loopToken = new CancellationToken();
|
|
97
115
|
let shouldBackoff = false;
|
|
116
|
+
let customBackoffDelay;
|
|
98
117
|
if (isDefined(cancellationSignal)) {
|
|
99
118
|
loopToken.inherit(cancellationSignal, { set: true, unset: false, complete: false, error: false });
|
|
100
119
|
}
|
|
101
120
|
const controller = {
|
|
102
|
-
backoff: () =>
|
|
121
|
+
backoff: (delay) => {
|
|
122
|
+
shouldBackoff = true;
|
|
123
|
+
customBackoffDelay = delay;
|
|
124
|
+
},
|
|
103
125
|
break: () => loopToken.set(),
|
|
104
126
|
};
|
|
105
127
|
while (loopToken.isUnset) {
|
|
106
|
-
await loopFunction(controller, loopToken.signal);
|
|
128
|
+
await loopFunction(controller, loopToken.signal, backoffHelper.status);
|
|
107
129
|
// Exit immediately if the loop function requested a break.
|
|
108
130
|
if (loopToken.isSet) {
|
|
109
131
|
return;
|
|
110
132
|
}
|
|
111
133
|
if (shouldBackoff) {
|
|
112
134
|
shouldBackoff = false;
|
|
113
|
-
const delay = backoffHelper.getNextDelay();
|
|
135
|
+
const delay = customBackoffDelay ?? backoffHelper.getNextDelay();
|
|
114
136
|
await cancelableTimeout(delay, loopToken.signal);
|
|
115
137
|
}
|
|
116
138
|
else {
|
package/utils/index.d.ts
CHANGED
|
@@ -39,6 +39,7 @@ export * from './random.js';
|
|
|
39
39
|
export * from './reactive-value-to-signal.js';
|
|
40
40
|
export * from './reflection.js';
|
|
41
41
|
export * from './repl.js';
|
|
42
|
+
export * from './retry-with-backoff.js';
|
|
42
43
|
export * from './set.js';
|
|
43
44
|
export * from './singleton.js';
|
|
44
45
|
export * from './sort.js';
|
package/utils/index.js
CHANGED
|
@@ -39,6 +39,7 @@ export * from './random.js';
|
|
|
39
39
|
export * from './reactive-value-to-signal.js';
|
|
40
40
|
export * from './reflection.js';
|
|
41
41
|
export * from './repl.js';
|
|
42
|
+
export * from './retry-with-backoff.js';
|
|
42
43
|
export * from './set.js';
|
|
43
44
|
export * from './singleton.js';
|
|
44
45
|
export * from './sort.js';
|
|
@@ -0,0 +1,22 @@
|
|
|
1
|
+
import type { CancellationSignal } from '../cancellation/token.js';
|
|
2
|
+
import { type BackoffOptions } from './backoff.js';
|
|
3
|
+
export type RetryWithBackoffOptions = {
|
|
4
|
+
/** Timeout for each individual attempt in milliseconds. */
|
|
5
|
+
timeout?: number;
|
|
6
|
+
/** Retry configuration. If not provided, it will not retry. */
|
|
7
|
+
retry?: {
|
|
8
|
+
/** Whether to retry when an error occurs. */
|
|
9
|
+
shouldRetry: (error: Error, attempt: number) => boolean | Promise<boolean>;
|
|
10
|
+
/** Backoff configuration for retries. */
|
|
11
|
+
backoff?: BackoffOptions;
|
|
12
|
+
};
|
|
13
|
+
/** Optional cancellation signal to abort the execution. */
|
|
14
|
+
signal?: CancellationSignal;
|
|
15
|
+
};
|
|
16
|
+
/**
|
|
17
|
+
* Executes a function with a specified policy including timeout and retries with backoff.
|
|
18
|
+
* @param func The function to execute. It receives a cancellation signal that is set when the attempt times out or is cancelled.
|
|
19
|
+
* @param options The retry options.
|
|
20
|
+
* @returns The result of the function.
|
|
21
|
+
*/
|
|
22
|
+
export declare function retryWithBackoff<T>(func: (signal: CancellationSignal) => T | Promise<T>, options?: RetryWithBackoffOptions): Promise<T>;
|
|
@@ -0,0 +1,64 @@
|
|
|
1
|
+
import { HttpError } from '../http/http.error.js';
|
|
2
|
+
import { isDefined, isInstanceOf, isNumber, isString } from '../utils/type-guards.js';
|
|
3
|
+
import { backoffLoop } from './backoff.js';
|
|
4
|
+
import { withTimeout } from './timing.js';
|
|
5
|
+
/**
|
|
6
|
+
* Executes a function with a specified policy including timeout and retries with backoff.
|
|
7
|
+
* @param func The function to execute. It receives a cancellation signal that is set when the attempt times out or is cancelled.
|
|
8
|
+
* @param options The retry options.
|
|
9
|
+
* @returns The result of the function.
|
|
10
|
+
*/
|
|
11
|
+
export async function retryWithBackoff(func, options = {}) {
|
|
12
|
+
const { timeout, retry, signal } = options;
|
|
13
|
+
let result;
|
|
14
|
+
let lastError;
|
|
15
|
+
await backoffLoop(async (controller, loopSignal, status) => {
|
|
16
|
+
try {
|
|
17
|
+
const attemptSignal = isDefined(signal)
|
|
18
|
+
? signal.createChild().inherit(loopSignal)
|
|
19
|
+
: loopSignal;
|
|
20
|
+
if (isDefined(timeout)) {
|
|
21
|
+
result = await withTimeout(timeout, async () => await func(attemptSignal));
|
|
22
|
+
}
|
|
23
|
+
else {
|
|
24
|
+
result = await func(attemptSignal);
|
|
25
|
+
}
|
|
26
|
+
controller.break();
|
|
27
|
+
}
|
|
28
|
+
catch (error) {
|
|
29
|
+
lastError = error;
|
|
30
|
+
if (isDefined(retry) && await retry.shouldRetry(lastError, status.attempt)) {
|
|
31
|
+
const retryAfter = tryGetRetryAfterDelay(lastError);
|
|
32
|
+
controller.backoff(retryAfter);
|
|
33
|
+
}
|
|
34
|
+
else {
|
|
35
|
+
throw error;
|
|
36
|
+
}
|
|
37
|
+
}
|
|
38
|
+
}, {
|
|
39
|
+
...retry?.backoff,
|
|
40
|
+
cancellationSignal: signal,
|
|
41
|
+
});
|
|
42
|
+
return result;
|
|
43
|
+
}
|
|
44
|
+
function tryGetRetryAfterDelay(error) {
|
|
45
|
+
if (isInstanceOf(error, HttpError)) {
|
|
46
|
+
const retryAfter = error.responseInstance?.headers.tryGetSingle('Retry-After');
|
|
47
|
+
if (isDefined(retryAfter)) {
|
|
48
|
+
if (isNumber(retryAfter)) {
|
|
49
|
+
return retryAfter * 1000;
|
|
50
|
+
}
|
|
51
|
+
if (isString(retryAfter)) {
|
|
52
|
+
const seconds = Number.parseInt(retryAfter, 10);
|
|
53
|
+
if (!Number.isNaN(seconds)) {
|
|
54
|
+
return seconds * 1000;
|
|
55
|
+
}
|
|
56
|
+
const date = Date.parse(retryAfter);
|
|
57
|
+
if (!Number.isNaN(date)) {
|
|
58
|
+
return Math.max(0, date - Date.now());
|
|
59
|
+
}
|
|
60
|
+
}
|
|
61
|
+
}
|
|
62
|
+
}
|
|
63
|
+
return undefined;
|
|
64
|
+
}
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
export {};
|
|
@@ -0,0 +1,41 @@
|
|
|
1
|
+
import { describe, expect, it } from 'vitest';
|
|
2
|
+
import { BackoffHelper } from '../backoff.js';
|
|
3
|
+
describe('BackoffHelper', () => {
|
|
4
|
+
it('should support full jitter', () => {
|
|
5
|
+
const helper = new BackoffHelper({
|
|
6
|
+
initialDelay: 1000,
|
|
7
|
+
jitter: 1,
|
|
8
|
+
jitterStrategy: 'full',
|
|
9
|
+
});
|
|
10
|
+
for (let i = 0; i < 100; i++) {
|
|
11
|
+
const delay = helper.getNextDelay();
|
|
12
|
+
// Full jitter should be between 0 and the base delay (1000 for first call, then 2000, 4000, etc.)
|
|
13
|
+
expect(delay).toBeGreaterThanOrEqual(0);
|
|
14
|
+
expect(delay).toBeLessThanOrEqual(1000 * Math.pow(2, i));
|
|
15
|
+
}
|
|
16
|
+
});
|
|
17
|
+
it('should support proportional jitter (default)', () => {
|
|
18
|
+
const helper = new BackoffHelper({
|
|
19
|
+
initialDelay: 1000,
|
|
20
|
+
jitter: 0.1,
|
|
21
|
+
jitterStrategy: 'proportional',
|
|
22
|
+
});
|
|
23
|
+
for (let i = 0; i < 1; i++) {
|
|
24
|
+
const delay = helper.getNextDelay();
|
|
25
|
+
// Proportional jitter should be 1000 ± 100 (900 to 1100)
|
|
26
|
+
expect(delay).toBeGreaterThanOrEqual(900);
|
|
27
|
+
expect(delay).toBeLessThanOrEqual(1100);
|
|
28
|
+
}
|
|
29
|
+
});
|
|
30
|
+
it('should provide status emission', () => {
|
|
31
|
+
const helper = new BackoffHelper({ initialDelay: 1000 });
|
|
32
|
+
expect(helper.status.attempt).toBe(0);
|
|
33
|
+
expect(helper.status.currentDelay).toBe(1000);
|
|
34
|
+
helper.getNextDelay();
|
|
35
|
+
expect(helper.status.attempt).toBe(1);
|
|
36
|
+
expect(helper.status.currentDelay).toBe(2000);
|
|
37
|
+
helper.reset();
|
|
38
|
+
expect(helper.status.attempt).toBe(0);
|
|
39
|
+
expect(helper.status.currentDelay).toBe(1000);
|
|
40
|
+
});
|
|
41
|
+
});
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
export {};
|
|
@@ -0,0 +1,49 @@
|
|
|
1
|
+
import { TimeoutError } from '../../errors/timeout.error.js';
|
|
2
|
+
import { describe, expect, it } from 'vitest';
|
|
3
|
+
import { retryWithBackoff } from '../retry-with-backoff.js';
|
|
4
|
+
describe('retryWithBackoff', () => {
|
|
5
|
+
it('should return result on success', async () => {
|
|
6
|
+
const result = await retryWithBackoff(async () => 'success');
|
|
7
|
+
expect(result).toBe('success');
|
|
8
|
+
});
|
|
9
|
+
it('should retry on failure if shouldRetry returns true', async () => {
|
|
10
|
+
let attempts = 0;
|
|
11
|
+
const result = await retryWithBackoff(async () => {
|
|
12
|
+
attempts++;
|
|
13
|
+
if (attempts < 3) {
|
|
14
|
+
throw new Error('fail');
|
|
15
|
+
}
|
|
16
|
+
return 'success';
|
|
17
|
+
}, {
|
|
18
|
+
retry: {
|
|
19
|
+
shouldRetry: (_error, attempt) => attempt < 5,
|
|
20
|
+
backoff: { initialDelay: 0 },
|
|
21
|
+
},
|
|
22
|
+
});
|
|
23
|
+
expect(result).toBe('success');
|
|
24
|
+
expect(attempts).toBe(3);
|
|
25
|
+
});
|
|
26
|
+
it('should throw after max retries', async () => {
|
|
27
|
+
let attempts = 0;
|
|
28
|
+
const execute = retryWithBackoff(async () => {
|
|
29
|
+
attempts++;
|
|
30
|
+
throw new Error('fail');
|
|
31
|
+
}, {
|
|
32
|
+
retry: {
|
|
33
|
+
shouldRetry: (_error, attempt) => attempt < 2,
|
|
34
|
+
backoff: { initialDelay: 0 },
|
|
35
|
+
},
|
|
36
|
+
});
|
|
37
|
+
await expect(execute).rejects.toThrow('fail');
|
|
38
|
+
expect(attempts).toBe(3); // 0, 1, 2
|
|
39
|
+
});
|
|
40
|
+
it('should timeout if execution takes too long', async () => {
|
|
41
|
+
const execute = retryWithBackoff(async () => {
|
|
42
|
+
await new Promise((resolve) => setTimeout(resolve, 100));
|
|
43
|
+
return 'success';
|
|
44
|
+
}, {
|
|
45
|
+
timeout: 50,
|
|
46
|
+
});
|
|
47
|
+
await expect(execute).rejects.toThrow(TimeoutError);
|
|
48
|
+
});
|
|
49
|
+
});
|