@volontariapp/post-processors 1.0.0-snap-856af13 → 1.0.0-snap-fb20f49
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/constants/config.default.d.ts +27 -0
- package/dist/constants/config.default.d.ts.map +1 -0
- package/dist/constants/config.default.js +29 -0
- package/dist/constants/config.default.js.map +1 -0
- package/dist/constants/index.d.ts +2 -0
- package/dist/constants/index.d.ts.map +1 -0
- package/dist/constants/index.js +2 -0
- package/dist/constants/index.js.map +1 -0
- package/dist/core/helpers/index.d.ts +3 -0
- package/dist/core/helpers/index.d.ts.map +1 -0
- package/dist/core/helpers/index.js +3 -0
- package/dist/core/helpers/index.js.map +1 -0
- package/dist/core/{redis-stream.helper.d.ts → helpers/redis-stream.helper.d.ts} +2 -2
- package/dist/core/helpers/redis-stream.helper.d.ts.map +1 -0
- package/dist/core/helpers/redis-stream.helper.js.map +1 -0
- package/dist/core/helpers/retry.helper.d.ts +24 -0
- package/dist/core/helpers/retry.helper.d.ts.map +1 -0
- package/dist/core/helpers/retry.helper.js +104 -0
- package/dist/core/helpers/retry.helper.js.map +1 -0
- package/dist/core/index.d.ts +7 -4
- package/dist/core/index.d.ts.map +1 -1
- package/dist/core/index.js +7 -4
- package/dist/core/index.js.map +1 -1
- package/dist/core/processors/base.post-processor.d.ts +46 -0
- package/dist/core/processors/base.post-processor.d.ts.map +1 -0
- package/dist/core/processors/base.post-processor.js +422 -0
- package/dist/core/processors/base.post-processor.js.map +1 -0
- package/dist/core/{batch.post-processor.d.ts → processors/batch.post-processor.d.ts} +3 -2
- package/dist/core/processors/batch.post-processor.d.ts.map +1 -0
- package/dist/core/{batch.post-processor.js → processors/batch.post-processor.js} +56 -5
- package/dist/core/processors/batch.post-processor.js.map +1 -0
- package/dist/core/processors/index.d.ts +4 -0
- package/dist/core/processors/index.d.ts.map +1 -0
- package/dist/core/processors/index.js +4 -0
- package/dist/core/processors/index.js.map +1 -0
- package/dist/core/{single.post-processor.d.ts → processors/single.post-processor.d.ts} +3 -1
- package/dist/core/processors/single.post-processor.d.ts.map +1 -0
- package/dist/core/processors/single.post-processor.js +120 -0
- package/dist/core/processors/single.post-processor.js.map +1 -0
- package/dist/core/validators/circuit-breaker.d.ts +27 -0
- package/dist/core/validators/circuit-breaker.d.ts.map +1 -0
- package/dist/core/validators/circuit-breaker.js +83 -0
- package/dist/core/validators/circuit-breaker.js.map +1 -0
- package/dist/core/validators/index.d.ts +2 -0
- package/dist/core/validators/index.d.ts.map +1 -0
- package/dist/core/validators/index.js +2 -0
- package/dist/core/validators/index.js.map +1 -0
- package/dist/core/validators/options-validator.d.ts +5 -0
- package/dist/core/validators/options-validator.d.ts.map +1 -0
- package/dist/core/validators/options-validator.js +90 -0
- package/dist/core/validators/options-validator.js.map +1 -0
- package/dist/enums/circuit-breaker-state.enum.d.ts +6 -0
- package/dist/enums/circuit-breaker-state.enum.d.ts.map +1 -0
- package/dist/enums/circuit-breaker-state.enum.js +7 -0
- package/dist/enums/circuit-breaker-state.enum.js.map +1 -0
- package/dist/enums/index.d.ts +2 -0
- package/dist/enums/index.d.ts.map +1 -0
- package/dist/enums/index.js +2 -0
- package/dist/enums/index.js.map +1 -0
- package/dist/index.d.ts +2 -0
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +2 -0
- package/dist/index.js.map +1 -1
- package/dist/interfaces/index.d.ts +7 -3
- package/dist/interfaces/index.d.ts.map +1 -1
- package/dist/interfaces/{batch-event-item.interface.d.ts → processors/batch-event-item.interface.d.ts} +1 -1
- package/dist/interfaces/processors/batch-event-item.interface.d.ts.map +1 -0
- package/dist/interfaces/processors/batch-event-item.interface.js.map +1 -0
- package/dist/interfaces/processors/normalized-post-processor-options.interface.d.ts +6 -0
- package/dist/interfaces/processors/normalized-post-processor-options.interface.d.ts.map +1 -0
- package/dist/interfaces/processors/normalized-post-processor-options.interface.js +2 -0
- package/dist/interfaces/processors/normalized-post-processor-options.interface.js.map +1 -0
- package/dist/interfaces/processors/pending-message-info.interface.d.ts.map +1 -0
- package/dist/interfaces/processors/pending-message-info.interface.js.map +1 -0
- package/dist/interfaces/processors/post-processor-options.interface.d.ts +21 -0
- package/dist/interfaces/processors/post-processor-options.interface.d.ts.map +1 -0
- package/dist/interfaces/processors/post-processor-options.interface.js.map +1 -0
- package/dist/interfaces/retry/retry-metadata.interface.d.ts +6 -0
- package/dist/interfaces/retry/retry-metadata.interface.d.ts.map +1 -0
- package/dist/interfaces/retry/retry-metadata.interface.js +2 -0
- package/dist/interfaces/retry/retry-metadata.interface.js.map +1 -0
- package/dist/interfaces/retry/retry-options.interface.d.ts +8 -0
- package/dist/interfaces/retry/retry-options.interface.d.ts.map +1 -0
- package/dist/interfaces/retry/retry-options.interface.js +2 -0
- package/dist/interfaces/retry/retry-options.interface.js.map +1 -0
- package/dist/interfaces/validators/circuit-breaker-config.interface.d.ts +6 -0
- package/dist/interfaces/validators/circuit-breaker-config.interface.d.ts.map +1 -0
- package/dist/interfaces/validators/circuit-breaker-config.interface.js +2 -0
- package/dist/interfaces/validators/circuit-breaker-config.interface.js.map +1 -0
- package/dist/test/data-source.d.ts.map +1 -0
- package/dist/test/data-source.js.map +1 -0
- package/dist/test/migrations/1776783577425-CreateJobsOutbox.d.ts.map +1 -0
- package/dist/test/migrations/1776783577425-CreateJobsOutbox.js.map +1 -0
- package/dist/test/migrations/1778328780881-InitialSchemaJobAudit.d.ts.map +1 -0
- package/dist/test/migrations/1778328780881-InitialSchemaJobAudit.js.map +1 -0
- package/dist/test/specs/batch-post-processor/batch-post-processor.int.spec.js +92 -4
- package/dist/test/specs/batch-post-processor/batch-post-processor.int.spec.js.map +1 -1
- package/dist/test/specs/batch-post-processor/batch-post-processor.unit.spec.js +174 -7
- package/dist/test/specs/batch-post-processor/batch-post-processor.unit.spec.js.map +1 -1
- package/dist/test/specs/helpers/redis-stream-helper.int.spec.d.ts +2 -0
- package/dist/test/specs/helpers/redis-stream-helper.int.spec.d.ts.map +1 -0
- package/dist/test/specs/helpers/redis-stream-helper.int.spec.js +56 -0
- package/dist/test/specs/helpers/redis-stream-helper.int.spec.js.map +1 -0
- package/dist/test/specs/helpers/redis-stream-helper.unit.spec.d.ts +2 -0
- package/dist/test/specs/helpers/redis-stream-helper.unit.spec.d.ts.map +1 -0
- package/dist/test/specs/helpers/redis-stream-helper.unit.spec.js +102 -0
- package/dist/test/specs/helpers/redis-stream-helper.unit.spec.js.map +1 -0
- package/dist/test/specs/helpers/retry-helper.int.spec.d.ts +2 -0
- package/dist/test/specs/helpers/retry-helper.int.spec.d.ts.map +1 -0
- package/dist/test/specs/helpers/retry-helper.int.spec.js +101 -0
- package/dist/test/specs/helpers/retry-helper.int.spec.js.map +1 -0
- package/dist/test/specs/helpers/retry-helper.unit.spec.d.ts +2 -0
- package/dist/test/specs/helpers/retry-helper.unit.spec.d.ts.map +1 -0
- package/dist/test/specs/helpers/retry-helper.unit.spec.js +202 -0
- package/dist/test/specs/helpers/retry-helper.unit.spec.js.map +1 -0
- package/dist/test/specs/integration/circuit-breaker.integration.spec.d.ts +2 -0
- package/dist/test/specs/integration/circuit-breaker.integration.spec.d.ts.map +1 -0
- package/dist/test/specs/integration/circuit-breaker.integration.spec.js +112 -0
- package/dist/test/specs/integration/circuit-breaker.integration.spec.js.map +1 -0
- package/dist/test/specs/single-post-processor/single-post-processor-retry.int.spec.d.ts +2 -0
- package/dist/test/specs/single-post-processor/single-post-processor-retry.int.spec.d.ts.map +1 -0
- package/dist/test/specs/single-post-processor/single-post-processor-retry.int.spec.js +142 -0
- package/dist/test/specs/single-post-processor/single-post-processor-retry.int.spec.js.map +1 -0
- package/dist/test/specs/single-post-processor/single-post-processor.int.spec.js +49 -7
- package/dist/test/specs/single-post-processor/single-post-processor.int.spec.js.map +1 -1
- package/dist/test/specs/single-post-processor/single-post-processor.unit.spec.js +110 -22
- package/dist/test/specs/single-post-processor/single-post-processor.unit.spec.js.map +1 -1
- package/dist/test/specs/validators/options-validator.unit.spec.d.ts +2 -0
- package/dist/test/specs/validators/options-validator.unit.spec.d.ts.map +1 -0
- package/dist/test/specs/validators/options-validator.unit.spec.js +290 -0
- package/dist/test/specs/validators/options-validator.unit.spec.js.map +1 -0
- package/dist/test/utils/classes/test-batch-post-processor.class.d.ts +2 -1
- package/dist/test/utils/classes/test-batch-post-processor.class.d.ts.map +1 -1
- package/dist/test/utils/classes/test-batch-post-processor.class.js +4 -1
- package/dist/test/utils/classes/test-batch-post-processor.class.js.map +1 -1
- package/dist/test/utils/classes/test-post-processor.class.d.ts +2 -1
- package/dist/test/utils/classes/test-post-processor.class.d.ts.map +1 -1
- package/dist/test/utils/classes/test-post-processor.class.js +4 -1
- package/dist/test/utils/classes/test-post-processor.class.js.map +1 -1
- package/dist/test/utils/helpers/wait.helper.d.ts +2 -0
- package/dist/test/utils/helpers/wait.helper.d.ts.map +1 -0
- package/dist/test/utils/helpers/wait.helper.js +11 -0
- package/dist/test/utils/helpers/wait.helper.js.map +1 -0
- package/dist/test/utils/index.d.ts +1 -0
- package/dist/test/utils/index.d.ts.map +1 -1
- package/dist/test/utils/index.js +1 -0
- package/dist/test/utils/index.js.map +1 -1
- package/dist/test/utils/mocks/redis-call.mock.d.ts.map +1 -1
- package/dist/test/utils/mocks/redis-call.mock.js +9 -0
- package/dist/test/utils/mocks/redis-call.mock.js.map +1 -1
- package/dist/types/index.d.ts +1 -0
- package/dist/types/index.d.ts.map +1 -1
- package/dist/types/index.js +1 -1
- package/dist/types/index.js.map +1 -1
- package/dist/types/parse-event.types.d.ts +13 -0
- package/dist/types/parse-event.types.d.ts.map +1 -0
- package/dist/types/parse-event.types.js +2 -0
- package/dist/types/parse-event.types.js.map +1 -0
- package/package.json +1 -1
- package/dist/core/base.post-processor.d.ts +0 -27
- package/dist/core/base.post-processor.d.ts.map +0 -1
- package/dist/core/base.post-processor.js +0 -206
- package/dist/core/base.post-processor.js.map +0 -1
- package/dist/core/batch.post-processor.d.ts.map +0 -1
- package/dist/core/batch.post-processor.js.map +0 -1
- package/dist/core/redis-stream.helper.d.ts.map +0 -1
- package/dist/core/redis-stream.helper.js.map +0 -1
- package/dist/core/single.post-processor.d.ts.map +0 -1
- package/dist/core/single.post-processor.js +0 -58
- package/dist/core/single.post-processor.js.map +0 -1
- package/dist/data-source.d.ts.map +0 -1
- package/dist/data-source.js.map +0 -1
- package/dist/interfaces/batch-event-item.interface.d.ts.map +0 -1
- package/dist/interfaces/batch-event-item.interface.js.map +0 -1
- package/dist/interfaces/pending-message-info.interface.d.ts.map +0 -1
- package/dist/interfaces/pending-message-info.interface.js.map +0 -1
- package/dist/interfaces/post-processor-options.interface.d.ts +0 -11
- package/dist/interfaces/post-processor-options.interface.d.ts.map +0 -1
- package/dist/interfaces/post-processor-options.interface.js.map +0 -1
- package/dist/migrations/1776783577425-CreateJobsOutbox.d.ts.map +0 -1
- package/dist/migrations/1776783577425-CreateJobsOutbox.js.map +0 -1
- package/dist/migrations/1778328780881-InitialSchemaJobAudit.d.ts.map +0 -1
- package/dist/migrations/1778328780881-InitialSchemaJobAudit.js.map +0 -1
- /package/dist/core/{redis-stream.helper.js → helpers/redis-stream.helper.js} +0 -0
- /package/dist/interfaces/{batch-event-item.interface.js → processors/batch-event-item.interface.js} +0 -0
- /package/dist/interfaces/{pending-message-info.interface.d.ts → processors/pending-message-info.interface.d.ts} +0 -0
- /package/dist/interfaces/{pending-message-info.interface.js → processors/pending-message-info.interface.js} +0 -0
- /package/dist/interfaces/{post-processor-options.interface.js → processors/post-processor-options.interface.js} +0 -0
- /package/dist/{data-source.d.ts → test/data-source.d.ts} +0 -0
- /package/dist/{data-source.js → test/data-source.js} +0 -0
- /package/dist/{migrations → test/migrations}/1776783577425-CreateJobsOutbox.d.ts +0 -0
- /package/dist/{migrations → test/migrations}/1776783577425-CreateJobsOutbox.js +0 -0
- /package/dist/{migrations → test/migrations}/1778328780881-InitialSchemaJobAudit.d.ts +0 -0
- /package/dist/{migrations → test/migrations}/1778328780881-InitialSchemaJobAudit.js +0 -0
|
@@ -0,0 +1,101 @@
|
|
|
1
|
+
import { describe, it, expect, beforeEach, afterAll, beforeAll } from '@jest/globals';
|
|
2
|
+
import { Redis } from 'ioredis';
|
|
3
|
+
import { testRedisOptions } from '../../redis-config.js';
|
|
4
|
+
import { RetryHelper } from '../../../core/helpers/retry.helper.js';
|
|
5
|
+
describe('RetryHelper — Integration', () => {
|
|
6
|
+
let redis;
|
|
7
|
+
let helper;
|
|
8
|
+
beforeAll(() => {
|
|
9
|
+
redis = new Redis(testRedisOptions);
|
|
10
|
+
});
|
|
11
|
+
afterAll(async () => {
|
|
12
|
+
await redis.quit();
|
|
13
|
+
});
|
|
14
|
+
beforeEach(async () => {
|
|
15
|
+
await redis.flushdb();
|
|
16
|
+
helper = new RetryHelper({
|
|
17
|
+
maxRetries: 3,
|
|
18
|
+
initialDelayMs: 50,
|
|
19
|
+
maxDelayMs: 200,
|
|
20
|
+
backoffMultiplier: 2,
|
|
21
|
+
enableDlq: true,
|
|
22
|
+
});
|
|
23
|
+
});
|
|
24
|
+
it('should record, fetch, and clear retry metadata in a real Redis database', async () => {
|
|
25
|
+
const groupName = 'test-group';
|
|
26
|
+
const messageId = '1-0';
|
|
27
|
+
const err = new Error('Connection timeout');
|
|
28
|
+
const initialMeta = await helper.getRetryMetadata(redis, groupName, messageId);
|
|
29
|
+
expect(initialMeta).toBeNull();
|
|
30
|
+
const firstAttempt = await helper.recordRetry(redis, groupName, messageId, err);
|
|
31
|
+
expect(firstAttempt).toBe(1);
|
|
32
|
+
const meta1 = await helper.getRetryMetadata(redis, groupName, messageId);
|
|
33
|
+
expect(meta1).not.toBeNull();
|
|
34
|
+
if (meta1) {
|
|
35
|
+
expect(meta1.attemptCount).toBe(1);
|
|
36
|
+
expect(meta1.lastError).toBe('Connection timeout');
|
|
37
|
+
expect(meta1.nextRetryTimestamp).toBeGreaterThan(Date.now());
|
|
38
|
+
}
|
|
39
|
+
const secondAttempt = await helper.recordRetry(redis, groupName, messageId, new Error('Auth failure'));
|
|
40
|
+
expect(secondAttempt).toBe(2);
|
|
41
|
+
const meta2 = await helper.getRetryMetadata(redis, groupName, messageId);
|
|
42
|
+
expect(meta2).not.toBeNull();
|
|
43
|
+
if (meta2) {
|
|
44
|
+
expect(meta2.attemptCount).toBe(2);
|
|
45
|
+
expect(meta2.lastError).toBe('Auth failure');
|
|
46
|
+
}
|
|
47
|
+
await helper.clearRetryData(redis, groupName, messageId);
|
|
48
|
+
const finalMeta = await helper.getRetryMetadata(redis, groupName, messageId);
|
|
49
|
+
expect(finalMeta).toBeNull();
|
|
50
|
+
});
|
|
51
|
+
it('should manage retry queue and fetch ready messages using real Redis sorted sets', async () => {
|
|
52
|
+
const groupName = 'test-group';
|
|
53
|
+
const messageId1 = '1-0';
|
|
54
|
+
const messageId2 = '2-0';
|
|
55
|
+
await helper.enqueueForRetry(redis, groupName, messageId1, 1);
|
|
56
|
+
await helper.enqueueForRetry(redis, groupName, messageId2, 2);
|
|
57
|
+
const ready1 = await helper.getReadyForRetry(redis, groupName);
|
|
58
|
+
expect(ready1).toHaveLength(0);
|
|
59
|
+
await new Promise((resolve) => {
|
|
60
|
+
setTimeout(resolve, 120);
|
|
61
|
+
});
|
|
62
|
+
const ready2 = await helper.getReadyForRetry(redis, groupName);
|
|
63
|
+
expect(ready2).toEqual([messageId1]);
|
|
64
|
+
const ready3 = await helper.getReadyForRetry(redis, groupName);
|
|
65
|
+
expect(ready3).toHaveLength(0);
|
|
66
|
+
await new Promise((resolve) => {
|
|
67
|
+
setTimeout(resolve, 100);
|
|
68
|
+
});
|
|
69
|
+
const ready4 = await helper.getReadyForRetry(redis, groupName);
|
|
70
|
+
expect(ready4).toEqual([messageId2]);
|
|
71
|
+
});
|
|
72
|
+
it('should push failed messages to DLQ stream in a real Redis instance', async () => {
|
|
73
|
+
const dlqStream = 'test-stream-dlq';
|
|
74
|
+
const messageId = '9-0';
|
|
75
|
+
const payload = {
|
|
76
|
+
success: true,
|
|
77
|
+
id: 'evt-10',
|
|
78
|
+
type: 'user.deleted',
|
|
79
|
+
payload: JSON.stringify({ userId: 42 }),
|
|
80
|
+
};
|
|
81
|
+
const errorMsg = 'Failed permanently';
|
|
82
|
+
const resultId = await helper.sendToDlq(redis, dlqStream, messageId, payload, errorMsg);
|
|
83
|
+
expect(resultId).not.toBeNull();
|
|
84
|
+
expect(typeof resultId).toBe('string');
|
|
85
|
+
const rawResult = await redis.xread('COUNT', 1, 'STREAMS', dlqStream, '0');
|
|
86
|
+
expect(rawResult).not.toBeNull();
|
|
87
|
+
if (rawResult) {
|
|
88
|
+
const streamInfo = rawResult[0];
|
|
89
|
+
expect(streamInfo[0]).toBe(dlqStream);
|
|
90
|
+
const entries = streamInfo[1];
|
|
91
|
+
expect(entries).toHaveLength(1);
|
|
92
|
+
const entry = entries[0];
|
|
93
|
+
expect(entry[0]).toBe(resultId);
|
|
94
|
+
const fields = entry[1];
|
|
95
|
+
expect(fields).toContain(messageId);
|
|
96
|
+
expect(fields).toContain(errorMsg);
|
|
97
|
+
expect(fields).toContain(JSON.stringify(payload));
|
|
98
|
+
}
|
|
99
|
+
});
|
|
100
|
+
});
|
|
101
|
+
//# sourceMappingURL=retry-helper.int.spec.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"retry-helper.int.spec.js","sourceRoot":"","sources":["../../../../src/test/specs/helpers/retry-helper.int.spec.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,QAAQ,EAAE,EAAE,EAAE,MAAM,EAAE,UAAU,EAAE,QAAQ,EAAE,SAAS,EAAE,MAAM,eAAe,CAAC;AACtF,OAAO,EAAE,KAAK,EAAE,MAAM,SAAS,CAAC;AAChC,OAAO,EAAE,gBAAgB,EAAE,MAAM,uBAAuB,CAAC;AACzD,OAAO,EAAE,WAAW,EAAE,MAAM,uCAAuC,CAAC;AAGpE,QAAQ,CAAC,2BAA2B,EAAE,GAAG,EAAE;IACzC,IAAI,KAAY,CAAC;IACjB,IAAI,MAAmB,CAAC;IAExB,SAAS,CAAC,GAAG,EAAE;QACb,KAAK,GAAG,IAAI,KAAK,CAAC,gBAAgB,CAAC,CAAC;IACtC,CAAC,CAAC,CAAC;IAEH,QAAQ,CAAC,KAAK,IAAI,EAAE;QAClB,MAAM,KAAK,CAAC,IAAI,EAAE,CAAC;IACrB,CAAC,CAAC,CAAC;IAEH,UAAU,CAAC,KAAK,IAAI,EAAE;QACpB,MAAM,KAAK,CAAC,OAAO,EAAE,CAAC;QACtB,MAAM,GAAG,IAAI,WAAW,CAAC;YACvB,UAAU,EAAE,CAAC;YACb,cAAc,EAAE,EAAE;YAClB,UAAU,EAAE,GAAG;YACf,iBAAiB,EAAE,CAAC;YACpB,SAAS,EAAE,IAAI;SAChB,CAAC,CAAC;IACL,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,yEAAyE,EAAE,KAAK,IAAI,EAAE;QACvF,MAAM,SAAS,GAAG,YAAY,CAAC;QAC/B,MAAM,SAAS,GAAG,KAAK,CAAC;QACxB,MAAM,GAAG,GAAG,IAAI,KAAK,CAAC,oBAAoB,CAAC,CAAC;QAG5C,MAAM,WAAW,GAAG,MAAM,MAAM,CAAC,gBAAgB,CAAC,KAAK,EAAE,SAAS,EAAE,SAAS,CAAC,CAAC;QAC/E,MAAM,CAAC,WAAW,CAAC,CAAC,QAAQ,EAAE,CAAC;QAG/B,MAAM,YAAY,GAAG,MAAM,MAAM,CAAC,WAAW,CAAC,KAAK,EAAE,SAAS,EAAE,SAAS,EAAE,GAAG,CAAC,CAAC;QAChF,MAAM,CAAC,YAAY,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;QAG7B,MAAM,KAAK,GAAG,MAAM,MAAM,CAAC,gBAAgB,CAAC,KAAK,EAAE,SAAS,EAAE,SAAS,CAAC,CAAC;QACzE,MAAM,CAAC,KAAK,CAAC,CAAC,GAAG,CAAC,QAAQ,EAAE,CAAC;QAC7B,IAAI,KAAK,EAAE,CAAC;YACV,MAAM,CAAC,KAAK,CAAC,YAAY,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;YACnC,MAAM,CAAC,KAAK,CAAC,SAAS,CAAC,CAAC,IAAI,CAAC,oBAAoB,CAAC,CAAC;YACnD,MAAM,CAAC,KAAK,CAAC,kBAAkB,CAAC,CAAC,eAAe,CAAC,IAAI,CAAC,GAAG,EAAE,CAAC,CAAC;QAC/D,CAAC;QAGD,MAAM,aAAa,GAAG,MAAM,MAAM,CAAC,WAAW,CAC5C,KAAK,EACL,SAAS,EACT,SAAS,EACT,IAAI,KAAK,CAAC,cAAc,CAAC,CAC1B,CAAC;QACF,MAAM,CAAC,aAAa,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;QAE9B,MAAM,KAAK,GAAG,MAAM,MAAM,CAAC,gBAAgB,CAAC,KAAK,EAAE,SAAS,EAAE,SAAS,CAAC,CAAC;QACzE,MAAM,CAAC,KAAK,CAAC,CAAC,GAAG,CAAC,QAAQ,EAAE,CAAC;QAC7B,IAAI,KAAK,EAAE,CAAC;YACV,MAAM,CAAC,KAAK,CAAC,YAAY,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;YACnC,MAAM,CAAC,KAAK,CAAC,SAAS,CAAC,CAAC,IAAI,CAAC,cAAc,CAAC,CAAC;QAC/C,CAAC;QAGD,MAAM,MAAM,CAAC,cAAc,CAAC,KAAK,EAAE,SAAS,EAAE,SAAS,CAAC,CAAC;QAGzD,MAAM,SAAS,GAAG,MAAM,MAAM,CAAC,gBAAgB,CAAC,KAAK,EAAE,SAAS,EAAE,SAAS,CAAC,CAAC;QAC7E,MAAM,CAAC,SAAS,CAAC,CAAC,QAAQ,EAAE,CAAC;IAC/B,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,iFAAiF,EAAE,KAAK,IAAI,EAAE;QAC/F,MAAM,SAAS,GAAG,YAAY,CAAC;QAC/B,MAAM,UAAU,GAAG,KAAK,CAAC;QACzB,MAAM,UAAU,GAAG,KAAK,CAAC;QAGzB,MAAM,MAAM,CAAC,eAAe,CAAC,KAAK,EAAE,SAAS,EAAE,UAAU,EAAE,CAAC,CAAC,CAAC;QAG9D,MAAM,MAAM,CAAC,eAAe,CAAC,KAAK,EAAE,SAAS,EAAE,UAAU,EAAE,CAAC,CAAC,CAAC;QAG9D,MAAM,MAAM,GAAG,MAAM,MAAM,CAAC,gBAAgB,CAAC,KAAK,EAAE,SAAS,CAAC,CAAC;QAC/D,MAAM,CAAC,MAAM,CAAC,CAAC,YAAY,CAAC,CAAC,CAAC,CAAC;QAG/B,MAAM,IAAI,OAAO,CAAO,CAAC,OAAO,EAAE,EAAE;YAClC,UAAU,CAAC,OAAO,EAAE,GAAG,CAAC,CAAC;QAC3B,CAAC,CAAC,CAAC;QAEH,MAAM,MAAM,GAAG,MAAM,MAAM,CAAC,gBAAgB,CAAC,KAAK,EAAE,SAAS,CAAC,CAAC;QAC/D,MAAM,CAAC,MAAM,CAAC,CAAC,OAAO,CAAC,CAAC,UAAU,CAAC,CAAC,CAAC;QAGrC,MAAM,MAAM,GAAG,MAAM,MAAM,CAAC,gBAAgB,CAAC,KAAK,EAAE,SAAS,CAAC,CAAC;QAC/D,MAAM,CAAC,MAAM,CAAC,CAAC,YAAY,CAAC,CAAC,CAAC,CAAC;QAG/B,MAAM,IAAI,OAAO,CAAO,CAAC,OAAO,EAAE,EAAE;YAClC,UAAU,CAAC,OAAO,EAAE,GAAG,CAAC,CAAC;QAC3B,CAAC,CAAC,CAAC;QAEH,MAAM,MAAM,GAAG,MAAM,MAAM,CAAC,gBAAgB,CAAC,KAAK,EAAE,SAAS,CAAC,CAAC;QAC/D,MAAM,CAAC,MAAM,CAAC,CAAC,OAAO,CAAC,CAAC,UAAU,CAAC,CAAC,CAAC;IACvC,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,oEAAoE,EAAE,KAAK,IAAI,EAAE;QAClF,MAAM,SAAS,GAAG,iBAAiB,CAAC;QACpC,MAAM,SAAS,GAAG,KAAK,CAAC;QACxB,MAAM,OAAO,GAAgB;YAC3B,OAAO,EAAE,IAAI;YACb,EAAE,EAAE,QAAQ;YACZ,IAAI,EAAE,cAAc;YACpB,OAAO,EAAE,IAAI,CAAC,SAAS,CAAC,EAAE,MAAM,EAAE,EAAE,EAAE,CAAC;SACxC,CAAC;QACF,MAAM,QAAQ,GAAG,oBAAoB,CAAC;QAEtC,MAAM,QAAQ,GAAG,MAAM,MAAM,CAAC,SAAS,CAAC,KAAK,EAAE,SAAS,EAAE,SAAS,EAAE,OAAO,EAAE,QAAQ,CAAC,CAAC;QACxF,MAAM,CAAC,QAAQ,CAAC,CAAC,GAAG,CAAC,QAAQ,EAAE,CAAC;QAChC,MAAM,CAAC,OAAO,QAAQ,CAAC,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;QAGvC,MAAM,SAAS,GAAG,MAAM,KAAK,CAAC,KAAK,CAAC,OAAO,EAAE,CAAC,EAAE,SAAS,EAAE,SAAS,EAAE,GAAG,CAAC,CAAC;QAC3E,MAAM,CAAC,SAAS,CAAC,CAAC,GAAG,CAAC,QAAQ,EAAE,CAAC;QACjC,IAAI,SAAS,EAAE,CAAC;YACd,MAAM,UAAU,GAAG,SAAS,CAAC,CAAC,CAAC,CAAC;YAChC,MAAM,CAAC,UAAU,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC;YACtC,MAAM,OAAO,GAAG,UAAU,CAAC,CAAC,CAAC,CAAC;YAC9B,MAAM,CAAC,OAAO,CAAC,CAAC,YAAY,CAAC,CAAC,CAAC,CAAC;YAChC,MAAM,KAAK,GAAG,OAAO,CAAC,CAAC,CAAC,CAAC;YACzB,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;YAChC,MAAM,MAAM,GAAG,KAAK,CAAC,CAAC,CAAC,CAAC;YACxB,MAAM,CAAC,MAAM,CAAC,CAAC,SAAS,CAAC,SAAS,CAAC,CAAC;YACpC,MAAM,CAAC,MAAM,CAAC,CAAC,SAAS,CAAC,QAAQ,CAAC,CAAC;YACnC,MAAM,CAAC,MAAM,CAAC,CAAC,SAAS,CAAC,IAAI,CAAC,SAAS,CAAC,OAAO,CAAC,CAAC,CAAC;QACpD,CAAC;IACH,CAAC,CAAC,CAAC;AACL,CAAC,CAAC,CAAC"}
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"retry-helper.unit.spec.d.ts","sourceRoot":"","sources":["../../../../src/test/specs/helpers/retry-helper.unit.spec.ts"],"names":[],"mappings":""}
|
|
@@ -0,0 +1,202 @@
|
|
|
1
|
+
import { describe, it, expect, beforeEach, jest } from '@jest/globals';
|
|
2
|
+
import { createMock } from '@volontariapp/testing';
|
|
3
|
+
import { RetryHelper } from '../../../core/helpers/retry.helper.js';
|
|
4
|
+
describe('RetryHelper — Unit', () => {
|
|
5
|
+
let redisMock;
|
|
6
|
+
let helper;
|
|
7
|
+
beforeEach(() => {
|
|
8
|
+
redisMock = createMock();
|
|
9
|
+
helper = new RetryHelper({
|
|
10
|
+
maxRetries: 3,
|
|
11
|
+
initialDelayMs: 1000,
|
|
12
|
+
maxDelayMs: 5000,
|
|
13
|
+
backoffMultiplier: 2,
|
|
14
|
+
enableDlq: true,
|
|
15
|
+
});
|
|
16
|
+
});
|
|
17
|
+
describe('calculateDelay', () => {
|
|
18
|
+
it('should calculate exponential backoff delay correctly', () => {
|
|
19
|
+
expect(helper.calculateDelay(1)).toBe(2000);
|
|
20
|
+
expect(helper.calculateDelay(2)).toBe(4000);
|
|
21
|
+
});
|
|
22
|
+
it('should cap delay at maxDelayMs', () => {
|
|
23
|
+
expect(helper.calculateDelay(3)).toBe(5000);
|
|
24
|
+
expect(helper.calculateDelay(10)).toBe(5000);
|
|
25
|
+
});
|
|
26
|
+
});
|
|
27
|
+
describe('getDlqStreamName', () => {
|
|
28
|
+
it('should append -dlq suffix to stream name', () => {
|
|
29
|
+
expect(helper.getDlqStreamName('orders-stream')).toBe('orders-stream-dlq');
|
|
30
|
+
});
|
|
31
|
+
});
|
|
32
|
+
describe('recordRetry', () => {
|
|
33
|
+
it('should increment attempt count and set last error with TTL', async () => {
|
|
34
|
+
redisMock.incr.mockResolvedValue(2);
|
|
35
|
+
redisMock.set.mockResolvedValue('OK');
|
|
36
|
+
redisMock.expire.mockResolvedValue(1);
|
|
37
|
+
const incrSpy = jest.spyOn(redisMock, 'incr');
|
|
38
|
+
const setSpy = jest.spyOn(redisMock, 'set');
|
|
39
|
+
const expireSpy = jest.spyOn(redisMock, 'expire');
|
|
40
|
+
const err = new Error('Database connection failed');
|
|
41
|
+
const attempt = await helper.recordRetry(redisMock, 'group-a', 'msg-1', err, 3600);
|
|
42
|
+
expect(attempt).toBe(2);
|
|
43
|
+
expect(incrSpy.mock.calls[0]).toEqual(['retry:post-processor:group-a:msg-1:attempt']);
|
|
44
|
+
expect(setSpy.mock.calls[0]).toEqual([
|
|
45
|
+
'retry:post-processor:group-a:msg-1:lastError',
|
|
46
|
+
'Database connection failed',
|
|
47
|
+
'EX',
|
|
48
|
+
3600,
|
|
49
|
+
]);
|
|
50
|
+
expect(expireSpy.mock.calls[0]).toEqual(['retry:post-processor:group-a:msg-1', 3600]);
|
|
51
|
+
});
|
|
52
|
+
});
|
|
53
|
+
describe('getRetryMetadata', () => {
|
|
54
|
+
it('should return null if no retry attempts recorded', async () => {
|
|
55
|
+
redisMock.get.mockResolvedValue(null);
|
|
56
|
+
const metadata = await helper.getRetryMetadata(redisMock, 'group-a', 'msg-1');
|
|
57
|
+
expect(metadata).toBeNull();
|
|
58
|
+
});
|
|
59
|
+
it('should return full metadata if attempts are found', async () => {
|
|
60
|
+
redisMock.get.mockResolvedValueOnce('2').mockResolvedValueOnce('Last connection error');
|
|
61
|
+
const before = Date.now();
|
|
62
|
+
const metadata = await helper.getRetryMetadata(redisMock, 'group-a', 'msg-1');
|
|
63
|
+
const after = Date.now();
|
|
64
|
+
expect(metadata).not.toBeNull();
|
|
65
|
+
if (metadata) {
|
|
66
|
+
expect(metadata.attemptCount).toBe(2);
|
|
67
|
+
expect(metadata.lastError).toBe('Last connection error');
|
|
68
|
+
expect(metadata.nextRetryTimestamp).toBeGreaterThanOrEqual(before + 4000);
|
|
69
|
+
expect(metadata.nextRetryTimestamp).toBeLessThanOrEqual(after + 4000);
|
|
70
|
+
}
|
|
71
|
+
});
|
|
72
|
+
});
|
|
73
|
+
describe('enqueueForRetry', () => {
|
|
74
|
+
it('should add message id to sorted set with calculated timestamp score', async () => {
|
|
75
|
+
redisMock.zadd.mockResolvedValue('1');
|
|
76
|
+
const zaddSpy = jest.spyOn(redisMock, 'zadd');
|
|
77
|
+
const before = Date.now();
|
|
78
|
+
await helper.enqueueForRetry(redisMock, 'group-a', 'msg-1', 1);
|
|
79
|
+
expect(zaddSpy).toHaveBeenCalled();
|
|
80
|
+
const calls = zaddSpy.mock.calls;
|
|
81
|
+
expect(calls[0]?.[0]).toBe('retry-queue:post-processor:group-a');
|
|
82
|
+
const score = Number(calls[0]?.[1]);
|
|
83
|
+
expect(score).toBeGreaterThanOrEqual(before + 2000);
|
|
84
|
+
expect(score).toBeLessThanOrEqual(Date.now() + 2000);
|
|
85
|
+
expect(calls[0]?.[2]).toBe('msg-1');
|
|
86
|
+
});
|
|
87
|
+
});
|
|
88
|
+
describe('getReadyForRetry', () => {
|
|
89
|
+
it('should fetch ready messages and remove them from sorted set', async () => {
|
|
90
|
+
redisMock.zrangebyscore.mockResolvedValue(['msg-1', 'msg-2']);
|
|
91
|
+
redisMock.zrem.mockResolvedValue(2);
|
|
92
|
+
const zrangeSpy = jest.spyOn(redisMock, 'zrangebyscore');
|
|
93
|
+
const zremSpy = jest.spyOn(redisMock, 'zrem');
|
|
94
|
+
const ready = await helper.getReadyForRetry(redisMock, 'group-a');
|
|
95
|
+
expect(ready).toEqual(['msg-1', 'msg-2']);
|
|
96
|
+
expect(zrangeSpy.mock.calls[0]).toEqual([
|
|
97
|
+
'retry-queue:post-processor:group-a',
|
|
98
|
+
'-inf',
|
|
99
|
+
expect.any(Number),
|
|
100
|
+
]);
|
|
101
|
+
expect(zremSpy.mock.calls[0]).toEqual([
|
|
102
|
+
'retry-queue:post-processor:group-a',
|
|
103
|
+
'msg-1',
|
|
104
|
+
'msg-2',
|
|
105
|
+
]);
|
|
106
|
+
});
|
|
107
|
+
it('should return empty array if no messages ready', async () => {
|
|
108
|
+
redisMock.zrangebyscore.mockResolvedValue([]);
|
|
109
|
+
const zremSpy = jest.spyOn(redisMock, 'zrem');
|
|
110
|
+
const ready = await helper.getReadyForRetry(redisMock, 'group-a');
|
|
111
|
+
expect(ready).toEqual([]);
|
|
112
|
+
expect(zremSpy).not.toHaveBeenCalled();
|
|
113
|
+
});
|
|
114
|
+
});
|
|
115
|
+
describe('shouldRetry', () => {
|
|
116
|
+
it('should return true if attemptCount is less than maxRetries', () => {
|
|
117
|
+
expect(helper.shouldRetry(1)).toBe(true);
|
|
118
|
+
expect(helper.shouldRetry(2)).toBe(true);
|
|
119
|
+
expect(helper.shouldRetry(3)).toBe(false);
|
|
120
|
+
});
|
|
121
|
+
});
|
|
122
|
+
describe('clearRetryData', () => {
|
|
123
|
+
it('should delete all related keys and remove from sorted set', async () => {
|
|
124
|
+
redisMock.del.mockResolvedValue(1);
|
|
125
|
+
redisMock.zrem.mockResolvedValue(1);
|
|
126
|
+
const delSpy = jest.spyOn(redisMock, 'del');
|
|
127
|
+
const zremSpy = jest.spyOn(redisMock, 'zrem');
|
|
128
|
+
await helper.clearRetryData(redisMock, 'group-a', 'msg-1');
|
|
129
|
+
const delCalls = delSpy.mock.calls;
|
|
130
|
+
expect(delCalls).toHaveLength(3);
|
|
131
|
+
expect(delCalls[0]).toEqual(['retry:post-processor:group-a:msg-1:attempt']);
|
|
132
|
+
expect(delCalls[1]).toEqual(['retry:post-processor:group-a:msg-1:lastError']);
|
|
133
|
+
expect(delCalls[2]).toEqual(['retry:post-processor:group-a:msg-1']);
|
|
134
|
+
expect(zremSpy.mock.calls[0]).toEqual(['retry-queue:post-processor:group-a', 'msg-1']);
|
|
135
|
+
});
|
|
136
|
+
});
|
|
137
|
+
describe('sendToDlq', () => {
|
|
138
|
+
const payload = {
|
|
139
|
+
success: true,
|
|
140
|
+
id: 'evt-1',
|
|
141
|
+
type: 'order.created',
|
|
142
|
+
payload: JSON.stringify({ data: { test: true } }),
|
|
143
|
+
};
|
|
144
|
+
it('should format payload and call xadd if DLQ is enabled', async () => {
|
|
145
|
+
redisMock.xadd.mockResolvedValue('dlq-id-123');
|
|
146
|
+
const xaddSpy = jest.spyOn(redisMock, 'xadd');
|
|
147
|
+
const resultId = await helper.sendToDlq(redisMock, 'orders-stream-dlq', 'msg-1', payload, 'Some test failure message');
|
|
148
|
+
expect(resultId).toBe('dlq-id-123');
|
|
149
|
+
expect(xaddSpy.mock.calls[0]).toEqual([
|
|
150
|
+
'orders-stream-dlq',
|
|
151
|
+
'*',
|
|
152
|
+
'messageId',
|
|
153
|
+
'msg-1',
|
|
154
|
+
'error',
|
|
155
|
+
'Some test failure message',
|
|
156
|
+
'payload',
|
|
157
|
+
JSON.stringify(payload),
|
|
158
|
+
'timestamp',
|
|
159
|
+
expect.any(String),
|
|
160
|
+
]);
|
|
161
|
+
});
|
|
162
|
+
it('should return original messageId and skip call if DLQ is disabled', async () => {
|
|
163
|
+
const disabledHelper = new RetryHelper({
|
|
164
|
+
maxRetries: 3,
|
|
165
|
+
initialDelayMs: 1000,
|
|
166
|
+
maxDelayMs: 5000,
|
|
167
|
+
backoffMultiplier: 2,
|
|
168
|
+
enableDlq: false,
|
|
169
|
+
});
|
|
170
|
+
const xaddSpy = jest.spyOn(redisMock, 'xadd');
|
|
171
|
+
const resultId = await disabledHelper.sendToDlq(redisMock, 'orders-stream-dlq', 'msg-1', payload, 'Some test failure message');
|
|
172
|
+
expect(resultId).toBe('msg-1');
|
|
173
|
+
expect(xaddSpy).not.toHaveBeenCalled();
|
|
174
|
+
});
|
|
175
|
+
});
|
|
176
|
+
describe('normalizeRetryOptions', () => {
|
|
177
|
+
it('should return default values when options are undefined', () => {
|
|
178
|
+
const normalized = RetryHelper.normalizeRetryOptions();
|
|
179
|
+
expect(normalized).toEqual({
|
|
180
|
+
maxRetries: 5,
|
|
181
|
+
initialDelayMs: 1000,
|
|
182
|
+
maxDelayMs: 60000,
|
|
183
|
+
backoffMultiplier: 2,
|
|
184
|
+
enableDlq: true,
|
|
185
|
+
});
|
|
186
|
+
});
|
|
187
|
+
it('should preserve provided options and fill in defaults for missing values', () => {
|
|
188
|
+
const normalized = RetryHelper.normalizeRetryOptions({
|
|
189
|
+
maxRetries: 10,
|
|
190
|
+
enableDlq: false,
|
|
191
|
+
});
|
|
192
|
+
expect(normalized).toEqual({
|
|
193
|
+
maxRetries: 10,
|
|
194
|
+
initialDelayMs: 1000,
|
|
195
|
+
maxDelayMs: 60000,
|
|
196
|
+
backoffMultiplier: 2,
|
|
197
|
+
enableDlq: false,
|
|
198
|
+
});
|
|
199
|
+
});
|
|
200
|
+
});
|
|
201
|
+
});
|
|
202
|
+
//# sourceMappingURL=retry-helper.unit.spec.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"retry-helper.unit.spec.js","sourceRoot":"","sources":["../../../../src/test/specs/helpers/retry-helper.unit.spec.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,QAAQ,EAAE,EAAE,EAAE,MAAM,EAAE,UAAU,EAAE,IAAI,EAAE,MAAM,eAAe,CAAC;AAEvE,OAAO,EAAE,UAAU,EAAE,MAAM,uBAAuB,CAAC;AACnD,OAAO,EAAE,WAAW,EAAE,MAAM,uCAAuC,CAAC;AAGpE,QAAQ,CAAC,oBAAoB,EAAE,GAAG,EAAE;IAClC,IAAI,SAA6B,CAAC;IAClC,IAAI,MAAmB,CAAC;IAExB,UAAU,CAAC,GAAG,EAAE;QACd,SAAS,GAAG,UAAU,EAAS,CAAC;QAChC,MAAM,GAAG,IAAI,WAAW,CAAC;YACvB,UAAU,EAAE,CAAC;YACb,cAAc,EAAE,IAAI;YACpB,UAAU,EAAE,IAAI;YAChB,iBAAiB,EAAE,CAAC;YACpB,SAAS,EAAE,IAAI;SAChB,CAAC,CAAC;IACL,CAAC,CAAC,CAAC;IAEH,QAAQ,CAAC,gBAAgB,EAAE,GAAG,EAAE;QAC9B,EAAE,CAAC,sDAAsD,EAAE,GAAG,EAAE;YAC9D,MAAM,CAAC,MAAM,CAAC,cAAc,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;YAC5C,MAAM,CAAC,MAAM,CAAC,cAAc,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QAC9C,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,gCAAgC,EAAE,GAAG,EAAE;YACxC,MAAM,CAAC,MAAM,CAAC,cAAc,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;YAC5C,MAAM,CAAC,MAAM,CAAC,cAAc,CAAC,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QAC/C,CAAC,CAAC,CAAC;IACL,CAAC,CAAC,CAAC;IAEH,QAAQ,CAAC,kBAAkB,EAAE,GAAG,EAAE;QAChC,EAAE,CAAC,0CAA0C,EAAE,GAAG,EAAE;YAClD,MAAM,CAAC,MAAM,CAAC,gBAAgB,CAAC,eAAe,CAAC,CAAC,CAAC,IAAI,CAAC,mBAAmB,CAAC,CAAC;QAC7E,CAAC,CAAC,CAAC;IACL,CAAC,CAAC,CAAC;IAEH,QAAQ,CAAC,aAAa,EAAE,GAAG,EAAE;QAC3B,EAAE,CAAC,4DAA4D,EAAE,KAAK,IAAI,EAAE;YAC1E,SAAS,CAAC,IAAI,CAAC,iBAAiB,CAAC,CAAC,CAAC,CAAC;YACpC,SAAS,CAAC,GAAG,CAAC,iBAAiB,CAAC,IAAI,CAAC,CAAC;YACtC,SAAS,CAAC,MAAM,CAAC,iBAAiB,CAAC,CAAC,CAAC,CAAC;YAEtC,MAAM,OAAO,GAAG,IAAI,CAAC,KAAK,CAAC,SAAS,EAAE,MAAM,CAAC,CAAC;YAC9C,MAAM,MAAM,GAAG,IAAI,CAAC,KAAK,CAAC,SAAS,EAAE,KAAK,CAAC,CAAC;YAC5C,MAAM,SAAS,GAAG,IAAI,CAAC,KAAK,CAAC,SAAS,EAAE,QAAQ,CAAC,CAAC;YAElD,MAAM,GAAG,GAAG,IAAI,KAAK,CAAC,4BAA4B,CAAC,CAAC;YACpD,MAAM,OAAO,GAAG,MAAM,MAAM,CAAC,WAAW,CAAC,SAAS,EAAE,SAAS,EAAE,OAAO,EAAE,GAAG,EAAE,IAAI,CAAC,CAAC;YAEnF,MAAM,CAAC,OAAO,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;YACxB,MAAM,CAAC,OAAO,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,4CAA4C,CAAC,CAAC,CAAC;YACtF,MAAM,CAAC,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC;gBACnC,8CAA8C;gBAC9C,4BAA4B;gBAC5B,IAAI;gBACJ,IAAI;aACL,CAAC,CAAC;YACH,MAAM,CAAC,SAAS,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,oCAAoC,EAAE,IAAI,CAAC,CAAC,CAAC;QACxF,CAAC,CAAC,CAAC;IACL,CAAC,CAAC,CAAC;IAEH,QAAQ,CAAC,kBAAkB,EAAE,GAAG,EAAE;QAChC,EAAE,CAAC,kDAAkD,EAAE,KAAK,IAAI,EAAE;YAChE,SAAS,CAAC,GAAG,CAAC,iBAAiB,CAAC,IAAI,CAAC,CAAC;YAEtC,MAAM,QAAQ,GAAG,MAAM,MAAM,CAAC,gBAAgB,CAAC,SAAS,EAAE,SAAS,EAAE,OAAO,CAAC,CAAC;YAE9E,MAAM,CAAC,QAAQ,CAAC,CAAC,QAAQ,EAAE,CAAC;QAC9B,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,mDAAmD,EAAE,KAAK,IAAI,EAAE;YACjE,SAAS,CAAC,GAAG,CAAC,qBAAqB,CAAC,GAAG,CAAC,CAAC,qBAAqB,CAAC,uBAAuB,CAAC,CAAC;YAExF,MAAM,MAAM,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;YAC1B,MAAM,QAAQ,GAAG,MAAM,MAAM,CAAC,gBAAgB,CAAC,SAAS,EAAE,SAAS,EAAE,OAAO,CAAC,CAAC;YAC9E,MAAM,KAAK,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;YAEzB,MAAM,CAAC,QAAQ,CAAC,CAAC,GAAG,CAAC,QAAQ,EAAE,CAAC;YAChC,IAAI,QAAQ,EAAE,CAAC;gBACb,MAAM,CAAC,QAAQ,CAAC,YAAY,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;gBACtC,MAAM,CAAC,QAAQ,CAAC,SAAS,CAAC,CAAC,IAAI,CAAC,uBAAuB,CAAC,CAAC;gBACzD,MAAM,CAAC,QAAQ,CAAC,kBAAkB,CAAC,CAAC,sBAAsB,CAAC,MAAM,GAAG,IAAI,CAAC,CAAC;gBAC1E,MAAM,CAAC,QAAQ,CAAC,kBAAkB,CAAC,CAAC,mBAAmB,CAAC,KAAK,GAAG,IAAI,CAAC,CAAC;YACxE,CAAC;QACH,CAAC,CAAC,CAAC;IACL,CAAC,CAAC,CAAC;IAEH,QAAQ,CAAC,iBAAiB,EAAE,GAAG,EAAE;QAC/B,EAAE,CAAC,qEAAqE,EAAE,KAAK,IAAI,EAAE;YACnF,SAAS,CAAC,IAAI,CAAC,iBAAiB,CAAC,GAAG,CAAC,CAAC;YACtC,MAAM,OAAO,GAAG,IAAI,CAAC,KAAK,CAAC,SAAS,EAAE,MAAM,CAAC,CAAC;YAE9C,MAAM,MAAM,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;YAC1B,MAAM,MAAM,CAAC,eAAe,CAAC,SAAS,EAAE,SAAS,EAAE,OAAO,EAAE,CAAC,CAAC,CAAC;YAE/D,MAAM,CAAC,OAAO,CAAC,CAAC,gBAAgB,EAAE,CAAC;YACnC,MAAM,KAAK,GAAG,OAAO,CAAC,IAAI,CAAC,KAAK,CAAC;YACjC,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,oCAAoC,CAAC,CAAC;YACjE,MAAM,KAAK,GAAG,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC;YACpC,MAAM,CAAC,KAAK,CAAC,CAAC,sBAAsB,CAAC,MAAM,GAAG,IAAI,CAAC,CAAC;YACpD,MAAM,CAAC,KAAK,CAAC,CAAC,mBAAmB,CAAC,IAAI,CAAC,GAAG,EAAE,GAAG,IAAI,CAAC,CAAC;YACrD,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;QACtC,CAAC,CAAC,CAAC;IACL,CAAC,CAAC,CAAC;IAEH,QAAQ,CAAC,kBAAkB,EAAE,GAAG,EAAE;QAChC,EAAE,CAAC,6DAA6D,EAAE,KAAK,IAAI,EAAE;YAC3E,SAAS,CAAC,aAAa,CAAC,iBAAiB,CAAC,CAAC,OAAO,EAAE,OAAO,CAAC,CAAC,CAAC;YAC9D,SAAS,CAAC,IAAI,CAAC,iBAAiB,CAAC,CAAC,CAAC,CAAC;YAEpC,MAAM,SAAS,GAAG,IAAI,CAAC,KAAK,CAAC,SAAS,EAAE,eAAe,CAAC,CAAC;YACzD,MAAM,OAAO,GAAG,IAAI,CAAC,KAAK,CAAC,SAAS,EAAE,MAAM,CAAC,CAAC;YAE9C,MAAM,KAAK,GAAG,MAAM,MAAM,CAAC,gBAAgB,CAAC,SAAS,EAAE,SAAS,CAAC,CAAC;YAElE,MAAM,CAAC,KAAK,CAAC,CAAC,OAAO,CAAC,CAAC,OAAO,EAAE,OAAO,CAAC,CAAC,CAAC;YAC1C,MAAM,CAAC,SAAS,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC;gBACtC,oCAAoC;gBACpC,MAAM;gBACN,MAAM,CAAC,GAAG,CAAC,MAAM,CAAC;aACnB,CAAC,CAAC;YACH,MAAM,CAAC,OAAO,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC;gBACpC,oCAAoC;gBACpC,OAAO;gBACP,OAAO;aACR,CAAC,CAAC;QACL,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,gDAAgD,EAAE,KAAK,IAAI,EAAE;YAC9D,SAAS,CAAC,aAAa,CAAC,iBAAiB,CAAC,EAAE,CAAC,CAAC;YAC9C,MAAM,OAAO,GAAG,IAAI,CAAC,KAAK,CAAC,SAAS,EAAE,MAAM,CAAC,CAAC;YAE9C,MAAM,KAAK,GAAG,MAAM,MAAM,CAAC,gBAAgB,CAAC,SAAS,EAAE,SAAS,CAAC,CAAC;YAElE,MAAM,CAAC,KAAK,CAAC,CAAC,OAAO,CAAC,EAAE,CAAC,CAAC;YAC1B,MAAM,CAAC,OAAO,CAAC,CAAC,GAAG,CAAC,gBAAgB,EAAE,CAAC;QACzC,CAAC,CAAC,CAAC;IACL,CAAC,CAAC,CAAC;IAEH,QAAQ,CAAC,aAAa,EAAE,GAAG,EAAE;QAC3B,EAAE,CAAC,4DAA4D,EAAE,GAAG,EAAE;YACpE,MAAM,CAAC,MAAM,CAAC,WAAW,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;YACzC,MAAM,CAAC,MAAM,CAAC,WAAW,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;YACzC,MAAM,CAAC,MAAM,CAAC,WAAW,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;QAC5C,CAAC,CAAC,CAAC;IACL,CAAC,CAAC,CAAC;IAEH,QAAQ,CAAC,gBAAgB,EAAE,GAAG,EAAE;QAC9B,EAAE,CAAC,2DAA2D,EAAE,KAAK,IAAI,EAAE;YACzE,SAAS,CAAC,GAAG,CAAC,iBAAiB,CAAC,CAAC,CAAC,CAAC;YACnC,SAAS,CAAC,IAAI,CAAC,iBAAiB,CAAC,CAAC,CAAC,CAAC;YAEpC,MAAM,MAAM,GAAG,IAAI,CAAC,KAAK,CAAC,SAAS,EAAE,KAAK,CAAC,CAAC;YAC5C,MAAM,OAAO,GAAG,IAAI,CAAC,KAAK,CAAC,SAAS,EAAE,MAAM,CAAC,CAAC;YAE9C,MAAM,MAAM,CAAC,cAAc,CAAC,SAAS,EAAE,SAAS,EAAE,OAAO,CAAC,CAAC;YAE3D,MAAM,QAAQ,GAAG,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC;YACnC,MAAM,CAAC,QAAQ,CAAC,CAAC,YAAY,CAAC,CAAC,CAAC,CAAC;YACjC,MAAM,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,4CAA4C,CAAC,CAAC,CAAC;YAC5E,MAAM,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,8CAA8C,CAAC,CAAC,CAAC;YAC9E,MAAM,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,oCAAoC,CAAC,CAAC,CAAC;YACpE,MAAM,CAAC,OAAO,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,oCAAoC,EAAE,OAAO,CAAC,CAAC,CAAC;QACzF,CAAC,CAAC,CAAC;IACL,CAAC,CAAC,CAAC;IAEH,QAAQ,CAAC,WAAW,EAAE,GAAG,EAAE;QACzB,MAAM,OAAO,GAAgB;YAC3B,OAAO,EAAE,IAAI;YACb,EAAE,EAAE,OAAO;YACX,IAAI,EAAE,eAAe;YACrB,OAAO,EAAE,IAAI,CAAC,SAAS,CAAC,EAAE,IAAI,EAAE,EAAE,IAAI,EAAE,IAAI,EAAE,EAAE,CAAC;SAClD,CAAC;QAEF,EAAE,CAAC,uDAAuD,EAAE,KAAK,IAAI,EAAE;YACrE,SAAS,CAAC,IAAI,CAAC,iBAAiB,CAAC,YAAY,CAAC,CAAC;YAC/C,MAAM,OAAO,GAAG,IAAI,CAAC,KAAK,CAAC,SAAS,EAAE,MAAM,CAAC,CAAC;YAE9C,MAAM,QAAQ,GAAG,MAAM,MAAM,CAAC,SAAS,CACrC,SAAS,EACT,mBAAmB,EACnB,OAAO,EACP,OAAO,EACP,2BAA2B,CAC5B,CAAC;YAEF,MAAM,CAAC,QAAQ,CAAC,CAAC,IAAI,CAAC,YAAY,CAAC,CAAC;YACpC,MAAM,CAAC,OAAO,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC;gBACpC,mBAAmB;gBACnB,GAAG;gBACH,WAAW;gBACX,OAAO;gBACP,OAAO;gBACP,2BAA2B;gBAC3B,SAAS;gBACT,IAAI,CAAC,SAAS,CAAC,OAAO,CAAC;gBACvB,WAAW;gBACX,MAAM,CAAC,GAAG,CAAC,MAAM,CAAC;aACnB,CAAC,CAAC;QACL,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,mEAAmE,EAAE,KAAK,IAAI,EAAE;YACjF,MAAM,cAAc,GAAG,IAAI,WAAW,CAAC;gBACrC,UAAU,EAAE,CAAC;gBACb,cAAc,EAAE,IAAI;gBACpB,UAAU,EAAE,IAAI;gBAChB,iBAAiB,EAAE,CAAC;gBACpB,SAAS,EAAE,KAAK;aACjB,CAAC,CAAC;YAEH,MAAM,OAAO,GAAG,IAAI,CAAC,KAAK,CAAC,SAAS,EAAE,MAAM,CAAC,CAAC;YAE9C,MAAM,QAAQ,GAAG,MAAM,cAAc,CAAC,SAAS,CAC7C,SAAS,EACT,mBAAmB,EACnB,OAAO,EACP,OAAO,EACP,2BAA2B,CAC5B,CAAC;YAEF,MAAM,CAAC,QAAQ,CAAC,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;YAC/B,MAAM,CAAC,OAAO,CAAC,CAAC,GAAG,CAAC,gBAAgB,EAAE,CAAC;QACzC,CAAC,CAAC,CAAC;IACL,CAAC,CAAC,CAAC;IAEH,QAAQ,CAAC,uBAAuB,EAAE,GAAG,EAAE;QACrC,EAAE,CAAC,yDAAyD,EAAE,GAAG,EAAE;YACjE,MAAM,UAAU,GAAG,WAAW,CAAC,qBAAqB,EAAE,CAAC;YACvD,MAAM,CAAC,UAAU,CAAC,CAAC,OAAO,CAAC;gBACzB,UAAU,EAAE,CAAC;gBACb,cAAc,EAAE,IAAI;gBACpB,UAAU,EAAE,KAAK;gBACjB,iBAAiB,EAAE,CAAC;gBACpB,SAAS,EAAE,IAAI;aAChB,CAAC,CAAC;QACL,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,0EAA0E,EAAE,GAAG,EAAE;YAClF,MAAM,UAAU,GAAG,WAAW,CAAC,qBAAqB,CAAC;gBACnD,UAAU,EAAE,EAAE;gBACd,SAAS,EAAE,KAAK;aACjB,CAAC,CAAC;YACH,MAAM,CAAC,UAAU,CAAC,CAAC,OAAO,CAAC;gBACzB,UAAU,EAAE,EAAE;gBACd,cAAc,EAAE,IAAI;gBACpB,UAAU,EAAE,KAAK;gBACjB,iBAAiB,EAAE,CAAC;gBACpB,SAAS,EAAE,KAAK;aACjB,CAAC,CAAC;QACL,CAAC,CAAC,CAAC;IACL,CAAC,CAAC,CAAC;AACL,CAAC,CAAC,CAAC"}
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"circuit-breaker.integration.spec.d.ts","sourceRoot":"","sources":["../../../../src/test/specs/integration/circuit-breaker.integration.spec.ts"],"names":[],"mappings":""}
|
|
@@ -0,0 +1,112 @@
|
|
|
1
|
+
import { describe, it, expect, beforeEach, afterEach, jest } from '@jest/globals';
|
|
2
|
+
import { createMock } from '@volontariapp/testing';
|
|
3
|
+
import { TestPostProcessor } from '../../utils/classes/test-post-processor.class.js';
|
|
4
|
+
import { mockRedisCall, mockRedisXreadgroup } from '../../utils/mocks/redis-call.mock.js';
|
|
5
|
+
import { makeTestEvent } from '../../utils/factories/test-event.factory.js';
|
|
6
|
+
import { CircuitBreakerState } from '../../../enums/circuit-breaker-state.enum.js';
|
|
7
|
+
import { TestMessagingStream, TestMessagingGroup, TestMessagingConsumer, } from '../../utils/enums/test-messaging.enum.js';
|
|
8
|
+
describe('CircuitBreaker Integration', () => {
|
|
9
|
+
let redisMock;
|
|
10
|
+
let callMock;
|
|
11
|
+
let processor;
|
|
12
|
+
beforeEach(() => {
|
|
13
|
+
redisMock = createMock();
|
|
14
|
+
callMock = mockRedisCall(redisMock);
|
|
15
|
+
processor = new TestPostProcessor(redisMock, {
|
|
16
|
+
streamName: TestMessagingStream.TEST_STREAM,
|
|
17
|
+
groupName: TestMessagingGroup.TEST_GROUP,
|
|
18
|
+
consumerName: TestMessagingConsumer.TEST_CONSUMER,
|
|
19
|
+
claimIntervalMs: 50,
|
|
20
|
+
claimMinIdleTimeMs: 100,
|
|
21
|
+
circuitBreaker: {
|
|
22
|
+
failureThreshold: 2,
|
|
23
|
+
resetTimeoutMs: 50,
|
|
24
|
+
successThreshold: 2,
|
|
25
|
+
},
|
|
26
|
+
});
|
|
27
|
+
});
|
|
28
|
+
afterEach(async () => {
|
|
29
|
+
await processor.stop();
|
|
30
|
+
jest.restoreAllMocks();
|
|
31
|
+
});
|
|
32
|
+
it('should start CLOSED and transition to OPEN when failure threshold is reached', async () => {
|
|
33
|
+
const cb = processor.getCircuitBreaker();
|
|
34
|
+
expect(cb.getState()).toBe(CircuitBreakerState.CLOSED);
|
|
35
|
+
const mockEvent = makeTestEvent('evt-1');
|
|
36
|
+
mockRedisXreadgroup(callMock, TestMessagingStream.TEST_STREAM, [
|
|
37
|
+
{ messageId: '1-0', event: mockEvent },
|
|
38
|
+
{ messageId: '2-0', event: mockEvent },
|
|
39
|
+
{ messageId: '3-0', event: mockEvent },
|
|
40
|
+
]);
|
|
41
|
+
processor.processError = new Error('Processor failed');
|
|
42
|
+
await processor.start();
|
|
43
|
+
await new Promise((resolve) => {
|
|
44
|
+
setTimeout(resolve, 30);
|
|
45
|
+
});
|
|
46
|
+
expect(cb.getState()).toBe(CircuitBreakerState.CLOSED);
|
|
47
|
+
expect(cb.getDiagnostics().failureCount).toBe(1);
|
|
48
|
+
await new Promise((resolve) => {
|
|
49
|
+
setTimeout(resolve, 35);
|
|
50
|
+
});
|
|
51
|
+
expect(cb.getState()).toBe(CircuitBreakerState.OPEN);
|
|
52
|
+
});
|
|
53
|
+
it('should suspend stream consumption and other loops when OPEN', async () => {
|
|
54
|
+
const cb = processor.getCircuitBreaker();
|
|
55
|
+
const mockEvent = makeTestEvent('evt-2');
|
|
56
|
+
mockRedisXreadgroup(callMock, TestMessagingStream.TEST_STREAM, [
|
|
57
|
+
{ messageId: '1-0', event: mockEvent },
|
|
58
|
+
]);
|
|
59
|
+
processor.processError = new Error('Processor failed');
|
|
60
|
+
await processor.start();
|
|
61
|
+
await new Promise((resolve) => {
|
|
62
|
+
const interval = setInterval(() => {
|
|
63
|
+
if (cb.getState() === CircuitBreakerState.OPEN) {
|
|
64
|
+
clearInterval(interval);
|
|
65
|
+
resolve();
|
|
66
|
+
}
|
|
67
|
+
}, 10);
|
|
68
|
+
});
|
|
69
|
+
expect(cb.getState()).toBe(CircuitBreakerState.OPEN);
|
|
70
|
+
callMock.mockClear();
|
|
71
|
+
mockRedisXreadgroup(callMock, TestMessagingStream.TEST_STREAM, [
|
|
72
|
+
{ messageId: '2-0', event: mockEvent },
|
|
73
|
+
]);
|
|
74
|
+
await new Promise((resolve) => {
|
|
75
|
+
setTimeout(resolve, 50);
|
|
76
|
+
});
|
|
77
|
+
const xreadCalls = callMock.mock.calls.filter((args) => args[0] === 'XREADGROUP');
|
|
78
|
+
expect(xreadCalls.length).toBe(0);
|
|
79
|
+
});
|
|
80
|
+
it('should transition to HALF_OPEN after reset timeout and CLOSE on success threshold met', async () => {
|
|
81
|
+
const cb = processor.getCircuitBreaker();
|
|
82
|
+
const mockEvent = makeTestEvent('evt-3');
|
|
83
|
+
mockRedisXreadgroup(callMock, TestMessagingStream.TEST_STREAM, [
|
|
84
|
+
{ messageId: '1-0', event: mockEvent },
|
|
85
|
+
]);
|
|
86
|
+
processor.processError = new Error('Processor failed');
|
|
87
|
+
await processor.start();
|
|
88
|
+
await new Promise((resolve) => {
|
|
89
|
+
const interval = setInterval(() => {
|
|
90
|
+
if (cb.getState() === CircuitBreakerState.OPEN) {
|
|
91
|
+
clearInterval(interval);
|
|
92
|
+
resolve();
|
|
93
|
+
}
|
|
94
|
+
}, 10);
|
|
95
|
+
});
|
|
96
|
+
expect(cb.getState()).toBe(CircuitBreakerState.OPEN);
|
|
97
|
+
processor.processError = null;
|
|
98
|
+
await new Promise((resolve) => {
|
|
99
|
+
setTimeout(resolve, 60);
|
|
100
|
+
});
|
|
101
|
+
mockRedisXreadgroup(callMock, TestMessagingStream.TEST_STREAM, [
|
|
102
|
+
{ messageId: '2-0', event: mockEvent },
|
|
103
|
+
{ messageId: '3-0', event: mockEvent },
|
|
104
|
+
]);
|
|
105
|
+
await new Promise((resolve) => {
|
|
106
|
+
setTimeout(resolve, 60);
|
|
107
|
+
});
|
|
108
|
+
expect(cb.getState()).toBe(CircuitBreakerState.CLOSED);
|
|
109
|
+
expect(cb.getDiagnostics().failureCount).toBe(0);
|
|
110
|
+
});
|
|
111
|
+
});
|
|
112
|
+
//# sourceMappingURL=circuit-breaker.integration.spec.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"circuit-breaker.integration.spec.js","sourceRoot":"","sources":["../../../../src/test/specs/integration/circuit-breaker.integration.spec.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,QAAQ,EAAE,EAAE,EAAE,MAAM,EAAE,UAAU,EAAE,SAAS,EAAE,IAAI,EAAE,MAAM,eAAe,CAAC;AAElF,OAAO,EAAE,UAAU,EAAE,MAAM,uBAAuB,CAAC;AACnD,OAAO,EAAE,iBAAiB,EAAE,MAAM,kDAAkD,CAAC;AACrF,OAAO,EAAE,aAAa,EAAE,mBAAmB,EAAE,MAAM,sCAAsC,CAAC;AAE1F,OAAO,EAAE,aAAa,EAAE,MAAM,6CAA6C,CAAC;AAC5E,OAAO,EAAE,mBAAmB,EAAE,MAAM,8CAA8C,CAAC;AACnF,OAAO,EACL,mBAAmB,EACnB,kBAAkB,EAClB,qBAAqB,GACtB,MAAM,0CAA0C,CAAC;AAElD,QAAQ,CAAC,4BAA4B,EAAE,GAAG,EAAE;IAC1C,IAAI,SAA6B,CAAC;IAClC,IAAI,QAEH,CAAC;IACF,IAAI,SAA4B,CAAC;IAEjC,UAAU,CAAC,GAAG,EAAE;QACd,SAAS,GAAG,UAAU,EAAS,CAAC;QAChC,QAAQ,GAAG,aAAa,CAAC,SAAS,CAAC,CAAC;QAEpC,SAAS,GAAG,IAAI,iBAAiB,CAAC,SAAS,EAAE;YAC3C,UAAU,EAAE,mBAAmB,CAAC,WAAW;YAC3C,SAAS,EAAE,kBAAkB,CAAC,UAAU;YACxC,YAAY,EAAE,qBAAqB,CAAC,aAAa;YACjD,eAAe,EAAE,EAAE;YACnB,kBAAkB,EAAE,GAAG;YACvB,cAAc,EAAE;gBACd,gBAAgB,EAAE,CAAC;gBACnB,cAAc,EAAE,EAAE;gBAClB,gBAAgB,EAAE,CAAC;aACpB;SACF,CAAC,CAAC;IACL,CAAC,CAAC,CAAC;IAEH,SAAS,CAAC,KAAK,IAAI,EAAE;QACnB,MAAM,SAAS,CAAC,IAAI,EAAE,CAAC;QACvB,IAAI,CAAC,eAAe,EAAE,CAAC;IACzB,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,8EAA8E,EAAE,KAAK,IAAI,EAAE;QAC5F,MAAM,EAAE,GAAG,SAAS,CAAC,iBAAiB,EAAE,CAAC;QACzC,MAAM,CAAC,EAAE,CAAC,QAAQ,EAAE,CAAC,CAAC,IAAI,CAAC,mBAAmB,CAAC,MAAM,CAAC,CAAC;QAEvD,MAAM,SAAS,GAAG,aAAa,CAAC,OAAO,CAAC,CAAC;QACzC,mBAAmB,CAAC,QAAQ,EAAE,mBAAmB,CAAC,WAAW,EAAE;YAC7D,EAAE,SAAS,EAAE,KAAK,EAAE,KAAK,EAAE,SAAS,EAAE;YACtC,EAAE,SAAS,EAAE,KAAK,EAAE,KAAK,EAAE,SAAS,EAAE;YACtC,EAAE,SAAS,EAAE,KAAK,EAAE,KAAK,EAAE,SAAS,EAAE;SACvC,CAAC,CAAC;QAEH,SAAS,CAAC,YAAY,GAAG,IAAI,KAAK,CAAC,kBAAkB,CAAC,CAAC;QAEvD,MAAM,SAAS,CAAC,KAAK,EAAE,CAAC;QACxB,MAAM,IAAI,OAAO,CAAO,CAAC,OAAO,EAAE,EAAE;YAClC,UAAU,CAAC,OAAO,EAAE,EAAE,CAAC,CAAC;QAC1B,CAAC,CAAC,CAAC;QAEH,MAAM,CAAC,EAAE,CAAC,QAAQ,EAAE,CAAC,CAAC,IAAI,CAAC,mBAAmB,CAAC,MAAM,CAAC,CAAC;QACvD,MAAM,CAAC,EAAE,CAAC,cAAc,EAAE,CAAC,YAAY,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;QAEjD,MAAM,IAAI,OAAO,CAAO,CAAC,OAAO,EAAE,EAAE;YAClC,UAAU,CAAC,OAAO,EAAE,EAAE,CAAC,CAAC;QAC1B,CAAC,CAAC,CAAC;QAEH,MAAM,CAAC,EAAE,CAAC,QAAQ,EAAE,CAAC,CAAC,IAAI,CAAC,mBAAmB,CAAC,IAAI,CAAC,CAAC;IACvD,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,6DAA6D,EAAE,KAAK,IAAI,EAAE;QAC3E,MAAM,EAAE,GAAG,SAAS,CAAC,iBAAiB,EAAE,CAAC;QACzC,MAAM,SAAS,GAAG,aAAa,CAAC,OAAO,CAAC,CAAC;QACzC,mBAAmB,CAAC,QAAQ,EAAE,mBAAmB,CAAC,WAAW,EAAE;YAC7D,EAAE,SAAS,EAAE,KAAK,EAAE,KAAK,EAAE,SAAS,EAAE;SACvC,CAAC,CAAC;QAEH,SAAS,CAAC,YAAY,GAAG,IAAI,KAAK,CAAC,kBAAkB,CAAC,CAAC;QACvD,MAAM,SAAS,CAAC,KAAK,EAAE,CAAC;QAExB,MAAM,IAAI,OAAO,CAAO,CAAC,OAAO,EAAE,EAAE;YAClC,MAAM,QAAQ,GAAG,WAAW,CAAC,GAAG,EAAE;gBAChC,IAAI,EAAE,CAAC,QAAQ,EAAE,KAAK,mBAAmB,CAAC,IAAI,EAAE,CAAC;oBAC/C,aAAa,CAAC,QAAQ,CAAC,CAAC;oBACxB,OAAO,EAAE,CAAC;gBACZ,CAAC;YACH,CAAC,EAAE,EAAE,CAAC,CAAC;QACT,CAAC,CAAC,CAAC;QAEH,MAAM,CAAC,EAAE,CAAC,QAAQ,EAAE,CAAC,CAAC,IAAI,CAAC,mBAAmB,CAAC,IAAI,CAAC,CAAC;QAErD,QAAQ,CAAC,SAAS,EAAE,CAAC;QACrB,mBAAmB,CAAC,QAAQ,EAAE,mBAAmB,CAAC,WAAW,EAAE;YAC7D,EAAE,SAAS,EAAE,KAAK,EAAE,KAAK,EAAE,SAAS,EAAE;SACvC,CAAC,CAAC;QAEH,MAAM,IAAI,OAAO,CAAO,CAAC,OAAO,EAAE,EAAE;YAClC,UAAU,CAAC,OAAO,EAAE,EAAE,CAAC,CAAC;QAC1B,CAAC,CAAC,CAAC;QAEH,MAAM,UAAU,GAAG,QAAQ,CAAC,IAAI,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,CAAC,CAAC,KAAK,YAAY,CAAC,CAAC;QAClF,MAAM,CAAC,UAAU,CAAC,MAAM,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;IACpC,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,uFAAuF,EAAE,KAAK,IAAI,EAAE;QACrG,MAAM,EAAE,GAAG,SAAS,CAAC,iBAAiB,EAAE,CAAC;QACzC,MAAM,SAAS,GAAG,aAAa,CAAC,OAAO,CAAC,CAAC;QACzC,mBAAmB,CAAC,QAAQ,EAAE,mBAAmB,CAAC,WAAW,EAAE;YAC7D,EAAE,SAAS,EAAE,KAAK,EAAE,KAAK,EAAE,SAAS,EAAE;SACvC,CAAC,CAAC;QAEH,SAAS,CAAC,YAAY,GAAG,IAAI,KAAK,CAAC,kBAAkB,CAAC,CAAC;QACvD,MAAM,SAAS,CAAC,KAAK,EAAE,CAAC;QAExB,MAAM,IAAI,OAAO,CAAO,CAAC,OAAO,EAAE,EAAE;YAClC,MAAM,QAAQ,GAAG,WAAW,CAAC,GAAG,EAAE;gBAChC,IAAI,EAAE,CAAC,QAAQ,EAAE,KAAK,mBAAmB,CAAC,IAAI,EAAE,CAAC;oBAC/C,aAAa,CAAC,QAAQ,CAAC,CAAC;oBACxB,OAAO,EAAE,CAAC;gBACZ,CAAC;YACH,CAAC,EAAE,EAAE,CAAC,CAAC;QACT,CAAC,CAAC,CAAC;QAEH,MAAM,CAAC,EAAE,CAAC,QAAQ,EAAE,CAAC,CAAC,IAAI,CAAC,mBAAmB,CAAC,IAAI,CAAC,CAAC;QAErD,SAAS,CAAC,YAAY,GAAG,IAAI,CAAC;QAE9B,MAAM,IAAI,OAAO,CAAO,CAAC,OAAO,EAAE,EAAE;YAClC,UAAU,CAAC,OAAO,EAAE,EAAE,CAAC,CAAC;QAC1B,CAAC,CAAC,CAAC;QAEH,mBAAmB,CAAC,QAAQ,EAAE,mBAAmB,CAAC,WAAW,EAAE;YAC7D,EAAE,SAAS,EAAE,KAAK,EAAE,KAAK,EAAE,SAAS,EAAE;YACtC,EAAE,SAAS,EAAE,KAAK,EAAE,KAAK,EAAE,SAAS,EAAE;SACvC,CAAC,CAAC;QAEH,MAAM,IAAI,OAAO,CAAO,CAAC,OAAO,EAAE,EAAE;YAClC,UAAU,CAAC,OAAO,EAAE,EAAE,CAAC,CAAC;QAC1B,CAAC,CAAC,CAAC;QAEH,MAAM,CAAC,EAAE,CAAC,QAAQ,EAAE,CAAC,CAAC,IAAI,CAAC,mBAAmB,CAAC,MAAM,CAAC,CAAC;QACvD,MAAM,CAAC,EAAE,CAAC,cAAc,EAAE,CAAC,YAAY,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;IACnD,CAAC,CAAC,CAAC;AACL,CAAC,CAAC,CAAC"}
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"single-post-processor-retry.int.spec.d.ts","sourceRoot":"","sources":["../../../../src/test/specs/single-post-processor/single-post-processor-retry.int.spec.ts"],"names":[],"mappings":""}
|
|
@@ -0,0 +1,142 @@
|
|
|
1
|
+
import { describe, it, expect, beforeEach, afterEach, beforeAll, afterAll, jest, } from '@jest/globals';
|
|
2
|
+
import { Redis } from 'ioredis';
|
|
3
|
+
import { testRedisOptions } from '../../redis-config.js';
|
|
4
|
+
import { TestPostProcessor, makeTestEvent, pushTestEventToStream, TestMessagingStream, TestMessagingGroup, TestMessagingConsumer, waitFor, } from '../../utils/index.js';
|
|
5
|
+
describe('SinglePostProcessor — Retry & Exponential Backoff', () => {
|
|
6
|
+
let redis;
|
|
7
|
+
let processor;
|
|
8
|
+
beforeAll(() => {
|
|
9
|
+
redis = new Redis(testRedisOptions);
|
|
10
|
+
});
|
|
11
|
+
afterAll(async () => {
|
|
12
|
+
await redis.quit();
|
|
13
|
+
});
|
|
14
|
+
beforeEach(async () => {
|
|
15
|
+
await redis.flushdb();
|
|
16
|
+
processor = new TestPostProcessor(redis, {
|
|
17
|
+
streamName: TestMessagingStream.TEST_STREAM,
|
|
18
|
+
groupName: TestMessagingGroup.TEST_GROUP,
|
|
19
|
+
consumerName: TestMessagingConsumer.TEST_CONSUMER,
|
|
20
|
+
claimIntervalMs: 100,
|
|
21
|
+
claimMinIdleTimeMs: 50,
|
|
22
|
+
blockMs: 50,
|
|
23
|
+
circuitBreaker: {
|
|
24
|
+
failureThreshold: 20,
|
|
25
|
+
resetTimeoutMs: 10000,
|
|
26
|
+
successThreshold: 1,
|
|
27
|
+
},
|
|
28
|
+
retry: {
|
|
29
|
+
maxRetries: 3,
|
|
30
|
+
initialDelayMs: 100,
|
|
31
|
+
maxDelayMs: 5000,
|
|
32
|
+
backoffMultiplier: 2,
|
|
33
|
+
enableDlq: true,
|
|
34
|
+
},
|
|
35
|
+
});
|
|
36
|
+
});
|
|
37
|
+
afterEach(async () => {
|
|
38
|
+
await processor.stop();
|
|
39
|
+
});
|
|
40
|
+
it('should retry a failed message with exponential backoff', async () => {
|
|
41
|
+
const eventPayload = makeTestEvent('evt-retry-1');
|
|
42
|
+
await pushTestEventToStream(redis, TestMessagingStream.TEST_STREAM, 'evt-retry-1', eventPayload);
|
|
43
|
+
processor.processError = new Error('Simulated processing error');
|
|
44
|
+
await processor.start();
|
|
45
|
+
const retryQueueKey = `retry-queue:post-processor:${TestMessagingGroup.TEST_GROUP}`;
|
|
46
|
+
await waitFor(async () => {
|
|
47
|
+
const retryCount = await redis.zcard(retryQueueKey);
|
|
48
|
+
return retryCount === 1;
|
|
49
|
+
});
|
|
50
|
+
expect(processor.processedEvents).toHaveLength(0);
|
|
51
|
+
processor.processError = null;
|
|
52
|
+
await waitFor(() => processor.processedEvents.length === 1);
|
|
53
|
+
expect(processor.processedEvents[0]?.id).toBe('evt-retry-1');
|
|
54
|
+
const finalRetryCount = await redis.zcard(retryQueueKey);
|
|
55
|
+
expect(finalRetryCount).toBe(0);
|
|
56
|
+
});
|
|
57
|
+
it('should respect exponential backoff delays', async () => {
|
|
58
|
+
const eventPayload = makeTestEvent('evt-backoff-1');
|
|
59
|
+
await pushTestEventToStream(redis, TestMessagingStream.TEST_STREAM, 'evt-backoff-1', eventPayload);
|
|
60
|
+
const retryHelper = processor['retryHelper'];
|
|
61
|
+
const calculateDelaySpy = jest.spyOn(retryHelper, 'calculateDelay');
|
|
62
|
+
processor.processError = new Error('Simulated processing error');
|
|
63
|
+
await processor.start();
|
|
64
|
+
await waitFor(() => calculateDelaySpy.mock.calls.length >= 4);
|
|
65
|
+
expect(calculateDelaySpy.mock.calls.length).toBeGreaterThanOrEqual(4);
|
|
66
|
+
expect(calculateDelaySpy).toHaveBeenNthCalledWith(1, 1);
|
|
67
|
+
expect(calculateDelaySpy.mock.results[0]?.value).toBe(200);
|
|
68
|
+
expect(calculateDelaySpy).toHaveBeenNthCalledWith(2, 1);
|
|
69
|
+
expect(calculateDelaySpy.mock.results[1]?.value).toBe(200);
|
|
70
|
+
expect(calculateDelaySpy).toHaveBeenNthCalledWith(3, 2);
|
|
71
|
+
expect(calculateDelaySpy.mock.results[2]?.value).toBe(400);
|
|
72
|
+
expect(calculateDelaySpy).toHaveBeenNthCalledWith(4, 2);
|
|
73
|
+
expect(calculateDelaySpy.mock.results[3]?.value).toBe(400);
|
|
74
|
+
calculateDelaySpy.mockRestore();
|
|
75
|
+
});
|
|
76
|
+
it('should send message to DLQ after max retries exceeded', async () => {
|
|
77
|
+
const eventPayload = makeTestEvent('evt-dlq-1');
|
|
78
|
+
const messageId = await pushTestEventToStream(redis, TestMessagingStream.TEST_STREAM, 'evt-dlq-1', eventPayload);
|
|
79
|
+
processor.processError = new Error('Persistent processing error');
|
|
80
|
+
await processor.start();
|
|
81
|
+
for (let i = 0; i < 5; i++) {
|
|
82
|
+
await new Promise((resolve) => setTimeout(resolve, 250));
|
|
83
|
+
}
|
|
84
|
+
processor.processError = null;
|
|
85
|
+
const dlqStreamName = `${TestMessagingStream.TEST_STREAM}-dlq`;
|
|
86
|
+
const dlqEntries = (await redis.xrange(dlqStreamName, '-', '+'));
|
|
87
|
+
expect(dlqEntries.length).toBeGreaterThan(0);
|
|
88
|
+
const dlqEntry = dlqEntries[0];
|
|
89
|
+
const dlqFields = dlqEntry[1];
|
|
90
|
+
const dlqObject = {};
|
|
91
|
+
for (let i = 0; i < dlqFields.length; i += 2) {
|
|
92
|
+
const key = dlqFields[i];
|
|
93
|
+
const val = dlqFields[i + 1];
|
|
94
|
+
dlqObject[key] = val;
|
|
95
|
+
}
|
|
96
|
+
expect(dlqObject.messageId).toBe(messageId);
|
|
97
|
+
const parsedPayload = JSON.parse(dlqObject.payload || '{}');
|
|
98
|
+
expect(parsedPayload.id).toBe('evt-dlq-1');
|
|
99
|
+
expect(dlqObject.error).toContain('Persistent processing error');
|
|
100
|
+
});
|
|
101
|
+
it('should clear retry data on successful processing', async () => {
|
|
102
|
+
const eventPayload = makeTestEvent('evt-retry-clear-1');
|
|
103
|
+
const messageId = await pushTestEventToStream(redis, TestMessagingStream.TEST_STREAM, 'evt-retry-clear-1', eventPayload);
|
|
104
|
+
processor.processError = new Error('Simulated error');
|
|
105
|
+
await processor.start();
|
|
106
|
+
const retryKey = `retry:post-processor:${TestMessagingGroup.TEST_GROUP}:${messageId}:attempt`;
|
|
107
|
+
await waitFor(async () => {
|
|
108
|
+
const attemptCount = await redis.get(retryKey);
|
|
109
|
+
return attemptCount === '1';
|
|
110
|
+
});
|
|
111
|
+
processor.processError = null;
|
|
112
|
+
await waitFor(() => processor.processedEvents.length === 1);
|
|
113
|
+
const attemptCount = await redis.get(retryKey);
|
|
114
|
+
expect(attemptCount).toBeNull();
|
|
115
|
+
});
|
|
116
|
+
it('should not retry when retry is disabled', async () => {
|
|
117
|
+
await processor.stop();
|
|
118
|
+
const noRetryProcessor = new TestPostProcessor(redis, {
|
|
119
|
+
streamName: TestMessagingStream.TEST_STREAM,
|
|
120
|
+
groupName: TestMessagingGroup.TEST_GROUP,
|
|
121
|
+
consumerName: TestMessagingConsumer.TEST_CONSUMER,
|
|
122
|
+
claimIntervalMs: 100,
|
|
123
|
+
blockMs: 50,
|
|
124
|
+
circuitBreaker: {
|
|
125
|
+
failureThreshold: 20,
|
|
126
|
+
},
|
|
127
|
+
retry: {
|
|
128
|
+
maxRetries: 0,
|
|
129
|
+
},
|
|
130
|
+
});
|
|
131
|
+
const eventPayload = makeTestEvent('evt-no-retry-1');
|
|
132
|
+
await pushTestEventToStream(redis, TestMessagingStream.TEST_STREAM, 'evt-no-retry-1', eventPayload);
|
|
133
|
+
noRetryProcessor.processError = new Error('Processing failed');
|
|
134
|
+
await noRetryProcessor.start();
|
|
135
|
+
await new Promise((resolve) => setTimeout(resolve, 300));
|
|
136
|
+
const dlqStreamName = `${TestMessagingStream.TEST_STREAM}-dlq`;
|
|
137
|
+
const dlqEntries = (await redis.xrange(dlqStreamName, '-', '+'));
|
|
138
|
+
expect(dlqEntries.length).toBeGreaterThan(0);
|
|
139
|
+
await noRetryProcessor.stop();
|
|
140
|
+
});
|
|
141
|
+
});
|
|
142
|
+
//# sourceMappingURL=single-post-processor-retry.int.spec.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"single-post-processor-retry.int.spec.js","sourceRoot":"","sources":["../../../../src/test/specs/single-post-processor/single-post-processor-retry.int.spec.ts"],"names":[],"mappings":"AAAA,OAAO,EACL,QAAQ,EACR,EAAE,EACF,MAAM,EACN,UAAU,EACV,SAAS,EACT,SAAS,EACT,QAAQ,EACR,IAAI,GACL,MAAM,eAAe,CAAC;AACvB,OAAO,EAAE,KAAK,EAAE,MAAM,SAAS,CAAC;AAChC,OAAO,EAAE,gBAAgB,EAAE,MAAM,uBAAuB,CAAC;AACzD,OAAO,EACL,iBAAiB,EACjB,aAAa,EACb,qBAAqB,EACrB,mBAAmB,EACnB,kBAAkB,EAClB,qBAAqB,EACrB,OAAO,GACR,MAAM,sBAAsB,CAAC;AAG9B,QAAQ,CAAC,mDAAmD,EAAE,GAAG,EAAE;IACjE,IAAI,KAAY,CAAC;IACjB,IAAI,SAA4B,CAAC;IAEjC,SAAS,CAAC,GAAG,EAAE;QACb,KAAK,GAAG,IAAI,KAAK,CAAC,gBAAgB,CAAC,CAAC;IACtC,CAAC,CAAC,CAAC;IAEH,QAAQ,CAAC,KAAK,IAAI,EAAE;QAClB,MAAM,KAAK,CAAC,IAAI,EAAE,CAAC;IACrB,CAAC,CAAC,CAAC;IAEH,UAAU,CAAC,KAAK,IAAI,EAAE;QACpB,MAAM,KAAK,CAAC,OAAO,EAAE,CAAC;QAEtB,SAAS,GAAG,IAAI,iBAAiB,CAAC,KAAK,EAAE;YACvC,UAAU,EAAE,mBAAmB,CAAC,WAAW;YAC3C,SAAS,EAAE,kBAAkB,CAAC,UAAU;YACxC,YAAY,EAAE,qBAAqB,CAAC,aAAa;YACjD,eAAe,EAAE,GAAG;YACpB,kBAAkB,EAAE,EAAE;YACtB,OAAO,EAAE,EAAE;YACX,cAAc,EAAE;gBACd,gBAAgB,EAAE,EAAE;gBACpB,cAAc,EAAE,KAAK;gBACrB,gBAAgB,EAAE,CAAC;aACpB;YACD,KAAK,EAAE;gBACL,UAAU,EAAE,CAAC;gBACb,cAAc,EAAE,GAAG;gBACnB,UAAU,EAAE,IAAI;gBAChB,iBAAiB,EAAE,CAAC;gBACpB,SAAS,EAAE,IAAI;aAChB;SACF,CAAC,CAAC;IACL,CAAC,CAAC,CAAC;IAEH,SAAS,CAAC,KAAK,IAAI,EAAE;QACnB,MAAM,SAAS,CAAC,IAAI,EAAE,CAAC;IACzB,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,wDAAwD,EAAE,KAAK,IAAI,EAAE;QACtE,MAAM,YAAY,GAAG,aAAa,CAAC,aAAa,CAAC,CAAC;QAClD,MAAM,qBAAqB,CACzB,KAAK,EACL,mBAAmB,CAAC,WAAW,EAC/B,aAAa,EACb,YAAY,CACb,CAAC;QAGF,SAAS,CAAC,YAAY,GAAG,IAAI,KAAK,CAAC,4BAA4B,CAAC,CAAC;QACjE,MAAM,SAAS,CAAC,KAAK,EAAE,CAAC;QAGxB,MAAM,aAAa,GAAG,8BAA8B,kBAAkB,CAAC,UAAU,EAAE,CAAC;QACpF,MAAM,OAAO,CAAC,KAAK,IAAI,EAAE;YACvB,MAAM,UAAU,GAAG,MAAM,KAAK,CAAC,KAAK,CAAC,aAAa,CAAC,CAAC;YACpD,OAAO,UAAU,KAAK,CAAC,CAAC;QAC1B,CAAC,CAAC,CAAC;QAEH,MAAM,CAAC,SAAS,CAAC,eAAe,CAAC,CAAC,YAAY,CAAC,CAAC,CAAC,CAAC;QAGlD,SAAS,CAAC,YAAY,GAAG,IAAI,CAAC;QAG9B,MAAM,OAAO,CAAC,GAAG,EAAE,CAAC,SAAS,CAAC,eAAe,CAAC,MAAM,KAAK,CAAC,CAAC,CAAC;QAE5D,MAAM,CAAC,SAAS,CAAC,eAAe,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,IAAI,CAAC,aAAa,CAAC,CAAC;QAG7D,MAAM,eAAe,GAAG,MAAM,KAAK,CAAC,KAAK,CAAC,aAAa,CAAC,CAAC;QACzD,MAAM,CAAC,eAAe,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;IAClC,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,2CAA2C,EAAE,KAAK,IAAI,EAAE;QACzD,MAAM,YAAY,GAAG,aAAa,CAAC,eAAe,CAAC,CAAC;QACpD,MAAM,qBAAqB,CACzB,KAAK,EACL,mBAAmB,CAAC,WAAW,EAC/B,eAAe,EACf,YAAY,CACb,CAAC;QAEF,MAAM,WAAW,GAAG,SAAS,CAAC,aAAa,CAAC,CAAC;QAC7C,MAAM,iBAAiB,GAAG,IAAI,CAAC,KAAK,CAAC,WAAW,EAAE,gBAAgB,CAAC,CAAC;QAEpE,SAAS,CAAC,YAAY,GAAG,IAAI,KAAK,CAAC,4BAA4B,CAAC,CAAC;QACjE,MAAM,SAAS,CAAC,KAAK,EAAE,CAAC;QAGxB,MAAM,OAAO,CAAC,GAAG,EAAE,CAAC,iBAAiB,CAAC,IAAI,CAAC,KAAK,CAAC,MAAM,IAAI,CAAC,CAAC,CAAC;QAG9D,MAAM,CAAC,iBAAiB,CAAC,IAAI,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC,sBAAsB,CAAC,CAAC,CAAC,CAAC;QAGtE,MAAM,CAAC,iBAAiB,CAAC,CAAC,uBAAuB,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC;QACxD,MAAM,CAAC,iBAAiB,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,KAAK,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;QAC3D,MAAM,CAAC,iBAAiB,CAAC,CAAC,uBAAuB,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC;QACxD,MAAM,CAAC,iBAAiB,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,KAAK,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;QAG3D,MAAM,CAAC,iBAAiB,CAAC,CAAC,uBAAuB,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC;QACxD,MAAM,CAAC,iBAAiB,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,KAAK,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;QAC3D,MAAM,CAAC,iBAAiB,CAAC,CAAC,uBAAuB,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC;QACxD,MAAM,CAAC,iBAAiB,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,KAAK,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;QAE3D,iBAAiB,CAAC,WAAW,EAAE,CAAC;IAClC,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,uDAAuD,EAAE,KAAK,IAAI,EAAE;QACrE,MAAM,YAAY,GAAG,aAAa,CAAC,WAAW,CAAC,CAAC;QAChD,MAAM,SAAS,GAAG,MAAM,qBAAqB,CAC3C,KAAK,EACL,mBAAmB,CAAC,WAAW,EAC/B,WAAW,EACX,YAAY,CACb,CAAC;QAEF,SAAS,CAAC,YAAY,GAAG,IAAI,KAAK,CAAC,6BAA6B,CAAC,CAAC;QAClE,MAAM,SAAS,CAAC,KAAK,EAAE,CAAC;QAGxB,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC;YAC3B,MAAM,IAAI,OAAO,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC,UAAU,CAAC,OAAO,EAAE,GAAG,CAAC,CAAC,CAAC;QAC3D,CAAC;QAED,SAAS,CAAC,YAAY,GAAG,IAAI,CAAC;QAG9B,MAAM,aAAa,GAAG,GAAG,mBAAmB,CAAC,WAAW,MAAM,CAAC;QAC/D,MAAM,UAAU,GAAG,CAAC,MAAM,KAAK,CAAC,MAAM,CAAC,aAAa,EAAE,GAAG,EAAE,GAAG,CAAC,CAAyB,CAAC;QAEzF,MAAM,CAAC,UAAU,CAAC,MAAM,CAAC,CAAC,eAAe,CAAC,CAAC,CAAC,CAAC;QAC7C,MAAM,QAAQ,GAAG,UAAU,CAAC,CAAC,CAAC,CAAC;QAC/B,MAAM,SAAS,GAAG,QAAQ,CAAC,CAAC,CAAC,CAAC;QAE9B,MAAM,SAAS,GAA2B,EAAE,CAAC;QAC7C,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,SAAS,CAAC,MAAM,EAAE,CAAC,IAAI,CAAC,EAAE,CAAC;YAC7C,MAAM,GAAG,GAAG,SAAS,CAAC,CAAC,CAAC,CAAC;YACzB,MAAM,GAAG,GAAG,SAAS,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC;YAC7B,SAAS,CAAC,GAAG,CAAC,GAAG,GAAG,CAAC;QACvB,CAAC;QAED,MAAM,CAAC,SAAS,CAAC,SAAS,CAAC,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC;QAC5C,MAAM,aAAa,GAAG,IAAI,CAAC,KAAK,CAAC,SAAS,CAAC,OAAO,IAAI,IAAI,CAAiB,CAAC;QAC5E,MAAM,CAAC,aAAa,CAAC,EAAE,CAAC,CAAC,IAAI,CAAC,WAAW,CAAC,CAAC;QAC3C,MAAM,CAAC,SAAS,CAAC,KAAK,CAAC,CAAC,SAAS,CAAC,6BAA6B,CAAC,CAAC;IACnE,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,kDAAkD,EAAE,KAAK,IAAI,EAAE;QAChE,MAAM,YAAY,GAAG,aAAa,CAAC,mBAAmB,CAAC,CAAC;QACxD,MAAM,SAAS,GAAG,MAAM,qBAAqB,CAC3C,KAAK,EACL,mBAAmB,CAAC,WAAW,EAC/B,mBAAmB,EACnB,YAAY,CACb,CAAC;QAEF,SAAS,CAAC,YAAY,GAAG,IAAI,KAAK,CAAC,iBAAiB,CAAC,CAAC;QACtD,MAAM,SAAS,CAAC,KAAK,EAAE,CAAC;QAExB,MAAM,QAAQ,GAAG,wBAAwB,kBAAkB,CAAC,UAAU,IAAI,SAAS,UAAU,CAAC;QAG9F,MAAM,OAAO,CAAC,KAAK,IAAI,EAAE;YACvB,MAAM,YAAY,GAAG,MAAM,KAAK,CAAC,GAAG,CAAC,QAAQ,CAAC,CAAC;YAC/C,OAAO,YAAY,KAAK,GAAG,CAAC;QAC9B,CAAC,CAAC,CAAC;QAEH,SAAS,CAAC,YAAY,GAAG,IAAI,CAAC;QAG9B,MAAM,OAAO,CAAC,GAAG,EAAE,CAAC,SAAS,CAAC,eAAe,CAAC,MAAM,KAAK,CAAC,CAAC,CAAC;QAE5D,MAAM,YAAY,GAAG,MAAM,KAAK,CAAC,GAAG,CAAC,QAAQ,CAAC,CAAC;QAC/C,MAAM,CAAC,YAAY,CAAC,CAAC,QAAQ,EAAE,CAAC;IAClC,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,yCAAyC,EAAE,KAAK,IAAI,EAAE;QACvD,MAAM,SAAS,CAAC,IAAI,EAAE,CAAC;QAGvB,MAAM,gBAAgB,GAAG,IAAI,iBAAiB,CAAC,KAAK,EAAE;YACpD,UAAU,EAAE,mBAAmB,CAAC,WAAW;YAC3C,SAAS,EAAE,kBAAkB,CAAC,UAAU;YACxC,YAAY,EAAE,qBAAqB,CAAC,aAAa;YACjD,eAAe,EAAE,GAAG;YACpB,OAAO,EAAE,EAAE;YACX,cAAc,EAAE;gBACd,gBAAgB,EAAE,EAAE;aACrB;YACD,KAAK,EAAE;gBACL,UAAU,EAAE,CAAC;aACd;SACF,CAAC,CAAC;QAEH,MAAM,YAAY,GAAG,aAAa,CAAC,gBAAgB,CAAC,CAAC;QACrD,MAAM,qBAAqB,CACzB,KAAK,EACL,mBAAmB,CAAC,WAAW,EAC/B,gBAAgB,EAChB,YAAY,CACb,CAAC;QAEF,gBAAgB,CAAC,YAAY,GAAG,IAAI,KAAK,CAAC,mBAAmB,CAAC,CAAC;QAC/D,MAAM,gBAAgB,CAAC,KAAK,EAAE,CAAC;QAC/B,MAAM,IAAI,OAAO,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC,UAAU,CAAC,OAAO,EAAE,GAAG,CAAC,CAAC,CAAC;QAGzD,MAAM,aAAa,GAAG,GAAG,mBAAmB,CAAC,WAAW,MAAM,CAAC;QAC/D,MAAM,UAAU,GAAG,CAAC,MAAM,KAAK,CAAC,MAAM,CAAC,aAAa,EAAE,GAAG,EAAE,GAAG,CAAC,CAAyB,CAAC;QACzF,MAAM,CAAC,UAAU,CAAC,MAAM,CAAC,CAAC,eAAe,CAAC,CAAC,CAAC,CAAC;QAE7C,MAAM,gBAAgB,CAAC,IAAI,EAAE,CAAC;IAChC,CAAC,CAAC,CAAC;AACL,CAAC,CAAC,CAAC"}
|