@volontariapp/post-processors 1.0.0-snap-856af13
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +15 -0
- package/dist/core/base.post-processor.d.ts +27 -0
- package/dist/core/base.post-processor.d.ts.map +1 -0
- package/dist/core/base.post-processor.js +206 -0
- package/dist/core/base.post-processor.js.map +1 -0
- package/dist/core/batch.post-processor.d.ts +13 -0
- package/dist/core/batch.post-processor.d.ts.map +1 -0
- package/dist/core/batch.post-processor.js +92 -0
- package/dist/core/batch.post-processor.js.map +1 -0
- package/dist/core/index.d.ts +5 -0
- package/dist/core/index.d.ts.map +1 -0
- package/dist/core/index.js +5 -0
- package/dist/core/index.js.map +1 -0
- package/dist/core/redis-stream.helper.d.ts +12 -0
- package/dist/core/redis-stream.helper.d.ts.map +1 -0
- package/dist/core/redis-stream.helper.js +44 -0
- package/dist/core/redis-stream.helper.js.map +1 -0
- package/dist/core/single.post-processor.d.ts +12 -0
- package/dist/core/single.post-processor.d.ts.map +1 -0
- package/dist/core/single.post-processor.js +58 -0
- package/dist/core/single.post-processor.js.map +1 -0
- package/dist/data-source.d.ts +5 -0
- package/dist/data-source.d.ts.map +1 -0
- package/dist/data-source.js +33 -0
- package/dist/data-source.js.map +1 -0
- package/dist/index.d.ts +4 -0
- package/dist/index.d.ts.map +1 -0
- package/dist/index.js +4 -0
- package/dist/index.js.map +1 -0
- package/dist/interfaces/batch-event-item.interface.d.ts +7 -0
- package/dist/interfaces/batch-event-item.interface.d.ts.map +1 -0
- package/dist/interfaces/batch-event-item.interface.js +2 -0
- package/dist/interfaces/batch-event-item.interface.js.map +1 -0
- package/dist/interfaces/index.d.ts +4 -0
- package/dist/interfaces/index.d.ts.map +1 -0
- package/dist/interfaces/index.js +2 -0
- package/dist/interfaces/index.js.map +1 -0
- package/dist/interfaces/pending-message-info.interface.d.ts +7 -0
- package/dist/interfaces/pending-message-info.interface.d.ts.map +1 -0
- package/dist/interfaces/pending-message-info.interface.js +2 -0
- package/dist/interfaces/pending-message-info.interface.js.map +1 -0
- package/dist/interfaces/post-processor-options.interface.d.ts +11 -0
- package/dist/interfaces/post-processor-options.interface.d.ts.map +1 -0
- package/dist/interfaces/post-processor-options.interface.js +2 -0
- package/dist/interfaces/post-processor-options.interface.js.map +1 -0
- package/dist/migrations/1776783577425-CreateJobsOutbox.d.ts +7 -0
- package/dist/migrations/1776783577425-CreateJobsOutbox.d.ts.map +1 -0
- package/dist/migrations/1776783577425-CreateJobsOutbox.js +25 -0
- package/dist/migrations/1776783577425-CreateJobsOutbox.js.map +1 -0
- package/dist/migrations/1778328780881-InitialSchemaJobAudit.d.ts +6 -0
- package/dist/migrations/1778328780881-InitialSchemaJobAudit.d.ts.map +1 -0
- package/dist/migrations/1778328780881-InitialSchemaJobAudit.js +93 -0
- package/dist/migrations/1778328780881-InitialSchemaJobAudit.js.map +1 -0
- package/dist/test/global-setup.d.ts +3 -0
- package/dist/test/global-setup.d.ts.map +1 -0
- package/dist/test/global-setup.js +34 -0
- package/dist/test/global-setup.js.map +1 -0
- package/dist/test/redis-config.d.ts +3 -0
- package/dist/test/redis-config.d.ts.map +1 -0
- package/dist/test/redis-config.js +8 -0
- package/dist/test/redis-config.js.map +1 -0
- package/dist/test/setup.d.ts +2 -0
- package/dist/test/setup.d.ts.map +1 -0
- package/dist/test/setup.js +2 -0
- package/dist/test/setup.js.map +1 -0
- package/dist/test/specs/batch-post-processor/batch-post-processor.int.spec.d.ts +2 -0
- package/dist/test/specs/batch-post-processor/batch-post-processor.int.spec.d.ts.map +1 -0
- package/dist/test/specs/batch-post-processor/batch-post-processor.int.spec.js +87 -0
- package/dist/test/specs/batch-post-processor/batch-post-processor.int.spec.js.map +1 -0
- package/dist/test/specs/batch-post-processor/batch-post-processor.unit.spec.d.ts +2 -0
- package/dist/test/specs/batch-post-processor/batch-post-processor.unit.spec.d.ts.map +1 -0
- package/dist/test/specs/batch-post-processor/batch-post-processor.unit.spec.js +62 -0
- package/dist/test/specs/batch-post-processor/batch-post-processor.unit.spec.js.map +1 -0
- package/dist/test/specs/single-post-processor/single-post-processor.int.spec.d.ts +2 -0
- package/dist/test/specs/single-post-processor/single-post-processor.int.spec.d.ts.map +1 -0
- package/dist/test/specs/single-post-processor/single-post-processor.int.spec.js +89 -0
- package/dist/test/specs/single-post-processor/single-post-processor.int.spec.js.map +1 -0
- package/dist/test/specs/single-post-processor/single-post-processor.unit.spec.d.ts +2 -0
- package/dist/test/specs/single-post-processor/single-post-processor.unit.spec.d.ts.map +1 -0
- package/dist/test/specs/single-post-processor/single-post-processor.unit.spec.js +131 -0
- package/dist/test/specs/single-post-processor/single-post-processor.unit.spec.js.map +1 -0
- package/dist/test/utils/classes/test-batch-post-processor.class.d.ts +11 -0
- package/dist/test/utils/classes/test-batch-post-processor.class.d.ts.map +1 -0
- package/dist/test/utils/classes/test-batch-post-processor.class.js +20 -0
- package/dist/test/utils/classes/test-batch-post-processor.class.js.map +1 -0
- package/dist/test/utils/classes/test-post-processor.class.d.ts +12 -0
- package/dist/test/utils/classes/test-post-processor.class.d.ts.map +1 -0
- package/dist/test/utils/classes/test-post-processor.class.js +20 -0
- package/dist/test/utils/classes/test-post-processor.class.js.map +1 -0
- package/dist/test/utils/enums/test-messaging.enum.d.ts +16 -0
- package/dist/test/utils/enums/test-messaging.enum.d.ts.map +1 -0
- package/dist/test/utils/enums/test-messaging.enum.js +20 -0
- package/dist/test/utils/enums/test-messaging.enum.js.map +1 -0
- package/dist/test/utils/factories/test-event.factory.d.ts +5 -0
- package/dist/test/utils/factories/test-event.factory.d.ts.map +1 -0
- package/dist/test/utils/factories/test-event.factory.js +49 -0
- package/dist/test/utils/factories/test-event.factory.js.map +1 -0
- package/dist/test/utils/index.d.ts +7 -0
- package/dist/test/utils/index.d.ts.map +1 -0
- package/dist/test/utils/index.js +7 -0
- package/dist/test/utils/index.js.map +1 -0
- package/dist/test/utils/mocks/redis-call.mock.d.ts +21 -0
- package/dist/test/utils/mocks/redis-call.mock.d.ts.map +1 -0
- package/dist/test/utils/mocks/redis-call.mock.js +73 -0
- package/dist/test/utils/mocks/redis-call.mock.js.map +1 -0
- package/dist/test/utils/types/test-messaging.types.d.ts +9 -0
- package/dist/test/utils/types/test-messaging.types.d.ts.map +1 -0
- package/dist/test/utils/types/test-messaging.types.js +2 -0
- package/dist/test/utils/types/test-messaging.types.js.map +1 -0
- package/dist/types/index.d.ts +2 -0
- package/dist/types/index.d.ts.map +1 -0
- package/dist/types/index.js +2 -0
- package/dist/types/index.js.map +1 -0
- package/dist/types/post-processor.types.d.ts +14 -0
- package/dist/types/post-processor.types.d.ts.map +1 -0
- package/dist/types/post-processor.types.js +2 -0
- package/dist/types/post-processor.types.js.map +1 -0
- package/package.json +62 -0
|
@@ -0,0 +1,7 @@
|
|
|
1
|
+
import type { EventMessagingType, EventRegistry, StreamEvent } from '@volontariapp/messaging';
|
|
2
|
+
import type { ExtractPayload } from '../types/post-processor.types.js';
|
|
3
|
+
export interface BatchEventItem<TKey extends EventMessagingType = EventMessagingType> {
|
|
4
|
+
event: StreamEvent<ExtractPayload<EventRegistry[TKey]>>;
|
|
5
|
+
messageId: string;
|
|
6
|
+
}
|
|
7
|
+
//# sourceMappingURL=batch-event-item.interface.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"batch-event-item.interface.d.ts","sourceRoot":"","sources":["../../src/interfaces/batch-event-item.interface.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,kBAAkB,EAAE,aAAa,EAAE,WAAW,EAAE,MAAM,yBAAyB,CAAC;AAC9F,OAAO,KAAK,EAAE,cAAc,EAAE,MAAM,kCAAkC,CAAC;AAEvE,MAAM,WAAW,cAAc,CAAC,IAAI,SAAS,kBAAkB,GAAG,kBAAkB;IAClF,KAAK,EAAE,WAAW,CAAC,cAAc,CAAC,aAAa,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;IACxD,SAAS,EAAE,MAAM,CAAC;CACnB"}
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"batch-event-item.interface.js","sourceRoot":"","sources":["../../src/interfaces/batch-event-item.interface.ts"],"names":[],"mappings":""}
|
|
@@ -0,0 +1,4 @@
|
|
|
1
|
+
export type { PostProcessorOptions } from './post-processor-options.interface.js';
|
|
2
|
+
export type { BatchEventItem } from './batch-event-item.interface.js';
|
|
3
|
+
export type { PendingMessageInfo } from './pending-message-info.interface.js';
|
|
4
|
+
//# sourceMappingURL=index.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/interfaces/index.ts"],"names":[],"mappings":"AAAA,YAAY,EAAE,oBAAoB,EAAE,MAAM,uCAAuC,CAAC;AAClF,YAAY,EAAE,cAAc,EAAE,MAAM,iCAAiC,CAAC;AACtE,YAAY,EAAE,kBAAkB,EAAE,MAAM,qCAAqC,CAAC"}
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"index.js","sourceRoot":"","sources":["../../src/interfaces/index.ts"],"names":[],"mappings":""}
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"pending-message-info.interface.d.ts","sourceRoot":"","sources":["../../src/interfaces/pending-message-info.interface.ts"],"names":[],"mappings":"AAAA,MAAM,WAAW,kBAAkB;IACjC,SAAS,EAAE,MAAM,CAAC;IAClB,YAAY,EAAE,MAAM,CAAC;IACrB,UAAU,EAAE,MAAM,CAAC;IACnB,aAAa,EAAE,MAAM,CAAC;CACvB"}
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"pending-message-info.interface.js","sourceRoot":"","sources":["../../src/interfaces/pending-message-info.interface.ts"],"names":[],"mappings":""}
|
|
@@ -0,0 +1,11 @@
|
|
|
1
|
+
export interface PostProcessorOptions {
|
|
2
|
+
streamName: string;
|
|
3
|
+
groupName: string;
|
|
4
|
+
consumerName?: string;
|
|
5
|
+
batchSize?: number;
|
|
6
|
+
blockMs?: number;
|
|
7
|
+
claimIntervalMs?: number;
|
|
8
|
+
claimMinIdleTimeMs?: number;
|
|
9
|
+
idempotencyTtlSeconds?: number;
|
|
10
|
+
}
|
|
11
|
+
//# sourceMappingURL=post-processor-options.interface.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"post-processor-options.interface.d.ts","sourceRoot":"","sources":["../../src/interfaces/post-processor-options.interface.ts"],"names":[],"mappings":"AAAA,MAAM,WAAW,oBAAoB;IAInC,UAAU,EAAE,MAAM,CAAC;IAKnB,SAAS,EAAE,MAAM,CAAC;IAMlB,YAAY,CAAC,EAAE,MAAM,CAAC;IAMtB,SAAS,CAAC,EAAE,MAAM,CAAC;IAMnB,OAAO,CAAC,EAAE,MAAM,CAAC;IAMjB,eAAe,CAAC,EAAE,MAAM,CAAC;IAMzB,kBAAkB,CAAC,EAAE,MAAM,CAAC;IAO5B,qBAAqB,CAAC,EAAE,MAAM,CAAC;CAChC"}
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"post-processor-options.interface.js","sourceRoot":"","sources":["../../src/interfaces/post-processor-options.interface.ts"],"names":[],"mappings":""}
|
|
@@ -0,0 +1,7 @@
|
|
|
1
|
+
import type { MigrationInterface, QueryRunner } from 'typeorm';
|
|
2
|
+
export declare class CreateJobsOutbox1776783577425 implements MigrationInterface {
|
|
3
|
+
name: string;
|
|
4
|
+
up(queryRunner: QueryRunner): Promise<void>;
|
|
5
|
+
down(queryRunner: QueryRunner): Promise<void>;
|
|
6
|
+
}
|
|
7
|
+
//# sourceMappingURL=1776783577425-CreateJobsOutbox.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"1776783577425-CreateJobsOutbox.d.ts","sourceRoot":"","sources":["../../src/migrations/1776783577425-CreateJobsOutbox.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,kBAAkB,EAAE,WAAW,EAAE,MAAM,SAAS,CAAC;AAE/D,qBAAa,6BAA8B,YAAW,kBAAkB;IACtE,IAAI,SAAmC;IAE1B,EAAE,CAAC,WAAW,EAAE,WAAW,GAAG,OAAO,CAAC,IAAI,CAAC;IAqB3C,IAAI,CAAC,WAAW,EAAE,WAAW,GAAG,OAAO,CAAC,IAAI,CAAC;CAG3D"}
|
|
@@ -0,0 +1,25 @@
|
|
|
1
|
+
export class CreateJobsOutbox1776783577425 {
|
|
2
|
+
name = 'CreateJobsOutbox1776783577425';
|
|
3
|
+
async up(queryRunner) {
|
|
4
|
+
await queryRunner.query('CREATE EXTENSION IF NOT EXISTS "uuid-ossp"');
|
|
5
|
+
await queryRunner.query(`CREATE TABLE "jobs_outbox" (
|
|
6
|
+
"id" uuid NOT NULL DEFAULT uuid_generate_v4(),
|
|
7
|
+
"status" character varying(20) NOT NULL DEFAULT 'PENDING',
|
|
8
|
+
"attempts" integer NOT NULL DEFAULT '0',
|
|
9
|
+
"lastError" text,
|
|
10
|
+
"type" character varying(100) NOT NULL,
|
|
11
|
+
"emitter" character varying(100) NOT NULL,
|
|
12
|
+
"updated_at" TIMESTAMP,
|
|
13
|
+
"created_at" TIMESTAMP NOT NULL DEFAULT now(),
|
|
14
|
+
"target" character varying(100) NOT NULL,
|
|
15
|
+
"payload" jsonb NOT NULL,
|
|
16
|
+
"scheduled_at" TIMESTAMP NOT NULL,
|
|
17
|
+
"traceId" uuid,
|
|
18
|
+
CONSTRAINT "PK_7172b6dd5767f423ecb44d9fb57" PRIMARY KEY ("id")
|
|
19
|
+
)`);
|
|
20
|
+
}
|
|
21
|
+
async down(queryRunner) {
|
|
22
|
+
await queryRunner.query('DROP TABLE "jobs_outbox"');
|
|
23
|
+
}
|
|
24
|
+
}
|
|
25
|
+
//# sourceMappingURL=1776783577425-CreateJobsOutbox.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"1776783577425-CreateJobsOutbox.js","sourceRoot":"","sources":["../../src/migrations/1776783577425-CreateJobsOutbox.ts"],"names":[],"mappings":"AAEA,MAAM,OAAO,6BAA6B;IACxC,IAAI,GAAG,+BAA+B,CAAC;IAEhC,KAAK,CAAC,EAAE,CAAC,WAAwB;QACtC,MAAM,WAAW,CAAC,KAAK,CAAC,4CAA4C,CAAC,CAAC;QACtE,MAAM,WAAW,CAAC,KAAK,CACrB;;;;;;;;;;;;;;QAcE,CACH,CAAC;IACJ,CAAC;IAEM,KAAK,CAAC,IAAI,CAAC,WAAwB;QACxC,MAAM,WAAW,CAAC,KAAK,CAAC,0BAA0B,CAAC,CAAC;IACtD,CAAC;CACF"}
|
|
@@ -0,0 +1,6 @@
|
|
|
1
|
+
import { type MigrationInterface, type QueryRunner } from 'typeorm';
|
|
2
|
+
export declare class InitialSchemaJobAudit1778328780881 implements MigrationInterface {
|
|
3
|
+
up(queryRunner: QueryRunner): Promise<void>;
|
|
4
|
+
down(queryRunner: QueryRunner): Promise<void>;
|
|
5
|
+
}
|
|
6
|
+
//# sourceMappingURL=1778328780881-InitialSchemaJobAudit.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"1778328780881-InitialSchemaJobAudit.d.ts","sourceRoot":"","sources":["../../src/migrations/1778328780881-InitialSchemaJobAudit.ts"],"names":[],"mappings":"AAAA,OAAO,EAAqB,KAAK,kBAAkB,EAAE,KAAK,WAAW,EAAE,MAAM,SAAS,CAAC;AAEvF,qBAAa,kCAAmC,YAAW,kBAAkB;IAC9D,EAAE,CAAC,WAAW,EAAE,WAAW,GAAG,OAAO,CAAC,IAAI,CAAC;IA2F3C,IAAI,CAAC,WAAW,EAAE,WAAW,GAAG,OAAO,CAAC,IAAI,CAAC;CAG3D"}
|
|
@@ -0,0 +1,93 @@
|
|
|
1
|
+
import { Table, TableIndex } from 'typeorm';
|
|
2
|
+
export class InitialSchemaJobAudit1778328780881 {
|
|
3
|
+
async up(queryRunner) {
|
|
4
|
+
await queryRunner.createTable(new Table({
|
|
5
|
+
name: 'job_audit',
|
|
6
|
+
columns: [
|
|
7
|
+
{
|
|
8
|
+
name: 'id',
|
|
9
|
+
type: 'uuid',
|
|
10
|
+
isPrimary: true,
|
|
11
|
+
isGenerated: true,
|
|
12
|
+
generationStrategy: 'uuid',
|
|
13
|
+
default: 'gen_random_uuid()',
|
|
14
|
+
},
|
|
15
|
+
{
|
|
16
|
+
name: 'job_id',
|
|
17
|
+
type: 'varchar',
|
|
18
|
+
length: '100',
|
|
19
|
+
isNullable: false,
|
|
20
|
+
isUnique: true,
|
|
21
|
+
},
|
|
22
|
+
{
|
|
23
|
+
name: 'job_type',
|
|
24
|
+
type: 'varchar',
|
|
25
|
+
length: '255',
|
|
26
|
+
isNullable: false,
|
|
27
|
+
},
|
|
28
|
+
{
|
|
29
|
+
name: 'status',
|
|
30
|
+
type: 'varchar',
|
|
31
|
+
length: '20',
|
|
32
|
+
default: "'PENDING'",
|
|
33
|
+
},
|
|
34
|
+
{
|
|
35
|
+
name: 'worker_id',
|
|
36
|
+
type: 'varchar',
|
|
37
|
+
length: '255',
|
|
38
|
+
isNullable: false,
|
|
39
|
+
},
|
|
40
|
+
{
|
|
41
|
+
name: 'current_attempt',
|
|
42
|
+
type: 'int',
|
|
43
|
+
default: 1,
|
|
44
|
+
},
|
|
45
|
+
{
|
|
46
|
+
name: 'started_at',
|
|
47
|
+
type: 'timestamp',
|
|
48
|
+
isNullable: true,
|
|
49
|
+
},
|
|
50
|
+
{
|
|
51
|
+
name: 'finished_at',
|
|
52
|
+
type: 'timestamp',
|
|
53
|
+
isNullable: true,
|
|
54
|
+
},
|
|
55
|
+
{
|
|
56
|
+
name: 'result_payload',
|
|
57
|
+
type: 'jsonb',
|
|
58
|
+
isNullable: true,
|
|
59
|
+
},
|
|
60
|
+
{
|
|
61
|
+
name: 'error_message',
|
|
62
|
+
type: 'text',
|
|
63
|
+
isNullable: true,
|
|
64
|
+
},
|
|
65
|
+
{
|
|
66
|
+
name: 'error_stack',
|
|
67
|
+
type: 'text',
|
|
68
|
+
isNullable: true,
|
|
69
|
+
},
|
|
70
|
+
{
|
|
71
|
+
name: 'created_at',
|
|
72
|
+
type: 'timestamp',
|
|
73
|
+
default: 'now()',
|
|
74
|
+
},
|
|
75
|
+
{
|
|
76
|
+
name: 'updated_at',
|
|
77
|
+
type: 'timestamp',
|
|
78
|
+
default: 'now()',
|
|
79
|
+
},
|
|
80
|
+
],
|
|
81
|
+
}), true);
|
|
82
|
+
await queryRunner.createIndices('job_audit', [
|
|
83
|
+
new TableIndex({ columnNames: ['job_id'], isUnique: true }),
|
|
84
|
+
new TableIndex({ columnNames: ['job_type'] }),
|
|
85
|
+
new TableIndex({ columnNames: ['worker_id'] }),
|
|
86
|
+
new TableIndex({ columnNames: ['status'] }),
|
|
87
|
+
]);
|
|
88
|
+
}
|
|
89
|
+
async down(queryRunner) {
|
|
90
|
+
await queryRunner.dropTable('job_audit');
|
|
91
|
+
}
|
|
92
|
+
}
|
|
93
|
+
//# sourceMappingURL=1778328780881-InitialSchemaJobAudit.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"1778328780881-InitialSchemaJobAudit.js","sourceRoot":"","sources":["../../src/migrations/1778328780881-InitialSchemaJobAudit.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,KAAK,EAAE,UAAU,EAA6C,MAAM,SAAS,CAAC;AAEvF,MAAM,OAAO,kCAAkC;IACtC,KAAK,CAAC,EAAE,CAAC,WAAwB;QACtC,MAAM,WAAW,CAAC,WAAW,CAC3B,IAAI,KAAK,CAAC;YACR,IAAI,EAAE,WAAW;YACjB,OAAO,EAAE;gBACP;oBACE,IAAI,EAAE,IAAI;oBACV,IAAI,EAAE,MAAM;oBACZ,SAAS,EAAE,IAAI;oBACf,WAAW,EAAE,IAAI;oBACjB,kBAAkB,EAAE,MAAM;oBAC1B,OAAO,EAAE,mBAAmB;iBAC7B;gBACD;oBACE,IAAI,EAAE,QAAQ;oBACd,IAAI,EAAE,SAAS;oBACf,MAAM,EAAE,KAAK;oBACb,UAAU,EAAE,KAAK;oBACjB,QAAQ,EAAE,IAAI;iBACf;gBACD;oBACE,IAAI,EAAE,UAAU;oBAChB,IAAI,EAAE,SAAS;oBACf,MAAM,EAAE,KAAK;oBACb,UAAU,EAAE,KAAK;iBAClB;gBACD;oBACE,IAAI,EAAE,QAAQ;oBACd,IAAI,EAAE,SAAS;oBACf,MAAM,EAAE,IAAI;oBACZ,OAAO,EAAE,WAAW;iBACrB;gBACD;oBACE,IAAI,EAAE,WAAW;oBACjB,IAAI,EAAE,SAAS;oBACf,MAAM,EAAE,KAAK;oBACb,UAAU,EAAE,KAAK;iBAClB;gBACD;oBACE,IAAI,EAAE,iBAAiB;oBACvB,IAAI,EAAE,KAAK;oBACX,OAAO,EAAE,CAAC;iBACX;gBACD;oBACE,IAAI,EAAE,YAAY;oBAClB,IAAI,EAAE,WAAW;oBACjB,UAAU,EAAE,IAAI;iBACjB;gBACD;oBACE,IAAI,EAAE,aAAa;oBACnB,IAAI,EAAE,WAAW;oBACjB,UAAU,EAAE,IAAI;iBACjB;gBACD;oBACE,IAAI,EAAE,gBAAgB;oBACtB,IAAI,EAAE,OAAO;oBACb,UAAU,EAAE,IAAI;iBACjB;gBACD;oBACE,IAAI,EAAE,eAAe;oBACrB,IAAI,EAAE,MAAM;oBACZ,UAAU,EAAE,IAAI;iBACjB;gBACD;oBACE,IAAI,EAAE,aAAa;oBACnB,IAAI,EAAE,MAAM;oBACZ,UAAU,EAAE,IAAI;iBACjB;gBACD;oBACE,IAAI,EAAE,YAAY;oBAClB,IAAI,EAAE,WAAW;oBACjB,OAAO,EAAE,OAAO;iBACjB;gBACD;oBACE,IAAI,EAAE,YAAY;oBAClB,IAAI,EAAE,WAAW;oBACjB,OAAO,EAAE,OAAO;iBACjB;aACF;SACF,CAAC,EACF,IAAI,CACL,CAAC;QAEF,MAAM,WAAW,CAAC,aAAa,CAAC,WAAW,EAAE;YAC3C,IAAI,UAAU,CAAC,EAAE,WAAW,EAAE,CAAC,QAAQ,CAAC,EAAE,QAAQ,EAAE,IAAI,EAAE,CAAC;YAC3D,IAAI,UAAU,CAAC,EAAE,WAAW,EAAE,CAAC,UAAU,CAAC,EAAE,CAAC;YAC7C,IAAI,UAAU,CAAC,EAAE,WAAW,EAAE,CAAC,WAAW,CAAC,EAAE,CAAC;YAC9C,IAAI,UAAU,CAAC,EAAE,WAAW,EAAE,CAAC,QAAQ,CAAC,EAAE,CAAC;SAC5C,CAAC,CAAC;IACL,CAAC;IAEM,KAAK,CAAC,IAAI,CAAC,WAAwB;QACxC,MAAM,WAAW,CAAC,SAAS,CAAC,WAAW,CAAC,CAAC;IAC3C,CAAC;CACF"}
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"global-setup.d.ts","sourceRoot":"","sources":["../../src/test/global-setup.ts"],"names":[],"mappings":";AAIA,wBAgCE"}
|
|
@@ -0,0 +1,34 @@
|
|
|
1
|
+
import { Redis } from 'ioredis';
|
|
2
|
+
const sleep = (ms) => new Promise((resolve) => setTimeout(resolve, ms));
|
|
3
|
+
export default async () => {
|
|
4
|
+
const maxAttempts = 20;
|
|
5
|
+
for (let attempt = 1; attempt <= maxAttempts; attempt += 1) {
|
|
6
|
+
try {
|
|
7
|
+
const redis = new Redis({
|
|
8
|
+
host: '127.0.0.1',
|
|
9
|
+
port: 6379,
|
|
10
|
+
password: 'password',
|
|
11
|
+
connectTimeout: 1000,
|
|
12
|
+
retryStrategy: () => null,
|
|
13
|
+
});
|
|
14
|
+
redis.on('error', () => { });
|
|
15
|
+
try {
|
|
16
|
+
await redis.ping();
|
|
17
|
+
await redis.quit();
|
|
18
|
+
}
|
|
19
|
+
catch (err) {
|
|
20
|
+
await redis.quit().catch(() => undefined);
|
|
21
|
+
throw err;
|
|
22
|
+
}
|
|
23
|
+
return;
|
|
24
|
+
}
|
|
25
|
+
catch (err) {
|
|
26
|
+
console.log(err);
|
|
27
|
+
if (attempt === maxAttempts) {
|
|
28
|
+
throw new Error('Test Redis is not reachable. Start it before running tests.');
|
|
29
|
+
}
|
|
30
|
+
await sleep(500);
|
|
31
|
+
}
|
|
32
|
+
}
|
|
33
|
+
};
|
|
34
|
+
//# sourceMappingURL=global-setup.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"global-setup.js","sourceRoot":"","sources":["../../src/test/global-setup.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,KAAK,EAAE,MAAM,SAAS,CAAC;AAEhC,MAAM,KAAK,GAAG,CAAC,EAAU,EAAE,EAAE,CAAC,IAAI,OAAO,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC,UAAU,CAAC,OAAO,EAAE,EAAE,CAAC,CAAC,CAAC;AAEhF,eAAe,KAAK,IAAI,EAAE;IACxB,MAAM,WAAW,GAAG,EAAE,CAAC;IAEvB,KAAK,IAAI,OAAO,GAAG,CAAC,EAAE,OAAO,IAAI,WAAW,EAAE,OAAO,IAAI,CAAC,EAAE,CAAC;QAC3D,IAAI,CAAC;YAEH,MAAM,KAAK,GAAG,IAAI,KAAK,CAAC;gBACtB,IAAI,EAAE,WAAW;gBACjB,IAAI,EAAE,IAAI;gBACV,QAAQ,EAAE,UAAU;gBACpB,cAAc,EAAE,IAAI;gBACpB,aAAa,EAAE,GAAG,EAAE,CAAC,IAAI;aAC1B,CAAC,CAAC;YACH,KAAK,CAAC,EAAE,CAAC,OAAO,EAAE,GAAG,EAAE,GAAE,CAAC,CAAC,CAAC;YAE5B,IAAI,CAAC;gBACH,MAAM,KAAK,CAAC,IAAI,EAAE,CAAC;gBACnB,MAAM,KAAK,CAAC,IAAI,EAAE,CAAC;YACrB,CAAC;YAAC,OAAO,GAAG,EAAE,CAAC;gBACb,MAAM,KAAK,CAAC,IAAI,EAAE,CAAC,KAAK,CAAC,GAAG,EAAE,CAAC,SAAS,CAAC,CAAC;gBAC1C,MAAM,GAAG,CAAC;YACZ,CAAC;YAED,OAAO;QACT,CAAC;QAAC,OAAO,GAAY,EAAE,CAAC;YACtB,OAAO,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;YACjB,IAAI,OAAO,KAAK,WAAW,EAAE,CAAC;gBAC5B,MAAM,IAAI,KAAK,CAAC,6DAA6D,CAAC,CAAC;YACjF,CAAC;YACD,MAAM,KAAK,CAAC,GAAG,CAAC,CAAC;QACnB,CAAC;IACH,CAAC;AACH,CAAC,CAAC"}
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"redis-config.d.ts","sourceRoot":"","sources":["../../src/test/redis-config.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,SAAS,CAAC;AAE5C,eAAO,MAAM,gBAAgB,EAAE,YAM9B,CAAC"}
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"redis-config.js","sourceRoot":"","sources":["../../src/test/redis-config.ts"],"names":[],"mappings":"AAEA,MAAM,CAAC,MAAM,gBAAgB,GAAiB;IAC5C,IAAI,EAAE,WAAW;IACjB,IAAI,EAAE,IAAI;IACV,EAAE,EAAE,CAAC;IACL,QAAQ,EAAE,UAAU;IACpB,aAAa,EAAE,GAAG,EAAE,CAAC,IAAI;CAC1B,CAAC"}
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"setup.d.ts","sourceRoot":"","sources":["../../src/test/setup.ts"],"names":[],"mappings":"AAAA,OAAO,kBAAkB,CAAC"}
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"setup.js","sourceRoot":"","sources":["../../src/test/setup.ts"],"names":[],"mappings":"AAAA,OAAO,kBAAkB,CAAC"}
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"batch-post-processor.int.spec.d.ts","sourceRoot":"","sources":["../../../../src/test/specs/batch-post-processor/batch-post-processor.int.spec.ts"],"names":[],"mappings":""}
|
|
@@ -0,0 +1,87 @@
|
|
|
1
|
+
import { describe, it, expect, beforeEach, afterEach, beforeAll, afterAll } from '@jest/globals';
|
|
2
|
+
import { Redis } from 'ioredis';
|
|
3
|
+
import { testRedisOptions } from '../../redis-config.js';
|
|
4
|
+
import { TestBatchPostProcessor, makeTestEvent, pushTestEventToStream, TestMessagingStream, TestMessagingGroup, TestMessagingConsumer, } from '../../utils/index.js';
|
|
5
|
+
describe('BatchPostProcessor — Integration', () => {
|
|
6
|
+
let redis;
|
|
7
|
+
let processor;
|
|
8
|
+
beforeAll(() => {
|
|
9
|
+
redis = new Redis(testRedisOptions);
|
|
10
|
+
});
|
|
11
|
+
afterAll(async () => {
|
|
12
|
+
await redis.quit();
|
|
13
|
+
});
|
|
14
|
+
beforeEach(async () => {
|
|
15
|
+
await redis.flushdb();
|
|
16
|
+
processor = new TestBatchPostProcessor(redis, {
|
|
17
|
+
streamName: TestMessagingStream.TEST_STREAM,
|
|
18
|
+
groupName: TestMessagingGroup.TEST_GROUP,
|
|
19
|
+
consumerName: TestMessagingConsumer.TEST_CONSUMER,
|
|
20
|
+
claimIntervalMs: 500,
|
|
21
|
+
claimMinIdleTimeMs: 100,
|
|
22
|
+
blockMs: 50,
|
|
23
|
+
batchSize: 10,
|
|
24
|
+
});
|
|
25
|
+
});
|
|
26
|
+
afterEach(async () => {
|
|
27
|
+
await processor.stop();
|
|
28
|
+
});
|
|
29
|
+
it('should successfully read and process multiple events as a single batch from a real Redis Stream', async () => {
|
|
30
|
+
const eventPayload1 = makeTestEvent('evt-batch-1');
|
|
31
|
+
const eventPayload2 = makeTestEvent('evt-batch-2');
|
|
32
|
+
await pushTestEventToStream(redis, TestMessagingStream.TEST_STREAM, 'evt-batch-1', eventPayload1);
|
|
33
|
+
await pushTestEventToStream(redis, TestMessagingStream.TEST_STREAM, 'evt-batch-2', eventPayload2);
|
|
34
|
+
await processor.start();
|
|
35
|
+
await new Promise((resolve) => setTimeout(resolve, 300));
|
|
36
|
+
expect(processor.processedBatches).toHaveLength(1);
|
|
37
|
+
expect(processor.processedBatches[0]).toHaveLength(2);
|
|
38
|
+
expect(processor.processedBatches[0]?.[0]?.event.id).toBe('evt-batch-1');
|
|
39
|
+
expect(processor.processedBatches[0]?.[1]?.event.id).toBe('evt-batch-2');
|
|
40
|
+
const pendingInfo = (await redis.call('XPENDING', TestMessagingStream.TEST_STREAM, TestMessagingGroup.TEST_GROUP));
|
|
41
|
+
const pendingCount = Number(pendingInfo[0]);
|
|
42
|
+
expect(pendingCount).toBe(0);
|
|
43
|
+
});
|
|
44
|
+
it('should not acknowledge batch if processing throws an error', async () => {
|
|
45
|
+
const eventPayload1 = makeTestEvent('evt-batch-fail1');
|
|
46
|
+
const eventPayload2 = makeTestEvent('evt-batch-fail2');
|
|
47
|
+
await pushTestEventToStream(redis, TestMessagingStream.TEST_STREAM, 'evt-batch-fail1', eventPayload1);
|
|
48
|
+
await pushTestEventToStream(redis, TestMessagingStream.TEST_STREAM, 'evt-batch-fail2', eventPayload2);
|
|
49
|
+
processor.processError = new Error('Integration batch processing error');
|
|
50
|
+
await processor.start();
|
|
51
|
+
await new Promise((resolve) => setTimeout(resolve, 300));
|
|
52
|
+
expect(processor.processedBatches).toHaveLength(0);
|
|
53
|
+
const pendingInfo = (await redis.call('XPENDING', TestMessagingStream.TEST_STREAM, TestMessagingGroup.TEST_GROUP));
|
|
54
|
+
const pendingCount = Number(pendingInfo[0]);
|
|
55
|
+
expect(pendingCount).toBe(2);
|
|
56
|
+
});
|
|
57
|
+
it('should skip already locked events in a batch and process only unlocked ones (idempotence)', async () => {
|
|
58
|
+
const eventPayload1 = makeTestEvent('evt-batch-idemp1');
|
|
59
|
+
const eventPayload2 = makeTestEvent('evt-batch-idemp2');
|
|
60
|
+
const msgId1 = await pushTestEventToStream(redis, TestMessagingStream.TEST_STREAM, 'evt-batch-idemp1', eventPayload1);
|
|
61
|
+
await pushTestEventToStream(redis, TestMessagingStream.TEST_STREAM, 'evt-batch-idemp2', eventPayload2);
|
|
62
|
+
const idempotencyKey1 = `idempotency:post-processor:${TestMessagingGroup.TEST_GROUP}:${msgId1}`;
|
|
63
|
+
await redis.set(idempotencyKey1, 'processing');
|
|
64
|
+
await processor.start();
|
|
65
|
+
await new Promise((resolve) => setTimeout(resolve, 300));
|
|
66
|
+
expect(processor.processedBatches).toHaveLength(1);
|
|
67
|
+
expect(processor.processedBatches[0]).toHaveLength(1);
|
|
68
|
+
expect(processor.processedBatches[0]?.[0]?.event.id).toBe('evt-batch-idemp2');
|
|
69
|
+
const pendingInfo = (await redis.call('XPENDING', TestMessagingStream.TEST_STREAM, TestMessagingGroup.TEST_GROUP));
|
|
70
|
+
expect(Number(pendingInfo[0])).toBe(0);
|
|
71
|
+
});
|
|
72
|
+
it('should release all acquired idempotency locks in a batch if processing fails', async () => {
|
|
73
|
+
const eventPayload1 = makeTestEvent('evt-batch-idemp-fail1');
|
|
74
|
+
const eventPayload2 = makeTestEvent('evt-batch-idemp-fail2');
|
|
75
|
+
const msgId1 = await pushTestEventToStream(redis, TestMessagingStream.TEST_STREAM, 'evt-batch-idemp-fail1', eventPayload1);
|
|
76
|
+
const msgId2 = await pushTestEventToStream(redis, TestMessagingStream.TEST_STREAM, 'evt-batch-idemp-fail2', eventPayload2);
|
|
77
|
+
processor.processError = new Error('Integration batch processing error');
|
|
78
|
+
await processor.start();
|
|
79
|
+
await new Promise((resolve) => setTimeout(resolve, 300));
|
|
80
|
+
expect(processor.processedBatches).toHaveLength(0);
|
|
81
|
+
const key1 = `idempotency:post-processor:${TestMessagingGroup.TEST_GROUP}:${msgId1}`;
|
|
82
|
+
const key2 = `idempotency:post-processor:${TestMessagingGroup.TEST_GROUP}:${msgId2}`;
|
|
83
|
+
expect(await redis.get(key1)).toBeNull();
|
|
84
|
+
expect(await redis.get(key2)).toBeNull();
|
|
85
|
+
});
|
|
86
|
+
});
|
|
87
|
+
//# sourceMappingURL=batch-post-processor.int.spec.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"batch-post-processor.int.spec.js","sourceRoot":"","sources":["../../../../src/test/specs/batch-post-processor/batch-post-processor.int.spec.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,QAAQ,EAAE,EAAE,EAAE,MAAM,EAAE,UAAU,EAAE,SAAS,EAAE,SAAS,EAAE,QAAQ,EAAE,MAAM,eAAe,CAAC;AACjG,OAAO,EAAE,KAAK,EAAE,MAAM,SAAS,CAAC;AAChC,OAAO,EAAE,gBAAgB,EAAE,MAAM,uBAAuB,CAAC;AACzD,OAAO,EACL,sBAAsB,EACtB,aAAa,EACb,qBAAqB,EACrB,mBAAmB,EACnB,kBAAkB,EAClB,qBAAqB,GAEtB,MAAM,sBAAsB,CAAC;AAE9B,QAAQ,CAAC,kCAAkC,EAAE,GAAG,EAAE;IAChD,IAAI,KAAY,CAAC;IACjB,IAAI,SAAiC,CAAC;IAEtC,SAAS,CAAC,GAAG,EAAE;QACb,KAAK,GAAG,IAAI,KAAK,CAAC,gBAAgB,CAAC,CAAC;IACtC,CAAC,CAAC,CAAC;IAEH,QAAQ,CAAC,KAAK,IAAI,EAAE;QAClB,MAAM,KAAK,CAAC,IAAI,EAAE,CAAC;IACrB,CAAC,CAAC,CAAC;IAEH,UAAU,CAAC,KAAK,IAAI,EAAE;QACpB,MAAM,KAAK,CAAC,OAAO,EAAE,CAAC;QAEtB,SAAS,GAAG,IAAI,sBAAsB,CAAC,KAAK,EAAE;YAC5C,UAAU,EAAE,mBAAmB,CAAC,WAAW;YAC3C,SAAS,EAAE,kBAAkB,CAAC,UAAU;YACxC,YAAY,EAAE,qBAAqB,CAAC,aAAa;YACjD,eAAe,EAAE,GAAG;YACpB,kBAAkB,EAAE,GAAG;YACvB,OAAO,EAAE,EAAE;YACX,SAAS,EAAE,EAAE;SACd,CAAC,CAAC;IACL,CAAC,CAAC,CAAC;IAEH,SAAS,CAAC,KAAK,IAAI,EAAE;QACnB,MAAM,SAAS,CAAC,IAAI,EAAE,CAAC;IACzB,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,iGAAiG,EAAE,KAAK,IAAI,EAAE;QAC/G,MAAM,aAAa,GAAG,aAAa,CAAC,aAAa,CAAC,CAAC;QACnD,MAAM,aAAa,GAAG,aAAa,CAAC,aAAa,CAAC,CAAC;QAEnD,MAAM,qBAAqB,CACzB,KAAK,EACL,mBAAmB,CAAC,WAAW,EAC/B,aAAa,EACb,aAAa,CACd,CAAC;QACF,MAAM,qBAAqB,CACzB,KAAK,EACL,mBAAmB,CAAC,WAAW,EAC/B,aAAa,EACb,aAAa,CACd,CAAC;QAEF,MAAM,SAAS,CAAC,KAAK,EAAE,CAAC;QAExB,MAAM,IAAI,OAAO,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC,UAAU,CAAC,OAAO,EAAE,GAAG,CAAC,CAAC,CAAC;QAEzD,MAAM,CAAC,SAAS,CAAC,gBAAgB,CAAC,CAAC,YAAY,CAAC,CAAC,CAAC,CAAC;QACnD,MAAM,CAAC,SAAS,CAAC,gBAAgB,CAAC,CAAC,CAAC,CAAC,CAAC,YAAY,CAAC,CAAC,CAAC,CAAC;QACtD,MAAM,CAAC,SAAS,CAAC,gBAAgB,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,KAAK,CAAC,EAAE,CAAC,CAAC,IAAI,CAAC,aAAa,CAAC,CAAC;QACzE,MAAM,CAAC,SAAS,CAAC,gBAAgB,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,KAAK,CAAC,EAAE,CAAC,CAAC,IAAI,CAAC,aAAa,CAAC,CAAC;QAEzE,MAAM,WAAW,GAAG,CAAC,MAAM,KAAK,CAAC,IAAI,CACnC,UAAU,EACV,mBAAmB,CAAC,WAAW,EAC/B,kBAAkB,CAAC,UAAU,CAC9B,CAAyB,CAAC;QAC3B,MAAM,YAAY,GAAG,MAAM,CAAC,WAAW,CAAC,CAAC,CAAC,CAAC,CAAC;QAC5C,MAAM,CAAC,YAAY,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;IAC/B,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,4DAA4D,EAAE,KAAK,IAAI,EAAE;QAC1E,MAAM,aAAa,GAAG,aAAa,CAAC,iBAAiB,CAAC,CAAC;QACvD,MAAM,aAAa,GAAG,aAAa,CAAC,iBAAiB,CAAC,CAAC;QAEvD,MAAM,qBAAqB,CACzB,KAAK,EACL,mBAAmB,CAAC,WAAW,EAC/B,iBAAiB,EACjB,aAAa,CACd,CAAC;QACF,MAAM,qBAAqB,CACzB,KAAK,EACL,mBAAmB,CAAC,WAAW,EAC/B,iBAAiB,EACjB,aAAa,CACd,CAAC;QAEF,SAAS,CAAC,YAAY,GAAG,IAAI,KAAK,CAAC,oCAAoC,CAAC,CAAC;QAEzE,MAAM,SAAS,CAAC,KAAK,EAAE,CAAC;QACxB,MAAM,IAAI,OAAO,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC,UAAU,CAAC,OAAO,EAAE,GAAG,CAAC,CAAC,CAAC;QAEzD,MAAM,CAAC,SAAS,CAAC,gBAAgB,CAAC,CAAC,YAAY,CAAC,CAAC,CAAC,CAAC;QAEnD,MAAM,WAAW,GAAG,CAAC,MAAM,KAAK,CAAC,IAAI,CACnC,UAAU,EACV,mBAAmB,CAAC,WAAW,EAC/B,kBAAkB,CAAC,UAAU,CAC9B,CAAyB,CAAC;QAC3B,MAAM,YAAY,GAAG,MAAM,CAAC,WAAW,CAAC,CAAC,CAAC,CAAC,CAAC;QAC5C,MAAM,CAAC,YAAY,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;IAC/B,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,2FAA2F,EAAE,KAAK,IAAI,EAAE;QACzG,MAAM,aAAa,GAAG,aAAa,CAAC,kBAAkB,CAAC,CAAC;QACxD,MAAM,aAAa,GAAG,aAAa,CAAC,kBAAkB,CAAC,CAAC;QAExD,MAAM,MAAM,GAAG,MAAM,qBAAqB,CACxC,KAAK,EACL,mBAAmB,CAAC,WAAW,EAC/B,kBAAkB,EAClB,aAAa,CACd,CAAC;QACF,MAAM,qBAAqB,CACzB,KAAK,EACL,mBAAmB,CAAC,WAAW,EAC/B,kBAAkB,EAClB,aAAa,CACd,CAAC;QAEF,MAAM,eAAe,GAAG,8BAA8B,kBAAkB,CAAC,UAAU,IAAI,MAAM,EAAE,CAAC;QAChG,MAAM,KAAK,CAAC,GAAG,CAAC,eAAe,EAAE,YAAY,CAAC,CAAC;QAE/C,MAAM,SAAS,CAAC,KAAK,EAAE,CAAC;QACxB,MAAM,IAAI,OAAO,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC,UAAU,CAAC,OAAO,EAAE,GAAG,CAAC,CAAC,CAAC;QAEzD,MAAM,CAAC,SAAS,CAAC,gBAAgB,CAAC,CAAC,YAAY,CAAC,CAAC,CAAC,CAAC;QACnD,MAAM,CAAC,SAAS,CAAC,gBAAgB,CAAC,CAAC,CAAC,CAAC,CAAC,YAAY,CAAC,CAAC,CAAC,CAAC;QACtD,MAAM,CAAC,SAAS,CAAC,gBAAgB,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,KAAK,CAAC,EAAE,CAAC,CAAC,IAAI,CAAC,kBAAkB,CAAC,CAAC;QAE9E,MAAM,WAAW,GAAG,CAAC,MAAM,KAAK,CAAC,IAAI,CACnC,UAAU,EACV,mBAAmB,CAAC,WAAW,EAC/B,kBAAkB,CAAC,UAAU,CAC9B,CAAyB,CAAC;QAC3B,MAAM,CAAC,MAAM,CAAC,WAAW,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;IACzC,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,8EAA8E,EAAE,KAAK,IAAI,EAAE;QAC5F,MAAM,aAAa,GAAG,aAAa,CAAC,uBAAuB,CAAC,CAAC;QAC7D,MAAM,aAAa,GAAG,aAAa,CAAC,uBAAuB,CAAC,CAAC;QAE7D,MAAM,MAAM,GAAG,MAAM,qBAAqB,CACxC,KAAK,EACL,mBAAmB,CAAC,WAAW,EAC/B,uBAAuB,EACvB,aAAa,CACd,CAAC;QACF,MAAM,MAAM,GAAG,MAAM,qBAAqB,CACxC,KAAK,EACL,mBAAmB,CAAC,WAAW,EAC/B,uBAAuB,EACvB,aAAa,CACd,CAAC;QAEF,SAAS,CAAC,YAAY,GAAG,IAAI,KAAK,CAAC,oCAAoC,CAAC,CAAC;QAEzE,MAAM,SAAS,CAAC,KAAK,EAAE,CAAC;QACxB,MAAM,IAAI,OAAO,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC,UAAU,CAAC,OAAO,EAAE,GAAG,CAAC,CAAC,CAAC;QAEzD,MAAM,CAAC,SAAS,CAAC,gBAAgB,CAAC,CAAC,YAAY,CAAC,CAAC,CAAC,CAAC;QAEnD,MAAM,IAAI,GAAG,8BAA8B,kBAAkB,CAAC,UAAU,IAAI,MAAM,EAAE,CAAC;QACrF,MAAM,IAAI,GAAG,8BAA8B,kBAAkB,CAAC,UAAU,IAAI,MAAM,EAAE,CAAC;QAErF,MAAM,CAAC,MAAM,KAAK,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC,CAAC,QAAQ,EAAE,CAAC;QACzC,MAAM,CAAC,MAAM,KAAK,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC,CAAC,QAAQ,EAAE,CAAC;IAC3C,CAAC,CAAC,CAAC;AACL,CAAC,CAAC,CAAC"}
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"batch-post-processor.unit.spec.d.ts","sourceRoot":"","sources":["../../../../src/test/specs/batch-post-processor/batch-post-processor.unit.spec.ts"],"names":[],"mappings":""}
|
|
@@ -0,0 +1,62 @@
|
|
|
1
|
+
import { describe, it, expect, beforeEach, afterEach, jest } from '@jest/globals';
|
|
2
|
+
import { createMock } from '@volontariapp/testing';
|
|
3
|
+
import { TestBatchPostProcessor, makeTestEvent, TestMessagingStream, TestMessagingGroup, TestMessagingConsumer, mockRedisCall, mockRedisXreadgroup, } from '../../utils/index.js';
|
|
4
|
+
describe('BatchPostProcessor', () => {
|
|
5
|
+
let redisMock;
|
|
6
|
+
let processor;
|
|
7
|
+
let callMock;
|
|
8
|
+
beforeEach(() => {
|
|
9
|
+
jest.clearAllMocks();
|
|
10
|
+
redisMock = createMock();
|
|
11
|
+
callMock = mockRedisCall(redisMock);
|
|
12
|
+
processor = new TestBatchPostProcessor(redisMock, {
|
|
13
|
+
streamName: TestMessagingStream.TEST_STREAM,
|
|
14
|
+
groupName: TestMessagingGroup.TEST_GROUP,
|
|
15
|
+
consumerName: TestMessagingConsumer.TEST_CONSUMER,
|
|
16
|
+
claimIntervalMs: 50,
|
|
17
|
+
claimMinIdleTimeMs: 100,
|
|
18
|
+
});
|
|
19
|
+
});
|
|
20
|
+
afterEach(async () => {
|
|
21
|
+
await processor.stop();
|
|
22
|
+
});
|
|
23
|
+
describe('runLoop in batch', () => {
|
|
24
|
+
it('should read multiple pending messages and process them as a single batch', async () => {
|
|
25
|
+
const mockEvent1 = makeTestEvent('evt-1');
|
|
26
|
+
const mockEvent2 = makeTestEvent('evt-2');
|
|
27
|
+
mockRedisXreadgroup(callMock, TestMessagingStream.TEST_STREAM, [
|
|
28
|
+
{ messageId: '1-0', event: mockEvent1 },
|
|
29
|
+
{ messageId: '2-0', event: mockEvent2 },
|
|
30
|
+
]);
|
|
31
|
+
const processSpy = jest.spyOn(processor, 'processEvents');
|
|
32
|
+
await processor.start();
|
|
33
|
+
await new Promise((resolve) => setTimeout(resolve, 50));
|
|
34
|
+
expect(processSpy).toHaveBeenCalledTimes(1);
|
|
35
|
+
expect(processSpy).toHaveBeenCalledWith([
|
|
36
|
+
expect.objectContaining({
|
|
37
|
+
event: expect.objectContaining({ id: 'evt-1' }),
|
|
38
|
+
messageId: '1-0',
|
|
39
|
+
}),
|
|
40
|
+
expect.objectContaining({
|
|
41
|
+
event: expect.objectContaining({ id: 'evt-2' }),
|
|
42
|
+
messageId: '2-0',
|
|
43
|
+
}),
|
|
44
|
+
]);
|
|
45
|
+
expect(callMock).toHaveBeenCalledWith('XACK', TestMessagingStream.TEST_STREAM, TestMessagingGroup.TEST_GROUP, '1-0');
|
|
46
|
+
expect(callMock).toHaveBeenCalledWith('XACK', TestMessagingStream.TEST_STREAM, TestMessagingGroup.TEST_GROUP, '2-0');
|
|
47
|
+
});
|
|
48
|
+
it('should not acknowledge batch if processEvents throws an error', async () => {
|
|
49
|
+
const mockEvent = makeTestEvent('evt-1');
|
|
50
|
+
mockRedisXreadgroup(callMock, TestMessagingStream.TEST_STREAM, [
|
|
51
|
+
{ messageId: '1-0', event: mockEvent },
|
|
52
|
+
]);
|
|
53
|
+
const processSpy = jest.spyOn(processor, 'processEvents');
|
|
54
|
+
processor.processError = new Error('Batch processing failed');
|
|
55
|
+
await processor.start();
|
|
56
|
+
await new Promise((resolve) => setTimeout(resolve, 50));
|
|
57
|
+
expect(processSpy).toHaveBeenCalledTimes(1);
|
|
58
|
+
expect(callMock).not.toHaveBeenCalledWith('XACK', expect.any(String), expect.any(String), expect.any(String));
|
|
59
|
+
});
|
|
60
|
+
});
|
|
61
|
+
});
|
|
62
|
+
//# sourceMappingURL=batch-post-processor.unit.spec.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"batch-post-processor.unit.spec.js","sourceRoot":"","sources":["../../../../src/test/specs/batch-post-processor/batch-post-processor.unit.spec.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,QAAQ,EAAE,EAAE,EAAE,MAAM,EAAE,UAAU,EAAE,SAAS,EAAE,IAAI,EAAE,MAAM,eAAe,CAAC;AAElF,OAAO,EAAE,UAAU,EAAE,MAAM,uBAAuB,CAAC;AACnD,OAAO,EACL,sBAAsB,EACtB,aAAa,EACb,mBAAmB,EACnB,kBAAkB,EAClB,qBAAqB,EACrB,aAAa,EACb,mBAAmB,GACpB,MAAM,sBAAsB,CAAC;AAE9B,QAAQ,CAAC,oBAAoB,EAAE,GAAG,EAAE;IAClC,IAAI,SAA6B,CAAC;IAClC,IAAI,SAAiC,CAAC;IACtC,IAAI,QAA0C,CAAC;IAE/C,UAAU,CAAC,GAAG,EAAE;QACd,IAAI,CAAC,aAAa,EAAE,CAAC;QACrB,SAAS,GAAG,UAAU,EAAS,CAAC;QAChC,QAAQ,GAAG,aAAa,CAAC,SAAS,CAAC,CAAC;QAEpC,SAAS,GAAG,IAAI,sBAAsB,CAAC,SAAS,EAAE;YAChD,UAAU,EAAE,mBAAmB,CAAC,WAAW;YAC3C,SAAS,EAAE,kBAAkB,CAAC,UAAU;YACxC,YAAY,EAAE,qBAAqB,CAAC,aAAa;YACjD,eAAe,EAAE,EAAE;YACnB,kBAAkB,EAAE,GAAG;SACxB,CAAC,CAAC;IACL,CAAC,CAAC,CAAC;IAEH,SAAS,CAAC,KAAK,IAAI,EAAE;QACnB,MAAM,SAAS,CAAC,IAAI,EAAE,CAAC;IACzB,CAAC,CAAC,CAAC;IAEH,QAAQ,CAAC,kBAAkB,EAAE,GAAG,EAAE;QAChC,EAAE,CAAC,0EAA0E,EAAE,KAAK,IAAI,EAAE;YACxF,MAAM,UAAU,GAAG,aAAa,CAAC,OAAO,CAAC,CAAC;YAC1C,MAAM,UAAU,GAAG,aAAa,CAAC,OAAO,CAAC,CAAC;YAE1C,mBAAmB,CAAC,QAAQ,EAAE,mBAAmB,CAAC,WAAW,EAAE;gBAC7D,EAAE,SAAS,EAAE,KAAK,EAAE,KAAK,EAAE,UAAU,EAAE;gBACvC,EAAE,SAAS,EAAE,KAAK,EAAE,KAAK,EAAE,UAAU,EAAE;aACxC,CAAC,CAAC;YAEH,MAAM,UAAU,GAAG,IAAI,CAAC,KAAK,CAAC,SAAS,EAAE,eAAe,CAAC,CAAC;YAE1D,MAAM,SAAS,CAAC,KAAK,EAAE,CAAC;YACxB,MAAM,IAAI,OAAO,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC,UAAU,CAAC,OAAO,EAAE,EAAE,CAAC,CAAC,CAAC;YAExD,MAAM,CAAC,UAAU,CAAC,CAAC,qBAAqB,CAAC,CAAC,CAAC,CAAC;YAC5C,MAAM,CAAC,UAAU,CAAC,CAAC,oBAAoB,CAAC;gBACtC,MAAM,CAAC,gBAAgB,CAAC;oBACtB,KAAK,EAAE,MAAM,CAAC,gBAAgB,CAAC,EAAE,EAAE,EAAE,OAAO,EAAE,CAAC;oBAC/C,SAAS,EAAE,KAAK;iBACjB,CAAC;gBACF,MAAM,CAAC,gBAAgB,CAAC;oBACtB,KAAK,EAAE,MAAM,CAAC,gBAAgB,CAAC,EAAE,EAAE,EAAE,OAAO,EAAE,CAAC;oBAC/C,SAAS,EAAE,KAAK;iBACjB,CAAC;aACH,CAAC,CAAC;YAEH,MAAM,CAAC,QAAQ,CAAC,CAAC,oBAAoB,CACnC,MAAM,EACN,mBAAmB,CAAC,WAAW,EAC/B,kBAAkB,CAAC,UAAU,EAC7B,KAAK,CACN,CAAC;YACF,MAAM,CAAC,QAAQ,CAAC,CAAC,oBAAoB,CACnC,MAAM,EACN,mBAAmB,CAAC,WAAW,EAC/B,kBAAkB,CAAC,UAAU,EAC7B,KAAK,CACN,CAAC;QACJ,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,+DAA+D,EAAE,KAAK,IAAI,EAAE;YAC7E,MAAM,SAAS,GAAG,aAAa,CAAC,OAAO,CAAC,CAAC;YAEzC,mBAAmB,CAAC,QAAQ,EAAE,mBAAmB,CAAC,WAAW,EAAE;gBAC7D,EAAE,SAAS,EAAE,KAAK,EAAE,KAAK,EAAE,SAAS,EAAE;aACvC,CAAC,CAAC;YAEH,MAAM,UAAU,GAAG,IAAI,CAAC,KAAK,CAAC,SAAS,EAAE,eAAe,CAAC,CAAC;YAC1D,SAAS,CAAC,YAAY,GAAG,IAAI,KAAK,CAAC,yBAAyB,CAAC,CAAC;YAE9D,MAAM,SAAS,CAAC,KAAK,EAAE,CAAC;YACxB,MAAM,IAAI,OAAO,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC,UAAU,CAAC,OAAO,EAAE,EAAE,CAAC,CAAC,CAAC;YAExD,MAAM,CAAC,UAAU,CAAC,CAAC,qBAAqB,CAAC,CAAC,CAAC,CAAC;YAC5C,MAAM,CAAC,QAAQ,CAAC,CAAC,GAAG,CAAC,oBAAoB,CACvC,MAAM,EACN,MAAM,CAAC,GAAG,CAAC,MAAM,CAAC,EAClB,MAAM,CAAC,GAAG,CAAC,MAAM,CAAC,EAClB,MAAM,CAAC,GAAG,CAAC,MAAM,CAAC,CACnB,CAAC;QACJ,CAAC,CAAC,CAAC;IACL,CAAC,CAAC,CAAC;AACL,CAAC,CAAC,CAAC"}
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"single-post-processor.int.spec.d.ts","sourceRoot":"","sources":["../../../../src/test/specs/single-post-processor/single-post-processor.int.spec.ts"],"names":[],"mappings":""}
|
|
@@ -0,0 +1,89 @@
|
|
|
1
|
+
import { describe, it, expect, beforeEach, afterEach, beforeAll, afterAll } from '@jest/globals';
|
|
2
|
+
import { Redis } from 'ioredis';
|
|
3
|
+
import { testRedisOptions } from '../../redis-config.js';
|
|
4
|
+
import { TestPostProcessor, makeTestEvent, pushTestEventToStream, TestMessagingStream, TestMessagingGroup, TestMessagingConsumer, } from '../../utils/index.js';
|
|
5
|
+
describe('SinglePostProcessor — Integration', () => {
|
|
6
|
+
let redis;
|
|
7
|
+
let processor;
|
|
8
|
+
beforeAll(() => {
|
|
9
|
+
redis = new Redis(testRedisOptions);
|
|
10
|
+
});
|
|
11
|
+
afterAll(async () => {
|
|
12
|
+
await redis.quit();
|
|
13
|
+
});
|
|
14
|
+
beforeEach(async () => {
|
|
15
|
+
await redis.flushdb();
|
|
16
|
+
processor = new TestPostProcessor(redis, {
|
|
17
|
+
streamName: TestMessagingStream.TEST_STREAM,
|
|
18
|
+
groupName: TestMessagingGroup.TEST_GROUP,
|
|
19
|
+
consumerName: TestMessagingConsumer.TEST_CONSUMER,
|
|
20
|
+
claimIntervalMs: 500,
|
|
21
|
+
claimMinIdleTimeMs: 100,
|
|
22
|
+
blockMs: 50,
|
|
23
|
+
});
|
|
24
|
+
});
|
|
25
|
+
afterEach(async () => {
|
|
26
|
+
await processor.stop();
|
|
27
|
+
});
|
|
28
|
+
it('should successfully read, process, and acknowledge events from a real Redis Stream', async () => {
|
|
29
|
+
const eventPayload = makeTestEvent('evt-int-1');
|
|
30
|
+
await pushTestEventToStream(redis, TestMessagingStream.TEST_STREAM, 'evt-int-1', eventPayload);
|
|
31
|
+
await processor.start();
|
|
32
|
+
await new Promise((resolve) => setTimeout(resolve, 200));
|
|
33
|
+
expect(processor.processedEvents).toHaveLength(1);
|
|
34
|
+
expect(processor.processedEvents[0]?.id).toBe('evt-int-1');
|
|
35
|
+
const pendingInfo = (await redis.call('XPENDING', TestMessagingStream.TEST_STREAM, TestMessagingGroup.TEST_GROUP));
|
|
36
|
+
const pendingCount = Number(pendingInfo[0]);
|
|
37
|
+
expect(pendingCount).toBe(0);
|
|
38
|
+
});
|
|
39
|
+
it('should not acknowledge message if processing throws an error', async () => {
|
|
40
|
+
const eventPayload = makeTestEvent('evt-int-fail');
|
|
41
|
+
await pushTestEventToStream(redis, TestMessagingStream.TEST_STREAM, 'evt-int-fail', eventPayload);
|
|
42
|
+
processor.processError = new Error('Integration processing error');
|
|
43
|
+
await processor.start();
|
|
44
|
+
await new Promise((resolve) => setTimeout(resolve, 200));
|
|
45
|
+
expect(processor.processedEvents).toHaveLength(0);
|
|
46
|
+
const pendingInfo = (await redis.call('XPENDING', TestMessagingStream.TEST_STREAM, TestMessagingGroup.TEST_GROUP));
|
|
47
|
+
const pendingCount = Number(pendingInfo[0]);
|
|
48
|
+
expect(pendingCount).toBe(1);
|
|
49
|
+
});
|
|
50
|
+
it('should claim and process idle pending messages from other consumers', async () => {
|
|
51
|
+
const eventPayload = makeTestEvent('evt-int-claim');
|
|
52
|
+
await redis.call('XGROUP', 'CREATE', TestMessagingStream.TEST_STREAM, TestMessagingGroup.TEST_GROUP, '0', 'MKSTREAM');
|
|
53
|
+
await pushTestEventToStream(redis, TestMessagingStream.TEST_STREAM, 'evt-int-claim', eventPayload);
|
|
54
|
+
await redis.call('XREADGROUP', 'GROUP', TestMessagingGroup.TEST_GROUP, 'other-consumer', 'COUNT', 1, 'STREAMS', TestMessagingStream.TEST_STREAM, '>');
|
|
55
|
+
const initialPending = (await redis.call('XPENDING', TestMessagingStream.TEST_STREAM, TestMessagingGroup.TEST_GROUP));
|
|
56
|
+
expect(Number(initialPending[0])).toBe(1);
|
|
57
|
+
await new Promise((resolve) => setTimeout(resolve, 200));
|
|
58
|
+
await processor.start();
|
|
59
|
+
await new Promise((resolve) => setTimeout(resolve, 800));
|
|
60
|
+
expect(processor.processedEvents).toHaveLength(1);
|
|
61
|
+
expect(processor.processedEvents[0]?.id).toBe('evt-int-claim');
|
|
62
|
+
const finalPending = (await redis.call('XPENDING', TestMessagingStream.TEST_STREAM, TestMessagingGroup.TEST_GROUP));
|
|
63
|
+
expect(Number(finalPending[0])).toBe(0);
|
|
64
|
+
});
|
|
65
|
+
it('should skip processing and acknowledge if message is already locked (idempotence)', async () => {
|
|
66
|
+
const eventPayload = makeTestEvent('evt-int-idempotence');
|
|
67
|
+
const messageId = await pushTestEventToStream(redis, TestMessagingStream.TEST_STREAM, 'evt-int-idempotence', eventPayload);
|
|
68
|
+
const idempotencyKey = `idempotency:post-processor:${TestMessagingGroup.TEST_GROUP}:${messageId}`;
|
|
69
|
+
await redis.set(idempotencyKey, 'processing');
|
|
70
|
+
await processor.start();
|
|
71
|
+
await new Promise((resolve) => setTimeout(resolve, 200));
|
|
72
|
+
expect(processor.processedEvents).toHaveLength(0);
|
|
73
|
+
const pendingInfo = (await redis.call('XPENDING', TestMessagingStream.TEST_STREAM, TestMessagingGroup.TEST_GROUP));
|
|
74
|
+
const pendingCount = Number(pendingInfo[0]);
|
|
75
|
+
expect(pendingCount).toBe(0);
|
|
76
|
+
});
|
|
77
|
+
it('should release idempotency lock if processing throws an error', async () => {
|
|
78
|
+
const eventPayload = makeTestEvent('evt-int-idempotence-fail');
|
|
79
|
+
const messageId = await pushTestEventToStream(redis, TestMessagingStream.TEST_STREAM, 'evt-int-idempotence-fail', eventPayload);
|
|
80
|
+
processor.processError = new Error('Integration processing error');
|
|
81
|
+
await processor.start();
|
|
82
|
+
await new Promise((resolve) => setTimeout(resolve, 200));
|
|
83
|
+
expect(processor.processedEvents).toHaveLength(0);
|
|
84
|
+
const idempotencyKey = `idempotency:post-processor:${TestMessagingGroup.TEST_GROUP}:${messageId}`;
|
|
85
|
+
const lockVal = await redis.get(idempotencyKey);
|
|
86
|
+
expect(lockVal).toBeNull();
|
|
87
|
+
});
|
|
88
|
+
});
|
|
89
|
+
//# sourceMappingURL=single-post-processor.int.spec.js.map
|