@flowcore/data-pump 0.1.5 → 0.1.7

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (15) hide show
  1. package/CHANGELOG.md +16 -0
  2. package/README.md +34 -18
  3. package/esm/lib/data-pump-create.d.ts +3 -2
  4. package/esm/lib/data-pump-create.d.ts.map +1 -1
  5. package/esm/lib/data-pump-create.js +2 -1
  6. package/esm/lib/data-pump.d.ts +3 -2
  7. package/esm/lib/data-pump.d.ts.map +1 -1
  8. package/esm/lib/data-pump.js +25 -5
  9. package/package.json +1 -1
  10. package/script/lib/data-pump-create.d.ts +3 -2
  11. package/script/lib/data-pump-create.d.ts.map +1 -1
  12. package/script/lib/data-pump-create.js +2 -1
  13. package/script/lib/data-pump.d.ts +3 -2
  14. package/script/lib/data-pump.d.ts.map +1 -1
  15. package/script/lib/data-pump.js +25 -5
package/CHANGELOG.md CHANGED
@@ -1,5 +1,21 @@
1
1
  # Changelog
2
2
 
3
+ ## [0.1.7](https://github.com/flowcore-io/data-pump/compare/v0.1.6...v0.1.7) (2025-02-17)
4
+
5
+
6
+ ### Bug Fixes
7
+
8
+ * rename to maxRedeliveryCount ([5ad7484](https://github.com/flowcore-io/data-pump/commit/5ad74848f55e60f37a4a29b40641d72ae9d8596c))
9
+
10
+ ## [0.1.6](https://github.com/flowcore-io/data-pump/compare/v0.1.5...v0.1.6) (2025-02-17)
11
+
12
+
13
+ ### Bug Fixes
14
+
15
+ * add onEventsFail ([00c59ff](https://github.com/flowcore-io/data-pump/commit/00c59ffae41dd64260e99bb439da553865bc5c5b))
16
+ * set onFailedEvents ([09f8015](https://github.com/flowcore-io/data-pump/commit/09f8015e2171c587f3e23edd44313cdebe1fcc4f))
17
+ * update readme ([b078763](https://github.com/flowcore-io/data-pump/commit/b078763a77994a701929500ac8c09fff1629b941))
18
+
3
19
  ## [0.1.5](https://github.com/flowcore-io/data-pump/compare/v0.1.4...v0.1.5) (2025-02-14)
4
20
 
5
21
 
package/README.md CHANGED
@@ -3,25 +3,41 @@
3
3
  ## Usage example
4
4
 
5
5
  ```ts
6
- import { createDataPump } from "@flowcore/data-pump"
6
+ import { createDataPump } from "@flowcore/data-pump"
7
+ import { oidcClient } from "@flowcore/oidc-client"
7
8
 
8
- const dataPump = createDataPump({
9
- auth: {
10
- clientId: "",
11
- clientSecret: "",
12
- },
13
- dataSource: {
14
- tenant: "tenant",
15
- dataCore: "data-core",
16
- flowType: "data.0",
17
- eventTypes: ["data.created.0", "data.updated.0", "data.deleted.0"],
18
- },
19
- processor: {
20
- onEvents: async (events) => {
21
- await new Promise((resolve) => setTimeout(resolve, 100))
22
- console.log(`Got ${events.length} events`)
23
- },
24
- logger: console,
9
+ const oidcClient = oidcClient({
10
+ clientId: "",
11
+ clientSecret: "",
12
+ })
13
+
14
+ const dataPump = createDataPump({
15
+ auth: {
16
+ oidcClient,
17
+ },
18
+ dataSource: {
19
+ tenant: "tenant",
20
+ dataCore: "data-core",
21
+ flowType: "data.0",
22
+ eventTypes: ["data.created.0", "data.updated.0", "data.deleted.0"],
23
+ },
24
+ processor: {
25
+ onEvents: async (events) => {
26
+ await new Promise((resolve) => setTimeout(resolve, 100))
27
+ console.log(`Got ${events.length} events`)
28
+ return true
29
+ },
30
+ onFailedEvents: async (events) => {
31
+ console.error(`Failed ${events.length} events`)
32
+ },
33
+ },
34
+ buffer: {
35
+ size: 10_000,
36
+ threshold: 1_000,
37
+ maxRedeliveryCount: 1,
38
+ achknowledgeTimeoutMs: 1_000,
39
+ },
40
+ logger: console,
25
41
  })
26
42
 
27
43
  await dataPump.start()
package/esm/lib/data-pump-create.d.ts CHANGED
@@ -25,13 +25,14 @@ export interface DataPumpClientOptions {
25
25
  buffer?: {
26
26
  size?: number;
27
27
  threshold?: number;
28
- maxDeliveryCount?: number;
28
+ maxRedeliveryCount?: number;
29
29
  achknowledgeTimeoutMs?: number;
30
30
  };
31
31
  stateManager?: DataPumpStateManager;
32
32
  processor?: {
33
33
  concurrency?: number;
34
- onEvents: (events: FlowcoreEvent[]) => Promise<void>;
34
+ onEvents: (events: FlowcoreEvent[]) => Promise<boolean> | boolean;
35
+ onFailedEvents?: (events: FlowcoreEvent[]) => Promise<void> | void;
35
36
  };
36
37
  natsServers?: string[];
37
38
  }
package/esm/lib/data-pump-create.d.ts.map CHANGED
@@ -1 +1 @@
1
- {"version":3,"file":"data-pump-create.d.ts","sourceRoot":"","sources":["../../src/lib/data-pump-create.ts"],"names":[],"mappings":"AAAA,OAAO,EAAkB,KAAK,aAAa,EAAE,MAAM,iDAAiD,CAAA;AACpG,OAAO,EAAE,QAAQ,EAAE,KAAK,oBAAoB,EAAE,KAAK,MAAM,EAAE,MAAM,gBAAgB,CAAA;AAIjF,eAAO,MAAM,UAAU,EAAE,MAKxB,CAAA;AAED,UAAU,mCAAmC;IAC3C,UAAU,EAAE;QACV,QAAQ,EAAE,MAAM,OAAO,CAAC;YAAE,WAAW,EAAE,MAAM,CAAA;SAAE,CAAC,CAAA;KACjD,CAAA;CACF;AAED,UAAU,+BAA+B;IACvC,MAAM,EAAE,MAAM,CAAA;IACd,QAAQ,EAAE,MAAM,CAAA;CACjB;AAED,KAAK,yBAAyB,GAAG,mCAAmC,GAAG,+BAA+B,CAAA;AAEtG,MAAM,WAAW,qBAAqB;IACpC,MAAM,CAAC,EAAE,MAAM,CAAA;IACf,IAAI,EAAE,yBAAyB,CAAA;IAC/B,UAAU,EAAE;QACV,MAAM,EAAE,MAAM,CAAA;QACd,QAAQ,EAAE,MAAM,CAAA;QAChB,QAAQ,EAAE,MAAM,CAAA;QAChB,UAAU,EAAE,MAAM,EAAE,CAAA;KACrB,CAAA;IACD,MAAM,CAAC,EAAE;QACP,IAAI,CAAC,EAAE,MAAM,CAAA;QACb,SAAS,CAAC,EAAE,MAAM,CAAA;QAClB,gBAAgB,CAAC,EAAE,MAAM,CAAA;QACzB,qBAAqB,CAAC,EAAE,MAAM,CAAA;KAC/B,CAAA;IACD,YAAY,CAAC,EAAE,oBAAoB,CAAA;IACnC,SAAS,CAAC,EAAE;QACV,WAAW,CAAC,EAAE,MAAM,CAAA;QACpB,QAAQ,EAAE,CAAC,MAAM,EAAE,aAAa,EAAE,KAAK,OAAO,CAAC,IAAI,CAAC,CAAA;KACrD,CAAA;IACD,WAAW,CAAC,EAAE,MAAM,EAAE,CAAA;CACvB;AAED,wBAAgB,cAAc,CAAC,OAAO,EAAE,qBAAqB,GAAG,QAAQ,CAuDvE"}
1
+ {"version":3,"file":"data-pump-create.d.ts","sourceRoot":"","sources":["../../src/lib/data-pump-create.ts"],"names":[],"mappings":"AAAA,OAAO,EAAkB,KAAK,aAAa,EAAE,MAAM,iDAAiD,CAAA;AACpG,OAAO,EAAE,QAAQ,EAAE,KAAK,oBAAoB,EAAE,KAAK,MAAM,EAAE,MAAM,gBAAgB,CAAA;AAIjF,eAAO,MAAM,UAAU,EAAE,MAKxB,CAAA;AAED,UAAU,mCAAmC;IAC3C,UAAU,EAAE;QACV,QAAQ,EAAE,MAAM,OAAO,CAAC;YAAE,WAAW,EAAE,MAAM,CAAA;SAAE,CAAC,CAAA;KACjD,CAAA;CACF;AAED,UAAU,+BAA+B;IACvC,MAAM,EAAE,MAAM,CAAA;IACd,QAAQ,EAAE,MAAM,CAAA;CACjB;AAED,KAAK,yBAAyB,GAAG,mCAAmC,GAAG,+BAA+B,CAAA;AAEtG,MAAM,WAAW,qBAAqB;IACpC,MAAM,CAAC,EAAE,MAAM,CAAA;IACf,IAAI,EAAE,yBAAyB,CAAA;IAC/B,UAAU,EAAE;QACV,MAAM,EAAE,MAAM,CAAA;QACd,QAAQ,EAAE,MAAM,CAAA;QAChB,QAAQ,EAAE,MAAM,CAAA;QAChB,UAAU,EAAE,MAAM,EAAE,CAAA;KACrB,CAAA;IACD,MAAM,CAAC,EAAE;QACP,IAAI,CAAC,EAAE,MAAM,CAAA;QACb,SAAS,CAAC,EAAE,MAAM,CAAA;QAClB,kBAAkB,CAAC,EAAE,MAAM,CAAA;QAC3B,qBAAqB,CAAC,EAAE,MAAM,CAAA;KAC/B,CAAA;IACD,YAAY,CAAC,EAAE,oBAAoB,CAAA;IACnC,SAAS,CAAC,EAAE;QACV,WAAW,CAAC,EAAE,MAAM,CAAA;QACpB,QAAQ,EAAE,CAAC,MAAM,EAAE,aAAa,EAAE,KAAK,OAAO,CAAC,OAAO,CAAC,GAAG,OAAO,CAAA;QACjE,cAAc,CAAC,EAAE,CAAC,MAAM,EAAE,aAAa,EAAE,KAAK,OAAO,CAAC,IAAI,CAAC,GAAG,IAAI,CAAA;KACnE,CAAA;IACD,WAAW,CAAC,EAAE,MAAM,EAAE,CAAA;CACvB;AAED,wBAAgB,cAAc,CAAC,OAAO,EAAE,qBAAqB,GAAG,QAAQ,CAwDvE"}
package/esm/lib/data-pump-create.js CHANGED
@@ -39,13 +39,14 @@ export function createDataPump(options) {
39
39
  const buffer = {
40
40
  size: bufferSize,
41
41
  threshold: options.buffer?.threshold ?? Math.round(bufferSize * 0.1),
42
- maxDeliveryCount: options.buffer?.maxDeliveryCount ?? 3,
42
+ maxRedeliveryCount: options.buffer?.maxRedeliveryCount ?? 3,
43
43
  achknowledgeTimeoutMs: options.buffer?.achknowledgeTimeoutMs ?? 10_000,
44
44
  };
45
45
  const processor = options.processor
46
46
  ? {
47
47
  concurrency: options.processor.concurrency ?? 1,
48
48
  onEvents: options.processor.onEvents,
49
+ onFailedEvents: options.processor.onFailedEvents,
49
50
  }
50
51
  : undefined;
51
52
  const dataPump = new DataPump({
package/esm/lib/data-pump.d.ts CHANGED
@@ -19,12 +19,13 @@ interface DataPumpOptions {
19
19
  buffer: {
20
20
  size: number;
21
21
  threshold: number;
22
- maxDeliveryCount: number;
22
+ maxRedeliveryCount: number;
23
23
  achknowledgeTimeoutMs: number;
24
24
  };
25
25
  processor?: {
26
26
  concurrency: number;
27
- onEvents: (events: FlowcoreEvent[]) => Promise<void>;
27
+ onEvents: (events: FlowcoreEvent[]) => Promise<boolean> | boolean;
28
+ onFailedEvents?: (events: FlowcoreEvent[]) => Promise<void> | void;
28
29
  };
29
30
  }
30
31
  export interface DataPumpState {
package/esm/lib/data-pump.d.ts.map CHANGED
@@ -1 +1 @@
1
- {"version":3,"file":"data-pump.d.ts","sourceRoot":"","sources":["../../src/lib/data-pump.ts"],"names":[],"mappings":"AAAA,OAAO,EAIL,KAAK,cAAc,EACnB,KAAK,aAAa,EAEnB,MAAM,iDAAiD,CAAA;AAiBxD,MAAM,WAAW,MAAM;IACrB,KAAK,CAAC,OAAO,EAAE,MAAM,EAAE,GAAG,IAAI,EAAE,OAAO,EAAE,GAAG,IAAI,CAAA;IAChD,IAAI,CAAC,OAAO,EAAE,MAAM,EAAE,GAAG,IAAI,EAAE,OAAO,EAAE,GAAG,IAAI,CAAA;IAC/C,IAAI,CAAC,OAAO,EAAE,MAAM,EAAE,GAAG,IAAI,EAAE,OAAO,EAAE,GAAG,IAAI,CAAA;IAC/C,KAAK,CAAC,OAAO,EAAE,MAAM,GAAG,KAAK,EAAE,GAAG,IAAI,EAAE,OAAO,EAAE,GAAG,IAAI,CAAA;CACzD;AAED,UAAU,eAAe;IACvB,cAAc,EAAE,cAAc,CAAA;IAC9B,YAAY,EAAE,oBAAoB,CAAA;IAClC,MAAM,CAAC,EAAE,MAAM,CAAA;IACf,QAAQ,EAAE,gBAAgB,CAAA;IAC1B,UAAU,EAAE;QACV,MAAM,EAAE,MAAM,CAAA;QACd,QAAQ,EAAE,MAAM,CAAA;QAChB,QAAQ,EAAE,MAAM,CAAA;QAChB,UAAU,EAAE,MAAM,EAAE,CAAA;KACrB,CAAA;IACD,MAAM,EAAE;QACN,IAAI,EAAE,MAAM,CAAA;QACZ,SAAS,EAAE,MAAM,CAAA;QACjB,gBAAgB,EAAE,MAAM,CAAA;QACxB,qBAAqB,EAAE,MAAM,CAAA;KAC9B,CAAA;IACD,SAAS,CAAC,EAAE;QACV,WAAW,EAAE,MAAM,CAAA;QACnB,QAAQ,EAAE,CAAC,MAAM,EAAE,aAAa,EAAE,KAAK,OAAO,CAAC,IAAI,CAAC,CAAA;KACrD,CAAA;CACF;AAED,MAAM,WAAW,aAAa;IAC5B,UAAU,EAAE,MAAM,CAAA;IAClB,OAAO,CAAC,EAAE,MAAM,CAAA;CACjB;AAED,MAAM,WAAW,oBAAoB;IACnC,QAAQ,IAAI,OAAO,CAAC,aAAa,GAAG,IAAI,CAAC,GAAG,aAAa,GAAG,IAAI,CAAA;IAChE,QAAQ,CAAC,KAAK,EAAE,aAAa,GAAG,OAAO,CAAC,IAAI,CAAC,GAAG,IAAI,CAAA;CACrD;AAED,MAAM,WAAW,gBAAgB;IAC/B,IAAI,CAAC,UAAU,EAAE,eAAe,CAAC,YAAY,CAAC,GAAG;QAAE,UAAU,EAAE,MAAM,CAAA;KAAE,EAAE,MAAM,CAAC,EAAE,WAAW,GAAG,OAAO,CAAC,IAAI,CAAC,CAAA;CAC9G;AAED,qBAAa,QAAQ;IAWP,OAAO,CAAC,OAAO;IAV3B,OAAO,CAAC,MAAM,CAAC,CAAQ;IACvB,OAAO,CAAC,KAAK,CAAe;IAC5B,OAAO,CAAC,YAAY,CAAe;IACnC,OAAO,CAAC,MAAM,CAA4B;IAC1C,OAAO,CAAC,YAAY,CAA8C;IAClE,OAAO,CAAC,MAAM,CAA8C;IAC5D,OAAO,CAAC,OAAO,CAAQ;IACvB,OAAO,CAAC,uBAAuB,CAAC,CAAiB;IACjD,OAAO,CAAC,UAAU,CAAC,CAAQ;gBAEP,OAAO,EAAE,eAAe;IAO/B,KAAK;IAiCX,IAAI;YAWG,SAAS;YAmBT,eAAe;IAwF7B,OAAO,CAAC,MAAM;IAgBd,OAAO,CAAC,WAAW;IAcnB,OAAO,KAAK,WAAW,GAMtB;IAED,OAAO,CAAC,oBAAoB;YAUd,iBAAiB;YAoBjB,SAAS;YAsCT,gBAAgB;YAsBhB,eAAe;YAUf,aAAa;YAab,sBAAsB;IAepC,OAAO,CAAC,gBAAgB;YAIV,aAAa;IAmBpB,IAAI,CAAC,MAAM,EAAE,MAAM,GAAG,OAAO,CAAC,aAAa,EAAE,CAAC;IAO9C,WAAW,CAAC,QAAQ,EAAE,MAAM,EAAE,GAAG,OAAO,CAAC,IAAI,CAAC;CAMtD"}
1
+ {"version":3,"file":"data-pump.d.ts","sourceRoot":"","sources":["../../src/lib/data-pump.ts"],"names":[],"mappings":"AAAA,OAAO,EAIL,KAAK,cAAc,EACnB,KAAK,aAAa,EAEnB,MAAM,iDAAiD,CAAA;AAiBxD,MAAM,WAAW,MAAM;IACrB,KAAK,CAAC,OAAO,EAAE,MAAM,EAAE,GAAG,IAAI,EAAE,OAAO,EAAE,GAAG,IAAI,CAAA;IAChD,IAAI,CAAC,OAAO,EAAE,MAAM,EAAE,GAAG,IAAI,EAAE,OAAO,EAAE,GAAG,IAAI,CAAA;IAC/C,IAAI,CAAC,OAAO,EAAE,MAAM,EAAE,GAAG,IAAI,EAAE,OAAO,EAAE,GAAG,IAAI,CAAA;IAC/C,KAAK,CAAC,OAAO,EAAE,MAAM,GAAG,KAAK,EAAE,GAAG,IAAI,EAAE,OAAO,EAAE,GAAG,IAAI,CAAA;CACzD;AAED,UAAU,eAAe;IACvB,cAAc,EAAE,cAAc,CAAA;IAC9B,YAAY,EAAE,oBAAoB,CAAA;IAClC,MAAM,CAAC,EAAE,MAAM,CAAA;IACf,QAAQ,EAAE,gBAAgB,CAAA;IAC1B,UAAU,EAAE;QACV,MAAM,EAAE,MAAM,CAAA;QACd,QAAQ,EAAE,MAAM,CAAA;QAChB,QAAQ,EAAE,MAAM,CAAA;QAChB,UAAU,EAAE,MAAM,EAAE,CAAA;KACrB,CAAA;IACD,MAAM,EAAE;QACN,IAAI,EAAE,MAAM,CAAA;QACZ,SAAS,EAAE,MAAM,CAAA;QACjB,kBAAkB,EAAE,MAAM,CAAA;QAC1B,qBAAqB,EAAE,MAAM,CAAA;KAC9B,CAAA;IACD,SAAS,CAAC,EAAE;QACV,WAAW,EAAE,MAAM,CAAA;QACnB,QAAQ,EAAE,CAAC,MAAM,EAAE,aAAa,EAAE,KAAK,OAAO,CAAC,OAAO,CAAC,GAAG,OAAO,CAAA;QACjE,cAAc,CAAC,EAAE,CAAC,MAAM,EAAE,aAAa,EAAE,KAAK,OAAO,CAAC,IAAI,CAAC,GAAG,IAAI,CAAA;KACnE,CAAA;CACF;AAED,MAAM,WAAW,aAAa;IAC5B,UAAU,EAAE,MAAM,CAAA;IAClB,OAAO,CAAC,EAAE,MAAM,CAAA;CACjB;AAED,MAAM,WAAW,oBAAoB;IACnC,QAAQ,IAAI,OAAO,CAAC,aAAa,GAAG,IAAI,CAAC,GAAG,aAAa,GAAG,IAAI,CAAA;IAChE,QAAQ,CAAC,KAAK,EAAE,aAAa,GAAG,OAAO,CAAC,IAAI,CAAC,GAAG,IAAI,CAAA;CACrD;AAED,MAAM,WAAW,gBAAgB;IAC/B,IAAI,CAAC,UAAU,EAAE,eAAe,CAAC,YAAY,CAAC,GAAG;QAAE,UAAU,EAAE,MAAM,CAAA;KAAE,EAAE,MAAM,CAAC,EAAE,WAAW,GAAG,OAAO,CAAC,IAAI,CAAC,CAAA;CAC9G;AAED,qBAAa,QAAQ;IAWP,OAAO,CAAC,OAAO;IAV3B,OAAO,CAAC,MAAM,CAAC,CAAQ;IACvB,OAAO,CAAC,KAAK,CAAe;IAC5B,OAAO,CAAC,YAAY,CAAe;IACnC,OAAO,CAAC,MAAM,CAA4B;IAC1C,OAAO,CAAC,YAAY,CAA8C;IAClE,OAAO,CAAC,MAAM,CAA8C;IAC5D,OAAO,CAAC,OAAO,CAAQ;IACvB,OAAO,CAAC,uBAAuB,CAAC,CAAiB;IACjD,OAAO,CAAC,UAAU,CAAC,CAAQ;gBAEP,OAAO,EAAE,eAAe;IAO/B,KAAK;IAiCX,IAAI;YAWG,SAAS;YAqBT,eAAe;YAwFf,MAAM;IAsCpB,OAAO,CAAC,WAAW;IAcnB,OAAO,KAAK,WAAW,GAMtB;IAED,OAAO,CAAC,oBAAoB;YAUd,iBAAiB;YAoBjB,SAAS;YAsCT,gBAAgB;YAsBhB,eAAe;YAUf,aAAa;YAab,sBAAsB;IAepC,OAAO,CAAC,gBAAgB;YAIV,aAAa;IAmBpB,IAAI,CAAC,MAAM,EAAE,MAAM,GAAG,OAAO,CAAC,aAAa,EAAE,CAAC;IAO9C,WAAW,CAAC,QAAQ,EAAE,MAAM,EAAE,GAAG,OAAO,CAAC,IAAI,CAAC;CAMtD"}
package/esm/lib/data-pump.js CHANGED
@@ -122,12 +122,14 @@ export class DataPump {
122
122
  if (!this.running) {
123
123
  break;
124
124
  }
125
- await this.options.processor.onEvents(events);
126
- await this.acknowledgeInner(events.map((event) => event.eventId));
125
+ const success = await this.options.processor.onEvents(events);
126
+ if (success) {
127
+ await this.acknowledgeInner(events.map((event) => event.eventId));
128
+ }
127
129
  }
128
130
  catch (error) {
129
131
  this.logger?.error("Error in processor", { error });
130
- this.stop();
132
+ await this.stop();
131
133
  }
132
134
  }
133
135
  }
@@ -199,19 +201,37 @@ export class DataPump {
199
201
  }
200
202
  return this.eventBufferLoop();
201
203
  }
202
- reOpen(eventIds, deliveryId) {
204
+ async reOpen(eventIds, deliveryId) {
203
205
  if (!eventIds.length) {
204
206
  return;
205
207
  }
206
- this.buffer.forEach((event) => {
208
+ const failedEvents = [];
209
+ const reopenedEvents = [];
210
+ let lastEvent;
211
+ this.buffer = this.buffer.filter((event) => {
212
+ if (event.deliveryId === deliveryId && this.options.buffer.maxRedeliveryCount > -1 &&
213
+ event.deliveryCount > this.options.buffer.maxRedeliveryCount) {
214
+ failedEvents.push(event.event);
215
+ lastEvent = event.event;
216
+ return false;
217
+ }
207
218
  if (event.deliveryId === deliveryId && eventIds.includes(event.event.eventId)) {
208
219
  event.status = FlowcoreBufferEventStatus.OPEN;
209
220
  event.deliveryId = "";
221
+ reopenedEvents.push(event.event);
210
222
  }
223
+ return true;
211
224
  });
225
+ if (reopenedEvents.length) {
226
+ this.waiter?.();
227
+ }
212
228
  if (this.bufferWaiter && this.buffer.length <= this.options.buffer.size - this.options.buffer.threshold) {
213
229
  this.bufferWaiter();
214
230
  }
231
+ if (failedEvents.length) {
232
+ await this.options.processor?.onFailedEvents?.(failedEvents);
233
+ await this.updateState(lastEvent?.eventId);
234
+ }
215
235
  }
216
236
  updateState(eventId) {
217
237
  const firstEvent = this.buffer[0];
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "@flowcore/data-pump",
3
- "version": "0.1.5",
3
+ "version": "0.1.7",
4
4
  "description": "Flowcore Data Pump",
5
5
  "homepage": "https://github.com/flowcore-io/flowcore-data-pump#readme",
6
6
  "repository": {
package/script/lib/data-pump-create.d.ts CHANGED
@@ -25,13 +25,14 @@ export interface DataPumpClientOptions {
25
25
  buffer?: {
26
26
  size?: number;
27
27
  threshold?: number;
28
- maxDeliveryCount?: number;
28
+ maxRedeliveryCount?: number;
29
29
  achknowledgeTimeoutMs?: number;
30
30
  };
31
31
  stateManager?: DataPumpStateManager;
32
32
  processor?: {
33
33
  concurrency?: number;
34
- onEvents: (events: FlowcoreEvent[]) => Promise<void>;
34
+ onEvents: (events: FlowcoreEvent[]) => Promise<boolean> | boolean;
35
+ onFailedEvents?: (events: FlowcoreEvent[]) => Promise<void> | void;
35
36
  };
36
37
  natsServers?: string[];
37
38
  }
package/script/lib/data-pump-create.d.ts.map CHANGED
@@ -1 +1 @@
1
- {"version":3,"file":"data-pump-create.d.ts","sourceRoot":"","sources":["../../src/lib/data-pump-create.ts"],"names":[],"mappings":"AAAA,OAAO,EAAkB,KAAK,aAAa,EAAE,MAAM,iDAAiD,CAAA;AACpG,OAAO,EAAE,QAAQ,EAAE,KAAK,oBAAoB,EAAE,KAAK,MAAM,EAAE,MAAM,gBAAgB,CAAA;AAIjF,eAAO,MAAM,UAAU,EAAE,MAKxB,CAAA;AAED,UAAU,mCAAmC;IAC3C,UAAU,EAAE;QACV,QAAQ,EAAE,MAAM,OAAO,CAAC;YAAE,WAAW,EAAE,MAAM,CAAA;SAAE,CAAC,CAAA;KACjD,CAAA;CACF;AAED,UAAU,+BAA+B;IACvC,MAAM,EAAE,MAAM,CAAA;IACd,QAAQ,EAAE,MAAM,CAAA;CACjB;AAED,KAAK,yBAAyB,GAAG,mCAAmC,GAAG,+BAA+B,CAAA;AAEtG,MAAM,WAAW,qBAAqB;IACpC,MAAM,CAAC,EAAE,MAAM,CAAA;IACf,IAAI,EAAE,yBAAyB,CAAA;IAC/B,UAAU,EAAE;QACV,MAAM,EAAE,MAAM,CAAA;QACd,QAAQ,EAAE,MAAM,CAAA;QAChB,QAAQ,EAAE,MAAM,CAAA;QAChB,UAAU,EAAE,MAAM,EAAE,CAAA;KACrB,CAAA;IACD,MAAM,CAAC,EAAE;QACP,IAAI,CAAC,EAAE,MAAM,CAAA;QACb,SAAS,CAAC,EAAE,MAAM,CAAA;QAClB,gBAAgB,CAAC,EAAE,MAAM,CAAA;QACzB,qBAAqB,CAAC,EAAE,MAAM,CAAA;KAC/B,CAAA;IACD,YAAY,CAAC,EAAE,oBAAoB,CAAA;IACnC,SAAS,CAAC,EAAE;QACV,WAAW,CAAC,EAAE,MAAM,CAAA;QACpB,QAAQ,EAAE,CAAC,MAAM,EAAE,aAAa,EAAE,KAAK,OAAO,CAAC,IAAI,CAAC,CAAA;KACrD,CAAA;IACD,WAAW,CAAC,EAAE,MAAM,EAAE,CAAA;CACvB;AAED,wBAAgB,cAAc,CAAC,OAAO,EAAE,qBAAqB,GAAG,QAAQ,CAuDvE"}
1
+ {"version":3,"file":"data-pump-create.d.ts","sourceRoot":"","sources":["../../src/lib/data-pump-create.ts"],"names":[],"mappings":"AAAA,OAAO,EAAkB,KAAK,aAAa,EAAE,MAAM,iDAAiD,CAAA;AACpG,OAAO,EAAE,QAAQ,EAAE,KAAK,oBAAoB,EAAE,KAAK,MAAM,EAAE,MAAM,gBAAgB,CAAA;AAIjF,eAAO,MAAM,UAAU,EAAE,MAKxB,CAAA;AAED,UAAU,mCAAmC;IAC3C,UAAU,EAAE;QACV,QAAQ,EAAE,MAAM,OAAO,CAAC;YAAE,WAAW,EAAE,MAAM,CAAA;SAAE,CAAC,CAAA;KACjD,CAAA;CACF;AAED,UAAU,+BAA+B;IACvC,MAAM,EAAE,MAAM,CAAA;IACd,QAAQ,EAAE,MAAM,CAAA;CACjB;AAED,KAAK,yBAAyB,GAAG,mCAAmC,GAAG,+BAA+B,CAAA;AAEtG,MAAM,WAAW,qBAAqB;IACpC,MAAM,CAAC,EAAE,MAAM,CAAA;IACf,IAAI,EAAE,yBAAyB,CAAA;IAC/B,UAAU,EAAE;QACV,MAAM,EAAE,MAAM,CAAA;QACd,QAAQ,EAAE,MAAM,CAAA;QAChB,QAAQ,EAAE,MAAM,CAAA;QAChB,UAAU,EAAE,MAAM,EAAE,CAAA;KACrB,CAAA;IACD,MAAM,CAAC,EAAE;QACP,IAAI,CAAC,EAAE,MAAM,CAAA;QACb,SAAS,CAAC,EAAE,MAAM,CAAA;QAClB,kBAAkB,CAAC,EAAE,MAAM,CAAA;QAC3B,qBAAqB,CAAC,EAAE,MAAM,CAAA;KAC/B,CAAA;IACD,YAAY,CAAC,EAAE,oBAAoB,CAAA;IACnC,SAAS,CAAC,EAAE;QACV,WAAW,CAAC,EAAE,MAAM,CAAA;QACpB,QAAQ,EAAE,CAAC,MAAM,EAAE,aAAa,EAAE,KAAK,OAAO,CAAC,OAAO,CAAC,GAAG,OAAO,CAAA;QACjE,cAAc,CAAC,EAAE,CAAC,MAAM,EAAE,aAAa,EAAE,KAAK,OAAO,CAAC,IAAI,CAAC,GAAG,IAAI,CAAA;KACnE,CAAA;IACD,WAAW,CAAC,EAAE,MAAM,EAAE,CAAA;CACvB;AAED,wBAAgB,cAAc,CAAC,OAAO,EAAE,qBAAqB,GAAG,QAAQ,CAwDvE"}
package/script/lib/data-pump-create.js CHANGED
@@ -43,13 +43,14 @@ function createDataPump(options) {
43
43
  const buffer = {
44
44
  size: bufferSize,
45
45
  threshold: options.buffer?.threshold ?? Math.round(bufferSize * 0.1),
46
- maxDeliveryCount: options.buffer?.maxDeliveryCount ?? 3,
46
+ maxRedeliveryCount: options.buffer?.maxRedeliveryCount ?? 3,
47
47
  achknowledgeTimeoutMs: options.buffer?.achknowledgeTimeoutMs ?? 10_000,
48
48
  };
49
49
  const processor = options.processor
50
50
  ? {
51
51
  concurrency: options.processor.concurrency ?? 1,
52
52
  onEvents: options.processor.onEvents,
53
+ onFailedEvents: options.processor.onFailedEvents,
53
54
  }
54
55
  : undefined;
55
56
  const dataPump = new data_pump_js_1.DataPump({
package/script/lib/data-pump.d.ts CHANGED
@@ -19,12 +19,13 @@ interface DataPumpOptions {
19
19
  buffer: {
20
20
  size: number;
21
21
  threshold: number;
22
- maxDeliveryCount: number;
22
+ maxRedeliveryCount: number;
23
23
  achknowledgeTimeoutMs: number;
24
24
  };
25
25
  processor?: {
26
26
  concurrency: number;
27
- onEvents: (events: FlowcoreEvent[]) => Promise<void>;
27
+ onEvents: (events: FlowcoreEvent[]) => Promise<boolean> | boolean;
28
+ onFailedEvents?: (events: FlowcoreEvent[]) => Promise<void> | void;
28
29
  };
29
30
  }
30
31
  export interface DataPumpState {
package/script/lib/data-pump.d.ts.map CHANGED
@@ -1 +1 @@
1
- {"version":3,"file":"data-pump.d.ts","sourceRoot":"","sources":["../../src/lib/data-pump.ts"],"names":[],"mappings":"AAAA,OAAO,EAIL,KAAK,cAAc,EACnB,KAAK,aAAa,EAEnB,MAAM,iDAAiD,CAAA;AAiBxD,MAAM,WAAW,MAAM;IACrB,KAAK,CAAC,OAAO,EAAE,MAAM,EAAE,GAAG,IAAI,EAAE,OAAO,EAAE,GAAG,IAAI,CAAA;IAChD,IAAI,CAAC,OAAO,EAAE,MAAM,EAAE,GAAG,IAAI,EAAE,OAAO,EAAE,GAAG,IAAI,CAAA;IAC/C,IAAI,CAAC,OAAO,EAAE,MAAM,EAAE,GAAG,IAAI,EAAE,OAAO,EAAE,GAAG,IAAI,CAAA;IAC/C,KAAK,CAAC,OAAO,EAAE,MAAM,GAAG,KAAK,EAAE,GAAG,IAAI,EAAE,OAAO,EAAE,GAAG,IAAI,CAAA;CACzD;AAED,UAAU,eAAe;IACvB,cAAc,EAAE,cAAc,CAAA;IAC9B,YAAY,EAAE,oBAAoB,CAAA;IAClC,MAAM,CAAC,EAAE,MAAM,CAAA;IACf,QAAQ,EAAE,gBAAgB,CAAA;IAC1B,UAAU,EAAE;QACV,MAAM,EAAE,MAAM,CAAA;QACd,QAAQ,EAAE,MAAM,CAAA;QAChB,QAAQ,EAAE,MAAM,CAAA;QAChB,UAAU,EAAE,MAAM,EAAE,CAAA;KACrB,CAAA;IACD,MAAM,EAAE;QACN,IAAI,EAAE,MAAM,CAAA;QACZ,SAAS,EAAE,MAAM,CAAA;QACjB,gBAAgB,EAAE,MAAM,CAAA;QACxB,qBAAqB,EAAE,MAAM,CAAA;KAC9B,CAAA;IACD,SAAS,CAAC,EAAE;QACV,WAAW,EAAE,MAAM,CAAA;QACnB,QAAQ,EAAE,CAAC,MAAM,EAAE,aAAa,EAAE,KAAK,OAAO,CAAC,IAAI,CAAC,CAAA;KACrD,CAAA;CACF;AAED,MAAM,WAAW,aAAa;IAC5B,UAAU,EAAE,MAAM,CAAA;IAClB,OAAO,CAAC,EAAE,MAAM,CAAA;CACjB;AAED,MAAM,WAAW,oBAAoB;IACnC,QAAQ,IAAI,OAAO,CAAC,aAAa,GAAG,IAAI,CAAC,GAAG,aAAa,GAAG,IAAI,CAAA;IAChE,QAAQ,CAAC,KAAK,EAAE,aAAa,GAAG,OAAO,CAAC,IAAI,CAAC,GAAG,IAAI,CAAA;CACrD;AAED,MAAM,WAAW,gBAAgB;IAC/B,IAAI,CAAC,UAAU,EAAE,eAAe,CAAC,YAAY,CAAC,GAAG;QAAE,UAAU,EAAE,MAAM,CAAA;KAAE,EAAE,MAAM,CAAC,EAAE,WAAW,GAAG,OAAO,CAAC,IAAI,CAAC,CAAA;CAC9G;AAED,qBAAa,QAAQ;IAWP,OAAO,CAAC,OAAO;IAV3B,OAAO,CAAC,MAAM,CAAC,CAAQ;IACvB,OAAO,CAAC,KAAK,CAAe;IAC5B,OAAO,CAAC,YAAY,CAAe;IACnC,OAAO,CAAC,MAAM,CAA4B;IAC1C,OAAO,CAAC,YAAY,CAA8C;IAClE,OAAO,CAAC,MAAM,CAA8C;IAC5D,OAAO,CAAC,OAAO,CAAQ;IACvB,OAAO,CAAC,uBAAuB,CAAC,CAAiB;IACjD,OAAO,CAAC,UAAU,CAAC,CAAQ;gBAEP,OAAO,EAAE,eAAe;IAO/B,KAAK;IAiCX,IAAI;YAWG,SAAS;YAmBT,eAAe;IAwF7B,OAAO,CAAC,MAAM;IAgBd,OAAO,CAAC,WAAW;IAcnB,OAAO,KAAK,WAAW,GAMtB;IAED,OAAO,CAAC,oBAAoB;YAUd,iBAAiB;YAoBjB,SAAS;YAsCT,gBAAgB;YAsBhB,eAAe;YAUf,aAAa;YAab,sBAAsB;IAepC,OAAO,CAAC,gBAAgB;YAIV,aAAa;IAmBpB,IAAI,CAAC,MAAM,EAAE,MAAM,GAAG,OAAO,CAAC,aAAa,EAAE,CAAC;IAO9C,WAAW,CAAC,QAAQ,EAAE,MAAM,EAAE,GAAG,OAAO,CAAC,IAAI,CAAC;CAMtD"}
1
+ {"version":3,"file":"data-pump.d.ts","sourceRoot":"","sources":["../../src/lib/data-pump.ts"],"names":[],"mappings":"AAAA,OAAO,EAIL,KAAK,cAAc,EACnB,KAAK,aAAa,EAEnB,MAAM,iDAAiD,CAAA;AAiBxD,MAAM,WAAW,MAAM;IACrB,KAAK,CAAC,OAAO,EAAE,MAAM,EAAE,GAAG,IAAI,EAAE,OAAO,EAAE,GAAG,IAAI,CAAA;IAChD,IAAI,CAAC,OAAO,EAAE,MAAM,EAAE,GAAG,IAAI,EAAE,OAAO,EAAE,GAAG,IAAI,CAAA;IAC/C,IAAI,CAAC,OAAO,EAAE,MAAM,EAAE,GAAG,IAAI,EAAE,OAAO,EAAE,GAAG,IAAI,CAAA;IAC/C,KAAK,CAAC,OAAO,EAAE,MAAM,GAAG,KAAK,EAAE,GAAG,IAAI,EAAE,OAAO,EAAE,GAAG,IAAI,CAAA;CACzD;AAED,UAAU,eAAe;IACvB,cAAc,EAAE,cAAc,CAAA;IAC9B,YAAY,EAAE,oBAAoB,CAAA;IAClC,MAAM,CAAC,EAAE,MAAM,CAAA;IACf,QAAQ,EAAE,gBAAgB,CAAA;IAC1B,UAAU,EAAE;QACV,MAAM,EAAE,MAAM,CAAA;QACd,QAAQ,EAAE,MAAM,CAAA;QAChB,QAAQ,EAAE,MAAM,CAAA;QAChB,UAAU,EAAE,MAAM,EAAE,CAAA;KACrB,CAAA;IACD,MAAM,EAAE;QACN,IAAI,EAAE,MAAM,CAAA;QACZ,SAAS,EAAE,MAAM,CAAA;QACjB,kBAAkB,EAAE,MAAM,CAAA;QAC1B,qBAAqB,EAAE,MAAM,CAAA;KAC9B,CAAA;IACD,SAAS,CAAC,EAAE;QACV,WAAW,EAAE,MAAM,CAAA;QACnB,QAAQ,EAAE,CAAC,MAAM,EAAE,aAAa,EAAE,KAAK,OAAO,CAAC,OAAO,CAAC,GAAG,OAAO,CAAA;QACjE,cAAc,CAAC,EAAE,CAAC,MAAM,EAAE,aAAa,EAAE,KAAK,OAAO,CAAC,IAAI,CAAC,GAAG,IAAI,CAAA;KACnE,CAAA;CACF;AAED,MAAM,WAAW,aAAa;IAC5B,UAAU,EAAE,MAAM,CAAA;IAClB,OAAO,CAAC,EAAE,MAAM,CAAA;CACjB;AAED,MAAM,WAAW,oBAAoB;IACnC,QAAQ,IAAI,OAAO,CAAC,aAAa,GAAG,IAAI,CAAC,GAAG,aAAa,GAAG,IAAI,CAAA;IAChE,QAAQ,CAAC,KAAK,EAAE,aAAa,GAAG,OAAO,CAAC,IAAI,CAAC,GAAG,IAAI,CAAA;CACrD;AAED,MAAM,WAAW,gBAAgB;IAC/B,IAAI,CAAC,UAAU,EAAE,eAAe,CAAC,YAAY,CAAC,GAAG;QAAE,UAAU,EAAE,MAAM,CAAA;KAAE,EAAE,MAAM,CAAC,EAAE,WAAW,GAAG,OAAO,CAAC,IAAI,CAAC,CAAA;CAC9G;AAED,qBAAa,QAAQ;IAWP,OAAO,CAAC,OAAO;IAV3B,OAAO,CAAC,MAAM,CAAC,CAAQ;IACvB,OAAO,CAAC,KAAK,CAAe;IAC5B,OAAO,CAAC,YAAY,CAAe;IACnC,OAAO,CAAC,MAAM,CAA4B;IAC1C,OAAO,CAAC,YAAY,CAA8C;IAClE,OAAO,CAAC,MAAM,CAA8C;IAC5D,OAAO,CAAC,OAAO,CAAQ;IACvB,OAAO,CAAC,uBAAuB,CAAC,CAAiB;IACjD,OAAO,CAAC,UAAU,CAAC,CAAQ;gBAEP,OAAO,EAAE,eAAe;IAO/B,KAAK;IAiCX,IAAI;YAWG,SAAS;YAqBT,eAAe;YAwFf,MAAM;IAsCpB,OAAO,CAAC,WAAW;IAcnB,OAAO,KAAK,WAAW,GAMtB;IAED,OAAO,CAAC,oBAAoB;YAUd,iBAAiB;YAoBjB,SAAS;YAsCT,gBAAgB;YAsBhB,eAAe;YAUf,aAAa;YAab,sBAAsB;IAepC,OAAO,CAAC,gBAAgB;YAIV,aAAa;IAmBpB,IAAI,CAAC,MAAM,EAAE,MAAM,GAAG,OAAO,CAAC,aAAa,EAAE,CAAC;IAO9C,WAAW,CAAC,QAAQ,EAAE,MAAM,EAAE,GAAG,OAAO,CAAC,IAAI,CAAC;CAMtD"}
package/script/lib/data-pump.js CHANGED
@@ -125,12 +125,14 @@ class DataPump {
125
125
  if (!this.running) {
126
126
  break;
127
127
  }
128
- await this.options.processor.onEvents(events);
129
- await this.acknowledgeInner(events.map((event) => event.eventId));
128
+ const success = await this.options.processor.onEvents(events);
129
+ if (success) {
130
+ await this.acknowledgeInner(events.map((event) => event.eventId));
131
+ }
130
132
  }
131
133
  catch (error) {
132
134
  this.logger?.error("Error in processor", { error });
133
- this.stop();
135
+ await this.stop();
134
136
  }
135
137
  }
136
138
  }
@@ -202,19 +204,37 @@ class DataPump {
202
204
  }
203
205
  return this.eventBufferLoop();
204
206
  }
205
- reOpen(eventIds, deliveryId) {
207
+ async reOpen(eventIds, deliveryId) {
206
208
  if (!eventIds.length) {
207
209
  return;
208
210
  }
209
- this.buffer.forEach((event) => {
211
+ const failedEvents = [];
212
+ const reopenedEvents = [];
213
+ let lastEvent;
214
+ this.buffer = this.buffer.filter((event) => {
215
+ if (event.deliveryId === deliveryId && this.options.buffer.maxRedeliveryCount > -1 &&
216
+ event.deliveryCount > this.options.buffer.maxRedeliveryCount) {
217
+ failedEvents.push(event.event);
218
+ lastEvent = event.event;
219
+ return false;
220
+ }
210
221
  if (event.deliveryId === deliveryId && eventIds.includes(event.event.eventId)) {
211
222
  event.status = FlowcoreBufferEventStatus.OPEN;
212
223
  event.deliveryId = "";
224
+ reopenedEvents.push(event.event);
213
225
  }
226
+ return true;
214
227
  });
228
+ if (reopenedEvents.length) {
229
+ this.waiter?.();
230
+ }
215
231
  if (this.bufferWaiter && this.buffer.length <= this.options.buffer.size - this.options.buffer.threshold) {
216
232
  this.bufferWaiter();
217
233
  }
234
+ if (failedEvents.length) {
235
+ await this.options.processor?.onFailedEvents?.(failedEvents);
236
+ await this.updateState(lastEvent?.eventId);
237
+ }
218
238
  }
219
239
  updateState(eventId) {
220
240
  const firstEvent = this.buffer[0];