@flowcore/data-pump 0.1.8 → 0.1.10

package/CHANGELOG.md

@@ -1,5 +1,19 @@
 # Changelog
 
+## [0.1.10](https://github.com/flowcore-io/data-pump/compare/v0.1.9...v0.1.10) (2025-02-19)
+
+
+### Bug Fixes
+
+* add stop at option ([e0d90eb](https://github.com/flowcore-io/data-pump/commit/e0d90eb48dea096ee8cc9602b9b3acf9a706d7f8))
+
+## [0.1.9](https://github.com/flowcore-io/data-pump/compare/v0.1.8...v0.1.9) (2025-02-19)
+
+
+### Bug Fixes
+
+* add getDate to time uuid util ([03d7d2e](https://github.com/flowcore-io/data-pump/commit/03d7d2eb8d621368aa08ade148dea7e4561e90e1))
+
 ## [0.1.8](https://github.com/flowcore-io/data-pump/compare/v0.1.7...v0.1.8) (2025-02-19)
 
 

package/esm/_dnt.polyfills.d.ts

@@ -0,0 +1,50 @@
+declare global {
+    interface Array<T> {
+        /**
+         * Returns the value of the last element in the array where predicate is true, and undefined
+         * otherwise.
+         * @param predicate find calls predicate once for each element of the array, in ascending
+         * order, until it finds one where predicate returns true. If such an element is found, find
+         * immediately returns that element value. Otherwise, find returns undefined.
+         * @param thisArg If provided, it will be used as the this value for each invocation of
+         * predicate. If it is not provided, undefined is used instead.
+         */
+        findLast<S extends T>(predicate: (this: void, value: T, index: number, obj: T[]) => value is S, thisArg?: any): S | undefined;
+        findLast(predicate: (value: T, index: number, obj: T[]) => unknown, thisArg?: any): T | undefined;
+        /**
+         * Returns the index of the last element in the array where predicate is true, and -1
+         * otherwise.
+         * @param predicate find calls predicate once for each element of the array, in ascending
+         * order, until it finds one where predicate returns true. If such an element is found,
+         * findIndex immediately returns that element index. Otherwise, findIndex returns -1.
+         * @param thisArg If provided, it will be used as the this value for each invocation of
+         * predicate. If it is not provided, undefined is used instead.
+         */
+        findLastIndex(predicate: (value: T, index: number, obj: T[]) => unknown, thisArg?: any): number;
+    }
+    interface Uint8Array {
+        /**
+         * Returns the value of the last element in the array where predicate is true, and undefined
+         * otherwise.
+         * @param predicate findLast calls predicate once for each element of the array, in descending
+         * order, until it finds one where predicate returns true. If such an element is found, findLast
+         * immediately returns that element value. Otherwise, findLast returns undefined.
+         * @param thisArg If provided, it will be used as the this value for each invocation of
+         * predicate. If it is not provided, undefined is used instead.
+         */
+        findLast<S extends number>(predicate: (value: number, index: number, array: Uint8Array) => value is S, thisArg?: any): S | undefined;
+        findLast(predicate: (value: number, index: number, array: Uint8Array) => unknown, thisArg?: any): number | undefined;
+        /**
+         * Returns the index of the last element in the array where predicate is true, and -1
+         * otherwise.
+         * @param predicate findLastIndex calls predicate once for each element of the array, in descending
+         * order, until it finds one where predicate returns true. If such an element is found,
+         * findLastIndex immediately returns that element index. Otherwise, findLastIndex returns -1.
+         * @param thisArg If provided, it will be used as the this value for each invocation of
+         * predicate. If it is not provided, undefined is used instead.
+         */
+        findLastIndex(predicate: (value: number, index: number, array: Uint8Array) => unknown, thisArg?: any): number;
+    }
+}
+export {};
+//# sourceMappingURL=_dnt.polyfills.d.ts.map

package/esm/_dnt.polyfills.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"_dnt.polyfills.d.ts","sourceRoot":"","sources":["../src/_dnt.polyfills.ts"],"names":[],"mappings":"AACA,OAAO,CAAC,MAAM,CAAC;IACb,UAAU,KAAK,CAAC,CAAC;QACf;;;;;;;;WAQG;QACH,QAAQ,CAAC,CAAC,SAAS,CAAC,EAClB,SAAS,EAAE,CAAC,IAAI,EAAE,IAAI,EAAE,KAAK,EAAE,CAAC,EAAE,KAAK,EAAE,MAAM,EAAE,GAAG,EAAE,CAAC,EAAE,KAAK,KAAK,IAAI,CAAC,EACxE,OAAO,CAAC,EAAE,GAAG,GACZ,CAAC,GAAG,SAAS,CAAC;QACjB,QAAQ,CACN,SAAS,EAAE,CAAC,KAAK,EAAE,CAAC,EAAE,KAAK,EAAE,MAAM,EAAE,GAAG,EAAE,CAAC,EAAE,KAAK,OAAO,EACzD,OAAO,CAAC,EAAE,GAAG,GACZ,CAAC,GAAG,SAAS,CAAC;QAEjB;;;;;;;;WAQG;QACH,aAAa,CACX,SAAS,EAAE,CAAC,KAAK,EAAE,CAAC,EAAE,KAAK,EAAE,MAAM,EAAE,GAAG,EAAE,CAAC,EAAE,KAAK,OAAO,EACzD,OAAO,CAAC,EAAE,GAAG,GACZ,MAAM,CAAC;KACX;IACD,UAAU,UAAU;QAClB;;;;;;;;WAQG;QACH,QAAQ,CAAC,CAAC,SAAS,MAAM,EACvB,SAAS,EAAE,CACP,KAAK,EAAE,MAAM,EACb,KAAK,EAAE,MAAM,EACb,KAAK,EAAE,UAAU,KAChB,KAAK,IAAI,CAAC,EACf,OAAO,CAAC,EAAE,GAAG,GACZ,CAAC,GAAG,SAAS,CAAC;QACjB,QAAQ,CACJ,SAAS,EAAE,CAAC,KAAK,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,EAAE,KAAK,EAAE,UAAU,KAAK,OAAO,EACvE,OAAO,CAAC,EAAE,GAAG,GACd,MAAM,GAAG,SAAS,CAAC;QAEtB;;;;;;;;WAQG;QACH,aAAa,CACT,SAAS,EAAE,CAAC,KAAK,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,EAAE,KAAK,EAAE,UAAU,KAAK,OAAO,EACvE,OAAO,CAAC,EAAE,GAAG,GACd,MAAM,CAAC;KACX;CACF;AA4CD,OAAO,EAAE,CAAC"}

package/esm/_dnt.polyfills.js

@@ -0,0 +1,37 @@
+function findLastIndex(self, callbackfn, that) {
+    const boundFunc = that === undefined ? callbackfn : callbackfn.bind(that);
+    let index = self.length - 1;
+    while (index >= 0) {
+        const result = boundFunc(self[index], index, self);
+        if (result) {
+            return index;
+        }
+        index--;
+    }
+    return -1;
+}
+function findLast(self, callbackfn, that) {
+    const index = self.findLastIndex(callbackfn, that);
+    return index === -1 ? undefined : self[index];
+}
+if (!Array.prototype.findLastIndex) {
+    Array.prototype.findLastIndex = function (callbackfn, that) {
+        return findLastIndex(this, callbackfn, that);
+    };
+}
+if (!Array.prototype.findLast) {
+    Array.prototype.findLast = function (callbackfn, that) {
+        return findLast(this, callbackfn, that);
+    };
+}
+if (!Uint8Array.prototype.findLastIndex) {
+    Uint8Array.prototype.findLastIndex = function (callbackfn, that) {
+        return findLastIndex(this, callbackfn, that);
+    };
+}
+if (!Uint8Array.prototype.findLast) {
+    Uint8Array.prototype.findLast = function (callbackfn, that) {
+        return findLast(this, callbackfn, that);
+    };
+}
+export {};

package/esm/lib/data-pump-create.d.ts

@@ -36,7 +36,9 @@ export interface DataPumpClientOptions {
         concurrency?: number;
         onEvents: (events: FlowcoreEvent[]) => Promise<boolean> | boolean;
         onFailedEvents?: (events: FlowcoreEvent[]) => Promise<void> | void;
+        onDone?: () => Promise<void> | void;
     };
+    stopAt?: Date;
     natsServers?: string[];
 }
 export declare function createDataPump(options: DataPumpClientOptions): DataPump;

package/esm/lib/data-pump-create.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"data-pump-create.d.ts","sourceRoot":"","sources":["../../src/lib/data-pump-create.ts"],"names":[],"mappings":"AAAA,OAAO,EAAkB,KAAK,aAAa,EAAE,MAAM,iDAAiD,CAAA;AACpG,OAAO,EAAE,QAAQ,EAAE,KAAK,oBAAoB,EAAE,KAAK,MAAM,EAAE,MAAM,gBAAgB,CAAA;AAIjF,eAAO,MAAM,UAAU,EAAE,MAKxB,CAAA;AAED,UAAU,mCAAmC;IAC3C,UAAU,EAAE;QACV,QAAQ,EAAE,MAAM,OAAO,CAAC;YAAE,WAAW,EAAE,MAAM,CAAA;SAAE,CAAC,CAAA;KACjD,CAAA;CACF;AAED,UAAU,+BAA+B;IACvC,MAAM,EAAE,MAAM,CAAA;IACd,QAAQ,EAAE,MAAM,CAAA;CACjB;AAED,KAAK,yBAAyB,GAAG,mCAAmC,GAAG,+BAA+B,CAAA;AAEtG;;GAEG;AACH,MAAM,WAAW,qBAAqB;IACpC,MAAM,CAAC,EAAE,MAAM,CAAA;IACf,IAAI,EAAE,yBAAyB,CAAA;IAC/B,UAAU,EAAE;QACV,MAAM,EAAE,MAAM,CAAA;QACd,QAAQ,EAAE,MAAM,CAAA;QAChB,QAAQ,EAAE,MAAM,CAAA;QAChB,UAAU,EAAE,MAAM,EAAE,CAAA;KACrB,CAAA;IACD,MAAM,CAAC,EAAE;QACP,IAAI,CAAC,EAAE,MAAM,CAAA;QACb,SAAS,CAAC,EAAE,MAAM,CAAA;QAClB,kBAAkB,CAAC,EAAE,MAAM,CAAA;QAC3B,qBAAqB,CAAC,EAAE,MAAM,CAAA;KAC/B,CAAA;IACD,YAAY,CAAC,EAAE,oBAAoB,CAAA;IACnC,SAAS,CAAC,EAAE;QACV,WAAW,CAAC,EAAE,MAAM,CAAA;QACpB,QAAQ,EAAE,CAAC,MAAM,EAAE,aAAa,EAAE,KAAK,OAAO,CAAC,OAAO,CAAC,GAAG,OAAO,CAAA;QACjE,cAAc,CAAC,EAAE,CAAC,MAAM,EAAE,aAAa,EAAE,KAAK,OAAO,CAAC,IAAI,CAAC,GAAG,IAAI,CAAA;KACnE,CAAA;IACD,WAAW,CAAC,EAAE,MAAM,EAAE,CAAA;CACvB;AAED,wBAAgB,cAAc,CAAC,OAAO,EAAE,qBAAqB,GAAG,QAAQ,CAwDvE"}
+{"version":3,"file":"data-pump-create.d.ts","sourceRoot":"","sources":["../../src/lib/data-pump-create.ts"],"names":[],"mappings":"AAAA,OAAO,EAAkB,KAAK,aAAa,EAAE,MAAM,iDAAiD,CAAA;AACpG,OAAO,EAAE,QAAQ,EAAE,KAAK,oBAAoB,EAAE,KAAK,MAAM,EAAE,MAAM,gBAAgB,CAAA;AAIjF,eAAO,MAAM,UAAU,EAAE,MAKxB,CAAA;AAED,UAAU,mCAAmC;IAC3C,UAAU,EAAE;QACV,QAAQ,EAAE,MAAM,OAAO,CAAC;YAAE,WAAW,EAAE,MAAM,CAAA;SAAE,CAAC,CAAA;KACjD,CAAA;CACF;AAED,UAAU,+BAA+B;IACvC,MAAM,EAAE,MAAM,CAAA;IACd,QAAQ,EAAE,MAAM,CAAA;CACjB;AAED,KAAK,yBAAyB,GAAG,mCAAmC,GAAG,+BAA+B,CAAA;AAEtG;;GAEG;AACH,MAAM,WAAW,qBAAqB;IACpC,MAAM,CAAC,EAAE,MAAM,CAAA;IACf,IAAI,EAAE,yBAAyB,CAAA;IAC/B,UAAU,EAAE;QACV,MAAM,EAAE,MAAM,CAAA;QACd,QAAQ,EAAE,MAAM,CAAA;QAChB,QAAQ,EAAE,MAAM,CAAA;QAChB,UAAU,EAAE,MAAM,EAAE,CAAA;KACrB,CAAA;IACD,MAAM,CAAC,EAAE;QACP,IAAI,CAAC,EAAE,MAAM,CAAA;QACb,SAAS,CAAC,EAAE,MAAM,CAAA;QAClB,kBAAkB,CAAC,EAAE,MAAM,CAAA;QAC3B,qBAAqB,CAAC,EAAE,MAAM,CAAA;KAC/B,CAAA;IACD,YAAY,CAAC,EAAE,oBAAoB,CAAA;IACnC,SAAS,CAAC,EAAE;QACV,WAAW,CAAC,EAAE,MAAM,CAAA;QACpB,QAAQ,EAAE,CAAC,MAAM,EAAE,aAAa,EAAE,KAAK,OAAO,CAAC,OAAO,CAAC,GAAG,OAAO,CAAA;QACjE,cAAc,CAAC,EAAE,CAAC,MAAM,EAAE,aAAa,EAAE,KAAK,OAAO,CAAC,IAAI,CAAC,GAAG,IAAI,CAAA;QAClE,MAAM,CAAC,EAAE,MAAM,OAAO,CAAC,IAAI,CAAC,GAAG,IAAI,CAAA;KACpC,CAAA;IACD,MAAM,CAAC,EAAE,IAAI,CAAA;IACb,WAAW,CAAC,EAAE,MAAM,EAAE,CAAA;CACvB;AAED,wBAAgB,cAAc,CAAC,OAAO,EAAE,qBAAqB,GAAG,QAAQ,CA0DvE"}

package/esm/lib/data-pump-create.js

@@ -47,6 +47,7 @@ export function createDataPump(options) {
             concurrency: options.processor.concurrency ?? 1,
             onEvents: options.processor.onEvents,
             onFailedEvents: options.processor.onFailedEvents,
+            onDone: options.processor.onDone,
         }
         : undefined;
     const dataPump = new DataPump({
@@ -57,6 +58,7 @@ export function createDataPump(options) {
         processor,
         notifier: notifier.getNotifier(),
         logger: options.logger,
+        stopAt: options.stopAt,
     });
     return dataPump;
 }

package/esm/lib/data-pump.d.ts

@@ -26,15 +26,18 @@ interface DataPumpOptions {
         concurrency: number;
         onEvents: (events: FlowcoreEvent[]) => Promise<boolean> | boolean;
         onFailedEvents?: (events: FlowcoreEvent[]) => Promise<void> | void;
+        onDone?: () => Promise<void> | void;
     };
+    stopAt?: Date;
 }
 export interface DataPumpState {
     timeBucket: string;
     eventId?: string;
 }
 export interface DataPumpStateManager {
-    getState(): Promise<DataPumpState | null> | DataPumpState | null;
-    setState(state: DataPumpState): Promise<void> | void;
+    getState: () => Promise<DataPumpState | null> | DataPumpState | null;
+    setState?: (state: DataPumpState) => Promise<void> | void;
+    stopAt?: () => Date;
 }
 export interface DataPumpNotifier {
     wait(dataSource: DataPumpOptions["dataSource"] & {
@@ -52,6 +55,7 @@ export declare class DataPump {
     private running;
     private notifierAbortController?;
     private dataCoreId?;
+    private stopAt?;
     constructor(options: DataPumpOptions);
     start(): Promise<void>;
     stop(): void;

package/esm/lib/data-pump.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"data-pump.d.ts","sourceRoot":"","sources":["../../src/lib/data-pump.ts"],"names":[],"mappings":"AAAA,OAAO,EAIL,KAAK,cAAc,EACnB,KAAK,aAAa,EAEnB,MAAM,iDAAiD,CAAA;AAiBxD,MAAM,WAAW,MAAM;IACrB,KAAK,CAAC,OAAO,EAAE,MAAM,EAAE,GAAG,IAAI,EAAE,OAAO,EAAE,GAAG,IAAI,CAAA;IAChD,IAAI,CAAC,OAAO,EAAE,MAAM,EAAE,GAAG,IAAI,EAAE,OAAO,EAAE,GAAG,IAAI,CAAA;IAC/C,IAAI,CAAC,OAAO,EAAE,MAAM,EAAE,GAAG,IAAI,EAAE,OAAO,EAAE,GAAG,IAAI,CAAA;IAC/C,KAAK,CAAC,OAAO,EAAE,MAAM,GAAG,KAAK,EAAE,GAAG,IAAI,EAAE,OAAO,EAAE,GAAG,IAAI,CAAA;CACzD;AAED,UAAU,eAAe;IACvB,cAAc,EAAE,cAAc,CAAA;IAC9B,YAAY,EAAE,oBAAoB,CAAA;IAClC,MAAM,CAAC,EAAE,MAAM,CAAA;IACf,QAAQ,EAAE,gBAAgB,CAAA;IAC1B,UAAU,EAAE;QACV,MAAM,EAAE,MAAM,CAAA;QACd,QAAQ,EAAE,MAAM,CAAA;QAChB,QAAQ,EAAE,MAAM,CAAA;QAChB,UAAU,EAAE,MAAM,EAAE,CAAA;KACrB,CAAA;IACD,MAAM,EAAE;QACN,IAAI,EAAE,MAAM,CAAA;QACZ,SAAS,EAAE,MAAM,CAAA;QACjB,kBAAkB,EAAE,MAAM,CAAA;QAC1B,qBAAqB,EAAE,MAAM,CAAA;KAC9B,CAAA;IACD,SAAS,CAAC,EAAE;QACV,WAAW,EAAE,MAAM,CAAA;QACnB,QAAQ,EAAE,CAAC,MAAM,EAAE,aAAa,EAAE,KAAK,OAAO,CAAC,OAAO,CAAC,GAAG,OAAO,CAAA;QACjE,cAAc,CAAC,EAAE,CAAC,MAAM,EAAE,aAAa,EAAE,KAAK,OAAO,CAAC,IAAI,CAAC,GAAG,IAAI,CAAA;KACnE,CAAA;CACF;AAED,MAAM,WAAW,aAAa;IAC5B,UAAU,EAAE,MAAM,CAAA;IAClB,OAAO,CAAC,EAAE,MAAM,CAAA;CACjB;AAED,MAAM,WAAW,oBAAoB;IACnC,QAAQ,IAAI,OAAO,CAAC,aAAa,GAAG,IAAI,CAAC,GAAG,aAAa,GAAG,IAAI,CAAA;IAChE,QAAQ,CAAC,KAAK,EAAE,aAAa,GAAG,OAAO,CAAC,IAAI,CAAC,GAAG,IAAI,CAAA;CACrD;AAED,MAAM,WAAW,gBAAgB;IAC/B,IAAI,CAAC,UAAU,EAAE,eAAe,CAAC,YAAY,CAAC,GAAG;QAAE,UAAU,EAAE,MAAM,CAAA;KAAE,EAAE,MAAM,CAAC,EAAE,WAAW,GAAG,OAAO,CAAC,IAAI,CAAC,CAAA;CAC9G;AAED,qBAAa,QAAQ;IAWP,OAAO,CAAC,OAAO;IAV3B,OAAO,CAAC,MAAM,CAAC,CAAQ;IACvB,OAAO,CAAC,KAAK,CAAe;IAC5B,OAAO,CAAC,YAAY,CAAe;IACnC,OAAO,CAAC,MAAM,CAA4B;IAC1C,OAAO,CAAC,YAAY,CAA8C;IAClE,OAAO,CAAC,MAAM,CAA8C;IAC5D,OAAO,CAAC,OAAO,CAAQ;IACvB,OAAO,CAAC,uBAAuB,CAAC,CAAiB;IACjD,OAAO,CAAC,UAAU,CAAC,CAAQ;gBAEP,OAAO,EAAE,eAAe;IAO/B,KAAK;IAiCX,IAAI;YAWG,SAAS;YAqBT,eAAe;YAwFf,MAAM;IAsCpB,OAAO,CAAC,WAAW;IAcnB,OAAO,KAAK,WAAW,GAMtB;IAED,OAAO,CAAC,oBAAoB;YAUd,iBAAiB;YAoBjB,SAAS;YAsCT,gBAAgB;YAsBhB,eAAe;YAUf,aAAa;YAab,sBAAsB;IAepC,OAAO,CAAC,gBAAgB;YAIV,aAAa;IAmBpB,IAAI,CAAC,MAAM,EAAE,MAAM,GAAG,OAAO,CAAC,aAAa,EAAE,CAAC;IAO9C,WAAW,CAAC,QAAQ,EAAE,MAAM,EAAE,GAAG,OAAO,CAAC,IAAI,CAAC;CAMtD"}
+{"version":3,"file":"data-pump.d.ts","sourceRoot":"","sources":["../../src/lib/data-pump.ts"],"names":[],"mappings":"AAAA,OAAO,EAIL,KAAK,cAAc,EACnB,KAAK,aAAa,EAEnB,MAAM,iDAAiD,CAAA;AAiBxD,MAAM,WAAW,MAAM;IACrB,KAAK,CAAC,OAAO,EAAE,MAAM,EAAE,GAAG,IAAI,EAAE,OAAO,EAAE,GAAG,IAAI,CAAA;IAChD,IAAI,CAAC,OAAO,EAAE,MAAM,EAAE,GAAG,IAAI,EAAE,OAAO,EAAE,GAAG,IAAI,CAAA;IAC/C,IAAI,CAAC,OAAO,EAAE,MAAM,EAAE,GAAG,IAAI,EAAE,OAAO,EAAE,GAAG,IAAI,CAAA;IAC/C,KAAK,CAAC,OAAO,EAAE,MAAM,GAAG,KAAK,EAAE,GAAG,IAAI,EAAE,OAAO,EAAE,GAAG,IAAI,CAAA;CACzD;AAED,UAAU,eAAe;IACvB,cAAc,EAAE,cAAc,CAAA;IAC9B,YAAY,EAAE,oBAAoB,CAAA;IAClC,MAAM,CAAC,EAAE,MAAM,CAAA;IACf,QAAQ,EAAE,gBAAgB,CAAA;IAC1B,UAAU,EAAE;QACV,MAAM,EAAE,MAAM,CAAA;QACd,QAAQ,EAAE,MAAM,CAAA;QAChB,QAAQ,EAAE,MAAM,CAAA;QAChB,UAAU,EAAE,MAAM,EAAE,CAAA;KACrB,CAAA;IACD,MAAM,EAAE;QACN,IAAI,EAAE,MAAM,CAAA;QACZ,SAAS,EAAE,MAAM,CAAA;QACjB,kBAAkB,EAAE,MAAM,CAAA;QAC1B,qBAAqB,EAAE,MAAM,CAAA;KAC9B,CAAA;IACD,SAAS,CAAC,EAAE;QACV,WAAW,EAAE,MAAM,CAAA;QACnB,QAAQ,EAAE,CAAC,MAAM,EAAE,aAAa,EAAE,KAAK,OAAO,CAAC,OAAO,CAAC,GAAG,OAAO,CAAA;QACjE,cAAc,CAAC,EAAE,CAAC,MAAM,EAAE,aAAa,EAAE,KAAK,OAAO,CAAC,IAAI,CAAC,GAAG,IAAI,CAAA;QAClE,MAAM,CAAC,EAAE,MAAM,OAAO,CAAC,IAAI,CAAC,GAAG,IAAI,CAAA;KACpC,CAAA;IACD,MAAM,CAAC,EAAE,IAAI,CAAA;CACd;AAED,MAAM,WAAW,aAAa;IAC5B,UAAU,EAAE,MAAM,CAAA;IAClB,OAAO,CAAC,EAAE,MAAM,CAAA;CACjB;AAED,MAAM,WAAW,oBAAoB;IACnC,QAAQ,EAAE,MAAM,OAAO,CAAC,aAAa,GAAG,IAAI,CAAC,GAAG,aAAa,GAAG,IAAI,CAAA;IACpE,QAAQ,CAAC,EAAE,CAAC,KAAK,EAAE,aAAa,KAAK,OAAO,CAAC,IAAI,CAAC,GAAG,IAAI,CAAA;IACzD,MAAM,CAAC,EAAE,MAAM,IAAI,CAAA;CACpB;AAED,MAAM,WAAW,gBAAgB;IAC/B,IAAI,CAAC,UAAU,EAAE,eAAe,CAAC,YAAY,CAAC,GAAG;QAAE,UAAU,EAAE,MAAM,CAAA;KAAE,EAAE,MAAM,CAAC,EAAE,WAAW,GAAG,OAAO,CAAC,IAAI,CAAC,CAAA;CAC9G;AAED,qBAAa,QAAQ;IAgBP,OAAO,CAAC,OAAO;IAf3B,OAAO,CAAC,MAAM,CAAC,CAAQ;IACvB,OAAO,CAAC,KAAK,CAAe;IAC5B,OAAO,CAAC,YAAY,CAAe;IACnC,OAAO,CAAC,MAAM,CAA4B;IAC1C,OAAO,CAAC,YAAY,CAA8C;IAClE,OAAO,CAAC,MAAM,CAA8C;IAC5D,OAAO,CAAC,OAAO,CAAQ;IACvB,OAAO,CAAC,uBAAuB,CAAC,CAAiB;IACjD,OAAO,CAAC,UAAU,CAAC,CAAQ;IAC3B,OAAO,CAAC,MAAM,CAAC,CAId;gBAEmB,OAAO,EAAE,eAAe;IAO/B,KAAK;IAiDX,IAAI;YAWG,SAAS;YAqBT,eAAe;YAqGf,MAAM;IAsCpB,OAAO,CAAC,WAAW;IAcnB,OAAO,KAAK,WAAW,GAMtB;IAED,OAAO,CAAC,oBAAoB;YAgBd,iBAAiB;YAoBjB,SAAS;YAsCT,gBAAgB;YA2BhB,eAAe;YAUf,aAAa;YAab,sBAAsB;IAepC,OAAO,CAAC,gBAAgB;YAIV,aAAa;IAmBpB,IAAI,CAAC,MAAM,EAAE,MAAM,GAAG,OAAO,CAAC,aAAa,EAAE,CAAC;IAO9C,WAAW,CAAC,QAAQ,EAAE,MAAM,EAAE,GAAG,OAAO,CAAC,IAAI,CAAC;CAMtD"}

package/esm/lib/data-pump.js

@@ -69,6 +69,12 @@ export class DataPump {
             writable: true,
             value: void 0
         });
+        Object.defineProperty(this, "stopAt", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: void 0
+        });
         this.state = {
             timeBucket: this.getNowTimeBucket(),
         };
@@ -79,6 +85,17 @@ export class DataPump {
             return;
         }
         await this.updateTimeBuckets();
+        if (this.options.stopAt) {
+            const timeBucket = format(startOfHour(utc(this.options.stopAt)), "yyyyMMddHH0000");
+            const lastTimeBucket = this.getClosestTimeBucket(timeBucket, true);
+            const lastEventId = TimeUuid.fromDate(this.options.stopAt).toString();
+            this.stopAt = {
+                timeBucket: lastTimeBucket,
+                eventId: lastEventId,
+                eventBufferReached: false,
+            };
+            console.log("stop at", this.stopAt);
+        }
         const newState = await this.options.stateManager.getState();
         if (!newState) {
             this.state.timeBucket = this.getClosestTimeBucket(this.getNowTimeBucket());
@@ -88,6 +105,9 @@ export class DataPump {
             this.state.timeBucket = this.getClosestTimeBucket(newState.timeBucket);
             this.state.eventId = newState.eventId;
         }
+        if (this.stopAt?.timeBucket && this.state.timeBucket > this.stopAt.timeBucket) {
+            throw new Error("Stop at time is lower than current state");
+        }
         this.running = true;
         this.logger?.debug("Starting event buffer loop");
         this.eventBufferLoop()
@@ -138,6 +158,10 @@ export class DataPump {
         if (!this.running) {
             return;
         }
+        if (this.stopAt?.eventBufferReached) {
+            await new Promise((resolve) => setTimeout(resolve, 10_000));
+            return this.eventBufferLoop();
+        }
         // Buffer is full, wait for some space
         if (this.buffer.length >= this.options.buffer.size - this.options.buffer.threshold && this.running) {
             this.logger?.debug("Buffer is full, waiting for some space");
@@ -159,6 +183,7 @@ export class DataPump {
             timeBucket: this.state.timeBucket,
             afterEventId: this.state.eventId,
             pageSize: eventsToFetch,
+            toEventId: this.stopAt?.eventId,
         });
         const result = await this.options.flowcoreClient.execute(command);
         const time = Date.now() - timeStart;
@@ -181,6 +206,12 @@ export class DataPump {
         this.state.eventId = result.events[result.events.length - 1]?.eventId ?? this.state.eventId;
         // If we don't have a next cursor move to next time bucket
         if (!result.nextCursor) {
+            // Check if we reached the stop at
+            if (this.state.timeBucket === this.stopAt?.timeBucket) {
+                this.stopAt.eventBufferReached = true;
+                this.logger?.debug("Reached stop at");
+                return this.eventBufferLoop();
+            }
             // Get next time bucket
             const nextTimeBucketIndex = this.timeBuckets.indexOf(this.state.timeBucket) + 1;
             // There is no next time bucket
@@ -238,13 +269,13 @@ export class DataPump {
         if (!firstEvent) {
             if (eventId) {
                 const timeUuid = TimeUuid.fromString(eventId);
-                return this.options.stateManager.setState({ timeBucket: timeUuid.getTimeBucket(), eventId });
+                return this.options.stateManager.setState?.({ timeBucket: timeUuid.getTimeBucket(), eventId });
             }
             return;
         }
         const timeUuid = TimeUuid.fromString(firstEvent.event.eventId);
         const timeBucket = firstEvent.event.timeBucket;
-        return this.options.stateManager.setState({ timeBucket, eventId: timeUuid.getBefore().toString() });
+        return this.options.stateManager.setState?.({ timeBucket, eventId: timeUuid.getBefore().toString() });
     }
     get timeBuckets() {
         const timeBucketNow = this.getNowTimeBucket();
@@ -253,10 +284,14 @@ export class DataPump {
         }
         return this._timeBuckets;
     }
-    getClosestTimeBucket(timeBucket) {
+    getClosestTimeBucket(timeBucket, getBefore = false) {
         if (!timeBucket.match(/^\d{14}$/)) {
             throw new Error(`Invalid timebucket: ${timeBucket}`);
         }
+        if (getBefore) {
+            return (this.timeBuckets.findLast((t) => Number.parseFloat(t) <= Number.parseFloat(timeBucket)) ??
+                this.timeBuckets[this.timeBuckets.length - 1]);
+        }
         return (this.timeBuckets.find((t) => Number.parseFloat(t) >= Number.parseFloat(timeBucket)) ??
             this.timeBuckets[this.timeBuckets.length - 1]);
     }
@@ -329,6 +364,10 @@ export class DataPump {
         else {
             await this.updateState();
         }
+        if (!this.buffer.length && this.stopAt?.eventBufferReached) {
+            await this.options.processor?.onDone?.();
+            await this.stop();
+        }
     }
     async waitForNotifier() {
         this.logger?.debug("No more events, waiting for notifier...");

package/esm/lib/time-uuid.d.ts

@@ -10,5 +10,6 @@ export declare class TimeUuid {
     static fromNow(): TimeUuid;
     toString(): string;
     getTimeBucket(): string;
+    getDate(): Date;
 }
 //# sourceMappingURL=time-uuid.d.ts.map

package/esm/lib/time-uuid.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"time-uuid.d.ts","sourceRoot":"","sources":["../../src/lib/time-uuid.ts"],"names":[],"mappings":"AAEA,OAAO,KAAK,EAAE,MAAM,EAAE,MAAM,aAAa,CAAA;AACzC,OAAO,EAAE,KAAK,EAAE,MAAM,kBAAkB,CAAA;AAGxC,qBAAa,QAAQ;IACP,OAAO,CAAC,QAAQ,CAAC,QAAQ;gBAAR,QAAQ,EAAE,KAAK,CAAC,QAAQ;WAEvC,QAAQ,CAAC,IAAI,EAAE,IAAI,EAAE,KAAK,CAAC,EAAE,MAAM,EAAE,MAAM,CAAC,EAAE,MAAM,GAAG,MAAM,EAAE,OAAO,CAAC,EAAE,MAAM,GAAG,MAAM,GAAG,QAAQ;WAInG,UAAU,CAAC,KAAK,EAAE,MAAM,GAAG,QAAQ;IAIjD,OAAO,CAAC,gBAAgB;IAIjB,SAAS,IAAI,QAAQ;WASd,OAAO,IAAI,QAAQ;IAI1B,QAAQ,IAAI,MAAM;IAIlB,aAAa,IAAI,MAAM;CAG/B"}
+{"version":3,"file":"time-uuid.d.ts","sourceRoot":"","sources":["../../src/lib/time-uuid.ts"],"names":[],"mappings":"AAEA,OAAO,KAAK,EAAE,MAAM,EAAE,MAAM,aAAa,CAAA;AACzC,OAAO,EAAE,KAAK,EAAE,MAAM,kBAAkB,CAAA;AAGxC,qBAAa,QAAQ;IACP,OAAO,CAAC,QAAQ,CAAC,QAAQ;gBAAR,QAAQ,EAAE,KAAK,CAAC,QAAQ;WAEvC,QAAQ,CAAC,IAAI,EAAE,IAAI,EAAE,KAAK,CAAC,EAAE,MAAM,EAAE,MAAM,CAAC,EAAE,MAAM,GAAG,MAAM,EAAE,OAAO,CAAC,EAAE,MAAM,GAAG,MAAM,GAAG,QAAQ;WAInG,UAAU,CAAC,KAAK,EAAE,MAAM,GAAG,QAAQ;IAIjD,OAAO,CAAC,gBAAgB;IAIjB,SAAS,IAAI,QAAQ;WASd,OAAO,IAAI,QAAQ;IAI1B,QAAQ,IAAI,MAAM;IAIlB,aAAa,IAAI,MAAM;IAIvB,OAAO,IAAI,IAAI;CAGvB"}

package/esm/lib/time-uuid.js

@@ -37,4 +37,7 @@ export class TimeUuid {
     getTimeBucket() {
         return format(startOfHour(this.timeUuid.getDate()), "yyyyMMddHH0000");
     }
+    getDate() {
+        return this.timeUuid.getDate();
+    }
 }

package/esm/mod.d.ts

@@ -1,3 +1,4 @@
+import "./_dnt.polyfills.js";
 export * from "./lib/data-pump.js";
 export * from "./lib/data-pump-create.js";
 export * from "./lib/time-uuid.js";

package/esm/mod.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"mod.d.ts","sourceRoot":"","sources":["../src/mod.ts"],"names":[],"mappings":"AAAA,cAAc,oBAAoB,CAAA;AAClC,cAAc,2BAA2B,CAAA;AAEzC,cAAc,oBAAoB,CAAA"}
+{"version":3,"file":"mod.d.ts","sourceRoot":"","sources":["../src/mod.ts"],"names":[],"mappings":"AAAA,OAAO,qBAAqB,CAAC;AAC7B,cAAc,oBAAoB,CAAA;AAClC,cAAc,2BAA2B,CAAA;AAEzC,cAAc,oBAAoB,CAAA"}

package/esm/mod.js

@@ -1,3 +1,4 @@
+import "./_dnt.polyfills.js";
 export * from "./lib/data-pump.js";
 export * from "./lib/data-pump-create.js";
 export * from "./lib/time-uuid.js";

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@flowcore/data-pump",
-  "version": "0.1.8",
+  "version": "0.1.10",
   "description": "Flowcore Data Pump",
   "homepage": "https://github.com/flowcore-io/flowcore-data-pump#readme",
   "repository": {

package/script/_dnt.polyfills.d.ts

@@ -0,0 +1,50 @@
+declare global {
+    interface Array<T> {
+        /**
+         * Returns the value of the last element in the array where predicate is true, and undefined
+         * otherwise.
+         * @param predicate find calls predicate once for each element of the array, in ascending
+         * order, until it finds one where predicate returns true. If such an element is found, find
+         * immediately returns that element value. Otherwise, find returns undefined.
+         * @param thisArg If provided, it will be used as the this value for each invocation of
+         * predicate. If it is not provided, undefined is used instead.
+         */
+        findLast<S extends T>(predicate: (this: void, value: T, index: number, obj: T[]) => value is S, thisArg?: any): S | undefined;
+        findLast(predicate: (value: T, index: number, obj: T[]) => unknown, thisArg?: any): T | undefined;
+        /**
+         * Returns the index of the last element in the array where predicate is true, and -1
+         * otherwise.
+         * @param predicate find calls predicate once for each element of the array, in ascending
+         * order, until it finds one where predicate returns true. If such an element is found,
+         * findIndex immediately returns that element index. Otherwise, findIndex returns -1.
+         * @param thisArg If provided, it will be used as the this value for each invocation of
+         * predicate. If it is not provided, undefined is used instead.
+         */
+        findLastIndex(predicate: (value: T, index: number, obj: T[]) => unknown, thisArg?: any): number;
+    }
+    interface Uint8Array {
+        /**
+         * Returns the value of the last element in the array where predicate is true, and undefined
+         * otherwise.
+         * @param predicate findLast calls predicate once for each element of the array, in descending
+         * order, until it finds one where predicate returns true. If such an element is found, findLast
+         * immediately returns that element value. Otherwise, findLast returns undefined.
+         * @param thisArg If provided, it will be used as the this value for each invocation of
+         * predicate. If it is not provided, undefined is used instead.
+         */
+        findLast<S extends number>(predicate: (value: number, index: number, array: Uint8Array) => value is S, thisArg?: any): S | undefined;
+        findLast(predicate: (value: number, index: number, array: Uint8Array) => unknown, thisArg?: any): number | undefined;
+        /**
+         * Returns the index of the last element in the array where predicate is true, and -1
+         * otherwise.
+         * @param predicate findLastIndex calls predicate once for each element of the array, in descending
+         * order, until it finds one where predicate returns true. If such an element is found,
+         * findLastIndex immediately returns that element index. Otherwise, findLastIndex returns -1.
+         * @param thisArg If provided, it will be used as the this value for each invocation of
+         * predicate. If it is not provided, undefined is used instead.
+         */
+        findLastIndex(predicate: (value: number, index: number, array: Uint8Array) => unknown, thisArg?: any): number;
+    }
+}
+export {};
+//# sourceMappingURL=_dnt.polyfills.d.ts.map

package/script/_dnt.polyfills.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"_dnt.polyfills.d.ts","sourceRoot":"","sources":["../src/_dnt.polyfills.ts"],"names":[],"mappings":"AACA,OAAO,CAAC,MAAM,CAAC;IACb,UAAU,KAAK,CAAC,CAAC;QACf;;;;;;;;WAQG;QACH,QAAQ,CAAC,CAAC,SAAS,CAAC,EAClB,SAAS,EAAE,CAAC,IAAI,EAAE,IAAI,EAAE,KAAK,EAAE,CAAC,EAAE,KAAK,EAAE,MAAM,EAAE,GAAG,EAAE,CAAC,EAAE,KAAK,KAAK,IAAI,CAAC,EACxE,OAAO,CAAC,EAAE,GAAG,GACZ,CAAC,GAAG,SAAS,CAAC;QACjB,QAAQ,CACN,SAAS,EAAE,CAAC,KAAK,EAAE,CAAC,EAAE,KAAK,EAAE,MAAM,EAAE,GAAG,EAAE,CAAC,EAAE,KAAK,OAAO,EACzD,OAAO,CAAC,EAAE,GAAG,GACZ,CAAC,GAAG,SAAS,CAAC;QAEjB;;;;;;;;WAQG;QACH,aAAa,CACX,SAAS,EAAE,CAAC,KAAK,EAAE,CAAC,EAAE,KAAK,EAAE,MAAM,EAAE,GAAG,EAAE,CAAC,EAAE,KAAK,OAAO,EACzD,OAAO,CAAC,EAAE,GAAG,GACZ,MAAM,CAAC;KACX;IACD,UAAU,UAAU;QAClB;;;;;;;;WAQG;QACH,QAAQ,CAAC,CAAC,SAAS,MAAM,EACvB,SAAS,EAAE,CACP,KAAK,EAAE,MAAM,EACb,KAAK,EAAE,MAAM,EACb,KAAK,EAAE,UAAU,KAChB,KAAK,IAAI,CAAC,EACf,OAAO,CAAC,EAAE,GAAG,GACZ,CAAC,GAAG,SAAS,CAAC;QACjB,QAAQ,CACJ,SAAS,EAAE,CAAC,KAAK,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,EAAE,KAAK,EAAE,UAAU,KAAK,OAAO,EACvE,OAAO,CAAC,EAAE,GAAG,GACd,MAAM,GAAG,SAAS,CAAC;QAEtB;;;;;;;;WAQG;QACH,aAAa,CACT,SAAS,EAAE,CAAC,KAAK,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,EAAE,KAAK,EAAE,UAAU,KAAK,OAAO,EACvE,OAAO,CAAC,EAAE,GAAG,GACd,MAAM,CAAC;KACX;CACF;AA4CD,OAAO,EAAE,CAAC"}

package/script/_dnt.polyfills.js

@@ -0,0 +1,38 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+function findLastIndex(self, callbackfn, that) {
+    const boundFunc = that === undefined ? callbackfn : callbackfn.bind(that);
+    let index = self.length - 1;
+    while (index >= 0) {
+        const result = boundFunc(self[index], index, self);
+        if (result) {
+            return index;
+        }
+        index--;
+    }
+    return -1;
+}
+function findLast(self, callbackfn, that) {
+    const index = self.findLastIndex(callbackfn, that);
+    return index === -1 ? undefined : self[index];
+}
+if (!Array.prototype.findLastIndex) {
+    Array.prototype.findLastIndex = function (callbackfn, that) {
+        return findLastIndex(this, callbackfn, that);
+    };
+}
+if (!Array.prototype.findLast) {
+    Array.prototype.findLast = function (callbackfn, that) {
+        return findLast(this, callbackfn, that);
+    };
+}
+if (!Uint8Array.prototype.findLastIndex) {
+    Uint8Array.prototype.findLastIndex = function (callbackfn, that) {
+        return findLastIndex(this, callbackfn, that);
+    };
+}
+if (!Uint8Array.prototype.findLast) {
+    Uint8Array.prototype.findLast = function (callbackfn, that) {
+        return findLast(this, callbackfn, that);
+    };
+}

package/script/lib/data-pump-create.d.ts

@@ -36,7 +36,9 @@ export interface DataPumpClientOptions {
         concurrency?: number;
         onEvents: (events: FlowcoreEvent[]) => Promise<boolean> | boolean;
         onFailedEvents?: (events: FlowcoreEvent[]) => Promise<void> | void;
+        onDone?: () => Promise<void> | void;
     };
+    stopAt?: Date;
     natsServers?: string[];
 }
 export declare function createDataPump(options: DataPumpClientOptions): DataPump;

package/script/lib/data-pump-create.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"data-pump-create.d.ts","sourceRoot":"","sources":["../../src/lib/data-pump-create.ts"],"names":[],"mappings":"AAAA,OAAO,EAAkB,KAAK,aAAa,EAAE,MAAM,iDAAiD,CAAA;AACpG,OAAO,EAAE,QAAQ,EAAE,KAAK,oBAAoB,EAAE,KAAK,MAAM,EAAE,MAAM,gBAAgB,CAAA;AAIjF,eAAO,MAAM,UAAU,EAAE,MAKxB,CAAA;AAED,UAAU,mCAAmC;IAC3C,UAAU,EAAE;QACV,QAAQ,EAAE,MAAM,OAAO,CAAC;YAAE,WAAW,EAAE,MAAM,CAAA;SAAE,CAAC,CAAA;KACjD,CAAA;CACF;AAED,UAAU,+BAA+B;IACvC,MAAM,EAAE,MAAM,CAAA;IACd,QAAQ,EAAE,MAAM,CAAA;CACjB;AAED,KAAK,yBAAyB,GAAG,mCAAmC,GAAG,+BAA+B,CAAA;AAEtG;;GAEG;AACH,MAAM,WAAW,qBAAqB;IACpC,MAAM,CAAC,EAAE,MAAM,CAAA;IACf,IAAI,EAAE,yBAAyB,CAAA;IAC/B,UAAU,EAAE;QACV,MAAM,EAAE,MAAM,CAAA;QACd,QAAQ,EAAE,MAAM,CAAA;QAChB,QAAQ,EAAE,MAAM,CAAA;QAChB,UAAU,EAAE,MAAM,EAAE,CAAA;KACrB,CAAA;IACD,MAAM,CAAC,EAAE;QACP,IAAI,CAAC,EAAE,MAAM,CAAA;QACb,SAAS,CAAC,EAAE,MAAM,CAAA;QAClB,kBAAkB,CAAC,EAAE,MAAM,CAAA;QAC3B,qBAAqB,CAAC,EAAE,MAAM,CAAA;KAC/B,CAAA;IACD,YAAY,CAAC,EAAE,oBAAoB,CAAA;IACnC,SAAS,CAAC,EAAE;QACV,WAAW,CAAC,EAAE,MAAM,CAAA;QACpB,QAAQ,EAAE,CAAC,MAAM,EAAE,aAAa,EAAE,KAAK,OAAO,CAAC,OAAO,CAAC,GAAG,OAAO,CAAA;QACjE,cAAc,CAAC,EAAE,CAAC,MAAM,EAAE,aAAa,EAAE,KAAK,OAAO,CAAC,IAAI,CAAC,GAAG,IAAI,CAAA;KACnE,CAAA;IACD,WAAW,CAAC,EAAE,MAAM,EAAE,CAAA;CACvB;AAED,wBAAgB,cAAc,CAAC,OAAO,EAAE,qBAAqB,GAAG,QAAQ,CAwDvE"}
+{"version":3,"file":"data-pump-create.d.ts","sourceRoot":"","sources":["../../src/lib/data-pump-create.ts"],"names":[],"mappings":"AAAA,OAAO,EAAkB,KAAK,aAAa,EAAE,MAAM,iDAAiD,CAAA;AACpG,OAAO,EAAE,QAAQ,EAAE,KAAK,oBAAoB,EAAE,KAAK,MAAM,EAAE,MAAM,gBAAgB,CAAA;AAIjF,eAAO,MAAM,UAAU,EAAE,MAKxB,CAAA;AAED,UAAU,mCAAmC;IAC3C,UAAU,EAAE;QACV,QAAQ,EAAE,MAAM,OAAO,CAAC;YAAE,WAAW,EAAE,MAAM,CAAA;SAAE,CAAC,CAAA;KACjD,CAAA;CACF;AAED,UAAU,+BAA+B;IACvC,MAAM,EAAE,MAAM,CAAA;IACd,QAAQ,EAAE,MAAM,CAAA;CACjB;AAED,KAAK,yBAAyB,GAAG,mCAAmC,GAAG,+BAA+B,CAAA;AAEtG;;GAEG;AACH,MAAM,WAAW,qBAAqB;IACpC,MAAM,CAAC,EAAE,MAAM,CAAA;IACf,IAAI,EAAE,yBAAyB,CAAA;IAC/B,UAAU,EAAE;QACV,MAAM,EAAE,MAAM,CAAA;QACd,QAAQ,EAAE,MAAM,CAAA;QAChB,QAAQ,EAAE,MAAM,CAAA;QAChB,UAAU,EAAE,MAAM,EAAE,CAAA;KACrB,CAAA;IACD,MAAM,CAAC,EAAE;QACP,IAAI,CAAC,EAAE,MAAM,CAAA;QACb,SAAS,CAAC,EAAE,MAAM,CAAA;QAClB,kBAAkB,CAAC,EAAE,MAAM,CAAA;QAC3B,qBAAqB,CAAC,EAAE,MAAM,CAAA;KAC/B,CAAA;IACD,YAAY,CAAC,EAAE,oBAAoB,CAAA;IACnC,SAAS,CAAC,EAAE;QACV,WAAW,CAAC,EAAE,MAAM,CAAA;QACpB,QAAQ,EAAE,CAAC,MAAM,EAAE,aAAa,EAAE,KAAK,OAAO,CAAC,OAAO,CAAC,GAAG,OAAO,CAAA;QACjE,cAAc,CAAC,EAAE,CAAC,MAAM,EAAE,aAAa,EAAE,KAAK,OAAO,CAAC,IAAI,CAAC,GAAG,IAAI,CAAA;QAClE,MAAM,CAAC,EAAE,MAAM,OAAO,CAAC,IAAI,CAAC,GAAG,IAAI,CAAA;KACpC,CAAA;IACD,MAAM,CAAC,EAAE,IAAI,CAAA;IACb,WAAW,CAAC,EAAE,MAAM,EAAE,CAAA;CACvB;AAED,wBAAgB,cAAc,CAAC,OAAO,EAAE,qBAAqB,GAAG,QAAQ,CA0DvE"}

package/script/lib/data-pump-create.js

@@ -51,6 +51,7 @@ function createDataPump(options) {
             concurrency: options.processor.concurrency ?? 1,
             onEvents: options.processor.onEvents,
             onFailedEvents: options.processor.onFailedEvents,
+            onDone: options.processor.onDone,
         }
         : undefined;
     const dataPump = new data_pump_js_1.DataPump({
@@ -61,6 +62,7 @@ function createDataPump(options) {
         processor,
         notifier: notifier.getNotifier(),
         logger: options.logger,
+        stopAt: options.stopAt,
     });
     return dataPump;
 }

package/script/lib/data-pump.d.ts

@@ -26,15 +26,18 @@ interface DataPumpOptions {
         concurrency: number;
         onEvents: (events: FlowcoreEvent[]) => Promise<boolean> | boolean;
         onFailedEvents?: (events: FlowcoreEvent[]) => Promise<void> | void;
+        onDone?: () => Promise<void> | void;
     };
+    stopAt?: Date;
 }
 export interface DataPumpState {
     timeBucket: string;
     eventId?: string;
 }
 export interface DataPumpStateManager {
-    getState(): Promise<DataPumpState | null> | DataPumpState | null;
-    setState(state: DataPumpState): Promise<void> | void;
+    getState: () => Promise<DataPumpState | null> | DataPumpState | null;
+    setState?: (state: DataPumpState) => Promise<void> | void;
+    stopAt?: () => Date;
 }
 export interface DataPumpNotifier {
     wait(dataSource: DataPumpOptions["dataSource"] & {
@@ -52,6 +55,7 @@ export declare class DataPump {
     private running;
     private notifierAbortController?;
     private dataCoreId?;
+    private stopAt?;
     constructor(options: DataPumpOptions);
     start(): Promise<void>;
     stop(): void;

package/script/lib/data-pump.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"data-pump.d.ts","sourceRoot":"","sources":["../../src/lib/data-pump.ts"],"names":[],"mappings":"AAAA,OAAO,EAIL,KAAK,cAAc,EACnB,KAAK,aAAa,EAEnB,MAAM,iDAAiD,CAAA;AAiBxD,MAAM,WAAW,MAAM;IACrB,KAAK,CAAC,OAAO,EAAE,MAAM,EAAE,GAAG,IAAI,EAAE,OAAO,EAAE,GAAG,IAAI,CAAA;IAChD,IAAI,CAAC,OAAO,EAAE,MAAM,EAAE,GAAG,IAAI,EAAE,OAAO,EAAE,GAAG,IAAI,CAAA;IAC/C,IAAI,CAAC,OAAO,EAAE,MAAM,EAAE,GAAG,IAAI,EAAE,OAAO,EAAE,GAAG,IAAI,CAAA;IAC/C,KAAK,CAAC,OAAO,EAAE,MAAM,GAAG,KAAK,EAAE,GAAG,IAAI,EAAE,OAAO,EAAE,GAAG,IAAI,CAAA;CACzD;AAED,UAAU,eAAe;IACvB,cAAc,EAAE,cAAc,CAAA;IAC9B,YAAY,EAAE,oBAAoB,CAAA;IAClC,MAAM,CAAC,EAAE,MAAM,CAAA;IACf,QAAQ,EAAE,gBAAgB,CAAA;IAC1B,UAAU,EAAE;QACV,MAAM,EAAE,MAAM,CAAA;QACd,QAAQ,EAAE,MAAM,CAAA;QAChB,QAAQ,EAAE,MAAM,CAAA;QAChB,UAAU,EAAE,MAAM,EAAE,CAAA;KACrB,CAAA;IACD,MAAM,EAAE;QACN,IAAI,EAAE,MAAM,CAAA;QACZ,SAAS,EAAE,MAAM,CAAA;QACjB,kBAAkB,EAAE,MAAM,CAAA;QAC1B,qBAAqB,EAAE,MAAM,CAAA;KAC9B,CAAA;IACD,SAAS,CAAC,EAAE;QACV,WAAW,EAAE,MAAM,CAAA;QACnB,QAAQ,EAAE,CAAC,MAAM,EAAE,aAAa,EAAE,KAAK,OAAO,CAAC,OAAO,CAAC,GAAG,OAAO,CAAA;QACjE,cAAc,CAAC,EAAE,CAAC,MAAM,EAAE,aAAa,EAAE,KAAK,OAAO,CAAC,IAAI,CAAC,GAAG,IAAI,CAAA;KACnE,CAAA;CACF;AAED,MAAM,WAAW,aAAa;IAC5B,UAAU,EAAE,MAAM,CAAA;IAClB,OAAO,CAAC,EAAE,MAAM,CAAA;CACjB;AAED,MAAM,WAAW,oBAAoB;IACnC,QAAQ,IAAI,OAAO,CAAC,aAAa,GAAG,IAAI,CAAC,GAAG,aAAa,GAAG,IAAI,CAAA;IAChE,QAAQ,CAAC,KAAK,EAAE,aAAa,GAAG,OAAO,CAAC,IAAI,CAAC,GAAG,IAAI,CAAA;CACrD;AAED,MAAM,WAAW,gBAAgB;IAC/B,IAAI,CAAC,UAAU,EAAE,eAAe,CAAC,YAAY,CAAC,GAAG;QAAE,UAAU,EAAE,MAAM,CAAA;KAAE,EAAE,MAAM,CAAC,EAAE,WAAW,GAAG,OAAO,CAAC,IAAI,CAAC,CAAA;CAC9G;AAED,qBAAa,QAAQ;IAWP,OAAO,CAAC,OAAO;IAV3B,OAAO,CAAC,MAAM,CAAC,CAAQ;IACvB,OAAO,CAAC,KAAK,CAAe;IAC5B,OAAO,CAAC,YAAY,CAAe;IACnC,OAAO,CAAC,MAAM,CAA4B;IAC1C,OAAO,CAAC,YAAY,CAA8C;IAClE,OAAO,CAAC,MAAM,CAA8C;IAC5D,OAAO,CAAC,OAAO,CAAQ;IACvB,OAAO,CAAC,uBAAuB,CAAC,CAAiB;IACjD,OAAO,CAAC,UAAU,CAAC,CAAQ;gBAEP,OAAO,EAAE,eAAe;IAO/B,KAAK;IAiCX,IAAI;YAWG,SAAS;YAqBT,eAAe;YAwFf,MAAM;IAsCpB,OAAO,CAAC,WAAW;IAcnB,OAAO,KAAK,WAAW,GAMtB;IAED,OAAO,CAAC,oBAAoB;YAUd,iBAAiB;YAoBjB,SAAS;YAsCT,gBAAgB;YAsBhB,eAAe;YAUf,aAAa;YAab,sBAAsB;IAepC,OAAO,CAAC,gBAAgB;YAIV,aAAa;IAmBpB,IAAI,CAAC,MAAM,EAAE,MAAM,GAAG,OAAO,CAAC,aAAa,EAAE,CAAC;IAO9C,WAAW,CAAC,QAAQ,EAAE,MAAM,EAAE,GAAG,OAAO,CAAC,IAAI,CAAC;CAMtD"}
+{"version":3,"file":"data-pump.d.ts","sourceRoot":"","sources":["../../src/lib/data-pump.ts"],"names":[],"mappings":"AAAA,OAAO,EAIL,KAAK,cAAc,EACnB,KAAK,aAAa,EAEnB,MAAM,iDAAiD,CAAA;AAiBxD,MAAM,WAAW,MAAM;IACrB,KAAK,CAAC,OAAO,EAAE,MAAM,EAAE,GAAG,IAAI,EAAE,OAAO,EAAE,GAAG,IAAI,CAAA;IAChD,IAAI,CAAC,OAAO,EAAE,MAAM,EAAE,GAAG,IAAI,EAAE,OAAO,EAAE,GAAG,IAAI,CAAA;IAC/C,IAAI,CAAC,OAAO,EAAE,MAAM,EAAE,GAAG,IAAI,EAAE,OAAO,EAAE,GAAG,IAAI,CAAA;IAC/C,KAAK,CAAC,OAAO,EAAE,MAAM,GAAG,KAAK,EAAE,GAAG,IAAI,EAAE,OAAO,EAAE,GAAG,IAAI,CAAA;CACzD;AAED,UAAU,eAAe;IACvB,cAAc,EAAE,cAAc,CAAA;IAC9B,YAAY,EAAE,oBAAoB,CAAA;IAClC,MAAM,CAAC,EAAE,MAAM,CAAA;IACf,QAAQ,EAAE,gBAAgB,CAAA;IAC1B,UAAU,EAAE;QACV,MAAM,EAAE,MAAM,CAAA;QACd,QAAQ,EAAE,MAAM,CAAA;QAChB,QAAQ,EAAE,MAAM,CAAA;QAChB,UAAU,EAAE,MAAM,EAAE,CAAA;KACrB,CAAA;IACD,MAAM,EAAE;QACN,IAAI,EAAE,MAAM,CAAA;QACZ,SAAS,EAAE,MAAM,CAAA;QACjB,kBAAkB,EAAE,MAAM,CAAA;QAC1B,qBAAqB,EAAE,MAAM,CAAA;KAC9B,CAAA;IACD,SAAS,CAAC,EAAE;QACV,WAAW,EAAE,MAAM,CAAA;QACnB,QAAQ,EAAE,CAAC,MAAM,EAAE,aAAa,EAAE,KAAK,OAAO,CAAC,OAAO,CAAC,GAAG,OAAO,CAAA;QACjE,cAAc,CAAC,EAAE,CAAC,MAAM,EAAE,aAAa,EAAE,KAAK,OAAO,CAAC,IAAI,CAAC,GAAG,IAAI,CAAA;QAClE,MAAM,CAAC,EAAE,MAAM,OAAO,CAAC,IAAI,CAAC,GAAG,IAAI,CAAA;KACpC,CAAA;IACD,MAAM,CAAC,EAAE,IAAI,CAAA;CACd;AAED,MAAM,WAAW,aAAa;IAC5B,UAAU,EAAE,MAAM,CAAA;IAClB,OAAO,CAAC,EAAE,MAAM,CAAA;CACjB;AAED,MAAM,WAAW,oBAAoB;IACnC,QAAQ,EAAE,MAAM,OAAO,CAAC,aAAa,GAAG,IAAI,CAAC,GAAG,aAAa,GAAG,IAAI,CAAA;IACpE,QAAQ,CAAC,EAAE,CAAC,KAAK,EAAE,aAAa,KAAK,OAAO,CAAC,IAAI,CAAC,GAAG,IAAI,CAAA;IACzD,MAAM,CAAC,EAAE,MAAM,IAAI,CAAA;CACpB;AAED,MAAM,WAAW,gBAAgB;IAC/B,IAAI,CAAC,UAAU,EAAE,eAAe,CAAC,YAAY,CAAC,GAAG;QAAE,UAAU,EAAE,MAAM,CAAA;KAAE,EAAE,MAAM,CAAC,EAAE,WAAW,GAAG,OAAO,CAAC,IAAI,CAAC,CAAA;CAC9G;AAED,qBAAa,QAAQ;IAgBP,OAAO,CAAC,OAAO;IAf3B,OAAO,CAAC,MAAM,CAAC,CAAQ;IACvB,OAAO,CAAC,KAAK,CAAe;IAC5B,OAAO,CAAC,YAAY,CAAe;IACnC,OAAO,CAAC,MAAM,CAA4B;IAC1C,OAAO,CAAC,YAAY,CAA8C;IAClE,OAAO,CAAC,MAAM,CAA8C;IAC5D,OAAO,CAAC,OAAO,CAAQ;IACvB,OAAO,CAAC,uBAAuB,CAAC,CAAiB;IACjD,OAAO,CAAC,UAAU,CAAC,CAAQ;IAC3B,OAAO,CAAC,MAAM,CAAC,CAId;gBAEmB,OAAO,EAAE,eAAe;IAO/B,KAAK;IAiDX,IAAI;YAWG,SAAS;YAqBT,eAAe;YAqGf,MAAM;IAsCpB,OAAO,CAAC,WAAW;IAcnB,OAAO,KAAK,WAAW,GAMtB;IAED,OAAO,CAAC,oBAAoB;YAgBd,iBAAiB;YAoBjB,SAAS;YAsCT,gBAAgB;YA2BhB,eAAe;YAUf,aAAa;YAab,sBAAsB;IAepC,OAAO,CAAC,gBAAgB;YAIV,aAAa;IAmBpB,IAAI,CAAC,MAAM,EAAE,MAAM,GAAG,OAAO,CAAC,aAAa,EAAE,CAAC;IAO9C,WAAW,CAAC,QAAQ,EAAE,MAAM,EAAE,GAAG,OAAO,CAAC,IAAI,CAAC;CAMtD"}

package/script/lib/data-pump.js

@@ -72,6 +72,12 @@ class DataPump {
             writable: true,
             value: void 0
         });
+        Object.defineProperty(this, "stopAt", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: void 0
+        });
         this.state = {
             timeBucket: this.getNowTimeBucket(),
         };
@@ -82,6 +88,17 @@ class DataPump {
             return;
         }
         await this.updateTimeBuckets();
+        if (this.options.stopAt) {
+            const timeBucket = (0, date_fns_1.format)((0, date_fns_1.startOfHour)((0, utc_1.utc)(this.options.stopAt)), "yyyyMMddHH0000");
+            const lastTimeBucket = this.getClosestTimeBucket(timeBucket, true);
+            const lastEventId = time_uuid_js_1.TimeUuid.fromDate(this.options.stopAt).toString();
+            this.stopAt = {
+                timeBucket: lastTimeBucket,
+                eventId: lastEventId,
+                eventBufferReached: false,
+            };
+            console.log("stop at", this.stopAt);
+        }
         const newState = await this.options.stateManager.getState();
         if (!newState) {
             this.state.timeBucket = this.getClosestTimeBucket(this.getNowTimeBucket());
@@ -91,6 +108,9 @@ class DataPump {
             this.state.timeBucket = this.getClosestTimeBucket(newState.timeBucket);
             this.state.eventId = newState.eventId;
         }
+        if (this.stopAt?.timeBucket && this.state.timeBucket > this.stopAt.timeBucket) {
+            throw new Error("Stop at time is lower than current state");
+        }
         this.running = true;
         this.logger?.debug("Starting event buffer loop");
         this.eventBufferLoop()
@@ -141,6 +161,10 @@ class DataPump {
         if (!this.running) {
             return;
         }
+        if (this.stopAt?.eventBufferReached) {
+            await new Promise((resolve) => setTimeout(resolve, 10_000));
+            return this.eventBufferLoop();
+        }
         // Buffer is full, wait for some space
         if (this.buffer.length >= this.options.buffer.size - this.options.buffer.threshold && this.running) {
             this.logger?.debug("Buffer is full, waiting for some space");
@@ -162,6 +186,7 @@ class DataPump {
             timeBucket: this.state.timeBucket,
             afterEventId: this.state.eventId,
             pageSize: eventsToFetch,
+            toEventId: this.stopAt?.eventId,
         });
         const result = await this.options.flowcoreClient.execute(command);
         const time = Date.now() - timeStart;
@@ -184,6 +209,12 @@ class DataPump {
         this.state.eventId = result.events[result.events.length - 1]?.eventId ?? this.state.eventId;
         // If we don't have a next cursor move to next time bucket
         if (!result.nextCursor) {
+            // Check if we reached the stop at
+            if (this.state.timeBucket === this.stopAt?.timeBucket) {
+                this.stopAt.eventBufferReached = true;
+                this.logger?.debug("Reached stop at");
+                return this.eventBufferLoop();
+            }
             // Get next time bucket
             const nextTimeBucketIndex = this.timeBuckets.indexOf(this.state.timeBucket) + 1;
             // There is no next time bucket
@@ -241,13 +272,13 @@ class DataPump {
         if (!firstEvent) {
             if (eventId) {
                 const timeUuid = time_uuid_js_1.TimeUuid.fromString(eventId);
-                return this.options.stateManager.setState({ timeBucket: timeUuid.getTimeBucket(), eventId });
+                return this.options.stateManager.setState?.({ timeBucket: timeUuid.getTimeBucket(), eventId });
             }
             return;
         }
         const timeUuid = time_uuid_js_1.TimeUuid.fromString(firstEvent.event.eventId);
         const timeBucket = firstEvent.event.timeBucket;
-        return this.options.stateManager.setState({ timeBucket, eventId: timeUuid.getBefore().toString() });
+        return this.options.stateManager.setState?.({ timeBucket, eventId: timeUuid.getBefore().toString() });
     }
     get timeBuckets() {
         const timeBucketNow = this.getNowTimeBucket();
@@ -256,10 +287,14 @@ class DataPump {
         }
         return this._timeBuckets;
     }
-    getClosestTimeBucket(timeBucket) {
+    getClosestTimeBucket(timeBucket, getBefore = false) {
         if (!timeBucket.match(/^\d{14}$/)) {
             throw new Error(`Invalid timebucket: ${timeBucket}`);
         }
+        if (getBefore) {
+            return (this.timeBuckets.findLast((t) => Number.parseFloat(t) <= Number.parseFloat(timeBucket)) ??
+                this.timeBuckets[this.timeBuckets.length - 1]);
+        }
         return (this.timeBuckets.find((t) => Number.parseFloat(t) >= Number.parseFloat(timeBucket)) ??
             this.timeBuckets[this.timeBuckets.length - 1]);
     }
@@ -332,6 +367,10 @@ class DataPump {
         else {
             await this.updateState();
         }
+        if (!this.buffer.length && this.stopAt?.eventBufferReached) {
+            await this.options.processor?.onDone?.();
+            await this.stop();
+        }
     }
     async waitForNotifier() {
         this.logger?.debug("No more events, waiting for notifier...");

package/script/lib/time-uuid.d.ts

@@ -10,5 +10,6 @@ export declare class TimeUuid {
     static fromNow(): TimeUuid;
     toString(): string;
     getTimeBucket(): string;
+    getDate(): Date;
 }
 //# sourceMappingURL=time-uuid.d.ts.map

package/script/lib/time-uuid.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"time-uuid.d.ts","sourceRoot":"","sources":["../../src/lib/time-uuid.ts"],"names":[],"mappings":"AAEA,OAAO,KAAK,EAAE,MAAM,EAAE,MAAM,aAAa,CAAA;AACzC,OAAO,EAAE,KAAK,EAAE,MAAM,kBAAkB,CAAA;AAGxC,qBAAa,QAAQ;IACP,OAAO,CAAC,QAAQ,CAAC,QAAQ;gBAAR,QAAQ,EAAE,KAAK,CAAC,QAAQ;WAEvC,QAAQ,CAAC,IAAI,EAAE,IAAI,EAAE,KAAK,CAAC,EAAE,MAAM,EAAE,MAAM,CAAC,EAAE,MAAM,GAAG,MAAM,EAAE,OAAO,CAAC,EAAE,MAAM,GAAG,MAAM,GAAG,QAAQ;WAInG,UAAU,CAAC,KAAK,EAAE,MAAM,GAAG,QAAQ;IAIjD,OAAO,CAAC,gBAAgB;IAIjB,SAAS,IAAI,QAAQ;WASd,OAAO,IAAI,QAAQ;IAI1B,QAAQ,IAAI,MAAM;IAIlB,aAAa,IAAI,MAAM;CAG/B"}
+{"version":3,"file":"time-uuid.d.ts","sourceRoot":"","sources":["../../src/lib/time-uuid.ts"],"names":[],"mappings":"AAEA,OAAO,KAAK,EAAE,MAAM,EAAE,MAAM,aAAa,CAAA;AACzC,OAAO,EAAE,KAAK,EAAE,MAAM,kBAAkB,CAAA;AAGxC,qBAAa,QAAQ;IACP,OAAO,CAAC,QAAQ,CAAC,QAAQ;gBAAR,QAAQ,EAAE,KAAK,CAAC,QAAQ;WAEvC,QAAQ,CAAC,IAAI,EAAE,IAAI,EAAE,KAAK,CAAC,EAAE,MAAM,EAAE,MAAM,CAAC,EAAE,MAAM,GAAG,MAAM,EAAE,OAAO,CAAC,EAAE,MAAM,GAAG,MAAM,GAAG,QAAQ;WAInG,UAAU,CAAC,KAAK,EAAE,MAAM,GAAG,QAAQ;IAIjD,OAAO,CAAC,gBAAgB;IAIjB,SAAS,IAAI,QAAQ;WASd,OAAO,IAAI,QAAQ;IAI1B,QAAQ,IAAI,MAAM;IAIlB,aAAa,IAAI,MAAM;IAIvB,OAAO,IAAI,IAAI;CAGvB"}

package/script/lib/time-uuid.js

@@ -40,5 +40,8 @@ class TimeUuid {
     getTimeBucket() {
         return (0, date_fns_2.format)((0, date_fns_2.startOfHour)(this.timeUuid.getDate()), "yyyyMMddHH0000");
     }
+    getDate() {
+        return this.timeUuid.getDate();
+    }
 }
 exports.TimeUuid = TimeUuid;

package/script/mod.d.ts

@@ -1,3 +1,4 @@
+import "./_dnt.polyfills.js";
 export * from "./lib/data-pump.js";
 export * from "./lib/data-pump-create.js";
 export * from "./lib/time-uuid.js";

package/script/mod.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"mod.d.ts","sourceRoot":"","sources":["../src/mod.ts"],"names":[],"mappings":"AAAA,cAAc,oBAAoB,CAAA;AAClC,cAAc,2BAA2B,CAAA;AAEzC,cAAc,oBAAoB,CAAA"}
+{"version":3,"file":"mod.d.ts","sourceRoot":"","sources":["../src/mod.ts"],"names":[],"mappings":"AAAA,OAAO,qBAAqB,CAAC;AAC7B,cAAc,oBAAoB,CAAA;AAClC,cAAc,2BAA2B,CAAA;AAEzC,cAAc,oBAAoB,CAAA"}

package/script/mod.js

@@ -14,6 +14,7 @@ var __exportStar = (this && this.__exportStar) || function(m, exports) {
     for (var p in m) if (p !== "default" && !Object.prototype.hasOwnProperty.call(exports, p)) __createBinding(exports, m, p);
 };
 Object.defineProperty(exports, "__esModule", { value: true });
+require("./_dnt.polyfills.js");
 __exportStar(require("./lib/data-pump.js"), exports);
 __exportStar(require("./lib/data-pump-create.js"), exports);
 __exportStar(require("./lib/time-uuid.js"), exports);