@flowcore/data-pump 0.2.0 → 0.3.0

package/esm/lib/data-pump.d.ts

@@ -1,4 +1,4 @@
-import { type FlowcoreClient, type FlowcoreEvent } from "../deps/jsr.io/@flowcore/sdk/1.11.10/src/mod.js";
+import { type FlowcoreClient, type FlowcoreEvent } from "../deps/jsr.io/@flowcore/sdk/1.19.1/src/mod.js";
 export interface Logger {
     debug(message: string, ...meta: unknown[]): void;
     info(message: string, ...meta: unknown[]): void;

package/esm/lib/data-pump.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"data-pump.d.ts","sourceRoot":"","sources":["../../src/lib/data-pump.ts"],"names":[],"mappings":"AAAA,OAAO,EAIL,KAAK,cAAc,EACnB,KAAK,aAAa,EAEnB,MAAM,iDAAiD,CAAA;AAiBxD,MAAM,WAAW,MAAM;IACrB,KAAK,CAAC,OAAO,EAAE,MAAM,EAAE,GAAG,IAAI,EAAE,OAAO,EAAE,GAAG,IAAI,CAAA;IAChD,IAAI,CAAC,OAAO,EAAE,MAAM,EAAE,GAAG,IAAI,EAAE,OAAO,EAAE,GAAG,IAAI,CAAA;IAC/C,IAAI,CAAC,OAAO,EAAE,MAAM,EAAE,GAAG,IAAI,EAAE,OAAO,EAAE,GAAG,IAAI,CAAA;IAC/C,KAAK,CAAC,OAAO,EAAE,MAAM,GAAG,KAAK,EAAE,GAAG,IAAI,EAAE,OAAO,EAAE,GAAG,IAAI,CAAA;CACzD;AAED,UAAU,eAAe;IACvB,cAAc,EAAE,cAAc,CAAA;IAC9B,YAAY,EAAE,oBAAoB,CAAA;IAClC,MAAM,CAAC,EAAE,MAAM,CAAA;IACf,QAAQ,EAAE,gBAAgB,CAAA;IAC1B,UAAU,EAAE;QACV,MAAM,EAAE,MAAM,CAAA;QACd,QAAQ,EAAE,MAAM,CAAA;QAChB,QAAQ,EAAE,MAAM,CAAA;QAChB,UAAU,EAAE,MAAM,EAAE,CAAA;KACrB,CAAA;IACD,MAAM,EAAE;QACN,IAAI,EAAE,MAAM,CAAA;QACZ,SAAS,EAAE,MAAM,CAAA;QACjB,kBAAkB,EAAE,MAAM,CAAA;QAC1B,qBAAqB,EAAE,MAAM,CAAA;KAC9B,CAAA;IACD,SAAS,CAAC,EAAE;QACV,WAAW,EAAE,MAAM,CAAA;QACnB,QAAQ,EAAE,CAAC,MAAM,EAAE,aAAa,EAAE,KAAK,OAAO,CAAC,OAAO,CAAC,GAAG,OAAO,CAAA;QACjE,cAAc,CAAC,EAAE,CAAC,MAAM,EAAE,aAAa,EAAE,KAAK,OAAO,CAAC,IAAI,CAAC,GAAG,IAAI,CAAA;QAClE,MAAM,CAAC,EAAE,MAAM,OAAO,CAAC,IAAI,CAAC,GAAG,IAAI,CAAA;KACpC,CAAA;IACD,MAAM,CAAC,EAAE,IAAI,CAAA;CACd;AAED,MAAM,WAAW,aAAa;IAC5B,UAAU,EAAE,MAAM,CAAA;IAClB,OAAO,CAAC,EAAE,MAAM,CAAA;CACjB;AAED,MAAM,WAAW,oBAAoB;IACnC,QAAQ,EAAE,MAAM,OAAO,CAAC,aAAa,GAAG,IAAI,CAAC,GAAG,aAAa,GAAG,IAAI,CAAA;IACpE,QAAQ,CAAC,EAAE,CAAC,KAAK,EAAE,aAAa,KAAK,OAAO,CAAC,IAAI,CAAC,GAAG,IAAI,CAAA;IACzD,MAAM,CAAC,EAAE,MAAM,IAAI,CAAA;CACpB;AAED,MAAM,WAAW,gBAAgB;IAC/B,IAAI,CAAC,UAAU,EAAE,eAAe,CAAC,YAAY,CAAC,GAAG;QAAE,UAAU,EAAE,MAAM,CAAA;KAAE,EAAE,MAAM,CAAC,EAAE,WAAW,GAAG,OAAO,CAAC,IAAI,CAAC,CAAA;CAC9G;AAED,qBAAa,QAAQ;IAgBP,OAAO,CAAC,OAAO;IAf3B,OAAO,CAAC,MAAM,CAAC,CAAQ;IACvB,OAAO,CAAC,KAAK,CAAe;IAC5B,OAAO,CAAC,YAAY,CAAe;IACnC,OAAO,CAAC,MAAM,CAA4B;IAC1C,OAAO,CAAC,YAAY,CAA8C;IAClE,OAAO,CAAC,MAAM,CAA8C;IAC5D,OAAO,CAAC,OAAO,CAAQ;IACvB,OAAO,CAAC,uBAAuB,CAAC,CAAiB;IACjD,OAAO,CAAC,UAAU,CAAC,CAAQ;IAC3B,OAAO,CAAC,MAAM,CAAC,CAId;gBAEmB,OAAO,EAAE,eAAe;IAO/B,KAAK;IAiDX,IAAI;YAWG,SAAS;YAqBT,eAAe;YAsGf,MAAM;IAsCpB,OAAO,CAAC,WAAW;IAenB,OAAO,KAAK,WAAW,GAMtB;IAED,OAAO,CAAC,oBAAoB;YAgBd,iBAAiB;YAoBjB,SAAS;YA4CT,gBAAgB;YA2BhB,eAAe;YAUf,aAAa;YAab,sBAAsB;IAepC,OAAO,CAAC,gBAAgB;YAIV,aAAa;IAmBpB,IAAI,CAAC,MAAM,EAAE,MAAM,GAAG,OAAO,CAAC,aAAa,EAAE,CAAC;IAO9C,WAAW,CAAC,QAAQ,EAAE,MAAM,EAAE,GAAG,OAAO,CAAC,IAAI,CAAC;CAMtD"}
+{"version":3,"file":"data-pump.d.ts","sourceRoot":"","sources":["../../src/lib/data-pump.ts"],"names":[],"mappings":"AAAA,OAAO,EAIL,KAAK,cAAc,EACnB,KAAK,aAAa,EAEnB,MAAM,gDAAgD,CAAA;AAiBvD,MAAM,WAAW,MAAM;IACrB,KAAK,CAAC,OAAO,EAAE,MAAM,EAAE,GAAG,IAAI,EAAE,OAAO,EAAE,GAAG,IAAI,CAAA;IAChD,IAAI,CAAC,OAAO,EAAE,MAAM,EAAE,GAAG,IAAI,EAAE,OAAO,EAAE,GAAG,IAAI,CAAA;IAC/C,IAAI,CAAC,OAAO,EAAE,MAAM,EAAE,GAAG,IAAI,EAAE,OAAO,EAAE,GAAG,IAAI,CAAA;IAC/C,KAAK,CAAC,OAAO,EAAE,MAAM,GAAG,KAAK,EAAE,GAAG,IAAI,EAAE,OAAO,EAAE,GAAG,IAAI,CAAA;CACzD;AAED,UAAU,eAAe;IACvB,cAAc,EAAE,cAAc,CAAA;IAC9B,YAAY,EAAE,oBAAoB,CAAA;IAClC,MAAM,CAAC,EAAE,MAAM,CAAA;IACf,QAAQ,EAAE,gBAAgB,CAAA;IAC1B,UAAU,EAAE;QACV,MAAM,EAAE,MAAM,CAAA;QACd,QAAQ,EAAE,MAAM,CAAA;QAChB,QAAQ,EAAE,MAAM,CAAA;QAChB,UAAU,EAAE,MAAM,EAAE,CAAA;KACrB,CAAA;IACD,MAAM,EAAE;QACN,IAAI,EAAE,MAAM,CAAA;QACZ,SAAS,EAAE,MAAM,CAAA;QACjB,kBAAkB,EAAE,MAAM,CAAA;QAC1B,qBAAqB,EAAE,MAAM,CAAA;KAC9B,CAAA;IACD,SAAS,CAAC,EAAE;QACV,WAAW,EAAE,MAAM,CAAA;QACnB,QAAQ,EAAE,CAAC,MAAM,EAAE,aAAa,EAAE,KAAK,OAAO,CAAC,OAAO,CAAC,GAAG,OAAO,CAAA;QACjE,cAAc,CAAC,EAAE,CAAC,MAAM,EAAE,aAAa,EAAE,KAAK,OAAO,CAAC,IAAI,CAAC,GAAG,IAAI,CAAA;QAClE,MAAM,CAAC,EAAE,MAAM,OAAO,CAAC,IAAI,CAAC,GAAG,IAAI,CAAA;KACpC,CAAA;IACD,MAAM,CAAC,EAAE,IAAI,CAAA;CACd;AAED,MAAM,WAAW,aAAa;IAC5B,UAAU,EAAE,MAAM,CAAA;IAClB,OAAO,CAAC,EAAE,MAAM,CAAA;CACjB;AAED,MAAM,WAAW,oBAAoB;IACnC,QAAQ,EAAE,MAAM,OAAO,CAAC,aAAa,GAAG,IAAI,CAAC,GAAG,aAAa,GAAG,IAAI,CAAA;IACpE,QAAQ,CAAC,EAAE,CAAC,KAAK,EAAE,aAAa,KAAK,OAAO,CAAC,IAAI,CAAC,GAAG,IAAI,CAAA;IACzD,MAAM,CAAC,EAAE,MAAM,IAAI,CAAA;CACpB;AAED,MAAM,WAAW,gBAAgB;IAC/B,IAAI,CAAC,UAAU,EAAE,eAAe,CAAC,YAAY,CAAC,GAAG;QAAE,UAAU,EAAE,MAAM,CAAA;KAAE,EAAE,MAAM,CAAC,EAAE,WAAW,GAAG,OAAO,CAAC,IAAI,CAAC,CAAA;CAC9G;AAED,qBAAa,QAAQ;IAgBP,OAAO,CAAC,OAAO;IAf3B,OAAO,CAAC,MAAM,CAAC,CAAQ;IACvB,OAAO,CAAC,KAAK,CAAe;IAC5B,OAAO,CAAC,YAAY,CAAe;IACnC,OAAO,CAAC,MAAM,CAA4B;IAC1C,OAAO,CAAC,YAAY,CAA8C;IAClE,OAAO,CAAC,MAAM,CAA8C;IAC5D,OAAO,CAAC,OAAO,CAAQ;IACvB,OAAO,CAAC,uBAAuB,CAAC,CAAiB;IACjD,OAAO,CAAC,UAAU,CAAC,CAAQ;IAC3B,OAAO,CAAC,MAAM,CAAC,CAId;gBAEmB,OAAO,EAAE,eAAe;IAO/B,KAAK;IAiDX,IAAI;YAWG,SAAS;YAqBT,eAAe;YAsGf,MAAM;IAsCpB,OAAO,CAAC,WAAW;IAenB,OAAO,KAAK,WAAW,GAMtB;IAED,OAAO,CAAC,oBAAoB;YAgBd,iBAAiB;YAoBjB,SAAS;YA4CT,gBAAgB;YA2BhB,eAAe;YAUf,aAAa;YAab,sBAAsB;IAepC,OAAO,CAAC,gBAAgB;YAIV,aAAa;IAmBpB,IAAI,CAAC,MAAM,EAAE,MAAM,GAAG,OAAO,CAAC,aAAa,EAAE,CAAC;IAO9C,WAAW,CAAC,QAAQ,EAAE,MAAM,EAAE,GAAG,OAAO,CAAC,IAAI,CAAC;CAMtD"}

package/esm/lib/data-pump.js

@@ -1,5 +1,5 @@
-import { DataCoreFetchCommand, EventsFetchCommand, EventsFetchTimeBucketsByNamesCommand, TenantFetchCommand, } from "../deps/jsr.io/@flowcore/sdk/1.11.10/src/mod.js";
-import { TimeUuid } from "../deps/jsr.io/@flowcore/time-uuid/0.1.1/src/mod.js";
+import { DataCoreFetchCommand, EventsFetchCommand, EventsFetchTimeBucketsByNamesCommand, TenantFetchCommand, } from "../deps/jsr.io/@flowcore/sdk/1.19.1/src/mod.js";
+import { TimeUuid } from "../deps/jsr.io/@flowcore/time-uuid/0.1.2/src/mod.js";
 import { format, startOfHour } from "date-fns";
 import { utc } from "@date-fns/utc";
 var FlowcoreBufferEventStatus;
@@ -94,7 +94,7 @@ export class DataPump {
                 eventId: lastEventId,
                 eventBufferReached: false,
             };
-            console.log("stop at", this.stopAt);
+            this.logger?.debug("stop at", `${this.stopAt.timeBucket} @ ${this.stopAt.eventId}`);
         }
         const newState = await this.options.stateManager.getState();
         if (!newState) {

package/esm/lib/ws-notifier.js

@@ -1,5 +1,5 @@
 import { Subject } from "rxjs";
-import { NotificationClient } from "../deps/jsr.io/@flowcore/sdk/1.11.10/src/mod.js";
+import { NotificationClient } from "../deps/jsr.io/@flowcore/sdk/1.19.1/src/mod.js";
 export class WebSocketNotifier {
     constructor(options) {
         Object.defineProperty(this, "authOptions", {

package/esm/mod.d.ts

@@ -1,4 +1,7 @@
 import "./_dnt.polyfills.js";
 export * from "./lib/data-pump.js";
 export * from "./lib/data-pump-create.js";
+export * from "./data-pump/data-pump.js";
+export * from "./data-pump/metrics.js";
+export * from "./data-pump/types.js";
 //# sourceMappingURL=mod.d.ts.map

package/esm/mod.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"mod.d.ts","sourceRoot":"","sources":["../src/mod.ts"],"names":[],"mappings":"AAAA,OAAO,qBAAqB,CAAC;AAC7B,cAAc,oBAAoB,CAAA;AAClC,cAAc,2BAA2B,CAAA"}
+{"version":3,"file":"mod.d.ts","sourceRoot":"","sources":["../src/mod.ts"],"names":[],"mappings":"AAAA,OAAO,qBAAqB,CAAC;AAC7B,cAAc,oBAAoB,CAAA;AAClC,cAAc,2BAA2B,CAAA;AAEzC,cAAc,0BAA0B,CAAA;AACxC,cAAc,wBAAwB,CAAA;AACtC,cAAc,sBAAsB,CAAA"}

package/esm/mod.js

@@ -1,3 +1,6 @@
 import "./_dnt.polyfills.js";
 export * from "./lib/data-pump.js";
 export * from "./lib/data-pump-create.js";
+export * from "./data-pump/data-pump.js";
+export * from "./data-pump/metrics.js";
+export * from "./data-pump/types.js";

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@flowcore/data-pump",
-  "version": "0.2.0",
+  "version": "0.3.0",
   "description": "Flowcore Data Pump",
   "homepage": "https://github.com/flowcore-io/flowcore-data-pump#readme",
   "repository": {
@@ -25,6 +25,7 @@
     "date-fns": "^4.1.0",
     "long": "^5.3.1",
     "nats": "^2.29.1",
+    "prom-client": "^15.1.3",
     "rxjs": "^7.8.1",
     "@deno/shim-deno": "~0.18.0",
     "ws": "^8.13.0"

package/script/data-pump/data-pump.d.ts

@@ -0,0 +1,48 @@
+import type { FlowcoreEvent } from "../deps/jsr.io/@flowcore/sdk/1.19.1/src/mod.js";
+import type { FlowcoreDataPumpAuth, FlowcoreDataPumpDataSource, FlowcoreDataPumpProcessor, FlowcoreDataPumpState, FlowcoreDataPumpStateManager, FlowcoreLogger } from "./types.js";
+export interface FlowcoreDataPumpOptions {
+    auth: FlowcoreDataPumpAuth;
+    dataSource: FlowcoreDataPumpDataSource;
+    stateManager: FlowcoreDataPumpStateManager;
+    bufferSize?: number;
+    maxRedeliveryCount?: number;
+    achknowledgeTimeoutMs?: number;
+    processor?: FlowcoreDataPumpProcessor;
+    natsServers?: string[];
+    logger?: FlowcoreLogger;
+    stopAt?: Date;
+}
+export declare class FlowcoreDataPump {
+    private readonly dataSource;
+    private readonly notifier;
+    private stateManager;
+    private readonly options;
+    private readonly logger?;
+    private running;
+    private restartTo?;
+    private abortController?;
+    private buffer;
+    private bufferState;
+    private stopAtState?;
+    private constructor();
+    get isRunning(): boolean;
+    static create(options: FlowcoreDataPumpOptions): FlowcoreDataPump;
+    start(callback?: (error?: Error) => void): Promise<void>;
+    restart(state: FlowcoreDataPumpState, stopAt?: Date | null): void;
+    stop(isRestart?: boolean): void;
+    private updateState;
+    private loop;
+    reserve(amount: number): Promise<FlowcoreEvent[]>;
+    acknowledge(eventIds: string[]): Promise<void>;
+    private reOpen;
+    private processLoop;
+    private updateMetricsGauges;
+    private incMetricsCounter;
+    private waiterEvents?;
+    private waitForEvents;
+    private waiterBufferThreshold?;
+    private waitForBufferThreshold;
+    private waiterBufferEmpty?;
+    private waitForBufferEmpty;
+}
+//# sourceMappingURL=data-pump.d.ts.map

package/script/data-pump/data-pump.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"data-pump.d.ts","sourceRoot":"","sources":["../../src/data-pump/data-pump.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAE,aAAa,EAAE,MAAM,gDAAgD,CAAA;AAMnF,OAAO,KAAK,EACV,oBAAoB,EACpB,0BAA0B,EAC1B,yBAAyB,EACzB,qBAAqB,EACrB,4BAA4B,EAC5B,cAAc,EACf,MAAM,YAAY,CAAA;AAEnB,MAAM,WAAW,uBAAuB;IACtC,IAAI,EAAE,oBAAoB,CAAA;IAC1B,UAAU,EAAE,0BAA0B,CAAA;IACtC,YAAY,EAAE,4BAA4B,CAAA;IAC1C,UAAU,CAAC,EAAE,MAAM,CAAA;IACnB,kBAAkB,CAAC,EAAE,MAAM,CAAA;IAC3B,qBAAqB,CAAC,EAAE,MAAM,CAAA;IAC9B,SAAS,CAAC,EAAE,yBAAyB,CAAA;IACrC,WAAW,CAAC,EAAE,MAAM,EAAE,CAAA;IACtB,MAAM,CAAC,EAAE,cAAc,CAAA;IACvB,MAAM,CAAC,EAAE,IAAI,CAAA;CACd;AAkBD,qBAAa,gBAAgB;IASzB,OAAO,CAAC,QAAQ,CAAC,UAAU;IAC3B,OAAO,CAAC,QAAQ,CAAC,QAAQ;IACzB,OAAO,CAAC,YAAY;IACpB,OAAO,CAAC,QAAQ,CAAC,OAAO;IACxB,OAAO,CAAC,QAAQ,CAAC,MAAM,CAAC;IAZ1B,OAAO,CAAC,OAAO,CAAQ;IACvB,OAAO,CAAC,SAAS,CAAC,CAAuB;IACzC,OAAO,CAAC,eAAe,CAAC,CAAiB;IACzC,OAAO,CAAC,MAAM,CAAmC;IACjD,OAAO,CAAC,WAAW,CAAuB;IAC1C,OAAO,CAAC,WAAW,CAAC,CAAuB;IAE3C,OAAO;IAaP,IAAW,SAAS,IAAI,OAAO,CAE9B;WAEa,MAAM,CAAC,OAAO,EAAE,uBAAuB,GAAG,gBAAgB;IA2B3D,KAAK,CAAC,QAAQ,CAAC,EAAE,CAAC,KAAK,CAAC,EAAE,KAAK,KAAK,IAAI,GAAG,OAAO,CAAC,IAAI,CAAC;IAuC9D,OAAO,CAAC,KAAK,EAAE,qBAAqB,EAAE,MAAM,CAAC,EAAE,IAAI,GAAG,IAAI,GAAG,IAAI;IAQjE,IAAI,CAAC,SAAS,UAAQ,GAAG,IAAI;IAWpC,OAAO,CAAC,WAAW;YAaL,IAAI;IAmEL,OAAO,CAAC,MAAM,EAAE,MAAM,GAAG,OAAO,CAAC,aAAa,EAAE,CAAC;IAoCjD,WAAW,CAAC,QAAQ,EAAE,MAAM,EAAE;YA0B7B,MAAM;YAgDN,WAAW;IAiBzB,OAAO,CAAC,mBAAmB;IA4D3B,OAAO,CAAC,iBAAiB;IAyCzB,OAAO,CAAC,YAAY,CAAC,CAAY;YACnB,aAAa;IAO3B,OAAO,CAAC,qBAAqB,CAAC,CAAY;YAC5B,sBAAsB;IAOpC,OAAO,CAAC,iBAAiB,CAAC,CAAY;YACxB,kBAAkB;CAOjC"}

package/script/data-pump/data-pump.js

@@ -0,0 +1,433 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.FlowcoreDataPump = void 0;
+const utc_1 = require("@date-fns/utc");
+const mod_js_1 = require("../deps/jsr.io/@flowcore/time-uuid/0.1.2/src/mod.js");
+const date_fns_1 = require("date-fns");
+const data_source_js_1 = require("./data-source.js");
+const metrics_js_1 = require("./metrics.js");
+const notifier_js_1 = require("./notifier.js");
+class FlowcoreDataPump {
+    constructor(dataSource, notifier, stateManager, options, logger) {
+        Object.defineProperty(this, "dataSource", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: dataSource
+        });
+        Object.defineProperty(this, "notifier", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: notifier
+        });
+        Object.defineProperty(this, "stateManager", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: stateManager
+        });
+        Object.defineProperty(this, "options", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: options
+        });
+        Object.defineProperty(this, "logger", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: logger
+        });
+        Object.defineProperty(this, "running", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: false
+        });
+        Object.defineProperty(this, "restartTo", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: void 0
+        });
+        Object.defineProperty(this, "abortController", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: void 0
+        });
+        Object.defineProperty(this, "buffer", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: []
+        });
+        Object.defineProperty(this, "bufferState", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: void 0
+        });
+        Object.defineProperty(this, "stopAtState", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: void 0
+        });
+        // #endregion
+        // #region Waiters
+        Object.defineProperty(this, "waiterEvents", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: void 0
+        });
+        Object.defineProperty(this, "waiterBufferThreshold", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: void 0
+        });
+        Object.defineProperty(this, "waiterBufferEmpty", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: void 0
+        });
+        this.bufferState = {
+            timeBucket: (0, date_fns_1.format)((0, date_fns_1.startOfHour)((0, utc_1.utc)(new Date())), "yyyyMMddHH0000"),
+            eventId: mod_js_1.TimeUuid.now().toString(),
+        };
+    }
+    get isRunning() {
+        return this.running;
+    }
+    static create(options) {
+        const dataSource = new data_source_js_1.FlowcoreDataSource({
+            auth: options.auth,
+            dataSource: options.dataSource,
+        });
+        const notifier = new notifier_js_1.FlowcoreNotifier({
+            auth: options.auth,
+            dataSource: options.dataSource,
+            natsServers: options.natsServers,
+            logger: options.logger,
+        });
+        return new FlowcoreDataPump(dataSource, notifier, options.stateManager, {
+            bufferSize: options.bufferSize ?? 1000,
+            bufferThreshold: Math.ceil((options.bufferSize ?? 1000) * 0.1),
+            maxRedeliveryCount: options.maxRedeliveryCount ?? 3,
+            achknowledgeTimeoutMs: options.achknowledgeTimeoutMs ?? 5_000,
+            processor: options.processor,
+            stopAt: options.stopAt,
+        }, options.logger);
+    }
+    async start(callback) {
+        if (this.running) {
+            throw new Error("Data pump already running");
+        }
+        this.running = true;
+        this.updateMetricsGauges();
+        const currentState = await this.stateManager.getState();
+        const timeBucket = currentState
+            ? await this.dataSource.getNextTimeBucket(currentState.timeBucket)
+            : (0, date_fns_1.format)((0, date_fns_1.startOfHour)((0, utc_1.utc)(new Date())), "yyyyMMddHH0000");
+        this.bufferState = {
+            timeBucket: timeBucket ?? (0, date_fns_1.format)((0, date_fns_1.startOfHour)((0, utc_1.utc)(new Date())), "yyyyMMddHH0000"),
+            eventId: currentState ? currentState.eventId : mod_js_1.TimeUuid.now().toString(),
+        };
+        if (this.options.stopAt) {
+            this.stopAtState = {
+                timeBucket: (await this.dataSource.getClosestTimeBucket((0, date_fns_1.format)((0, date_fns_1.startOfHour)((0, utc_1.utc)(this.options.stopAt)), "yyyyMMddHH0000"))) ?? (0, date_fns_1.format)((0, date_fns_1.startOfHour)((0, utc_1.utc)(new Date())), "yyyyMMddHH0000"),
+                eventId: mod_js_1.TimeUuid.fromDate(this.options.stopAt).toString(),
+            };
+        }
+        if (this.options.processor) {
+            this.processLoop().catch((error) => {
+                this.logger?.error("Error in processor", { error });
+                this.stop();
+            });
+        }
+        if (!callback) {
+            return this.loop();
+        }
+        this.loop()
+            .then(() => callback())
+            .catch((error) => callback(error));
+    }
+    restart(state, stopAt) {
+        this.restartTo = state;
+        if (stopAt !== undefined) {
+            this.options.stopAt = stopAt ?? undefined;
+        }
+        this.stop(true);
+    }
+    stop(isRestart = false) {
+        this.running = false;
+        this.buffer = [];
+        this.updateMetricsGauges();
+        this.abortController?.abort();
+        this.waiterBufferThreshold?.();
+        if (!isRestart) {
+            this.waiterEvents?.();
+        }
+    }
+    updateState(eventId) {
+        if (!this.stateManager.setState) {
+            return;
+        }
+        const stateEventId = eventId ?? this.buffer[0]?.event.eventId;
+        if (!stateEventId) {
+            return;
+        }
+        const date = mod_js_1.TimeUuid.fromString(stateEventId).getDate();
+        const timeBucket = (0, date_fns_1.format)((0, date_fns_1.startOfHour)((0, utc_1.utc)(date)), "yyyyMMddHH0000");
+        return this.stateManager.setState?.({ timeBucket, eventId: stateEventId });
+    }
+    async loop() {
+        do {
+            if (this.bufferState.timeBucket === this.stopAtState?.timeBucket) {
+                this.logger?.info("Stopping at stopAt state");
+                await this.waitForBufferEmpty();
+                this.stop();
+                break;
+            }
+            const amountToFetch = this.options.bufferSize - this.buffer.length;
+            if (amountToFetch <= 0) {
+                this.logger?.info("Buffer is full, waiting for space");
+                await this.waitForBufferThreshold();
+                continue;
+            }
+            this.logger?.debug(`fetching ${amountToFetch} events from ${this.bufferState.timeBucket}(${this.bufferState.eventId})`);
+            const events = await this.dataSource.getEvents(this.bufferState, amountToFetch, this.stopAtState?.eventId);
+            if (!this.running) {
+                break;
+            }
+            this.logger?.debug(`fetched ${events.length} events`);
+            this.buffer.push(...events.map((event) => ({ event, status: "open", deliveryCount: 0 })));
+            this.updateMetricsGauges();
+            events.length && this.waiterEvents?.();
+            this.bufferState.eventId = events[events.length - 1]?.eventId ?? this.bufferState.eventId;
+            if (events.length !== amountToFetch) {
+                const timeBucket = await this.dataSource.getNextTimeBucket(this.bufferState.timeBucket);
+                if (!this.running) {
+                    break;
+                }
+                if (timeBucket) {
+                    this.bufferState.timeBucket = timeBucket;
+                }
+                else {
+                    const previousTimeBucket = this.bufferState.timeBucket;
+                    this.bufferState.timeBucket = (0, date_fns_1.format)((0, date_fns_1.startOfHour)((0, utc_1.utc)(new Date())), "yyyyMMddHH0000");
+                    if (previousTimeBucket === this.bufferState.timeBucket && !events.length) {
+                        this.logger?.info("Going live...");
+                        this.abortController = new AbortController();
+                        await this.notifier.wait(this.abortController.signal);
+                    }
+                }
+            }
+        } while (this.running);
+        if (this.restartTo) {
+            await this.dataSource.getTimeBuckets(true);
+            this.restartTo.timeBucket = (await this.dataSource.getClosestTimeBucket(this.restartTo.timeBucket)) ??
+                (0, date_fns_1.format)((0, date_fns_1.startOfHour)((0, utc_1.utc)(new Date())), "yyyyMMddHH0000");
+            this.bufferState = this.restartTo;
+            this.restartTo = undefined;
+            this.running = true;
+            return this.loop();
+        }
+        this.logger?.info("Data pump stopped");
+    }
+    // #region Puller
+    async reserve(amount) {
+        if (!this.running) {
+            return [];
+        }
+        const events = [];
+        const deliveryId = crypto.randomUUID();
+        for (const event of this.buffer) {
+            if (event.status === "open") {
+                event.status = "reserved";
+                event.deliveryId = deliveryId;
+                event.deliveryCount++;
+                events.push(event.event);
+                this.incMetricsCounter("pulled", event.event.eventType, JSON.stringify(event.event).length);
+                if (events.length === amount) {
+                    break;
+                }
+            }
+        }
+        if (!events.length) {
+            await this.waitForEvents();
+            return this.reserve(amount);
+        }
+        this.updateMetricsGauges();
+        setTimeout(() => {
+            this.reOpen(events.map((event) => event.eventId), deliveryId);
+        }, this.options.achknowledgeTimeoutMs);
+        return events;
+    }
+    async acknowledge(eventIds) {
+        if (!this.running) {
+            return;
+        }
+        const lastEventInBuffer = this.buffer[this.buffer.length - 1];
+        this.buffer = this.buffer.filter((event) => {
+            if (eventIds.includes(event.event.eventId)) {
+                this.incMetricsCounter("acknowledged", event.event.eventType, 1);
+                return false;
+            }
+            return true;
+        });
+        if (this.buffer.length <= this.options.bufferSize - this.options.bufferThreshold) {
+            this.waiterBufferThreshold?.();
+        }
+        await this.updateState(this.buffer.length ? undefined : lastEventInBuffer?.event.eventId);
+        this.updateMetricsGauges();
+        if (!this.buffer.length) {
+            this.waiterBufferEmpty?.();
+        }
+    }
+    async reOpen(eventIds, deliveryId) {
+        let lastEvent;
+        const failedEvents = [];
+        const reopenedEvents = [];
+        this.buffer = this.buffer.filter((event) => {
+            if (event.deliveryId !== deliveryId || !eventIds.includes(event.event.eventId)) {
+                return true;
+            }
+            if (this.options.maxRedeliveryCount > -1 && event.deliveryCount > this.options.maxRedeliveryCount) {
+                this.incMetricsCounter("failed", event.event.eventType, 1);
+                failedEvents.push(event.event);
+                lastEvent = event.event;
+                return false;
+            }
+            event.status = "open";
+            event.deliveryId = undefined;
+            reopenedEvents.push(event.event);
+            return true;
+        });
+        this.updateMetricsGauges();
+        if (reopenedEvents.length) {
+            this.logger?.info(`Reopened ${reopenedEvents.length} events`);
+            await this.waiterEvents?.();
+        }
+        if (!failedEvents.length) {
+            return;
+        }
+        this.logger?.info(`Failed ${failedEvents.length} events`);
+        if (this.buffer.length <= this.options.bufferSize - this.options.bufferThreshold) {
+            this.waiterBufferThreshold?.();
+        }
+        await this.updateState(this.buffer.length ? undefined : lastEvent?.eventId);
+        if (!this.buffer.length) {
+            this.waiterBufferEmpty?.();
+        }
+    }
+    // #endregion
+    // #region Pusher
+    async processLoop() {
+        while (this.running) {
+            try {
+                const events = await this.reserve(this.options.processor?.concurrency ?? 1);
+                await this.options.processor?.handler(events);
+                this.logger?.debug(`Processed ${events.length} events`);
+                await this.acknowledge(events.map((event) => event.eventId));
+            }
+            catch (error) {
+                this.logger?.error(error);
+            }
+        }
+    }
+    // #endregion
+    // #region Metrics
+    updateMetricsGauges() {
+        const stats = new Map();
+        for (const eventType of this.dataSource.eventTypes) {
+            stats.set(eventType, {
+                eventCount: 0,
+                eventReservedCount: 0,
+                eventSizeBytes: 0,
+            });
+        }
+        for (const item of this.buffer) {
+            const stat = stats.get(item.event.eventType);
+            if (!stat) {
+                continue;
+            }
+            stat.eventCount++;
+            if (item.status === "reserved") {
+                stat.eventReservedCount++;
+            }
+            stat.eventSizeBytes += JSON.stringify(item.event.payload).length;
+        }
+        for (const [eventType, stat] of stats) {
+            metrics_js_1.metrics.bufferEventCountGauge.set({
+                tenant: this.dataSource.tenant,
+                data_core: this.dataSource.dataCore,
+                flow_type: this.dataSource.flowType,
+                event_type: eventType,
+            }, stat.eventCount);
+            metrics_js_1.metrics.bufferReservedEventCountGauge.set({
+                tenant: this.dataSource.tenant,
+                data_core: this.dataSource.dataCore,
+                flow_type: this.dataSource.flowType,
+                event_type: eventType,
+            }, stat.eventReservedCount);
+            metrics_js_1.metrics.bufferSizeBytesGauge.set({
+                tenant: this.dataSource.tenant,
+                data_core: this.dataSource.dataCore,
+                flow_type: this.dataSource.flowType,
+                event_type: eventType,
+            }, stat.eventSizeBytes);
+        }
+    }
+    incMetricsCounter(name, eventType, value) {
+        switch (name) {
+            case "acknowledged":
+                metrics_js_1.metrics.eventsAcknowledgedCounter.inc({
+                    tenant: this.dataSource.tenant,
+                    data_core: this.dataSource.dataCore,
+                    flow_type: this.dataSource.flowType,
+                    event_type: eventType,
+                }, value);
+                break;
+            case "failed":
+                metrics_js_1.metrics.eventsFailedCounter.inc({
+                    tenant: this.dataSource.tenant,
+                    data_core: this.dataSource.dataCore,
+                    flow_type: this.dataSource.flowType,
+                    event_type: eventType,
+                }, value);
+                break;
+            case "pulled":
+                metrics_js_1.metrics.eventsPulledSizeBytesCounter.inc({
+                    tenant: this.dataSource.tenant,
+                    data_core: this.dataSource.dataCore,
+                    flow_type: this.dataSource.flowType,
+                    event_type: eventType,
+                }, value);
+                break;
+        }
+    }
+    async waitForEvents() {
+        const promise = new Promise((resolve) => {
+            this.waiterEvents = resolve;
+        });
+        await promise;
+    }
+    async waitForBufferThreshold() {
+        const promise = new Promise((resolve) => {
+            this.waiterBufferThreshold = resolve;
+        });
+        await promise;
+    }
+    async waitForBufferEmpty() {
+        const promise = new Promise((resolve) => {
+            this.waiterBufferEmpty = resolve;
+        });
+        await promise;
+    }
+}
+exports.FlowcoreDataPump = FlowcoreDataPump;

package/script/data-pump/data-source.d.ts

@@ -0,0 +1,42 @@
+import type { FlowcoreDataPumpAuth, FlowcoreDataPumpDataSource, FlowcoreDataPumpState } from "./types.js";
+export interface FlowcoreDataSourceOptions {
+    auth: FlowcoreDataPumpAuth;
+    dataSource: FlowcoreDataPumpDataSource;
+}
+export declare class FlowcoreDataSource {
+    private readonly options;
+    private flowcoreClient;
+    private tenantId?;
+    private dataCoreId?;
+    private flowTypeId?;
+    private eventTypeIds?;
+    private timeBuckets?;
+    constructor(options: FlowcoreDataSourceOptions);
+    get tenant(): string;
+    get dataCore(): string;
+    get flowType(): string;
+    get eventTypes(): string[];
+    getTenantId(): Promise<string>;
+    getDataCoreId(): Promise<string>;
+    getFlowTypeId(): Promise<string>;
+    getEventTypeIds(): Promise<string[]>;
+    getTimeBuckets(force?: boolean): Promise<string[]>;
+    getNextTimeBucket(timeBucket: string): Promise<string | null>;
+    getClosestTimeBucket(timeBucket: string, getBefore?: boolean): Promise<string | null>;
+    getEvents(from: FlowcoreDataPumpState, amount: number, toEventId?: string): Promise<{
+        tenant: string;
+        dataCoreId: string;
+        payload: {
+            [x: string]: unknown;
+        };
+        eventType: string;
+        eventId: string;
+        timeBucket: string;
+        flowType: string;
+        metadata: {
+            [x: string]: unknown;
+        };
+        validTime: string;
+    }[]>;
+}
+//# sourceMappingURL=data-source.d.ts.map

package/script/data-pump/data-source.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"data-source.d.ts","sourceRoot":"","sources":["../../src/data-pump/data-source.ts"],"names":[],"mappings":"AAUA,OAAO,KAAK,EAAE,oBAAoB,EAAE,0BAA0B,EAAE,qBAAqB,EAAE,MAAM,YAAY,CAAA;AACzG,MAAM,WAAW,yBAAyB;IACxC,IAAI,EAAE,oBAAoB,CAAA;IAC1B,UAAU,EAAE,0BAA0B,CAAA;CACvC;AAED,qBAAa,kBAAkB;IAQjB,OAAO,CAAC,QAAQ,CAAC,OAAO;IAPpC,OAAO,CAAC,cAAc,CAAgB;IACtC,OAAO,CAAC,QAAQ,CAAC,CAAQ;IACzB,OAAO,CAAC,UAAU,CAAC,CAAQ;IAC3B,OAAO,CAAC,UAAU,CAAC,CAAQ;IAC3B,OAAO,CAAC,YAAY,CAAC,CAAU;IAC/B,OAAO,CAAC,WAAW,CAAC,CAAU;gBAED,OAAO,EAAE,yBAAyB;IAI/D,IAAW,MAAM,WAEhB;IAED,IAAW,QAAQ,WAElB;IAED,IAAW,QAAQ,WAElB;IAED,IAAW,UAAU,aAEpB;IAEY,WAAW;IAYX,aAAa;IAab,aAAa;IAcb,eAAe;IAYf,cAAc,CAAC,KAAK,UAAQ;IAoB5B,iBAAiB,CAAC,UAAU,EAAE,MAAM,GAAG,OAAO,CAAC,MAAM,GAAG,IAAI,CAAC;IAa7D,oBAAoB,CAAC,UAAU,EAAE,MAAM,EAAE,SAAS,UAAQ,GAAG,OAAO,CAAC,MAAM,GAAG,IAAI,CAAC;IAiBnF,SAAS,CAAC,IAAI,EAAE,qBAAqB,EAAE,MAAM,EAAE,MAAM,EAAE,SAAS,CAAC,EAAE,MAAM;;;;;;;;;;;;;;;CAYvF"}