@flowcore/data-pump 0.1.2

package/script/deps/jsr.io/@flowcore/sdk/1.11.4/src/mod.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"mod.d.ts","sourceRoot":"","sources":["../../../../../../../src/deps/jsr.io/@flowcore/sdk/1.11.4/src/mod.ts"],"names":[],"mappings":"AAAA,cAAc,qBAAqB,CAAA;AACnC,cAAc,qBAAqB,CAAA;AACnC,cAAc,6BAA6B,CAAA;AAC3C,cAAc,iCAAiC,CAAA;AAC/C,cAAc,sBAAsB,CAAA;AACpC,cAAc,uBAAuB,CAAA;AACrC,cAAc,kCAAkC,CAAA"}

package/script/deps/jsr.io/@flowcore/sdk/1.11.4/src/mod.js

@@ -0,0 +1,23 @@
+"use strict";
+var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    var desc = Object.getOwnPropertyDescriptor(m, k);
+    if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
+      desc = { enumerable: true, get: function() { return m[k]; } };
+    }
+    Object.defineProperty(o, k2, desc);
+}) : (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    o[k2] = m[k];
+}));
+var __exportStar = (this && this.__exportStar) || function(m, exports) {
+    for (var p in m) if (p !== "default" && !Object.prototype.hasOwnProperty.call(exports, p)) __createBinding(exports, m, p);
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+__exportStar(require("./commands/index.js"), exports);
+__exportStar(require("./common/command.js"), exports);
+__exportStar(require("./common/flowcore-client.js"), exports);
+__exportStar(require("./common/notification-client.js"), exports);
+__exportStar(require("./contracts/index.js"), exports);
+__exportStar(require("./exceptions/index.js"), exports);
+__exportStar(require("./utils/parse-response-helper.js"), exports);

package/script/deps/jsr.io/@flowcore/sdk/1.11.4/src/utils/logger.d.ts

@@ -0,0 +1,8 @@
+export type Logger = {
+    debug: (message: string, meta?: Record<string, unknown>) => void;
+    info: (message: string, meta?: Record<string, unknown>) => void;
+    warn: (message: string, meta?: Record<string, unknown>) => void;
+    error: (message: string | Error, meta?: Record<string, unknown>) => void;
+};
+export declare const defaultLogger: Logger;
+//# sourceMappingURL=logger.d.ts.map

package/script/deps/jsr.io/@flowcore/sdk/1.11.4/src/utils/logger.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"logger.d.ts","sourceRoot":"","sources":["../../../../../../../../src/deps/jsr.io/@flowcore/sdk/1.11.4/src/utils/logger.ts"],"names":[],"mappings":"AAAA,MAAM,MAAM,MAAM,GAAG;IACnB,KAAK,EAAE,CAAC,OAAO,EAAE,MAAM,EAAE,IAAI,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,KAAK,IAAI,CAAA;IAChE,IAAI,EAAE,CAAC,OAAO,EAAE,MAAM,EAAE,IAAI,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,KAAK,IAAI,CAAA;IAC/D,IAAI,EAAE,CAAC,OAAO,EAAE,MAAM,EAAE,IAAI,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,KAAK,IAAI,CAAA;IAC/D,KAAK,EAAE,CAAC,OAAO,EAAE,MAAM,GAAG,KAAK,EAAE,IAAI,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,KAAK,IAAI,CAAA;CACzE,CAAA;AAED,eAAO,MAAM,aAAa,EAAE,MAa3B,CAAA"}

package/script/deps/jsr.io/@flowcore/sdk/1.11.4/src/utils/logger.js

@@ -0,0 +1,17 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.defaultLogger = void 0;
+exports.defaultLogger = {
+    debug: (message, meta) => {
+        console.debug(message, meta);
+    },
+    info: (message, meta) => {
+        console.info(message, meta);
+    },
+    warn: (message, meta) => {
+        console.warn(message, meta);
+    },
+    error: (message, meta) => {
+        console.error(message, meta);
+    },
+};

package/script/deps/jsr.io/@flowcore/sdk/1.11.4/src/utils/parse-response-helper.d.ts

@@ -0,0 +1,6 @@
+import type { Static, TSchema } from "@sinclair/typebox";
+/**
+ * Helper method for parsing the response
+ */
+export declare const parseResponseHelper: <T extends TSchema>(schema: T, response: unknown) => Static<T>;
+//# sourceMappingURL=parse-response-helper.d.ts.map

package/script/deps/jsr.io/@flowcore/sdk/1.11.4/src/utils/parse-response-helper.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"parse-response-helper.d.ts","sourceRoot":"","sources":["../../../../../../../../src/deps/jsr.io/@flowcore/sdk/1.11.4/src/utils/parse-response-helper.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAE,MAAM,EAAE,OAAO,EAAE,MAAM,mBAAmB,CAAA;AAGxD;;GAEG;AACH,eAAO,MAAM,mBAAmB,GAAI,CAAC,SAAS,OAAO,UAAU,CAAC,YAAY,OAAO,KAAG,MAAM,CAAC,CAAC,CAU7F,CAAA"}

package/script/deps/jsr.io/@flowcore/sdk/1.11.4/src/utils/parse-response-helper.js

@@ -0,0 +1,20 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.parseResponseHelper = void 0;
+const value_1 = require("@sinclair/typebox/value");
+const invalid_response_js_1 = require("../exceptions/invalid-response.js");
+/**
+ * Helper method for parsing the response
+ */
+const parseResponseHelper = (schema, response) => {
+    if (!value_1.Value.Check(schema, response)) {
+        const parseErrors = value_1.Value.Errors(schema, response);
+        const errors = {};
+        for (const error of parseErrors) {
+            errors[error.path] = error.message;
+        }
+        throw new invalid_response_js_1.InvalidResponseException("Invalid response", errors);
+    }
+    return response;
+};
+exports.parseResponseHelper = parseResponseHelper;

package/script/lib/data-pump-client.d.ts

@@ -0,0 +1,30 @@
+import { type FlowcoreEvent } from "../deps/jsr.io/@flowcore/sdk/1.11.4/src/mod.js";
+import { DataPump, type DataPumpStateManager, type Logger } from "./data-pump.js";
+export interface DataPumpClientOptions {
+    logger?: Logger;
+    auth: {
+        clientId: string;
+        clientSecret: string;
+        authUrl?: string;
+    };
+    dataSource: {
+        tenant: string;
+        dataCore: string;
+        flowType: string;
+        eventTypes: string[];
+    };
+    buffer?: {
+        size?: number;
+        threshold?: number;
+        maxDeliveryCount?: number;
+        achknowledgeTimeoutMs?: number;
+    };
+    stateManager?: DataPumpStateManager;
+    processor?: {
+        concurrency?: number;
+        onEvents: (events: FlowcoreEvent[]) => Promise<void>;
+    };
+    natsServers?: string[];
+}
+export declare function createDataPump(options: DataPumpClientOptions): DataPump;
+//# sourceMappingURL=data-pump-client.d.ts.map

package/script/lib/data-pump-client.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"data-pump-client.d.ts","sourceRoot":"","sources":["../../src/lib/data-pump-client.ts"],"names":[],"mappings":"AACA,OAAO,EAAkB,KAAK,aAAa,EAAE,MAAM,gDAAgD,CAAA;AACnG,OAAO,EAAE,QAAQ,EAAE,KAAK,oBAAoB,EAAE,KAAK,MAAM,EAAE,MAAM,gBAAgB,CAAA;AAMjF,MAAM,WAAW,qBAAqB;IACpC,MAAM,CAAC,EAAE,MAAM,CAAA;IACf,IAAI,EAAE;QACJ,QAAQ,EAAE,MAAM,CAAA;QAChB,YAAY,EAAE,MAAM,CAAA;QACpB,OAAO,CAAC,EAAE,MAAM,CAAA;KACjB,CAAA;IACD,UAAU,EAAE;QACV,MAAM,EAAE,MAAM,CAAA;QACd,QAAQ,EAAE,MAAM,CAAA;QAChB,QAAQ,EAAE,MAAM,CAAA;QAChB,UAAU,EAAE,MAAM,EAAE,CAAA;KACrB,CAAA;IACD,MAAM,CAAC,EAAE;QACP,IAAI,CAAC,EAAE,MAAM,CAAA;QACb,SAAS,CAAC,EAAE,MAAM,CAAA;QAClB,gBAAgB,CAAC,EAAE,MAAM,CAAA;QACzB,qBAAqB,CAAC,EAAE,MAAM,CAAA;KAC/B,CAAA;IACD,YAAY,CAAC,EAAE,oBAAoB,CAAA;IACnC,SAAS,CAAC,EAAE;QACV,WAAW,CAAC,EAAE,MAAM,CAAA;QACpB,QAAQ,EAAE,CAAC,MAAM,EAAE,aAAa,EAAE,KAAK,OAAO,CAAC,IAAI,CAAC,CAAA;KACrD,CAAA;IACD,WAAW,CAAC,EAAE,MAAM,EAAE,CAAA;CACvB;AAED,wBAAgB,cAAc,CAAC,OAAO,EAAE,qBAAqB,GAAG,QAAQ,CAuDvE"}

package/script/lib/data-pump-client.js

@@ -0,0 +1,58 @@
+"use strict";
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.createDataPump = createDataPump;
+const sdk_oidc_client_1 = __importDefault(require("@flowcore/sdk-oidc-client"));
+const mod_js_1 = require("../deps/jsr.io/@flowcore/sdk/1.11.4/src/mod.js");
+const data_pump_js_1 = require("./data-pump.js");
+const ws_notifier_js_1 = require("./ws-notifier.js");
+const nats_notifier_js_1 = require("./nats-notifier.js");
+const defaultAuthUrl = "https://auth.flowcore.io/realms/flowcore/.well-known/openid-configuration";
+function createDataPump(options) {
+    const authClient = new sdk_oidc_client_1.default.OidcClient(options.auth.clientId, options.auth.clientSecret, options.auth.authUrl ?? defaultAuthUrl);
+    const flowcoreClient = new mod_js_1.FlowcoreClient({
+        getBearerToken: async () => (await authClient.getToken()).accessToken ?? null,
+    });
+    const stateManager = options.stateManager ?? {
+        getState: () => null,
+        setState: () => { },
+    };
+    const notifier = options.natsServers
+        ? new nats_notifier_js_1.NatsNotifier({
+            servers: options.natsServers,
+        })
+        : new ws_notifier_js_1.WebSocketNotifier({
+            auth: {
+                clientId: options.auth.clientId,
+                clientSecret: options.auth.clientSecret,
+                authUrl: options.auth.authUrl ?? defaultAuthUrl,
+            },
+            dataSource: options.dataSource,
+            logger: options.logger,
+        });
+    const bufferSize = options.buffer?.size ?? 1000;
+    const buffer = {
+        size: bufferSize,
+        threshold: options.buffer?.threshold ?? Math.round(bufferSize * 0.1),
+        maxDeliveryCount: options.buffer?.maxDeliveryCount ?? 3,
+        achknowledgeTimeoutMs: options.buffer?.achknowledgeTimeoutMs ?? 10_000,
+    };
+    const processor = options.processor
+        ? {
+            concurrency: options.processor.concurrency ?? 1,
+            onEvents: options.processor.onEvents,
+        }
+        : undefined;
+    const dataPump = new data_pump_js_1.DataPump({
+        flowcoreClient,
+        stateManager,
+        dataSource: options.dataSource,
+        buffer,
+        processor,
+        notifier: notifier.getNotifier(),
+        logger: options.logger,
+    });
+    return dataPump;
+}

package/script/lib/data-pump.d.ts

@@ -0,0 +1,75 @@
+import { type FlowcoreClient, type FlowcoreEvent } from "../deps/jsr.io/@flowcore/sdk/1.11.4/src/mod.js";
+export interface Logger {
+    debug(message: string, ...meta: unknown[]): void;
+    info(message: string, ...meta: unknown[]): void;
+    warn(message: string, ...meta: unknown[]): void;
+    error(message: string | Error, ...meta: unknown[]): void;
+}
+interface DataPumpOptions {
+    flowcoreClient: FlowcoreClient;
+    stateManager: DataPumpStateManager;
+    logger?: Logger;
+    notifier: DataPumpNotifier;
+    dataSource: {
+        tenant: string;
+        dataCore: string;
+        flowType: string;
+        eventTypes: string[];
+    };
+    buffer: {
+        size: number;
+        threshold: number;
+        maxDeliveryCount: number;
+        achknowledgeTimeoutMs: number;
+    };
+    processor?: {
+        concurrency: number;
+        onEvents: (events: FlowcoreEvent[]) => Promise<void>;
+    };
+}
+export interface DataPumpState {
+    timeBucket: string;
+    eventId?: string;
+}
+export interface DataPumpStateManager {
+    getState(): Promise<DataPumpState | null> | DataPumpState | null;
+    setState(state: DataPumpState): Promise<void> | void;
+}
+export interface DataPumpNotifier {
+    wait(dataSource: DataPumpOptions["dataSource"] & {
+        dataCoreId: string;
+    }, signal?: AbortSignal): Promise<void>;
+}
+export declare class DataPump {
+    private options;
+    private logger?;
+    private state;
+    private _timeBuckets;
+    private buffer;
+    private bufferWaiter;
+    private waiter;
+    private running;
+    private notifierAbortController?;
+    private dataCoreId?;
+    constructor(options: DataPumpOptions);
+    start(): Promise<void>;
+    stop(): void;
+    private proceccor;
+    private eventBufferLoop;
+    private reOpen;
+    private updateState;
+    private get timeBuckets();
+    private getClosestTimeBucket;
+    private updateTimeBuckets;
+    private pullInner;
+    private acknowledgeInner;
+    private waitForNotifier;
+    private waitForEvents;
+    private waitForBufferThreshold;
+    private getNowTimeBucket;
+    private getDataCoreId;
+    pull(amount: number): Promise<FlowcoreEvent[]>;
+    acknowledge(eventIds: string[]): Promise<void>;
+}
+export {};
+//# sourceMappingURL=data-pump.d.ts.map

package/script/lib/data-pump.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"data-pump.d.ts","sourceRoot":"","sources":["../../src/lib/data-pump.ts"],"names":[],"mappings":"AAAA,OAAO,EAIL,KAAK,cAAc,EACnB,KAAK,aAAa,EAEnB,MAAM,gDAAgD,CAAA;AAiBvD,MAAM,WAAW,MAAM;IACrB,KAAK,CAAC,OAAO,EAAE,MAAM,EAAE,GAAG,IAAI,EAAE,OAAO,EAAE,GAAG,IAAI,CAAA;IAChD,IAAI,CAAC,OAAO,EAAE,MAAM,EAAE,GAAG,IAAI,EAAE,OAAO,EAAE,GAAG,IAAI,CAAA;IAC/C,IAAI,CAAC,OAAO,EAAE,MAAM,EAAE,GAAG,IAAI,EAAE,OAAO,EAAE,GAAG,IAAI,CAAA;IAC/C,KAAK,CAAC,OAAO,EAAE,MAAM,GAAG,KAAK,EAAE,GAAG,IAAI,EAAE,OAAO,EAAE,GAAG,IAAI,CAAA;CACzD;AAED,UAAU,eAAe;IACvB,cAAc,EAAE,cAAc,CAAA;IAC9B,YAAY,EAAE,oBAAoB,CAAA;IAClC,MAAM,CAAC,EAAE,MAAM,CAAA;IACf,QAAQ,EAAE,gBAAgB,CAAA;IAC1B,UAAU,EAAE;QACV,MAAM,EAAE,MAAM,CAAA;QACd,QAAQ,EAAE,MAAM,CAAA;QAChB,QAAQ,EAAE,MAAM,CAAA;QAChB,UAAU,EAAE,MAAM,EAAE,CAAA;KACrB,CAAA;IACD,MAAM,EAAE;QACN,IAAI,EAAE,MAAM,CAAA;QACZ,SAAS,EAAE,MAAM,CAAA;QACjB,gBAAgB,EAAE,MAAM,CAAA;QACxB,qBAAqB,EAAE,MAAM,CAAA;KAC9B,CAAA;IACD,SAAS,CAAC,EAAE;QACV,WAAW,EAAE,MAAM,CAAA;QACnB,QAAQ,EAAE,CAAC,MAAM,EAAE,aAAa,EAAE,KAAK,OAAO,CAAC,IAAI,CAAC,CAAA;KACrD,CAAA;CACF;AAED,MAAM,WAAW,aAAa;IAC5B,UAAU,EAAE,MAAM,CAAA;IAClB,OAAO,CAAC,EAAE,MAAM,CAAA;CACjB;AAED,MAAM,WAAW,oBAAoB;IACnC,QAAQ,IAAI,OAAO,CAAC,aAAa,GAAG,IAAI,CAAC,GAAG,aAAa,GAAG,IAAI,CAAA;IAChE,QAAQ,CAAC,KAAK,EAAE,aAAa,GAAG,OAAO,CAAC,IAAI,CAAC,GAAG,IAAI,CAAA;CACrD;AAED,MAAM,WAAW,gBAAgB;IAC/B,IAAI,CAAC,UAAU,EAAE,eAAe,CAAC,YAAY,CAAC,GAAG;QAAE,UAAU,EAAE,MAAM,CAAA;KAAE,EAAE,MAAM,CAAC,EAAE,WAAW,GAAG,OAAO,CAAC,IAAI,CAAC,CAAA;CAC9G;AAED,qBAAa,QAAQ;IAWP,OAAO,CAAC,OAAO;IAV3B,OAAO,CAAC,MAAM,CAAC,CAAQ;IACvB,OAAO,CAAC,KAAK,CAAe;IAC5B,OAAO,CAAC,YAAY,CAAe;IACnC,OAAO,CAAC,MAAM,CAA4B;IAC1C,OAAO,CAAC,YAAY,CAA8C;IAClE,OAAO,CAAC,MAAM,CAA8C;IAC5D,OAAO,CAAC,OAAO,CAAQ;IACvB,OAAO,CAAC,uBAAuB,CAAC,CAAiB;IACjD,OAAO,CAAC,UAAU,CAAC,CAAQ;gBAEP,OAAO,EAAE,eAAe;IAO/B,KAAK;IAiCX,IAAI;YAWG,SAAS;YAmBT,eAAe;IAwF7B,OAAO,CAAC,MAAM;IAgBd,OAAO,CAAC,WAAW;IAUnB,OAAO,KAAK,WAAW,GAMtB;IAED,OAAO,CAAC,oBAAoB;YAUd,iBAAiB;YAoBjB,SAAS;YAsCT,gBAAgB;YAiBhB,eAAe;YAUf,aAAa;YAab,sBAAsB;IAepC,OAAO,CAAC,gBAAgB;YAIV,aAAa;IAmBpB,IAAI,CAAC,MAAM,EAAE,MAAM,GAAG,OAAO,CAAC,aAAa,EAAE,CAAC;IAO9C,WAAW,CAAC,QAAQ,EAAE,MAAM,EAAE,GAAG,OAAO,CAAC,IAAI,CAAC;CAMtD"}

package/script/lib/data-pump.js

@@ -0,0 +1,368 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.DataPump = void 0;
+const mod_js_1 = require("../deps/jsr.io/@flowcore/sdk/1.11.4/src/mod.js");
+const time_uuid_js_1 = require("./time-uuid.js");
+const date_fns_1 = require("date-fns");
+const utc_1 = require("@date-fns/utc");
+var FlowcoreBufferEventStatus;
+(function (FlowcoreBufferEventStatus) {
+    FlowcoreBufferEventStatus["OPEN"] = "open";
+    FlowcoreBufferEventStatus["DELIVERED"] = "delivered";
+})(FlowcoreBufferEventStatus || (FlowcoreBufferEventStatus = {}));
+class DataPump {
+    constructor(options) {
+        Object.defineProperty(this, "options", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: options
+        });
+        Object.defineProperty(this, "logger", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: void 0
+        });
+        Object.defineProperty(this, "state", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: void 0
+        });
+        Object.defineProperty(this, "_timeBuckets", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: []
+        });
+        Object.defineProperty(this, "buffer", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: []
+        });
+        Object.defineProperty(this, "bufferWaiter", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: null
+        });
+        Object.defineProperty(this, "waiter", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: null
+        });
+        Object.defineProperty(this, "running", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: false
+        });
+        Object.defineProperty(this, "notifierAbortController", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: void 0
+        });
+        Object.defineProperty(this, "dataCoreId", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: void 0
+        });
+        this.state = {
+            timeBucket: this.getNowTimeBucket(),
+        };
+        this.logger = options.logger ?? undefined;
+    }
+    async start() {
+        if (this.running) {
+            return;
+        }
+        await this.updateTimeBuckets();
+        const newState = await this.options.stateManager.getState();
+        if (!newState) {
+            this.state.timeBucket = this.getClosestTimeBucket(this.getNowTimeBucket());
+            this.state.eventId = time_uuid_js_1.TimeUuid.fromNow().toString();
+        }
+        else {
+            this.state.timeBucket = this.getClosestTimeBucket(newState.timeBucket);
+            this.state.eventId = newState.eventId;
+        }
+        this.running = true;
+        this.logger?.debug("Starting event buffer loop");
+        this.eventBufferLoop()
+            .then(() => this.logger?.debug("Event buffer loop stopped"))
+            .catch((error) => this.logger?.error("Error in event buffer loop", { error }))
+            .finally(() => this.stop());
+        if (this.options.processor) {
+            this.logger?.debug("Starting processor");
+            this.proceccor()
+                .then(() => this.logger?.debug("Processor stopped"))
+                .catch((error) => this.logger?.error("Error in processor", { error }))
+                .finally(() => this.stop());
+        }
+    }
+    stop() {
+        if (!this.running) {
+            return;
+        }
+        this.logger?.debug("Stopping data pump");
+        this.running = false;
+        this.notifierAbortController?.abort("stopping");
+        this.bufferWaiter?.(true);
+        this.waiter?.(true);
+    }
+    async proceccor() {
+        if (!this.options.processor) {
+            throw new Error("Processor not set");
+        }
+        while (this.running) {
+            try {
+                const events = await this.pullInner(this.options.processor.concurrency);
+                if (!this.running) {
+                    break;
+                }
+                await this.options.processor.onEvents(events);
+                await this.acknowledgeInner(events.map((event) => event.eventId));
+            }
+            catch (error) {
+                this.logger?.error("Error in processor", { error });
+                this.stop();
+            }
+        }
+    }
+    async eventBufferLoop() {
+        // Stop buffer loop if we are not running
+        if (!this.running) {
+            return;
+        }
+        // Buffer is full, wait for some space
+        if (this.buffer.length >= this.options.buffer.size - this.options.buffer.threshold && this.running) {
+            this.logger?.debug("Buffer is full, waiting for some space");
+            await this.waitForBufferThreshold();
+        }
+        // Stop buffer loop if we are not running
+        if (!this.running) {
+            return;
+        }
+        // How many events should we fetch
+        const eventsToFetch = this.options.buffer.size - this.buffer.length;
+        this.logger?.debug(`fetching ${eventsToFetch} events from ${this.state.timeBucket} @ ${this.state.eventId ?? "-"}`);
+        const timeStart = Date.now();
+        const command = new mod_js_1.EventsFetchCommand({
+            tenant: this.options.dataSource.tenant,
+            dataCoreId: await this.getDataCoreId(),
+            flowType: this.options.dataSource.flowType,
+            eventTypes: this.options.dataSource.eventTypes,
+            timeBucket: this.state.timeBucket,
+            afterEventId: this.state.eventId,
+            pageSize: eventsToFetch,
+        });
+        const result = await this.options.flowcoreClient.execute(command);
+        const time = Date.now() - timeStart;
+        // Stop buffer loop if we are not running
+        if (!this.running) {
+            return;
+        }
+        const events = result.events.map((event) => ({
+            event,
+            status: FlowcoreBufferEventStatus.OPEN,
+            deliveryCount: 0,
+            deliveryTime: 0,
+            deliveryId: "",
+        }));
+        this.buffer.push(...events);
+        // Check if we have a waiter and if we have events, notify the waiter
+        events.length && this.waiter?.();
+        this.logger?.debug(`Got ${result.events.length} events from ${this.state.timeBucket} @ ${this.state.eventId ?? "-"} in ${time}ms. Buffer size ${this.buffer.length}`);
+        // Update buffer state
+        this.state.eventId = result.events[result.events.length - 1]?.eventId ?? this.state.eventId;
+        // If we don't have a next cursor move to next time bucket
+        if (!result.nextCursor) {
+            // Get next time bucket
+            const nextTimeBucketIndex = this.timeBuckets.indexOf(this.state.timeBucket) + 1;
+            // There is no next time bucket
+            if (!this.timeBuckets[nextTimeBucketIndex]) {
+                if (result.events.length) {
+                    // If we got events, we can try to get more one more time
+                    this.logger?.debug("Try to get more events");
+                }
+                else {
+                    // If we didn't get events, we wait for notifier
+                    await this.waitForNotifier();
+                }
+            }
+            else {
+                // Set next time bucket
+                this.state.timeBucket = this.timeBuckets[nextTimeBucketIndex];
+            }
+        }
+        return this.eventBufferLoop();
+    }
+    reOpen(eventIds, deliveryId) {
+        if (!eventIds.length) {
+            return;
+        }
+        this.buffer.forEach((event) => {
+            if (event.deliveryId === deliveryId && eventIds.includes(event.event.eventId)) {
+                event.status = FlowcoreBufferEventStatus.OPEN;
+                event.deliveryId = "";
+            }
+        });
+        if (this.bufferWaiter && this.buffer.length <= this.options.buffer.size - this.options.buffer.threshold) {
+            this.bufferWaiter();
+        }
+    }
+    updateState() {
+        const firstEvent = this.buffer[0];
+        if (!firstEvent) {
+            return;
+        }
+        const timeUuid = time_uuid_js_1.TimeUuid.fromString(firstEvent.event.eventId);
+        const timeBucket = firstEvent.event.timeBucket;
+        return this.options.stateManager.setState({ timeBucket, eventId: timeUuid.getBefore().toString() });
+    }
+    get timeBuckets() {
+        const timeBucketNow = this.getNowTimeBucket();
+        if (this._timeBuckets[this._timeBuckets.length - 1] !== timeBucketNow) {
+            return [...this._timeBuckets, timeBucketNow];
+        }
+        return this._timeBuckets;
+    }
+    getClosestTimeBucket(timeBucket) {
+        if (!timeBucket.match(/^\d{14}$/)) {
+            throw new Error(`Invalid timebucket: ${timeBucket}`);
+        }
+        return (this.timeBuckets.find((t) => Number.parseFloat(t) >= Number.parseFloat(timeBucket)) ??
+            this.timeBuckets[this.timeBuckets.length - 1]);
+    }
+    async updateTimeBuckets() {
+        this._timeBuckets = [];
+        let first = true;
+        let cursor;
+        while (cursor !== undefined || first) {
+            first = false;
+            const command = new mod_js_1.EventsFetchTimeBucketsByNamesCommand({
+                tenant: this.options.dataSource.tenant,
+                dataCoreId: await this.getDataCoreId(),
+                flowType: this.options.dataSource.flowType,
+                eventTypes: this.options.dataSource.eventTypes,
+                pageSize: 10_000,
+            });
+            // sdkCommandCount.labels("EventsFetchTimeBucketsByNamesCommand").inc(1)
+            const result = await this.options.flowcoreClient.execute(command);
+            this._timeBuckets.push(...result.timeBuckets);
+            cursor = result.nextCursor;
+        }
+    }
+    async pullInner(amount) {
+        if (!this.running) {
+            throw new Error("Data pump not running");
+        }
+        const deliveryId = crypto.randomUUID();
+        const events = [];
+        for (const event of this.buffer) {
+            if (event.status === FlowcoreBufferEventStatus.OPEN) {
+                event.status = FlowcoreBufferEventStatus.DELIVERED;
+                event.deliveryCount++;
+                event.deliveryId = deliveryId;
+                events.push(event.event);
+                if (events.length >= amount) {
+                    break;
+                }
+            }
+        }
+        if (!events.length) {
+            await this.waitForEvents();
+            if (!this.running) {
+                return [];
+            }
+            return this.pullInner(amount);
+        }
+        setTimeout(() => {
+            this.reOpen(events.map((event) => event.eventId), deliveryId);
+        }, this.options.buffer.achknowledgeTimeoutMs);
+        return events;
+    }
+    async acknowledgeInner(eventIds) {
+        if (!eventIds.length) {
+            return;
+        }
+        // this.logger.info("Acknowledging events", { events: eventIds.length })
+        this.buffer = this.buffer.filter((event) => {
+            if (eventIds.includes(event.event.eventId)) {
+                return false;
+            }
+            return true;
+        });
+        if (this.bufferWaiter && this.buffer.length <= this.options.buffer.size - this.options.buffer.threshold) {
+            this.bufferWaiter();
+        }
+        await this.updateState();
+    }
+    async waitForNotifier() {
+        this.logger?.debug("No more events, waiting for notifier...");
+        this.notifierAbortController = new AbortController();
+        await this.options.notifier.wait({ ...this.options.dataSource, dataCoreId: await this.getDataCoreId() }, this.notifierAbortController.signal);
+        this.notifierAbortController = undefined;
+    }
+    async waitForEvents() {
+        this.logger?.debug("No events, waiting for events");
+        await new Promise((resolve) => {
+            this.waiter = (stopping) => {
+                if (!stopping) {
+                    this.logger?.debug("Notifying waiter that we have events");
+                }
+                resolve();
+            };
+        });
+        this.waiter = null;
+    }
+    async waitForBufferThreshold() {
+        if (this.buffer.length < this.options.buffer.size - this.options.buffer.threshold) {
+            return;
+        }
+        await new Promise((resolve) => {
+            this.bufferWaiter = (stopping) => {
+                if (!stopping) {
+                    this.logger?.debug("Notifying buffer waiter that we are under threshold");
+                }
+                resolve();
+            };
+        });
+        this.bufferWaiter = null;
+    }
+    getNowTimeBucket() {
+        return (0, date_fns_1.format)((0, date_fns_1.startOfHour)((0, utc_1.utc)(new Date())), "yyyyMMddHH0000");
+    }
+    async getDataCoreId() {
+        if (this.dataCoreId) {
+            return this.dataCoreId;
+        }
+        const tenant = await this.options.flowcoreClient.execute(new mod_js_1.TenantFetchCommand({
+            tenant: this.options.dataSource.tenant,
+        }));
+        const dataCore = await this.options.flowcoreClient.execute(new mod_js_1.DataCoreFetchCommand({
+            dataCore: this.options.dataSource.dataCore,
+            tenantId: tenant.id,
+        }));
+        this.dataCoreId = dataCore.id;
+        return this.dataCoreId;
+    }
+    pull(amount) {
+        if (this.options.processor) {
+            throw new Error("Pull is not supported when processor is set");
+        }
+        return this.pullInner(amount);
+    }
+    acknowledge(eventIds) {
+        if (this.options.processor) {
+            throw new Error("Acknowledge is not supported when processor is set");
+        }
+        return this.acknowledgeInner(eventIds);
+    }
+}
+exports.DataPump = DataPump;

package/script/lib/nats-notifier.d.ts

@@ -0,0 +1,18 @@
+import type { DataPumpNotifier } from "./data-pump.js";
+interface NatsClientOptions {
+    servers: string[];
+    timeoutMs?: number;
+}
+export declare class NatsNotifier {
+    private readonly options;
+    private nats;
+    private timeoutMs;
+    constructor(options: NatsClientOptions);
+    private start;
+    private stop;
+    private awaitMessage;
+    private getSubscription;
+    getNotifier(): DataPumpNotifier;
+}
+export {};
+//# sourceMappingURL=nats-notifier.d.ts.map

package/script/lib/nats-notifier.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"nats-notifier.d.ts","sourceRoot":"","sources":["../../src/lib/nats-notifier.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAE,gBAAgB,EAAE,MAAM,gBAAgB,CAAA;AAEtD,UAAU,iBAAiB;IACzB,OAAO,EAAE,MAAM,EAAE,CAAA;IACjB,SAAS,CAAC,EAAE,MAAM,CAAA;CACnB;AAED,qBAAa,YAAY;IAGX,OAAO,CAAC,QAAQ,CAAC,OAAO;IAFpC,OAAO,CAAC,IAAI,CAAiC;IAC7C,OAAO,CAAC,SAAS,CAAQ;gBACI,OAAO,EAAE,iBAAiB;YAIzC,KAAK;YAOL,IAAI;IAOlB,OAAO,CAAC,YAAY;IAoCpB,OAAO,CAAC,eAAe;IAgBhB,WAAW,IAAI,gBAAgB;CAavC"}