@flowcore/data-pump 0.11.0 → 0.12.0

package/script/lib/data-pump-create.d.ts

@@ -1,46 +0,0 @@
-import { type FlowcoreEvent } from "../deps/jsr.io/@flowcore/sdk/1.26.3/src/mod.js";
-import { DataPump, type DataPumpStateManager, type Logger } from "./data-pump.js";
-export declare const noOpLogger: Logger;
-interface DataPumpClientOptionsAuthOidcClient {
-    oidcClient: {
-        getToken: () => Promise<{
-            accessToken: string;
-        }>;
-    };
-}
-interface DataPumpClientOptionsAuthApiKey {
-    apiKey: string;
-    apiKeyId: string;
-}
-type DataPumpClientOptionsAuth = DataPumpClientOptionsAuthOidcClient | DataPumpClientOptionsAuthApiKey;
-/**
- * Create data pump client options
- */
-export interface DataPumpClientOptions {
-    logger?: Logger;
-    auth: DataPumpClientOptionsAuth;
-    dataSource: {
-        tenant: string;
-        dataCore: string;
-        flowType: string;
-        eventTypes: string[];
-    };
-    buffer?: {
-        size?: number;
-        threshold?: number;
-        maxRedeliveryCount?: number;
-        achknowledgeTimeoutMs?: number;
-    };
-    stateManager?: DataPumpStateManager;
-    processor?: {
-        concurrency?: number;
-        onEvents: (events: FlowcoreEvent[]) => Promise<boolean> | boolean;
-        onFailedEvents?: (events: FlowcoreEvent[]) => Promise<void> | void;
-        onDone?: () => Promise<void> | void;
-    };
-    stopAt?: Date;
-    natsServers?: string[];
-}
-export declare function createDataPump(options: DataPumpClientOptions): DataPump;
-export {};
-//# sourceMappingURL=data-pump-create.d.ts.map

package/script/lib/data-pump-create.d.ts.map

@@ -1 +0,0 @@
-{"version":3,"file":"data-pump-create.d.ts","sourceRoot":"","sources":["../../src/lib/data-pump-create.ts"],"names":[],"mappings":"AAAA,OAAO,EAAkB,KAAK,aAAa,EAAE,MAAM,gDAAgD,CAAA;AACnG,OAAO,EAAE,QAAQ,EAAE,KAAK,oBAAoB,EAAE,KAAK,MAAM,EAAE,MAAM,gBAAgB,CAAA;AAIjF,eAAO,MAAM,UAAU,EAAE,MAKxB,CAAA;AAED,UAAU,mCAAmC;IAC3C,UAAU,EAAE;QACV,QAAQ,EAAE,MAAM,OAAO,CAAC;YAAE,WAAW,EAAE,MAAM,CAAA;SAAE,CAAC,CAAA;KACjD,CAAA;CACF;AAED,UAAU,+BAA+B;IACvC,MAAM,EAAE,MAAM,CAAA;IACd,QAAQ,EAAE,MAAM,CAAA;CACjB;AAED,KAAK,yBAAyB,GAAG,mCAAmC,GAAG,+BAA+B,CAAA;AAEtG;;GAEG;AACH,MAAM,WAAW,qBAAqB;IACpC,MAAM,CAAC,EAAE,MAAM,CAAA;IACf,IAAI,EAAE,yBAAyB,CAAA;IAC/B,UAAU,EAAE;QACV,MAAM,EAAE,MAAM,CAAA;QACd,QAAQ,EAAE,MAAM,CAAA;QAChB,QAAQ,EAAE,MAAM,CAAA;QAChB,UAAU,EAAE,MAAM,EAAE,CAAA;KACrB,CAAA;IACD,MAAM,CAAC,EAAE;QACP,IAAI,CAAC,EAAE,MAAM,CAAA;QACb,SAAS,CAAC,EAAE,MAAM,CAAA;QAClB,kBAAkB,CAAC,EAAE,MAAM,CAAA;QAC3B,qBAAqB,CAAC,EAAE,MAAM,CAAA;KAC/B,CAAA;IACD,YAAY,CAAC,EAAE,oBAAoB,CAAA;IACnC,SAAS,CAAC,EAAE;QACV,WAAW,CAAC,EAAE,MAAM,CAAA;QACpB,QAAQ,EAAE,CAAC,MAAM,EAAE,aAAa,EAAE,KAAK,OAAO,CAAC,OAAO,CAAC,GAAG,OAAO,CAAA;QACjE,cAAc,CAAC,EAAE,CAAC,MAAM,EAAE,aAAa,EAAE,KAAK,OAAO,CAAC,IAAI,CAAC,GAAG,IAAI,CAAA;QAClE,MAAM,CAAC,EAAE,MAAM,OAAO,CAAC,IAAI,CAAC,GAAG,IAAI,CAAA;KACpC,CAAA;IACD,MAAM,CAAC,EAAE,IAAI,CAAA;IACb,WAAW,CAAC,EAAE,MAAM,EAAE,CAAA;CACvB;AAED,wBAAgB,cAAc,CAAC,OAAO,EAAE,qBAAqB,GAAG,QAAQ,CA0DvE"}

package/script/lib/data-pump-create.js

@@ -1,68 +0,0 @@
-"use strict";
-Object.defineProperty(exports, "__esModule", { value: true });
-exports.noOpLogger = void 0;
-exports.createDataPump = createDataPump;
-const mod_js_1 = require("../deps/jsr.io/@flowcore/sdk/1.26.3/src/mod.js");
-const data_pump_js_1 = require("./data-pump.js");
-const ws_notifier_js_1 = require("./ws-notifier.js");
-const nats_notifier_js_1 = require("./nats-notifier.js");
-exports.noOpLogger = {
-    debug: () => { },
-    info: () => { },
-    warn: () => { },
-    error: console.error,
-};
-function createDataPump(options) {
-    let flowcoreClient;
-    if ("oidcClient" in options.auth) {
-        const authClient = options.auth.oidcClient;
-        flowcoreClient = new mod_js_1.FlowcoreClient({
-            getBearerToken: async () => (await authClient.getToken()).accessToken ?? null,
-        });
-    }
-    else {
-        flowcoreClient = new mod_js_1.FlowcoreClient({
-            apiKey: options.auth.apiKey,
-            apiKeyId: options.auth.apiKeyId,
-        });
-    }
-    const stateManager = options.stateManager ?? {
-        getState: () => null,
-        setState: () => { },
-    };
-    const notifier = options.natsServers
-        ? new nats_notifier_js_1.NatsNotifier({
-            servers: options.natsServers,
-        })
-        : new ws_notifier_js_1.WebSocketNotifier({
-            auth: options.auth,
-            dataSource: options.dataSource,
-            logger: options.logger,
-        });
-    const bufferSize = options.buffer?.size ?? 1000;
-    const buffer = {
-        size: bufferSize,
-        threshold: options.buffer?.threshold ?? Math.round(bufferSize * 0.1),
-        maxRedeliveryCount: options.buffer?.maxRedeliveryCount ?? 3,
-        achknowledgeTimeoutMs: options.buffer?.achknowledgeTimeoutMs ?? 10_000,
-    };
-    const processor = options.processor
-        ? {
-            concurrency: options.processor.concurrency ?? 1,
-            onEvents: options.processor.onEvents,
-            onFailedEvents: options.processor.onFailedEvents,
-            onDone: options.processor.onDone,
-        }
-        : undefined;
-    const dataPump = new data_pump_js_1.DataPump({
-        flowcoreClient,
-        stateManager,
-        dataSource: options.dataSource,
-        buffer,
-        processor,
-        notifier: notifier.getNotifier(),
-        logger: options.logger,
-        stopAt: options.stopAt,
-    });
-    return dataPump;
-}

package/script/lib/data-pump.d.ts

@@ -1,80 +0,0 @@
-import { type FlowcoreClient, type FlowcoreEvent } from "../deps/jsr.io/@flowcore/sdk/1.26.3/src/mod.js";
-export interface Logger {
-    debug(message: string, ...meta: unknown[]): void;
-    info(message: string, ...meta: unknown[]): void;
-    warn(message: string, ...meta: unknown[]): void;
-    error(message: string | Error, ...meta: unknown[]): void;
-}
-interface DataPumpOptions {
-    flowcoreClient: FlowcoreClient;
-    stateManager: DataPumpStateManager;
-    logger?: Logger;
-    notifier: DataPumpNotifier;
-    dataSource: {
-        tenant: string;
-        dataCore: string;
-        flowType: string;
-        eventTypes: string[];
-    };
-    buffer: {
-        size: number;
-        threshold: number;
-        maxRedeliveryCount: number;
-        achknowledgeTimeoutMs: number;
-    };
-    processor?: {
-        concurrency: number;
-        onEvents: (events: FlowcoreEvent[]) => Promise<boolean> | boolean;
-        onFailedEvents?: (events: FlowcoreEvent[]) => Promise<void> | void;
-        onDone?: () => Promise<void> | void;
-    };
-    stopAt?: Date;
-}
-export interface DataPumpState {
-    timeBucket: string;
-    eventId?: string;
-}
-export interface DataPumpStateManager {
-    getState: () => Promise<DataPumpState | null> | DataPumpState | null;
-    setState?: (state: DataPumpState) => Promise<void> | void;
-    stopAt?: () => Date;
-}
-export interface DataPumpNotifier {
-    wait(dataSource: DataPumpOptions["dataSource"] & {
-        dataCoreId: string;
-    }, signal?: AbortSignal): Promise<void>;
-}
-export declare class DataPump {
-    private options;
-    private logger?;
-    private state;
-    private _timeBuckets;
-    private buffer;
-    private bufferWaiter;
-    private waiter;
-    private running;
-    private notifierAbortController?;
-    private dataCoreId?;
-    private stopAt?;
-    constructor(options: DataPumpOptions);
-    start(): Promise<void>;
-    stop(): void;
-    private proceccor;
-    private eventBufferLoop;
-    private reOpen;
-    private updateState;
-    private get timeBuckets();
-    private getClosestTimeBucket;
-    private updateTimeBuckets;
-    private pullInner;
-    private acknowledgeInner;
-    private waitForNotifier;
-    private waitForEvents;
-    private waitForBufferThreshold;
-    private getNowTimeBucket;
-    private getDataCoreId;
-    pull(amount: number): Promise<FlowcoreEvent[]>;
-    acknowledge(eventIds: string[]): Promise<void>;
-}
-export {};
-//# sourceMappingURL=data-pump.d.ts.map

package/script/lib/data-pump.d.ts.map

@@ -1 +0,0 @@
-{"version":3,"file":"data-pump.d.ts","sourceRoot":"","sources":["../../src/lib/data-pump.ts"],"names":[],"mappings":"AAAA,OAAO,EAIL,KAAK,cAAc,EACnB,KAAK,aAAa,EACnB,MAAM,gDAAgD,CAAA;AAiBvD,MAAM,WAAW,MAAM;IACrB,KAAK,CAAC,OAAO,EAAE,MAAM,EAAE,GAAG,IAAI,EAAE,OAAO,EAAE,GAAG,IAAI,CAAA;IAChD,IAAI,CAAC,OAAO,EAAE,MAAM,EAAE,GAAG,IAAI,EAAE,OAAO,EAAE,GAAG,IAAI,CAAA;IAC/C,IAAI,CAAC,OAAO,EAAE,MAAM,EAAE,GAAG,IAAI,EAAE,OAAO,EAAE,GAAG,IAAI,CAAA;IAC/C,KAAK,CAAC,OAAO,EAAE,MAAM,GAAG,KAAK,EAAE,GAAG,IAAI,EAAE,OAAO,EAAE,GAAG,IAAI,CAAA;CACzD;AAED,UAAU,eAAe;IACvB,cAAc,EAAE,cAAc,CAAA;IAC9B,YAAY,EAAE,oBAAoB,CAAA;IAClC,MAAM,CAAC,EAAE,MAAM,CAAA;IACf,QAAQ,EAAE,gBAAgB,CAAA;IAC1B,UAAU,EAAE;QACV,MAAM,EAAE,MAAM,CAAA;QACd,QAAQ,EAAE,MAAM,CAAA;QAChB,QAAQ,EAAE,MAAM,CAAA;QAChB,UAAU,EAAE,MAAM,EAAE,CAAA;KACrB,CAAA;IACD,MAAM,EAAE;QACN,IAAI,EAAE,MAAM,CAAA;QACZ,SAAS,EAAE,MAAM,CAAA;QACjB,kBAAkB,EAAE,MAAM,CAAA;QAC1B,qBAAqB,EAAE,MAAM,CAAA;KAC9B,CAAA;IACD,SAAS,CAAC,EAAE;QACV,WAAW,EAAE,MAAM,CAAA;QACnB,QAAQ,EAAE,CAAC,MAAM,EAAE,aAAa,EAAE,KAAK,OAAO,CAAC,OAAO,CAAC,GAAG,OAAO,CAAA;QACjE,cAAc,CAAC,EAAE,CAAC,MAAM,EAAE,aAAa,EAAE,KAAK,OAAO,CAAC,IAAI,CAAC,GAAG,IAAI,CAAA;QAClE,MAAM,CAAC,EAAE,MAAM,OAAO,CAAC,IAAI,CAAC,GAAG,IAAI,CAAA;KACpC,CAAA;IACD,MAAM,CAAC,EAAE,IAAI,CAAA;CACd;AAED,MAAM,WAAW,aAAa;IAC5B,UAAU,EAAE,MAAM,CAAA;IAClB,OAAO,CAAC,EAAE,MAAM,CAAA;CACjB;AAED,MAAM,WAAW,oBAAoB;IACnC,QAAQ,EAAE,MAAM,OAAO,CAAC,aAAa,GAAG,IAAI,CAAC,GAAG,aAAa,GAAG,IAAI,CAAA;IACpE,QAAQ,CAAC,EAAE,CAAC,KAAK,EAAE,aAAa,KAAK,OAAO,CAAC,IAAI,CAAC,GAAG,IAAI,CAAA;IACzD,MAAM,CAAC,EAAE,MAAM,IAAI,CAAA;CACpB;AAED,MAAM,WAAW,gBAAgB;IAC/B,IAAI,CAAC,UAAU,EAAE,eAAe,CAAC,YAAY,CAAC,GAAG;QAAE,UAAU,EAAE,MAAM,CAAA;KAAE,EAAE,MAAM,CAAC,EAAE,WAAW,GAAG,OAAO,CAAC,IAAI,CAAC,CAAA;CAC9G;AAED,qBAAa,QAAQ;IAgBP,OAAO,CAAC,OAAO;IAf3B,OAAO,CAAC,MAAM,CAAC,CAAQ;IACvB,OAAO,CAAC,KAAK,CAAe;IAC5B,OAAO,CAAC,YAAY,CAAe;IACnC,OAAO,CAAC,MAAM,CAA4B;IAC1C,OAAO,CAAC,YAAY,CAA8C;IAClE,OAAO,CAAC,MAAM,CAA8C;IAC5D,OAAO,CAAC,OAAO,CAAQ;IACvB,OAAO,CAAC,uBAAuB,CAAC,CAAiB;IACjD,OAAO,CAAC,UAAU,CAAC,CAAQ;IAC3B,OAAO,CAAC,MAAM,CAAC,CAId;gBAEmB,OAAO,EAAE,eAAe;IAO/B,KAAK;IAiDX,IAAI;YAWG,SAAS;YAqBT,eAAe;YAsGf,MAAM;IAsCpB,OAAO,CAAC,WAAW;IAenB,OAAO,KAAK,WAAW,GAMtB;IAED,OAAO,CAAC,oBAAoB;YAgBd,iBAAiB;YAoBjB,SAAS;YA4CT,gBAAgB;YA2BhB,eAAe;YAUf,aAAa;YAab,sBAAsB;IAepC,OAAO,CAAC,gBAAgB;YAIV,aAAa;IAcpB,IAAI,CAAC,MAAM,EAAE,MAAM,GAAG,OAAO,CAAC,aAAa,EAAE,CAAC;IAO9C,WAAW,CAAC,QAAQ,EAAE,MAAM,EAAE,GAAG,OAAO,CAAC,IAAI,CAAC;CAMtD"}

package/script/lib/data-pump.js

@@ -1,441 +0,0 @@
-"use strict";
-Object.defineProperty(exports, "__esModule", { value: true });
-exports.DataPump = void 0;
-const mod_js_1 = require("../deps/jsr.io/@flowcore/sdk/1.26.3/src/mod.js");
-const mod_js_2 = require("../deps/jsr.io/@flowcore/time-uuid/0.1.2/src/mod.js");
-const date_fns_1 = require("date-fns");
-const utc_1 = require("@date-fns/utc");
-var FlowcoreBufferEventStatus;
-(function (FlowcoreBufferEventStatus) {
-    FlowcoreBufferEventStatus["OPEN"] = "open";
-    FlowcoreBufferEventStatus["DELIVERED"] = "delivered";
-})(FlowcoreBufferEventStatus || (FlowcoreBufferEventStatus = {}));
-class DataPump {
-    constructor(options) {
-        Object.defineProperty(this, "options", {
-            enumerable: true,
-            configurable: true,
-            writable: true,
-            value: options
-        });
-        Object.defineProperty(this, "logger", {
-            enumerable: true,
-            configurable: true,
-            writable: true,
-            value: void 0
-        });
-        Object.defineProperty(this, "state", {
-            enumerable: true,
-            configurable: true,
-            writable: true,
-            value: void 0
-        });
-        Object.defineProperty(this, "_timeBuckets", {
-            enumerable: true,
-            configurable: true,
-            writable: true,
-            value: []
-        });
-        Object.defineProperty(this, "buffer", {
-            enumerable: true,
-            configurable: true,
-            writable: true,
-            value: []
-        });
-        Object.defineProperty(this, "bufferWaiter", {
-            enumerable: true,
-            configurable: true,
-            writable: true,
-            value: null
-        });
-        Object.defineProperty(this, "waiter", {
-            enumerable: true,
-            configurable: true,
-            writable: true,
-            value: null
-        });
-        Object.defineProperty(this, "running", {
-            enumerable: true,
-            configurable: true,
-            writable: true,
-            value: false
-        });
-        Object.defineProperty(this, "notifierAbortController", {
-            enumerable: true,
-            configurable: true,
-            writable: true,
-            value: void 0
-        });
-        Object.defineProperty(this, "dataCoreId", {
-            enumerable: true,
-            configurable: true,
-            writable: true,
-            value: void 0
-        });
-        Object.defineProperty(this, "stopAt", {
-            enumerable: true,
-            configurable: true,
-            writable: true,
-            value: void 0
-        });
-        this.state = {
-            timeBucket: this.getNowTimeBucket(),
-        };
-        this.logger = options.logger ?? undefined;
-    }
-    async start() {
-        if (this.running) {
-            return;
-        }
-        await this.updateTimeBuckets();
-        if (this.options.stopAt) {
-            const timeBucket = (0, date_fns_1.format)((0, date_fns_1.startOfHour)((0, utc_1.utc)(this.options.stopAt)), "yyyyMMddHH0000");
-            const lastTimeBucket = this.getClosestTimeBucket(timeBucket, true);
-            const lastEventId = mod_js_2.TimeUuid.fromDate(this.options.stopAt).toString();
-            this.stopAt = {
-                timeBucket: lastTimeBucket,
-                eventId: lastEventId,
-                eventBufferReached: false,
-            };
-            this.logger?.debug("stop at", `${this.stopAt.timeBucket} @ ${this.stopAt.eventId}`);
-        }
-        const newState = await this.options.stateManager.getState();
-        if (!newState) {
-            this.state.timeBucket = this.getClosestTimeBucket(this.getNowTimeBucket());
-            this.state.eventId = mod_js_2.TimeUuid.now().toString();
-        }
-        else {
-            this.state.timeBucket = this.getClosestTimeBucket(newState.timeBucket);
-            this.state.eventId = newState.eventId;
-        }
-        if (this.stopAt?.timeBucket && this.state.timeBucket > this.stopAt.timeBucket) {
-            throw new Error("Stop at time is lower than current state");
-        }
-        this.running = true;
-        this.logger?.debug("Starting event buffer loop");
-        this.eventBufferLoop()
-            .then(() => this.logger?.debug("Event buffer loop stopped"))
-            .catch((error) => this.logger?.error("Error in event buffer loop", { error }))
-            .finally(() => this.stop());
-        if (this.options.processor) {
-            this.logger?.debug("Starting processor");
-            this.proceccor()
-                .then(() => this.logger?.debug("Processor stopped"))
-                .catch((error) => this.logger?.error("Error in processor", { error }))
-                .finally(() => this.stop());
-        }
-    }
-    stop() {
-        if (!this.running) {
-            return;
-        }
-        this.logger?.debug("Stopping data pump");
-        this.running = false;
-        this.notifierAbortController?.abort("stopping");
-        this.bufferWaiter?.(true);
-        this.waiter?.(true);
-    }
-    async proceccor() {
-        if (!this.options.processor) {
-            throw new Error("Processor not set");
-        }
-        while (this.running) {
-            try {
-                const events = await this.pullInner(this.options.processor.concurrency);
-                if (!this.running) {
-                    break;
-                }
-                const success = await this.options.processor.onEvents(events);
-                if (success) {
-                    await this.acknowledgeInner(events.map((event) => event.eventId));
-                }
-            }
-            catch (error) {
-                this.logger?.error("Error in processor", { error });
-                await this.stop();
-            }
-        }
-    }
-    async eventBufferLoop() {
-        // Stop buffer loop if we are not running
-        if (!this.running) {
-            return;
-        }
-        if (this.stopAt?.eventBufferReached) {
-            await new Promise((resolve) => setTimeout(resolve, 10_000));
-            return this.eventBufferLoop();
-        }
-        // Buffer is full, wait for some space
-        if (this.buffer.length >= this.options.buffer.size - this.options.buffer.threshold && this.running) {
-            this.logger?.debug("Buffer is full, waiting for some space");
-            await this.waitForBufferThreshold();
-        }
-        // Stop buffer loop if we are not running
-        if (!this.running) {
-            return;
-        }
-        // How many events should we fetch
-        const eventsToFetch = this.options.buffer.size - this.buffer.length;
-        this.logger?.info(`fetching events from ${this.state.timeBucket} @ ${this.state.eventId ?? "-"}`);
-        const timeStart = Date.now();
-        const command = new mod_js_1.EventsFetchCommand({
-            tenant: this.options.dataSource.tenant,
-            dataCoreId: await this.getDataCoreId(),
-            flowType: this.options.dataSource.flowType,
-            eventTypes: this.options.dataSource.eventTypes,
-            timeBucket: this.state.timeBucket,
-            afterEventId: this.state.eventId,
-            pageSize: eventsToFetch,
-            toEventId: this.stopAt?.eventId,
-        });
-        const result = await this.options.flowcoreClient.execute(command);
-        const time = Date.now() - timeStart;
-        // Stop buffer loop if we are not running
-        if (!this.running) {
-            return;
-        }
-        const events = result.events.map((event) => ({
-            event,
-            status: FlowcoreBufferEventStatus.OPEN,
-            deliveryCount: 0,
-            deliveryTime: 0,
-            deliveryId: "",
-        }));
-        this.buffer.push(...events);
-        // Check if we have a waiter and if we have events, notify the waiter
-        events.length && this.waiter?.();
-        this.logger?.info(`Got ${result.events.length} events from ${this.state.timeBucket} @ ${this.state.eventId ?? "-"} in ${time}ms. Buffer size ${this.buffer.length}`);
-        // Update buffer state
-        this.state.eventId = result.events[result.events.length - 1]?.eventId ?? this.state.eventId;
-        // If we don't have a next cursor move to next time bucket
-        if (!result.nextCursor) {
-            // Check if we reached the stop at
-            if (this.state.timeBucket === this.stopAt?.timeBucket) {
-                this.stopAt.eventBufferReached = true;
-                this.logger?.debug("Reached stop at");
-                this.waiter?.();
-                return this.eventBufferLoop();
-            }
-            // Get next time bucket
-            const nextTimeBucketIndex = this.timeBuckets.indexOf(this.state.timeBucket) + 1;
-            // There is no next time bucket
-            if (!this.timeBuckets[nextTimeBucketIndex]) {
-                if (result.events.length) {
-                    // If we got events, we can try to get more one more time
-                    this.logger?.debug("Try to get more events");
-                }
-                else {
-                    // If we didn't get events, we wait for notifier
-                    await this.waitForNotifier();
-                }
-            }
-            else {
-                // Set next time bucket
-                this.state.timeBucket = this.timeBuckets[nextTimeBucketIndex];
-            }
-        }
-        return this.eventBufferLoop();
-    }
-    async reOpen(eventIds, deliveryId) {
-        if (!eventIds.length) {
-            return;
-        }
-        const failedEvents = [];
-        const reopenedEvents = [];
-        let lastEvent;
-        this.buffer = this.buffer.filter((event) => {
-            if (event.deliveryId === deliveryId && this.options.buffer.maxRedeliveryCount > -1 &&
-                event.deliveryCount > this.options.buffer.maxRedeliveryCount) {
-                failedEvents.push(event.event);
-                lastEvent = event.event;
-                return false;
-            }
-            if (event.deliveryId === deliveryId && eventIds.includes(event.event.eventId)) {
-                event.status = FlowcoreBufferEventStatus.OPEN;
-                event.deliveryId = "";
-                reopenedEvents.push(event.event);
-            }
-            return true;
-        });
-        if (reopenedEvents.length) {
-            this.waiter?.();
-        }
-        if (this.bufferWaiter && this.buffer.length <= this.options.buffer.size - this.options.buffer.threshold) {
-            this.bufferWaiter();
-        }
-        if (failedEvents.length) {
-            await this.options.processor?.onFailedEvents?.(failedEvents);
-            await this.updateState(lastEvent?.eventId);
-        }
-    }
-    updateState(eventId) {
-        const firstEvent = this.buffer[0];
-        if (!firstEvent) {
-            if (eventId) {
-                const date = mod_js_2.TimeUuid.fromString(eventId).getDate();
-                const timeBucket = (0, date_fns_1.format)((0, date_fns_1.startOfHour)((0, utc_1.utc)(date)), "yyyyMMddHH0000");
-                return this.options.stateManager.setState?.({ timeBucket, eventId });
-            }
-            return;
-        }
-        const timeUuid = mod_js_2.TimeUuid.fromString(firstEvent.event.eventId);
-        const timeBucket = firstEvent.event.timeBucket;
-        return this.options.stateManager.setState?.({ timeBucket, eventId: timeUuid.getBefore().toString() });
-    }
-    get timeBuckets() {
-        const timeBucketNow = this.getNowTimeBucket();
-        if (this._timeBuckets[this._timeBuckets.length - 1] !== timeBucketNow) {
-            return [...this._timeBuckets, timeBucketNow];
-        }
-        return this._timeBuckets;
-    }
-    getClosestTimeBucket(timeBucket, getBefore = false) {
-        if (!timeBucket.match(/^\d{14}$/)) {
-            throw new Error(`Invalid timebucket: ${timeBucket}`);
-        }
-        if (getBefore) {
-            return (this.timeBuckets.findLast((t) => Number.parseFloat(t) <= Number.parseFloat(timeBucket)) ??
-                this.timeBuckets[this.timeBuckets.length - 1]);
-        }
-        return (this.timeBuckets.find((t) => Number.parseFloat(t) >= Number.parseFloat(timeBucket)) ??
-            this.timeBuckets[this.timeBuckets.length - 1]);
-    }
-    async updateTimeBuckets() {
-        this._timeBuckets = [];
-        let first = true;
-        let cursor;
-        while (cursor !== undefined || first) {
-            first = false;
-            const command = new mod_js_1.EventsFetchTimeBucketsByNamesCommand({
-                tenant: this.options.dataSource.tenant,
-                dataCoreId: await this.getDataCoreId(),
-                flowType: this.options.dataSource.flowType,
-                eventTypes: this.options.dataSource.eventTypes,
-                pageSize: 10_000,
-            });
-            // sdkCommandCount.labels("EventsFetchTimeBucketsByNamesCommand").inc(1)
-            const result = await this.options.flowcoreClient.execute(command);
-            this._timeBuckets.push(...result.timeBuckets);
-            cursor = result.nextCursor;
-        }
-    }
-    async pullInner(amount) {
-        if (!this.running) {
-            throw new Error("Data pump not running");
-        }
-        if (this.stopAt?.eventBufferReached && this.buffer.length === 0) {
-            await this.options.processor?.onDone?.();
-            await this.stop();
-            return [];
-        }
-        const deliveryId = crypto.randomUUID();
-        const events = [];
-        for (const event of this.buffer) {
-            if (event.status === FlowcoreBufferEventStatus.OPEN) {
-                event.status = FlowcoreBufferEventStatus.DELIVERED;
-                event.deliveryCount++;
-                event.deliveryId = deliveryId;
-                events.push(event.event);
-                if (events.length >= amount) {
-                    break;
-                }
-            }
-        }
-        if (!events.length) {
-            await this.waitForEvents();
-            if (!this.running) {
-                return [];
-            }
-            return this.pullInner(amount);
-        }
-        setTimeout(() => {
-            this.reOpen(events.map((event) => event.eventId), deliveryId);
-        }, this.options.buffer.achknowledgeTimeoutMs);
-        return events;
-    }
-    async acknowledgeInner(eventIds) {
-        if (!eventIds.length) {
-            return;
-        }
-        // this.logger.info("Acknowledging events", { events: eventIds.length })
-        const lastEventInBuffer = this.buffer[this.buffer.length - 1];
-        this.buffer = this.buffer.filter((event) => {
-            if (eventIds.includes(event.event.eventId)) {
-                return false;
-            }
-            return true;
-        });
-        if (this.bufferWaiter && this.buffer.length <= this.options.buffer.size - this.options.buffer.threshold) {
-            this.bufferWaiter();
-        }
-        if (!this.buffer.length) {
-            await this.updateState(lastEventInBuffer?.event.eventId);
-        }
-        else {
-            await this.updateState();
-        }
-        if (!this.buffer.length && this.stopAt?.eventBufferReached) {
-            await this.options.processor?.onDone?.();
-            await this.stop();
-        }
-    }
-    async waitForNotifier() {
-        this.logger?.info("No more events, waiting for notifier...");
-        this.notifierAbortController = new AbortController();
-        await this.options.notifier.wait({ ...this.options.dataSource, dataCoreId: await this.getDataCoreId() }, this.notifierAbortController.signal);
-        this.notifierAbortController = undefined;
-    }
-    async waitForEvents() {
-        this.logger?.debug("No events, waiting for events");
-        await new Promise((resolve) => {
-            this.waiter = (stopping) => {
-                if (!stopping) {
-                    this.logger?.debug("Notifying waiter that we have events");
-                }
-                resolve();
-            };
-        });
-        this.waiter = null;
-    }
-    async waitForBufferThreshold() {
-        if (this.buffer.length < this.options.buffer.size - this.options.buffer.threshold) {
-            return;
-        }
-        await new Promise((resolve) => {
-            this.bufferWaiter = (stopping) => {
-                if (!stopping) {
-                    this.logger?.debug("Notifying buffer waiter that we are under threshold");
-                }
-                resolve();
-            };
-        });
-        this.bufferWaiter = null;
-    }
-    getNowTimeBucket() {
-        return (0, date_fns_1.format)((0, date_fns_1.startOfHour)((0, utc_1.utc)(new Date())), "yyyyMMddHH0000");
-    }
-    async getDataCoreId() {
-        if (this.dataCoreId) {
-            return this.dataCoreId;
-        }
-        const dataCore = await this.options.flowcoreClient.execute(new mod_js_1.DataCoreFetchCommand({
-            dataCore: this.options.dataSource.dataCore,
-            tenant: this.options.dataSource.tenant,
-        }));
-        this.dataCoreId = dataCore.id;
-        return this.dataCoreId;
-    }
-    pull(amount) {
-        if (this.options.processor) {
-            throw new Error("Pull is not supported when processor is set");
-        }
-        return this.pullInner(amount);
-    }
-    acknowledge(eventIds) {
-        if (this.options.processor) {
-            throw new Error("Acknowledge is not supported when processor is set");
-        }
-        return this.acknowledgeInner(eventIds);
-    }
-}
-exports.DataPump = DataPump;

package/script/lib/nats-notifier.d.ts

@@ -1,18 +0,0 @@
-import type { DataPumpNotifier } from "./data-pump.js";
-interface NatsClientOptions {
-    servers: string[];
-    timeoutMs?: number;
-}
-export declare class NatsNotifier {
-    private readonly options;
-    private nats;
-    private timeoutMs;
-    constructor(options: NatsClientOptions);
-    private start;
-    private stop;
-    private awaitMessage;
-    private getSubscription;
-    getNotifier(): DataPumpNotifier;
-}
-export {};
-//# sourceMappingURL=nats-notifier.d.ts.map

package/script/lib/nats-notifier.d.ts.map

@@ -1 +0,0 @@
-{"version":3,"file":"nats-notifier.d.ts","sourceRoot":"","sources":["../../src/lib/nats-notifier.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAE,gBAAgB,EAAE,MAAM,gBAAgB,CAAA;AAEtD,UAAU,iBAAiB;IACzB,OAAO,EAAE,MAAM,EAAE,CAAA;IACjB,SAAS,CAAC,EAAE,MAAM,CAAA;CACnB;AAED,qBAAa,YAAY;IAGX,OAAO,CAAC,QAAQ,CAAC,OAAO;IAFpC,OAAO,CAAC,IAAI,CAAiC;IAC7C,OAAO,CAAC,SAAS,CAAQ;gBACI,OAAO,EAAE,iBAAiB;YAIzC,KAAK;YAOL,IAAI;IAOlB,OAAO,CAAC,YAAY;IAoCpB,OAAO,CAAC,eAAe;IAgBhB,WAAW,IAAI,gBAAgB;CAavC"}