@flowcore/pathways 0.16.4 → 0.17.1

package/script/deps/jsr.io/@flowcore/data-pump/0.16.5/src/data-pump/data-pump.d.ts

@@ -0,0 +1,74 @@
+import type { FlowcoreEvent } from "../../../../sdk/1.71.0/src/mod.js";
+import { FlowcoreDataSource } from "./data-source.js";
+import type { FlowcoreDataPumpAuth, FlowcoreDataPumpDataSource, FlowcoreDataPumpProcessor, FlowcoreDataPumpState, FlowcoreDataPumpStateManager, FlowcoreLogger } from "./types.js";
+interface FlowcoreDataPumpNotifierNatsOptions {
+    type: "nats";
+    servers: string[];
+}
+interface FlowcoreDataPumpNotifierWebsocketOptions {
+    type: "websocket";
+}
+interface FlowcoreDataPumpNotifierPollerOptions {
+    type: "poller";
+    /**
+     * The interval in milliseconds to poll the data pump state (min 1000 ms)
+     */
+    intervalMs: number;
+}
+type FlowcoreDataPumpNotifierOptions = FlowcoreDataPumpNotifierNatsOptions | FlowcoreDataPumpNotifierWebsocketOptions | FlowcoreDataPumpNotifierPollerOptions;
+export interface FlowcoreDataPumpOptions {
+    auth: FlowcoreDataPumpAuth;
+    dataSource: FlowcoreDataPumpDataSource;
+    stateManager: FlowcoreDataPumpStateManager;
+    bufferSize?: number;
+    maxRedeliveryCount?: number;
+    achknowledgeTimeoutMs?: number;
+    includeSensitiveData?: boolean;
+    processor?: FlowcoreDataPumpProcessor;
+    notifier?: FlowcoreDataPumpNotifierOptions;
+    logger?: FlowcoreLogger;
+    stopAt?: Date;
+    baseUrlOverride?: string;
+    noTranslation?: boolean;
+    directMode?: boolean;
+}
+export declare class FlowcoreDataPump {
+    readonly dataSource: FlowcoreDataSource;
+    private readonly notifier;
+    private stateManager;
+    private readonly options;
+    private readonly logger?;
+    private nextCursor?;
+    private running;
+    private restartTo?;
+    private abortController?;
+    private buffer;
+    private bufferState;
+    private stopAtState?;
+    private isLive;
+    private finallyFailedHandler?;
+    private constructor();
+    get isRunning(): boolean;
+    static create(options: FlowcoreDataPumpOptions, dataSourceOverride?: FlowcoreDataSource): FlowcoreDataPump;
+    start(callback?: (error?: Error) => void): Promise<void>;
+    restart(state: FlowcoreDataPumpState, stopAt?: Date | null): void;
+    stop(isRestart?: boolean): void;
+    private updateState;
+    private loop;
+    reserve(amount: number): Promise<FlowcoreEvent[]>;
+    onFinalyFailed(handler: (events: FlowcoreEvent[]) => Promise<void> | void): void;
+    acknowledge(eventIds: string[]): Promise<void>;
+    fail(eventIds: string[]): Promise<void>;
+    private reOpen;
+    private processLoop;
+    private updateMetricsGauges;
+    private incMetricsCounter;
+    private waiterEvents?;
+    private waitForEvents;
+    private waiterBufferThreshold?;
+    private waitForBufferThreshold;
+    private waiterBufferEmpty?;
+    private waitForBufferEmpty;
+}
+export {};
+//# sourceMappingURL=data-pump.d.ts.map

package/script/deps/jsr.io/@flowcore/data-pump/0.16.5/src/data-pump/data-pump.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"data-pump.d.ts","sourceRoot":"","sources":["../../../../../../../../src/deps/jsr.io/@flowcore/data-pump/0.16.5/src/data-pump/data-pump.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAE,aAAa,EAAE,MAAM,mCAAmC,CAAA;AAGtE,OAAO,EAAE,kBAAkB,EAAE,MAAM,kBAAkB,CAAA;AAGrD,OAAO,KAAK,EACR,oBAAoB,EACpB,0BAA0B,EAC1B,yBAAyB,EACzB,qBAAqB,EACrB,4BAA4B,EAC5B,cAAc,EACjB,MAAM,YAAY,CAAA;AAEnB,UAAU,mCAAmC;IAC3C,IAAI,EAAE,MAAM,CAAA;IACZ,OAAO,EAAE,MAAM,EAAE,CAAA;CAClB;AAED,UAAU,wCAAwC;IAChD,IAAI,EAAE,WAAW,CAAA;CAClB;AAED,UAAU,qCAAqC;IAC7C,IAAI,EAAE,QAAQ,CAAA;IACd;;OAEG;IACH,UAAU,EAAE,MAAM,CAAA;CACnB;AAED,KAAK,+BAA+B,GAChC,mCAAmC,GACnC,wCAAwC,GACxC,qCAAqC,CAAA;AAEzC,MAAM,WAAW,uBAAuB;IACtC,IAAI,EAAE,oBAAoB,CAAA;IAC1B,UAAU,EAAE,0BAA0B,CAAA;IACtC,YAAY,EAAE,4BAA4B,CAAA;IAC1C,UAAU,CAAC,EAAE,MAAM,CAAA;IACnB,kBAAkB,CAAC,EAAE,MAAM,CAAA;IAC3B,qBAAqB,CAAC,EAAE,MAAM,CAAA;IAC9B,oBAAoB,CAAC,EAAE,OAAO,CAAA;IAC9B,SAAS,CAAC,EAAE,yBAAyB,CAAA;IACrC,QAAQ,CAAC,EAAE,+BAA+B,CAAA;IAC1C,MAAM,CAAC,EAAE,cAAc,CAAA;IACvB,MAAM,CAAC,EAAE,IAAI,CAAA;IACb,eAAe,CAAC,EAAE,MAAM,CAAA;IACxB,aAAa,CAAC,EAAE,OAAO,CAAA;IACvB,UAAU,CAAC,EAAE,OAAO,CAAA;CACrB;AAmBD,qBAAa,gBAAgB;aAYT,UAAU,EAAE,kBAAkB;IAC9C,OAAO,CAAC,QAAQ,CAAC,QAAQ;IACzB,OAAO,CAAC,YAAY;IACpB,OAAO,CAAC,QAAQ,CAAC,OAAO;IACxB,OAAO,CAAC,QAAQ,CAAC,MAAM,CAAC;IAf1B,OAAO,CAAC,UAAU,CAAC,CAAQ;IAC3B,OAAO,CAAC,OAAO,CAAQ;IACvB,OAAO,CAAC,SAAS,CAAC,CAAuB;IACzC,OAAO,CAAC,eAAe,CAAC,CAAiB;IACzC,OAAO,CAAC,MAAM,CAAmC;IACjD,OAAO,CAAC,WAAW,CAAuB;IAC1C,OAAO,CAAC,WAAW,CAAC,CAAuB;IAC3C,OAAO,CAAC,MAAM,CAAQ;IACtB,OAAO,CAAC,oBAAoB,CAAC,CAAmD;IAEhF,OAAO;IAaP,IAAW,SAAS,IAAI,OAAO,CAE9B;WAEa,MAAM,CAAC,OAAO,EAAE,uBAAuB,EAAE,kBAAkB,CAAC,EAAE,kBAAkB,GAAG,gBAAgB;IA0CpG,KAAK,CAAC,QAAQ,CAAC,EAAE,CAAC,KAAK,CAAC,EAAE,KAAK,KAAK,IAAI,GAAG,OAAO,CAAC,IAAI,CAAC;IA2C9D,OAAO,CAAC,KAAK,EAAE,qBAAqB,EAAE,MAAM,CAAC,EAAE,IAAI,GAAG,IAAI,GAAG,IAAI;IAQjE,IAAI,CAAC,SAAS,UAAQ,GAAG,IAAI;IAWpC,OAAO,CAAC,WAAW;YAaL,IAAI;IAmFL,OAAO,CAAC,MAAM,EAAE,MAAM,GAAG,OAAO,CAAC,aAAa,EAAE,CAAC;IAqCvD,cAAc,CAAC,OAAO,EAAE,CAAC,MAAM,EAAE,aAAa,EAAE,KAAK,OAAO,CAAC,IAAI,CAAC,GAAG,IAAI;IAInE,WAAW,CAAC,QAAQ,EAAE,MAAM,EAAE;IA0B9B,IAAI,CAAC,QAAQ,EAAE,MAAM,EAAE;YA4BtB,MAAM;YAkDN,WAAW;IAiBzB,OAAO,CAAC,mBAAmB;IA4D3B,OAAO,CAAC,iBAAiB;IA0CzB,OAAO,CAAC,YAAY,CAAC,CAAY;YACnB,aAAa;IAO3B,OAAO,CAAC,qBAAqB,CAAC,CAAY;YAC5B,sBAAsB;IAOpC,OAAO,CAAC,iBAAiB,CAAC,CAAY;YACxB,kBAAkB;CAWjC"}

package/script/deps/jsr.io/@flowcore/data-pump/0.16.5/src/data-pump/data-pump.js

@@ -0,0 +1,505 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.FlowcoreDataPump = void 0;
+const utc_1 = require("@date-fns/utc");
+const mod_js_1 = require("../../../../time-uuid/0.1.2/src/mod.js");
+const date_fns_1 = require("date-fns");
+const data_source_js_1 = require("./data-source.js");
+const metrics_js_1 = require("./metrics.js");
+const notifier_js_1 = require("./notifier.js");
+class FlowcoreDataPump {
+    constructor(dataSource, notifier, stateManager, options, logger) {
+        Object.defineProperty(this, "dataSource", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: dataSource
+        });
+        Object.defineProperty(this, "notifier", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: notifier
+        });
+        Object.defineProperty(this, "stateManager", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: stateManager
+        });
+        Object.defineProperty(this, "options", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: options
+        });
+        Object.defineProperty(this, "logger", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: logger
+        });
+        Object.defineProperty(this, "nextCursor", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: void 0
+        });
+        Object.defineProperty(this, "running", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: false
+        });
+        Object.defineProperty(this, "restartTo", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: void 0
+        });
+        Object.defineProperty(this, "abortController", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: void 0
+        });
+        Object.defineProperty(this, "buffer", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: []
+        });
+        Object.defineProperty(this, "bufferState", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: void 0
+        });
+        Object.defineProperty(this, "stopAtState", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: void 0
+        });
+        Object.defineProperty(this, "isLive", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: false
+        });
+        Object.defineProperty(this, "finallyFailedHandler", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: void 0
+        });
+        // #endregion
+        // #region Waiters
+        Object.defineProperty(this, "waiterEvents", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: void 0
+        });
+        Object.defineProperty(this, "waiterBufferThreshold", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: void 0
+        });
+        Object.defineProperty(this, "waiterBufferEmpty", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: void 0
+        });
+        this.bufferState = {
+            timeBucket: (0, date_fns_1.format)((0, date_fns_1.startOfHour)((0, utc_1.utc)(new Date())), "yyyyMMddHH0000"),
+            eventId: mod_js_1.TimeUuid.now().toString(),
+        };
+    }
+    get isRunning() {
+        return this.running;
+    }
+    static create(options, dataSourceOverride) {
+        if ("apiKey" in options.auth && !options.auth.apiKeyId) {
+            const parts = options.auth.apiKey.split("_");
+            if (parts.length !== 3 || parts[0] !== "fc") {
+                throw new Error("Invalid API key");
+            }
+            options.auth.apiKeyId = parts[1];
+        }
+        const dataSource = dataSourceOverride ?? new data_source_js_1.FlowcoreDataSource({
+            auth: options.auth,
+            dataSource: options.dataSource,
+            baseUrlOverride: options.baseUrlOverride,
+            noTranslation: options.noTranslation,
+            directMode: options.directMode,
+        });
+        const notifier = new notifier_js_1.FlowcoreNotifier({
+            auth: options.auth,
+            dataSource: options.dataSource,
+            natsServers: options.notifier?.type === "nats" ? options.notifier.servers : undefined,
+            pollerIntervalMs: options.notifier?.type === "poller" ? options.notifier.intervalMs : undefined,
+            logger: options.logger,
+            directMode: options.directMode,
+            noTranslation: options.noTranslation,
+        });
+        return new FlowcoreDataPump(dataSource, notifier, options.stateManager, {
+            bufferSize: options.bufferSize ?? 1000,
+            bufferThreshold: Math.ceil((options.bufferSize ?? 1000) * 0.1),
+            maxRedeliveryCount: options.maxRedeliveryCount ?? 3,
+            achknowledgeTimeoutMs: options.achknowledgeTimeoutMs ?? 5_000,
+            includeSensitiveData: options.includeSensitiveData ?? false,
+            processor: options.processor,
+            stopAt: options.stopAt,
+        }, options.logger);
+    }
+    async start(callback) {
+        this.isLive = false;
+        if (this.running) {
+            throw new Error("Data pump already running");
+        }
+        this.running = true;
+        this.nextCursor = undefined;
+        this.updateMetricsGauges();
+        const currentState = await this.stateManager.getState();
+        const timeBucket = currentState
+            ? await this.dataSource.getClosestTimeBucket(currentState.timeBucket)
+            : (0, date_fns_1.format)((0, date_fns_1.startOfHour)((0, utc_1.utc)(new Date())), "yyyyMMddHH0000");
+        this.bufferState = {
+            timeBucket: timeBucket ?? (0, date_fns_1.format)((0, date_fns_1.startOfHour)((0, utc_1.utc)(new Date())), "yyyyMMddHH0000"),
+            eventId: currentState ? currentState.eventId : mod_js_1.TimeUuid.now().toString(),
+        };
+        if (this.options.stopAt) {
+            this.stopAtState = {
+                timeBucket: (await this.dataSource.getClosestTimeBucket((0, date_fns_1.format)((0, date_fns_1.startOfHour)((0, utc_1.utc)(this.options.stopAt)), "yyyyMMddHH0000"), true)) ?? (0, date_fns_1.format)((0, date_fns_1.startOfHour)((0, utc_1.utc)(new Date())), "yyyyMMddHH0000"),
+                eventId: mod_js_1.TimeUuid.fromDate(this.options.stopAt).toString(),
+            };
+        }
+        if (this.options.processor) {
+            this.processLoop().catch((error) => {
+                this.logger?.error("Error in processor", { error });
+                this.stop();
+            });
+        }
+        if (!callback) {
+            return this.loop();
+        }
+        void this.loop()
+            .then(() => callback())
+            .catch((error) => callback(error));
+    }
+    restart(state, stopAt) {
+        this.restartTo = state;
+        if (stopAt !== undefined) {
+            this.options.stopAt = stopAt ?? undefined;
+        }
+        this.stop(true);
+    }
+    stop(isRestart = false) {
+        this.running = false;
+        this.buffer = [];
+        this.updateMetricsGauges();
+        this.abortController?.abort();
+        this.waiterBufferThreshold?.();
+        if (!isRestart) {
+            this.waiterEvents?.();
+        }
+    }
+    updateState(eventId) {
+        if (!this.stateManager.setState) {
+            return;
+        }
+        const stateEventId = eventId ?? this.buffer[0]?.event.eventId;
+        if (!stateEventId) {
+            return;
+        }
+        const date = mod_js_1.TimeUuid.fromString(stateEventId).getDate();
+        const timeBucket = (0, date_fns_1.format)((0, date_fns_1.startOfHour)((0, utc_1.utc)(date)), "yyyyMMddHH0000");
+        return this.stateManager.setState?.({ timeBucket, eventId: stateEventId });
+    }
+    async loop() {
+        do {
+            const amountToFetch = this.options.bufferSize - this.buffer.length;
+            if (amountToFetch <= 0) {
+                this.logger?.info("Buffer is full, waiting for space");
+                await this.waitForBufferThreshold();
+                continue;
+            }
+            this.logger?.debug(`fetching ${amountToFetch} events from ${this.bufferState.timeBucket}(${this.nextCursor})`);
+            const { events, nextCursor } = await this.dataSource.getEvents(this.bufferState, amountToFetch, this.stopAtState?.eventId, this.nextCursor, this.options.includeSensitiveData);
+            if (!this.running) {
+                break;
+            }
+            this.logger?.debug(`fetched ${events.length} events`);
+            this.buffer.push(...events.map((event) => ({ event, status: "open", deliveryCount: 0 })));
+            this.nextCursor = nextCursor;
+            this.updateMetricsGauges();
+            events.length && this.waiterEvents?.();
+            this.bufferState.eventId = events[events.length - 1]?.eventId ?? this.bufferState.eventId;
+            if (this.stopAtState?.timeBucket && this.bufferState.timeBucket >= this.stopAtState.timeBucket && !events.length) {
+                this.logger?.info("Stopping at stopAt state");
+                await this.waitForBufferEmpty();
+                this.stop();
+                break;
+            }
+            if (!this.nextCursor) {
+                const timeBucket = await this.dataSource.getNextTimeBucket(this.bufferState.timeBucket);
+                if (!this.running) {
+                    break;
+                }
+                if (timeBucket) {
+                    this.bufferState.timeBucket = timeBucket;
+                }
+                else {
+                    const previousTimeBucket = this.bufferState.timeBucket;
+                    this.bufferState.timeBucket = (0, date_fns_1.format)((0, date_fns_1.startOfHour)((0, utc_1.utc)(new Date())), "yyyyMMddHH0000");
+                    if (previousTimeBucket === this.bufferState.timeBucket && !events.length) {
+                        this.isLive = true;
+                        this.logger?.debug("Going live...");
+                        this.abortController = new AbortController();
+                        await this.notifier.wait(this.abortController.signal);
+                    }
+                    else if (this.isLive) {
+                        await new Promise((resolve) => setTimeout(resolve, 1000));
+                    }
+                }
+            }
+        } while (this.running);
+        if (this.restartTo) {
+            await this.dataSource.getTimeBuckets(true);
+            this.restartTo.timeBucket = (await this.dataSource.getClosestTimeBucket(this.restartTo.timeBucket)) ??
+                (0, date_fns_1.format)((0, date_fns_1.startOfHour)((0, utc_1.utc)(new Date())), "yyyyMMddHH0000");
+            this.nextCursor = undefined;
+            this.bufferState = this.restartTo;
+            this.restartTo = undefined;
+            this.running = true;
+            return this.loop();
+        }
+        this.logger?.info("Data pump stopped");
+    }
+    // #region Puller
+    async reserve(amount) {
+        if (!this.running) {
+            return [];
+        }
+        const events = [];
+        const deliveryId = crypto.randomUUID();
+        for (const event of this.buffer) {
+            if (event.status === "open") {
+                event.status = "reserved";
+                event.deliveryId = deliveryId;
+                event.deliveryCount++;
+                events.push(event.event);
+                this.incMetricsCounter("pulled", event.event.eventType, JSON.stringify(event.event).length);
+                if (events.length === amount) {
+                    break;
+                }
+            }
+        }
+        if (!events.length) {
+            await this.waitForEvents();
+            return this.reserve(amount);
+        }
+        this.updateMetricsGauges();
+        setTimeout(() => {
+            this.reOpen(events.map((event) => event.eventId), deliveryId);
+        }, this.options.achknowledgeTimeoutMs);
+        return events;
+    }
+    onFinalyFailed(handler) {
+        this.finallyFailedHandler = handler;
+    }
+    async acknowledge(eventIds) {
+        if (!this.running) {
+            return;
+        }
+        const lastEventInBuffer = this.buffer[this.buffer.length - 1];
+        this.buffer = this.buffer.filter((event) => {
+            if (eventIds.includes(event.event.eventId)) {
+                this.incMetricsCounter("acknowledged", event.event.eventType, 1);
+                return false;
+            }
+            return true;
+        });
+        if (this.buffer.length <= this.options.bufferSize - this.options.bufferThreshold) {
+            this.waiterBufferThreshold?.();
+        }
+        await this.updateState(this.buffer.length ? undefined : lastEventInBuffer?.event.eventId);
+        this.updateMetricsGauges();
+        if (!this.buffer.length) {
+            this.waiterBufferEmpty?.();
+        }
+    }
+    async fail(eventIds) {
+        if (!this.running || !eventIds.length) {
+            return;
+        }
+        const lastEventInBuffer = this.buffer[this.buffer.length - 1];
+        const failedEvents = [];
+        this.buffer = this.buffer.filter((event) => {
+            if (eventIds.includes(event.event.eventId)) {
+                this.incMetricsCounter("failed", event.event.eventType, 1);
+                failedEvents.push(event.event);
+                return false;
+            }
+            return true;
+        });
+        this.logger?.info(`Failed ${failedEvents.length} events`);
+        void this.options.processor?.failedHandler?.(failedEvents);
+        if (this.buffer.length <= this.options.bufferSize - this.options.bufferThreshold) {
+            this.waiterBufferThreshold?.();
+        }
+        await this.updateState(this.buffer.length ? undefined : lastEventInBuffer?.event.eventId);
+        if (!this.buffer.length) {
+            this.waiterBufferEmpty?.();
+        }
+    }
+    async reOpen(eventIds, deliveryId) {
+        let lastEvent;
+        const failedEvents = [];
+        const reopenedEvents = [];
+        this.buffer = this.buffer.filter((event) => {
+            if (event.deliveryId !== deliveryId || !eventIds.includes(event.event.eventId)) {
+                return true;
+            }
+            if (this.options.maxRedeliveryCount > -1 && event.deliveryCount > this.options.maxRedeliveryCount) {
+                this.incMetricsCounter("failed", event.event.eventType, 1);
+                failedEvents.push(event.event);
+                lastEvent = event.event;
+                return false;
+            }
+            event.status = "open";
+            event.deliveryId = undefined;
+            reopenedEvents.push(event.event);
+            return true;
+        });
+        this.updateMetricsGauges();
+        if (reopenedEvents.length) {
+            this.logger?.info(`Reopened ${reopenedEvents.length} events`);
+            await this.waiterEvents?.();
+        }
+        if (!failedEvents.length) {
+            return;
+        }
+        this.logger?.info(`Failed ${failedEvents.length} events`);
+        void this.options.processor?.failedHandler?.(failedEvents);
+        void this.finallyFailedHandler?.(failedEvents);
+        if (this.buffer.length <= this.options.bufferSize - this.options.bufferThreshold) {
+            this.waiterBufferThreshold?.();
+        }
+        await this.updateState(this.buffer.length ? undefined : lastEvent?.eventId);
+        if (!this.buffer.length) {
+            this.waiterBufferEmpty?.();
+        }
+    }
+    // #endregion
+    // #region Pusher
+    async processLoop() {
+        while (this.running) {
+            try {
+                const events = await this.reserve(this.options.processor?.concurrency ?? 1);
+                await this.options.processor?.handler(events);
+                await this.acknowledge(events.map((event) => event.eventId));
+            }
+            catch (error) {
+                const errorMessage = error instanceof Error ? error.message : "Unknown error";
+                this.logger?.error(`Failed to process events: ${errorMessage}`);
+            }
+        }
+    }
+    // #endregion
+    // #region Metrics
+    updateMetricsGauges() {
+        const stats = new Map();
+        for (const eventType of this.dataSource.eventTypes) {
+            stats.set(eventType, {
+                eventCount: 0,
+                eventReservedCount: 0,
+                eventSizeBytes: 0,
+            });
+        }
+        for (const item of this.buffer) {
+            const stat = stats.get(item.event.eventType);
+            if (!stat) {
+                continue;
+            }
+            stat.eventCount++;
+            if (item.status === "reserved") {
+                stat.eventReservedCount++;
+            }
+            stat.eventSizeBytes += JSON.stringify(item.event.payload).length;
+        }
+        for (const [eventType, stat] of stats) {
+            metrics_js_1.metrics.bufferEventCountGauge.set({
+                tenant: this.dataSource.tenant,
+                data_core: this.dataSource.dataCore,
+                flow_type: this.dataSource.flowType,
+                event_type: eventType,
+            }, stat.eventCount);
+            metrics_js_1.metrics.bufferReservedEventCountGauge.set({
+                tenant: this.dataSource.tenant,
+                data_core: this.dataSource.dataCore,
+                flow_type: this.dataSource.flowType,
+                event_type: eventType,
+            }, stat.eventReservedCount);
+            metrics_js_1.metrics.bufferSizeBytesGauge.set({
+                tenant: this.dataSource.tenant,
+                data_core: this.dataSource.dataCore,
+                flow_type: this.dataSource.flowType,
+                event_type: eventType,
+            }, stat.eventSizeBytes);
+        }
+    }
+    incMetricsCounter(name, eventType, value) {
+        switch (name) {
+            case "acknowledged":
+                metrics_js_1.metrics.eventsAcknowledgedCounter.inc({
+                    tenant: this.dataSource.tenant,
+                    data_core: this.dataSource.dataCore,
+                    flow_type: this.dataSource.flowType,
+                    event_type: eventType,
+                }, value);
+                break;
+            case "failed":
+                metrics_js_1.metrics.eventsFailedCounter.inc({
+                    tenant: this.dataSource.tenant,
+                    data_core: this.dataSource.dataCore,
+                    flow_type: this.dataSource.flowType,
+                    event_type: eventType,
+                }, value);
+                break;
+            case "pulled":
+                metrics_js_1.metrics.eventsPulledSizeBytesCounter.inc({
+                    tenant: this.dataSource.tenant,
+                    data_core: this.dataSource.dataCore,
+                    flow_type: this.dataSource.flowType,
+                    event_type: eventType,
+                }, value);
+                break;
+        }
+    }
+    async waitForEvents() {
+        const promise = new Promise((resolve) => {
+            this.waiterEvents = resolve;
+        });
+        await promise;
+    }
+    async waitForBufferThreshold() {
+        const promise = new Promise((resolve) => {
+            this.waiterBufferThreshold = resolve;
+        });
+        await promise;
+    }
+    async waitForBufferEmpty() {
+        if (!this.buffer.length) {
+            return;
+        }
+        const promise = new Promise((resolve) => {
+            this.waiterBufferEmpty = resolve;
+        });
+        await promise;
+    }
+}
+exports.FlowcoreDataPump = FlowcoreDataPump;