@flowcore/data-pump 0.21.4 → 0.23.0

package/script/data-pump/data-pump.js

@@ -1,642 +0,0 @@
-"use strict";
-Object.defineProperty(exports, "__esModule", { value: true });
-exports.FlowcoreDataPump = void 0;
-const utc_1 = require("@date-fns/utc");
-const mod_js_1 = require("../deps/jsr.io/@flowcore/time-uuid/0.1.2/src/mod.js");
-const date_fns_1 = require("date-fns");
-const data_source_js_1 = require("./data-source.js");
-const metrics_js_1 = require("./metrics.js");
-const notifier_js_1 = require("./notifier.js");
-const pulse_js_1 = require("./pulse.js");
-class FlowcoreDataPump {
-    constructor(dataSource, notifier, stateManager, options, logger) {
-        Object.defineProperty(this, "dataSource", {
-            enumerable: true,
-            configurable: true,
-            writable: true,
-            value: dataSource
-        });
-        Object.defineProperty(this, "notifier", {
-            enumerable: true,
-            configurable: true,
-            writable: true,
-            value: notifier
-        });
-        Object.defineProperty(this, "stateManager", {
-            enumerable: true,
-            configurable: true,
-            writable: true,
-            value: stateManager
-        });
-        Object.defineProperty(this, "options", {
-            enumerable: true,
-            configurable: true,
-            writable: true,
-            value: options
-        });
-        Object.defineProperty(this, "logger", {
-            enumerable: true,
-            configurable: true,
-            writable: true,
-            value: logger
-        });
-        Object.defineProperty(this, "nextCursor", {
-            enumerable: true,
-            configurable: true,
-            writable: true,
-            value: void 0
-        });
-        Object.defineProperty(this, "running", {
-            enumerable: true,
-            configurable: true,
-            writable: true,
-            value: false
-        });
-        Object.defineProperty(this, "restartTo", {
-            enumerable: true,
-            configurable: true,
-            writable: true,
-            value: void 0
-        });
-        Object.defineProperty(this, "abortController", {
-            enumerable: true,
-            configurable: true,
-            writable: true,
-            value: void 0
-        });
-        Object.defineProperty(this, "buffer", {
-            enumerable: true,
-            configurable: true,
-            writable: true,
-            value: []
-        });
-        Object.defineProperty(this, "bufferState", {
-            enumerable: true,
-            configurable: true,
-            writable: true,
-            value: void 0
-        });
-        Object.defineProperty(this, "stopAtState", {
-            enumerable: true,
-            configurable: true,
-            writable: true,
-            value: void 0
-        });
-        Object.defineProperty(this, "isLive", {
-            enumerable: true,
-            configurable: true,
-            writable: true,
-            value: false
-        });
-        Object.defineProperty(this, "finallyFailedHandler", {
-            enumerable: true,
-            configurable: true,
-            writable: true,
-            value: void 0
-        });
-        Object.defineProperty(this, "pulseEmitter", {
-            enumerable: true,
-            configurable: true,
-            writable: true,
-            value: void 0
-        });
-        Object.defineProperty(this, "startedAt", {
-            enumerable: true,
-            configurable: true,
-            writable: true,
-            value: 0
-        });
-        Object.defineProperty(this, "acknowledgedCount", {
-            enumerable: true,
-            configurable: true,
-            writable: true,
-            value: 0
-        });
-        Object.defineProperty(this, "failedCount", {
-            enumerable: true,
-            configurable: true,
-            writable: true,
-            value: 0
-        });
-        Object.defineProperty(this, "pulledCount", {
-            enumerable: true,
-            configurable: true,
-            writable: true,
-            value: 0
-        });
-        Object.defineProperty(this, "processLoopRestartAttempts", {
-            enumerable: true,
-            configurable: true,
-            writable: true,
-            value: 0
-        });
-        Object.defineProperty(this, "mainLoopRestartAttempts", {
-            enumerable: true,
-            configurable: true,
-            writable: true,
-            value: 0
-        });
-        // #endregion
-        // #region Waiters
-        Object.defineProperty(this, "waiterEvents", {
-            enumerable: true,
-            configurable: true,
-            writable: true,
-            value: void 0
-        });
-        Object.defineProperty(this, "waiterBufferThreshold", {
-            enumerable: true,
-            configurable: true,
-            writable: true,
-            value: void 0
-        });
-        Object.defineProperty(this, "waiterBufferEmpty", {
-            enumerable: true,
-            configurable: true,
-            writable: true,
-            value: void 0
-        });
-        this.bufferState = {
-            timeBucket: (0, date_fns_1.format)((0, date_fns_1.startOfHour)((0, utc_1.utc)(new Date())), "yyyyMMddHH0000"),
-            eventId: mod_js_1.TimeUuid.now().toString(),
-        };
-    }
-    get isRunning() {
-        return this.running;
-    }
-    getSnapshot() {
-        if (!this.running)
-            return null;
-        const reserved = this.buffer.filter((b) => b.status === "reserved").length;
-        const sizeBytes = this.buffer.reduce((sum, b) => sum + JSON.stringify(b.event.payload).length, 0);
-        return {
-            pathwayId: this.pulseEmitter ? "" : "", // set by caller
-            flowType: this.dataSource.flowType,
-            timeBucket: this.bufferState.timeBucket,
-            eventId: this.bufferState.eventId,
-            isLive: this.isLive,
-            bufferDepth: this.buffer.length,
-            bufferReserved: reserved,
-            bufferSizeBytes: sizeBytes,
-            acknowledgedTotal: this.acknowledgedCount,
-            failedTotal: this.failedCount,
-            pulledTotal: this.pulledCount,
-            uptimeMs: this.startedAt ? Date.now() - this.startedAt : 0,
-        };
-    }
-    static create(options, dataSourceOverride) {
-        if ("apiKey" in options.auth && !options.auth.apiKeyId) {
-            const parts = options.auth.apiKey.split("_");
-            if (parts.length !== 3 || parts[0] !== "fc") {
-                throw new Error("Invalid API key");
-            }
-            options.auth.apiKeyId = parts[1];
-        }
-        const dataSource = dataSourceOverride ?? new data_source_js_1.FlowcoreDataSource({
-            auth: options.auth,
-            dataSource: options.dataSource,
-            baseUrlOverride: options.baseUrlOverride,
-            noTranslation: options.noTranslation,
-            directMode: options.directMode,
-        });
-        const notifier = new notifier_js_1.FlowcoreNotifier({
-            auth: options.auth,
-            dataSource: options.dataSource,
-            natsServers: options.notifier?.type === "nats" ? options.notifier.servers : undefined,
-            pollerIntervalMs: options.notifier?.type === "poller" ? options.notifier.intervalMs : undefined,
-            logger: options.logger,
-            directMode: options.directMode,
-            noTranslation: options.noTranslation,
-        });
-        const pump = new FlowcoreDataPump(dataSource, notifier, options.stateManager, {
-            bufferSize: options.bufferSize ?? 1000,
-            bufferThreshold: Math.ceil((options.bufferSize ?? 1000) * 0.1),
-            maxRedeliveryCount: options.maxRedeliveryCount ?? 3,
-            achknowledgeTimeoutMs: options.achknowledgeTimeoutMs ?? 5_000,
-            includeSensitiveData: options.includeSensitiveData ?? false,
-            processor: options.processor,
-            stopAt: options.stopAt,
-        }, options.logger);
-        if (options.pulse) {
-            const pathwayId = options.pulse.pathwayId;
-            const sourceId = options.pulse.sourceId;
-            pump.pulseEmitter = new pulse_js_1.PulseEmitter({
-                url: options.pulse.url,
-                intervalMs: options.pulse.intervalMs,
-                auth: options.auth,
-                logger: options.logger,
-                successLogLevel: options.pulse.successLogLevel,
-                failureLogLevel: options.pulse.failureLogLevel,
-            }, () => {
-                const snapshot = pump.getSnapshot();
-                if (snapshot) {
-                    snapshot.pathwayId = pathwayId;
-                    snapshot.sourceId = sourceId;
-                }
-                return snapshot;
-            });
-        }
-        return pump;
-    }
-    async start(callback) {
-        this.isLive = false;
-        if (this.running) {
-            throw new Error("Data pump already running");
-        }
-        this.running = true;
-        this.startedAt = Date.now();
-        this.nextCursor = undefined;
-        this.updateMetricsGauges();
-        this.pulseEmitter?.start();
-        const currentState = await this.stateManager.getState();
-        const timeBucket = currentState
-            ? await this.dataSource.getClosestTimeBucket(currentState.timeBucket)
-            : (0, date_fns_1.format)((0, date_fns_1.startOfHour)((0, utc_1.utc)(new Date())), "yyyyMMddHH0000");
-        this.bufferState = {
-            timeBucket: timeBucket ?? (0, date_fns_1.format)((0, date_fns_1.startOfHour)((0, utc_1.utc)(new Date())), "yyyyMMddHH0000"),
-            eventId: currentState ? currentState.eventId : mod_js_1.TimeUuid.now().toString(),
-        };
-        if (this.options.stopAt) {
-            this.stopAtState = {
-                timeBucket: (await this.dataSource.getClosestTimeBucket((0, date_fns_1.format)((0, date_fns_1.startOfHour)((0, utc_1.utc)(this.options.stopAt)), "yyyyMMddHH0000"), true)) ?? (0, date_fns_1.format)((0, date_fns_1.startOfHour)((0, utc_1.utc)(new Date())), "yyyyMMddHH0000"),
-                eventId: mod_js_1.TimeUuid.fromDate(this.options.stopAt).toString(),
-            };
-        }
-        if (this.options.processor) {
-            this.startProcessLoop();
-        }
-        if (!callback) {
-            return this.loop();
-        }
-        this.startMainLoop(callback);
-    }
-    startMainLoop(callback) {
-        this.loop()
-            .then(() => {
-            this.mainLoopRestartAttempts = 0;
-            callback?.();
-        })
-            .catch((error) => {
-            this.logger?.error("Error in fetch loop", { error });
-            if (!this.running) {
-                callback?.(error);
-                return;
-            }
-            this.mainLoopRestartAttempts++;
-            const delay = Math.min(1_000 * Math.pow(2, this.mainLoopRestartAttempts - 1), 30_000);
-            this.logger?.warn(`Restarting fetch loop in ${delay}ms (attempt ${this.mainLoopRestartAttempts})`);
-            setTimeout(() => {
-                if (!this.running) {
-                    callback?.(error);
-                    return;
-                }
-                this.startMainLoop(callback);
-            }, delay);
-        });
-    }
-    restart(state, stopAt) {
-        if (typeof state.timeBucket !== "string" || !state.timeBucket.match(/^\d{14}$/)) {
-            throw new Error(`Invalid timebucket: ${state.timeBucket}`);
-        }
-        this.restartTo = state;
-        if (stopAt !== undefined) {
-            this.options.stopAt = stopAt ?? undefined;
-        }
-        this.stop(true);
-    }
-    stop(isRestart = false) {
-        this.running = false;
-        this.processLoopRestartAttempts = 0;
-        this.mainLoopRestartAttempts = 0;
-        this.buffer = [];
-        this.updateMetricsGauges();
-        this.pulseEmitter?.stop();
-        this.abortController?.abort();
-        this.waiterBufferThreshold?.();
-        if (!isRestart) {
-            this.waiterEvents?.();
-        }
-    }
-    updateState(eventId) {
-        if (!this.stateManager.setState) {
-            return;
-        }
-        const stateEventId = eventId ?? this.buffer[0]?.event.eventId;
-        if (!stateEventId) {
-            return;
-        }
-        const date = mod_js_1.TimeUuid.fromString(stateEventId).getDate();
-        const timeBucket = (0, date_fns_1.format)((0, date_fns_1.startOfHour)((0, utc_1.utc)(date)), "yyyyMMddHH0000");
-        return this.stateManager.setState?.({ timeBucket, eventId: stateEventId });
-    }
-    async loop() {
-        do {
-            const amountToFetch = this.options.bufferSize - this.buffer.length;
-            if (amountToFetch <= 0) {
-                this.logger?.info("Buffer is full, waiting for space");
-                await this.waitForBufferThreshold();
-                continue;
-            }
-            this.logger?.debug(`fetching ${amountToFetch} events from ${this.bufferState.timeBucket}(${this.nextCursor})`);
-            const { events, nextCursor } = await this.dataSource.getEvents(this.bufferState, amountToFetch, this.stopAtState?.eventId, this.nextCursor, this.options.includeSensitiveData);
-            if (!this.running) {
-                break;
-            }
-            this.logger?.debug(`fetched ${events.length} events`);
-            this.pulledCount += events.length;
-            this.mainLoopRestartAttempts = 0;
-            this.buffer.push(...events.map((event) => ({ event, status: "open", deliveryCount: 0 })));
-            this.nextCursor = nextCursor;
-            this.updateMetricsGauges();
-            events.length && this.waiterEvents?.();
-            this.bufferState.eventId = events[events.length - 1]?.eventId ?? this.bufferState.eventId;
-            if (this.stopAtState?.timeBucket && this.bufferState.timeBucket >= this.stopAtState.timeBucket && !events.length) {
-                this.logger?.info("Stopping at stopAt state");
-                await this.waitForBufferEmpty();
-                this.stop();
-                break;
-            }
-            if (!this.nextCursor) {
-                const timeBucket = await this.dataSource.getNextTimeBucket(this.bufferState.timeBucket);
-                if (!this.running) {
-                    break;
-                }
-                if (timeBucket) {
-                    this.bufferState.timeBucket = timeBucket;
-                }
-                else {
-                    const previousTimeBucket = this.bufferState.timeBucket;
-                    this.bufferState.timeBucket = (0, date_fns_1.format)((0, date_fns_1.startOfHour)((0, utc_1.utc)(new Date())), "yyyyMMddHH0000");
-                    if (previousTimeBucket === this.bufferState.timeBucket && !events.length) {
-                        this.isLive = true;
-                        this.logger?.debug("Going live...");
-                        this.abortController = new AbortController();
-                        await this.notifier.wait(this.abortController.signal);
-                    }
-                    else if (this.isLive) {
-                        await new Promise((resolve) => setTimeout(resolve, 1000));
-                    }
-                }
-            }
-        } while (this.running);
-        if (this.restartTo) {
-            try {
-                await this.dataSource.getTimeBuckets(true);
-                this.restartTo.timeBucket = (await this.dataSource.getClosestTimeBucket(this.restartTo.timeBucket)) ??
-                    (0, date_fns_1.format)((0, date_fns_1.startOfHour)((0, utc_1.utc)(new Date())), "yyyyMMddHH0000");
-                this.nextCursor = undefined;
-                this.bufferState = this.restartTo;
-                this.restartTo = undefined;
-                this.running = true;
-                return this.loop();
-            }
-            catch (error) {
-                this.logger?.error("Failed to consume restartTo, dropping it", { error });
-                this.restartTo = undefined;
-                return;
-            }
-        }
-        this.logger?.info("Data pump stopped");
-    }
-    // #region Puller
-    async reserve(amount) {
-        if (!this.running) {
-            return [];
-        }
-        const events = [];
-        const deliveryId = crypto.randomUUID();
-        for (const event of this.buffer) {
-            if (event.status === "open") {
-                event.status = "reserved";
-                event.deliveryId = deliveryId;
-                event.deliveryCount++;
-                events.push(event.event);
-                this.incMetricsCounter("pulled", event.event.eventType, JSON.stringify(event.event).length);
-                if (events.length === amount) {
-                    break;
-                }
-            }
-        }
-        if (!events.length) {
-            await this.waitForEvents();
-            return this.reserve(amount);
-        }
-        this.updateMetricsGauges();
-        setTimeout(() => {
-            this.reOpen(events.map((event) => event.eventId), deliveryId);
-        }, this.options.achknowledgeTimeoutMs);
-        return events;
-    }
-    onFinalyFailed(handler) {
-        this.finallyFailedHandler = handler;
-    }
-    async acknowledge(eventIds) {
-        if (!this.running) {
-            return;
-        }
-        const lastEventInBuffer = this.buffer[this.buffer.length - 1];
-        this.buffer = this.buffer.filter((event) => {
-            if (eventIds.includes(event.event.eventId)) {
-                this.incMetricsCounter("acknowledged", event.event.eventType, 1);
-                this.acknowledgedCount++;
-                return false;
-            }
-            return true;
-        });
-        if (this.buffer.length <= this.options.bufferSize - this.options.bufferThreshold) {
-            this.waiterBufferThreshold?.();
-        }
-        await this.updateState(this.buffer.length ? undefined : lastEventInBuffer?.event.eventId);
-        this.updateMetricsGauges();
-        if (!this.buffer.length) {
-            this.waiterBufferEmpty?.();
-        }
-    }
-    async fail(eventIds) {
-        if (!this.running || !eventIds.length) {
-            return;
-        }
-        const lastEventInBuffer = this.buffer[this.buffer.length - 1];
-        const failedEvents = [];
-        this.buffer = this.buffer.filter((event) => {
-            if (eventIds.includes(event.event.eventId)) {
-                this.incMetricsCounter("failed", event.event.eventType, 1);
-                this.failedCount++;
-                failedEvents.push(event.event);
-                return false;
-            }
-            return true;
-        });
-        this.logger?.info(`Failed ${failedEvents.length} events`);
-        void this.options.processor?.failedHandler?.(failedEvents);
-        if (this.buffer.length <= this.options.bufferSize - this.options.bufferThreshold) {
-            this.waiterBufferThreshold?.();
-        }
-        await this.updateState(this.buffer.length ? undefined : lastEventInBuffer?.event.eventId);
-        if (!this.buffer.length) {
-            this.waiterBufferEmpty?.();
-        }
-    }
-    async reOpen(eventIds, deliveryId) {
-        let lastEvent;
-        const failedEvents = [];
-        const reopenedEvents = [];
-        this.buffer = this.buffer.filter((event) => {
-            if (event.deliveryId !== deliveryId || !eventIds.includes(event.event.eventId)) {
-                return true;
-            }
-            if (this.options.maxRedeliveryCount > -1 && event.deliveryCount > this.options.maxRedeliveryCount) {
-                this.incMetricsCounter("failed", event.event.eventType, 1);
-                failedEvents.push(event.event);
-                lastEvent = event.event;
-                return false;
-            }
-            event.status = "open";
-            event.deliveryId = undefined;
-            reopenedEvents.push(event.event);
-            return true;
-        });
-        this.updateMetricsGauges();
-        if (reopenedEvents.length) {
-            this.logger?.info(`Reopened ${reopenedEvents.length} events`);
-            await this.waiterEvents?.();
-        }
-        if (!failedEvents.length) {
-            return;
-        }
-        this.logger?.info(`Failed ${failedEvents.length} events`);
-        void this.options.processor?.failedHandler?.(failedEvents);
-        void this.finallyFailedHandler?.(failedEvents);
-        if (this.buffer.length <= this.options.bufferSize - this.options.bufferThreshold) {
-            this.waiterBufferThreshold?.();
-        }
-        await this.updateState(this.buffer.length ? undefined : lastEvent?.eventId);
-        if (!this.buffer.length) {
-            this.waiterBufferEmpty?.();
-        }
-    }
-    // #endregion
-    // #region Pusher
-    startProcessLoop() {
-        this.processLoop().catch((error) => {
-            this.logger?.error("Error in processor", { error });
-            if (!this.running)
-                return;
-            this.processLoopRestartAttempts++;
-            const delay = Math.min(1_000 * Math.pow(2, this.processLoopRestartAttempts - 1), 30_000);
-            this.logger?.warn(`Restarting process loop in ${delay}ms (attempt ${this.processLoopRestartAttempts})`);
-            setTimeout(() => {
-                if (!this.running)
-                    return;
-                this.startProcessLoop();
-            }, delay);
-        });
-    }
-    async processLoop() {
-        while (this.running) {
-            try {
-                const events = await this.reserve(this.options.processor?.concurrency ?? 1);
-                await this.options.processor?.handler(events);
-                await this.acknowledge(events.map((event) => event.eventId));
-                this.processLoopRestartAttempts = 0;
-            }
-            catch (error) {
-                const errorMessage = error instanceof Error ? error.message : "Unknown error";
-                this.logger?.error(`Failed to process events: ${errorMessage}`);
-            }
-        }
-    }
-    // #endregion
-    // #region Metrics
-    updateMetricsGauges() {
-        const stats = new Map();
-        for (const eventType of this.dataSource.eventTypes) {
-            stats.set(eventType, {
-                eventCount: 0,
-                eventReservedCount: 0,
-                eventSizeBytes: 0,
-            });
-        }
-        for (const item of this.buffer) {
-            const stat = stats.get(item.event.eventType);
-            if (!stat) {
-                continue;
-            }
-            stat.eventCount++;
-            if (item.status === "reserved") {
-                stat.eventReservedCount++;
-            }
-            stat.eventSizeBytes += JSON.stringify(item.event.payload).length;
-        }
-        for (const [eventType, stat] of stats) {
-            metrics_js_1.metrics.bufferEventCountGauge.set({
-                tenant: this.dataSource.tenant,
-                data_core: this.dataSource.dataCore,
-                flow_type: this.dataSource.flowType,
-                event_type: eventType,
-            }, stat.eventCount);
-            metrics_js_1.metrics.bufferReservedEventCountGauge.set({
-                tenant: this.dataSource.tenant,
-                data_core: this.dataSource.dataCore,
-                flow_type: this.dataSource.flowType,
-                event_type: eventType,
-            }, stat.eventReservedCount);
-            metrics_js_1.metrics.bufferSizeBytesGauge.set({
-                tenant: this.dataSource.tenant,
-                data_core: this.dataSource.dataCore,
-                flow_type: this.dataSource.flowType,
-                event_type: eventType,
-            }, stat.eventSizeBytes);
-        }
-    }
-    incMetricsCounter(name, eventType, value) {
-        switch (name) {
-            case "acknowledged":
-                metrics_js_1.metrics.eventsAcknowledgedCounter.inc({
-                    tenant: this.dataSource.tenant,
-                    data_core: this.dataSource.dataCore,
-                    flow_type: this.dataSource.flowType,
-                    event_type: eventType,
-                }, value);
-                break;
-            case "failed":
-                metrics_js_1.metrics.eventsFailedCounter.inc({
-                    tenant: this.dataSource.tenant,
-                    data_core: this.dataSource.dataCore,
-                    flow_type: this.dataSource.flowType,
-                    event_type: eventType,
-                }, value);
-                break;
-            case "pulled":
-                metrics_js_1.metrics.eventsPulledSizeBytesCounter.inc({
-                    tenant: this.dataSource.tenant,
-                    data_core: this.dataSource.dataCore,
-                    flow_type: this.dataSource.flowType,
-                    event_type: eventType,
-                }, value);
-                break;
-        }
-    }
-    async waitForEvents() {
-        const promise = new Promise((resolve) => {
-            this.waiterEvents = resolve;
-        });
-        await promise;
-    }
-    async waitForBufferThreshold() {
-        const promise = new Promise((resolve) => {
-            this.waiterBufferThreshold = resolve;
-        });
-        await promise;
-    }
-    async waitForBufferEmpty() {
-        if (!this.buffer.length) {
-            return;
-        }
-        const promise = new Promise((resolve) => {
-            this.waiterBufferEmpty = resolve;
-        });
-        await promise;
-    }
-}
-exports.FlowcoreDataPump = FlowcoreDataPump;