@flowcore/pathways 0.2.1

package/script/pathways/builder.js

@@ -0,0 +1,534 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.PathwaysBuilder = void 0;
+const value_1 = require("@sinclair/typebox/value");
+const rxjs_1 = require("rxjs");
+const flowcore_transformer_core_sdk_js_1 = require("../compatibility/flowcore-transformer-core.sdk.js");
+const internal_pathway_state_js_1 = require("./internal-pathway.state.js");
+const logger_js_1 = require("./logger.js");
+const DEFAULT_PATHWAY_TIMEOUT_MS = 10000;
+const DEFAULT_MAX_RETRIES = 3;
+const DEFAULT_RETRY_DELAY_MS = 500;
+class PathwaysBuilder {
+    constructor({ baseUrl, tenant, dataCore, apiKey, pathwayTimeoutMs, logger, }) {
+        Object.defineProperty(this, "pathways", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: {}
+        });
+        Object.defineProperty(this, "handlers", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: {}
+        });
+        Object.defineProperty(this, "beforeObservable", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: {}
+        });
+        Object.defineProperty(this, "afterObservers", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: {}
+        });
+        Object.defineProperty(this, "errorObservers", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: {}
+        });
+        Object.defineProperty(this, "globalErrorSubject", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: new rxjs_1.Subject()
+        });
+        Object.defineProperty(this, "writers", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: {}
+        });
+        Object.defineProperty(this, "schemas", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: {}
+        });
+        Object.defineProperty(this, "writable", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: {}
+        });
+        Object.defineProperty(this, "timeouts", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: {}
+        });
+        Object.defineProperty(this, "maxRetries", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: {}
+        });
+        Object.defineProperty(this, "retryDelays", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: {}
+        });
+        Object.defineProperty(this, "filePathways", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: new Set()
+        });
+        Object.defineProperty(this, "webhookBuilderFactory", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: void 0
+        });
+        Object.defineProperty(this, "pathwayState", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: new internal_pathway_state_js_1.InternalPathwayState()
+        });
+        Object.defineProperty(this, "pathwayTimeoutMs", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: DEFAULT_PATHWAY_TIMEOUT_MS
+        });
+        // Audit-related properties
+        Object.defineProperty(this, "auditHandler", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: void 0
+        });
+        Object.defineProperty(this, "userIdResolver", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: void 0
+        });
+        // Logger instance (but not using it yet due to TypeScript errors)
+        Object.defineProperty(this, "logger", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: void 0
+        });
+        // Initialize logger (use NoopLogger if none provided)
+        this.logger = logger ?? new logger_js_1.NoopLogger();
+        this.logger.debug('Initializing PathwaysBuilder', {
+            baseUrl,
+            tenant,
+            dataCore,
+            pathwayTimeoutMs
+        });
+        this.webhookBuilderFactory = new flowcore_transformer_core_sdk_js_1.WebhookBuilder({
+            baseUrl,
+            tenant,
+            dataCore,
+            apiKey,
+        })
+            .withRetry({
+            maxAttempts: 5,
+            attemptDelayMs: 250,
+        })
+            .factory();
+        if (pathwayTimeoutMs) {
+            this.pathwayTimeoutMs = pathwayTimeoutMs;
+        }
+    }
+    withPathwayState(state) {
+        this.logger.debug('Setting custom pathway state');
+        this.pathwayState = state;
+        return this;
+    }
+    /**
+     * Configures the PathwaysBuilder to use audit functionality
+     * @param handler The handler function that receives pathway and event information
+     * @param userIdResolver An async function that resolves to the current user ID
+     * @returns The PathwaysBuilder instance with audit configured
+     */
+    withAudit(handler, userIdResolver) {
+        this.logger.debug('Configuring audit functionality');
+        this.auditHandler = handler;
+        this.userIdResolver = userIdResolver;
+        return this;
+    }
+    /**
+     * Process a pathway event with error handling and retries
+     * @param pathway The pathway to process
+     * @param data The event data to process
+     * @returns Promise that resolves when processing is complete
+     */
+    async process(pathway, data) {
+        const pathwayStr = String(pathway);
+        this.logger.debug(`Processing pathway event`, {
+            pathway: pathwayStr,
+            eventId: data.eventId
+        });
+        if (!this.pathways[pathway]) {
+            const error = `Pathway ${pathwayStr} not found`;
+            this.logger.error(error);
+            throw new Error(error);
+        }
+        // Call audit handler if configured
+        if (this.auditHandler) {
+            this.logger.debug(`Calling audit handler for pathway`, {
+                pathway: pathwayStr,
+                eventId: data.eventId
+            });
+            this.auditHandler(pathwayStr, data);
+        }
+        if (this.handlers[pathway]) {
+            let retryCount = 0;
+            const maxRetries = this.maxRetries[pathway] ?? DEFAULT_MAX_RETRIES;
+            const retryDelayMs = this.retryDelays[pathway] ?? DEFAULT_RETRY_DELAY_MS;
+            this.logger.debug(`Emitting 'before' event`, {
+                pathway: pathwayStr,
+                eventId: data.eventId
+            });
+            this.beforeObservable[pathway].next(data);
+            while (true) {
+                try {
+                    this.logger.debug(`Executing handler for pathway`, {
+                        pathway: pathwayStr,
+                        eventId: data.eventId,
+                        attempt: retryCount + 1
+                    });
+                    // Execute the handler
+                    const handle = this.handlers[pathway](data);
+                    await handle;
+                    // If successful, emit success event and mark as processed
+                    this.logger.debug(`Handler executed successfully, emitting 'after' event`, {
+                        pathway: pathwayStr,
+                        eventId: data.eventId
+                    });
+                    this.afterObservers[pathway].next(data);
+                    await this.pathwayState.setProcessed(data.eventId);
+                    this.logger.info(`Successfully processed pathway event`, {
+                        pathway: pathwayStr,
+                        eventId: data.eventId
+                    });
+                    return;
+                }
+                catch (error) {
+                    // Create error object if needed
+                    const errorObj = error instanceof Error ? error : new Error(String(error));
+                    this.logger.error(`Error processing pathway event`, {
+                        pathway: pathwayStr,
+                        eventId: data.eventId,
+                        error: errorObj.message,
+                        retryCount,
+                        maxRetries
+                    });
+                    // Emit error event with both error and event data
+                    this.errorObservers[pathway].next({ event: data, error: errorObj });
+                    // Also emit to global error subject
+                    this.globalErrorSubject.next({
+                        pathway: pathwayStr,
+                        event: data,
+                        error: errorObj
+                    });
+                    // Check if we should retry
+                    if (retryCount < maxRetries) {
+                        retryCount++;
+                        const nextDelay = retryDelayMs * retryCount;
+                        this.logger.debug(`Retrying pathway event processing`, {
+                            pathway: pathwayStr,
+                            eventId: data.eventId,
+                            attempt: retryCount,
+                            maxRetries,
+                            nextDelay
+                        });
+                        // Wait for delay before retrying
+                        await new Promise(resolve => setTimeout(resolve, nextDelay));
+                        continue;
+                    }
+                    // If we've exhausted retries, mark as processed to avoid hanging
+                    this.logger.warn(`Max retries exceeded for pathway event, marking as processed`, {
+                        pathway: pathwayStr,
+                        eventId: data.eventId,
+                        retryCount,
+                        maxRetries
+                    });
+                    await this.pathwayState.setProcessed(data.eventId);
+                    throw error;
+                }
+            }
+        }
+        else {
+            // No handler, just emit events and mark as processed
+            this.logger.debug(`No handler for pathway, emitting events and marking as processed`, {
+                pathway: pathwayStr,
+                eventId: data.eventId
+            });
+            this.beforeObservable[pathway].next(data);
+            this.afterObservers[pathway].next(data);
+            await this.pathwayState.setProcessed(data.eventId);
+        }
+    }
+    register(contract) {
+        const path = `${contract.flowType}/${contract.eventType}`;
+        const writable = contract.writable ?? true;
+        this.logger.debug(`Registering pathway`, {
+            pathway: path,
+            flowType: contract.flowType,
+            eventType: contract.eventType,
+            writable,
+            isFilePathway: contract.isFilePathway,
+            timeoutMs: contract.timeoutMs,
+            maxRetries: contract.maxRetries,
+            retryDelayMs: contract.retryDelayMs
+        });
+        // deno-lint-ignore no-explicit-any
+        this.pathways[path] = true;
+        this.beforeObservable[path] = new rxjs_1.Subject();
+        this.afterObservers[path] = new rxjs_1.Subject();
+        this.errorObservers[path] = new rxjs_1.Subject();
+        if (writable) {
+            if (contract.isFilePathway) {
+                this.filePathways.add(path);
+                this.writers[path] = this.webhookBuilderFactory()
+                    .buildFileWebhook(contract.flowType, contract.eventType).send;
+            }
+            else {
+                this.writers[path] = this.webhookBuilderFactory()
+                    .buildWebhook(contract.flowType, contract.eventType).send;
+            }
+        }
+        if (contract.timeoutMs) {
+            this.timeouts[path] = contract.timeoutMs;
+        }
+        if (contract.maxRetries !== undefined) {
+            this.maxRetries[path] = contract.maxRetries;
+        }
+        if (contract.retryDelayMs !== undefined) {
+            this.retryDelays[path] = contract.retryDelayMs;
+        }
+        this.schemas[path] = contract.schema;
+        this.writable[path] = writable;
+        this.logger.info(`Pathway registered successfully`, {
+            pathway: path,
+            flowType: contract.flowType,
+            eventType: contract.eventType,
+            writable
+        });
+        return this;
+    }
+    get(path) {
+        this.logger.debug(`Getting pathway`, { pathway: String(path) });
+        return this.pathways[path];
+    }
+    handle(path, handler) {
+        const pathStr = String(path);
+        this.logger.debug(`Setting handler for pathway`, { pathway: pathStr });
+        const pathway = this.pathways[path];
+        if (!pathway) {
+            const error = `Pathway ${pathStr} not found`;
+            this.logger.error(error);
+            throw new Error(error);
+        }
+        if (this.handlers[path]) {
+            const error = `Someone is already handling pathway ${pathStr} in this instance`;
+            this.logger.error(error);
+            throw new Error(error);
+        }
+        this.handlers[path] = handler;
+        this.logger.info(`Handler set for pathway`, { pathway: pathStr });
+    }
+    /**
+     * Subscribe to pathway events (before or after processing)
+     * @param path The pathway to subscribe to
+     * @param handler The handler function for the events
+     * @param type The event type to subscribe to (before, after, or all)
+     */
+    subscribe(path, handler, type = "before") {
+        const pathStr = String(path);
+        this.logger.debug(`Subscribing to pathway events`, {
+            pathway: pathStr,
+            type
+        });
+        if (!this.pathways[path]) {
+            const error = `Pathway ${pathStr} not found`;
+            this.logger.error(error);
+            throw new Error(error);
+        }
+        if (type === "before") {
+            this.beforeObservable[path].subscribe(handler);
+        }
+        else if (type === "after") {
+            this.afterObservers[path].subscribe(handler);
+        }
+        else if (type === "all") {
+            // Subscribe to both before and after events
+            this.beforeObservable[path].subscribe(handler);
+            this.afterObservers[path].subscribe(handler);
+        }
+        this.logger.debug(`Subscribed to pathway events`, {
+            pathway: pathStr,
+            type
+        });
+    }
+    /**
+     * Subscribe to errors for a specific pathway
+     * @param path The pathway to subscribe to errors for
+     * @param handler The handler function that receives the error and event
+     */
+    onError(path, handler) {
+        const pathStr = String(path);
+        this.logger.debug(`Subscribing to pathway errors`, { pathway: pathStr });
+        if (!this.pathways[path]) {
+            const error = `Pathway ${pathStr} not found`;
+            this.logger.error(error);
+            throw new Error(error);
+        }
+        this.errorObservers[path].subscribe(({ event, error }) => handler(error, event));
+        this.logger.debug(`Subscribed to pathway errors`, { pathway: pathStr });
+    }
+    /**
+     * Subscribe to errors for all pathways
+     * @param handler The handler function that receives the error, event, and pathway name
+     */
+    onAnyError(handler) {
+        this.logger.debug(`Subscribing to all pathway errors`);
+        this.globalErrorSubject.subscribe(({ pathway, event, error }) => handler(error, event, pathway));
+        this.logger.debug(`Subscribed to all pathway errors`);
+    }
+    /**
+     * Writes data to a pathway with optional audit metadata
+     * @param path The pathway to write to
+     * @param data The data to write
+     * @param metadata Optional metadata to include with the event
+     * @param options Optional write options
+     * @returns A promise that resolves to the event ID(s)
+     */
+    async write(path, data, metadata, options) {
+        const pathStr = String(path);
+        this.logger.debug(`Writing to pathway`, {
+            pathway: pathStr,
+            metadata,
+            options: {
+                fireAndForget: options?.fireAndForget
+            }
+        });
+        if (!this.pathways[path]) {
+            const error = `Pathway ${pathStr} not found`;
+            this.logger.error(error);
+            throw new Error(error);
+        }
+        if (!this.writable[path]) {
+            const error = `Pathway ${pathStr} is not writable`;
+            this.logger.error(error);
+            throw new Error(error);
+        }
+        const schema = this.schemas[path];
+        if (!value_1.Value.Check(schema, data)) {
+            const error = `Invalid data for pathway ${pathStr}`;
+            this.logger.error(error, {
+                pathway: pathStr,
+                schema: schema.toString()
+            });
+            throw new Error(error);
+        }
+        // Create a copy of the metadata to avoid modifying the original
+        const finalMetadata = metadata ? { ...metadata } : {};
+        // Process audit metadata if audit is configured
+        if (this.userIdResolver) {
+            this.logger.debug(`Resolving user ID for audit metadata`, { pathway: pathStr });
+            const userId = await this.userIdResolver();
+            // Determine the audit mode: default is "user" unless explicitly specified as "system"
+            const auditMode = finalMetadata?.["audit/mode"] || "user";
+            this.logger.debug(`Adding audit metadata`, {
+                pathway: pathStr,
+                auditMode,
+                userId
+            });
+            // Add appropriate audit metadata based on mode
+            if (auditMode === "system") {
+                finalMetadata["audit/user-id"] = "system";
+                finalMetadata["audit/on-behalf-of"] = userId;
+                finalMetadata["audit/mode"] = "system";
+            }
+            else {
+                finalMetadata["audit/user-id"] = userId;
+                finalMetadata["audit/mode"] = "user"; // Always set mode for user
+            }
+        }
+        let eventIds = [];
+        if (this.filePathways.has(path)) {
+            this.logger.debug(`Writing file data to pathway`, { pathway: pathStr });
+            const fileData = data;
+            eventIds = await this.writers[path](fileData, finalMetadata, options);
+        }
+        else {
+            this.logger.debug(`Writing webhook data to pathway`, { pathway: pathStr });
+            eventIds = await this.writers[path](data, finalMetadata, options);
+        }
+        this.logger.info(`Successfully wrote to pathway`, {
+            pathway: pathStr,
+            eventIds: Array.isArray(eventIds) ? eventIds : [eventIds],
+            fireAndForget: options?.fireAndForget
+        });
+        if (!options?.fireAndForget) {
+            this.logger.debug(`Waiting for pathway to be processed`, {
+                pathway: pathStr,
+                eventIds: Array.isArray(eventIds) ? eventIds : [eventIds]
+            });
+            await Promise.all(Array.isArray(eventIds)
+                ? eventIds.map(id => this.waitForPathwayToBeProcessed(id))
+                : [this.waitForPathwayToBeProcessed(eventIds)]);
+        }
+        return eventIds;
+    }
+    async waitForPathwayToBeProcessed(eventId) {
+        const startTime = Date.now();
+        const timeoutMs = this.timeouts[eventId] ?? this.pathwayTimeoutMs;
+        this.logger.debug(`Waiting for event to be processed`, {
+            eventId,
+            timeoutMs
+        });
+        let attempts = 0;
+        while (!(await this.pathwayState.isProcessed(eventId))) {
+            attempts++;
+            const elapsedTime = Date.now() - startTime;
+            if (elapsedTime > timeoutMs) {
+                const error = `Pathway processing timed out after ${timeoutMs}ms for event ${eventId}`;
+                this.logger.error(error, {
+                    eventId,
+                    timeoutMs,
+                    elapsedTime,
+                    attempts
+                });
+                throw new Error(error);
+            }
+            if (attempts % 10 === 0) { // Log every 10 attempts (1 second)
+                this.logger.debug(`Still waiting for event to be processed`, {
+                    eventId,
+                    elapsedTime,
+                    attempts,
+                    timeoutMs
+                });
+            }
+            await new Promise((resolve) => setTimeout(resolve, 100));
+        }
+        this.logger.debug(`Event has been processed`, {
+            eventId,
+            elapsedTime: Date.now() - startTime,
+            attempts
+        });
+    }
+}
+exports.PathwaysBuilder = PathwaysBuilder;

package/script/pathways/index.d.ts

@@ -0,0 +1,17 @@
+/**
+ * Main export module for the pathways functionality
+ *
+ * Exports all components needed to build and manage pathways including:
+ * - Pathway builder
+ * - State management
+ * - Storage adapters (KV, Postgres)
+ * - Logging
+ * - Type definitions
+ */
+export * from "./builder.js";
+export * from "./internal-pathway.state.js";
+export * from "./kv/kv-adapter.js";
+export * from "./logger.js";
+export * from "./postgres/index.js";
+export * from "./types.js";
+//# sourceMappingURL=index.d.ts.map

package/script/pathways/index.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/pathways/index.ts"],"names":[],"mappings":"AAAA;;;;;;;;;GASG;AACH,cAAc,cAAc,CAAC;AAC7B,cAAc,6BAA6B,CAAC;AAC5C,cAAc,oBAAoB,CAAC;AACnC,cAAc,aAAa,CAAC;AAC5B,cAAc,qBAAqB,CAAC;AACpC,cAAc,YAAY,CAAC"}

package/script/pathways/index.js

@@ -0,0 +1,32 @@
+"use strict";
+var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    var desc = Object.getOwnPropertyDescriptor(m, k);
+    if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
+      desc = { enumerable: true, get: function() { return m[k]; } };
+    }
+    Object.defineProperty(o, k2, desc);
+}) : (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    o[k2] = m[k];
+}));
+var __exportStar = (this && this.__exportStar) || function(m, exports) {
+    for (var p in m) if (p !== "default" && !Object.prototype.hasOwnProperty.call(exports, p)) __createBinding(exports, m, p);
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+/**
+ * Main export module for the pathways functionality
+ *
+ * Exports all components needed to build and manage pathways including:
+ * - Pathway builder
+ * - State management
+ * - Storage adapters (KV, Postgres)
+ * - Logging
+ * - Type definitions
+ */
+__exportStar(require("./builder.js"), exports);
+__exportStar(require("./internal-pathway.state.js"), exports);
+__exportStar(require("./kv/kv-adapter.js"), exports);
+__exportStar(require("./logger.js"), exports);
+__exportStar(require("./postgres/index.js"), exports);
+__exportStar(require("./types.js"), exports);

package/script/pathways/internal-pathway.state.d.ts

@@ -0,0 +1,41 @@
+import type { PathwayState } from "./types.js";
+/**
+ * Internal implementation of PathwayState interface that uses KV storage
+ * for tracking processed events to prevent duplicate processing
+ *
+ * @implements {PathwayState}
+ */
+export declare class InternalPathwayState implements PathwayState {
+    /**
+     * Default time-to-live for processed event records (5 minutes)
+     * @private
+     */
+    private static readonly DEFAULT_TTL_MS;
+    /**
+     * The KV adapter instance for storage
+     * @private
+     */
+    private kv;
+    /**
+     * Gets or initializes the KV adapter
+     *
+     * @private
+     * @returns {Promise<KvAdapter>} The KV adapter instance
+     */
+    private getKv;
+    /**
+     * Checks if an event has already been processed
+     *
+     * @param {string} eventId - The ID of the event to check
+     * @returns {Promise<boolean>} True if the event has been processed, false otherwise
+     */
+    isProcessed(eventId: string): Promise<boolean>;
+    /**
+     * Marks an event as processed
+     *
+     * @param {string} eventId - The ID of the event to mark as processed
+     * @returns {Promise<void>}
+     */
+    setProcessed(eventId: string): Promise<void>;
+}
+//# sourceMappingURL=internal-pathway.state.d.ts.map

package/script/pathways/internal-pathway.state.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"internal-pathway.state.d.ts","sourceRoot":"","sources":["../../src/pathways/internal-pathway.state.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,YAAY,CAAC;AAE/C;;;;;GAKG;AACH,qBAAa,oBAAqB,YAAW,YAAY;IACvD;;;OAGG;IACH,OAAO,CAAC,MAAM,CAAC,QAAQ,CAAC,cAAc,CAAiB;IAEvD;;;OAGG;IACH,OAAO,CAAC,EAAE,CAA0B;IAEpC;;;;;OAKG;YACW,KAAK;IAOnB;;;;;OAKG;IACG,WAAW,CAAC,OAAO,EAAE,MAAM,GAAG,OAAO,CAAC,OAAO,CAAC;IAMpD;;;;;OAKG;IACG,YAAY,CAAC,OAAO,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC;CAInD"}