@friggframework/core 2.0.0-next.12 → 2.0.0-next.120

package/handlers/app-definition-loader.js

@@ -0,0 +1,103 @@
+const { findNearestBackendPackageJson } = require('@friggframework/core/utils');
+const path = require('node:path');
+const fs = require('fs-extra');
+const { resolveTelemetryConfig } = require('../telemetry/telemetry-config');
+const { registerDeniedKeys } = require('../logs/denied-keys');
+const {
+    extractCredentialFieldsFromModules,
+} = require('../database/encryption/encryption-schema-registry');
+const {
+    getModulesDefinitionFromIntegrationClasses,
+} = require('../integrations/utils/map-integration-dto');
+
+// Handlers without a database never load the encryption registry, so the
+// credential leaf keys are registered for redaction here too.
+function registerCredentialLogKeys(appDefinition, integrations) {
+    try {
+        extractCredentialFieldsFromModules(
+            getModulesDefinitionFromIntegrationClasses(integrations)
+        );
+    } catch {
+        // An odd integration shape must not stop the app from loading.
+    }
+    const schema = appDefinition.encryption?.schema;
+    if (schema && typeof schema === 'object') {
+        for (const config of Object.values(schema)) {
+            registerDeniedKeys(config?.fields);
+        }
+    }
+}
+
+/**
+ * Loads the App definition from the nearest backend package
+ * @function loadAppDefinition
+ * @description Searches for the nearest backend package.json, loads the corresponding index.js file,
+ * and extracts the application definition containing integrations and user configuration.
+ * @returns {{integrations: Array<object>, userConfig: object | null, adminScripts: Array<object>, reports: Array<object>, admin: object, telemetry: object, logging: object | null}} An object containing the application definition.
+ * @throws {Error} Throws error if backend package.json cannot be found.
+ * @throws {Error} Throws error if index.js file cannot be found in the backend directory.
+ * @example
+ * const { integrations, userConfig, telemetry } = loadAppDefinition();
+ * console.log(`Found ${integrations.length} integrations`);
+ */
+function loadAppDefinition() {
+    const backendPath = findNearestBackendPackageJson();
+    if (!backendPath) {
+        throw new Error('Could not find backend package.json');
+    }
+
+    const backendDir = path.dirname(backendPath);
+    const backendFilePath = path.join(backendDir, 'index.js');
+    if (!fs.existsSync(backendFilePath)) {
+        throw new Error('Could not find index.js');
+    }
+
+    const backendJsFile = require(backendFilePath);
+    const appDefinition = backendJsFile.Definition;
+
+    const {
+        integrations = [],
+        user: userConfig = null,
+        adminScripts = [],
+        reports = [],
+        admin = {},
+        logging = null,
+    } = appDefinition;
+
+    registerCredentialLogKeys(appDefinition, integrations);
+
+    // Degrade consistently: an invalid telemetry block must never take down a
+    // router bundle that loads the app definition at module scope (telemetry is
+    // never allowed to break a handler). The singletons apply the same
+    // fall-back, so all consumers behave identically.
+    let telemetry;
+    try {
+        telemetry = resolveTelemetryConfig(appDefinition);
+    } catch (error) {
+        console.warn(
+            `[Frigg][telemetry] invalid telemetry config, defaulting to disabled: ${
+                error && error.message
+            }`
+        );
+        telemetry = {
+            exporter: { type: 'none' },
+            northStar: null,
+            sampleRatio: 1,
+            subscribers: [],
+        };
+    }
+
+    return {
+        integrations,
+        userConfig,
+        adminScripts,
+        reports,
+        admin,
+        telemetry,
+        logging,
+    };
+}
+
+module.exports = {
+    loadAppDefinition,
+};

package/handlers/app-handler-helpers.js

@@ -1,11 +1,14 @@
-const { createHandler, flushDebugLog } = require('@friggframework/core');
+const { createHandler } = require('@friggframework/core');
+const { getLogger } = require('../logs');
+const { summarizeExpressRequest } = require('../logs/summarize-event');
 const express = require('express');
 const bodyParser = require('body-parser');
 const cors = require('cors');
 const Boom = require('@hapi/boom');
-const loadUserManager = require('./routers/middleware/loadUser');
 const serverlessHttp = require('serverless-http');
 
+const log = getLogger('frigg.http');
+
 const createApp = (applyMiddleware) => {
     const app = express();
 
@@ -14,15 +17,15 @@ const createApp = (applyMiddleware) => {
     app.use(
         cors({
             origin: '*',
+            allowedHeaders: '*',
+            methods: '*',
             credentials: true,
         })
     );
 
-    app.use(loadUserManager);
-
     if (applyMiddleware) applyMiddleware(app);
 
-    // Handle sending error response and logging server errors to console
+    // The express boundary: send the error response and log it one time.
     app.use((err, req, res, next) => {
         const boomError = err.isBoom ? err : Boom.boomify(err);
         const {
@@ -30,9 +33,20 @@ const createApp = (applyMiddleware) => {
         } = boomError;
 
         if (statusCode >= 500) {
-            flushDebugLog(boomError);
+            log.error('Request failed', {
+                eventName: 'frigg.http.request_failed',
+                statusCode,
+                invocation: summarizeExpressRequest(req),
+                error: boomError,
+            });
             res.status(statusCode).json({ error: 'Internal Server Error' });
         } else {
+            // A client error needs no stack; the logger scrubs the reason.
+            log.warn('Request rejected', {
+                eventName: 'frigg.http.request_rejected',
+                statusCode,
+                reason: boomError.message,
+            });
             res.status(statusCode).json({ error: err.message });
         }
     });

package/handlers/backend-utils.js

@@ -1,38 +1,49 @@
-const { createFriggBackend, Worker } = require('@friggframework/core');
-const {
-    findNearestBackendPackageJson,
-} = require('../../devtools/frigg-cli/utils/backend-path');
-const path = require('node:path');
-const fs = require('fs-extra');
-
-const backendPath = findNearestBackendPackageJson();
-if (!backendPath) {
-    throw new Error('Could not find backend package.json');
-}
-
-const backendDir = path.dirname(backendPath);
-const backendFilePath = path.join(backendDir, 'index.js');
-if (!fs.existsSync(backendFilePath)) {
-    throw new Error('Could not find index.js');
-}
-
-const backendJsFile = require(backendFilePath);
 const { Router } = require('express');
-const appDefinition = backendJsFile.Definition;
+const { runInContext, LOGGER_SCOPE_KEY } = require('../logs/context');
+const { Worker } = require('@friggframework/core');
+const {
+    IntegrationEventDispatcher,
+} = require('./integration-event-dispatcher');
+const {
+    GetIntegrationInstance,
+} = require('../integrations/use-cases/get-integration-instance');
+const { ModuleFactory } = require('../modules/module-factory');
+const {
+    createProcessRepository,
+} = require('../integrations/repositories/process-repository-factory');
+const {
+    createIntegrationRepository,
+} = require('../integrations/repositories/integration-repository-factory');
+const {
+    createModuleRepository,
+} = require('../modules/repositories/module-repository-factory');
+const {
+    getModulesDefinitionFromIntegrationClasses,
+} = require('../integrations/utils/map-integration-dto');
+const { processNotFound } = require('../integrations/use-cases/process-errors');
 
-const backend = createFriggBackend(appDefinition);
 const loadRouterFromObject = (IntegrationClass, routerObject) => {
     const router = Router();
     const { path, method, event } = routerObject;
+
     console.log(
         `Registering ${method} ${path} for ${IntegrationClass.Definition.name}`
     );
+
     router[method.toLowerCase()](path, async (req, res, next) => {
         try {
-            const integration = new IntegrationClass({});
-            await integration.loadModules();
-            await integration.registerEventHandlers();
-            const result = await integration.send(event, {req, res, next});
+            const integrationInstance = new IntegrationClass();
+            // initialize() registers dynamic user actions AND merges any Tier 3
+            // Integration Extension events into instance.events before dispatch.
+            await integrationInstance.initialize();
+            const dispatcher = new IntegrationEventDispatcher(
+                integrationInstance
+            );
+            // Logs only: a route :integrationId is unauthenticated input.
+            const result = await runInContext(
+                { [LOGGER_SCOPE_KEY]: { integrationId: req.params?.integrationId } },
+                () => dispatcher.dispatchHttp({ event, req, res, next })
+            );
             res.json(result);
         } catch (error) {
             next(error);
@@ -41,38 +52,252 @@ const loadRouterFromObject = (IntegrationClass, routerObject) => {
 
     return router;
 };
+
+const initializeRepositories = () => {
+    const processRepository = createProcessRepository();
+    const integrationRepository = createIntegrationRepository();
+    const moduleRepository = createModuleRepository();
+
+    return { processRepository, integrationRepository, moduleRepository };
+};
+
+const createModuleFactoryWithDefinitions = (
+    moduleRepository,
+    integrationClasses
+) => {
+    const moduleDefinitions =
+        getModulesDefinitionFromIntegrationClasses(integrationClasses);
+
+    return new ModuleFactory({
+        moduleRepository,
+        moduleDefinitions,
+    });
+};
+
+const loadIntegrationForWebhook = async (integrationId) => {
+    const { loadAppDefinition } = require('./app-definition-loader');
+    const { integrations: integrationClasses } = loadAppDefinition();
+
+    const { integrationRepository, moduleRepository } =
+        initializeRepositories();
+
+    const moduleFactory = createModuleFactoryWithDefinitions(
+        moduleRepository,
+        integrationClasses
+    );
+
+    const getIntegrationInstance = new GetIntegrationInstance({
+        integrationRepository,
+        integrationClasses,
+        moduleFactory,
+    });
+
+    let integrationRecord;
+    try {
+        integrationRecord = await integrationRepository.findIntegrationById(
+            integrationId
+        );
+    } catch (error) {
+        if (error.message?.includes('not found')) {
+            return null;
+        }
+        throw error;
+    }
+
+    const instance = await getIntegrationInstance.execute(
+        integrationId,
+        integrationRecord.userId
+    );
+
+    return instance;
+};
+
+// Returns false only when the integration is confirmed gone; a transient
+// lookup failure returns true so genuine errors keep their normal retry path.
+const integrationExists = async (integrationId) => {
+    const integrationRepository = createIntegrationRepository();
+    try {
+        const record = await integrationRepository.findIntegrationById(
+            integrationId
+        );
+        return Boolean(record);
+    } catch (error) {
+        if (error.message?.includes('not found')) {
+            return false;
+        }
+        return true;
+    }
+};
+
+const loadIntegrationForProcess = async (processId, integrationClass) => {
+    const { processRepository, integrationRepository, moduleRepository } =
+        initializeRepositories();
+
+    const moduleFactory = createModuleFactoryWithDefinitions(moduleRepository, [
+        integrationClass,
+    ]);
+
+    const getIntegrationInstance = new GetIntegrationInstance({
+        integrationRepository,
+        integrationClasses: [integrationClass],
+        moduleFactory,
+    });
+
+    if (!processId) {
+        throw new Error('processId is required in queue message data');
+    }
+
+    const process = await processRepository.findById(processId);
+
+    if (!process) {
+        throw processNotFound(`Process not found: ${processId}`);
+    }
+
+    const instance = await getIntegrationInstance.execute(
+        process.integrationId,
+        process.userId
+    );
+
+    return instance;
+};
+
 const createQueueWorker = (integrationClass) => {
+    const integrationName = integrationClass.Definition.name;
+
     class QueueWorker extends Worker {
-        async _run(params, context) {
+        async _run(params, context, delivery) {
+            const logCtx = {
+                integration: integrationName,
+                event: params.event,
+                processId: params.data?.processId,
+                integrationId: params.data?.integrationId,
+            };
+
             try {
-                let instance;
-                if (!params.integrationId) {
-                    instance = new integrationClass({});
-                    await instance.loadModules();
-                    // await instance.loadUserActions();
-                    await instance.registerEventHandlers();
+                let integrationInstance;
+
+                // Prioritize processId first (for sync handler compatibility),
+                // then integrationId (for ANY event type that needs hydration),
+                // fallback to unhydrated instance
+                if (params.data?.processId) {
+                    console.log(`[QueueWorker] hydrating by processId`, logCtx);
+                    integrationInstance = await loadIntegrationForProcess(
+                        params.data.processId,
+                        integrationClass
+                    );
+                    console.log(`[QueueWorker] hydrated`, {
+                        ...logCtx,
+                        integrationStatus: integrationInstance?.status,
+                        hydratedIntegrationId: integrationInstance?.id,
+                    });
+                    if (
+                        ['DISABLED', 'ERROR', 'IN_DELETION'].includes(
+                            integrationInstance?.status
+                        )
+                    ) {
+                        console.warn(
+                            `[${integrationName}] Integration for process ${params.data.processId} is ${integrationInstance.status}. Discarding ${params.event} message.`
+                        );
+                        return;
+                    }
+                } else if (params.data?.integrationId) {
                     console.log(
-                        `${params.event} for ${integrationClass.Definition.name} integration with no integrationId`
+                        `[QueueWorker] hydrating by integrationId`,
+                        logCtx
+                    );
+                    integrationInstance = await loadIntegrationForWebhook(
+                        params.data.integrationId
                     );
+                    if (!integrationInstance) {
+                        console.warn(
+                            `[${integrationName}] Integration ${params.data.integrationId} no longer exists. Discarding ${params.event} message.`
+                        );
+                        return;
+                    }
+                    console.log(`[QueueWorker] hydrated`, {
+                        ...logCtx,
+                        integrationStatus: integrationInstance?.status,
+                    });
+                    if (
+                        ['DISABLED', 'ERROR', 'IN_DELETION'].includes(
+                            integrationInstance.status
+                        )
+                    ) {
+                        console.warn(
+                            `[${integrationName}] Integration ${params.data.integrationId} is ${integrationInstance.status}. Discarding ${params.event} message.`
+                        );
+                        return;
+                    }
                 } else {
-                    instance =
-                        await integrationClass.getInstanceFromIntegrationId({
-                            integrationId: params.integrationId,
-                        });
+                    // Instantiates a DRY integration class without database records.
+                    // There will be cases where we need to use helpers that the api modules can export.
+                    // Like for HubSpot, the answer is to do a reverse lookup for the integration by the entity external ID (HubSpot Portal ID),
+                    // and then you'll have the integration ID available to hydrate from.
                     console.log(
-                        `${params.event} for ${instance.integration.config.type} of integrationId: ${params.integrationId}`
+                        `[QueueWorker] no processId/integrationId — running dry instance`,
+                        logCtx
                     );
+                    integrationInstance = new integrationClass();
+                    // Merge Tier 3 Integration Extension events into instance.events
+                    // so extension-contributed queue events can be dispatched.
+                    await integrationInstance.initialize();
                 }
-                const res = await instance.send(params.event, {
+
+                const dispatcher = new IntegrationEventDispatcher(
+                    integrationInstance
+                );
+
+                console.log(
+                    `[QueueWorker] dispatching ${params.event}`,
+                    logCtx
+                );
+                const result = await dispatcher.dispatchJob({
+                    event: params.event,
                     data: params.data,
-                    context,
+                    context: context,
+                    delivery,
                 });
-                return res;
-            } catch (error) {
-                console.error(
-                    `Error in ${params.event} for ${integrationClass.Definition.name}:`,
-                    error
+                console.log(
+                    `[QueueWorker] ${params.event} dispatched ok`,
+                    logCtx
                 );
+                return result;
+            } catch (error) {
+                // Integration deleted mid-flight: no retry can succeed once
+                // it's gone, so discard instead of sending it to the DLQ.
+                if (
+                    params.data?.integrationId &&
+                    !(await integrationExists(params.data.integrationId))
+                ) {
+                    console.warn(
+                        `[${integrationName}] Integration ${params.data.integrationId} was deleted mid-flight — discarding ${params.event} message (no retry)`
+                    );
+                    return;
+                }
+
+                // 4xx HTTP errors are permanent — the requester already
+                // attempted token refresh (401) and backoff (429/5xx).
+                // By the time a 4xx reaches here, retrying won't help.
+                // 408 (timeout) and 429 (rate limit) are excluded — both are transient.
+                const status = error.statusCode;
+                if (
+                    status &&
+                    status >= 400 &&
+                    status < 500 &&
+                    status !== 408 &&
+                    status !== 429
+                ) {
+                    error.isHaltError = true;
+                    console.warn(
+                        `[${integrationName}] Permanent ${status} error for ${params.event} — message will be discarded (no retry)`,
+                        {
+                            ...logCtx,
+                            errorName: error.name,
+                            errorMessage: error.message,
+                        }
+                    );
+                }
+
                 throw error;
             }
         }
@@ -81,7 +306,8 @@ const createQueueWorker = (integrationClass) => {
 };
 
 module.exports = {
-    ...backend,
     loadRouterFromObject,
     createQueueWorker,
+    integrationExists,
+    loadIntegrationForWebhook,
 };