@friggframework/core 2.0.0--canary.5ff5209.0 → 2.0.0--canary.590.094a71b.0

package/core/Worker.js

@@ -20,15 +20,45 @@ class Worker {
         const records = get(params, 'Records');
         const batchItemFailures = [];
 
+        console.log(
+            `[Worker] run: processing ${records.length} record(s)`
+        );
+
         for (const record of records) {
+            // Log record entry with SQS-provided attributes useful for tracing
+            // delivery history (ApproximateReceiveCount for retries, etc.).
+            let parsedEvent;
+            try {
+                parsedEvent = JSON.parse(record.body)?.event;
+            } catch {
+                parsedEvent = undefined;
+            }
+            console.log(`[Worker] record begin`, {
+                messageId: record.messageId,
+                event: parsedEvent,
+                receiveCount: record.attributes?.ApproximateReceiveCount,
+            });
+
             try {
                 const runParams = JSON.parse(record.body);
                 this._validateParams(runParams);
                 await this._run(runParams, context);
+                console.log(`[Worker] record success`, {
+                    messageId: record.messageId,
+                    event: runParams?.event,
+                });
             } catch (error) {
                 if (error.isHaltError) {
                     // HaltError means "discard this message, don't retry".
                     // Treat as success so SQS deletes it from the queue.
+                    // Logged explicitly — silent discards made prod debugging
+                    // extremely hard; keep this visible.
+                    console.warn(`[Worker] record halted (discarded, no retry)`, {
+                        messageId: record.messageId,
+                        event: parsedEvent,
+                        reason: error.message,
+                        statusCode: error.statusCode,
+                    });
                     continue;
                 }
                 console.error(`[Worker] Failed to process record ${record.messageId}:`, error);
@@ -36,6 +66,12 @@ class Worker {
             }
         }
 
+        if (batchItemFailures.length > 0) {
+            console.warn(
+                `[Worker] run: returning ${batchItemFailures.length} batchItemFailure(s) of ${records.length}`
+            );
+        }
+
         return { batchItemFailures };
     }
 

package/core/create-handler.js

@@ -5,6 +5,45 @@
 const { initDebugLog, flushDebugLog } = require('../logs');
 const { secretsToEnv } = require('./secrets-to-env');
 
+// Best-effort extraction of correlation identifiers from a Lambda event.
+// For SQS: pulls messageIds + parsed event/processId/integrationId from each
+// record body. For HTTP: pulls method+path. Never throws.
+const summarizeLambdaEvent = (event) => {
+    if (!event) return {};
+    if (Array.isArray(event.Records)) {
+        return {
+            source: 'sqs',
+            records: event.Records.map((r) => {
+                let parsed = {};
+                try {
+                    const body = JSON.parse(r.body);
+                    parsed = {
+                        event: body?.event,
+                        processId: body?.data?.processId,
+                        integrationId: body?.data?.integrationId,
+                    };
+                } catch {
+                    // ignore unparseable bodies
+                }
+                return {
+                    messageId: r.messageId,
+                    receiveCount: r.attributes?.ApproximateReceiveCount,
+                    ...parsed,
+                };
+            }),
+        };
+    }
+    if (event.httpMethod || event.requestContext?.http) {
+        return {
+            source: 'http',
+            method:
+                event.httpMethod || event.requestContext?.http?.method,
+            path: event.path || event.rawPath,
+        };
+    }
+    return { source: 'other' };
+};
+
 const createHandler = (optionByName = {}) => {
     const {
         eventName = 'Event',
@@ -17,7 +56,18 @@ const createHandler = (optionByName = {}) => {
     }
 
     return async (event, context) => {
+        const eventSummary = summarizeLambdaEvent(event);
+
         try {
+            console.info(
+                `[createHandler] ${eventName}: handler entry`,
+                {
+                    eventName,
+                    awsRequestId: context?.awsRequestId,
+                    ...eventSummary,
+                }
+            );
+
             initDebugLog(eventName, event);
 
             const requestMethod = event.httpMethod;
@@ -62,7 +112,21 @@ const createHandler = (optionByName = {}) => {
             // Handle server-to-server responses.
 
             // Halt errors are logged but suceed and won't be retried.
+            // Log explicitly — silent suppression here previously made stuck
+            // messages invisible to observability tooling. Include
+            // eventSummary so operators can correlate across concurrent
+            // invocations (processId / messageIds / HTTP path).
             if (error.isHaltError === true) {
+                console.warn(
+                    `[createHandler] ${eventName}: halt error suppressed (no retry)`,
+                    {
+                        eventName,
+                        errorName: error.name,
+                        errorMessage: error.message,
+                        statusCode: error.statusCode,
+                        ...eventSummary,
+                    }
+                );
                 return;
             }
 

package/database/encryption/encryption-schema-registry.js

@@ -42,6 +42,17 @@ const CORE_ENCRYPTION_SCHEMA = {
 
 let customSchema = {};
 
+/**
+ * Per-model write-side opt-out: fields registered here are NOT encrypted on
+ * write, but ARE still decrypted on read so legacy encrypted rows continue to
+ * deserialize. Lets apps migrate a model from encrypted to plain JSON without
+ * a data migration — touched rows naturally rewrite as plain on the next save,
+ * untouched rows stay encrypted-but-readable forever.
+ *
+ * Shape: `{ ModelName: ['field.path', ...] }`
+ */
+let encryptionOptOut = {};
+
 /**
  * Validates a custom encryption schema
  * @returns {{valid: boolean, errors: string[]}}
@@ -218,6 +229,14 @@ function loadCustomEncryptionSchema() {
             registerCustomSchema(customSchema);
         }
 
+        // Load app-level encryption opt-out — apps can declare fields they
+        // don't want encrypted on write (decryption on read still works,
+        // so legacy data remains readable).
+        const disable = appDefinition.encryption?.disable;
+        if (disable && Object.keys(disable).length > 0) {
+            registerEncryptionOptOut(disable);
+        }
+
         // Load module-level encryption schemas from integrations
         const integrations = appDefinition.integrations;
         if (integrations && Array.isArray(integrations)) {
@@ -240,6 +259,115 @@ function getEncryptedFields(modelName) {
     return [...new Set(allFields)];
 }
 
+/**
+ * Validates an encryption opt-out config.
+ *
+ * Unlike custom schema validation, opt-out IS allowed to target paths that
+ * already live in CORE_ENCRYPTION_SCHEMA — that's the entire point.
+ *
+ * @param {Object} optOut - Map of `{ ModelName: ['field.path', ...] }`
+ * @returns {{valid: boolean, errors: string[]}}
+ */
+function validateOptOut(optOut) {
+    const errors = [];
+
+    if (!optOut || typeof optOut !== 'object') {
+        errors.push('Encryption opt-out must be an object');
+        return { valid: false, errors };
+    }
+
+    for (const [modelName, fields] of Object.entries(optOut)) {
+        if (typeof modelName !== 'string' || !modelName) {
+            errors.push(`Invalid model name in opt-out: ${modelName}`);
+            continue;
+        }
+
+        if (!Array.isArray(fields)) {
+            errors.push(
+                `Model "${modelName}" opt-out must be an array of field paths`
+            );
+            continue;
+        }
+
+        for (const fieldPath of fields) {
+            if (typeof fieldPath !== 'string' || !fieldPath) {
+                errors.push(
+                    `Model "${modelName}" has invalid opt-out field path: ${fieldPath}`
+                );
+            }
+        }
+    }
+
+    return { valid: errors.length === 0, errors };
+}
+
+/**
+ * Registers an encryption opt-out config. Listed fields will be skipped during
+ * encryption on write while still being eligible for decryption on read (so
+ * legacy encrypted rows still deserialize correctly).
+ *
+ * Intended call site: `appDefinition.encryption.disable` via
+ * `loadCustomEncryptionSchema`.
+ *
+ * @param {Object} optOut - Map of `{ ModelName: ['field.path', ...] }`
+ * @throws {Error} If opt-out validation fails
+ */
+function registerEncryptionOptOut(optOut) {
+    if (!optOut || Object.keys(optOut).length === 0) {
+        return;
+    }
+
+    const validation = validateOptOut(optOut);
+    if (!validation.valid) {
+        throw new Error(
+            `Invalid encryption opt-out:\n- ${validation.errors.join('\n- ')}`
+        );
+    }
+
+    encryptionOptOut = { ...optOut };
+    logger.info(
+        `Registered encryption opt-out for models: ${Object.keys(
+            encryptionOptOut
+        ).join(', ')}`
+    );
+}
+
+/**
+ * Returns the field paths that should be encrypted when writing the given
+ * model. This is `getEncryptedFields` minus any paths the app has opted out
+ * of via `registerEncryptionOptOut`.
+ *
+ * Use this in the encrypt-on-write path of the FieldEncryptionService.
+ */
+function getFieldsToEncryptOnWrite(modelName) {
+    const allFields = getEncryptedFields(modelName);
+    const optedOut = new Set(encryptionOptOut[modelName] || []);
+    if (optedOut.size === 0) return allFields;
+    return allFields.filter((path) => !optedOut.has(path));
+}
+
+/**
+ * Returns the field paths that should be checked for decryption when reading
+ * the given model. Always includes opted-out paths so legacy encrypted rows
+ * remain readable after an app opts a field out.
+ *
+ * `FieldEncryptionService._isEncrypted` already short-circuits for plain JSON
+ * values, so listing more fields than necessary here is harmless.
+ *
+ * Use this in the decrypt-on-read path of the FieldEncryptionService.
+ */
+function getFieldsToDecryptOnRead(modelName) {
+    return getEncryptedFields(modelName);
+}
+
+/**
+ * Clears any registered encryption opt-outs. Test-helper; not intended for
+ * runtime use.
+ */
+function resetEncryptionOptOut() {
+    encryptionOptOut = {};
+}
+
 function hasEncryptedFields(modelName) {
     return getEncryptedFields(modelName).length > 0;
 }
@@ -257,12 +385,17 @@ function resetCustomSchema() {
 module.exports = {
     CORE_ENCRYPTION_SCHEMA,
     getEncryptedFields,
+    getFieldsToEncryptOnWrite,
+    getFieldsToDecryptOnRead,
     hasEncryptedFields,
     getEncryptedModels,
     registerCustomSchema,
+    registerEncryptionOptOut,
     loadCustomEncryptionSchema,
     loadModuleEncryptionSchemas,
     extractCredentialFieldsFromModules,
     validateCustomSchema,
+    validateOptOut,
     resetCustomSchema,
+    resetEncryptionOptOut,
 };

package/database/encryption/field-encryption-service.js

@@ -17,12 +17,40 @@ class FieldEncryptionService {
         this.schema = schema;
     }
 
+    /**
+     * Resolve the field paths to encrypt on write. Prefers
+     * `schema.getFieldsToEncryptOnWrite` (which respects app opt-outs); falls
+     * back to `schema.getEncryptedFields` for backwards compatibility with
+     * older schema adapters.
+     * @private
+     */
+    _getWriteFields(modelName) {
+        if (typeof this.schema.getFieldsToEncryptOnWrite === 'function') {
+            return this.schema.getFieldsToEncryptOnWrite(modelName);
+        }
+        return this.schema.getEncryptedFields(modelName);
+    }
+
+    /**
+     * Resolve the field paths to attempt decryption on read. Prefers
+     * `schema.getFieldsToDecryptOnRead` (which IGNORES opt-outs so legacy
+     * encrypted rows still deserialize); falls back to
+     * `schema.getEncryptedFields` for backwards compatibility.
+     * @private
+     */
+    _getReadFields(modelName) {
+        if (typeof this.schema.getFieldsToDecryptOnRead === 'function') {
+            return this.schema.getFieldsToDecryptOnRead(modelName);
+        }
+        return this.schema.getEncryptedFields(modelName);
+    }
+
     async encryptFields(modelName, document) {
         if (!document || typeof document !== 'object') {
             return document;
         }
 
-        const fields = this.schema.getEncryptedFields(modelName);
+        const fields = this._getWriteFields(modelName);
         if (fields.length === 0) {
             return document;
         }
@@ -58,7 +86,7 @@ class FieldEncryptionService {
             return document;
         }
 
-        const fields = this.schema.getEncryptedFields(modelName);
+        const fields = this._getReadFields(modelName);
         if (fields.length === 0) {
             return document;
         }

package/database/encryption/prisma-encryption-extension.js

@@ -3,7 +3,11 @@
  * Intercepts Prisma queries to encrypt on write and decrypt on read.
  */
 
-const { getEncryptedFields } = require('./encryption-schema-registry');
+const {
+    getEncryptedFields,
+    getFieldsToEncryptOnWrite,
+    getFieldsToDecryptOnRead,
+} = require('./encryption-schema-registry');
 const { FieldEncryptionService } = require('./field-encryption-service');
 
 function createEncryptionExtension({ cryptor, enabled = true }) {
@@ -19,7 +23,11 @@ function createEncryptionExtension({ cryptor, enabled = true }) {
 
     const encryptionService = new FieldEncryptionService({
         cryptor,
-        schema: { getEncryptedFields },
+        schema: {
+            getEncryptedFields,
+            getFieldsToEncryptOnWrite,
+            getFieldsToDecryptOnRead,
+        },
     });
 
     return {

package/handlers/backend-utils.js

@@ -31,6 +31,9 @@ const loadRouterFromObject = (IntegrationClass, routerObject) => {
     router[method.toLowerCase()](path, async (req, res, next) => {
         try {
             const integrationInstance = new IntegrationClass();
+            // initialize() registers dynamic user actions AND merges any Tier 3
+            // Integration Extension events into instance.events before dispatch.
+            await integrationInstance.initialize();
             const dispatcher = new IntegrationEventDispatcher(
                 integrationInstance
             );
@@ -141,8 +144,17 @@ const loadIntegrationForProcess = async (processId, integrationClass) => {
 };
 
 const createQueueWorker = (integrationClass) => {
+    const integrationName = integrationClass.Definition.name;
+
     class QueueWorker extends Worker {
         async _run(params, context) {
+            const logCtx = {
+                integration: integrationName,
+                event: params.event,
+                processId: params.data?.processId,
+                integrationId: params.data?.integrationId,
+            };
+
             try {
                 let integrationInstance;
 
@@ -150,29 +162,46 @@ const createQueueWorker = (integrationClass) => {
                 // then integrationId (for ANY event type that needs hydration),
                 // fallback to unhydrated instance
                 if (params.data?.processId) {
+                    console.log(
+                        `[QueueWorker] hydrating by processId`,
+                        logCtx
+                    );
                     integrationInstance = await loadIntegrationForProcess(
                         params.data.processId,
                         integrationClass
                     );
+                    console.log(`[QueueWorker] hydrated`, {
+                        ...logCtx,
+                        integrationStatus: integrationInstance?.status,
+                        hydratedIntegrationId: integrationInstance?.id,
+                    });
                     if (['DISABLED', 'ERROR'].includes(integrationInstance?.status)) {
                         console.warn(
-                            `[${integrationClass.Definition.name}] Integration for process ${params.data.processId} is ${integrationInstance.status}. Discarding ${params.event} message.`
+                            `[${integrationName}] Integration for process ${params.data.processId} is ${integrationInstance.status}. Discarding ${params.event} message.`
                         );
                         return;
                     }
                 } else if (params.data?.integrationId) {
+                    console.log(
+                        `[QueueWorker] hydrating by integrationId`,
+                        logCtx
+                    );
                     integrationInstance = await loadIntegrationForWebhook(
                         params.data.integrationId
                     );
                     if (!integrationInstance) {
                         console.warn(
-                            `[${integrationClass.Definition.name}] Integration ${params.data.integrationId} no longer exists. Discarding ${params.event} message.`
+                            `[${integrationName}] Integration ${params.data.integrationId} no longer exists. Discarding ${params.event} message.`
                         );
                         return;
                     }
+                    console.log(`[QueueWorker] hydrated`, {
+                        ...logCtx,
+                        integrationStatus: integrationInstance?.status,
+                    });
                     if (['DISABLED', 'ERROR'].includes(integrationInstance.status)) {
                         console.warn(
-                            `[${integrationClass.Definition.name}] Integration ${params.data.integrationId} is ${integrationInstance.status}. Discarding ${params.event} message.`
+                            `[${integrationName}] Integration ${params.data.integrationId} is ${integrationInstance.status}. Discarding ${params.event} message.`
                         );
                         return;
                     }
@@ -181,21 +210,31 @@ const createQueueWorker = (integrationClass) => {
                     // There will be cases where we need to use helpers that the api modules can export.
                     // Like for HubSpot, the answer is to do a reverse lookup for the integration by the entity external ID (HubSpot Portal ID),
                     // and then you'll have the integration ID available to hydrate from.
+                    console.log(
+                        `[QueueWorker] no processId/integrationId — running dry instance`,
+                        logCtx
+                    );
                     integrationInstance = new integrationClass();
+                    // Merge Tier 3 Integration Extension events into instance.events
+                    // so extension-contributed queue events can be dispatched.
+                    await integrationInstance.initialize();
                 }
 
                 const dispatcher = new IntegrationEventDispatcher(
                     integrationInstance
                 );
 
-                return await dispatcher.dispatchJob({
+                console.log(`[QueueWorker] dispatching ${params.event}`, logCtx);
+                const result = await dispatcher.dispatchJob({
                     event: params.event,
                     data: params.data,
                     context: context,
                 });
+                console.log(`[QueueWorker] ${params.event} dispatched ok`, logCtx);
+                return result;
             } catch (error) {
                 console.error(
-                    `Error in ${params.event} for ${integrationClass.Definition.name}:`,
+                    `Error in ${params.event} for ${integrationName}:`,
                     error
                 );
 
@@ -207,7 +246,12 @@ const createQueueWorker = (integrationClass) => {
                 if (status && status >= 400 && status < 500 && status !== 408 && status !== 429) {
                     error.isHaltError = true;
                     console.warn(
-                        `[${integrationClass.Definition.name}] Permanent ${status} error for ${params.event} — message will be discarded (no retry)`
+                        `[${integrationName}] Permanent ${status} error for ${params.event} — message will be discarded (no retry)`,
+                        {
+                            ...logCtx,
+                            errorName: error.name,
+                            errorMessage: error.message,
+                        }
                     );
                 }
 

package/handlers/routers/integration-defined-routers.js

@@ -2,24 +2,47 @@ const { createAppHandler } = require('./../app-handler-helpers');
 const {
     loadAppDefinition,
 } = require('../app-definition-loader');
-const { Router } = require('express');
+const express = require('express');
+const { Router } = express;
 const { loadRouterFromObject } = require('../backend-utils');
+const { getExtensionRoutes } = require('../../integrations/extension');
 
 const handlers = {};
 const { integrations: integrationClasses } = loadAppDefinition();
 
+const routeKey = (method, path) => `${(method || 'ANY').toUpperCase()} ${path}`;
+
 //todo: this should be in a use case class
 for (const IntegrationClass of integrationClasses) {
     const router = Router();
     const basePath = `/api/${IntegrationClass.Definition.name}-integration`;
+    // Track (method, path) tuples to fail fast on conflicts between Definition.routes,
+    // extension routes, or two extensions claiming the same path.
+    const claimedRoutes = new Map();
+    const claim = (method, path, source) => {
+        const key = routeKey(method, path);
+        if (claimedRoutes.has(key)) {
+            const prev = claimedRoutes.get(key);
+            throw new Error(
+                `Integration "${IntegrationClass.Definition.name}" route conflict: ` +
+                    `${key} declared by ${prev} and ${source}`
+            );
+        }
+        claimedRoutes.set(key, source);
+    };
 
     console.log(`\n│ Configuring routes for ${IntegrationClass.Definition.name} Integration:`);
 
-    for (const routeDef of IntegrationClass.Definition.routes) {
+    const routes = IntegrationClass.Definition.routes || [];
+    for (const routeDef of routes) {
         if (typeof routeDef === 'function') {
             router.use(basePath, routeDef(IntegrationClass));
             console.log(`│ ANY ${basePath}/* (function handler)`);
+        } else if (routeDef instanceof express.Router) {
+            router.use(basePath, routeDef);
+            console.log(`│ ANY ${basePath}/* (express router)`);
         } else if (typeof routeDef === 'object') {
+            claim(routeDef.method, routeDef.path, 'Definition.routes');
             router.use(
                 basePath,
                 loadRouterFromObject(IntegrationClass, routeDef)
@@ -27,11 +50,25 @@ for (const IntegrationClass of integrationClasses) {
             const method = (routeDef.method || 'ANY').toUpperCase();
             const fullPath = `${basePath}${routeDef.path}`;
             console.log(`│ ${method} ${fullPath}`);
-        } else if (routeDef instanceof express.Router) {
-            router.use(basePath, routeDef);
-            console.log(`│ ANY ${basePath}/* (express router)`);
         }
     }
+
+    // Tier 3 Integration Extension routes — see EXTENSIONS.md
+    for (const extRoute of getExtensionRoutes(IntegrationClass)) {
+        claim(
+            extRoute.method,
+            extRoute.path,
+            `extension "${extRoute.extensionName}" (binding "${extRoute.bindingName}")`
+        );
+        router.use(
+            basePath,
+            loadRouterFromObject(IntegrationClass, extRoute)
+        );
+        const method = extRoute.method.toUpperCase();
+        console.log(
+            `│ ${method} ${basePath}${extRoute.path}  (extension: ${extRoute.extensionName})`
+        );
+    }
     console.log('│');
 
     handlers[`${IntegrationClass.Definition.name}`] = {

package/handlers/workers/integration-defined-workers.js

@@ -1,6 +1,13 @@
 const { createHandler } = require('@friggframework/core');
 const { loadAppDefinition } = require('../app-definition-loader');
 const { createQueueWorker } = require('../backend-utils');
+// TODO(Phase 2): mount extension-declared workers in addition to the per-integration
+// default queue worker. Today, getExtensionWorkers(IntegrationClass) returns the
+// declared workers but they are not yet bound to dedicated SQS sources. Extension-
+// contributed *events* still flow through the default queue worker below because
+// _mergeExtensions() registers them in instance.events, so end-to-end webhook
+// delivery for Tier 3 extensions works without this Phase 2 work.
+// const { getExtensionWorkers } = require('../../integrations/extension');
 
 const handlers = {};
 const { integrations: integrationClasses } = loadAppDefinition();