@friggframework/core 2.0.0-next.111 → 2.0.0-next.113

package/handlers/routers/websocket.js

@@ -1,5 +1,8 @@
 const { createHandler } = require('@friggframework/core');
-const { createWebsocketConnectionRepository } = require('../../database/websocket-connection-repository-factory');
+const { createWebsocketConnectionRepository } = require('../../websocket/repositories/websocket-connection-repository-factory');
+const { getLogger } = require('../../logs');
+
+const log = getLogger('frigg.websocket');
 
 const websocketConnectionRepository = createWebsocketConnectionRepository();
 
@@ -11,10 +14,16 @@ const handleWebSocketConnection = async (event, context) => {
             try {
                 const connectionId = event.requestContext.connectionId;
                 await websocketConnectionRepository.createConnection(connectionId);
-                console.log(`Stored new connection: ${connectionId}`);
+                log.info('Stored new connection', {
+                    eventName: 'frigg.websocket.connected',
+                    connectionId,
+                });
                 return { statusCode: 200, body: 'Connected.' };
             } catch (error) {
-                console.error('Error storing connection:', error);
+                log.error('Error storing connection', {
+                    eventName: 'frigg.websocket.connect_failed',
+                    error,
+                });
                 return { statusCode: 500, body: 'Error connecting.' };
             }
 
@@ -23,17 +32,28 @@ const handleWebSocketConnection = async (event, context) => {
             try {
                 const connectionId = event.requestContext.connectionId;
                 await websocketConnectionRepository.deleteConnection(connectionId);
-                console.log(`Removed connection: ${connectionId}`);
+                log.info('Removed connection', {
+                    eventName: 'frigg.websocket.disconnected',
+                    connectionId,
+                });
                 return { statusCode: 200, body: 'Disconnected.' };
             } catch (error) {
-                console.error('Error removing connection:', error);
+                log.error('Error removing connection', {
+                    eventName: 'frigg.websocket.disconnect_failed',
+                    error,
+                });
                 return { statusCode: 500, body: 'Error disconnecting.' };
             }
 
         case 'MESSAGE':
             // Handle incoming message
-            const message = JSON.parse(event.body);
-            console.log('Received message:', message);
+            // Parsed only to keep the old failure on a body that is not JSON.
+            JSON.parse(event.body);
+            log.info('Received message', {
+                eventName: 'frigg.websocket.message_received',
+                connectionId: event.requestContext.connectionId,
+                bodyLength: event.body?.length ?? 0,
+            });
 
             // Process the message and send a response
             const responseMessage = { message: 'Message received' };

package/handlers/workers/db-migration.js

@@ -55,6 +55,9 @@ const {
 
 // Inject prisma-runner as dependency
 const prismaRunner = require('../../database/utils/prisma-runner');
+const { getLogger } = require('../../logs');
+
+const log = getLogger('frigg.database.migration');
 
 // Use S3 repository for migration status tracking (no User table dependency)
 const bucketName = process.env.S3_BUCKET_NAME || process.env.MIGRATION_STATUS_BUCKET;
@@ -144,7 +147,6 @@ exports.handler = async (event, context) => {
     console.log('========================================');
     console.log('Database Migration Lambda Started');
     console.log('========================================');
-    console.log('Event:', JSON.stringify(event, null, 2));
     console.log('Context:', JSON.stringify({
         requestId: context.requestId,
         functionName: context.functionName,
@@ -157,6 +159,15 @@ exports.handler = async (event, context) => {
     // Check for action parameter (direct invocation for status checks)
     const action = event.action || 'migrate'; // Default to migration
 
+    // targetStage, not stage: the record's own stage field would win.
+    log.info('Database migration invoked', {
+        eventName: 'frigg.database.migration.invoked',
+        migrationId,
+        dbType,
+        targetStage: stage,
+        action,
+    });
+
     // Handle checkStatus action
     if (action === 'checkStatus') {
         console.log(`\n========================================`);

package/handlers/workers/dlq-processor.js

@@ -5,6 +5,18 @@
  * This handler MUST NOT throw. If it throws, the message goes back to
  * the DLQ and creates an infinite loop. All errors are caught and logged.
  */
+const crypto = require('node:crypto');
+const { getLogger } = require('../../logs');
+const {
+    summarizeLambdaEvent,
+    toScopeInvocation,
+} = require('../../logs/summarize-event');
+const {
+    runInvocationScope,
+    runMessageScope,
+} = require('../../core/invocation-scope');
+
+const log = getLogger('frigg.queue.dlq');
 
 function extractQueueName(eventSourceARN) {
     if (!eventSourceARN) return 'UNKNOWN';
@@ -12,50 +24,64 @@ function extractQueueName(eventSourceARN) {
     return parts[parts.length - 1] || 'UNKNOWN';
 }
 
-function parseMessageBody(body) {
+// The body can hold credentials, so only its size and digest are logged.
+function describeBody(body) {
+    const text = typeof body === 'string' ? body : String(body ?? '');
+    let parsed = true;
     try {
-        const parsed = JSON.parse(body);
-        return {
-            event: parsed.event || 'UNKNOWN',
-            integrationId: parsed.data?.integrationId || null,
-            processId: parsed.data?.processId || null,
-            data: parsed.data,
-        };
-    } catch (error) {
-        console.warn('[DLQ] Failed to parse message body', { error: error.message, body });
-        return {
-            event: 'UNKNOWN',
-            integrationId: null,
-            processId: null,
-            rawBody: body,
-        };
+        JSON.parse(text);
+    } catch {
+        parsed = false;
     }
+    return {
+        parsed,
+        bodyLength: Buffer.byteLength(text),
+        bodySha256: crypto.createHash('sha256').update(text).digest('hex'),
+    };
 }
 
-async function dlqProcessor(event) {
+function logRecord(record) {
+    const { parsed, ...body } = describeBody(record.body);
+    if (!parsed) {
+        log.warn('DLQ message body is not JSON', {
+            eventName: 'frigg.queue.dlq.body_unparsed',
+            ...body,
+        });
+    }
+    // messageId, receiveCount and the body ids come from the message scope.
+    log.error('Message reached the DLQ', {
+        eventName: 'frigg.queue.dlq.message_failed',
+        sentTimestamp: record.attributes?.SentTimestamp,
+        sourceQueue: extractQueueName(record.eventSourceARN),
+        ...body,
+    });
+}
+
+async function dlqProcessor(event, context) {
     if (!event?.Records?.length) return { batchItemFailures: [] };
 
-    for (const record of event.Records) {
-        try {
-            const parsed = parseMessageBody(record.body);
-
-            console.error('[DLQ] Failed message', {
-                messageId: record.messageId,
-                event: parsed.event,
-                integrationId: parsed.integrationId,
-                processId: parsed.processId,
-                receiveCount: record.attributes?.ApproximateReceiveCount,
-                sentTimestamp: record.attributes?.SentTimestamp,
-                sourceQueue: extractQueueName(record.eventSourceARN),
-                ...(parsed.rawBody !== undefined && { rawBody: parsed.rawBody }),
-            });
-        } catch (error) {
-            console.error('[DLQ] Error processing DLQ record', {
-                messageId: record.messageId,
-                error: error.message,
-            });
-        }
-    }
+    await runInvocationScope(
+        {
+            requestId: context?.awsRequestId,
+            handlerName: 'dlqProcessor',
+            invocation: toScopeInvocation(summarizeLambdaEvent(event)),
+        },
+        async () => {
+            for (const record of event.Records) {
+                await runMessageScope(record, async () => {
+                    try {
+                        logRecord(record);
+                    } catch (error) {
+                        log.error('DLQ record failed', {
+                            eventName: 'frigg.queue.dlq.record_failed',
+                            error,
+                        });
+                    }
+                });
+            }
+        },
+        { shouldUseDatabase: false, context }
+    );
 
     return { batchItemFailures: [] };
 }

package/index.js

@@ -11,6 +11,7 @@ const {
     Worker,
     loadInstalledModules,
     createHandler,
+    runInvocationScope,
 } = require('./core/index');
 const {
     prisma,
@@ -78,7 +79,17 @@ const {
 } = require('./telemetry/index');
 const { createUsageRepository } = require('./usage/index');
 const { TimeoutCatcher } = require('./lambda/index');
-const { debug, initDebugLog, flushDebugLog } = require('./logs/index');
+const {
+    getLogger,
+    createMemorySink,
+    resetLoggerForTests,
+    serializeError,
+    redactValue,
+    toSanitizedSurrogate,
+    debug,
+    initDebugLog,
+    flushDebugLog,
+} = require('./logs/index');
 const {
     Credential,
     Entity,
@@ -108,6 +119,7 @@ module.exports = {
     Worker,
     loadInstalledModules,
     createHandler,
+    runInvocationScope,
 
     // database
     prisma,
@@ -175,6 +187,12 @@ module.exports = {
     TimeoutCatcher,
 
     // logs
+    getLogger,
+    createMemorySink,
+    resetLoggerForTests,
+    serializeError,
+    redactValue,
+    toSanitizedSurrogate,
     debug,
     initDebugLog,
     flushDebugLog,

package/integrations/integration-base.js

@@ -21,6 +21,7 @@ const { validateExtensionBinding } = require('./extension');
 const { getTelemetry } = require('../telemetry/telemetry-runtime');
 const { instrumentHandler } = require('../telemetry/instrument-handler');
 const { bindTelemetryContext } = require('../telemetry/bind-telemetry-context');
+const { getLogger } = require('../logs');
 
 const constantsToBeMigrated = {
     defaultEvents: {
@@ -64,6 +65,10 @@ class IntegrationBase {
         this.getTelemetryContext()
     );
 
+    logger = getLogger(
+        `integration.${this.constructor.Definition?.name ?? 'unknown'}`
+    ).child(() => this.getLoggerBindings());
+
     static getOptionDetails() {
         const options = new Options({
             module: Object.values(this.Definition.modules)[0], // This is a placeholder until we revamp the frontend
@@ -254,6 +259,14 @@ class IntegrationBase {
         };
     }
 
+    // Stage and appName are record resource fields; binding them would
+    // only land in droppedKeys.
+    getLoggerBindings() {
+        const { integrationId, integrationType, userId, version } =
+            this.getTelemetryContext();
+        return { integrationId, integrationType, userId, version };
+    }
+
     assertHydrated(message = 'Integration instance is not hydrated') {
         if (!this.isHydrated) {
             throw new Error(message);
@@ -411,15 +424,15 @@ class IntegrationBase {
      */
     async reconcileAuthStatus(authPassed) {
         if (!authPassed) {
-            console.log(
-                `[Frigg] Integration ${this.id} failed to authenticate`
-            );
+            this.logger.warn('Integration failed to authenticate', {
+                eventName: `${this.logger.name}.auth_failed`,
+            });
             await this.persistStatus('ERROR');
         }
         if (authPassed && this.status === 'ERROR') {
-            console.log(
-                `[Frigg] auth confirmed for integration ${this.id} — clearing ERROR → ENABLED`
-            );
+            this.logger.info('Auth confirmed, clearing ERROR', {
+                eventName: `${this.logger.name}.auth_confirmed`,
+            });
             await this.persistStatus('ENABLED');
         }
     }
@@ -615,12 +628,22 @@ class IntegrationBase {
         this[key] = module;
     }
 
+    // The raw error can echo a request with its credentials, and messages
+    // reach end users, so only a fixed text is stored.
     addError(error) {
         if (!this.messages.errors) {
             this.messages.errors = [];
         }
-        this.messages.errors.push(error);
+        this.messages.errors.push({
+            title: 'Integration Error',
+            message: `Integration ${this.id} hit an error. Contact support.`,
+            timestamp: Date.now(),
+        });
         this.status = 'ERROR';
+        this.logger.error('Integration error recorded', {
+            eventName: `${this.logger.name}.error_recorded`,
+            error,
+        });
     }
 
     addWarning(warning) {
@@ -639,9 +662,10 @@ class IntegrationBase {
     async persistStatus(status) {
         await this.updateIntegrationStatus.execute(this.id, status);
         this.status = status;
-        console.log(
-            `[Frigg] Integration ${this.id} status changed to ${status}`
-        );
+        this.logger.info('Integration status changed', {
+            eventName: `${this.logger.name}.status_changed`,
+            integrationStatus: status,
+        });
     }
 
     /**
@@ -753,10 +777,14 @@ class IntegrationBase {
                 // it wins. Warn if the binding tried to wire an override that's now ignored.
                 if (this.events[eventName]) {
                     if (typeof handlers[eventName] === 'string') {
-                        console.warn(
-                            `[Frigg] Integration "${integrationName}" binding "${bindingName}": ` +
-                                `handler "${handlers[eventName]}" for event "${eventName}" is ignored because ` +
-                                `this.events["${eventName}"] was already set (subclass constructor or earlier merge)`
+                        this.logger.warn(
+                            'Binding handler is ignored: the event is already set',
+                            {
+                                eventName: `${this.logger.name}.extension_handler_shadowed`,
+                                bindingName,
+                                handler: handlers[eventName],
+                                event: eventName,
+                            }
                         );
                     }
                     continue;
@@ -858,19 +886,12 @@ class IntegrationBase {
             if (this.status === 'ERROR') return;
 
             const moduleName = notifier?.name;
-            const detail =
-                object?.reason || object?.statusCode
-                    ? ` (status ${object?.statusCode ?? '?'}: ${
-                          object?.reason ?? 'no reason given'
-                      })`
-                    : '';
-            console.log(
-                `[Frigg] Module ${
-                    moduleName || '?'
-                } reported invalid credentials for integration ${
-                    this.id
-                } — marking ERROR${detail}`
-            );
+            this.logger.warn('Module reported invalid credentials, marking ERROR', {
+                eventName: `${this.logger.name}.credentials_invalidated`,
+                moduleName,
+                statusCode: object?.statusCode,
+                reason: object?.reason,
+            });
             await this._recordCredentialRejection(
                 moduleName,
                 object?.statusCode
@@ -881,13 +902,10 @@ class IntegrationBase {
 
         if (delegateString === 'CREDENTIAL_VALIDATED') {
             if (this.status !== 'ERROR') return;
-            console.log(
-                `[Frigg] Module ${
-                    notifier?.name || '?'
-                } reported valid credentials for integration ${
-                    this.id
-                } — clearing ERROR → ENABLED`
-            );
+            this.logger.info('Module reported valid credentials, clearing ERROR', {
+                eventName: `${this.logger.name}.credentials_validated`,
+                moduleName: notifier?.name,
+            });
             await this.persistStatus('ENABLED');
         }
     }
@@ -909,10 +927,10 @@ class IntegrationBase {
                 Date.now()
             );
         } catch (error) {
-            console.error(
-                `[Frigg] Failed to record credential rejection for integration ${this.id}:`,
-                error
-            );
+            this.logger.error('Failed to record credential rejection', {
+                eventName: `${this.logger.name}.credential_rejection_record_failed`,
+                error,
+            });
         }
     }
 }

package/integrations/integration-router.js

@@ -2,6 +2,7 @@ const express = require('express');
 const { get } = require('../assertions');
 const Boom = require('@hapi/boom');
 const catchAsyncError = require('express-async-handler');
+const { getLogger } = require('../logs');
 const {
     createIntegrationRepository,
 } = require('./repositories/integration-repository-factory');
@@ -70,6 +71,8 @@ const {
     ProcessAuthorizationCallback,
 } = require('../modules/use-cases/process-authorization-callback');
 
+const log = getLogger('frigg.integrations');
+
 function createIntegrationRouter() {
     const { integrations: integrationClasses, userConfig } =
         loadAppDefinition();
@@ -536,29 +539,27 @@ function setEntityRoutes(router, authenticateUser, useCases) {
                 params.data && typeof params.data === 'object'
                     ? Object.keys(params.data)
                     : [];
-            console.log(
-                `[Frigg] POST /api/authorize userId=${userId} entityType=${params.entityType} dataKeys=${JSON.stringify(dataKeys)}`
-            );
+            log.debug('Authorize requested', {
+                userId,
+                entityType: params.entityType,
+                dataKeys,
+            });
 
-            try {
-                const entityDetails =
-                    await processAuthorizationCallback.execute(
-                        userId,
-                        params.entityType,
-                        params.data
-                    );
+            const entityDetails = await processAuthorizationCallback.execute(
+                userId,
+                params.entityType,
+                params.data
+            );
 
-                console.log(
-                    `[Frigg] POST /api/authorize success userId=${userId} entityType=${params.entityType} credentialId=${entityDetails?.credential_id} entityId=${entityDetails?.entity_id}`
-                );
+            log.info('Entity authorized', {
+                eventName: 'frigg.integrations.authorized',
+                userId,
+                entityType: params.entityType,
+                credentialId: entityDetails?.credential_id,
+                entityId: entityDetails?.entity_id,
+            });
 
-                res.json(entityDetails);
-            } catch (err) {
-                console.error(
-                    `[Frigg] POST /api/authorize failed userId=${userId} entityType=${params.entityType} error=${err?.message || err}`
-                );
-                throw err;
-            }
+            res.json(entityDetails);
         })
     );
 

package/integrations/repositories/integration-mapping-query-pipeline.js

@@ -0,0 +1,152 @@
+const SORT_KEY = '__sortKey';
+const MONGO_DIRECTIONS = { asc: 1, desc: -1 };
+const SORT_TYPE_RANKS = [
+    ['string'],
+    ['int', 'long', 'double', 'decimal'],
+    ['bool'],
+    ['array'],
+    ['object'],
+];
+const SCALAR_TYPES = SORT_TYPE_RANKS.slice(0, 3).flat();
+
+const isType = (expression, type) => ({ $eq: [{ $type: expression }, type] });
+const isAbsent = (expression) => ({
+    $eq: [{ $ifNull: [expression, null] }, null],
+});
+const mappingField = (segments) => `$mapping.${segments.join('.')}`;
+
+const CONDITION_EXPRESSIONS = {
+    exists: (operand) => ({ $ne: [{ $ifNull: [operand, null] }, null] }),
+    notExists: (operand) => isAbsent(operand),
+    in: (operand, values) => ({
+        $and: [
+            isType(operand, 'string'),
+            { $in: [operand, { $literal: values }] },
+        ],
+    }),
+    notStartsWith: (operand, prefix) => ({
+        $cond: [
+            isType(operand, 'string'),
+            {
+                $ne: [
+                    { $substrCP: [operand, 0, [...prefix].length] },
+                    { $literal: prefix },
+                ],
+            },
+            true,
+        ],
+    }),
+};
+
+/**
+ * Aggregation stages that answer a validated queryMappings query on the
+ * MongoDB wire protocol.
+ *
+ * @param {*} integrationId - The value IntegrationMapping.integrationId is stored as
+ * @param {ReturnType<import('./integration-mapping-query').validateMappingQuery>} query
+ * @returns {{match: Object, page: Object[], sort: Object}} `match` selects
+ *   the matching documents; `page` sorts, skips and limits them; `sort` is
+ *   the $sort stage inside `page`
+ */
+function buildMappingQueryStages(
+    integrationId,
+    { where, orderBy, skip, take, omit }
+) {
+    const match = {
+        $match: {
+            integrationId,
+            $expr: {
+                $and: [
+                    isType('$mapping', 'object'),
+                    ...where.map(entryExpression),
+                ],
+            },
+        },
+    };
+    const sort = { $sort: sortSpec(orderBy) };
+    const page = [
+        ...(orderBy
+            ? [{ $addFields: { [SORT_KEY]: sortKey(orderBy.path) } }]
+            : []),
+        ...(omit.length > 0 ? [{ $project: omitProjection(omit) }] : []),
+        sort,
+        ...(skip > 0 ? [{ $skip: skip }] : []),
+        { $limit: take },
+    ];
+    return { match, page, sort };
+}
+
+function entryExpression(entry) {
+    if (!entry.anyOf) return conditionExpression(entry);
+    return { $or: entry.anyOf.map(conditionExpression) };
+}
+
+function conditionExpression({ field, path, op, value }) {
+    const operand = field === 'sourceId' ? '$sourceId' : resolvePath(path);
+    return CONDITION_EXPRESSIONS[op](operand, value);
+}
+
+/**
+ * The value at a mapping path; null when an enclosing value is not an object.
+ */
+function resolvePath(segments) {
+    const enclosing = segments
+        .slice(0, -1)
+        .map((_, i) =>
+            isType(mappingField(segments.slice(0, i + 1)), 'object')
+        );
+    if (enclosing.length === 0) return mappingField(segments);
+    return {
+        $cond: [
+            enclosing.length === 1 ? enclosing[0] : { $and: enclosing },
+            mappingField(segments),
+            null,
+        ],
+    };
+}
+
+/**
+ * Orders values like jsonb: strings < numbers < booleans < arrays < objects,
+ * scalars by value within their type. Null and absent values sort last in
+ * both directions.
+ */
+function sortKey(path) {
+    const type = { $type: '$$value' };
+    return {
+        $let: {
+            vars: { value: resolvePath(path) },
+            in: {
+                missing: isAbsent('$$value'),
+                rank: {
+                    $switch: {
+                        branches: SORT_TYPE_RANKS.map((types, i) => ({
+                            case: { $in: [type, types] },
+                            then: i + 1,
+                        })),
+                        default: 0,
+                    },
+                },
+                value: {
+                    $cond: [{ $in: [type, SCALAR_TYPES] }, '$$value', null],
+                },
+            },
+        },
+    };
+}
+
+function sortSpec(orderBy) {
+    if (!orderBy) return { _id: 1 };
+    const direction = MONGO_DIRECTIONS[orderBy.direction];
+    return {
+        [`${SORT_KEY}.missing`]: 1,
+        [`${SORT_KEY}.rank`]: direction,
+        [`${SORT_KEY}.value`]: direction,
+        _id: direction,
+    };
+}
+
+function omitProjection(omit) {
+    return Object.fromEntries(omit.map((key) => [`mapping.${key}`, 0]));
+}
+
+module.exports = { buildMappingQueryStages };