@friggframework/core 2.0.0-next.112 → 2.0.0-next.113

package/logs/sinks.js

@@ -0,0 +1,93 @@
+const fs = require('node:fs');
+const { deepFreeze } = require('./record');
+
+const DEFAULT_MAX_RETRIES = 10;
+const MAX_SLEEP_MS = 20;
+
+function sleepSync(ms) {
+    Atomics.wait(new Int32Array(new SharedArrayBuffer(4)), 0, 0, ms);
+}
+
+function defaultWriteStderr(line) {
+    fs.writeSync(2, line);
+}
+
+function safeCode(code) {
+    return String(code ?? 'UNKNOWN').replace(/[^A-Za-z0-9_]/g, '').slice(0, 32) || 'UNKNOWN';
+}
+
+function createStdoutSink({
+    fd = 1,
+    writeSync = fs.writeSync,
+    writeStderr = defaultWriteStderr,
+    sleep = sleepSync,
+    maxRetries = DEFAULT_MAX_RETRIES,
+} = {}) {
+    const reportFailure = (code) => {
+        try {
+            writeStderr(`frigg.logger.write_failed code=${safeCode(code)}\n`);
+        } catch {
+            // Nothing is left to report to.
+        }
+    };
+
+    return {
+        name: 'stdout',
+        write(record) {
+            let buffer;
+            try {
+                buffer = Buffer.from(`${JSON.stringify(record)}\n`);
+            } catch {
+                reportFailure('SERIALIZE');
+                return;
+            }
+            let offset = 0;
+            let retries = 0;
+            // EAGAIN and a zero-byte write share one bounded retry budget.
+            const retry = () => {
+                if (retries >= maxRetries) {
+                    reportFailure('EAGAIN');
+                    return false;
+                }
+                retries += 1;
+                sleep(Math.min(retries, MAX_SLEEP_MS));
+                return true;
+            };
+            while (offset < buffer.length) {
+                let written;
+                try {
+                    written = writeSync(fd, buffer, offset, buffer.length - offset);
+                } catch (error) {
+                    if (error?.code !== 'EAGAIN') {
+                        reportFailure(error?.code);
+                        return;
+                    }
+                    if (!retry()) return;
+                    continue;
+                }
+                if (written > 0) offset += written;
+                else if (!retry()) return;
+            }
+        },
+    };
+}
+
+function createMemorySink({ install = true } = {}) {
+    const records = [];
+    const sink = {
+        name: 'memory',
+        records,
+        write(record) {
+            records.push(deepFreeze(JSON.parse(JSON.stringify(record))));
+        },
+        clear() {
+            records.length = 0;
+        },
+    };
+    if (install) {
+        require('./logger-runtime').setSinks([sink]);
+    }
+    return sink;
+}
+
+module.exports = { createStdoutSink, createMemorySink };

package/logs/summarize-event.js

@@ -0,0 +1,163 @@
+const { redactUrl } = require('./redact');
+
+// Best-effort extraction of the logical event/processId/integrationId from a
+// JSON message body: `data.*` first, then the top level. Used only for log
+// correlation. Never throws.
+function summarizeMessageBody(bodyStr) {
+    try {
+        const parsed = JSON.parse(bodyStr);
+        return {
+            event: parsed?.event,
+            processId: parsed?.data?.processId ?? parsed?.processId,
+            integrationId: parsed?.data?.integrationId ?? parsed?.integrationId,
+        };
+    } catch {
+        return {};
+    }
+}
+
+function objectKeys(value) {
+    return value && typeof value === 'object' && !Array.isArray(value)
+        ? Object.keys(value)
+        : [];
+}
+
+function rawQueryKeys(rawQueryString) {
+    if (typeof rawQueryString !== 'string' || !rawQueryString) return [];
+    try {
+        return [...new URLSearchParams(rawQueryString).keys()];
+    } catch {
+        return [];
+    }
+}
+
+function unique(list) {
+    return [...new Set(list)];
+}
+
+// The route template keeps ids out of the field: REST v1 `resource`, or the
+// path part of an HTTP API v2 `routeKey` such as `GET /api/{id}`.
+function routeTemplate(event, path) {
+    if (typeof event.resource === 'string' && event.resource) {
+        return event.resource;
+    }
+    if (typeof event.routeKey === 'string') {
+        const space = event.routeKey.indexOf(' ');
+        if (space > 0) return event.routeKey.slice(space + 1);
+    }
+    return path;
+}
+
+function summarizeHttp(event) {
+    const path = event.path || event.rawPath;
+    const summary = {
+        source: 'http',
+        method: event.httpMethod || event.requestContext?.http?.method,
+        path,
+        route: routeTemplate(event, path),
+        queryKeys: unique([
+            ...objectKeys(event.queryStringParameters),
+            ...objectKeys(event.multiValueQueryStringParameters),
+            ...rawQueryKeys(event.rawQueryString),
+        ]),
+        headerNames: unique(
+            [...objectKeys(event.headers), ...objectKeys(event.multiValueHeaders)].map(
+                (name) => name.toLowerCase()
+            )
+        ),
+    };
+    if (typeof event.routeKey === 'string' && event.routeKey) {
+        summary.routeKey = event.routeKey;
+    }
+    return summary;
+}
+
+// Best-effort extraction of correlation identifiers from a Lambda event.
+// For SQS: messageIds plus the parsed event/processId/integrationId of each
+// record body. For HTTP: method, route, query keys and header names. Never
+// throws and never returns a body, a query value or a header value.
+function summarizeLambdaEvent(event) {
+    try {
+        if (!event || typeof event !== 'object') return {};
+        if (Array.isArray(event.Records)) {
+            return {
+                source: 'sqs',
+                records: event.Records.map((r) => ({
+                    messageId: r?.messageId,
+                    receiveCount: r?.attributes?.ApproximateReceiveCount,
+                    ...summarizeMessageBody(r?.body),
+                })),
+            };
+        }
+        if (event.httpMethod || event.requestContext?.http) {
+            return summarizeHttp(event);
+        }
+        return { source: 'other' };
+    } catch {
+        return { source: 'other' };
+    }
+}
+
+// The bounded form that goes into the logger scope: no per-record array.
+function toScopeInvocation(summary) {
+    if (!summary || typeof summary !== 'object' || !summary.source) {
+        return { source: 'other' };
+    }
+    if (summary.source === 'sqs') {
+        return {
+            source: 'sqs',
+            recordCount: Array.isArray(summary.records) ? summary.records.length : 0,
+        };
+    }
+    if (summary.source === 'http') {
+        const invocation = {
+            source: 'http',
+            method: summary.method,
+            route: summary.route,
+        };
+        if (summary.routeKey) invocation.routeKey = summary.routeKey;
+        return invocation;
+    }
+    return { source: summary.source };
+}
+
+// Per-request detail for the entry and failure records, never for the
+// scope. A raw path can hold a token, so it goes through redactUrl.
+function toRequestDetails(summary) {
+    if (!summary || summary.source !== 'http') return {};
+    return {
+        path: redactUrl(summary.path ?? ''),
+        queryKeys: summary.queryKeys ?? [],
+        headerNames: summary.headerNames ?? [],
+    };
+}
+
+// The full redacted request summary (ADR-048 §6: it goes under `invocation`).
+function toRequestInvocation(summary) {
+    return { ...toScopeInvocation(summary), ...toRequestDetails(summary) };
+}
+
+// The same shape from an express request, for the express error boundary.
+function summarizeExpressRequest(req) {
+    try {
+        if (!req || typeof req !== 'object') return { source: 'http' };
+        return {
+            source: 'http',
+            method: req.method,
+            path: redactUrl(req.path ?? ''),
+            queryKeys: objectKeys(req.query),
+            headerNames: unique(objectKeys(req.headers).map((name) => name.toLowerCase())),
+        };
+    } catch {
+        return { source: 'http' };
+    }
+}
+
+module.exports = {
+    summarizeLambdaEvent,
+    summarizeMessageBody,
+    toScopeInvocation,
+    toRequestDetails,
+    toRequestInvocation,
+    summarizeExpressRequest,
+};

package/modules/module.js

@@ -1,6 +1,6 @@
 const { Delegate } = require('../core');
 const _ = require('lodash');
-const { flushDebugLog } = require('../logs');
+const { getLogger } = require('../logs');
 const { ModuleConstants } = require('./ModuleConstants');
 const {
     createCredentialRepository,
@@ -32,6 +32,14 @@ class Module extends Delegate {
         this.credential = entityObj?.credential;
         this.definition = definition;
         this.name = this.definition.moduleName;
+        this.logger = getLogger(`module.${this.name ?? 'unknown'}`).child(() => ({
+            entityId: this.entity?.id,
+            credentialId:
+                this.credential?.id ??
+                (typeof this.credential === 'string'
+                    ? this.credential
+                    : undefined),
+        }));
         this.modelName = this.definition.modelName;
         this.apiClass = this.definition.API;
 
@@ -49,6 +57,7 @@ class Module extends Delegate {
         const apiParams = {
             ...this.definition.env,
             delegate: this,
+            logger: this.logger,
             ...(state ? { state } : {}),
             ...(this.credential?.data
                 ? this.apiParamsFromCredential(this.credential.data)
@@ -100,7 +109,10 @@ class Module extends Delegate {
         try {
             if (await this.testAuthRequest(this.api)) validAuth = true;
         } catch (e) {
-            flushDebugLog(e);
+            this.logger.warn('testAuth failed', {
+                eventName: `${this.logger.name}.test_auth_failed`,
+                error: e,
+            });
         }
         return validAuth;
     }
@@ -113,9 +125,9 @@ class Module extends Delegate {
         const apiParams = this.apiParamsFromCredential(this.api);
 
         if (!apiParams.refresh_token && this.api.isRefreshable) {
-            console.warn(
-                `[Frigg] No refresh_token in apiParams for module ${this.name}.`
-            );
+            this.logger.warn('No refresh_token in apiParams', {
+                eventName: `${this.logger.name}.refresh_token_missing`,
+            });
         }
 
         Object.assign(credentialDetails.details, apiParams);
@@ -133,10 +145,10 @@ class Module extends Delegate {
                     moduleName: this.name,
                 });
             } catch (err) {
-                console.error(
-                    `[Frigg] Failed to propagate CREDENTIAL_VALIDATED for module ${this.name}:`,
-                    err?.message || err
-                );
+                this.logger.error('Failed to propagate CREDENTIAL_VALIDATED', {
+                    eventName: `${this.logger.name}.credential_validated_propagation_failed`,
+                    error: err,
+                });
             }
         }
     }
@@ -177,12 +189,11 @@ class Module extends Delegate {
         if (!this.credential.id) return;
 
         if (diagnosticInfo) {
-            console.error(
-                `[Frigg] Module ${this.name} credentials rejected (status ${
-                    diagnosticInfo.statusCode ?? '?'
-                }):`,
-                diagnosticInfo.message ?? diagnosticInfo
-            );
+            this.logger.warn('Credentials rejected', {
+                eventName: `${this.logger.name}.credentials_rejected`,
+                statusCode: diagnosticInfo.statusCode,
+                error: diagnosticInfo,
+            });
         }
 
         await this.credentialRepository.updateAuthenticationStatus(
@@ -216,16 +227,17 @@ class Module extends Delegate {
                 }),
             });
         } catch (err) {
-            console.error(
-                `[Frigg] Failed to propagate CREDENTIAL_INVALIDATED for module ${this.name}:`,
-                err?.message || err
-            );
+            this.logger.error('Failed to propagate CREDENTIAL_INVALIDATED', {
+                eventName: `${this.logger.name}.credential_invalidated_propagation_failed`,
+                error: err,
+            });
         }
     }
 
     async deauthorize() {
         //todo: Check if this is correct, we're instantiating a new api without params (credentials, tokens, etc...)
         this.api = new this.apiClass();
+        this.api.logger = this.logger;
 
         // Remove persisted credential (if any)
         if (this.entity?.credential) {
@@ -234,6 +246,7 @@ class Module extends Delegate {
 
             // Delete credential via repository
             await this.credentialRepository.deleteCredentialById(credentialId);
+            this.credential = undefined;
 
             // Unset credential reference on the Entity document
             const entityId = this.entity.id;

package/modules/requester/oauth-2.js

@@ -136,13 +136,13 @@ class OAuth2Requester extends Requester {
             this.refresh_token = newRefreshToken;
         } else {
             if (this.refresh_token) {
-                console.log(
-                    '[Frigg] No refresh_token in response, preserving existing'
-                );
+                this.logger.debug('No refresh_token in response, preserving existing', {
+                    eventName: `${this.logger.name}.refresh_token_preserved`,
+                });
             } else {
-                console.log(
-                    '[Frigg] Current refresh_token is null and no new refresh_token in response'
-                );
+                this.logger.debug('No refresh_token in response and none held', {
+                    eventName: `${this.logger.name}.refresh_token_absent`,
+                });
             }
         }
         const accessExpiresIn = get(params, 'expires_in', null);
@@ -265,7 +265,9 @@ class OAuth2Requester extends Requester {
                 'Content-Type': 'application/x-www-form-urlencoded',
             },
         };
-        console.log('[Frigg] Refreshing access token with options');
+        this.logger.debug('Refreshing access token', {
+            eventName: `${this.logger.name}.token_refresh_request`,
+        });
         const response = await this._post(options, false);
         await this.setTokens(response);
         return response;
@@ -323,12 +325,14 @@ class OAuth2Requester extends Requester {
         }
 
         try {
-            console.log('[Frigg] Starting token refresh', {
-                grant_type: this.grant_type,
-                has_refresh_token: !!this.refresh_token,
-                has_client_id: !!this.client_id,
-                has_client_secret: !!this.client_secret,
-                has_token_uri: !!this.tokenUri,
+            // Field names avoid the denied suffixes (token, secret), which
+            // would redact these booleans.
+            this.logger.debug('Starting token refresh', {
+                eventName: `${this.logger.name}.token_refresh_started`,
+                grantType: this.grant_type,
+                refreshTokenPresent: !!this.refresh_token,
+                clientIdPresent: !!this.client_id,
+                clientSecretPresent: !!this.client_secret,
                 tokenUri: this.tokenUri,
             });
 
@@ -343,15 +347,17 @@ class OAuth2Requester extends Requester {
                 const tokenRes = await this.getTokenFromClientCredentials();
                 if (!tokenRes) return false;
             }
-            console.log('[Frigg] Token refresh succeeded');
+            this.logger.info('Token refresh succeeded', {
+                eventName: `${this.logger.name}.token_refreshed`,
+            });
             return true;
         } catch (error) {
             const moduleName = this.delegate?.name ?? 'unknown module';
-            console.error(`[Frigg] Token refresh failed for ${moduleName}`, {
-                error_message: error?.message,
-                error_name: error?.name,
-                response_status: error?.response?.status,
-                response_data: error?.response?.data,
+            this.logger.debug('Token refresh failed', {
+                eventName: `${this.logger.name}.token_refresh_failed`,
+                statusCode:
+                    error?.statusCode ?? error?.status ?? error?.response?.status,
+                reason: error?.message,
             });
 
             if (!this._isDefinitiveAuthRejection(error)) {
@@ -405,9 +411,9 @@ class OAuth2Requester extends Requester {
         for (const delayMs of this.credentialReloadBackoffMs) {
             await new Promise((resolve) => setTimeout(resolve, delayMs));
             if (await this._adoptNewerCredential()) {
-                console.log(
-                    '[Frigg] Adopted a newer credential after a refresh rejection',
-                    { module: this._telemetryModuleLabel() }
+                this.logger.info(
+                    'Adopted a newer credential after a refresh rejection',
+                    { eventName: `${this.logger.name}.credential_adopted` }
                 );
                 this.telemetry?.count?.('frigg.auth.refresh_race_recovered', 1, {
                     module: this._telemetryModuleLabel(),

package/modules/requester/requester.js

@@ -4,8 +4,22 @@ const { Delegate } = require('../../core');
 const { FetchError } = require('../../errors');
 const { get } = require('../../assertions');
 const { getTelemetry } = require('../../telemetry/telemetry-runtime');
+const { getLogger } = require('../../logs');
+const { redactUrl } = require('../../logs/redact');
+const { toSanitizedSurrogate } = require('../../logs/serialize');
+const { getLoggerScope } = require('../../logs/context');
 
 const DEFAULT_REQUEST_TIMEOUT_MS = 60_000;
+
+// A node-fetch error message holds the raw URL, and util.inspect prints the
+// cause chain, so the FetchError keeps only a sanitized copy.
+function sanitizedCause(err) {
+    const surrogate = toSanitizedSurrogate(err);
+    for (const key of ['errno', 'type']) {
+        if (typeof err?.[key] === 'string') surrogate[key] = err[key];
+    }
+    return surrogate;
+}
 const MAX_AUTH_RETRIES = 3;
 
 // This context marks the async call chain that holds the refresh slot, and
@@ -64,6 +78,18 @@ class Requester extends Delegate {
         // `this.telemetry` to attribute out-of-band requests — setup/OAuth calls
         // made before an integration context exists aren't rolled up otherwise.
         this.telemetry = (params && params.telemetry) || getTelemetry();
+
+        // Not `get(params, 'logger')`: it throws when the key is missing.
+        this._logger = params?.logger ?? null;
+    }
+
+    // Resolved per read, so a delegate set after construction names the logger.
+    get logger() {
+        return this._logger ?? getLogger(`module.${this._telemetryModuleLabel()}`);
+    }
+
+    set logger(logger) {
+        this._logger = logger ?? null;
     }
 
     /**
@@ -141,11 +167,14 @@ class Requester extends Delegate {
 
         return telemetry.span('frigg.apimodule.request', async (span) => {
             if (span && typeof span.setAttributes === 'function') {
+                const { requestId, messageId } = getLoggerScope();
                 span.setAttributes({
                     'frigg.module': module,
                     'http.request.method': method,
                     // Redacted (no query/userinfo) — never emit raw URLs.
                     'url.path': safeUrl,
+                    ...(requestId !== undefined && { requestId }),
+                    ...(messageId !== undefined && { messageId }),
                 });
             }
             // Redacted url (unbounded) rides the bus-only context for North Star
@@ -255,15 +284,12 @@ class Requester extends Delegate {
                 const fetchError = await FetchError.create({
                     resource: encodedUrl,
                     init: options,
-                    responseBody: isTimeout
-                        ? `Request timed out after ${timeoutMs}ms`
-                        : e,
+                    cause: sanitizedCause(e),
                 });
                 if (isTimeout) {
                     // Flag + machine-readable fields so callers can
                     // distinguish a timeout from a generic network error
-                    // without parsing the message (which FetchError
-                    // sanitizes outside of STAGE=dev).
+                    // without parsing the message.
                     fetchError.isTimeout = true;
                     fetchError.timeoutMs = timeoutMs;
                 }
@@ -361,6 +387,7 @@ class Requester extends Delegate {
             // the response body (response.text()) — timer must still be
             // alive to catch a stalled body stream.
             if (status >= 400) {
+                this._logRequestFailed(encodedUrl, options, status);
                 const fetchError = await FetchError.create({
                     resource: encodedUrl,
                     init: options,
@@ -387,12 +414,25 @@ class Requester extends Delegate {
             // If the abort fired during body consumption, node-fetch emits
             // the error as an AbortError on the body stream. Surface the
             // same isTimeout flag callers use for header-phase timeouts.
-            throw this._maybeFlagTimeoutDuringBodyRead(e, timeoutMs);
+            const flagged = this._maybeFlagTimeoutDuringBodyRead(e, timeoutMs);
+            throw this._wrapFetchLibraryError(flagged, encodedUrl, options);
         } finally {
             clearRequestTimer();
         }
     }
 
+    _logRequestFailed(encodedUrl, options, status) {
+        const logger = this.logger;
+        if (!logger.isLevelEnabled('DEBUG')) return;
+        logger.debug('Request failed', {
+            eventName: `${logger.name}.request_failed`,
+            method: (options.method || 'GET').toUpperCase(),
+            url: redactUrl(encodedUrl),
+            statusCode: status,
+            headerNames: Object.keys(options.headers || {}),
+        });
+    }
+
     async _invalidateAuth(encodedUrl, options, response) {
         const fetchError = await FetchError.create({
             resource: encodedUrl,
@@ -403,6 +443,29 @@ class Requester extends Delegate {
         return fetchError;
     }
 
+    // node-fetch names its own error class FetchError too, and its message
+    // holds the raw URL. Only our class passes through unwrapped.
+    _wrapFetchLibraryError(err, encodedUrl, options) {
+        if (!err || typeof err !== 'object' || err instanceof FetchError) {
+            return err;
+        }
+        const isFetchLibraryError =
+            err.name === 'FetchError' ||
+            err.name === 'AbortError' ||
+            typeof err.type === 'string';
+        if (!isFetchLibraryError) return err;
+        const wrapped = new FetchError({
+            resource: encodedUrl,
+            init: options,
+            cause: sanitizedCause(err),
+        });
+        if (err.isTimeout) {
+            wrapped.isTimeout = true;
+            wrapped.timeoutMs = err.timeoutMs;
+        }
+        return wrapped;
+    }
+
     _maybeFlagTimeoutDuringBodyRead(err, timeoutMs) {
         if (!err || typeof err !== 'object') return err;
         if (err.isTimeout) return err;

package/package.json

@@ -1,7 +1,7 @@
 {
     "name": "@friggframework/core",
     "prettier": "@friggframework/prettier-config",
-    "version": "2.0.0-next.112",
+    "version": "2.0.0-next.113",
     "dependencies": {
         "@aws-sdk/client-apigatewaymanagementapi": "^3.588.0",
         "@aws-sdk/client-kms": "^3.588.0",
@@ -22,7 +22,6 @@
         "body-parser": "^1.20.5",
         "bson": "^4.7.2",
         "chalk": "^4.1.2",
-        "common-tags": "^1.8.2",
         "cors": "^2.8.5",
         "dotenv": "^16.4.7",
         "express": "^4.22.2",
@@ -48,9 +47,9 @@
         }
     },
     "devDependencies": {
-        "@friggframework/eslint-config": "2.0.0-next.112",
-        "@friggframework/prettier-config": "2.0.0-next.112",
-        "@friggframework/test": "2.0.0-next.112",
+        "@friggframework/eslint-config": "2.0.0-next.113",
+        "@friggframework/prettier-config": "2.0.0-next.113",
+        "@friggframework/test": "2.0.0-next.113",
         "@prisma/client": "^6.19.3",
         "@types/lodash": "4.17.15",
         "@typescript-eslint/eslint-plugin": "^8.0.0",
@@ -90,5 +89,5 @@
     "publishConfig": {
         "access": "public"
     },
-    "gitHead": "2346e78e4c68226a6647d472857c055d84b8f2e8"
+    "gitHead": "783d9f59b16663376c313b1bb4874d5efcb54a5a"
 }

package/queues/queue-delivery.js

@@ -0,0 +1,41 @@
+const INTEGRATION_QUEUE_MAX_RECEIVE_COUNT = 3;
+const QUEUE_MAX_RECEIVE_COUNT_ENV = 'FRIGG_QUEUE_MAX_RECEIVE_COUNT';
+
+const toPositiveInteger = (value) => {
+    const number = Number(value);
+    return Number.isInteger(number) && number > 0 ? number : undefined;
+};
+
+/**
+ * @typedef {Object} QueueDelivery
+ * @property {number|undefined} receiveCount SQS ApproximateReceiveCount of this delivery.
+ * @property {number|undefined} maxReceiveCount Receives allowed before the queue dead-letters the message.
+ * @property {boolean} isLastAttempt True only when both counts are known and receiveCount >= maxReceiveCount.
+ */
+
+/**
+ * @param {Object} record SQS event record
+ * @param {Object} [env=process.env]
+ * @returns {QueueDelivery}
+ */
+const readQueueDelivery = (record, env = process.env) => {
+    const receiveCount = toPositiveInteger(
+        record?.attributes?.ApproximateReceiveCount
+    );
+    const maxReceiveCount = toPositiveInteger(env[QUEUE_MAX_RECEIVE_COUNT_ENV]);
+
+    return {
+        receiveCount,
+        maxReceiveCount,
+        isLastAttempt:
+            receiveCount !== undefined &&
+            maxReceiveCount !== undefined &&
+            receiveCount >= maxReceiveCount,
+    };
+};
+
+module.exports = {
+    INTEGRATION_QUEUE_MAX_RECEIVE_COUNT,
+    QUEUE_MAX_RECEIVE_COUNT_ENV,
+    readQueueDelivery,
+};