@friggframework/core 2.0.0-next.11 → 2.0.0-next.110

package/core/Worker.js

@@ -1,10 +1,9 @@
-const AWS = require('aws-sdk');
+const { SQSClient, GetQueueUrlCommand, SendMessageCommand } = require('@aws-sdk/client-sqs');
 const _ = require('lodash');
 const { RequiredPropertyError } = require('../errors');
 const { get } = require('../assertions');
 
-AWS.config.update({ region: process.env.AWS_REGION });
-const sqs = new AWS.SQS({ apiVersion: '2012-11-05' });
+const sqs = new SQSClient({ region: process.env.AWS_REGION });
 
 class Worker {
     async getQueueURL(params) {
@@ -12,25 +11,68 @@ class Worker {
         // let params = {
         //     QueueName:  process.env.QueueName
         // };
-        return new Promise((resolve, reject) => {
-            sqs.getQueueUrl(params, (err, data) => {
-                if (err) {
-                    reject(err);
-                } else {
-                    resolve(data.QueueUrl);
-                }
-            });
-        });
+        const command = new GetQueueUrlCommand(params);
+        const data = await sqs.send(command);
+        return data.QueueUrl;
     }
 
     async run(params, context = {}) {
         const records = get(params, 'Records');
+        const batchItemFailures = [];
+
+        console.log(
+            `[Worker] run: processing ${records.length} record(s)`
+        );
 
         for (const record of records) {
-            const runParams = JSON.parse(record.body);
-            this._validateParams(runParams);
-            await this._run(runParams, context);
+            // Log record entry with SQS-provided attributes useful for tracing
+            // delivery history (ApproximateReceiveCount for retries, etc.).
+            let parsedEvent;
+            try {
+                parsedEvent = JSON.parse(record.body)?.event;
+            } catch {
+                parsedEvent = undefined;
+            }
+            console.log(`[Worker] record begin`, {
+                messageId: record.messageId,
+                event: parsedEvent,
+                receiveCount: record.attributes?.ApproximateReceiveCount,
+            });
+
+            try {
+                const runParams = JSON.parse(record.body);
+                this._validateParams(runParams);
+                await this._run(runParams, context);
+                console.log(`[Worker] record success`, {
+                    messageId: record.messageId,
+                    event: runParams?.event,
+                });
+            } catch (error) {
+                if (error.isHaltError) {
+                    // HaltError means "discard this message, don't retry".
+                    // Treat as success so SQS deletes it from the queue.
+                    // Logged explicitly — silent discards made prod debugging
+                    // extremely hard; keep this visible.
+                    console.warn(`[Worker] record halted (discarded, no retry)`, {
+                        messageId: record.messageId,
+                        event: parsedEvent,
+                        reason: error.message,
+                        statusCode: error.statusCode,
+                    });
+                    continue;
+                }
+                console.error(`[Worker] Failed to process record ${record.messageId}:`, error);
+                batchItemFailures.push({ itemIdentifier: record.messageId });
+            }
         }
+
+        if (batchItemFailures.length > 0) {
+            console.warn(
+                `[Worker] run: returning ${batchItemFailures.length} batchItemFailure(s) of ${records.length}`
+            );
+        }
+
+        return { batchItemFailures };
     }
 
     async _run(params, context = {}) {
@@ -54,15 +96,9 @@ class Worker {
     }
 
     async sendAsyncSQSMessage(params) {
-        return new Promise((resolve, reject) => {
-            sqs.sendMessage(params, (err, data) => {
-                if (err) {
-                    reject(err);
-                } else {
-                    resolve(data.MessageId);
-                }
-            });
-        });
+        const command = new SendMessageCommand(params);
+        const data = await sqs.send(command);
+        return data.MessageId;
     }
 
     // Throw an exception if the params do not validate

package/core/create-handler.js

@@ -1,9 +1,133 @@
 // This line should be at the top of the webpacked output, so be sure to require createHandler first in any handlers.  "Soon" sourcemaps will be built into Node... after that, this package won't be needed.
-require('source-map-support').install();
+// REMOVING FOR NOW UNTIL WE ADD WEBPACK BACK IN
+// require('source-map-support').install();
 
-const { connectToDatabase } = require('../database/mongo');
 const { initDebugLog, flushDebugLog } = require('../logs');
 const { secretsToEnv } = require('./secrets-to-env');
+const { parametersToEnv } = require('./parameters-to-env');
+const {
+    getTelemetry,
+    getUsageRollupSubscriber,
+    getPluginTelemetrySubscribers,
+} = require('../telemetry/telemetry-runtime');
+
+// Bounds the tail latency telemetry adds to every warm invocation. Kept low so
+// an unreachable OTLP endpoint (e.g. a VPC Lambda with no NAT/egress) costs at
+// most this, not multiple seconds. Override with OTEL_FLUSH_TIMEOUT_MS.
+const DEFAULT_FLUSH_TIMEOUT_MS =
+    Number(process.env.OTEL_FLUSH_TIMEOUT_MS) || 500;
+
+/**
+ * Fold the invocation's buffered usage counters into the durable store, then
+ * clear the buffer. On an SQS redelivery (ApproximateReceiveCount > 1) we
+ * DISCARD rather than flush — the prior delivery already counted, and the usage
+ * accuracy contract is "approximate, skip obvious redeliveries". Fully guarded.
+ */
+async function flushUsageRollup(subscriber, eventSummary, shouldUseDatabase) {
+    if (!subscriber) return;
+    try {
+        // Persisting usage requires a DB connection. DB-free handlers (e.g. the
+        // webhook-receipt route) never called connectPrisma, so drop the buffer
+        // instead of issuing a connectionless Prisma write.
+        if (!shouldUseDatabase) {
+            subscriber.discard();
+            return;
+        }
+        // Discard only when EVERY record in the batch is a redelivery. The buffer
+        // is invocation-scoped (not per-message), so discarding on *any*
+        // redelivery would drop the fresh records' counts too (silent
+        // under-count). For a mixed batch we flush: preserving fresh counts and
+        // at worst re-counting the one redelivered record is strictly better than
+        // losing fresh data for an approximate store. (Integration queue workers
+        // are batchSize:1 today, so a batch is all-or-nothing; this keeps it
+        // correct if batchSize is ever raised.)
+        const records = Array.isArray(eventSummary?.records)
+            ? eventSummary.records
+            : [];
+        const allRedelivered =
+            records.length > 0 &&
+            records.every((r) => Number(r.receiveCount) > 1);
+        if (allRedelivered) {
+            subscriber.discard();
+        } else {
+            await subscriber.flush();
+        }
+    } catch (_) {
+        // Usage rollup must never break the handler.
+    }
+}
+
+/**
+ * Flush telemetry before the Lambda container freezes. Because
+ * `callbackWaitsForEmptyEventLoop=false` (below) stops the event loop the moment
+ * the handler returns, OTel's timer-driven batch processors would never fire —
+ * so spans/metrics must be flushed synchronously here. Bounded by a timeout so a
+ * stalled exporter can never block the response, and fully guarded so a flush
+ * failure never breaks the handler.
+ */
+async function flushTelemetry(telemetry, timeoutMs) {
+    try {
+        if (
+            !telemetry ||
+            typeof telemetry.isEnabled !== 'function' ||
+            !telemetry.isEnabled()
+        ) {
+            return;
+        }
+        let timer;
+        const deadline = new Promise((resolve) => {
+            timer = setTimeout(resolve, timeoutMs);
+        });
+        try {
+            await Promise.race([
+                Promise.resolve(telemetry.forceFlush()),
+                deadline,
+            ]);
+        } finally {
+            clearTimeout(timer);
+        }
+    } catch (_) {
+        // Telemetry flush must never break the handler.
+    }
+}
+
+// Best-effort extraction of correlation identifiers from a Lambda event.
+// For SQS: pulls messageIds + parsed event/processId/integrationId from each
+// record body. For HTTP: pulls method+path. Never throws.
+const summarizeLambdaEvent = (event) => {
+    if (!event) return {};
+    if (Array.isArray(event.Records)) {
+        return {
+            source: 'sqs',
+            records: event.Records.map((r) => {
+                let parsed = {};
+                try {
+                    const body = JSON.parse(r.body);
+                    parsed = {
+                        event: body?.event,
+                        processId: body?.data?.processId,
+                        integrationId: body?.data?.integrationId,
+                    };
+                } catch {
+                    // ignore unparseable bodies
+                }
+                return {
+                    messageId: r.messageId,
+                    receiveCount: r.attributes?.ApproximateReceiveCount,
+                    ...parsed,
+                };
+            }),
+        };
+    }
+    if (event.httpMethod || event.requestContext?.http) {
+        return {
+            source: 'http',
+            method: event.httpMethod || event.requestContext?.http?.method,
+            path: event.path || event.rawPath,
+        };
+    }
+    return { source: 'other' };
+};
 
 const createHandler = (optionByName = {}) => {
     const {
@@ -11,6 +135,9 @@ const createHandler = (optionByName = {}) => {
         isUserFacingResponse = true,
         method,
         shouldUseDatabase = true,
+        telemetry,
+        flushTimeoutMs = DEFAULT_FLUSH_TIMEOUT_MS,
+        usageRollup,
     } = optionByName;
 
     if (!method) {
@@ -18,7 +145,25 @@ const createHandler = (optionByName = {}) => {
     }
 
     return async (event, context) => {
+        const eventSummary = summarizeLambdaEvent(event);
+        const activeTelemetry = telemetry || getTelemetry();
+        const activeUsageRollup =
+            usageRollup !== undefined
+                ? usageRollup
+                : getUsageRollupSubscriber();
+
+        // Wire adopter-declared telemetry subscribers once per cold start.
+        // Memoized in the singleton, so this is a cheap
+        // no-op after the first invocation.
+        getPluginTelemetrySubscribers();
+
         try {
+            console.info(`[createHandler] ${eventName}: handler entry`, {
+                eventName,
+                awsRequestId: context?.awsRequestId,
+                ...eventSummary,
+            });
+
             initDebugLog(eventName, event);
 
             const requestMethod = event.httpMethod;
@@ -30,13 +175,18 @@ const createHandler = (optionByName = {}) => {
             // If enabled (i.e. if SECRET_ARN is set in process.env) Fetch secrets from AWS Secrets Manager, and set them as environment variables.
             await secretsToEnv();
 
-            // Helps mongoose reuse the connection.  Lowers response times.
-            context.callbackWaitsForEmptyEventLoop = false;
+            // If enabled (i.e. if SSM_PARAMETER_PREFIX and FRIGG_SSM_OFFLOADED_KEYS are set) fetch offloaded params from SSM Parameter Store into process.env.
+            await parametersToEnv();
 
+            // Lazy-required so DB-free handlers never load the Prisma client.
             if (shouldUseDatabase) {
-                await connectToDatabase();
+                const { connectPrisma } = require('../database/prisma');
+                await connectPrisma();
             }
 
+            // Helps reuse the database connection.  Lowers response times.
+            context.callbackWaitsForEmptyEventLoop = false;
+
             // Run the Lambda
             return await method(event, context);
         } catch (error) {
@@ -44,6 +194,18 @@ const createHandler = (optionByName = {}) => {
 
             // Don't leak implementation details to end users.
             if (isUserFacingResponse) {
+                // Allow client-safe errors to pass through with their actual message
+                if (error.isClientSafe === true) {
+                    const statusCode = error.statusCode || 400;
+                    return {
+                        statusCode,
+                        body: JSON.stringify({
+                            error: error.message,
+                        }),
+                    };
+                }
+
+                // Hide other errors with generic message
                 return {
                     statusCode: 500,
                     body: JSON.stringify({
@@ -55,12 +217,34 @@ const createHandler = (optionByName = {}) => {
             // Handle server-to-server responses.
 
             // Halt errors are logged but suceed and won't be retried.
+            // Log explicitly — silent suppression here previously made stuck
+            // messages invisible to observability tooling. Include
+            // eventSummary so operators can correlate across concurrent
+            // invocations (processId / messageIds / HTTP path).
             if (error.isHaltError === true) {
+                console.warn(
+                    `[createHandler] ${eventName}: halt error suppressed (no retry)`,
+                    {
+                        eventName,
+                        errorName: error.name,
+                        errorMessage: error.message,
+                        statusCode: error.statusCode,
+                        ...eventSummary,
+                    }
+                );
                 return;
             }
 
             // Here we can just rethrow and let AWS build the response.
             throw error;
+        } finally {
+            // Flush telemetry + usage before the container freezes.
+            await flushTelemetry(activeTelemetry, flushTimeoutMs);
+            await flushUsageRollup(
+                activeUsageRollup,
+                eventSummary,
+                shouldUseDatabase
+            );
         }
     };
 };

package/core/parameters-to-env.js

@@ -0,0 +1,257 @@
+// Runtime loader that hydrates process.env from SSM Parameter Store. Used to
+// offload env vars that would otherwise blow past Lambda's 4KB env limit.
+// Real process.env always wins over SSM (a documented local-debugging escape
+// hatch); only keys this loader sets are ever refreshed.
+
+const DEFAULT_CACHE_TTL_SECONDS = 300;
+const BATCH_SIZE = 10;
+const THROTTLE_BACKOFF_MS = [100, 200, 400];
+const REFRESH_RETRY_MS = 30_000;
+
+let client = null;
+let cacheExpiresAt = null;
+let inflight = null;
+const ownedKeys = new Set();
+
+const sleep = (ms) => new Promise((resolve) => setTimeout(resolve, ms));
+
+const chunk = (items, size) => {
+    const batches = [];
+    for (let i = 0; i < items.length; i += size) {
+        batches.push(items.slice(i, i + size));
+    }
+    return batches;
+};
+
+const getCacheTtlSeconds = () => {
+    const raw = process.env.FRIGG_SSM_CACHE_TTL;
+    if (raw === undefined || raw === '') {
+        return DEFAULT_CACHE_TTL_SECONDS;
+    }
+    const parsed = Number(raw);
+    return Number.isNaN(parsed) ? DEFAULT_CACHE_TTL_SECONDS : parsed;
+};
+
+const parseKeys = (raw) =>
+    raw
+        .split(',')
+        .map((key) => key.trim())
+        .filter(Boolean);
+
+const sendWithRetry = async (ssmClient, command) => {
+    let attempt = 0;
+    for (;;) {
+        try {
+            return await ssmClient.send(command);
+        } catch (err) {
+            if (
+                err.name === 'ThrottlingException' &&
+                attempt < THROTTLE_BACKOFF_MS.length
+            ) {
+                await sleep(THROTTLE_BACKOFF_MS[attempt]);
+                attempt += 1;
+                continue;
+            }
+            throw err;
+        }
+    }
+};
+
+/**
+ * Fetch offloaded parameters and return a plain { key: value } map. Fetch-only:
+ * no process.env mutation, no caching, no ownership tracking. Shared by the
+ * runtime loader and the INIT-phase preload (ssm-preload.js). Throws (listing
+ * every missing name) if any requested key is absent from SSM.
+ */
+const fetchOffloadedParameters = async (prefix, keys, { region } = {}) => {
+    const { SSMClient, GetParametersCommand } = require('@aws-sdk/client-ssm');
+    const ssmClient = new SSMClient({
+        region: region || process.env.AWS_REGION,
+    });
+    const nameFor = (key) => `${prefix}/${key}`;
+
+    const found = new Map();
+    for (const batch of chunk(keys, BATCH_SIZE)) {
+        const { Parameters = [] } = await sendWithRetry(
+            ssmClient,
+            new GetParametersCommand({
+                Names: batch.map(nameFor),
+                WithDecryption: true,
+            })
+        );
+        for (const param of Parameters) {
+            found.set(param.Name, param);
+        }
+    }
+
+    const missing = keys.map(nameFor).filter((name) => !found.has(name));
+    if (missing.length > 0) {
+        throw new Error(
+            `missing SSM parameters under ${prefix}: ${missing.join(', ')}`
+        );
+    }
+
+    const values = {};
+    for (const key of keys) {
+        values[key] = found.get(nameFor(key)).Value;
+    }
+    return values;
+};
+
+const loadParameters = async (prefix, keys) => {
+    const { SSMClient, GetParametersCommand } = require('@aws-sdk/client-ssm');
+    if (!client) {
+        client = new SSMClient({ region: process.env.AWS_REGION });
+    }
+
+    // A key already living in process.env wins over SSM, except keys we set
+    // ourselves — those we refresh so later invocations pick up rotations.
+    const keysToFetch = keys.filter(
+        (key) => ownedKeys.has(key) || process.env[key] === undefined
+    );
+    const nameFor = (key) => `${prefix}/${key}`;
+
+    // A refresh means every key already has a served value; a warm container
+    // must keep serving stale values through an SSM blip instead of erroring.
+    const isRefresh =
+        keysToFetch.length > 0 &&
+        keysToFetch.every((key) => ownedKeys.has(key));
+
+    const found = new Map();
+    try {
+        for (const batch of chunk(keysToFetch, BATCH_SIZE)) {
+            const { Parameters = [] } = await sendWithRetry(
+                client,
+                new GetParametersCommand({
+                    Names: batch.map(nameFor),
+                    WithDecryption: true,
+                })
+            );
+            for (const param of Parameters) {
+                found.set(param.Name, param);
+            }
+        }
+
+        const missing = keysToFetch
+            .map(nameFor)
+            .filter((name) => !found.has(name));
+        if (missing.length > 0) {
+            throw new Error(
+                `parametersToEnv: missing SSM parameters under ${prefix}: ${missing.join(
+                    ', '
+                )}`
+            );
+        }
+    } catch (err) {
+        if (!isRefresh) {
+            throw err;
+        }
+        console.warn(
+            `parametersToEnv: refresh failed, keeping cached values: ${err.message}`
+        );
+        cacheExpiresAt = Date.now() + REFRESH_RETRY_MS;
+        return;
+    }
+
+    const loaded = keysToFetch.map((key) => {
+        const param = found.get(nameFor(key));
+        process.env[key] = param.Value;
+        ownedKeys.add(key);
+        return `${param.Name} (v${param.Version})`;
+    });
+
+    const ttlSeconds = getCacheTtlSeconds();
+    cacheExpiresAt =
+        ttlSeconds === 0 ? Infinity : Date.now() + ttlSeconds * 1000;
+
+    if (loaded.length > 0) {
+        console.log('parametersToEnv: loaded', loaded);
+    }
+};
+
+/**
+ * Hydrate process.env from SSM Parameter Store.
+ *
+ * No-op unless both SSM_PARAMETER_PREFIX and FRIGG_SSM_OFFLOADED_KEYS are set.
+ * Results are cached for FRIGG_SSM_CACHE_TTL seconds (default 300; 0 = forever).
+ * A failed fetch is never cached, so the next invocation retries.
+ */
+const parametersToEnv = async () => {
+    const prefix = process.env.SSM_PARAMETER_PREFIX;
+    const rawKeys = process.env.FRIGG_SSM_OFFLOADED_KEYS;
+    if (!prefix || !rawKeys) {
+        return;
+    }
+    const keys = parseKeys(rawKeys);
+    if (keys.length === 0) {
+        return;
+    }
+
+    if (cacheExpiresAt !== null && Date.now() < cacheExpiresAt) {
+        return;
+    }
+
+    if (!inflight) {
+        inflight = loadParameters(prefix, keys).finally(() => {
+            inflight = null;
+        });
+    }
+    return inflight;
+};
+
+/**
+ * Adopt keys the INIT preload (ssm-preload.mjs) already populated into
+ * process.env, so the handler-time loader treats them as its own: it seeds
+ * the cache TTL and marks the keys owned so the TTL-refresh path re-fetches
+ * them. The preload runs in-process (NODE_OPTIONS=--import), sharing this
+ * module singleton. Pass ONLY the keys the preload actually set (not keys
+ * already present as real env vars) so the real-env-wins rule is preserved.
+ */
+const adoptPreloadedKeys = (keys) => {
+    for (const key of keys) {
+        ownedKeys.add(key);
+    }
+    const ttlSeconds = getCacheTtlSeconds();
+    cacheExpiresAt =
+        ttlSeconds === 0 ? Infinity : Date.now() + ttlSeconds * 1000;
+};
+
+/**
+ * INIT-phase preload used by ssm-preload.mjs: fetch offloaded parameters and
+ * set them into process.env, then adopt the keys it set so the handler-time
+ * loader refreshes them on TTL. Real env wins — a key already present is never
+ * fetched or overwritten, so a console override keeps working even if that
+ * key's parameter is absent from SSM (fetching it would fail INIT). Returns the
+ * keys actually set (empty when every key was already in real env). Throws,
+ * listing the names, only for a genuinely-needed parameter that is missing.
+ */
+const preloadOffloadedParameters = async (prefix, keys, options = {}) => {
+    const keysToFetch = keys.filter((key) => process.env[key] === undefined);
+    if (keysToFetch.length === 0) {
+        return [];
+    }
+
+    const values = await fetchOffloadedParameters(prefix, keysToFetch, options);
+    const setKeys = [];
+    for (const [key, value] of Object.entries(values)) {
+        process.env[key] = value;
+        setKeys.push(key);
+    }
+    adoptPreloadedKeys(setKeys);
+    return setKeys;
+};
+
+const _resetCache = () => {
+    client = null;
+    cacheExpiresAt = null;
+    inflight = null;
+    ownedKeys.clear();
+};
+
+module.exports = {
+    parametersToEnv,
+    fetchOffloadedParameters,
+    preloadOffloadedParameters,
+    adoptPreloadedKeys,
+    _resetCache,
+};

package/core/ssm-preload.mjs

@@ -0,0 +1,34 @@
+// INIT-phase SSM loader (ADR-027).
+//
+// Loaded via NODE_OPTIONS=--import BEFORE the Lambda handler and any api-module
+// is required, so SSM-offloaded values are present in process.env at
+// module-load time — when api-modules capture their OAuth client credentials
+// in a top-level `const Definition = { env: { client_secret: process.env.X } }`.
+// The runtime loader (parameters-to-env.js) runs inside the handler, which is
+// too late for those module-load reads; this preload closes that gap.
+//
+// Top-level await here is awaited by Node before the entry module loads, so the
+// fetch completes first. A fetch failure rejects the preload, failing Lambda
+// INIT loudly (fail-fast) rather than starting with undefined credentials.
+
+import { createRequire } from 'module';
+
+const require = createRequire(import.meta.url);
+
+const prefix = process.env.SSM_PARAMETER_PREFIX;
+const rawKeys = process.env.FRIGG_SSM_OFFLOADED_KEYS;
+
+if (prefix && rawKeys) {
+    const keys = rawKeys
+        .split(',')
+        .map((key) => key.trim())
+        .filter(Boolean);
+
+    if (keys.length > 0) {
+        // The fetch/real-env-wins/adopt logic lives in the CJS module (tested
+        // there); this preload is a thin INIT-phase shim over it.
+        const { preloadOffloadedParameters } = require('./parameters-to-env');
+        const setKeys = await preloadOffloadedParameters(prefix, keys);
+        console.log(`frigg-ssm-preload: loaded ${setKeys.length} parameter(s) at INIT under ${prefix}`);
+    }
+}

package/credential/repositories/credential-active-type.js

@@ -0,0 +1,32 @@
+const UNKNOWN_TYPE = 'unknown';
+
+/**
+ * Type is derived from the related Entity.moduleName since Credential carries
+ * no type column. Counted once per distinct linked moduleName; none → `unknown`.
+ * Reads only the non-encrypted projection — never `data`/secrets.
+ */
+function tallyActiveCredentialsByType(credentials = []) {
+    const counts = new Map();
+
+    for (const credential of credentials) {
+        const modules = new Set(
+            (credential?.entities || [])
+                .map((entity) => entity?.moduleName)
+                .filter(
+                    (moduleName) => moduleName != null && moduleName !== ''
+                )
+        );
+        const types = modules.size > 0 ? [...modules] : [UNKNOWN_TYPE];
+
+        for (const type of types) {
+            counts.set(type, (counts.get(type) || 0) + 1);
+        }
+    }
+
+    return [...counts].map(([integrationType, count]) => ({
+        integrationType,
+        count,
+    }));
+}
+
+module.exports = { tallyActiveCredentialsByType, UNKNOWN_TYPE };