@intelligems/sst 2.49.6-ig.1 → 2.49.6-ig.10

package/runtime/iot.js

@@ -1,17 +1,44 @@
 import { useBus } from "../bus.js";
-import { useIOT } from "../iot.js";
+import { useIOT, useIOTControl } from "../iot.js";
 import { lazy } from "../util/lazy.js";
+import { logInvokeTrace } from "./worker-pool-logging.js";
+import { logIot } from "./debug-bridge-logging.js";
+import { logEventTrace } from "./event-trace-logging.js";
 export const useIOTBridge = lazy(async () => {
     const bus = useBus();
     const iot = await useIOT();
+    // The ack goes out on its own socket so it can never queue behind a response
+    // body. See `useIOTControl`.
+    const control = await useIOTControl();
     const topic = `${iot.prefix}/events`;
     bus.subscribe("function.success", async (evt) => {
-        iot.publish(topic + "/" + evt.properties.workerID, "function.success", evt.properties);
+        const { workerID, requestID } = evt.properties;
+        logIot(`reqId=${requestID?.slice(0, 8)} Publishing function.success to worker ${workerID.slice(0, 8)}`);
+        const startTime = Date.now();
+        await iot.publish(topic + "/" + workerID, "function.success", evt.properties);
+        logIot(`reqId=${requestID?.slice(0, 8)} function.success published in ${Date.now() - startTime}ms`);
     });
     bus.subscribe("function.error", async (evt) => {
-        iot.publish(topic + "/" + evt.properties.workerID, "function.error", evt.properties);
+        const { workerID, requestID } = evt.properties;
+        logIot(`reqId=${requestID?.slice(0, 8)} Publishing function.error to worker ${workerID.slice(0, 8)}`);
+        const startTime = Date.now();
+        await iot.publish(topic + "/" + workerID, "function.error", evt.properties);
+        logIot(`reqId=${requestID?.slice(0, 8)} function.error published in ${Date.now() - startTime}ms`);
     });
     bus.subscribe("function.ack", async (evt) => {
-        iot.publish(topic + "/" + evt.properties.workerID, "function.ack", evt.properties);
+        const { workerID, requestID, functionID } = evt.properties;
+        logIot(`reqId=${requestID?.slice(0, 8)} Publishing function.ack to worker ${workerID.slice(0, 8)}`);
+        logInvokeTrace("IOT_ACK_START", workerID, `worker=${workerID.slice(0, 8)}`);
+        const startTime = Date.now();
+        await control.publish(topic + "/" + workerID, "function.ack", evt.properties);
+        const elapsed = Date.now() - startTime;
+        logIot(`reqId=${requestID?.slice(0, 8)} function.ack published in ${elapsed}ms`);
+        logInvokeTrace("IOT_ACK_DONE", workerID, `worker=${workerID.slice(0, 8)}`);
+        logEventTrace("IOT_ACK", {
+            requestID,
+            functionID,
+            workerID,
+            elapsed,
+        });
     });
 });

package/runtime/mono-build-config.d.ts

@@ -0,0 +1,58 @@
+/**
+ * Global mono build configuration for SST dev mode.
+ *
+ * Mono build mode bundles all Lambda handlers into a single file (.mono-build/index.mjs)
+ * instead of building each handler individually. This significantly speeds up dev mode
+ * by sharing compilation work across all handlers.
+ *
+ * Detection latches on: the first check that finds the bundle enables mono
+ * build for the rest of the session. It deliberately does NOT latch off, so a
+ * caller arriving before the bundle has been written cannot disable it.
+ */
+export declare const useMonoBuildConfig: () => {
+    /**
+     * Whether mono build mode is enabled. Re-checked until the bundle is
+     * found, so a check made before it was written does not stick.
+     * When true, all Node.js handlers use the shared .mono-build bundle.
+     */
+    readonly enabled: boolean;
+    /**
+     * The mono bundle directory (.mono-build)
+     */
+    dir: string;
+    /**
+     * The mono bundle entry file (.mono-build/index.mjs)
+     */
+    entryFile: string;
+    /**
+     * The handler string to use for mono build mode
+     */
+    handler: string;
+    /**
+     * Check if a build output path represents a mono build.
+     * This is useful when you have a build result and need to determine its type.
+     */
+    isMonoBuildPath(buildOut: string): boolean;
+    /**
+     * Get the pool key for a function based on mono build status.
+     * For mono build: shared key (all functions share workers)
+     * For non-mono build: per-function key
+     */
+    getPoolKey(functionID: string, runtime: string, buildOut: string): {
+        key: string;
+        isShared: boolean;
+    };
+};
+/**
+ * Quick check for mono build mode without full config initialization.
+ * Use this for simple boolean checks where you don't need the full config.
+ */
+export declare function isMonoBuildEnabled(): boolean;
+/**
+ * Get the mono build directory path.
+ */
+export declare function getMonoBuildDir(): string;
+/**
+ * Check if a path represents a mono build output.
+ */
+export declare function isMonoBuildPath(buildOut: string): boolean;

package/runtime/mono-build-config.js

@@ -0,0 +1,104 @@
+import path from "path";
+import fsSync from "fs";
+import { useProject } from "../project.js";
+import { lazy } from "../util/lazy.js";
+import { Logger } from "../logger.js";
+/**
+ * Global mono build configuration for SST dev mode.
+ *
+ * Mono build mode bundles all Lambda handlers into a single file (.mono-build/index.mjs)
+ * instead of building each handler individually. This significantly speeds up dev mode
+ * by sharing compilation work across all handlers.
+ *
+ * Detection latches on: the first check that finds the bundle enables mono
+ * build for the rest of the session. It deliberately does NOT latch off, so a
+ * caller arriving before the bundle has been written cannot disable it.
+ */
+export const useMonoBuildConfig = lazy(() => {
+    const project = useProject();
+    const monoBundleDir = path.join(project.paths.root, ".mono-build");
+    const monoBundlePath = path.join(monoBundleDir, "index.mjs");
+    /**
+     * Latches on true, never on false.
+     *
+     * This was a single `existsSync` at first call, cached for the session by
+     * `lazy`. Dev start deletes `.mono-build/index.mjs` and then rebuilds it, so
+     * any caller landing in that window — typically an invocation that beat the
+     * bundle to disk — pinned this to false for the whole session. Every handler
+     * then took the per-function esbuild path, which does not carry the mono
+     * build's `external` list, and failed to resolve packages the mono bundle
+     * never opens. The bundle finishing changed nothing, because the flag had
+     * already been decided, so the session stayed broken until restarted.
+     *
+     * Re-checking until it is found costs one `existsSync` per call for the few
+     * seconds before the bundle lands, and nothing afterwards.
+     */
+    let enabled = false;
+    const isEnabled = () => {
+        if (!enabled && fsSync.existsSync(monoBundlePath)) {
+            enabled = true;
+            Logger.debug("Mono build mode enabled:", monoBundlePath);
+        }
+        return enabled;
+    };
+    isEnabled();
+    return {
+        /**
+         * Whether mono build mode is enabled. Re-checked until the bundle is
+         * found, so a check made before it was written does not stick.
+         * When true, all Node.js handlers use the shared .mono-build bundle.
+         */
+        get enabled() {
+            return isEnabled();
+        },
+        /**
+         * The mono bundle directory (.mono-build)
+         */
+        dir: monoBundleDir,
+        /**
+         * The mono bundle entry file (.mono-build/index.mjs)
+         */
+        entryFile: monoBundlePath,
+        /**
+         * The handler string to use for mono build mode
+         */
+        handler: "index.handler",
+        /**
+         * Check if a build output path represents a mono build.
+         * This is useful when you have a build result and need to determine its type.
+         */
+        isMonoBuildPath(buildOut) {
+            return isEnabled() && buildOut.includes(".mono-build");
+        },
+        /**
+         * Get the pool key for a function based on mono build status.
+         * For mono build: shared key (all functions share workers)
+         * For non-mono build: per-function key
+         */
+        getPoolKey(functionID, runtime, buildOut) {
+            if (isEnabled() && buildOut.includes(".mono-build")) {
+                return { key: `${runtime}:mono-build`, isShared: true };
+            }
+            return { key: `${runtime}:${functionID}`, isShared: false };
+        },
+    };
+});
+/**
+ * Quick check for mono build mode without full config initialization.
+ * Use this for simple boolean checks where you don't need the full config.
+ */
+export function isMonoBuildEnabled() {
+    return useMonoBuildConfig().enabled;
+}
+/**
+ * Get the mono build directory path.
+ */
+export function getMonoBuildDir() {
+    return useMonoBuildConfig().dir;
+}
+/**
+ * Check if a path represents a mono build output.
+ */
+export function isMonoBuildPath(buildOut) {
+    return useMonoBuildConfig().isMonoBuildPath(buildOut);
+}

package/runtime/request-utils.d.ts

@@ -0,0 +1,21 @@
+/**
+ * Extracts a request path from a Lambda event for logging purposes.
+ * Handles API Gateway v1, v2, and other event formats.
+ */
+export declare function getRequestPath(event: unknown): string;
+/**
+ * Extracts correlation ID from headers for request tracing.
+ * Checks common correlation header names.
+ *
+ * Frontend can send: X-Correlation-ID, X-Request-ID, or X-Trace-ID
+ */
+export declare function getCorrelationId(event: unknown): string | undefined;
+/**
+ * Extracts API Gateway Request ID from requestContext.
+ * This ID is visible in browser dev tools (x-amzn-requestid response header).
+ */
+export declare function getApiGatewayRequestId(event: unknown): string | undefined;
+/**
+ * Extracts HTTP method from event
+ */
+export declare function getHttpMethod(event: unknown): string | undefined;

package/runtime/request-utils.js

@@ -0,0 +1,102 @@
+/**
+ * Extracts a request path from a Lambda event for logging purposes.
+ * Handles API Gateway v1, v2, and other event formats.
+ */
+export function getRequestPath(event) {
+    if (!event || typeof event !== "object") {
+        return "[unknown]";
+    }
+    const evt = event;
+    // API Gateway v2 (HTTP API)
+    if (typeof evt.rawPath === "string") {
+        return evt.rawPath;
+    }
+    // API Gateway v1 (REST API)
+    if (typeof evt.path === "string") {
+        return evt.path;
+    }
+    // Warmup requests
+    if ("ding" in evt || "warmer" in evt || evt.__sst_warmup === true) {
+        return "[warmup]";
+    }
+    // SQS, SNS, or other event types - no path
+    return "[event]";
+}
+/**
+ * Extracts correlation ID from headers for request tracing.
+ * Checks common correlation header names.
+ *
+ * Frontend can send: X-Correlation-ID, X-Request-ID, or X-Trace-ID
+ */
+export function getCorrelationId(event) {
+    if (!event || typeof event !== "object") {
+        return undefined;
+    }
+    const evt = event;
+    // Get headers - handle both v1 and v2 API Gateway formats
+    let headers;
+    // API Gateway v2 uses lowercase headers
+    if (evt.headers && typeof evt.headers === "object") {
+        headers = evt.headers;
+    }
+    if (!headers)
+        return undefined;
+    // Check common correlation header names (case-insensitive)
+    const correlationHeaders = [
+        "x-correlation-id",
+        "x-request-id",
+        "x-trace-id",
+        "correlation-id",
+        "request-id",
+    ];
+    // Normalize header keys to lowercase for comparison
+    const normalizedHeaders = {};
+    for (const [key, value] of Object.entries(headers)) {
+        normalizedHeaders[key.toLowerCase()] = value;
+    }
+    for (const headerName of correlationHeaders) {
+        const value = normalizedHeaders[headerName];
+        if (value) {
+            return value;
+        }
+    }
+    return undefined;
+}
+/**
+ * Extracts API Gateway Request ID from requestContext.
+ * This ID is visible in browser dev tools (x-amzn-requestid response header).
+ */
+export function getApiGatewayRequestId(event) {
+    if (!event || typeof event !== "object") {
+        return undefined;
+    }
+    const evt = event;
+    // Check requestContext.requestId (both v1 and v2)
+    const requestContext = evt.requestContext;
+    if (requestContext?.requestId && typeof requestContext.requestId === "string") {
+        return requestContext.requestId;
+    }
+    return undefined;
+}
+/**
+ * Extracts HTTP method from event
+ */
+export function getHttpMethod(event) {
+    if (!event || typeof event !== "object") {
+        return undefined;
+    }
+    const evt = event;
+    // API Gateway v2
+    const requestContext = evt.requestContext;
+    if (requestContext?.http && typeof requestContext.http === "object") {
+        const http = requestContext.http;
+        if (typeof http.method === "string") {
+            return http.method;
+        }
+    }
+    // API Gateway v1
+    if (typeof evt.httpMethod === "string") {
+        return evt.httpMethod;
+    }
+    return undefined;
+}

package/runtime/runtime.d.ts

@@ -4,6 +4,7 @@ declare module "../bus.js" {
         "function.ack": {
             workerID: string;
             functionID: string;
+            requestID: string;
         };
         "function.invoked": {
             workerID: string;

package/runtime/server.d.ts

@@ -1,6 +1,9 @@
+import { Events } from "../bus.js";
 export declare const useRuntimeServerConfig: () => Promise<{
     API_VERSION: string;
     port: number;
     url: string;
 }>;
-export declare const useRuntimeServer: () => Promise<void>;
+export declare const useRuntimeServer: () => Promise<{
+    routeInvocation: (targetWorkerID: string, invocation: Events["function.invoked"]) => void;
+}>;

package/runtime/server.js

@@ -2,9 +2,13 @@ import express from "express";
 import { useBus } from "../bus.js";
 import { Logger } from "../logger.js";
 import { useRuntimeWorkers } from "./workers.js";
+import { useFunctions } from "../constructs/Function.js";
 import https from "https";
 import getPort from "get-port";
 import { lazy } from "../util/lazy.js";
+import { getRequestPath } from "./request-utils.js";
+import { logServer } from "./debug-bridge-logging.js";
+import { logEventTrace } from "./event-trace-logging.js";
 export const useRuntimeServerConfig = lazy(async () => {
     const port = await getPort({
         port: 12557,
@@ -31,54 +35,100 @@ export const useRuntimeServer = lazy(async () => {
             workersWaiting.set(workerID, resolve);
         });
     }
-    workers.subscribe("worker.exited", async (evt) => {
-        const waiting = workersWaiting.get(evt.properties.workerID);
-        if (!waiting)
-            return;
-        workersWaiting.delete(evt.properties.workerID);
-    });
-    bus.subscribe("function.invoked", async (evt) => {
-        const worker = workersWaiting.get(evt.properties.workerID);
-        if (worker) {
-            workersWaiting.delete(evt.properties.workerID);
-            worker(evt.properties);
+    // Route an invocation to a specific workerID (used by workers.ts for pooled workers)
+    function routeInvocation(targetWorkerID, invocation) {
+        const requestPath = getRequestPath(invocation.event);
+        const requestID = invocation.requestID;
+        const waiting = workersWaiting.get(targetWorkerID);
+        if (waiting) {
+            workersWaiting.delete(targetWorkerID);
+            logServer(`path=${requestPath} reqId=${requestID.slice(0, 8)} Worker ${targetWorkerID.slice(0, 8)} was waiting, delivering immediately`);
+            waiting(invocation);
             return;
         }
-        let arr = invocationsQueued.get(evt.properties.workerID);
+        let arr = invocationsQueued.get(targetWorkerID);
         if (!arr) {
             arr = [];
-            invocationsQueued.set(evt.properties.workerID, arr);
+            invocationsQueued.set(targetWorkerID, arr);
         }
-        arr.push(evt.properties);
+        arr.push(invocation);
+        logServer(`path=${requestPath} reqId=${requestID.slice(0, 8)} Worker ${targetWorkerID.slice(0, 8)} not waiting, QUEUED (queueSize=${arr.length})`);
+    }
+    workers.subscribe("worker.exited", async (evt) => {
+        const waiting = workersWaiting.get(evt.properties.workerID);
+        if (!waiting)
+            return;
+        workersWaiting.delete(evt.properties.workerID);
     });
+    // Note: function.invoked routing is handled by workers.ts via routeInvocation()
+    // This ensures correct routing for both pooled and non-pooled workers
     app.post(`/:workerID/${cfg.API_VERSION}/runtime/init/error`, express.json({
         strict: false,
         type: ["application/json", "application/*+json"],
         limit: "10mb",
     }), async (req, res) => {
-        const worker = workers.fromID(req.params.workerID);
+        const pooledWorkerID = req.params.workerID;
+        const worker = workers.fromID(pooledWorkerID);
+        if (!worker) {
+            res.status(404).send();
+            return;
+        }
+        // Get AWS workerID for IoT routing (if pooled)
+        const awsWorkerID = workers.isPooled(pooledWorkerID)
+            ? workers.getAwsWorkerID(pooledWorkerID) || pooledWorkerID
+            : pooledWorkerID;
+        const requestID = workers.getCurrentRequestID(pooledWorkerID);
+        logEventTrace("WORKER_END", {
+            requestID: requestID || "unknown",
+            functionID: worker.functionID,
+            workerID: pooledWorkerID,
+            status: "error",
+            errorType: req.body?.errorType || "init_error",
+        });
         bus.publish("function.error", {
-            requestID: workers.getCurrentRequestID(worker.workerID),
-            workerID: worker.workerID,
+            requestID,
+            workerID: awsWorkerID,
             functionID: worker.functionID,
             ...req.body,
         });
+        // Return pooled worker to pool
+        workers.onResponse(pooledWorkerID);
         res.json("ok");
     });
     app.get(`/:workerID/${cfg.API_VERSION}/runtime/invocation/next`, async (req, res) => {
-        Logger.debug("Worker", req.params.workerID, "is waiting for next invocation");
-        const payload = await next(req.params.workerID);
-        Logger.debug("Worker", req.params.workerID, "sending next payload");
+        const workerID = req.params.workerID;
+        logServer(`Worker ${workerID.slice(0, 8)} requesting /next`);
+        Logger.debug("Worker", workerID, "is waiting for next invocation");
+        const waitStart = Date.now();
+        const payload = await next(workerID);
+        const requestPath = getRequestPath(payload.event);
+        const requestID = payload.context.awsRequestId;
+        logServer(`path=${requestPath} reqId=${requestID.slice(0, 8)} Worker ${workerID.slice(0, 8)} received payload after waiting ${Date.now() - waitStart}ms`);
+        Logger.debug("Worker", workerID, "sending next payload");
+        // Get function properties for per-invocation context
+        // This is essential for mono-build shared workers that serve multiple functions
+        const funcProps = useFunctions().fromID(payload.functionID);
         res.set({
-            "Lambda-Runtime-Aws-Request-Id": payload.context.awsRequestId,
+            "Lambda-Runtime-Aws-Request-Id": requestID,
             "Lambda-Runtime-Deadline-Ms": Date.now() + payload.deadline,
             "Lambda-Runtime-Invoked-Function-Arn": payload.context.invokedFunctionArn,
             "Lambda-Runtime-Client-Context": JSON.stringify(payload.context.clientContext || null),
             "Lambda-Runtime-Cognito-Identity": JSON.stringify(payload.context.identity || null),
             "Lambda-Runtime-Log-Group-Name": payload.context.logGroupName,
             "Lambda-Runtime-Log-Stream-Name": payload.context.logStreamName,
+            // Pass function ID for mono-build shared pool: allows per-invocation dispatch
+            "Lambda-Runtime-Sst-Function-Id": payload.functionID,
+            // Per-invocation function context for mono-build shared workers
+            "Lambda-Runtime-Function-Name": funcProps?.functionName || "",
+            "Lambda-Runtime-Function-Version": "$LATEST",
+            "Lambda-Runtime-Function-Memory-Size": String(funcProps?.memorySize || 1024),
+        });
+        // Wrap event with env for per-invocation environment variable application
+        // This prevents env leakage when workers are reused across different functions
+        res.json({
+            event: payload.event,
+            env: payload.env,
         });
-        res.json(payload.event);
     });
     app.post(`/:workerID/${cfg.API_VERSION}/runtime/invocation/:awsRequestId/response`, express.json({
         strict: false,
@@ -87,14 +137,35 @@ export const useRuntimeServer = lazy(async () => {
         },
         limit: "10mb",
     }), (req, res) => {
-        Logger.debug("Worker", req.params.workerID, "got response", req.body);
-        const worker = workers.fromID(req.params.workerID);
+        const pooledWorkerID = req.params.workerID;
+        const requestID = req.params.awsRequestId;
+        logServer(`reqId=${requestID.slice(0, 8)} Worker ${pooledWorkerID.slice(0, 8)} posting /response`);
+        Logger.debug("Worker", pooledWorkerID, "got response", req.body);
+        const worker = workers.fromID(pooledWorkerID);
+        if (!worker) {
+            logServer(`reqId=${requestID.slice(0, 8)} ERROR: Worker ${pooledWorkerID.slice(0, 8)} not found`);
+            res.status(404).send();
+            return;
+        }
+        // Get AWS workerID for IoT routing (if pooled)
+        const awsWorkerID = workers.isPooled(pooledWorkerID)
+            ? workers.getAwsWorkerID(pooledWorkerID) || pooledWorkerID
+            : pooledWorkerID;
+        logServer(`reqId=${requestID.slice(0, 8)} Publishing function.success awsWorkerID=${awsWorkerID.slice(0, 8)}`);
+        logEventTrace("WORKER_END", {
+            requestID,
+            functionID: worker.functionID,
+            workerID: pooledWorkerID,
+            status: "success",
+        });
         bus.publish("function.success", {
-            workerID: worker.workerID,
+            workerID: awsWorkerID,
             functionID: worker.functionID,
-            requestID: req.params.awsRequestId,
+            requestID: requestID,
             body: req.body,
         });
+        // Return pooled worker to pool
+        workers.onResponse(pooledWorkerID);
         res.status(202).send();
     });
     app.all(`/proxy*`, express.raw({
@@ -135,16 +206,38 @@ export const useRuntimeServer = lazy(async () => {
         type: ["application/json", "application/*+json"],
         limit: "10mb",
     }), (req, res) => {
-        const worker = workers.fromID(req.params.workerID);
+        const pooledWorkerID = req.params.workerID;
+        const worker = workers.fromID(pooledWorkerID);
+        if (!worker) {
+            res.status(404).send();
+            return;
+        }
+        // Get AWS workerID for IoT routing (if pooled)
+        const awsWorkerID = workers.isPooled(pooledWorkerID)
+            ? workers.getAwsWorkerID(pooledWorkerID) || pooledWorkerID
+            : pooledWorkerID;
+        const requestID = req.params.awsRequestId;
+        logEventTrace("WORKER_END", {
+            requestID,
+            functionID: worker.functionID,
+            workerID: pooledWorkerID,
+            status: "error",
+            errorType: req.body.errorType,
+        });
         bus.publish("function.error", {
-            workerID: worker.workerID,
+            workerID: awsWorkerID,
             functionID: worker.functionID,
             errorType: req.body.errorType,
             errorMessage: req.body.errorMessage,
-            requestID: req.params.awsRequestId,
+            requestID,
             trace: req.body.trace,
         });
+        // Return pooled worker to pool
+        workers.onResponse(pooledWorkerID);
         res.status(202).send();
     });
     app.listen(cfg.port);
+    return {
+        routeInvocation,
+    };
 });

package/runtime/worker-pool-logging.d.ts

@@ -0,0 +1,29 @@
+export declare const POOL_SIZE: number;
+export declare const IDLE_TIMEOUT: number;
+/**
+ * Set the function name resolver callback
+ * Called by workers.ts to provide access to useFunctions()
+ */
+export declare function setFunctionNameResolver(resolver: (functionID: string) => string): void;
+/**
+ * Pool debug logging helper - writes to file with bottleneck detection
+ */
+export declare function logPool(action: string, details?: Record<string, any>): void;
+/**
+ * Track request lifecycle for metrics - called at request start
+ */
+export declare function trackRequestStart(functionID: string, isPoolHit: boolean): void;
+/**
+ * Track request lifecycle for metrics - called at request end
+ */
+export declare function trackRequestEnd(functionID: string): void;
+/**
+ * Write session end summary and close log stream
+ * Called from workers.ts on process exit
+ */
+export declare function writeSessionEndSummary(): void;
+/**
+ * Log invocation trace events to .sst/invoke-trace.log
+ * Always enabled for debugging invocation flow
+ */
+export declare function logInvokeTrace(stage: string, requestID: string, details?: string): void;