@intelligems/sst 2.49.3 → 2.49.6-ig.10

package/runtime/server.js

@@ -2,9 +2,13 @@ import express from "express";
 import { useBus } from "../bus.js";
 import { Logger } from "../logger.js";
 import { useRuntimeWorkers } from "./workers.js";
+import { useFunctions } from "../constructs/Function.js";
 import https from "https";
 import getPort from "get-port";
 import { lazy } from "../util/lazy.js";
+import { getRequestPath } from "./request-utils.js";
+import { logServer } from "./debug-bridge-logging.js";
+import { logEventTrace } from "./event-trace-logging.js";
 export const useRuntimeServerConfig = lazy(async () => {
     const port = await getPort({
         port: 12557,
@@ -31,54 +35,100 @@ export const useRuntimeServer = lazy(async () => {
             workersWaiting.set(workerID, resolve);
         });
     }
-    workers.subscribe("worker.exited", async (evt) => {
-        const waiting = workersWaiting.get(evt.properties.workerID);
-        if (!waiting)
-            return;
-        workersWaiting.delete(evt.properties.workerID);
-    });
-    bus.subscribe("function.invoked", async (evt) => {
-        const worker = workersWaiting.get(evt.properties.workerID);
-        if (worker) {
-            workersWaiting.delete(evt.properties.workerID);
-            worker(evt.properties);
+    // Route an invocation to a specific workerID (used by workers.ts for pooled workers)
+    function routeInvocation(targetWorkerID, invocation) {
+        const requestPath = getRequestPath(invocation.event);
+        const requestID = invocation.requestID;
+        const waiting = workersWaiting.get(targetWorkerID);
+        if (waiting) {
+            workersWaiting.delete(targetWorkerID);
+            logServer(`path=${requestPath} reqId=${requestID.slice(0, 8)} Worker ${targetWorkerID.slice(0, 8)} was waiting, delivering immediately`);
+            waiting(invocation);
             return;
         }
-        let arr = invocationsQueued.get(evt.properties.workerID);
+        let arr = invocationsQueued.get(targetWorkerID);
         if (!arr) {
             arr = [];
-            invocationsQueued.set(evt.properties.workerID, arr);
+            invocationsQueued.set(targetWorkerID, arr);
         }
-        arr.push(evt.properties);
+        arr.push(invocation);
+        logServer(`path=${requestPath} reqId=${requestID.slice(0, 8)} Worker ${targetWorkerID.slice(0, 8)} not waiting, QUEUED (queueSize=${arr.length})`);
+    }
+    workers.subscribe("worker.exited", async (evt) => {
+        const waiting = workersWaiting.get(evt.properties.workerID);
+        if (!waiting)
+            return;
+        workersWaiting.delete(evt.properties.workerID);
     });
+    // Note: function.invoked routing is handled by workers.ts via routeInvocation()
+    // This ensures correct routing for both pooled and non-pooled workers
     app.post(`/:workerID/${cfg.API_VERSION}/runtime/init/error`, express.json({
         strict: false,
         type: ["application/json", "application/*+json"],
         limit: "10mb",
     }), async (req, res) => {
-        const worker = workers.fromID(req.params.workerID);
+        const pooledWorkerID = req.params.workerID;
+        const worker = workers.fromID(pooledWorkerID);
+        if (!worker) {
+            res.status(404).send();
+            return;
+        }
+        // Get AWS workerID for IoT routing (if pooled)
+        const awsWorkerID = workers.isPooled(pooledWorkerID)
+            ? workers.getAwsWorkerID(pooledWorkerID) || pooledWorkerID
+            : pooledWorkerID;
+        const requestID = workers.getCurrentRequestID(pooledWorkerID);
+        logEventTrace("WORKER_END", {
+            requestID: requestID || "unknown",
+            functionID: worker.functionID,
+            workerID: pooledWorkerID,
+            status: "error",
+            errorType: req.body?.errorType || "init_error",
+        });
         bus.publish("function.error", {
-            requestID: workers.getCurrentRequestID(worker.workerID),
-            workerID: worker.workerID,
+            requestID,
+            workerID: awsWorkerID,
             functionID: worker.functionID,
             ...req.body,
         });
+        // Return pooled worker to pool
+        workers.onResponse(pooledWorkerID);
         res.json("ok");
     });
     app.get(`/:workerID/${cfg.API_VERSION}/runtime/invocation/next`, async (req, res) => {
-        Logger.debug("Worker", req.params.workerID, "is waiting for next invocation");
-        const payload = await next(req.params.workerID);
-        Logger.debug("Worker", req.params.workerID, "sending next payload");
+        const workerID = req.params.workerID;
+        logServer(`Worker ${workerID.slice(0, 8)} requesting /next`);
+        Logger.debug("Worker", workerID, "is waiting for next invocation");
+        const waitStart = Date.now();
+        const payload = await next(workerID);
+        const requestPath = getRequestPath(payload.event);
+        const requestID = payload.context.awsRequestId;
+        logServer(`path=${requestPath} reqId=${requestID.slice(0, 8)} Worker ${workerID.slice(0, 8)} received payload after waiting ${Date.now() - waitStart}ms`);
+        Logger.debug("Worker", workerID, "sending next payload");
+        // Get function properties for per-invocation context
+        // This is essential for mono-build shared workers that serve multiple functions
+        const funcProps = useFunctions().fromID(payload.functionID);
         res.set({
-            "Lambda-Runtime-Aws-Request-Id": payload.context.awsRequestId,
+            "Lambda-Runtime-Aws-Request-Id": requestID,
             "Lambda-Runtime-Deadline-Ms": Date.now() + payload.deadline,
             "Lambda-Runtime-Invoked-Function-Arn": payload.context.invokedFunctionArn,
             "Lambda-Runtime-Client-Context": JSON.stringify(payload.context.clientContext || null),
             "Lambda-Runtime-Cognito-Identity": JSON.stringify(payload.context.identity || null),
             "Lambda-Runtime-Log-Group-Name": payload.context.logGroupName,
             "Lambda-Runtime-Log-Stream-Name": payload.context.logStreamName,
+            // Pass function ID for mono-build shared pool: allows per-invocation dispatch
+            "Lambda-Runtime-Sst-Function-Id": payload.functionID,
+            // Per-invocation function context for mono-build shared workers
+            "Lambda-Runtime-Function-Name": funcProps?.functionName || "",
+            "Lambda-Runtime-Function-Version": "$LATEST",
+            "Lambda-Runtime-Function-Memory-Size": String(funcProps?.memorySize || 1024),
+        });
+        // Wrap event with env for per-invocation environment variable application
+        // This prevents env leakage when workers are reused across different functions
+        res.json({
+            event: payload.event,
+            env: payload.env,
         });
-        res.json(payload.event);
     });
     app.post(`/:workerID/${cfg.API_VERSION}/runtime/invocation/:awsRequestId/response`, express.json({
         strict: false,
@@ -87,14 +137,35 @@ export const useRuntimeServer = lazy(async () => {
         },
         limit: "10mb",
     }), (req, res) => {
-        Logger.debug("Worker", req.params.workerID, "got response", req.body);
-        const worker = workers.fromID(req.params.workerID);
+        const pooledWorkerID = req.params.workerID;
+        const requestID = req.params.awsRequestId;
+        logServer(`reqId=${requestID.slice(0, 8)} Worker ${pooledWorkerID.slice(0, 8)} posting /response`);
+        Logger.debug("Worker", pooledWorkerID, "got response", req.body);
+        const worker = workers.fromID(pooledWorkerID);
+        if (!worker) {
+            logServer(`reqId=${requestID.slice(0, 8)} ERROR: Worker ${pooledWorkerID.slice(0, 8)} not found`);
+            res.status(404).send();
+            return;
+        }
+        // Get AWS workerID for IoT routing (if pooled)
+        const awsWorkerID = workers.isPooled(pooledWorkerID)
+            ? workers.getAwsWorkerID(pooledWorkerID) || pooledWorkerID
+            : pooledWorkerID;
+        logServer(`reqId=${requestID.slice(0, 8)} Publishing function.success awsWorkerID=${awsWorkerID.slice(0, 8)}`);
+        logEventTrace("WORKER_END", {
+            requestID,
+            functionID: worker.functionID,
+            workerID: pooledWorkerID,
+            status: "success",
+        });
         bus.publish("function.success", {
-            workerID: worker.workerID,
+            workerID: awsWorkerID,
             functionID: worker.functionID,
-            requestID: req.params.awsRequestId,
+            requestID: requestID,
             body: req.body,
         });
+        // Return pooled worker to pool
+        workers.onResponse(pooledWorkerID);
         res.status(202).send();
     });
     app.all(`/proxy*`, express.raw({
@@ -135,16 +206,38 @@ export const useRuntimeServer = lazy(async () => {
         type: ["application/json", "application/*+json"],
         limit: "10mb",
     }), (req, res) => {
-        const worker = workers.fromID(req.params.workerID);
+        const pooledWorkerID = req.params.workerID;
+        const worker = workers.fromID(pooledWorkerID);
+        if (!worker) {
+            res.status(404).send();
+            return;
+        }
+        // Get AWS workerID for IoT routing (if pooled)
+        const awsWorkerID = workers.isPooled(pooledWorkerID)
+            ? workers.getAwsWorkerID(pooledWorkerID) || pooledWorkerID
+            : pooledWorkerID;
+        const requestID = req.params.awsRequestId;
+        logEventTrace("WORKER_END", {
+            requestID,
+            functionID: worker.functionID,
+            workerID: pooledWorkerID,
+            status: "error",
+            errorType: req.body.errorType,
+        });
         bus.publish("function.error", {
-            workerID: worker.workerID,
+            workerID: awsWorkerID,
             functionID: worker.functionID,
             errorType: req.body.errorType,
             errorMessage: req.body.errorMessage,
-            requestID: req.params.awsRequestId,
+            requestID,
             trace: req.body.trace,
         });
+        // Return pooled worker to pool
+        workers.onResponse(pooledWorkerID);
         res.status(202).send();
     });
     app.listen(cfg.port);
+    return {
+        routeInvocation,
+    };
 });

package/runtime/worker-pool-logging.d.ts

@@ -0,0 +1,29 @@
+export declare const POOL_SIZE: number;
+export declare const IDLE_TIMEOUT: number;
+/**
+ * Set the function name resolver callback
+ * Called by workers.ts to provide access to useFunctions()
+ */
+export declare function setFunctionNameResolver(resolver: (functionID: string) => string): void;
+/**
+ * Pool debug logging helper - writes to file with bottleneck detection
+ */
+export declare function logPool(action: string, details?: Record<string, any>): void;
+/**
+ * Track request lifecycle for metrics - called at request start
+ */
+export declare function trackRequestStart(functionID: string, isPoolHit: boolean): void;
+/**
+ * Track request lifecycle for metrics - called at request end
+ */
+export declare function trackRequestEnd(functionID: string): void;
+/**
+ * Write session end summary and close log stream
+ * Called from workers.ts on process exit
+ */
+export declare function writeSessionEndSummary(): void;
+/**
+ * Log invocation trace events to .sst/invoke-trace.log
+ * Always enabled for debugging invocation flow
+ */
+export declare function logInvokeTrace(stage: string, requestID: string, details?: string): void;

package/runtime/worker-pool-logging.js

@@ -0,0 +1,185 @@
+import { createDebugFileLogger } from "./debug-file-logger.js";
+// Configuration
+const SST_BUILD_CONCURRENCY = parseInt(process.env.SST_BUILD_CONCURRENCY || "4", 10);
+export const POOL_SIZE = parseInt(process.env.SST_WORKER_POOL_SIZE || "10", 10);
+export const IDLE_TIMEOUT = parseInt(process.env.SST_WORKER_IDLE_TIMEOUT || "60000", 10);
+const DEBUG_POOL = process.env.SST_DEBUG_POOL === "true";
+const DEBUG_POOL_FILE = process.env.SST_DEBUG_POOL_FILE || ".sst/worker-pool.log";
+const INVOKE_TRACE_FILE = ".sst/invoke-trace.log";
+const metrics = {
+    coldStarts: new Map(),
+    concurrentRequests: new Map(),
+    peakConcurrent: new Map(),
+    totalRequests: new Map(),
+    poolHits: new Map(),
+    avgResponseTime: new Map(),
+    responseCount: new Map(),
+};
+// Create pool logger using the abstract logger
+let poolLogger = null;
+function getPoolLogger() {
+    if (!DEBUG_POOL)
+        return null;
+    if (!poolLogger) {
+        poolLogger = createDebugFileLogger({
+            filePath: DEBUG_POOL_FILE,
+            sessionName: "POOL",
+            sessionHeader: `POOL_SIZE=${POOL_SIZE} IDLE_TIMEOUT=${IDLE_TIMEOUT}ms BUILD_CONCURRENCY=${SST_BUILD_CONCURRENCY}`,
+            width: 100,
+        });
+    }
+    return poolLogger;
+}
+// Function name resolver - set by workers.ts
+let functionNameResolver = (id) => id.slice(0, 25);
+/**
+ * Set the function name resolver callback
+ * Called by workers.ts to provide access to useFunctions()
+ */
+export function setFunctionNameResolver(resolver) {
+    functionNameResolver = resolver;
+}
+// Extract readable function name from handler path
+function getFunctionName(functionID) {
+    return functionNameResolver(functionID);
+}
+// Calculate bottleneck indicators
+function getBottleneckFlags(functionID) {
+    const flags = [];
+    const concurrent = metrics.concurrentRequests.get(functionID) || 0;
+    const total = metrics.totalRequests.get(functionID) || 0;
+    const hits = metrics.poolHits.get(functionID) || 0;
+    const hitRate = total > 0 ? (hits / total) * 100 : 0;
+    const coldStarts = metrics.coldStarts.get(functionID) || 0;
+    // SATURATED: All pool slots in use
+    if (concurrent >= POOL_SIZE) {
+        flags.push("SATURATED");
+    }
+    // HIGH_LOAD: >70% pool utilization
+    else if (concurrent >= POOL_SIZE * 0.7) {
+        flags.push("HIGH_LOAD");
+    }
+    // COLD_START: Low hit rate indicates frequent cold starts
+    if (total >= 5 && hitRate < 30) {
+        flags.push("LOW_REUSE");
+    }
+    // BOTTLENECK: High cold start ratio
+    if (total >= 3 && coldStarts / total > 0.5) {
+        flags.push("COLD_HEAVY");
+    }
+    return flags.length > 0 ? " [" + flags.join(" ") + "]" : "";
+}
+/**
+ * Pool debug logging helper - writes to file with bottleneck detection
+ */
+export function logPool(action, details = {}) {
+    const logger = getPoolLogger();
+    if (!logger)
+        return;
+    const funcName = details.functionID
+        ? getFunctionName(details.functionID)
+        : "";
+    const bottleneckFlags = details.functionID &&
+        ["CREATE", "REUSE", "POOL_MISS", "RESPONSE"].includes(action)
+        ? getBottleneckFlags(details.functionID)
+        : "";
+    // Build metrics string for key actions
+    let metricsStr = "";
+    if (details.functionID && ["RESPONSE", "RETURN_TO_POOL"].includes(action)) {
+        const concurrent = metrics.concurrentRequests.get(details.functionID) || 0;
+        const total = metrics.totalRequests.get(details.functionID) || 0;
+        const hits = metrics.poolHits.get(details.functionID) || 0;
+        const hitRate = total > 0 ? ((hits / total) * 100).toFixed(0) : "0";
+        metricsStr = `concurrent=${concurrent} hitRate=${hitRate}%`;
+    }
+    // Build the log details
+    const logDetails = {};
+    // Add function name first if present
+    if (funcName) {
+        logDetails.func = funcName;
+    }
+    // Add all other details except functionID
+    for (const [k, v] of Object.entries(details)) {
+        if (k !== "functionID") {
+            logDetails[k] = v;
+        }
+    }
+    // Add metrics if present
+    if (metricsStr) {
+        logDetails.metrics = metricsStr;
+    }
+    // Add bottleneck flags
+    if (bottleneckFlags) {
+        logDetails.status = bottleneckFlags.trim();
+    }
+    logger.log(action, logDetails);
+}
+/**
+ * Track request lifecycle for metrics - called at request start
+ */
+export function trackRequestStart(functionID, isPoolHit) {
+    const current = (metrics.concurrentRequests.get(functionID) || 0) + 1;
+    metrics.concurrentRequests.set(functionID, current);
+    metrics.totalRequests.set(functionID, (metrics.totalRequests.get(functionID) || 0) + 1);
+    const peak = metrics.peakConcurrent.get(functionID) || 0;
+    if (current > peak) {
+        metrics.peakConcurrent.set(functionID, current);
+    }
+    if (isPoolHit) {
+        metrics.poolHits.set(functionID, (metrics.poolHits.get(functionID) || 0) + 1);
+    }
+    else {
+        metrics.coldStarts.set(functionID, (metrics.coldStarts.get(functionID) || 0) + 1);
+    }
+}
+/**
+ * Track request lifecycle for metrics - called at request end
+ */
+export function trackRequestEnd(functionID) {
+    const current = metrics.concurrentRequests.get(functionID) || 0;
+    if (current > 0) {
+        metrics.concurrentRequests.set(functionID, current - 1);
+    }
+}
+/**
+ * Write session end summary and close log stream
+ * Called from workers.ts on process exit
+ */
+export function writeSessionEndSummary() {
+    const logger = getPoolLogger();
+    if (!logger)
+        return;
+    let summary = "";
+    for (const [funcID, total] of metrics.totalRequests) {
+        const hits = metrics.poolHits.get(funcID) || 0;
+        const cold = metrics.coldStarts.get(funcID) || 0;
+        const peak = metrics.peakConcurrent.get(funcID) || 0;
+        const hitRate = total > 0 ? ((hits / total) * 100).toFixed(1) : "0";
+        const funcName = getFunctionName(funcID);
+        summary += `  ${funcName.padEnd(25)} total=${total} poolHits=${hits} coldStarts=${cold} peakConcurrent=${peak} hitRate=${hitRate}%\n`;
+    }
+    logger.close(summary);
+}
+// Invocation trace logging - uses the abstract logger
+let traceLogger = null;
+function getTraceLogger() {
+    if (!traceLogger) {
+        traceLogger = createDebugFileLogger({
+            filePath: INVOKE_TRACE_FILE,
+            sessionName: "TRACE",
+            width: 80,
+        });
+    }
+    return traceLogger;
+}
+/**
+ * Log invocation trace events to .sst/invoke-trace.log
+ * Always enabled for debugging invocation flow
+ */
+export function logInvokeTrace(stage, requestID, details) {
+    const logger = getTraceLogger();
+    logger.log(stage, {
+        req: requestID.slice(0, 8),
+        ...(details ? { info: details } : {}),
+    });
+}

package/runtime/workers.d.ts

@@ -18,20 +18,52 @@ declare module "../bus.js" {
             requestID: string;
             message: string;
         };
+        "worker.reused": {
+            workerID: string;
+            functionID: string;
+            pooledWorkerID: string;
+        };
+        "warmup.start": {
+            count: number;
+        };
+        "warmup.progress": {
+            completed: number;
+            total: number;
+            success: number;
+            failed: number;
+        };
+        "warmup.complete": {
+            success: number;
+            failed: number;
+            elapsedMs: number;
+        };
     }
 }
-interface Worker {
-    workerID: string;
-    functionID: string;
-}
 export declare const useRuntimeWorkers: () => Promise<{
-    fromID(workerID: string): Worker;
+    fromID(workerID: string): {
+        workerID: string;
+        functionID: string;
+    };
     getCurrentRequestID(workerID: string): string | undefined;
     stdout(workerID: string, message: string): void;
     exited(workerID: string): void;
-    subscribe: <Type extends "worker.started" | "worker.stopped" | "worker.exited" | "worker.stdout">(type: Type, cb: (payload: import("../bus.js").EventPayload<Type>) => void) => {
+    onResponse(pooledWorkerID: string): void;
+    getAwsWorkerID(pooledWorkerID: string): string | undefined;
+    isPooled(workerID: string): boolean;
+    subscribe: <Type extends "worker.started" | "worker.stopped" | "worker.exited" | "worker.stdout" | "worker.reused">(type: Type, cb: (payload: import("../bus.js").EventPayload<Type>) => void) => {
         type: keyof import("../bus.js").Events;
         cb: (payload: any) => void;
     };
+    /**
+     * Trigger warmup by invoking Lambda functions with warmup payloads.
+     * This sends real requests through the IoT bridge, which naturally creates workers.
+     * @param count Number of workers to warm up (default: 15)
+     */
+    triggerWarmup(count?: number): Promise<{
+        warmed: number;
+        elapsed?: undefined;
+    } | {
+        warmed: number;
+        elapsed: number;
+    }>;
 }>;
-export {};