@intelligems/sst 2.49.6-ig.4 → 2.49.6-ig.6

package/runtime/server.js

@@ -2,9 +2,13 @@ import express from "express";
 import { useBus } from "../bus.js";
 import { Logger } from "../logger.js";
 import { useRuntimeWorkers } from "./workers.js";
+import { useFunctions } from "../constructs/Function.js";
 import https from "https";
 import getPort from "get-port";
 import { lazy } from "../util/lazy.js";
+import { getRequestPath } from "./request-utils.js";
+import { logServer } from "./debug-bridge-logging.js";
+import { logEventTrace } from "./event-trace-logging.js";
 export const useRuntimeServerConfig = lazy(async () => {
     const port = await getPort({
         port: 12557,
@@ -33,9 +37,12 @@ export const useRuntimeServer = lazy(async () => {
     }
     // Route an invocation to a specific workerID (used by workers.ts for pooled workers)
     function routeInvocation(targetWorkerID, invocation) {
+        const requestPath = getRequestPath(invocation.event);
+        const requestID = invocation.requestID;
         const waiting = workersWaiting.get(targetWorkerID);
         if (waiting) {
             workersWaiting.delete(targetWorkerID);
+            logServer(`path=${requestPath} reqId=${requestID.slice(0, 8)} Worker ${targetWorkerID.slice(0, 8)} was waiting, delivering immediately`);
             waiting(invocation);
             return;
         }
@@ -45,6 +52,7 @@ export const useRuntimeServer = lazy(async () => {
             invocationsQueued.set(targetWorkerID, arr);
         }
         arr.push(invocation);
+        logServer(`path=${requestPath} reqId=${requestID.slice(0, 8)} Worker ${targetWorkerID.slice(0, 8)} not waiting, QUEUED (queueSize=${arr.length})`);
     }
     workers.subscribe("worker.exited", async (evt) => {
         const waiting = workersWaiting.get(evt.properties.workerID);
@@ -69,8 +77,16 @@ export const useRuntimeServer = lazy(async () => {
         const awsWorkerID = workers.isPooled(pooledWorkerID)
             ? workers.getAwsWorkerID(pooledWorkerID) || pooledWorkerID
             : pooledWorkerID;
+        const requestID = workers.getCurrentRequestID(pooledWorkerID);
+        logEventTrace("WORKER_END", {
+            requestID: requestID || "unknown",
+            functionID: worker.functionID,
+            workerID: pooledWorkerID,
+            status: "error",
+            errorType: req.body?.errorType || "init_error",
+        });
         bus.publish("function.error", {
-            requestID: workers.getCurrentRequestID(pooledWorkerID),
+            requestID,
             workerID: awsWorkerID,
             functionID: worker.functionID,
             ...req.body,
@@ -80,11 +96,20 @@ export const useRuntimeServer = lazy(async () => {
         res.json("ok");
     });
     app.get(`/:workerID/${cfg.API_VERSION}/runtime/invocation/next`, async (req, res) => {
-        Logger.debug("Worker", req.params.workerID, "is waiting for next invocation");
-        const payload = await next(req.params.workerID);
-        Logger.debug("Worker", req.params.workerID, "sending next payload");
+        const workerID = req.params.workerID;
+        logServer(`Worker ${workerID.slice(0, 8)} requesting /next`);
+        Logger.debug("Worker", workerID, "is waiting for next invocation");
+        const waitStart = Date.now();
+        const payload = await next(workerID);
+        const requestPath = getRequestPath(payload.event);
+        const requestID = payload.context.awsRequestId;
+        logServer(`path=${requestPath} reqId=${requestID.slice(0, 8)} Worker ${workerID.slice(0, 8)} received payload after waiting ${Date.now() - waitStart}ms`);
+        Logger.debug("Worker", workerID, "sending next payload");
+        // Get function properties for per-invocation context
+        // This is essential for mono-build shared workers that serve multiple functions
+        const funcProps = useFunctions().fromID(payload.functionID);
         res.set({
-            "Lambda-Runtime-Aws-Request-Id": payload.context.awsRequestId,
+            "Lambda-Runtime-Aws-Request-Id": requestID,
             "Lambda-Runtime-Deadline-Ms": Date.now() + payload.deadline,
             "Lambda-Runtime-Invoked-Function-Arn": payload.context.invokedFunctionArn,
             "Lambda-Runtime-Client-Context": JSON.stringify(payload.context.clientContext || null),
@@ -93,6 +118,10 @@ export const useRuntimeServer = lazy(async () => {
             "Lambda-Runtime-Log-Stream-Name": payload.context.logStreamName,
             // Pass function ID for mono-build shared pool: allows per-invocation dispatch
             "Lambda-Runtime-Sst-Function-Id": payload.functionID,
+            // Per-invocation function context for mono-build shared workers
+            "Lambda-Runtime-Function-Name": funcProps?.functionName || "",
+            "Lambda-Runtime-Function-Version": "$LATEST",
+            "Lambda-Runtime-Function-Memory-Size": String(funcProps?.memorySize || 1024),
         });
         // Wrap event with env for per-invocation environment variable application
         // This prevents env leakage when workers are reused across different functions
@@ -109,9 +138,12 @@ export const useRuntimeServer = lazy(async () => {
         limit: "10mb",
     }), (req, res) => {
         const pooledWorkerID = req.params.workerID;
+        const requestID = req.params.awsRequestId;
+        logServer(`reqId=${requestID.slice(0, 8)} Worker ${pooledWorkerID.slice(0, 8)} posting /response`);
         Logger.debug("Worker", pooledWorkerID, "got response", req.body);
         const worker = workers.fromID(pooledWorkerID);
         if (!worker) {
+            logServer(`reqId=${requestID.slice(0, 8)} ERROR: Worker ${pooledWorkerID.slice(0, 8)} not found`);
             res.status(404).send();
             return;
         }
@@ -119,10 +151,17 @@ export const useRuntimeServer = lazy(async () => {
         const awsWorkerID = workers.isPooled(pooledWorkerID)
             ? workers.getAwsWorkerID(pooledWorkerID) || pooledWorkerID
             : pooledWorkerID;
+        logServer(`reqId=${requestID.slice(0, 8)} Publishing function.success awsWorkerID=${awsWorkerID.slice(0, 8)}`);
+        logEventTrace("WORKER_END", {
+            requestID,
+            functionID: worker.functionID,
+            workerID: pooledWorkerID,
+            status: "success",
+        });
         bus.publish("function.success", {
             workerID: awsWorkerID,
             functionID: worker.functionID,
-            requestID: req.params.awsRequestId,
+            requestID: requestID,
             body: req.body,
         });
         // Return pooled worker to pool
@@ -177,12 +216,20 @@ export const useRuntimeServer = lazy(async () => {
         const awsWorkerID = workers.isPooled(pooledWorkerID)
             ? workers.getAwsWorkerID(pooledWorkerID) || pooledWorkerID
             : pooledWorkerID;
+        const requestID = req.params.awsRequestId;
+        logEventTrace("WORKER_END", {
+            requestID,
+            functionID: worker.functionID,
+            workerID: pooledWorkerID,
+            status: "error",
+            errorType: req.body.errorType,
+        });
         bus.publish("function.error", {
             workerID: awsWorkerID,
             functionID: worker.functionID,
             errorType: req.body.errorType,
             errorMessage: req.body.errorMessage,
-            requestID: req.params.awsRequestId,
+            requestID,
             trace: req.body.trace,
         });
         // Return pooled worker to pool

package/runtime/worker-pool-logging.js

@@ -1,5 +1,4 @@
-import fs from "fs";
-import path from "path";
+import { createDebugFileLogger } from "./debug-file-logger.js";
 // Configuration
 const SST_BUILD_CONCURRENCY = parseInt(process.env.SST_BUILD_CONCURRENCY || "4", 10);
 export const POOL_SIZE = parseInt(process.env.SST_WORKER_POOL_SIZE || "10", 10);
@@ -16,7 +15,21 @@ const metrics = {
     avgResponseTime: new Map(),
     responseCount: new Map(),
 };
-let logStream = null;
+// Create pool logger using the abstract logger
+let poolLogger = null;
+function getPoolLogger() {
+    if (!DEBUG_POOL)
+        return null;
+    if (!poolLogger) {
+        poolLogger = createDebugFileLogger({
+            filePath: DEBUG_POOL_FILE,
+            sessionName: "POOL",
+            sessionHeader: `POOL_SIZE=${POOL_SIZE} IDLE_TIMEOUT=${IDLE_TIMEOUT}ms BUILD_CONCURRENCY=${SST_BUILD_CONCURRENCY}`,
+            width: 100,
+        });
+    }
+    return poolLogger;
+}
 // Function name resolver - set by workers.ts
 let functionNameResolver = (id) => id.slice(0, 25);
 /**
@@ -26,23 +39,6 @@ let functionNameResolver = (id) => id.slice(0, 25);
 export function setFunctionNameResolver(resolver) {
     functionNameResolver = resolver;
 }
-function initLogFile() {
-    if (!DEBUG_POOL || logStream)
-        return;
-    try {
-        const logDir = path.dirname(DEBUG_POOL_FILE);
-        if (!fs.existsSync(logDir)) {
-            fs.mkdirSync(logDir, { recursive: true });
-        }
-        logStream = fs.createWriteStream(DEBUG_POOL_FILE, { flags: "w" });
-        logStream.write("\n" + "=".repeat(100) + "\n" +
-            "[SESSION START] " + new Date().toISOString() + " | POOL_SIZE=" + POOL_SIZE + " IDLE_TIMEOUT=" + IDLE_TIMEOUT + "ms BUILD_CONCURRENCY=" + SST_BUILD_CONCURRENCY + "\n" +
-            "=".repeat(100) + "\n");
-    }
-    catch {
-        // Fall back to no logging if file creation fails
-    }
-}
 // Extract readable function name from handler path
 function getFunctionName(functionID) {
     return functionNameResolver(functionID);
@@ -57,30 +53,29 @@ function getBottleneckFlags(functionID) {
     const coldStarts = metrics.coldStarts.get(functionID) || 0;
     // SATURATED: All pool slots in use
     if (concurrent >= POOL_SIZE) {
-        flags.push("🚨 SATURATED");
+        flags.push("SATURATED");
     }
     // HIGH_LOAD: >70% pool utilization
     else if (concurrent >= POOL_SIZE * 0.7) {
-        flags.push("⚠️ HIGH_LOAD");
+        flags.push("HIGH_LOAD");
     }
     // COLD_START: Low hit rate indicates frequent cold starts
     if (total >= 5 && hitRate < 30) {
-        flags.push("❄️ LOW_REUSE");
+        flags.push("LOW_REUSE");
     }
     // BOTTLENECK: High cold start ratio
     if (total >= 3 && coldStarts / total > 0.5) {
-        flags.push("🔥 COLD_HEAVY");
+        flags.push("COLD_HEAVY");
     }
-    return flags.length > 0 ? " " + flags.join(" ") : "";
+    return flags.length > 0 ? " [" + flags.join(" ") + "]" : "";
 }
 /**
  * Pool debug logging helper - writes to file with bottleneck detection
  */
 export function logPool(action, details = {}) {
-    if (!DEBUG_POOL)
+    const logger = getPoolLogger();
+    if (!logger)
         return;
-    initLogFile();
-    const timestamp = new Date().toISOString().slice(11, 23); // HH:MM:SS.mmm
     const funcName = details.functionID
         ? getFunctionName(details.functionID)
         : "";
@@ -95,20 +90,29 @@ export function logPool(action, details = {}) {
         const total = metrics.totalRequests.get(details.functionID) || 0;
         const hits = metrics.poolHits.get(details.functionID) || 0;
         const hitRate = total > 0 ? ((hits / total) * 100).toFixed(0) : "0";
-        metricsStr = " | concurrent=" + concurrent + " hitRate=" + hitRate + "%";
-    }
-    // Format details, replacing functionID with funcName
-    const filteredDetails = { ...details };
-    delete filteredDetails.functionID;
-    const detailStr = Object.entries(filteredDetails)
-        .map(([k, v]) => k + "=" + (typeof v === "object" ? JSON.stringify(v) : v))
-        .join(" ");
-    const actionPadded = action.padEnd(14);
-    const funcPadded = funcName.padEnd(25);
-    const line = "[" + timestamp + "] " + actionPadded + " " + funcPadded + " " + detailStr + metricsStr + bottleneckFlags + "\n";
-    if (logStream) {
-        logStream.write(line);
+        metricsStr = `concurrent=${concurrent} hitRate=${hitRate}%`;
     }
+    // Build the log details
+    const logDetails = {};
+    // Add function name first if present
+    if (funcName) {
+        logDetails.func = funcName;
+    }
+    // Add all other details except functionID
+    for (const [k, v] of Object.entries(details)) {
+        if (k !== "functionID") {
+            logDetails[k] = v;
+        }
+    }
+    // Add metrics if present
+    if (metricsStr) {
+        logDetails.metrics = metricsStr;
+    }
+    // Add bottleneck flags
+    if (bottleneckFlags) {
+        logDetails.status = bottleneckFlags.trim();
+    }
+    logger.log(action, logDetails);
 }
 /**
  * Track request lifecycle for metrics - called at request start
@@ -142,49 +146,40 @@ export function trackRequestEnd(functionID) {
  * Called from workers.ts on process exit
  */
 export function writeSessionEndSummary() {
-    if (!DEBUG_POOL || !logStream)
+    const logger = getPoolLogger();
+    if (!logger)
         return;
-    let summary = "\n" + "─".repeat(100) + "\n[SESSION END] " + new Date().toISOString() + "\n";
+    let summary = "";
     for (const [funcID, total] of metrics.totalRequests) {
         const hits = metrics.poolHits.get(funcID) || 0;
         const cold = metrics.coldStarts.get(funcID) || 0;
         const peak = metrics.peakConcurrent.get(funcID) || 0;
         const hitRate = total > 0 ? ((hits / total) * 100).toFixed(1) : "0";
         const funcName = getFunctionName(funcID);
-        summary += "  " + funcName.padEnd(25) + " total=" + total + " poolHits=" + hits + " coldStarts=" + cold + " peakConcurrent=" + peak + " hitRate=" + hitRate + "%\n";
+        summary += `  ${funcName.padEnd(25)} total=${total} poolHits=${hits} coldStarts=${cold} peakConcurrent=${peak} hitRate=${hitRate}%\n`;
     }
-    summary += "─".repeat(100) + "\n";
-    logStream.write(summary);
-    logStream.end();
+    logger.close(summary);
 }
-// Invocation trace logging - always enabled, writes to separate file
-let traceStream = null;
-function initTraceFile() {
-    if (traceStream)
-        return;
-    try {
-        const logDir = path.dirname(INVOKE_TRACE_FILE);
-        if (!fs.existsSync(logDir)) {
-            fs.mkdirSync(logDir, { recursive: true });
-        }
-        traceStream = fs.createWriteStream(INVOKE_TRACE_FILE, { flags: "w" });
-        traceStream.write("\n" + "=".repeat(80) + "\n" +
-            "[TRACE START] " + new Date().toISOString() + "\n" +
-            "=".repeat(80) + "\n");
-    }
-    catch {
-        // Fall back to no logging if file creation fails
+// Invocation trace logging - uses the abstract logger
+let traceLogger = null;
+function getTraceLogger() {
+    if (!traceLogger) {
+        traceLogger = createDebugFileLogger({
+            filePath: INVOKE_TRACE_FILE,
+            sessionName: "TRACE",
+            width: 80,
+        });
     }
+    return traceLogger;
 }
 /**
  * Log invocation trace events to .sst/invoke-trace.log
  * Always enabled for debugging invocation flow
  */
 export function logInvokeTrace(stage, requestID, details) {
-    initTraceFile();
-    if (!traceStream)
-        return;
-    const timestamp = new Date().toISOString().slice(11, 23); // HH:MM:SS.mmm
-    const line = `[${timestamp}] ${stage.padEnd(20)} req=${requestID.slice(0, 8)} ${details || ""}\n`;
-    traceStream.write(line);
+    const logger = getTraceLogger();
+    logger.log(stage, {
+        req: requestID.slice(0, 8),
+        ...(details ? { info: details } : {}),
+    });
 }

package/runtime/workers.d.ts

@@ -23,6 +23,20 @@ declare module "../bus.js" {
             functionID: string;
             pooledWorkerID: string;
         };
+        "warmup.start": {
+            count: number;
+        };
+        "warmup.progress": {
+            completed: number;
+            total: number;
+            success: number;
+            failed: number;
+        };
+        "warmup.complete": {
+            success: number;
+            failed: number;
+            elapsedMs: number;
+        };
     }
 }
 export declare const useRuntimeWorkers: () => Promise<{
@@ -40,4 +54,16 @@ export declare const useRuntimeWorkers: () => Promise<{
         type: keyof import("../bus.js").Events;
         cb: (payload: any) => void;
     };
+    /**
+     * Trigger warmup by invoking Lambda functions with warmup payloads.
+     * This sends real requests through the IoT bridge, which naturally creates workers.
+     * @param count Number of workers to warm up (default: 15)
+     */
+    triggerWarmup(count?: number): Promise<{
+        warmed: number;
+        elapsed?: undefined;
+    } | {
+        warmed: number;
+        elapsed: number;
+    }>;
 }>;