@intelligems/sst 2.49.6-ig.4 → 2.49.6-ig.6

package/runtime/workers.js

@@ -8,6 +8,10 @@ import { useFunctions } from "../constructs/Function.js";
 import { lazy } from "../util/lazy.js";
 import { Logger } from "../logger.js";
 import { POOL_SIZE, IDLE_TIMEOUT, logPool, logInvokeTrace, trackRequestStart, trackRequestEnd, setFunctionNameResolver, writeSessionEndSummary, } from "./worker-pool-logging.js";
+import { useMonoBuildConfig, isMonoBuildPath } from "./mono-build-config.js";
+import { getRequestPath, getCorrelationId, getApiGatewayRequestId } from "./request-utils.js";
+import { logWorkers } from "./debug-bridge-logging.js";
+import { logEventTrace } from "./event-trace-logging.js";
 // Track workers marked as stale (should not return to pool after completion)
 const staleWorkers = new Set();
 const bundleMtimes = new Map();
@@ -23,7 +27,7 @@ process.on("exit", () => {
 });
 // Get bundle rebuild timestamp for staleness checking
 function getBundleMtime(buildOut) {
-    if (buildOut.includes(".mono-build")) {
+    if (isMonoBuildPath(buildOut)) {
         if (bundleMtimes.has(buildOut)) {
             return bundleMtimes.get(buildOut);
         }
@@ -72,11 +76,8 @@ function getBundleMtime(buildOut) {
 // Helper: Get pool key for worker lookup
 // For mono-build, uses shared key so any warm worker can serve any handler
 function getPoolKey(functionID, runtime, buildOut) {
-    const isMonoBuild = buildOut.includes(".mono-build");
-    if (isMonoBuild) {
-        return { key: `${runtime}:mono-build`, isShared: true };
-    }
-    return { key: `${runtime}:${functionID}`, isShared: false };
+    // Use the global mono build config for pool key calculation
+    return useMonoBuildConfig().getPoolKey(functionID, runtime, buildOut);
 }
 // Extract readable function name from handler path
 function getFunctionName(functionID) {
@@ -318,9 +319,9 @@ export const useRuntimeWorkers = lazy(async () => {
         const props = useFunctions().fromID(functionID);
         if (!props)
             return;
-        // Get build to check if mono-build
+        // Get build to check if mono-build using global config
         const build = await builder.artifact(functionID);
-        const isMonoBuild = build?.out.includes(".mono-build") ?? false;
+        const isMonoBuild = build ? isMonoBuildPath(build.out) : false;
         if (isMonoBuild) {
             // For mono-build: clear the entire shared pool since all functions share the same bundle
             const sharedPoolKey = `${props.runtime}:mono-build`;
@@ -391,14 +392,28 @@ export const useRuntimeWorkers = lazy(async () => {
     const lastRequestId = new Map();
     // Main invocation handler
     bus.subscribe("function.invoked", async (evt) => {
-        const { workerID: awsWorkerID, functionID, requestID, env, } = evt.properties;
+        const { workerID: awsWorkerID, functionID, requestID, env, event, } = evt.properties;
         const startTime = Date.now();
-        logInvokeTrace("INVOKE_RECEIVED", requestID, `func=${functionID.slice(-40)}`);
+        const requestPath = getRequestPath(event);
+        // Check if this is a warmup request - force-create new workers for these
+        // Matches the warmer format: { ding: true } or { warmer: true }
+        const isWarmupRequest = event && typeof event === 'object' &&
+            ('ding' in event || 'warmer' in event || event.__sst_warmup === true);
+        const warmupId = isWarmupRequest ? (event.warmupId ?? event.index) : undefined;
+        logWorkers(`path=${requestPath} reqId=${requestID.slice(0, 8)} RECEIVED func=${functionID.slice(-30)}`);
+        if (isWarmupRequest) {
+            logInvokeTrace("WARMUP_RECEIVED", requestID, `warmupId=${warmupId}`);
+        }
+        else {
+            logInvokeTrace("INVOKE_RECEIVED", requestID, `func=${functionID.slice(-40)}`);
+        }
         // Send ack immediately
-        bus.publish("function.ack", { functionID, workerID: awsWorkerID });
+        bus.publish("function.ack", { functionID, workerID: awsWorkerID, requestID });
+        logWorkers(`path=${requestPath} reqId=${requestID.slice(0, 8)} ACK sent elapsed=${Date.now() - startTime}ms`);
         logInvokeTrace("ACK_PUBLISHED", requestID, `elapsed=${Date.now() - startTime}ms`);
         const props = useFunctions().fromID(functionID);
         if (!props) {
+            logWorkers(`path=${requestPath} reqId=${requestID.slice(0, 8)} ERROR: Function not found`);
             Logger.debug("Function not found:", functionID);
             bus.publish("function.error", {
                 workerID: awsWorkerID,
@@ -412,6 +427,7 @@ export const useRuntimeWorkers = lazy(async () => {
         }
         const handler = handlers.for(props.runtime);
         if (!handler) {
+            logWorkers(`path=${requestPath} reqId=${requestID.slice(0, 8)} ERROR: No handler for runtime ${props.runtime}`);
             Logger.debug("No handler for runtime:", props.runtime);
             bus.publish("function.error", {
                 workerID: awsWorkerID,
@@ -423,10 +439,15 @@ export const useRuntimeWorkers = lazy(async () => {
             });
             return;
         }
+        logWorkers(`path=${requestPath} reqId=${requestID.slice(0, 8)} Getting build artifact...`);
         logInvokeTrace("BUILD_ARTIFACT_START", requestID);
+        const buildStartTime = Date.now();
         const build = await builder.artifact(functionID);
+        const buildElapsed = Date.now() - buildStartTime;
+        logWorkers(`path=${requestPath} reqId=${requestID.slice(0, 8)} Build artifact took ${buildElapsed}ms`);
         logInvokeTrace("BUILD_ARTIFACT_DONE", requestID, build ? `out=${build.out.slice(-30)}` : "NO_BUILD");
         if (!build) {
+            logWorkers(`path=${requestPath} reqId=${requestID.slice(0, 8)} ERROR: Build artifact not ready`);
             Logger.debug("Build artifact not ready for:", functionID);
             bus.publish("function.error", {
                 workerID: awsWorkerID,
@@ -444,13 +465,16 @@ export const useRuntimeWorkers = lazy(async () => {
             // === POOLED PATH ===
             // Get pool key: shared for mono-build, per-function otherwise
             const { key: poolKey, isShared } = getPoolKey(functionID, props.runtime, build.out);
-            let pooledWorker = getIdleWorker(poolKey, functionID, build.out);
+            logWorkers(`path=${requestPath} reqId=${requestID.slice(0, 8)} Looking for pooled worker, poolKey=${poolKey.slice(0, 20)}`);
+            // For warmup requests, always create new workers (never reuse from pool)
+            let pooledWorker = isWarmupRequest ? undefined : getIdleWorker(poolKey, functionID, build.out);
             let isReuse = false;
             if (pooledWorker) {
                 isReuse = true;
                 // Update functionID for cross-function reuse (mono-build)
                 pooledWorker.functionID = functionID;
                 trackRequestStart(functionID, true);
+                logWorkers(`path=${requestPath} reqId=${requestID.slice(0, 8)} REUSING pooled worker ${pooledWorker.pooledWorkerID.slice(0, 8)}`);
             }
             else {
                 // Create new pooled worker
@@ -466,6 +490,7 @@ export const useRuntimeWorkers = lazy(async () => {
                     bundlePath: build.out,
                     bundleMtime,
                 };
+                logWorkers(`path=${requestPath} reqId=${requestID.slice(0, 8)} CREATING new pooled worker ${pooledWorkerID.slice(0, 8)}`);
             }
             // Set up mappings
             workerIDMapping.set(awsWorkerID, pooledWorker.pooledWorkerID);
@@ -476,7 +501,7 @@ export const useRuntimeWorkers = lazy(async () => {
                 // Start new worker with pooledWorkerID (cold start)
                 trackRequestStart(functionID, false);
                 const currentPoolSize = workerPool.get(poolKey)?.length || 0;
-                logPool("CREATE", {
+                logPool(isWarmupRequest ? "WARMUP_CREATE" : "CREATE", {
                     pooledWorkerID: pooledWorker.pooledWorkerID.slice(0, 8),
                     functionID,
                     runtime: props.runtime,
@@ -485,8 +510,11 @@ export const useRuntimeWorkers = lazy(async () => {
                     isSharedPool: isShared,
                     currentPoolSize,
                     activeWorkers: activeWorkers.size,
+                    ...(isWarmupRequest && { warmupId }),
                 });
+                logWorkers(`path=${requestPath} reqId=${requestID.slice(0, 8)} Starting worker ${pooledWorker.pooledWorkerID.slice(0, 8)}...`);
                 logInvokeTrace("WORKER_START", requestID, `pooled=${pooledWorker.pooledWorkerID.slice(0, 8)}`);
+                const workerStartTime = Date.now();
                 try {
                     await handler.startWorker({
                         ...build,
@@ -495,15 +523,29 @@ export const useRuntimeWorkers = lazy(async () => {
                         environment: env,
                         url: `${serverConfig.url}/${pooledWorker.pooledWorkerID}/${serverConfig.API_VERSION}`,
                         runtime: props.runtime,
+                        isMonoBuild: isShared,
                     });
                     startedWorkers.add(pooledWorker.pooledWorkerID);
+                    const workerStartElapsed = Date.now() - workerStartTime;
+                    logWorkers(`path=${requestPath} reqId=${requestID.slice(0, 8)} Worker started in ${workerStartElapsed}ms`);
                     logInvokeTrace("WORKER_STARTED", requestID);
+                    logEventTrace("WORKER_START", {
+                        requestID,
+                        functionID,
+                        workerID: pooledWorker.pooledWorkerID,
+                        path: requestPath,
+                        correlationId: getCorrelationId(event),
+                        apiGwReqId: getApiGatewayRequestId(event),
+                        elapsed: workerStartElapsed,
+                        reused: false,
+                    });
                     bus.publish("worker.started", {
                         workerID: awsWorkerID,
                         functionID,
                     });
                 }
                 catch (ex) {
+                    logWorkers(`path=${requestPath} reqId=${requestID.slice(0, 8)} ERROR: Failed to start worker: ${ex.message}`);
                     Logger.debug("Failed to start pooled worker", ex);
                     bus.publish("function.error", {
                         workerID: awsWorkerID,
@@ -524,6 +566,15 @@ export const useRuntimeWorkers = lazy(async () => {
             }
             else {
                 logInvokeTrace("WORKER_REUSE", requestID, `pooled=${pooledWorker.pooledWorkerID.slice(0, 8)}`);
+                logEventTrace("WORKER_START", {
+                    requestID,
+                    functionID,
+                    workerID: pooledWorker.pooledWorkerID,
+                    path: requestPath,
+                    correlationId: getCorrelationId(event),
+                    apiGwReqId: getApiGatewayRequestId(event),
+                    reused: true,
+                });
                 bus.publish("worker.reused", {
                     workerID: awsWorkerID,
                     functionID,
@@ -532,6 +583,7 @@ export const useRuntimeWorkers = lazy(async () => {
             }
             // Route invocation to the pooled worker
             const server = await getServer();
+            logWorkers(`path=${requestPath} reqId=${requestID.slice(0, 8)} Routing invocation to worker ${pooledWorker.pooledWorkerID.slice(0, 8)} elapsed=${Date.now() - startTime}ms`);
             logInvokeTrace("ROUTE_INVOCATION", requestID);
             server.routeInvocation(pooledWorker.pooledWorkerID, evt.properties);
         }
@@ -549,6 +601,7 @@ export const useRuntimeWorkers = lazy(async () => {
                     environment: env,
                     url: `${serverConfig.url}/${awsWorkerID}/${serverConfig.API_VERSION}`,
                     runtime: props.runtime,
+                    isMonoBuild: isMonoBuildPath(build.out),
                 });
                 workers.set(awsWorkerID, { workerID: awsWorkerID, functionID });
                 bus.publish("worker.started", { workerID: awsWorkerID, functionID });
@@ -598,23 +651,49 @@ export const useRuntimeWorkers = lazy(async () => {
             if (pooled) {
                 const requestID = lastRequestId.get(workerID);
                 if (requestID) {
+                    const trimmedMessage = message.trim();
+                    // Log messages that contain [LOG] prefix
+                    if (trimmedMessage.includes("[LOG]")) {
+                        logEventTrace("WORKER_LOG", {
+                            requestID,
+                            functionID: pooled.functionID,
+                            workerID,
+                            message: trimmedMessage,
+                        });
+                    }
                     bus.publish("worker.stdout", {
                         workerID,
                         functionID: pooled.functionID,
-                        message: message.trim(),
+                        message: trimmedMessage,
                         requestID,
                     });
                 }
                 return;
             }
+            // Check if this is a preWarm worker (started but not yet active)
+            if (startedWorkers.has(workerID)) {
+                // During preWarm, ignore output since there's no request context
+                return;
+            }
             // Non-pooled worker
             const worker = workers.get(workerID);
             if (!worker)
                 return;
+            const trimmedMessage = message.trim();
+            const requestID = lastRequestId.get(workerID);
+            // Log messages that contain [LOG] prefix
+            if (trimmedMessage.includes("[LOG]") && requestID) {
+                logEventTrace("WORKER_LOG", {
+                    requestID,
+                    functionID: worker.functionID,
+                    workerID,
+                    message: trimmedMessage,
+                });
+            }
             bus.publish("worker.stdout", {
                 ...worker,
-                message: message.trim(),
-                requestID: lastRequestId.get(workerID),
+                message: trimmedMessage,
+                requestID: requestID,
             });
         },
         exited(workerID) {
@@ -677,5 +756,123 @@ export const useRuntimeWorkers = lazy(async () => {
             return activeWorkers.has(workerID) || startedWorkers.has(workerID);
         },
         subscribe: bus.forward("worker.started", "worker.stopped", "worker.exited", "worker.stdout", "worker.reused"),
+        /**
+         * Trigger warmup by invoking Lambda functions with warmup payloads.
+         * This sends real requests through the IoT bridge, which naturally creates workers.
+         * @param count Number of workers to warm up (default: 15)
+         */
+        async triggerWarmup(count = 15) {
+            const functions = useFunctions();
+            const allFunctions = functions.all;
+            // Find a nodejs function to use as the warmup target
+            let targetFunction = null;
+            for (const [id, props] of Object.entries(allFunctions)) {
+                if (!props.runtime?.startsWith("nodejs"))
+                    continue;
+                if (!isPoolableRuntime(props.runtime))
+                    continue;
+                if (!props.functionName)
+                    continue;
+                targetFunction = { id, props, functionName: props.functionName };
+                break;
+            }
+            if (!targetFunction) {
+                logPool("WARMUP_SKIP", {
+                    reason: "no nodejs function found",
+                });
+                return { warmed: 0 };
+            }
+            const { functionName } = targetFunction;
+            logPool("WARMUP_START", {
+                count,
+                functionName,
+            });
+            // Publish warmup start event
+            bus.publish("warmup.start", { count });
+            const startTime = Date.now();
+            let success = 0;
+            let failed = 0;
+            let completed = 0;
+            // Use the shared AWS client
+            const { useAWSClient } = await import("../credentials.js");
+            const { LambdaClient, InvokeCommand } = await import("@aws-sdk/client-lambda");
+            const lambda = useAWSClient(LambdaClient);
+            // Helper to publish progress
+            const publishProgress = () => {
+                bus.publish("warmup.progress", {
+                    completed,
+                    total: count,
+                    success,
+                    failed,
+                });
+            };
+            // Phase 1: Invoke first warmup to populate V8 compile cache
+            try {
+                const result = await lambda.send(new InvokeCommand({
+                    FunctionName: functionName,
+                    InvocationType: "RequestResponse",
+                    Payload: JSON.stringify({
+                        ding: true,
+                        concurrency: count,
+                        index: 0,
+                    }),
+                }));
+                if (result.StatusCode === 200) {
+                    success++;
+                }
+                else {
+                    failed++;
+                }
+            }
+            catch (ex) {
+                failed++;
+            }
+            completed++;
+            publishProgress();
+            // Phase 2: Invoke remaining warmups in parallel
+            if (count > 1) {
+                const results = await Promise.all(Array.from({ length: count - 1 }, (_, i) => i + 1).map(async (i) => {
+                    try {
+                        const result = await lambda.send(new InvokeCommand({
+                            FunctionName: functionName,
+                            InvocationType: "RequestResponse",
+                            Payload: JSON.stringify({
+                                ding: true,
+                                concurrency: count,
+                                index: i,
+                            }),
+                        }));
+                        const ok = result.StatusCode === 200;
+                        if (ok)
+                            success++;
+                        else
+                            failed++;
+                        completed++;
+                        publishProgress();
+                        return ok;
+                    }
+                    catch {
+                        failed++;
+                        completed++;
+                        publishProgress();
+                        return false;
+                    }
+                }));
+            }
+            const elapsed = Date.now() - startTime;
+            logPool("WARMUP_DONE", {
+                success,
+                failed,
+                elapsedMs: elapsed,
+                avgMs: success > 0 ? Math.round(elapsed / success) : 0,
+            });
+            // Publish warmup complete event
+            bus.publish("warmup.complete", {
+                success,
+                failed,
+                elapsedMs: elapsed,
+            });
+            return { warmed: success, elapsed };
+        },
     };
 });