@514labs/moose-lib 0.6.349 → 0.6.351

package/dist/moose-runner.js

@@ -2217,6 +2217,103 @@ var Cluster = class {
 init_sqlHelpers();
 init_internal();
 init_compiler_config();
+
+// src/utils/structured-logging.ts
+var util = __toESM(require("util"));
+var import_async_hooks = require("async_hooks");
+function setupStructuredConsole(getContextField, contextFieldName) {
+  const contextStorage = new import_async_hooks.AsyncLocalStorage();
+  const originalConsole = {
+    log: console.log,
+    info: console.info,
+    warn: console.warn,
+    error: console.error,
+    debug: console.debug
+  };
+  console.log = createStructuredConsoleWrapper(
+    contextStorage,
+    getContextField,
+    contextFieldName,
+    originalConsole.log,
+    "info"
+  );
+  console.info = createStructuredConsoleWrapper(
+    contextStorage,
+    getContextField,
+    contextFieldName,
+    originalConsole.info,
+    "info"
+  );
+  console.warn = createStructuredConsoleWrapper(
+    contextStorage,
+    getContextField,
+    contextFieldName,
+    originalConsole.warn,
+    "warn"
+  );
+  console.error = createStructuredConsoleWrapper(
+    contextStorage,
+    getContextField,
+    contextFieldName,
+    originalConsole.error,
+    "error"
+  );
+  console.debug = createStructuredConsoleWrapper(
+    contextStorage,
+    getContextField,
+    contextFieldName,
+    originalConsole.debug,
+    "debug"
+  );
+  return contextStorage;
+}
+function safeStringify(arg) {
+  if (typeof arg === "object" && arg !== null) {
+    if (arg instanceof Error) {
+      return util.inspect(arg, { depth: 2, breakLength: Infinity });
+    }
+    try {
+      return JSON.stringify(arg);
+    } catch (e) {
+      return util.inspect(arg, { depth: 2, breakLength: Infinity });
+    }
+  }
+  if (typeof arg === "string") {
+    return arg;
+  }
+  return util.inspect(arg);
+}
+function createStructuredConsoleWrapper(contextStorage, getContextField, contextFieldName, originalMethod, level) {
+  return (...args) => {
+    const context = contextStorage.getStore();
+    if (!context) {
+      originalMethod(...args);
+      return;
+    }
+    let ctxValue;
+    try {
+      ctxValue = getContextField(context);
+    } catch {
+      ctxValue = "unknown";
+    }
+    try {
+      const message = args.map((arg) => safeStringify(arg)).join(" ");
+      process.stderr.write(
+        JSON.stringify({
+          __moose_structured_log__: true,
+          level,
+          message,
+          [contextFieldName]: ctxValue,
+          timestamp: (/* @__PURE__ */ new Date()).toISOString()
+        }) + "\n"
+      );
+    } catch {
+      originalMethod(...args);
+    }
+  };
+}
+
+// src/consumption-apis/runner.ts
 var toClientConfig2 = (config) => ({
   ...config,
   useSSL: config.useSSL ? "true" : "false"
@@ -2225,12 +2322,21 @@ var createPath = (apisDir, path5, useCompiled2) => {
   const extension = useCompiled2 ? ".js" : ".ts";
   return `${apisDir}${path5}${extension}`;
 };
-var httpLogger = (req, res, startMs) => {
-  console.log(
+var httpLogger = (req, res, startMs, apiName) => {
+  const logFn = () => console.log(
     `${req.method} ${req.url} ${res.statusCode} ${Date.now() - startMs}ms`
   );
+  if (apiName) {
+    apiContextStorage.run({ apiName }, logFn);
+  } else {
+    logFn();
+  }
 };
 var modulesCache = /* @__PURE__ */ new Map();
+var apiContextStorage = setupStructuredConsole(
+  (ctx) => ctx.apiName,
+  "api_name"
+);
 var apiHandler = async (publicKey, clickhouseClient, temporalClient, enforceAuth, jwtConfig) => {
   const useCompiled2 = shouldUseCompiled();
   const sourceDir = getSourceDir();
@@ -2238,6 +2344,7 @@ var apiHandler = async (publicKey, clickhouseClient, temporalClient, enforceAuth
   const apis = await getApis2();
   return async (req, res) => {
     const start = Date.now();
+    let matchedApiName;
     try {
       const url = new URL(req.url || "", "http://localhost");
       const fileName = url.pathname;
@@ -2289,43 +2396,66 @@ var apiHandler = async (publicKey, clickhouseClient, temporalClient, enforceAuth
         },
         {}
       );
-      let userFuncModule = modulesCache.get(pathName);
-      if (userFuncModule === void 0) {
-        let apiName = fileName.replace(/^\/+|\/+$/g, "");
+      const versionParam = url.searchParams.get("version");
+      const cacheKey = versionParam ? `${pathName}:${versionParam}` : pathName;
+      let userFuncModule;
+      const cachedEntry = modulesCache.get(cacheKey);
+      if (cachedEntry !== void 0) {
+        userFuncModule = cachedEntry.module;
+        matchedApiName = cachedEntry.apiName;
+      } else {
+        let lookupName = fileName.replace(/^\/+|\/+$/g, "");
         let version = null;
-        userFuncModule = apis.get(apiName);
+        userFuncModule = apis.get(lookupName);
+        if (userFuncModule) {
+          matchedApiName = lookupName;
+        }
         if (!userFuncModule) {
           version = url.searchParams.get("version");
-          if (!version && apiName.includes("/")) {
-            const pathParts = apiName.split("/");
+          if (!version && lookupName.includes("/")) {
+            const pathParts = lookupName.split("/");
             if (pathParts.length >= 2) {
-              userFuncModule = apis.get(apiName);
-              if (!userFuncModule) {
-                apiName = pathParts[0];
-                version = pathParts.slice(1).join("/");
-              }
+              lookupName = pathParts[0];
+              version = pathParts.slice(1).join("/");
             }
           }
           if (!userFuncModule && version) {
-            const versionedKey = `${apiName}:${version}`;
+            const versionedKey = `${lookupName}:${version}`;
             userFuncModule = apis.get(versionedKey);
+            if (userFuncModule) {
+              matchedApiName = lookupName;
+            }
+          }
+          if (!userFuncModule) {
+            userFuncModule = apis.get(lookupName);
+            if (userFuncModule) {
+              matchedApiName = lookupName;
+            }
           }
         }
-        if (!userFuncModule) {
+        if (!userFuncModule || matchedApiName === void 0) {
           const availableApis = Array.from(apis.keys()).map(
             (key) => key.replace(":", "/")
           );
-          const errorMessage = version ? `API ${apiName} with version ${version} not found. Available APIs: ${availableApis.join(", ")}` : `API ${apiName} not found. Available APIs: ${availableApis.join(", ")}`;
+          const errorMessage = version ? `API ${lookupName} with version ${version} not found. Available APIs: ${availableApis.join(", ")}` : `API ${lookupName} not found. Available APIs: ${availableApis.join(", ")}`;
           throw new Error(errorMessage);
         }
-        modulesCache.set(pathName, userFuncModule);
-        console.log(`[API] | Executing API: ${apiName}`);
+        modulesCache.set(cacheKey, {
+          module: userFuncModule,
+          apiName: matchedApiName
+        });
+        apiContextStorage.run({ apiName: matchedApiName }, () => {
+          console.log(`[API] | Executing API: ${matchedApiName}`);
+        });
       }
       const queryClient = new QueryClient(clickhouseClient, fileName);
-      let result = await userFuncModule(paramsObject, {
-        client: new MooseClient(queryClient, temporalClient),
-        sql,
-        jwt: jwtPayload
+      const apiName = matchedApiName;
+      const result = await apiContextStorage.run({ apiName }, async () => {
+        return await userFuncModule(paramsObject, {
+          client: new MooseClient(queryClient, temporalClient),
+          sql,
+          jwt: jwtPayload
+        });
       });
       let body;
       let status;
@@ -2341,27 +2471,32 @@ var apiHandler = async (publicKey, clickhouseClient, temporalClient, enforceAuth
       }
       if (status) {
         res.writeHead(status, { "Content-Type": "application/json" });
-        httpLogger(req, res, start);
+        httpLogger(req, res, start, apiName);
       } else {
         res.writeHead(200, { "Content-Type": "application/json" });
-        httpLogger(req, res, start);
+        httpLogger(req, res, start, apiName);
       }
       res.end(body);
     } catch (error) {
-      console.log("error in path ", req.url, error);
+      const logError2 = () => console.log("error in path ", req.url, error);
+      if (matchedApiName) {
+        apiContextStorage.run({ apiName: matchedApiName }, logError2);
+      } else {
+        logError2();
+      }
       if (Object.getPrototypeOf(error).constructor.name === "TypeGuardError") {
         res.writeHead(400, { "Content-Type": "application/json" });
         res.end(JSON.stringify({ error: error.message }));
-        httpLogger(req, res, start);
+        httpLogger(req, res, start, matchedApiName);
       }
       if (error instanceof Error) {
         res.writeHead(500, { "Content-Type": "application/json" });
         res.end(JSON.stringify({ error: error.message }));
-        httpLogger(req, res, start);
+        httpLogger(req, res, start, matchedApiName);
       } else {
         res.writeHead(500, { "Content-Type": "application/json" });
         res.end();
-        httpLogger(req, res, start);
+        httpLogger(req, res, start, matchedApiName);
       }
     }
   };
@@ -2533,6 +2668,10 @@ var SESSION_TIMEOUT_CONSUMER = 3e4;
 var HEARTBEAT_INTERVAL_CONSUMER = 3e3;
 var DEFAULT_MAX_STREAMING_CONCURRENCY = 100;
 var CONSUMER_MAX_BATCH_SIZE = 1e3;
+var functionContextStorage = setupStructuredConsole(
+  (ctx) => ctx.functionName,
+  "function_name"
+);
 var MAX_STREAMING_CONCURRENCY = process3.env.MAX_STREAMING_CONCURRENCY ? parseInt(process3.env.MAX_STREAMING_CONCURRENCY, 10) : DEFAULT_MAX_STREAMING_CONCURRENCY;
 var metricsLog = (log) => {
   const req = http3.request({
@@ -2858,56 +2997,63 @@ var startConsumer = async (args, logger2, metrics, _parallelism, consumer, produ
       if (!isRunning() || isStale()) {
         return;
       }
-      metrics.count_in += batch.messages.length;
-      cliLog({
-        action: "Received",
-        message: `${logger2.logPrefix} ${batch.messages.length} message(s)`
-      });
-      logger2.log(`Received ${batch.messages.length} message(s)`);
-      let index = 0;
-      const readableStream = import_node_stream2.Readable.from(batch.messages);
-      const processedMessages = await readableStream.map(
-        async (message) => {
-          index++;
-          if (batch.messages.length > DEFAULT_MAX_STREAMING_CONCURRENCY && index % DEFAULT_MAX_STREAMING_CONCURRENCY || index - 1 === batch.messages.length) {
-            await heartbeat();
+      const functionName = logger2.logPrefix;
+      await functionContextStorage.run({ functionName }, async () => {
+        metrics.count_in += batch.messages.length;
+        cliLog({
+          action: "Received",
+          message: `${logger2.logPrefix.replace("__", " -> ")} ${batch.messages.length} message(s)`
+        });
+        logger2.log(`Received ${batch.messages.length} message(s)`);
+        let index = 0;
+        const readableStream = import_node_stream2.Readable.from(batch.messages);
+        const processedMessages = await readableStream.map(
+          async (message) => {
+            index++;
+            if (batch.messages.length > DEFAULT_MAX_STREAMING_CONCURRENCY && index % DEFAULT_MAX_STREAMING_CONCURRENCY || index - 1 === batch.messages.length) {
+              await heartbeat();
+            }
+            return handleMessage(
+              logger2,
+              streamingFunctions,
+              message,
+              producer,
+              fieldMutations,
+              args.logPayloads
+            );
+          },
+          {
+            concurrency: MAX_STREAMING_CONCURRENCY
           }
-          return handleMessage(
+        ).toArray();
+        const filteredMessages = processedMessages.flat().filter((msg) => msg !== void 0 && msg.value !== void 0);
+        if (args.targetTopic === void 0 || processedMessages.length === 0) {
+          return;
+        }
+        await heartbeat();
+        if (filteredMessages.length > 0) {
+          await sendMessages(
             logger2,
-            streamingFunctions,
-            message,
+            metrics,
+            args.targetTopic,
             producer,
-            fieldMutations,
-            args.logPayloads
+            filteredMessages
           );
-        },
-        {
-          concurrency: MAX_STREAMING_CONCURRENCY
         }
-      ).toArray();
-      const filteredMessages = processedMessages.flat().filter((msg) => msg !== void 0 && msg.value !== void 0);
-      if (args.targetTopic === void 0 || processedMessages.length === 0) {
-        return;
-      }
-      await heartbeat();
-      if (filteredMessages.length > 0) {
-        await sendMessages(
-          logger2,
-          metrics,
-          args.targetTopic,
-          producer,
-          filteredMessages
-        );
-      }
+      });
     }
   });
   logger2.log("Consumer is running...");
 };
 var buildLogger = (args, workerId) => {
-  const targetLabel = args.targetTopic?.name ? ` -> ${args.targetTopic.name}` : " (consumer)";
-  const logPrefix = `${args.sourceTopic.name}${targetLabel} (worker ${workerId})`;
+  const sourceBaseName = topicNameToStreamName(args.sourceTopic);
+  const targetBaseName = args.targetTopic ? topicNameToStreamName(args.targetTopic) : void 0;
+  const functionName = targetBaseName ? `${sourceBaseName}__${targetBaseName}` : sourceBaseName;
+  const logPrefix = `${functionName} (worker ${workerId})`;
   return {
-    logPrefix,
+    // logPrefix is used for structured logging (function_name field)
+    // Must match source_primitive.name format for log correlation
+    logPrefix: functionName,
     log: (message) => {
       console.log(`${logPrefix}: ${message}`);
     },
@@ -2972,82 +3118,88 @@ var runStreamingFunctions = async (args) => {
     maxWorkerCount: args.maxSubscriberCount,
     workerStart: async (worker, parallelism) => {
       const logger2 = buildLogger(args, worker.id);
-      const metrics = {
-        count_in: 0,
-        count_out: 0,
-        bytes: 0
-      };
-      setTimeout(() => sendMessageMetrics(logger2, metrics), 1e3);
-      const clientIdPrefix = HOSTNAME ? `${HOSTNAME}-` : "";
-      const processId = `${clientIdPrefix}${streamingFuncId}-ts-${worker.id}`;
-      const kafka = await getKafkaClient(
-        {
-          clientId: processId,
-          broker: args.broker,
-          securityProtocol: args.securityProtocol,
-          saslUsername: args.saslUsername,
-          saslPassword: args.saslPassword,
-          saslMechanism: args.saslMechanism
-        },
-        logger2
-      );
-      const consumer = kafka.consumer({
-        kafkaJS: {
-          groupId: streamingFuncId,
-          sessionTimeout: SESSION_TIMEOUT_CONSUMER,
-          heartbeatInterval: HEARTBEAT_INTERVAL_CONSUMER,
-          retry: {
-            retries: MAX_RETRIES_CONSUMER
+      const functionName = logger2.logPrefix;
+      return await functionContextStorage.run({ functionName }, async () => {
+        const metrics = {
+          count_in: 0,
+          count_out: 0,
+          bytes: 0
+        };
+        setTimeout(() => sendMessageMetrics(logger2, metrics), 1e3);
+        const clientIdPrefix = HOSTNAME ? `${HOSTNAME}-` : "";
+        const processId = `${clientIdPrefix}${streamingFuncId}-ts-${worker.id}`;
+        const kafka = await getKafkaClient(
+          {
+            clientId: processId,
+            broker: args.broker,
+            securityProtocol: args.securityProtocol,
+            saslUsername: args.saslUsername,
+            saslPassword: args.saslPassword,
+            saslMechanism: args.saslMechanism
           },
-          autoCommit: true,
-          autoCommitInterval: AUTO_COMMIT_INTERVAL_MS,
-          fromBeginning: true
-        },
-        "js.consumer.max.batch.size": CONSUMER_MAX_BATCH_SIZE
-      });
-      const maxMessageBytes = args.targetTopic?.max_message_bytes || 1024 * 1024;
-      const producer = kafka.producer(
-        createProducerConfig(maxMessageBytes)
-      );
-      try {
-        logger2.log("Starting producer...");
-        await startProducer(logger2, producer);
+          logger2
+        );
+        const consumer = kafka.consumer({
+          kafkaJS: {
+            groupId: streamingFuncId,
+            sessionTimeout: SESSION_TIMEOUT_CONSUMER,
+            heartbeatInterval: HEARTBEAT_INTERVAL_CONSUMER,
+            retry: {
+              retries: MAX_RETRIES_CONSUMER
+            },
+            autoCommit: true,
+            autoCommitInterval: AUTO_COMMIT_INTERVAL_MS,
+            fromBeginning: true
+          },
+          "js.consumer.max.batch.size": CONSUMER_MAX_BATCH_SIZE
+        });
+        const maxMessageBytes = args.targetTopic?.max_message_bytes || 1024 * 1024;
+        const producer = kafka.producer(
+          createProducerConfig(maxMessageBytes)
+        );
         try {
-          logger2.log("Starting consumer...");
-          await startConsumer(
-            args,
-            logger2,
-            metrics,
-            parallelism,
-            consumer,
-            producer,
-            streamingFuncId
-          );
+          logger2.log("Starting producer...");
+          await startProducer(logger2, producer);
+          try {
+            logger2.log("Starting consumer...");
+            await startConsumer(
+              args,
+              logger2,
+              metrics,
+              parallelism,
+              consumer,
+              producer,
+              streamingFuncId
+            );
+          } catch (e) {
+            logger2.error("Failed to start kafka consumer: ");
+            if (e instanceof Error) {
+              logError(logger2, e);
+            }
+            throw e;
+          }
         } catch (e) {
-          logger2.error("Failed to start kafka consumer: ");
+          logger2.error("Failed to start kafka producer: ");
           if (e instanceof Error) {
             logError(logger2, e);
           }
           throw e;
         }
-      } catch (e) {
-        logger2.error("Failed to start kafka producer: ");
-        if (e instanceof Error) {
-          logError(logger2, e);
-        }
-        throw e;
-      }
-      return [logger2, producer, consumer];
+        return [logger2, producer, consumer];
+      });
     },
     workerStop: async ([logger2, producer, consumer]) => {
-      logger2.log(`Received SIGTERM, shutting down gracefully...`);
-      logger2.log("Stopping consumer first...");
-      await stopConsumer(logger2, consumer, args.sourceTopic);
-      logger2.log("Waiting for in-flight messages to complete...");
-      await new Promise((resolve3) => setTimeout(resolve3, 2e3));
-      logger2.log("Stopping producer...");
-      await stopProducer(logger2, producer);
-      logger2.log("Graceful shutdown completed");
+      const functionName = logger2.logPrefix;
+      await functionContextStorage.run({ functionName }, async () => {
+        logger2.log(`Received SIGTERM, shutting down gracefully...`);
+        logger2.log("Stopping consumer first...");
+        await stopConsumer(logger2, consumer, args.sourceTopic);
+        logger2.log("Waiting for in-flight messages to complete...");
+        await new Promise((resolve3) => setTimeout(resolve3, 2e3));
+        logger2.log("Stopping producer...");
+        await stopProducer(logger2, producer);
+        logger2.log("Graceful shutdown completed");
+      });
     }
   });
   cluster2.start();
@@ -3103,6 +3255,10 @@ var import_activity = require("@temporalio/activity");
 var import_workflow3 = require("@temporalio/workflow");
 init_internal();
 init_json();
+var taskContextStorage = setupStructuredConsole(
+  (ctx) => ctx.taskName,
+  "task_name"
+);
 var activities = {
   async hasWorkflow(name) {
     try {
@@ -3184,10 +3340,16 @@ var activities = {
       const revivedInputData = inputData ? JSON.parse(JSON.stringify(inputData), jsonDateReviver) : inputData;
       try {
         startPeriodicHeartbeat();
-        const result = await Promise.race([
-          fullTask.config.run({ state: taskState, input: revivedInputData }),
-          context.cancelled
-        ]);
+        const taskIdentifier = workflow.name;
+        const result = await taskContextStorage.run(
+          { taskName: taskIdentifier },
+          async () => {
+            return await Promise.race([
+              fullTask.config.run({ state: taskState, input: revivedInputData }),
+              context.cancelled
+            ]);
+          }
+        );
         return result;
       } catch (error) {
         if ((0, import_workflow3.isCancellation)(error)) {