@514labs/moose-lib 0.6.449 → 0.6.450

package/dist/moose-runner.js

@@ -3375,14 +3375,16 @@ var init_internal = __esm({
               name: destinationName,
               version: config.version,
               metadata: config.metadata,
-              sourceFile: config.sourceFile
+              sourceFile: config.sourceFile,
+              deadLetterQueue: config.deadLetterQueue?.name
             });
           });
         });
         stream._consumers.forEach((consumer) => {
           consumers.push({
             version: consumer.config.version,
-            sourceFile: consumer.config.sourceFile
+            sourceFile: consumer.config.sourceFile,
+            deadLetterQueue: consumer.config.deadLetterQueue?.name
           });
         });
         topics[stream.name] = {
@@ -4477,7 +4479,7 @@ var stopConsumer = async (logger2, consumer, sourceTopic) => {
     }
   }
 };
-var handleMessage = async (logger2, streamingFunctionWithConfigList, message, producer, fieldMutations, logPayloads) => {
+var handleMessage = async (logger2, streamingFunctionWithConfigList, message, producer, fieldMutations, logPayloads, dlqTopicName) => {
   if (message.value === void 0 || message.value === null) {
     logger2.log(`Received message with no value, skipping...`);
     return void 0;
@@ -4497,12 +4499,10 @@ var handleMessage = async (logger2, streamingFunctionWithConfigList, message, pr
         try {
           return await fn(parsedData);
         } catch (e) {
-          const deadLetterQueue = config.deadLetterQueue;
-          if (deadLetterQueue) {
+          if (dlqTopicName) {
             const deadLetterRecord = {
               originalRecord: {
                 ...parsedData,
-                // Include original Kafka message metadata
                 __sourcePartition: message.partition,
                 __sourceOffset: message.offset,
                 __sourceTimestamp: message.timestamp
@@ -4514,12 +4514,12 @@ var handleMessage = async (logger2, streamingFunctionWithConfigList, message, pr
             };
             cliLog({
               action: "DeadLetter",
-              message: `Sending message to DLQ ${deadLetterQueue.name}: ${e instanceof Error ? e.message : String(e)}`,
+              message: `Sending message to DLQ ${dlqTopicName}: ${e instanceof Error ? e.message : String(e)}`,
               message_type: "Error"
             });
             try {
               await producer.send({
-                topic: deadLetterQueue.name,
+                topic: dlqTopicName,
                 messages: [{ value: JSON.stringify(deadLetterRecord) }]
               });
             } catch (dlqError) {
@@ -4536,15 +4536,14 @@ var handleMessage = async (logger2, streamingFunctionWithConfigList, message, pr
         }
       })
     );
-    const processedMessages = transformedData.map((userFunctionOutput, i) => {
-      const [_, config] = streamingFunctionWithConfigList[i];
+    const processedMessages = transformedData.map((userFunctionOutput) => {
       if (userFunctionOutput) {
         if (Array.isArray(userFunctionOutput)) {
           return userFunctionOutput.flat().filter((item) => item !== void 0 && item !== null).map((item) => ({
             value: JSON.stringify(item),
             originalValue: parsedData,
             originalMessage: message,
-            dlq: config.deadLetterQueue ?? void 0
+            dlqTopicName
           }));
         } else {
           return [
@@ -4552,7 +4551,7 @@ var handleMessage = async (logger2, streamingFunctionWithConfigList, message, pr
               value: JSON.stringify(userFunctionOutput),
               originalValue: parsedData,
               originalMessage: message,
-              dlq: config.deadLetterQueue ?? void 0
+              dlqTopicName
             }
           ];
         }
@@ -4580,11 +4579,10 @@ var handleDLQForFailedMessages = async (logger2, producer, messages, error) => {
   let messagesWithoutDLQ = 0;
   let dlqErrors = 0;
   for (const msg of messages) {
-    if (msg.dlq && msg.originalValue) {
+    if (msg.dlqTopicName && msg.originalValue) {
       const deadLetterRecord = {
         originalRecord: {
           ...msg.originalValue,
-          // Include original Kafka message metadata
           __sourcePartition: msg.originalMessage.partition,
           __sourceOffset: msg.originalMessage.offset,
           __sourceTimestamp: msg.originalMessage.timestamp
@@ -4596,21 +4594,21 @@ var handleDLQForFailedMessages = async (logger2, producer, messages, error) => {
       };
       cliLog({
         action: "DeadLetter",
-        message: `Sending failed message to DLQ ${msg.dlq.name}: ${error instanceof Error ? error.message : String(error)}`,
+        message: `Sending failed message to DLQ ${msg.dlqTopicName}: ${error instanceof Error ? error.message : String(error)}`,
         message_type: "Error"
       });
       try {
         await producer.send({
-          topic: msg.dlq.name,
+          topic: msg.dlqTopicName,
           messages: [{ value: JSON.stringify(deadLetterRecord) }]
         });
-        logger2.log(`Sent failed message to DLQ ${msg.dlq.name}`);
+        logger2.log(`Sent failed message to DLQ ${msg.dlqTopicName}`);
         messagesHandledByDLQ++;
       } catch (dlqError) {
         logger2.error(`Failed to send to DLQ: ${dlqError}`);
         dlqErrors++;
       }
-    } else if (!msg.dlq) {
+    } else if (!msg.dlqTopicName) {
       messagesWithoutDLQ++;
       logger2.warn(`Cannot send to DLQ: no DLQ configured for message`);
     } else {
@@ -4765,7 +4763,8 @@ var startConsumer = async (args, logger2, metrics, _parallelism, consumer, produ
               message,
               producer,
               fieldMutations,
-              args.logPayloads
+              args.logPayloads,
+              args.dlqTopic?.name
             );
           },
           {
@@ -5424,11 +5423,15 @@ program.command("consumption-apis").description("Run consumption APIs").argument
 program.command("streaming-functions").description("Run streaming functions").argument("<source-topic>", "Source topic configuration as JSON").argument("<function-file-path>", "Path to the function file").argument(
   "<broker>",
   "Kafka broker address(es) - comma-separated for multiple brokers (e.g., 'broker1:9092, broker2:9092'). Whitespace around commas is automatically trimmed."
-).argument("<max-subscriber-count>", "Maximum number of subscribers").option("--target-topic <target-topic>", "Target topic configuration as JSON").option("--sasl-username <username>", "SASL username").option("--sasl-password <password>", "SASL password").option("--sasl-mechanism <mechanism>", "SASL mechanism").option("--security-protocol <protocol>", "Security protocol").option("--log-payloads", "Log payloads for debugging", false).action(
+).argument("<max-subscriber-count>", "Maximum number of subscribers").option("--target-topic <target-topic>", "Target topic configuration as JSON").option(
+  "--dlq-topic <dlq-topic>",
+  "Dead letter queue topic configuration as JSON"
+).option("--sasl-username <username>", "SASL username").option("--sasl-password <password>", "SASL password").option("--sasl-mechanism <mechanism>", "SASL mechanism").option("--security-protocol <protocol>", "Security protocol").option("--log-payloads", "Log payloads for debugging", false).action(
   (sourceTopic, functionFilePath, broker, maxSubscriberCount, options) => {
     const config = {
       sourceTopic: JSON.parse(sourceTopic),
       targetTopic: options.targetTopic ? JSON.parse(options.targetTopic) : void 0,
+      dlqTopic: options.dlqTopic ? JSON.parse(options.dlqTopic) : void 0,
       functionFilePath,
       broker,
       maxSubscriberCount: parseInt(maxSubscriberCount),