@514labs/moose-lib 0.6.256-ci-1-g6ca86038 → 0.6.256-ci-4-g0ca62054

package/dist/moose-runner.mjs

@@ -26,6 +26,22 @@ function isTruthy(value) {
       return false;
   }
 }
+function createProducerConfig(maxMessageBytes) {
+  return {
+    kafkaJS: {
+      idempotent: false,
+      // Not needed for at-least-once delivery
+      acks: ACKs,
+      retry: {
+        retries: MAX_RETRIES_PRODUCER,
+        maxRetryTime: MAX_RETRY_TIME_MS
+      }
+    },
+    "linger.ms": 0,
+    // This is to make sure at least once delivery with immediate feedback on the send
+    ...maxMessageBytes && { "message.max.bytes": maxMessageBytes }
+  };
+}
 var Kafka, compilerLog, getClickhouseClient, cliLog, MAX_RETRIES, MAX_RETRY_TIME_MS, RETRY_INITIAL_TIME_MS, MAX_RETRIES_PRODUCER, ACKs, parseBrokerString, logError, buildSaslConfig, getKafkaClient;
 var init_commons = __esm({
   "src/commons.ts"() {
@@ -1803,156 +1819,7 @@ var MAX_RETRIES_CONSUMER = 150;
 var SESSION_TIMEOUT_CONSUMER = 3e4;
 var HEARTBEAT_INTERVAL_CONSUMER = 3e3;
 var DEFAULT_MAX_STREAMING_CONCURRENCY = 100;
-var KAFKAJS_BYTE_MESSAGE_OVERHEAD = 500;
-var isMessageTooLargeError = (error) => {
-  if (KafkaJS2.isKafkaJSError && error instanceof Error && KafkaJS2.isKafkaJSError(error)) {
-    return error.type === "ERR_MSG_SIZE_TOO_LARGE" || error.code === 10 || error.cause !== void 0 && isMessageTooLargeError(error.cause);
-  }
-  if (error && typeof error === "object") {
-    const err = error;
-    return err.type === "ERR_MSG_SIZE_TOO_LARGE" || err.code === 10 || err.cause !== void 0 && isMessageTooLargeError(err.cause);
-  }
-  return false;
-};
-var splitBatch = (messages, maxChunkSize) => {
-  if (messages.length <= 1) {
-    return [messages];
-  }
-  const chunks = [];
-  let currentChunk = [];
-  let currentSize = 0;
-  for (const message of messages) {
-    const messageSize = Buffer2.byteLength(message.value, "utf8") + KAFKAJS_BYTE_MESSAGE_OVERHEAD;
-    if (currentSize + messageSize > maxChunkSize && currentChunk.length > 0) {
-      chunks.push(currentChunk);
-      currentChunk = [message];
-      currentSize = messageSize;
-    } else {
-      currentChunk.push(message);
-      currentSize += messageSize;
-    }
-  }
-  if (currentChunk.length > 0) {
-    chunks.push(currentChunk);
-  }
-  return chunks;
-};
-var sendChunkWithRetry = async (logger2, targetTopic, producer, messages, currentMaxSize, maxRetries = 3) => {
-  const currentMessages = messages;
-  let attempts = 0;
-  while (attempts < maxRetries) {
-    try {
-      await producer.send({
-        topic: targetTopic.name,
-        messages: currentMessages
-      });
-      logger2.log(
-        `Successfully sent ${currentMessages.length} messages to ${targetTopic.name}`
-      );
-      return;
-    } catch (error) {
-      if (isMessageTooLargeError(error) && currentMessages.length > 1) {
-        logger2.warn(
-          `Got MESSAGE_TOO_LARGE error, splitting batch of ${currentMessages.length} messages and retrying (${maxRetries - attempts} attempts left)`
-        );
-        const newMaxSize = Math.floor(currentMaxSize / 2);
-        const splitChunks = splitBatch(currentMessages, newMaxSize);
-        for (const chunk of splitChunks) {
-          await sendChunkWithRetry(
-            logger2,
-            targetTopic,
-            producer,
-            chunk,
-            newMaxSize,
-            // this error does not count as one failed attempt
-            maxRetries - attempts
-          );
-        }
-        return;
-      } else {
-        attempts++;
-        if (attempts >= maxRetries) {
-          let messagesHandledByDLQ = 0;
-          let messagesWithoutDLQ = 0;
-          const dlqErrors = [];
-          for (const failedMessage of currentMessages) {
-            const dlqTopic = failedMessage.dlq;
-            if (dlqTopic && failedMessage.originalValue) {
-              const dlqTopicName = dlqTopic.name;
-              const deadLetterRecord = {
-                originalRecord: {
-                  ...failedMessage.originalValue,
-                  // Include original Kafka message metadata
-                  __sourcePartition: failedMessage.originalMessage.partition,
-                  __sourceOffset: failedMessage.originalMessage.offset,
-                  __sourceTimestamp: failedMessage.originalMessage.timestamp
-                },
-                errorMessage: error instanceof Error ? error.message : String(error),
-                errorType: error instanceof Error ? error.constructor.name : "Unknown",
-                failedAt: /* @__PURE__ */ new Date(),
-                source: "transform"
-              };
-              cliLog({
-                action: "DeadLetter",
-                message: `Sending failed message to DLQ ${dlqTopicName}: ${error instanceof Error ? error.message : String(error)}`,
-                message_type: "Error"
-              });
-              try {
-                await producer.send({
-                  topic: dlqTopicName,
-                  messages: [{ value: JSON.stringify(deadLetterRecord) }]
-                });
-                logger2.log(`Sent failed message to DLQ ${dlqTopicName}`);
-                messagesHandledByDLQ++;
-              } catch (dlqError) {
-                const errorMsg = `Failed to send message to DLQ: ${dlqError}`;
-                logger2.error(errorMsg);
-                dlqErrors.push(errorMsg);
-              }
-            } else if (!dlqTopic) {
-              messagesWithoutDLQ++;
-              logger2.warn(
-                `Cannot send to DLQ: no DLQ configured for message (batch has mixed DLQ configurations)`
-              );
-            } else {
-              messagesWithoutDLQ++;
-              logger2.warn(
-                `Cannot send to DLQ: original message value not available`
-              );
-            }
-          }
-          const allMessagesHandled = messagesHandledByDLQ === currentMessages.length && messagesWithoutDLQ === 0 && dlqErrors.length === 0;
-          if (allMessagesHandled) {
-            logger2.log(
-              `All ${messagesHandledByDLQ} failed message(s) sent to DLQ, not throwing original error`
-            );
-            return;
-          }
-          if (messagesWithoutDLQ > 0) {
-            logger2.error(
-              `Cannot handle batch failure: ${messagesWithoutDLQ} message(s) have no DLQ configured`
-            );
-          }
-          if (dlqErrors.length > 0) {
-            logger2.error(
-              `Some messages failed to send to DLQ: ${dlqErrors.join(", ")}`
-            );
-          }
-          if (messagesHandledByDLQ > 0) {
-            logger2.warn(
-              `Partial DLQ success: ${messagesHandledByDLQ}/${currentMessages.length} message(s) sent to DLQ, but throwing due to incomplete batch handling`
-            );
-          }
-          throw error;
-        }
-        logger2.warn(
-          `Send ${currentMessages.length} messages failed (attempt ${attempts}/${maxRetries}), retrying: ${error}`
-        );
-        await new Promise((resolve2) => setTimeout(resolve2, 100 * attempts));
-      }
-    }
-  }
-};
+var CONSUMER_MAX_BATCH_SIZE = 1e3;
 var MAX_STREAMING_CONCURRENCY = process3.env.MAX_STREAMING_CONCURRENCY ? parseInt(process3.env.MAX_STREAMING_CONCURRENCY, 10) : DEFAULT_MAX_STREAMING_CONCURRENCY;
 var metricsLog = (log) => {
   const req = http3.request({
@@ -2098,57 +1965,95 @@ var handleMessage = async (logger2, streamingFunctionWithConfigList, message, pr
   }
   return void 0;
 };
-var sendMessages = async (logger2, metrics, targetTopic, producer, messages) => {
-  try {
-    let chunk = [];
-    let chunkSize = 0;
-    const maxMessageSize = targetTopic.max_message_bytes || 1024 * 1024;
-    for (const message of messages) {
-      const messageSize = Buffer2.byteLength(message.value, "utf8") + KAFKAJS_BYTE_MESSAGE_OVERHEAD;
-      if (chunkSize + messageSize > maxMessageSize) {
-        logger2.log(
-          `Sending ${chunkSize} bytes of a transformed record batch to ${targetTopic.name}`
-        );
-        await sendChunkWithRetry(
-          logger2,
-          targetTopic,
-          producer,
-          chunk,
-          maxMessageSize
-        );
-        logger2.log(
-          `Sent ${chunk.length} transformed records to ${targetTopic.name}`
-        );
-        chunk = [message];
-        chunkSize = messageSize;
-      } else {
-        chunk.push(message);
-        metrics.bytes += Buffer2.byteLength(message.value, "utf8");
-        chunkSize += messageSize;
+var handleDLQForFailedMessages = async (logger2, producer, messages, error) => {
+  let messagesHandledByDLQ = 0;
+  let messagesWithoutDLQ = 0;
+  let dlqErrors = 0;
+  for (const msg of messages) {
+    if (msg.dlq && msg.originalValue) {
+      const deadLetterRecord = {
+        originalRecord: {
+          ...msg.originalValue,
+          // Include original Kafka message metadata
+          __sourcePartition: msg.originalMessage.partition,
+          __sourceOffset: msg.originalMessage.offset,
+          __sourceTimestamp: msg.originalMessage.timestamp
+        },
+        errorMessage: error instanceof Error ? error.message : String(error),
+        errorType: error instanceof Error ? error.constructor.name : "Unknown",
+        failedAt: /* @__PURE__ */ new Date(),
+        source: "transform"
+      };
+      cliLog({
+        action: "DeadLetter",
+        message: `Sending failed message to DLQ ${msg.dlq.name}: ${error instanceof Error ? error.message : String(error)}`,
+        message_type: "Error"
+      });
+      try {
+        await producer.send({
+          topic: msg.dlq.name,
+          messages: [{ value: JSON.stringify(deadLetterRecord) }]
+        });
+        logger2.log(`Sent failed message to DLQ ${msg.dlq.name}`);
+        messagesHandledByDLQ++;
+      } catch (dlqError) {
+        logger2.error(`Failed to send to DLQ: ${dlqError}`);
+        dlqErrors++;
       }
+    } else if (!msg.dlq) {
+      messagesWithoutDLQ++;
+      logger2.warn(`Cannot send to DLQ: no DLQ configured for message`);
+    } else {
+      messagesWithoutDLQ++;
+      logger2.warn(`Cannot send to DLQ: original message value not available`);
     }
-    metrics.count_out += chunk.length;
-    if (chunk.length > 0) {
-      logger2.log(
-        `Sending ${chunkSize} bytes of a transformed record batch to ${targetTopic.name}`
-      );
-      await sendChunkWithRetry(
-        logger2,
-        targetTopic,
-        producer,
-        chunk,
-        maxMessageSize
-      );
-      logger2.log(
-        `Sent final ${chunk.length} transformed data to ${targetTopic.name}`
+  }
+  const allMessagesHandled = messagesHandledByDLQ === messages.length && messagesWithoutDLQ === 0 && dlqErrors === 0;
+  if (allMessagesHandled) {
+    logger2.log(
+      `All ${messagesHandledByDLQ} failed message(s) sent to DLQ, suppressing original error`
+    );
+  } else if (messagesHandledByDLQ > 0) {
+    logger2.warn(
+      `Partial DLQ success: ${messagesHandledByDLQ}/${messages.length} message(s) sent to DLQ`
+    );
+    if (messagesWithoutDLQ > 0) {
+      logger2.error(
+        `Cannot handle batch failure: ${messagesWithoutDLQ} message(s) have no DLQ configured or missing original value`
       );
     }
+    if (dlqErrors > 0) {
+      logger2.error(`${dlqErrors} message(s) failed to send to DLQ`);
+    }
+  }
+  return allMessagesHandled;
+};
+var sendMessages = async (logger2, metrics, targetTopic, producer, messages) => {
+  if (messages.length === 0) return;
+  try {
+    await producer.send({
+      topic: targetTopic.name,
+      messages
+    });
+    for (const msg of messages) {
+      metrics.bytes += Buffer2.byteLength(msg.value, "utf8");
+    }
+    metrics.count_out += messages.length;
+    logger2.log(`Sent ${messages.length} messages to ${targetTopic.name}`);
   } catch (e) {
     logger2.error(`Failed to send transformed data`);
     if (e instanceof Error) {
       logError(logger2, e);
     }
-    throw e;
+    const allHandledByDLQ = await handleDLQForFailedMessages(
+      logger2,
+      producer,
+      messages,
+      e
+    );
+    if (!allHandledByDLQ) {
+      throw e;
+    }
   }
 };
 var sendMessageMetrics = (logger2, metrics) => {
@@ -2388,18 +2293,13 @@ var runStreamingFunctions = async (args) => {
           autoCommit: true,
           autoCommitInterval: AUTO_COMMIT_INTERVAL_MS,
           fromBeginning: true
-        }
-      });
-      const producer = kafka.producer({
-        kafkaJS: {
-          idempotent: true,
-          acks: ACKs,
-          retry: {
-            retries: MAX_RETRIES_PRODUCER,
-            maxRetryTime: MAX_RETRY_TIME_MS
-          }
-        }
+        },
+        "js.consumer.max.batch.size": CONSUMER_MAX_BATCH_SIZE
       });
+      const maxMessageBytes = args.targetTopic?.max_message_bytes || 1024 * 1024;
+      const producer = kafka.producer(
+        createProducerConfig(maxMessageBytes)
+      );
       try {
         logger2.log("Starting producer...");
         await startProducer(logger2, producer);