@514labs/moose-lib 0.6.252-ci-1-g901efb04 → 0.6.252-ci-3-g37e54b29

package/dist/moose-runner.mjs

@@ -26,22 +26,6 @@ function isTruthy(value) {
       return false;
   }
 }
-function createProducerConfig(maxMessageBytes) {
-  return {
-    kafkaJS: {
-      idempotent: false,
-      // Not needed for at-least-once delivery
-      acks: ACKs,
-      retry: {
-        retries: MAX_RETRIES_PRODUCER,
-        maxRetryTime: MAX_RETRY_TIME_MS
-      }
-    },
-    "linger.ms": 0,
-    // Send immediately - batching happens at application level
-    ...maxMessageBytes && { "message.max.bytes": maxMessageBytes }
-  };
-}
 var Kafka, compilerLog, getClickhouseClient, cliLog, MAX_RETRIES, MAX_RETRY_TIME_MS, RETRY_INITIAL_TIME_MS, MAX_RETRIES_PRODUCER, ACKs, parseBrokerString, logError, buildSaslConfig, getKafkaClient;
 var init_commons = __esm({
   "src/commons.ts"() {
@@ -1819,6 +1803,156 @@ var MAX_RETRIES_CONSUMER = 150;
 var SESSION_TIMEOUT_CONSUMER = 3e4;
 var HEARTBEAT_INTERVAL_CONSUMER = 3e3;
 var DEFAULT_MAX_STREAMING_CONCURRENCY = 100;
+var KAFKAJS_BYTE_MESSAGE_OVERHEAD = 500;
+var isMessageTooLargeError = (error) => {
+  if (KafkaJS2.isKafkaJSError && error instanceof Error && KafkaJS2.isKafkaJSError(error)) {
+    return error.type === "ERR_MSG_SIZE_TOO_LARGE" || error.code === 10 || error.cause !== void 0 && isMessageTooLargeError(error.cause);
+  }
+  if (error && typeof error === "object") {
+    const err = error;
+    return err.type === "ERR_MSG_SIZE_TOO_LARGE" || err.code === 10 || err.cause !== void 0 && isMessageTooLargeError(err.cause);
+  }
+  return false;
+};
+var splitBatch = (messages, maxChunkSize) => {
+  if (messages.length <= 1) {
+    return [messages];
+  }
+  const chunks = [];
+  let currentChunk = [];
+  let currentSize = 0;
+  for (const message of messages) {
+    const messageSize = Buffer2.byteLength(message.value, "utf8") + KAFKAJS_BYTE_MESSAGE_OVERHEAD;
+    if (currentSize + messageSize > maxChunkSize && currentChunk.length > 0) {
+      chunks.push(currentChunk);
+      currentChunk = [message];
+      currentSize = messageSize;
+    } else {
+      currentChunk.push(message);
+      currentSize += messageSize;
+    }
+  }
+  if (currentChunk.length > 0) {
+    chunks.push(currentChunk);
+  }
+  return chunks;
+};
+var sendChunkWithRetry = async (logger2, targetTopic, producer, messages, currentMaxSize, maxRetries = 3) => {
+  const currentMessages = messages;
+  let attempts = 0;
+  while (attempts < maxRetries) {
+    try {
+      await producer.send({
+        topic: targetTopic.name,
+        messages: currentMessages
+      });
+      logger2.log(
+        `Successfully sent ${currentMessages.length} messages to ${targetTopic.name}`
+      );
+      return;
+    } catch (error) {
+      if (isMessageTooLargeError(error) && currentMessages.length > 1) {
+        logger2.warn(
+          `Got MESSAGE_TOO_LARGE error, splitting batch of ${currentMessages.length} messages and retrying (${maxRetries - attempts} attempts left)`
+        );
+        const newMaxSize = Math.floor(currentMaxSize / 2);
+        const splitChunks = splitBatch(currentMessages, newMaxSize);
+        for (const chunk of splitChunks) {
+          await sendChunkWithRetry(
+            logger2,
+            targetTopic,
+            producer,
+            chunk,
+            newMaxSize,
+            // this error does not count as one failed attempt
+            maxRetries - attempts
+          );
+        }
+        return;
+      } else {
+        attempts++;
+        if (attempts >= maxRetries) {
+          let messagesHandledByDLQ = 0;
+          let messagesWithoutDLQ = 0;
+          const dlqErrors = [];
+          for (const failedMessage of currentMessages) {
+            const dlqTopic = failedMessage.dlq;
+            if (dlqTopic && failedMessage.originalValue) {
+              const dlqTopicName = dlqTopic.name;
+              const deadLetterRecord = {
+                originalRecord: {
+                  ...failedMessage.originalValue,
+                  // Include original Kafka message metadata
+                  __sourcePartition: failedMessage.originalMessage.partition,
+                  __sourceOffset: failedMessage.originalMessage.offset,
+                  __sourceTimestamp: failedMessage.originalMessage.timestamp
+                },
+                errorMessage: error instanceof Error ? error.message : String(error),
+                errorType: error instanceof Error ? error.constructor.name : "Unknown",
+                failedAt: /* @__PURE__ */ new Date(),
+                source: "transform"
+              };
+              cliLog({
+                action: "DeadLetter",
+                message: `Sending failed message to DLQ ${dlqTopicName}: ${error instanceof Error ? error.message : String(error)}`,
+                message_type: "Error"
+              });
+              try {
+                await producer.send({
+                  topic: dlqTopicName,
+                  messages: [{ value: JSON.stringify(deadLetterRecord) }]
+                });
+                logger2.log(`Sent failed message to DLQ ${dlqTopicName}`);
+                messagesHandledByDLQ++;
+              } catch (dlqError) {
+                const errorMsg = `Failed to send message to DLQ: ${dlqError}`;
+                logger2.error(errorMsg);
+                dlqErrors.push(errorMsg);
+              }
+            } else if (!dlqTopic) {
+              messagesWithoutDLQ++;
+              logger2.warn(
+                `Cannot send to DLQ: no DLQ configured for message (batch has mixed DLQ configurations)`
+              );
+            } else {
+              messagesWithoutDLQ++;
+              logger2.warn(
+                `Cannot send to DLQ: original message value not available`
+              );
+            }
+          }
+          const allMessagesHandled = messagesHandledByDLQ === currentMessages.length && messagesWithoutDLQ === 0 && dlqErrors.length === 0;
+          if (allMessagesHandled) {
+            logger2.log(
+              `All ${messagesHandledByDLQ} failed message(s) sent to DLQ, not throwing original error`
+            );
+            return;
+          }
+          if (messagesWithoutDLQ > 0) {
+            logger2.error(
+              `Cannot handle batch failure: ${messagesWithoutDLQ} message(s) have no DLQ configured`
+            );
+          }
+          if (dlqErrors.length > 0) {
+            logger2.error(
+              `Some messages failed to send to DLQ: ${dlqErrors.join(", ")}`
+            );
+          }
+          if (messagesHandledByDLQ > 0) {
+            logger2.warn(
+              `Partial DLQ success: ${messagesHandledByDLQ}/${currentMessages.length} message(s) sent to DLQ, but throwing due to incomplete batch handling`
+            );
+          }
+          throw error;
+        }
+        logger2.warn(
+          `Send ${currentMessages.length} messages failed (attempt ${attempts}/${maxRetries}), retrying: ${error}`
+        );
+        await new Promise((resolve2) => setTimeout(resolve2, 100 * attempts));
+      }
+    }
+  }
+};
 var MAX_STREAMING_CONCURRENCY = process3.env.MAX_STREAMING_CONCURRENCY ? parseInt(process3.env.MAX_STREAMING_CONCURRENCY, 10) : DEFAULT_MAX_STREAMING_CONCURRENCY;
 var metricsLog = (log) => {
   const req = http3.request({
@@ -1964,71 +2098,56 @@ var handleMessage = async (logger2, streamingFunctionWithConfigList, message, pr
   }
   return void 0;
 };
-var handleDLQForFailedMessages = async (logger2, producer, messages, error) => {
-  let messagesHandledByDLQ = 0;
-  let messagesWithoutDLQ = 0;
-  for (const msg of messages) {
-    if (msg.dlq && msg.originalValue) {
-      const deadLetterRecord = {
-        originalRecord: {
-          ...msg.originalValue,
-          // Include original Kafka message metadata
-          __sourcePartition: msg.originalMessage.partition,
-          __sourceOffset: msg.originalMessage.offset,
-          __sourceTimestamp: msg.originalMessage.timestamp
-        },
-        errorMessage: error instanceof Error ? error.message : String(error),
-        errorType: error instanceof Error ? error.constructor.name : "Unknown",
-        failedAt: /* @__PURE__ */ new Date(),
-        source: "transform"
-      };
-      cliLog({
-        action: "DeadLetter",
-        message: `Sending failed message to DLQ ${msg.dlq.name}: ${error instanceof Error ? error.message : String(error)}`,
-        message_type: "Error"
-      });
-      try {
-        await producer.send({
-          topic: msg.dlq.name,
-          messages: [{ value: JSON.stringify(deadLetterRecord) }]
-        });
-        logger2.log(`Sent failed message to DLQ ${msg.dlq.name}`);
-        messagesHandledByDLQ++;
-      } catch (dlqError) {
-        logger2.error(`Failed to send to DLQ: ${dlqError}`);
-      }
-    } else if (!msg.dlq) {
-      messagesWithoutDLQ++;
-      logger2.warn(`Cannot send to DLQ: no DLQ configured for message`);
-    } else {
-      messagesWithoutDLQ++;
-      logger2.warn(`Cannot send to DLQ: original message value not available`);
-    }
-  }
-  if (messagesHandledByDLQ > 0 && messagesWithoutDLQ > 0) {
-    logger2.warn(
-      `Partial DLQ success: ${messagesHandledByDLQ}/${messages.length} message(s) sent to DLQ`
-    );
-  }
-};
 var sendMessages = async (logger2, metrics, targetTopic, producer, messages) => {
-  if (messages.length === 0) return;
-  for (const msg of messages) {
-    metrics.bytes += Buffer2.byteLength(msg.value, "utf8");
-  }
-  metrics.count_out += messages.length;
   try {
-    await producer.send({
-      topic: targetTopic.name,
-      messages
-    });
-    logger2.log(`Sent ${messages.length} messages to ${targetTopic.name}`);
+    let chunk = [];
+    let chunkSize = 0;
+    const maxMessageSize = targetTopic.max_message_bytes || 1024 * 1024;
+    for (const message of messages) {
+      const messageSize = Buffer2.byteLength(message.value, "utf8") + KAFKAJS_BYTE_MESSAGE_OVERHEAD;
+      if (chunkSize + messageSize > maxMessageSize) {
+        logger2.log(
+          `Sending ${chunkSize} bytes of a transformed record batch to ${targetTopic.name}`
+        );
+        await sendChunkWithRetry(
+          logger2,
+          targetTopic,
+          producer,
+          chunk,
+          maxMessageSize
+        );
+        logger2.log(
+          `Sent ${chunk.length} transformed records to ${targetTopic.name}`
+        );
+        chunk = [message];
+        chunkSize = messageSize;
+      } else {
+        chunk.push(message);
+        metrics.bytes += Buffer2.byteLength(message.value, "utf8");
+        chunkSize += messageSize;
+      }
+    }
+    metrics.count_out += chunk.length;
+    if (chunk.length > 0) {
+      logger2.log(
+        `Sending ${chunkSize} bytes of a transformed record batch to ${targetTopic.name}`
+      );
+      await sendChunkWithRetry(
+        logger2,
+        targetTopic,
+        producer,
+        chunk,
+        maxMessageSize
+      );
+      logger2.log(
+        `Sent final ${chunk.length} transformed data to ${targetTopic.name}`
+      );
+    }
   } catch (e) {
     logger2.error(`Failed to send transformed data`);
     if (e instanceof Error) {
       logError(logger2, e);
     }
-    await handleDLQForFailedMessages(logger2, producer, messages, e);
     throw e;
   }
 };
@@ -2271,10 +2390,16 @@ var runStreamingFunctions = async (args) => {
           fromBeginning: true
         }
       });
-      const maxMessageBytes = args.targetTopic?.max_message_bytes || 1024 * 1024;
-      const producer = kafka.producer(
-        createProducerConfig(maxMessageBytes)
-      );
+      const producer = kafka.producer({
+        kafkaJS: {
+          idempotent: true,
+          acks: ACKs,
+          retry: {
+            retries: MAX_RETRIES_PRODUCER,
+            maxRetryTime: MAX_RETRY_TIME_MS
+          }
+        }
+      });
       try {
         logger2.log("Starting producer...");
         await startProducer(logger2, producer);