@514labs/moose-lib 0.6.252-ci-3-gf88b2b20 → 0.6.252-ci-5-g3da4ba34

package/dist/moose-runner.js

@@ -39,22 +39,6 @@ function isTruthy(value) {
       return false;
   }
 }
-function createProducerConfig(maxMessageBytes) {
-  return {
-    kafkaJS: {
-      idempotent: false,
-      // Not needed for at-least-once delivery
-      acks: ACKs,
-      retry: {
-        retries: MAX_RETRIES_PRODUCER,
-        maxRetryTime: MAX_RETRY_TIME_MS
-      }
-    },
-    "linger.ms": 0,
-    // Send immediately - batching happens at application level
-    ...maxMessageBytes && { "message.max.bytes": maxMessageBytes }
-  };
-}
 var import_http, import_client, import_kafka_javascript, Kafka, compilerLog, getClickhouseClient, cliLog, MAX_RETRIES, MAX_RETRY_TIME_MS, RETRY_INITIAL_TIME_MS, MAX_RETRIES_PRODUCER, ACKs, parseBrokerString, logError, buildSaslConfig, getKafkaClient;
 var init_commons = __esm({
   "src/commons.ts"() {
@@ -1338,6 +1322,18 @@ function convertIcebergS3EngineConfig(config) {
     compression: config.compression
   };
 }
+function convertKafkaEngineConfig(config) {
+  if (!("engine" in config) || config.engine !== "Kafka" /* Kafka */) {
+    return void 0;
+  }
+  return {
+    engine: "Kafka",
+    brokerList: config.brokerList,
+    topicList: config.topicList,
+    groupName: config.groupName,
+    format: config.format
+  };
+}
 function convertTableConfigToEngineConfig(config) {
   const engine = extractEngineValue(config);
   const basicConfig = convertBasicEngineConfig(engine, config);
@@ -1363,6 +1359,9 @@ function convertTableConfigToEngineConfig(config) {
   if (engine === "IcebergS3" /* IcebergS3 */) {
     return convertIcebergS3EngineConfig(config);
   }
+  if (engine === "Kafka" /* Kafka */) {
+    return convertKafkaEngineConfig(config);
+  }
   return void 0;
 }
 var toInfraMap = (registry) => {
@@ -1832,6 +1831,156 @@ var MAX_RETRIES_CONSUMER = 150;
 var SESSION_TIMEOUT_CONSUMER = 3e4;
 var HEARTBEAT_INTERVAL_CONSUMER = 3e3;
 var DEFAULT_MAX_STREAMING_CONCURRENCY = 100;
+var KAFKAJS_BYTE_MESSAGE_OVERHEAD = 500;
+var isMessageTooLargeError = (error) => {
+  if (import_kafka_javascript2.KafkaJS.isKafkaJSError && error instanceof Error && import_kafka_javascript2.KafkaJS.isKafkaJSError(error)) {
+    return error.type === "ERR_MSG_SIZE_TOO_LARGE" || error.code === 10 || error.cause !== void 0 && isMessageTooLargeError(error.cause);
+  }
+  if (error && typeof error === "object") {
+    const err = error;
+    return err.type === "ERR_MSG_SIZE_TOO_LARGE" || err.code === 10 || err.cause !== void 0 && isMessageTooLargeError(err.cause);
+  }
+  return false;
+};
+var splitBatch = (messages, maxChunkSize) => {
+  if (messages.length <= 1) {
+    return [messages];
+  }
+  const chunks = [];
+  let currentChunk = [];
+  let currentSize = 0;
+  for (const message of messages) {
+    const messageSize = import_node_buffer.Buffer.byteLength(message.value, "utf8") + KAFKAJS_BYTE_MESSAGE_OVERHEAD;
+    if (currentSize + messageSize > maxChunkSize && currentChunk.length > 0) {
+      chunks.push(currentChunk);
+      currentChunk = [message];
+      currentSize = messageSize;
+    } else {
+      currentChunk.push(message);
+      currentSize += messageSize;
+    }
+  }
+  if (currentChunk.length > 0) {
+    chunks.push(currentChunk);
+  }
+  return chunks;
+};
+var sendChunkWithRetry = async (logger2, targetTopic, producer, messages, currentMaxSize, maxRetries = 3) => {
+  const currentMessages = messages;
+  let attempts = 0;
+  while (attempts < maxRetries) {
+    try {
+      await producer.send({
+        topic: targetTopic.name,
+        messages: currentMessages
+      });
+      logger2.log(
+        `Successfully sent ${currentMessages.length} messages to ${targetTopic.name}`
+      );
+      return;
+    } catch (error) {
+      if (isMessageTooLargeError(error) && currentMessages.length > 1) {
+        logger2.warn(
+          `Got MESSAGE_TOO_LARGE error, splitting batch of ${currentMessages.length} messages and retrying (${maxRetries - attempts} attempts left)`
+        );
+        const newMaxSize = Math.floor(currentMaxSize / 2);
+        const splitChunks = splitBatch(currentMessages, newMaxSize);
+        for (const chunk of splitChunks) {
+          await sendChunkWithRetry(
+            logger2,
+            targetTopic,
+            producer,
+            chunk,
+            newMaxSize,
+            // this error does not count as one failed attempt
+            maxRetries - attempts
+          );
+        }
+        return;
+      } else {
+        attempts++;
+        if (attempts >= maxRetries) {
+          let messagesHandledByDLQ = 0;
+          let messagesWithoutDLQ = 0;
+          const dlqErrors = [];
+          for (const failedMessage of currentMessages) {
+            const dlqTopic = failedMessage.dlq;
+            if (dlqTopic && failedMessage.originalValue) {
+              const dlqTopicName = dlqTopic.name;
+              const deadLetterRecord = {
+                originalRecord: {
+                  ...failedMessage.originalValue,
+                  // Include original Kafka message metadata
+                  __sourcePartition: failedMessage.originalMessage.partition,
+                  __sourceOffset: failedMessage.originalMessage.offset,
+                  __sourceTimestamp: failedMessage.originalMessage.timestamp
+                },
+                errorMessage: error instanceof Error ? error.message : String(error),
+                errorType: error instanceof Error ? error.constructor.name : "Unknown",
+                failedAt: /* @__PURE__ */ new Date(),
+                source: "transform"
+              };
+              cliLog({
+                action: "DeadLetter",
+                message: `Sending failed message to DLQ ${dlqTopicName}: ${error instanceof Error ? error.message : String(error)}`,
+                message_type: "Error"
+              });
+              try {
+                await producer.send({
+                  topic: dlqTopicName,
+                  messages: [{ value: JSON.stringify(deadLetterRecord) }]
+                });
+                logger2.log(`Sent failed message to DLQ ${dlqTopicName}`);
+                messagesHandledByDLQ++;
+              } catch (dlqError) {
+                const errorMsg = `Failed to send message to DLQ: ${dlqError}`;
+                logger2.error(errorMsg);
+                dlqErrors.push(errorMsg);
+              }
+            } else if (!dlqTopic) {
+              messagesWithoutDLQ++;
+              logger2.warn(
+                `Cannot send to DLQ: no DLQ configured for message (batch has mixed DLQ configurations)`
+              );
+            } else {
+              messagesWithoutDLQ++;
+              logger2.warn(
+                `Cannot send to DLQ: original message value not available`
+              );
+            }
+          }
+          const allMessagesHandled = messagesHandledByDLQ === currentMessages.length && messagesWithoutDLQ === 0 && dlqErrors.length === 0;
+          if (allMessagesHandled) {
+            logger2.log(
+              `All ${messagesHandledByDLQ} failed message(s) sent to DLQ, not throwing original error`
+            );
+            return;
+          }
+          if (messagesWithoutDLQ > 0) {
+            logger2.error(
+              `Cannot handle batch failure: ${messagesWithoutDLQ} message(s) have no DLQ configured`
+            );
+          }
+          if (dlqErrors.length > 0) {
+            logger2.error(
+              `Some messages failed to send to DLQ: ${dlqErrors.join(", ")}`
+            );
+          }
+          if (messagesHandledByDLQ > 0) {
+            logger2.warn(
+              `Partial DLQ success: ${messagesHandledByDLQ}/${currentMessages.length} message(s) sent to DLQ, but throwing due to incomplete batch handling`
+            );
+          }
+          throw error;
+        }
+        logger2.warn(
+          `Send ${currentMessages.length} messages failed (attempt ${attempts}/${maxRetries}), retrying: ${error}`
+        );
+        await new Promise((resolve2) => setTimeout(resolve2, 100 * attempts));
+      }
+    }
+  }
+};
 var MAX_STREAMING_CONCURRENCY = process3.env.MAX_STREAMING_CONCURRENCY ? parseInt(process3.env.MAX_STREAMING_CONCURRENCY, 10) : DEFAULT_MAX_STREAMING_CONCURRENCY;
 var metricsLog = (log) => {
   const req = http3.request({
@@ -1977,95 +2126,57 @@ var handleMessage = async (logger2, streamingFunctionWithConfigList, message, pr
   }
   return void 0;
 };
-var handleDLQForFailedMessages = async (logger2, producer, messages, error) => {
-  let messagesHandledByDLQ = 0;
-  let messagesWithoutDLQ = 0;
-  let dlqErrors = 0;
-  for (const msg of messages) {
-    if (msg.dlq && msg.originalValue) {
-      const deadLetterRecord = {
-        originalRecord: {
-          ...msg.originalValue,
-          // Include original Kafka message metadata
-          __sourcePartition: msg.originalMessage.partition,
-          __sourceOffset: msg.originalMessage.offset,
-          __sourceTimestamp: msg.originalMessage.timestamp
-        },
-        errorMessage: error instanceof Error ? error.message : String(error),
-        errorType: error instanceof Error ? error.constructor.name : "Unknown",
-        failedAt: /* @__PURE__ */ new Date(),
-        source: "transform"
-      };
-      cliLog({
-        action: "DeadLetter",
-        message: `Sending failed message to DLQ ${msg.dlq.name}: ${error instanceof Error ? error.message : String(error)}`,
-        message_type: "Error"
-      });
-      try {
-        await producer.send({
-          topic: msg.dlq.name,
-          messages: [{ value: JSON.stringify(deadLetterRecord) }]
-        });
-        logger2.log(`Sent failed message to DLQ ${msg.dlq.name}`);
-        messagesHandledByDLQ++;
-      } catch (dlqError) {
-        logger2.error(`Failed to send to DLQ: ${dlqError}`);
-        dlqErrors++;
+var sendMessages = async (logger2, metrics, targetTopic, producer, messages) => {
+  try {
+    let chunk = [];
+    let chunkSize = 0;
+    const maxMessageSize = targetTopic.max_message_bytes || 1024 * 1024;
+    for (const message of messages) {
+      const messageSize = import_node_buffer.Buffer.byteLength(message.value, "utf8") + KAFKAJS_BYTE_MESSAGE_OVERHEAD;
+      if (chunkSize + messageSize > maxMessageSize) {
+        logger2.log(
+          `Sending ${chunkSize} bytes of a transformed record batch to ${targetTopic.name}`
+        );
+        await sendChunkWithRetry(
+          logger2,
+          targetTopic,
+          producer,
+          chunk,
+          maxMessageSize
+        );
+        logger2.log(
+          `Sent ${chunk.length} transformed records to ${targetTopic.name}`
+        );
+        chunk = [message];
+        chunkSize = messageSize;
+      } else {
+        chunk.push(message);
+        metrics.bytes += import_node_buffer.Buffer.byteLength(message.value, "utf8");
+        chunkSize += messageSize;
       }
-    } else if (!msg.dlq) {
-      messagesWithoutDLQ++;
-      logger2.warn(`Cannot send to DLQ: no DLQ configured for message`);
-    } else {
-      messagesWithoutDLQ++;
-      logger2.warn(`Cannot send to DLQ: original message value not available`);
     }
-  }
-  const allMessagesHandled = messagesHandledByDLQ === messages.length && messagesWithoutDLQ === 0 && dlqErrors === 0;
-  if (allMessagesHandled) {
-    logger2.log(
-      `All ${messagesHandledByDLQ} failed message(s) sent to DLQ, suppressing original error`
-    );
-  } else if (messagesHandledByDLQ > 0) {
-    logger2.warn(
-      `Partial DLQ success: ${messagesHandledByDLQ}/${messages.length} message(s) sent to DLQ`
-    );
-    if (messagesWithoutDLQ > 0) {
-      logger2.error(
-        `Cannot handle batch failure: ${messagesWithoutDLQ} message(s) have no DLQ configured or missing original value`
+    metrics.count_out += chunk.length;
+    if (chunk.length > 0) {
+      logger2.log(
+        `Sending ${chunkSize} bytes of a transformed record batch to ${targetTopic.name}`
+      );
+      await sendChunkWithRetry(
+        logger2,
+        targetTopic,
+        producer,
+        chunk,
+        maxMessageSize
+      );
+      logger2.log(
+        `Sent final ${chunk.length} transformed data to ${targetTopic.name}`
       );
     }
-    if (dlqErrors > 0) {
-      logger2.error(`${dlqErrors} message(s) failed to send to DLQ`);
-    }
-  }
-  return allMessagesHandled;
-};
-var sendMessages = async (logger2, metrics, targetTopic, producer, messages) => {
-  if (messages.length === 0) return;
-  try {
-    await producer.send({
-      topic: targetTopic.name,
-      messages
-    });
-    for (const msg of messages) {
-      metrics.bytes += import_node_buffer.Buffer.byteLength(msg.value, "utf8");
-    }
-    metrics.count_out += messages.length;
-    logger2.log(`Sent ${messages.length} messages to ${targetTopic.name}`);
   } catch (e) {
     logger2.error(`Failed to send transformed data`);
     if (e instanceof Error) {
       logError(logger2, e);
     }
-    const allHandledByDLQ = await handleDLQForFailedMessages(
-      logger2,
-      producer,
-      messages,
-      e
-    );
-    if (!allHandledByDLQ) {
-      throw e;
-    }
+    throw e;
   }
 };
 var sendMessageMetrics = (logger2, metrics) => {
@@ -2307,10 +2418,16 @@ var runStreamingFunctions = async (args) => {
           fromBeginning: true
         }
       });
-      const maxMessageBytes = args.targetTopic?.max_message_bytes || 1024 * 1024;
-      const producer = kafka.producer(
-        createProducerConfig(maxMessageBytes)
-      );
+      const producer = kafka.producer({
+        kafkaJS: {
+          idempotent: true,
+          acks: ACKs,
+          retry: {
+            retries: MAX_RETRIES_PRODUCER,
+            maxRetryTime: MAX_RETRY_TIME_MS
+          }
+        }
+      });
       try {
         logger2.log("Starting producer...");
         await startProducer(logger2, producer);