@514labs/moose-lib 0.6.276-ci-6-g278c5539 → 0.6.276-ci-1-gfe86cd2c

package/dist/index.mjs

@@ -18,6 +18,8 @@ var __export = (target, all) => {
 var commons_exports = {};
 __export(commons_exports, {
   ACKs: () => ACKs,
+  CODES: () => CODES,
+  KafkaConsumer: () => KafkaConsumer,
   MAX_RETRIES: () => MAX_RETRIES,
   MAX_RETRIES_PRODUCER: () => MAX_RETRIES_PRODUCER,
   MAX_RETRY_TIME_MS: () => MAX_RETRY_TIME_MS,
@@ -26,6 +28,7 @@ __export(commons_exports, {
   antiCachePath: () => antiCachePath,
   cliLog: () => cliLog,
   compilerLog: () => compilerLog,
+  createNativeKafkaConsumer: () => createNativeKafkaConsumer,
   createProducerConfig: () => createProducerConfig,
   getClickhouseClient: () => getClickhouseClient,
   getFileName: () => getFileName,
@@ -36,7 +39,11 @@ __export(commons_exports, {
 });
 import http from "http";
 import { createClient } from "@clickhouse/client";
-import { KafkaJS } from "@confluentinc/kafka-javascript";
+import {
+  KafkaConsumer,
+  KafkaJS,
+  CODES
+} from "@confluentinc/kafka-javascript";
 function isTruthy(value) {
   if (!value) return false;
   switch (value.trim().toLowerCase()) {
@@ -74,7 +81,7 @@ async function getKafkaProducer(cfg, logger, maxMessageBytes) {
   await producer.connect();
   return producer;
 }
-var Kafka, compilerLog, antiCachePath, getFileName, getClickhouseClient, cliLog, MAX_RETRIES, MAX_RETRY_TIME_MS, RETRY_INITIAL_TIME_MS, MAX_RETRIES_PRODUCER, RETRY_FACTOR_PRODUCER, ACKs, parseBrokerString, logError, buildSaslConfig, getKafkaClient;
+var Kafka, compilerLog, antiCachePath, getFileName, getClickhouseClient, cliLog, MAX_RETRIES, MAX_RETRY_TIME_MS, RETRY_INITIAL_TIME_MS, MAX_RETRIES_PRODUCER, RETRY_FACTOR_PRODUCER, ACKs, parseBrokerString, logError, buildSaslConfig, getKafkaClient, buildNativeSaslConfig, createNativeKafkaConsumer;
 var init_commons = __esm({
   "src/commons.ts"() {
     "use strict";
@@ -178,6 +185,59 @@ var init_commons = __esm({
         }
       });
     };
+    buildNativeSaslConfig = (logger, cfg) => {
+      if (!cfg.saslMechanism || !cfg.saslUsername || !cfg.saslPassword) {
+        return {};
+      }
+      const mechanism = cfg.saslMechanism.toUpperCase();
+      const validMechanisms = ["PLAIN", "SCRAM-SHA-256", "SCRAM-SHA-512"];
+      if (!validMechanisms.includes(mechanism)) {
+        logger.warn(`Unsupported SASL mechanism: ${cfg.saslMechanism}`);
+        return {};
+      }
+      return {
+        "sasl.mechanisms": mechanism,
+        "sasl.username": cfg.saslUsername,
+        "sasl.password": cfg.saslPassword
+      };
+    };
+    createNativeKafkaConsumer = (cfg, logger, rebalanceCb) => {
+      const brokers = parseBrokerString(cfg.broker || "");
+      if (brokers.length === 0) {
+        throw new Error(`No valid broker addresses found in: "${cfg.broker}"`);
+      }
+      logger.log(
+        `Creating native KafkaConsumer with brokers: ${brokers.join(", ")}`
+      );
+      logger.log(`Security protocol: ${cfg.securityProtocol || "plaintext"}`);
+      logger.log(`Client ID: ${cfg.clientId}`);
+      logger.log(`Group ID: ${cfg.groupId}`);
+      const saslConfig = buildNativeSaslConfig(logger, cfg);
+      const consumerConfig = {
+        // Connection
+        "bootstrap.servers": brokers.join(","),
+        "client.id": cfg.clientId,
+        // Group management
+        "group.id": cfg.groupId,
+        "session.timeout.ms": cfg.sessionTimeoutMs ?? 3e4,
+        "heartbeat.interval.ms": cfg.heartbeatIntervalMs ?? 3e3,
+        "max.poll.interval.ms": cfg.maxPollIntervalMs ?? 3e5,
+        // Offset management
+        "enable.auto.commit": cfg.autoCommit ?? true,
+        "auto.commit.interval.ms": cfg.autoCommitIntervalMs ?? 5e3,
+        // Security
+        ...cfg.securityProtocol === "SASL_SSL" && {
+          "security.protocol": "sasl_ssl"
+        },
+        ...saslConfig,
+        // Rebalance callback
+        ...rebalanceCb && { rebalance_cb: rebalanceCb }
+      };
+      const topicConfig = {
+        "auto.offset.reset": cfg.autoOffsetReset ?? "earliest"
+      };
+      return new KafkaConsumer(consumerConfig, topicConfig);
+    };
   }
 });
 
@@ -3334,6 +3394,7 @@ export {
   ACKs,
   Api,
   ApiHelpers,
+  CODES,
   CSV_DELIMITERS,
   ClickHouseEngines,
   ConsumptionApi,
@@ -3345,6 +3406,7 @@ export {
   ETLPipeline,
   IngestApi,
   IngestPipeline,
+  KafkaConsumer,
   LifeCycle,
   MAX_RETRIES,
   MAX_RETRIES_PRODUCER,
@@ -3372,6 +3434,7 @@ export {
   createClickhouseParameter,
   createConsumptionApi,
   createMaterializedView,
+  createNativeKafkaConsumer,
   createProducerConfig,
   dropView,
   expressMiddleware,