@514labs/moose-lib 0.6.276-ci-6-g278c5539 → 0.6.276-ci-1-gfe86cd2c

package/dist/index.d.mts

@@ -2,7 +2,8 @@ export { C as ClickHouseByteSize, q as ClickHouseCodec, j as ClickHouseDecimal,
 import { K as ApiUtil, a4 as MooseClient } from './index-BtkwFbT9.mjs';
 export { A as Aggregated, h as Api, i as ApiConfig, ad as ApiHelpers, a5 as Blocks, a6 as ClickHouseEngines, C as ConsumptionApi, ae as ConsumptionHelpers, N as ConsumptionUtil, e as DeadLetter, D as DeadLetterModel, f as DeadLetterQueue, l as ETLPipeline, m as ETLPipelineConfig, E as EgressConfig, F as FrameworkApp, Q as IdentifierBrandedString, I as IngestApi, g as IngestConfig, j as IngestPipeline, L as LifeCycle, M as MaterializedView, R as NonIdentifierBrandedString, a as OlapConfig, O as OlapTable, aa as QueryClient, X as RawValue, b as S3QueueTableSettings, S as SimpleAggregated, Z as Sql, k as SqlResource, c as Stream, d as StreamConfig, T as Task, U as Value, V as View, n as WebApp, o as WebAppConfig, p as WebAppHandler, W as Workflow, ab as WorkflowClient, a2 as createClickhouseParameter, a8 as createMaterializedView, a7 as dropView, x as getApi, w as getApis, v as getIngestApi, u as getIngestApis, z as getSqlResource, y as getSqlResources, t as getStream, s as getStreams, r as getTable, q as getTables, ac as getTemporalClient, a1 as getValueFromParameter, J as getWebApp, H as getWebApps, G as getWorkflow, B as getWorkflows, af as joinQueries, a3 as mapToClickHouseType, a9 as populateTable, P as quoteIdentifier, Y as sql, $ as toQuery, a0 as toQueryPreview, _ as toStaticQuery } from './index-BtkwFbT9.mjs';
 import * as _clickhouse_client from '@clickhouse/client';
-import { KafkaJS } from '@confluentinc/kafka-javascript';
+import { KafkaJS, LibrdKafkaError, TopicPartition, KafkaConsumer } from '@confluentinc/kafka-javascript';
+export { CODES, ConsumerGlobalConfig, KafkaConsumer, LibrdKafkaError, Message as NativeKafkaMessage, TopicPartition, TopicPartitionOffset } from '@confluentinc/kafka-javascript';
 import http from 'http';
 import { IsTuple } from 'typia/lib/typings/IsTuple';
 import { Readable } from 'node:stream';
@@ -15,6 +16,7 @@ import 'jose';
 declare const Kafka: typeof KafkaJS.Kafka;
 type Kafka = KafkaJS.Kafka;
 type Producer = KafkaJS.Producer;
+
 declare const compilerLog: (message: string) => void;
 declare const antiCachePath: (path: string) => string;
 declare const getFileName: (filePath: string) => string;
@@ -95,6 +97,29 @@ declare const logError: (logger: Logger, e: Error) => void;
  * Use this to construct producers/consumers with custom options.
  */
 declare const getKafkaClient: (cfg: KafkaClientConfig, logger: Logger) => Promise<Kafka>;
+/**
+ * Configuration for native KafkaConsumer
+ */
+interface NativeConsumerConfig extends KafkaClientConfig {
+    groupId: string;
+    sessionTimeoutMs?: number;
+    heartbeatIntervalMs?: number;
+    maxPollIntervalMs?: number;
+    autoCommit?: boolean;
+    autoCommitIntervalMs?: number;
+    autoOffsetReset?: "smallest" | "earliest" | "largest" | "latest" | "error";
+    maxBatchSize?: number;
+}
+/**
+ * Creates a native KafkaConsumer instance (using librdkafka directly).
+ * This provides lower-level control and potentially better performance than the KafkaJS wrapper.
+ *
+ * @param cfg - Consumer configuration
+ * @param logger - Logger instance
+ * @param rebalanceCb - Optional callback for rebalance events
+ * @returns Configured but not yet connected KafkaConsumer
+ */
+declare const createNativeKafkaConsumer: (cfg: NativeConsumerConfig, logger: Logger, rebalanceCb?: (err: LibrdKafkaError, assignments: TopicPartition[]) => void) => KafkaConsumer;
 
 /**
  * @module secrets
@@ -554,4 +579,4 @@ type DataModelConfig<T> = Partial<{
     parallelism?: number;
 }>;
 
-export { ACKs, ApiUtil, type CSVParsingConfig, CSV_DELIMITERS, type CliLogData, DEFAULT_CSV_CONFIG, DEFAULT_JSON_CONFIG, type DataModelConfig, DataSource, type DataSourceConfig, type ExpressRequestWithMoose, type ExtractionResult, type JSONParsingConfig, type KafkaClientConfig, type Logger, MAX_RETRIES, MAX_RETRIES_PRODUCER, MAX_RETRY_TIME_MS, MOOSE_RUNTIME_ENV_PREFIX, MooseCache, MooseClient, type Producer, RETRY_FACTOR_PRODUCER, RETRY_INITIAL_TIME_MS, type StripDateIntersection, type TaskConfig, type TaskDefinition, type TaskFunction, antiCachePath, cliLog, compilerLog, createApi, createConsumptionApi, createProducerConfig, expressMiddleware, getClickhouseClient, getFileName, getKafkaClient, getKafkaProducer, getMooseClients, getMooseUtils, isValidCSVDelimiter, logError, mapTstoJs, mooseEnvSecrets, mooseRuntimeEnv, parseCSV, parseJSON, parseJSONWithDates };
+export { ACKs, ApiUtil, type CSVParsingConfig, CSV_DELIMITERS, type CliLogData, DEFAULT_CSV_CONFIG, DEFAULT_JSON_CONFIG, type DataModelConfig, DataSource, type DataSourceConfig, type ExpressRequestWithMoose, type ExtractionResult, type JSONParsingConfig, type KafkaClientConfig, type Logger, MAX_RETRIES, MAX_RETRIES_PRODUCER, MAX_RETRY_TIME_MS, MOOSE_RUNTIME_ENV_PREFIX, MooseCache, MooseClient, type NativeConsumerConfig, type Producer, RETRY_FACTOR_PRODUCER, RETRY_INITIAL_TIME_MS, type StripDateIntersection, type TaskConfig, type TaskDefinition, type TaskFunction, antiCachePath, cliLog, compilerLog, createApi, createConsumptionApi, createNativeKafkaConsumer, createProducerConfig, expressMiddleware, getClickhouseClient, getFileName, getKafkaClient, getKafkaProducer, getMooseClients, getMooseUtils, isValidCSVDelimiter, logError, mapTstoJs, mooseEnvSecrets, mooseRuntimeEnv, parseCSV, parseJSON, parseJSONWithDates };

package/dist/index.d.ts

@@ -2,7 +2,8 @@ export { C as ClickHouseByteSize, q as ClickHouseCodec, j as ClickHouseDecimal,
 import { K as ApiUtil, a4 as MooseClient } from './index-BtkwFbT9.js';
 export { A as Aggregated, h as Api, i as ApiConfig, ad as ApiHelpers, a5 as Blocks, a6 as ClickHouseEngines, C as ConsumptionApi, ae as ConsumptionHelpers, N as ConsumptionUtil, e as DeadLetter, D as DeadLetterModel, f as DeadLetterQueue, l as ETLPipeline, m as ETLPipelineConfig, E as EgressConfig, F as FrameworkApp, Q as IdentifierBrandedString, I as IngestApi, g as IngestConfig, j as IngestPipeline, L as LifeCycle, M as MaterializedView, R as NonIdentifierBrandedString, a as OlapConfig, O as OlapTable, aa as QueryClient, X as RawValue, b as S3QueueTableSettings, S as SimpleAggregated, Z as Sql, k as SqlResource, c as Stream, d as StreamConfig, T as Task, U as Value, V as View, n as WebApp, o as WebAppConfig, p as WebAppHandler, W as Workflow, ab as WorkflowClient, a2 as createClickhouseParameter, a8 as createMaterializedView, a7 as dropView, x as getApi, w as getApis, v as getIngestApi, u as getIngestApis, z as getSqlResource, y as getSqlResources, t as getStream, s as getStreams, r as getTable, q as getTables, ac as getTemporalClient, a1 as getValueFromParameter, J as getWebApp, H as getWebApps, G as getWorkflow, B as getWorkflows, af as joinQueries, a3 as mapToClickHouseType, a9 as populateTable, P as quoteIdentifier, Y as sql, $ as toQuery, a0 as toQueryPreview, _ as toStaticQuery } from './index-BtkwFbT9.js';
 import * as _clickhouse_client from '@clickhouse/client';
-import { KafkaJS } from '@confluentinc/kafka-javascript';
+import { KafkaJS, LibrdKafkaError, TopicPartition, KafkaConsumer } from '@confluentinc/kafka-javascript';
+export { CODES, ConsumerGlobalConfig, KafkaConsumer, LibrdKafkaError, Message as NativeKafkaMessage, TopicPartition, TopicPartitionOffset } from '@confluentinc/kafka-javascript';
 import http from 'http';
 import { IsTuple } from 'typia/lib/typings/IsTuple';
 import { Readable } from 'node:stream';
@@ -15,6 +16,7 @@ import 'jose';
 declare const Kafka: typeof KafkaJS.Kafka;
 type Kafka = KafkaJS.Kafka;
 type Producer = KafkaJS.Producer;
+
 declare const compilerLog: (message: string) => void;
 declare const antiCachePath: (path: string) => string;
 declare const getFileName: (filePath: string) => string;
@@ -95,6 +97,29 @@ declare const logError: (logger: Logger, e: Error) => void;
  * Use this to construct producers/consumers with custom options.
  */
 declare const getKafkaClient: (cfg: KafkaClientConfig, logger: Logger) => Promise<Kafka>;
+/**
+ * Configuration for native KafkaConsumer
+ */
+interface NativeConsumerConfig extends KafkaClientConfig {
+    groupId: string;
+    sessionTimeoutMs?: number;
+    heartbeatIntervalMs?: number;
+    maxPollIntervalMs?: number;
+    autoCommit?: boolean;
+    autoCommitIntervalMs?: number;
+    autoOffsetReset?: "smallest" | "earliest" | "largest" | "latest" | "error";
+    maxBatchSize?: number;
+}
+/**
+ * Creates a native KafkaConsumer instance (using librdkafka directly).
+ * This provides lower-level control and potentially better performance than the KafkaJS wrapper.
+ *
+ * @param cfg - Consumer configuration
+ * @param logger - Logger instance
+ * @param rebalanceCb - Optional callback for rebalance events
+ * @returns Configured but not yet connected KafkaConsumer
+ */
+declare const createNativeKafkaConsumer: (cfg: NativeConsumerConfig, logger: Logger, rebalanceCb?: (err: LibrdKafkaError, assignments: TopicPartition[]) => void) => KafkaConsumer;
 
 /**
  * @module secrets
@@ -554,4 +579,4 @@ type DataModelConfig<T> = Partial<{
     parallelism?: number;
 }>;
 
-export { ACKs, ApiUtil, type CSVParsingConfig, CSV_DELIMITERS, type CliLogData, DEFAULT_CSV_CONFIG, DEFAULT_JSON_CONFIG, type DataModelConfig, DataSource, type DataSourceConfig, type ExpressRequestWithMoose, type ExtractionResult, type JSONParsingConfig, type KafkaClientConfig, type Logger, MAX_RETRIES, MAX_RETRIES_PRODUCER, MAX_RETRY_TIME_MS, MOOSE_RUNTIME_ENV_PREFIX, MooseCache, MooseClient, type Producer, RETRY_FACTOR_PRODUCER, RETRY_INITIAL_TIME_MS, type StripDateIntersection, type TaskConfig, type TaskDefinition, type TaskFunction, antiCachePath, cliLog, compilerLog, createApi, createConsumptionApi, createProducerConfig, expressMiddleware, getClickhouseClient, getFileName, getKafkaClient, getKafkaProducer, getMooseClients, getMooseUtils, isValidCSVDelimiter, logError, mapTstoJs, mooseEnvSecrets, mooseRuntimeEnv, parseCSV, parseJSON, parseJSONWithDates };
+export { ACKs, ApiUtil, type CSVParsingConfig, CSV_DELIMITERS, type CliLogData, DEFAULT_CSV_CONFIG, DEFAULT_JSON_CONFIG, type DataModelConfig, DataSource, type DataSourceConfig, type ExpressRequestWithMoose, type ExtractionResult, type JSONParsingConfig, type KafkaClientConfig, type Logger, MAX_RETRIES, MAX_RETRIES_PRODUCER, MAX_RETRY_TIME_MS, MOOSE_RUNTIME_ENV_PREFIX, MooseCache, MooseClient, type NativeConsumerConfig, type Producer, RETRY_FACTOR_PRODUCER, RETRY_INITIAL_TIME_MS, type StripDateIntersection, type TaskConfig, type TaskDefinition, type TaskFunction, antiCachePath, cliLog, compilerLog, createApi, createConsumptionApi, createNativeKafkaConsumer, createProducerConfig, expressMiddleware, getClickhouseClient, getFileName, getKafkaClient, getKafkaProducer, getMooseClients, getMooseUtils, isValidCSVDelimiter, logError, mapTstoJs, mooseEnvSecrets, mooseRuntimeEnv, parseCSV, parseJSON, parseJSONWithDates };

package/dist/index.js

@@ -34,6 +34,8 @@ var __toCommonJS = (mod) => __copyProps(__defProp({}, "__esModule", { value: tru
 var commons_exports = {};
 __export(commons_exports, {
   ACKs: () => ACKs,
+  CODES: () => import_kafka_javascript.CODES,
+  KafkaConsumer: () => import_kafka_javascript.KafkaConsumer,
   MAX_RETRIES: () => MAX_RETRIES,
   MAX_RETRIES_PRODUCER: () => MAX_RETRIES_PRODUCER,
   MAX_RETRY_TIME_MS: () => MAX_RETRY_TIME_MS,
@@ -42,6 +44,7 @@ __export(commons_exports, {
   antiCachePath: () => antiCachePath,
   cliLog: () => cliLog,
   compilerLog: () => compilerLog,
+  createNativeKafkaConsumer: () => createNativeKafkaConsumer,
   createProducerConfig: () => createProducerConfig,
   getClickhouseClient: () => getClickhouseClient,
   getFileName: () => getFileName,
@@ -87,7 +90,7 @@ async function getKafkaProducer(cfg, logger, maxMessageBytes) {
   await producer.connect();
   return producer;
 }
-var import_http, import_client, import_kafka_javascript, Kafka, compilerLog, antiCachePath, getFileName, getClickhouseClient, cliLog, MAX_RETRIES, MAX_RETRY_TIME_MS, RETRY_INITIAL_TIME_MS, MAX_RETRIES_PRODUCER, RETRY_FACTOR_PRODUCER, ACKs, parseBrokerString, logError, buildSaslConfig, getKafkaClient;
+var import_http, import_client, import_kafka_javascript, Kafka, compilerLog, antiCachePath, getFileName, getClickhouseClient, cliLog, MAX_RETRIES, MAX_RETRY_TIME_MS, RETRY_INITIAL_TIME_MS, MAX_RETRIES_PRODUCER, RETRY_FACTOR_PRODUCER, ACKs, parseBrokerString, logError, buildSaslConfig, getKafkaClient, buildNativeSaslConfig, createNativeKafkaConsumer;
 var init_commons = __esm({
   "src/commons.ts"() {
     "use strict";
@@ -194,6 +197,59 @@ var init_commons = __esm({
         }
       });
     };
+    buildNativeSaslConfig = (logger, cfg) => {
+      if (!cfg.saslMechanism || !cfg.saslUsername || !cfg.saslPassword) {
+        return {};
+      }
+      const mechanism = cfg.saslMechanism.toUpperCase();
+      const validMechanisms = ["PLAIN", "SCRAM-SHA-256", "SCRAM-SHA-512"];
+      if (!validMechanisms.includes(mechanism)) {
+        logger.warn(`Unsupported SASL mechanism: ${cfg.saslMechanism}`);
+        return {};
+      }
+      return {
+        "sasl.mechanisms": mechanism,
+        "sasl.username": cfg.saslUsername,
+        "sasl.password": cfg.saslPassword
+      };
+    };
+    createNativeKafkaConsumer = (cfg, logger, rebalanceCb) => {
+      const brokers = parseBrokerString(cfg.broker || "");
+      if (brokers.length === 0) {
+        throw new Error(`No valid broker addresses found in: "${cfg.broker}"`);
+      }
+      logger.log(
+        `Creating native KafkaConsumer with brokers: ${brokers.join(", ")}`
+      );
+      logger.log(`Security protocol: ${cfg.securityProtocol || "plaintext"}`);
+      logger.log(`Client ID: ${cfg.clientId}`);
+      logger.log(`Group ID: ${cfg.groupId}`);
+      const saslConfig = buildNativeSaslConfig(logger, cfg);
+      const consumerConfig = {
+        // Connection
+        "bootstrap.servers": brokers.join(","),
+        "client.id": cfg.clientId,
+        // Group management
+        "group.id": cfg.groupId,
+        "session.timeout.ms": cfg.sessionTimeoutMs ?? 3e4,
+        "heartbeat.interval.ms": cfg.heartbeatIntervalMs ?? 3e3,
+        "max.poll.interval.ms": cfg.maxPollIntervalMs ?? 3e5,
+        // Offset management
+        "enable.auto.commit": cfg.autoCommit ?? true,
+        "auto.commit.interval.ms": cfg.autoCommitIntervalMs ?? 5e3,
+        // Security
+        ...cfg.securityProtocol === "SASL_SSL" && {
+          "security.protocol": "sasl_ssl"
+        },
+        ...saslConfig,
+        // Rebalance callback
+        ...rebalanceCb && { rebalance_cb: rebalanceCb }
+      };
+      const topicConfig = {
+        "auto.offset.reset": cfg.autoOffsetReset ?? "earliest"
+      };
+      return new import_kafka_javascript.KafkaConsumer(consumerConfig, topicConfig);
+    };
   }
 });
 
@@ -387,6 +443,7 @@ __export(index_exports, {
   ACKs: () => ACKs,
   Api: () => Api,
   ApiHelpers: () => ApiHelpers,
+  CODES: () => import_kafka_javascript.CODES,
   CSV_DELIMITERS: () => CSV_DELIMITERS,
   ClickHouseEngines: () => ClickHouseEngines,
   ConsumptionApi: () => ConsumptionApi,
@@ -398,6 +455,7 @@ __export(index_exports, {
   ETLPipeline: () => ETLPipeline,
   IngestApi: () => IngestApi,
   IngestPipeline: () => IngestPipeline,
+  KafkaConsumer: () => import_kafka_javascript.KafkaConsumer,
   LifeCycle: () => LifeCycle,
   MAX_RETRIES: () => MAX_RETRIES,
   MAX_RETRIES_PRODUCER: () => MAX_RETRIES_PRODUCER,
@@ -425,6 +483,7 @@ __export(index_exports, {
   createClickhouseParameter: () => createClickhouseParameter,
   createConsumptionApi: () => createConsumptionApi,
   createMaterializedView: () => createMaterializedView,
+  createNativeKafkaConsumer: () => createNativeKafkaConsumer,
   createProducerConfig: () => createProducerConfig,
   dropView: () => dropView,
   expressMiddleware: () => expressMiddleware,
@@ -3436,6 +3495,7 @@ var DataSource = class {
   ACKs,
   Api,
   ApiHelpers,
+  CODES,
   CSV_DELIMITERS,
   ClickHouseEngines,
   ConsumptionApi,
@@ -3447,6 +3507,7 @@ var DataSource = class {
   ETLPipeline,
   IngestApi,
   IngestPipeline,
+  KafkaConsumer,
   LifeCycle,
   MAX_RETRIES,
   MAX_RETRIES_PRODUCER,
@@ -3474,6 +3535,7 @@ var DataSource = class {
   createClickhouseParameter,
   createConsumptionApi,
   createMaterializedView,
+  createNativeKafkaConsumer,
   createProducerConfig,
   dropView,
   expressMiddleware,