@514labs/moose-lib 0.6.276-ci-1-gfe86cd2c → 0.6.276-ci-3-g7a7bb18a

package/dist/index.d.mts

@@ -2,8 +2,7 @@ export { C as ClickHouseByteSize, q as ClickHouseCodec, j as ClickHouseDecimal,
 import { K as ApiUtil, a4 as MooseClient } from './index-BtkwFbT9.mjs';
 export { A as Aggregated, h as Api, i as ApiConfig, ad as ApiHelpers, a5 as Blocks, a6 as ClickHouseEngines, C as ConsumptionApi, ae as ConsumptionHelpers, N as ConsumptionUtil, e as DeadLetter, D as DeadLetterModel, f as DeadLetterQueue, l as ETLPipeline, m as ETLPipelineConfig, E as EgressConfig, F as FrameworkApp, Q as IdentifierBrandedString, I as IngestApi, g as IngestConfig, j as IngestPipeline, L as LifeCycle, M as MaterializedView, R as NonIdentifierBrandedString, a as OlapConfig, O as OlapTable, aa as QueryClient, X as RawValue, b as S3QueueTableSettings, S as SimpleAggregated, Z as Sql, k as SqlResource, c as Stream, d as StreamConfig, T as Task, U as Value, V as View, n as WebApp, o as WebAppConfig, p as WebAppHandler, W as Workflow, ab as WorkflowClient, a2 as createClickhouseParameter, a8 as createMaterializedView, a7 as dropView, x as getApi, w as getApis, v as getIngestApi, u as getIngestApis, z as getSqlResource, y as getSqlResources, t as getStream, s as getStreams, r as getTable, q as getTables, ac as getTemporalClient, a1 as getValueFromParameter, J as getWebApp, H as getWebApps, G as getWorkflow, B as getWorkflows, af as joinQueries, a3 as mapToClickHouseType, a9 as populateTable, P as quoteIdentifier, Y as sql, $ as toQuery, a0 as toQueryPreview, _ as toStaticQuery } from './index-BtkwFbT9.mjs';
 import * as _clickhouse_client from '@clickhouse/client';
-import { KafkaJS, LibrdKafkaError, TopicPartition, KafkaConsumer } from '@confluentinc/kafka-javascript';
-export { CODES, ConsumerGlobalConfig, KafkaConsumer, LibrdKafkaError, Message as NativeKafkaMessage, TopicPartition, TopicPartitionOffset } from '@confluentinc/kafka-javascript';
+import { KafkaJS } from '@514labs/kafka-javascript';
 import http from 'http';
 import { IsTuple } from 'typia/lib/typings/IsTuple';
 import { Readable } from 'node:stream';
@@ -16,7 +15,6 @@ import 'jose';
 declare const Kafka: typeof KafkaJS.Kafka;
 type Kafka = KafkaJS.Kafka;
 type Producer = KafkaJS.Producer;
-
 declare const compilerLog: (message: string) => void;
 declare const antiCachePath: (path: string) => string;
 declare const getFileName: (filePath: string) => string;
@@ -97,29 +95,6 @@ declare const logError: (logger: Logger, e: Error) => void;
  * Use this to construct producers/consumers with custom options.
  */
 declare const getKafkaClient: (cfg: KafkaClientConfig, logger: Logger) => Promise<Kafka>;
-/**
- * Configuration for native KafkaConsumer
- */
-interface NativeConsumerConfig extends KafkaClientConfig {
-    groupId: string;
-    sessionTimeoutMs?: number;
-    heartbeatIntervalMs?: number;
-    maxPollIntervalMs?: number;
-    autoCommit?: boolean;
-    autoCommitIntervalMs?: number;
-    autoOffsetReset?: "smallest" | "earliest" | "largest" | "latest" | "error";
-    maxBatchSize?: number;
-}
-/**
- * Creates a native KafkaConsumer instance (using librdkafka directly).
- * This provides lower-level control and potentially better performance than the KafkaJS wrapper.
- *
- * @param cfg - Consumer configuration
- * @param logger - Logger instance
- * @param rebalanceCb - Optional callback for rebalance events
- * @returns Configured but not yet connected KafkaConsumer
- */
-declare const createNativeKafkaConsumer: (cfg: NativeConsumerConfig, logger: Logger, rebalanceCb?: (err: LibrdKafkaError, assignments: TopicPartition[]) => void) => KafkaConsumer;
 
 /**
  * @module secrets
@@ -579,4 +554,4 @@ type DataModelConfig<T> = Partial<{
     parallelism?: number;
 }>;
 
-export { ACKs, ApiUtil, type CSVParsingConfig, CSV_DELIMITERS, type CliLogData, DEFAULT_CSV_CONFIG, DEFAULT_JSON_CONFIG, type DataModelConfig, DataSource, type DataSourceConfig, type ExpressRequestWithMoose, type ExtractionResult, type JSONParsingConfig, type KafkaClientConfig, type Logger, MAX_RETRIES, MAX_RETRIES_PRODUCER, MAX_RETRY_TIME_MS, MOOSE_RUNTIME_ENV_PREFIX, MooseCache, MooseClient, type NativeConsumerConfig, type Producer, RETRY_FACTOR_PRODUCER, RETRY_INITIAL_TIME_MS, type StripDateIntersection, type TaskConfig, type TaskDefinition, type TaskFunction, antiCachePath, cliLog, compilerLog, createApi, createConsumptionApi, createNativeKafkaConsumer, createProducerConfig, expressMiddleware, getClickhouseClient, getFileName, getKafkaClient, getKafkaProducer, getMooseClients, getMooseUtils, isValidCSVDelimiter, logError, mapTstoJs, mooseEnvSecrets, mooseRuntimeEnv, parseCSV, parseJSON, parseJSONWithDates };
+export { ACKs, ApiUtil, type CSVParsingConfig, CSV_DELIMITERS, type CliLogData, DEFAULT_CSV_CONFIG, DEFAULT_JSON_CONFIG, type DataModelConfig, DataSource, type DataSourceConfig, type ExpressRequestWithMoose, type ExtractionResult, type JSONParsingConfig, type KafkaClientConfig, type Logger, MAX_RETRIES, MAX_RETRIES_PRODUCER, MAX_RETRY_TIME_MS, MOOSE_RUNTIME_ENV_PREFIX, MooseCache, MooseClient, type Producer, RETRY_FACTOR_PRODUCER, RETRY_INITIAL_TIME_MS, type StripDateIntersection, type TaskConfig, type TaskDefinition, type TaskFunction, antiCachePath, cliLog, compilerLog, createApi, createConsumptionApi, createProducerConfig, expressMiddleware, getClickhouseClient, getFileName, getKafkaClient, getKafkaProducer, getMooseClients, getMooseUtils, isValidCSVDelimiter, logError, mapTstoJs, mooseEnvSecrets, mooseRuntimeEnv, parseCSV, parseJSON, parseJSONWithDates };

package/dist/index.d.ts

@@ -2,8 +2,7 @@ export { C as ClickHouseByteSize, q as ClickHouseCodec, j as ClickHouseDecimal,
 import { K as ApiUtil, a4 as MooseClient } from './index-BtkwFbT9.js';
 export { A as Aggregated, h as Api, i as ApiConfig, ad as ApiHelpers, a5 as Blocks, a6 as ClickHouseEngines, C as ConsumptionApi, ae as ConsumptionHelpers, N as ConsumptionUtil, e as DeadLetter, D as DeadLetterModel, f as DeadLetterQueue, l as ETLPipeline, m as ETLPipelineConfig, E as EgressConfig, F as FrameworkApp, Q as IdentifierBrandedString, I as IngestApi, g as IngestConfig, j as IngestPipeline, L as LifeCycle, M as MaterializedView, R as NonIdentifierBrandedString, a as OlapConfig, O as OlapTable, aa as QueryClient, X as RawValue, b as S3QueueTableSettings, S as SimpleAggregated, Z as Sql, k as SqlResource, c as Stream, d as StreamConfig, T as Task, U as Value, V as View, n as WebApp, o as WebAppConfig, p as WebAppHandler, W as Workflow, ab as WorkflowClient, a2 as createClickhouseParameter, a8 as createMaterializedView, a7 as dropView, x as getApi, w as getApis, v as getIngestApi, u as getIngestApis, z as getSqlResource, y as getSqlResources, t as getStream, s as getStreams, r as getTable, q as getTables, ac as getTemporalClient, a1 as getValueFromParameter, J as getWebApp, H as getWebApps, G as getWorkflow, B as getWorkflows, af as joinQueries, a3 as mapToClickHouseType, a9 as populateTable, P as quoteIdentifier, Y as sql, $ as toQuery, a0 as toQueryPreview, _ as toStaticQuery } from './index-BtkwFbT9.js';
 import * as _clickhouse_client from '@clickhouse/client';
-import { KafkaJS, LibrdKafkaError, TopicPartition, KafkaConsumer } from '@confluentinc/kafka-javascript';
-export { CODES, ConsumerGlobalConfig, KafkaConsumer, LibrdKafkaError, Message as NativeKafkaMessage, TopicPartition, TopicPartitionOffset } from '@confluentinc/kafka-javascript';
+import { KafkaJS } from '@514labs/kafka-javascript';
 import http from 'http';
 import { IsTuple } from 'typia/lib/typings/IsTuple';
 import { Readable } from 'node:stream';
@@ -16,7 +15,6 @@ import 'jose';
 declare const Kafka: typeof KafkaJS.Kafka;
 type Kafka = KafkaJS.Kafka;
 type Producer = KafkaJS.Producer;
-
 declare const compilerLog: (message: string) => void;
 declare const antiCachePath: (path: string) => string;
 declare const getFileName: (filePath: string) => string;
@@ -97,29 +95,6 @@ declare const logError: (logger: Logger, e: Error) => void;
  * Use this to construct producers/consumers with custom options.
  */
 declare const getKafkaClient: (cfg: KafkaClientConfig, logger: Logger) => Promise<Kafka>;
-/**
- * Configuration for native KafkaConsumer
- */
-interface NativeConsumerConfig extends KafkaClientConfig {
-    groupId: string;
-    sessionTimeoutMs?: number;
-    heartbeatIntervalMs?: number;
-    maxPollIntervalMs?: number;
-    autoCommit?: boolean;
-    autoCommitIntervalMs?: number;
-    autoOffsetReset?: "smallest" | "earliest" | "largest" | "latest" | "error";
-    maxBatchSize?: number;
-}
-/**
- * Creates a native KafkaConsumer instance (using librdkafka directly).
- * This provides lower-level control and potentially better performance than the KafkaJS wrapper.
- *
- * @param cfg - Consumer configuration
- * @param logger - Logger instance
- * @param rebalanceCb - Optional callback for rebalance events
- * @returns Configured but not yet connected KafkaConsumer
- */
-declare const createNativeKafkaConsumer: (cfg: NativeConsumerConfig, logger: Logger, rebalanceCb?: (err: LibrdKafkaError, assignments: TopicPartition[]) => void) => KafkaConsumer;
 
 /**
  * @module secrets
@@ -579,4 +554,4 @@ type DataModelConfig<T> = Partial<{
     parallelism?: number;
 }>;
 
-export { ACKs, ApiUtil, type CSVParsingConfig, CSV_DELIMITERS, type CliLogData, DEFAULT_CSV_CONFIG, DEFAULT_JSON_CONFIG, type DataModelConfig, DataSource, type DataSourceConfig, type ExpressRequestWithMoose, type ExtractionResult, type JSONParsingConfig, type KafkaClientConfig, type Logger, MAX_RETRIES, MAX_RETRIES_PRODUCER, MAX_RETRY_TIME_MS, MOOSE_RUNTIME_ENV_PREFIX, MooseCache, MooseClient, type NativeConsumerConfig, type Producer, RETRY_FACTOR_PRODUCER, RETRY_INITIAL_TIME_MS, type StripDateIntersection, type TaskConfig, type TaskDefinition, type TaskFunction, antiCachePath, cliLog, compilerLog, createApi, createConsumptionApi, createNativeKafkaConsumer, createProducerConfig, expressMiddleware, getClickhouseClient, getFileName, getKafkaClient, getKafkaProducer, getMooseClients, getMooseUtils, isValidCSVDelimiter, logError, mapTstoJs, mooseEnvSecrets, mooseRuntimeEnv, parseCSV, parseJSON, parseJSONWithDates };
+export { ACKs, ApiUtil, type CSVParsingConfig, CSV_DELIMITERS, type CliLogData, DEFAULT_CSV_CONFIG, DEFAULT_JSON_CONFIG, type DataModelConfig, DataSource, type DataSourceConfig, type ExpressRequestWithMoose, type ExtractionResult, type JSONParsingConfig, type KafkaClientConfig, type Logger, MAX_RETRIES, MAX_RETRIES_PRODUCER, MAX_RETRY_TIME_MS, MOOSE_RUNTIME_ENV_PREFIX, MooseCache, MooseClient, type Producer, RETRY_FACTOR_PRODUCER, RETRY_INITIAL_TIME_MS, type StripDateIntersection, type TaskConfig, type TaskDefinition, type TaskFunction, antiCachePath, cliLog, compilerLog, createApi, createConsumptionApi, createProducerConfig, expressMiddleware, getClickhouseClient, getFileName, getKafkaClient, getKafkaProducer, getMooseClients, getMooseUtils, isValidCSVDelimiter, logError, mapTstoJs, mooseEnvSecrets, mooseRuntimeEnv, parseCSV, parseJSON, parseJSONWithDates };

package/dist/index.js

@@ -34,8 +34,6 @@ var __toCommonJS = (mod) => __copyProps(__defProp({}, "__esModule", { value: tru
 var commons_exports = {};
 __export(commons_exports, {
   ACKs: () => ACKs,
-  CODES: () => import_kafka_javascript.CODES,
-  KafkaConsumer: () => import_kafka_javascript.KafkaConsumer,
   MAX_RETRIES: () => MAX_RETRIES,
   MAX_RETRIES_PRODUCER: () => MAX_RETRIES_PRODUCER,
   MAX_RETRY_TIME_MS: () => MAX_RETRY_TIME_MS,
@@ -44,7 +42,6 @@ __export(commons_exports, {
   antiCachePath: () => antiCachePath,
   cliLog: () => cliLog,
   compilerLog: () => compilerLog,
-  createNativeKafkaConsumer: () => createNativeKafkaConsumer,
   createProducerConfig: () => createProducerConfig,
   getClickhouseClient: () => getClickhouseClient,
   getFileName: () => getFileName,
@@ -90,13 +87,13 @@ async function getKafkaProducer(cfg, logger, maxMessageBytes) {
   await producer.connect();
   return producer;
 }
-var import_http, import_client, import_kafka_javascript, Kafka, compilerLog, antiCachePath, getFileName, getClickhouseClient, cliLog, MAX_RETRIES, MAX_RETRY_TIME_MS, RETRY_INITIAL_TIME_MS, MAX_RETRIES_PRODUCER, RETRY_FACTOR_PRODUCER, ACKs, parseBrokerString, logError, buildSaslConfig, getKafkaClient, buildNativeSaslConfig, createNativeKafkaConsumer;
+var import_http, import_client, import_kafka_javascript, Kafka, compilerLog, antiCachePath, getFileName, getClickhouseClient, cliLog, MAX_RETRIES, MAX_RETRY_TIME_MS, RETRY_INITIAL_TIME_MS, MAX_RETRIES_PRODUCER, RETRY_FACTOR_PRODUCER, ACKs, parseBrokerString, logError, buildSaslConfig, getKafkaClient;
 var init_commons = __esm({
   "src/commons.ts"() {
     "use strict";
     import_http = __toESM(require("http"));
     import_client = require("@clickhouse/client");
-    import_kafka_javascript = require("@confluentinc/kafka-javascript");
+    import_kafka_javascript = require("@514labs/kafka-javascript");
     ({ Kafka } = import_kafka_javascript.KafkaJS);
     compilerLog = (message) => {
       if (!isTruthy(process.env.MOOSE_DISABLE_COMPILER_LOGS)) {
@@ -197,59 +194,6 @@ var init_commons = __esm({
         }
       });
     };
-    buildNativeSaslConfig = (logger, cfg) => {
-      if (!cfg.saslMechanism || !cfg.saslUsername || !cfg.saslPassword) {
-        return {};
-      }
-      const mechanism = cfg.saslMechanism.toUpperCase();
-      const validMechanisms = ["PLAIN", "SCRAM-SHA-256", "SCRAM-SHA-512"];
-      if (!validMechanisms.includes(mechanism)) {
-        logger.warn(`Unsupported SASL mechanism: ${cfg.saslMechanism}`);
-        return {};
-      }
-      return {
-        "sasl.mechanisms": mechanism,
-        "sasl.username": cfg.saslUsername,
-        "sasl.password": cfg.saslPassword
-      };
-    };
-    createNativeKafkaConsumer = (cfg, logger, rebalanceCb) => {
-      const brokers = parseBrokerString(cfg.broker || "");
-      if (brokers.length === 0) {
-        throw new Error(`No valid broker addresses found in: "${cfg.broker}"`);
-      }
-      logger.log(
-        `Creating native KafkaConsumer with brokers: ${brokers.join(", ")}`
-      );
-      logger.log(`Security protocol: ${cfg.securityProtocol || "plaintext"}`);
-      logger.log(`Client ID: ${cfg.clientId}`);
-      logger.log(`Group ID: ${cfg.groupId}`);
-      const saslConfig = buildNativeSaslConfig(logger, cfg);
-      const consumerConfig = {
-        // Connection
-        "bootstrap.servers": brokers.join(","),
-        "client.id": cfg.clientId,
-        // Group management
-        "group.id": cfg.groupId,
-        "session.timeout.ms": cfg.sessionTimeoutMs ?? 3e4,
-        "heartbeat.interval.ms": cfg.heartbeatIntervalMs ?? 3e3,
-        "max.poll.interval.ms": cfg.maxPollIntervalMs ?? 3e5,
-        // Offset management
-        "enable.auto.commit": cfg.autoCommit ?? true,
-        "auto.commit.interval.ms": cfg.autoCommitIntervalMs ?? 5e3,
-        // Security
-        ...cfg.securityProtocol === "SASL_SSL" && {
-          "security.protocol": "sasl_ssl"
-        },
-        ...saslConfig,
-        // Rebalance callback
-        ...rebalanceCb && { rebalance_cb: rebalanceCb }
-      };
-      const topicConfig = {
-        "auto.offset.reset": cfg.autoOffsetReset ?? "earliest"
-      };
-      return new import_kafka_javascript.KafkaConsumer(consumerConfig, topicConfig);
-    };
   }
 });
 
@@ -443,7 +387,6 @@ __export(index_exports, {
   ACKs: () => ACKs,
   Api: () => Api,
   ApiHelpers: () => ApiHelpers,
-  CODES: () => import_kafka_javascript.CODES,
   CSV_DELIMITERS: () => CSV_DELIMITERS,
   ClickHouseEngines: () => ClickHouseEngines,
   ConsumptionApi: () => ConsumptionApi,
@@ -455,7 +398,6 @@ __export(index_exports, {
   ETLPipeline: () => ETLPipeline,
   IngestApi: () => IngestApi,
   IngestPipeline: () => IngestPipeline,
-  KafkaConsumer: () => import_kafka_javascript.KafkaConsumer,
   LifeCycle: () => LifeCycle,
   MAX_RETRIES: () => MAX_RETRIES,
   MAX_RETRIES_PRODUCER: () => MAX_RETRIES_PRODUCER,
@@ -483,7 +425,6 @@ __export(index_exports, {
   createClickhouseParameter: () => createClickhouseParameter,
   createConsumptionApi: () => createConsumptionApi,
   createMaterializedView: () => createMaterializedView,
-  createNativeKafkaConsumer: () => createNativeKafkaConsumer,
   createProducerConfig: () => createProducerConfig,
   dropView: () => dropView,
   expressMiddleware: () => expressMiddleware,
@@ -3495,7 +3436,6 @@ var DataSource = class {
   ACKs,
   Api,
   ApiHelpers,
-  CODES,
   CSV_DELIMITERS,
   ClickHouseEngines,
   ConsumptionApi,
@@ -3507,7 +3447,6 @@ var DataSource = class {
   ETLPipeline,
   IngestApi,
   IngestPipeline,
-  KafkaConsumer,
   LifeCycle,
   MAX_RETRIES,
   MAX_RETRIES_PRODUCER,
@@ -3535,7 +3474,6 @@ var DataSource = class {
   createClickhouseParameter,
   createConsumptionApi,
   createMaterializedView,
-  createNativeKafkaConsumer,
   createProducerConfig,
   dropView,
   expressMiddleware,