@514labs/moose-lib 0.6.252-ci-3-gf88b2b20 → 0.6.252-ci-1-g40bb1da4

package/dist/index.d.mts

@@ -44,25 +44,6 @@ declare const RETRY_INITIAL_TIME_MS = 100;
 declare const MAX_RETRIES_PRODUCER = 150;
 declare const RETRY_FACTOR_PRODUCER = 0.2;
 declare const ACKs = -1;
-/**
- * Creates the base producer configuration for Kafka.
- * Used by both the SDK stream publishing and streaming function workers.
- *
- * @param maxMessageBytes - Optional max message size in bytes (synced with topic config)
- * @returns Producer configuration object for the Confluent Kafka client
- */
-declare function createProducerConfig(maxMessageBytes?: number): {
-    "message.max.bytes"?: number | undefined;
-    kafkaJS: {
-        idempotent: boolean;
-        acks: number;
-        retry: {
-            retries: number;
-            maxRetryTime: number;
-        };
-    };
-    "linger.ms": number;
-};
 type KafkaClientConfig = {
     clientId: string;
     broker: string;
@@ -74,12 +55,8 @@ type KafkaClientConfig = {
 /**
  * Dynamically creates and connects a KafkaJS producer using the provided configuration.
  * Returns a connected producer instance.
- *
- * @param cfg - Kafka client configuration
- * @param logger - Logger instance
- * @param maxMessageBytes - Optional max message size in bytes (synced with topic config)
  */
-declare function getKafkaProducer(cfg: KafkaClientConfig, logger: Logger, maxMessageBytes?: number): Promise<Producer>;
+declare function getKafkaProducer(cfg: KafkaClientConfig, logger: Logger): Promise<Producer>;
 /**
  * Interface for logging functionality
  */
@@ -554,4 +531,4 @@ type DataModelConfig<T> = Partial<{
     parallelism?: number;
 }>;
 
-export { ACKs, ApiUtil, type CSVParsingConfig, CSV_DELIMITERS, type CliLogData, DEFAULT_CSV_CONFIG, DEFAULT_JSON_CONFIG, type DataModelConfig, DataSource, type DataSourceConfig, type ExpressRequestWithMoose, type ExtractionResult, type JSONParsingConfig, type KafkaClientConfig, type Logger, MAX_RETRIES, MAX_RETRIES_PRODUCER, MAX_RETRY_TIME_MS, MOOSE_RUNTIME_ENV_PREFIX, MooseCache, MooseClient, type Producer, RETRY_FACTOR_PRODUCER, RETRY_INITIAL_TIME_MS, type StripDateIntersection, type TaskConfig, type TaskDefinition, type TaskFunction, antiCachePath, cliLog, compilerLog, createApi, createConsumptionApi, createProducerConfig, expressMiddleware, getClickhouseClient, getFileName, getKafkaClient, getKafkaProducer, getMooseClients, getMooseUtils, isValidCSVDelimiter, logError, mapTstoJs, mooseEnvSecrets, mooseRuntimeEnv, parseCSV, parseJSON, parseJSONWithDates };
+export { ACKs, ApiUtil, type CSVParsingConfig, CSV_DELIMITERS, type CliLogData, DEFAULT_CSV_CONFIG, DEFAULT_JSON_CONFIG, type DataModelConfig, DataSource, type DataSourceConfig, type ExpressRequestWithMoose, type ExtractionResult, type JSONParsingConfig, type KafkaClientConfig, type Logger, MAX_RETRIES, MAX_RETRIES_PRODUCER, MAX_RETRY_TIME_MS, MOOSE_RUNTIME_ENV_PREFIX, MooseCache, MooseClient, type Producer, RETRY_FACTOR_PRODUCER, RETRY_INITIAL_TIME_MS, type StripDateIntersection, type TaskConfig, type TaskDefinition, type TaskFunction, antiCachePath, cliLog, compilerLog, createApi, createConsumptionApi, expressMiddleware, getClickhouseClient, getFileName, getKafkaClient, getKafkaProducer, getMooseClients, getMooseUtils, isValidCSVDelimiter, logError, mapTstoJs, mooseEnvSecrets, mooseRuntimeEnv, parseCSV, parseJSON, parseJSONWithDates };

package/dist/index.d.ts

@@ -44,25 +44,6 @@ declare const RETRY_INITIAL_TIME_MS = 100;
 declare const MAX_RETRIES_PRODUCER = 150;
 declare const RETRY_FACTOR_PRODUCER = 0.2;
 declare const ACKs = -1;
-/**
- * Creates the base producer configuration for Kafka.
- * Used by both the SDK stream publishing and streaming function workers.
- *
- * @param maxMessageBytes - Optional max message size in bytes (synced with topic config)
- * @returns Producer configuration object for the Confluent Kafka client
- */
-declare function createProducerConfig(maxMessageBytes?: number): {
-    "message.max.bytes"?: number | undefined;
-    kafkaJS: {
-        idempotent: boolean;
-        acks: number;
-        retry: {
-            retries: number;
-            maxRetryTime: number;
-        };
-    };
-    "linger.ms": number;
-};
 type KafkaClientConfig = {
     clientId: string;
     broker: string;
@@ -74,12 +55,8 @@ type KafkaClientConfig = {
 /**
  * Dynamically creates and connects a KafkaJS producer using the provided configuration.
  * Returns a connected producer instance.
- *
- * @param cfg - Kafka client configuration
- * @param logger - Logger instance
- * @param maxMessageBytes - Optional max message size in bytes (synced with topic config)
  */
-declare function getKafkaProducer(cfg: KafkaClientConfig, logger: Logger, maxMessageBytes?: number): Promise<Producer>;
+declare function getKafkaProducer(cfg: KafkaClientConfig, logger: Logger): Promise<Producer>;
 /**
  * Interface for logging functionality
  */
@@ -554,4 +531,4 @@ type DataModelConfig<T> = Partial<{
     parallelism?: number;
 }>;
 
-export { ACKs, ApiUtil, type CSVParsingConfig, CSV_DELIMITERS, type CliLogData, DEFAULT_CSV_CONFIG, DEFAULT_JSON_CONFIG, type DataModelConfig, DataSource, type DataSourceConfig, type ExpressRequestWithMoose, type ExtractionResult, type JSONParsingConfig, type KafkaClientConfig, type Logger, MAX_RETRIES, MAX_RETRIES_PRODUCER, MAX_RETRY_TIME_MS, MOOSE_RUNTIME_ENV_PREFIX, MooseCache, MooseClient, type Producer, RETRY_FACTOR_PRODUCER, RETRY_INITIAL_TIME_MS, type StripDateIntersection, type TaskConfig, type TaskDefinition, type TaskFunction, antiCachePath, cliLog, compilerLog, createApi, createConsumptionApi, createProducerConfig, expressMiddleware, getClickhouseClient, getFileName, getKafkaClient, getKafkaProducer, getMooseClients, getMooseUtils, isValidCSVDelimiter, logError, mapTstoJs, mooseEnvSecrets, mooseRuntimeEnv, parseCSV, parseJSON, parseJSONWithDates };
+export { ACKs, ApiUtil, type CSVParsingConfig, CSV_DELIMITERS, type CliLogData, DEFAULT_CSV_CONFIG, DEFAULT_JSON_CONFIG, type DataModelConfig, DataSource, type DataSourceConfig, type ExpressRequestWithMoose, type ExtractionResult, type JSONParsingConfig, type KafkaClientConfig, type Logger, MAX_RETRIES, MAX_RETRIES_PRODUCER, MAX_RETRY_TIME_MS, MOOSE_RUNTIME_ENV_PREFIX, MooseCache, MooseClient, type Producer, RETRY_FACTOR_PRODUCER, RETRY_INITIAL_TIME_MS, type StripDateIntersection, type TaskConfig, type TaskDefinition, type TaskFunction, antiCachePath, cliLog, compilerLog, createApi, createConsumptionApi, expressMiddleware, getClickhouseClient, getFileName, getKafkaClient, getKafkaProducer, getMooseClients, getMooseUtils, isValidCSVDelimiter, logError, mapTstoJs, mooseEnvSecrets, mooseRuntimeEnv, parseCSV, parseJSON, parseJSONWithDates };

package/dist/index.js

@@ -42,7 +42,6 @@ __export(commons_exports, {
   antiCachePath: () => antiCachePath,
   cliLog: () => cliLog,
   compilerLog: () => compilerLog,
-  createProducerConfig: () => createProducerConfig,
   getClickhouseClient: () => getClickhouseClient,
   getFileName: () => getFileName,
   getKafkaClient: () => getKafkaClient,
@@ -65,25 +64,18 @@ function isTruthy(value) {
 function mapTstoJs(filePath) {
   return filePath.replace(/\.ts$/, ".js").replace(/\.cts$/, ".cjs").replace(/\.mts$/, ".mjs");
 }
-function createProducerConfig(maxMessageBytes) {
-  return {
+async function getKafkaProducer(cfg, logger) {
+  const kafka = await getKafkaClient(cfg, logger);
+  const producer = kafka.producer({
     kafkaJS: {
-      idempotent: false,
-      // Not needed for at-least-once delivery
+      idempotent: true,
       acks: ACKs,
       retry: {
         retries: MAX_RETRIES_PRODUCER,
         maxRetryTime: MAX_RETRY_TIME_MS
       }
-    },
-    "linger.ms": 0,
-    // Send immediately - batching happens at application level
-    ...maxMessageBytes && { "message.max.bytes": maxMessageBytes }
-  };
-}
-async function getKafkaProducer(cfg, logger, maxMessageBytes) {
-  const kafka = await getKafkaClient(cfg, logger);
-  const producer = kafka.producer(createProducerConfig(maxMessageBytes));
+    }
+  });
   await producer.connect();
   return producer;
 }
@@ -425,7 +417,6 @@ __export(index_exports, {
   createClickhouseParameter: () => createClickhouseParameter,
   createConsumptionApi: () => createConsumptionApi,
   createMaterializedView: () => createMaterializedView,
-  createProducerConfig: () => createProducerConfig,
   dropView: () => dropView,
   expressMiddleware: () => expressMiddleware,
   getApi: () => getApi,
@@ -3407,7 +3398,6 @@ var DataSource = class {
   createClickhouseParameter,
   createConsumptionApi,
   createMaterializedView,
-  createProducerConfig,
   dropView,
   expressMiddleware,
   getApi,