@514labs/moose-lib 0.6.252-ci-4-gb8a461bd → 0.6.252-ci-2-g41538689

package/dist/{index-xC52kbse.d.mts → index-CNlTyF6R.d.mts}

@@ -190,7 +190,6 @@ declare enum ClickHouseEngines {
     Buffer = "Buffer",
     Distributed = "Distributed",
     IcebergS3 = "IcebergS3",
-    Kafka = "Kafka",
     ReplicatedMergeTree = "ReplicatedMergeTree",
     ReplicatedReplacingMergeTree = "ReplicatedReplacingMergeTree",
     ReplicatedAggregatingMergeTree = "ReplicatedAggregatingMergeTree",
@@ -638,105 +637,6 @@ type DistributedConfig<T> = Omit<BaseOlapConfig<T>, "orderByFields" | "orderByEx
     /** Optional: Policy name for data distribution */
     policyName?: string;
 };
-/**
- * Kafka engine table settings (alterable properties via ALTER TABLE MODIFY SETTING)
- *
- * These settings control the behavior of the Kafka consumer and can be modified
- * after table creation without recreating the table.
- *
- * Reference: https://clickhouse.com/docs/en/engines/table-engines/integrations/kafka
- */
-/**
- * Kafka engine table settings (alterable properties via ALTER TABLE MODIFY SETTING)
- *
- * Reference: https://clickhouse.com/docs/engines/table-engines/integrations/kafka
- */
-interface KafkaTableSettings {
-    /** Security protocol for Kafka connection */
-    kafka_security_protocol?: "PLAINTEXT" | "SSL" | "SASL_PLAINTEXT" | "SASL_SSL";
-    /** SASL mechanism for authentication */
-    kafka_sasl_mechanism?: "GSSAPI" | "PLAIN" | "SCRAM-SHA-256" | "SCRAM-SHA-512" | "OAUTHBEARER";
-    /** SASL username */
-    kafka_sasl_username?: string;
-    /** SASL password */
-    kafka_sasl_password?: string;
-    /** Schema definition (required for formats like Avro, Cap'n Proto) */
-    kafka_schema?: string;
-    /** Number of parallel consumers (default: 1, max: number of topic partitions) */
-    kafka_num_consumers?: string;
-    /** Maximum batch size (in messages) for poll. Default: max_insert_block_size */
-    kafka_max_block_size?: string;
-    /** Number of broken messages to skip per block before throwing an exception. Default: 0 */
-    kafka_skip_broken_messages?: string;
-    /** Commit offsets after each inserted batch: "0" (manual) or "1" (auto-commit per batch). Default: "0" */
-    kafka_commit_every_batch?: string;
-    /** Client identifier passed to Kafka broker */
-    kafka_client_id?: string;
-    /** Timeout for polling Kafka in milliseconds. Default: 0 */
-    kafka_poll_timeout_ms?: string;
-    /** Maximum batch size for poll. Default: 0 */
-    kafka_poll_max_batch_size?: string;
-    /** Interval between flushes in milliseconds. Default: 0 */
-    kafka_flush_interval_ms?: string;
-    /** Consumer reschedule interval in milliseconds. Default: 0 */
-    kafka_consumer_reschedule_ms?: string;
-    /** Use dedicated thread per consumer: "0" (shared) or "1" (dedicated). Default: "0" */
-    kafka_thread_per_consumer?: string;
-    /** Error handling mode: 'default' (stop on error) or 'stream' (write errors to separate stream). Default: 'default' */
-    kafka_handle_error_mode?: "default" | "stream";
-    /** Commit on SELECT queries: "0" (false) or "1" (true). Default: "0" */
-    kafka_commit_on_select?: string;
-    /** Maximum rows per message for row-based formats. Default: 1 */
-    kafka_max_rows_per_message?: string;
-    /** Compression codec for producing messages (e.g., 'gzip', 'snappy', 'lz4', 'zstd') */
-    kafka_compression_codec?: string;
-    /** Compression level. Default: -1 (use codec default) */
-    kafka_compression_level?: string;
-}
-/**
- * Configuration for Kafka engine - streaming data ingestion from Kafka topics
- *
- * The Kafka engine creates an internal consumer that reads messages from topics.
- * Typically used with materialized views to persist data into MergeTree tables.
- *
- * @template T The data type of the records stored in the table.
- *
- * @example
- * ```typescript
- * import { mooseRuntimeEnv } from "@514labs/moose-lib";
- *
- * const eventStream = new OlapTable<Event>("event_stream", {
- *   engine: ClickHouseEngines.Kafka,
- *   brokerList: "kafka-1:9092,kafka-2:9092",
- *   topicList: "events",
- *   groupName: "moose_consumer",
- *   format: "JSONEachRow",
- *   settings: {
- *     kafka_num_consumers: "3",
- *     kafka_sasl_mechanism: "SCRAM-SHA-256",
- *     kafka_security_protocol: "SASL_PLAINTEXT",
- *     kafka_sasl_username: mooseRuntimeEnv.get("KAFKA_USERNAME"),
- *     kafka_sasl_password: mooseRuntimeEnv.get("KAFKA_PASSWORD"),
- *   },
- * });
- * ```
- */
-type KafkaConfig<T> = Omit<BaseOlapConfig<T>, "orderByFields" | "orderByExpression" | "partitionBy" | "sampleByExpression"> & {
-    engine: ClickHouseEngines.Kafka;
-    /** Kafka broker addresses (comma-separated, e.g., 'kafka-1:9092,kafka-2:9092') */
-    brokerList: string;
-    /** Kafka topics to consume from (comma-separated) */
-    topicList: string;
-    /** Consumer group identifier */
-    groupName: string;
-    /** Message format (e.g., 'JSONEachRow', 'CSV', 'Avro') */
-    format: string;
-    /**
-     * Kafka settings (kafka_schema, kafka_num_consumers, security, tuning params)
-     * All other Kafka parameters must be specified here.
-     */
-    settings?: KafkaTableSettings;
-};
 /**
  * Configuration for IcebergS3 engine - read-only Iceberg table access
  *
@@ -779,7 +679,7 @@ type IcebergS3Config<T> = Omit<BaseOlapConfig<T>, "orderByFields" | "orderByExpr
  * @template T The data type of the records stored in the table.
  */
 type LegacyOlapConfig<T> = BaseOlapConfig<T>;
-type EngineConfig<T> = MergeTreeConfig<T> | ReplacingMergeTreeConfig<T> | AggregatingMergeTreeConfig<T> | SummingMergeTreeConfig<T> | ReplicatedMergeTreeConfig<T> | ReplicatedReplacingMergeTreeConfig<T> | ReplicatedAggregatingMergeTreeConfig<T> | ReplicatedSummingMergeTreeConfig<T> | S3QueueConfig<T> | S3Config<T> | BufferConfig<T> | DistributedConfig<T> | IcebergS3Config<T> | KafkaConfig<T>;
+type EngineConfig<T> = MergeTreeConfig<T> | ReplacingMergeTreeConfig<T> | AggregatingMergeTreeConfig<T> | SummingMergeTreeConfig<T> | ReplicatedMergeTreeConfig<T> | ReplicatedReplacingMergeTreeConfig<T> | ReplicatedAggregatingMergeTreeConfig<T> | ReplicatedSummingMergeTreeConfig<T> | S3QueueConfig<T> | S3Config<T> | BufferConfig<T> | DistributedConfig<T> | IcebergS3Config<T>;
 /**
  * Union of all engine-specific configurations (new API)
  * @template T The data type of the records stored in the table.

package/dist/{index-xC52kbse.d.ts → index-CNlTyF6R.d.ts}

@@ -190,7 +190,6 @@ declare enum ClickHouseEngines {
     Buffer = "Buffer",
     Distributed = "Distributed",
     IcebergS3 = "IcebergS3",
-    Kafka = "Kafka",
     ReplicatedMergeTree = "ReplicatedMergeTree",
     ReplicatedReplacingMergeTree = "ReplicatedReplacingMergeTree",
     ReplicatedAggregatingMergeTree = "ReplicatedAggregatingMergeTree",
@@ -638,105 +637,6 @@ type DistributedConfig<T> = Omit<BaseOlapConfig<T>, "orderByFields" | "orderByEx
     /** Optional: Policy name for data distribution */
     policyName?: string;
 };
-/**
- * Kafka engine table settings (alterable properties via ALTER TABLE MODIFY SETTING)
- *
- * These settings control the behavior of the Kafka consumer and can be modified
- * after table creation without recreating the table.
- *
- * Reference: https://clickhouse.com/docs/en/engines/table-engines/integrations/kafka
- */
-/**
- * Kafka engine table settings (alterable properties via ALTER TABLE MODIFY SETTING)
- *
- * Reference: https://clickhouse.com/docs/engines/table-engines/integrations/kafka
- */
-interface KafkaTableSettings {
-    /** Security protocol for Kafka connection */
-    kafka_security_protocol?: "PLAINTEXT" | "SSL" | "SASL_PLAINTEXT" | "SASL_SSL";
-    /** SASL mechanism for authentication */
-    kafka_sasl_mechanism?: "GSSAPI" | "PLAIN" | "SCRAM-SHA-256" | "SCRAM-SHA-512" | "OAUTHBEARER";
-    /** SASL username */
-    kafka_sasl_username?: string;
-    /** SASL password */
-    kafka_sasl_password?: string;
-    /** Schema definition (required for formats like Avro, Cap'n Proto) */
-    kafka_schema?: string;
-    /** Number of parallel consumers (default: 1, max: number of topic partitions) */
-    kafka_num_consumers?: string;
-    /** Maximum batch size (in messages) for poll. Default: max_insert_block_size */
-    kafka_max_block_size?: string;
-    /** Number of broken messages to skip per block before throwing an exception. Default: 0 */
-    kafka_skip_broken_messages?: string;
-    /** Commit offsets after each inserted batch: "0" (manual) or "1" (auto-commit per batch). Default: "0" */
-    kafka_commit_every_batch?: string;
-    /** Client identifier passed to Kafka broker */
-    kafka_client_id?: string;
-    /** Timeout for polling Kafka in milliseconds. Default: 0 */
-    kafka_poll_timeout_ms?: string;
-    /** Maximum batch size for poll. Default: 0 */
-    kafka_poll_max_batch_size?: string;
-    /** Interval between flushes in milliseconds. Default: 0 */
-    kafka_flush_interval_ms?: string;
-    /** Consumer reschedule interval in milliseconds. Default: 0 */
-    kafka_consumer_reschedule_ms?: string;
-    /** Use dedicated thread per consumer: "0" (shared) or "1" (dedicated). Default: "0" */
-    kafka_thread_per_consumer?: string;
-    /** Error handling mode: 'default' (stop on error) or 'stream' (write errors to separate stream). Default: 'default' */
-    kafka_handle_error_mode?: "default" | "stream";
-    /** Commit on SELECT queries: "0" (false) or "1" (true). Default: "0" */
-    kafka_commit_on_select?: string;
-    /** Maximum rows per message for row-based formats. Default: 1 */
-    kafka_max_rows_per_message?: string;
-    /** Compression codec for producing messages (e.g., 'gzip', 'snappy', 'lz4', 'zstd') */
-    kafka_compression_codec?: string;
-    /** Compression level. Default: -1 (use codec default) */
-    kafka_compression_level?: string;
-}
-/**
- * Configuration for Kafka engine - streaming data ingestion from Kafka topics
- *
- * The Kafka engine creates an internal consumer that reads messages from topics.
- * Typically used with materialized views to persist data into MergeTree tables.
- *
- * @template T The data type of the records stored in the table.
- *
- * @example
- * ```typescript
- * import { mooseRuntimeEnv } from "@514labs/moose-lib";
- *
- * const eventStream = new OlapTable<Event>("event_stream", {
- *   engine: ClickHouseEngines.Kafka,
- *   brokerList: "kafka-1:9092,kafka-2:9092",
- *   topicList: "events",
- *   groupName: "moose_consumer",
- *   format: "JSONEachRow",
- *   settings: {
- *     kafka_num_consumers: "3",
- *     kafka_sasl_mechanism: "SCRAM-SHA-256",
- *     kafka_security_protocol: "SASL_PLAINTEXT",
- *     kafka_sasl_username: mooseRuntimeEnv.get("KAFKA_USERNAME"),
- *     kafka_sasl_password: mooseRuntimeEnv.get("KAFKA_PASSWORD"),
- *   },
- * });
- * ```
- */
-type KafkaConfig<T> = Omit<BaseOlapConfig<T>, "orderByFields" | "orderByExpression" | "partitionBy" | "sampleByExpression"> & {
-    engine: ClickHouseEngines.Kafka;
-    /** Kafka broker addresses (comma-separated, e.g., 'kafka-1:9092,kafka-2:9092') */
-    brokerList: string;
-    /** Kafka topics to consume from (comma-separated) */
-    topicList: string;
-    /** Consumer group identifier */
-    groupName: string;
-    /** Message format (e.g., 'JSONEachRow', 'CSV', 'Avro') */
-    format: string;
-    /**
-     * Kafka settings (kafka_schema, kafka_num_consumers, security, tuning params)
-     * All other Kafka parameters must be specified here.
-     */
-    settings?: KafkaTableSettings;
-};
 /**
  * Configuration for IcebergS3 engine - read-only Iceberg table access
  *
@@ -779,7 +679,7 @@ type IcebergS3Config<T> = Omit<BaseOlapConfig<T>, "orderByFields" | "orderByExpr
  * @template T The data type of the records stored in the table.
  */
 type LegacyOlapConfig<T> = BaseOlapConfig<T>;
-type EngineConfig<T> = MergeTreeConfig<T> | ReplacingMergeTreeConfig<T> | AggregatingMergeTreeConfig<T> | SummingMergeTreeConfig<T> | ReplicatedMergeTreeConfig<T> | ReplicatedReplacingMergeTreeConfig<T> | ReplicatedAggregatingMergeTreeConfig<T> | ReplicatedSummingMergeTreeConfig<T> | S3QueueConfig<T> | S3Config<T> | BufferConfig<T> | DistributedConfig<T> | IcebergS3Config<T> | KafkaConfig<T>;
+type EngineConfig<T> = MergeTreeConfig<T> | ReplacingMergeTreeConfig<T> | AggregatingMergeTreeConfig<T> | SummingMergeTreeConfig<T> | ReplicatedMergeTreeConfig<T> | ReplicatedReplacingMergeTreeConfig<T> | ReplicatedAggregatingMergeTreeConfig<T> | ReplicatedSummingMergeTreeConfig<T> | S3QueueConfig<T> | S3Config<T> | BufferConfig<T> | DistributedConfig<T> | IcebergS3Config<T>;
 /**
  * Union of all engine-specific configurations (new API)
  * @template T The data type of the records stored in the table.

package/dist/index.d.mts

@@ -1,6 +1,6 @@
-export { C as ClickHouseByteSize, q as ClickHouseCodec, j as ClickHouseDecimal, n as ClickHouseDefault, k as ClickHouseFixedStringSize, l as ClickHouseFloat, a as ClickHouseInt, m as ClickHouseJson, e as ClickHouseLineString, p as ClickHouseMaterialized, f as ClickHouseMultiLineString, h as ClickHouseMultiPolygon, b as ClickHouseNamedTuple, c as ClickHousePoint, g as ClickHousePolygon, i as ClickHousePrecision, d as ClickHouseRing, o as ClickHouseTTL, D as DateTime, r as DateTime64, t as DateTime64String, s as DateTimeString, E as Decimal, F as FixedString, u as Float32, v as Float64, w as Int16, x as Int32, y as Int64, I as Int8, J as JWT, K as Key, L as LowCardinality, z as UInt16, A as UInt32, B as UInt64, U as UInt8, W as WithDefault } from './browserCompatible-CYDIQRRZ.mjs';
-import { K as ApiUtil, a4 as MooseClient } from './index-xC52kbse.mjs';
-export { A as Aggregated, h as Api, i as ApiConfig, ad as ApiHelpers, a5 as Blocks, a6 as ClickHouseEngines, C as ConsumptionApi, ae as ConsumptionHelpers, N as ConsumptionUtil, e as DeadLetter, D as DeadLetterModel, f as DeadLetterQueue, l as ETLPipeline, m as ETLPipelineConfig, E as EgressConfig, F as FrameworkApp, Q as IdentifierBrandedString, I as IngestApi, g as IngestConfig, j as IngestPipeline, L as LifeCycle, M as MaterializedView, R as NonIdentifierBrandedString, a as OlapConfig, O as OlapTable, aa as QueryClient, X as RawValue, b as S3QueueTableSettings, S as SimpleAggregated, Z as Sql, k as SqlResource, c as Stream, d as StreamConfig, T as Task, U as Value, V as View, n as WebApp, o as WebAppConfig, p as WebAppHandler, W as Workflow, ab as WorkflowClient, a2 as createClickhouseParameter, a8 as createMaterializedView, a7 as dropView, x as getApi, w as getApis, v as getIngestApi, u as getIngestApis, z as getSqlResource, y as getSqlResources, t as getStream, s as getStreams, r as getTable, q as getTables, ac as getTemporalClient, a1 as getValueFromParameter, J as getWebApp, H as getWebApps, G as getWorkflow, B as getWorkflows, af as joinQueries, a3 as mapToClickHouseType, a9 as populateTable, P as quoteIdentifier, Y as sql, $ as toQuery, a0 as toQueryPreview, _ as toStaticQuery } from './index-xC52kbse.mjs';
+export { C as ClickHouseByteSize, q as ClickHouseCodec, j as ClickHouseDecimal, n as ClickHouseDefault, k as ClickHouseFixedStringSize, l as ClickHouseFloat, a as ClickHouseInt, m as ClickHouseJson, e as ClickHouseLineString, p as ClickHouseMaterialized, f as ClickHouseMultiLineString, h as ClickHouseMultiPolygon, b as ClickHouseNamedTuple, c as ClickHousePoint, g as ClickHousePolygon, i as ClickHousePrecision, d as ClickHouseRing, o as ClickHouseTTL, D as DateTime, r as DateTime64, t as DateTime64String, s as DateTimeString, E as Decimal, F as FixedString, u as Float32, v as Float64, w as Int16, x as Int32, y as Int64, I as Int8, J as JWT, K as Key, L as LowCardinality, z as UInt16, A as UInt32, B as UInt64, U as UInt8, W as WithDefault } from './browserCompatible-BTN82akc.mjs';
+import { K as ApiUtil, a4 as MooseClient } from './index-CNlTyF6R.mjs';
+export { A as Aggregated, h as Api, i as ApiConfig, ad as ApiHelpers, a5 as Blocks, a6 as ClickHouseEngines, C as ConsumptionApi, ae as ConsumptionHelpers, N as ConsumptionUtil, e as DeadLetter, D as DeadLetterModel, f as DeadLetterQueue, l as ETLPipeline, m as ETLPipelineConfig, E as EgressConfig, F as FrameworkApp, Q as IdentifierBrandedString, I as IngestApi, g as IngestConfig, j as IngestPipeline, L as LifeCycle, M as MaterializedView, R as NonIdentifierBrandedString, a as OlapConfig, O as OlapTable, aa as QueryClient, X as RawValue, b as S3QueueTableSettings, S as SimpleAggregated, Z as Sql, k as SqlResource, c as Stream, d as StreamConfig, T as Task, U as Value, V as View, n as WebApp, o as WebAppConfig, p as WebAppHandler, W as Workflow, ab as WorkflowClient, a2 as createClickhouseParameter, a8 as createMaterializedView, a7 as dropView, x as getApi, w as getApis, v as getIngestApi, u as getIngestApis, z as getSqlResource, y as getSqlResources, t as getStream, s as getStreams, r as getTable, q as getTables, ac as getTemporalClient, a1 as getValueFromParameter, J as getWebApp, H as getWebApps, G as getWorkflow, B as getWorkflows, af as joinQueries, a3 as mapToClickHouseType, a9 as populateTable, P as quoteIdentifier, Y as sql, $ as toQuery, a0 as toQueryPreview, _ as toStaticQuery } from './index-CNlTyF6R.mjs';
 import * as _clickhouse_client from '@clickhouse/client';
 import { KafkaJS } from '@confluentinc/kafka-javascript';
 import http from 'http';
@@ -44,6 +44,25 @@ declare const RETRY_INITIAL_TIME_MS = 100;
 declare const MAX_RETRIES_PRODUCER = 150;
 declare const RETRY_FACTOR_PRODUCER = 0.2;
 declare const ACKs = -1;
+/**
+ * Creates the base producer configuration for Kafka.
+ * Used by both the SDK stream publishing and streaming function workers.
+ *
+ * @param maxMessageBytes - Optional max message size in bytes (synced with topic config)
+ * @returns Producer configuration object for the Confluent Kafka client
+ */
+declare function createProducerConfig(maxMessageBytes?: number): {
+    "message.max.bytes"?: number | undefined;
+    kafkaJS: {
+        idempotent: boolean;
+        acks: number;
+        retry: {
+            retries: number;
+            maxRetryTime: number;
+        };
+    };
+    "linger.ms": number;
+};
 type KafkaClientConfig = {
     clientId: string;
     broker: string;
@@ -55,8 +74,12 @@ type KafkaClientConfig = {
 /**
  * Dynamically creates and connects a KafkaJS producer using the provided configuration.
  * Returns a connected producer instance.
+ *
+ * @param cfg - Kafka client configuration
+ * @param logger - Logger instance
+ * @param maxMessageBytes - Optional max message size in bytes (synced with topic config)
  */
-declare function getKafkaProducer(cfg: KafkaClientConfig, logger: Logger): Promise<Producer>;
+declare function getKafkaProducer(cfg: KafkaClientConfig, logger: Logger, maxMessageBytes?: number): Promise<Producer>;
 /**
  * Interface for logging functionality
  */
@@ -531,4 +554,4 @@ type DataModelConfig<T> = Partial<{
     parallelism?: number;
 }>;
 
-export { ACKs, ApiUtil, type CSVParsingConfig, CSV_DELIMITERS, type CliLogData, DEFAULT_CSV_CONFIG, DEFAULT_JSON_CONFIG, type DataModelConfig, DataSource, type DataSourceConfig, type ExpressRequestWithMoose, type ExtractionResult, type JSONParsingConfig, type KafkaClientConfig, type Logger, MAX_RETRIES, MAX_RETRIES_PRODUCER, MAX_RETRY_TIME_MS, MOOSE_RUNTIME_ENV_PREFIX, MooseCache, MooseClient, type Producer, RETRY_FACTOR_PRODUCER, RETRY_INITIAL_TIME_MS, type StripDateIntersection, type TaskConfig, type TaskDefinition, type TaskFunction, antiCachePath, cliLog, compilerLog, createApi, createConsumptionApi, expressMiddleware, getClickhouseClient, getFileName, getKafkaClient, getKafkaProducer, getMooseClients, getMooseUtils, isValidCSVDelimiter, logError, mapTstoJs, mooseEnvSecrets, mooseRuntimeEnv, parseCSV, parseJSON, parseJSONWithDates };
+export { ACKs, ApiUtil, type CSVParsingConfig, CSV_DELIMITERS, type CliLogData, DEFAULT_CSV_CONFIG, DEFAULT_JSON_CONFIG, type DataModelConfig, DataSource, type DataSourceConfig, type ExpressRequestWithMoose, type ExtractionResult, type JSONParsingConfig, type KafkaClientConfig, type Logger, MAX_RETRIES, MAX_RETRIES_PRODUCER, MAX_RETRY_TIME_MS, MOOSE_RUNTIME_ENV_PREFIX, MooseCache, MooseClient, type Producer, RETRY_FACTOR_PRODUCER, RETRY_INITIAL_TIME_MS, type StripDateIntersection, type TaskConfig, type TaskDefinition, type TaskFunction, antiCachePath, cliLog, compilerLog, createApi, createConsumptionApi, createProducerConfig, expressMiddleware, getClickhouseClient, getFileName, getKafkaClient, getKafkaProducer, getMooseClients, getMooseUtils, isValidCSVDelimiter, logError, mapTstoJs, mooseEnvSecrets, mooseRuntimeEnv, parseCSV, parseJSON, parseJSONWithDates };

package/dist/index.d.ts

@@ -1,6 +1,6 @@
-export { C as ClickHouseByteSize, q as ClickHouseCodec, j as ClickHouseDecimal, n as ClickHouseDefault, k as ClickHouseFixedStringSize, l as ClickHouseFloat, a as ClickHouseInt, m as ClickHouseJson, e as ClickHouseLineString, p as ClickHouseMaterialized, f as ClickHouseMultiLineString, h as ClickHouseMultiPolygon, b as ClickHouseNamedTuple, c as ClickHousePoint, g as ClickHousePolygon, i as ClickHousePrecision, d as ClickHouseRing, o as ClickHouseTTL, D as DateTime, r as DateTime64, t as DateTime64String, s as DateTimeString, E as Decimal, F as FixedString, u as Float32, v as Float64, w as Int16, x as Int32, y as Int64, I as Int8, J as JWT, K as Key, L as LowCardinality, z as UInt16, A as UInt32, B as UInt64, U as UInt8, W as WithDefault } from './browserCompatible-C_H_-QHB.js';
-import { K as ApiUtil, a4 as MooseClient } from './index-xC52kbse.js';
-export { A as Aggregated, h as Api, i as ApiConfig, ad as ApiHelpers, a5 as Blocks, a6 as ClickHouseEngines, C as ConsumptionApi, ae as ConsumptionHelpers, N as ConsumptionUtil, e as DeadLetter, D as DeadLetterModel, f as DeadLetterQueue, l as ETLPipeline, m as ETLPipelineConfig, E as EgressConfig, F as FrameworkApp, Q as IdentifierBrandedString, I as IngestApi, g as IngestConfig, j as IngestPipeline, L as LifeCycle, M as MaterializedView, R as NonIdentifierBrandedString, a as OlapConfig, O as OlapTable, aa as QueryClient, X as RawValue, b as S3QueueTableSettings, S as SimpleAggregated, Z as Sql, k as SqlResource, c as Stream, d as StreamConfig, T as Task, U as Value, V as View, n as WebApp, o as WebAppConfig, p as WebAppHandler, W as Workflow, ab as WorkflowClient, a2 as createClickhouseParameter, a8 as createMaterializedView, a7 as dropView, x as getApi, w as getApis, v as getIngestApi, u as getIngestApis, z as getSqlResource, y as getSqlResources, t as getStream, s as getStreams, r as getTable, q as getTables, ac as getTemporalClient, a1 as getValueFromParameter, J as getWebApp, H as getWebApps, G as getWorkflow, B as getWorkflows, af as joinQueries, a3 as mapToClickHouseType, a9 as populateTable, P as quoteIdentifier, Y as sql, $ as toQuery, a0 as toQueryPreview, _ as toStaticQuery } from './index-xC52kbse.js';
+export { C as ClickHouseByteSize, q as ClickHouseCodec, j as ClickHouseDecimal, n as ClickHouseDefault, k as ClickHouseFixedStringSize, l as ClickHouseFloat, a as ClickHouseInt, m as ClickHouseJson, e as ClickHouseLineString, p as ClickHouseMaterialized, f as ClickHouseMultiLineString, h as ClickHouseMultiPolygon, b as ClickHouseNamedTuple, c as ClickHousePoint, g as ClickHousePolygon, i as ClickHousePrecision, d as ClickHouseRing, o as ClickHouseTTL, D as DateTime, r as DateTime64, t as DateTime64String, s as DateTimeString, E as Decimal, F as FixedString, u as Float32, v as Float64, w as Int16, x as Int32, y as Int64, I as Int8, J as JWT, K as Key, L as LowCardinality, z as UInt16, A as UInt32, B as UInt64, U as UInt8, W as WithDefault } from './browserCompatible-CoxWbneN.js';
+import { K as ApiUtil, a4 as MooseClient } from './index-CNlTyF6R.js';
+export { A as Aggregated, h as Api, i as ApiConfig, ad as ApiHelpers, a5 as Blocks, a6 as ClickHouseEngines, C as ConsumptionApi, ae as ConsumptionHelpers, N as ConsumptionUtil, e as DeadLetter, D as DeadLetterModel, f as DeadLetterQueue, l as ETLPipeline, m as ETLPipelineConfig, E as EgressConfig, F as FrameworkApp, Q as IdentifierBrandedString, I as IngestApi, g as IngestConfig, j as IngestPipeline, L as LifeCycle, M as MaterializedView, R as NonIdentifierBrandedString, a as OlapConfig, O as OlapTable, aa as QueryClient, X as RawValue, b as S3QueueTableSettings, S as SimpleAggregated, Z as Sql, k as SqlResource, c as Stream, d as StreamConfig, T as Task, U as Value, V as View, n as WebApp, o as WebAppConfig, p as WebAppHandler, W as Workflow, ab as WorkflowClient, a2 as createClickhouseParameter, a8 as createMaterializedView, a7 as dropView, x as getApi, w as getApis, v as getIngestApi, u as getIngestApis, z as getSqlResource, y as getSqlResources, t as getStream, s as getStreams, r as getTable, q as getTables, ac as getTemporalClient, a1 as getValueFromParameter, J as getWebApp, H as getWebApps, G as getWorkflow, B as getWorkflows, af as joinQueries, a3 as mapToClickHouseType, a9 as populateTable, P as quoteIdentifier, Y as sql, $ as toQuery, a0 as toQueryPreview, _ as toStaticQuery } from './index-CNlTyF6R.js';
 import * as _clickhouse_client from '@clickhouse/client';
 import { KafkaJS } from '@confluentinc/kafka-javascript';
 import http from 'http';
@@ -44,6 +44,25 @@ declare const RETRY_INITIAL_TIME_MS = 100;
 declare const MAX_RETRIES_PRODUCER = 150;
 declare const RETRY_FACTOR_PRODUCER = 0.2;
 declare const ACKs = -1;
+/**
+ * Creates the base producer configuration for Kafka.
+ * Used by both the SDK stream publishing and streaming function workers.
+ *
+ * @param maxMessageBytes - Optional max message size in bytes (synced with topic config)
+ * @returns Producer configuration object for the Confluent Kafka client
+ */
+declare function createProducerConfig(maxMessageBytes?: number): {
+    "message.max.bytes"?: number | undefined;
+    kafkaJS: {
+        idempotent: boolean;
+        acks: number;
+        retry: {
+            retries: number;
+            maxRetryTime: number;
+        };
+    };
+    "linger.ms": number;
+};
 type KafkaClientConfig = {
     clientId: string;
     broker: string;
@@ -55,8 +74,12 @@ type KafkaClientConfig = {
 /**
  * Dynamically creates and connects a KafkaJS producer using the provided configuration.
  * Returns a connected producer instance.
+ *
+ * @param cfg - Kafka client configuration
+ * @param logger - Logger instance
+ * @param maxMessageBytes - Optional max message size in bytes (synced with topic config)
  */
-declare function getKafkaProducer(cfg: KafkaClientConfig, logger: Logger): Promise<Producer>;
+declare function getKafkaProducer(cfg: KafkaClientConfig, logger: Logger, maxMessageBytes?: number): Promise<Producer>;
 /**
  * Interface for logging functionality
  */
@@ -531,4 +554,4 @@ type DataModelConfig<T> = Partial<{
     parallelism?: number;
 }>;
 
-export { ACKs, ApiUtil, type CSVParsingConfig, CSV_DELIMITERS, type CliLogData, DEFAULT_CSV_CONFIG, DEFAULT_JSON_CONFIG, type DataModelConfig, DataSource, type DataSourceConfig, type ExpressRequestWithMoose, type ExtractionResult, type JSONParsingConfig, type KafkaClientConfig, type Logger, MAX_RETRIES, MAX_RETRIES_PRODUCER, MAX_RETRY_TIME_MS, MOOSE_RUNTIME_ENV_PREFIX, MooseCache, MooseClient, type Producer, RETRY_FACTOR_PRODUCER, RETRY_INITIAL_TIME_MS, type StripDateIntersection, type TaskConfig, type TaskDefinition, type TaskFunction, antiCachePath, cliLog, compilerLog, createApi, createConsumptionApi, expressMiddleware, getClickhouseClient, getFileName, getKafkaClient, getKafkaProducer, getMooseClients, getMooseUtils, isValidCSVDelimiter, logError, mapTstoJs, mooseEnvSecrets, mooseRuntimeEnv, parseCSV, parseJSON, parseJSONWithDates };
+export { ACKs, ApiUtil, type CSVParsingConfig, CSV_DELIMITERS, type CliLogData, DEFAULT_CSV_CONFIG, DEFAULT_JSON_CONFIG, type DataModelConfig, DataSource, type DataSourceConfig, type ExpressRequestWithMoose, type ExtractionResult, type JSONParsingConfig, type KafkaClientConfig, type Logger, MAX_RETRIES, MAX_RETRIES_PRODUCER, MAX_RETRY_TIME_MS, MOOSE_RUNTIME_ENV_PREFIX, MooseCache, MooseClient, type Producer, RETRY_FACTOR_PRODUCER, RETRY_INITIAL_TIME_MS, type StripDateIntersection, type TaskConfig, type TaskDefinition, type TaskFunction, antiCachePath, cliLog, compilerLog, createApi, createConsumptionApi, createProducerConfig, expressMiddleware, getClickhouseClient, getFileName, getKafkaClient, getKafkaProducer, getMooseClients, getMooseUtils, isValidCSVDelimiter, logError, mapTstoJs, mooseEnvSecrets, mooseRuntimeEnv, parseCSV, parseJSON, parseJSONWithDates };

package/dist/index.js

@@ -42,6 +42,7 @@ __export(commons_exports, {
   antiCachePath: () => antiCachePath,
   cliLog: () => cliLog,
   compilerLog: () => compilerLog,
+  createProducerConfig: () => createProducerConfig,
   getClickhouseClient: () => getClickhouseClient,
   getFileName: () => getFileName,
   getKafkaClient: () => getKafkaClient,
@@ -64,18 +65,25 @@ function isTruthy(value) {
 function mapTstoJs(filePath) {
   return filePath.replace(/\.ts$/, ".js").replace(/\.cts$/, ".cjs").replace(/\.mts$/, ".mjs");
 }
-async function getKafkaProducer(cfg, logger) {
-  const kafka = await getKafkaClient(cfg, logger);
-  const producer = kafka.producer({
+function createProducerConfig(maxMessageBytes) {
+  return {
     kafkaJS: {
-      idempotent: true,
+      idempotent: false,
+      // Not needed for at-least-once delivery
       acks: ACKs,
       retry: {
         retries: MAX_RETRIES_PRODUCER,
         maxRetryTime: MAX_RETRY_TIME_MS
       }
-    }
-  });
+    },
+    "linger.ms": 0,
+    // Send immediately - batching happens at application level
+    ...maxMessageBytes && { "message.max.bytes": maxMessageBytes }
+  };
+}
+async function getKafkaProducer(cfg, logger, maxMessageBytes) {
+  const kafka = await getKafkaClient(cfg, logger);
+  const producer = kafka.producer(createProducerConfig(maxMessageBytes));
   await producer.connect();
   return producer;
 }
@@ -417,6 +425,7 @@ __export(index_exports, {
   createClickhouseParameter: () => createClickhouseParameter,
   createConsumptionApi: () => createConsumptionApi,
   createMaterializedView: () => createMaterializedView,
+  createProducerConfig: () => createProducerConfig,
   dropView: () => dropView,
   expressMiddleware: () => expressMiddleware,
   getApi: () => getApi,
@@ -716,7 +725,6 @@ var ClickHouseEngines = /* @__PURE__ */ ((ClickHouseEngines2) => {
   ClickHouseEngines2["Buffer"] = "Buffer";
   ClickHouseEngines2["Distributed"] = "Distributed";
   ClickHouseEngines2["IcebergS3"] = "IcebergS3";
-  ClickHouseEngines2["Kafka"] = "Kafka";
   ClickHouseEngines2["ReplicatedMergeTree"] = "ReplicatedMergeTree";
   ClickHouseEngines2["ReplicatedReplacingMergeTree"] = "ReplicatedReplacingMergeTree";
   ClickHouseEngines2["ReplicatedAggregatingMergeTree"] = "ReplicatedAggregatingMergeTree";
@@ -3399,6 +3407,7 @@ var DataSource = class {
   createClickhouseParameter,
   createConsumptionApi,
   createMaterializedView,
+  createProducerConfig,
   dropView,
   expressMiddleware,
   getApi,