@514labs/moose-lib 0.6.252-ci-3-g37e54b29 → 0.6.252-ci-3-gf88b2b20

package/dist/{index-rECDLgTX.d.mts → index-CNlTyF6R.d.mts}

@@ -1894,7 +1894,6 @@ interface FrameworkApp {
     handle?: (req: http.IncomingMessage, res: http.ServerResponse, next?: (err?: any) => void) => void;
     callback?: () => WebAppHandler;
     routing?: (req: http.IncomingMessage, res: http.ServerResponse) => void;
-    ready?: () => PromiseLike<unknown>;
 }
 interface WebAppConfig {
     mountPath: string;

package/dist/{index-rECDLgTX.d.ts → index-CNlTyF6R.d.ts}

@@ -1894,7 +1894,6 @@ interface FrameworkApp {
     handle?: (req: http.IncomingMessage, res: http.ServerResponse, next?: (err?: any) => void) => void;
     callback?: () => WebAppHandler;
     routing?: (req: http.IncomingMessage, res: http.ServerResponse) => void;
-    ready?: () => PromiseLike<unknown>;
 }
 interface WebAppConfig {
     mountPath: string;

package/dist/index.d.mts

@@ -1,6 +1,6 @@
-export { C as ClickHouseByteSize, q as ClickHouseCodec, j as ClickHouseDecimal, n as ClickHouseDefault, k as ClickHouseFixedStringSize, l as ClickHouseFloat, a as ClickHouseInt, m as ClickHouseJson, e as ClickHouseLineString, p as ClickHouseMaterialized, f as ClickHouseMultiLineString, h as ClickHouseMultiPolygon, b as ClickHouseNamedTuple, c as ClickHousePoint, g as ClickHousePolygon, i as ClickHousePrecision, d as ClickHouseRing, o as ClickHouseTTL, D as DateTime, r as DateTime64, t as DateTime64String, s as DateTimeString, E as Decimal, F as FixedString, u as Float32, v as Float64, w as Int16, x as Int32, y as Int64, I as Int8, J as JWT, K as Key, L as LowCardinality, z as UInt16, A as UInt32, B as UInt64, U as UInt8, W as WithDefault } from './browserCompatible-DnYA4Zgi.mjs';
-import { K as ApiUtil, a4 as MooseClient } from './index-rECDLgTX.mjs';
-export { A as Aggregated, h as Api, i as ApiConfig, ad as ApiHelpers, a5 as Blocks, a6 as ClickHouseEngines, C as ConsumptionApi, ae as ConsumptionHelpers, N as ConsumptionUtil, e as DeadLetter, D as DeadLetterModel, f as DeadLetterQueue, l as ETLPipeline, m as ETLPipelineConfig, E as EgressConfig, F as FrameworkApp, Q as IdentifierBrandedString, I as IngestApi, g as IngestConfig, j as IngestPipeline, L as LifeCycle, M as MaterializedView, R as NonIdentifierBrandedString, a as OlapConfig, O as OlapTable, aa as QueryClient, X as RawValue, b as S3QueueTableSettings, S as SimpleAggregated, Z as Sql, k as SqlResource, c as Stream, d as StreamConfig, T as Task, U as Value, V as View, n as WebApp, o as WebAppConfig, p as WebAppHandler, W as Workflow, ab as WorkflowClient, a2 as createClickhouseParameter, a8 as createMaterializedView, a7 as dropView, x as getApi, w as getApis, v as getIngestApi, u as getIngestApis, z as getSqlResource, y as getSqlResources, t as getStream, s as getStreams, r as getTable, q as getTables, ac as getTemporalClient, a1 as getValueFromParameter, J as getWebApp, H as getWebApps, G as getWorkflow, B as getWorkflows, af as joinQueries, a3 as mapToClickHouseType, a9 as populateTable, P as quoteIdentifier, Y as sql, $ as toQuery, a0 as toQueryPreview, _ as toStaticQuery } from './index-rECDLgTX.mjs';
+export { C as ClickHouseByteSize, q as ClickHouseCodec, j as ClickHouseDecimal, n as ClickHouseDefault, k as ClickHouseFixedStringSize, l as ClickHouseFloat, a as ClickHouseInt, m as ClickHouseJson, e as ClickHouseLineString, p as ClickHouseMaterialized, f as ClickHouseMultiLineString, h as ClickHouseMultiPolygon, b as ClickHouseNamedTuple, c as ClickHousePoint, g as ClickHousePolygon, i as ClickHousePrecision, d as ClickHouseRing, o as ClickHouseTTL, D as DateTime, r as DateTime64, t as DateTime64String, s as DateTimeString, E as Decimal, F as FixedString, u as Float32, v as Float64, w as Int16, x as Int32, y as Int64, I as Int8, J as JWT, K as Key, L as LowCardinality, z as UInt16, A as UInt32, B as UInt64, U as UInt8, W as WithDefault } from './browserCompatible-BTN82akc.mjs';
+import { K as ApiUtil, a4 as MooseClient } from './index-CNlTyF6R.mjs';
+export { A as Aggregated, h as Api, i as ApiConfig, ad as ApiHelpers, a5 as Blocks, a6 as ClickHouseEngines, C as ConsumptionApi, ae as ConsumptionHelpers, N as ConsumptionUtil, e as DeadLetter, D as DeadLetterModel, f as DeadLetterQueue, l as ETLPipeline, m as ETLPipelineConfig, E as EgressConfig, F as FrameworkApp, Q as IdentifierBrandedString, I as IngestApi, g as IngestConfig, j as IngestPipeline, L as LifeCycle, M as MaterializedView, R as NonIdentifierBrandedString, a as OlapConfig, O as OlapTable, aa as QueryClient, X as RawValue, b as S3QueueTableSettings, S as SimpleAggregated, Z as Sql, k as SqlResource, c as Stream, d as StreamConfig, T as Task, U as Value, V as View, n as WebApp, o as WebAppConfig, p as WebAppHandler, W as Workflow, ab as WorkflowClient, a2 as createClickhouseParameter, a8 as createMaterializedView, a7 as dropView, x as getApi, w as getApis, v as getIngestApi, u as getIngestApis, z as getSqlResource, y as getSqlResources, t as getStream, s as getStreams, r as getTable, q as getTables, ac as getTemporalClient, a1 as getValueFromParameter, J as getWebApp, H as getWebApps, G as getWorkflow, B as getWorkflows, af as joinQueries, a3 as mapToClickHouseType, a9 as populateTable, P as quoteIdentifier, Y as sql, $ as toQuery, a0 as toQueryPreview, _ as toStaticQuery } from './index-CNlTyF6R.mjs';
 import * as _clickhouse_client from '@clickhouse/client';
 import { KafkaJS } from '@confluentinc/kafka-javascript';
 import http from 'http';
@@ -44,6 +44,25 @@ declare const RETRY_INITIAL_TIME_MS = 100;
 declare const MAX_RETRIES_PRODUCER = 150;
 declare const RETRY_FACTOR_PRODUCER = 0.2;
 declare const ACKs = -1;
+/**
+ * Creates the base producer configuration for Kafka.
+ * Used by both the SDK stream publishing and streaming function workers.
+ *
+ * @param maxMessageBytes - Optional max message size in bytes (synced with topic config)
+ * @returns Producer configuration object for the Confluent Kafka client
+ */
+declare function createProducerConfig(maxMessageBytes?: number): {
+    "message.max.bytes"?: number | undefined;
+    kafkaJS: {
+        idempotent: boolean;
+        acks: number;
+        retry: {
+            retries: number;
+            maxRetryTime: number;
+        };
+    };
+    "linger.ms": number;
+};
 type KafkaClientConfig = {
     clientId: string;
     broker: string;
@@ -55,8 +74,12 @@ type KafkaClientConfig = {
 /**
  * Dynamically creates and connects a KafkaJS producer using the provided configuration.
  * Returns a connected producer instance.
+ *
+ * @param cfg - Kafka client configuration
+ * @param logger - Logger instance
+ * @param maxMessageBytes - Optional max message size in bytes (synced with topic config)
  */
-declare function getKafkaProducer(cfg: KafkaClientConfig, logger: Logger): Promise<Producer>;
+declare function getKafkaProducer(cfg: KafkaClientConfig, logger: Logger, maxMessageBytes?: number): Promise<Producer>;
 /**
  * Interface for logging functionality
  */
@@ -531,4 +554,4 @@ type DataModelConfig<T> = Partial<{
     parallelism?: number;
 }>;
 
-export { ACKs, ApiUtil, type CSVParsingConfig, CSV_DELIMITERS, type CliLogData, DEFAULT_CSV_CONFIG, DEFAULT_JSON_CONFIG, type DataModelConfig, DataSource, type DataSourceConfig, type ExpressRequestWithMoose, type ExtractionResult, type JSONParsingConfig, type KafkaClientConfig, type Logger, MAX_RETRIES, MAX_RETRIES_PRODUCER, MAX_RETRY_TIME_MS, MOOSE_RUNTIME_ENV_PREFIX, MooseCache, MooseClient, type Producer, RETRY_FACTOR_PRODUCER, RETRY_INITIAL_TIME_MS, type StripDateIntersection, type TaskConfig, type TaskDefinition, type TaskFunction, antiCachePath, cliLog, compilerLog, createApi, createConsumptionApi, expressMiddleware, getClickhouseClient, getFileName, getKafkaClient, getKafkaProducer, getMooseClients, getMooseUtils, isValidCSVDelimiter, logError, mapTstoJs, mooseEnvSecrets, mooseRuntimeEnv, parseCSV, parseJSON, parseJSONWithDates };
+export { ACKs, ApiUtil, type CSVParsingConfig, CSV_DELIMITERS, type CliLogData, DEFAULT_CSV_CONFIG, DEFAULT_JSON_CONFIG, type DataModelConfig, DataSource, type DataSourceConfig, type ExpressRequestWithMoose, type ExtractionResult, type JSONParsingConfig, type KafkaClientConfig, type Logger, MAX_RETRIES, MAX_RETRIES_PRODUCER, MAX_RETRY_TIME_MS, MOOSE_RUNTIME_ENV_PREFIX, MooseCache, MooseClient, type Producer, RETRY_FACTOR_PRODUCER, RETRY_INITIAL_TIME_MS, type StripDateIntersection, type TaskConfig, type TaskDefinition, type TaskFunction, antiCachePath, cliLog, compilerLog, createApi, createConsumptionApi, createProducerConfig, expressMiddleware, getClickhouseClient, getFileName, getKafkaClient, getKafkaProducer, getMooseClients, getMooseUtils, isValidCSVDelimiter, logError, mapTstoJs, mooseEnvSecrets, mooseRuntimeEnv, parseCSV, parseJSON, parseJSONWithDates };

package/dist/index.d.ts

@@ -1,6 +1,6 @@
-export { C as ClickHouseByteSize, q as ClickHouseCodec, j as ClickHouseDecimal, n as ClickHouseDefault, k as ClickHouseFixedStringSize, l as ClickHouseFloat, a as ClickHouseInt, m as ClickHouseJson, e as ClickHouseLineString, p as ClickHouseMaterialized, f as ClickHouseMultiLineString, h as ClickHouseMultiPolygon, b as ClickHouseNamedTuple, c as ClickHousePoint, g as ClickHousePolygon, i as ClickHousePrecision, d as ClickHouseRing, o as ClickHouseTTL, D as DateTime, r as DateTime64, t as DateTime64String, s as DateTimeString, E as Decimal, F as FixedString, u as Float32, v as Float64, w as Int16, x as Int32, y as Int64, I as Int8, J as JWT, K as Key, L as LowCardinality, z as UInt16, A as UInt32, B as UInt64, U as UInt8, W as WithDefault } from './browserCompatible-B_aEflr_.js';
-import { K as ApiUtil, a4 as MooseClient } from './index-rECDLgTX.js';
-export { A as Aggregated, h as Api, i as ApiConfig, ad as ApiHelpers, a5 as Blocks, a6 as ClickHouseEngines, C as ConsumptionApi, ae as ConsumptionHelpers, N as ConsumptionUtil, e as DeadLetter, D as DeadLetterModel, f as DeadLetterQueue, l as ETLPipeline, m as ETLPipelineConfig, E as EgressConfig, F as FrameworkApp, Q as IdentifierBrandedString, I as IngestApi, g as IngestConfig, j as IngestPipeline, L as LifeCycle, M as MaterializedView, R as NonIdentifierBrandedString, a as OlapConfig, O as OlapTable, aa as QueryClient, X as RawValue, b as S3QueueTableSettings, S as SimpleAggregated, Z as Sql, k as SqlResource, c as Stream, d as StreamConfig, T as Task, U as Value, V as View, n as WebApp, o as WebAppConfig, p as WebAppHandler, W as Workflow, ab as WorkflowClient, a2 as createClickhouseParameter, a8 as createMaterializedView, a7 as dropView, x as getApi, w as getApis, v as getIngestApi, u as getIngestApis, z as getSqlResource, y as getSqlResources, t as getStream, s as getStreams, r as getTable, q as getTables, ac as getTemporalClient, a1 as getValueFromParameter, J as getWebApp, H as getWebApps, G as getWorkflow, B as getWorkflows, af as joinQueries, a3 as mapToClickHouseType, a9 as populateTable, P as quoteIdentifier, Y as sql, $ as toQuery, a0 as toQueryPreview, _ as toStaticQuery } from './index-rECDLgTX.js';
+export { C as ClickHouseByteSize, q as ClickHouseCodec, j as ClickHouseDecimal, n as ClickHouseDefault, k as ClickHouseFixedStringSize, l as ClickHouseFloat, a as ClickHouseInt, m as ClickHouseJson, e as ClickHouseLineString, p as ClickHouseMaterialized, f as ClickHouseMultiLineString, h as ClickHouseMultiPolygon, b as ClickHouseNamedTuple, c as ClickHousePoint, g as ClickHousePolygon, i as ClickHousePrecision, d as ClickHouseRing, o as ClickHouseTTL, D as DateTime, r as DateTime64, t as DateTime64String, s as DateTimeString, E as Decimal, F as FixedString, u as Float32, v as Float64, w as Int16, x as Int32, y as Int64, I as Int8, J as JWT, K as Key, L as LowCardinality, z as UInt16, A as UInt32, B as UInt64, U as UInt8, W as WithDefault } from './browserCompatible-CoxWbneN.js';
+import { K as ApiUtil, a4 as MooseClient } from './index-CNlTyF6R.js';
+export { A as Aggregated, h as Api, i as ApiConfig, ad as ApiHelpers, a5 as Blocks, a6 as ClickHouseEngines, C as ConsumptionApi, ae as ConsumptionHelpers, N as ConsumptionUtil, e as DeadLetter, D as DeadLetterModel, f as DeadLetterQueue, l as ETLPipeline, m as ETLPipelineConfig, E as EgressConfig, F as FrameworkApp, Q as IdentifierBrandedString, I as IngestApi, g as IngestConfig, j as IngestPipeline, L as LifeCycle, M as MaterializedView, R as NonIdentifierBrandedString, a as OlapConfig, O as OlapTable, aa as QueryClient, X as RawValue, b as S3QueueTableSettings, S as SimpleAggregated, Z as Sql, k as SqlResource, c as Stream, d as StreamConfig, T as Task, U as Value, V as View, n as WebApp, o as WebAppConfig, p as WebAppHandler, W as Workflow, ab as WorkflowClient, a2 as createClickhouseParameter, a8 as createMaterializedView, a7 as dropView, x as getApi, w as getApis, v as getIngestApi, u as getIngestApis, z as getSqlResource, y as getSqlResources, t as getStream, s as getStreams, r as getTable, q as getTables, ac as getTemporalClient, a1 as getValueFromParameter, J as getWebApp, H as getWebApps, G as getWorkflow, B as getWorkflows, af as joinQueries, a3 as mapToClickHouseType, a9 as populateTable, P as quoteIdentifier, Y as sql, $ as toQuery, a0 as toQueryPreview, _ as toStaticQuery } from './index-CNlTyF6R.js';
 import * as _clickhouse_client from '@clickhouse/client';
 import { KafkaJS } from '@confluentinc/kafka-javascript';
 import http from 'http';
@@ -44,6 +44,25 @@ declare const RETRY_INITIAL_TIME_MS = 100;
 declare const MAX_RETRIES_PRODUCER = 150;
 declare const RETRY_FACTOR_PRODUCER = 0.2;
 declare const ACKs = -1;
+/**
+ * Creates the base producer configuration for Kafka.
+ * Used by both the SDK stream publishing and streaming function workers.
+ *
+ * @param maxMessageBytes - Optional max message size in bytes (synced with topic config)
+ * @returns Producer configuration object for the Confluent Kafka client
+ */
+declare function createProducerConfig(maxMessageBytes?: number): {
+    "message.max.bytes"?: number | undefined;
+    kafkaJS: {
+        idempotent: boolean;
+        acks: number;
+        retry: {
+            retries: number;
+            maxRetryTime: number;
+        };
+    };
+    "linger.ms": number;
+};
 type KafkaClientConfig = {
     clientId: string;
     broker: string;
@@ -55,8 +74,12 @@ type KafkaClientConfig = {
 /**
  * Dynamically creates and connects a KafkaJS producer using the provided configuration.
  * Returns a connected producer instance.
+ *
+ * @param cfg - Kafka client configuration
+ * @param logger - Logger instance
+ * @param maxMessageBytes - Optional max message size in bytes (synced with topic config)
  */
-declare function getKafkaProducer(cfg: KafkaClientConfig, logger: Logger): Promise<Producer>;
+declare function getKafkaProducer(cfg: KafkaClientConfig, logger: Logger, maxMessageBytes?: number): Promise<Producer>;
 /**
  * Interface for logging functionality
  */
@@ -531,4 +554,4 @@ type DataModelConfig<T> = Partial<{
     parallelism?: number;
 }>;
 
-export { ACKs, ApiUtil, type CSVParsingConfig, CSV_DELIMITERS, type CliLogData, DEFAULT_CSV_CONFIG, DEFAULT_JSON_CONFIG, type DataModelConfig, DataSource, type DataSourceConfig, type ExpressRequestWithMoose, type ExtractionResult, type JSONParsingConfig, type KafkaClientConfig, type Logger, MAX_RETRIES, MAX_RETRIES_PRODUCER, MAX_RETRY_TIME_MS, MOOSE_RUNTIME_ENV_PREFIX, MooseCache, MooseClient, type Producer, RETRY_FACTOR_PRODUCER, RETRY_INITIAL_TIME_MS, type StripDateIntersection, type TaskConfig, type TaskDefinition, type TaskFunction, antiCachePath, cliLog, compilerLog, createApi, createConsumptionApi, expressMiddleware, getClickhouseClient, getFileName, getKafkaClient, getKafkaProducer, getMooseClients, getMooseUtils, isValidCSVDelimiter, logError, mapTstoJs, mooseEnvSecrets, mooseRuntimeEnv, parseCSV, parseJSON, parseJSONWithDates };
+export { ACKs, ApiUtil, type CSVParsingConfig, CSV_DELIMITERS, type CliLogData, DEFAULT_CSV_CONFIG, DEFAULT_JSON_CONFIG, type DataModelConfig, DataSource, type DataSourceConfig, type ExpressRequestWithMoose, type ExtractionResult, type JSONParsingConfig, type KafkaClientConfig, type Logger, MAX_RETRIES, MAX_RETRIES_PRODUCER, MAX_RETRY_TIME_MS, MOOSE_RUNTIME_ENV_PREFIX, MooseCache, MooseClient, type Producer, RETRY_FACTOR_PRODUCER, RETRY_INITIAL_TIME_MS, type StripDateIntersection, type TaskConfig, type TaskDefinition, type TaskFunction, antiCachePath, cliLog, compilerLog, createApi, createConsumptionApi, createProducerConfig, expressMiddleware, getClickhouseClient, getFileName, getKafkaClient, getKafkaProducer, getMooseClients, getMooseUtils, isValidCSVDelimiter, logError, mapTstoJs, mooseEnvSecrets, mooseRuntimeEnv, parseCSV, parseJSON, parseJSONWithDates };

package/dist/index.js

@@ -42,6 +42,7 @@ __export(commons_exports, {
   antiCachePath: () => antiCachePath,
   cliLog: () => cliLog,
   compilerLog: () => compilerLog,
+  createProducerConfig: () => createProducerConfig,
   getClickhouseClient: () => getClickhouseClient,
   getFileName: () => getFileName,
   getKafkaClient: () => getKafkaClient,
@@ -64,18 +65,25 @@ function isTruthy(value) {
 function mapTstoJs(filePath) {
   return filePath.replace(/\.ts$/, ".js").replace(/\.cts$/, ".cjs").replace(/\.mts$/, ".mjs");
 }
-async function getKafkaProducer(cfg, logger) {
-  const kafka = await getKafkaClient(cfg, logger);
-  const producer = kafka.producer({
+function createProducerConfig(maxMessageBytes) {
+  return {
     kafkaJS: {
-      idempotent: true,
+      idempotent: false,
+      // Not needed for at-least-once delivery
       acks: ACKs,
       retry: {
         retries: MAX_RETRIES_PRODUCER,
         maxRetryTime: MAX_RETRY_TIME_MS
       }
-    }
-  });
+    },
+    "linger.ms": 0,
+    // Send immediately - batching happens at application level
+    ...maxMessageBytes && { "message.max.bytes": maxMessageBytes }
+  };
+}
+async function getKafkaProducer(cfg, logger, maxMessageBytes) {
+  const kafka = await getKafkaClient(cfg, logger);
+  const producer = kafka.producer(createProducerConfig(maxMessageBytes));
   await producer.connect();
   return producer;
 }
@@ -417,6 +425,7 @@ __export(index_exports, {
   createClickhouseParameter: () => createClickhouseParameter,
   createConsumptionApi: () => createConsumptionApi,
   createMaterializedView: () => createMaterializedView,
+  createProducerConfig: () => createProducerConfig,
   dropView: () => dropView,
   expressMiddleware: () => expressMiddleware,
   getApi: () => getApi,
@@ -2617,19 +2626,7 @@ var WebApp = class {
       return app.callback();
     }
     if (typeof app.routing === "function") {
-      let isReady = false;
-      let readyPromise = null;
-      if (typeof app.ready === "function") {
-        readyPromise = app.ready().then(() => {
-          isReady = true;
-        });
-      } else {
-        isReady = true;
-      }
-      return async (req, res) => {
-        if (!isReady && readyPromise) {
-          await readyPromise;
-        }
+      return (req, res) => {
         app.routing(req, res);
       };
     }
@@ -3410,6 +3407,7 @@ var DataSource = class {
   createClickhouseParameter,
   createConsumptionApi,
   createMaterializedView,
+  createProducerConfig,
   dropView,
   expressMiddleware,
   getApi,