@infersec/conduit 1.109.1 → 1.109.2

package/dist/cli.js

@@ -126399,6 +126399,17 @@ async function streamResponse({ apiURL, configuration, logger, requestID, reques
 function encodeBinaryChunk(chunk) {
     return chunk.toString("base64");
 }
+const FLUSH_MAX_BATCH = 16;
+const FLUSH_WINDOW_MS = 60;
+/**
+ * Decides whether the chunk queue must flush at a chunk arrival. Age-based,
+ * so cadence adapts to generation speed: slow streams flush almost
+ * immediately, fast streams accumulate into larger batches. The max batch
+ * guards bursts faster than any flush roundtrip can absorb.
+ */
+function shouldFlushChunkBatch({ ageMs, count, isTerminal, maxBatch, windowMs }) {
+    return isTerminal || ageMs >= windowMs || count >= maxBatch;
+}
 async function sendChunkStream({ apiURL, configuration, requestID, logger }) {
     const streamURL = `${apiURL}/conduit/api/v1/source/${configuration.inferenceSourceID}/requests/${requestID}/stream`;
     const maxFlushAttempts = 3;
@@ -126407,6 +126418,7 @@ async function sendChunkStream({ apiURL, configuration, requestID, logger }) {
     let isClosed = false;
     let activeAbortController = null;
     let consecutiveFlushFailures = 0;
+    let pendingSince = null;
     const chunks = [];
     const sendChunk = async (payload) => {
         if (isAborted || isClosed) {
@@ -126422,9 +126434,22 @@ async function sendChunkStream({ apiURL, configuration, requestID, logger }) {
             ...response,
             sequence: payload.sequence
         });
+        const now = Date.now();
+        if (pendingSince === null) {
+            pendingSince = now;
+        }
         chunks.push(Buffer.from(chunk + "\n"));
-        if (payload.data === null || chunks.length >= 10) {
+        if (shouldFlushChunkBatch({
+            ageMs: now - pendingSince,
+            count: chunks.length,
+            isTerminal: payload.data === null,
+            maxBatch: FLUSH_MAX_BATCH,
+            windowMs: FLUSH_WINDOW_MS
+        })) {
             await flushChunks();
+            if (chunks.length === 0) {
+                pendingSince = null;
+            }
         }
     };
     const flushChunks = async () => {
@@ -126493,6 +126518,7 @@ async function sendChunkStream({ apiURL, configuration, requestID, logger }) {
             activeAbortController.abort();
         }
         chunks.length = 0;
+        pendingSince = null;
         if (error) {
             logger.error("Chunk stream aborted", {
                 error: asError(error)

package/dist/cli.sea.cjs

@@ -126414,6 +126414,17 @@ async function streamResponse({ apiURL, configuration, logger, requestID, reques
 function encodeBinaryChunk(chunk) {
     return chunk.toString("base64");
 }
+const FLUSH_MAX_BATCH = 16;
+const FLUSH_WINDOW_MS = 60;
+/**
+ * Decides whether the chunk queue must flush at a chunk arrival. Age-based,
+ * so cadence adapts to generation speed: slow streams flush almost
+ * immediately, fast streams accumulate into larger batches. The max batch
+ * guards bursts faster than any flush roundtrip can absorb.
+ */
+function shouldFlushChunkBatch({ ageMs, count, isTerminal, maxBatch, windowMs }) {
+    return isTerminal || ageMs >= windowMs || count >= maxBatch;
+}
 async function sendChunkStream({ apiURL, configuration, requestID, logger }) {
     const streamURL = `${apiURL}/conduit/api/v1/source/${configuration.inferenceSourceID}/requests/${requestID}/stream`;
     const maxFlushAttempts = 3;
@@ -126422,6 +126433,7 @@ async function sendChunkStream({ apiURL, configuration, requestID, logger }) {
     let isClosed = false;
     let activeAbortController = null;
     let consecutiveFlushFailures = 0;
+    let pendingSince = null;
     const chunks = [];
     const sendChunk = async (payload) => {
         if (isAborted || isClosed) {
@@ -126437,9 +126449,22 @@ async function sendChunkStream({ apiURL, configuration, requestID, logger }) {
             ...response,
             sequence: payload.sequence
         });
+        const now = Date.now();
+        if (pendingSince === null) {
+            pendingSince = now;
+        }
         chunks.push(Buffer.from(chunk + "\n"));
-        if (payload.data === null || chunks.length >= 10) {
+        if (shouldFlushChunkBatch({
+            ageMs: now - pendingSince,
+            count: chunks.length,
+            isTerminal: payload.data === null,
+            maxBatch: FLUSH_MAX_BATCH,
+            windowMs: FLUSH_WINDOW_MS
+        })) {
             await flushChunks();
+            if (chunks.length === 0) {
+                pendingSince = null;
+            }
         }
     };
     const flushChunks = async () => {
@@ -126508,6 +126533,7 @@ async function sendChunkStream({ apiURL, configuration, requestID, logger }) {
             activeAbortController.abort();
         }
         chunks.length = 0;
+        pendingSince = null;
         if (error) {
             logger.error("Chunk stream aborted", {
                 error: asError(error)

package/dist/sse/handler.d.ts

@@ -13,3 +13,16 @@ export declare function handleSSERequests({ apiURL, configuration, logger, onReq
     onRequestStart?: (request: ServerToClientAPIRequest) => Promise<void> | void;
     signal?: AbortSignal;
 }): Promise<void>;
+/**
+ * Decides whether the chunk queue must flush at a chunk arrival. Age-based,
+ * so cadence adapts to generation speed: slow streams flush almost
+ * immediately, fast streams accumulate into larger batches. The max batch
+ * guards bursts faster than any flush roundtrip can absorb.
+ */
+export declare function shouldFlushChunkBatch({ ageMs, count, isTerminal, maxBatch, windowMs }: {
+    ageMs: number;
+    count: number;
+    isTerminal: boolean;
+    maxBatch: number;
+    windowMs: number;
+}): boolean;

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@infersec/conduit",
   "description": "End user conduit agent for connecting local LLMs to the cloud.",
-  "version": "1.109.1",
+  "version": "1.109.2",
   "bin": {
     "infersec-conduit": "./dist/cli.js"
   },