@infersec/conduit 1.109.0 → 1.109.2

package/dist/cli.js

@@ -69402,6 +69402,7 @@ function validateAndExtract(type, data, schemas) {
 function containsStatusOnly(payload) {
     return !!payload && typeof payload === "object" && "statusText" in payload;
 }
+const KEEPALIVE_INTERVAL_MS = 15_000;
 function sendTextStreamResponse({ logger, res, stream }) {
     const failStream = (error) => {
         if (res.writableEnded || res.destroyed)
@@ -69427,10 +69428,38 @@ function sendTextStreamResponse({ logger, res, stream }) {
         })}\n\n`);
         res.end();
     };
+    let keepalive = null;
+    let streamEnded = false;
+    const stopKeepalive = () => {
+        if (keepalive !== null) {
+            clearInterval(keepalive);
+            keepalive = null;
+        }
+    };
+    res.on("close", stopKeepalive);
+    res.flushHeaders();
+    stream.on("close", () => {
+        stopKeepalive();
+        if (!streamEnded && !res.writableEnded && !res.destroyed) {
+            failStream(new Error("Upstream stream closed prematurely"));
+        }
+    });
+    stream.on("end", () => {
+        streamEnded = true;
+        stopKeepalive();
+    });
     stream.on("error", error => {
+        stopKeepalive();
         failStream(error instanceof Error ? error : new Error(String(error)));
     });
     stream.pipe(res);
+    keepalive = setInterval(() => {
+        if (res.writableEnded || res.destroyed) {
+            stopKeepalive();
+            return;
+        }
+        res.write(": keepalive\n\n");
+    }, KEEPALIVE_INTERVAL_MS);
 }
 function implementSingleEndpoint({ endpoint, handler, logger, method, mount, route }) {
     const expressRoute = getHTTPMethod(mount, method);
@@ -126370,6 +126399,17 @@ async function streamResponse({ apiURL, configuration, logger, requestID, reques
 function encodeBinaryChunk(chunk) {
     return chunk.toString("base64");
 }
+const FLUSH_MAX_BATCH = 16;
+const FLUSH_WINDOW_MS = 60;
+/**
+ * Decides whether the chunk queue must flush at a chunk arrival. Age-based,
+ * so cadence adapts to generation speed: slow streams flush almost
+ * immediately, fast streams accumulate into larger batches. The max batch
+ * guards bursts faster than any flush roundtrip can absorb.
+ */
+function shouldFlushChunkBatch({ ageMs, count, isTerminal, maxBatch, windowMs }) {
+    return isTerminal || ageMs >= windowMs || count >= maxBatch;
+}
 async function sendChunkStream({ apiURL, configuration, requestID, logger }) {
     const streamURL = `${apiURL}/conduit/api/v1/source/${configuration.inferenceSourceID}/requests/${requestID}/stream`;
     const maxFlushAttempts = 3;
@@ -126378,6 +126418,7 @@ async function sendChunkStream({ apiURL, configuration, requestID, logger }) {
     let isClosed = false;
     let activeAbortController = null;
     let consecutiveFlushFailures = 0;
+    let pendingSince = null;
     const chunks = [];
     const sendChunk = async (payload) => {
         if (isAborted || isClosed) {
@@ -126393,9 +126434,22 @@ async function sendChunkStream({ apiURL, configuration, requestID, logger }) {
             ...response,
             sequence: payload.sequence
         });
+        const now = Date.now();
+        if (pendingSince === null) {
+            pendingSince = now;
+        }
         chunks.push(Buffer.from(chunk + "\n"));
-        if (payload.data === null || chunks.length >= 10) {
+        if (shouldFlushChunkBatch({
+            ageMs: now - pendingSince,
+            count: chunks.length,
+            isTerminal: payload.data === null,
+            maxBatch: FLUSH_MAX_BATCH,
+            windowMs: FLUSH_WINDOW_MS
+        })) {
             await flushChunks();
+            if (chunks.length === 0) {
+                pendingSince = null;
+            }
         }
     };
     const flushChunks = async () => {
@@ -126464,6 +126518,7 @@ async function sendChunkStream({ apiURL, configuration, requestID, logger }) {
             activeAbortController.abort();
         }
         chunks.length = 0;
+        pendingSince = null;
         if (error) {
             logger.error("Chunk stream aborted", {
                 error: asError(error)

package/dist/cli.sea.cjs

@@ -69417,6 +69417,7 @@ function validateAndExtract(type, data, schemas) {
 function containsStatusOnly(payload) {
     return !!payload && typeof payload === "object" && "statusText" in payload;
 }
+const KEEPALIVE_INTERVAL_MS = 15_000;
 function sendTextStreamResponse({ logger, res, stream }) {
     const failStream = (error) => {
         if (res.writableEnded || res.destroyed)
@@ -69442,10 +69443,38 @@ function sendTextStreamResponse({ logger, res, stream }) {
         })}\n\n`);
         res.end();
     };
+    let keepalive = null;
+    let streamEnded = false;
+    const stopKeepalive = () => {
+        if (keepalive !== null) {
+            clearInterval(keepalive);
+            keepalive = null;
+        }
+    };
+    res.on("close", stopKeepalive);
+    res.flushHeaders();
+    stream.on("close", () => {
+        stopKeepalive();
+        if (!streamEnded && !res.writableEnded && !res.destroyed) {
+            failStream(new Error("Upstream stream closed prematurely"));
+        }
+    });
+    stream.on("end", () => {
+        streamEnded = true;
+        stopKeepalive();
+    });
     stream.on("error", error => {
+        stopKeepalive();
         failStream(error instanceof Error ? error : new Error(String(error)));
     });
     stream.pipe(res);
+    keepalive = setInterval(() => {
+        if (res.writableEnded || res.destroyed) {
+            stopKeepalive();
+            return;
+        }
+        res.write(": keepalive\n\n");
+    }, KEEPALIVE_INTERVAL_MS);
 }
 function implementSingleEndpoint({ endpoint, handler, logger, method, mount, route }) {
     const expressRoute = getHTTPMethod(mount, method);
@@ -126385,6 +126414,17 @@ async function streamResponse({ apiURL, configuration, logger, requestID, reques
 function encodeBinaryChunk(chunk) {
     return chunk.toString("base64");
 }
+const FLUSH_MAX_BATCH = 16;
+const FLUSH_WINDOW_MS = 60;
+/**
+ * Decides whether the chunk queue must flush at a chunk arrival. Age-based,
+ * so cadence adapts to generation speed: slow streams flush almost
+ * immediately, fast streams accumulate into larger batches. The max batch
+ * guards bursts faster than any flush roundtrip can absorb.
+ */
+function shouldFlushChunkBatch({ ageMs, count, isTerminal, maxBatch, windowMs }) {
+    return isTerminal || ageMs >= windowMs || count >= maxBatch;
+}
 async function sendChunkStream({ apiURL, configuration, requestID, logger }) {
     const streamURL = `${apiURL}/conduit/api/v1/source/${configuration.inferenceSourceID}/requests/${requestID}/stream`;
     const maxFlushAttempts = 3;
@@ -126393,6 +126433,7 @@ async function sendChunkStream({ apiURL, configuration, requestID, logger }) {
     let isClosed = false;
     let activeAbortController = null;
     let consecutiveFlushFailures = 0;
+    let pendingSince = null;
     const chunks = [];
     const sendChunk = async (payload) => {
         if (isAborted || isClosed) {
@@ -126408,9 +126449,22 @@ async function sendChunkStream({ apiURL, configuration, requestID, logger }) {
             ...response,
             sequence: payload.sequence
         });
+        const now = Date.now();
+        if (pendingSince === null) {
+            pendingSince = now;
+        }
         chunks.push(Buffer.from(chunk + "\n"));
-        if (payload.data === null || chunks.length >= 10) {
+        if (shouldFlushChunkBatch({
+            ageMs: now - pendingSince,
+            count: chunks.length,
+            isTerminal: payload.data === null,
+            maxBatch: FLUSH_MAX_BATCH,
+            windowMs: FLUSH_WINDOW_MS
+        })) {
             await flushChunks();
+            if (chunks.length === 0) {
+                pendingSince = null;
+            }
         }
     };
     const flushChunks = async () => {
@@ -126479,6 +126533,7 @@ async function sendChunkStream({ apiURL, configuration, requestID, logger }) {
             activeAbortController.abort();
         }
         chunks.length = 0;
+        pendingSince = null;
         if (error) {
             logger.error("Chunk stream aborted", {
                 error: asError(error)

package/dist/sse/handler.d.ts

@@ -13,3 +13,16 @@ export declare function handleSSERequests({ apiURL, configuration, logger, onReq
     onRequestStart?: (request: ServerToClientAPIRequest) => Promise<void> | void;
     signal?: AbortSignal;
 }): Promise<void>;
+/**
+ * Decides whether the chunk queue must flush at a chunk arrival. Age-based,
+ * so cadence adapts to generation speed: slow streams flush almost
+ * immediately, fast streams accumulate into larger batches. The max batch
+ * guards bursts faster than any flush roundtrip can absorb.
+ */
+export declare function shouldFlushChunkBatch({ ageMs, count, isTerminal, maxBatch, windowMs }: {
+    ageMs: number;
+    count: number;
+    isTerminal: boolean;
+    maxBatch: number;
+    windowMs: number;
+}): boolean;

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@infersec/conduit",
   "description": "End user conduit agent for connecting local LLMs to the cloud.",
-  "version": "1.109.0",
+  "version": "1.109.2",
   "bin": {
     "infersec-conduit": "./dist/cli.js"
   },