flexinference 1.5.0 → 1.5.1

package/CHANGELOG.md

@@ -1,5 +1,15 @@
 # Changelog
 
+## 1.5.1
+
+Small robustness fixes for the transport, no API changes. A non-streaming request that hits
+its total `timeout` now throws a clear timeout error instead of a bare abort that looked like
+a caller cancellation. A streamed response flushes its decoder and delivers the final frame
+even when the stream ends without a trailing blank line, so the last event is never dropped,
+and the streaming buffer is capped so a stream that never sends a separator cannot grow
+without bound. The `start_within` type now matches the runtime rule, so a value the client
+would reject at run time no longer type-checks.
+
 ## 1.5.0
 
 Reworks request timeouts so a healthy stream is never cut off yet a hung request never hangs

package/dist/index.js

@@ -132,11 +132,18 @@ const FLEX_FIRST_BYTE_MARGIN_MS = 30_000; // headroom over the flex deadline for
 const DEFAULT_IDLE_MS = 60_000; // max silence between streamed chunks before the stream is treated as hung
 const DEFAULT_TOTAL_MS = 600_000; // non-streaming wall-clock budget (streaming has none)
 const ERROR_BODY_READ_MS = 30_000; // bound the read of a non-2xx error body
+// Cap the streaming buffer above the largest legitimate single frame (a terminal
+// response object is well under this). A stream that never sends a blank-line separator
+// would otherwise grow the buffer without bound; error clearly instead. See T2.
+const MAX_SSE_BUFFER_CHARS = 8 * 1024 * 1024;
 const MIN_DEADLINE_SECONDS = 5;
 const MAX_DEADLINE_SECONDS = 600;
 const SECONDS_PER_HOUR = 3600;
 const SECONDS_PER_MINUTE = 60;
-const DURATION_RE = /^(\d{2})h-(\d{2})m-(\d{2})s$/;
+// One or two digits per field so the runtime accepts the same shapes the StartWithin
+// template-literal type does (e.g. "0h-5m-0s"): `${number}` in the type imposes no digit
+// count, so a two-digit-only regex would reject values that type-check. See T4.
+const DURATION_RE = /^(\d{1,2})h-(\d{1,2})m-(\d{1,2})s$/;
 const START_WITHIN_VALUES = '"default", "priority", "auto", or a duration "HHh-MMm-SSs"';
 // Runtime guard so plain-JS callers (who get no compile-time check) still fail
 // locally with a clear message instead of a provider round-trip. TypeScript callers
@@ -157,7 +164,7 @@ function assertStartWithin(body) {
     const match = DURATION_RE.exec(value);
     if (match === null) {
         throw new Error(`FlexInference: \`start_within\` "${value}" is not ${START_WITHIN_VALUES}. ` +
-            `A duration is two digits each for hours, minutes, and seconds, for example "00h-05m-00s" for a 5-minute deadline.`);
+            `A duration is one or two digits each for hours, minutes, and seconds, for example "00h-05m-00s" for a 5-minute deadline.`);
     }
     const seconds = Number(match[1]) * SECONDS_PER_HOUR + Number(match[2]) * SECONDS_PER_MINUTE + Number(match[3]);
     if (seconds < MIN_DEADLINE_SECONDS || seconds > MAX_DEADLINE_SECONDS) {
@@ -168,6 +175,7 @@ function assertStartWithin(body) {
 // (turned into a clear message) versus the caller's own AbortSignal (propagated as-is).
 const FIRST_BYTE_TIMEOUT = Symbol("first-byte-timeout");
 const IDLE_TIMEOUT = Symbol("idle-timeout");
+const TOTAL_TIMEOUT = Symbol("total-timeout");
 /**
  * Seconds for a flex-duration `start_within`, or null for a tier value ("default" etc.),
  * a missing field, or a malformed one. Used to auto-size the first-byte wait: the router
@@ -189,12 +197,23 @@ function flexDeadlineSeconds(body) {
 // and the abort listener. Non-streaming only: a stream is drained by streamSSE instead.
 async function readJson(open) {
     const { res, controller, cleanup, totalMs } = open;
+    // Abort with a dedicated reason so a fired total timeout can be told apart from the
+    // caller cancelling on purpose, and translated into a clear message (parity with the
+    // first-byte and idle timeouts, and with the Python SDK). See T1.
     const totalTimer = setTimeout(() => {
-        controller.abort();
+        controller.abort(TOTAL_TIMEOUT);
     }, totalMs);
     try {
         return (await res.json());
     }
+    catch (err) {
+        if (controller.signal.reason === TOTAL_TIMEOUT) {
+            throw new Error(`FlexInference: no full response from the router within ${String(totalMs)}ms ` +
+                "(total timeout). Raise `timeout` for very large non-streaming responses, or " +
+                "stream the request so a long generation is governed by the idle timeout instead.");
+        }
+        throw err;
+    }
     finally {
         clearTimeout(totalTimer);
         cleanup();
@@ -378,6 +397,19 @@ class Messages {
         return readJson(open);
     }
 }
+function parseSSEFrame(frame) {
+    const dataLines = [];
+    for (const line of frame.split("\n")) {
+        if (line.startsWith("data:"))
+            dataLines.push(line.slice(5).replace(/^ /, ""));
+    }
+    if (dataLines.length === 0)
+        return null;
+    const data = dataLines.join("\n");
+    if (data === "[DONE]")
+        return { kind: "done" };
+    return { kind: "value", value: JSON.parse(data) };
+}
 async function* streamSSE(open) {
     const { res, controller, idleMs, cleanup } = open;
     if (!res.body) {
@@ -411,7 +443,18 @@ async function* streamSSE(open) {
             }
             const { done, value } = chunk;
             if (done) {
+                // The stream ended. Flush any bytes the decoder held back for an incomplete
+                // multibyte sequence, then emit a final frame that arrived without its trailing
+                // blank line so the last piece of the answer is not dropped. See T3.
                 exhausted = true;
+                buffer += decoder.decode();
+                const frame = buffer;
+                buffer = "";
+                if (frame.length > 0) {
+                    const parsed = parseSSEFrame(frame);
+                    if (parsed !== null && parsed.kind === "value")
+                        yield parsed.value;
+                }
                 break;
             }
             buffer += decoder.decode(value, { stream: true }).replace(/\r/g, "");
@@ -419,19 +462,21 @@ async function* streamSSE(open) {
             while ((sep = buffer.indexOf("\n\n")) !== -1) {
                 const frame = buffer.slice(0, sep);
                 buffer = buffer.slice(sep + 2);
-                const dataLines = [];
-                for (const line of frame.split("\n")) {
-                    if (line.startsWith("data:"))
-                        dataLines.push(line.slice(5).replace(/^ /, ""));
-                }
-                if (dataLines.length === 0)
+                const parsed = parseSSEFrame(frame);
+                if (parsed === null)
                     continue;
-                const data = dataLines.join("\n");
-                if (data === "[DONE]") {
+                if (parsed.kind === "done") {
                     exhausted = true;
                     return;
                 }
-                yield JSON.parse(data);
+                yield parsed.value;
+            }
+            // With every complete frame drained, whatever remains is a single unterminated
+            // frame. If it grows past the cap the stream is never sending a separator; error
+            // clearly instead of buffering without bound. See T2.
+            if (buffer.length > MAX_SSE_BUFFER_CHARS) {
+                throw new Error(`FlexInference: stream buffer exceeded ${String(MAX_SSE_BUFFER_CHARS)} characters ` +
+                    "without a frame boundary; aborting to avoid unbounded memory.");
             }
         }
     }

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "flexinference",
-  "version": "1.5.0",
+  "version": "1.5.1",
   "description": "Official TypeScript SDK for FlexInference - a deadline-aware, OpenAI-compatible inference router.",
   "license": "MIT",
   "author": "Aditya Perswal <adityaperswal@gmail.com>",
@@ -34,7 +34,7 @@
     "build": "tsc -p tsconfig.json",
     "typecheck": "tsc -p tsconfig.json --noEmit",
     "test:types": "tsc -p tsconfig.test.json --noEmit",
-    "test": "node test/smoke.mjs && npm run test:types",
+    "test": "node test/smoke.mjs && node test/fixes.test.mjs && npm run test:types",
     "lint": "eslint .",
     "lint:fix": "eslint . --fix",
     "format": "prettier --write .",