flexinference 1.5.0 → 1.5.2

package/CHANGELOG.md

@@ -1,5 +1,15 @@
 # Changelog
 
+## 1.5.1
+
+Small robustness fixes for the transport, no API changes. A non-streaming request that hits
+its total `timeout` now throws a clear timeout error instead of a bare abort that looked like
+a caller cancellation. A streamed response flushes its decoder and delivers the final frame
+even when the stream ends without a trailing blank line, so the last event is never dropped,
+and the streaming buffer is capped so a stream that never sends a separator cannot grow
+without bound. The `start_within` type now requires the runtime's two-digit duration
+shape, so values like `0h-5m-0s` no longer type-check.
+
 ## 1.5.0
 
 Reworks request timeouts so a healthy stream is never cut off yet a hung request never hangs
@@ -72,11 +82,11 @@ required and aligns its values with OpenAI's `service_tier` vocabulary.
 
 ### Breaking changes
 
-- **`start_within` is now required on every request.** It is typed as
-  `StartWithin = "default" | "priority" | "auto" | \`${number}h-${number}m-${number}s\``,
-so omitting it (or passing a value outside those shapes) is a **compile-time** error.
-A duration that fits the shape but is malformed or out of range is caught at **runtime**;
-plain-JS callers get every check at **runtime** (a thrown `Error` before any network call).
+- **`start_within` is now required on every request.** It is typed as `"default"`,
+  `"priority"`, `"auto"`, or a duration shaped like `"00h-05m-00s"`, so omitting it
+  (or passing a value outside those shapes) is a **compile-time** error.
+  A duration that fits the shape but is malformed or out of range is caught at **runtime**;
+  plain-JS callers get every check at **runtime** (a thrown `Error` before any network call).
 - **Accepted values changed.** `start_within` now takes `"default"`, `"priority"`,
   `"auto"`, or a duration `"HHh-MMm-SSs"` (5s-10m). The old `"standard"` value is
   removed; use `"default"` (it maps to the same OpenAI service tier).

package/README.md

@@ -104,7 +104,7 @@ console.log(messageText(res));
 
 Streaming and non-streaming requests are timed differently, so a long generation is never cut off yet a hung request never hangs forever:
 
-- **Non-streaming** requests have a total wall-clock budget: `timeout`, default **600000 ms (10 min)**, from send through the full body.
+- **Non-streaming** requests have a body-read budget after response headers arrive: `timeout`, default **600000 ms (10 min)**. The worst case is roughly `firstByteTimeout + timeout`.
 - **Streaming** requests have **no total cap** (a healthy stream runs as long as tokens keep arriving). They are bounded by two clocks: `firstByteTimeout` (wait for the first response, default **60000 ms**) and `idleTimeout` (max silence between chunks before the stream is treated as hung, default **60000 ms**).
 
 The first-byte wait is **auto-raised for a flex `start_within`** (the router withholds response headers until the flex race resolves), so a long deadline just works. Raise `firstByteTimeout` yourself only for very large non-flex contexts whose first token is slow.

package/dist/index.d.ts

@@ -5,7 +5,12 @@ type Schemas = components["schemas"];
  * those OpenAI service tiers and proxy any model; a duration "HHh-MMm-SSs" (5s-10m)
  * runs the flex race on a flex-capable model. An arbitrary string needs `as StartWithin`.
  */
-export type StartWithin = "default" | "priority" | "auto" | `${number}h-${number}m-${number}s`;
+type Digit = "0" | "1" | "2" | "3" | "4" | "5" | "6" | "7" | "8" | "9";
+type Second00To59 = "00" | "01" | "02" | "03" | "04" | "05" | "06" | "07" | "08" | "09" | `1${Digit}` | `2${Digit}` | `3${Digit}` | `4${Digit}` | `5${Digit}`;
+type Second05To59 = Exclude<Second00To59, "00" | "01" | "02" | "03" | "04">;
+type Minute01To09 = `0${Exclude<Digit, "0">}`;
+type Duration = `00h-00m-${Second05To59}s` | `00h-${Minute01To09}m-${Second00To59}s` | "00h-10m-00s";
+export type StartWithin = "default" | "priority" | "auto" | Duration;
 /** Make `start_within` a required, typed field on a generated request body. */
 type WithStartWithin<T> = Omit<T, "start_within"> & {
     start_within: StartWithin;
@@ -78,6 +83,7 @@ export declare class FlexInferenceError extends Error {
  * FlexInferenceError)` handlers keep working; narrow to this to handle 402 specially.
  */
 export declare class PaymentRequiredError extends FlexInferenceError {
+    constructor(status: number, body: unknown, fallback: string);
 }
 export interface ClientOptions {
     apiKey: string;
@@ -86,9 +92,10 @@ export interface ClientOptions {
     /** Override the global fetch (e.g. for tests or non-standard runtimes). */
     fetch?: typeof fetch;
     /**
-     * Non-streaming wall-clock budget in ms (from send through the full body). Defaults to
-     * 600000 (10 min). Streaming requests ignore this: they are governed by `firstByteTimeout`
-     * and `idleTimeout` and have no total cap, so a healthy long stream is never cut off.
+     * Non-streaming body-read budget in ms, after response headers arrive. Defaults to
+     * 600000 (10 min). The worst case is roughly `firstByteTimeout + timeout`.
+     * Streaming requests ignore this: they are governed by `firstByteTimeout` and
+     * `idleTimeout` and have no total cap, so a healthy long stream is never cut off.
      */
     timeout?: number;
     /**

package/dist/index.js

@@ -120,6 +120,10 @@ export class FlexInferenceError extends Error {
  * FlexInferenceError)` handlers keep working; narrow to this to handle 402 specially.
  */
 export class PaymentRequiredError extends FlexInferenceError {
+    constructor(status, body, fallback) {
+        super(status, body, fallback);
+        this.name = "PaymentRequiredError";
+    }
 }
 const DEFAULT_BASE_URL = "https://api.flexinference.com/v1";
 // A streaming request is governed by two clocks and no total: how long we wait for the
@@ -132,42 +136,114 @@ const FLEX_FIRST_BYTE_MARGIN_MS = 30_000; // headroom over the flex deadline for
 const DEFAULT_IDLE_MS = 60_000; // max silence between streamed chunks before the stream is treated as hung
 const DEFAULT_TOTAL_MS = 600_000; // non-streaming wall-clock budget (streaming has none)
 const ERROR_BODY_READ_MS = 30_000; // bound the read of a non-2xx error body
+// Cap the streaming buffer above the largest legitimate single frame (a terminal
+// response object is well under this). A stream that never sends a blank-line separator
+// would otherwise grow the buffer without bound; error clearly instead.
+const MAX_SSE_BUFFER_CHARS = 8 * 1024 * 1024;
 const MIN_DEADLINE_SECONDS = 5;
 const MAX_DEADLINE_SECONDS = 600;
 const SECONDS_PER_HOUR = 3600;
 const SECONDS_PER_MINUTE = 60;
+// Two digits per field so the runtime matches the router, Python SDK, and StartWithin type.
 const DURATION_RE = /^(\d{2})h-(\d{2})m-(\d{2})s$/;
 const START_WITHIN_VALUES = '"default", "priority", "auto", or a duration "HHh-MMm-SSs"';
+function startWithinValue(body) {
+    const value = typeof body === "object" && body !== null
+        ? body["start_within"]
+        : undefined;
+    return value === undefined ? { kind: "missing" } : { kind: "present", value };
+}
+function parseStartWithin(value) {
+    if (typeof value !== "string")
+        return { kind: "invalid", reason: "type", value };
+    if (value === "default" || value === "priority" || value === "auto") {
+        return { kind: "tier", value };
+    }
+    const match = DURATION_RE.exec(value);
+    if (match === null)
+        return { kind: "invalid", reason: "malformed", value };
+    const seconds = Number(match[1]) * SECONDS_PER_HOUR +
+        Number(match[2]) * SECONDS_PER_MINUTE +
+        Number(match[3]);
+    return { kind: "duration", value, seconds };
+}
 // Runtime guard so plain-JS callers (who get no compile-time check) still fail
 // locally with a clear message instead of a provider round-trip. TypeScript callers
 // are already protected by the required StartWithin field above.
 function assertStartWithin(body) {
-    const value = typeof body === "object" && body !== null
-        ? body["start_within"]
-        : undefined;
-    if (value === undefined) {
+    const sw = startWithinValue(body);
+    if (sw.kind === "missing") {
         throw new Error(`FlexInference: \`start_within\` is required on every request. Set it to ${START_WITHIN_VALUES}, ` +
             `for example "start_within": "default" for standard-tier behavior or "00h-00m-30s" for a 30-second flex race.`);
     }
-    if (typeof value !== "string") {
+    const parsed = parseStartWithin(sw.value);
+    if (parsed.kind === "invalid" && parsed.reason === "type") {
         throw new Error(`FlexInference: \`start_within\` must be a string (${START_WITHIN_VALUES}), for example "default".`);
     }
-    if (value === "default" || value === "priority" || value === "auto")
-        return;
-    const match = DURATION_RE.exec(value);
-    if (match === null) {
-        throw new Error(`FlexInference: \`start_within\` "${value}" is not ${START_WITHIN_VALUES}. ` +
+    if (parsed.kind === "invalid") {
+        throw new Error(`FlexInference: \`start_within\` "${parsed.value}" is not ${START_WITHIN_VALUES}. ` +
             `A duration is two digits each for hours, minutes, and seconds, for example "00h-05m-00s" for a 5-minute deadline.`);
     }
-    const seconds = Number(match[1]) * SECONDS_PER_HOUR + Number(match[2]) * SECONDS_PER_MINUTE + Number(match[3]);
-    if (seconds < MIN_DEADLINE_SECONDS || seconds > MAX_DEADLINE_SECONDS) {
+    if (parsed.kind === "duration" &&
+        (parsed.seconds < MIN_DEADLINE_SECONDS || parsed.seconds > MAX_DEADLINE_SECONDS)) {
         throw new Error("FlexInference: `start_within` duration must be between 5s and 10m.");
     }
 }
+// Per-endpoint required fields beyond the model/start_within envelope. Mirrors the Python
+// _RequestSpec table in _validation.py: responses/interactions require `input` (presence only);
+// chat requires `messages` (array); messages requires `messages` (array) + `max_tokens` (number).
+const RESPONSES_REQUEST_SPEC = {
+    required: [{ name: "input", kind: "present" }],
+};
+const CHAT_REQUEST_SPEC = {
+    required: [{ name: "messages", kind: "array" }],
+};
+const INTERACTIONS_REQUEST_SPEC = {
+    required: [{ name: "input", kind: "present" }],
+};
+const MESSAGES_REQUEST_SPEC = {
+    required: [
+        { name: "messages", kind: "array" },
+        { name: "max_tokens", kind: "finiteNumber" },
+    ],
+};
+// Read one field by reference off the raw body without copying it. `undefined` (or a non-object
+// body) is treated as missing; `null` counts as present, matching Python's `name in body`
+// presence check (a JSON `null` is a present key, and JSON serialization drops `undefined`).
+function requestField(body, name) {
+    if (!isRecord(body))
+        return { kind: "missing" };
+    const value = body[name];
+    return value === undefined ? { kind: "missing" } : { kind: "present", value };
+}
+// Pre-flight guard: model, then start_within, then the endpoint's required fields in spec order.
+// Pure - it only throws earlier for requests the router/Python already reject; it never mutates,
+// clones, or reshapes the body, so unknown/extra fields pass straight through to `send`.
+function validateRequest(spec, body) {
+    const model = requestField(body, "model");
+    if (model.kind === "missing" || typeof model.value !== "string" || model.value === "") {
+        throw new Error("FlexInference: `model` is required and must be a non-empty string.");
+    }
+    assertStartWithin(body);
+    for (const field of spec.required) {
+        const value = requestField(body, field.name);
+        if (value.kind === "missing") {
+            throw new Error(`FlexInference: \`${field.name}\` is required for this endpoint.`);
+        }
+        if (field.kind === "array" && !Array.isArray(value.value)) {
+            throw new Error(`FlexInference: \`${field.name}\` must be an array.`);
+        }
+        if (field.kind === "finiteNumber" &&
+            (typeof value.value !== "number" || !Number.isFinite(value.value))) {
+            throw new Error(`FlexInference: \`${field.name}\` must be a finite number.`);
+        }
+    }
+}
 // Distinct abort reasons so an aborted fetch/read can be told apart: a timeout we fired
 // (turned into a clear message) versus the caller's own AbortSignal (propagated as-is).
 const FIRST_BYTE_TIMEOUT = Symbol("first-byte-timeout");
 const IDLE_TIMEOUT = Symbol("idle-timeout");
+const TOTAL_TIMEOUT = Symbol("total-timeout");
 /**
  * Seconds for a flex-duration `start_within`, or null for a tier value ("default" etc.),
  * a missing field, or a malformed one. Used to auto-size the first-byte wait: the router
@@ -175,31 +251,45 @@ const IDLE_TIMEOUT = Symbol("idle-timeout");
  * clock must cover the whole deadline plus the fallback path.
  */
 function flexDeadlineSeconds(body) {
-    const value = typeof body === "object" && body !== null
-        ? body["start_within"]
-        : undefined;
-    if (typeof value !== "string")
-        return null;
-    const match = DURATION_RE.exec(value);
-    if (match === null)
+    const sw = startWithinValue(body);
+    if (sw.kind === "missing")
         return null;
-    return (Number(match[1]) * SECONDS_PER_HOUR + Number(match[2]) * SECONDS_PER_MINUTE + Number(match[3]));
+    const parsed = parseStartWithin(sw.value);
+    return parsed.kind === "duration" ? parsed.seconds : null;
 }
 // Read a non-streamed JSON body under the total wall-clock budget, then release the timer
 // and the abort listener. Non-streaming only: a stream is drained by streamSSE instead.
 async function readJson(open) {
     const { res, controller, cleanup, totalMs } = open;
+    // Abort with a dedicated reason so a fired total timeout can be told apart from the
+    // caller cancelling on purpose, and translated into a clear message (parity with the
+    // first-byte and idle timeouts, and with the Python SDK).
     const totalTimer = setTimeout(() => {
-        controller.abort();
+        controller.abort(TOTAL_TIMEOUT);
     }, totalMs);
     try {
         return (await res.json());
     }
+    catch (err) {
+        if (controller.signal.reason === TOTAL_TIMEOUT) {
+            throw new Error(`FlexInference: no full response from the router within ${String(totalMs)}ms ` +
+                "(total timeout). Raise `timeout` for very large non-streaming responses, or " +
+                "stream the request so a long generation is governed by the idle timeout instead.");
+        }
+        throw err;
+    }
     finally {
         clearTimeout(totalTimer);
         cleanup();
     }
 }
+async function createEndpoint(send, path, spec, body, options) {
+    validateRequest(spec, body);
+    const open = await send(path, body, options);
+    if (body.stream)
+        return streamSSE(open);
+    return readJson(open);
+}
 export class FlexInference {
     responses;
     chat;
@@ -326,11 +416,7 @@ class Responses {
         this.send = send;
     }
     async create(body, options) {
-        assertStartWithin(body);
-        const open = await this.send("/responses", body, options);
-        if (body.stream)
-            return streamSSE(open);
-        return readJson(open);
+        return createEndpoint(this.send, "/responses", RESPONSES_REQUEST_SPEC, body, options);
     }
 }
 class Chat {
@@ -345,11 +431,7 @@ class ChatCompletions {
         this.send = send;
     }
     async create(body, options) {
-        assertStartWithin(body);
-        const open = await this.send("/chat/completions", body, options);
-        if (body.stream)
-            return streamSSE(open);
-        return readJson(open);
+        return createEndpoint(this.send, "/chat/completions", CHAT_REQUEST_SPEC, body, options);
     }
 }
 class Interactions {
@@ -358,11 +440,7 @@ class Interactions {
         this.send = send;
     }
     async create(body, options) {
-        assertStartWithin(body);
-        const open = await this.send("/interactions", body, options);
-        if (body.stream)
-            return streamSSE(open);
-        return readJson(open);
+        return createEndpoint(this.send, "/interactions", INTERACTIONS_REQUEST_SPEC, body, options);
     }
 }
 class Messages {
@@ -371,13 +449,22 @@ class Messages {
         this.send = send;
     }
     async create(body, options) {
-        assertStartWithin(body);
-        const open = await this.send("/messages", body, options);
-        if (body.stream)
-            return streamSSE(open);
-        return readJson(open);
+        return createEndpoint(this.send, "/messages", MESSAGES_REQUEST_SPEC, body, options);
     }
 }
+function parseSSEFrame(frame) {
+    const dataLines = [];
+    for (const line of frame.split("\n")) {
+        if (line.startsWith("data:"))
+            dataLines.push(line.slice(5).replace(/^ /, ""));
+    }
+    if (dataLines.length === 0)
+        return null;
+    const data = dataLines.join("\n");
+    if (data === "[DONE]")
+        return { kind: "done" };
+    return { kind: "value", value: JSON.parse(data) };
+}
 async function* streamSSE(open) {
     const { res, controller, idleMs, cleanup } = open;
     if (!res.body) {
@@ -411,7 +498,18 @@ async function* streamSSE(open) {
             }
             const { done, value } = chunk;
             if (done) {
+                // The stream ended. Flush any bytes the decoder held back for an incomplete
+                // multibyte sequence, then emit a final frame that arrived without its trailing
+                // blank line so the last piece of the answer is not dropped.
                 exhausted = true;
+                buffer += decoder.decode();
+                const frame = buffer;
+                buffer = "";
+                if (frame.length > 0) {
+                    const parsed = parseSSEFrame(frame);
+                    if (parsed !== null && parsed.kind === "value")
+                        yield parsed.value;
+                }
                 break;
             }
             buffer += decoder.decode(value, { stream: true }).replace(/\r/g, "");
@@ -419,19 +517,21 @@ async function* streamSSE(open) {
             while ((sep = buffer.indexOf("\n\n")) !== -1) {
                 const frame = buffer.slice(0, sep);
                 buffer = buffer.slice(sep + 2);
-                const dataLines = [];
-                for (const line of frame.split("\n")) {
-                    if (line.startsWith("data:"))
-                        dataLines.push(line.slice(5).replace(/^ /, ""));
-                }
-                if (dataLines.length === 0)
+                const parsed = parseSSEFrame(frame);
+                if (parsed === null)
                     continue;
-                const data = dataLines.join("\n");
-                if (data === "[DONE]") {
+                if (parsed.kind === "done") {
                     exhausted = true;
                     return;
                 }
-                yield JSON.parse(data);
+                yield parsed.value;
+            }
+            // With every complete frame drained, whatever remains is a single unterminated
+            // frame. If it grows past the cap the stream is never sending a separator; error
+            // clearly instead of buffering without bound.
+            if (buffer.length > MAX_SSE_BUFFER_CHARS) {
+                throw new Error(`FlexInference: stream buffer exceeded ${String(MAX_SSE_BUFFER_CHARS)} characters ` +
+                    "without a frame boundary; aborting to avoid unbounded memory.");
             }
         }
     }

package/dist/types.d.ts

@@ -5398,6 +5398,8 @@ export interface components {
                 type: string;
                 code?: string | null;
                 param?: string | null;
+                /** @description Additive deep link into the docs for this exact code. Present (a URL or null) on FlexInference-layer errors; absent on verbatim upstream errors. */
+                doc_url?: string | null;
             };
         };
         /** Interaction Text Part */

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "flexinference",
-  "version": "1.5.0",
+  "version": "1.5.2",
   "description": "Official TypeScript SDK for FlexInference - a deadline-aware, OpenAI-compatible inference router.",
   "license": "MIT",
   "author": "Aditya Perswal <adityaperswal@gmail.com>",
@@ -34,7 +34,7 @@
     "build": "tsc -p tsconfig.json",
     "typecheck": "tsc -p tsconfig.json --noEmit",
     "test:types": "tsc -p tsconfig.test.json --noEmit",
-    "test": "node test/smoke.mjs && npm run test:types",
+    "test": "node test/smoke.mjs && node test/fixes.test.mjs && node test/parity.test.mjs && npm run test:types",
     "lint": "eslint .",
     "lint:fix": "eslint . --fix",
     "format": "prettier --write .",