omnigateway 0.1.2 → 0.1.3

package/bin/omni.js

@@ -608,6 +608,7 @@ async function* decodeAnthropic(messages) {
   let cacheWriteTokens = 0;
   let outputTokens = 0;
   let stopReason = "endTurn";
+  let terminal = false;
   for await (const msg of messages) {
     const d = json(msg.data);
     if (d === null)
@@ -673,6 +674,7 @@ async function* decodeAnthropic(messages) {
         break;
       }
       case "message_stop":
+        terminal = true;
         yield {
           type: "end",
           stopReason,
@@ -680,6 +682,7 @@ async function* decodeAnthropic(messages) {
         };
         break;
       case "error": {
+        terminal = true;
         const code = ERROR_TYPE[String(d.error?.type)] ?? "UPSTREAM";
         yield {
           type: "error",
@@ -693,6 +696,14 @@ async function* decodeAnthropic(messages) {
         break;
     }
   }
+  if (!terminal) {
+    yield {
+      type: "error",
+      code: "UPSTREAM",
+      message: "upstream stream ended before message_stop",
+      retryable: RETRYABLE.UPSTREAM
+    };
+  }
 }
 
 // packages/providers/src/anthropic/wire.ts
@@ -805,9 +816,10 @@ var anthropicAdapter = {
       ["anthropic-version", API_VERSION],
       ["Accept", req.request.stream ? "text/event-stream" : "application/json"]
     ];
+    const betas = new Set(req.request.betas ?? []);
     if (oauth) {
       protocol.push(["Authorization", `Bearer ${req.credentials.accessToken}`]);
-      protocol.push(["anthropic-beta", OAUTH_BETA]);
+      betas.add(OAUTH_BETA);
     } else if (req.credentials.apiKey !== null) {
       protocol.push(["x-api-key", req.credentials.apiKey]);
     } else {
@@ -815,6 +827,8 @@ var anthropicAdapter = {
         provider: "anthropic"
       });
     }
+    if (betas.size > 0)
+      protocol.push(["anthropic-beta", [...betas].join(",")]);
     const profile = PROFILES.anthropic;
     const headers = orderHeaders(mergeHeaders(profile.headers, protocol), profile.order);
     const bodyString = signAnthropicBody(JSON.stringify(orderFields(withSystem, BODY_ORDER.anthropic)));
@@ -1032,18 +1046,16 @@ async function* decodeChat(messages) {
   let started = false;
   let textOpen = false;
   let textIndex;
-  let ended = false;
+  let done = false;
   let stopReason = "endTurn";
   let usage = { inputTokens: 0, outputTokens: 0, cacheReadTokens: 0, cacheWriteTokens: 0 };
   const toolIndex = new Map;
   let nextIndex = 0;
-  const emitEnd = () => {
-    ended = true;
-    return { type: "end", stopReason, usage };
-  };
   for await (const msg of messages) {
-    if (msg.data === "[DONE]")
+    if (msg.data === "[DONE]") {
+      done = true;
       break;
+    }
     const d = json2(msg.data);
     if (d === null)
       continue;
@@ -1098,19 +1110,21 @@ async function* decodeChat(messages) {
     }
     if (typeof choice.finish_reason === "string") {
       stopReason = FINISH[choice.finish_reason] ?? "endTurn";
-      if (textOpen)
-        yield { type: "blockEnd", index: textIndex ?? 0 };
-      for (const index of toolIndex.values())
-        yield { type: "blockEnd", index };
-      yield emitEnd();
     }
   }
-  if (!ended) {
+  if (done) {
     if (textOpen)
       yield { type: "blockEnd", index: textIndex ?? 0 };
     for (const index of toolIndex.values())
       yield { type: "blockEnd", index };
-    yield emitEnd();
+    yield { type: "end", stopReason, usage };
+  } else {
+    yield {
+      type: "error",
+      code: "UPSTREAM",
+      message: "upstream stream ended before [DONE]",
+      retryable: RETRYABLE.UPSTREAM
+    };
   }
 }
 
@@ -1265,6 +1279,7 @@ async function* decodeResponses(messages) {
     return assigned;
   };
   let sawToolCall = false;
+  let terminal = false;
   const ownsBlock = new Set;
   for await (const msg of messages) {
     const d = json3(msg.data);
@@ -1340,6 +1355,7 @@ async function* decodeResponses(messages) {
       }
       case "response.completed":
       case "response.incomplete": {
+        terminal = true;
         const r = d.response ?? {};
         const reason = r.incomplete_details?.reason;
         let stopReason = sawToolCall ? "toolUse" : "endTurn";
@@ -1361,6 +1377,7 @@ async function* decodeResponses(messages) {
       }
       case "response.failed":
       case "error": {
+        terminal = true;
         const err = d.response?.error ?? d.error ?? {};
         const code = ERROR_CODE[String(err.code ?? err.type)] ?? "UPSTREAM";
         yield {
@@ -1375,6 +1392,14 @@ async function* decodeResponses(messages) {
         break;
     }
   }
+  if (!terminal) {
+    yield {
+      type: "error",
+      code: "UPSTREAM",
+      message: "upstream stream ended before response completion",
+      retryable: RETRYABLE.UPSTREAM
+    };
+  }
 }
 
 // packages/providers/src/openai/wire.ts

package/gateway.js

@@ -5866,6 +5866,7 @@ async function* decodeAnthropic(messages) {
   let cacheWriteTokens = 0;
   let outputTokens = 0;
   let stopReason = "endTurn";
+  let terminal = false;
   for await (const msg of messages) {
     const d = json(msg.data);
     if (d === null)
@@ -5931,6 +5932,7 @@ async function* decodeAnthropic(messages) {
         break;
       }
       case "message_stop":
+        terminal = true;
         yield {
           type: "end",
           stopReason,
@@ -5938,6 +5940,7 @@ async function* decodeAnthropic(messages) {
         };
         break;
       case "error": {
+        terminal = true;
         const code = ERROR_TYPE[String(d.error?.type)] ?? "UPSTREAM";
         yield {
           type: "error",
@@ -5951,6 +5954,14 @@ async function* decodeAnthropic(messages) {
         break;
     }
   }
+  if (!terminal) {
+    yield {
+      type: "error",
+      code: "UPSTREAM",
+      message: "upstream stream ended before message_stop",
+      retryable: RETRYABLE.UPSTREAM
+    };
+  }
 }
 
 // packages/providers/src/anthropic/wire.ts
@@ -6063,9 +6074,10 @@ var anthropicAdapter = {
       ["anthropic-version", API_VERSION],
       ["Accept", req.request.stream ? "text/event-stream" : "application/json"]
     ];
+    const betas = new Set(req.request.betas ?? []);
     if (oauth) {
       protocol.push(["Authorization", `Bearer ${req.credentials.accessToken}`]);
-      protocol.push(["anthropic-beta", OAUTH_BETA]);
+      betas.add(OAUTH_BETA);
     } else if (req.credentials.apiKey !== null) {
       protocol.push(["x-api-key", req.credentials.apiKey]);
     } else {
@@ -6073,6 +6085,8 @@ var anthropicAdapter = {
         provider: "anthropic"
       });
     }
+    if (betas.size > 0)
+      protocol.push(["anthropic-beta", [...betas].join(",")]);
     const profile = PROFILES.anthropic;
     const headers = orderHeaders(mergeHeaders(profile.headers, protocol), profile.order);
     const bodyString = signAnthropicBody(JSON.stringify(orderFields(withSystem, BODY_ORDER.anthropic)));
@@ -6290,18 +6304,16 @@ async function* decodeChat(messages) {
   let started = false;
   let textOpen = false;
   let textIndex;
-  let ended = false;
+  let done = false;
   let stopReason = "endTurn";
   let usage = { inputTokens: 0, outputTokens: 0, cacheReadTokens: 0, cacheWriteTokens: 0 };
   const toolIndex = new Map;
   let nextIndex = 0;
-  const emitEnd = () => {
-    ended = true;
-    return { type: "end", stopReason, usage };
-  };
   for await (const msg of messages) {
-    if (msg.data === "[DONE]")
+    if (msg.data === "[DONE]") {
+      done = true;
       break;
+    }
     const d = json2(msg.data);
     if (d === null)
       continue;
@@ -6356,19 +6368,21 @@ async function* decodeChat(messages) {
     }
     if (typeof choice.finish_reason === "string") {
       stopReason = FINISH[choice.finish_reason] ?? "endTurn";
-      if (textOpen)
-        yield { type: "blockEnd", index: textIndex ?? 0 };
-      for (const index of toolIndex.values())
-        yield { type: "blockEnd", index };
-      yield emitEnd();
     }
   }
-  if (!ended) {
+  if (done) {
     if (textOpen)
       yield { type: "blockEnd", index: textIndex ?? 0 };
     for (const index of toolIndex.values())
       yield { type: "blockEnd", index };
-    yield emitEnd();
+    yield { type: "end", stopReason, usage };
+  } else {
+    yield {
+      type: "error",
+      code: "UPSTREAM",
+      message: "upstream stream ended before [DONE]",
+      retryable: RETRYABLE.UPSTREAM
+    };
   }
 }
 
@@ -6523,6 +6537,7 @@ async function* decodeResponses(messages) {
     return assigned;
   };
   let sawToolCall = false;
+  let terminal = false;
   const ownsBlock = new Set;
   for await (const msg of messages) {
     const d = json3(msg.data);
@@ -6598,6 +6613,7 @@ async function* decodeResponses(messages) {
       }
       case "response.completed":
       case "response.incomplete": {
+        terminal = true;
         const r = d.response ?? {};
         const reason = r.incomplete_details?.reason;
         let stopReason = sawToolCall ? "toolUse" : "endTurn";
@@ -6619,6 +6635,7 @@ async function* decodeResponses(messages) {
       }
       case "response.failed":
       case "error": {
+        terminal = true;
         const err = d.response?.error ?? d.error ?? {};
         const code = ERROR_CODE[String(err.code ?? err.type)] ?? "UPSTREAM";
         yield {
@@ -6633,6 +6650,14 @@ async function* decodeResponses(messages) {
         break;
     }
   }
+  if (!terminal) {
+    yield {
+      type: "error",
+      code: "UPSTREAM",
+      message: "upstream stream ended before response completion",
+      retryable: RETRYABLE.UPSTREAM
+    };
+  }
 }
 
 // packages/providers/src/openai/wire.ts
@@ -38319,12 +38344,18 @@ async function dispatch(request2, deps, signal) {
               }), dispatchSignal);
               for (const d of result.degradations)
                 log.degradations.push(d);
+              let terminal = false;
+              const pending = [];
               for await (const event of result.events) {
                 if (event.type === "blockDelta" && !committed) {
                   committed = true;
                   log.ttftMs = deps.now() - startedAt;
+                  for (const buffered of pending)
+                    yield buffered;
+                  pending.length = 0;
                 }
                 if (event.type === "end") {
+                  terminal = true;
                   log.inputTokens = event.usage.inputTokens;
                   log.outputTokens = event.usage.outputTokens;
                   log.cacheReadTokens = event.usage.cacheReadTokens;
@@ -38332,6 +38363,7 @@ async function dispatch(request2, deps, signal) {
                   log.costUsd = priceOf(candidate, event.usage);
                 }
                 if (event.type === "error") {
+                  terminal = true;
                   if (!committed && RETRYABLE[event.code]) {
                     throw new GatewayError(event.code, event.message);
                   }
@@ -38348,7 +38380,21 @@ async function dispatch(request2, deps, signal) {
                   log.durationMs = deps.now() - startedAt;
                   return;
                 }
-                yield event;
+                if (!committed && event.type !== "end") {
+                  pending.push(event);
+                } else {
+                  if (!committed) {
+                    for (const buffered of pending)
+                      yield buffered;
+                    pending.length = 0;
+                  }
+                  yield event;
+                }
+                if (event.type === "end")
+                  break;
+              }
+              if (!terminal) {
+                throw new GatewayError("UPSTREAM", "upstream stream ended without a terminal event");
               }
               await persistHealth(recordSuccess(healthFor(candidate), {
                 settings: snapshot.settings,
@@ -38602,6 +38648,7 @@ var chunk = (id, created, model, choice, usage) => ({
 async function* openaiStream(events, requestId, created) {
   let model = "";
   let roleSent = false;
+  let completed = false;
   const toolIndex = new Map;
   const emit = (data) => ({ event: "message", data: JSON.stringify(data) });
   for await (const event of events) {
@@ -38663,6 +38710,7 @@ async function* openaiStream(events, requestId, created) {
       case "blockEnd":
         break;
       case "end":
+        completed = true;
         yield emit(chunk(requestId, created, model, { index: 0, delta: {}, finish_reason: FINISH2[event.stopReason] }, {
           prompt_tokens: event.usage.inputTokens,
           completion_tokens: event.usage.outputTokens,
@@ -38676,7 +38724,8 @@ async function* openaiStream(events, requestId, created) {
       }
     }
   }
-  yield { event: "message", data: "[DONE]" };
+  if (completed)
+    yield { event: "message", data: "[DONE]" };
 }
 function openaiResponse(collected, requestId, created) {
   const text = collected.content.flatMap((b) => b.type === "text" ? [b.text] : []).join("");
@@ -38855,7 +38904,14 @@ function toIrReasoning(thinking, effort) {
       return { mode: "off" };
   }
 }
-function parseAnthropicRequest(body2) {
+function readBetas(headers) {
+  const raw = headers?.get("anthropic-beta");
+  if (raw === undefined || raw === null)
+    return [];
+  const names = raw.split(",").map((name) => name.trim()).filter((name) => name.length > 0);
+  return [...new Set(names)];
+}
+function parseAnthropicRequest(body2, headers) {
   if (typeof body2 !== "object" || body2 === null) {
     throw new GatewayError("BAD_REQUEST", "request body must be a JSON object");
   }
@@ -38896,6 +38952,9 @@ function parseAnthropicRequest(body2) {
   const extras = extraFields(body2, KNOWN);
   if (extras !== undefined)
     request2.vendor = { anthropic: extras };
+  const betas = readBetas(headers);
+  if (betas.length > 0)
+    request2.betas = betas;
   return validateRequest(request2);
 }
 
@@ -39129,7 +39188,7 @@ async function handle(deps, rateLimiter, surface, request2) {
     keyId = key.id;
     rateLimiter.consume(key.id, key.rateLimitPerMin);
     const body2 = await request2.json();
-    const chatRequest = surface === "anthropic" ? parseAnthropicRequest(body2) : parseOpenAIRequest(body2);
+    const chatRequest = surface === "anthropic" ? parseAnthropicRequest(body2, request2.headers) : parseOpenAIRequest(body2);
     if (key.modelAllowlist !== null && !key.modelAllowlist.includes(chatRequest.model)) {
       throw new GatewayError("AUTH", `model "${chatRequest.model}" is not allowed for this API key`);
     }

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "omnigateway",
-  "version": "0.1.2",
+  "version": "0.1.3",
   "description": "Self-hosted AI gateway with Anthropic- and OpenAI-compatible APIs, an admin console, and a CLI",
   "license": "MIT",
   "author": "Harismawan <mail@harismawan.com>",