@inhuman.tools/harness 0.2.0 → 0.3.0

package/dist/providers/openai-compat/actor.js

@@ -40,7 +40,7 @@ const COMPLETION_FAILED_STATUS = /(?:openai-compat completion failed: |\bHTTP )(
 const TRANSIENT_CODE = /^(?:ECONNRESET|ETIMEDOUT|ENOTFOUND|ECONNREFUSED|EAI_AGAIN|EPIPE|EHOSTUNREACH|ENETUNREACH|EPROTO|UND_ERR_)/;
 const TRANSIENT_TEXT = /fetch failed|Failed to fetch|ECONNRESET|ETIMEDOUT|ENOTFOUND|ECONNREFUSED|EAI_AGAIN|Connect Timeout|Headers Timeout|other side closed|UND_ERR_|socket hang up/i;
 /** Parsed SSE `data:` only. Socket bytes reset undici's body timer, including pings (`request-timeout.ts`). */
-const STREAM_STALL_MS = 60_000;
+const STREAM_STALL_MS = 180_000;
 /** Longest wait between attempts, whether it comes from our backoff or from the server. */
 const BACKOFF_CAP_MS = 5 * 60 * 1000;
 /** A 429 or 503 that named how long to wait. The message is the ordinary HTTP failure text. */
@@ -336,20 +336,6 @@ function visibleThought(raw, summary) {
         return { thought: summary, kind: "summary" };
     return { thought: "", kind: null };
 }
-/**
- * Responses `incomplete` + `max_output_tokens` after a reasoning item is the
- * model spending the budget on thought and stopping before a message. Not a
- * failed generation — ReACT continues. Other incomplete reasons stay fatal.
- */
-function incompleteThoughtExhausted(reason, thought) {
-    if (reason !== "max_output_tokens")
-        return false;
-    return (thought.raw.trim() !== "" ||
-        thought.summary.trim() !== "" ||
-        thought.encrypted !== "" ||
-        thought.kind != null ||
-        (thought.reasoningTokens ?? 0) > 0);
-}
 const THINKING_PART_TYPES = new Set(["thinking", "reasoning", "redacted_thinking", "reasoning_text"]);
 function isThinkingPart(part) {
     const type = readField(part, "type");
@@ -424,7 +410,107 @@ function chunkMessageReasoning(raw) {
     const alt = readField(message, "reasoning");
     return typeof alt === "string" && alt !== "" ? alt : "";
 }
+/** A no-progress deadline is terminal, unlike explicitly configured transient retries. */
+async function withProgressDeadline(pending, signal, ms, phase, cancel) {
+    let timer;
+    let onAbort;
+    const deadline = new Promise((_resolve, reject) => {
+        onAbort = () => reject(signal.reason ?? new Error("completion aborted"));
+        if (signal.aborted) {
+            onAbort();
+            return;
+        }
+        signal.addEventListener("abort", onAbort, { once: true });
+        timer = setTimeout(() => {
+            const reason = new StreamStallError(ms);
+            reason.message = `${phase} stalled after ${ms}ms`;
+            reject(reason);
+            cancel(reason);
+        }, ms);
+        timer.unref();
+    });
+    try {
+        return await Promise.race([pending, deadline]);
+    }
+    finally {
+        clearTimeout(timer);
+        if (onAbort)
+            signal.removeEventListener("abort", onAbort);
+    }
+}
+function readBulkText(response, signal, ms) {
+    return withProgressDeadline(response.text(), signal, ms, "response body", (reason) => {
+        // The fetch signal owns network cancellation. A synthetic response may not
+        // observe it; deadline rejection must not depend on its cancel implementation.
+        void response.body?.cancel(reason).catch(() => undefined);
+    });
+}
+/** Only model output or an authoritative terminal can extend the no-progress deadline. */
+function streamFrameProgress(frame) {
+    if (typeof frame !== "object" || frame === null)
+        return false;
+    const text = (value) => typeof value === "string" && value.length > 0;
+    const type = readField(frame, "type");
+    if (type === "error" ||
+        type === "response.completed" ||
+        type === "response.done" ||
+        type === "response.failed" ||
+        type === "response.incomplete")
+        return true;
+    if (typeof type === "string" && type.startsWith("response.") && type.endsWith(".delta"))
+        return text(readField(frame, "delta"));
+    if (type === "response.output_item.added" || type === "response.output_item.done") {
+        const item = readField(frame, "item");
+        if (typeof item !== "object" || item === null)
+            return false;
+        const content = splitMessageContent(readField(item, "content"));
+        return (text(readField(item, "name")) ||
+            text(readField(item, "arguments")) ||
+            text(readField(item, "encrypted_content")) ||
+            text(reasoningPartText(readField(item, "summary"))) ||
+            text(reasoningPartText(readField(item, "content"))) ||
+            text(content.text) ||
+            text(content.thought));
+    }
+    const choices = readField(frame, "choices");
+    if (!Array.isArray(choices))
+        return false;
+    return choices.some((choice) => {
+        if (typeof choice !== "object" || choice === null)
+            return false;
+        if (text(readField(choice, "finish_reason")))
+            return true;
+        const delta = readField(choice, "delta");
+        if (typeof delta !== "object" || delta === null)
+            return false;
+        const content = splitMessageContent(readField(delta, "content"));
+        if (text(content.text) ||
+            text(content.thought) ||
+            content.sawThinking ||
+            text(readField(delta, "reasoning_content")) ||
+            text(readField(delta, "reasoning")) ||
+            text(readField(delta, "thinking")))
+            return true;
+        const reasoningDetails = readField(delta, "reasoning_details");
+        if (Array.isArray(reasoningDetails) &&
+            reasoningDetails.some((entry) => typeof entry === "object" &&
+                entry !== null &&
+                (text(readField(entry, "text")) || text(readField(entry, "summary")) || text(readField(entry, "data")))))
+            return true;
+        const calls = readField(delta, "tool_calls");
+        return (Array.isArray(calls) &&
+            calls.some((call) => {
+                if (typeof call !== "object" || call === null)
+                    return false;
+                if (text(readField(call, "id")))
+                    return true;
+                const fn = readField(call, "function");
+                return (typeof fn === "object" && fn !== null && (text(readField(fn, "name")) || text(readField(fn, "arguments"))));
+            }));
+    });
+}
 export async function* readSseJson(body, signal, stallMs) {
+    signal.throwIfAborted();
     if (!body)
         return;
     const reader = body.getReader();
@@ -432,6 +518,18 @@ export async function* readSseJson(body, signal, stallMs) {
     let buf = "";
     let armTimer;
     let stalled;
+    let rejectInterrupted;
+    const interrupted = new Promise((_resolve, reject) => {
+        rejectInterrupted = reject;
+    });
+    const onAbort = () => {
+        rejectInterrupted(signal.reason ?? new Error("completion aborted"));
+        void reader.cancel(signal.reason).catch(() => undefined);
+    };
+    if (signal.aborted)
+        onAbort();
+    else
+        signal.addEventListener("abort", onAbort, { once: true });
     const clearArm = () => {
         if (armTimer === undefined)
             return;
@@ -443,6 +541,7 @@ export async function* readSseJson(body, signal, stallMs) {
         clearArm();
         const timer = setTimeout(() => {
             stalled = new StreamStallError(stallMs);
+            rejectInterrupted(stalled);
             void reader.cancel(stalled).catch(() => undefined);
         }, stallMs);
         timer.unref();
@@ -450,7 +549,7 @@ export async function* readSseJson(body, signal, stallMs) {
     };
     const read = async () => {
         try {
-            const next = await reader.read();
+            const next = await Promise.race([reader.read(), interrupted]);
             if (stalled !== undefined)
                 throw stalled;
             return next;
@@ -470,11 +569,11 @@ export async function* readSseJson(body, signal, stallMs) {
             buf += decoder.decode(value, { stream: true });
             let yielded = false;
             for (;;) {
-                const sep = buf.indexOf("\n\n");
-                if (sep === -1)
+                const separator = /\r?\n\r?\n/.exec(buf);
+                if (separator === null)
                     break;
-                const event = buf.slice(0, sep);
-                buf = buf.slice(sep + 2);
+                const event = buf.slice(0, separator.index);
+                buf = buf.slice(separator.index + separator[0].length);
                 for (const line of event.split(/\r?\n/)) {
                     const trimmed = line.trimEnd();
                     if (!trimmed.startsWith("data:"))
@@ -486,8 +585,10 @@ export async function* readSseJson(body, signal, stallMs) {
                         continue;
                     }
                     try {
-                        yield parseJsonUnknown(data);
-                        yielded = true;
+                        const parsed = parseJsonUnknown(data);
+                        yield parsed;
+                        if (streamFrameProgress(parsed))
+                            yielded = true;
                     }
                     catch {
                         // skip malformed line
@@ -516,6 +617,8 @@ export async function* readSseJson(body, signal, stallMs) {
     }
     finally {
         clearArm();
+        signal.removeEventListener("abort", onAbort);
+        void reader.cancel().catch(() => undefined);
         try {
             reader.releaseLock();
         }
@@ -621,14 +724,14 @@ export class OpenAiCompatActor extends ExpectationActor {
                     const posted = Date.now();
                     trace("harness:complete", `post ${this.expectation.uuid} connection=${this.driverConnection.connectionId} model=${this.effectiveModel()} attempt=${n}`);
                     try {
-                        response = await this.driverConnection.authorizedFetch(url, {
+                        response = await withProgressDeadline(this.driverConnection.authorizedFetch(url, {
                             method: "POST",
                             headers: affinity.headers,
                             body: JSON.stringify(payload),
-                        }, abort.signal);
+                        }, abort.signal), abort.signal, this.stallMs, "completion request", (reason) => abort.abort(reason));
                     }
                     catch (error) {
-                        if (error instanceof ConnectionMiss)
+                        if (error instanceof ConnectionMiss || error instanceof StreamStallError)
                             throw error;
                         throw wrapFetchFailure(error, {
                             why: "completion",
@@ -637,7 +740,7 @@ export class OpenAiCompatActor extends ExpectationActor {
                         });
                     }
                     if (!response.ok) {
-                        const errorBody = await response.text();
+                        const errorBody = await withProgressDeadline(response.text(), abort.signal, this.stallMs, "error response body", (reason) => abort.abort(reason));
                         const detail = errorBody.trim().slice(0, 240);
                         const message = describeHttpFailure({
                             why: "completion",
@@ -658,6 +761,8 @@ export class OpenAiCompatActor extends ExpectationActor {
                         throw new Error(message);
                     }
                     const answer = await this.readAnswer(response, abort.signal);
+                    if (this.signal.aborted)
+                        return;
                     trace("harness:complete", `answer ${this.expectation.uuid} ms=${Date.now() - posted} status=${response.status} chars=${answer?.content.length ?? 0}`);
                     invariant(answer !== null, "openai-compat completion failed");
                     this.expectation.stampAnswer({
@@ -692,6 +797,8 @@ export class OpenAiCompatActor extends ExpectationActor {
                         return;
                     if (error instanceof ExpectationInvariant)
                         throw error;
+                    if (error instanceof StreamStallError)
+                        abort.abort(error);
                     if (!isTransientCompletionError(error)) {
                         const reason = error instanceof Error ? error.message : String(error);
                         this.expectation.failReason = reason;
@@ -968,12 +1075,12 @@ export class OpenAiCompatChatActor extends OpenAiCompatActor {
     async readAnswer(response, signal) {
         const contentType = response.headers.get("content-type")?.toLowerCase() ?? "";
         if (contentType.includes("application/json")) {
-            return this.#roundFromBulk(await response.json());
+            return this.#roundFromBulk(parseJsonUnknown(await readBulkText(response, signal, this.stallMs)));
         }
         if (contentType.includes("text/event-stream") || contentType.includes("event-stream")) {
             return this.#consumeStream(readSseJson(response.body, signal, this.stallMs));
         }
-        const text = await response.text();
+        const text = await readBulkText(response, signal, this.stallMs);
         const head = text.trimStart();
         if (head.startsWith("{") || head.startsWith("[")) {
             try {
@@ -997,6 +1104,9 @@ export class OpenAiCompatChatActor extends OpenAiCompatActor {
         const choice = Array.isArray(choices) && typeof choices[0] === "object" && choices[0] !== null ? choices[0] : undefined;
         const message = choice == null ? undefined : readField(choice, "message");
         const messageRec = typeof message === "object" && message !== null ? message : undefined;
+        const terminal = choice == null ? undefined : readField(choice, "finish_reason");
+        if (messageRec == null || typeof terminal !== "string" || terminal === "")
+            throw new Error("Chat completion response is missing its message or terminal finish reason");
         const reasoningContent = messageRec == null ? undefined : readField(messageRec, "reasoning_content");
         const reasoningAlt = messageRec == null ? undefined : readField(messageRec, "reasoning");
         const reasoning = typeof reasoningContent === "string" && reasoningContent.length > 0
@@ -1054,6 +1164,7 @@ export class OpenAiCompatChatActor extends OpenAiCompatActor {
         const partial = new Set();
         const dropped = [];
         let finishReason;
+        let sawTerminal = false;
         const onEvent = (ev) => {
             switch (ev.type) {
                 case "reasoning_delta": {
@@ -1109,6 +1220,8 @@ export class OpenAiCompatChatActor extends OpenAiCompatActor {
             if (typeof raw === "object" && raw !== null) {
                 const rec = raw;
                 const choice = rec.choices?.[0];
+                if (typeof choice?.finish_reason === "string" && choice.finish_reason !== "")
+                    sawTerminal = true;
                 const delta = choice?.delta;
                 let chunk = rec;
                 if (delta != null) {
@@ -1144,7 +1257,11 @@ export class OpenAiCompatChatActor extends OpenAiCompatActor {
             }
             if (thoughtAcc === "")
                 thoughtAcc = chunkMessageReasoning(raw);
+            if (sawTerminal)
+                break;
         }
+        if (!sawTerminal)
+            throw new Error("Chat completion stream ended without a terminal finish reason");
         for (const ev of session.close())
             onEvent(ev);
         const settled = settleCalls(calls, partial, dropped, finishReason);
@@ -1258,8 +1375,19 @@ export class OpenAiCompatResponsesActor extends OpenAiCompatActor {
     async readAnswer(response, signal) {
         const contentType = response.headers.get("content-type") ?? "";
         if (contentType.toLowerCase().includes("application/json") && !contentType.toLowerCase().includes("event-stream")) {
-            const payload = await response.json();
+            const payload = parseJsonUnknown(await readBulkText(response, signal, this.stallMs));
             this.stampConsumed(payload);
+            const status = typeof payload === "object" && payload !== null ? readField(payload, "status") : undefined;
+            if (status !== "completed" && status !== "incomplete") {
+                const error = readField(payload, "error");
+                const message = typeof error === "object" && error !== null ? readField(error, "message") : undefined;
+                throw new Error(typeof message === "string" ? message : `Responses request ended with status ${status}`);
+            }
+            const incomplete = typeof payload === "object" && payload !== null ? readField(payload, "incomplete_details") : undefined;
+            const incompleteReason = typeof incomplete === "object" && incomplete !== null ? readField(incomplete, "reason") : undefined;
+            if (status === "incomplete" && incompleteReason !== "max_output_tokens" && incompleteReason !== "content_filter")
+                throw new Error(`Responses request incomplete: ${String(incompleteReason)}`);
+            const finishReason = status === "incomplete" ? (incompleteReason === "content_filter" ? "content_filter" : "length") : undefined;
             const parts = [];
             const rawParts = [];
             const summaryParts = [];
@@ -1319,7 +1447,13 @@ export class OpenAiCompatResponsesActor extends OpenAiCompatActor {
             this.#stampReasoningEncrypted(encrypted);
             this.stampThoughtKind(thoughtKind);
             this.speech(text, visible.thought);
-            return { content: text, reasoning: visible.thought, thoughtKind, calls };
+            return {
+                content: text,
+                reasoning: visible.thought,
+                thoughtKind,
+                calls,
+                ...(finishReason == null ? {} : { finishReason }),
+            };
         }
         let acc = {
             text: "",
@@ -1422,19 +1556,11 @@ export class OpenAiCompatResponsesActor extends OpenAiCompatActor {
                             }
                         }
                     }
-                    if (incompleteThoughtExhausted(details, {
-                        raw: acc.raw,
-                        summary: acc.summary,
-                        encrypted,
-                        kind: this.expectation.responseThoughtKind,
-                        reasoningTokens: this.expectation.reasoningTokens,
-                    })) {
+                    if (details === "max_output_tokens") {
                         sawTerminal = true;
                         finish = "length";
                         break;
                     }
-                    // Speech cut off by max_output_tokens stays a failed round (chatgpt-complete.test); only
-                    // the filter is a stop of its own.
                     if (details === "content_filter") {
                         sawTerminal = true;
                         finish = "content_filter";
@@ -1493,6 +1619,8 @@ export class OpenAiCompatResponsesActor extends OpenAiCompatActor {
                         emitSpeech();
                     }
             }
+            if (sawTerminal)
+                break;
         }
         if (!sawTerminal)
             throw new Error("Responses stream ended before terminal response event");