@remnic/core 9.69.38 → 9.69.40

package/src/fallback-llm-codex-subscription.ts

@@ -1,11 +1,10 @@
 /**
- * Codex-subscription helpers for FallbackLlmClient (issue #2833).
- *
- * Terminal error classification, the `api: "codex-cli"` provider attempt,
- * and Codex runner/lifecycle helpers live here so fallback-llm.ts stays
- * generic chain orchestration.
+ * Helpers extracted from FallbackLlmClient so fallback-llm.ts stays under
+ * the 1200-line cap: Codex-subscription lifecycle (issue #2833) plus
+ * generic chain helpers used by the same client.
  */
 
+import path from "node:path";
 import { raceAbort } from "./abort-error.js";
 import { callCodexCliFallback, isCodexCliFallbackRunnerRegistered } from "./cli-fallback.js";
 import type { CodexCliFallbackRunner } from "./cli-fallback.js";
@@ -16,10 +15,14 @@ import {
   CodexSubscriptionTimeoutError,
   assertNoApiKeyConfig,
   ensureCodexSubscriptionRunnerRegistered,
+  getCodexSubscriptionRunnerForOwner,
   getCodexSubscriptionShutdownSignal,
   isDefaultRegisteredCodexSubscriptionRunner,
 } from "./providers/codex-subscription.js";
-import type { ModelProviderConfig } from "./types.js";
+import type { GetRuntimeAuthForModelFn, ResolveApiKeyFn } from "./resolve-provider-secret.js";
+import { resolveHomeDir } from "./runtime/env.js";
+import type { GatewayConfig, ModelProviderConfig } from "./types.js";
+import { expandTildePath } from "./utils/path.js";
 
 type CodexSubscriptionAttemptResult = {
   content: string;
@@ -151,7 +154,7 @@ export async function tryCodexSubscriptionProvider(
 
 export function abortReason(signal: AbortSignal | undefined): Error {
   const reason = signal?.reason;
-  return reason instanceof Error ? reason : new Error("fallback LLM request aborted");
+  return reason instanceof Error ? reason : new Error("task LLM request aborted");
 }
 
 export function withCodexRuntimeShutdown<T extends { signal?: AbortSignal }>(
@@ -166,3 +169,186 @@ export function withCodexRuntimeShutdown<T extends { signal?: AbortSignal }>(
     signal: options.signal ? AbortSignal.any([options.signal, shutdown]) : shutdown,
   };
 }
+
+export function isUnsupportedJsonSchemaError(error: unknown): boolean {
+  const message = error instanceof Error ? error.message : String(error);
+  // Require an HTTP status plus explicit schema/format context. A bare
+  // "unsupported" (e.g. "'temperature' is unsupported with this model") must
+  // NOT trigger the schema-stripping retry, or we pay a duplicate request
+  // for unrelated provider errors.
+  if (!/\b(?:400|404|422)\b/.test(message)) {
+    return false;
+  }
+  if (/(?:response[_ ]?format|json[_ ]?schema|structured[_ ]?output)/i.test(message)) {
+    return true;
+  }
+  // "unsupported" only counts when adjacent to schema/format terminology.
+  return /\bunsupported\b[\s\S]{0,40}\b(?:schema|format)\b/i.test(message);
+}
+
+export function extractResponsesOutputText(data: {
+  output_text?: string;
+  output?: Array<{
+    type?: string;
+    text?: string;
+    content?: Array<{
+      type?: string;
+      text?: string;
+    }>;
+  }>;
+}): string | null {
+  if (typeof data.output_text === "string" && data.output_text.trim().length > 0) {
+    return data.output_text;
+  }
+
+  const chunks: string[] = [];
+  for (const item of data.output ?? []) {
+    if (typeof item.text === "string" && item.text.trim().length > 0) {
+      chunks.push(item.text);
+    }
+    for (const part of item.content ?? []) {
+      if (
+        (part.type === "output_text" || part.type === "text") &&
+        typeof part.text === "string" &&
+        part.text.trim().length > 0
+      ) {
+        chunks.push(part.text);
+      }
+    }
+  }
+
+  const joined = chunks.join("\n").trim();
+  return joined.length > 0 ? joined : null;
+}
+
+export type StructuredParseFailureReason =
+  | "no_models"
+  | "empty"
+  | "http_error"
+  | "schema_rejection"
+  | "timeout";
+
+export interface StructuredParseFailure {
+  result: null;
+  failureReason: StructuredParseFailureReason;
+  /** Model string of the last attempt, when one was selected. Not `modelUsed`. */
+  attemptedModel?: string;
+  /** HTTP status when known. Never a response body. */
+  httpStatus?: number;
+  /** Coarse class: http_4xx / http_5xx / timeout / network / empty / schema_rejection / no_models. */
+  errorClass?: string;
+}
+
+export type StructuredParseResult<T> =
+  | { result: T; modelUsed: string }
+  | StructuredParseFailure;
+
+export interface FallbackLlmRuntimeContext {
+  agentDir?: string;
+  getRuntimeAuthForModel?: GetRuntimeAuthForModelFn | null;
+  resolveApiKeyForProvider?: ResolveApiKeyFn | null;
+  workspaceDir?: string;
+  /** Per-runtime Codex child owner. Shutdown must terminate only this runner. */
+  codexSubscriptionRunner?: CodexCliFallbackRunner;
+}
+
+type GatewayBackedRuntimeConfig = {
+  providerApiKeyResolver?: ResolveApiKeyFn | null;
+  runtimeAuthForModelResolver?: GetRuntimeAuthForModelFn | null;
+  workspaceDir?: string;
+};
+
+export function fallbackLlmRuntimeContextFromConfig(
+  config: Pick<
+    GatewayBackedRuntimeConfig,
+    "providerApiKeyResolver" | "runtimeAuthForModelResolver" | "workspaceDir"
+  >,
+  overrides: FallbackLlmRuntimeContext = {},
+): FallbackLlmRuntimeContext {
+  return {
+    workspaceDir: config.workspaceDir,
+    resolveApiKeyForProvider: config.providerApiKeyResolver,
+    getRuntimeAuthForModel: config.runtimeAuthForModelResolver,
+    codexSubscriptionRunner: getCodexSubscriptionRunnerForOwner(config),
+    ...overrides,
+  };
+}
+
+const HTTP_STATUS_IN_MESSAGE = /\b([1-5]\d{2})\b/;
+
+export function isTimeoutError(err: unknown): boolean {
+  const msg = err instanceof Error ? err.message : String(err);
+  return /timed out after \d+ms/i.test(msg) || /timed out before request started/i.test(msg);
+}
+
+function isEmptyProviderResponse(err: unknown): boolean {
+  const msg = err instanceof Error ? err.message : String(err);
+  return /^Empty response from /i.test(msg);
+}
+
+function finiteHttpStatus(value: unknown): number | undefined {
+  if (typeof value !== "number" || !Number.isFinite(value) || !Number.isInteger(value)) return undefined;
+  if (value < 100 || value > 599) return undefined;
+  return value;
+}
+
+function httpStatusFromProviderError(err: unknown): number | undefined {
+  if (err && typeof err === "object") {
+    if ("status" in err) {
+      const status = finiteHttpStatus(err.status);
+      if (status !== undefined) return status;
+    }
+    if ("statusCode" in err) {
+      const status = finiteHttpStatus(err.statusCode);
+      if (status !== undefined) return status;
+    }
+  }
+  const msg = err instanceof Error ? err.message : String(err);
+  const match = HTTP_STATUS_IN_MESSAGE.exec(msg);
+  if (!match) return undefined;
+  return finiteHttpStatus(Number(match[1]));
+}
+
+function errorClassForHttpStatus(status: number): string {
+  if (status >= 500) return "http_5xx";
+  if (status >= 400) return "http_4xx";
+  return `http_${status}`;
+}
+
+export function classifyThrownProviderError(err: unknown): Omit<StructuredParseFailure, "result"> {
+  if (isTimeoutError(err)) {
+    return { failureReason: "timeout", errorClass: "timeout" };
+  }
+  if (isEmptyProviderResponse(err)) {
+    return { failureReason: "empty", errorClass: "empty" };
+  }
+  const httpStatus = httpStatusFromProviderError(err);
+  if (httpStatus !== undefined) {
+    return {
+      failureReason: "http_error",
+      httpStatus,
+      errorClass: errorClassForHttpStatus(httpStatus),
+    };
+  }
+  return { failureReason: "http_error", errorClass: "network" };
+}
+
+export function normalizeRuntimePath(value: unknown): string | undefined {
+  if (typeof value !== "string") return undefined;
+  const trimmed = value.trim();
+  return trimmed.length > 0 ? expandTildePath(trimmed) : undefined;
+}
+
+export function readGatewayWorkspaceDir(gatewayConfig: GatewayConfig | undefined): string | undefined {
+  if (!gatewayConfig || typeof gatewayConfig !== "object") return undefined;
+  const raw = gatewayConfig as Record<string, unknown>;
+  return (
+    normalizeRuntimePath(raw.workspaceDir) ??
+    normalizeRuntimePath(raw.workspacePath) ??
+    normalizeRuntimePath(raw.workspace)
+  );
+}
+
+export function defaultOpenClawWorkspaceDir(): string {
+  return path.join(resolveHomeDir(), ".openclaw", "workspace");
+}

package/src/fallback-llm-parse-failure.test.ts

@@ -0,0 +1,156 @@
+/**
+ * Distinct structured-parse failure classes (issue #2968).
+ *
+ * HTTP 200 empty, invalid JSON, schema-invalid JSON, HTTP errors, and the
+ * abort/timeout race must not collapse into one reason. Fingerprints stay
+ * retryable because these map to provider_retryable / parse_empty, which the
+ * extraction-run coordinator never marks processed.
+ */
+import assert from "node:assert/strict";
+import test from "node:test";
+import { z } from "zod";
+
+import { classifyFallbackParseFailure } from "./extraction-error-classification.js";
+import { FallbackLlmClient } from "./fallback-llm.js";
+import { clearModelsJsonCache } from "./models-json.js";
+import { clearSecretCache } from "./resolve-provider-secret.js";
+
+const Schema = z.object({ ok: z.literal(true) });
+
+function makeClient(): FallbackLlmClient {
+  return new FallbackLlmClient({
+    agents: { defaults: { model: { primary: "openai/test-model" } } },
+    models: {
+      providers: {
+        openai: {
+          baseUrl: "https://openai.example/v1",
+          api: "openai-completions",
+          apiKey: "key",
+          models: [],
+        },
+      },
+    },
+  });
+}
+
+async function parse(
+  fetchImpl: typeof fetch,
+  options: { timeoutMs?: number; signal?: AbortSignal } = {},
+) {
+  const originalFetch = globalThis.fetch;
+  globalThis.fetch = fetchImpl;
+  const client = makeClient();
+  try {
+    return await client.parseWithSchemaDetailed(
+      [
+        { role: "system", content: "Return JSON." },
+        { role: "user", content: "Do the thing." },
+      ],
+      { parse: (data: unknown) => Schema.parse(data) },
+      { temperature: 0, maxTokens: 16, ...options },
+    );
+  } finally {
+    globalThis.fetch = originalFetch;
+    clearModelsJsonCache();
+    clearSecretCache();
+  }
+}
+
+function jsonResponse(body: unknown, status = 200): Response {
+  return new Response(JSON.stringify(body), {
+    status,
+    headers: { "Content-Type": "application/json" },
+  });
+}
+
+test("parseWithSchemaDetailed: HTTP 200 empty content is empty, not http_error", { concurrency: false }, async () => {
+  const detailed = await parse(async () =>
+    jsonResponse({ choices: [{ message: { content: "" } }] }),
+  );
+  assert.equal(detailed.result, null);
+  if (detailed.result !== null) throw new Error("expected failure");
+  assert.equal(detailed.failureReason, "empty");
+  assert.equal(detailed.errorClass, "empty");
+  assert.equal(classifyFallbackParseFailure(detailed.failureReason), "parse_empty");
+});
+
+test("parseWithSchemaDetailed: invalid JSON is empty, not schema_rejection", { concurrency: false }, async () => {
+  const detailed = await parse(async () =>
+    jsonResponse({ choices: [{ message: { content: "not-json {{{" } }] }),
+  );
+  assert.equal(detailed.result, null);
+  if (detailed.result !== null) throw new Error("expected failure");
+  assert.equal(detailed.failureReason, "empty");
+  assert.equal(detailed.errorClass, "empty");
+  assert.notEqual(detailed.failureReason, "schema_rejection");
+  assert.notEqual(detailed.failureReason, "http_error");
+  assert.equal(classifyFallbackParseFailure(detailed.failureReason), "parse_empty");
+});
+
+test("parseWithSchemaDetailed: schema-invalid JSON is schema_rejection", { concurrency: false }, async () => {
+  const detailed = await parse(async () =>
+    jsonResponse({ choices: [{ message: { content: '{"ok":false}' } }] }),
+  );
+  assert.equal(detailed.result, null);
+  if (detailed.result !== null) throw new Error("expected failure");
+  assert.equal(detailed.failureReason, "schema_rejection");
+  assert.equal(detailed.errorClass, "schema_rejection");
+  assert.equal(detailed.attemptedModel, "openai/test-model");
+  assert.equal(classifyFallbackParseFailure(detailed.failureReason), "parse_empty");
+});
+
+test("parseWithSchemaDetailed: HTTP error is http_error with redacted status class", { concurrency: false }, async () => {
+  const detailed = await parse(
+    async () =>
+      new Response("upstream error: auth rejected by upstream (401 Unauthorized)", {
+        status: 502,
+        headers: { "Content-Type": "text/plain" },
+      }),
+  );
+  assert.equal(detailed.result, null);
+  if (detailed.result !== null) throw new Error("expected failure");
+  assert.equal(detailed.failureReason, "http_error");
+  assert.equal(detailed.httpStatus, 502);
+  assert.equal(detailed.errorClass, "http_5xx");
+  assert.equal(detailed.attemptedModel, "openai/test-model");
+  assert.equal(classifyFallbackParseFailure(detailed.failureReason), "provider_retryable");
+  const serialized = JSON.stringify(detailed);
+  assert.equal(serialized.includes("auth rejected"), false);
+  assert.equal(serialized.includes("Unauthorized"), false);
+});
+
+test("parseWithSchemaDetailed: abort/timeout race is timeout, not http_error", { concurrency: false }, async () => {
+  const detailed = await parse(
+    async (_url, init) => {
+      const signal = init?.signal;
+      return await new Promise<Response>((_resolve, reject) => {
+        const onAbort = (): void => {
+          reject(signal?.reason ?? new Error("aborted"));
+        };
+        if (signal?.aborted) {
+          onAbort();
+          return;
+        }
+        signal?.addEventListener("abort", onAbort, { once: true });
+      });
+    },
+    { timeoutMs: 25 },
+  );
+  assert.equal(detailed.result, null);
+  if (detailed.result !== null) throw new Error("expected failure");
+  assert.equal(detailed.failureReason, "timeout");
+  assert.equal(detailed.errorClass, "timeout");
+  assert.notEqual(detailed.failureReason, "http_error");
+  assert.equal(classifyFallbackParseFailure(detailed.failureReason), "provider_retryable");
+});
+
+test("parseWithSchemaDetailed: success shape still uses modelUsed as discriminant", { concurrency: false }, async () => {
+  const detailed = await parse(async () =>
+    jsonResponse({ choices: [{ message: { content: '{"ok":true}' } }] }),
+  );
+  assert.equal("modelUsed" in detailed, true);
+  assert.notEqual(detailed.result, null);
+  if (detailed.result === null) throw new Error("expected success");
+  assert.deepEqual(detailed.result, { ok: true });
+  assert.equal(detailed.modelUsed, "openai/test-model");
+});

package/src/fallback-llm.test.ts

@@ -3,6 +3,7 @@ import path from "node:path";
 import test from "node:test";
 
 import { FallbackLlmClient, gatewayTaskChainOptions } from "./fallback-llm.js";
+import { classifyAnalysisProviderError } from "./activity/timeline/analysis-provider.js";
 import { __codexCliFallbackTestHooks } from "./cli-fallback.js";
 import { initLogger, resetLogger } from "./logger.js";
 import { clearModelsJsonCache, __setModelsJsonForTest } from "./models-json.js";
@@ -1944,3 +1945,60 @@ test("fallback llm retries without responsesJsonSchema when the Responses API re
     clearSecretCache();
   }
 });
+
+test("chatCompletionDetailed preserves a 429 rate-limit failure cause (issue #2891)", { concurrency: false }, async () => {
+  clearModelsJsonCache();
+  clearSecretCache();
+  const llm = new FallbackLlmClient({
+    agents: { defaults: { model: { primary: "openai/test-model" } } },
+    models: { providers: { openai: { baseUrl: "https://openai.example/v1", api: "openai-completions", apiKey: "key", models: [] } } },
+  });
+  const originalFetch = globalThis.fetch;
+  globalThis.fetch = (async () =>
+    new Response(JSON.stringify({ error: { message: "rate limit exceeded" } }), { status: 429 })) as typeof fetch;
+  try {
+    const failureDiag: { lastError?: unknown } = {};
+    assert.equal(
+      await llm.chatCompletion([{ role: "user", content: "test" }], { failureDiag }),
+      null,
+    );
+    const lastError = failureDiag.lastError;
+    assert.ok(lastError instanceof Error, "expected the terminal provider error to survive the chain");
+    assert.equal("status" in lastError ? lastError.status : undefined, 429);
+    assert.equal(classifyAnalysisProviderError(lastError), "rate_limited");
+  } finally {
+    globalThis.fetch = originalFetch;
+    clearModelsJsonCache();
+    clearSecretCache();
+  }
+});
+
+test("timeout records TimeoutError on failureDiag (issue #2891)", { concurrency: false }, async () => {
+  clearModelsJsonCache();
+  clearSecretCache();
+  const restoreRunner = __codexCliFallbackTestHooks.setRunCodexCliForTest(
+    async (request) =>
+      await new Promise<never>(() => {
+        request.options.signal?.addEventListener("abort", () => undefined, { once: true });
+      }),
+  );
+  const llm = new FallbackLlmClient({
+    agents: { defaults: { model: { primary: "codex-cli/gpt-custom" } } },
+    models: { providers: { "codex-cli": { baseUrl: "", api: "codex-cli", apiKey: "codex-test-key", models: [] } } },
+  });
+  try {
+    const failureDiag: { lastError?: unknown } = {};
+    assert.equal(
+      await llm.chatCompletion([{ role: "user", content: "test" }], { timeoutMs: 30, failureDiag }),
+      null,
+    );
+    const lastError = failureDiag.lastError;
+    assert.ok(lastError instanceof Error, "expected the outer timeout to survive as lastError");
+    assert.equal(lastError.name, "TimeoutError");
+    assert.equal(classifyAnalysisProviderError(lastError), "timeout");
+  } finally {
+    restoreRunner();
+    clearModelsJsonCache();
+    clearSecretCache();
+  }
+});