@bitkyc08/opencodex 2.35.0 → 2.36.0-preview.20260830

package/src/server/responses/empty-completion-guard.ts

@@ -1,4 +1,5 @@
 import type { AdapterEvent, OcxConfig, OcxUsage } from "../../types";
+import { sanitizeLogMetadataString } from "../../lib/redact";
 
 /**
  * Empty-completion guard for Responses turns (port of codex-router's
@@ -21,6 +22,21 @@ import type { AdapterEvent, OcxConfig, OcxUsage } from "../../types";
  */
 export const EMPTY_COMPLETION_RETRY_ENV = "OCX_EMPTY_COMPLETION_RETRY";
 
+/**
+ * The observability notice for a turn that ended empty with the retry guard off.
+ *
+ * Both labels are caller-supplied: the request names its provider and model. Interpolated raw,
+ * a model name carrying newlines or terminal escapes writes additional lines into whatever
+ * reads this warning, so a caller could forge log records it never produced. Both are reduced
+ * to bounded single-line metadata first.
+ */
+export function emptyCompletionNotice(providerName: unknown, modelId: unknown): string {
+  const provider = sanitizeLogMetadataString(providerName) ?? "unknown";
+  const model = sanitizeLogMetadataString(modelId) ?? "unknown";
+  return `[opencodex] ${provider}/${model} completed with no output text and no tool call. `
+    + "Set \"emptyCompletionRetry\": true to retry such turns once.";
+}
+
 /** Retained pre-content events are bounded independently by count and encoded size. */
 export const EMPTY_COMPLETION_MAX_BUFFERED_EVENTS = 1_024;
 export const EMPTY_COMPLETION_MAX_BUFFERED_BYTES = 1_048_576;

package/src/server/responses/fetch-helpers.ts

@@ -108,6 +108,48 @@ export function providerFetch(
 
 
 
+/**
+ * Wrap a provider fetch so `onDispatch` fires immediately before the send, not before pacing.
+ *
+ * `fetchWithHeaderTimeout` awaits `waitForPacing` and only then calls the executor, so a caller
+ * that signals at the call site records a dispatch even when a rejected pacing wait means nothing
+ * reached the network. That matters when the signal bounds later recovery: the request would lose
+ * its fallback on the strength of a send that never happened.
+ *
+ * The pacing surface is preserved deliberately. `waitForPacing` and `unpacedFetch` are read off
+ * the executor by `fetchWithHeaderTimeout`, so a plain function wrapper would silently drop
+ * provider pacing and double-send the slot.
+ */
+export function storedPoolReplayDispatchNotifier(
+  executor: ProviderFetch,
+  onDispatch: (() => void) | undefined,
+): ProviderFetch {
+  if (!onDispatch) return executor;
+  let notified = false;
+  const notifyOnce = (): void => {
+    if (notified) return;
+    notified = true;
+    onDispatch();
+  };
+  const unpacedSource = executor.unpacedFetch ?? executor;
+  const unpaced = Object.assign(
+    (input: Parameters<typeof globalThis.fetch>[0], init?: RequestInit) => {
+      notifyOnce();
+      return unpacedSource(input, init);
+    },
+    { preconnect: unpacedSource.preconnect },
+  ) as ProviderFetch["unpacedFetch"];
+  const wrapped = async (input: Parameters<typeof globalThis.fetch>[0], init?: RequestInit) => {
+    await executor.waitForPacing?.(init?.signal ?? undefined);
+    return unpaced!(input, init);
+  };
+  return Object.assign(wrapped, {
+    preconnect: executor.preconnect,
+    waitForPacing: executor.waitForPacing,
+    unpacedFetch: unpaced,
+  }) as ProviderFetch;
+}
+
 export async function fetchWithHeaderTimeout(
   url: string,
   init: Omit<RequestInit, "signal">,

package/src/server/responses/policy-fallback.ts

@@ -116,16 +116,21 @@ export async function handleResponsesWithPolicyFallback(
 ): Promise<Response> {
   const runCore = deps.runCore ?? handleResponsesCore;
   let requestBodyReadNotified = false;
-  const coreOptions: CoreOptions = options.onRequestBodyRead
-    ? {
-      ...options,
+  let storedPool401ReplayDispatched = false;
+  const coreOptions: CoreOptions = {
+    ...options,
+    ...(options.onRequestBodyRead ? {
       onRequestBodyRead: () => {
         if (requestBodyReadNotified) return;
         requestBodyReadNotified = true;
         options.onRequestBodyRead?.();
       },
-    }
-    : options;
+    } : {}),
+    onStoredPool401ReplayDispatched: () => {
+      storedPool401ReplayDispatched = true;
+      options.onStoredPool401ReplayDispatched?.();
+    },
+  };
   let rawBody: Record<string, unknown> | null = null;
   try {
     const parsed = await readJsonRequestBody(req.clone());
@@ -150,7 +155,7 @@ export async function handleResponsesWithPolicyFallback(
     candidateKey({ provider: initialTrace.selected.provider, model: initialTrace.selected.model }),
   ]);
 
-  while (await shouldHopPolicyCandidate(response, req.signal)) {
+  while (!storedPool401ReplayDispatched && await shouldHopPolicyCandidate(response, req.signal)) {
     if (req.signal.aborted) return response;
     const next = rankPolicyFallbackCandidates(initialTrace, tried)[0];
     if (!next) return response;

package/src/server/responses-undeclared-tool-guard.ts

@@ -1,8 +1,14 @@
-import { namespacedToolName, normalizeDeclaredToolName } from "../types";
+import {
+  CODE_MODE_EXEC_TOOL_NAME,
+  namespacedToolName,
+  normalizeDeclaredToolName,
+} from "../types";
 import { sseDataPayload, type SseBlockRewrite } from "./sse-payload-rewrite";
 
 /** Item types the client executes through a request-declared wire name. */
 const CLIENT_EXECUTED_CALL_TYPES = new Set(["function_call", "custom_tool_call"]);
+/** Codex groups ordinary top-level tools here; unlike an MCP namespace, it has no wire prefix. */
+const BUILTIN_FUNCTIONS_NAMESPACE = "functions";
 
 /**
  * Hosted declarations whose response items the PROVIDER executes, keyed by the request
@@ -79,11 +85,18 @@ function addWireToolName(names: Set<string>, tool: unknown, namespace?: string):
       ? nestedFunction.name
       : undefined;
   if (!name) return;
-  names.add(name);
   // Codex routes MCP calls by an explicit `namespace` field, so the same tool is reachable
   // as a bare inner name or as the flattened form; accept both rather than guess which
   // coordinate system this provider echoes back.
-  if (namespace) names.add(namespacedToolName(namespace, name));
+  if (!namespace || namespace === BUILTIN_FUNCTIONS_NAMESPACE) {
+    names.add(name);
+    return;
+  }
+  names.add(namespacedToolName(namespace, name));
+  // `exec` is the one name that also switches on nested-helper normalization, so a bare alias
+  // for a namespaced MCP tool would silently authorize `exec_command`/`shell_command`/
+  // `apply_patch` the request never declared. Every other inner name keeps the bare alias.
+  if (name !== CODE_MODE_EXEC_TOOL_NAME) names.add(name);
 }
 
 /**

package/src/server/startup-health-cache.ts

@@ -9,13 +9,47 @@ import type { OcxConfig } from "../types";
 import { truncateRetainedUtf8 } from "../lib/admission";
 
 const CACHE_TTL_MS = 30_000;
-const PROBE_TIMEOUT_MS = 5_000;
-const INITIAL_PROBE_WAIT_MS = 5_500;
+const PROBE_TIMEOUT_MS = probeTimeoutMs();
+const INITIAL_PROBE_WAIT_MS = PROBE_TIMEOUT_MS + 500;
 const MAX_DIAGNOSTIC_VALUE_BYTES = 8 * 1024;
+
+/**
+ * How long the isolated probe child gets before its reading is abandoned.
+ *
+ * The child is a full Bun CLI start that then runs `diagnoseService()`, and on
+ * Windows that means shelling out to `sc.exe` / `schtasks.exe` — external
+ * processes whose latency is set by the service-control manager, not by us.
+ * Under load those overran the flat 5s, the probe was abandoned, and the
+ * endpoint answered `diagnosticStale: true` for a machine it could have read.
+ * That is a real dashboard regression, not only a test failure: it downgrades a
+ * `protected` host to `at-risk` and recommends a repair command for a healthy
+ * service.
+ *
+ * Raising it only on Windows keeps the tighter bound everywhere else. It stays a
+ * bound in both cases: a wedged probe is still abandoned, and the caller still
+ * receives the previous reading rather than waiting on it.
+ */
+function probeTimeoutMs(): number {
+  return process.platform === "win32" ? 15_000 : 5_000;
+}
+
+/** The probe bound, so a test's own budget cannot fall below what it must wait for. */
+export function startupHealthProbeTimeoutMs(): number {
+  return PROBE_TIMEOUT_MS;
+}
 let cached: { timestamp: number; value: StartupHealth } | null = null;
 let inflight: Promise<StartupHealth> | null = null;
 let generation = 0;
 
+export interface StartupHealthCacheDeps {
+  now?: () => number;
+  probe?: (config: Pick<OcxConfig, "codexAutoStart">) => Promise<StartupHealth>;
+  waitForProbe?: (
+    probe: Promise<StartupHealth>,
+    timeoutMs: number,
+  ) => Promise<StartupHealth | null>;
+}
+
 export function markStartupHealthDiagnosticStale(value: StartupHealth): StartupHealth {
   if (!value.localRoutingDependency) return { ...value, diagnosticStale: true };
   return {
@@ -94,11 +128,16 @@ function runProbe(config: Pick<OcxConfig, "codexAutoStart">): Promise<StartupHea
   });
 }
 
-function refreshInBackground(config: Pick<OcxConfig, "codexAutoStart">): void {
+function refreshInBackground(
+  config: Pick<OcxConfig, "codexAutoStart">,
+  deps: StartupHealthCacheDeps,
+): void {
   if (inflight) return;
   const startedGeneration = generation;
-  const probe = runProbe(config).then(value => {
-    if (startedGeneration === generation) cached = { timestamp: Date.now(), value };
+  const probe = (deps.probe ?? runProbe)(config).then(value => {
+    if (startedGeneration === generation) {
+      cached = { timestamp: (deps.now ?? Date.now)(), value };
+    }
     return value;
   });
   inflight = probe.finally(() => {
@@ -107,18 +146,25 @@ function refreshInBackground(config: Pick<OcxConfig, "codexAutoStart">): void {
 }
 
 /** Stale-while-revalidate: service-manager probes never hold open a model/UI request. */
-export async function getCachedStartupHealth(config: Pick<OcxConfig, "codexAutoStart">): Promise<StartupHealth> {
-  if (cached && Date.now() - cached.timestamp < CACHE_TTL_MS) return cached.value;
-  refreshInBackground(config);
+export async function getCachedStartupHealth(
+  config: Pick<OcxConfig, "codexAutoStart">,
+  deps: StartupHealthCacheDeps = {},
+): Promise<StartupHealth> {
+  const now = deps.now ?? Date.now;
+  if (cached && now() - cached.timestamp < CACHE_TTL_MS) return cached.value;
+  refreshInBackground(config, deps);
   // An expired or empty read is an explicit protection check. Wait for the
   // isolated probe instead of presenting a synthetic failure while that probe
   // is still running. The probe remains child-process isolated and hard-capped
-  // at 5s; stale state is returned only if that bounded probe cannot settle.
+  // by the platform-specific deadline; stale state is returned only if that
+  // bounded probe cannot settle.
   if (inflight) {
-    const settled = await Promise.race([
-      inflight,
-      new Promise<null>(resolve => setTimeout(() => resolve(null), INITIAL_PROBE_WAIT_MS)),
-    ]);
+    const settled = await (deps.waitForProbe
+      ? deps.waitForProbe(inflight, INITIAL_PROBE_WAIT_MS)
+      : Promise.race([
+          inflight,
+          new Promise<null>(resolve => setTimeout(() => resolve(null), INITIAL_PROBE_WAIT_MS)),
+        ]));
     if (settled) return settled;
   }
   return cached ? markStartupHealthDiagnosticStale(cached.value) : conservativeFallback(config);

package/src/service-manager-probe.ts

@@ -32,6 +32,19 @@ import { WINSW_SERVICE_ID } from "./lib/winsw";
 /** Short: this runs inside admission, and a slow answer is the same as none. */
 export const SERVICE_PROBE_TIMEOUT_MS = 2_000;
 
+/**
+ * The one query that is allowed to be slow: the full `schtasks` listing.
+ *
+ * 2s is the right budget for a targeted query and the wrong one for enumerating
+ * every task on the machine — measured at 12.3s on a host with 401 of them, which
+ * killed the listing and left ownership unprovable (#2914). This is not a general
+ * relaxation: the targeted queries keep the 2s ceiling, and after the
+ * locale-independent absence check above, a healthy host decides before the
+ * listing runs at all. Only a host that has already exhausted the cheap evidence
+ * pays this, and for it the alternative is not a fast answer but no answer.
+ */
+export const SERVICE_PROBE_LISTING_TIMEOUT_MS = 20_000;
+
 export type ServiceManagerBackend = "launchd" | "systemd" | "scheduler" | "winsw";
 
 export interface ServiceManagerClaim {
@@ -70,7 +83,11 @@ export interface ProbeRunner {
  * Windows probe runner: preserves schtasks stdout/stderr as raw bytes so the
  * UTF-16LE task XML is not corrupted by a UTF-8 decode.
  */
-export type RawProbeRunner = (file: string, args: readonly string[]) => {
+export type RawProbeRunner = (
+  file: string,
+  args: readonly string[],
+  options?: { readonly timeoutMs?: number },
+) => {
   status: number | null;
   stdout: Buffer;
   stderr: Buffer;
@@ -78,6 +95,57 @@ export type RawProbeRunner = (file: string, args: readonly string[]) => {
   spawnFailed: boolean;
 };
 
+type RawProbeResult = ReturnType<RawProbeRunner>;
+
+/**
+ * Startup-local memo for the expensive full Task Scheduler listing.
+ *
+ * The targeted `/tn ... /xml` query is deliberately NOT cached: it is the
+ * race-sensitive evidence that a task appeared between two ownership checks.
+ * A listing may be reused only when that fresh targeted query returned exactly
+ * the same bytes and status as the query that caused the listing. If the
+ * targeted evidence changes, the old absence proof is stale and another
+ * listing is required rather than turning uncertainty into absence.
+ *
+ * Only a SUCCESSFUL listing is retained. A stall or spawn failure is not
+ * evidence of anything, and caching it made one transient 20s timeout poison the
+ * rest of the startup: the targeted query is byte-identical on the next
+ * inspection, so the identity check passed, the listing was never retried, and
+ * ownership stayed unprovable for the whole run — refusing the write that #2914
+ * exists to allow.
+ */
+export interface WindowsTaskListingCache {
+  getOrRun(targetedQuery: RawProbeResult, run: () => RawProbeResult): RawProbeResult;
+}
+
+function rawProbeIdentity(result: RawProbeResult): string {
+  return [
+    result.status === null ? "null" : String(result.status),
+    result.timedOut ? "1" : "0",
+    result.spawnFailed ? "1" : "0",
+    result.stdout.toString("base64"),
+    result.stderr.toString("base64"),
+  ].join("\u0000");
+}
+
+/** Create one bounded cache for the synchronous ownership phase of one startup. */
+export function createWindowsTaskListingCache(): WindowsTaskListingCache {
+  let identity: string | null = null;
+  let result: RawProbeResult | null = null;
+  return {
+    getOrRun(targetedQuery, run) {
+      const nextIdentity = rawProbeIdentity(targetedQuery);
+      if (result !== null && identity === nextIdentity) return result;
+      const next = run();
+      if (!next.timedOut && !next.spawnFailed && next.status === 0) {
+        identity = nextIdentity;
+        result = next;
+      }
+      return next;
+    },
+  };
+}
+
 export const defaultProbeRunner: ProbeRunner = (file, args) => {
   const result = spawnSync(file, [...args], {
     encoding: "utf8",
@@ -94,11 +162,11 @@ export const defaultProbeRunner: ProbeRunner = (file, args) => {
   };
 };
 
-export const defaultRawProbeRunner: RawProbeRunner = (file, args) => {
+export const defaultRawProbeRunner: RawProbeRunner = (file, args, options) => {
   const result = spawnSync(file, [...args], {
     encoding: "buffer",
     windowsHide: true,
-    timeout: SERVICE_PROBE_TIMEOUT_MS,
+    timeout: options?.timeoutMs ?? SERVICE_PROBE_TIMEOUT_MS,
   });
   return {
     status: result.status,
@@ -122,6 +190,8 @@ export interface ProbeDeps {
   readonly winswStatus?: () => "started" | "stopped" | "nonexistent" | "unknown";
   /** Test seam for redirected Windows legacy-codepage output. */
   readonly windowsLocale?: string;
+  /** Startup-local full-listing cache; targeted task queries always bypass it. */
+  readonly windowsTaskListingCache?: WindowsTaskListingCache;
 }
 
 const LABEL = "com.opencodex.proxy";
@@ -323,7 +393,7 @@ function inspectLaunchd(deps: Required<Pick<ProbeDeps, "run" | "uid" | "home">>)
   };
 }
 
-function systemdProperty(out: string, key: string): string | null {
+export function systemdProperty(out: string, key: string): string | null {
   for (const line of out.split("\n")) {
     const match = line.match(new RegExp(`^${key}=(.*)$`));
     if (match) return match[1].trim();
@@ -535,7 +605,22 @@ function windowsTaskListContains(body: string, taskName: string): boolean {
   });
 }
 
-/** English hosts provide a decisive fast path; other locales fall back to a full listing. */
+/**
+ * The English message: a fast path on an English host, and nothing more.
+ *
+ * It cannot match a localized host — on zh-CN schtasks answers with the CP936
+ * bytes of `错误: 系统找不到指定的文件。` — which is why absence there has to be
+ * settled by the locale-neutral listing below (#2914). Adding more translated
+ * substrings would only cover the languages someone thought of, and each one is
+ * a chance to read a DIFFERENT refusal as absence.
+ *
+ * Deriving the host's own not-found wording from a control query looks like the
+ * general fix and is not: schtasks exits 1 for both "not found" and "access
+ * denied", so a locked-down host answers the control and the real query
+ * identically, and comparing them yields a false `absent` — the one direction
+ * that lets an unattended write proceed into a home another process owns.
+ * `tests/codex-service-manager-probe.test.ts` covers exactly that host.
+ */
 const SCHTASKS_TASK_NOT_FOUND_EN = /cannot find the file specified/i;
 
 /**
@@ -547,7 +632,8 @@ const SCHTASKS_TASK_NOT_FOUND_EN = /cannot find the file specified/i;
  * fallback, and only a successful list without our task proves absence.
  */
 function probeWindowsTaskRegistration(
-  deps: Required<Pick<ProbeDeps, "runRaw">> & Pick<ProbeDeps, "windowsLocale">,
+  deps: Required<Pick<ProbeDeps, "runRaw">>
+    & Pick<ProbeDeps, "windowsLocale" | "windowsTaskListingCache">,
 ): {
   registered: "present" | "absent" | "unknown";
   registeredXml: string;
@@ -574,7 +660,14 @@ function probeWindowsTaskRegistration(
     return { registered: "absent", registeredXml: "" };
   }
 
-  const listed = deps.runRaw(schtasks, ["/query", "/fo", "CSV", "/nh"]);
+  const runListing = () => deps.runRaw(
+    schtasks,
+    ["/query", "/fo", "CSV", "/nh"],
+    { timeoutMs: SERVICE_PROBE_LISTING_TIMEOUT_MS },
+  );
+  const listed = deps.windowsTaskListingCache
+    ? deps.windowsTaskListingCache.getOrRun(queried, runListing)
+    : runListing();
   if (listed.spawnFailed || listed.timedOut || listed.status !== 0) {
     return { registered: "unknown", registeredXml: "" };
   }
@@ -618,7 +711,7 @@ function probeWinswRegistration(
 
 function inspectWindows(
   deps: Required<Pick<ProbeDeps, "runRaw" | "home">>
-    & Pick<ProbeDeps, "configDir" | "winswStatus" | "windowsLocale">,
+    & Pick<ProbeDeps, "configDir" | "winswStatus" | "windowsLocale" | "windowsTaskListingCache">,
 ): ServiceManagerInstallation {
   const configDir = windowsConfigDirPath(deps);
   const taskXmlPath = join(configDir, "opencodex-service-task.xml");
@@ -681,7 +774,19 @@ function inspectWindows(
   }
 
   if (task === "absent") {
-    return schedulerRegistered
+    if (!schedulerRegistered) return { kind: "absent" };
+    if (!registration.registeredXml.trim()) {
+      return unknown("Task Scheduler is registered but its definition XML could not be read");
+    }
+    const launcherArg = windowsTaskArguments(registration.registeredXml);
+    if (!launcherArg) {
+      return unknown("Task Scheduler holds opencodex-proxy but its task XML is missing");
+    }
+    const launcherPath = /"([^"]+)"/.exec(launcherArg)?.[1];
+    if (!launcherPath) {
+      return unknown("Task Scheduler holds opencodex-proxy but its task XML is missing");
+    }
+    return windowsPathInsideConfigDir(launcherPath, configDir)
       ? unknown("Task Scheduler holds opencodex-proxy but its task XML is missing")
       : { kind: "absent" };
   }
@@ -886,6 +991,7 @@ export function inspectServiceManagerInstallation(deps: ProbeDeps = {}): Service
       configDir: deps.configDir,
       winswStatus: deps.winswStatus,
       windowsLocale: deps.windowsLocale,
+      windowsTaskListingCache: deps.windowsTaskListingCache,
     });
   }
   return unknown(`no service manager probe for platform ${platform}`);