@opengsd/gsd-pi 1.2.0 → 1.3.0-dev.72e3af2a

package/src/resources/extensions/gsd/bootstrap/agent-end-recovery.ts

@@ -1,6 +1,8 @@
 // gsd-pi + src/resources/extensions/gsd/bootstrap/agent-end-recovery.ts - Handles provider and agent-end recovery for GSD auto-mode.
 
 import type { ExtensionAPI, ExtensionContext } from "@gsd/pi-coding-agent";
+import { mkdirSync, readdirSync, writeFileSync } from "node:fs";
+import { join } from "node:path";
 
 import type { AgentEndEvent, ErrorContext } from "../auto/types.js";
 import { logWarning } from "../workflow-logger.js";
@@ -11,7 +13,7 @@ import {
   maybeHandleEmptyIntentTurn,
   resetEmptyTurnCounter,
 } from "../guided-flow.js";
-import { clearPathCache } from "../paths.js";
+import { clearPathCache, gsdRoot } from "../paths.js";
 import {
   getAutoDashboardData,
   getAutoModeStartModel,
@@ -19,6 +21,7 @@ import {
   isAutoCompletionStopInProgress,
   pauseAuto,
   setCurrentDispatchedModelId,
+  setCurrentUnitModelForRecovery,
 } from "../auto.js";
 import { getNextFallbackModel, resolveModelWithFallbacksForUnit } from "../preferences.js";
 import { pauseAutoForProviderError } from "../provider-error-pause.js";
@@ -32,7 +35,8 @@ import { shouldIgnoreAgentEndForActiveUnit } from "../auto/unit-runner-events.js
 import { resolveModelId } from "../auto-model-selection.js";
 import { resolveProjectRoot } from "../worktree.js";
 import { clearDiscussionFlowState } from "./write-gate.js";
-import { clearGuidedUnitContext } from "../guided-unit-context.js";
+import { scheduleFallbackContinuation } from "./fallback-continuation.js";
+import { clearGuidedUnitContext, getGuidedUnitContext, type GuidedUnitContext } from "../guided-unit-context.js";
 import { resumeAutoAfterProviderDelay } from "./provider-error-resume.js";
 import {
   classifyError,
@@ -41,12 +45,10 @@ import {
   isTransient,
   type ErrorClass,
 } from "../error-classifier.js";
-import { blockModel, isModelBlocked } from "../blocked-models.js";
+import { blockModel, blockModelUntil, isModelBlocked, isModelTemporarilyUnavailable } from "../blocked-models.js";
 import { getProjectGSDPreferencesPath } from "../preferences.js";
-import {
-  formatProviderErrorGuidance,
-  resolveProviderErrorGuidance,
-} from "../provider-error-guidance.js";
+import { resolveProviderErrorGuidance } from "../provider-error-guidance.js";
+import { formatGuidance } from "../guidance.js";
 
 const retryState = createRetryState();
 const MAX_NETWORK_RETRIES = 2;
@@ -116,9 +118,14 @@ export function isUserInitiatedAbortMessage(message: string | undefined | null):
 export function shouldDeferTransientErrorToCoreRetry(
   cls: ErrorClass,
   rawErrorMsg: string,
+  deferCheckMsg: string = rawErrorMsg,
 ): boolean {
   if (!isTransient(cls) || cls.kind === "rate-limit") return false;
-  return !/retry failed after \d+ attempts:/i.test(rawErrorMsg);
+  // Empty rawErrorMsg means the SDK terminated the session without providing an
+  // error string — core is done, not mid-retry.  GSD must schedule its own
+  // retry rather than silently deferring to a core that has already exited.
+  if (!rawErrorMsg) return false;
+  return !/retry failed after \d+ attempts:/i.test(deferCheckMsg);
 }
 
 type ProviderModelFallbackParams = {
@@ -145,15 +152,17 @@ async function tryProviderModelFallback(params: ProviderModelFallbackParams): Pr
       const nextModelId = getNextFallbackModel(cursorModelId, modelConfig);
       if (!nextModelId) break;
       const candidate = resolveModelId(nextModelId, availableModels, rejectedProvider);
-      if (candidate && !isModelBlocked(basePath, candidate.provider, candidate.id)) {
+      if (
+        candidate &&
+        !isModelBlocked(basePath, candidate.provider, candidate.id) &&
+        !isModelTemporarilyUnavailable(basePath, candidate.provider, candidate.id)
+      ) {
         const ok = await pi.setModel(candidate, { persist: false });
         if (ok) {
+          setCurrentUnitModelForRecovery(candidate);
           setCurrentDispatchedModelId({ provider: candidate.provider, id: candidate.id });
           switchedNotify(`${candidate.provider}/${candidate.id}`);
-          pi.sendMessage(
-            { customType: "gsd-auto-timeout-recovery", content: "Continue execution.", display: false },
-            { triggerTurn: true },
-          );
+          scheduleFallbackContinuation(pi);
           return true;
         }
       }
@@ -165,7 +174,8 @@ async function tryProviderModelFallback(params: ProviderModelFallbackParams): Pr
   if (
     sessionModel &&
     !(sessionModel.provider === rejectedProvider && sessionModel.id === rejectedId) &&
-    !isModelBlocked(basePath, sessionModel.provider, sessionModel.id)
+    !isModelBlocked(basePath, sessionModel.provider, sessionModel.id) &&
+    !isModelTemporarilyUnavailable(basePath, sessionModel.provider, sessionModel.id)
   ) {
     const startModel = availableModels.find(
       (m) => m.provider === sessionModel.provider && m.id === sessionModel.id,
@@ -173,12 +183,10 @@ async function tryProviderModelFallback(params: ProviderModelFallbackParams): Pr
     if (startModel) {
       const ok = await pi.setModel(startModel, { persist: false });
       if (ok) {
+        setCurrentUnitModelForRecovery(startModel);
         setCurrentDispatchedModelId({ provider: startModel.provider, id: startModel.id });
         switchedNotify(`${startModel.provider}/${startModel.id}`);
-        pi.sendMessage(
-          { customType: "gsd-auto-timeout-recovery", content: "Continue execution.", display: false },
-          { triggerTurn: true },
-        );
+        scheduleFallbackContinuation(pi);
         return true;
       }
     }
@@ -387,6 +395,108 @@ export function suppressTerminalDeletedWorktreeMessageEnd(event: MessageEndLike)
   return true;
 }
 
+function modelLabel(ctx: ExtensionContext): string {
+  const provider = ctx.model?.provider;
+  const id = ctx.model?.id;
+  return provider && id ? `${provider}/${id}` : "unknown model";
+}
+
+function isFatalManualGuidedTerminalFailure(lastMsg: unknown): boolean {
+  if (!isObjectRecord(lastMsg) || !("stopReason" in lastMsg)) return false;
+  if (lastMsg.stopReason === "error") {
+    const rawErrorMsg = ("errorMessage" in lastMsg && lastMsg.errorMessage) ? String(lastMsg.errorMessage) : "";
+    if (isUserInitiatedAbortMessage(rawErrorMsg)) return false;
+    return true;
+  }
+  if (lastMsg.stopReason !== "aborted") return false;
+
+  const content = "content" in lastMsg ? lastMsg.content : undefined;
+  const hasErrorMessage = "errorMessage" in lastMsg && !!lastMsg.errorMessage;
+  return hasErrorMessage || !_hasEmptyAgentEndContent(content);
+}
+
+function terminalFailureDetail(lastMsg: unknown): string {
+  if (!isObjectRecord(lastMsg)) return "Provider turn ended with an unknown terminal error.";
+  const rawErrorMsg = ("errorMessage" in lastMsg && lastMsg.errorMessage) ? String(lastMsg.errorMessage) : "";
+  const content = "content" in lastMsg ? lastMsg.content : undefined;
+  const displayMsg = resolveAgentEndErrorDisplay(rawErrorMsg, content).replace(/\s+/g, " ").trim();
+  if (displayMsg) return displayMsg.length > 300 ? `${displayMsg.slice(0, 300)}...` : displayMsg;
+  return lastMsg.stopReason === "aborted"
+    ? "Provider turn aborted with error context."
+    : "Provider stream ended with stopReason=error.";
+}
+
+function nextManualActivitySequence(activityDir: string): string {
+  let maxSeq = 0;
+  try {
+    for (const file of readdirSync(activityDir)) {
+      const match = /^(\d+)-/.exec(file);
+      if (match) maxSeq = Math.max(maxSeq, Number.parseInt(match[1]!, 10));
+    }
+  } catch {
+    return "001";
+  }
+  return String(maxSeq + 1).padStart(3, "0");
+}
+
+function writeManualGuidedTerminalErrorActivity(
+  basePath: string,
+  unitType: string,
+  model: string,
+  detail: string,
+  stopReason: unknown,
+): void {
+  const activityDir = join(gsdRoot(basePath), "activity");
+  mkdirSync(activityDir, { recursive: true });
+  const seq = nextManualActivitySequence(activityDir);
+  const safeUnitType = unitType.replace(/[^a-z0-9_.-]+/gi, "-");
+  const markerPath = join(activityDir, `${seq}-${safeUnitType}-manual-terminal-provider-error.jsonl`);
+  const message = `Manual guided ${unitType} turn ended with provider ${String(stopReason)} on ${model}: ${detail}`;
+  writeFileSync(
+    markerPath,
+    JSON.stringify({
+      type: "message",
+      message: {
+        role: "toolResult",
+        toolCallId: "manual-guided-terminal-provider-error",
+        toolName: "provider",
+        isError: true,
+        content: [{ type: "text", text: message }],
+      },
+    }) + "\n",
+    "utf-8",
+  );
+}
+
+function observeManualDiscussTerminalError(
+  ctx: ExtensionContext,
+  lastMsg: unknown,
+  guidedUnit: GuidedUnitContext | null,
+): void {
+  if (!guidedUnit?.unitType.startsWith("discuss-")) return;
+  if (!isFatalManualGuidedTerminalFailure(lastMsg)) return;
+
+  const model = modelLabel(ctx);
+  const detail = terminalFailureDetail(lastMsg);
+  ctx.ui.notify(
+    `Manual /gsd discuss ${guidedUnit.unitType} ended with a provider error on ${model}: ${detail}`,
+    "warning",
+  );
+
+  try {
+    writeManualGuidedTerminalErrorActivity(
+      guidedUnit.basePath,
+      guidedUnit.unitType,
+      model,
+      detail,
+      isObjectRecord(lastMsg) ? lastMsg.stopReason : "unknown",
+    );
+  } catch (err) {
+    const message = err instanceof Error ? err.message : String(err);
+    logWarning("bootstrap", `Failed to write manual guided terminal-error activity marker: ${message}`);
+  }
+}
+
 async function pauseTransientWithBackoff(
   cls: ErrorClass,
   pi: ExtensionAPI,
@@ -436,7 +546,9 @@ export async function handleAgentEnd(
   // rejected" loop even though the files are on disk.
   clearPathCache();
   const basePath = resolveAgentEndBasePath();
-  clearGuidedUnitContext(basePath);
+  const lastMsg = event.messages[event.messages.length - 1];
+  const guidedUnit = basePath ? getGuidedUnitContext(basePath) ?? getGuidedUnitContext() : getGuidedUnitContext();
+  clearGuidedUnitContext(guidedUnit?.basePath ?? basePath);
 
   try {
     if (await checkDeepProjectSetupAfterTurn(event, ctx, basePath)) {
@@ -465,13 +577,15 @@ export async function handleAgentEnd(
   // discussions (where isAutoActive may be false) still get recovered.
   if (maybeHandleEmptyIntentTurn(event, isAutoActive(), basePath)) return;
 
-  if (!isAutoActive()) return;
+  if (!isAutoActive()) {
+    observeManualDiscussTerminalError(ctx, lastMsg, guidedUnit);
+    return;
+  }
 
   if (shouldIgnoreAgentEndForActiveUnit(event)) {
     return;
   }
 
-  const lastMsg = event.messages[event.messages.length - 1];
   if (isSessionSwitchInFlight()) {
     _handleSessionSwitchAgentEnd(lastMsg, resolveAgentEndCancelled);
     return;
@@ -554,9 +668,9 @@ export async function handleAgentEnd(
       });
       return;
     }
-    // #3588: When errorMessage is uninformative, extract the real error from
-    // the assistant message text content for display purposes only.
-    // Classification still uses rawErrorMsg to avoid false positives from prose.
+    // #3588/#956: When errorMessage is uninformative, extract the real error
+    // from assistant text. Prefer rawErrorMsg for classification to avoid
+    // prose false-positives, but use display text when rawErrorMsg is empty.
     const displayMsg = resolveAgentEndErrorDisplay(
       rawErrorMsg,
       "content" in lastMsg ? lastMsg.content : undefined,
@@ -575,8 +689,8 @@ export async function handleAgentEnd(
     const errorDetail = displayMsg ? `: ${displayMsg}` : "";
     const explicitRetryAfterMs = ("retryAfterMs" in lastMsg && typeof lastMsg.retryAfterMs === "number") ? lastMsg.retryAfterMs : undefined;
 
-    // ── 1. Classify using rawErrorMsg to avoid prose false-positives ────
-    const cls = classifyError(rawErrorMsg, explicitRetryAfterMs);
+    // ── 1. Classify, preserving non-empty errorMessage precedence ──────
+    const cls = classifyError(rawErrorMsg || displayMsg, explicitRetryAfterMs);
 
     // ── 1a. Unsupported-model: provider rejected this model for the current
     //        account/plan at request time (#4513).  Persist a block so the
@@ -627,7 +741,7 @@ export async function handleAgentEnd(
         preferencesPath: dash.basePath ? getProjectGSDPreferencesPath(dash.basePath) : undefined,
         hasConfiguredFallbacks: (modelConfig?.fallbacks.length ?? 0) > 0,
       });
-      const guidanceText = formatProviderErrorGuidance(guidance);
+      const guidanceText = formatGuidance(guidance);
 
       await pauseForProviderModelRejection(ctx, pi, {
         errorDetail,
@@ -654,11 +768,32 @@ export async function handleAgentEnd(
     // Core retries transient failures in-session after this handler.
     // Keep that behavior for non-rate-limit classes to avoid pause/retry races,
     // but let rate-limit continue into model fallback logic below (#4373).
-    if (shouldDeferTransientErrorToCoreRetry(cls, rawErrorMsg)) {
+    if (shouldDeferTransientErrorToCoreRetry(cls, rawErrorMsg, rawErrorMsg || displayMsg)) {
       return;
     }
 
+    // ── Tool-schema overload: the active model repeatedly emitted tool-call
+    //    arguments that fail schema validation (e.g. a model whose tool-call
+    //    grammar can't satisfy GSD's schemas). Before hard-pausing, try the
+    //    unit's configured fallbacks and the auto-mode start model — the model
+    //    the user actually selected. This lets auto-mode recover by switching
+    //    back to a schema-capable model instead of aborting outright (#813).
+    //    Do not persistently block the model: schema-overload is request-shape
+    //    specific, mirroring the model-error path above.
     if (cls.kind === "tool-schema") {
+      const dash = getAutoDashboardData();
+      const switched = await tryProviderModelFallback({
+        ctx,
+        pi,
+        rejectedProvider: ctx.model?.provider,
+        rejectedId: ctx.model?.id,
+        basePath: dash.basePath,
+        unitType: dash.currentUnit?.type,
+        switchedNotify: (label) =>
+          ctx.ui.notify(`Switched to ${label} after repeated tool schema validation failures.`, "warning"),
+      });
+      if (switched) return;
+
       await pauseAutoForProviderError(ctx.ui, errorDetail, () => pauseAuto(ctx, pi, {
         message: `Tool schema error${errorDetail}`,
         category: "tool-schema",
@@ -671,13 +806,27 @@ export async function handleAgentEnd(
       return;
     }
 
-    // Cap rate-limit backoff for CLI-style providers (openai-codex, google-gemini-cli)
+    // Cap rate-limit backoff for CLI-style providers (openai-codex, google-gemini-cli, google-antigravity)
     // which use per-user quotas with shorter windows (#2922).
     if (cls.kind === "rate-limit") {
       const currentProvider = ctx.model?.provider;
-      if (currentProvider === "openai-codex" || currentProvider === "google-gemini-cli") {
+      if (
+        currentProvider === "openai-codex"
+        || currentProvider === "google-gemini-cli"
+        || currentProvider === "google-antigravity"
+      ) {
         cls.retryAfterMs = Math.min(cls.retryAfterMs, 30_000);
       }
+      const dash = getAutoDashboardData();
+      if (dash.basePath && ctx.model?.provider && ctx.model?.id) {
+        blockModelUntil(
+          dash.basePath,
+          ctx.model.provider,
+          ctx.model.id,
+          Date.now() + cls.retryAfterMs,
+          rawErrorMsg || displayMsg || "rate limit",
+        );
+      }
     }
 
     // ── 2. Decide & Act ──────────────────────────────────────────────────
@@ -712,47 +861,22 @@ export async function handleAgentEnd(
     // --- Transient errors: try model fallback first, then pause ---
     // Rate limits are often per-model, so switching models can bypass them.
     if (cls.kind === "rate-limit" || cls.kind === "network" || cls.kind === "server" || cls.kind === "connection" || cls.kind === "stream") {
-      // Try model fallback
       const dash = getAutoDashboardData();
-      if (dash.currentUnit) {
-        const modelConfig = resolveModelWithFallbacksForUnit(dash.currentUnit.type);
-        if (modelConfig && modelConfig.fallbacks.length > 0) {
-          const availableModels = ctx.modelRegistry.getAvailable();
-          const nextModelId = getNextFallbackModel(ctx.model?.id, modelConfig);
-          if (nextModelId) {
-            retryState.networkRetryCount = 0;
-            retryState.currentRetryModelId = undefined;
-            const modelToSet = resolveModelId(nextModelId, availableModels, ctx.model?.provider);
-            if (modelToSet) {
-              const ok = await pi.setModel(modelToSet, { persist: false });
-              if (ok) {
-                setCurrentDispatchedModelId({ provider: modelToSet.provider, id: modelToSet.id });
-                ctx.ui.notify(`Model error${errorDetail}. Switched to fallback: ${nextModelId} and resuming.`, "warning");
-                pi.sendMessage({ customType: "gsd-auto-timeout-recovery", content: "Continue execution.", display: false }, { triggerTurn: true });
-                return;
-              }
-            }
-          }
-        }
-      }
-
-      // Try restoring session model
-      const sessionModel = getAutoModeStartModel();
-      if (sessionModel) {
-        if (ctx.model?.id !== sessionModel.id || ctx.model?.provider !== sessionModel.provider) {
-          const startModel = ctx.modelRegistry.getAvailable().find((m) => m.provider === sessionModel.provider && m.id === sessionModel.id);
-          if (startModel) {
-            const ok = await pi.setModel(startModel, { persist: false });
-            if (ok) {
-              setCurrentDispatchedModelId({ provider: startModel.provider, id: startModel.id });
-              retryState.networkRetryCount = 0;
-              retryState.currentRetryModelId = undefined;
-              ctx.ui.notify(`Model error${errorDetail}. Restored session model: ${sessionModel.provider}/${sessionModel.id} and resuming.`, "warning");
-              pi.sendMessage({ customType: "gsd-auto-timeout-recovery", content: "Continue execution.", display: false }, { triggerTurn: true });
-              return;
-            }
-          }
-        }
+      const switched = await tryProviderModelFallback({
+        ctx,
+        pi,
+        rejectedProvider: ctx.model?.provider,
+        rejectedId: ctx.model?.id,
+        basePath: dash.basePath,
+        unitType: dash.currentUnit?.type,
+        switchedNotify: (label) => {
+          retryState.networkRetryCount = 0;
+          retryState.currentRetryModelId = undefined;
+          ctx.ui.notify(`Model error${errorDetail}. Switched to fallback: ${label} and resuming.`, "warning");
+        },
+      });
+      if (switched) {
+        return;
       }
     }
 

package/src/resources/extensions/gsd/bootstrap/core-session-tools.ts

@@ -0,0 +1,43 @@
+// Project/App: gsd-pi
+// File Purpose: Canonical core session tool surface shared by auto-mode scoping and loop guards.
+
+/** Base tools available during auto-mode execution before unit-specific GSD tools are added. */
+export const MINIMAL_AUTO_BASE_TOOL_NAMES = [
+  "ask_user_questions",
+  "bash",
+  "bg_shell",
+  "edit",
+  "find",
+  "glob",
+  "grep",
+  "fetch_page",
+  "search-the-web",
+  "ls",
+  "read",
+  "subagent",
+  "write",
+  "ToolSearch",
+] as const;
+
+/** Tools excluded from the high per-turn loop-guard cap (strict or orchestration). */
+const NON_REPEATABLE_FROM_MINIMAL_BASE = new Set<string>([
+  "ask_user_questions",
+  "subagent",
+  "ToolSearch",
+]);
+
+/** Additional core tools not in MINIMAL_AUTO_BASE but routinely multi-called per turn. */
+const EXTRA_INHERENTLY_REPEATABLE = [
+  "multi_edit",
+  "todo_write",
+  "notebook_edit",
+  "search_and_read",
+] as const;
+
+/** Core session tools that may be invoked many times per agent turn. */
+export const INHERENTLY_REPEATABLE_TOOL_NAMES = [
+  ...MINIMAL_AUTO_BASE_TOOL_NAMES.filter((name) => !NON_REPEATABLE_FROM_MINIMAL_BASE.has(name)),
+  ...EXTRA_INHERENTLY_REPEATABLE,
+] as const;
+
+export const INHERENTLY_REPEATABLE_TOOL_SET = new Set<string>(INHERENTLY_REPEATABLE_TOOL_NAMES);