gsd-pi 2.10.12 → 2.12.0

package/packages/pi-coding-agent/src/cli/args.ts

@@ -290,6 +290,7 @@ ${chalk.bold("Environment Variables:")}
   AI_GATEWAY_API_KEY               - Vercel AI Gateway API key
   ZAI_API_KEY                      - ZAI API key
   MISTRAL_API_KEY                  - Mistral API key
+  OLLAMA_API_KEY                   - Ollama Cloud API key
   MINIMAX_API_KEY                  - MiniMax API key
   OPENCODE_API_KEY                 - OpenCode Zen/OpenCode Go API key
   KIMI_API_KEY                     - Kimi For Coding API key

package/packages/pi-coding-agent/src/core/agent-session.ts

@@ -67,9 +67,9 @@ import {
 	type TurnEndEvent,
 	type TurnStartEvent,
 	wrapRegisteredTools,
-	wrapToolsWithExtensions,
 } from "./extensions/index.js";
 import type { BashExecutionMessage, CustomMessage } from "./messages.js";
+import { FallbackResolver } from "./fallback-resolver.js";
 import type { ModelRegistry } from "./model-registry.js";
 import { expandPromptTemplate, type PromptTemplate } from "./prompt-templates.js";
 import type { ResourceExtensionPaths, ResourceLoader } from "./resource-loader.js";
@@ -120,7 +120,10 @@ export type AgentSessionEvent =
 			errorMessage?: string;
 	  }
 	| { type: "auto_retry_start"; attempt: number; maxAttempts: number; delayMs: number; errorMessage: string }
-	| { type: "auto_retry_end"; success: boolean; attempt: number; finalError?: string };
+	| { type: "auto_retry_end"; success: boolean; attempt: number; finalError?: string }
+	| { type: "fallback_provider_switch"; from: string; to: string; reason: string }
+	| { type: "fallback_provider_restored"; provider: string; reason: string }
+	| { type: "fallback_chain_exhausted"; reason: string };
 
 /** Listener function for agent session events */
 export type AgentSessionEventListener = (event: AgentSessionEvent) => void;
@@ -267,6 +270,9 @@ export class AgentSession {
 	// Model registry for API key resolution
 	private _modelRegistry: ModelRegistry;
 
+	// Provider fallback resolver
+	private _fallbackResolver: FallbackResolver;
+
 	// Tool registry for extension getTools/setTools
 	private _toolRegistry: Map<string, AgentTool> = new Map();
 	private _toolPromptSnippets: Map<string, string> = new Map();
@@ -284,6 +290,11 @@ export class AgentSession {
 		this._customTools = config.customTools ?? [];
 		this._cwd = config.cwd;
 		this._modelRegistry = config.modelRegistry;
+		this._fallbackResolver = new FallbackResolver(
+			this.settingsManager,
+			this._modelRegistry.authStorage,
+			this._modelRegistry,
+		);
 		this._extensionRunnerRef = config.extensionRunnerRef;
 		this._initialActiveToolNames = config.initialActiveToolNames;
 		this._baseToolsOverride = config.baseToolsOverride;
@@ -292,6 +303,11 @@ export class AgentSession {
 		// (session persistence, extensions, auto-compaction, retry logic)
 		this._unsubscribeAgent = this.agent.subscribe(this._handleAgentEvent);
 
+		// Install tool hooks that await the event queue before emitting extension events.
+		// This ensures extensions always see settled state (e.g., assistant message appended)
+		// even when tools execute in parallel.
+		this._installAgentToolHooks();
+
 		this._buildRuntime({
 			activeToolNames: this._initialActiveToolNames,
 			includeAllExtensionTools: true,
@@ -303,6 +319,11 @@ export class AgentSession {
 		return this._modelRegistry;
 	}
 
+	/** Fallback resolver for cross-provider fallback */
+	get fallbackResolver(): FallbackResolver {
+		return this._fallbackResolver;
+	}
+
 	// =========================================================================
 	// Event Subscription
 	// =========================================================================
@@ -460,6 +481,73 @@ export class AgentSession {
 		}
 	}
 
+	/**
+	 * Install beforeToolCall/afterToolCall hooks on the Agent.
+	 *
+	 * These hooks await `_agentEventQueue` before emitting extension events,
+	 * ensuring that all prior events (including `message_end` which appends
+	 * the assistant message) have fully settled. This prevents a race condition
+	 * in parallel tool execution where extension `tool_call` handlers could
+	 * see stale agent state.
+	 */
+	private _installAgentToolHooks(): void {
+		this.agent.setBeforeToolCall(async ({ toolCall, args }) => {
+			// Wait for all queued agent events to settle before emitting to extensions
+			await this._agentEventQueue;
+
+			if (!this._extensionRunner?.hasHandlers("tool_call")) return undefined;
+
+			try {
+				const callResult = await this._extensionRunner.emitToolCall({
+					type: "tool_call",
+					toolName: toolCall.name,
+					toolCallId: toolCall.id,
+					input: args as Record<string, unknown>,
+				});
+
+				if (callResult?.block) {
+					return {
+						block: true,
+						reason: callResult.reason || "Tool execution was blocked by an extension",
+					};
+				}
+			} catch (err) {
+				if (err instanceof Error) {
+					return { block: true, reason: err.message };
+				}
+				return { block: true, reason: `Extension failed, blocking execution: ${String(err)}` };
+			}
+
+			return undefined;
+		});
+
+		this.agent.setAfterToolCall(async ({ toolCall, args, result, isError }) => {
+			// Wait for all queued agent events to settle
+			await this._agentEventQueue;
+
+			if (!this._extensionRunner?.hasHandlers("tool_result")) return undefined;
+
+			const resultResult = await this._extensionRunner.emitToolResult({
+				type: "tool_result",
+				toolName: toolCall.name,
+				toolCallId: toolCall.id,
+				input: args as Record<string, unknown>,
+				content: result.content,
+				details: result.details,
+				isError,
+			});
+
+			if (resultResult) {
+				return {
+					content: resultResult.content ?? undefined,
+					details: resultResult.details ?? undefined,
+				};
+			}
+
+			return undefined;
+		});
+	}
+
 	/** Extract text content from a message */
 	private _getUserMessageText(message: Message): string {
 		if (message.role !== "user") return "";
@@ -868,6 +956,19 @@ export class AgentSession {
 			);
 		}
 
+		// Check if a higher-priority provider in the fallback chain has recovered
+		const restoration = await this._fallbackResolver.checkForRestoration(this.model);
+		if (restoration) {
+			const previousProvider = `${this.model.provider}/${this.model.id}`;
+			this.agent.setModel(restoration.model);
+			this.sessionManager.appendModelChange(restoration.model.provider, restoration.model.id);
+			this._emit({
+				type: "fallback_provider_restored",
+				provider: `${restoration.model.provider}/${restoration.model.id}`,
+				reason: `Restored from ${previousProvider}`,
+			});
+		}
+
 		// Validate API key
 		const apiKey = await this._modelRegistry.getApiKey(this.model, this.sessionId);
 		if (!apiKey) {
@@ -2151,12 +2252,10 @@ export class AgentSession {
 			toolRegistry.set(tool.name, tool);
 		}
 
-		if (this._extensionRunner) {
-			const wrappedAllTools = wrapToolsWithExtensions(Array.from(toolRegistry.values()), this._extensionRunner);
-			this._toolRegistry = new Map(wrappedAllTools.map((tool) => [tool.name, tool]));
-		} else {
-			this._toolRegistry = toolRegistry;
-		}
+		// Tool interception (tool_call/tool_result extension events) is handled by
+		// beforeToolCall/afterToolCall hooks installed in _installAgentToolHooks(),
+		// which await _agentEventQueue for safe parallel execution.
+		this._toolRegistry = toolRegistry;
 
 		const nextActiveToolNames = options?.activeToolNames
 			? [...options.activeToolNames]
@@ -2354,20 +2453,66 @@ export class AgentSession {
 				return true;
 			}
 
-			// All credentials are backed off. For quota-exhausted errors the backoff is very
-			// long (30+ min), so retrying immediately is futile and will only produce
-			// confusing secondary errors (e.g. "Authentication failed"). Give up now and
-			// surface the original quota error to the user.
-			if (errorType === "quota_exhausted") {
-				this._emit({
-					type: "auto_retry_end",
-					success: false,
-					attempt: this._retryAttempt,
-					finalError: message.errorMessage,
-				});
-				this._retryAttempt = 0;
-				this._resolveRetry();
-				return false;
+			// All credentials are backed off. Try cross-provider fallback before giving up.
+			if (isCredentialError) {
+				const fallbackResult = await this._fallbackResolver.findFallback(
+					this.model,
+					errorType,
+				);
+
+				if (fallbackResult) {
+					// Swap to fallback model — don't persist to settings
+					const previousProvider = this.model.provider;
+					this.agent.setModel(fallbackResult.model);
+					this.sessionManager.appendModelChange(fallbackResult.model.provider, fallbackResult.model.id);
+
+					// Remove error message from agent state
+					const msgs = this.agent.state.messages;
+					if (msgs.length > 0 && msgs[msgs.length - 1].role === "assistant") {
+						this.agent.replaceMessages(msgs.slice(0, -1));
+					}
+
+					this._emit({
+						type: "fallback_provider_switch",
+						from: `${previousProvider}/${this.model?.id}`,
+						to: `${fallbackResult.model.provider}/${fallbackResult.model.id}`,
+						reason: fallbackResult.reason,
+					});
+
+					this._emit({
+						type: "auto_retry_start",
+						attempt: this._retryAttempt + 1,
+						maxAttempts: settings.maxRetries,
+						delayMs: 0,
+						errorMessage: `${message.errorMessage} (${fallbackResult.reason})`,
+					});
+
+					// Retry immediately with fallback provider - don't increment _retryAttempt
+					setTimeout(() => {
+						this.agent.continue().catch(() => {
+							// Retry failed - will be caught by next agent_end
+						});
+					}, 0);
+
+					return true;
+				}
+
+				// No fallback available either
+				if (errorType === "quota_exhausted") {
+					this._emit({
+						type: "fallback_chain_exhausted",
+						reason: `All providers exhausted for ${this.model.provider}/${this.model.id}`,
+					});
+					this._emit({
+						type: "auto_retry_end",
+						success: false,
+						attempt: this._retryAttempt,
+						finalError: message.errorMessage,
+					});
+					this._retryAttempt = 0;
+					this._resolveRetry();
+					return false;
+				}
 			}
 		}
 

package/packages/pi-coding-agent/src/core/auth-storage.ts

@@ -248,6 +248,13 @@ export class AuthStorage {
 	 */
 	private credentialBackoff: Map<string, Map<number, number>> = new Map();
 
+	/**
+	 * Provider-level backoff tracking.
+	 * Set when all credentials for a provider are backed off.
+	 * Map<provider, backoffExpiresAt>
+	 */
+	private providerBackoff: Map<string, number> = new Map();
+
 	private constructor(private storage: AuthStorageBackend) {
 		this.reload();
 	}
@@ -398,6 +405,7 @@ export class AuthStorage {
 		delete this.data[provider];
 		this.providerRoundRobinIndex.delete(provider);
 		this.credentialBackoff.delete(provider);
+		this.providerBackoff.delete(provider);
 		this.persistProviderChange(provider, undefined);
 	}
 
@@ -484,6 +492,43 @@ export class AuthStorage {
 		return true;
 	}
 
+	/**
+	 * Mark an entire provider as exhausted.
+	 * Called when all credentials for a provider are backed off.
+	 */
+	markProviderExhausted(provider: string, errorType: UsageLimitErrorType): void {
+		const backoffMs = getBackoffDuration(errorType);
+		this.providerBackoff.set(provider, Date.now() + backoffMs);
+	}
+
+	/**
+	 * Check if a provider is currently available (not backed off at provider level).
+	 */
+	isProviderAvailable(provider: string): boolean {
+		const expiresAt = this.providerBackoff.get(provider);
+		if (expiresAt === undefined) return true;
+		if (Date.now() >= expiresAt) {
+			this.providerBackoff.delete(provider);
+			return true;
+		}
+		return false;
+	}
+
+	/**
+	 * Get milliseconds remaining until provider backoff expires.
+	 * Returns 0 if provider is available.
+	 */
+	getProviderBackoffRemaining(provider: string): number {
+		const expiresAt = this.providerBackoff.get(provider);
+		if (expiresAt === undefined) return 0;
+		const remaining = expiresAt - Date.now();
+		if (remaining <= 0) {
+			this.providerBackoff.delete(provider);
+			return 0;
+		}
+		return remaining;
+	}
+
 	/**
 	 * Check if a credential index is currently backed off.
 	 */

package/packages/pi-coding-agent/src/core/fallback-resolver.test.ts

@@ -0,0 +1,229 @@
+// GSD Provider Fallback Resolver Tests
+// Copyright (c) 2026 Jeremy McSpadden <jeremy@fluxlabs.net>
+
+import { describe, it, beforeEach, mock } from "node:test";
+import assert from "node:assert/strict";
+import { FallbackResolver } from "./fallback-resolver.js";
+import type { Api, Model } from "@gsd/pi-ai";
+import type { AuthStorage } from "./auth-storage.js";
+import type { ModelRegistry } from "./model-registry.js";
+import type { FallbackChainEntry, SettingsManager } from "./settings-manager.js";
+
+function createMockModel(provider: string, id: string): Model<Api> {
+	return {
+		id,
+		name: id,
+		api: "openai-completions" as Api,
+		provider,
+		baseUrl: `https://api.${provider}.com`,
+		reasoning: false,
+		input: ["text"],
+		cost: { input: 0, output: 0, cacheRead: 0, cacheWrite: 0 },
+		contextWindow: 128000,
+		maxTokens: 16384,
+	} as Model<Api>;
+}
+
+const zaiModel = createMockModel("zai", "glm-5");
+const alibabaModel = createMockModel("alibaba", "glm-5");
+const openaiModel = createMockModel("openai", "gpt-4.1");
+
+const defaultChain: FallbackChainEntry[] = [
+	{ provider: "zai", model: "glm-5", priority: 1 },
+	{ provider: "alibaba", model: "glm-5", priority: 2 },
+	{ provider: "openai", model: "gpt-4.1", priority: 3 },
+];
+
+function createResolver(overrides?: {
+	enabled?: boolean;
+	isProviderAvailable?: (provider: string) => boolean;
+	hasAuth?: (provider: string) => boolean;
+	find?: (provider: string, modelId: string) => Model<Api> | undefined;
+}) {
+	const settingsManager = {
+		getFallbackSettings: () => ({
+			enabled: overrides?.enabled ?? true,
+			chains: { coding: defaultChain },
+		}),
+	} as unknown as SettingsManager;
+
+	const authStorage = {
+		markProviderExhausted: mock.fn(),
+		isProviderAvailable: overrides?.isProviderAvailable ?? (() => true),
+		hasAuth: overrides?.hasAuth ?? (() => true),
+	} as unknown as AuthStorage;
+
+	const modelRegistry = {
+		find: overrides?.find ?? ((provider: string, modelId: string) => {
+			if (provider === "zai" && modelId === "glm-5") return zaiModel;
+			if (provider === "alibaba" && modelId === "glm-5") return alibabaModel;
+			if (provider === "openai" && modelId === "gpt-4.1") return openaiModel;
+			return undefined;
+		}),
+	} as unknown as ModelRegistry;
+
+	return { resolver: new FallbackResolver(settingsManager, authStorage, modelRegistry), authStorage };
+}
+
+// ─── findFallback ────────────────────────────────────────────────────────────
+
+describe("FallbackResolver — findFallback", () => {
+	it("returns next available provider when current fails", async () => {
+		const { resolver } = createResolver();
+		const result = await resolver.findFallback(zaiModel, "quota_exhausted");
+
+		assert.notEqual(result, null);
+		assert.equal(result!.model.provider, "alibaba");
+		assert.equal(result!.model.id, "glm-5");
+		assert.equal(result!.chainName, "coding");
+	});
+
+	it("marks current provider as exhausted", async () => {
+		const { resolver, authStorage } = createResolver();
+		await resolver.findFallback(zaiModel, "rate_limit");
+
+		const fn = authStorage.markProviderExhausted as any;
+		assert.equal(fn.mock.calls.length, 1);
+		assert.equal(fn.mock.calls[0].arguments[0], "zai");
+		assert.equal(fn.mock.calls[0].arguments[1], "rate_limit");
+	});
+
+	it("skips backed-off providers", async () => {
+		const { resolver } = createResolver({
+			isProviderAvailable: (provider: string) => provider !== "alibaba",
+		});
+
+		const result = await resolver.findFallback(zaiModel, "quota_exhausted");
+
+		assert.notEqual(result, null);
+		assert.equal(result!.model.provider, "openai");
+		assert.equal(result!.model.id, "gpt-4.1");
+	});
+
+	it("returns null when all providers are backed off", async () => {
+		const { resolver } = createResolver({
+			isProviderAvailable: () => false,
+		});
+
+		const result = await resolver.findFallback(zaiModel, "quota_exhausted");
+		assert.equal(result, null);
+	});
+
+	it("returns null when fallback is disabled", async () => {
+		const { resolver } = createResolver({ enabled: false });
+		const result = await resolver.findFallback(zaiModel, "quota_exhausted");
+		assert.equal(result, null);
+	});
+
+	it("returns null when model is not in any chain", async () => {
+		const { resolver } = createResolver();
+		const unknownModel = createMockModel("unknown", "some-model");
+		const result = await resolver.findFallback(unknownModel, "quota_exhausted");
+		assert.equal(result, null);
+	});
+
+	it("skips providers without auth", async () => {
+		const { resolver } = createResolver({
+			hasAuth: (provider: string) => provider !== "alibaba",
+		});
+
+		const result = await resolver.findFallback(zaiModel, "quota_exhausted");
+
+		assert.notEqual(result, null);
+		assert.equal(result!.model.provider, "openai");
+	});
+
+	it("skips providers with no model in registry", async () => {
+		const { resolver } = createResolver({
+			find: (provider: string, modelId: string) => {
+				if (provider === "alibaba") return undefined;
+				if (provider === "openai" && modelId === "gpt-4.1") return openaiModel;
+				return undefined;
+			},
+		});
+
+		const result = await resolver.findFallback(zaiModel, "quota_exhausted");
+
+		assert.notEqual(result, null);
+		assert.equal(result!.model.provider, "openai");
+	});
+});
+
+// ─── checkForRestoration ─────────────────────────────────────────────────────
+
+describe("FallbackResolver — checkForRestoration", () => {
+	it("returns higher-priority provider when recovered", async () => {
+		const { resolver } = createResolver();
+		const result = await resolver.checkForRestoration(alibabaModel);
+
+		assert.notEqual(result, null);
+		assert.equal(result!.model.provider, "zai");
+		assert.equal(result!.model.id, "glm-5");
+	});
+
+	it("returns null when already at highest priority", async () => {
+		const { resolver } = createResolver();
+		const result = await resolver.checkForRestoration(zaiModel);
+		assert.equal(result, null);
+	});
+
+	it("returns null when higher-priority provider is still backed off", async () => {
+		const { resolver } = createResolver({
+			isProviderAvailable: (provider: string) => provider !== "zai",
+		});
+
+		const result = await resolver.checkForRestoration(alibabaModel);
+		assert.equal(result, null);
+	});
+
+	it("returns null when fallback is disabled", async () => {
+		const { resolver } = createResolver({ enabled: false });
+		const result = await resolver.checkForRestoration(alibabaModel);
+		assert.equal(result, null);
+	});
+});
+
+// ─── getBestAvailable ────────────────────────────────────────────────────────
+
+describe("FallbackResolver — getBestAvailable", () => {
+	it("returns highest-priority available provider", async () => {
+		const { resolver } = createResolver();
+		const result = await resolver.getBestAvailable("coding");
+
+		assert.notEqual(result, null);
+		assert.equal(result!.model.provider, "zai");
+	});
+
+	it("skips backed-off providers", async () => {
+		const { resolver } = createResolver({
+			isProviderAvailable: (provider: string) => provider !== "zai",
+		});
+
+		const result = await resolver.getBestAvailable("coding");
+
+		assert.notEqual(result, null);
+		assert.equal(result!.model.provider, "alibaba");
+	});
+
+	it("returns null for unknown chain", async () => {
+		const { resolver } = createResolver();
+		const result = await resolver.getBestAvailable("nonexistent");
+		assert.equal(result, null);
+	});
+});
+
+// ─── findChainsForModel ──────────────────────────────────────────────────────
+
+describe("FallbackResolver — findChainsForModel", () => {
+	it("finds chains containing a model", () => {
+		const { resolver } = createResolver();
+		const chains = resolver.findChainsForModel("zai", "glm-5");
+		assert.deepEqual(chains, ["coding"]);
+	});
+
+	it("returns empty array for model not in any chain", () => {
+		const { resolver } = createResolver();
+		const chains = resolver.findChainsForModel("unknown", "model");
+		assert.deepEqual(chains, []);
+	});
+});