@oh-my-pi/pi-coding-agent 16.0.3 → 16.0.5

package/src/utils/image-vision-fallback.ts

@@ -0,0 +1,197 @@
+/**
+ * Vision fallback for text-only models. When a user attaches an image to a model
+ * that cannot accept image input, this:
+ *  1. saves each image under the session `local://` root (for later analysis), and
+ *  2. asks a vision-capable model to describe it and injects that description as
+ *     a text block in place of the image:
+ *
+ *     <image path="local://image-<hash>.png">
+ *     <description>
+ *     </image>
+ *
+ * Without this the provider layer drops the image entirely (NON_VISION_IMAGE_PLACEHOLDER).
+ */
+import * as path from "node:path";
+import {
+	type AgentTelemetry,
+	type AgentTelemetryConfig,
+	instrumentedCompleteSimple,
+	resolveTelemetry,
+} from "@oh-my-pi/pi-agent-core";
+import type { Api, completeSimple, ImageContent, Model, TextContent } from "@oh-my-pi/pi-ai";
+import { logger, prompt, toError } from "@oh-my-pi/pi-utils";
+import { extractTextContent } from "../commit/utils";
+import type { ModelRegistry } from "../config/model-registry";
+import { expandRoleAlias, getModelMatchPreferences, resolveModelFromString } from "../config/model-resolver";
+import type { Settings } from "../config/settings";
+import { type LocalProtocolOptions, resolveLocalRoot } from "../internal-urls";
+import describeUserPrompt from "../prompts/tools/image-attachment-describe.md" with { type: "text" };
+import describeSystemPrompt from "../prompts/tools/image-attachment-describe-system.md" with { type: "text" };
+
+/** Telemetry tag for the oneshot vision-description calls. */
+const ONESHOT_KIND = "image_attachment_describe";
+
+const NO_VISION_MODEL_NOTE =
+	"[No vision-capable model is configured, so this image could not be described automatically. " +
+	"The image was saved; configure a vision model role (modelRoles.vision) and use the inspect_image tool to analyze it.]";
+
+const DESCRIPTION_UNAVAILABLE_NOTE =
+	"[Image description unavailable: the vision model returned no usable text. The image was saved for further analysis.]";
+
+/** Registry surface needed to resolve a vision model and authorize requests. */
+export type VisionFallbackRegistry = Pick<ModelRegistry, "getAvailable" | "getApiKey" | "resolver"> &
+	Partial<Pick<ModelRegistry, "resolveCanonicalModel" | "getCanonicalVariants" | "getCanonicalId">>;
+
+export interface DescribeAttachedImagesDeps {
+	/** Active (text-only) model the prompt is destined for. */
+	activeModel: Model<Api>;
+	modelRegistry: VisionFallbackRegistry;
+	settings: Settings;
+	/** Inputs for resolving the session-scoped `local://` root. */
+	localProtocolOptions: LocalProtocolOptions;
+	/** `provider/id` of the active model; a last-resort vision-model candidate (filtered to image-capable). */
+	activeModelString?: string;
+	telemetryConfig?: AgentTelemetryConfig;
+	sessionId?: string;
+	/** Test seam: overrides the underlying completeSimple call. */
+	completeImpl?: typeof completeSimple;
+}
+
+/** Map an image MIME type to a file extension for the saved artifact. */
+function extensionForMime(mimeType: string): string {
+	const subtype = mimeType.split("/")[1]?.toLowerCase() ?? "";
+	switch (subtype) {
+		case "jpeg":
+		case "jpg":
+			return "jpg";
+		case "png":
+			return "png";
+		case "gif":
+			return "gif";
+		case "webp":
+			return "webp";
+		default: {
+			const sanitized = subtype.replace(/[^a-z0-9]/g, "");
+			return sanitized || "png";
+		}
+	}
+}
+
+/** Content-addressed file name so re-pasting the same image reuses one artifact. */
+function imageFileName(image: ImageContent): string {
+	const hash = Bun.hash(image.data).toString(16);
+	return `image-${hash}.${extensionForMime(image.mimeType)}`;
+}
+
+/** Persist an image under the local root; returns its `local://` URL. */
+async function saveImage(image: ImageContent, localRoot: string): Promise<string> {
+	const fileName = imageFileName(image);
+	const filePath = path.join(localRoot, fileName);
+	// Content-addressed: identical bytes overwrite themselves harmlessly. Bun.write creates parent dirs.
+	await Bun.write(filePath, Buffer.from(image.data, "base64"));
+	return `local://${fileName}`;
+}
+
+function formatImageBlock(localUrl: string, description: string): string {
+	return `<image path="${localUrl}">\n${description}\n</image>`;
+}
+
+/**
+ * Resolve a vision-capable model, mirroring the inspect_image priority
+ * (`pi/vision` → `pi/default` → active → first image-capable available), but
+ * never returning a text-only model.
+ */
+function resolveVisionModel(deps: DescribeAttachedImagesDeps): Model<Api> | undefined {
+	const available = deps.modelRegistry.getAvailable();
+	if (available.length === 0) return undefined;
+	const preferences = getModelMatchPreferences(deps.settings);
+	const resolvePattern = (pattern: string | undefined): Model<Api> | undefined => {
+		if (!pattern) return undefined;
+		const expanded = expandRoleAlias(pattern, deps.settings);
+		const model = resolveModelFromString(expanded, available, preferences, deps.modelRegistry);
+		return model?.input.includes("image") ? model : undefined;
+	};
+	return (
+		resolvePattern("pi/vision") ??
+		resolvePattern("pi/default") ??
+		resolvePattern(deps.activeModelString) ??
+		available.find(model => model.input.includes("image"))
+	);
+}
+
+/** Run one vision-description round-trip; returns trimmed text or `null` on any failure. */
+async function describeImage(
+	image: ImageContent,
+	visionModel: Model<Api>,
+	deps: DescribeAttachedImagesDeps,
+	telemetry: AgentTelemetry | undefined,
+	signal: AbortSignal | undefined,
+): Promise<string | null> {
+	try {
+		const response = await instrumentedCompleteSimple(
+			visionModel,
+			{
+				systemPrompt: [prompt.render(describeSystemPrompt)],
+				messages: [
+					{
+						role: "user",
+						content: [
+							{ type: "image", data: image.data, mimeType: image.mimeType },
+							{ type: "text", text: prompt.render(describeUserPrompt) },
+						],
+						timestamp: Date.now(),
+					},
+				],
+			},
+			{ apiKey: deps.modelRegistry.resolver(visionModel, deps.sessionId), signal },
+			{ telemetry, oneshotKind: ONESHOT_KIND, completeImpl: deps.completeImpl },
+		);
+		if (response.stopReason === "error" || response.stopReason === "aborted") {
+			logger.warn("image attachment description did not complete", {
+				stopReason: response.stopReason,
+				model: `${visionModel.provider}/${visionModel.id}`,
+			});
+			return null;
+		}
+		const text = extractTextContent(response).trim();
+		return text.length > 0 ? text : null;
+	} catch (err) {
+		logger.warn("image attachment description failed", {
+			error: toError(err).message,
+			model: `${visionModel.provider}/${visionModel.id}`,
+		});
+		return null;
+	}
+}
+
+/**
+ * Save each attached image under `local://` and replace it with a descriptive
+ * text block. Returns one {@link TextContent} per input image, in order. Never
+ * throws for an individual image: a failed description falls back to a note while
+ * the saved-path block is still emitted.
+ */
+export async function describeAttachedImagesForTextModel(
+	images: readonly ImageContent[],
+	deps: DescribeAttachedImagesDeps,
+	signal?: AbortSignal,
+): Promise<TextContent[]> {
+	const localRoot = resolveLocalRoot(deps.localProtocolOptions);
+	const visionModel = resolveVisionModel(deps);
+	const apiKey = visionModel ? await deps.modelRegistry.getApiKey(visionModel, deps.sessionId) : undefined;
+	const canDescribe = Boolean(visionModel && apiKey);
+	const telemetry = resolveTelemetry(deps.telemetryConfig, deps.sessionId);
+
+	return Promise.all(
+		images.map(async (image): Promise<TextContent> => {
+			const localUrl = await saveImage(image, localRoot);
+			let description: string;
+			if (canDescribe && visionModel) {
+				description =
+					(await describeImage(image, visionModel, deps, telemetry, signal)) ?? DESCRIPTION_UNAVAILABLE_NOTE;
+			} else {
+				description = NO_VISION_MODEL_NOTE;
+			}
+			return { type: "text", text: formatImageBlock(localUrl, description) };
+		}),
+	);
+}

package/src/web/search/index.ts

@@ -8,6 +8,7 @@ import type { AgentTool, AgentToolContext, AgentToolResult, AgentToolUpdateCallb
 import type { AuthStorage } from "@oh-my-pi/pi-ai";
 import { prompt } from "@oh-my-pi/pi-utils";
 import { z } from "zod/v4";
+import { settings } from "../../config/settings";
 import type { CustomTool, CustomToolContext, RenderResultOptions } from "../../extensibility/custom-tools/types";
 import type { Theme } from "../../modes/theme/theme";
 import webSearchSystemPrompt from "../../prompts/system/web-search.md" with { type: "text" };
@@ -153,6 +154,16 @@ async function executeSearch(
 		};
 	}
 
+	// Invariant across providers; read once and tolerate an uninitialized
+	// Settings singleton (e.g. `omp q ...` CLI path, unit tests) so the
+	// provider-fallback loop never aborts before any provider runs.
+	let antigravityEndpointMode: "auto" | "production" | "sandbox" | undefined;
+	try {
+		antigravityEndpointMode = settings.get("providers.antigravityEndpoint");
+	} catch {
+		antigravityEndpointMode = undefined;
+	}
+
 	const failures: Array<{ provider: SearchProvider; error: unknown }> = [];
 	let lastProvider = providers[0];
 	for (const provider of providers) {
@@ -169,6 +180,7 @@ async function executeSearch(
 				signal,
 				authStorage,
 				sessionId,
+				antigravityEndpointMode,
 			});
 
 			if (!hasRenderableSearchContent(response)) {

package/src/web/search/providers/base.ts

@@ -51,6 +51,7 @@ export interface SearchParams {
 	 * caller's agent session when available; otherwise omit.
 	 */
 	sessionId?: string;
+	antigravityEndpointMode?: "auto" | "production" | "sandbox";
 }
 
 /** Base class for web search providers. */

package/src/web/search/providers/gemini.ts

@@ -52,6 +52,7 @@ export interface GeminiSearchParams extends GeminiToolParams {
 	authStorage: AuthStorage;
 	sessionId?: string;
 	fetch?: FetchImpl;
+	antigravityEndpointMode?: "auto" | "production" | "sandbox";
 }
 
 export function buildGeminiRequestTools(params: GeminiToolParams): Array<Record<string, Record<string, unknown>>> {
@@ -163,6 +164,7 @@ async function callGeminiSearch(
 	toolParams: GeminiToolParams,
 	fetchImpl: FetchImpl | undefined,
 	signal: AbortSignal | undefined,
+	mode?: "auto" | "production" | "sandbox",
 ): Promise<{
 	answer: string;
 	sources: SearchSource[];
@@ -171,7 +173,19 @@ async function callGeminiSearch(
 	model: string;
 	usage?: { inputTokens: number; outputTokens: number; totalTokens: number };
 }> {
-	const endpoints = auth.isAntigravity ? ANTIGRAVITY_ENDPOINT_FALLBACKS : [DEFAULT_ENDPOINT];
+	let endpoints: string[];
+	if (auth.isAntigravity) {
+		const m = mode ?? "auto";
+		if (m === "sandbox") {
+			endpoints = [ANTIGRAVITY_SANDBOX_ENDPOINT];
+		} else if (m === "production") {
+			endpoints = [ANTIGRAVITY_DAILY_ENDPOINT];
+		} else {
+			endpoints = [...ANTIGRAVITY_ENDPOINT_FALLBACKS];
+		}
+	} else {
+		endpoints = [DEFAULT_ENDPOINT];
+	}
 	const headers = auth.isAntigravity ? { "User-Agent": getAntigravityUserAgent() } : getGeminiCliHeaders();
 
 	const requestMetadata = auth.isAntigravity
@@ -187,12 +201,7 @@ async function callGeminiSearch(
 
 	const normalizedSystemPrompt = systemPrompt?.toWellFormed();
 	const systemInstructionParts: Array<{ text: string }> = [
-		...(auth.isAntigravity
-			? [
-					{ text: ANTIGRAVITY_SYSTEM_INSTRUCTION },
-					{ text: `Please ignore following [ignore]${ANTIGRAVITY_SYSTEM_INSTRUCTION}[/ignore]` },
-				]
-			: []),
+		...(auth.isAntigravity ? [{ text: ANTIGRAVITY_SYSTEM_INSTRUCTION }] : []),
 		...(normalizedSystemPrompt ? [{ text: normalizedSystemPrompt }] : []),
 	];
 
@@ -238,16 +247,45 @@ async function callGeminiSearch(
 		body: JSON.stringify(requestBody),
 		signal: withHardTimeout(signal),
 	});
-	const urlFor = (attempt: number) =>
-		`${endpoints[Math.min(attempt, endpoints.length - 1)]}/v1internal:streamGenerateContent?alt=sse`;
-
-	const response = await fetchWithRetry(urlFor, {
-		...buildInit(),
-		fetch: fetchImpl,
-		maxAttempts: MAX_RETRIES + 1,
-		defaultDelayMs: attempt => BASE_DELAY_MS * 2 ** attempt,
-		maxDelayMs: RATE_LIMIT_BUDGET_MS,
-	});
+
+	let response: Response | undefined;
+
+	for (let i = 0; i < endpoints.length; i++) {
+		const endpoint = endpoints[i];
+		const isLastEndpoint = i === endpoints.length - 1;
+		try {
+			response = await fetchWithRetry(() => `${endpoint}/v1internal:streamGenerateContent?alt=sse`, {
+				...buildInit(),
+				fetch: fetchImpl,
+				maxAttempts: isLastEndpoint ? MAX_RETRIES + 1 : 1,
+				defaultDelayMs: attempt => BASE_DELAY_MS * 2 ** attempt,
+				maxDelayMs: RATE_LIMIT_BUDGET_MS,
+			});
+
+			if (response.ok) {
+				break;
+			}
+
+			if (response.status === 429 || (response.status >= 500 && response.status < 600)) {
+				if (!isLastEndpoint) {
+					continue;
+				}
+			}
+			break;
+		} catch (error) {
+			if (isLastEndpoint) {
+				throw error;
+			}
+		}
+	}
+
+	if (!response?.ok) {
+		const errorText = response ? await response.text() : "Network error";
+		const status = response?.status ?? 502;
+		const classified = classifyProviderHttpError("gemini", status, errorText);
+		if (classified) throw classified;
+		throw new SearchProviderError("gemini", `Gemini Cloud Code API error (${status}): ${errorText}`, status);
+	}
 
 	if (!response.ok) {
 		const errorText = await response.text();
@@ -410,7 +448,6 @@ export async function searchGemini(params: GeminiSearchParams): Promise<SearchRe
 			// re-resolved access may omit projectId, in which case the seed's
 			// project is still the right tenant for the credential. The
 			// `fetchWithRetry` transport backoff stays INSIDE this attempt — auth
-			// retry wraps transport retry.
 			callGeminiSearch(
 				{
 					accessToken: access.accessToken,
@@ -428,6 +465,7 @@ export async function searchGemini(params: GeminiSearchParams): Promise<SearchRe
 				},
 				params.fetch,
 				params.signal,
+				params.antigravityEndpointMode,
 			),
 		{ sessionId: params.sessionId, signal: params.signal, seed: seed.access },
 	);