@oh-my-pi/pi-coding-agent 15.10.10 → 15.10.12

package/src/web/scrapers/types.ts

@@ -1,6 +1,7 @@
 /**
  * Shared types and utilities for web-fetch handlers
  */
+import { scheduler } from "node:timers/promises";
 import { ptree } from "@oh-my-pi/pi-utils";
 import type TurndownService from "turndown";
 
@@ -70,6 +71,12 @@ export interface LoadPageOptions {
 	body?: string;
 	maxBytes?: number;
 	signal?: AbortSignal;
+	/**
+	 * Return true to skip reading the response body for this content type
+	 * (lowercased mime, no params). The caller is expected to re-fetch the
+	 * payload as binary; this avoids streaming + decoding huge binaries twice.
+	 */
+	skipBodyForContentType?: (contentType: string) => boolean;
 }
 
 export interface LoadPageResult {
@@ -78,6 +85,51 @@ export interface LoadPageResult {
 	finalUrl: string;
 	ok: boolean;
 	status?: number;
+	/** True when the body was cut mid-stream at maxBytes. */
+	truncated?: boolean;
+	/** Last transport-level error message when ok is false. */
+	error?: string;
+	/** True when the body read was skipped via skipBodyForContentType. */
+	bodySkipped?: boolean;
+}
+
+const RETRY_AFTER_MAX_MS = 10_000;
+
+/** Parse a Retry-After header (seconds or HTTP-date) into a bounded delay. */
+function parseRetryAfterMs(value: string | null): number {
+	if (!value) return 1_000;
+	const seconds = Number(value);
+	if (Number.isFinite(seconds)) return Math.min(Math.max(seconds, 0) * 1000, RETRY_AFTER_MAX_MS);
+	const date = Date.parse(value);
+	if (!Number.isNaN(date)) return Math.min(Math.max(date - Date.now(), 0), RETRY_AFTER_MAX_MS);
+	return 1_000;
+}
+
+function charsetFromContentType(header: string): string | undefined {
+	return /charset\s*=\s*"?([\w-]+)"?/i.exec(header)?.[1];
+}
+
+/**
+ * Decode a response body honoring the declared charset (Content-Type header,
+ * then a cheap <meta charset> sniff), falling back to UTF-8.
+ */
+function decodeBody(bytes: Buffer, contentTypeHeader: string): string {
+	let label = charsetFromContentType(contentTypeHeader);
+	if (!label) {
+		// All charsets we can decode are ASCII-compatible in the prefix, so a
+		// latin1 view of the first 2KB is enough to find a <meta charset>.
+		label = /<meta[^>]+charset\s*=\s*["']?([\w-]+)/i.exec(bytes.subarray(0, 2048).toString("latin1"))?.[1];
+	}
+	if (label && !/^utf-?8$/i.test(label)) {
+		try {
+			// Bun.Encoding's union is narrower than the runtime, which accepts
+			// WHATWG labels (shift_jis, euc-kr, gbk, big5, …); unknowns throw here.
+			return new TextDecoder(label as Bun.Encoding).decode(bytes);
+		} catch {
+			// Unknown/unsupported label — fall back to UTF-8.
+		}
+	}
+	return bytes.toString("utf-8");
 }
 
 /**
@@ -86,6 +138,8 @@ export interface LoadPageResult {
 export async function loadPage(url: string, options: LoadPageOptions = {}): Promise<LoadPageResult> {
 	const { timeout = 20, headers = {}, maxBytes = MAX_BYTES, signal, method = "GET", body } = options;
 
+	let lastError: string | undefined;
+	let retried429 = false;
 	for (let attempt = 0; attempt < USER_AGENTS.length; attempt++) {
 		if (signal?.aborted) {
 			throw new ToolAbortError();
@@ -114,9 +168,31 @@ export async function loadPage(url: string, options: LoadPageOptions = {}): Prom
 
 			const response = await fetch(url, requestInit);
 
-			const contentType = response.headers.get("content-type")?.split(";")[0]?.trim().toLowerCase() ?? "";
+			const rawContentType = response.headers.get("content-type") ?? "";
+			const contentType = rawContentType.split(";")[0]?.trim().toLowerCase() ?? "";
 			const finalUrl = response.url;
 
+			if (response.status === 429 && !retried429) {
+				// Rate limited: retry once, honoring a bounded Retry-After. The
+				// wait observes the caller's signal so an Esc during the backoff
+				// does not stall for up to the full delay.
+				retried429 = true;
+				const delayMs = parseRetryAfterMs(response.headers.get("retry-after"));
+				void response.body?.cancel().catch(() => {});
+				try {
+					await scheduler.wait(delayMs, { signal });
+				} catch {
+					throw new ToolAbortError();
+				}
+				attempt--; // Reuse the same user agent for the retry.
+				continue;
+			}
+
+			if (response.ok && options.skipBodyForContentType?.(contentType)) {
+				void response.body?.cancel().catch(() => {});
+				return { content: "", contentType, finalUrl, ok: true, status: response.status, bodySkipped: true };
+			}
+
 			const reader = response.body?.getReader();
 			if (!reader) {
 				return { content: "", contentType, finalUrl, ok: false, status: response.status };
@@ -124,6 +200,7 @@ export async function loadPage(url: string, options: LoadPageOptions = {}): Prom
 
 			const chunks: Uint8Array[] = [];
 			let totalSize = 0;
+			let truncated = false;
 
 			while (true) {
 				const { done, value } = await reader.read();
@@ -133,32 +210,34 @@ export async function loadPage(url: string, options: LoadPageOptions = {}): Prom
 				totalSize += value.length;
 
 				if (totalSize > maxBytes) {
-					reader.cancel();
+					truncated = true;
+					void reader.cancel().catch(() => {});
 					break;
 				}
 			}
 
-			const content = Buffer.concat(chunks).toString("utf-8");
+			const content = decodeBody(Buffer.concat(chunks), rawContentType);
 			if (isBotBlocked(response.status, content) && attempt < USER_AGENTS.length - 1) {
 				continue;
 			}
 
 			if (!response.ok) {
-				return { content, contentType, finalUrl, ok: false, status: response.status };
+				return { content, contentType, finalUrl, ok: false, status: response.status, truncated };
 			}
 
-			return { content, contentType, finalUrl, ok: true, status: response.status };
-		} catch {
+			return { content, contentType, finalUrl, ok: true, status: response.status, truncated };
+		} catch (error) {
 			if (signal?.aborted) {
 				throw new ToolAbortError();
 			}
+			lastError = error instanceof Error ? error.message : String(error);
 			if (attempt === USER_AGENTS.length - 1) {
-				return { content: "", contentType: "", finalUrl: url, ok: false };
+				return { content: "", contentType: "", finalUrl: url, ok: false, error: lastError };
 			}
 		}
 	}
 
-	return { content: "", contentType: "", finalUrl: url, ok: false };
+	return { content: "", contentType: "", finalUrl: url, ok: false, error: lastError };
 }
 
 /** Module-level Turndown instance — built lazily on first use. */

package/src/web/scrapers/wikipedia.ts

@@ -1,4 +1,3 @@
-import { parseHTML } from "linkedom";
 import type { RenderResult, SpecialHandler } from "./types";
 import { buildResult, loadPage } from "./types";
 
@@ -45,6 +44,7 @@ export const handleWikipedia: SpecialHandler = async (
 		const contentResult = await loadPage(contentUrl, { timeout, signal });
 
 		if (contentResult.ok) {
+			const { parseHTML } = await import("linkedom");
 			const doc = parseHTML(contentResult.content).document;
 
 			// Extract main content sections

package/src/web/scrapers/youtube.ts

@@ -288,12 +288,17 @@ export const handleYouTube: SpecialHandler = async (
 			}
 		}
 	} finally {
-		throwIfAborted(signal);
 		// Cleanup temp files (fire-and-forget with error suppression)
 		Array.fromAsync(new Bun.Glob(`${tmpBase}*`).scan({ absolute: true }))
 			.then(tmpFiles => Promise.all(tmpFiles.map(f => fs.unlink(f).catch(() => {}))))
 			.catch(() => {});
 	}
+	// Only a user-initiated abort is fatal; the per-fetch time budget expiring
+	// just means partial metadata/transcript, which we surface as a note.
+	throwIfAborted(userSignal);
+	if (signal?.aborted) {
+		notes.push("Fetch time budget exhausted; metadata/transcript may be incomplete");
+	}
 
 	// Build markdown output
 	let md = `# ${title}\n\n`;

package/src/web/search/index.ts

@@ -150,7 +150,7 @@ async function executeSearch(
 		lastProvider = provider;
 		try {
 			const response = await provider.search({
-				query: params.query.replace(/202\d/g, String(new Date().getFullYear())), // LUL
+				query: params.query,
 				limit: params.limit,
 				recency: params.recency,
 				systemPrompt: webSearchSystemPrompt,

package/src/web/search/providers/codex.ts

@@ -7,8 +7,9 @@
  * SQLite store, never POSTs the broker sentinel to an OpenAI token endpoint.
  */
 import * as os from "node:os";
-import { type AuthStorage, type FetchImpl, getBundledModels } from "@oh-my-pi/pi-ai";
+import type { AuthStorage, FetchImpl } from "@oh-my-pi/pi-ai";
 import { decodeJwt } from "@oh-my-pi/pi-ai/oauth/openai-codex";
+import { getBundledModels } from "@oh-my-pi/pi-catalog/models";
 import { $env, readSseJson } from "@oh-my-pi/pi-utils";
 import packageJson from "../../../../package.json" with { type: "json" };
 import type { SearchResponse, SearchSource } from "../../../web/search/types";

package/src/web/search/providers/gemini.ts

@@ -8,13 +8,12 @@
  * sibling SQLite store and never POSTs the broker sentinel to a Google token
  * endpoint.
  */
+import type { AuthStorage, FetchImpl } from "@oh-my-pi/pi-ai";
 import {
 	ANTIGRAVITY_SYSTEM_INSTRUCTION,
-	type AuthStorage,
-	type FetchImpl,
 	getAntigravityUserAgent,
 	getGeminiCliHeaders,
-} from "@oh-my-pi/pi-ai";
+} from "@oh-my-pi/pi-catalog/wire/gemini-headers";
 import { fetchWithRetry } from "@oh-my-pi/pi-utils";
 
 import type { SearchCitation, SearchResponse, SearchSource } from "../../../web/search/types";

package/src/web/search/render.ts

@@ -65,7 +65,7 @@ function renderSearchErrorPanel(message: string, providerLabel: string | undefin
 	const body = theme.fg("error", `Error: ${replaceTabs(message)}`);
 	const outputBlock = new CachedOutputBlock();
 	return markFramedBlockComponent({
-		render(width: number): string[] {
+		render(width: number): readonly string[] {
 			return outputBlock.render({ header, state: "error", sections: [{ lines: [body] }], width }, theme);
 		},
 		invalidate() {
@@ -154,23 +154,25 @@ export function renderSearchResult(
 	const outputBlock = new CachedOutputBlock();
 
 	return markFramedBlockComponent({
-		render(width: number): string[] {
+		render(width: number): readonly string[] {
 			// Read mutable state at render time
 			const { expanded } = options;
 
 			// Answer lines: full markdown when expanded, capped markdown preview when collapsed.
 			const answerWidth = Math.max(20, width - 3);
 			const renderedAnswer = answerMarkdown ? answerMarkdown.render(answerWidth) : [];
-			let answerLines: string[];
+			let answerLines: readonly string[];
 			if (renderedAnswer.length === 0) {
 				answerLines = [theme.fg("muted", "No answer text returned")];
 			} else if (args?.maxAnswerLines !== undefined && !expanded) {
 				// CLI compact mode (`omp q`) caps the answer; the TUI passes no cap and shows it in full.
-				answerLines = renderedAnswer.slice(0, args.maxAnswerLines);
-				const remaining = renderedAnswer.length - answerLines.length;
+				// `renderedAnswer` is the Markdown component's shared cache — slice copies before appending.
+				const capped = renderedAnswer.slice(0, args.maxAnswerLines);
+				const remaining = renderedAnswer.length - capped.length;
 				if (remaining > 0) {
-					answerLines.push(theme.fg("muted", formatMoreItems(remaining, "line")));
+					capped.push(theme.fg("muted", formatMoreItems(remaining, "line")));
 				}
+				answerLines = capped;
 			} else {
 				answerLines = renderedAnswer;
 			}

package/dist/types/config/model-equivalence.d.ts

@@ -1,24 +0,0 @@
-import { type Api, type Model } from "@oh-my-pi/pi-ai";
-export type CanonicalModelSource = "override" | "bundled" | "heuristic" | "fallback";
-export interface ModelEquivalenceConfig {
-    overrides?: Record<string, string>;
-    exclude?: string[];
-}
-export interface CanonicalModelVariant {
-    canonicalId: string;
-    selector: string;
-    model: Model<Api>;
-    source: CanonicalModelSource;
-}
-export interface CanonicalModelRecord {
-    id: string;
-    name: string;
-    variants: CanonicalModelVariant[];
-}
-export interface CanonicalModelIndex {
-    records: CanonicalModelRecord[];
-    byId: Map<string, CanonicalModelRecord>;
-    bySelector: Map<string, string>;
-}
-export declare function formatCanonicalVariantSelector(model: Model<Api>): string;
-export declare function buildCanonicalModelIndex(models: readonly Model<Api>[], equivalence?: ModelEquivalenceConfig): CanonicalModelIndex;

package/dist/types/config/model-id-affixes.d.ts

@@ -1,12 +0,0 @@
-export declare function getModelLikeIdSegments(modelId: string): string[];
-export declare function getLongestModelLikeIdSegment(modelId: string): string | undefined;
-/**
- * Strip reseller / wrapper tags that are injected as bracketed affixes around an
- * upstream model id, e.g.
- *   "[Kiro] claude-opus-4-8"                -> "claude-opus-4-8"
- *   "[gcli转] gemini-3.1-pro-preview [假流]" -> "gemini-3.1-pro-preview"
- *
- * Candidates are returned most-stripped first: both ends, then leading-only, then trailing-only.
- */
-export declare function getBracketStrippedModelIdCandidates(modelId: string): string[];
-export declare function stripBracketedModelIdAffixes(modelId: string): string | undefined;

package/dist/types/config/model-provider-priority.d.ts

@@ -1 +0,0 @@
-export declare function buildModelProviderPriorityRank(configuredProviderOrder?: readonly string[]): Map<string, number>;

package/dist/types/exec/idle-timeout-watchdog.d.ts

@@ -1,18 +0,0 @@
-export type ExecutionAbortReason = "idle-timeout" | "signal";
-export interface IdleTimeoutWatchdogOptions {
-    timeoutMs?: number;
-    signal?: AbortSignal;
-    hardTimeoutGraceMs: number;
-    onAbort?: (reason: ExecutionAbortReason) => void;
-}
-export declare class IdleTimeoutWatchdog {
-    #private;
-    constructor(options: IdleTimeoutWatchdogOptions);
-    get abortedBySignal(): boolean;
-    get hardTimeoutPromise(): Promise<"hard-timeout">;
-    get signal(): AbortSignal;
-    get timedOut(): boolean;
-    touch(): void;
-    dispose(): void;
-}
-export declare function formatIdleTimeoutMessage(timeoutMs?: number): string;