@oh-my-pi/pi-coding-agent 16.2.5 → 16.2.7

package/src/tools/read.ts

@@ -14,7 +14,15 @@ import type { ImageContent, TextContent } from "@oh-my-pi/pi-ai";
 import { glob, type SummaryResult, summarizeCode } from "@oh-my-pi/pi-natives";
 import type { Component } from "@oh-my-pi/pi-tui";
 import { Text } from "@oh-my-pi/pi-tui";
-import { getRemoteDir, type ImageMetadata, logger, prompt, readImageMetadata, untilAborted } from "@oh-my-pi/pi-utils";
+import {
+	getRemoteDir,
+	type ImageMetadata,
+	isProbablyBinary,
+	logger,
+	prompt,
+	readImageMetadata,
+	untilAborted,
+} from "@oh-my-pi/pi-utils";
 import { type } from "arktype";
 import { LRUCache } from "lru-cache/raw";
 import {
@@ -2314,6 +2322,25 @@ export class ReadTool implements AgentTool<typeof readSchema, ReadToolDetails> {
 				content = [{ type: "text", text: `[Cannot read ${ext} file: conversion failed]` }];
 			}
 		} else {
+			// Binary sniff before any UTF-8 text materialization. A binary file
+			// (font, object, archive, packed blob) decodes to NUL/control bytes and
+			// U+FFFD mojibake that corrupts the terminal and burns context. Images,
+			// notebooks, and markit-convertible documents were already routed above;
+			// everything reaching here is meant to be plain text. `:raw` stays the
+			// explicit escape hatch for reading bytes verbatim. This single guard
+			// covers both the multi-range and single-range disk paths below.
+			if (!isRawSelector(parsed) && (await isProbablyBinary(absolutePath))) {
+				return toolResult<ReadToolDetails>({ resolvedPath: absolutePath, suffixResolution })
+					.text(
+						prependSuffixResolutionNotice(
+							`[Cannot read binary file '${formatPathRelativeToCwd(absolutePath, this.session.cwd)}' (${formatBytes(fileSize)}); not valid UTF-8 text. Use ':raw' to read bytes verbatim.]`,
+							suffixResolution,
+						),
+					)
+					.sourcePath(absolutePath)
+					.done();
+			}
+
 			if (
 				parsed.kind === "none" &&
 				this.session.settings.get("read.summarize.enabled") &&
@@ -2449,33 +2476,6 @@ export class ReadTool implements AgentTool<typeof readSchema, ReadToolDetails> {
 					// counts in `truncation` keep reflecting the source, not the trimmed
 					// view — column truncation surfaces separately via `.limits()`.
 					const rawSelector = isRawSelector(parsed);
-					// Binary sniff: NUL bytes mean the file is not displayable text
-					// (binary, or UTF-16 which has NULs in the ASCII range) — emit a
-					// notice instead of mojibake filling the line budget. `:raw`
-					// stays an explicit escape hatch.
-					//
-					// `collectedLines` covers the common case where at least one
-					// physical line terminates within the byte budget. Binary blobs
-					// without newlines (videos, archives, packed JSON) leave it
-					// empty; their bytes only land in `firstLinePreview`, which the
-					// `firstLineExceedsLimit` branch below would otherwise emit
-					// verbatim. Sniffing the preview here keeps the refusal uniform.
-					if (!rawSelector) {
-						const hasNul = (text: string): boolean => text.includes("\u0000");
-						const binaryDetected =
-							collectedLines.some(hasNul) || (firstLinePreview !== undefined && hasNul(firstLinePreview.text));
-						if (binaryDetected) {
-							return toolResult<ReadToolDetails>({ resolvedPath: absolutePath, suffixResolution })
-								.text(
-									prependSuffixResolutionNotice(
-										`[Cannot read binary file '${formatPathRelativeToCwd(absolutePath, this.session.cwd)}' (${formatBytes(fileSize)}); content contains NUL bytes (binary or UTF-16 encoded)]`,
-										suffixResolution,
-									),
-								)
-								.sourcePath(absolutePath)
-								.done();
-						}
-					}
 					const maxColumns = resolveOutputMaxColumns(this.session.settings);
 					// Column truncation is display-only. `collectedLines` MUST stay
 					// byte-for-byte with the on-disk content so the snapshot recorded

package/src/utils/file-mentions.ts

@@ -10,7 +10,7 @@ import path from "node:path";
 import { formatHashlineHeader, formatNumberedLines, type SnapshotStore } from "@oh-my-pi/hashline";
 import type { AgentMessage } from "@oh-my-pi/pi-agent-core";
 import type { ImageContent } from "@oh-my-pi/pi-ai";
-import { formatAge, formatBytes, readImageMetadata } from "@oh-my-pi/pi-utils";
+import { formatAge, formatBytes, isProbablyBinary, readImageMetadata } from "@oh-my-pi/pi-utils";
 import { canonicalSnapshotKey } from "../edit/file-snapshot-store";
 import { normalizeToLF } from "../edit/normalize";
 import type { FileMentionMessage } from "../session/messages";
@@ -257,6 +257,15 @@ export async function generateFileMentionMessages(
 				});
 				continue;
 			}
+			if (await isProbablyBinary(absolutePath)) {
+				files.push({
+					path: resolvedPath,
+					content: `(skipped auto-read: binary file, ${formatBytes(stat.size)})`,
+					byteSize: stat.size,
+					skippedReason: "binary",
+				});
+				continue;
+			}
 
 			const content = await Bun.file(absolutePath).text();
 			const snapshotStore = options?.useHashLines ? options.snapshotStore : undefined;

package/src/utils/git.ts

@@ -74,8 +74,20 @@ export interface StatusOptions {
 	readonly z?: boolean;
 }
 
+export interface CommitAuthor {
+	readonly date?: string;
+	readonly email: string;
+	readonly name: string;
+}
+
+export interface CommitDetails {
+	readonly author: CommitAuthor;
+	readonly message: string;
+}
+
 export interface CommitOptions {
 	readonly allowEmpty?: boolean;
+	readonly author?: CommitAuthor;
 	readonly files?: readonly string[];
 	readonly signal?: AbortSignal;
 }
@@ -91,6 +103,7 @@ export interface PatchOptions {
 	readonly cached?: boolean;
 	readonly check?: boolean;
 	readonly env?: Record<string, string | undefined>;
+	readonly threeWay?: boolean;
 	readonly signal?: AbortSignal;
 }
 
@@ -359,6 +372,7 @@ function buildApplyArgs(patchPath: string, options: PatchOptions): string[] {
 	const args = ["apply"];
 	if (options.check) args.push("--check");
 	if (options.cached) args.push("--cached");
+	if (options.threeWay) args.push("--3way");
 	args.push("--binary", patchPath);
 	return args;
 }
@@ -1122,6 +1136,10 @@ export const stage = {
 /** Create a commit with the given message (passed via stdin). */
 export async function commit(cwd: string, message: string, options: CommitOptions = {}): Promise<GitCommandResult> {
 	const args = ["commit", "-F", "-"];
+	if (options.author) {
+		args.push(`--author=${options.author.name} <${options.author.email}>`);
+		if (options.author.date) args.push(`--date=${options.author.date}`);
+	}
 	if (options.allowEmpty) args.push("--allow-empty");
 	if (options.files?.length) args.push("--", ...options.files);
 	return runChecked(cwd, args, { signal: options.signal, stdin: message });
@@ -1195,6 +1213,19 @@ export const show = Object.assign(
 	},
 );
 
+/** Read commit message and author metadata for replay/rewrite flows. */
+export async function commitDetails(cwd: string, revision: string, signal?: AbortSignal): Promise<CommitDetails> {
+	const raw = await runText(cwd, ["show", "-s", "--format=%an%x00%ae%x00%aI%x00%B", revision], {
+		readOnly: true,
+		signal,
+	});
+	const [name = "", email = "", date = "", ...messageParts] = raw.split("\0");
+	return {
+		author: { date, email, name },
+		message: messageParts.join("\0").replace(/\n$/, ""),
+	};
+}
+
 // ════════════════════════════════════════════════════════════════════════════
 // API: log
 // ════════════════════════════════════════════════════════════════════════════
@@ -1212,6 +1243,13 @@ export const log = {
 	},
 };
 
+export const revList = {
+	/** Commits in `base..head`, oldest first. */
+	async range(cwd: string, base: string, head: string, signal?: AbortSignal): Promise<string[]> {
+		return splitLines(await runText(cwd, ["rev-list", "--reverse", `${base}..${head}`], { readOnly: true, signal }));
+	},
+};
+
 // ════════════════════════════════════════════════════════════════════════════
 // API: branch
 // ════════════════════════════════════════════════════════════════════════════

package/src/web/search/index.ts

@@ -138,14 +138,20 @@ async function executeSearch(
 	options: ExecuteSearchOptions,
 ): Promise<{ content: Array<{ type: "text"; text: string }>; details: SearchRenderDetails }> {
 	const { authStorage, sessionId, signal } = options;
-	const providers =
-		params.provider && params.provider !== "auto"
-			? await getSearchProvider(params.provider).then(async provider =>
-					(await provider.isExplicitlyAvailable(authStorage))
-						? [provider]
-						: resolveProviderChain(authStorage, "auto"),
-				)
-			: await resolveProviderChain(authStorage);
+	const explicitProvider = params.provider;
+	let providers: SearchProvider[];
+	if (explicitProvider && explicitProvider !== "auto") {
+		const provider = await getSearchProvider(explicitProvider);
+		providers = (await provider.isExplicitlyAvailable(authStorage))
+			? [provider]
+			: await resolveProviderChain(authStorage, "auto");
+	} else if (explicitProvider === "auto") {
+		// Explicit `--provider auto` bypasses the configured preferred provider
+		// for this invocation; exclusions still apply.
+		providers = await resolveProviderChain(authStorage, "auto");
+	} else {
+		providers = await resolveProviderChain(authStorage);
+	}
 	if (providers.length === 0) {
 		const message = "No web search provider configured.";
 		return {

package/src/web/search/providers/duckduckgo.ts

@@ -6,122 +6,194 @@ import type { SearchParams } from "./base";
 import { SearchProvider } from "./base";
 import { classifyProviderHttpError, withHardTimeout } from "./utils";
 
-const DUCKDUCKGO_SEARCH_URL = "https://api.duckduckgo.com/";
+/**
+ * DuckDuckGo's no-JS HTML search frontend. POST `q=…` to receive a static
+ * results page we can parse without a real browser. The Instant Answer API
+ * (`api.duckduckgo.com`) was tried first but it only returns content for
+ * Wikipedia/Wolfram-Alpha-style topics — empty for the vast majority of
+ * agent queries (see #3799).
+ */
+const DUCKDUCKGO_HTML_URL = "https://html.duckduckgo.com/html/";
 const DEFAULT_NUM_RESULTS = 10;
 const MAX_NUM_RESULTS = 20;
 
-interface DuckDuckGoTopic {
-	FirstURL?: string | null;
-	Text?: string | null;
-	Topics?: DuckDuckGoTopic[] | null;
+/**
+ * Recency → DDG `df` form param. DDG accepts single letters for the time
+ * filter; queries without a `df` value return the unfiltered default.
+ */
+const RECENCY_TO_DDG_DF: Record<NonNullable<SearchParams["recency"]>, string> = {
+	day: "d",
+	week: "w",
+	month: "m",
+	year: "y",
+};
+
+/**
+ * Browser-like UA so DDG serves the standard results page instead of the
+ * mobile-only or noscript variants. DDG returns HTTP 202 plus an anomaly
+ * modal when it suspects automation; we surface that as a clear error so
+ * the orchestrator can fall through to the next provider with context.
+ */
+const BROWSER_USER_AGENT =
+	"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/149.0.0.0 Safari/537.36";
+
+interface ParsedResult {
+	title: string;
+	url: string;
+	snippet?: string;
 }
 
-interface DuckDuckGoResponse {
-	AbstractText?: string | null;
-	AbstractURL?: string | null;
-	AbstractSource?: string | null;
-	Answer?: string | null;
-	Definition?: string | null;
-	Heading?: string | null;
-	Results?: DuckDuckGoTopic[] | null;
-	RelatedTopics?: DuckDuckGoTopic[] | null;
-}
-
-function cleanText(value: string | null | undefined): string | undefined {
-	const cleaned = value
-		?.replace(/<[^>]*>/g, " ")
+/**
+ * Decode an HTML-encoded fragment lifted from DDG markup. Strips inline tags
+ * (the results page wraps query terms in `<b>`), unescapes the small set of
+ * named entities DDG emits, and normalises whitespace.
+ */
+function decodeHtmlText(value: string): string {
+	return value
+		.replace(/<[^>]*>/g, " ")
+		.replace(/&#(\d+);/g, (_, code) => String.fromCharCode(Number(code)))
+		.replace(/&#x([0-9a-f]+);/gi, (_, code) => String.fromCharCode(Number.parseInt(code, 16)))
 		.replace(/&nbsp;/gi, " ")
 		.replace(/&amp;/gi, "&")
 		.replace(/&lt;/gi, "<")
 		.replace(/&gt;/gi, ">")
 		.replace(/&quot;/gi, '"')
-		.replace(/&#39;/gi, "'")
+		.replace(/&#39;|&apos;/gi, "'")
 		.replace(/\s+/g, " ")
 		.trim();
-	return cleaned ? cleaned : undefined;
 }
 
-function addSource(sources: SearchSource[], source: SearchSource): void {
-	if (!source.url || sources.some(existing => existing.url === source.url)) return;
-	sources.push(source);
+/**
+ * Resolve a DDG result href back to the underlying target URL.
+ *
+ * DDG routes outbound clicks through `//duckduckgo.com/l/?uddg=<encoded>` so
+ * it can record analytics; we want the unwrapped URL. Handles three shapes
+ * the page mixes in practice: redirect wrappers, protocol-relative links,
+ * and (rarely) absolute URLs on sponsored or instant answer rows.
+ */
+function unwrapResultUrl(href: string): string | undefined {
+	if (!href) return undefined;
+	const decoded = href.replace(/&amp;/gi, "&");
+	const wrapMatch = decoded.match(/[?&]uddg=([^&]+)/);
+	if (wrapMatch) {
+		try {
+			return decodeURIComponent(wrapMatch[1]);
+		} catch {
+			return undefined;
+		}
+	}
+	if (decoded.startsWith("//")) return `https:${decoded}`;
+	if (decoded.startsWith("http://") || decoded.startsWith("https://")) return decoded;
+	return undefined;
 }
 
-function addTopicSource(sources: SearchSource[], topic: DuckDuckGoTopic): void {
-	const url = topic.FirstURL?.trim();
-	if (!url) return;
-	const text = cleanText(topic.Text);
-	addSource(sources, {
-		title: text ?? url,
-		url,
-		snippet: text,
-	});
+/**
+ * Walk the HTML page and pull out result blocks in document order.
+ *
+ * DDG renders each result inside a `<div class="result …">` container with
+ * `<a class="result__a" …>` for the title link and an optional sibling
+ * `<a class="result__snippet">` (or `<div class="result__snippet">` in some
+ * variants) for the preview text. Sponsored placements, missing snippets,
+ * and the trailing pagination row are tolerated.
+ */
+function parseHtmlResults(html: string): ParsedResult[] {
+	const results: ParsedResult[] = [];
+	const blockRe =
+		/<div\b[^>]*\bclass="[^"]*\bresult\b[^"]*"[^>]*>([\s\S]*?)(?=<div\b[^>]*\bclass="[^"]*\bresult\b|<div\b[^>]*\bclass="[^"]*\bnav-link\b|$)/g;
+	const titleRe = /<a\b[^>]*\bclass="[^"]*\bresult__a\b[^"]*"[^>]*\bhref="([^"]+)"[^>]*>([\s\S]*?)<\/a>/;
+	const snippetRe = /<(?:a|div|span)\b[^>]*\bclass="[^"]*\bresult__snippet\b[^"]*"[^>]*>([\s\S]*?)<\/(?:a|div|span)>/;
+	for (const match of html.matchAll(blockRe)) {
+		const block = match[1];
+		const title = titleRe.exec(block);
+		if (!title) continue;
+		const url = unwrapResultUrl(title[1]);
+		if (!url) continue;
+		const titleText = decodeHtmlText(title[2]);
+		if (!titleText) continue;
+		const snippet = snippetRe.exec(block);
+		const snippetText = snippet ? decodeHtmlText(snippet[1]) : undefined;
+		results.push({ title: titleText, url, snippet: snippetText || undefined });
+	}
+	return results;
 }
 
-function collectTopicSources(sources: SearchSource[], topics: readonly DuckDuckGoTopic[] | null | undefined): void {
-	if (!topics) return;
-	for (const topic of topics) {
-		addTopicSource(sources, topic);
-		collectTopicSources(sources, topic.Topics);
-	}
+/**
+ * `true` when the page DDG returned is the bot-challenge modal instead of
+ * real results. DDG mixes status codes (200 vs 202) on these so the body
+ * check is the reliable signal.
+ */
+function isAnomalyResponse(html: string): boolean {
+	return html.includes("anomaly-modal") || html.includes("anomaly.js");
 }
 
-async function callDuckDuckGoSearch(params: SearchParams): Promise<DuckDuckGoResponse> {
-	const queryString = [
-		["q", params.query],
-		["format", "json"],
-		["no_redirect", "1"],
-		["no_html", "1"],
-		["skip_disambig", "1"],
-		["t", "oh-my-pi"],
-	]
-		.map(([key, value]) => `${encodeURIComponent(key)}=${encodeURIComponent(value)}`)
-		.join("&");
-	const response = await (params.fetch ?? fetch)(`${DUCKDUCKGO_SEARCH_URL}?${queryString}`, {
-		method: "GET",
+async function callDuckDuckGoHtml(params: SearchParams): Promise<string> {
+	const form = new URLSearchParams({ q: params.query, kl: "us-en" });
+	const df = params.recency ? RECENCY_TO_DDG_DF[params.recency] : undefined;
+	if (df) form.set("df", df);
+	// Add b: "" parameter as specified in the browser fetch template to match real browser form submission
+	form.set("b", "");
+
+	const response = await (params.fetch ?? fetch)(DUCKDUCKGO_HTML_URL, {
+		method: "POST",
+		body: form.toString(),
+		headers: {
+			Accept:
+				"text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.7",
+			"Accept-Language": "en,en-US;q=0.9",
+			"Cache-Control": "max-age=0",
+			"Content-Type": "application/x-www-form-urlencoded",
+			Priority: "u=0, i",
+			"Sec-Ch-Ua": '"Google Chrome";v="149", "Chromium";v="149", "Not)A;Brand";v="24"',
+			"Sec-Ch-Ua-Mobile": "?0",
+			"Sec-Ch-Ua-Platform": '"macOS"',
+			"Sec-Fetch-Dest": "document",
+			"Sec-Fetch-Mode": "navigate",
+			"Sec-Fetch-Site": "same-origin",
+			"Sec-Fetch-User": "?1",
+			"Upgrade-Insecure-Requests": "1",
+			"User-Agent": BROWSER_USER_AGENT,
+			Referer: "https://html.duckduckgo.com/",
+		},
 		signal: withHardTimeout(params.signal),
 	});
 
-	if (!response.ok) {
-		const errorText = await response.text();
-		const classified = classifyProviderHttpError("duckduckgo", response.status, errorText);
+	const body = await response.text();
+	if (!response.ok && response.status !== 202) {
+		const classified = classifyProviderHttpError("duckduckgo", response.status, body);
 		if (classified) throw classified;
+		throw new SearchProviderError("duckduckgo", `DuckDuckGo HTML error (${response.status})`, response.status);
+	}
+
+	if (isAnomalyResponse(body)) {
 		throw new SearchProviderError(
 			"duckduckgo",
-			`DuckDuckGo API error (${response.status}): ${errorText}`,
-			response.status,
+			"DuckDuckGo blocked the request with a bot-detection challenge. DuckDuckGo throttles repeat searches from datacenter and shared-egress IPs; configure another provider (e.g. Brave, Tavily, SearXNG) or retry from a residential network.",
+			429,
 		);
 	}
 
-	return (await response.json()) as DuckDuckGoResponse;
+	return body;
 }
 
-/** Execute DuckDuckGo Instant Answer API search. */
+/** Execute a DuckDuckGo web search via the no-JS HTML frontend. */
 export async function searchDuckDuckGo(params: SearchParams): Promise<SearchResponse> {
 	const numResults = clampNumResults(params.numSearchResults ?? params.limit, DEFAULT_NUM_RESULTS, MAX_NUM_RESULTS);
-	const data = await callDuckDuckGoSearch(params);
-	const answer = cleanText(data.AbstractText) ?? cleanText(data.Answer) ?? cleanText(data.Definition);
-	const sources: SearchSource[] = [];
+	const html = await callDuckDuckGoHtml(params);
+	const parsed = parseHtmlResults(html);
 
-	const abstractUrl = data.AbstractURL?.trim();
-	if (abstractUrl) {
-		addSource(sources, {
-			title: cleanText(data.AbstractSource) ?? cleanText(data.Heading) ?? abstractUrl,
-			url: abstractUrl,
-			snippet: cleanText(data.AbstractText),
-		});
+	const sources: SearchSource[] = [];
+	const seen = new Set<string>();
+	for (const result of parsed) {
+		if (seen.has(result.url)) continue;
+		seen.add(result.url);
+		sources.push({ title: result.title, url: result.url, snippet: result.snippet });
+		if (sources.length >= numResults) break;
 	}
 
-	collectTopicSources(sources, data.Results);
-	collectTopicSources(sources, data.RelatedTopics);
-
-	return {
-		provider: "duckduckgo",
-		answer,
-		sources: sources.slice(0, numResults),
-	};
+	return { provider: "duckduckgo", sources };
 }
 
-/** Search provider for DuckDuckGo Instant Answer API. */
+/** Search provider for DuckDuckGo (no API key required). */
 export class DuckDuckGoProvider extends SearchProvider {
 	readonly id = "duckduckgo";
 	readonly label = "DuckDuckGo";