@oh-my-pi/pi-coding-agent 16.0.7 → 16.0.9

package/src/tools/fetch.ts

@@ -20,12 +20,11 @@ import { CachedOutputBlock, markFramedBlockComponent } from "../tui/output-block
 import { webpExclusionForModel } from "../utils/image-loading";
 import { formatDimensionNote, resizeImage } from "../utils/image-resize";
 import { ensureTool } from "../utils/tools-manager";
+import { type ArchiveFormat, listArchiveRoot, sniffArchiveFormat } from "../utils/zip";
 import { extractWithParallel, findParallelApiKey, getParallelExtractContent } from "../web/parallel";
-import { specialHandlers } from "../web/scrapers";
-import type { RenderResult } from "../web/scrapers/types";
+import type { RenderResult, SpecialHandler } from "../web/scrapers/types";
 import { finalizeOutput, loadPage, looksLikeHtml, MAX_BYTES, MAX_OUTPUT_CHARS } from "../web/scrapers/types";
 import { convertWithMarkit, fetchBinary } from "../web/scrapers/utils";
-import { type ArchiveFormat, listArchiveRoot, sniffArchiveFormat } from "./archive-reader";
 import { applyListLimit } from "./list-limit";
 import { formatStyledArtifactReference, type OutputMeta } from "./output-meta";
 import { type LineRange, parseLineRanges } from "./path-utils";
@@ -51,34 +50,9 @@ const CONVERTIBLE_MIMES = new Set([
 	"application/vnd.openxmlformats-officedocument.spreadsheetml.sheet",
 	"application/rtf",
 	"application/epub+zip",
-	"image/png",
-	"image/jpeg",
-	"image/gif",
-	"image/webp",
-	"audio/mpeg",
-	"audio/wav",
-	"audio/ogg",
 ]);
 
-const CONVERTIBLE_EXTENSIONS = new Set([
-	".pdf",
-	".doc",
-	".docx",
-	".ppt",
-	".pptx",
-	".xls",
-	".xlsx",
-	".rtf",
-	".epub",
-	".png",
-	".jpg",
-	".jpeg",
-	".gif",
-	".webp",
-	".mp3",
-	".wav",
-	".ogg",
-]);
+const CONVERTIBLE_EXTENSIONS = new Set([".pdf", ".doc", ".docx", ".ppt", ".pptx", ".xls", ".xlsx", ".rtf", ".epub"]);
 
 const NOTEBOOK_MIMES = new Set(["application/x-ipynb+json"]);
 const NOTEBOOK_EXTENSIONS = new Set([".ipynb"]);
@@ -1044,6 +1018,18 @@ async function tryRenderBinaryPayload(
 // Unified Special Handler Dispatch
 // =============================================================================
 
+let specialHandlersPromise: Promise<SpecialHandler[]> | undefined;
+
+/**
+ * Lazily load the site-specific scraper handlers. The scrapers barrel eagerly
+ * imports ~80 site modules, none of which are needed until the first fetch that
+ * requires a special handler, so we keep them out of the cold-startup graph.
+ */
+function loadSpecialHandlers(): Promise<SpecialHandler[]> {
+	specialHandlersPromise ??= import("../web/scrapers").then(m => m.specialHandlers);
+	return specialHandlersPromise;
+}
+
 /**
  * Try all special handlers
  */
@@ -1053,6 +1039,7 @@ async function handleSpecialUrls(
 	signal: AbortSignal | undefined,
 	storage: AgentStorage | null,
 ): Promise<FetchRenderResult | null> {
+	const specialHandlers = await loadSpecialHandlers();
 	for (const handler of specialHandlers) {
 		if (signal?.aborted) {
 			throw new ToolAbortError();
@@ -1144,45 +1131,25 @@ async function renderUrl(
 			notes.push(
 				`Image MIME type ${imageMimeType} is unsupported for inline model serialization; returning text metadata only`,
 			);
-			const shouldTryConvertibleFallback = isConvertible(mime, extHint);
-			if (shouldTryConvertibleFallback) {
-				notes.push("Attempting binary conversion fallback for unsupported image MIME type");
-			} else {
-				notes.push("Falling back to textual rendering from initial response");
-			}
-			skipConvertibleBinaryRetry = !shouldTryConvertibleFallback;
+			notes.push("Falling back to textual rendering from initial response");
+			skipConvertibleBinaryRetry = true;
 		} else {
 			const binary = await fetchBinary(finalUrl, timeout, signal);
 			if (binary.ok) {
 				notes.push("Fetched image binary");
-				const conversionExtension = getExtensionHint(finalUrl, binary.contentDisposition) || extHint;
-				let convertedText: string | null = null;
-				const converted = await convertWithMarkit(binary.buffer, conversionExtension, timeout, signal);
-				if (converted.ok) {
-					if (converted.content.trim().length > 50) {
-						notes.push("Converted with markit");
-						convertedText = converted.content;
-					} else {
-						notes.push("markit conversion produced no usable output");
-					}
-				} else if (converted.error) {
-					notes.push(`markit conversion failed: ${converted.error}`);
-				} else {
-					notes.push("markit conversion failed");
-				}
 
 				if (binary.buffer.byteLength > MAX_INLINE_IMAGE_SOURCE_BYTES) {
 					notes.push(
 						`Image exceeds inline source limit (${binary.buffer.byteLength} bytes > ${MAX_INLINE_IMAGE_SOURCE_BYTES} bytes)`,
 					);
 					const output = finalizeOutput(
-						convertedText ?? `Fetched image content (${imageMimeType}), but it is too large to inline render.`,
+						`Fetched image content (${imageMimeType}), but it is too large to inline render.`,
 					);
 					return {
 						url,
 						finalUrl,
 						contentType: imageMimeType,
-						method: convertedText ? "markit" : "image-too-large",
+						method: "image-too-large",
 						content: output.content,
 						fetchedAt,
 						truncated: output.truncated,
@@ -1199,15 +1166,13 @@ async function renderUrl(
 				if (!isDecodedImage) {
 					notes.push(`Fetched payload could not be decoded as ${imageMimeType}; returning text metadata only`);
 					const output = finalizeOutput(
-						convertedText ??
-							rawContent ??
-							`Fetched payload was labeled ${imageMimeType}, but bytes were not a valid image.`,
+						rawContent ?? `Fetched payload was labeled ${imageMimeType}, but bytes were not a valid image.`,
 					);
 					return {
 						url,
 						finalUrl,
 						contentType: imageMimeType,
-						method: convertedText ? "markit" : "image-invalid",
+						method: "image-invalid",
 						content: output.content,
 						fetchedAt,
 						truncated: output.truncated,
@@ -1219,13 +1184,13 @@ async function renderUrl(
 						`Image exceeds inline output limit after resize (${resized.buffer.length} bytes > ${MAX_INLINE_IMAGE_OUTPUT_BYTES} bytes)`,
 					);
 					const output = finalizeOutput(
-						convertedText ?? `Fetched image content (${imageMimeType}), but it is too large to inline render.`,
+						`Fetched image content (${imageMimeType}), but it is too large to inline render.`,
 					);
 					return {
 						url,
 						finalUrl,
 						contentType: imageMimeType,
-						method: convertedText ? "markit" : "image-too-large",
+						method: "image-too-large",
 						content: output.content,
 						fetchedAt,
 						truncated: output.truncated,
@@ -1234,7 +1199,7 @@ async function renderUrl(
 				}
 
 				const dimensionNote = formatDimensionNote(resized);
-				let imageSummary = convertedText ?? `Fetched image content (${resized.mimeType}).`;
+				let imageSummary = `Fetched image content (${resized.mimeType}).`;
 				if (dimensionNote) {
 					imageSummary += `\n${dimensionNote}`;
 				}

package/src/tools/read.ts

@@ -48,8 +48,8 @@ import {
 	webpExclusionForModel,
 } from "../utils/image-loading";
 import { convertFileWithMarkit } from "../utils/markit";
+import { type ArchiveReader, formatArchiveEntryLines, openArchive, parseArchivePathCandidates } from "../utils/zip";
 import { buildDirectoryTree, type DirectoryTree } from "../workspace-tree";
-import { type ArchiveReader, formatArchiveEntryLines, openArchive, parseArchivePathCandidates } from "./archive-reader";
 import {
 	type ConflictEntry,
 	type ConflictScope,

package/src/tools/search.ts

@@ -28,13 +28,8 @@ import {
 	uriHyperlink,
 } from "../tui";
 import { resolveFileDisplayMode } from "../utils/file-display-mode";
+import { type ArchiveReader, type ExtractedArchiveFile, openArchive, parseArchivePathCandidates } from "../utils/zip";
 import type { ToolSession } from ".";
-import {
-	type ArchiveReader,
-	type ExtractedArchiveFile,
-	openArchive,
-	parseArchivePathCandidates,
-} from "./archive-reader";
 import { createFileRecorder, formatResultPath } from "./file-recorder";
 import { classifyGroupedLines, formatGroupedFiles, groupLineIndicesByBlank } from "./grouped-file-output";
 import { formatMatchLine } from "./match-line-format";

package/src/tools/write.ts

@@ -20,8 +20,14 @@ import writeDescription from "../prompts/tools/write.md" with { type: "text" };
 import type { ToolSession } from "../sdk";
 import { fileHyperlink, framedBlock, renderStatusLine } from "../tui";
 import { resolveFileDisplayMode } from "../utils/file-display-mode";
+import {
+	type ArchiveMemberContent,
+	archiveFormatFromPath,
+	parseArchivePathCandidates,
+	readArchiveEntries,
+	writeArchive,
+} from "../utils/zip";
 import { truncateForPrompt } from "./approval";
-import { parseArchivePathCandidates } from "./archive-reader";
 import { assertEditableFile } from "./auto-generated-guard";
 import {
 	type ConflictEntry,
@@ -65,12 +71,6 @@ import { toolResult } from "./tool-result";
 const LOOSE_HASHLINE_HEADER_RE = /^\s*\[[^#\r\n]+#[^ \t\r\n]*\]\s*$/;
 const EXECUTABLE_NOTICE = "[Notice: Made executable via chmod +x]";
 
-let fflateModulePromise: Promise<typeof import("fflate")> | undefined;
-async function loadFflate(): Promise<typeof import("fflate")> {
-	if (!fflateModulePromise) fflateModulePromise = import("fflate");
-	return fflateModulePromise;
-}
-
 const writeSchema = type({
 	path: type("string").describe("file path"),
 	content: type("string").describe("file content"),
@@ -369,9 +369,10 @@ export class WriteTool implements AgentTool<typeof writeSchema, WriteToolDetails
 		const finalPath = resolvedArchivePath.exists
 			? await fs.realpath(resolvedArchivePath.absolutePath).catch(() => resolvedArchivePath.absolutePath)
 			: resolvedArchivePath.absolutePath;
-		const lowerPath = finalPath.toLowerCase();
-		const isZip = lowerPath.endsWith(".zip");
-		const isGzip = lowerPath.endsWith(".tar.gz") || lowerPath.endsWith(".tgz");
+		// A realpath swap can land on a name without an archive extension; a
+		// whole-archive rewrite then defaults to an uncompressed tar, matching the
+		// previous `isZip`/`isGzip`/else fallthrough.
+		const format = archiveFormatFromPath(finalPath) ?? "tar";
 		// Rewrites are whole-archive: write to a temp file and rename so a
 		// crash/disk-full mid-write can't destroy the original archive.
 		const tmpPath = `${finalPath}.tmp-${process.pid}`;
@@ -381,67 +382,26 @@ export class WriteTool implements AgentTool<typeof writeSchema, WriteToolDetails
 			await fs.mkdir(parentDir, { recursive: true });
 		}
 
-		if (isZip) {
-			const zipEntries: Record<string, Uint8Array> = {};
-
-			if (resolvedArchivePath.exists) {
-				try {
-					const bytes = await Bun.file(resolvedArchivePath.absolutePath).bytes();
-					const { unzipSync } = await loadFflate();
-					const existing = unzipSync(new Uint8Array(bytes));
-					for (const [entryPath, data] of Object.entries(existing)) {
-						zipEntries[entryPath.replace(/\\/g, "/")] = data;
-					}
-				} catch (error) {
-					throw new ToolError(error instanceof Error ? error.message : String(error));
-				}
-			}
-
-			zipEntries[resolvedArchivePath.archiveSubPath] = new TextEncoder().encode(content);
-
+		const entries = new Map<string, ArchiveMemberContent>();
+		if (resolvedArchivePath.exists) {
 			try {
-				const { zipSync } = await loadFflate();
-				const zipBuffer = zipSync(zipEntries);
-				await Bun.write(tmpPath, zipBuffer);
-				await fs.rename(tmpPath, finalPath);
-			} catch (error) {
-				await fs.rm(tmpPath, { force: true }).catch(() => {});
-				throw new ToolError(error instanceof Error ? error.message : String(error));
-			}
-		} else {
-			const archiveEntries: Record<string, string | File> = {};
-			if (resolvedArchivePath.exists) {
-				let archive: Bun.Archive;
-				try {
-					archive = new Bun.Archive(await Bun.file(resolvedArchivePath.absolutePath).bytes());
-				} catch (error) {
-					throw new ToolError(error instanceof Error ? error.message : String(error));
+				const existing = await readArchiveEntries({ bytes: await Bun.file(finalPath).bytes(), format });
+				for (const [entryPath, data] of existing) {
+					entries.set(entryPath, data);
 				}
-
-				let files: Map<string, File>;
-				try {
-					files = await archive.files();
-				} catch (error) {
-					throw new ToolError(error instanceof Error ? error.message : String(error));
-				}
-
-				for (const [entryPath, file] of files) {
-					archiveEntries[entryPath.replace(/\\/g, "/")] = file;
-				}
-			}
-
-			archiveEntries[resolvedArchivePath.archiveSubPath] = content;
-
-			try {
-				// `Bun.Archive.write` never infers compression from the extension;
-				// request gzip explicitly so `.tar.gz`/`.tgz` stay compressed.
-				await Bun.Archive.write(tmpPath, archiveEntries, isGzip ? { compress: "gzip" } : undefined);
-				await fs.rename(tmpPath, finalPath);
 			} catch (error) {
-				await fs.rm(tmpPath, { force: true }).catch(() => {});
 				throw new ToolError(error instanceof Error ? error.message : String(error));
 			}
 		}
+		entries.set(resolvedArchivePath.archiveSubPath, content);
+
+		try {
+			await writeArchive(tmpPath, format, entries);
+			await fs.rename(tmpPath, finalPath);
+		} catch (error) {
+			await fs.rm(tmpPath, { force: true }).catch(() => {});
+			throw new ToolError(error instanceof Error ? error.message : String(error));
+		}
 
 		invalidateFsScanAfterWrite(resolvedArchivePath.absolutePath);
 		const outputPath = `${formatPathRelativeToCwd(resolvedArchivePath.absolutePath, this.session.cwd)}:${

package/src/utils/markit.ts

@@ -1,6 +1,7 @@
 import { logger, untilAborted } from "@oh-my-pi/pi-utils";
-import type { Markit, StreamInfo } from "markit-ai";
+import type { Markit, StreamInfo } from "../markit";
 import { ToolAbortError } from "../tools/tool-errors";
+import { loadEmbeddedMupdfWasm } from "./mupdf-wasm-embed";
 
 export interface MarkitConversionResult {
 	content: string;
@@ -21,10 +22,7 @@ export interface MarkitFileConversionOptions {
 interface MuPdfWasmModuleConfig {
 	print?: (...values: unknown[]) => void;
 	printErr?: (...values: unknown[]) => void;
-}
-
-declare global {
-	var $libmupdf_wasm_Module: MuPdfWasmModuleConfig | undefined;
+	wasmBinary?: Uint8Array;
 }
 
 function logMuPdfWasmOutput(stream: "stdout" | "stderr", values: unknown[]): void {
@@ -32,17 +30,35 @@ function logMuPdfWasmOutput(stream: "stdout" | "stderr", values: unknown[]): voi
 	logger.debug("mupdf wasm output", { stream, message });
 }
 
+// `$libmupdf_wasm_Module` is declared globally (as `any`) by the mupdf package.
+// Install print hooks before the WASM module initializes so its stdout/stderr
+// route to the file logger instead of corrupting the TUI.
 function installMuPdfWasmLogger(): void {
-	const moduleConfig = globalThis.$libmupdf_wasm_Module ?? {};
-	moduleConfig.print = (...values) => logMuPdfWasmOutput("stdout", values);
-	moduleConfig.printErr = (...values) => logMuPdfWasmOutput("stderr", values);
+	const moduleConfig: MuPdfWasmModuleConfig = globalThis.$libmupdf_wasm_Module ?? {};
+	moduleConfig.print = (...values: unknown[]) => logMuPdfWasmOutput("stdout", values);
+	moduleConfig.printErr = (...values: unknown[]) => logMuPdfWasmOutput("stderr", values);
+	globalThis.$libmupdf_wasm_Module = moduleConfig;
+}
+
+// Hand the WASM module its bytes directly when the compiled binary embedded them
+// (scripts/embed-mupdf-wasm.ts); a single-file binary has no node_modules for
+// mupdf to read `mupdf-wasm.wasm` from. Source/npm builds get undefined here and
+// mupdf loads its own wasm. Must run before the mupdf module evaluates.
+function installEmbeddedMupdfWasm(): void {
+	const wasmBinary = loadEmbeddedMupdfWasm();
+	if (!wasmBinary) return;
+	const moduleConfig: MuPdfWasmModuleConfig = globalThis.$libmupdf_wasm_Module ?? {};
+	moduleConfig.wasmBinary = wasmBinary;
 	globalThis.$libmupdf_wasm_Module = moduleConfig;
 }
 
 installMuPdfWasmLogger();
 
 let markit: () => Markit | Promise<Markit> = async () => {
-	const promise = import("markit-ai").then(({ Markit }) => {
+	// Lazy: keep the document engine (mammoth/mupdf) off the startup
+	// import graph — it loads only when a document is first converted.
+	installEmbeddedMupdfWasm();
+	const promise = import("../markit").then(({ Markit }) => {
 		const instance = new Markit();
 		markit = () => instance;
 		return instance;

package/src/utils/mupdf-wasm-embed.ts

@@ -0,0 +1,12 @@
+// AUTOGENERATED -- managed by scripts/embed-mupdf-wasm.ts. Do not edit by hand.
+//
+// Compiled single-file binaries cannot let mupdf resolve its `mupdf-wasm.wasm`
+// sibling from the read-only bunfs, so the binary build (scripts/build-binary.ts
+// and scripts/ci-release-build-binaries.ts) regenerates this module to embed the
+// wasm bytes via `with { type: "file" }` and copies the wasm next to it. Source
+// checkouts, `bun test`, and the npm `dist/cli.js` bundle keep mupdf external and
+// load the wasm from node_modules, so this placeholder returns undefined and the
+// build resets back to it afterward.
+export function loadEmbeddedMupdfWasm(): Uint8Array | undefined {
+	return undefined;
+}

package/src/utils/tools-manager.ts

@@ -2,6 +2,7 @@ import * as fs from "node:fs";
 import * as os from "node:os";
 import * as path from "node:path";
 import { $which, APP_NAME, getToolsDir, logger, ptree, TempDir } from "@oh-my-pi/pi-utils";
+import { extractArchive } from "./zip";
 
 const TOOLS_DIR = getToolsDir();
 const TOOL_DOWNLOAD_TIMEOUT_MS = 120_000;
@@ -220,17 +221,7 @@ async function downloadTool(tool: ToolName, signal?: AbortSignal): Promise<strin
 		}
 
 		try {
-			const archive = new Bun.Archive(await Bun.file(archivePath).arrayBuffer());
-			const files = await archive.files();
-			const extractRoot = path.resolve(tmp.path());
-
-			for (const [filePath, file] of files) {
-				const outputPath = path.resolve(extractRoot, filePath);
-				if (!outputPath.startsWith(extractRoot + path.sep)) {
-					throw new Error(`Archive entry escapes extraction dir: ${filePath}`);
-				}
-				await Bun.write(outputPath, file);
-			}
+			await extractArchive(archivePath, tmp.path());
 		} catch (err) {
 			throw new Error(`Failed to extract ${assetName}: ${err instanceof Error ? err.message : String(err)}`);
 		}

package/src/utils/turndown.ts

@@ -0,0 +1,83 @@
+import TurndownService from "turndown";
+import { gfm } from "turndown-plugin-gfm";
+
+type TurndownListParent = {
+	nodeName: string;
+	getAttribute(name: string): string | null;
+	children: ArrayLike<unknown>;
+};
+
+/**
+ * Build a Turndown instance configured for GFM with the fixes omp relies on:
+ * `~~strikethrough~~`, unescaped heading periods, and single-space list markers.
+ *
+ * Shared by the web scrapers (HTML → markdown) and the markit document engine
+ * (`src/markit`). The rule set must stay identical across both call sites.
+ */
+export function createTurndown(): TurndownService {
+	const turndown = new TurndownService({
+		headingStyle: "atx",
+		codeBlockStyle: "fenced",
+		bulletListMarker: "-",
+	});
+	turndown.use(gfm);
+	// GFM spec uses ~~ (double tilde), not ~ (single)
+	turndown.addRule("strikethrough", {
+		filter: ["del", "s", "strike"],
+		replacement(content) {
+			return `~~${content}~~`;
+		},
+	});
+	// Unescape the backslash turndown inserts before periods in headings ("1." -> "1\.")
+	turndown.addRule("heading", {
+		filter: ["h1", "h2", "h3", "h4", "h5", "h6"],
+		replacement(content, node) {
+			const level = Number(node.nodeName.charAt(1));
+			const prefix = "#".repeat(level);
+			const cleaned = content.replace(/\\([.])/g, "$1").trim();
+			return `\n\n${prefix} ${cleaned}\n\n`;
+		},
+	});
+	// Single space after the marker (turndown hardcodes three)
+	turndown.addRule("listItem", {
+		filter: "li",
+		replacement(content, node, options) {
+			const body = content.replace(/^\n+/, "").replace(/\n+$/, "\n").replace(/\n/gm, "\n  ");
+			const parent = node.parentNode as unknown as TurndownListParent | null;
+			let prefix = `${options.bulletListMarker} `;
+			if (parent?.nodeName === "OL") {
+				const start = parent.getAttribute("start");
+				const index = Array.prototype.indexOf.call(parent.children, node);
+				prefix = `${(start ? Number(start) : 1) + index}. `;
+			}
+			return prefix + body + (node.nextSibling ? "\n" : "");
+		},
+	});
+	return turndown;
+}
+
+/**
+ * Normalize HTML tables so turndown-plugin-gfm can render them:
+ * - strip `<p>` tags inside `<td>`/`<th>` cells (joining paragraphs with a space)
+ * - wrap the first row in `<thead>` when missing
+ */
+export function normalizeTablesHtml(html: string): string {
+	let result = html.replace(
+		/<(td|th)([^>]*)>([\s\S]*?)<\/(td|th)>/gi,
+		(_match, tag: string, attrs: string, inner: string, closeTag: string) => {
+			const stripped = inner
+				.replace(/^\s*<p>/i, "")
+				.replace(/<\/p>\s*$/i, "")
+				.replace(/<\/p>\s*<p>/gi, " ");
+			return `<${tag}${attrs}>${stripped}</${closeTag}>`;
+		},
+	);
+	result = result.replace(
+		/<table([^>]*)>\s*(?:<tbody>\s*)?(<tr[\s\S]*?<\/tr>)([\s\S]*?)<\/(?:tbody>\s*<\/)?table>/gi,
+		(_match, attrs: string, firstRow: string, rest: string) => {
+			const theadRow = firstRow.replace(/<td/gi, "<th").replace(/<\/td>/gi, "</th>");
+			return `<table${attrs}><thead>${theadRow}</thead><tbody>${rest}</tbody></table>`;
+		},
+	);
+	return result;
+}