@linxiraos/pi-utils 1.0.0

package/src/tls-fetch.ts

@@ -0,0 +1,178 @@
+/**
+ * `NODE_EXTRA_CA_CERTS` shim for Bun's `fetch`.
+ *
+ * Node's TLS layer honours `NODE_EXTRA_CA_CERTS` natively, but Bun's
+ * `fetch` does not, and both the provider streams (`openai-responses`,
+ * `openai-completions`, `openai-codex-responses`, `ollama-chat`, ...) and
+ * catalog model discovery (`/models` probes) route every request through
+ * Bun's runtime. Without this wrapper, corporate relays and private
+ * gateways behind a custom CA bundle fail with
+ * `unknown certificate verification error` even when the env var is set.
+ *
+ * The wrapper merges the resolved CA bundle into Bun's `RequestInit.tls.ca`.
+ * Bun's `tls.ca` REPLACES the default trust store when set, so the wrapper
+ * always seeds {@link tls.rootCertificates} when the caller has not already
+ * curated their own CA list.
+ */
+import * as fs from "node:fs";
+import * as tls from "node:tls";
+import { $env } from "./env";
+import { isEnoent } from "./fs-error";
+
+/**
+ * `fetch`-compatible function. Accepts any callable matching the standard
+ * fetch signature; `preconnect` is optional because non-Bun runtimes
+ * (browsers, test mocks) won't expose it.
+ */
+export type FetchImpl = ((input: string | URL | Request, init?: RequestInit) => Promise<Response>) & {
+	preconnect?: typeof globalThis.fetch.preconnect;
+};
+
+/**
+ * `NODE_EXTRA_CA_CERTS` was set but unusable (path does not exist). This is
+ * a config/contract error, not a transient transport fault — it is never
+ * retried.
+ */
+export class ExtraCaError extends Error {
+	constructor(message: string, options?: { cause?: unknown }) {
+		super(message, options?.cause === undefined ? undefined : { cause: options.cause });
+		this.name = "ExtraCaError";
+	}
+}
+
+/** Bun extension to `RequestInit` for the TLS options we touch. */
+type BunTlsOptions = {
+	ca?: string | string[];
+	cert?: string;
+	key?: string;
+	rejectUnauthorized?: boolean;
+	serverName?: string;
+	ciphers?: string;
+};
+
+type BunTlsRequestInit = RequestInit & { tls?: BunTlsOptions };
+
+const EXTRA_CA_FETCH_MARKER = Symbol("omp.extraCaFetch");
+type ExtraCaFetch = FetchImpl & { [EXTRA_CA_FETCH_MARKER]?: true };
+
+/**
+ * Cached resolution of `NODE_EXTRA_CA_CERTS`. Keyed on the env value plus
+ * the file mtime for path values so on-disk cert rotation (short-lived
+ * corporate bundles) invalidates the cache instead of pinning the first
+ * read forever.
+ */
+let cacheKey: string | undefined;
+let cacheValue: string | undefined;
+
+/**
+ * Returns the PEM bytes referenced by `NODE_EXTRA_CA_CERTS`, or `undefined`
+ * when the env var is unset/empty.
+ *
+ * Accepts the same shapes Node accepts plus an inline-PEM escape hatch:
+ * - Inline PEM (`-----BEGIN CERTIFICATE-----...`). Literal `\n` escapes in
+ *   the env value are expanded so callers can ship single-line PEMs through
+ *   shell exports.
+ * - File path. Anything that does not contain a PEM header is treated as a
+ *   path, matching Node's "extensionless filename is still a path" contract.
+ *   `ENOENT` becomes {@link ExtraCaError}; other I/O errors bubble.
+ */
+function resolveExtraCa(): string | undefined {
+	const raw = $env.NODE_EXTRA_CA_CERTS?.trim();
+	if (!raw) return undefined;
+
+	let key: string;
+	if (raw.includes("-----BEGIN")) {
+		key = raw;
+	} else {
+		try {
+			key = `${raw}@${fs.statSync(raw).mtimeMs}`;
+		} catch {
+			key = raw;
+		}
+	}
+	if (key === cacheKey) return cacheValue;
+
+	if (raw.includes("-----BEGIN")) {
+		cacheValue = raw.replace(/\\n/g, "\n");
+	} else {
+		try {
+			cacheValue = fs.readFileSync(raw, "utf8");
+		} catch (error) {
+			if (isEnoent(error)) {
+				throw new ExtraCaError(`NODE_EXTRA_CA_CERTS path does not exist: ${raw}`);
+			}
+			throw error;
+		}
+	}
+	cacheKey = key;
+	return cacheValue;
+}
+
+/** Test seam: drop the cached PEM so a follow-up call re-reads the env. */
+export function __resetExtraCaCache(): void {
+	cacheKey = undefined;
+	cacheValue = undefined;
+}
+
+/**
+ * Merge `extraCa` into `init.tls.ca`. When the caller has not supplied a CA
+ * list, the system root store is included alongside the extra bundle —
+ * Bun's `tls.ca` replaces the default trust store, so omitting roots would
+ * break every public host. When the caller already curated a list (e.g.
+ * Anthropic Foundry's mTLS options, which already seed
+ * `tls.rootCertificates`), only the extra CA is appended.
+ */
+function withExtraCaInit(init: RequestInit | undefined, extraCa: string): RequestInit {
+	const existingTls = (init as BunTlsRequestInit | undefined)?.tls;
+	const existingCa = existingTls?.ca;
+	let mergedCa: string[];
+	if (existingCa === undefined) {
+		mergedCa = [...tls.rootCertificates, extraCa];
+	} else if (Array.isArray(existingCa)) {
+		mergedCa = [...existingCa, extraCa];
+	} else {
+		mergedCa = [existingCa, extraCa];
+	}
+	return { ...init, tls: { ...existingTls, ca: mergedCa } } as RequestInit;
+}
+
+/**
+ * Wrap `fetchImpl` so every call honours `NODE_EXTRA_CA_CERTS`. Idempotent:
+ * a fetch already wrapped is returned unchanged so repeated composition
+ * (Anthropic auth-retry replays, request-debug fan-out) never stacks
+ * wrappers. When the env var is unset the original fetch is returned, so
+ * default deployments pay nothing.
+ */
+export function wrapFetchForExtraCa(fetchImpl: FetchImpl): FetchImpl {
+	const maybeWrapped = fetchImpl as ExtraCaFetch;
+	if (maybeWrapped[EXTRA_CA_FETCH_MARKER]) return fetchImpl;
+	// Peek once at construction — if the env var is unset, skip the wrapper
+	// entirely so the hot fetch path stays a single function call. The env
+	// is evaluated again per request below to catch in-process updates from
+	// tests (`__resetExtraCaCache` + env mutation).
+	if (!$env.NODE_EXTRA_CA_CERTS?.trim()) return fetchImpl;
+
+	const wrapped = Object.assign(
+		async (input: string | URL | Request, init?: RequestInit): Promise<Response> => {
+			const extraCa = resolveExtraCa();
+			return extraCa ? fetchImpl(input, withExtraCaInit(init, extraCa)) : fetchImpl(input, init);
+		},
+		fetchImpl.preconnect ? { preconnect: fetchImpl.preconnect } : {},
+		{ [EXTRA_CA_FETCH_MARKER]: true as const },
+	);
+	return wrapped;
+}
+
+/**
+ * Convenience for options-bag composition (e.g. the stream-entry path in
+ * `@linxiraos/pi-ai`'s `stream.ts`, which mirrors `withRequestDebugFetch` so
+ * the proxy/debug/extra-CA wrappers compose uniformly). No-op when the env
+ * var is unset.
+ */
+export function withExtraCaFetch<T extends { fetch?: FetchImpl } | undefined>(options: T): T {
+	if (!$env.NODE_EXTRA_CA_CERTS?.trim()) return options;
+	const fetchImpl = options?.fetch ?? (globalThis.fetch as FetchImpl);
+	const wrapped = wrapFetchForExtraCa(fetchImpl);
+	if (wrapped === fetchImpl && options?.fetch !== undefined) return options;
+	return { ...(options ?? {}), fetch: wrapped } as T;
+}

package/src/turndown/gfm.ts

@@ -0,0 +1,106 @@
+import type { TurndownNode, TurndownPlugin, TurndownServiceLike } from "./types";
+
+function descendantElements(node: TurndownNode, name: string): TurndownNode[] {
+	const matches: TurndownNode[] = [];
+	for (const child of Array.from(node.children)) {
+		if (child.nodeName === name) matches.push(child);
+		if (child.nodeName !== "TABLE") matches.push(...descendantElements(child, name));
+	}
+	return matches;
+}
+
+function isHeadingTable(node: TurndownNode): boolean {
+	if (node.nodeName !== "TABLE") return false;
+	const firstRow = descendantElements(node, "TR")[0];
+	if (!firstRow) return false;
+	const cells = Array.from(firstRow.children);
+	return cells.length > 0 && cells.every(cell => cell.nodeName === "TH");
+}
+
+function cellAlignment(cell: TurndownNode): "left" | "center" | "right" | undefined {
+	const explicit = cell.getAttribute("align")?.toLowerCase();
+	if (explicit === "left" || explicit === "center" || explicit === "right") return explicit;
+	const style = cell.getAttribute("style") ?? "";
+	const styled = /(?:^|;)\s*text-align\s*:\s*(left|center|right)\s*(?:;|$)/i.exec(style)?.[1]?.toLowerCase();
+	return styled === "left" || styled === "center" || styled === "right" ? styled : undefined;
+}
+
+function alignmentMarker(cell: TurndownNode): string {
+	const alignment = cellAlignment(cell);
+	if (alignment === "left") return ":--";
+	if (alignment === "center") return ":-:";
+	if (alignment === "right") return "--:";
+	return "---";
+}
+
+function renderTable(service: TurndownServiceLike, table: TurndownNode): string {
+	const rows = descendantElements(table, "TR");
+	const rendered: string[] = [];
+	for (let rowIndex = 0; rowIndex < rows.length; rowIndex++) {
+		const row = rows[rowIndex];
+		if (!row) continue;
+		const cells = Array.from(row.children).filter(cell => cell.nodeName === "TH" || cell.nodeName === "TD");
+		const values = cells.map(cell => service.convertChildren(cell).trim());
+		rendered.push(`| ${values.join(" | ")} |`);
+		if (rowIndex === 0) rendered.push(`| ${cells.map(alignmentMarker).join(" | ")} |`);
+	}
+	return `\n\n${rendered.join("\n")}\n\n`;
+}
+
+/** Install GitHub fenced code blocks selected by `highlight-source-*` wrappers. */
+export const highlightedCodeBlock: TurndownPlugin = service => {
+	service.addRule("highlightedCodeBlock", {
+		filter(node) {
+			return node.nodeName === "DIV" && /(?:^|\s)highlight-source-([^\s]+)/.test(node.getAttribute("class") ?? "");
+		},
+		replacement(_content, node, options) {
+			const language = /(?:^|\s)highlight-source-([^\s]+)/.exec(node.getAttribute("class") ?? "")?.[1] ?? "";
+			const text = descendantElements(node, "PRE")[0]?.textContent ?? "";
+			return `\n\n${options.fence}${language}\n${text}\n${options.fence}\n\n`;
+		},
+	});
+};
+
+/** Install GFM strikethrough conversion. */
+export const strikethrough: TurndownPlugin = service => {
+	service.addRule("strikethrough", {
+		filter: ["del", "s", "strike"],
+		replacement(content) {
+			return `~${content}~`;
+		},
+	});
+};
+
+/** Install GFM task-list checkbox conversion. */
+export const taskListItems: TurndownPlugin = service => {
+	service.addRule("taskListItems", {
+		filter(node) {
+			return (
+				node.nodeName === "INPUT" &&
+				node.parentNode?.nodeName === "LI" &&
+				(node.getAttribute("type") ?? "").toLowerCase() === "checkbox"
+			);
+		},
+		replacement(_content, node) {
+			return node.hasAttribute("checked") ? "[x] " : "[ ] ";
+		},
+	});
+};
+
+/** Install GFM table conversion for tables with a heading row. */
+export const tables: TurndownPlugin = service => {
+	service.addRule("table", {
+		filter: isHeadingTable,
+		replacement(_content, node) {
+			return renderTable(service, node);
+		},
+	});
+};
+
+/** Install every GFM rule used by the coding agent. */
+export const gfm: TurndownPlugin = service => {
+	highlightedCodeBlock(service);
+	strikethrough(service);
+	tables(service);
+	taskListItems(service);
+};

package/src/turndown/html.ts

@@ -0,0 +1,257 @@
+import type { TurndownNode } from "./types";
+
+const VOID_ELEMENTS: Readonly<Record<string, true>> = {
+	AREA: true,
+	BASE: true,
+	BR: true,
+	COL: true,
+	EMBED: true,
+	HR: true,
+	IMG: true,
+	INPUT: true,
+	LINK: true,
+	META: true,
+	PARAM: true,
+	SOURCE: true,
+	TRACK: true,
+	WBR: true,
+};
+
+const NAMED_ENTITIES: Readonly<Record<string, string>> = {
+	amp: "&",
+	apos: "'",
+	copy: "©",
+	gt: ">",
+	hellip: "…",
+	laquo: "«",
+	lt: "<",
+	mdash: "—",
+	nbsp: " ",
+	ndash: "–",
+	quot: '"',
+	raquo: "»",
+	reg: "®",
+};
+
+function decodeEntities(value: string): string {
+	return value.replace(/&(#(?:x[\da-f]+|\d+)|[a-z][\da-z]+);?/gi, (entity, name: string) => {
+		if (name.charAt(0) === "#") {
+			const hexadecimal = name.charAt(1).toLowerCase() === "x";
+			const number = Number.parseInt(name.slice(hexadecimal ? 2 : 1), hexadecimal ? 16 : 10);
+			if (!Number.isFinite(number) || number <= 0 || number > 0x10ffff) return entity;
+			try {
+				return String.fromCodePoint(number);
+			} catch {
+				return "�";
+			}
+		}
+		return NAMED_ENTITIES[name.toLowerCase()] ?? entity;
+	});
+}
+
+function escapeAttribute(value: string): string {
+	return value.replace(/&/g, "&amp;").replace(/"/g, "&quot;");
+}
+
+abstract class HtmlNode implements TurndownNode {
+	parentNode: HtmlNode | null = null;
+	readonly childNodes: HtmlNode[] = [];
+	abstract readonly nodeType: number;
+	abstract readonly nodeName: string;
+
+	get children(): HtmlNode[] {
+		return this.childNodes.filter(child => child.nodeType === 1);
+	}
+
+	get firstChild(): HtmlNode | null {
+		return this.childNodes[0] ?? null;
+	}
+
+	get lastChild(): HtmlNode | null {
+		return this.childNodes[this.childNodes.length - 1] ?? null;
+	}
+
+	get previousSibling(): HtmlNode | null {
+		if (!this.parentNode) return null;
+		const index = this.parentNode.childNodes.indexOf(this);
+		return index > 0 ? (this.parentNode.childNodes[index - 1] ?? null) : null;
+	}
+
+	get nextSibling(): HtmlNode | null {
+		if (!this.parentNode) return null;
+		const index = this.parentNode.childNodes.indexOf(this);
+		return index >= 0 ? (this.parentNode.childNodes[index + 1] ?? null) : null;
+	}
+
+	get textContent(): string {
+		return this.childNodes.map(child => child.textContent).join("");
+	}
+
+	abstract get outerHTML(): string;
+
+	getAttribute(_name: string): string | null {
+		return null;
+	}
+
+	hasAttribute(_name: string): boolean {
+		return false;
+	}
+
+	append(child: HtmlNode): void {
+		child.parentNode = this;
+		this.childNodes.push(child);
+	}
+}
+
+class HtmlText extends HtmlNode {
+	readonly nodeType = 3;
+	readonly nodeName = "#text";
+	readonly value: string;
+
+	constructor(value: string) {
+		super();
+		this.value = value;
+	}
+
+	override get textContent(): string {
+		return this.value;
+	}
+
+	get outerHTML(): string {
+		return this.value.replace(/&/g, "&amp;").replace(/</g, "&lt;").replace(/>/g, "&gt;");
+	}
+}
+
+class HtmlElement extends HtmlNode {
+	readonly nodeType = 1;
+	readonly nodeName: string;
+	readonly #attributes: Map<string, string>;
+
+	constructor(name: string, attributes: Map<string, string>) {
+		super();
+		this.nodeName = name.toUpperCase();
+		this.#attributes = attributes;
+	}
+
+	override getAttribute(name: string): string | null {
+		return this.#attributes.get(name.toLowerCase()) ?? null;
+	}
+
+	override hasAttribute(name: string): boolean {
+		return this.#attributes.has(name.toLowerCase());
+	}
+
+	get outerHTML(): string {
+		const tag = this.nodeName.toLowerCase();
+		const attributes = [...this.#attributes]
+			.map(([name, value]) => (value === "" ? name : `${name}="${escapeAttribute(value)}"`))
+			.join(" ");
+		const opening = `<${tag}${attributes ? ` ${attributes}` : ""}>`;
+		if (VOID_ELEMENTS[this.nodeName]) return opening;
+		return `${opening}${this.childNodes.map(child => child.outerHTML).join("")}</${tag}>`;
+	}
+}
+
+class HtmlFragment extends HtmlNode {
+	readonly nodeType = 11;
+	readonly nodeName = "#document-fragment";
+
+	get outerHTML(): string {
+		return this.childNodes.map(child => child.outerHTML).join("");
+	}
+}
+
+function parseAttributes(source: string): Map<string, string> {
+	const attributes = new Map<string, string>();
+	const pattern = /([^\s=/>]+)(?:\s*=\s*(?:"([^"]*)"|'([^']*)'|([^\s"'=<>`]+)))?/g;
+	for (const match of source.matchAll(pattern)) {
+		const name = match[1]?.toLowerCase();
+		if (!name) continue;
+		attributes.set(name, decodeEntities(match[2] ?? match[3] ?? match[4] ?? ""));
+	}
+	return attributes;
+}
+
+function* htmlTokens(html: string): Generator<string> {
+	let cursor = 0;
+	while (cursor < html.length) {
+		if (html.charAt(cursor) !== "<") {
+			const nextTag = html.indexOf("<", cursor);
+			const end = nextTag < 0 ? html.length : nextTag;
+			yield html.slice(cursor, end);
+			cursor = end;
+			continue;
+		}
+		if (!/[/!a-z]/i.test(html.charAt(cursor + 1))) {
+			yield "<";
+			cursor++;
+			continue;
+		}
+		if (html.startsWith("<!--", cursor)) {
+			const commentEnd = html.indexOf("-->", cursor + 4);
+			const end = commentEnd < 0 ? html.length : commentEnd + 3;
+			yield html.slice(cursor, end);
+			cursor = end;
+			continue;
+		}
+		let quote: string | undefined;
+		let tagEnd = cursor + 1;
+		for (; tagEnd < html.length; tagEnd++) {
+			const character = html.charAt(tagEnd);
+			if (quote) {
+				if (character === quote) quote = undefined;
+			} else if (character === '"' || character === "'") {
+				quote = character;
+			} else if (character === ">") {
+				break;
+			}
+		}
+		if (tagEnd >= html.length) {
+			yield "<";
+			cursor++;
+			continue;
+		}
+		yield html.slice(cursor, tagEnd + 1);
+		cursor = tagEnd + 1;
+	}
+}
+
+/** Parse an HTML fragment into the small DOM subset needed by Turndown. */
+export function parseHtmlFragment(html: string): TurndownNode {
+	const root = new HtmlFragment();
+	const stack: HtmlNode[] = [root];
+	for (const token of htmlTokens(html)) {
+		const parent = stack[stack.length - 1] ?? root;
+		if (token.startsWith("<!--") || /^<!/i.test(token)) continue;
+		if (!token.startsWith("<")) {
+			parent.append(new HtmlText(decodeEntities(token)));
+			continue;
+		}
+		const closing = /^<\/\s*([a-z][\w:-]*)[^>]*>$/i.exec(token);
+		if (closing) {
+			const name = closing[1]?.toUpperCase();
+			for (let index = stack.length - 1; index > 0; index--) {
+				if (stack[index]?.nodeName !== name) continue;
+				stack.length = index;
+				break;
+			}
+			continue;
+		}
+		const opening = /^<\s*([a-z][\w:-]*)([\s\S]*?)\/?\s*>$/i.exec(token);
+		if (!opening?.[1]) {
+			parent.append(new HtmlText("<"));
+			continue;
+		}
+		const element = new HtmlElement(opening[1], parseAttributes(opening[2] ?? ""));
+		parent.append(element);
+		if (!VOID_ELEMENTS[element.nodeName] && !/\/\s*>$/.test(token)) stack.push(element);
+	}
+	return root;
+}
+
+/** Serialize a standards-shaped or internal HTML node. */
+export function serializeNode(node: TurndownNode): string {
+	if (typeof node.outerHTML === "string") return node.outerHTML;
+	if (node.nodeType === 3) return node.textContent ?? "";
+	return Array.from(node.childNodes).map(serializeNode).join("");
+}