@oh-my-pi/pi-utils 14.9.9 → 15.0.1

package/package.json

@@ -1,7 +1,7 @@
 {
 	"type": "module",
 	"name": "@oh-my-pi/pi-utils",
-	"version": "14.9.9",
+	"version": "15.0.1",
 	"description": "Shared utilities for pi packages",
 	"homepage": "https://github.com/can1357/oh-my-pi",
 	"author": "Can Boluk",
@@ -37,11 +37,11 @@
 		"winston-daily-rotate-file": "^5.0.0"
 	},
 	"devDependencies": {
-		"@types/bun": "^1.3.13",
-		"@oh-my-pi/pi-natives": "14.9.9"
+		"@types/bun": "^1.3.14",
+		"@oh-my-pi/pi-natives": "15.0.1"
 	},
 	"engines": {
-		"bun": ">=1.3.7"
+		"bun": ">=1.3.14"
 	},
 	"files": [
 		"src"

package/src/abortable.ts

@@ -10,19 +10,6 @@ export class AbortError extends Error {
 	}
 }
 
-/**
- * Sleep for a given number of milliseconds, respecting abort signal.
- *
- * Uses setTimeout (not Bun.sleep) so that vitest fake timers can intercept it in tests.
- */
-export function abortableSleep(ms: number, signal?: AbortSignal): Promise<void> {
-	return untilAborted(signal, () => {
-		const { promise, resolve } = Promise.withResolvers<void>();
-		setTimeout(resolve, ms);
-		return promise;
-	});
-}
-
 /**
  * Creates an abortable stream from a given stream and signal.
  *

package/src/dirs.ts

@@ -324,6 +324,17 @@ export function getGpuCachePath(): string {
 	return dirs.rootSubdir("gpu_cache.json", "cache");
 }
 
+/**
+ * Get the GitHub view cache database path (~/.omp/cache/github-cache.db).
+ * Honors the `OMP_GITHUB_CACHE_DB` env var when set so tests can isolate the
+ * cache file without touching the rest of the config root.
+ */
+export function getGithubCacheDbPath(): string {
+	const override = process.env.OMP_GITHUB_CACHE_DB;
+	if (override) return override;
+	return dirs.rootSubdir(path.join("cache", "github-cache.db"), "cache");
+}
+
 /** Get the natives directory (~/.omp/natives). */
 export function getNativesDir(): string {
 	return dirs.rootSubdir("natives", "cache");

package/src/fetch-retry.ts

@@ -0,0 +1,290 @@
+import { scheduler } from "node:timers/promises";
+
+// "reset after 1h2m3s" / "10m15s" / "39s"
+const QUOTA_RESET_PATTERN = /reset after (?:(\d+)h)?(?:(\d+)m)?(\d+(?:\.\d+)?)s/i;
+// "Please retry in 250ms" / "Please retry in 12s"
+const PLEASE_RETRY_PATTERN = /Please retry in ([0-9.]+)(ms|s)/i;
+// JSON field: "retryDelay": "34.074824224s"
+const RETRY_DELAY_FIELD_PATTERN = /"retryDelay":\s*"([0-9.]+)(ms|s)"/i;
+// "try again in 250ms" / "try again in 12s" / "try again in 12sec"
+const TRY_AGAIN_PATTERN = /try again in\s+(\d+(?:\.\d+)?)\s*(ms|s)(?:ec)?/i;
+
+/**
+ * Server-suggested retry delay extraction. Merges the patterns historically used
+ * by the OpenAI Codex and Google Gemini retry helpers.
+ *
+ * Header sources (checked in order):
+ *  - `Retry-After` (numeric seconds, or HTTP date)
+ *  - `x-ratelimit-reset` (Unix epoch seconds)
+ *  - `x-ratelimit-reset-after` (seconds)
+ *
+ * Body patterns:
+ *  - `Your quota will reset after 18h31m10s` / `10m15s` / `39s`
+ *  - `Please retry in 250ms` / `Please retry in 12s`
+ *  - `"retryDelay": "34.074824224s"` (JSON error detail field)
+ *  - `try again in 250ms` / `try again in 12s` / `try again in 12sec`
+ *
+ * Returns `undefined` if no signal is found.
+ */
+export function extractRetryHint(source: Response | Headers | null | undefined, body?: string): number | undefined {
+	const headers = source instanceof Headers ? source : (source?.headers ?? undefined);
+	if (headers) {
+		const retryAfter = headers.get("retry-after");
+		if (retryAfter) {
+			const seconds = Number(retryAfter);
+			if (Number.isFinite(seconds)) return Math.max(0, seconds * 1000);
+			const parsedDate = Date.parse(retryAfter);
+			if (!Number.isNaN(parsedDate)) return Math.max(0, parsedDate - Date.now());
+		}
+		const rateLimitReset = headers.get("x-ratelimit-reset");
+		if (rateLimitReset) {
+			const resetSeconds = Number.parseInt(rateLimitReset, 10);
+			if (!Number.isNaN(resetSeconds)) {
+				const delta = resetSeconds * 1000 - Date.now();
+				if (delta > 0) return delta;
+			}
+		}
+		const rateLimitResetAfter = headers.get("x-ratelimit-reset-after");
+		if (rateLimitResetAfter) {
+			const seconds = Number(rateLimitResetAfter);
+			if (Number.isFinite(seconds) && seconds > 0) return seconds * 1000;
+		}
+	}
+
+	if (!body) return undefined;
+
+	const quotaMatch = QUOTA_RESET_PATTERN.exec(body);
+	if (quotaMatch) {
+		const hours = quotaMatch[1] ? Number.parseInt(quotaMatch[1], 10) : 0;
+		const minutes = quotaMatch[2] ? Number.parseInt(quotaMatch[2], 10) : 0;
+		const seconds = Number.parseFloat(quotaMatch[3]!);
+		if (!Number.isNaN(seconds)) {
+			const totalMs = ((hours * 60 + minutes) * 60 + seconds) * 1000;
+			if (totalMs > 0) return totalMs;
+		}
+	}
+	for (const pattern of [PLEASE_RETRY_PATTERN, RETRY_DELAY_FIELD_PATTERN, TRY_AGAIN_PATTERN]) {
+		const match = pattern.exec(body);
+		if (match?.[1]) {
+			const value = Number.parseFloat(match[1]);
+			if (Number.isFinite(value) && value > 0) {
+				return match[2]!.toLowerCase() === "ms" ? value : value * 1000;
+			}
+		}
+	}
+	return undefined;
+}
+
+export interface FetchWithRetryOptions extends RequestInit {
+	/** Total fetch attempts (initial + retries). Default `5`. */
+	maxAttempts?: number;
+	/**
+	 * Per-delay cap. Server-provided `Retry-After` hints exceeding this return
+	 * the current response immediately — caller deals with the `!response.ok`.
+	 * Default `60_000`.
+	 */
+	maxDelayMs?: number;
+	/**
+	 * Fallback delay schedule when no server hint is present. Number, array
+	 * (indexed by attempt, clamped to last), or function. Default exponential
+	 * `500ms * 2 ** attempt` capped at `maxDelayMs`.
+	 */
+	defaultDelayMs?: number | readonly number[] | ((attempt: number) => number);
+	/**
+	 * Optional per-attempt overlay merged into the base `RequestInit` each try.
+	 * Headers from the overlay shallow-merge over the base. Useful for auth
+	 * token refresh or user-agent rotation.
+	 */
+	prepareInit?: (attempt: number) => RequestInit | Promise<RequestInit>;
+}
+
+const DEFAULT_MAX_DELAY_MS = 60_000;
+const DEFAULT_MAX_ATTEMPTS = 5;
+
+/**
+ * Fetch with bounded retries and sensible defaults. Retries on any
+ * `isRetryableStatus` (5xx, 408, 429) and on transient network errors. Server
+ * `Retry-After`/quota hints are honoured up to `maxDelayMs`; a hint that exceeds
+ * the cap returns the current response so the caller can fail fast. Aborts on
+ * `init.signal` propagate as `"Request was aborted"`.
+ *
+ * The caller is responsible for inspecting `!response.ok` once the call returns.
+ */
+export async function fetchWithRetry(
+	url: string | URL | ((attempt: number) => string | URL),
+	options: FetchWithRetryOptions = {},
+): Promise<Response> {
+	const {
+		maxAttempts = DEFAULT_MAX_ATTEMPTS,
+		maxDelayMs = DEFAULT_MAX_DELAY_MS,
+		defaultDelayMs,
+		prepareInit,
+		...baseInit
+	} = options;
+	const signal = baseInit.signal as AbortSignal | undefined;
+
+	for (let attempt = 0; ; attempt++) {
+		if (signal?.aborted) throw new Error("Request was aborted");
+		const requestUrl = typeof url === "function" ? url(attempt) : url;
+		const init = prepareInit ? mergeInit(baseInit, await prepareInit(attempt)) : baseInit;
+
+		let response: Response;
+		try {
+			response = await fetch(requestUrl, init);
+		} catch (error) {
+			if (signal?.aborted) throw new Error("Request was aborted");
+			const wrapped = wrapNetworkError(error);
+			if (attempt + 1 >= maxAttempts) throw wrapped;
+			await scheduler.wait(resolveDefaultDelay(defaultDelayMs, attempt, maxDelayMs), { signal });
+			continue;
+		}
+
+		if (!isRetryableStatus(response.status)) return response;
+		if (attempt + 1 >= maxAttempts) return response;
+
+		const hint = extractRetryHint(response, await response.clone().text());
+		if (hint !== undefined && hint > maxDelayMs) return response;
+
+		const delayMs = Math.min(hint ?? resolveDefaultDelay(defaultDelayMs, attempt, maxDelayMs), maxDelayMs);
+		await scheduler.wait(delayMs, { signal });
+	}
+}
+
+function mergeInit(base: RequestInit, overlay: RequestInit): RequestInit {
+	const merged: RequestInit = { ...base, ...overlay };
+	if (base.headers || overlay.headers) {
+		const baseHeaders = new Headers(base.headers ?? undefined);
+		const overlayHeaders = new Headers(overlay.headers ?? undefined);
+		overlayHeaders.forEach((value, key) => {
+			baseHeaders.set(key, value);
+		});
+		merged.headers = baseHeaders;
+	}
+	return merged;
+}
+
+function wrapNetworkError(error: unknown): Error {
+	if (error instanceof Error) {
+		if (error.name === "AbortError" || error.message === "Request was aborted") {
+			return new Error("Request was aborted");
+		}
+		if (error.message === "fetch failed" && error.cause instanceof Error) {
+			return new Error(`Network error: ${error.cause.message}`);
+		}
+		return error;
+	}
+	return new Error(String(error));
+}
+
+function resolveDefaultDelay(
+	option: FetchWithRetryOptions["defaultDelayMs"],
+	attempt: number,
+	maxDelayMs: number,
+): number {
+	if (option === undefined) return Math.min(500 * 2 ** attempt, maxDelayMs);
+	if (typeof option === "number") return Math.min(option, maxDelayMs);
+	if (typeof option === "function") return Math.min(option(attempt), maxDelayMs);
+	return Math.min(option[Math.min(attempt, option.length - 1)] ?? 0, maxDelayMs);
+}
+
+/**
+ * Inspect an arbitrary error value (or its `cause` chain, up to depth 2) for an
+ * HTTP status code. Reads `status`, `statusCode`, and `response.status` fields,
+ * coerces string values, and falls back to scanning the error message for
+ * common patterns like `error (429)` or `HTTP 503`.
+ */
+export function extractHttpStatusFromError(error: unknown): number | undefined {
+	return extractHttpStatusFromErrorInternal(error, 0);
+}
+
+type HttpErrorLike = {
+	message?: string;
+	name?: string;
+	status?: number | string;
+	statusCode?: number | string;
+	response?: { status?: number | string };
+	cause?: unknown;
+};
+
+function extractHttpStatusFromErrorInternal(error: unknown, depth: number): number | undefined {
+	if (!error || typeof error !== "object" || depth > 2) return undefined;
+	const info = error as HttpErrorLike;
+	const rawStatus = info.status ?? info.statusCode ?? info.response?.status;
+
+	let status: number | undefined;
+	if (typeof rawStatus === "number" && Number.isFinite(rawStatus)) {
+		status = rawStatus;
+	} else if (typeof rawStatus === "string") {
+		const parsed = Number(rawStatus);
+		if (Number.isFinite(parsed)) status = parsed;
+	}
+	if (status !== undefined && status >= 100 && status <= 599) return status;
+
+	if (info.message) {
+		const extracted = extractStatusFromMessage(info.message);
+		if (extracted !== undefined) return extracted;
+	}
+	if (info.cause) return extractHttpStatusFromErrorInternal(info.cause, depth + 1);
+	return undefined;
+}
+
+const STATUS_MESSAGE_PATTERNS = [
+	/error\s*\((\d{3})\)/i,
+	/status\s*[:=]?\s*(\d{3})/i,
+	/\bhttp\s*(\d{3})\b/i,
+	/\b(\d{3})\s*(?:status|error)\b/i,
+] as const;
+
+function extractStatusFromMessage(message: string): number | undefined {
+	for (const pattern of STATUS_MESSAGE_PATTERNS) {
+		const match = pattern.exec(message);
+		if (!match) continue;
+		const value = Number(match[1]);
+		if (Number.isFinite(value) && value >= 100 && value <= 599) return value;
+	}
+	return undefined;
+}
+
+/**
+ * `true` if the given HTTP status code is one we treat as transient: 408
+ * (Request Timeout), 429 (Too Many Requests), or any 5xx (server error).
+ */
+export function isRetryableStatus(status: number): boolean {
+	return status >= 500 || status === 408 || status === 429;
+}
+
+/**
+ * `true` if the message describes an unexpected socket closure — Bun and some
+ * proxies surface these for any HTTP/2 stream reset.
+ */
+export function isUnexpectedSocketCloseMessage(message: string): boolean {
+	return /\b(?:the\s+)?socket connection (?:was )?closed unexpectedly\b/i.test(message);
+}
+
+const TRANSIENT_MESSAGE_PATTERN =
+	/overloaded|rate.?limit|too many requests|service.?unavailable|server error|internal error|connection.?error|unable to connect|fetch failed|network error|stream stall|other side closed/i;
+
+const VALIDATION_MESSAGE_PATTERN =
+	/invalid|validation|bad request|unsupported|schema|missing required|not found|unauthorized|forbidden/i;
+
+/**
+ * Identify errors that should be retried: aborts/timeouts in the error name or
+ * message, retryable HTTP statuses (see `isRetryableStatus`), unexpected socket
+ * closes, and the standard transient phrases. 4xx statuses other than 408/429
+ * and validation-shaped messages short-circuit to `false`.
+ */
+export function isRetryableError(error: unknown): boolean {
+	const info = error as { message?: string; name?: string } | null;
+	const message = info?.message ?? "";
+	const name = info?.name ?? "";
+	if (name === "AbortError" || /timeout|timed out|aborted/i.test(message)) return true;
+
+	const status = extractHttpStatusFromError(error);
+	if (status !== undefined) {
+		if (isRetryableStatus(status)) return true;
+		if (status >= 400 && status < 500) return false;
+	}
+
+	if (VALIDATION_MESSAGE_PATTERN.test(message)) return false;
+	return isUnexpectedSocketCloseMessage(message) || TRANSIENT_MESSAGE_PATTERN.test(message);
+}

package/src/index.ts

@@ -1,8 +1,9 @@
-export { abortableSleep, createAbortableStream, once, untilAborted } from "./abortable";
+export { createAbortableStream, once, untilAborted } from "./abortable";
 export * from "./async";
 export * from "./color";
 export * from "./dirs";
 export * from "./env";
+export * from "./fetch-retry";
 export * from "./format";
 export * from "./frontmatter";
 export * from "./fs-error";

package/src/prompt.ts

@@ -8,7 +8,7 @@ export type PromptRenderPhase = "pre-render" | "post-render";
 export interface PromptFormatOptions {
 	renderPhase?: PromptRenderPhase;
 	replaceAsciiSymbols?: boolean;
-	boldRfc2119Keywords?: boolean;
+	normalizeRfc2119?: boolean;
 }
 
 // Opening XML tag (not self-closing, not closing)
@@ -22,21 +22,27 @@ const TABLE_ROW = /^\|.*\|$/;
 // Table separator (|---|---|)
 const TABLE_SEP = /^\|[-:\s|]+\|$/;
 
-/** RFC 2119 keywords used in prompts. */
-const RFC2119_KEYWORDS = /\b(?:MUST NOT|SHOULD NOT|SHALL NOT|RECOMMENDED|REQUIRED|OPTIONAL|SHOULD|SHALL|MUST|MAY)\b/g;
-
-function boldRfc2119Keywords(line: string): string {
-	return line.replace(RFC2119_KEYWORDS, (match, offset, source) => {
-		const isAlreadyBold =
-			source[offset - 2] === "*" &&
-			source[offset - 1] === "*" &&
-			source[offset + match.length] === "*" &&
-			source[offset + match.length + 1] === "*";
-		if (isAlreadyBold) {
-			return match;
-		}
-		return `**${match}**`;
-	});
+/**
+ * RFC 2119 keywords (plus project aliases NEVER/AVOID) wrapped in markdown bold
+ * — `**MUST**`, `**MUST NOT**`, `**NEVER**`, etc.
+ */
+const RFC2119_BOLD = /\*\*(MUST NOT|SHOULD NOT|RECOMMENDED|REQUIRED|OPTIONAL|SHOULD|MUST|MAY|NEVER|AVOID)\*\*/g;
+
+/**
+ * Normalize RFC 2119 markers per project convention:
+ *   - Strip `**KEYWORD**` bold (visual noise, no semantics).
+ *   - Alias `MUST NOT` → `NEVER` and `SHOULD NOT` → `AVOID` (single-token equivalents).
+ * Skips spans inside inline code (`` `…` ``) so alias definitions can be quoted literally.
+ */
+function normalizeRfc2119(line: string): string {
+	const segments = line.split("`");
+	for (let i = 0; i < segments.length; i += 2) {
+		segments[i] = segments[i]
+			.replace(RFC2119_BOLD, "$1")
+			.replace(/\bMUST NOT\b/g, "NEVER")
+			.replace(/\bSHOULD NOT\b/g, "AVOID");
+	}
+	return segments.join("`");
 }
 
 /** Compact a table row by trimming cell padding */
@@ -60,6 +66,13 @@ function compactTableSep(line: string): string {
 	return `|${normalized.join("|")}|`;
 }
 
+const HTML_COMMENT_OPEN = "<!--";
+const HTML_COMMENT_CLOSE = "-->";
+
+type HtmlCommentState = {
+	inHtmlComment: boolean;
+};
+
 function replaceCommonAsciiSymbols(line: string): string {
 	return line
 		.replace(/\.{3}/g, "…")
@@ -71,16 +84,59 @@ function replaceCommonAsciiSymbols(line: string): string {
 		.replace(/>=/g, "≥");
 }
 
+function replaceCommonAsciiSymbolsOutsideHtmlComments(line: string, state: HtmlCommentState): string {
+	if (!state.inHtmlComment && !line.includes(HTML_COMMENT_OPEN) && !line.includes(HTML_COMMENT_CLOSE)) {
+		return replaceCommonAsciiSymbols(line);
+	}
+
+	let result = "";
+	let cursor = 0;
+
+	while (cursor < line.length) {
+		if (state.inHtmlComment) {
+			const closeIndex = line.indexOf(HTML_COMMENT_CLOSE, cursor);
+			if (closeIndex === -1) {
+				return result + line.slice(cursor);
+			}
+			result += line.slice(cursor, closeIndex + HTML_COMMENT_CLOSE.length);
+			cursor = closeIndex + HTML_COMMENT_CLOSE.length;
+			state.inHtmlComment = false;
+			continue;
+		}
+
+		const openIndex = line.indexOf(HTML_COMMENT_OPEN, cursor);
+		if (openIndex === -1) {
+			result += replaceCommonAsciiSymbols(line.slice(cursor));
+			return result;
+		}
+
+		result += replaceCommonAsciiSymbols(line.slice(cursor, openIndex));
+		const closeIndex = line.indexOf(HTML_COMMENT_CLOSE, openIndex + HTML_COMMENT_OPEN.length);
+		if (closeIndex === -1) {
+			result += line.slice(openIndex);
+			state.inHtmlComment = true;
+			return result;
+		}
+
+		result += line.slice(openIndex, closeIndex + HTML_COMMENT_CLOSE.length);
+		cursor = closeIndex + HTML_COMMENT_CLOSE.length;
+	}
+
+	return result;
+}
+
 export function format(content: string, options: PromptFormatOptions = {}): string {
 	const {
 		renderPhase = "post-render",
 		replaceAsciiSymbols = false,
-		boldRfc2119Keywords: shouldBoldRfc2119 = false,
+		normalizeRfc2119: shouldNormalizeRfc2119 = false,
 	} = options;
 	const isPreRender = renderPhase === "pre-render";
 	const lines = content.split("\n");
 	const result: string[] = [];
 	let inCodeBlock = false;
+
+	const htmlCommentState: HtmlCommentState = { inHtmlComment: false };
 	const topLevelTags: string[] = [];
 
 	for (let i = 0; i < lines.length; i++) {
@@ -98,7 +154,7 @@ export function format(content: string, options: PromptFormatOptions = {}): stri
 		}
 
 		if (replaceAsciiSymbols) {
-			line = replaceCommonAsciiSymbols(line);
+			line = replaceCommonAsciiSymbolsOutsideHtmlComments(line, htmlCommentState);
 		}
 		trimmedStart = line.trimStart();
 		const trimmed = line.trim();
@@ -125,8 +181,8 @@ export function format(content: string, options: PromptFormatOptions = {}): stri
 			line = `${leadingWhitespace}${compactTableRow(trimmedStart)}`;
 		}
 
-		if (shouldBoldRfc2119) {
-			line = boldRfc2119Keywords(line);
+		if (shouldNormalizeRfc2119) {
+			line = normalizeRfc2119(line);
 		}
 
 		if (trimmed === "") {