@oh-my-pi/pi-utils 18.1.22 → 18.2.1

package/src/stream.ts

@@ -4,6 +4,7 @@ import { abortableSource } from "./abortable";
 import { parseStreamingJson } from "./json-parse";
 
 const LF = 0x0a;
+const CR = 0x0d;
 
 export async function* readLines(stream: ReadableStream<Uint8Array>, signal?: AbortSignal): AsyncGenerator<Uint8Array> {
 	const buffer = new ConcatSink();
@@ -56,13 +57,19 @@ export async function* readJsonl<T>(stream: ReadableStream<Uint8Array>, signal?:
 	}
 }
 
-// =============================================================================
-// SSE (Server-Sent Events)
-// =============================================================================
-
-class ConcatSink {
+/**
+ * Amortized byte accumulator for chunked stream readers.
+ *
+ * Holds the unconsumed tail of a stream in a single growing `Buffer` so that
+ * appending N chunks costs O(total bytes) instead of re-copying the whole
+ * prefix per chunk. Backs {@link readLines}, {@link readJsonl} and
+ * {@link readSseEvents}; also usable directly when a reader needs its own
+ * framing loop (see `consume` and `flush`).
+ */
+export class ConcatSink {
 	#space?: Buffer;
 	#length = 0;
+	#skipLeadingLf = false;
 
 	#ensureCapacity(size: number): Buffer {
 		const space = this.#space;
@@ -100,11 +107,26 @@ class ConcatSink {
 		return this.#length === 0;
 	}
 
+	/**
+	 * The buffered bytes as a live view — invalidated by the next `append`,
+	 * `reset` or `consume`.
+	 */
 	flush(): Uint8Array | undefined {
 		if (!this.#length) return undefined;
 		return this.#space!.subarray(0, this.#length);
 	}
 
+	/** Drop the first `count` buffered bytes, keeping the remainder. */
+	consume(count: number) {
+		if (count <= 0) return;
+		if (count >= this.#length) {
+			this.#length = 0;
+			return;
+		}
+		this.#space!.copyWithin(0, count, this.#length);
+		this.#length -= count;
+	}
+
 	clear() {
 		this.#length = 0;
 	}
@@ -133,19 +155,27 @@ class ConcatSink {
 	}
 
 	appendAndFlushText(chunk: Uint8Array, decoder: TextDecoder): string | undefined {
-		const lastNewline = chunk.lastIndexOf(LF);
-		if (lastNewline === -1) {
-			this.append(chunk);
+		let start = 0;
+		if (this.#skipLeadingLf) {
+			if (chunk.length === 0) return undefined;
+			this.#skipLeadingLf = false;
+			if (chunk[0] === LF) start = 1;
+		}
+
+		const lastLineEnd = Math.max(chunk.lastIndexOf(LF), chunk.lastIndexOf(CR));
+		if (lastLineEnd < start) {
+			if (start < chunk.length) this.append(chunk.subarray(start));
 			return undefined;
 		}
 
-		const completeEnd = lastNewline + 1;
+		const completeEnd = lastLineEnd + 1;
+		this.#skipLeadingLf = chunk[lastLineEnd] === CR && completeEnd === chunk.length;
 		let text: string;
 		if (this.isEmpty) {
-			const complete = completeEnd === chunk.length ? chunk : chunk.subarray(0, completeEnd);
+			const complete = start === 0 && completeEnd === chunk.length ? chunk : chunk.subarray(start, completeEnd);
 			text = decoder.decode(complete);
 		} else {
-			this.append(completeEnd === chunk.length ? chunk : chunk.subarray(0, completeEnd));
+			this.append(chunk.subarray(start, completeEnd));
 			text = decoder.decode(this.flush());
 			this.clear();
 		}
@@ -197,6 +227,10 @@ class ConcatSink {
 	}
 }
 
+// =============================================================================
+// SSE (Server-Sent Events)
+// =============================================================================
+
 /**
  * Stream parsed JSON objects from SSE `data:` lines.
  *
@@ -236,11 +270,24 @@ function isRecoverableTrailingJson(data: string): boolean {
 	return typeof recovered === "object" && recovered !== null;
 }
 
-export async function* readSseJson<T>(
+/**
+ * One dispatched `data:` frame from {@link readSseFrames}: either the parsed JSON
+ * value, or the text of a frame `JSON.parse` rejected together with the
+ * `SyntaxError` it raised (so the strict reader can rethrow it unchanged).
+ */
+type SseFrame<T> = { ok: true; value: T } | { ok: false; raw: string; error: SyntaxError };
+
+/**
+ * Shared `data:`-line framing for {@link readSseJson} and
+ * {@link readSseJsonOrText}: skips empty events, stops at the OpenAI `[DONE]`
+ * sentinel, notifies the diagnostic observer, and treats a container-shaped
+ * stream tail as a clean end of iteration.
+ */
+async function* readSseFrames<T>(
 	stream: ReadableStream<Uint8Array>,
 	signal?: AbortSignal,
 	onEvent?: SseEventObserver,
-): AsyncGenerator<T> {
+): AsyncGenerator<SseFrame<T>> {
 	for await (const sse of readSseEvents(stream, signal)) {
 		const isTrailing = trailingEvents.has(sse);
 		notifySseEventObserver(onEvent, sse);
@@ -250,16 +297,60 @@ export async function* readSseJson<T>(
 			continue;
 		}
 		try {
-			yield JSON.parse(data) as T;
+			yield { ok: true, value: JSON.parse(data) as T };
 		} catch (err) {
 			if (err instanceof SyntaxError && isTrailing && isRecoverableTrailingJson(data)) {
 				return;
 			}
+			if (err instanceof SyntaxError) {
+				yield { ok: false, raw: data, error: err };
+				continue;
+			}
 			throw err;
 		}
 	}
 }
 
+export async function* readSseJson<T>(
+	stream: ReadableStream<Uint8Array>,
+	signal?: AbortSignal,
+	onEvent?: SseEventObserver,
+): AsyncGenerator<T> {
+	for await (const frame of readSseFrames<T>(stream, signal, onEvent)) {
+		if (!frame.ok) throw frame.error;
+		yield frame.value;
+	}
+}
+
+/**
+ * Like {@link readSseJson}, but a `data:` frame that is not valid JSON is yielded
+ * as its raw text instead of raising a `SyntaxError`. Cut-off container-shaped
+ * stream tails stay recoverable, exactly as they are in {@link readSseJson}.
+ *
+ * Consumers that only understand objects must treat a `string` yield as a
+ * transport-level failure (for example a `429 Too Many Requests` or an HTML
+ * throttle page from a reverse proxy that already committed to the stream). This
+ * exists because `readSseJson`'s baseline consumers span unrelated transports
+ * whose error handling a text yield would subtly change; new call sites opt in.
+ *
+ * Note that the text lane is only the frames `JSON.parse` *rejected*: a frame
+ * carrying a JSON-encoded string (`data: "429 Too Many Requests"`) parses, so it
+ * is yielded as that string and is indistinguishable from a rejected frame by
+ * type alone. Consumers branching on `typeof === "string"` therefore see both,
+ * which is the safe direction — each is classified as text rather than trusted as
+ * an event object.
+ */
+export async function* readSseJsonOrText<T>(
+	stream: ReadableStream<Uint8Array>,
+	signal?: AbortSignal,
+	onEvent?: SseEventObserver,
+): AsyncGenerator<T | string> {
+	for await (const frame of readSseFrames<T>(stream, signal, onEvent)) {
+		if (!frame.ok) yield frame.raw;
+		else yield frame.value;
+	}
+}
+
 /**
  * A single Server-Sent Event dispatched on a blank-line boundary.
  *
@@ -294,7 +385,7 @@ interface SseEventState {
 }
 
 // Complete lines are decoded in one batch per source chunk. Each batch ends on
-// LF, which cannot split a multi-byte UTF-8 sequence.
+// an ASCII line-ending byte, which cannot split a multi-byte UTF-8 sequence.
 const SSE_DECODER = new TextDecoder("utf-8");
 
 function flushSseEvent(state: SseEventState): ServerSentEvent | null {
@@ -318,11 +409,6 @@ function flushSseEvent(state: SseEventState): ServerSentEvent | null {
 }
 
 function pushSseLine(line: string, state: SseEventState): ServerSentEvent | null {
-	// Complete-line batches split on LF only; strip a trailing CR so CRLF sources
-	// don't leak `\r` into field values.
-	if (line.charCodeAt(line.length - 1) === 0x0d /* '\r' */) {
-		line = line.slice(0, -1);
-	}
 	if (line.length === 0) return flushSseEvent(state);
 
 	// Comment line: keep in `raw` for diagnostic context, skip parsing.
@@ -398,13 +484,21 @@ export async function* readSseEvents(
 			if (text === undefined) continue;
 			let start = 0;
 			while (start < text.length) {
-				const newline = text.indexOf("\n", start);
-				const event = pushSseLine(text.slice(start, newline), state);
+				let lineEnd = start;
+				while (lineEnd < text.length) {
+					const code = text.charCodeAt(lineEnd);
+					if (code === LF || code === CR) break;
+					lineEnd++;
+				}
+				const event = pushSseLine(text.slice(start, lineEnd), state);
 				if (event) yield event;
-				start = newline + 1;
+				if (text.charCodeAt(lineEnd) === CR && text.charCodeAt(lineEnd + 1) === LF) {
+					lineEnd++;
+				}
+				start = lineEnd + 1;
 			}
 		}
-		// Treat any trailing partial line (no terminating LF) as a complete line.
+		// Treat any trailing partial line (no terminating line ending) as complete.
 		if (!lineBuffer.isEmpty) {
 			const tail = lineBuffer.flush();
 			if (tail) {

package/src/which.ts

@@ -11,6 +11,7 @@
 import * as fs from "node:fs";
 import * as os from "node:os";
 import * as path from "node:path";
+import { isFullyQualifiedPath } from "./path";
 
 type CacheKey = string | bigint | number;
 
@@ -179,6 +180,12 @@ export interface WhichOptions extends Bun.WhichOptions {
 	 * Defaults to `WhichCachePolicy.Fresh`.
 	 */
 	cache?: WhichCachePolicy;
+	/**
+	 * Only search absolute directory entries in PATH, ignoring relative entries
+	 * (e.g. `.` or `./bin`) and empty components to prevent resolving against
+	 * an untrusted working directory.
+	 */
+	requireAbsolutePaths?: boolean;
 }
 
 // Darwin-specific "which" shim: consult Xcode/CLT toolchain directories after $PATH.
@@ -192,6 +199,15 @@ function darwinWhich(command: string, options?: Bun.WhichOptions): string | null
 	return null;
 }
 
+function filterAbsoluteSearchPath(rawPath: string | undefined): string | null {
+	if (!rawPath) return null;
+	const safePath = rawPath
+		.split(path.delimiter)
+		.filter(dir => dir.length > 0 && isFullyQualifiedPath(dir))
+		.join(path.delimiter);
+	return safePath || null;
+}
+
 // Which function that incorporates Darwin Xcode logic if platform reports as 'darwin'.
 // Look `Bun.which` up per call rather than capturing it at import, so a `Bun.which`
 // stub installed later (the per-test seam) is honoured on every platform.
@@ -219,8 +235,15 @@ function cacheKey(command: string, options?: Bun.WhichOptions): CacheKey {
  */
 export function $which(command: string, options?: WhichOptions): string | null {
 	const cachePolicy = options?.cache ?? WhichCachePolicy.Cached;
-	const lookupOptions =
+	let lookupOptions =
 		options?.PATH !== undefined || process.env.PATH === undefined ? options : { ...options, PATH: process.env.PATH };
+
+	if (options?.requireAbsolutePaths) {
+		const safePath = filterAbsoluteSearchPath(lookupOptions?.PATH);
+		if (!safePath) return null;
+		lookupOptions = { ...lookupOptions, PATH: safePath };
+	}
+
 	let key: CacheKey | undefined;
 
 	if (cachePolicy !== WhichCachePolicy.Bypass) {
@@ -232,6 +255,9 @@ export function $which(command: string, options?: WhichOptions): string | null {
 	}
 
 	const result = whichFresh(command, lookupOptions);
+	if (result && options?.requireAbsolutePaths && !isFullyQualifiedPath(result)) {
+		return null;
+	}
 	if (key != null && cachePolicy !== WhichCachePolicy.ReadOnly) {
 		toolCache.set(key, result);
 	}

package/src/xml.ts

@@ -104,6 +104,12 @@ class XmlReader {
 				this.#readProcessingInstruction(document);
 				continue;
 			}
+			if (this.#xml.startsWith("</", this.#position)) {
+				// Stray end tag with no element open: skip it instead of parsing it as a new element,
+				// whose empty name would leave #readElement stuck on the `/`.
+				this.#skipThrough(">");
+				continue;
+			}
 			if (this.#xml[this.#position] === "<") {
 				const element = this.#readElement("");
 				this.#addValue(document, element.name, element.name, element.value, element.leaf, null);
@@ -138,6 +144,11 @@ class XmlReader {
 				this.#position++;
 				this.#skipWhitespace();
 				value = this.#readAttributeValue();
+			} else if (attributeName === "") {
+				// Markup #readName cannot consume (the stray `/` in `<a / >`): skip the character
+				// so the loop always makes progress.
+				this.#position++;
+				continue;
 			}
 			attributes.push([attributeName, value]);
 		}
@@ -153,6 +164,11 @@ class XmlReader {
 					this.#position = close < 0 ? this.#xml.length : close + 1;
 					break;
 				}
+				if (this.#xml.startsWith("</", this.#position)) {
+					// End tag for an ancestor (an unclosed `<br>` inside `<p>`, or plain mismatched
+					// markup): close this element implicitly and leave the tag to its owner.
+					break;
+				}
 				if (this.#xml.startsWith("<![CDATA[", this.#position)) {
 					const end = this.#xml.indexOf("]]>", this.#position + 9);
 					const raw = this.#xml.slice(this.#position + 9, end < 0 ? this.#xml.length : end);

package/dist/types/glob.d.ts

@@ -1,28 +0,0 @@
-export interface GlobPathsOptions {
-    /** Base directory for glob patterns. Defaults to getProjectDir(). */
-    cwd?: string;
-    /** Glob exclusion patterns. */
-    exclude?: string[];
-    /** Abort signal to cancel the glob. */
-    signal?: AbortSignal;
-    /** Timeout in milliseconds for the glob operation. */
-    timeoutMs?: number;
-    /** Include dotfiles when true. */
-    dot?: boolean;
-    /** Only return files (skip directories). Default: true. */
-    onlyFiles?: boolean;
-    /** Respect .gitignore files when true. Walks up directory tree to find all applicable .gitignore files. */
-    gitignore?: boolean;
-}
-/**
- * Load .gitignore patterns from a directory and its parents.
- * Walks up the directory tree to find all applicable .gitignore files.
- * Returns glob-compatible exclude patterns.
- */
-export declare function loadGitignorePatterns(baseDir: string): Promise<string[]>;
-/**
- * Resolve filesystem paths matching glob patterns with optional exclude filters.
- * Returns paths relative to the provided cwd (or getProjectDir()).
- * Errors and abort/timeouts are surfaced to the caller.
- */
-export declare function globPaths(patterns: string | string[], options?: GlobPathsOptions): Promise<string[]>;

package/src/glob.ts

@@ -1,189 +0,0 @@
-import * as path from "node:path";
-import { Glob } from "bun";
-import { getProjectDir } from "./dirs";
-
-export interface GlobPathsOptions {
-	/** Base directory for glob patterns. Defaults to getProjectDir(). */
-	cwd?: string;
-	/** Glob exclusion patterns. */
-	exclude?: string[];
-	/** Abort signal to cancel the glob. */
-	signal?: AbortSignal;
-	/** Timeout in milliseconds for the glob operation. */
-	timeoutMs?: number;
-	/** Include dotfiles when true. */
-	dot?: boolean;
-	/** Only return files (skip directories). Default: true. */
-	onlyFiles?: boolean;
-	/** Respect .gitignore files when true. Walks up directory tree to find all applicable .gitignore files. */
-	gitignore?: boolean;
-}
-
-/** Patterns always excluded (.git is never useful in glob results). */
-const ALWAYS_IGNORED = ["**/.git", "**/.git/**"];
-
-/** node_modules exclusion patterns (skipped if pattern explicitly references node_modules). */
-const NODE_MODULES_IGNORED = ["**/node_modules", "**/node_modules/**"];
-
-/**
- * Parse a single .gitignore file and return glob-compatible exclude patterns.
- * @param content - Raw content of the .gitignore file
- * @param gitignoreDir - Absolute path to the directory containing the .gitignore
- * @param baseDir - Absolute path to the glob's cwd (for relativizing rooted patterns)
- */
-function parseGitignorePatterns(content: string, gitignoreDir: string, baseDir: string): string[] {
-	const patterns: string[] = [];
-
-	for (const rawLine of content.split("\n")) {
-		const line = rawLine.trim();
-		// Skip empty lines and comments
-		if (!line || line.startsWith("#")) {
-			continue;
-		}
-		// Skip negation patterns (unsupported for simple exclude)
-		if (line.startsWith("!")) {
-			continue;
-		}
-
-		let pattern = line;
-
-		// Handle trailing slash (directory-only match)
-		// For glob exclude, we treat it as matching the dir and its contents
-		const isDirectoryOnly = pattern.endsWith("/");
-		if (isDirectoryOnly) {
-			pattern = pattern.slice(0, -1);
-		}
-
-		// Handle rooted patterns (start with /)
-		if (pattern.startsWith("/")) {
-			// Rooted pattern: relative to the .gitignore location
-			const absolutePattern = path.join(gitignoreDir, pattern.slice(1));
-			const relativeToBase = path.relative(baseDir, absolutePattern);
-			if (relativeToBase.startsWith("..")) {
-				// Pattern is outside the search directory, skip
-				continue;
-			}
-			pattern = relativeToBase.replace(/\\/g, "/");
-			if (isDirectoryOnly) {
-				patterns.push(pattern);
-				patterns.push(`${pattern}/**`);
-			} else {
-				patterns.push(pattern);
-			}
-		} else {
-			// Unrooted pattern: match anywhere in the tree
-			if (pattern.includes("/")) {
-				// Contains slash: match from any directory level
-				patterns.push(`**/${pattern}`);
-				if (isDirectoryOnly) {
-					patterns.push(`**/${pattern}/**`);
-				}
-			} else {
-				// No slash: match file/dir name anywhere
-				patterns.push(`**/${pattern}`);
-				if (isDirectoryOnly) {
-					patterns.push(`**/${pattern}/**`);
-				}
-			}
-		}
-	}
-
-	return patterns;
-}
-
-/**
- * Load .gitignore patterns from a directory and its parents.
- * Walks up the directory tree to find all applicable .gitignore files.
- * Returns glob-compatible exclude patterns.
- */
-export async function loadGitignorePatterns(baseDir: string): Promise<string[]> {
-	const patterns: string[] = [];
-	const absoluteBase = path.resolve(baseDir);
-
-	let current = absoluteBase;
-	const maxDepth = 50; // Prevent infinite loops
-
-	for (let i = 0; i < maxDepth; i++) {
-		const gitignorePath = path.join(current, ".gitignore");
-
-		try {
-			const content = await Bun.file(gitignorePath).text();
-			const filePatterns = parseGitignorePatterns(content, current, absoluteBase);
-			patterns.push(...filePatterns);
-		} catch {
-			// .gitignore doesn't exist or can't be read, continue
-		}
-
-		const parent = path.dirname(current);
-		if (parent === current) {
-			// Reached filesystem root
-			break;
-		}
-		current = parent;
-	}
-
-	return patterns;
-}
-
-/**
- * Resolve filesystem paths matching glob patterns with optional exclude filters.
- * Returns paths relative to the provided cwd (or getProjectDir()).
- * Errors and abort/timeouts are surfaced to the caller.
- */
-export async function globPaths(patterns: string | string[], options: GlobPathsOptions = {}): Promise<string[]> {
-	const { cwd, exclude, signal, timeoutMs, dot, onlyFiles = true, gitignore } = options;
-
-	// Build exclude list: always exclude .git, exclude node_modules unless pattern references it
-	const patternArray = Array.isArray(patterns) ? patterns : [patterns];
-	const mentionsNodeModules = patternArray.some(p => p.includes("node_modules"));
-
-	const baseExclude = mentionsNodeModules ? [...ALWAYS_IGNORED] : [...ALWAYS_IGNORED, ...NODE_MODULES_IGNORED];
-	let effectiveExclude = exclude ? [...baseExclude, ...exclude] : baseExclude;
-
-	if (gitignore) {
-		const gitignorePatterns = await loadGitignorePatterns(cwd ?? getProjectDir());
-		effectiveExclude = [...effectiveExclude, ...gitignorePatterns];
-	}
-
-	const base = cwd ?? getProjectDir();
-	const allResults: string[] = [];
-
-	// Combine timeout and abort signals
-	const timeoutSignal = timeoutMs ? AbortSignal.timeout(timeoutMs) : undefined;
-	const combinedSignal =
-		signal && timeoutSignal ? AbortSignal.any([signal, timeoutSignal]) : (signal ?? timeoutSignal);
-
-	for (const pattern of patternArray) {
-		const glob = new Glob(pattern);
-		const scanOptions = {
-			cwd: base,
-			dot,
-			onlyFiles,
-			throwErrorOnBrokenSymlink: false,
-		};
-
-		for await (const entry of glob.scan(scanOptions)) {
-			if (combinedSignal?.aborted) {
-				const reason = combinedSignal.reason;
-				if (reason instanceof Error) throw reason;
-				throw new DOMException("Aborted", "AbortError");
-			}
-
-			// Check exclusion patterns
-			const normalized = entry.replace(/\\/g, "/");
-			let excluded = false;
-			for (const excludePattern of effectiveExclude) {
-				const excludeGlob = new Glob(excludePattern);
-				if (excludeGlob.match(normalized)) {
-					excluded = true;
-					break;
-				}
-			}
-			if (!excluded) {
-				allResults.push(normalized);
-			}
-		}
-	}
-
-	return allResults;
-}