@oh-my-pi/pi-utils 11.8.3 → 11.10.0

package/package.json

@@ -1,6 +1,6 @@
 {
 	"name": "@oh-my-pi/pi-utils",
-	"version": "11.8.3",
+	"version": "11.10.0",
 	"description": "Shared utilities for pi packages",
 	"type": "module",
 	"main": "./src/index.ts",

package/src/stream.ts

@@ -51,6 +51,118 @@ export function sanitizeText(text: string): string {
 }
 
 const LF = 0x0a;
+const CR = 0x0d;
+const decoder = new TextDecoder();
+
+type JsonlChunkResult = {
+	values: unknown[];
+	error: unknown;
+	read: number;
+	done: boolean;
+};
+
+function hasBunJsonlParseChunk(): boolean {
+	return typeof Bun !== "undefined" && typeof Bun.JSONL !== "undefined" && typeof Bun.JSONL.parseChunk === "function";
+}
+
+function parseJsonLine(lineBytes: Uint8Array): unknown {
+	let end = lineBytes.length;
+	if (end > 0 && lineBytes[end - 1] === CR) {
+		end--;
+	}
+	const text = decoder.decode(lineBytes.subarray(0, end)).trim();
+	if (text.length === 0) return undefined;
+	return JSON.parse(text);
+}
+
+function parseJsonlChunkFallbackBytes(bytes: Uint8Array, beg = 0, end = bytes.length): JsonlChunkResult {
+	const values: unknown[] = [];
+	let lineStart = beg;
+
+	for (let i = beg; i < end; i++) {
+		if (bytes[i] !== LF) continue;
+		const line = bytes.subarray(lineStart, i);
+		try {
+			const parsed = parseJsonLine(line);
+			if (parsed !== undefined) values.push(parsed);
+		} catch (error) {
+			return { values, error, read: lineStart, done: false };
+		}
+		lineStart = i + 1;
+	}
+
+	if (lineStart >= end) {
+		return { values, error: null, read: end, done: true };
+	}
+
+	const tail = bytes.subarray(lineStart, end);
+	const tailText = decoder.decode(tail).trim();
+	if (tailText.length === 0) {
+		return { values, error: null, read: end, done: true };
+	}
+	try {
+		values.push(JSON.parse(tailText));
+		return { values, error: null, read: end, done: true };
+	} catch {
+		// In streaming mode this is usually a partial line/object.
+		return { values, error: null, read: lineStart, done: false };
+	}
+}
+
+function parseJsonlChunkFallbackString(buffer: string): JsonlChunkResult {
+	const values: unknown[] = [];
+	let lineStart = 0;
+
+	for (let i = 0; i < buffer.length; i++) {
+		if (buffer.charCodeAt(i) !== LF) continue;
+		const rawLine = buffer.slice(lineStart, i);
+		const line = rawLine.endsWith("\r") ? rawLine.slice(0, -1) : rawLine;
+		const trimmed = line.trim();
+		if (trimmed.length > 0) {
+			try {
+				values.push(JSON.parse(trimmed));
+			} catch (error) {
+				return { values, error, read: lineStart, done: false };
+			}
+		}
+		lineStart = i + 1;
+	}
+
+	if (lineStart >= buffer.length) {
+		return { values, error: null, read: buffer.length, done: true };
+	}
+
+	const tail = buffer.slice(lineStart).trim();
+	if (tail.length === 0) {
+		return { values, error: null, read: buffer.length, done: true };
+	}
+	try {
+		values.push(JSON.parse(tail));
+		return { values, error: null, read: buffer.length, done: true };
+	} catch {
+		return { values, error: null, read: lineStart, done: false };
+	}
+}
+
+function parseJsonlChunkCompat(input: Uint8Array, beg?: number, end?: number): JsonlChunkResult;
+function parseJsonlChunkCompat(input: string): JsonlChunkResult;
+function parseJsonlChunkCompat(input: Uint8Array | string, beg?: number, end?: number): JsonlChunkResult {
+	if (hasBunJsonlParseChunk()) {
+		if (typeof input === "string") {
+			const { values, error, read, done } = Bun.JSONL.parseChunk(input);
+			return { values, error, read, done };
+		}
+		const start = beg ?? 0;
+		const stop = end ?? input.length;
+		const { values, error, read, done } = Bun.JSONL.parseChunk(input, start, stop);
+		return { values, error, read, done };
+	}
+
+	if (typeof input === "string") {
+		return parseJsonlChunkFallbackString(input);
+	}
+	return parseJsonlChunkFallbackBytes(input, beg, end);
+}
 
 export async function* readLines(stream: ReadableStream<Uint8Array>, signal?: AbortSignal): AsyncGenerator<Uint8Array> {
 	const buffer = new ConcatSink();
@@ -86,7 +198,7 @@ export async function* readJsonl<T>(stream: ReadableStream<Uint8Array>, signal?:
 			const tail = buffer.flush();
 			if (tail) {
 				buffer.clear();
-				const { values, error, done } = Bun.JSONL.parseChunk(tail, 0, tail.length);
+				const { values, error, done } = parseJsonlChunkCompat(tail, 0, tail.length);
 				if (values.length > 0) {
 					yield* values as T[];
 				}
@@ -205,7 +317,7 @@ class ConcatSink {
 	}
 	*pullJSONL<T>(chunk: Uint8Array, beg: number, end: number) {
 		if (this.isEmpty) {
-			const { values, error, read, done } = Bun.JSONL.parseChunk(chunk, beg, end);
+			const { values, error, read, done } = parseJsonlChunkCompat(chunk, beg, end);
 			if (values.length > 0) {
 				yield* values as T[];
 			}
@@ -222,7 +334,7 @@ class ConcatSink {
 		space.set(chunk.subarray(beg, end), offset);
 		this.#length = total;
 
-		const { values, error, read, done } = Bun.JSONL.parseChunk(space.subarray(0, total), 0, total);
+		const { values, error, read, done } = parseJsonlChunkCompat(space.subarray(0, total), 0, total);
 		if (values.length > 0) {
 			yield* values as T[];
 		}
@@ -324,7 +436,7 @@ export function parseJsonlLenient<T>(buffer: string): T[] {
 	let entries: T[] | undefined;
 
 	while (buffer.length > 0) {
-		const { values, error, read, done } = Bun.JSONL.parseChunk(buffer);
+		const { values, error, read, done } = parseJsonlChunkCompat(buffer);
 		if (values.length > 0) {
 			const ext = values as T[];
 			if (!entries) {