@linxiraos/pi-utils 1.1.8 → 1.1.9

package/src/json-parse.ts

@@ -1,11 +1,21 @@
-const QUOTE = 0x22;
-const BACKSLASH = 0x5c;
-const U = 0x75;
-const SQUOTE = 0x27;
+import {
+	BACKSLASH,
+	COLON,
+	COMMA,
+	isHexDigit,
+	isNumberStart,
+	isWhitespace,
+	JsonLexer,
+	LBRACE,
+	LBRACKET,
+	QUOTE,
+	RBRACE,
+	RBRACKET,
+	SQUOTE,
+	VALID_ESCAPE_CHAR,
+} from "./json-lexer";
 
-// Valid chars after `\`: " \ / b f n r t u
-const VALID_ESCAPE_CHAR = new Uint8Array(128);
-for (const ch of '"\\/bfnrtu') VALID_ESCAPE_CHAR[ch.charCodeAt(0)] = 1;
+const U = 0x75;
 
 const CONTROL_ESCAPES: readonly string[] = (() => {
 	const e: string[] = [];
@@ -20,47 +30,6 @@ const CONTROL_ESCAPES: readonly string[] = (() => {
 	return e;
 })();
 
-const HEX4_RE = /^[0-9a-fA-F]{4}$/;
-
-function isHexDigit(cp: number): boolean {
-	return (cp >= 0x30 && cp <= 0x39) || ((cp | 0x20) >= 0x61 && (cp | 0x20) <= 0x66);
-}
-
-function isWhitespace(cp: number): boolean {
-	return cp === 0x20 || cp === 0x09 || cp === 0x0a || cp === 0x0d;
-}
-
-function isIdentChar(cp: number): boolean {
-	return (
-		(cp >= 0x30 && cp <= 0x39) ||
-		((cp | 0x20) >= 0x61 && (cp | 0x20) <= 0x7a) ||
-		cp === 0x5f /* _ */ ||
-		cp === 0x24 /* $ */
-	);
-}
-
-/** Bareword literals: standard JSON plus Python `True`/`False`/`None`. */
-const KEYWORDS: readonly (readonly [string, unknown])[] = [
-	["true", true],
-	["false", false],
-	["null", null],
-	["True", true],
-	["False", false],
-	["None", null],
-];
-
-/**
- * JS-only atoms never recovered as bareword strings — a tool must not execute
- * with a non-finite or undefined argument masquerading as a string.
- */
-const NON_RECOVERABLE_BAREWORDS: Record<string, true> = {
-	NaN: true,
-	Infinity: true,
-	"-Infinity": true,
-	"+Infinity": true,
-	undefined: true,
-};
-
 /**
  * Sentinel returned by partial-mode value parsing when an atomic value
  * (number / keyword) is incomplete at the streaming edge, so the enclosing
@@ -160,139 +129,109 @@ export function repairJson(json: string): string {
 }
 
 /**
- * Recursive-descent parser for a forgiving superset of JSON. Beyond strict JSON
- * it accepts, and normalizes, the malformations LLM tool-call bodies leak in
- * practice:
- *
- * - single-quoted strings and unquoted object keys (JSON5);
- * - trailing / stray commas, and `//` + block comments;
- * - Python literals `True` / `False` / `None` and JS `NaN` / `Infinity`;
- * - raw control characters and invalid `\x` escapes inside strings (kept literally);
- * - unescaped quotes inside strings — a quote only closes a string when followed
- *   by a value terminator, recovering apostrophes such as `'it's'`;
- * - unquoted string values in object/array value position (strict mode only) —
- *   an unrecognized bareword such as `{"paths": packages/foo/*}` is recovered as
- *   a string up to the next `,` / `}` / `]` / newline.
+ * Recursive-descent tree builder over the tolerant {@link JsonLexer} grammar
+ * (single quotes, unquoted keys, comments, Python literals, stray commas,
+ * literal invalid escapes, apostrophe recovery, bareword values).
  *
- * In `partial` mode an unterminated string/object/array (or a value cut off at
- * end-of-input) is auto-closed with whatever was parsed so far — for streaming.
- * In strict mode, end-of-input mid-value and trailing garbage both throw, so a
- * final parse never silently accepts a half-formed tool call.
+ * In `partial` mode (lexer mode `streaming`) an unterminated string/object/array
+ * (or a value cut off at end-of-input) is auto-closed with whatever was parsed
+ * so far, and an incomplete trailing atom rolls the enclosing container back
+ * to its last valid prefix. In strict mode, end-of-input mid-value and trailing
+ * garbage both throw, so a final parse never silently accepts a half-formed
+ * tool call.
  */
 class RelaxedJson {
-	readonly #s: string;
-	readonly #n: number;
+	readonly #lex: JsonLexer;
 	readonly #partial: boolean;
-	#i = 0;
 
 	constructor(source: string, partial: boolean) {
-		this.#s = source;
-		this.#n = source.length;
+		this.#lex = new JsonLexer(source, partial ? "streaming" : "strict");
 		this.#partial = partial;
 	}
 
 	parse(): unknown {
-		this.#ws();
-		if (this.#i >= this.#n) {
+		const lex = this.#lex;
+		lex.ws();
+		if (lex.atEnd) {
 			if (this.#partial) return undefined;
 			throw new SyntaxError("Unexpected end of JSON input");
 		}
 		const value = this.#value(false);
 		if (value === INCOMPLETE) return undefined;
-		this.#ws();
-		if (!this.#partial && this.#i < this.#n) {
-			throw new SyntaxError(`Unexpected trailing characters at position ${this.#i}`);
+		lex.ws();
+		if (!this.#partial && !lex.atEnd) {
+			throw new SyntaxError(`Unexpected trailing characters at position ${lex.pos}`);
 		}
 		return value;
 	}
 
-	#ws(): void {
-		const s = this.#s;
-		for (;;) {
-			while (this.#i < this.#n && isWhitespace(s.charCodeAt(this.#i))) this.#i++;
-			if (this.#i + 1 < this.#n && s.charCodeAt(this.#i) === 0x2f /* / */) {
-				const next = s.charCodeAt(this.#i + 1);
-				if (next === 0x2f /* / line comment */) {
-					this.#i += 2;
-					while (this.#i < this.#n && s.charCodeAt(this.#i) !== 0x0a) this.#i++;
-					continue;
-				}
-				if (next === 0x2a /* * block comment */) {
-					this.#i += 2;
-					while (
-						this.#i + 1 < this.#n &&
-						!(s.charCodeAt(this.#i) === 0x2a && s.charCodeAt(this.#i + 1) === 0x2f)
-					) {
-						this.#i++;
-					}
-					this.#i = Math.min(this.#i + 2, this.#n);
-					continue;
-				}
-			}
-			break;
-		}
-	}
-
 	#value(allowBareword: boolean): unknown {
-		const s = this.#s;
-		const c = s[this.#i];
-		if (c === "{") return this.#object();
-		if (c === "[") return this.#array();
-		if (c === '"' || c === "'") return this.#string(s.charCodeAt(this.#i));
-		const cc = s.charCodeAt(this.#i);
-		if (cc === 0x2d /* - */ || cc === 0x2b /* + */ || cc === 0x2e /* . */ || (cc >= 0x30 && cc <= 0x39)) {
-			// JS-only NaN / Infinity are deliberately not accepted: a tool must not
-			// execute with a non-finite numeric arg; they fall through #number's
-			// NaN guard (strict throw / partial rollback) like other bad tokens.
-			return this.#number();
+		const lex = this.#lex;
+		const c = lex.peek();
+		if (c === LBRACE) return this.#object();
+		if (c === LBRACKET) return this.#array();
+		if (c === QUOTE || c === SQUOTE) return lex.string(c).value;
+		// JS-only NaN / Infinity are deliberately not accepted: a tool must not
+		// execute with a non-finite numeric arg; they fail the lexer's finite
+		// guard (strict throw / partial rollback) like other bad tokens.
+		if (isNumberStart(c)) return lex.number() ?? INCOMPLETE;
+		const keyword = lex.keyword();
+		if (keyword !== undefined) return keyword;
+		if (this.#partial) {
+			// Incomplete / unrecognized atomic token at the streaming edge — signal the
+			// caller to roll back to the last valid prefix instead of committing junk.
+			lex.pos = lex.src.length;
+			return INCOMPLETE;
 		}
-		return this.#keyword(allowBareword);
+		if (allowBareword) return lex.bareword();
+		throw new SyntaxError(`Unexpected token at position ${lex.pos}`);
 	}
 
 	#object(): Record<string, unknown> {
-		this.#i++; // consume {
+		const lex = this.#lex;
+		lex.pos++; // consume {
 		const out: Record<string, unknown> = {};
 		for (;;) {
-			this.#ws();
-			if (this.#i >= this.#n) {
+			lex.ws();
+			if (lex.atEnd) {
 				if (this.#partial) return out;
 				throw new SyntaxError("Unterminated object");
 			}
-			const c = this.#s[this.#i];
-			if (c === "}") {
-				this.#i++;
+			const c = lex.peek();
+			if (c === RBRACE) {
+				lex.pos++;
 				return out;
 			}
-			if (c === ",") {
+			if (c === COMMA) {
 				// Tolerate leading / doubled / trailing commas.
-				this.#i++;
+				lex.pos++;
 				continue;
 			}
 			const key = this.#key();
-			this.#ws();
-			if (this.#i < this.#n && this.#s[this.#i] === ":") {
-				this.#i++;
+			lex.ws();
+			if (lex.peek() === COLON) {
+				lex.pos++;
 			} else if (this.#partial) {
 				return out;
 			} else {
 				throw new SyntaxError("Expected ':' in object");
 			}
-			this.#ws();
-			if (this.#i >= this.#n) {
+			lex.ws();
+			if (lex.atEnd) {
 				if (this.#partial) return out;
 				throw new SyntaxError("Expected value after ':'");
 			}
 			const value = this.#value(true);
 			if (value === INCOMPLETE) return out;
 			out[key] = value;
-			this.#ws();
-			const d = this.#i < this.#n ? this.#s[this.#i] : "";
-			if (d === ",") {
-				this.#i++;
+			lex.ws();
+			const d = lex.peek();
+			if (d === COMMA) {
+				lex.pos++;
 				continue;
 			}
-			if (d === "}") {
-				this.#i++;
+			if (d === RBRACE) {
+				lex.pos++;
 				return out;
 			}
 			if (this.#partial) return out;
@@ -301,34 +240,35 @@ class RelaxedJson {
 	}
 
 	#array(): unknown[] {
-		this.#i++; // consume [
+		const lex = this.#lex;
+		lex.pos++; // consume [
 		const out: unknown[] = [];
 		for (;;) {
-			this.#ws();
-			if (this.#i >= this.#n) {
+			lex.ws();
+			if (lex.atEnd) {
 				if (this.#partial) return out;
 				throw new SyntaxError("Unterminated array");
 			}
-			const c = this.#s[this.#i];
-			if (c === "]") {
-				this.#i++;
+			const c = lex.peek();
+			if (c === RBRACKET) {
+				lex.pos++;
 				return out;
 			}
-			if (c === ",") {
-				this.#i++;
+			if (c === COMMA) {
+				lex.pos++;
 				continue;
 			}
 			const value = this.#value(true);
 			if (value === INCOMPLETE) return out;
 			out.push(value);
-			this.#ws();
-			const d = this.#i < this.#n ? this.#s[this.#i] : "";
-			if (d === ",") {
-				this.#i++;
+			lex.ws();
+			const d = lex.peek();
+			if (d === COMMA) {
+				lex.pos++;
 				continue;
 			}
-			if (d === "]") {
-				this.#i++;
+			if (d === RBRACKET) {
+				lex.pos++;
 				return out;
 			}
 			if (this.#partial) return out;
@@ -337,209 +277,12 @@ class RelaxedJson {
 	}
 
 	#key(): string {
-		const c = this.#s[this.#i];
-		if (c === '"' || c === "'") return this.#string(this.#s.charCodeAt(this.#i));
-		// Unquoted identifier key: read until a structural delimiter / whitespace.
-		const start = this.#i;
-		while (this.#i < this.#n) {
-			const ch = this.#s[this.#i];
-			if (ch === ":" || ch === "," || ch === "}" || isWhitespace(this.#s.charCodeAt(this.#i))) break;
-			this.#i++;
-		}
-		if (this.#i === start) {
-			if (this.#partial) return "";
-			throw new SyntaxError("Expected object key");
-		}
-		return this.#s.slice(start, this.#i);
-	}
-
-	#string(quote: number): string {
-		const s = this.#s;
-		const n = this.#n;
-		let i = this.#i + 1; // skip opening quote
-		let out = "";
-		let runStart = i;
-		while (i < n) {
-			const cc = s.charCodeAt(i);
-			if (cc !== BACKSLASH && cc !== quote) {
-				i++;
-				continue;
-			}
-			if (cc === quote) {
-				// Apostrophe / inner-quote recovery (a quote that isn't followed by a
-				// value terminator is literal) is safe for single quotes and in partial
-				// mode. For double quotes in strict mode, close on the first unescaped
-				// quote like standard JSON so malformed structure fails loudly instead
-				// of silently swallowing commas/colons into one string.
-				const lenient = quote === SQUOTE || this.#partial;
-				if (!lenient || this.#closesString(i + 1)) {
-					out += s.slice(runStart, i);
-					this.#i = i + 1;
-					return out;
-				}
-				// Unescaped inner quote (e.g. apostrophe in `'it's'`) — keep it literal.
-				i++;
-				continue;
-			}
-			// Backslash escape.
-			out += s.slice(runStart, i);
-			i++;
-			if (i >= n) {
-				out += "\\";
-				runStart = i;
-				break;
-			}
-			const esc = s.charCodeAt(i);
-			switch (esc) {
-				case QUOTE:
-					out += '"';
-					break;
-				case SQUOTE:
-					out += "'";
-					break;
-				case BACKSLASH:
-					out += "\\";
-					break;
-				case 0x2f:
-					out += "/";
-					break;
-				case 0x62:
-					out += "\b";
-					break;
-				case 0x66:
-					out += "\f";
-					break;
-				case 0x6e:
-					out += "\n";
-					break;
-				case 0x72:
-					out += "\r";
-					break;
-				case 0x74:
-					out += "\t";
-					break;
-				case U: {
-					const hex = s.slice(i + 1, i + 5);
-					if (HEX4_RE.test(hex)) {
-						out += String.fromCharCode(parseInt(hex, 16));
-						i += 4;
-					} else {
-						out += "\\u"; // invalid \u — keep literal
-					}
-					break;
-				}
-				default:
-					out += `\\${s[i]}`; // invalid escape — keep backslash literal
-			}
-			i++;
-			runStart = i;
-		}
-		out += s.slice(runStart, i);
-		if (this.#partial) {
-			this.#i = i;
-			return out;
-		}
-		throw new SyntaxError("Unterminated string");
-	}
-
-	/** A quote closes a string only when the next non-space char ends a value. */
-	#closesString(from: number): boolean {
-		const s = this.#s;
-		let k = from;
-		while (k < this.#n && isWhitespace(s.charCodeAt(k))) k++;
-		if (k >= this.#n) return true;
-		const c = s[k];
-		return c === "," || c === "}" || c === "]" || c === ":";
-	}
-
-	#number(): unknown {
-		const s = this.#s;
-		const start = this.#i;
-		while (this.#i < this.#n) {
-			const ch = s[this.#i];
-			if (
-				(ch >= "0" && ch <= "9") ||
-				ch === "-" ||
-				ch === "+" ||
-				ch === "." ||
-				ch === "e" ||
-				ch === "E" ||
-				ch === "x" ||
-				ch === "X" ||
-				(ch >= "a" && ch <= "f") ||
-				(ch >= "A" && ch <= "F")
-			) {
-				this.#i++;
-			} else {
-				break;
-			}
-		}
-		const token = s.slice(start, this.#i);
-		const num = Number(token);
-		if (Number.isNaN(num)) {
-			if (this.#partial) return INCOMPLETE;
-			throw new SyntaxError(`Invalid number: ${token}`);
-		}
-		return num;
-	}
-
-	#keyword(allowBareword: boolean): unknown {
-		const s = this.#s;
-		const i = this.#i;
-		for (const [word, value] of KEYWORDS) {
-			// Require a non-identifier boundary so `Truex` / `nullish` are not misread
-			// as the keyword followed by junk.
-			if (s.startsWith(word, i) && !isIdentChar(s.charCodeAt(i + word.length))) {
-				this.#i += word.length;
-				return value;
-			}
-		}
-		if (this.#partial) {
-			// Incomplete / unrecognized atomic token at the streaming edge — signal the
-			// caller to roll back to the last valid prefix instead of committing junk.
-			this.#i = this.#n;
-			return INCOMPLETE;
-		}
-		if (allowBareword) return this.#bareword();
-		throw new SyntaxError(`Unexpected token at position ${this.#i}`);
-	}
-
-	/**
-	 * Strict-mode recovery of an unquoted string value, e.g.
-	 * `{"paths": packages/foo/*}`: consume until `,` / `}` / `]` / newline and
-	 * trim trailing whitespace. Recovery still throws — so a final parse never
-	 * accepts a half-formed or non-finite argument — when the token:
-	 * - hits end-of-input before a delimiter (truncated value);
-	 * - contains a `"`, `{`, `[`, or a key-like `:` — this parser accepts
-	 *   unquoted keys, so a missed comma (`{"a": foo "b": 1}`, `{a: foo b: 1}`)
-	 *   would otherwise silently swallow the following field. A colon followed
-	 *   by `/` or `\` stays literal so URL and Windows-path values recover;
-	 * - is a non-finite atom ({@link NON_RECOVERABLE_BAREWORDS}).
-	 */
-	#bareword(): string {
-		const s = this.#s;
-		const start = this.#i;
-		let i = start;
-		while (i < this.#n) {
-			const cc = s.charCodeAt(i);
-			if (cc === 0x2c /* , */ || cc === 0x7d /* } */ || cc === 0x5d /* ] */ || cc === 0x0a || cc === 0x0d) break;
-			if (
-				cc === QUOTE ||
-				cc === 0x7b /* { */ ||
-				cc === 0x5b /* [ */ ||
-				(cc === 0x3a /* : */ && s.charCodeAt(i + 1) !== 0x2f /* / */ && s.charCodeAt(i + 1) !== 0x5c) /* \ */
-			) {
-				throw new SyntaxError(`Unexpected token at position ${start}`);
-			}
-			i++;
-		}
-		if (i >= this.#n) throw new SyntaxError(`Unexpected token at position ${start}`);
-		let end = i;
-		while (end > start && isWhitespace(s.charCodeAt(end - 1))) end--;
-		const word = s.slice(start, end);
-		if (NON_RECOVERABLE_BAREWORDS[word]) throw new SyntaxError(`Unexpected token at position ${start}`);
-		this.#i = i;
-		return word;
+		const lex = this.#lex;
+		const c = lex.peek();
+		if (c === QUOTE || c === SQUOTE) return lex.string(c).value;
+		const key = lex.unquotedKey();
+		if (key.length === 0 && !this.#partial) throw new SyntaxError("Expected object key");
+		return key;
 	}
 }
 

package/src/tls-fetch.ts

@@ -76,7 +76,7 @@ let cacheValue: string | undefined;
  *   path, matching Node's "extensionless filename is still a path" contract.
  *   `ENOENT` becomes {@link ExtraCaError}; other I/O errors bubble.
  */
-function resolveExtraCa(): string | undefined {
+export function resolveExtraCa(): string | undefined {
 	const raw = $env.NODE_EXTRA_CA_CERTS?.trim();
 	if (!raw) return undefined;
 
@@ -122,7 +122,7 @@ export function __resetExtraCaCache(): void {
  * Anthropic Foundry's mTLS options, which already seed
  * `tls.rootCertificates`), only the extra CA is appended.
  */
-function withExtraCaInit(init: RequestInit | undefined, extraCa: string): RequestInit {
+export function withExtraCaInit(init: RequestInit | undefined, extraCa: string): RequestInit {
 	const existingTls = (init as BunTlsRequestInit | undefined)?.tls;
 	const existingCa = existingTls?.ca;
 	let mergedCa: string[];
@@ -162,17 +162,3 @@ export function wrapFetchForExtraCa(fetchImpl: FetchImpl): FetchImpl {
 	);
 	return wrapped;
 }
-
-/**
- * Convenience for options-bag composition (e.g. the stream-entry path in
- * `@linxiraos/pi-ai`'s `stream.ts`, which mirrors `withRequestDebugFetch` so
- * the proxy/debug/extra-CA wrappers compose uniformly). No-op when the env
- * var is unset.
- */
-export function withExtraCaFetch<T extends { fetch?: FetchImpl } | undefined>(options: T): T {
-	if (!$env.NODE_EXTRA_CA_CERTS?.trim()) return options;
-	const fetchImpl = options?.fetch ?? (globalThis.fetch as FetchImpl);
-	const wrapped = wrapFetchForExtraCa(fetchImpl);
-	if (wrapped === fetchImpl && options?.fetch !== undefined) return options;
-	return { ...options, fetch: wrapped } as T;
-}