@asterxsk/kiln 0.1.0

package/agent/extensions/pi-web-access/data-uri-sanitize.ts

@@ -0,0 +1,406 @@
+import { Buffer } from "node:buffer";
+import { createHash } from "node:crypto";
+
+// Replaces inline RFC 2397 data: URIs in extracted text with explicit bounded
+// omission markers so base64 payloads never reach model-visible tool results,
+// the fetch cache, or session persistence. Carved out of the original
+// persistence-safety layer; only the text-sanitization path is retained.
+
+const MAX_DATA_URI_HEADER_CHARS = 1024;
+const MAX_MIME_CHARS = 127;
+const MAX_MARKER_SOURCE_CHARS = 180;
+
+export type DataUriEncoding = "base64" | "percent-encoded";
+export type DigestBasis = "decoded" | "encoded";
+
+export interface DataUriOmission {
+	ordinal: number;
+	sourcePath: string;
+	mimeType: string;
+	encoding: DataUriEncoding;
+	encodedBytes: number;
+	decodedBytes: number | null;
+	decodeError?: string;
+	sha256: string;
+	digestBasis: DigestBasis;
+	retrieval: "not-retained";
+}
+
+
+interface InternalDataUriOmission extends DataUriOmission {
+	markerStart: number;
+	markerEnd: number;
+}
+
+interface SanitizationState {
+	nextOrdinal: number;
+	omissions: InternalDataUriOmission[];
+}
+
+
+function sha256Bytes(value: string | Uint8Array): string {
+	return createHash("sha256").update(value).digest("hex");
+}
+
+
+function asciiLowerCode(code: number): number {
+	return code >= 65 && code <= 90 ? code + 32 : code;
+}
+
+function matchesDataScheme(text: string, index: number): boolean {
+	return index + 5 <= text.length
+		&& asciiLowerCode(text.charCodeAt(index)) === 100
+		&& asciiLowerCode(text.charCodeAt(index + 1)) === 97
+		&& asciiLowerCode(text.charCodeAt(index + 2)) === 116
+		&& asciiLowerCode(text.charCodeAt(index + 3)) === 97
+		&& text.charCodeAt(index + 4) === 58;
+}
+
+function isSchemeBoundary(text: string, index: number): boolean {
+	if (index === 0) return true;
+	const code = text.charCodeAt(index - 1);
+	const alphaNumeric = (code >= 48 && code <= 57)
+		|| (code >= 65 && code <= 90)
+		|| (code >= 97 && code <= 122);
+	return !alphaNumeric && code !== 43 && code !== 45 && code !== 46;
+}
+
+function findNextDataScheme(text: string, from: number): number {
+	for (let i = from; i + 5 <= text.length; i++) {
+		if (matchesDataScheme(text, i) && isSchemeBoundary(text, i)) return i;
+	}
+	return -1;
+}
+
+type DataUriEnclosure =
+	| { kind: "quote"; close: "\"" | "'" | "`" }
+	| { kind: "parentheses" }
+	| { kind: "angle" }
+	| null;
+
+interface ScannedDataUriCandidate {
+	end: number;
+	comma: number;
+	enclosure: DataUriEnclosure;
+}
+
+function dataUriEnclosure(text: string, start: number): DataUriEnclosure {
+	if (start === 0) return null;
+	const immediateIndex = start - 1;
+	const immediate = text[immediateIndex];
+	if (immediate === "\"" || immediate === "'" || immediate === "`") {
+		const beforeQuote = immediateIndex > 0 ? text[immediateIndex - 1] : "";
+		const likelyOpeningQuote = immediateIndex === 0
+			|| beforeQuote === "="
+			|| beforeQuote === "("
+			|| beforeQuote === "["
+			|| beforeQuote === "{"
+			|| beforeQuote === ":"
+			|| beforeQuote === ","
+			|| /\s/.test(beforeQuote);
+		if (likelyOpeningQuote) return { kind: "quote", close: immediate };
+	}
+
+	let openerIndex = immediateIndex;
+	while (openerIndex >= 0 && (text[openerIndex] === " " || text[openerIndex] === "\t")) openerIndex--;
+	const opener = text[openerIndex];
+	if (opener === "(") return { kind: "parentheses" };
+	if (opener === "<") return { kind: "angle" };
+	return null;
+}
+
+function isBareDataUriTerminator(ch: string): boolean {
+	const code = ch.charCodeAt(0);
+	return code <= 32
+		|| code === 127
+		|| ch === "\""
+		|| ch === "`"
+		|| ch === "<"
+		|| ch === ">";
+}
+
+/** Scan one candidate exactly once. Enclosures consume malformed whitespace and
+ * balanced inner parentheses so invalid payload suffixes cannot escape. */
+function scanDataUriCandidate(text: string, start: number): ScannedDataUriCandidate {
+	const enclosure = dataUriEnclosure(text, start);
+	let comma = -1;
+	let depth = enclosure?.kind === "parentheses" ? 1 : 0;
+	let index = start + 5;
+
+	for (; index < text.length; index++) {
+		const ch = text[index];
+		if (enclosure?.kind === "quote") {
+			if (ch === enclosure.close) break;
+		} else if (enclosure?.kind === "parentheses") {
+			if (ch === "(") {
+				depth++;
+			} else if (ch === ")") {
+				depth--;
+				if (depth === 0) break;
+			}
+		} else if (enclosure?.kind === "angle") {
+			if (ch === ">") break;
+		} else if (isBareDataUriTerminator(ch)) {
+			break;
+		}
+		if (comma < 0 && ch === ",") comma = index;
+	}
+
+	return { end: index, comma, enclosure };
+}
+
+function headerEndsWithBase64(text: string, headerStart: number, comma: number): boolean {
+	let end = comma;
+	while (end > headerStart && (text[end - 1] === " " || text[end - 1] === "\t")) end--;
+	const token = "base64";
+	if (end - headerStart < token.length + 1) return false;
+	const tokenStart = end - token.length;
+	for (let i = 0; i < token.length; i++) {
+		if (asciiLowerCode(text.charCodeAt(tokenStart + i)) !== token.charCodeAt(i)) return false;
+	}
+	return tokenStart > headerStart && text[tokenStart - 1] === ";";
+}
+
+function isMimeTokenChar(code: number): boolean {
+	if (code < 33 || code > 126) return false;
+	// RFC 2045 token = printable ASCII excluding tspecials. Slash is handled
+	// separately as the single type/subtype separator.
+	switch (code) {
+		case 34: // "
+		case 40: // (
+		case 41: // )
+		case 44: // ,
+		case 47: // /
+		case 58: // :
+		case 59: // ;
+		case 60: // <
+		case 61: // =
+		case 62: // >
+		case 63: // ?
+		case 64: // @
+		case 91: // [
+		case 92: // backslash
+		case 93: // ]
+			return false;
+		default:
+			return true;
+	}
+}
+
+function normalizeMimeType(text: string, headerStart: number, comma: number): string {
+	let end = headerStart;
+	while (end < comma && text[end] !== ";") end++;
+	if (end === headerStart) return "text/plain";
+	if (end - headerStart > MAX_MIME_CHARS) return "application/octet-stream";
+
+	let slashCount = 0;
+	for (let i = headerStart; i < end; i++) {
+		const code = text.charCodeAt(i);
+		if (code === 47) {
+			slashCount++;
+			if (i === headerStart || i === end - 1) return "application/octet-stream";
+			continue;
+		}
+		if (!isMimeTokenChar(code)) return "application/octet-stream";
+	}
+	if (slashCount !== 1) return "application/octet-stream";
+	return text.slice(headerStart, end).toLowerCase();
+}
+
+function hexValue(code: number): number {
+	if (code >= 48 && code <= 57) return code - 48;
+	if (code >= 65 && code <= 70) return code - 55;
+	if (code >= 97 && code <= 102) return code - 87;
+	return -1;
+}
+
+function decodePercentEncoded(payload: string): { bytes: Buffer } | { error: string } {
+	const output = Buffer.allocUnsafe(Buffer.byteLength(payload, "utf8"));
+	let outputOffset = 0;
+	let cursor = 0;
+
+	while (cursor < payload.length) {
+		const percent = payload.indexOf("%", cursor);
+		const runEnd = percent < 0 ? payload.length : percent;
+		if (runEnd > cursor) {
+			const run = Buffer.from(payload.slice(cursor, runEnd), "utf8");
+			run.copy(output, outputOffset);
+			outputOffset += run.length;
+		}
+		if (percent < 0) break;
+		if (percent + 2 >= payload.length) return { error: "invalid-percent-escape" };
+		const high = hexValue(payload.charCodeAt(percent + 1));
+		const low = hexValue(payload.charCodeAt(percent + 2));
+		if (high < 0 || low < 0) return { error: "invalid-percent-escape" };
+		output[outputOffset++] = (high << 4) | low;
+		cursor = percent + 3;
+	}
+
+	return { bytes: output.subarray(0, outputOffset) };
+}
+
+function isBase64AlphabetByte(byte: number): boolean {
+	return (byte >= 65 && byte <= 90)
+		|| (byte >= 97 && byte <= 122)
+		|| (byte >= 48 && byte <= 57)
+		|| byte === 43
+		|| byte === 47;
+}
+
+function decodeBase64Payload(payload: string): { bytes: Buffer } | { error: string } {
+	const percentDecoded = decodePercentEncoded(payload);
+	if ("error" in percentDecoded) return percentDecoded;
+	const encoded = percentDecoded.bytes;
+	let paddingStart = encoded.length;
+	let paddingCount = 0;
+
+	for (let i = 0; i < encoded.length; i++) {
+		const byte = encoded[i];
+		if (byte > 127) return { error: "non-ascii-base64" };
+		if (byte === 61) {
+			if (paddingStart === encoded.length) paddingStart = i;
+			paddingCount++;
+			continue;
+		}
+		if (paddingStart !== encoded.length) return { error: "invalid-base64-padding" };
+		if (!isBase64AlphabetByte(byte)) return { error: "invalid-base64-character" };
+	}
+
+	if (paddingCount > 2) return { error: "invalid-base64-padding" };
+	if (paddingCount > 0 && encoded.length % 4 !== 0) return { error: "invalid-base64-padding" };
+	if (paddingCount === 0 && encoded.length % 4 === 1) return { error: "invalid-base64-length" };
+	if (paddingCount === 1 && paddingStart % 4 !== 3) return { error: "invalid-base64-padding" };
+	if (paddingCount === 2 && paddingStart % 4 !== 2) return { error: "invalid-base64-padding" };
+
+	return { bytes: Buffer.from(encoded.toString("ascii"), "base64") };
+}
+
+function safeMarkerSource(sourcePath: string): string {
+	const source = sourcePath || "value";
+	let safe = "";
+	for (const ch of source) {
+		const code = ch.charCodeAt(0);
+		const allowed = (code >= 48 && code <= 57)
+			|| (code >= 65 && code <= 90)
+			|| (code >= 97 && code <= 122)
+			|| ch === "."
+			|| ch === "_"
+			|| ch === "-"
+			|| ch === "["
+			|| ch === "]"
+			|| ch === "*"
+			|| ch === "$";
+		safe += allowed ? ch : "_";
+	}
+	if (safe.length <= MAX_MARKER_SOURCE_CHARS) return safe;
+	const suffix = sha256Bytes(source).slice(0, 12);
+	return `${safe.slice(0, MAX_MARKER_SOURCE_CHARS - suffix.length - 1)}~${suffix}`;
+}
+
+function buildDataUriMarker(omission: DataUriOmission): string {
+	const decoded = omission.decodedBytes === null ? "unknown" : String(omission.decodedBytes);
+	const decodeError = omission.decodeError ? `; decodeError=${omission.decodeError}` : "";
+	return `[pi-web-access inline data URI omitted; ordinal=${omission.ordinal}; source=${omission.sourcePath}; mime=${omission.mimeType}; encoding=${omission.encoding}; encodedBytes=${omission.encodedBytes}; decodedBytes=${decoded}${decodeError}; sha256=${omission.sha256}; digestBasis=${omission.digestBasis}; retrieval=not-retained]`;
+}
+
+function sanitizeTextInternal(text: string, sourcePath: string, state: SanitizationState): string {
+	let scanFrom = 0;
+	let retainedFrom = 0;
+	let outputLength = 0;
+	const pieces: string[] = [];
+
+	while (scanFrom < text.length) {
+		const start = findNextDataScheme(text, scanFrom);
+		if (start < 0) break;
+		const candidate = scanDataUriCandidate(text, start);
+		const end = candidate.end;
+		const headerStart = start + 5;
+		const missingComma = candidate.comma < 0;
+		// A bare prose label such as "data: value" is not an inline URI. Enclosed
+		// or non-empty comma-less candidates are malformed URIs and are removed.
+		if (missingComma && end === headerStart && candidate.enclosure === null) {
+			scanFrom = end;
+			continue;
+		}
+		const headerEnd = missingComma ? end : candidate.comma;
+		const headerLength = headerEnd - headerStart;
+		const encoding: DataUriEncoding = headerEndsWithBase64(text, headerStart, headerEnd)
+			? "base64"
+			: "percent-encoded";
+		const mimeType = normalizeMimeType(text, headerStart, headerEnd);
+		// Without the required comma, the post-scheme bytes are ambiguous. Count
+		// and digest the whole omitted segment on the encoded basis rather than
+		// retaining any header/payload-looking suffix.
+		const payload = missingComma
+			? text.slice(headerStart, end)
+			: text.slice(candidate.comma + 1, end);
+		const encodedBytes = Buffer.byteLength(payload, "utf8");
+
+		let decodedBytes: number | null = null;
+		let decodeError: string | undefined;
+		let digestBasis: DigestBasis = "encoded";
+		let digest = "";
+
+		if (missingComma) {
+			decodeError = "missing-comma";
+			digest = sha256Bytes(payload);
+		} else if (headerLength > MAX_DATA_URI_HEADER_CHARS) {
+			decodeError = "header-too-long";
+			digest = sha256Bytes(payload);
+		} else {
+			const decoded = encoding === "base64"
+				? decodeBase64Payload(payload)
+				: decodePercentEncoded(payload);
+			if ("error" in decoded) {
+				decodeError = decoded.error;
+				digest = sha256Bytes(payload);
+			} else {
+				decodedBytes = decoded.bytes.length;
+				digestBasis = "decoded";
+				digest = sha256Bytes(decoded.bytes);
+			}
+		}
+
+		const omission: DataUriOmission = {
+			ordinal: state.nextOrdinal++,
+			sourcePath: safeMarkerSource(sourcePath),
+			mimeType,
+			encoding,
+			encodedBytes,
+			decodedBytes,
+			...(decodeError ? { decodeError } : {}),
+			sha256: digest,
+			digestBasis,
+			retrieval: "not-retained",
+		};
+		const marker = buildDataUriMarker(omission);
+		const prefix = text.slice(retainedFrom, start);
+		pieces.push(prefix, marker);
+		outputLength += prefix.length;
+		const markerStart = outputLength;
+		outputLength += marker.length;
+		state.omissions.push({ ...omission, markerStart, markerEnd: outputLength });
+		retainedFrom = end;
+		scanFrom = end;
+	}
+
+	if (state.omissions.length === 0 && retainedFrom === 0) return text;
+	pieces.push(text.slice(retainedFrom));
+	return pieces.join("");
+}
+
+
+/** Replace every recognized RFC 2397-style inline data URI in one string. */
+export function sanitizeInlineDataUris(
+	text: string,
+	sourcePath: string,
+): { text: string; omissions: DataUriOmission[] } {
+	const state: SanitizationState = { nextOrdinal: 1, omissions: [] };
+	const sanitized = sanitizeTextInternal(text, sourcePath, state);
+	return {
+		text: sanitized,
+		omissions: state.omissions.map(({ markerStart: _start, markerEnd: _end, ...omission }) => omission),
+	};
+}
+
+/** Sanitize every string leaf in a JSON-compatible value with one ordinal sequence. */