@f5-sales-demo/pi-utils 19.51.2

package/src/snowflake.ts

@@ -0,0 +1,136 @@
+// 16-bit hex lookup table (65536 entries) for fast conversion
+const HEX4 = Array.from({ length: 65536 }, (_, i) => i.toString(16).padStart(4, "0"));
+
+function randu32() {
+	return crypto.getRandomValues(new Uint32Array(1))[0];
+}
+
+const EPOCH = 1420070400000;
+const MAX_SEQ = 0x3fffff;
+
+// Snowflake as a hex string (16 chars, zero-padded).
+//
+// Since this is not distributed (no machine ID needed), we use an extended
+// 22-bit sequence instead of the standard 10-bit machine ID + 12-bit sequence.
+//
+type Snowflake = string & { readonly __brand: unique symbol };
+
+namespace Snowflake {
+	// Hex string validation pattern (16 lowercase hex chars).
+	//
+	export const PATTERN = /^[0-9a-f]{16}$/;
+
+	// Epoch timestamp.
+	//
+	export const EPOCH_TIMESTAMP = EPOCH;
+
+	// Maximum sequence number.
+	//
+	export const MAX_SEQUENCE = MAX_SEQ;
+
+	// Parses a hex string or bigint to bigint.
+	//
+	function toBigInt(value: Snowflake): bigint {
+		const hi = Number.parseInt(value.substring(0, 8), 16);
+		const lo = Number.parseInt(value.substring(8, 16), 16);
+		return (BigInt(hi) << 32n) | BigInt(lo);
+	}
+
+	// Formats a sequence and timestamp into a snowflake hex string.
+	//
+	export function formatParts(dt: number, seq: number): Snowflake {
+		// Split dt into hi/lo to avoid exceeding Number.MAX_SAFE_INTEGER.
+		// dt is ~39 bits; dt<<22 would be ~61 bits, so we split at bit 10:
+		//   lo32 = (dtLo << 22) | seq   (10+22 = 32 bits, no overlap)
+		//   hi32 = dtHi                 (~29 bits)
+		const dtLo = dt % 1024;
+		const hi = (dt - dtLo) / 1024; // dt >>> 10
+		const lo = ((dtLo << 22) | seq) >>> 0;
+		const hi1 = (hi >>> 16) & 0xffff;
+		const hi2 = hi & 0xffff;
+		const lo1 = (lo >>> 16) & 0xffff;
+		const lo2 = lo & 0xffff;
+		return `${HEX4[hi1]}${HEX4[hi2]}${HEX4[lo1]}${HEX4[lo2]}` as Snowflake;
+	}
+
+	// Snowflake generator type.
+	//
+	export class Source {
+		#seq = 0;
+		constructor(sequence: number = randu32() & MAX_SEQ) {
+			this.#seq = sequence & MAX_SEQ;
+		}
+
+		// Sequence number.
+		//
+		get sequence() {
+			return this.#seq & MAX_SEQ;
+		}
+		set sequence(v: number) {
+			this.#seq = v & MAX_SEQ;
+		}
+		reset() {
+			this.#seq = 0;
+		}
+
+		// Generates the next value as a hex string.
+		//
+		generate(timestamp: number): Snowflake {
+			const seq = (this.#seq + 1) & MAX_SEQ;
+			const dt = timestamp - EPOCH;
+			this.#seq = seq;
+			return formatParts(dt, seq);
+		}
+	}
+
+	// Gets the next snowflake given the timestamp.
+	//
+	const defaultSource = new Source();
+	export function next(timestamp = Date.now()): Snowflake {
+		return defaultSource.generate(timestamp);
+	}
+
+	// Validates a snowflake hex string.
+	//
+	export function valid(value: string): value is Snowflake {
+		return value.length === 16 && PATTERN.test(value);
+	}
+
+	// Returns the upper/lower boundaries for the given timestamp.
+	//
+	export function lowerbound(timelike: Date | number | Snowflake): Snowflake {
+		switch (typeof timelike) {
+			case "object": // Date
+				return formatParts(timelike.getTime() - EPOCH, 0);
+			case "number":
+				return formatParts(timelike - EPOCH, 0);
+			case "string": // Snowflake hex string
+				return timelike;
+		}
+	}
+	export function upperbound(timelike: Date | number | Snowflake): Snowflake {
+		switch (typeof timelike) {
+			case "object": // Date
+				return formatParts(timelike.getTime() - EPOCH, MAX_SEQ);
+			case "number":
+				return formatParts(timelike - EPOCH, MAX_SEQ);
+			case "string": // Snowflake hex string
+				return timelike;
+		}
+	}
+
+	// Returns the individual bits given the snowflake.
+	//
+	export function getSequence(value: Snowflake) {
+		return Number.parseInt(value.substring(8, 16), 16) & MAX_SEQ;
+	}
+	export function getTimestamp(value: Snowflake) {
+		const n = toBigInt(value) >> 22n;
+		return Number(n + BigInt(EPOCH));
+	}
+	export function getDate(value: Snowflake) {
+		return new Date(getTimestamp(value));
+	}
+}
+
+export { Snowflake };

package/src/stream.ts

@@ -0,0 +1,394 @@
+import { createAbortableStream } from "./abortable";
+
+/**
+ * Default ceiling for fully buffering a stream into memory (50 MiB). Reading an
+ * unbounded subprocess/HTTP stream into one buffer can exhaust the heap and crash
+ * the process with `RangeError: Out of memory`; this bounds that.
+ */
+export const DEFAULT_MAX_OUTPUT_BYTES = 50 * 1024 * 1024;
+
+/** Thrown when a capped stream read exceeds its byte limit. */
+export class OutputTooLargeError extends Error {
+	constructor(
+		readonly maxBytes: number,
+		readonly source?: string,
+	) {
+		super(`Output exceeded ${maxBytes} bytes${source ? ` (${source})` : ""}`);
+		this.name = "OutputTooLargeError";
+	}
+}
+
+export interface ReadStreamCappedOptions {
+	/** Maximum bytes to buffer before throwing OutputTooLargeError (default 50 MiB). */
+	maxBytes?: number;
+	/** Short label (e.g. the command/URL) included in the error and any log. */
+	source?: string;
+	/** Abort signal; aborting cancels the reader and rejects. */
+	signal?: AbortSignal;
+}
+
+/**
+ * Read a ReadableStream fully into a single Uint8Array, enforcing a hard byte
+ * cap. On exceeding the cap the reader is cancelled and `OutputTooLargeError` is
+ * thrown — the size is checked BEFORE retaining each chunk, so memory never
+ * exceeds the cap (unlike `new Response(stream).text()`, which buffers unbounded).
+ */
+export async function readStreamCapped(
+	stream: ReadableStream<Uint8Array>,
+	options: ReadStreamCappedOptions = {},
+): Promise<Uint8Array> {
+	const { maxBytes = DEFAULT_MAX_OUTPUT_BYTES, source, signal } = options;
+	const reader = stream.getReader();
+	const chunks: Uint8Array[] = [];
+	let total = 0;
+	try {
+		while (true) {
+			if (signal?.aborted) {
+				await reader.cancel();
+				throw signal.reason instanceof Error ? signal.reason : new Error("aborted");
+			}
+			const { done, value } = await reader.read();
+			if (done) break;
+			if (!value || value.byteLength === 0) continue;
+			total += value.byteLength;
+			if (total > maxBytes) {
+				await reader.cancel();
+				throw new OutputTooLargeError(maxBytes, source);
+			}
+			chunks.push(value);
+		}
+	} finally {
+		reader.releaseLock();
+	}
+
+	const out = new Uint8Array(total);
+	let offset = 0;
+	for (const chunk of chunks) {
+		out.set(chunk, offset);
+		offset += chunk.byteLength;
+	}
+	return out;
+}
+
+/** Like {@link readStreamCapped} but decodes the result as UTF-8 text. */
+export async function readStreamCappedText(
+	stream: ReadableStream<Uint8Array>,
+	options: ReadStreamCappedOptions = {},
+): Promise<string> {
+	return new TextDecoder().decode(await readStreamCapped(stream, options));
+}
+
+const LF = 0x0a;
+type JsonlChunkResult = {
+	values: unknown[];
+	error: unknown;
+	read: number;
+	done: boolean;
+};
+
+function parseJsonlChunkCompat(input: Uint8Array, beg?: number, end?: number): JsonlChunkResult;
+function parseJsonlChunkCompat(input: string): JsonlChunkResult;
+function parseJsonlChunkCompat(input: Uint8Array | string, beg?: number, end?: number): JsonlChunkResult {
+	if (typeof input === "string") {
+		const { values, error, read, done } = Bun.JSONL.parseChunk(input);
+		return { values, error, read, done };
+	}
+	const start = beg ?? 0;
+	const stop = end ?? input.length;
+	const { values, error, read, done } = Bun.JSONL.parseChunk(input, start, stop);
+	return { values, error, read, done };
+}
+
+export async function* readLines(stream: ReadableStream<Uint8Array>, signal?: AbortSignal): AsyncGenerator<Uint8Array> {
+	const buffer = new ConcatSink();
+	const source = createAbortableStream(stream, signal);
+	try {
+		for await (const chunk of source) {
+			for (const line of buffer.appendAndFlushLines(chunk)) {
+				yield line;
+			}
+		}
+		if (!buffer.isEmpty) {
+			const tail = buffer.flush();
+			if (tail) {
+				buffer.clear();
+				yield tail;
+			}
+		}
+	} catch (err) {
+		// Abort errors are expected — just stop the generator.
+		if (signal?.aborted) return;
+		throw err;
+	}
+}
+
+export async function* readJsonl<T>(stream: ReadableStream<Uint8Array>, signal?: AbortSignal): AsyncGenerator<T> {
+	const buffer = new ConcatSink();
+	const source = createAbortableStream(stream, signal);
+	try {
+		for await (const chunk of source) {
+			yield* buffer.pullJSONL<T>(chunk, 0, chunk.length);
+		}
+		if (!buffer.isEmpty) {
+			const tail = buffer.flush();
+			if (tail) {
+				buffer.clear();
+				const { values, error, done } = parseJsonlChunkCompat(tail, 0, tail.length);
+				if (values.length > 0) {
+					yield* values as T[];
+				}
+				if (error) throw error;
+				if (!done) {
+					throw new Error("JSONL stream ended unexpectedly");
+				}
+			}
+		}
+	} catch (err) {
+		// Abort errors are expected — just stop the generator.
+		if (signal?.aborted) return;
+		throw err;
+	}
+}
+
+// =============================================================================
+// SSE (Server-Sent Events)
+// =============================================================================
+
+/** Byte lookup table: 1 = whitespace, 0 = not. */
+const WS = new Uint8Array(256);
+WS[0x09] = 1; // tab
+WS[0x0a] = 1; // LF
+WS[0x0d] = 1; // CR
+WS[0x20] = 1; // space
+
+const createPattern = (prefix: string) => {
+	const pre = Buffer.from(prefix, "utf-8");
+	return {
+		strip(buf: Uint8Array): number | null {
+			const n = pre.length;
+			if (buf.length < n) return null;
+			if (pre.equals(buf.subarray(0, n))) {
+				return n;
+			}
+			return null;
+		},
+	};
+};
+
+const PAT_DATA = createPattern("data:");
+
+const PAT_DONE = createPattern("[DONE]");
+
+class ConcatSink {
+	#space?: Buffer;
+	#length = 0;
+
+	#ensureCapacity(size: number): Buffer {
+		const space = this.#space;
+		if (space && space.length >= size) return space;
+		const nextSize = space ? Math.max(size, space.length * 2) : size;
+		const next = Buffer.allocUnsafe(nextSize);
+		if (space && this.#length > 0) {
+			space.copy(next, 0, 0, this.#length);
+		}
+		this.#space = next;
+		return next;
+	}
+
+	append(chunk: Uint8Array) {
+		const n = chunk.length;
+		if (!n) return;
+		const offset = this.#length;
+		const space = this.#ensureCapacity(offset + n);
+		space.set(chunk, offset);
+		this.#length += n;
+	}
+
+	reset(chunk: Uint8Array) {
+		const n = chunk.length;
+		if (!n) {
+			this.#length = 0;
+			return;
+		}
+		const space = this.#ensureCapacity(n);
+		space.set(chunk, 0);
+		this.#length = n;
+	}
+
+	get isEmpty(): boolean {
+		return this.#length === 0;
+	}
+
+	flush(): Uint8Array | undefined {
+		if (!this.#length) return undefined;
+		return this.#space!.subarray(0, this.#length);
+	}
+
+	clear() {
+		this.#length = 0;
+	}
+
+	*appendAndFlushLines(chunk: Uint8Array) {
+		let pos = 0;
+		while (pos < chunk.length) {
+			const nl = chunk.indexOf(LF, pos);
+			if (nl === -1) {
+				this.append(chunk.subarray(pos));
+				return;
+			}
+			const suffix = chunk.subarray(pos, nl);
+			pos = nl + 1;
+			if (this.isEmpty) {
+				yield suffix;
+			} else {
+				this.append(suffix);
+				const payload = this.flush();
+				if (payload) {
+					yield payload;
+					this.clear();
+				}
+			}
+		}
+	}
+	*pullJSONL<T>(chunk: Uint8Array, beg: number, end: number) {
+		if (this.isEmpty) {
+			const { values, error, read, done } = parseJsonlChunkCompat(chunk, beg, end);
+			if (values.length > 0) {
+				yield* values as T[];
+			}
+			if (error) throw error;
+			if (done) return;
+			this.reset(chunk.subarray(read, end));
+			return;
+		}
+
+		const offset = this.#length;
+		const n = end - beg;
+		const total = offset + n;
+		const space = this.#ensureCapacity(total);
+		space.set(chunk.subarray(beg, end), offset);
+		this.#length = total;
+
+		const { values, error, read, done } = parseJsonlChunkCompat(space.subarray(0, total), 0, total);
+		if (values.length > 0) {
+			yield* values as T[];
+		}
+		if (error) throw error;
+		if (done) {
+			this.#length = 0;
+			return;
+		}
+		const rem = total - read;
+		if (rem < total) {
+			space.copyWithin(0, read, total);
+		}
+		this.#length = rem;
+	}
+}
+
+const kDoneError = new Error("SSE stream done");
+
+/**
+ * Stream parsed JSON objects from SSE `data:` lines.
+ *
+ * @example
+ * ```ts
+ * for await (const obj of readSseJson(response.body!)) {
+ *   console.log(obj);
+ * }
+ * ```
+ */
+export async function* readSseJson<T>(stream: ReadableStream<Uint8Array>, signal?: AbortSignal): AsyncGenerator<T> {
+	const lineBuffer = new ConcatSink();
+	const jsonBuffer = new ConcatSink();
+
+	// pipeThrough with { signal } makes the stream abort-aware: the pipe
+	// cancels the source and errors the output when the signal fires,
+	// so for-await-of exits cleanly without manual reader/listener management.
+	stream = createAbortableStream(stream, signal);
+	try {
+		const processLine = function* (line: Uint8Array) {
+			// Strip trailing spaces including \r.
+			let end = line.length;
+			while (end && WS[line[end - 1]]) {
+				--end;
+			}
+			if (!end) return; // blank line
+
+			const trimmed = end === line.length ? line : line.subarray(0, end);
+
+			// Check "data:" prefix and optional space afterwards.
+			let beg = PAT_DATA.strip(trimmed);
+			if (beg === null) return;
+			while (beg < end && WS[trimmed[beg]]) {
+				++beg;
+			}
+			if (beg >= end) return;
+
+			// Fast-path: the OpenAI-style done marker isn't JSON.
+			const donePrefix = PAT_DONE.strip(trimmed.subarray(beg, end));
+			if (donePrefix !== null && donePrefix === end - beg) {
+				throw kDoneError;
+			}
+
+			yield* jsonBuffer.pullJSONL<T>(trimmed, beg, end);
+		};
+		for await (const chunk of stream) {
+			for (const line of lineBuffer.appendAndFlushLines(chunk)) {
+				yield* processLine(line);
+			}
+		}
+		if (!lineBuffer.isEmpty) {
+			const tail = lineBuffer.flush();
+			if (tail) {
+				lineBuffer.clear();
+				yield* processLine(tail);
+			}
+		}
+	} catch (err) {
+		if (err === kDoneError) return;
+		// Abort errors are expected — just stop the generator.
+		if (signal?.aborted) return;
+		throw err;
+	}
+	if (!jsonBuffer.isEmpty) {
+		throw new Error("SSE stream ended unexpectedly");
+	}
+}
+
+/**
+ * Parse a complete JSONL string, skipping malformed lines instead of throwing.
+ *
+ * Uses `Bun.JSONL.parseChunk` internally. On parse errors, the malformed
+ * region is skipped up to the next newline and parsing continues.
+ *
+ * @example
+ * ```ts
+ * const entries = parseJsonlLenient<MyType>(fileContents);
+ * ```
+ */
+export function parseJsonlLenient<T>(buffer: string): T[] {
+	let entries: T[] | undefined;
+
+	while (buffer.length > 0) {
+		const { values, error, read, done } = parseJsonlChunkCompat(buffer);
+		if (values.length > 0) {
+			const ext = values as T[];
+			if (!entries) {
+				entries = ext;
+			} else {
+				entries.push(...ext);
+			}
+		}
+		if (error) {
+			const nextNewline = buffer.indexOf("\n", read);
+			if (nextNewline === -1) break;
+			buffer = buffer.substring(nextNewline + 1);
+			continue;
+		}
+		if (read === 0) break;
+		buffer = buffer.substring(read);
+		if (done) break;
+	}
+	return entries ?? [];
+}