@npy/fetch 0.1.3 → 0.1.4

package/io/readers.js

@@ -0,0 +1,333 @@
+import { parseMaxBytes } from "./_utils.js";
+import { CRLF_BYTES } from "../_internal/consts.js";
+import { Bytes, DelimiterCodec, read } from "@fuman/io";
+import { ConnectionClosedError } from "@fuman/net";
+//#region src/io/readers.ts
+var invalidHeaderCharRegex = /[^\t\x20-\x7e\x80-\xff]/g;
+function sanitizeHeaderValue(v) {
+	return v.replace(invalidHeaderCharRegex, (m) => encodeURI(m));
+}
+var LineReader = class {
+	#src;
+	#buf;
+	#codec = new DelimiterCodec(CRLF_BYTES, { strategy: "discard" });
+	#eof = false;
+	#readChunkSize;
+	#maxBufferedBytes;
+	#maxLineSize;
+	#closed = false;
+	close;
+	static Options;
+	constructor(src, opts = {}) {
+		this.#src = src;
+		this.#buf = Bytes.alloc(opts.bufferSize);
+		this.#readChunkSize = opts.readChunkSize ?? 16 * 1024;
+		this.#maxBufferedBytes = opts.maxBufferedBytes ?? 256 * 1024;
+		this.#maxLineSize = opts.maxLineSize ?? 64 * 1024;
+		this.close = this.#close.bind(this);
+	}
+	async read(into) {
+		if (this.#closed) return 0;
+		if (this.#buf.available > 0) {
+			const n = Math.min(into.length, this.#buf.available);
+			into.set(this.#buf.readSync(n));
+			this.#buf.reclaim();
+			return n;
+		}
+		if (this.#eof) return 0;
+		try {
+			return await this.#src.read(into);
+		} catch (e) {
+			if (e instanceof ConnectionClosedError) {
+				this.#eof = true;
+				return 0;
+			}
+			throw e;
+		}
+	}
+	async readLine() {
+		if (this.#closed) return null;
+		for (;;) {
+			const frame = this.#codec.decode(this.#buf, this.#eof);
+			if (frame !== null) {
+				this.#buf.reclaim();
+				return read.rawString(frame, frame.length);
+			}
+			if (this.#eof) return null;
+			if (this.#buf.available > this.#maxLineSize) throw new Error(`line too large (> ${this.#maxLineSize} bytes)`);
+			if (this.#buf.available > this.#maxBufferedBytes) throw new Error(`buffer too large while searching for delimiter (> ${this.#maxBufferedBytes} bytes)`);
+			await this.#pull();
+		}
+	}
+	async readHeaders(opts = {}) {
+		const maxHeaderSize = opts.maxHeaderSize ?? 64 * 1024;
+		const acc = /* @__PURE__ */ new Map();
+		const validator = new Headers();
+		let lastKey = null;
+		let firstLine = true;
+		let consumed = 0;
+		for (;;) {
+			const line = await this.readLine();
+			if (line === null) throw new Error("Unexpected EOF while reading HTTP headers");
+			consumed += line.length + 2;
+			if (consumed > maxHeaderSize) throw new Error(`HTTP headers too large (> ${maxHeaderSize} bytes)`);
+			if (line === "") break;
+			if (firstLine && (line[0] === " " || line[0] === "	")) throw new Error(`malformed HTTP header initial line: ${line}`);
+			firstLine = false;
+			if (line[0] === " " || line[0] === "	") {
+				if (!lastKey) throw new Error(`malformed HTTP header continuation line: ${line}`);
+				const arr = acc.get(lastKey);
+				if (!arr || arr.length === 0) throw new Error(`malformed HTTP header continuation line: ${line}`);
+				const piece = sanitizeHeaderValue(line.trim());
+				arr[arr.length - 1] = `${arr[arr.length - 1]} ${piece}`.trim();
+				continue;
+			}
+			const idx = line.indexOf(":");
+			if (idx === -1) throw new Error(`malformed HTTP header line: ${line}`);
+			const rawName = line.slice(0, idx).trim();
+			if (rawName === "") {
+				lastKey = null;
+				continue;
+			}
+			const name = rawName.toLowerCase();
+			const value = sanitizeHeaderValue(line.slice(idx + 1).trim());
+			try {
+				validator.append(name, value);
+			} catch {
+				lastKey = null;
+				continue;
+			}
+			const arr = acc.get(name);
+			if (arr) arr.push(value);
+			else acc.set(name, [value]);
+			lastKey = name;
+		}
+		const headers = new Headers();
+		for (const [k, values] of acc) for (const v of values) try {
+			headers.append(k, v);
+		} catch {}
+		return headers;
+	}
+	async #pull() {
+		const into = this.#buf.writeSync(this.#readChunkSize);
+		try {
+			const n = await this.#src.read(into);
+			this.#buf.disposeWriteSync(n);
+			if (n === 0) this.#eof = true;
+		} catch (e) {
+			this.#buf.disposeWriteSync(0);
+			if (e instanceof ConnectionClosedError) {
+				this.#eof = true;
+				return;
+			}
+			throw e;
+		} finally {
+			this.#buf.reclaim();
+		}
+	}
+	async #close() {
+		if (this.#closed) return;
+		this.#closed = true;
+		await this.#src.close();
+	}
+};
+var BodyReader = class {
+	#src;
+	#remaining;
+	#maxResponseSize;
+	#readSoFar = 0;
+	#closed = false;
+	close;
+	static Options;
+	constructor(src, contentLength, opts = {}) {
+		this.#src = src;
+		this.#remaining = contentLength;
+		this.#maxResponseSize = parseMaxBytes(opts.maxBodySize);
+		this.close = this.#close.bind(this);
+	}
+	async read(into) {
+		if (this.#closed) return 0;
+		if (this.#remaining === 0) return 0;
+		if (this.#maxResponseSize != null) {
+			const remainingLimit = this.#maxResponseSize - this.#readSoFar;
+			if (remainingLimit <= 0) throw new Error(`body too large (> ${this.#maxResponseSize} bytes)`);
+			if (into.length > remainingLimit) into = into.subarray(0, remainingLimit);
+		}
+		let max = into.length;
+		if (this.#remaining != null) max = Math.min(max, this.#remaining);
+		if (max === 0) return 0;
+		const view = max === into.length ? into : into.subarray(0, max);
+		let n = 0;
+		try {
+			n = await this.#src.read(view);
+		} catch (e) {
+			if (e instanceof ConnectionClosedError) n = 0;
+			else throw e;
+		}
+		if (n === 0) {
+			if (this.#remaining != null) throw new Error("Unexpected EOF while reading fixed-length body");
+			return 0;
+		}
+		this.#readSoFar += n;
+		if (this.#remaining != null) this.#remaining -= n;
+		return n;
+	}
+	async #close() {
+		if (this.#closed) return;
+		this.#closed = true;
+		await this.#src.close();
+	}
+};
+var ChunkedBodyReader = class {
+	#src;
+	#buf;
+	#codec = new DelimiterCodec(CRLF_BYTES, { strategy: "discard" });
+	#readChunkSize;
+	#maxLineSize;
+	#maxChunkSize;
+	#maxResponseSize;
+	#readSoFar = 0;
+	#eof = false;
+	#closed = false;
+	#state = { kind: "size" };
+	close;
+	static Options;
+	constructor(src, opts = {}) {
+		this.#src = src;
+		this.#buf = Bytes.alloc(opts.bufferSize);
+		this.#readChunkSize = opts.readChunkSize ?? 16 * 1024;
+		this.#maxLineSize = opts.maxLineSize ?? 64 * 1024;
+		this.#maxChunkSize = opts.maxChunkSize ?? 16 * 1024 * 1024;
+		this.#maxResponseSize = parseMaxBytes(opts.maxBodySize);
+		this.close = this.#close.bind(this);
+	}
+	async read(into) {
+		if (this.#closed) return 0;
+		for (;;) {
+			if (this.#state.kind === "done") return 0;
+			let view = into;
+			if (this.#maxResponseSize != null) {
+				const remainingLimit = this.#maxResponseSize - this.#readSoFar;
+				if (remainingLimit <= 0) throw new Error(`body too large (> ${this.#maxResponseSize} bytes)`);
+				if (view.length > remainingLimit) view = view.subarray(0, remainingLimit);
+			}
+			if (view.length === 0) return 0;
+			if (this.#state.kind === "data") {
+				if (this.#state.remaining === 0) {
+					this.#state = { kind: "crlf" };
+					continue;
+				}
+				if (this.#buf.available > 0) {
+					const n = Math.min(view.length, this.#state.remaining, this.#buf.available);
+					view.set(this.#buf.readSync(n));
+					this.#buf.reclaim();
+					this.#readSoFar += n;
+					this.#state = {
+						kind: "data",
+						remaining: this.#state.remaining - n
+					};
+					return n;
+				}
+				const max = Math.min(view.length, this.#state.remaining);
+				const slice = max === view.length ? view : view.subarray(0, max);
+				const n = await this.#readFromSrc(slice);
+				if (n === 0) throw new Error("Unexpected EOF while reading chunked body");
+				this.#readSoFar += n;
+				this.#state = {
+					kind: "data",
+					remaining: this.#state.remaining - n
+				};
+				return n;
+			}
+			if (this.#state.kind === "size") {
+				const line = await this.#readLine();
+				if (line === null) throw new Error("Unexpected EOF while reading chunk size");
+				const semi = line.indexOf(";");
+				const token = (semi === -1 ? line : line.slice(0, semi)).trim();
+				if (token === "") throw new Error(`invalid chunk size line: ${line}`);
+				const size = Number.parseInt(token, 16);
+				if (!Number.isFinite(size) || Number.isNaN(size) || size < 0) throw new Error(`invalid chunk size: ${token}`);
+				if (size > this.#maxChunkSize) throw new Error(`chunk too large (> ${this.#maxChunkSize} bytes)`);
+				if (this.#maxResponseSize != null) {
+					if (size > this.#maxResponseSize - this.#readSoFar) throw new Error(`body too large (> ${this.#maxResponseSize} bytes)`);
+				}
+				this.#state = size === 0 ? { kind: "trailers" } : {
+					kind: "data",
+					remaining: size
+				};
+				continue;
+			}
+			if (this.#state.kind === "crlf") {
+				await this.#consumeCrlf();
+				this.#state = { kind: "size" };
+				continue;
+			}
+			if (this.#state.kind === "trailers") for (;;) {
+				const line = await this.#readLine();
+				if (line === null) throw new Error("Unexpected EOF while reading chunked trailers");
+				if (line === "") {
+					this.#state = { kind: "done" };
+					return 0;
+				}
+			}
+		}
+	}
+	async #readFromSrc(into) {
+		if (this.#eof) return 0;
+		try {
+			const n = await this.#src.read(into);
+			if (n === 0) this.#eof = true;
+			return n;
+		} catch (e) {
+			if (e instanceof ConnectionClosedError) {
+				this.#eof = true;
+				return 0;
+			}
+			throw e;
+		}
+	}
+	async #pull() {
+		const into = this.#buf.writeSync(this.#readChunkSize);
+		try {
+			const n = await this.#readFromSrc(into);
+			this.#buf.disposeWriteSync(n);
+		} catch (e) {
+			this.#buf.disposeWriteSync(0);
+			if (e instanceof ConnectionClosedError) {
+				this.#eof = true;
+				return;
+			}
+			throw e;
+		} finally {
+			this.#buf.reclaim();
+		}
+	}
+	async #readLine() {
+		for (;;) {
+			const frame = this.#codec.decode(this.#buf, this.#eof);
+			if (frame !== null) {
+				this.#buf.reclaim();
+				return read.rawString(frame, frame.length);
+			}
+			if (this.#eof) return null;
+			if (this.#buf.available > this.#maxLineSize) throw new Error(`chunk line too large (> ${this.#maxLineSize} bytes)`);
+			await this.#pull();
+		}
+	}
+	async #consumeCrlf() {
+		while (this.#buf.available < 2) {
+			if (this.#eof) throw new Error("Unexpected EOF while reading chunk terminator");
+			await this.#pull();
+		}
+		const two = this.#buf.readSync(2);
+		this.#buf.reclaim();
+		if (two[0] !== CRLF_BYTES[0] || two[1] !== CRLF_BYTES[1]) throw new Error("Invalid chunked encoding: missing CRLF after chunk data");
+	}
+	async #close() {
+		if (this.#closed) return;
+		this.#closed = true;
+		await this.#src.close();
+	}
+};
+//#endregion
+export { BodyReader, ChunkedBodyReader, LineReader, sanitizeHeaderValue };

package/io/writers.cjs

@@ -0,0 +1,196 @@
+require("../_virtual/_rolldown/runtime.cjs");
+const require_streams = require("../_internal/streams.cjs");
+const require_encoding = require("../encoding.cjs");
+const require__utils = require("./_utils.cjs");
+const require_readers = require("./readers.cjs");
+const require_guards = require("../_internal/guards.cjs");
+const require_body = require("../body.cjs");
+const require_buf_writer = require("./buf-writer.cjs");
+let _fuman_node = require("@fuman/node");
+let _fuman_io = require("@fuman/io");
+//#region src/io/writers.ts
+function toRequestTarget(url) {
+	return (url.pathname?.startsWith("/") ? url.pathname : "/") + (url.search || "");
+}
+function encodeHead(into, head) {
+	_fuman_io.write.rawString(into, `${head.method.toUpperCase()} ${head.target} HTTP/1.1\r
+`);
+	for (const [k, v] of head.headers) _fuman_io.write.rawString(into, `${k}: ${require_readers.sanitizeHeaderValue(v)}\r
+`);
+	_fuman_io.write.rawString(into, "\r\n");
+}
+function prepareBody(headers, init) {
+	const state = require_body.extractBody(init);
+	if (state.body != null && !headers.has("content-type")) headers.set("content-type", state.contentType ?? "application/octet-stream");
+	const body = state.body;
+	if (body == null) return { kind: "none" };
+	if (body instanceof Uint8Array) return {
+		kind: "bytes",
+		bytes: body,
+		length: body.byteLength
+	};
+	if (require_guards.isReadableStream(body)) return {
+		kind: "stream",
+		stream: body,
+		length: state.contentLength
+	};
+	if (require_guards.isFumanReadable(body)) return {
+		kind: "stream",
+		stream: (0, _fuman_io.fumanReadableToWeb)(body),
+		length: state.contentLength
+	};
+	return {
+		kind: "stream",
+		stream: (0, _fuman_node.nodeReadableToWeb)(body),
+		length: state.contentLength
+	};
+}
+function finalizeDelimitation(headers, body) {
+	if (body.kind === "none") return { chunked: false };
+	const te = require__utils.parseTransferEncoding(headers);
+	if (te.has) {
+		headers.delete("content-length");
+		if (!te.chunked) {
+			const tokens = [...te.codings, "chunked"].filter(Boolean);
+			headers.set("transfer-encoding", tokens.join(", "));
+		}
+		return { chunked: true };
+	}
+	const knownLength = body.kind === "bytes" ? body.length : body.length ?? null;
+	if (knownLength != null) {
+		const existing = require__utils.parseContentLength(headers);
+		if (existing != null && existing !== knownLength) throw new Error(`Conflicting content-length: header=${existing} body=${knownLength}`);
+		if (existing == null) headers.set("content-length", String(knownLength));
+		return { chunked: false };
+	}
+	if (require__utils.parseContentLength(headers) != null) return { chunked: false };
+	headers.set("transfer-encoding", "chunked");
+	headers.delete("content-length");
+	return { chunked: true };
+}
+function createBufferedConnWriter(dst, opts) {
+	const bufferSize = opts.writeBufferSize ?? 16 * 1024;
+	const directWriteThreshold = opts.directWriteThreshold ?? 64 * 1024;
+	const bufWriter = new require_buf_writer.BufWriter(dst, bufferSize);
+	const flush = async () => {
+		await bufWriter.flush();
+	};
+	const writeBytes = async (bytes) => {
+		if (bytes.length === 0) return;
+		if (bytes.length >= directWriteThreshold) {
+			await bufWriter.flush();
+			await dst.write(bytes);
+			return;
+		}
+		await bufWriter.write(bytes);
+	};
+	const writeRawString = async (str) => {
+		if (str.length === 0) return;
+		_fuman_io.write.rawString(bufWriter, str);
+	};
+	return {
+		flush,
+		writeBytes,
+		writeRawString,
+		directWriteThreshold
+	};
+}
+async function writeBody(dst, body, chunked, opts, signal) {
+	const bw = createBufferedConnWriter(dst, opts);
+	const writeChunk = async (chunk) => {
+		if (signal?.aborted) throw signal.reason ?? /* @__PURE__ */ new Error("Request aborted");
+		if (!chunked) {
+			await bw.writeBytes(chunk);
+			return;
+		}
+		if (chunk.length === 0) return;
+		await bw.writeRawString(chunk.length.toString(16));
+		await bw.writeRawString("\r\n");
+		await bw.writeBytes(chunk);
+		await bw.writeRawString("\r\n");
+	};
+	if (body.kind === "bytes") await writeChunk(body.bytes);
+	else for await (const chunk of body.stream) await writeChunk(chunk);
+	if (chunked) await bw.writeRawString(`0\r
+\r
+`);
+	await bw.flush();
+}
+async function writeCoalesced(dst, scratch, head, bodyBytes, chunked) {
+	scratch.reset();
+	encodeHead(scratch, head);
+	if (!chunked) {
+		_fuman_io.write.bytes(scratch, bodyBytes);
+		await dst.write(scratch.result());
+		scratch.reset();
+		return;
+	}
+	_fuman_io.write.rawString(scratch, bodyBytes.length.toString(16));
+	_fuman_io.write.rawString(scratch, "\r\n");
+	_fuman_io.write.bytes(scratch, bodyBytes);
+	_fuman_io.write.rawString(scratch, `\r
+0\r
+\r
+`);
+	await dst.write(scratch.result());
+	scratch.reset();
+}
+function createRequestWriter(dst, opts = {}) {
+	const scratch = _fuman_io.Bytes.alloc(opts.writeBufferSize ?? 16 * 1024);
+	const write = async (req) => {
+		if (req.signal?.aborted) throw req.signal.reason ?? /* @__PURE__ */ new Error("Request aborted");
+		const method = req.method.toUpperCase();
+		const headers = req.headers ? new Headers(req.headers) : new Headers();
+		const url = req.url;
+		if (!headers.has("host")) headers.set("host", url.host);
+		if (!headers.has("date")) headers.set("date", (/* @__PURE__ */ new Date()).toUTCString());
+		const target = toRequestTarget(url);
+		let body = prepareBody(headers, req.body ?? null);
+		const ceRaw = headers.get("content-encoding") ?? void 0;
+		const ceEncoders = require_encoding.createEncoders(ceRaw);
+		if (body.kind !== "none" && ceEncoders.length > 0) {
+			body = {
+				kind: "stream",
+				stream: require_encoding.encodeStream(body.kind === "stream" ? body.stream : require_streams.bytesToStream(body.bytes), ceRaw),
+				length: null
+			};
+			headers.delete("content-length");
+		}
+		const teInfo = require__utils.parseTransferEncoding(headers);
+		if (body.kind !== "none" && teInfo.has && teInfo.codings.length > 0) {
+			body = {
+				kind: "stream",
+				stream: require_encoding.encodeStream(body.kind === "stream" ? body.stream : require_streams.bytesToStream(body.bytes), teInfo.codings),
+				length: null
+			};
+			headers.delete("content-length");
+		}
+		const { chunked } = finalizeDelimitation(headers, body);
+		const head = {
+			method,
+			target,
+			headers
+		};
+		if (body.kind === "bytes") {
+			const max = opts.coalesceBodyMaxBytes ?? 64 * 1024;
+			if (body.bytes.length <= max) {
+				await writeCoalesced(dst, scratch, head, body.bytes, chunked);
+				return;
+			}
+		}
+		scratch.reset();
+		encodeHead(scratch, head);
+		await dst.write(scratch.result());
+		scratch.reset();
+		if (body.kind === "none") return;
+		if (body.kind === "bytes" && !chunked) {
+			if (req.signal?.aborted) throw req.signal.reason ?? /* @__PURE__ */ new Error("Request aborted");
+			await dst.write(body.bytes);
+			return;
+		}
+		await writeBody(dst, body, chunked, opts, req.signal);
+	};
+	return { write };
+}
+//#endregion
+exports.createRequestWriter = createRequestWriter;

package/io/writers.d.cts

@@ -0,0 +1,22 @@
+import { IWritable } from '@fuman/io';
+import { BodyInit } from '../body';
+type Destination = IWritable;
+export declare namespace Writers {
+    interface Options {
+        writeBufferSize?: number;
+        directWriteThreshold?: number;
+        coalesceBodyMaxBytes?: number;
+    }
+    interface Request {
+        url: URL;
+        method: string;
+        headers?: Headers;
+        body?: BodyInit | null;
+        signal?: AbortSignal;
+    }
+    interface Writer {
+        write(req: Request): Promise<void>;
+    }
+}
+export declare function createRequestWriter(dst: Destination, opts?: Writers.Options): Writers.Writer;
+export {};

package/io/writers.d.ts

@@ -0,0 +1,22 @@
+import { IWritable } from '@fuman/io';
+import { BodyInit } from '../body';
+type Destination = IWritable;
+export declare namespace Writers {
+    interface Options {
+        writeBufferSize?: number;
+        directWriteThreshold?: number;
+        coalesceBodyMaxBytes?: number;
+    }
+    interface Request {
+        url: URL;
+        method: string;
+        headers?: Headers;
+        body?: BodyInit | null;
+        signal?: AbortSignal;
+    }
+    interface Writer {
+        write(req: Request): Promise<void>;
+    }
+}
+export declare function createRequestWriter(dst: Destination, opts?: Writers.Options): Writers.Writer;
+export {};