@stll/fetch 0.1.0 → 0.2.0

package/README.md

@@ -6,26 +6,32 @@
 
 ```ts
 import { fetchWithTimeout } from "@stll/fetch";
+import { readBodyWithIdleDeadline } from "@stll/fetch/body-reader";
 
 // Abort when no response headers arrive within 5 seconds.
 const response = await fetchWithTimeout(url, {
   timeout: { type: "headers", ms: 5_000 },
 });
 
-// Abort when the headers, or any later body chunk, take longer than
-// 10 seconds: a slow but progressing download keeps going.
-const stream = await fetchWithTimeout(url, {
-  timeout: { type: "idle", ms: 10_000 },
+// Bound headers separately, then observe chunks in the caller's own read.
+const download = await fetchWithTimeout(url, {
+  timeout: { type: "headers", ms: 10_000 },
   signal: controller.signal, // optional; either signal aborts the request
 });
+const bytes = await readBodyWithIdleDeadline(download, {
+  idleMs: 10_000,
+  signal: controller.signal,
+});
 ```
 
-The timeout is required by the type. `headers` bounds the wait for the
-response; `idle` also restarts on every body chunk and errors the body stream
-when it stalls. A timeout rejects (or errors the body) with a `TimeoutError`
-`DOMException`, and the caller's `signal` and a `Request`'s own signal still
-abort the request. `timeoutMs: number` is a total deadline over the request
-and its body.
+The timeout is required by the type. `headers` bounds only the wait for the
+response; `total` bounds the request and its body consumption through the
+transport's own signal. Both policies return the transport's response
+unchanged. `readBodyWithIdleDeadline` consumes the caller's own body reader,
+restarts its deadline after every chunk, and cancels a stalled body with a
+`TimeoutError` `DOMException`. The caller's `signal` and a `Request`'s own
+signal still abort the request.
+`timeoutMs: number` remains the legacy numeric form of the total policy.
 
 `createFetchWithTimeout(fetcher)` returns the same function over another
 `fetch` implementation (a proxy-aware client, or a fake in tests).

package/dist/body-reader.d.ts

@@ -0,0 +1,22 @@
+//#region src/body-reader.d.ts
+declare const BodyByteLimitError_base: import("better-result").TaggedErrorClass<"BodyByteLimitError">;
+export declare class BodyByteLimitError extends BodyByteLimitError_base<{
+  message: string;
+  limit: number;
+}> {}
+type BodyIdleDeadlineOptions = {
+  idleMs: number;
+  maxBytes?: number | undefined;
+  signal?: AbortSignal | undefined;
+};
+/**
+ * Yields response chunks while bounding the wait for each successive chunk.
+ * @yields Each byte chunk from the transport body.
+ */
+export declare function iterateBodyWithIdleDeadline(response: Response, { idleMs, maxBytes, signal }: BodyIdleDeadlineOptions): AsyncGenerator<Uint8Array, void, void>;
+/** Reads a response body while bounding the wait for each successive chunk. */
+export declare const readBodyWithIdleDeadline: (response: Response, options: BodyIdleDeadlineOptions) => Promise<Uint8Array>;
+export declare const readBodyTextWithIdleDeadline: (response: Response, options: BodyIdleDeadlineOptions) => Promise<string>;
+export declare const readBodyJsonWithIdleDeadline: (response: Response, options: BodyIdleDeadlineOptions) => Promise<unknown>;
+//#endregion
+export {};

package/dist/body-reader.js

@@ -0,0 +1,78 @@
+import { t as raceAbort } from "./race-abort-wtommv8G.js";
+import { TaggedError } from "better-result";
+//#region src/body-reader.ts
+var BodyByteLimitError = class extends TaggedError("BodyByteLimitError") {};
+const invalidIdleMs = (idleMs) => {
+	if (!Number.isFinite(idleMs) || idleMs < 0 || idleMs > 2147483647) return new DOMException("Invalid body idle deadline duration", "NotSupportedError");
+};
+const responseReader = (body) => body.getReader();
+/**
+* Yields response chunks while bounding the wait for each successive chunk.
+* @yields Each byte chunk from the transport body.
+*/
+async function* iterateBodyWithIdleDeadline(response, { idleMs, maxBytes, signal }) {
+	const invalidDuration = invalidIdleMs(idleMs);
+	if (invalidDuration !== void 0) await Promise.reject(invalidDuration);
+	if (signal?.aborted === true) {
+		response.body?.cancel(signal.reason).catch((cancellationError) => {});
+		signal.throwIfAborted();
+	}
+	if (maxBytes !== void 0 && (!Number.isSafeInteger(maxBytes) || maxBytes < 0)) await Promise.reject(new DOMException("Invalid response body byte limit", "NotSupportedError"));
+	const body = response.body;
+	if (body === null) return;
+	const reader = responseReader(body);
+	let exitReason;
+	let byteLength = 0;
+	try {
+		for (;;) {
+			if (signal?.aborted === true) exitReason = signal.reason;
+			signal?.throwIfAborted();
+			const deadline = new AbortController();
+			const timeoutError = new DOMException("Response body stalled", "TimeoutError");
+			const timer = setTimeout(() => deadline.abort(timeoutError), idleMs);
+			const combined = signal === void 0 ? deadline.signal : AbortSignal.any([signal, deadline.signal]);
+			let next;
+			try {
+				next = await raceAbort(reader.read(), combined);
+			} finally {
+				exitReason ??= combined.reason;
+				clearTimeout(timer);
+			}
+			if (next.done) return;
+			byteLength += next.value.byteLength;
+			if (maxBytes !== void 0 && byteLength > maxBytes) {
+				const limitError = new BodyByteLimitError({
+					message: `Response body exceeds the ${maxBytes}-byte limit`,
+					limit: maxBytes
+				});
+				exitReason = limitError;
+				await Promise.reject(limitError);
+			}
+			yield next.value;
+		}
+	} finally {
+		reader.cancel(exitReason).then(() => reader.releaseLock()).catch((cancellationError) => {
+			reader.releaseLock();
+		});
+	}
+}
+/** Reads a response body while bounding the wait for each successive chunk. */
+const readBodyWithIdleDeadline = async (response, options) => {
+	const chunks = [];
+	let byteLength = 0;
+	for await (const chunk of iterateBodyWithIdleDeadline(response, options)) {
+		chunks.push(chunk);
+		byteLength += chunk.byteLength;
+	}
+	const bytes = new Uint8Array(byteLength);
+	let offset = 0;
+	for (const chunk of chunks) {
+		bytes.set(chunk, offset);
+		offset += chunk.byteLength;
+	}
+	return bytes;
+};
+const readBodyTextWithIdleDeadline = async (response, options) => new TextDecoder().decode(await readBodyWithIdleDeadline(response, options));
+const readBodyJsonWithIdleDeadline = async (response, options) => JSON.parse(await readBodyTextWithIdleDeadline(response, options));
+//#endregion
+export { BodyByteLimitError, iterateBodyWithIdleDeadline, readBodyJsonWithIdleDeadline, readBodyTextWithIdleDeadline, readBodyWithIdleDeadline };

package/dist/{index-DNw-Gzhr.d.ts → index-C-MNDO3-.d.ts}

@@ -3,10 +3,10 @@ declare const isConnectionFailure: (cause: unknown) => boolean;
 //#endregion
 //#region src/index.d.ts
 type FetchTimeout = {
-  type: "headers";
+  type: "total";
   ms: number;
 } | {
-  type: "idle";
+  type: "headers";
   ms: number;
 };
 type TransportRequestInit = RequestInit & {
@@ -18,9 +18,9 @@ type FetchWithTimeoutInit = Omit<RequestInit, "signal"> & {
   signal?: AbortSignal | undefined;
 } & ({
   timeout: FetchTimeout;
-  timeoutMs?: never;
+  timeoutMs?: number;
 } | {
-  /** Choose a headers or idle timeout for new callers. */
+  /** Use an explicit timeout policy for new callers. */
   timeoutMs: number;
   timeout?: never;
 });

package/dist/index.d.ts

@@ -1,2 +1,2 @@
-import { a as createFetchWithTimeout, i as TransportRequestInit, n as FetchWithTimeoutInit, o as fetchWithTimeout, r as Fetcher, s as isConnectionFailure, t as FetchTimeout } from "./index-DNw-Gzhr.js";
+import { a as createFetchWithTimeout, i as TransportRequestInit, n as FetchWithTimeoutInit, o as fetchWithTimeout, r as Fetcher, s as isConnectionFailure, t as FetchTimeout } from "./index-C-MNDO3-.js";
 export { FetchTimeout, FetchWithTimeoutInit, Fetcher, TransportRequestInit, createFetchWithTimeout, fetchWithTimeout, isConnectionFailure };

package/dist/index.js

@@ -53,14 +53,14 @@ const isConnectionFailure = (cause) => {
 const executeFetchWithTimeout = async (fetcher, input, { timeout, timeoutMs, signal, ...init }) => {
 	const callerSignals = [signal, input instanceof Request ? input.signal : void 0].filter((candidate) => candidate !== void 0);
 	const activeFetcher = fetcher ?? globalThis.fetch;
-	if (timeout === void 0) {
-		const total = AbortSignal.timeout(timeoutMs);
-		return await activeFetcher(input, {
-			...init,
-			signal: AbortSignal.any([...callerSignals, total])
-		});
-	}
-	if (!Number.isFinite(timeout.ms) || timeout.ms < 0 || timeout.ms > 2147483647) throw new DOMException("Invalid fetch timeout duration", "NotSupportedError");
+	const duration = timeout?.ms ?? timeoutMs;
+	if (typeof duration !== "number" || !Number.isFinite(duration) || duration < 0 || duration > 2147483647) throw new DOMException("Invalid fetch timeout duration", "NotSupportedError");
+	if (timeout === void 0 || timeout.type === "total") return await activeFetcher(input, {
+		...init,
+		signal: AbortSignal.any([...callerSignals, AbortSignal.timeout(duration)]),
+		...timeout === void 0 ? {} : { timeout },
+		timeoutMs: duration
+	});
 	const controller = new AbortController();
 	const combined = AbortSignal.any([...callerSignals, controller.signal]);
 	const expire = () => controller.abort(new DOMException("Fetch timed out", "TimeoutError"));
@@ -69,86 +69,18 @@ const executeFetchWithTimeout = async (fetcher, input, { timeout, timeoutMs, sig
 		clearTimeout(timer);
 		timer = void 0;
 	};
-	const reset = () => {
-		clear();
-		timer = setTimeout(expire, timeout.ms);
-	};
-	reset();
+	timer = setTimeout(expire, duration);
 	const response = await Promise.resolve().then(async () => await activeFetcher(input, {
 		...init,
-		signal: combined
+		signal: combined,
+		timeout,
+		timeoutMs: duration
 	})).finally(clear);
 	if (combined.aborted) {
 		await Promise.allSettled([response.body?.cancel(combined.reason)]);
 		throw combined.reason;
 	}
-	if (timeout.type === "headers" || response.body === null) return response;
-	const reader = response.body.getReader();
-	let stopped = false;
-	let cancellation;
-	let abort;
-	const finish = () => {
-		stopped = true;
-		clear();
-		combined.removeEventListener("abort", abort);
-	};
-	const body = new ReadableStream({
-		start(stream) {
-			abort = () => {
-				if (stopped) return;
-				finish();
-				stream.error(combined.reason);
-				cancellation = reader.cancel(combined.reason).then(() => reader.releaseLock(), () => reader.releaseLock());
-			};
-			combined.addEventListener("abort", abort, { once: true });
-			if (combined.aborted) {
-				abort();
-				return;
-			}
-		},
-		async pull(stream) {
-			if (stopped) {
-				await cancellation;
-				return;
-			}
-			reset();
-			await reader.read().finally(clear).then(({ done, value }) => {
-				if (stopped) return;
-				if (done) {
-					finish();
-					reader.releaseLock();
-					stream.close();
-					return;
-				}
-				stream.enqueue(value);
-			}, (error) => {
-				if (stopped) return;
-				finish();
-				reader.releaseLock();
-				stream.error(error);
-			});
-			await cancellation;
-		},
-		async cancel(reason) {
-			if (stopped) {
-				await cancellation;
-				return;
-			}
-			finish();
-			await reader.cancel(reason).finally(() => reader.releaseLock());
-		}
-	});
-	const wrapped = new Response(body, {
-		status: response.status,
-		statusText: response.statusText,
-		headers: response.headers
-	});
-	Object.defineProperties(wrapped, {
-		url: { value: response.url },
-		redirected: { value: response.redirected },
-		type: { value: response.type }
-	});
-	return wrapped;
+	return response;
 };
 /** Creates a wrapper with the supplied transport and an explicit timeout policy. */
 const createFetchWithTimeout = (fetcher) => async (input, init) => await executeFetchWithTimeout(fetcher, input, init);

package/dist/publisher.d.ts

@@ -1,5 +1,5 @@
-import { i as TransportRequestInit } from "./index-DNw-Gzhr.js";
 import { Result } from "better-result";
+import { i as TransportRequestInit } from "./index-C-MNDO3-.js";
 //#region src/publisher.d.ts
 export type PublisherPolicy = {
   type: "obey";
@@ -121,4 +121,5 @@ type ReadBoundedBodyOptions = {
 };
 export declare const readBoundedBody: (response: Response, { maxBytes, signal }: ReadBoundedBodyOptions) => Promise<Result<Uint8Array, PublisherBodyLimitError>>;
 export declare const createPolitePublisherFetch: <Target extends PublisherTarget>({ validateTarget, fetcher, clock, robotsTtlMs, robotsRetryMs, robotsLoadTimeoutMs, defaultMinimumIntervalMs, onTelemetry }: PublisherFetcherOptions<Target>) => (input: string | URL, options?: PublisherFetchOptions) => Promise<Result<PublisherFetchResult, PublisherPolicyError>>;
-//#endregion
+//#endregion
+export {};

package/dist/publisher.js

@@ -1,3 +1,4 @@
+import { t as raceAbort } from "./race-abort-wtommv8G.js";
 import { Result, TaggedError } from "better-result";
 //#region src/publisher.ts
 const ROBOTS_MAX_BYTES = 524288;
@@ -193,19 +194,6 @@ const evaluatePublisherRobots = (groups, productToken, path) => {
 		crawlDelayMs: Math.max(0, ...selected.map(({ crawlDelayMs }) => crawlDelayMs ?? 0))
 	};
 };
-const abortReason = (signal) => signal.reason instanceof Error ? signal.reason : new DOMException("The operation was aborted", "AbortError");
-const raceAbort = async (work, signal) => {
-	if (!signal) return await work;
-	const { promise: aborted, reject } = Promise.withResolvers();
-	const onAbort = () => reject(abortReason(signal));
-	if (signal.aborted) onAbort();
-	else signal.addEventListener("abort", onAbort, { once: true });
-	try {
-		return await Promise.race([work, aborted]);
-	} finally {
-		signal.removeEventListener("abort", onAbort);
-	}
-};
 var PublisherBodyLimitError = class extends TaggedError("publisher_body_limit") {};
 const readBoundedBody = async (response, { maxBytes, signal }) => {
 	if (!response.body) return Result.ok(/* @__PURE__ */ new Uint8Array());

package/dist/race-abort-wtommv8G.js

@@ -0,0 +1,16 @@
+//#region src/race-abort.ts
+const abortReason = (signal) => signal.reason ?? new DOMException("The operation was aborted", "AbortError");
+const raceAbort = async (work, signal) => {
+	if (signal === void 0) return await work;
+	const { promise: aborted, reject } = Promise.withResolvers();
+	const onAbort = () => reject(abortReason(signal));
+	if (signal.aborted) onAbort();
+	else signal.addEventListener("abort", onAbort, { once: true });
+	try {
+		return await Promise.race([work, aborted]);
+	} finally {
+		signal.removeEventListener("abort", onAbort);
+	}
+};
+//#endregion
+export { raceAbort as t };

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@stll/fetch",
-  "version": "0.1.0",
-  "description": "Fetch with a mandatory timeout (headers or idle body window), connection-failure classification, and a robots-aware polite publisher fetch",
+  "version": "0.2.0",
+  "description": "Fetch with mandatory total or header timeouts, idle-bounded body reads, connection-failure classification, and a robots-aware polite publisher fetch",
   "keywords": [
     "abort",
     "fetch",
@@ -33,6 +33,10 @@
     "./publisher": {
       "types": "./dist/publisher.d.ts",
       "import": "./dist/publisher.js"
+    },
+    "./body-reader": {
+      "types": "./dist/body-reader.d.ts",
+      "import": "./dist/body-reader.js"
     }
   },
   "publishConfig": {
@@ -57,7 +61,7 @@
     "@stll/typescript-config": "0.0.0",
     "bun-types": "1.4.4",
     "fast-check": "^4.10.2",
-    "tsdown": "0.23.0"
+    "tsdown": "0.23.1"
   },
   "main": "./dist/index.js",
   "types": "./dist/index.d.ts"