@stll/fetch 0.1.1 → 0.3.0

package/README.md

@@ -6,26 +6,31 @@
 
 ```ts
 import { fetchWithTimeout } from "@stll/fetch";
+import { readBodyWithIdleDeadline } from "@stll/fetch/body-reader";
 
 // Abort when no response headers arrive within 5 seconds.
 const response = await fetchWithTimeout(url, {
   timeout: { type: "headers", ms: 5_000 },
 });
 
-// Abort when the headers, or any later body chunk, take longer than
-// 10 seconds: a slow but progressing download keeps going.
-const stream = await fetchWithTimeout(url, {
-  timeout: { type: "idle", ms: 10_000 },
+// Bound headers separately, then observe chunks in the caller's own read.
+const download = await fetchWithTimeout(url, {
+  timeout: { type: "headers", ms: 10_000 },
   signal: controller.signal, // optional; either signal aborts the request
 });
+const bytes = await readBodyWithIdleDeadline(download, {
+  idleMs: 10_000,
+  signal: controller.signal,
+});
 ```
 
-The timeout is required by the type. `headers` bounds the wait for the
-response; `idle` also restarts on every body chunk and errors the body stream
-when it stalls; `total` bounds the request and its body consumption through
-the transport's own signal and returns the transport's response unchanged. A
-timeout rejects (or errors the body) with a `TimeoutError` `DOMException`, and
-the caller's `signal` and a `Request`'s own signal still abort the request.
+The timeout is required by the type. `headers` bounds only the wait for the
+response; `total` bounds the request and its body consumption through the
+transport's own signal. Both policies return the transport's response
+unchanged. `readBodyWithIdleDeadline` consumes the caller's own body reader,
+restarts its deadline after every chunk, and cancels a stalled body with a
+`TimeoutError` `DOMException`. The caller's `signal` and a `Request`'s own
+signal still abort the request.
 `timeoutMs: number` remains the legacy numeric form of the total policy.
 
 `createFetchWithTimeout(fetcher)` returns the same function over another

package/dist/body-reader.d.ts

@@ -0,0 +1,20 @@
+declare const BodyByteLimitError_base: import("better-result").TaggedErrorClass<"BodyByteLimitError">;
+export declare class BodyByteLimitError extends BodyByteLimitError_base<{
+  message: string;
+  limit: number;
+}> {}
+type BodyIdleDeadlineOptions = {
+  idleMs: number;
+  maxBytes?: number | undefined;
+  signal?: AbortSignal | undefined;
+};
+/**
+ * Yields response chunks while bounding the wait for each successive chunk.
+ * @yields Each byte chunk from the transport body.
+ */
+export declare function iterateBodyWithIdleDeadline(response: Response, { idleMs, maxBytes, signal }: BodyIdleDeadlineOptions): AsyncGenerator<Uint8Array, void, void>;
+/** Reads a response body while bounding the wait for each successive chunk. */
+export declare const readBodyWithIdleDeadline: (response: Response, options: BodyIdleDeadlineOptions) => Promise<Uint8Array>;
+export declare const readBodyTextWithIdleDeadline: (response: Response, options: BodyIdleDeadlineOptions) => Promise<string>;
+export declare const readBodyJsonWithIdleDeadline: (response: Response, options: BodyIdleDeadlineOptions) => Promise<unknown>;
+export {};

package/dist/body-reader.js

@@ -0,0 +1,77 @@
+import { t as raceAbort } from "./race-abort-DhS8oTc3.js";
+import { TaggedError } from "better-result";
+// src/body-reader.ts
+var BodyByteLimitError = class extends TaggedError("BodyByteLimitError") {};
+const invalidIdleMs = (idleMs) => {
+	if (!Number.isFinite(idleMs) || idleMs < 0 || idleMs > 2147483647) return new DOMException("Invalid body idle deadline duration", "NotSupportedError");
+};
+const responseReader = (body) => body.getReader();
+/**
+* Yields response chunks while bounding the wait for each successive chunk.
+* @yields Each byte chunk from the transport body.
+*/
+async function* iterateBodyWithIdleDeadline(response, { idleMs, maxBytes, signal }) {
+	const invalidDuration = invalidIdleMs(idleMs);
+	if (invalidDuration !== void 0) await Promise.reject(invalidDuration);
+	if (signal?.aborted === true) {
+		response.body?.cancel(signal.reason).catch((cancellationError) => {});
+		signal.throwIfAborted();
+	}
+	if (maxBytes !== void 0 && (!Number.isSafeInteger(maxBytes) || maxBytes < 0)) await Promise.reject(new DOMException("Invalid response body byte limit", "NotSupportedError"));
+	const body = response.body;
+	if (body === null) return;
+	const reader = responseReader(body);
+	let exitReason;
+	let byteLength = 0;
+	try {
+		for (;;) {
+			if (signal?.aborted === true) exitReason = signal.reason;
+			signal?.throwIfAborted();
+			const deadline = new AbortController();
+			const timeoutError = new DOMException("Response body stalled", "TimeoutError");
+			const timer = setTimeout(() => deadline.abort(timeoutError), idleMs);
+			const combined = signal === void 0 ? deadline.signal : AbortSignal.any([signal, deadline.signal]);
+			let next;
+			try {
+				next = await raceAbort(reader.read(), combined);
+			} finally {
+				exitReason ??= combined.reason;
+				clearTimeout(timer);
+			}
+			if (next.done) return;
+			byteLength += next.value.byteLength;
+			if (maxBytes !== void 0 && byteLength > maxBytes) {
+				const limitError = new BodyByteLimitError({
+					message: `Response body exceeds the ${maxBytes}-byte limit`,
+					limit: maxBytes
+				});
+				exitReason = limitError;
+				await Promise.reject(limitError);
+			}
+			yield next.value;
+		}
+	} finally {
+		reader.cancel(exitReason).then(() => reader.releaseLock()).catch((cancellationError) => {
+			reader.releaseLock();
+		});
+	}
+}
+/** Reads a response body while bounding the wait for each successive chunk. */
+const readBodyWithIdleDeadline = async (response, options) => {
+	const chunks = [];
+	let byteLength = 0;
+	for await (const chunk of iterateBodyWithIdleDeadline(response, options)) {
+		chunks.push(chunk);
+		byteLength += chunk.byteLength;
+	}
+	const bytes = new Uint8Array(byteLength);
+	let offset = 0;
+	for (const chunk of chunks) {
+		bytes.set(chunk, offset);
+		offset += chunk.byteLength;
+	}
+	return bytes;
+};
+const readBodyTextWithIdleDeadline = async (response, options) => new TextDecoder().decode(await readBodyWithIdleDeadline(response, options));
+const readBodyJsonWithIdleDeadline = async (response, options) => JSON.parse(await readBodyTextWithIdleDeadline(response, options));
+export { BodyByteLimitError, iterateBodyWithIdleDeadline, readBodyJsonWithIdleDeadline, readBodyTextWithIdleDeadline, readBodyWithIdleDeadline };

package/dist/{index-nwAjbSQV.d.ts → index-pEynxKdA.d.ts}

@@ -1,19 +1,14 @@
-//#region src/connection-failure.d.ts
 declare const isConnectionFailure: (cause: unknown) => boolean;
-//#endregion
-//#region src/index.d.ts
 type FetchTimeout = {
   type: "total";
   ms: number;
 } | {
   type: "headers";
   ms: number;
-} | {
-  type: "idle";
-  ms: number;
 };
 type TransportRequestInit = RequestInit & {
-  timeout?: FetchTimeout;
+  /** `false` turns off the runtime's own idle limit; the signal owns the deadline. */
+  timeout?: FetchTimeout | false;
   timeoutMs?: number;
 };
 type Fetcher = (input: string | URL | Request, init?: TransportRequestInit) => Promise<Response>;
@@ -31,5 +26,4 @@ type FetchWithTimeoutInit = Omit<RequestInit, "signal"> & {
 declare const createFetchWithTimeout: (fetcher: Fetcher) => (input: string | URL | Request, init: FetchWithTimeoutInit) => Promise<Response>;
 /** Uses the current global fetch implementation at request time. */
 declare const fetchWithTimeout: (input: string | URL | Request, init: FetchWithTimeoutInit) => Promise<Response>;
-//#endregion
 export { createFetchWithTimeout as a, TransportRequestInit as i, FetchWithTimeoutInit as n, fetchWithTimeout as o, Fetcher as r, isConnectionFailure as s, FetchTimeout as t };

package/dist/index.d.ts

@@ -1,2 +1,2 @@
-import { a as createFetchWithTimeout, i as TransportRequestInit, n as FetchWithTimeoutInit, o as fetchWithTimeout, r as Fetcher, s as isConnectionFailure, t as FetchTimeout } from "./index-nwAjbSQV.js";
+import { a as createFetchWithTimeout, i as TransportRequestInit, n as FetchWithTimeoutInit, o as fetchWithTimeout, r as Fetcher, s as isConnectionFailure, t as FetchTimeout } from "./index-pEynxKdA.js";
 export { FetchTimeout, FetchWithTimeoutInit, Fetcher, TransportRequestInit, createFetchWithTimeout, fetchWithTimeout, isConnectionFailure };

package/dist/index.js

@@ -1,4 +1,4 @@
-//#region src/connection-failure.ts
+// src/connection-failure.ts
 const CONNECTION_ERROR_CODES = /* @__PURE__ */ new Set([
 	"ConnectionRefused",
 	"FailedToOpenSocket",
@@ -48,8 +48,7 @@ const isConnectionFailure = (cause) => {
 	if (codes.length > 0) return codes.some((code) => CONNECTION_ERROR_CODES.has(code) || code.startsWith("ERR_SSL_") || code.startsWith("ERR_TLS_"));
 	return messages.some((message) => /^(?:Unable to connect|Failed to fetch|fetch failed|NetworkError|Load failed|TLS handshake failed|DNS lookup failed)/iu.test(message));
 };
-//#endregion
-//#region src/index.ts
+// src/index.ts
 const executeFetchWithTimeout = async (fetcher, input, { timeout, timeoutMs, signal, ...init }) => {
 	const callerSignals = [signal, input instanceof Request ? input.signal : void 0].filter((candidate) => candidate !== void 0);
 	const activeFetcher = fetcher ?? globalThis.fetch;
@@ -58,10 +57,9 @@ const executeFetchWithTimeout = async (fetcher, input, { timeout, timeoutMs, sig
 	if (timeout === void 0 || timeout.type === "total") return await activeFetcher(input, {
 		...init,
 		signal: AbortSignal.any([...callerSignals, AbortSignal.timeout(duration)]),
-		...timeout === void 0 ? {} : { timeout },
+		timeout: false,
 		timeoutMs: duration
 	});
-	const policy = timeout;
 	const controller = new AbortController();
 	const combined = AbortSignal.any([...callerSignals, controller.signal]);
 	const expire = () => controller.abort(new DOMException("Fetch timed out", "TimeoutError"));
@@ -70,92 +68,21 @@ const executeFetchWithTimeout = async (fetcher, input, { timeout, timeoutMs, sig
 		clearTimeout(timer);
 		timer = void 0;
 	};
-	const reset = () => {
-		clear();
-		timer = setTimeout(expire, duration);
-	};
-	reset();
+	timer = setTimeout(expire, duration);
 	const response = await Promise.resolve().then(async () => await activeFetcher(input, {
 		...init,
 		signal: combined,
-		timeout: policy,
+		timeout,
 		timeoutMs: duration
 	})).finally(clear);
 	if (combined.aborted) {
 		await Promise.allSettled([response.body?.cancel(combined.reason)]);
 		throw combined.reason;
 	}
-	if (policy.type === "headers" || response.body === null) return response;
-	const reader = response.body.getReader();
-	let stopped = false;
-	let cancellation;
-	let abort;
-	const finish = () => {
-		stopped = true;
-		clear();
-		combined.removeEventListener("abort", abort);
-	};
-	const body = new ReadableStream({
-		start(stream) {
-			abort = () => {
-				if (stopped) return;
-				finish();
-				stream.error(combined.reason);
-				cancellation = reader.cancel(combined.reason).then(() => reader.releaseLock(), () => reader.releaseLock());
-			};
-			combined.addEventListener("abort", abort, { once: true });
-			if (combined.aborted) {
-				abort();
-				return;
-			}
-		},
-		async pull(stream) {
-			if (stopped) {
-				await cancellation;
-				return;
-			}
-			reset();
-			await reader.read().finally(clear).then(({ done, value }) => {
-				if (stopped) return;
-				if (done) {
-					finish();
-					reader.releaseLock();
-					stream.close();
-					return;
-				}
-				stream.enqueue(value);
-			}, (error) => {
-				if (stopped) return;
-				finish();
-				reader.releaseLock();
-				stream.error(error);
-			});
-			await cancellation;
-		},
-		async cancel(reason) {
-			if (stopped) {
-				await cancellation;
-				return;
-			}
-			finish();
-			await reader.cancel(reason).finally(() => reader.releaseLock());
-		}
-	});
-	const wrapped = new Response(body, {
-		status: response.status,
-		statusText: response.statusText,
-		headers: response.headers
-	});
-	Object.defineProperties(wrapped, {
-		url: { value: response.url },
-		redirected: { value: response.redirected },
-		type: { value: response.type }
-	});
-	return wrapped;
+	return response;
 };
 /** Creates a wrapper with the supplied transport and an explicit timeout policy. */
 const createFetchWithTimeout = (fetcher) => async (input, init) => await executeFetchWithTimeout(fetcher, input, init);
 /** Uses the current global fetch implementation at request time. */
 const fetchWithTimeout = async (input, init) => await executeFetchWithTimeout(void 0, input, init);
-//#endregion
 export { createFetchWithTimeout, fetchWithTimeout, isConnectionFailure };

package/dist/publisher.d.ts

@@ -1,6 +1,5 @@
 import { Result } from "better-result";
-import { i as TransportRequestInit } from "./index-nwAjbSQV.js";
-//#region src/publisher.d.ts
+import { i as TransportRequestInit } from "./index-pEynxKdA.js";
 export type PublisherPolicy = {
   type: "obey";
 } | {
@@ -121,5 +120,4 @@ type ReadBoundedBodyOptions = {
 };
 export declare const readBoundedBody: (response: Response, { maxBytes, signal }: ReadBoundedBodyOptions) => Promise<Result<Uint8Array, PublisherBodyLimitError>>;
 export declare const createPolitePublisherFetch: <Target extends PublisherTarget>({ validateTarget, fetcher, clock, robotsTtlMs, robotsRetryMs, robotsLoadTimeoutMs, defaultMinimumIntervalMs, onTelemetry }: PublisherFetcherOptions<Target>) => (input: string | URL, options?: PublisherFetchOptions) => Promise<Result<PublisherFetchResult, PublisherPolicyError>>;
-//#endregion
 export {};

package/dist/publisher.js

@@ -1,5 +1,6 @@
+import { t as raceAbort } from "./race-abort-DhS8oTc3.js";
 import { Result, TaggedError } from "better-result";
-//#region src/publisher.ts
+// src/publisher.ts
 const ROBOTS_MAX_BYTES = 524288;
 const ROBOTS_TTL_MS = 36e5;
 const ROBOTS_RETRY_MS = 6e4;
@@ -193,19 +194,6 @@ const evaluatePublisherRobots = (groups, productToken, path) => {
 		crawlDelayMs: Math.max(0, ...selected.map(({ crawlDelayMs }) => crawlDelayMs ?? 0))
 	};
 };
-const abortReason = (signal) => signal.reason instanceof Error ? signal.reason : new DOMException("The operation was aborted", "AbortError");
-const raceAbort = async (work, signal) => {
-	if (!signal) return await work;
-	const { promise: aborted, reject } = Promise.withResolvers();
-	const onAbort = () => reject(abortReason(signal));
-	if (signal.aborted) onAbort();
-	else signal.addEventListener("abort", onAbort, { once: true });
-	try {
-		return await Promise.race([work, aborted]);
-	} finally {
-		signal.removeEventListener("abort", onAbort);
-	}
-};
 var PublisherBodyLimitError = class extends TaggedError("publisher_body_limit") {};
 const readBoundedBody = async (response, { maxBytes, signal }) => {
 	if (!response.body) return Result.ok(/* @__PURE__ */ new Uint8Array());
@@ -666,5 +654,4 @@ const createPolitePublisherFetch = ({ validateTarget, fetcher, clock = {
 	};
 	return async (input, options = {}) => await walkPublisherRedirects(input, options, bindPublisherRuntimeToAbort(runtime, options.init?.signal ?? void 0));
 };
-//#endregion
 export { AgentExcludedError, CrawlDelayUnsupportedError, PublisherBodyLimitError, PublisherBodyNotReplayableError, PublisherBypassUnauditedError, PublisherRedirectLimitError, PublisherRequestAbortedError, PublisherTargetRefusedError, RobotsDisallowedError, RobotsUnavailableError, createPolitePublisherFetch, evaluatePublisherRobots, readBoundedBody };

package/dist/race-abort-DhS8oTc3.js

@@ -0,0 +1,15 @@
+// src/race-abort.ts
+const abortReason = (signal) => signal.reason ?? new DOMException("The operation was aborted", "AbortError");
+const raceAbort = async (work, signal) => {
+	if (signal === void 0) return await work;
+	const { promise: aborted, reject } = Promise.withResolvers();
+	const onAbort = () => reject(abortReason(signal));
+	if (signal.aborted) onAbort();
+	else signal.addEventListener("abort", onAbort, { once: true });
+	try {
+		return await Promise.race([work, aborted]);
+	} finally {
+		signal.removeEventListener("abort", onAbort);
+	}
+};
+export { raceAbort as t };

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@stll/fetch",
-  "version": "0.1.1",
-  "description": "Fetch with a mandatory timeout (headers or idle body window), connection-failure classification, and a robots-aware polite publisher fetch",
+  "version": "0.3.0",
+  "description": "Fetch with mandatory total or header timeouts, idle-bounded body reads, connection-failure classification, and a robots-aware polite publisher fetch",
   "keywords": [
     "abort",
     "fetch",
@@ -33,6 +33,10 @@
     "./publisher": {
       "types": "./dist/publisher.d.ts",
       "import": "./dist/publisher.js"
+    },
+    "./body-reader": {
+      "types": "./dist/body-reader.d.ts",
+      "import": "./dist/body-reader.js"
     }
   },
   "publishConfig": {
@@ -57,7 +61,7 @@
     "@stll/typescript-config": "0.0.0",
     "bun-types": "1.4.4",
     "fast-check": "^4.10.2",
-    "tsdown": "0.23.1"
+    "tsdown": "0.23.4"
   },
   "main": "./dist/index.js",
   "types": "./dist/index.d.ts"