@stll/fetch 0.0.1-placeholder.0 → 0.1.0

package/README.md

@@ -0,0 +1,48 @@
+# `@stll/fetch`
+
+`fetch` with a timeout you cannot forget, plus transport-failure helpers.
+
+## `fetchWithTimeout`
+
+```ts
+import { fetchWithTimeout } from "@stll/fetch";
+
+// Abort when no response headers arrive within 5 seconds.
+const response = await fetchWithTimeout(url, {
+  timeout: { type: "headers", ms: 5_000 },
+});
+
+// Abort when the headers, or any later body chunk, take longer than
+// 10 seconds: a slow but progressing download keeps going.
+const stream = await fetchWithTimeout(url, {
+  timeout: { type: "idle", ms: 10_000 },
+  signal: controller.signal, // optional; either signal aborts the request
+});
+```
+
+The timeout is required by the type. `headers` bounds the wait for the
+response; `idle` also restarts on every body chunk and errors the body stream
+when it stalls. A timeout rejects (or errors the body) with a `TimeoutError`
+`DOMException`, and the caller's `signal` and a `Request`'s own signal still
+abort the request. `timeoutMs: number` is a total deadline over the request
+and its body.
+
+`createFetchWithTimeout(fetcher)` returns the same function over another
+`fetch` implementation (a proxy-aware client, or a fake in tests).
+
+`isConnectionFailure(error)` reports whether an error, or any error in its
+`cause` chain, is a connection-level failure (refused, reset, DNS, TLS
+certificate, socket and connect timeouts) across Bun, Node and undici.
+
+## `@stll/fetch/publisher`
+
+This entry needs Bun: it waits between requests with `Bun.sleep`.
+
+`createPolitePublisherFetch({ validateTarget, fetcher })` fetches from public
+websites politely: it reads and caches each origin's `robots.txt`, honours
+`Disallow` and `Crawl-delay` rules, spaces requests per origin, follows at most
+five redirects (validating each hop with `validateTarget`), and returns refusals
+as typed `better-result` errors. Requests identify as
+`Stella/1.0 (+https://stll.app)`, the product token its robots evaluation
+uses. `readBoundedBody(response, { maxBytes, signal })` reads a body with a
+size cap.

package/dist/index-DNw-Gzhr.d.ts

@@ -0,0 +1,32 @@
+//#region src/connection-failure.d.ts
+declare const isConnectionFailure: (cause: unknown) => boolean;
+//#endregion
+//#region src/index.d.ts
+type FetchTimeout = {
+  type: "headers";
+  ms: number;
+} | {
+  type: "idle";
+  ms: number;
+};
+type TransportRequestInit = RequestInit & {
+  timeout?: FetchTimeout;
+  timeoutMs?: number;
+};
+type Fetcher = (input: string | URL | Request, init?: TransportRequestInit) => Promise<Response>;
+type FetchWithTimeoutInit = Omit<RequestInit, "signal"> & {
+  signal?: AbortSignal | undefined;
+} & ({
+  timeout: FetchTimeout;
+  timeoutMs?: never;
+} | {
+  /** Choose a headers or idle timeout for new callers. */
+  timeoutMs: number;
+  timeout?: never;
+});
+/** Creates a wrapper with the supplied transport and an explicit timeout policy. */
+declare const createFetchWithTimeout: (fetcher: Fetcher) => (input: string | URL | Request, init: FetchWithTimeoutInit) => Promise<Response>;
+/** Uses the current global fetch implementation at request time. */
+declare const fetchWithTimeout: (input: string | URL | Request, init: FetchWithTimeoutInit) => Promise<Response>;
+//#endregion
+export { createFetchWithTimeout as a, TransportRequestInit as i, FetchWithTimeoutInit as n, fetchWithTimeout as o, Fetcher as r, isConnectionFailure as s, FetchTimeout as t };

package/dist/index.d.ts

@@ -0,0 +1,2 @@
+import { a as createFetchWithTimeout, i as TransportRequestInit, n as FetchWithTimeoutInit, o as fetchWithTimeout, r as Fetcher, s as isConnectionFailure, t as FetchTimeout } from "./index-DNw-Gzhr.js";
+export { FetchTimeout, FetchWithTimeoutInit, Fetcher, TransportRequestInit, createFetchWithTimeout, fetchWithTimeout, isConnectionFailure };

package/dist/index.js

@@ -0,0 +1,158 @@
+//#region src/connection-failure.ts
+const CONNECTION_ERROR_CODES = /* @__PURE__ */ new Set([
+	"ConnectionRefused",
+	"FailedToOpenSocket",
+	"ConnectionClosed",
+	"ConnectionTimeout",
+	"SocketTimeout",
+	"EPIPE",
+	"ENOTFOUND",
+	"EAI_AGAIN",
+	"ECONNREFUSED",
+	"ECONNRESET",
+	"ETIMEDOUT",
+	"ESOCKETTIMEDOUT",
+	"EHOSTUNREACH",
+	"ENETUNREACH",
+	"ECONNABORTED",
+	"EPROTO",
+	"ERR_SOCKET_CLOSED",
+	"UND_ERR_CONNECT_TIMEOUT",
+	"UND_ERR_SOCKET",
+	"UND_ERR_HEADERS_TIMEOUT",
+	"UND_ERR_BODY_TIMEOUT",
+	"ERR_TLS_CERT_SIGNATURE_ALGORITHM_UNSUPPORTED",
+	"CERT_NOT_YET_VALID",
+	"SELF_SIGNED_CERT_IN_CHAIN",
+	"INVALID_CA",
+	"HOSTNAME_MISMATCH",
+	"ERR_TLS_CERT_ALTNAME_INVALID",
+	"CERT_HAS_EXPIRED",
+	"DEPTH_ZERO_SELF_SIGNED_CERT",
+	"UNABLE_TO_VERIFY_LEAF_SIGNATURE",
+	"UNABLE_TO_GET_ISSUER_CERT",
+	"UNABLE_TO_GET_ISSUER_CERT_LOCALLY"
+]);
+const isConnectionFailure = (cause) => {
+	const visited = /* @__PURE__ */ new Set();
+	const codes = [];
+	const messages = [];
+	let current = cause;
+	while ((typeof Error.isError === "function" && Error.isError(current) || current instanceof Error) && !visited.has(current)) {
+		visited.add(current);
+		if (current.name === "AbortError" || current.name === "TimeoutError") return false;
+		if ("code" in current && typeof current.code === "string") codes.push(current.code);
+		messages.push(current.message);
+		current = current.cause;
+	}
+	if (codes.length > 0) return codes.some((code) => CONNECTION_ERROR_CODES.has(code) || code.startsWith("ERR_SSL_") || code.startsWith("ERR_TLS_"));
+	return messages.some((message) => /^(?:Unable to connect|Failed to fetch|fetch failed|NetworkError|Load failed|TLS handshake failed|DNS lookup failed)/iu.test(message));
+};
+//#endregion
+//#region src/index.ts
+const executeFetchWithTimeout = async (fetcher, input, { timeout, timeoutMs, signal, ...init }) => {
+	const callerSignals = [signal, input instanceof Request ? input.signal : void 0].filter((candidate) => candidate !== void 0);
+	const activeFetcher = fetcher ?? globalThis.fetch;
+	if (timeout === void 0) {
+		const total = AbortSignal.timeout(timeoutMs);
+		return await activeFetcher(input, {
+			...init,
+			signal: AbortSignal.any([...callerSignals, total])
+		});
+	}
+	if (!Number.isFinite(timeout.ms) || timeout.ms < 0 || timeout.ms > 2147483647) throw new DOMException("Invalid fetch timeout duration", "NotSupportedError");
+	const controller = new AbortController();
+	const combined = AbortSignal.any([...callerSignals, controller.signal]);
+	const expire = () => controller.abort(new DOMException("Fetch timed out", "TimeoutError"));
+	let timer;
+	const clear = () => {
+		clearTimeout(timer);
+		timer = void 0;
+	};
+	const reset = () => {
+		clear();
+		timer = setTimeout(expire, timeout.ms);
+	};
+	reset();
+	const response = await Promise.resolve().then(async () => await activeFetcher(input, {
+		...init,
+		signal: combined
+	})).finally(clear);
+	if (combined.aborted) {
+		await Promise.allSettled([response.body?.cancel(combined.reason)]);
+		throw combined.reason;
+	}
+	if (timeout.type === "headers" || response.body === null) return response;
+	const reader = response.body.getReader();
+	let stopped = false;
+	let cancellation;
+	let abort;
+	const finish = () => {
+		stopped = true;
+		clear();
+		combined.removeEventListener("abort", abort);
+	};
+	const body = new ReadableStream({
+		start(stream) {
+			abort = () => {
+				if (stopped) return;
+				finish();
+				stream.error(combined.reason);
+				cancellation = reader.cancel(combined.reason).then(() => reader.releaseLock(), () => reader.releaseLock());
+			};
+			combined.addEventListener("abort", abort, { once: true });
+			if (combined.aborted) {
+				abort();
+				return;
+			}
+		},
+		async pull(stream) {
+			if (stopped) {
+				await cancellation;
+				return;
+			}
+			reset();
+			await reader.read().finally(clear).then(({ done, value }) => {
+				if (stopped) return;
+				if (done) {
+					finish();
+					reader.releaseLock();
+					stream.close();
+					return;
+				}
+				stream.enqueue(value);
+			}, (error) => {
+				if (stopped) return;
+				finish();
+				reader.releaseLock();
+				stream.error(error);
+			});
+			await cancellation;
+		},
+		async cancel(reason) {
+			if (stopped) {
+				await cancellation;
+				return;
+			}
+			finish();
+			await reader.cancel(reason).finally(() => reader.releaseLock());
+		}
+	});
+	const wrapped = new Response(body, {
+		status: response.status,
+		statusText: response.statusText,
+		headers: response.headers
+	});
+	Object.defineProperties(wrapped, {
+		url: { value: response.url },
+		redirected: { value: response.redirected },
+		type: { value: response.type }
+	});
+	return wrapped;
+};
+/** Creates a wrapper with the supplied transport and an explicit timeout policy. */
+const createFetchWithTimeout = (fetcher) => async (input, init) => await executeFetchWithTimeout(fetcher, input, init);
+/** Uses the current global fetch implementation at request time. */
+const fetchWithTimeout = async (input, init) => await executeFetchWithTimeout(void 0, input, init);
+//#endregion
+export { createFetchWithTimeout, fetchWithTimeout, isConnectionFailure };

package/dist/publisher.d.ts

@@ -0,0 +1,124 @@
+import { i as TransportRequestInit } from "./index-DNw-Gzhr.js";
+import { Result } from "better-result";
+//#region src/publisher.d.ts
+export type PublisherPolicy = {
+  type: "obey";
+} | {
+  type: "licensed";
+  basis: string;
+  evidenceUrl: string;
+};
+export type PublisherRobotsDecision = "allowed" | "allow_all_4xx" | "robots_disallowed" | "agent_excluded" | "robots_unavailable" | "licensed_bypass";
+type RefusalFields = {
+  message: string;
+  url: string;
+  origin: string;
+};
+declare const RobotsDisallowedError_base: import("better-result").TaggedErrorClass<"robots_disallowed">;
+export declare class RobotsDisallowedError extends RobotsDisallowedError_base<RefusalFields> {}
+declare const AgentExcludedError_base: import("better-result").TaggedErrorClass<"agent_excluded">;
+export declare class AgentExcludedError extends AgentExcludedError_base<RefusalFields> {}
+declare const RobotsUnavailableError_base: import("better-result").TaggedErrorClass<"robots_unavailable">;
+export declare class RobotsUnavailableError extends RobotsUnavailableError_base<RefusalFields & {
+  cause?: unknown;
+}> {}
+declare const CrawlDelayUnsupportedError_base: import("better-result").TaggedErrorClass<"crawl_delay_unsupported">;
+export declare class CrawlDelayUnsupportedError extends CrawlDelayUnsupportedError_base<RefusalFields & {
+  crawlDelayMs: number;
+  maximumMs: number;
+}> {}
+declare const PublisherRedirectLimitError_base: import("better-result").TaggedErrorClass<"publisher_redirect_limit">;
+export declare class PublisherRedirectLimitError extends PublisherRedirectLimitError_base<RefusalFields> {}
+declare const PublisherTargetRefusedError_base: import("better-result").TaggedErrorClass<"publisher_target_refused">;
+export declare class PublisherTargetRefusedError extends PublisherTargetRefusedError_base<RefusalFields & {
+  cause: unknown;
+}> {}
+declare const PublisherBodyNotReplayableError_base: import("better-result").TaggedErrorClass<"publisher_body_not_replayable">;
+export declare class PublisherBodyNotReplayableError extends PublisherBodyNotReplayableError_base<RefusalFields> {}
+declare const PublisherRequestAbortedError_base: import("better-result").TaggedErrorClass<"publisher_request_aborted">;
+export declare class PublisherRequestAbortedError extends PublisherRequestAbortedError_base<RefusalFields & {
+  cause: unknown;
+}> {}
+declare const PublisherBypassUnauditedError_base: import("better-result").TaggedErrorClass<"publisher_bypass_unaudited">;
+/** A licensed bypass is evidence that must be recorded; without a sink it is refused. */
+export declare class PublisherBypassUnauditedError extends PublisherBypassUnauditedError_base<RefusalFields & {
+  cause: unknown;
+}> {}
+export type PublisherPolicyError = RobotsDisallowedError | AgentExcludedError | RobotsUnavailableError | CrawlDelayUnsupportedError | PublisherRedirectLimitError | PublisherTargetRefusedError | PublisherBodyNotReplayableError | PublisherRequestAbortedError | PublisherBypassUnauditedError;
+export type PublisherFetchResult = {
+  response: Response;
+  robotsDecision: PublisherRobotsDecision;
+  delayAppliedMs: number;
+};
+export type PublisherTelemetry = {
+  event: "publisher_fetch_policy_bypass";
+  url: string;
+  origin: string;
+  refusal: PublisherPolicyError;
+  policy: PublisherPolicy;
+  delayMs: number;
+};
+type Clock = {
+  now: () => number;
+  sleep: (ms: number, signal?: AbortSignal) => Promise<void>;
+};
+type PublisherTarget = {
+  url: URL;
+};
+type PublisherFetcher<Target extends PublisherTarget> = (target: Target, init?: TransportRequestInit) => Promise<Response>;
+type PublisherFetcherOptions<Target extends PublisherTarget> = {
+  validateTarget: (url: URL) => Promise<Result<Target, unknown>>;
+  fetcher: PublisherFetcher<Target>;
+  clock?: Clock;
+  robotsTtlMs?: number;
+  robotsRetryMs?: number;
+  defaultMinimumIntervalMs?: number;
+  robotsLoadTimeoutMs?: number;
+  /**
+   * Audit sink for licensed bypasses, awaited before the bypass proceeds;
+   * without it, or when it throws or rejects, the bypass is refused.
+   */
+  onTelemetry?: (event: PublisherTelemetry) => void | Promise<void>;
+};
+type PublisherFetchOptions = {
+  init?: Omit<TransportRequestInit, "redirect">;
+  policy?: PublisherPolicy;
+};
+type CompiledPattern = {
+  anchored: boolean;
+  segments: {
+    literal: string;
+    prefix: number[];
+  }[];
+  specificity: number;
+};
+type Rule = {
+  type: "allow" | "disallow";
+  pattern: string;
+  compiled?: CompiledPattern;
+};
+type Group = {
+  agents: string[];
+  rules: Rule[];
+  crawlDelayMs?: number;
+};
+/** Every robots refusal a licensed policy may bypass, each only with an audit record. */
+export type PublisherBypassableRefusal = RobotsUnavailableError | CrawlDelayUnsupportedError | AgentExcludedError | RobotsDisallowedError;
+export declare const evaluatePublisherRobots: (groups: Group[], productToken: string, path: string) => {
+  allowed: boolean;
+  agentExcluded: boolean;
+  crawlDelayMs: number;
+};
+declare const PublisherBodyLimitError_base: import("better-result").TaggedErrorClass<"publisher_body_limit">;
+export declare class PublisherBodyLimitError extends PublisherBodyLimitError_base<{
+  message: string;
+  url: string;
+  maxBytes: number;
+}> {}
+type ReadBoundedBodyOptions = {
+  maxBytes: number;
+  signal: AbortSignal;
+};
+export declare const readBoundedBody: (response: Response, { maxBytes, signal }: ReadBoundedBodyOptions) => Promise<Result<Uint8Array, PublisherBodyLimitError>>;
+export declare const createPolitePublisherFetch: <Target extends PublisherTarget>({ validateTarget, fetcher, clock, robotsTtlMs, robotsRetryMs, robotsLoadTimeoutMs, defaultMinimumIntervalMs, onTelemetry }: PublisherFetcherOptions<Target>) => (input: string | URL, options?: PublisherFetchOptions) => Promise<Result<PublisherFetchResult, PublisherPolicyError>>;
+//#endregion

package/dist/publisher.js

@@ -0,0 +1,670 @@
+import { Result, TaggedError } from "better-result";
+//#region src/publisher.ts
+const ROBOTS_MAX_BYTES = 524288;
+const ROBOTS_TTL_MS = 36e5;
+const ROBOTS_RETRY_MS = 6e4;
+const DEFAULT_MINIMUM_INTERVAL_MS = 1e3;
+const MAX_CRAWL_DELAY_MS = 6e4;
+const ROBOTS_LOAD_TIMEOUT_MS = 1e4;
+const MAX_REDIRECTS = 5;
+const PRODUCT_TOKEN = "Stella";
+const PUBLISHER_USER_AGENT = `${PRODUCT_TOKEN}/1.0 (+https://stll.app)`;
+var RobotsDisallowedError = class extends TaggedError("robots_disallowed") {};
+var AgentExcludedError = class extends TaggedError("agent_excluded") {};
+var RobotsUnavailableError = class extends TaggedError("robots_unavailable") {};
+var CrawlDelayUnsupportedError = class extends TaggedError("crawl_delay_unsupported") {};
+var PublisherRedirectLimitError = class extends TaggedError("publisher_redirect_limit") {};
+var PublisherTargetRefusedError = class extends TaggedError("publisher_target_refused") {};
+var PublisherBodyNotReplayableError = class extends TaggedError("publisher_body_not_replayable") {};
+var PublisherRequestAbortedError = class extends TaggedError("publisher_request_aborted") {};
+/** A licensed bypass is evidence that must be recorded; without a sink it is refused. */
+var PublisherBypassUnauditedError = class extends TaggedError("publisher_bypass_unaudited") {};
+const publisherPolicyFor = (state, url, defaultMinimumIntervalMs) => {
+	if (state.type === "unavailable") return {
+		intervalMs: defaultMinimumIntervalMs,
+		refusal: new RobotsUnavailableError({
+			message: "robots.txt is unavailable",
+			url: url.href,
+			origin: url.origin,
+			cause: state.cause
+		})
+	};
+	if (state.type === "allow-all") return {
+		decision: "allow_all_4xx",
+		intervalMs: defaultMinimumIntervalMs
+	};
+	const evaluated = evaluatePublisherRobots(state.groups, PRODUCT_TOKEN, `${url.pathname}${url.search}`);
+	if (evaluated.crawlDelayMs > MAX_CRAWL_DELAY_MS) return {
+		intervalMs: evaluated.crawlDelayMs,
+		refusal: new CrawlDelayUnsupportedError({
+			message: "robots.txt crawl delay exceeds the supported maximum",
+			url: url.href,
+			origin: url.origin,
+			crawlDelayMs: evaluated.crawlDelayMs,
+			maximumMs: MAX_CRAWL_DELAY_MS
+		})
+	};
+	let refusal;
+	if (!evaluated.allowed) refusal = evaluated.agentExcluded ? new AgentExcludedError({
+		message: "robots.txt excludes the stella agent",
+		url: url.href,
+		origin: url.origin
+	}) : new RobotsDisallowedError({
+		message: "robots.txt disallows this URL",
+		url: url.href,
+		origin: url.origin
+	});
+	return {
+		intervalMs: Math.max(defaultMinimumIntervalMs, evaluated.crawlDelayMs),
+		...refusal ? { refusal } : {}
+	};
+};
+const normalizeOctets = (value) => {
+	const percentNormalized = value.replaceAll(/%[\da-f]{2}/giu, (encoded) => {
+		const octet = Number.parseInt(encoded.slice(1), 16);
+		const character = String.fromCodePoint(octet);
+		return /^[A-Za-z0-9._~-]$/u.test(character) ? character : `%${encoded.slice(1).toUpperCase()}`;
+	});
+	let normalized = "";
+	for (const character of percentNormalized) {
+		if ((character.codePointAt(0) ?? Number.POSITIVE_INFINITY) <= 127) {
+			normalized += character;
+			continue;
+		}
+		for (const octet of new TextEncoder().encode(character)) normalized += `%${octet.toString(16).toUpperCase().padStart(2, "0")}`;
+	}
+	return normalized;
+};
+const parseRobots = (body) => {
+	const groups = [];
+	let current;
+	let sawRule = false;
+	for (const rawLine of body.split(/\r\n|\r|\n/u)) {
+		const comment = rawLine.indexOf("#");
+		const line = rawLine.slice(0, comment === -1 ? void 0 : comment).trim();
+		const separator = line.indexOf(":");
+		if (separator === -1) continue;
+		const field = line.slice(0, separator).trim().toLowerCase();
+		const value = line.slice(separator + 1).trim();
+		if (field === "user-agent") {
+			if (!current || sawRule) {
+				current = {
+					agents: [],
+					rules: []
+				};
+				groups.push(current);
+				sawRule = false;
+			}
+			current.agents.push(value.toLowerCase());
+			continue;
+		}
+		if (!current || current.agents.length === 0) continue;
+		if (field === "allow" || field === "disallow") {
+			sawRule = true;
+			if (value !== "") current.rules.push({
+				type: field,
+				pattern: value,
+				compiled: compilePattern(value)
+			});
+			continue;
+		}
+		if (field === "crawl-delay") {
+			const seconds = Number(value);
+			if (Number.isFinite(seconds) && seconds >= 0) {
+				sawRule = true;
+				current.crawlDelayMs = seconds * 1e3;
+			}
+		}
+	}
+	return groups;
+};
+const selectedGroups = (groups, productToken) => {
+	const token = productToken.toLowerCase();
+	const named = groups.filter(({ agents }) => agents.includes(token));
+	return named.length > 0 ? named : groups.filter(({ agents }) => agents.includes("*"));
+};
+const compilePattern = (pattern) => {
+	const normalized = normalizeOctets(pattern);
+	const anchored = normalized.endsWith("$");
+	const body = anchored ? normalized.slice(0, -1) : normalized;
+	return {
+		anchored,
+		segments: body.split("*").map((literal) => {
+			const prefix = Array.from({ length: literal.length }, () => 0);
+			for (let index = 1, matched = 0; index < literal.length; index += 1) {
+				while (matched > 0 && literal[index] !== literal[matched]) matched = prefix.at(matched - 1) ?? 0;
+				if (literal[index] === literal[matched]) matched += 1;
+				prefix[index] = matched;
+			}
+			return {
+				literal,
+				prefix
+			};
+		}),
+		specificity: body.replaceAll(/%[\dA-F]{2}/gu, "%").length
+	};
+};
+const findSegment = (value, start, segment) => {
+	if (segment.literal.length === 0) return start;
+	for (let index = start, matched = 0; index < value.length; index += 1) {
+		while (matched > 0 && value[index] !== segment.literal[matched]) matched = segment.prefix.at(matched - 1) ?? 0;
+		if (value[index] === segment.literal[matched]) matched += 1;
+		if (matched === segment.literal.length) return index - matched + 1;
+	}
+	return -1;
+};
+const wildcardMatches = (compiled, value) => {
+	const first = compiled.segments.at(0);
+	if (!first) return false;
+	if (!value.startsWith(first.literal)) return false;
+	let position = first.literal.length;
+	const lastIndex = compiled.segments.length - 1;
+	for (let index = 1; index < lastIndex; index += 1) {
+		const segment = compiled.segments.at(index);
+		if (!segment) return false;
+		const found = findSegment(value, position, segment);
+		if (found === -1) return false;
+		position = found + segment.literal.length;
+	}
+	if (lastIndex === 0) return !compiled.anchored || position === value.length;
+	const last = compiled.segments.at(lastIndex);
+	if (!last) return false;
+	if (compiled.anchored) return value.length - last.literal.length >= position && value.endsWith(last.literal);
+	return findSegment(value, position, last) !== -1;
+};
+const evaluatePublisherRobots = (groups, productToken, path) => {
+	const selected = selectedGroups(groups, productToken);
+	const named = selected.some((group) => group.agents.some((agent) => agent !== "*" && productToken.toLowerCase().startsWith(agent)));
+	const rules = selected.flatMap(({ rules: groupRules }) => groupRules);
+	const normalizedPath = normalizeOctets(path);
+	const matches = rules.map((rule) => ({
+		...rule,
+		compiled: rule.compiled ?? compilePattern(rule.pattern)
+	})).filter(({ compiled }) => wildcardMatches(compiled, normalizedPath)).map((rule) => ({
+		...rule,
+		specificity: rule.compiled.specificity
+	}));
+	const longest = Math.max(-1, ...matches.map(({ specificity }) => specificity));
+	const winners = matches.filter(({ specificity }) => specificity === longest);
+	const allowed = longest < 0 || winners.some(({ type }) => type === "allow");
+	return {
+		allowed,
+		agentExcluded: named && rules.some(({ type, pattern }) => type === "disallow" && pattern === "/") && !allowed,
+		crawlDelayMs: Math.max(0, ...selected.map(({ crawlDelayMs }) => crawlDelayMs ?? 0))
+	};
+};
+const abortReason = (signal) => signal.reason instanceof Error ? signal.reason : new DOMException("The operation was aborted", "AbortError");
+const raceAbort = async (work, signal) => {
+	if (!signal) return await work;
+	const { promise: aborted, reject } = Promise.withResolvers();
+	const onAbort = () => reject(abortReason(signal));
+	if (signal.aborted) onAbort();
+	else signal.addEventListener("abort", onAbort, { once: true });
+	try {
+		return await Promise.race([work, aborted]);
+	} finally {
+		signal.removeEventListener("abort", onAbort);
+	}
+};
+var PublisherBodyLimitError = class extends TaggedError("publisher_body_limit") {};
+const readBoundedBody = async (response, { maxBytes, signal }) => {
+	if (!response.body) return Result.ok(/* @__PURE__ */ new Uint8Array());
+	const chunks = [];
+	let size = 0;
+	let exceeded = false;
+	const reader = response.body.getReader();
+	try {
+		while (true) {
+			const next = await raceAbort(reader.read(), signal);
+			if (next.done) break;
+			size += next.value.byteLength;
+			if (size > maxBytes) {
+				exceeded = true;
+				break;
+			}
+			chunks.push(next.value);
+		}
+	} finally {
+		try {
+			await raceAbort(reader.cancel(), signal);
+		} finally {
+			reader.releaseLock();
+		}
+	}
+	if (exceeded) return Result.err(new PublisherBodyLimitError({
+		message: "response body exceeds the byte limit",
+		url: response.url,
+		maxBytes
+	}));
+	const bytes = new Uint8Array(size);
+	let offset = 0;
+	for (const chunk of chunks) {
+		bytes.set(chunk, offset);
+		offset += chunk.byteLength;
+	}
+	return Result.ok(bytes);
+};
+const readRobotsText = async (response, signal) => {
+	const bytes = await readBoundedBody(response, {
+		maxBytes: ROBOTS_MAX_BYTES,
+		signal
+	});
+	return Result.isError(bytes) ? Result.err(new RobotsUnavailableError({
+		message: "robots.txt exceeds the byte limit",
+		url: response.url || "robots.txt",
+		origin: response.url ? new URL(response.url).origin : "unknown"
+	})) : Result.ok(new TextDecoder().decode(bytes.value));
+};
+const REDIRECT_STATUSES = /* @__PURE__ */ new Set([
+	301,
+	302,
+	303,
+	307,
+	308
+]);
+const isRedirect = (response) => REDIRECT_STATUSES.has(response.status);
+const validatePublisherTarget = async (url, validateTarget) => {
+	const validated = await validateTarget(url);
+	return Result.isError(validated) ? Result.err(new PublisherTargetRefusedError({
+		message: "Publisher target was refused",
+		url: url.href,
+		origin: url.origin,
+		cause: validated.error
+	})) : validated;
+};
+const redirectedRequestInit = ({ response, currentUrl, nextUrl, requestInit, headers }) => {
+	const method = requestInit?.method?.toUpperCase() ?? "GET";
+	const dropBody = response.status === 303 && method !== "HEAD" || (response.status === 301 || response.status === 302) && method === "POST";
+	if (!dropBody && requestInit?.body instanceof ReadableStream) return Result.err(new PublisherBodyNotReplayableError({
+		message: "Publisher redirect requires a replayable request body",
+		url: currentUrl.href,
+		origin: currentUrl.origin
+	}));
+	const redirectedHeaders = new Headers(headers);
+	if (nextUrl.origin !== currentUrl.origin) for (const name of [
+		"Authorization",
+		"Cookie",
+		"Proxy-Authorization"
+	]) redirectedHeaders.delete(name);
+	if (dropBody) for (const name of [
+		"Content-Type",
+		"Content-Length",
+		"Content-Encoding",
+		"Content-Language",
+		"Content-Location"
+	]) redirectedHeaders.delete(name);
+	return Result.ok({
+		...requestInit,
+		...dropBody ? {
+			body: null,
+			method: "GET"
+		} : {},
+		headers: redirectedHeaders
+	});
+};
+const discardBody = async (response, signal) => Result.tryPromise(async () => raceAbort(response.body?.cancel() ?? Promise.resolve(), signal));
+const loadRobots = async ({ fetcher, validateTarget, origin, now, robotsTtlMs, robotsRetryMs, robotsLoadTimeoutMs }) => {
+	const deadline = AbortSignal.timeout(robotsLoadTimeoutMs);
+	let target = new URL("/robots.txt", origin);
+	let finalResponse;
+	let failure;
+	for (let redirects = 0;; redirects += 1) {
+		const requestTarget = target;
+		const checked = await Result.tryPromise(async () => raceAbort(validatePublisherTarget(requestTarget, validateTarget), deadline));
+		if (Result.isError(checked)) {
+			failure = checked.error.cause;
+			break;
+		}
+		const validated = checked.value;
+		if (Result.isError(validated)) {
+			failure = validated.error;
+			break;
+		}
+		const fetched = await Result.tryPromise(async () => raceAbort(fetcher(validated.value, {
+			headers: { "User-Agent": PUBLISHER_USER_AGENT },
+			redirect: "manual",
+			signal: deadline
+		}), deadline));
+		if (Result.isError(fetched)) {
+			failure = fetched.error.cause;
+			break;
+		}
+		if (!isRedirect(fetched.value)) {
+			finalResponse = fetched.value;
+			break;
+		}
+		const cancelled = await discardBody(fetched.value, deadline);
+		if (Result.isError(cancelled)) {
+			failure = cancelled.error.cause;
+			break;
+		}
+		const location = fetched.value.headers.get("location");
+		if (!location || redirects >= MAX_REDIRECTS) {
+			failure = new RobotsUnavailableError({
+				message: location ? "robots.txt exceeded the redirect limit" : "robots.txt redirect has no location",
+				url: requestTarget.href,
+				origin
+			});
+			break;
+		}
+		if (!URL.canParse(location, requestTarget.href)) {
+			failure = new RobotsUnavailableError({
+				message: "robots.txt redirect location is invalid",
+				url: requestTarget.href,
+				origin
+			});
+			break;
+		}
+		target = new URL(location, requestTarget);
+	}
+	if (!finalResponse) {
+		if (PublisherTargetRefusedError.is(failure)) return {
+			type: "refused",
+			state: {
+				type: "unavailable",
+				cause: failure
+			}
+		};
+		return {
+			type: "cacheable",
+			robots: {
+				state: {
+					type: "unavailable",
+					cause: failure
+				},
+				expiresAt: now() + robotsRetryMs
+			}
+		};
+	}
+	if (finalResponse.status >= 400) {
+		const discarded = await discardBody(finalResponse, deadline);
+		if (Result.isError(discarded)) return {
+			type: "cacheable",
+			robots: {
+				state: {
+					type: "unavailable",
+					cause: discarded.error.cause
+				},
+				expiresAt: now() + robotsRetryMs
+			}
+		};
+	}
+	if (finalResponse.status >= 400 && finalResponse.status < 500) return {
+		type: "cacheable",
+		robots: {
+			state: { type: "allow-all" },
+			expiresAt: now() + robotsTtlMs
+		}
+	};
+	if (finalResponse.status >= 500) return {
+		type: "cacheable",
+		robots: {
+			state: { type: "unavailable" },
+			expiresAt: now() + robotsRetryMs
+		}
+	};
+	const body = await Result.tryPromise(async () => readRobotsText(finalResponse, deadline));
+	if (Result.isError(body)) return {
+		type: "cacheable",
+		robots: {
+			state: {
+				type: "unavailable",
+				cause: body.error.cause
+			},
+			expiresAt: now() + robotsRetryMs
+		}
+	};
+	if (Result.isError(body.value)) return {
+		type: "cacheable",
+		robots: {
+			state: {
+				type: "unavailable",
+				cause: body.value.error
+			},
+			expiresAt: now() + robotsRetryMs
+		}
+	};
+	return {
+		type: "cacheable",
+		robots: {
+			state: {
+				type: "rules",
+				groups: parseRobots(body.value.value)
+			},
+			expiresAt: now() + robotsTtlMs
+		}
+	};
+};
+const createRobotsCache = (options) => {
+	const settled = /* @__PURE__ */ new Map();
+	const pending = /* @__PURE__ */ new Map();
+	return async (origin) => {
+		const cached = settled.get(origin);
+		if (cached && cached.expiresAt > options.now()) return cached.state;
+		const activeLoad = pending.get(origin);
+		if (activeLoad) {
+			const activeResult = await activeLoad;
+			if (activeResult.type === "cacheable") return activeResult.robots.state;
+		}
+		const load = loadRobots({
+			...options,
+			origin
+		});
+		pending.set(origin, load);
+		const loaded = await load;
+		if (pending.get(origin) === load) pending.delete(origin);
+		if (loaded.type === "refused") return loaded.state;
+		settled.set(origin, loaded.robots);
+		return loaded.robots.state;
+	};
+};
+const createOriginScheduler = (clock) => {
+	const queues = /* @__PURE__ */ new Map();
+	const nextAllowedAt = /* @__PURE__ */ new Map();
+	return async (origin, intervalMs, signal) => {
+		const previous = queues.get(origin) ?? Promise.resolve();
+		let release;
+		const current = new Promise((resolve) => {
+			release = resolve;
+		});
+		queues.set(origin, previous.then(async () => current));
+		const acquired = await Result.tryPromise(async () => raceAbort(previous, signal));
+		if (Result.isError(acquired)) {
+			release();
+			const cause = acquired.error.cause;
+			return await Promise.reject(cause instanceof Error ? cause : new DOMException("The operation was aborted", "AbortError"));
+		}
+		const delay = Math.max(0, (nextAllowedAt.get(origin) ?? clock.now()) - clock.now());
+		try {
+			if (delay > 0) await raceAbort(clock.sleep(delay, signal), signal);
+			nextAllowedAt.set(origin, clock.now() + intervalMs);
+			return delay;
+		} finally {
+			release();
+		}
+	};
+};
+const abortBoundRuntimeBrand = Symbol("abort-bound-publisher-runtime");
+const bindPublisherRuntimeToAbort = (runtime, signal) => {
+	const onTelemetry = runtime.onTelemetry;
+	const bound = {
+		fetcher: async (...args) => await raceAbort(runtime.fetcher(...args), signal),
+		validateTarget: async (...args) => await raceAbort(runtime.validateTarget(...args), signal),
+		robotsFor: async (...args) => await raceAbort(runtime.robotsFor(...args), signal),
+		reserve: async (...args) => await raceAbort(runtime.reserve(...args), signal),
+		defaultMinimumIntervalMs: runtime.defaultMinimumIntervalMs,
+		onTelemetry: onTelemetry ? async (...args) => await raceAbort(onTelemetry(...args), signal) : void 0
+	};
+	return Object.assign(bound, { [abortBoundRuntimeBrand]: true });
+};
+const enforcePublisherPolicy = async (state, options, runtime) => {
+	const signal = state.requestInit?.signal ?? void 0;
+	const loaded = await Result.tryPromise(async () => runtime.robotsFor(state.url.origin));
+	if (Result.isError(loaded)) return Result.err(new PublisherRequestAbortedError({
+		message: "Publisher request was aborted while loading robots.txt",
+		url: state.url.href,
+		origin: state.url.origin,
+		cause: loaded.error.cause
+	}));
+	const robots = loaded.value;
+	if (robots.type === "unavailable" && PublisherTargetRefusedError.is(robots.cause)) return Result.err(robots.cause);
+	const evaluated = publisherPolicyFor(robots, state.url, runtime.defaultMinimumIntervalMs);
+	state.decision = evaluated.decision ?? state.decision;
+	const intervalMs = Math.min(evaluated.intervalMs, MAX_CRAWL_DELAY_MS);
+	const refusal = evaluated.refusal;
+	if (!refusal) return Result.ok(intervalMs);
+	const policy = state.url.origin === state.requestedOrigin ? options.policy : void 0;
+	if (policy?.type !== "licensed") return Result.err(refusal);
+	if (runtime.onTelemetry === void 0) return Result.err(new PublisherBypassUnauditedError({
+		message: "A licensed bypass needs an onTelemetry sink to record its evidence",
+		url: state.url.href,
+		origin: state.url.origin,
+		cause: refusal
+	}));
+	const sink = runtime.onTelemetry;
+	const recorded = await Result.tryPromise(async () => {
+		await sink({
+			event: "publisher_fetch_policy_bypass",
+			url: state.url.href,
+			origin: state.url.origin,
+			refusal,
+			policy,
+			delayMs: intervalMs
+		});
+	});
+	if (Result.isError(recorded)) {
+		if (signal?.aborted) return Result.err(new PublisherRequestAbortedError({
+			message: "Publisher request was aborted while recording its licensed bypass",
+			url: state.url.href,
+			origin: state.url.origin,
+			cause: recorded.error.cause
+		}));
+		return Result.err(new PublisherBypassUnauditedError({
+			message: "Recording the licensed bypass failed",
+			url: state.url.href,
+			origin: state.url.origin,
+			cause: recorded.error.cause
+		}));
+	}
+	state.decision = "licensed_bypass";
+	return Result.ok(intervalMs);
+};
+const redirectLimitResult = (state) => {
+	const refusal = new PublisherRedirectLimitError({
+		message: "publisher response exceeded the redirect limit",
+		url: state.url.href,
+		origin: state.url.origin
+	});
+	return Result.err(refusal);
+};
+const walkPublisherRedirects = async (input, options, runtime) => {
+	const url = new URL(input);
+	const state = {
+		requestedOrigin: url.origin,
+		url,
+		requestInit: options.init,
+		decision: "allowed",
+		delayAppliedMs: 0
+	};
+	for (let redirects = 0;; redirects += 1) {
+		const signal = state.requestInit?.signal ?? void 0;
+		const checked = await Result.tryPromise(async () => validatePublisherTarget(state.url, runtime.validateTarget));
+		if (Result.isError(checked)) return Result.err(new PublisherRequestAbortedError({
+			message: "Publisher request was aborted while validating its target",
+			url: state.url.href,
+			origin: state.url.origin,
+			cause: checked.error.cause
+		}));
+		const validated = checked.value;
+		if (Result.isError(validated)) return validated;
+		state.url = validated.value.url;
+		const interval = await enforcePublisherPolicy(state, options, runtime);
+		if (Result.isError(interval)) return interval;
+		const reserved = await Result.tryPromise(async () => runtime.reserve(state.url.origin, interval.value, state.requestInit?.signal ?? void 0));
+		if (Result.isError(reserved)) return Result.err(new PublisherRequestAbortedError({
+			message: "Publisher request was aborted while waiting",
+			url: state.url.href,
+			origin: state.url.origin,
+			cause: reserved.error.cause
+		}));
+		state.delayAppliedMs += reserved.value;
+		const headers = new Headers(state.requestInit?.headers);
+		headers.set("User-Agent", PUBLISHER_USER_AGENT);
+		const fetched = await Result.tryPromise(async () => runtime.fetcher(validated.value, {
+			...state.requestInit,
+			headers,
+			redirect: "manual"
+		}));
+		if (Result.isError(fetched)) {
+			if (!signal?.aborted) {
+				const cause = fetched.error.cause;
+				return await Promise.reject(cause instanceof Error ? cause : new Error("Publisher fetch failed", { cause }));
+			}
+			return Result.err(new PublisherRequestAbortedError({
+				message: "Publisher request was aborted while fetching the resource",
+				url: state.url.href,
+				origin: state.url.origin,
+				cause: fetched.error.cause
+			}));
+		}
+		const response = fetched.value;
+		const location = response.headers.get("location");
+		if (!isRedirect(response) || !location) return Result.ok({
+			response,
+			robotsDecision: state.decision,
+			delayAppliedMs: state.delayAppliedMs
+		});
+		const discarded = await discardBody(response, state.requestInit?.signal ?? void 0);
+		if (Result.isError(discarded)) return Result.err(new PublisherRequestAbortedError({
+			message: "Publisher request was aborted while discarding a redirect",
+			url: state.url.href,
+			origin: state.url.origin,
+			cause: discarded.error.cause
+		}));
+		if (redirects >= MAX_REDIRECTS) return redirectLimitResult(state);
+		if (!URL.canParse(location, state.url.href)) return Result.err(new PublisherTargetRefusedError({
+			message: "publisher redirect location is invalid",
+			url: state.url.href,
+			origin: state.url.origin,
+			cause: location
+		}));
+		const nextUrl = new URL(location, state.url);
+		const redirected = redirectedRequestInit({
+			response,
+			currentUrl: state.url,
+			nextUrl,
+			requestInit: state.requestInit,
+			headers
+		});
+		if (Result.isError(redirected)) return redirected;
+		state.requestInit = redirected.value;
+		state.url = nextUrl;
+	}
+};
+const createPolitePublisherFetch = ({ validateTarget, fetcher, clock = {
+	now: () => performance.now(),
+	sleep: async (ms, signal) => {
+		signal?.throwIfAborted();
+		await raceAbort(Bun.sleep(ms), signal);
+		signal?.throwIfAborted();
+	}
+}, robotsTtlMs = ROBOTS_TTL_MS, robotsRetryMs = ROBOTS_RETRY_MS, robotsLoadTimeoutMs = ROBOTS_LOAD_TIMEOUT_MS, defaultMinimumIntervalMs = DEFAULT_MINIMUM_INTERVAL_MS, onTelemetry }) => {
+	const runtime = {
+		fetcher,
+		validateTarget,
+		robotsFor: createRobotsCache({
+			fetcher,
+			validateTarget,
+			now: clock.now,
+			robotsTtlMs,
+			robotsRetryMs,
+			robotsLoadTimeoutMs
+		}),
+		reserve: createOriginScheduler(clock),
+		defaultMinimumIntervalMs,
+		onTelemetry: onTelemetry ? async (event) => await onTelemetry(event) : void 0
+	};
+	return async (input, options = {}) => await walkPublisherRedirects(input, options, bindPublisherRuntimeToAbort(runtime, options.init?.signal ?? void 0));
+};
+//#endregion
+export { AgentExcludedError, CrawlDelayUnsupportedError, PublisherBodyLimitError, PublisherBodyNotReplayableError, PublisherBypassUnauditedError, PublisherRedirectLimitError, PublisherRequestAbortedError, PublisherTargetRefusedError, RobotsDisallowedError, RobotsUnavailableError, createPolitePublisherFetch, evaluatePublisherRobots, readBoundedBody };

package/package.json

@@ -1,6 +1,64 @@
 {
   "name": "@stll/fetch",
-  "version": "0.0.1-placeholder.0",
-  "description": "Placeholder to bootstrap npm trusted publishing. Real releases are published from stella/stella via GitHub Actions.",
-  "license": "Apache-2.0"
+  "version": "0.1.0",
+  "description": "Fetch with a mandatory timeout (headers or idle body window), connection-failure classification, and a robots-aware polite publisher fetch",
+  "keywords": [
+    "abort",
+    "fetch",
+    "http",
+    "robots",
+    "timeout"
+  ],
+  "homepage": "https://github.com/stella/stella/tree/main/packages/fetch",
+  "bugs": {
+    "url": "https://github.com/stella/stella/issues"
+  },
+  "license": "Apache-2.0",
+  "repository": {
+    "type": "git",
+    "url": "https://github.com/stella/stella.git",
+    "directory": "packages/fetch"
+  },
+  "files": [
+    "dist",
+    "README.md"
+  ],
+  "type": "module",
+  "sideEffects": false,
+  "exports": {
+    ".": {
+      "types": "./dist/index.d.ts",
+      "import": "./dist/index.js"
+    },
+    "./publisher": {
+      "types": "./dist/publisher.d.ts",
+      "import": "./dist/publisher.js"
+    }
+  },
+  "publishConfig": {
+    "access": "public"
+  },
+  "scripts": {
+    "clean": "git clean -xdf dist .cache .turbo node_modules",
+    "build": "tsdown",
+    "pack:dry-run": "bun pm pack --dry-run",
+    "test": "bun test src",
+    "typecheck": "bun check --no-pretty --all --project=tsconfig.json",
+    "lint": "cd ../.. && bun --bun oxlint -c oxlint.config.ts --report-unused-disable-directives-severity=error --type-aware --type-check packages/fetch",
+    "lint:fix": "cd ../.. && bun --bun oxlint -c oxlint.config.ts --type-aware --fix packages/fetch",
+    "test:property": "bun test --preload @stll/property-testing/preload $(grep -rlE 'fc\\.assert|assertProperty' src --include='*.test.ts')",
+    "prepack": "bun run build"
+  },
+  "dependencies": {
+    "better-result": "3.0.1"
+  },
+  "devDependencies": {
+    "@stll/property-testing": "0.0.0",
+    "@stll/typescript-config": "0.0.0",
+    "bun-types": "1.4.4",
+    "fast-check": "^4.10.2",
+    "tsdown": "0.23.0"
+  },
+  "main": "./dist/index.js",
+  "types": "./dist/index.d.ts"
 }