@a-t-h-i/bot-lobby 0.6.2 → 0.6.4

package/src/web/fetch.ts

@@ -0,0 +1,232 @@
+/**
+ * Fetching a page for a model, safely: http(s) only, public addresses only
+ * (a page cannot send the researcher to localhost, the LAN or a cloud
+ * metadata endpoint, through a redirect either), a size cap, a time limit,
+ * and the body decoded by its declared charset.
+ * `BOT_LOBBY_WEB_ALLOW_PRIVATE=1` lifts the address rule (a local docs server).
+ */
+import { lookup as dnsLookup } from "node:dns/promises";
+import { isIP } from "node:net";
+
+export type Fetcher = typeof fetch;
+export type Lookup = (host: string) => Promise<string[]>;
+
+export interface FetchOptions {
+  signal?: AbortSignal;
+  /** Bytes read at most; the rest is cut off. */
+  maxBytes?: number;
+  timeoutMs?: number;
+  accept?: string;
+  method?: "GET" | "POST";
+  body?: string;
+  headers?: Record<string, string>;
+  /** Swappable for tests. */
+  fetch?: Fetcher;
+  lookup?: Lookup;
+  allowPrivate?: boolean;
+}
+
+export interface FetchedPage {
+  /** Where the page was finally read from, after redirects. */
+  url: string;
+  status: number;
+  statusText: string;
+  contentType: string;
+  headers: Headers;
+  /** The body as text; empty for a binary one (a PDF, an image), which is not read. */
+  text: string;
+  bytes: number;
+  truncated: boolean;
+  binary: boolean;
+  redirects: string[];
+}
+
+export const MAX_BYTES = 3 * 1024 * 1024;
+export const TIMEOUT_MS = 20_000;
+const MAX_REDIRECTS = 5;
+export const USER_AGENT = "Mozilla/5.0 (compatible; bot-lobby research; +https://github.com/a-t-h-i/bot-lobby)";
+const ACCEPT = "text/html,application/xhtml+xml,text/markdown;q=0.9,text/plain;q=0.8,application/json;q=0.7,*/*;q=0.5";
+
+/** True for an address that is not on the public internet. */
+export function isPrivateAddress(address: string): boolean {
+  const ip = address.replace(/^\[|\]$/g, "").toLowerCase();
+  const version = isIP(ip);
+  if (version === 4) {
+    const [a = 0, b = 0] = ip.split(".").map(Number);
+    return a === 0 || a === 10 || a === 127 || a >= 224 || (a === 100 && b >= 64 && b <= 127) || (a === 169 && b === 254) || (a === 172 && b >= 16 && b <= 31) || (a === 192 && b === 168) || (a === 192 && b === 0) || (a === 198 && (b === 18 || b === 19));
+  }
+  if (version === 6) {
+    if (ip === "::" || ip === "::1") return true;
+    const mapped = /^::ffff:(\d+\.\d+\.\d+\.\d+)$/.exec(ip) ?? /^64:ff9b::(\d+\.\d+\.\d+\.\d+)$/.exec(ip);
+    if (mapped) return isPrivateAddress(mapped[1]!);
+    if (/^::ffff:[0-9a-f]{1,4}:[0-9a-f]{1,4}$/.test(ip)) {
+      const [high = "0", low = "0"] = ip.slice(7).split(":");
+      const value = (Number.parseInt(high, 16) << 16) | Number.parseInt(low, 16);
+      return isPrivateAddress([value >>> 24, (value >>> 16) & 255, (value >>> 8) & 255, value & 255].join("."));
+    }
+    return /^(fc|fd|fe[89ab]|ff)/.test(ip);
+  }
+  return false;
+}
+
+const defaultLookup: Lookup = async (host) => (await dnsLookup(host, { all: true, verbatim: true })).map((entry) => entry.address);
+
+/**
+ * Why a URL may not be fetched, or undefined when it may: not http(s), a
+ * private or local host, or a name that resolves to one. A name that does not
+ * resolve here is let through (a proxy may resolve it; the fetch then says).
+ */
+export async function blockedReason(url: URL, lookup: Lookup = defaultLookup, allowPrivate = process.env.BOT_LOBBY_WEB_ALLOW_PRIVATE === "1"): Promise<string | undefined> {
+  if (url.protocol !== "http:" && url.protocol !== "https:") return `only http and https pages can be fetched, not ${url.protocol.replace(/:$/, "")}`;
+  if (url.username || url.password) return "URLs with credentials in them are not fetched";
+  if (allowPrivate) return undefined;
+  const host = url.hostname.replace(/^\[|\]$/g, "").toLowerCase().replace(/\.$/, "");
+  const local = "is a private or local address; only public pages are fetched (BOT_LOBBY_WEB_ALLOW_PRIVATE=1 allows it)";
+  if (isIP(host)) return isPrivateAddress(host) ? `${host} ${local}` : undefined;
+  if (host === "localhost" || /\.(localhost|local|internal|home\.arpa|lan)$/.test(host) || !host.includes(".")) return `${host} ${local}`;
+  let addresses: string[];
+  try {
+    addresses = await lookup(host);
+  } catch {
+    return undefined;
+  }
+  const inside = addresses.find(isPrivateAddress);
+  return inside ? `${host} resolves to ${inside}, which ${local}` : undefined;
+}
+
+function charsetOf(contentType: string, head: Uint8Array): string {
+  const declared = /charset\s*=\s*"?([\w.:-]+)/i.exec(contentType)?.[1];
+  if (declared) return declared;
+  if (!/html|xml/i.test(contentType)) return "utf-8";
+  const sniff = new TextDecoder("latin1").decode(head.subarray(0, 2048));
+  return /<meta[^>]+charset\s*=\s*["']?([\w.:-]+)/i.exec(sniff)?.[1] ?? "utf-8";
+}
+
+function decode(bytes: Uint8Array, charset: string): string {
+  try {
+    return new TextDecoder(charset).decode(bytes);
+  } catch {
+    return new TextDecoder("utf-8").decode(bytes);
+  }
+}
+
+async function readCapped(response: Response, maxBytes: number): Promise<{ bytes: Uint8Array; truncated: boolean }> {
+  if (!response.body) return { bytes: new Uint8Array(await response.arrayBuffer()).subarray(0, maxBytes), truncated: false };
+  const reader = response.body.getReader();
+  const chunks: Uint8Array[] = [];
+  let size = 0;
+  let truncated = false;
+  for (;;) {
+    const { done, value } = await reader.read();
+    if (done) break;
+    if (size + value.byteLength > maxBytes) {
+      chunks.push(value.subarray(0, maxBytes - size));
+      size = maxBytes;
+      truncated = true;
+      await reader.cancel().catch(() => {});
+      break;
+    }
+    chunks.push(value);
+    size += value.byteLength;
+  }
+  const bytes = new Uint8Array(size);
+  let offset = 0;
+  for (const chunk of chunks) {
+    bytes.set(chunk, offset);
+    offset += chunk.byteLength;
+  }
+  return { bytes, truncated };
+}
+
+const TEXTUAL = /^(text\/|application\/(json|xml|xhtml\+xml|javascript|x-javascript|ecmascript|x-yaml|yaml|toml)\b|[^;]*\+(json|xml)\b)/i;
+
+/** True for a content type read as text (an unlabelled body counts). */
+export function isTextual(contentType: string): boolean {
+  return !contentType.trim() || TEXTUAL.test(contentType.trim());
+}
+
+/** A friendlier message for why a fetch failed. */
+export function fetchError(error: unknown, url: string): Error {
+  const cause = (error as { cause?: { code?: string; message?: string } })?.cause;
+  const name = (error as { name?: string })?.name;
+  if (name === "TimeoutError") return new Error(`${url} did not answer in time`);
+  if (name === "AbortError") return new Error(`fetching ${url} was stopped`);
+  const code = cause?.code ?? "";
+  const reason = code === "ENOTFOUND" ? "the host does not exist" : code === "ECONNREFUSED" ? "the connection was refused" : code === "CERT_HAS_EXPIRED" || /CERT|SSL|TLS/.test(code) ? "its TLS certificate is not valid" : cause?.message ?? (error instanceof Error ? error.message : String(error));
+  return new Error(`could not fetch ${url}: ${reason}`);
+}
+
+/**
+ * Fetch a public page (following up to five redirects, each checked like the
+ * first) and read at most `maxBytes` of it as text. Throws with a readable
+ * reason when the URL is refused or unreachable; an HTTP error status is a
+ * page like any other, for the caller to judge.
+ */
+export async function fetchPage(input: string, options: FetchOptions = {}): Promise<FetchedPage> {
+  const fetcher = options.fetch ?? globalThis.fetch;
+  const lookup = options.lookup ?? defaultLookup;
+  const allowPrivate = options.allowPrivate ?? process.env.BOT_LOBBY_WEB_ALLOW_PRIVATE === "1";
+  let url: URL;
+  try {
+    url = new URL(input.trim());
+  } catch {
+    throw new Error(`"${input}" is not a URL`);
+  }
+  const timeout = AbortSignal.timeout(options.timeoutMs ?? TIMEOUT_MS);
+  const signal = options.signal ? AbortSignal.any([options.signal, timeout]) : timeout;
+  const redirects: string[] = [];
+  let method = options.method ?? "GET";
+  let body = options.body;
+  for (;;) {
+    const blocked = await blockedReason(url, lookup, allowPrivate);
+    if (blocked) throw new Error(`not fetched: ${blocked}`);
+    let response: Response;
+    try {
+      response = await fetcher(url.href, {
+        method,
+        ...(body !== undefined ? { body } : {}),
+        redirect: "manual",
+        signal,
+        headers: { "user-agent": USER_AGENT, accept: options.accept ?? ACCEPT, "accept-language": "en;q=0.9, *;q=0.5", ...options.headers },
+      });
+    } catch (error) {
+      throw fetchError(error, url.href);
+    }
+    const location = response.headers.get("location");
+    if (response.status >= 300 && response.status < 400 && location) {
+      await response.body?.cancel().catch(() => {});
+      if (redirects.length >= MAX_REDIRECTS) throw new Error(`${input} redirects more than ${MAX_REDIRECTS} times`);
+      redirects.push(url.href);
+      url = new URL(location, url);
+      if (response.status === 303 || ((response.status === 301 || response.status === 302) && method === "POST")) {
+        method = "GET";
+        body = undefined;
+      }
+      continue;
+    }
+    const contentType = response.headers.get("content-type") ?? "";
+    if (!isTextual(contentType)) {
+      await response.body?.cancel().catch(() => {});
+      const length = Number(response.headers.get("content-length") ?? 0);
+      return { url: url.href, status: response.status, statusText: response.statusText, contentType, headers: response.headers, text: "", bytes: Number.isFinite(length) ? length : 0, truncated: false, binary: true, redirects };
+    }
+    let read: { bytes: Uint8Array; truncated: boolean };
+    try {
+      read = await readCapped(response, options.maxBytes ?? MAX_BYTES);
+    } catch (error) {
+      throw fetchError(error, url.href);
+    }
+    return {
+      url: url.href,
+      status: response.status,
+      statusText: response.statusText,
+      contentType,
+      headers: response.headers,
+      text: decode(read.bytes, charsetOf(contentType, read.bytes)),
+      bytes: read.bytes.byteLength,
+      truncated: read.truncated,
+      binary: false,
+      redirects,
+    };
+  }
+}

package/src/web/html.ts

@@ -0,0 +1,183 @@
+/**
+ * A forgiving HTML parser, just enough to read web pages: tags into a tree
+ * (void elements, raw text in script/style, implied closes for p, li, td and
+ * the like, stray end tags ignored), entities decoded. No dependency; not a
+ * spec parser, and it does not need to be.
+ */
+
+export interface HtmlText {
+  type: "text";
+  text: string;
+}
+
+export interface HtmlElement {
+  type: "element";
+  name: string;
+  attrs: Record<string, string>;
+  children: HtmlNode[];
+}
+
+export type HtmlNode = HtmlText | HtmlElement;
+
+const VOID = new Set(["area", "base", "br", "col", "embed", "hr", "img", "input", "link", "meta", "param", "source", "track", "wbr"]);
+/** Elements whose content is text up to their end tag, never markup. */
+const RAW = new Set(["script", "style", "textarea", "title", "xmp", "noscript", "template"]);
+/** Opening one of these closes an open element of the listed names (up to the given boundaries). */
+const IMPLIED: Record<string, { closes: string[]; within: string[] }> = {
+  p: { closes: ["p"], within: ["div", "section", "article", "main", "body", "td", "th", "li", "blockquote"] },
+  li: { closes: ["li"], within: ["ul", "ol", "menu"] },
+  dt: { closes: ["dt", "dd"], within: ["dl"] },
+  dd: { closes: ["dt", "dd"], within: ["dl"] },
+  tr: { closes: ["tr", "td", "th"], within: ["table", "thead", "tbody", "tfoot"] },
+  td: { closes: ["td", "th"], within: ["tr", "table"] },
+  th: { closes: ["td", "th"], within: ["tr", "table"] },
+  thead: { closes: ["thead", "tbody", "tr", "td", "th"], within: ["table"] },
+  tbody: { closes: ["thead", "tbody", "tr", "td", "th"], within: ["table"] },
+  tfoot: { closes: ["thead", "tbody", "tr", "td", "th"], within: ["table"] },
+  option: { closes: ["option"], within: ["select", "datalist"] },
+};
+/** Block elements that end an open paragraph. */
+const CLOSES_P = new Set(["address", "article", "aside", "blockquote", "details", "div", "dl", "fieldset", "figure", "footer", "form", "h1", "h2", "h3", "h4", "h5", "h6", "header", "hr", "main", "nav", "ol", "pre", "section", "table", "ul"]);
+
+const NAMED: Record<string, string> = {
+  amp: "&", lt: "<", gt: ">", quot: "\"", apos: "'", nbsp: " ", ensp: " ", emsp: " ", thinsp: " ", shy: "",
+  copy: "©", reg: "®", trade: "™", hellip: "…", mdash: "—", ndash: "–", minus: "−",
+  lsquo: "‘", rsquo: "’", sbquo: "‚", ldquo: "“", rdquo: "”", bdquo: "„", laquo: "«", raquo: "»", lsaquo: "‹", rsaquo: "›",
+  bull: "•", middot: "·", times: "×", divide: "÷", deg: "°", plusmn: "±", para: "¶", sect: "§", dagger: "†", Dagger: "‡",
+  euro: "€", pound: "£", yen: "¥", cent: "¢", larr: "←", rarr: "→", uarr: "↑", darr: "↓", harr: "↔", rArr: "⇒", lArr: "⇐",
+  le: "≤", ge: "≥", ne: "≠", asymp: "≈", infin: "∞", micro: "µ", frac12: "½", frac14: "¼", frac34: "¾", sup2: "²", sup3: "³",
+  zwj: "", zwnj: "", lrm: "", rlm: "", check: "✓", star: "☆", hearts: "♥",
+};
+
+/** Decode HTML character references (named ones pages commonly use, and every numeric one). */
+export function decodeEntities(text: string): string {
+  if (!text.includes("&")) return text;
+  return text.replace(/&(#x[0-9a-f]+|#[0-9]+|[a-z][a-z0-9]*);?/gi, (match, ref: string) => {
+    if (ref[0] === "#") {
+      const code = ref[1] === "x" || ref[1] === "X" ? Number.parseInt(ref.slice(2), 16) : Number.parseInt(ref.slice(1), 10);
+      if (!Number.isFinite(code) || code <= 0 || code > 0x10ffff || (code >= 0xd800 && code <= 0xdfff)) return "�";
+      return String.fromCodePoint(code);
+    }
+    const named = NAMED[ref] ?? NAMED[ref.toLowerCase()];
+    return named ?? match;
+  });
+}
+
+function parseAttrs(source: string): Record<string, string> {
+  const attrs: Record<string, string> = {};
+  const pattern = /([^\s"'<>/=]+)(?:\s*=\s*(?:"([^"]*)"|'([^']*)'|([^\s"'=<>`]+)))?/g;
+  for (const match of source.matchAll(pattern)) {
+    const name = match[1]!.toLowerCase();
+    if (name in attrs) continue;
+    attrs[name] = decodeEntities(match[2] ?? match[3] ?? match[4] ?? "");
+  }
+  return attrs;
+}
+
+/** Parse HTML into a tree under a synthetic root element named `#root`. */
+export function parseHtml(html: string): HtmlElement {
+  const root: HtmlElement = { type: "element", name: "#root", attrs: {}, children: [] };
+  const stack: HtmlElement[] = [root];
+  const current = () => stack[stack.length - 1]!;
+  const openIndex = (names: string[], within: string[]): number => {
+    for (let index = stack.length - 1; index > 0; index -= 1) {
+      const name = stack[index]!.name;
+      if (names.includes(name)) return index;
+      if (within.includes(name)) return -1;
+    }
+    return -1;
+  };
+  const text = (value: string) => {
+    if (!value) return;
+    const parent = current();
+    const last = parent.children[parent.children.length - 1];
+    if (last?.type === "text") last.text += value;
+    else parent.children.push({ type: "text", text: value });
+  };
+
+  let position = 0;
+  const tag = /<(\/?)([a-zA-Z][a-zA-Z0-9:-]*)((?:[^>"']|"[^"]*"|'[^']*')*?)(\/?)>|<!--[\s\S]*?(?:-->|$)|<![^>]*>|<\?[^>]*>/g;
+  for (;;) {
+    tag.lastIndex = position;
+    const match = tag.exec(html);
+    if (!match) {
+      text(decodeEntities(html.slice(position)));
+      break;
+    }
+    text(decodeEntities(html.slice(position, match.index)));
+    position = match.index + match[0].length;
+    if (!match[2]) continue; // comment, doctype, processing instruction
+    const name = match[2].toLowerCase();
+    if (match[1]) {
+      // An end tag closes the nearest open element of that name; a stray one is ignored.
+      for (let index = stack.length - 1; index > 0; index -= 1) {
+        if (stack[index]!.name === name) {
+          stack.length = index;
+          break;
+        }
+      }
+      continue;
+    }
+    const implied = IMPLIED[name];
+    if (implied) {
+      const index = openIndex(implied.closes, implied.within);
+      if (index > 0) stack.length = index;
+    }
+    if (CLOSES_P.has(name)) {
+      const index = openIndex(["p"], ["div", "section", "article", "main", "body", "td", "th", "li", "blockquote", "button"]);
+      if (index > 0) stack.length = index;
+    }
+    const element: HtmlElement = { type: "element", name, attrs: parseAttrs(match[3] ?? ""), children: [] };
+    current().children.push(element);
+    if (RAW.has(name)) {
+      const end = html.toLowerCase().indexOf(`</${name}`, position);
+      const stop = end < 0 ? html.length : end;
+      const raw = html.slice(position, stop);
+      if (raw) element.children.push({ type: "text", text: name === "title" || name === "textarea" ? decodeEntities(raw) : raw });
+      const close = end < 0 ? html.length : html.indexOf(">", end);
+      position = close < 0 ? html.length : close + 1;
+      continue;
+    }
+    if (!VOID.has(name) && !match[4]) stack.push(element);
+  }
+  return root;
+}
+
+/** Every element under `node` (depth first, document order) that passes `test`. */
+export function findAll(node: HtmlElement, test: (element: HtmlElement) => boolean): HtmlElement[] {
+  const found: HtmlElement[] = [];
+  const walk = (element: HtmlElement) => {
+    for (const child of element.children) {
+      if (child.type !== "element") continue;
+      if (test(child)) found.push(child);
+      walk(child);
+    }
+  };
+  walk(node);
+  return found;
+}
+
+export function findFirst(node: HtmlElement, test: (element: HtmlElement) => boolean): HtmlElement | undefined {
+  for (const child of node.children) {
+    if (child.type !== "element") continue;
+    if (test(child)) return child;
+    const found = findFirst(child, test);
+    if (found) return found;
+  }
+  return undefined;
+}
+
+/** The text inside a node, whitespace collapsed. */
+export function textOf(node: HtmlNode): string {
+  const parts: string[] = [];
+  const walk = (current: HtmlNode) => {
+    if (current.type === "text") parts.push(current.text);
+    else if (current.name !== "script" && current.name !== "style") for (const child of current.children) walk(child);
+  };
+  walk(node);
+  return parts.join("").replace(/\s+/g, " ").trim();
+}
+
+export function hasClass(element: HtmlElement, name: string): boolean {
+  return (element.attrs.class ?? "").split(/\s+/).includes(name);
+}

package/src/web/read.ts

@@ -0,0 +1,113 @@
+/**
+ * Reading pages and checking sources: a fetched page as readable text with
+ * its title and dates (kept a few minutes, so reading on from an offset does
+ * not fetch it again), and a URL's standing as a citation (reachable, where
+ * it ends up, when it was published).
+ */
+import { fetchPage, type FetchOptions } from "./fetch.ts";
+import { htmlToMarkdown, pageMeta, type PageMeta } from "./extract.ts";
+import { parseHtml } from "./html.ts";
+
+export interface ReadPage extends PageMeta {
+  url: string;
+  /** The URL asked for, when a redirect moved it. */
+  requested?: string;
+  status: number;
+  contentType: string;
+  /** The page as Markdown or text. */
+  text: string;
+  /** The body was longer than was read. */
+  cut: boolean;
+}
+
+export interface SourceStatus extends PageMeta {
+  url: string;
+  ok: boolean;
+  status?: number;
+  statusText?: string;
+  finalUrl?: string;
+  contentType?: string;
+  /** The server's Last-Modified header, when it sends one. */
+  lastModified?: string;
+  error?: string;
+}
+
+const CACHE_MS = 10 * 60 * 1000;
+const CACHE_SIZE = 40;
+const cache = new Map<string, { page: ReadPage; at: number }>();
+
+export function clearPageCache(): void {
+  cache.clear();
+}
+
+const HTML = /html|xml/i;
+
+function looksLikeHtml(contentType: string, text: string): boolean {
+  if (/xhtml|html/i.test(contentType)) return true;
+  if (contentType.trim()) return false;
+  return /^\s*(<!doctype html|<html|<head|<body)/i.test(text);
+}
+
+function kilobytes(bytes: number): string {
+  return bytes >= 1024 * 1024 ? `${(bytes / 1024 / 1024).toFixed(1)} MB` : `${Math.max(1, Math.round(bytes / 1024))} KB`;
+}
+
+/** A page as text: HTML to Markdown, JSON pretty, other text as it is. Binary files are refused. */
+export async function readPage(url: string, options: FetchOptions = {}): Promise<ReadPage> {
+  const key = url.trim();
+  const hit = cache.get(key);
+  if (hit && Date.now() - hit.at < CACHE_MS) return hit.page;
+  const fetched = await fetchPage(key, options);
+  const type = fetched.contentType.split(";")[0]!.trim().toLowerCase();
+  if (fetched.binary) {
+    throw new Error(`${fetched.url} is ${type || "a binary file"}${fetched.bytes ? ` (${kilobytes(fetched.bytes)})` : ""}; only web pages and text can be read${type === "application/pdf" ? ". Look for an HTML version of it (an abstract page, docs, a release note)" : ""}`);
+  }
+  let page: ReadPage;
+  const base = { url: fetched.url, ...(fetched.redirects.length > 0 ? { requested: key } : {}), status: fetched.status, contentType: type, cut: fetched.truncated };
+  if (looksLikeHtml(type, fetched.text)) {
+    const readable = htmlToMarkdown(fetched.text, fetched.url);
+    const { markdown, ...meta } = readable;
+    page = { ...meta, ...base, text: markdown || readable.description || "" };
+  } else if (/json/.test(type)) {
+    let text = fetched.text;
+    try {
+      text = JSON.stringify(JSON.parse(fetched.text), null, 2);
+    } catch {
+      // Not valid JSON after all: shown as it came.
+    }
+    page = { ...base, text: `\`\`\`json\n${text}\n\`\`\`` };
+  } else {
+    page = { ...base, text: fetched.text.replace(/\r\n?/g, "\n") };
+  }
+  if (!page.modified) {
+    const lastModified = fetched.headers.get("last-modified");
+    if (lastModified && !Number.isNaN(Date.parse(lastModified))) page.modified = new Date(lastModified).toISOString().slice(0, 10);
+  }
+  if (fetched.status < 400) {
+    cache.set(key, { page, at: Date.now() });
+    if (cache.size > CACHE_SIZE) cache.delete(cache.keys().next().value!);
+  }
+  return page;
+}
+
+/** How each source stands: reachable or not, where it ends up, its title and dates. */
+export async function checkSource(url: string, options: FetchOptions = {}): Promise<SourceStatus> {
+  try {
+    const fetched = await fetchPage(url, { ...options, maxBytes: 512 * 1024 });
+    const type = fetched.contentType.split(";")[0]!.trim().toLowerCase();
+    const meta = HTML.test(type) || looksLikeHtml(type, fetched.text) ? pageMeta(parseHtml(fetched.text)) : {};
+    const lastModified = fetched.headers.get("last-modified") ?? undefined;
+    return {
+      url,
+      ok: fetched.status < 400,
+      status: fetched.status,
+      statusText: fetched.statusText,
+      ...(fetched.redirects.length > 0 ? { finalUrl: fetched.url } : {}),
+      contentType: type,
+      ...(lastModified ? { lastModified } : {}),
+      ...meta,
+    };
+  } catch (error) {
+    return { url, ok: false, error: error instanceof Error ? error.message : String(error) };
+  }
+}