@nebulacomponents/citable 1.19.0 → 1.21.0

package/src/connectors/webflow.js

@@ -1,7 +1,9 @@
 import { providerRequest } from './http.js';
 import { sha256, nowIso } from '../shared/io.js';
+import { collectionResult, errorMessage, paginationBoundary } from './collectionResult.js';
 
 const BASE = 'https://api.webflow.com/v2';
+const API_ORIGIN = new URL(BASE).origin;
 
 const METRICS = {
   pages: { unit: 'count', value_type: 'integer' },
@@ -24,6 +26,96 @@ function computeWebflowPayloadHash(payload) {
   return sha256(JSON.stringify(canonical));
 }
 
+function safeContinuation(url, { preserveOrigin = true } = {}) {
+  try {
+    const parsed = new URL(url);
+    return { ...(preserveOrigin ? { url: `${parsed.origin}${parsed.pathname}` } : { path: parsed.pathname }), query_redacted: parsed.search.length > 0 };
+  } catch {
+    return { url: '[invalid continuation]', query_redacted: true };
+  }
+}
+
+async function collectWebflowList(url, field, context) {
+  const maxPages = paginationBoundary(context);
+  const items = [];
+  const errors = [];
+  const limitations = [];
+  let providerTotal = null;
+  let pagesRequested = 0;
+  let pagesRetrieved = 0;
+  let offset = 0;
+  let cursor = null;
+  let nextUrl = null;
+  let continuationState = null;
+  let providerCompleteness = 'declared';
+  let connectorError = null;
+  while (pagesRequested < maxPages) {
+    pagesRequested += 1;
+    const query = cursor
+      ? `?limit=100&cursor=${encodeURIComponent(cursor)}`
+      : `?limit=100${offset ? `&offset=${offset}` : ''}`;
+    let result;
+    try {
+      if (nextUrl) {
+        let continuation;
+        try { continuation = new URL(nextUrl); } catch {
+          errors.push(`${field} page ${pagesRequested}: malformed continuation URL`);
+          continuationState = { next: safeContinuation(nextUrl), reason: 'invalid_url' };
+          break;
+        }
+        if (continuation.origin !== API_ORIGIN || continuation.protocol !== 'https:') {
+          errors.push(`${field} page ${pagesRequested}: continuation origin is outside the Webflow API`);
+          continuationState = { next: safeContinuation(nextUrl, { preserveOrigin: false }), reason: 'origin_mismatch' };
+          break;
+        }
+      }
+      result = await providerRequest(nextUrl || `${url}${query}`, context);
+      const pageItems = Array.isArray(result?.[field]) ? result[field] : [];
+      pagesRetrieved += 1;
+      items.push(...pageItems);
+      const pagination = result.pagination || {};
+      const reported = result.total ?? pagination.total ?? pagination.totalCount ?? null;
+      if (Number.isInteger(reported) && reported >= 0) providerTotal = reported;
+      const next = result.nextCursor ?? pagination.nextCursor ?? result.next?.cursor ?? null;
+      const providerNextUrl = typeof result.next === 'string' ? result.next : (typeof pagination.next === 'string' ? pagination.next : null);
+      const nextOffset = result.nextOffset ?? pagination.nextOffset ?? null;
+      nextUrl = null;
+      if (next !== null && (typeof next !== 'string' || !next.trim())) {
+        errors.push(`${field} page ${pagesRequested}: malformed continuation cursor`);
+        continuationState = { cursor: next };
+        break;
+      }
+      if (providerNextUrl !== null && (!/^https:\/\//i.test(providerNextUrl))) {
+        errors.push(`${field} page ${pagesRequested}: malformed continuation URL`);
+        continuationState = { next: safeContinuation(providerNextUrl, { preserveOrigin: false }) };
+        break;
+      }
+      if (next) cursor = String(next);
+      else if (providerNextUrl) { nextUrl = providerNextUrl; cursor = null; }
+      else if (nextOffset !== null && (!Number.isInteger(nextOffset) || nextOffset <= offset)) {
+        errors.push(`${field} page ${pagesRequested}: malformed continuation offset`);
+        continuationState = { offset: nextOffset };
+        break;
+      }
+      else if (providerTotal !== null && items.length >= providerTotal) break;
+      else if (Number.isInteger(nextOffset) && nextOffset >= 0) { offset = nextOffset; cursor = null; }
+      else if (pageItems.length >= 100 && (providerTotal === null || items.length < providerTotal)) offset += pageItems.length;
+      else { providerCompleteness = 'unknown'; break; }
+      if (providerTotal !== null && items.length >= providerTotal) break;
+    } catch (error) {
+      connectorError ||= error;
+      errors.push(`${field} page ${pagesRequested}: ${errorMessage(error)}`);
+      continuationState = cursor ? { cursor } : nextUrl ? { next: safeContinuation(nextUrl) } : { offset };
+      break;
+    }
+  }
+  if (!continuationState && pagesRequested >= maxPages && (providerTotal === null || items.length < providerTotal)) {
+    continuationState = cursor ? { cursor } : nextUrl ? { next: safeContinuation(nextUrl) } : { offset: offset + 100 };
+    limitations.push(`Webflow ${field} collection reached the ${maxPages}-page boundary.`);
+  }
+  return { items, providerTotal, providerCompleteness, connectorError, pagesRequested, pagesRetrieved, continuationState, limitations, errors };
+}
+
 export const webflowConnector = {
   provider: 'webflow',
   defaultCredentialEnv: 'WEBFLOW_API_TOKEN',
@@ -62,10 +154,26 @@ export const webflowConnector = {
     const siteId = connection.property_id;
     const rows = [];
     const metricNames = new Set(metrics.map((m) => m.external_name));
+    const allItems = [];
+    const errors = [];
+    const limitations = [];
+    let pagesRequested = 0;
+    let pagesRetrieved = 0;
+    let continuationState = null;
+    let providerTotal = 0;
+    let providerTotalKnown = true;
+    let connectorError = null;
 
     if (metricNames.has('pages')) {
-      const pageResult = await providerRequest(`${BASE}/sites/${encodeURIComponent(siteId)}/pages`, context);
-      const pages = pageResult.pages || [];
+      const pageResult = await collectWebflowList(`${BASE}/sites/${encodeURIComponent(siteId)}/pages`, 'pages', context);
+      const pages = pageResult.items;
+      allItems.push(...pages.map((item) => ({ ...item, _collection_type: 'pages' })));
+      errors.push(...pageResult.errors); limitations.push(...pageResult.limitations);
+      pagesRequested += pageResult.pagesRequested; pagesRetrieved += pageResult.pagesRetrieved;
+      continuationState ||= pageResult.continuationState;
+      connectorError ||= pageResult.connectorError;
+      if (pageResult.providerCompleteness === 'unknown') providerTotalKnown = false;
+      if (pageResult.providerTotal === null) providerTotalKnown = false; else providerTotal += pageResult.providerTotal;
       const pageMetric = metrics.find((m) => m.external_name === 'pages');
       rows.push({
         metric: pageMetric,
@@ -76,8 +184,15 @@ export const webflowConnector = {
     }
 
     if (metricNames.has('collections') || metricNames.has('items')) {
-      const collectionsResult = await providerRequest(`${BASE}/sites/${encodeURIComponent(siteId)}/collections`, context);
-      const collections = collectionsResult.collections || [];
+      const collectionsResult = await collectWebflowList(`${BASE}/sites/${encodeURIComponent(siteId)}/collections`, 'collections', context);
+      const collections = collectionsResult.items;
+      allItems.push(...collections.map((item) => ({ ...item, _collection_type: 'collections' })));
+      errors.push(...collectionsResult.errors); limitations.push(...collectionsResult.limitations);
+      pagesRequested += collectionsResult.pagesRequested; pagesRetrieved += collectionsResult.pagesRetrieved;
+      continuationState ||= collectionsResult.continuationState;
+      connectorError ||= collectionsResult.connectorError;
+      if (collectionsResult.providerCompleteness === 'unknown') providerTotalKnown = false;
+      if (collectionsResult.providerTotal === null) providerTotalKnown = false; else providerTotal += collectionsResult.providerTotal;
 
       if (metricNames.has('collections')) {
         const colMetric = metrics.find((m) => m.external_name === 'collections');
@@ -92,8 +207,15 @@ export const webflowConnector = {
       if (metricNames.has('items')) {
         let totalItems = 0;
         for (const col of collections) {
-          const itemsResult = await providerRequest(`${BASE}/collections/${encodeURIComponent(col.id)}/items?limit=100`, context);
-          totalItems += (itemsResult.items || []).length;
+          const itemsResult = await collectWebflowList(`${BASE}/collections/${encodeURIComponent(col.id)}/items`, 'items', context);
+          totalItems += itemsResult.items.length;
+          allItems.push(...itemsResult.items.map((item) => ({ ...item, _collection_type: 'items', _collection_id: col.id })));
+          errors.push(...itemsResult.errors); limitations.push(...itemsResult.limitations);
+          pagesRequested += itemsResult.pagesRequested; pagesRetrieved += itemsResult.pagesRetrieved;
+          continuationState ||= itemsResult.continuationState;
+          connectorError ||= itemsResult.connectorError;
+          if (itemsResult.providerCompleteness === 'unknown') providerTotalKnown = false;
+          if (itemsResult.providerTotal === null) providerTotalKnown = false; else providerTotal += itemsResult.providerTotal;
         }
         const itemMetric = metrics.find((m) => m.external_name === 'items');
         rows.push({
@@ -105,13 +227,21 @@ export const webflowConnector = {
       }
     }
 
+    const collection = collectionResult({
+      items: allItems,
+      paginationState: { pages_requested: pagesRequested, pages_retrieved: pagesRetrieved, boundary: { max_pages: paginationBoundary(context) } },
+      providerReportedTotal: providerTotalKnown ? providerTotal : null,
+      continuationState,
+      providerCompleteness: providerTotalKnown ? 'declared' : 'unknown',
+      limitations: [...new Set([...limitations, 'Webflow API enforces rate limits of 60 requests per minute.', 'Staged CMS changes require site publish to appear on live domains.'])],
+      errors,
+    });
     return {
       rows,
       cursor: endDate,
-      limitations: [
-        'Webflow API enforces rate limits of 60 requests per minute.',
-        'Staged CMS changes require site publish to appear on live domains.',
-      ],
+      limitations: collection.limitations,
+      collection,
+      ...(connectorError ? { connectorError } : {}),
     };
   },
 

package/src/connectors/wordpress.js

@@ -1,5 +1,6 @@
 import { providerRequest } from './http.js';
 import { sha256, nowIso } from '../shared/io.js';
+import { collectionResult, errorMessage, paginationBoundary } from './collectionResult.js';
 
 const METRICS = {
   published_posts: { unit: 'count', value_type: 'integer' },
@@ -25,6 +26,53 @@ function computeCmsPayloadHash(payload) {
   return sha256(JSON.stringify(canonical));
 }
 
+async function collectWordPressEndpoint(base, endpoint, context) {
+  const maxPages = paginationBoundary(context);
+  const items = [];
+  const limitations = [];
+  const errors = [];
+  let providerTotal = null;
+  let totalPages = null;
+  let page = 1;
+  let pagesRequested = 0;
+  let pagesRetrieved = 0;
+  let continuationState = null;
+  let providerCompleteness = 'declared';
+  let connectorError = null;
+
+  while (page <= maxPages) {
+    pagesRequested += 1;
+    const suffix = page === 1 ? '?per_page=100&status=publish' : `?per_page=100&status=publish&page=${page}`;
+    try {
+      const response = await providerRequest(`${base}/wp-json/wp/v2/${endpoint}${suffix}`, { ...context, rawResponse: true });
+      const pageItems = await response.json();
+      if (!Array.isArray(pageItems)) throw new Error(`WordPress ${endpoint} page ${page} did not return an array`);
+      pagesRetrieved += 1;
+      items.push(...pageItems.map((item) => ({ ...item, _collection_endpoint: endpoint })));
+      const headerTotal = Number(response.headers.get('x-wp-total'));
+      const headerPages = Number(response.headers.get('x-wp-totalpages'));
+      if (Number.isInteger(headerTotal) && headerTotal >= 0) providerTotal = headerTotal;
+      if (Number.isInteger(headerPages) && headerPages >= 1) totalPages = headerPages;
+      if (!totalPages) providerCompleteness = 'unknown';
+      if (pageItems.length === 0 || (totalPages && page >= totalPages)) break;
+      if (!totalPages && pageItems.length >= 100) {
+        continuationState = { endpoint, page: page + 1 };
+        limitations.push('WordPress did not report total pages for a full page of results.');
+        break;
+      }
+      page += 1;
+    } catch (error) {
+      connectorError ||= error;
+      errors.push(`${endpoint} page ${page}: ${errorMessage(error)}`);
+      continuationState = { endpoint, page };
+      break;
+    }
+  }
+  if (!continuationState && totalPages && page < totalPages) continuationState = { endpoint, page };
+  if (continuationState && page > maxPages) limitations.push(`WordPress ${endpoint} collection reached the ${maxPages}-page boundary.`);
+  return { items, providerTotal, providerCompleteness, connectorError, pagesRequested, pagesRetrieved, continuationState, limitations, errors };
+}
+
 export const wordpressConnector = {
   provider: 'wordpress',
   defaultCredentialEnv: 'WORDPRESS_AUTH_TOKEN',
@@ -78,35 +126,71 @@ export const wordpressConnector = {
     const rows = [];
     const metricNames = new Set(metrics.map((m) => m.external_name));
 
+    const collections = [];
+    const collectionErrors = [];
+    const collectionLimitations = [];
+    let pagesRequested = 0;
+    let pagesRetrieved = 0;
+    let continuationState = null;
+    let providerTotal = 0;
+    let providerTotalKnown = true;
+    let connectorError = null;
+
     if (metricNames.has('published_posts')) {
-      const posts = await providerRequest(`${base}/wp-json/wp/v2/posts?per_page=100&status=publish`, context);
+      const postsResult = await collectWordPressEndpoint(base, 'posts', context);
+      collections.push(...postsResult.items);
+      collectionErrors.push(...postsResult.errors);
+      collectionLimitations.push(...postsResult.limitations);
+      pagesRequested += postsResult.pagesRequested;
+      pagesRetrieved += postsResult.pagesRetrieved;
+      continuationState ||= postsResult.continuationState;
+      connectorError ||= postsResult.connectorError;
+      if (postsResult.providerCompleteness === 'unknown') providerTotalKnown = false;
+      if (postsResult.providerTotal === null) providerTotalKnown = false; else providerTotal += postsResult.providerTotal;
       const postMetric = metrics.find((m) => m.external_name === 'published_posts');
       rows.push({
         metric: postMetric,
-        value: Array.isArray(posts) ? posts.length : 0,
+        value: postsResult.items.length,
         dimensions: { date: endDate },
         observed_at: `${endDate}T00:00:00.000Z`,
       });
     }
 
     if (metricNames.has('published_pages')) {
-      const pages = await providerRequest(`${base}/wp-json/wp/v2/pages?per_page=100&status=publish`, context);
+      const pagesResult = await collectWordPressEndpoint(base, 'pages', context);
+      collections.push(...pagesResult.items);
+      collectionErrors.push(...pagesResult.errors);
+      collectionLimitations.push(...pagesResult.limitations);
+      pagesRequested += pagesResult.pagesRequested;
+      pagesRetrieved += pagesResult.pagesRetrieved;
+      continuationState ||= pagesResult.continuationState;
+      connectorError ||= pagesResult.connectorError;
+      if (pagesResult.providerCompleteness === 'unknown') providerTotalKnown = false;
+      if (pagesResult.providerTotal === null) providerTotalKnown = false; else providerTotal += pagesResult.providerTotal;
       const pageMetric = metrics.find((m) => m.external_name === 'published_pages');
       rows.push({
         metric: pageMetric,
-        value: Array.isArray(pages) ? pages.length : 0,
+        value: pagesResult.items.length,
         dimensions: { date: endDate },
         observed_at: `${endDate}T00:00:00.000Z`,
       });
     }
 
+    const collection = collectionResult({
+      items: collections,
+      paginationState: { pages_requested: pagesRequested, pages_retrieved: pagesRetrieved, boundary: { max_pages: paginationBoundary(context) } },
+      providerReportedTotal: providerTotalKnown ? providerTotal : null,
+      continuationState,
+      providerCompleteness: providerTotalKnown ? 'declared' : 'unknown',
+      limitations: [...collectionLimitations, 'WordPress REST API collection is bounded to accessible posts and pages.', 'Custom post types outside standard posts and pages require dedicated endpoint parameters.'],
+      errors: collectionErrors,
+    });
     return {
       rows,
       cursor: endDate,
-      limitations: [
-        'WordPress REST API collection is bounded to accessible posts and pages.',
-        'Custom post types outside standard posts and pages require dedicated endpoint parameters.',
-      ],
+      limitations: collection.limitations,
+      collection,
+      ...(connectorError ? { connectorError } : {}),
     };
   },
 

package/src/crawler/fetch.js

@@ -1,7 +1,12 @@
 import dns from 'node:dns';
+import http from 'node:http';
+import https from 'node:https';
 import net from 'node:net';
+import { Readable } from 'node:stream';
+import { createBrotliDecompress, createGunzip, createInflate } from 'node:zlib';
 
 const blockedAddresses = new net.BlockList();
+const GUARDED_LOOKUP_TRANSPORT = Symbol('citable.guardedLookupTransport');
 for (const [network, prefix] of [
   ['0.0.0.0', 8], ['10.0.0.0', 8], ['100.64.0.0', 10], ['127.0.0.0', 8],
   ['169.254.0.0', 16], ['172.16.0.0', 12], ['192.0.0.0', 24], ['192.0.2.0', 24],
@@ -22,18 +27,89 @@ function isBlockedAddress(address) {
 }
 
 export async function validatePublicUrl(url, { lookup = dns.promises.lookup } = {}) {
+  const parsed = parseNetworkUrl(url);
+  await resolvePublicAddresses(parsed.hostname, lookup);
+  return parsed;
+}
+
+function parseNetworkUrl(url) {
   const parsed = new URL(url);
   if (!['http:', 'https:'].includes(parsed.protocol)) throw new Error(`unsupported URL protocol: ${parsed.protocol}`);
   if (parsed.username || parsed.password) throw new Error('URL credentials are not permitted');
-  const hostname = parsed.hostname.replace(/^\[|\]$/g, '');
+  return parsed;
+}
+
+async function resolvePublicAddresses(rawHostname, lookup, options = {}) {
+  const hostname = rawHostname.replace(/^\[|\]$/g, '');
   const literalFamily = net.isIP(hostname);
   const addresses = literalFamily
     ? [{ address: hostname, family: literalFamily }]
-    : await lookup(hostname, { all: true, verbatim: true });
+    : await lookup(hostname, { ...options, all: true, verbatim: true });
   if (!addresses.length || addresses.some(({ address }) => isBlockedAddress(address))) {
     throw new Error(`refusing private, loopback, or non-public destination: ${hostname}`);
   }
-  return parsed;
+  return addresses;
+}
+
+function createGuardedLookup(lookup) {
+  return (hostname, options, callback) => {
+    const normalizedOptions = typeof options === 'number' ? { family: options } : (options ?? {});
+    resolvePublicAddresses(hostname, lookup, { family: normalizedOptions.family || 0 })
+      .then((addresses) => {
+        if (normalizedOptions.all) callback(null, addresses);
+        else callback(null, addresses[0].address, addresses[0].family);
+      })
+      .catch((error) => callback(error));
+  };
+}
+
+function fetchWithGuardedLookup(url, { headers, signal, lookup }) {
+  const parsed = new URL(url);
+  const transport = parsed.protocol === 'https:' ? https : http;
+  return new Promise((resolve, reject) => {
+    const request = transport.request(parsed, {
+      method: 'GET',
+      headers,
+      lookup,
+      servername: parsed.protocol === 'https:' ? parsed.hostname : undefined,
+      signal,
+    }, (response) => {
+      const responseHeaders = new Headers();
+      for (const [name, value] of Object.entries(response.headers)) {
+        if (Array.isArray(value)) for (const item of value) responseHeaders.append(name, item);
+        else if (value !== undefined) responseHeaders.set(name, value);
+      }
+      const encoding = String(response.headers['content-encoding'] ?? '').toLowerCase();
+      let body = response;
+      if (encoding === 'gzip' || encoding === 'x-gzip') body = response.pipe(createGunzip());
+      else if (encoding === 'deflate') body = response.pipe(createInflate());
+      else if (encoding === 'br') body = response.pipe(createBrotliDecompress());
+      const noBody = response.statusCode === 204 || response.statusCode === 205 || response.statusCode === 304;
+      resolve(new Response(noBody ? null : Readable.toWeb(body), {
+        status: response.statusCode,
+        statusText: response.statusMessage,
+        headers: responseHeaders,
+      }));
+    });
+    request.on('error', reject);
+    request.end();
+  });
+}
+
+/**
+ * Declare that a custom transport uses the `lookup` callback passed by fetchUrl
+ * for its actual socket connection. Undeclared transports are rejected before
+ * execution because URL preflight alone does not prevent DNS rebinding.
+ */
+export function declareGuardedLookupTransport(fetchImpl) {
+  if (typeof fetchImpl !== 'function') throw new TypeError('custom transport must be a function');
+  Object.defineProperty(fetchImpl, GUARDED_LOOKUP_TRANSPORT, {
+    value: true,
+    enumerable: false,
+    configurable: false,
+    writable: false,
+  });
+  return fetchImpl;
 }
 
 /** Enforce the public-network policy for each Playwright request. */
@@ -62,12 +138,12 @@ export async function handlePublicBrowserRoute(route, { lookup = dns.promises.lo
   }
 }
 
-async function readBodyLimited(res, maxBodyBytes) {
+async function readBodyLimited(res, maxBodyBytes, responseType = 'text') {
   const declared = Number(res.headers.get('content-length'));
   if (Number.isFinite(declared) && declared > maxBodyBytes) {
-    throw new Error(`response body exceeds ${maxBodyBytes} bytes`);
+    throw bodyLimitError(maxBodyBytes);
   }
-  if (!res.body) return '';
+  if (!res.body) return { body: responseType === 'buffer' ? Buffer.alloc(0) : '', bodyBytes: Buffer.alloc(0) };
   const reader = res.body.getReader();
   const chunks = [];
   let total = 0;
@@ -76,7 +152,7 @@ async function readBodyLimited(res, maxBodyBytes) {
       const { done, value } = await reader.read();
       if (done) break;
       total += value.byteLength;
-      if (total > maxBodyBytes) throw new Error(`response body exceeds ${maxBodyBytes} bytes`);
+      if (total > maxBodyBytes) throw bodyLimitError(maxBodyBytes);
       chunks.push(value);
     }
   } finally {
@@ -85,42 +161,127 @@ async function readBodyLimited(res, maxBodyBytes) {
   const bytes = new Uint8Array(total);
   let offset = 0;
   for (const chunk of chunks) { bytes.set(chunk, offset); offset += chunk.byteLength; }
-  return new TextDecoder().decode(bytes);
+  const bodyBytes = Buffer.from(bytes);
+  return { body: responseType === 'buffer' ? bodyBytes : new TextDecoder().decode(bytes), bodyBytes };
+}
+
+function bodyLimitError(maxBodyBytes) {
+  const error = new Error(`response body exceeds ${maxBodyBytes} bytes`);
+  error.code = 'FETCH_BODY_LIMIT';
+  return error;
+}
+
+function timeoutError(timeoutMs) {
+  const error = new Error(`request timeout after ${timeoutMs}ms`);
+  error.code = 'FETCH_TIMEOUT';
+  return error;
+}
+
+function errorCode(error) {
+  const code = typeof error?.code === 'string' && error.code ? error.code : null;
+  if (code && /^[A-Z][A-Z0-9_]{0,63}$/.test(code)) return code;
+  if (error?.name === 'AbortError') return 'FETCH_ABORTED';
+  return 'FETCH_ERROR';
+}
+
+function attemptUrl(url) {
+  const parsed = new URL(url);
+  parsed.hash = '';
+  if (parsed.search) parsed.search = '?[REDACTED]';
+  return parsed.href;
+}
+
+function isRetryableResponseStatus(status) {
+  return status === 500 || status === 502 || status === 503 || status === 504;
+}
+
+function codedFetchError(message, code) {
+  const error = new Error(message);
+  error.code = code;
+  return error;
+}
+
+function attachAttempts(error, attempts) {
+  const target = error instanceof Error ? error : new Error('request failed');
+  Object.defineProperty(target, 'attempts', {
+    value: attempts.map((attempt) => ({ ...attempt })),
+    enumerable: true,
+    configurable: true,
+  });
+  return target;
 }
 
 /** Fetch a public URL with bounded retries, redirects, time, and response size. */
 export async function fetchUrl(url, {
   userAgent = 'CitableAudit/0.1', maxRedirects = 10, timeoutMs = 20000,
   maxRetries = 3, retryDelayMs = 1000, maxBodyBytes = 5 * 1024 * 1024,
-  fetchImpl = globalThis.fetch, lookup = dns.promises.lookup,
+  responseType = 'text',
+  fetchImpl, lookup = dns.promises.lookup,
+  allowUnsafeCustomTransportForTest = false,
 } = {}) {
-  const requested = await validatePublicUrl(url, { lookup });
+  if (!['text', 'buffer'].includes(responseType)) throw new TypeError('responseType must be text or buffer');
+  const requested = parseNetworkUrl(url);
   const allowedOrigin = requested.origin;
   const chain = [];
+  const attempts = [];
+  let attemptNumber = 0;
   let current = requested.href;
+  const customFetch = fetchImpl && fetchImpl !== globalThis.fetch;
+  if (customFetch && allowUnsafeCustomTransportForTest !== true && fetchImpl[GUARDED_LOOKUP_TRANSPORT] !== true) {
+    throw new TypeError('custom transport must declare and implement the guarded lookup contract');
+  }
+  const effectiveFetch = customFetch ? fetchImpl : fetchWithGuardedLookup;
 
   for (let redirects = 0; redirects <= maxRedirects; redirects += 1) {
-    await validatePublicUrl(current, { lookup });
     let res;
     let lastError;
+    let responseStartedAtMs = null;
     for (let attempt = 0; attempt < Math.max(1, maxRetries); attempt += 1) {
+      attemptNumber += 1;
+      const startedAtMs = performance.now();
+      try {
+        await validatePublicUrl(current, { lookup });
+      } catch (error) {
+        const endedAtMs = performance.now();
+        attempts.push({
+          attempt: attemptNumber, url: attemptUrl(current), startedAtMs, endedAtMs,
+          elapsedMs: Math.max(0, endedAtMs - startedAtMs), outcome: 'error', status: null,
+          errorCode: errorCode(error), retryDecision: 'stop',
+        });
+        throw attachAttempts(error, attempts);
+      }
       const controller = new AbortController();
-      const timer = setTimeout(() => controller.abort(new Error(`request timeout after ${timeoutMs}ms`)), timeoutMs);
+      const timer = setTimeout(() => controller.abort(timeoutError(timeoutMs)), timeoutMs);
       try {
-        res = await fetchImpl(current, {
+        res = await effectiveFetch(current, {
           redirect: 'manual',
           headers: { 'user-agent': userAgent },
           signal: controller.signal,
+          lookup: createGuardedLookup(lookup),
         });
-        if (res.status < 500 || attempt === Math.max(1, maxRetries) - 1) {
+        const retry = isRetryableResponseStatus(res.status) && attempt < Math.max(1, maxRetries) - 1;
+        if (!retry) {
           lastError = null;
+          responseStartedAtMs = startedAtMs;
           break;
         }
+        const endedAtMs = performance.now();
+        attempts.push({
+          attempt: attemptNumber, url: attemptUrl(current), startedAtMs, endedAtMs,
+          elapsedMs: Math.max(0, endedAtMs - startedAtMs), outcome: 'response',
+          status: res.status, errorCode: null, retryDecision: 'retry',
+        });
         await res.body?.cancel();
         lastError = new Error(`server returned ${res.status}`);
       } catch (err) {
-        lastError = err;
-        if (controller.signal.aborted) throw controller.signal.reason ?? err;
+        lastError = controller.signal.aborted ? (controller.signal.reason ?? err) : err;
+        const retry = attempt < Math.max(1, maxRetries) - 1;
+        const endedAtMs = performance.now();
+        attempts.push({
+          attempt: attemptNumber, url: attemptUrl(current), startedAtMs, endedAtMs,
+          elapsedMs: Math.max(0, endedAtMs - startedAtMs), outcome: 'error', status: null,
+          errorCode: errorCode(lastError), retryDecision: retry ? 'retry' : 'stop',
+        });
       } finally {
         clearTimeout(timer);
       }
@@ -129,21 +290,56 @@ export async function fetchUrl(url, {
         await new Promise((resolve) => setTimeout(resolve, delay));
       }
     }
-    if (lastError) throw lastError;
+    if (lastError) throw attachAttempts(lastError, attempts);
 
     const headers = Object.fromEntries(res.headers.entries());
     if (res.status >= 300 && res.status < 400 && headers.location) {
+      const endedAtMs = performance.now();
+      attempts.push({
+        attempt: attemptNumber, url: attemptUrl(current), startedAtMs: responseStartedAtMs, endedAtMs,
+        elapsedMs: Math.max(0, endedAtMs - responseStartedAtMs), outcome: 'response',
+        status: res.status, errorCode: null,
+        retryDecision: redirects === maxRedirects ? 'stop' : 'redirect',
+      });
       if (redirects === maxRedirects) break;
-      const next = new URL(headers.location, current);
-      if (next.origin !== allowedOrigin) throw new Error(`redirect leaves audited origin: ${next.href}`);
-      await validatePublicUrl(next.href, { lookup });
-      chain.push({ url: current, status: res.status, location: headers.location });
+      let next;
+      try {
+        next = new URL(headers.location, current);
+      } catch {
+        throw attachAttempts(codedFetchError('invalid redirect location', 'FETCH_REDIRECT_INVALID'), attempts);
+      }
+      if (next.origin !== allowedOrigin) {
+        throw attachAttempts(codedFetchError('redirect leaves audited origin', 'FETCH_REDIRECT_ORIGIN'), attempts);
+      }
+      chain.push({ url: attemptUrl(current), status: res.status, location: attemptUrl(next.href) });
       current = next.href;
       await res.body?.cancel();
       continue;
     }
-    const body = await readBodyLimited(res, maxBodyBytes);
-    return { url: current, requestedUrl: url, status: res.status, headers, body, redirectChain: chain };
+    try {
+      const { body, bodyBytes } = await readBodyLimited(res, maxBodyBytes, responseType);
+      const endedAtMs = performance.now();
+      attempts.push({
+        attempt: attemptNumber, url: attemptUrl(current), startedAtMs: responseStartedAtMs, endedAtMs,
+        elapsedMs: Math.max(0, endedAtMs - responseStartedAtMs), outcome: 'response',
+        status: res.status, errorCode: null, retryDecision: res.status >= 500 ? 'stop' : 'complete',
+      });
+      return {
+        url: current, requestedUrl: url, status: res.status, headers, body, bodyBytes,
+        bodyComplete: true, redirectChain: chain, attempts,
+      };
+    } catch (error) {
+      const endedAtMs = performance.now();
+      attempts.push({
+        attempt: attemptNumber, url: attemptUrl(current), startedAtMs: responseStartedAtMs, endedAtMs,
+        elapsedMs: Math.max(0, endedAtMs - responseStartedAtMs), outcome: 'error', status: res.status,
+        errorCode: errorCode(error), retryDecision: 'stop',
+      });
+      throw attachAttempts(error, attempts);
+    }
   }
-  throw new Error(`redirect chain exceeded ${maxRedirects} hops for ${url}`);
+  throw attachAttempts(codedFetchError(
+    `redirect chain exceeded ${maxRedirects} hops`,
+    'FETCH_REDIRECT_LIMIT',
+  ), attempts);
 }