mixdog 0.9.3 → 0.9.4

package/src/runtime/search/lib/web-tools.mjs

@@ -1,11 +1,31 @@
-import fs, { readFileSync } from 'fs'
-import dns from 'dns'
-import net from 'net'
-import { Agent, fetch as undiciFetch } from 'undici'
+import fs from 'fs'
 
 import { Readability } from '@mozilla/readability'
 import { isWSL } from '../../shared/wsl.mjs'
 import { startChildGuardian } from '../../shared/child-guardian.mjs'
+import {
+  normalizeUrl,
+  assertPublicUrl,
+  assertResolvedIps,
+} from './ssrf-guard.mjs'
+import {
+  buildHeaders,
+  MAX_BODY_BYTES,
+  isFatalHttpPathPolicyError,
+  fetchPinnedForPausedRequest,
+  fetchHtml,
+  _metaRefreshTarget,
+} from './http-fetch.mjs'
+
+// Facade re-exports: SSRF-guard and HTTP-fetch clusters moved to dedicated
+// modules; keep the original public surface resolving unchanged for importers.
+export {
+  assertPublicUrl,
+  resolveAndValidate,
+  assertResolvedIps,
+  pinnedFetch,
+} from './ssrf-guard.mjs'
+export { isFatalHttpPathPolicyError } from './http-fetch.mjs'
 
 // Lazy heavy deps: importing jsdom (~400ms) and puppeteer-core (~130ms) at
 // module load added ~540ms to the first web search even when the request never
@@ -23,8 +43,6 @@ async function loadPuppeteer() {
   if (!_puppeteer) _puppeteer = (await import('puppeteer-core')).default
   return _puppeteer
 }
-
-const PKG_VERSION = (() => { try { return JSON.parse(readFileSync(new URL('../package.json', import.meta.url), 'utf8')).version } catch { return '0.0.1' } })()
 import {
   noteProviderFailure,
   noteProviderSuccess,
@@ -98,346 +116,6 @@ export function getScrapeCapabilities() {
   }
 }
 
-function normalizeUrl(url) {
-  const parsed = new URL(url)
-  parsed.hash = ''
-  return parsed.toString()
-}
-
-function assertPrivateIpv4(hostname) {
-  const ipv4Match = hostname.match(/^(\d{1,3})\.(\d{1,3})\.(\d{1,3})\.(\d{1,3})$/)
-  if (!ipv4Match) return
-  const [, a, b] = ipv4Match.map(Number)
-  if (a === 127 || a === 10 || a === 0 ||
-      (a === 172 && b >= 16 && b <= 31) ||
-      (a === 192 && b === 168) ||
-      (a === 169 && b === 254) ||
-      (a === 100 && b >= 64 && b <= 127) ||
-      (a === 198 && b >= 18 && b <= 19) ||
-      (a >= 224 && a <= 239) ||
-      (a >= 240)) {
-    throw new Error(`Blocked request to private address: ${hostname}`)
-  }
-}
-
-// Detect IPv4-mapped IPv6 (::ffff:/96) in BOTH dotted and hex forms and
-// return the embedded IPv4 as a dotted-quad string, or null when the input
-// is not an IPv4-mapped address. WHATWG URL canonicalises `[::ffff:127.0.0.1]`
-// to `[::ffff:7f00:1]`, so the hex form must be handled or assertPublicUrl /
-// _validateIpv6 will miss mapped loopback / private addresses.
-function _mappedIpv4FromIpv6(bare) {
-  const lower = bare.toLowerCase()
-  // Dotted form: ::ffff:a.b.c.d
-  const dotted = lower.match(/^::ffff:(\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3})$/)
-  if (dotted) return dotted[1]
-  // Hex form: ::ffff:HHHH:LLLL — low 32 bits of the /96 prefix carry the IPv4.
-  const hex = lower.match(/^::ffff:([0-9a-f]{1,4}):([0-9a-f]{1,4})$/)
-  if (hex) {
-    const high = parseInt(hex[1], 16)
-    const low = parseInt(hex[2], 16)
-    if (Number.isFinite(high) && Number.isFinite(low) && high <= 0xffff && low <= 0xffff) {
-      const a = (high >> 8) & 0xff
-      const b = high & 0xff
-      const c = (low >> 8) & 0xff
-      const d = low & 0xff
-      return `${a}.${b}.${c}.${d}`
-    }
-  }
-  return null
-}
-
-export function assertPublicUrl(url) {
-  const parsed = new URL(url)
-
-  // Block dangerous protocols
-  const blockedProtocols = ['file:', 'ftp:', 'data:', 'javascript:']
-  if (blockedProtocols.includes(parsed.protocol)) {
-    throw new Error(`Blocked non-HTTP protocol: ${parsed.protocol}`)
-  }
-  if (parsed.protocol !== 'http:' && parsed.protocol !== 'https:') {
-    throw new Error(`Blocked non-HTTP protocol: ${parsed.protocol}`)
-  }
-
-  const hostname = parsed.hostname.toLowerCase()
-
-  // Reject userinfo (user:pass@host) — credential-injection / SSRF vector
-  if (parsed.username || parsed.password) {
-    throw new Error(`Blocked URL with userinfo credentials: ${hostname}`)
-  }
-
-  // Localhost
-  if (hostname === 'localhost') {
-    throw new Error(`Blocked request to private address: ${hostname}`)
-  }
-
-  // IPv4 private/reserved ranges
-  assertPrivateIpv4(hostname)
-
-  // Strip brackets for IPv6 analysis (URL parser stores IPv6 without brackets in .hostname)
-  const bare = hostname.startsWith('[') ? hostname.slice(1, -1) : hostname
-
-  // IPv6 loopback
-  if (bare === '::1') {
-    throw new Error(`Blocked request to private address: ${hostname}`)
-  }
-
-  // IPv6 unspecified (::)
-  if (bare === '::') {
-    throw new Error(`Blocked request to private address: ${hostname}`)
-  }
-
-  // IPv6 multicast (ff00::/8)
-  if (/^ff/i.test(bare)) {
-    throw new Error(`Blocked request to private address: ${hostname}`)
-  }
-
-  // IPv4-mapped IPv6 — ::ffff:a.b.c.d
-  // Cover both dotted (::ffff:127.0.0.1) and hex (::ffff:7f00:1) forms —
-  // WHATWG URL canonicalises bracketed mapped literals to the hex shape.
-  const mappedIpv4 = _mappedIpv4FromIpv6(bare)
-  if (mappedIpv4) {
-    assertPrivateIpv4(mappedIpv4)
-  }
-
-  // IPv6 private (fc00::/7 — starts with fc or fd)
-  if (/^f[cd]/i.test(bare)) {
-    throw new Error(`Blocked request to private address: ${hostname}`)
-  }
-
-  // IPv6 link-local (fe80::/10 — starts with fe8, fe9, fea, feb)
-  if (/^fe[89ab]/i.test(bare)) {
-    throw new Error(`Blocked request to private address: ${hostname}`)
-  }
-}
-
-function _validateIpv6(ip) {
-  const lower = ip.toLowerCase()
-  if (lower === '::1') {
-    throw new Error(`Blocked request to private address: ${ip}`)
-  }
-  if (lower === '::') {
-    throw new Error(`Blocked request to private address: ${ip}`)
-  }
-  if (/^ff/i.test(lower)) {
-    throw new Error(`Blocked request to private address: ${ip}`)
-  }
-  if (/^f[cd]/i.test(lower)) {
-    throw new Error(`Blocked request to private address: ${ip}`)
-  }
-  if (/^fe[89ab]/i.test(lower)) {
-    throw new Error(`Blocked request to private address: ${ip}`)
-  }
-  // Cover both dotted and hex IPv4-mapped IPv6 forms — resolver output and
-  // WHATWG-canonicalised URL hostnames may arrive as `::ffff:7f00:1`.
-  const mappedIpv4 = _mappedIpv4FromIpv6(lower)
-  if (mappedIpv4) {
-    assertPrivateIpv4(mappedIpv4)
-  }
-}
-
-// Resolve hostname once, validate EVERY returned address (so a DNS round-robin
-// can't smuggle a private IP behind a public one), and return the de-duped
-// `{address, family}` list. The caller pins the real connection to one of
-// these addresses so a second uncontrolled resolution (DNS rebinding / TOCTOU)
-// cannot flip the IP between validation and connect.
-// Race a DNS promise against an abort signal so a hung resolver cannot
-// outlive the request's timeout budget. The signal is the same one that
-// bounds the outbound fetch (AbortSignal.timeout / requestTimeoutMs), so
-// DNS is bounded by the same deadline as the connection.
-function _abortRace(promise, signal, label) {
-  if (!signal) return promise
-  if (signal.aborted) return Promise.reject(signal.reason || new Error(`${label} aborted`))
-  return new Promise((resolve, reject) => {
-    const onAbort = () => reject(signal.reason || new Error(`${label} aborted`))
-    signal.addEventListener('abort', onAbort, { once: true })
-    promise.then(
-      (value) => { signal.removeEventListener('abort', onAbort); resolve(value) },
-      (err) => { signal.removeEventListener('abort', onAbort); reject(err) },
-    )
-  })
-}
-
-export async function resolveAndValidate(hostname, { signal } = {}) {
-  // Literal IPs bypass DNS entirely — validate directly.
-  if (net.isIP(hostname)) {
-    if (net.isIPv4(hostname)) {
-      assertPrivateIpv4(hostname)
-      return [{ address: hostname, family: 4 }]
-    }
-    _validateIpv6(hostname)
-    return [{ address: hostname, family: 6 }]
-  }
-
-  const addresses = []
-  const seen = new Set()
-  const push = (address, family) => {
-    const key = `${family}:${address}`
-    if (seen.has(key)) return
-    seen.add(key)
-    addresses.push({ address, family })
-  }
-
-  // dns.lookup mirrors what the platform resolver will hand to the connector;
-  // resolve4/resolve6 catch entries the stub resolver returns even when the
-  // OS lookup table would omit them.
-  let lookupAddrs = []
-  try {
-    lookupAddrs = await _abortRace(dns.promises.lookup(hostname, { all: true }), signal, 'dns.lookup')
-  } catch (err) {
-    if (err.code !== 'ENODATA' && err.code !== 'ENOTFOUND') throw err
-  }
-  for (const entry of lookupAddrs) {
-    if (entry.family === 4) assertPrivateIpv4(entry.address)
-    else _validateIpv6(entry.address)
-    push(entry.address, entry.family)
-  }
-
-  let v4Addrs = []
-  try {
-    v4Addrs = await _abortRace(dns.promises.resolve4(hostname), signal, 'dns.resolve4')
-  } catch (err) {
-    if (err.code !== 'ENODATA' && err.code !== 'ENOTFOUND') throw err
-  }
-  for (const ip of v4Addrs) {
-    assertPrivateIpv4(ip)
-    push(ip, 4)
-  }
-
-  let v6Addrs = []
-  try {
-    v6Addrs = await _abortRace(dns.promises.resolve6(hostname), signal, 'dns.resolve6')
-  } catch (err) {
-    if (err.code !== 'ENODATA' && err.code !== 'ENOTFOUND') throw err
-  }
-  for (const ip of v6Addrs) {
-    _validateIpv6(ip)
-    push(ip, 6)
-  }
-
-  return addresses
-}
-
-export async function assertResolvedIps(hostname) {
-  // Backward-compatible wrapper: callers that only need validation (e.g. the
-  // Puppeteer request interceptor, which cannot pin Chromium's connect) still
-  // get the same throw-on-private behaviour.
-  // Fail closed: an empty result (no DNS records, all lookups returned
-  // ENODATA/ENOTFOUND) must NOT be treated as success — the Puppeteer path
-  // would otherwise hand the raw hostname to Chromium for a second,
-  // unvalidated resolution.
-  // Callers pass `new URL(...).hostname`, which on Node/Bun keeps the
-  // brackets around IPv6 literals (e.g. `[2606:4700::1111]`). Strip them
-  // here so resolveAndValidate's net.isIP() path recognises the literal
-  // instead of falling through to a doomed DNS lookup on `[..]`.
-  const bare = _bareHost(hostname)
-  const addresses = await resolveAndValidate(bare)
-  if (!addresses || addresses.length === 0) {
-    throw new Error(`DNS returned no addresses for ${hostname}`)
-  }
-}
-
-// Bare hostname helper that strips IPv6 brackets — undici / WHATWG URL stores
-// IPv6 hostnames with the brackets included.
-function _bareHost(hostname) {
-  return hostname.startsWith('[') ? hostname.slice(1, -1) : hostname
-}
-
-// SSRF-hardened fetch: resolves the host ONCE, validates every returned
-// address, then connects to a single pre-validated IP via a per-request
-// undici Agent whose `connect.lookup` returns that IP only. This closes the
-// validate-then-fetch TOCTOU / DNS-rebinding window because the connector
-// never performs a second resolution against the live DNS — the Host header
-// (undici fills from the URL) and TLS SNI (likewise) are unaffected, so
-// virtual hosts and HTTPS certificate validation keep working against
-// legitimate public sites.
-export async function pinnedFetch(url, options = {}) {
-  const parsed = new URL(url)
-  const host = _bareHost(parsed.hostname)
-  // Bound the validating DNS lookups by the request's own abort signal so a
-  // hung resolver cannot outlive the fetch timeout.
-  const addresses = await resolveAndValidate(host, { signal: options.signal })
-  if (addresses.length === 0) {
-    throw new Error(`DNS returned no addresses for ${host}`)
-  }
-  // Deterministic: pin to the first validated address. Every entry in
-  // `addresses` already passed assertPrivateIpv4 / IPv6 checks, so picking any
-  // index is safe — first-match keeps behaviour stable across calls.
-  const pinned = addresses[0]
-  const dispatcher = new Agent({
-    connect: {
-      // Custom lookup invoked by undici's connector. We ignore the requested
-      // hostname argument and unconditionally hand back the pre-validated IP,
-      // so DNS rebinding cannot flip the address between assert and connect.
-      lookup: (_hostname, opts, cb) => {
-        if (opts && opts.all) {
-          cb(null, [{ address: pinned.address, family: pinned.family }])
-        } else {
-          cb(null, pinned.address, pinned.family)
-        }
-      },
-    },
-  })
-  // The per-request Agent owns a dedicated connection pool. If it is never
-  // closed it leaks the kept-alive socket until GC. Destroy it once the body
-  // is fully consumed, cancelled, or the request errors — wrapping the body
-  // stream so the dispatcher outlives streaming reads but is always reclaimed.
-  let response
-  try {
-    response = await undiciFetch(url, { ...options, dispatcher })
-  } catch (err) {
-    dispatcher.destroy().catch(() => {})
-    throw err
-  }
-  let cleaned = false
-  const cleanup = () => { if (!cleaned) { cleaned = true; dispatcher.destroy().catch(() => {}) } }
-  // If there's no body to stream, the response is already complete.
-  if (!response.body) {
-    cleanup()
-    return response
-  }
-  // Wrap the body in a ReadableStream that pulls from the original reader and
-  // destroys the dispatcher when the stream ends, errors, or the consumer
-  // cancels it. ReadableStream's underlying-source pull/cancel callbacks are
-  // reliably invoked, so the per-request Agent is always reclaimed instead of
-  // leaking its kept-alive socket until GC.
-  const reader = response.body.getReader()
-  const monitored = new ReadableStream({
-    async pull(controller) {
-      try {
-        const { done, value } = await reader.read()
-        if (done) {
-          controller.close()
-          cleanup()
-          return
-        }
-        controller.enqueue(value)
-      } catch (err) {
-        controller.error(err)
-        cleanup()
-      }
-    },
-    cancel(reason) {
-      reader.cancel(reason).catch(() => {})
-      cleanup()
-    },
-  })
-  return new Response(monitored, {
-    status: response.status,
-    statusText: response.statusText,
-    headers: response.headers,
-  })
-}
-
-function withTimeout(controller, timeoutMs) {
-  return setTimeout(() => controller.abort(), timeoutMs)
-}
-
-function buildHeaders() {
-  return {
-    'User-Agent': `mixdog-search/${PKG_VERSION}`,
-  }
-}
-
 function buildContentPayload(url, title, content, extractor, extra = {}) {
   // Whitespace-normalize extracted text so blank-line runs from page layout
   // don't eat the caller's maxLength window. Per-line interior spacing is
@@ -560,262 +238,6 @@ async function extractReadableArticle(url, html) {
   }
 }
 
-const REDIRECT_STATUSES = new Set([301, 302, 303, 307, 308])
-const MAX_REDIRECTS = 5
-// Hard cap on response body size (10 MB) to prevent memory DoS from a
-// hostile / misconfigured URL returning a huge body. Applied in two places:
-//   1. Content-Length pre-check (cheap reject before reading bytes).
-//   2. Streaming byte counter (covers chunked transfer / missing header).
-const MAX_BODY_BYTES = 10 * 1024 * 1024
-
-/** HTTP-path policy failures must not fall through to the Puppeteer extractor. */
-export function isFatalHttpPathPolicyError(error) {
-  const msg = error instanceof Error ? error.message : String(error)
-  if (/response body too large|page content too large|Content-Length=.*> cap=/i.test(msg)) return true
-  if (/Blocked non-text content-type/i.test(msg)) return true
-  if (/cross-host redirect blocked/i.test(msg)) return true
-  if (/Blocked request to private|Blocked non-HTTP|Blocked URL with userinfo/i.test(msg)) return true
-  if (/DNS returned no addresses/i.test(msg)) return true
-  if (/Too many redirects/i.test(msg)) return true
-  return false
-}
-
-async function readBodyWithCap(response, maxBytes) {
-  // Reject non-text content-types early; decode by content-type charset.
-  const contentType = (response.headers.get('content-type') || '').toLowerCase()
-  if (contentType) {
-    const isText = contentType.includes('text/') || contentType.includes('/html') ||
-      contentType.includes('/xml') || contentType.includes('/json') ||
-      contentType.includes('javascript') || contentType.includes('application/x-www-form-urlencoded')
-    if (!isText) {
-      // Cancel body before throwing so the underlying socket isn't held
-      // until GC — fetchHtml's caller would otherwise leak the connection.
-      try { await response.body?.cancel() } catch {}
-      throw new Error(`Blocked non-text content-type: ${contentType.split(';')[0].trim()}`)
-    }
-  }
-  const charsetMatch = contentType.match(/charset=([\w-]+)/i)
-  const charset = charsetMatch ? charsetMatch[1] : 'utf-8'
-
-  const contentLength = Number(response.headers.get('content-length') || 0)
-  if (contentLength > maxBytes) {
-    try { await response.body?.cancel() } catch {}
-    throw new Error(`response body too large: Content-Length=${contentLength} > cap=${maxBytes}`)
-  }
-  const reader = response.body?.getReader?.()
-  if (!reader) {
-    // Fallback for environments without a readable stream — post-check length.
-    const text = await response.text()
-    if (text.length > maxBytes) {
-      // response.text() already drained the body, but guard symmetrically.
-      try { await response.body?.cancel() } catch {}
-      throw new Error(`response body too large: ${text.length} bytes > cap=${maxBytes}`)
-    }
-    return text
-  }
-  const chunks = []
-  let total = 0
-  try {
-    while (true) {
-      const { done, value } = await reader.read()
-      if (done) break
-      total += value.byteLength
-      if (total > maxBytes) {
-        try { await reader.cancel() } catch {}
-        throw new Error(`response body too large: received ${total}+ bytes > cap=${maxBytes}`)
-      }
-      chunks.push(value)
-    }
-  } finally {
-    try { reader.releaseLock() } catch {}
-  }
-  const decoder = new TextDecoder(charset, { fatal: false })
-  let text = ''
-  for (const chunk of chunks) text += decoder.decode(chunk, { stream: true })
-  text += decoder.decode()
-  return text
-}
-
-/** Binary-safe body reader for CDP Fetch fulfillment (no text-only filter). */
-async function readBodyBytesWithCap(response, maxBytes) {
-  const contentLength = Number(response.headers.get('content-length') || 0)
-  if (contentLength > maxBytes) {
-    try { await response.body?.cancel() } catch {}
-    throw new Error(`response body too large: Content-Length=${contentLength} > cap=${maxBytes}`)
-  }
-  const reader = response.body?.getReader?.()
-  if (!reader) {
-    const buf = Buffer.from(await response.arrayBuffer())
-    if (buf.byteLength > maxBytes) {
-      try { await response.body?.cancel() } catch {}
-      throw new Error(`response body too large: ${buf.byteLength} bytes > cap=${maxBytes}`)
-    }
-    return buf
-  }
-  const chunks = []
-  let total = 0
-  try {
-    while (true) {
-      const { done, value } = await reader.read()
-      if (done) break
-      total += value.byteLength
-      if (total > maxBytes) {
-        try { await reader.cancel() } catch {}
-        throw new Error(`response body too large: received ${total}+ bytes > cap=${maxBytes}`)
-      }
-      chunks.push(value)
-    }
-  } finally {
-    try { reader.releaseLock() } catch {}
-  }
-  return Buffer.concat(chunks.map((c) => Buffer.from(c)))
-}
-
-const CDP_FORBIDDEN_RESPONSE_HEADERS = new Set([
-  'content-length',
-  'transfer-encoding',
-  // undici decodes gzip/br/deflate; body passed to fulfillRequest is plain bytes
-  'content-encoding',
-])
-
-function headersToCdpPairs(headers) {
-  const out = []
-  headers.forEach((value, name) => {
-    const lower = name.toLowerCase()
-    if (CDP_FORBIDDEN_RESPONSE_HEADERS.has(lower)) return
-    out.push({ name, value })
-  })
-  return out
-}
-
-/**
- * Pinned fetch for a paused Chromium request: validate each hop, follow redirects,
- * return bytes for Fetch.fulfillRequest. Chromium never performs its own DNS/connect.
- */
-async function fetchPinnedForPausedRequest(url, { signal, method = 'GET', headers = {}, body } = {}) {
-  const upperMethod = (method || 'GET').toUpperCase()
-  let currentUrl = url
-  for (let hops = 0; ; hops++) {
-    assertPublicUrl(currentUrl)
-    const response = await pinnedFetch(currentUrl, {
-      signal,
-      method: upperMethod,
-      headers,
-      body: hops === 0 ? body : undefined,
-      redirect: 'manual',
-    })
-    if (REDIRECT_STATUSES.has(response.status)) {
-      try { await response.body?.cancel() } catch {}
-      if (hops >= MAX_REDIRECTS) {
-        throw new Error(`Too many redirects (max ${MAX_REDIRECTS})`)
-      }
-      const location = response.headers.get('location')
-      if (!location) {
-        throw new Error(`Redirect ${response.status} without Location header`)
-      }
-      currentUrl = new URL(location, currentUrl).toString()
-      continue
-    }
-    const respBody = await readBodyBytesWithCap(response, MAX_BODY_BYTES)
-    return {
-      status: response.status,
-      responseHeaders: headersToCdpPairs(response.headers),
-      body: respBody,
-    }
-  }
-}
-
-async function fetchHtml(url, timeoutMs, signal) {
-  const controller = new AbortController()
-  const timer = withTimeout(controller, timeoutMs)
-  // Propagate an external (tool-call) abort into the local timeout controller
-  // so a cancelled web_fetch tears down the in-flight request promptly.
-  let onExternalAbort
-  if (signal) {
-    if (signal.aborted) controller.abort(signal.reason)
-    else {
-      onExternalAbort = () => controller.abort(signal.reason)
-      signal.addEventListener('abort', onExternalAbort, { once: true })
-    }
-  }
-  const originalHost = new URL(url).hostname.replace(/^www\./, '')
-  try {
-    let currentUrl = url
-    for (let hops = 0; ; hops++) {
-      // pinnedFetch resolves+validates the host once and forces the
-      // connection to the validated IP — closes the validate-then-fetch
-      // TOCTOU / DNS-rebinding window that bare `fetch` left open.
-      const response = await pinnedFetch(currentUrl, {
-        signal: controller.signal,
-        headers: buildHeaders(),
-        redirect: 'manual',
-      })
-      if (REDIRECT_STATUSES.has(response.status)) {
-        // Drain the redirect response body so the socket isn't held until GC.
-        try { await response.body?.cancel() } catch {}
-        if (hops >= MAX_REDIRECTS) {
-          throw new Error(`Too many redirects (max ${MAX_REDIRECTS})`)
-        }
-        const location = response.headers.get('location')
-        if (!location) {
-          throw new Error(`Redirect ${response.status} without Location header`)
-        }
-        const nextUrl = new URL(location, currentUrl).toString()
-        assertPublicUrl(nextUrl)
-        const nextHost = new URL(nextUrl).hostname.replace(/^www\./, '')
-        if (nextHost !== originalHost) {
-          throw new Error(`cross-host redirect blocked (redirected_to: ${nextUrl})`)
-        }
-        currentUrl = nextUrl
-        continue
-      }
-      if (!response.ok) {
-        // Drain the error response body before propagating.
-        try { await response.body?.cancel() } catch {}
-        const err = new Error(`HTTP ${response.status}`)
-        err.status = response.status
-        throw err
-      }
-      return await readBodyWithCap(response, MAX_BODY_BYTES)
-    }
-  } finally {
-    clearTimeout(timer)
-    if (onExternalAbort) signal.removeEventListener('abort', onExternalAbort)
-  }
-}
-
-// Parse a short-delay <meta http-equiv="refresh" content="N; url=..."> from
-// the document head. Browsers treat these as redirects, but fetchHtml only
-// follows HTTP-level (3xx) redirects — without this, a stub page like
-// tree-sitter.github.io (tiny body + meta refresh) is returned as the
-// "article". Long-delay refreshes (>5s) are page auto-reloads, not
-// redirects, and are deliberately NOT followed.
-function _metaRefreshTarget(html, baseUrl) {
-  const head = String(html || '').slice(0, 8192)
-  const tags = head.match(/<meta\b[^>]*>/gi) || []
-  for (const tag of tags) {
-    if (!/http-equiv\s*=\s*["']?refresh\b/i.test(tag)) continue
-    // Quote-aware capture: the attribute value may NEST the other quote kind
-    // (content="0; url='...'"), so a combined ["'] char class would cut the
-    // capture at the inner quote. Match each quote style to its own closer.
-    const m = /content\s*=\s*"([^"]*)"/i.exec(tag)
-      || /content\s*=\s*'([^']*)'/i.exec(tag)
-      || /content\s*=\s*([^\s>]+)/i.exec(tag)
-    if (!m) continue
-    const cm = /^\s*(\d+(?:\.\d+)?)\s*[;,]\s*url\s*=\s*['"]?([^'"]+?)['"]?\s*$/i.exec(m[1])
-    if (!cm) continue
-    const delay = Number(cm[1])
-    if (!Number.isFinite(delay) || delay > 5) continue
-    try {
-      const resolved = new URL(cm[2].trim(), baseUrl)
-      if (resolved.protocol !== 'http:' && resolved.protocol !== 'https:') continue
-      if (resolved.href === baseUrl) continue
-      return resolved.href
-    } catch { continue }
-  }
-  return null
-}
-
 async function scrapeWithReadability(url, timeoutMs, signal) {
   let currentUrl = url
   let html = await fetchHtml(currentUrl, timeoutMs, signal)

package/src/runtime/shared/atomic-file.mjs

@@ -376,7 +376,32 @@ export function writeFileAtomicSync(filePath, data, opts = {}) {
         }
         try { unlinkSync(tmp); } catch {}
       } else {
-        renameWithRetrySync(tmp, filePath, opts);
+        try {
+          renameWithRetrySync(tmp, filePath, opts);
+        } catch (err) {
+          // Opt-in fallback for callers whose target is a rename-lock hot
+          // spot on win32 (e.g. AV/indexer briefly holding the destination
+          // handle open), where rename-over keeps failing with EPERM/EACCES
+          // /EBUSY even after renameWithRetrySync's backoff loop. The write
+          // is already serialized by the caller's own withFileLockSync
+          // (opts.lock / updateJsonAtomicSync's lock), so a direct
+          // truncate+write of the target here is still race-safe against
+          // other writers using the same lock path; it just loses the
+          // rename's fully-atomic-swap guarantee for THIS write. Default
+          // behavior (throw) is unchanged for every other caller.
+          if (
+            opts.renameFallback === 'truncate'
+            && opts.secret !== true
+            && process.platform === 'win32'
+            && RETRY_CODES.has(err?.code)
+          ) {
+            const data = readFileSync(tmp);
+            writeFileSync(filePath, data, { mode: opts.mode !== undefined ? opts.mode : 0o600 });
+            try { unlinkSync(tmp); } catch {}
+          } else {
+            throw err;
+          }
+        }
       }
       if (opts.secret === true) {
         // Re-assert owner-only on the final path (createOnly's linkSync may