kaloko 0.20.0 → 0.20.1

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "kaloko",
-  "version": "0.20.0",
+  "version": "0.20.1",
   "description": "Your AI agents build; Kaloko walks every screen, puts it on one canvas and records who approved what.",
   "keywords": [
     "qa",

package/skills/kaloko/reference/stacks.md

@@ -80,6 +80,12 @@ Item sources:
   `sitemap-de.xml`; `en_US` finds `en-us`, `en` does not), only that
   locale's sitemaps are read; otherwise all of them. Use it when a site
   lists one sitemap per language, so the scenario does not hard-code paths.
+- `kaloko walk` reads robots.txt and sitemaps with the environment's gate
+  (headers, HTTP Basic, client certificates) and the walk's cookies, on the
+  environment's own origins only, so a staging behind Basic auth or mTLS
+  needs no public sitemap. A sitemap on another host (a CDN, an index
+  pointing elsewhere) or a redirect to another host gets none of them.
+  Redirects are followed by hand (at most 5); 60 s and 60 MB per file.
 - `crawl: { from: /clanky, links: "a.card", next: "a.next", pages: 20 }`
   collects links from a listing and follows its pagination.
 - `script: qa/steps/items.mjs` uses the default export
@@ -229,9 +235,12 @@ stack, set:
   locale).
 - A script that lists pages per locale instead: give each page the same
   `key` in every locale (for example the source path), so they are one item.
-- Hosted runs (`kaloko schedule add`) do not group variants yet: there each
-  locale reads its own list and HC1005 is not reported (`kaloko validate`
-  warns). Use `kaloko walk` or the scheduled GitHub workflow.
+- Hosted runs (`kaloko schedule add`) group variants the same way: same
+  ids, window, `missing` and HC1005. They read each source page's head for
+  its hreflang on the environment's own addresses (with the schedule's
+  password or cookies), and each read counts as one hosted page of the
+  monthly budget. Budget a stack of N source pages in L locales as
+  N reads + N × L pages at most.
 
 ## Cost and size: choose deliberately
 

package/src/cli.mjs

@@ -33,7 +33,7 @@ import { detectProject, initScenario, skeletonScenario, scenarioHeader, formatDe
 import { checkSlug, checkName, setConfigKeys, createProject, renameProject, syncProject, shareProjectNote } from './lib/projects-cli.mjs'
 import { recheckCaptures } from './lib/recheck.mjs'
 import { runFindings, findingsText } from './lib/findings.mjs'
-import { resolveItems, fieldsOf, DEFAULT_LIMIT, timeLeft, rotationState, rotationCoverage, sourceLocaleOf, variantReader } from './lib/stack.mjs'
+import { resolveItems, fieldsOf, DEFAULT_LIMIT, timeLeft, rotationState, rotationCoverage, sourceLocaleOf, stackReader, readsOutside } from './lib/stack.mjs'
 import { walkSiteChecks, siteChecksOn } from './lib/site.mjs'
 import { DEFAULTS as JEV_DEFAULTS } from './lib/jev.mjs'
 import { expandPacks } from './lib/packs.mjs'
@@ -818,8 +818,9 @@ async function walkStack ({ node, locale, page, env, run, scenario, scenarioDir,
   const sourceLocale = node.items?.variants ? sourceLocaleOf(node, scenario, locale) : locale
   const counter = state && node.items.variants && node.items.rotate === 'run' ? await rotationState(statePath('variants'), run.meta.id) : null
   const round = counter ? counter.run : state?.run || 0
-  // locale variants read the source pages for their hreflang with the environment's access (variantReader)
-  const reader = node.items?.variants ? await variantReader({ page, gate, origins: envOrigins(env, scenario, locale) }) : null
+  // sitemaps, robots.txt and the hreflang of source pages are read with the environment's access (stackReader): its
+  // gate and the walk's cookies on its own origins only
+  const reader = readsOutside(node.items) ? await stackReader({ page, gate, origins: envOrigins(env, scenario, locale) }) : null
   let resolved
   try { resolved = await resolveItems({ node, base: baseUrl(env, locale), sourceBase: baseUrl(env, sourceLocale), page, scenarioDir, env, locale, scenario, limit: Number(args.limit) || undefined, rotation: { run: round }, checked: state?.checked || null, only: list(args.items), cache, readPage: reader?.read || null }) } finally { await reader?.dispose() }
   const { items, total, truncated, all, rotation, partial, variants, resolve_failed: unresolved, resolve_error: unresolvedWhy } = resolved
@@ -1610,7 +1611,7 @@ const commands = {
     const pick = list(args.steps)
     // stacks with locale variants read their source list and each page's hreflang once per walk, for every locale
     const stackCache = new Map()
-    const stackGate = await gateForRequests(env, { root }) // the gate for requests outside the browser: link checks, hreflang of stack pages
+    const stackGate = await gateForRequests(env, { root }) // the gate for requests outside the browser: link checks, stack sitemaps and hreflang
     const locales = list(args.locales) || run.meta.scope.locales
     const scope = run.meta.scope
     // browser engines and device profiles (batch E): one pass each, capturing only its own keys; a run without

package/src/lib/scenario.mjs

@@ -120,7 +120,6 @@ export function scenarioWarnings (data, { projectRequired = null } = {}) {
   const out = []
   if (!data || !Array.isArray(data.nodes)) return out
   for (const node of data.nodes) {
-    if (node.kind === 'stack' && node.items?.variants) out.push(`node "${node.id}": hosted runs (kaloko schedule add) do not group locale variants yet: there each language reads its own list and missing translations (HC1005) are not reported; kaloko walk and the scheduled GitHub workflow do`)
     if (node.kind === 'stack' && node.items?.variants && (data.locales || []).length < 2) out.push(`node "${node.id}": items.variants: hreflang groups the language versions of a page, but the scenario walks ${(data.locales || []).length ? `one locale (${data.locales[0]})` : 'no locales'}; list the other locales under locales:`)
     if (node.kind === 'stack' && (node.packs || []).includes('errors')) out.push(`node "${node.id}": the errors pack is meant for error pages (404, 500, maintenance); on a stack of normal pages HC601 and HC605 stay not applicable. Remove it, or give the step expect_status when its pages are error pages`)
   }

package/src/lib/stack.mjs

@@ -8,7 +8,7 @@ import { pathToFileURL } from 'node:url'
 import { gunzipSync } from 'node:zlib'
 import { UserError, localized, readJson, writeJson } from './util.mjs'
 import { STACK_DEFAULT_FIELDS, STACK_SEMANTIC_SAMPLE, stackFieldsOf } from './manifest-core.mjs'
-import { variantItems, htmlAlternates, headerAlternates, sitemapAlternates, stripInert } from './variants-core.mjs'
+import { variantItems, sitemapAlternates, pageAlternates, readHeadFrom } from './variants-core.mjs'
 import { DEFAULT_LIMIT, SITEMAP_MAX_BYTES, SITEMAP_MAX_CHILDREN, ROTATE_MODES, itemId, sitemapLocs, robotsSitemaps, sitemapsForLocale, handoffOf, isoWeek, rotateWindow, patternFor as corePatternFor, withLocale, sitemapPages, selectItems } from './stack-core.mjs'
 
 /** How long items.resolve may take for one page (ms) before the page is walked without its edit link. */
@@ -94,7 +94,8 @@ export function rotationCoverage (all, checked) {
  * source page; `checked` and `only` use it). `variants` in the result: { mode, source_locale, locale, pages,
  * missing: [{ id, path, url, reason }], shared: [{ url, ids, paths }] } (`source: true` in the source locale). `cache`
  * (a Map the walk keeps for all its locales) reads the source list and each page's alternates once; `readPage` (a
- * pageReader with the environment's gate and the walk's cookies) reads the pages, else a plain one over `fetchImpl`.
+ * stackReader's read: the environment's gate and the walk's cookies) reads the sitemaps, robots.txt and the pages,
+ * else a plain pageReader over `fetchImpl`.
  */
 export async function resolveItems ({ node, base, page, scenarioDir, env, locale, limit, fetchImpl = fetch, rotation = {}, checked = null, scenario = null, only = null, resolveTimeout = RESOLVE_TIMEOUT, sourceBase = null, cache = null, readPage = null }) {
   const memo = cache || new Map()
@@ -111,7 +112,7 @@ export async function resolveItems ({ node, base, page, scenarioDir, env, locale
   const listLocale = mapped ? sourceLocale : locale
   const listBase = mapped ? (sourceBase || base) : base
   const listSrc = mapped ? withLocale(node.items || {}, listLocale) : src
-  const read = () => sourceEntries({ src: listSrc, locale: listLocale, base: listBase, page, scenarioDir, env, info: infoFor(listLocale), fetchImpl, withAlternates: variants })
+  const read = () => sourceEntries({ src: listSrc, locale: listLocale, base: listBase, page, scenarioDir, env, info: infoFor(listLocale), fetchImpl, readSource: readPage, withAlternates: variants })
   const { raw, alternates } = variants ? await cached(memo, `list|${scenario?.id || ''}|${node.id}|${listLocale}|${listBase}`, read) : await read()
   // 1. and 2.: match, exclude, deduplication, the window (stack-core.mjs, shared with hosted runs); with variants in
   // the source locale, so every locale walks the same window with the same ids
@@ -146,7 +147,7 @@ const urlKey = (u) => { try { return new URL(u).href } catch { return String(u)
  * { raw: [{ path, title?, key?, purpose?, edit_url?, content_ref? }], alternates: Map(url → the sitemap's xhtml:link
  * alternates of that page, read when `withAlternates`) }.
  */
-async function sourceEntries ({ src, locale, base, page, scenarioDir, env, info, fetchImpl, withAlternates = false }) {
+async function sourceEntries ({ src, locale, base, page, scenarioDir, env, info, fetchImpl, readSource = null, withAlternates = false }) {
   const origin = new URL(base).origin
   const abs = (p) => new URL(p, base.endsWith('/') ? base : base + '/').href.replace(/^(https?:\/\/[^/]+)\/\//, '$1/')
   const toUrl = (p) => (/^https?:\/\//.test(p) ? p : new URL(p, origin).href)
@@ -154,11 +155,15 @@ async function sourceEntries ({ src, locale, base, page, scenarioDir, env, info,
   if (src.urls) raw = src.urls.map((p) => ({ path: p }))
   else if (src.file) raw = await listFile(resolve(scenarioDir, src.file))
   else if (src.sitemap || src.robots) {
+    // through the environment's gate: its headers, Basic, client certificates and the walk's cookies on its own
+    // origins only; a sitemap on another host, or a redirect there, gets none of them
+    const reader = readSource || pageReader({ fetchImpl })
     const get = async (u, what = 'items.sitemap') => {
-      const r = await fetchImpl(u, { headers: { 'User-Agent': 'Kaloko', 'Accept-Encoding': 'gzip' } })
-      if (!r.ok) throw new UserError(`${what} ${u}: HTTP ${r.status}`)
-      if (what !== 'items.sitemap') return r.text()
-      const text = sitemapText(Buffer.from(await r.arrayBuffer()), u)
+      let r
+      try { r = await reader(u, { bytes: true }) } catch (e) { throw new UserError(`${what} ${u}: ${String(e?.message || e).split('\n')[0]}`) }
+      if (r.status < 200 || r.status >= 300) throw new UserError(`${what} ${u}: HTTP ${r.status}${r.url && r.url !== u ? ` (after a redirect to ${r.url})` : ''}`)
+      if (what !== 'items.sitemap') return r.body.toString('utf8')
+      const text = sitemapText(r.body, u)
       // with locale variants the xhtml:link alternates of each <url> are kept for pages that name none themselves
       if (withAlternates) for (const [k, v] of sitemapAlternates(text)) alternates.set(urlKey(k), v)
       return text
@@ -206,57 +211,76 @@ export const ALTERNATES_MAX = 1024 * 1024
 
 /**
  * A pageReader over a Playwright request client (APIRequestContext), for gates with client certificates, which fetch
- * cannot present: the client brings the certificates and the cookies; the gate's headers and HTTP Basic go only to
- * `origins`, and redirects are followed by hand (at most 5).
+ * cannot present: the client brings the certificates (Playwright presents each only to its origin); the gate's headers,
+ * HTTP Basic and the walk's cookies (`cookiesFor`) go only to `origins`, and redirects are followed by hand (at most 5).
+ * `want` as in pageReader (a sitemap: `{ bytes: true }`).
  */
-export function contextReader (request, { gate = null, origins = [], timeout = ALTERNATES_TIMEOUT, max = ALTERNATES_MAX } = {}) {
-  return async (url) => {
+export function contextReader (request, { gate = null, origins = [], cookiesFor = null, timeout = ALTERNATES_TIMEOUT, max = ALTERNATES_MAX } = {}) {
+  return async (url, want = {}) => {
+    const limit = want.max || (want.bytes ? SITEMAP_MAX_BYTES : max)
+    // one time limit for a sitemap with all its redirects, as with fetch; a page head has it per request
+    const deadline = want.bytes ? Date.now() + (want.timeout || SITEMAP_TIMEOUT) : null
     let at = url
     for (let hop = 0; hop <= 5; hop++) {
-      const gated = gateHeadersFor(gate, at, origins)
-      const r = await request.get(at, { headers: { 'User-Agent': 'Kaloko', Accept: 'text/html,application/xhtml+xml', ...gated }, timeout, maxRedirects: 0 })
+      const left = deadline ? deadline - Date.now() : (want.timeout || timeout)
+      if (left <= 0) throw new Error(`no answer in ${Math.round((want.timeout || SITEMAP_TIMEOUT) / 1000)} s (timeout)`)
+      const headers = { ...readHeaders(want), ...gateHeadersFor(gate, at, origins) }
+      if (cookiesFor && origins.includes(new URL(at).origin)) { const c = await cookiesFor(at); if (c) headers.Cookie = c }
+      const r = await request.get(at, { headers, timeout: Math.max(1, left), maxRedirects: 0 })
       const h = r.headers()
-      if (r.status() >= 300 && r.status() < 400 && h.location) { await r.dispose().catch(() => {}); at = new URL(h.location, at).href; continue }
-      const html = r.ok() ? (await r.body()).subarray(0, max).toString('utf8') : ''
-      await r.dispose().catch(() => {})
-      return { status: r.status(), url: at, link: h.link || null, html }
+      if (r.status() >= 300 && r.status() < 400 && h.location) { await r.dispose().catch(() => {}); at = nextHop(h.location, at); continue }
+      try {
+        if (!want.bytes) return { status: r.status(), url: at, link: h.link || null, html: r.ok() ? (await r.body()).subarray(0, limit).toString('utf8') : '' }
+        // the client reads the whole body before it answers: the limit is checked on what came
+        const body = r.ok() ? await r.body() : Buffer.alloc(0)
+        if (body.length > limit) throw new Error(`larger than ${Math.round(limit / 1048576)} MB`)
+        return { status: r.status(), url: at, link: h.link || null, body }
+      } finally { await r.dispose().catch(() => {}) }
     }
     throw new Error('more than 5 redirects')
   }
 }
 
-/** Options of the request client variantReader makes for client certificates: the gate's and the walk's cookies. */
-export const variantRequestOptions = (gate, cookies = []) => ({ clientCertificates: gate.clientCertificates, ...(cookies.length ? { storageState: { cookies, origins: [] } } : {}) })
+/**
+ * Options of the request client stackReader makes for client certificates: the certificates only. The walk's cookies
+ * go as a header on the environment's own origins (contextReader), not into the client's jar, which would send a
+ * domain cookie to a sibling host after a redirect.
+ */
+export const variantRequestOptions = (gate) => ({ clientCertificates: gate.clientCertificates })
+
+/** The next address of a redirect: http(s) only. */
+const nextHop = (location, from) => {
+  const u = new URL(location, from)
+  if (u.protocol !== 'https:' && u.protocol !== 'http:') throw new Error(`a redirect to ${u.protocol} is not followed`)
+  return u.href
+}
 
 /**
- * How a stack with locale variants reads its source pages for their hreflang: { read, dispose }. With the gate's
- * headers and HTTP Basic, and the walk's cookies, only on `origins`. A gate with client certificates gets a request
- * client of its own (`factory`: Playwright's `request`), made with the certificates and a copy of the page's cookies:
- * the browser context of a shared browser (connected over CDP) has neither, they live where the browser was started.
+ * How a stack reads its sitemaps, robots.txt and (with locale variants) its source pages for their hreflang:
+ * { read, dispose }. With the gate's headers and HTTP Basic, and the walk's cookies, only on `origins`. A gate with
+ * client certificates gets a request client of its own (`factory`: Playwright's `request`), made with the certificates:
+ * the browser context of a shared browser (connected over CDP) has none, they live where the browser was started.
  * `dispose` closes it after the reads.
  */
-export async function variantReader ({ page = null, gate = null, origins = [], factory = null, fetchImpl = fetch }) {
+export async function stackReader ({ page = null, gate = null, origins = [], factory = null, fetchImpl = fetch }) {
+  const cookiesFor = page ? async (url) => (await page.context().cookies(url).catch(() => [])).map((c) => `${c.name}=${c.value}`).join('; ') : null
   if (gate?.clientCertificates?.length) {
-    const cookies = page ? await page.context().cookies().catch(() => []) : []
     const pw = factory || (await import('playwright')).request
-    const request = await pw.newContext(variantRequestOptions(gate, cookies))
-    return { kind: 'certificates', read: contextReader(request, { gate, origins }), dispose: () => request.dispose().catch(() => {}) }
+    const request = await pw.newContext(variantRequestOptions(gate))
+    return { kind: 'certificates', read: contextReader(request, { gate, origins, cookiesFor }), dispose: () => request.dispose().catch(() => {}) }
   }
-  const cookiesFor = page ? async (url) => (await page.context().cookies(url).catch(() => [])).map((c) => `${c.name}=${c.value}`).join('; ') : null
   return { kind: 'fetch', read: pageReader({ fetchImpl, gate, origins, cookiesFor }), dispose: async () => {} }
 }
+/** The name of before, when only locale variants read through the gate. */
+export const variantReader = stackReader
+
+/** Whether a stack reads anything outside the browser: a sitemap, robots.txt or the hreflang of its pages. */
+export const readsOutside = (items = {}) => !!(items.sitemap || items.robots || items.variants)
 
 /** The head of a page as text: read until </head> (outside comments and scripts) or `max` bytes, then stopped. */
 export async function readHead (r, max = ALTERNATES_MAX) {
   if (!r.body?.getReader) return String(await r.text()).slice(0, max)
-  const reader = r.body.getReader(); const dec = new TextDecoder(); let text = ''; let bytes = 0
-  for (;;) {
-    const { done, value } = await reader.read()
-    if (done) break
-    bytes += value.length; text += dec.decode(value, { stream: true })
-    if (bytes >= max || /<\/head\s*>|<body[\s>]/i.test(stripInert(text))) { try { await reader.cancel() } catch {} break }
-  }
-  return text.slice(0, max)
+  return readHeadFrom(r.body.getReader(), max)
 }
 
 /**
@@ -272,21 +296,49 @@ export function gateHeadersFor (gate, url, origins = []) {
   return h
 }
 
+/** How long reading one sitemap or robots.txt may take (ms); its size is capped by SITEMAP_MAX_BYTES. */
+export const SITEMAP_TIMEOUT = 60000
+
+const readHeaders = (want) => want.bytes
+  ? { 'User-Agent': 'Kaloko', Accept: 'application/xml,text/xml,text/plain;q=0.9,*/*;q=0.8', 'Accept-Encoding': 'gzip' }
+  : { 'User-Agent': 'Kaloko', Accept: 'text/html,application/xhtml+xml' }
+
+/** The whole body as bytes, at most `max` (else an error, the rest is not read). */
+export async function readBytes (r, max = SITEMAP_MAX_BYTES) {
+  const big = () => new Error(`larger than ${Math.round(max / 1048576)} MB`)
+  if (!r.body?.getReader) { const b = Buffer.from(await r.arrayBuffer()); if (b.length > max) throw big(); return b }
+  const reader = r.body.getReader(); const parts = []; let bytes = 0
+  for (;;) {
+    const { done, value } = await reader.read()
+    if (done) break
+    bytes += value.length
+    if (bytes > max) { try { await reader.cancel() } catch {} throw big() }
+    parts.push(value)
+  }
+  return Buffer.concat(parts)
+}
+
 /**
  * Reads a page for its hreflang: { status, url (after redirects), link (the Link header), html (the head) }. The
  * gate (headers, Basic) and the walk's cookies (`cookiesFor(url)` → Cookie header) go only to `origins`; redirects
- * are followed by hand (at most 5) so they never carry them elsewhere; each request has `timeout` ms.
+ * are followed by hand (at most 5) so they never carry them elsewhere; each request has `timeout` ms. `want.bytes`
+ * reads a sitemap or robots.txt instead: the whole body as `body` (bytes, at most SITEMAP_MAX_BYTES, all hops within
+ * SITEMAP_TIMEOUT ms; `want.max` and `want.timeout` change them).
  */
 export function pageReader ({ fetchImpl = fetch, gate = null, origins = [], cookiesFor = null, timeout = ALTERNATES_TIMEOUT, max = ALTERNATES_MAX } = {}) {
-  return async (url) => {
+  return async (url, want = {}) => {
     let at = url
+    const whole = want.bytes ? AbortSignal.timeout(want.timeout || SITEMAP_TIMEOUT) : null
     for (let hop = 0; hop <= 5; hop++) {
-      const headers = { 'User-Agent': 'Kaloko', Accept: 'text/html,application/xhtml+xml', ...gateHeadersFor(gate, at, origins) }
+      const headers = { ...readHeaders(want), ...gateHeadersFor(gate, at, origins) }
       if (cookiesFor && origins.includes(new URL(at).origin)) { const c = await cookiesFor(at); if (c) headers.Cookie = c }
-      const r = await fetchImpl(at, { redirect: 'manual', headers, signal: AbortSignal.timeout(timeout) })
+      const r = await fetchImpl(at, { redirect: 'manual', headers, signal: whole || AbortSignal.timeout(timeout) })
       const loc = r.headers?.get?.('location')
-      if (r.status >= 300 && r.status < 400 && loc) { try { await r.body?.cancel() } catch {} at = new URL(loc, at).href; continue }
-      return { status: r.status, url: at, link: r.headers?.get?.('link') || null, html: r.ok ? await readHead(r, max) : '' }
+      if (r.status >= 300 && r.status < 400 && loc) { try { await r.body?.cancel() } catch {} at = nextHop(loc, at); continue }
+      const link = r.headers?.get?.('link') || null
+      if (!want.bytes) return { status: r.status, url: at, link, html: r.ok ? await readHead(r, max) : '' }
+      if (!r.ok) { try { await r.body?.cancel() } catch {} return { status: r.status, url: at, link, body: Buffer.alloc(0) } }
+      return { status: r.status, url: at, link, body: await readBytes(r, want.max || SITEMAP_MAX_BYTES) }
     }
     throw new Error('more than 5 redirects')
   }
@@ -298,22 +350,7 @@ export function pageReader ({ fetchImpl = fetch, gate = null, origins = [], cook
  * a timeout or a 5xx is tried once more; a result with an error is not kept, so another locale asks again.
  */
 export async function readAlternates (read, url, fallback = null, cache = new Map()) {
-  const got = await cached(cache, `alt|${url}`, async () => {
-    let alternates = []; let error = null; let at = url
-    for (let attempt = 0; attempt < 2; attempt++) {
-      try {
-        const r = await read(url)
-        at = r.url || url
-        if (r.status < 200 || r.status >= 300) { error = `HTTP ${r.status}`; if (r.status >= 500) continue; break }
-        error = null
-        alternates = htmlAlternates(r.html, at)
-        if (!alternates.length) alternates = headerAlternates(r.link, at)
-        break
-      } catch (e) { error = String(e?.message || e).split('\n')[0] }
-    }
-    if (!alternates.length && fallback?.length) return { alternates: fallback, error: null, url: at, from: 'sitemap' }
-    return { alternates, error, url: at }
-  })
+  const got = await cached(cache, `alt|${url}`, () => pageAlternates(read, url, fallback))
   if (got.error) cache.delete(`alt|${url}`)
   return got
 }

package/src/lib/variants-core.mjs

@@ -24,6 +24,53 @@ const unique = (list) => { const seen = new Set(); return list.filter((a) => a &
  */
 export const stripInert = (html) => String(html || '').replace(/<!--[\s\S]*?(?:-->|$)/g, '').replace(/<script\b[\s\S]*?(?:<\/script\s*>|$)/gi, '').replace(/<style\b[\s\S]*?(?:<\/style\s*>|$)/gi, '')
 
+/**
+ * Where the head of a page ends, found as the text arrives: `feed(chunk)` is true once `</head>` or `<body>` shows up
+ * outside comments, scripts and styles (the same rule as stripInert, then the search in htmlAlternates). Each call
+ * scans only the new text and a few characters before it, so a long head read in small chunks stays linear.
+ */
+export function headEnd () {
+  let text = ''; let pos = 0; let mode = null
+  const OPEN = /<!--|<script\b|<style\b|<\/head\s*>|<body[\s>]/gi
+  const CLOSE = { comment: /-->/g, script: /<\/script\s*>/gi, style: /<\/style\s*>/gi }
+  return (chunk) => {
+    text += chunk
+    for (;;) {
+      const re = mode ? CLOSE[mode] : OPEN
+      re.lastIndex = pos
+      const m = re.exec(text)
+      // nothing yet: keep the last characters, a token may be split between two chunks
+      if (!m) { pos = Math.max(pos, text.length - 16); return false }
+      // `<script` at the very end may still turn into `<scripts`: wait for the next character
+      if (!mode && m.index + m[0].length >= text.length && /^<(script|style)/i.test(m[0])) { pos = m.index; return false }
+      pos = m.index + m[0].length
+      if (mode) { mode = null; continue }
+      const t = m[0].toLowerCase()
+      if (t === '<!--') mode = 'comment'
+      else if (t.startsWith('<script')) mode = 'script'
+      else if (t.startsWith('<style')) mode = 'style'
+      else return true
+    }
+  }
+}
+
+/**
+ * The head of a page from a stream reader (`body.getReader()`): read until the head ends (headEnd) or `max` bytes,
+ * then the stream is cancelled. Shared by the CLI (stack.mjs readHead) and hosted runs (hosted-stacks.js).
+ */
+export async function readHeadFrom (reader, max) {
+  const dec = new TextDecoder(); const ended = headEnd(); let text = ''; let bytes = 0
+  for (;;) {
+    const { done, value } = await reader.read()
+    if (done) break
+    bytes += value.byteLength
+    const chunk = dec.decode(value, { stream: true })
+    text += chunk
+    if (bytes >= max || ended(chunk)) { try { await reader.cancel() } catch {} break }
+  }
+  return text.slice(0, max)
+}
+
 /** hreflang alternates in the head of an HTML page: [{ lang (lowercase, en-us), href (absolute) }]. */
 export function htmlAlternates (html, url) {
   const text = stripInert(html)
@@ -49,18 +96,30 @@ export function headerAlternates (link, url) {
   return unique(out)
 }
 
-/** The `<url>` entries of a sitemap with their `xhtml:link` alternates: Map(loc → [{ lang, href }]). */
-export function sitemapAlternates (xml) {
+/**
+ * The `<url>` entries of a sitemap that name alternates, unparsed: Map(loc → the entry's markup). The service keeps
+ * these for one sitemap at a time and parses only the pages it takes (urlAlternates).
+ */
+export function sitemapUrlEntries (xml) {
   const out = new Map()
   for (const m of String(xml || '').matchAll(/<url(?:\s[^>]*)?>([\s\S]*?)<\/url>/gi)) {
+    if (!/<(?:xhtml:)?link\b/i.test(m[1])) continue
     const loc = m[1].match(LOC)?.[1]
-    if (!loc) continue
-    const alts = unique([...m[1].matchAll(/<(?:xhtml:)?link\b[^>]*>/gi)].map((l) => { const a = attrsOf(l[0]); return /(^|\s)alternate(\s|$)/i.test(a.rel || '') ? alternate(a.hreflang, a.href) : null }))
-    if (alts.length) out.set(decode(loc), alts)
+    if (loc) out.set(decode(loc), m[1])
   }
   return out
 }
 
+/** The `xhtml:link` alternates of one `<url>` entry of a sitemap: [{ lang, href }]. */
+export const urlAlternates = (entry) => unique([...String(entry || '').matchAll(/<(?:xhtml:)?link\b[^>]*>/gi)].map((l) => { const a = attrsOf(l[0]); return /(^|\s)alternate(\s|$)/i.test(a.rel || '') ? alternate(a.hreflang, a.href) : null }))
+
+/** The `<url>` entries of a sitemap with their `xhtml:link` alternates: Map(loc → [{ lang, href }]). */
+export function sitemapAlternates (xml) {
+  const out = new Map()
+  for (const [loc, entry] of sitemapUrlEntries(xml)) { const alts = urlAlternates(entry); if (alts.length) out.set(loc, alts) }
+  return out
+}
+
 /**
  * The alternate that stands for `locale`: the same tag (en-us for en_US), else the bare language (en) for a locale
  * with a region, else the first regional tag of the language (sk-sk) for a locale without one. A locale with a script
@@ -84,6 +143,30 @@ export function alternateFor (alts, locale) {
   return parts.find(([, q]) => q?.lang === p.lang && !q.script)?.[0] || parts.find(([, q]) => q?.lang === p.lang)?.[0] || null
 }
 
+/**
+ * The hreflang alternates of one page as `read` (url → { status, url (after redirects), link, html }) gives it: from
+ * its HTML head, else its Link header, else `fallback` (the sitemap's xhtml:link entries for it):
+ * { alternates: [{ lang, href }], error, url, from? }. A network error, a timeout or a 5xx is tried once more, except
+ * an error `noRetry(e)` names (a refused address); an error `rethrow(e)` names (the service's spent round) is not
+ * caught. Shared by the CLI (stack.mjs readAlternates) and hosted runs (worker/src/hosted-stacks.js).
+ */
+export async function pageAlternates (read, url, fallback = null, { rethrow = () => false, noRetry = () => false } = {}) {
+  let alternates = []; let error = null; let at = url
+  for (let attempt = 0; attempt < 2; attempt++) {
+    try {
+      const r = await read(url)
+      at = r.url || url
+      if (r.status < 200 || r.status >= 300) { error = `HTTP ${r.status}`; if (r.status >= 500) continue; break }
+      error = null
+      alternates = htmlAlternates(r.html, at)
+      if (!alternates.length) alternates = headerAlternates(r.link, at)
+      break
+    } catch (e) { if (rethrow(e)) throw e; error = String(e?.message || e).split('\n')[0]; if (noRetry(e)) break }
+  }
+  if (!alternates.length && fallback?.length) return { alternates: fallback, error: null, url: at, from: 'sitemap' }
+  return { alternates, error, url: at }
+}
+
 const hostOf = (u) => { try { return new URL(u).host } catch { return null } }
 const sameLanguage = (a, b) => { const x = localeParts(a); const y = localeParts(b); return !!x && !!y && x.lang === y.lang }