kaloko 0.20.0 → 0.21.0

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "kaloko",
-  "version": "0.20.0",
+  "version": "0.21.0",
   "description": "Your AI agents build; Kaloko walks every screen, puts it on one canvas and records who approved what.",
   "keywords": [
     "qa",

package/schemas/scenario.v1.json

@@ -998,7 +998,7 @@
         },
         "full_page": {
           "type": "boolean",
-          "description": "Screenshot of the whole page (default true; false for stack pages: the first screen only)"
+          "description": "Screenshot of the whole page (default true, stack pages too); false keeps the first screen only"
         },
         "screenshot": {
           "enum": ["full", "thumb"],

package/skills/kaloko/reference/stacks.md

@@ -59,7 +59,7 @@ step script only to prepare each page, for example to dismiss a dialog
     - { id: author, label: Author, from: .author }
     - { id: cover, label: Cover, from: "article img@src" }
     - { id: links, label: Links, from: "count:main a" }
-  # capture: { full_page: true }            # default for stacks is the first screen only
+  # capture: { screenshot: thumb, full_page: false }  # whole pages by default; this is the smallest
   criteria:
     - { id: AC1, text: Has a cover image, check: { type: deterministic, assert: exists, selector: "article img" } }
     - { id: AC2, text: Reads as an article, check: { type: semantic, question: Does the page read as a complete magazine article? } }
@@ -80,6 +80,12 @@ Item sources:
   `sitemap-de.xml`; `en_US` finds `en-us`, `en` does not), only that
   locale's sitemaps are read; otherwise all of them. Use it when a site
   lists one sitemap per language, so the scenario does not hard-code paths.
+- `kaloko walk` reads robots.txt and sitemaps with the environment's gate
+  (headers, HTTP Basic, client certificates) and the walk's cookies, on the
+  environment's own origins only, so a staging behind Basic auth or mTLS
+  needs no public sitemap. A sitemap on another host (a CDN, an index
+  pointing elsewhere) or a redirect to another host gets none of them.
+  Redirects are followed by hand (at most 5); 60 s and 60 MB per file.
 - `crawl: { from: /clanky, links: "a.card", next: "a.next", pages: 20 }`
   collects links from a listing and follows its pagination.
 - `script: qa/steps/items.mjs` uses the default export
@@ -229,9 +235,12 @@ stack, set:
   locale).
 - A script that lists pages per locale instead: give each page the same
   `key` in every locale (for example the source path), so they are one item.
-- Hosted runs (`kaloko schedule add`) do not group variants yet: there each
-  locale reads its own list and HC1005 is not reported (`kaloko validate`
-  warns). Use `kaloko walk` or the scheduled GitHub workflow.
+- Hosted runs (`kaloko schedule add`) group variants the same way: same
+  ids, window, `missing` and HC1005. They read each source page's head for
+  its hreflang on the environment's own addresses (with the schedule's
+  password or cookies), and each read counts as one hosted page of the
+  monthly budget. Budget a stack of N source pages in L locales as
+  N reads + N × L pages at most.
 
 ## Cost and size: choose deliberately
 
@@ -244,20 +253,32 @@ stack, set:
   caps the calls of a whole evaluation (evaluation.md). Raise the sample only when the question
   really differs between pages, and tell the person what it costs
   (one call per page and viewport).
-- Stacks capture the first screen of each page (fast, small). Set
-  `capture: { full_page: true }` when the criteria are about the whole page.
-- Walking takes about 1–2 s per page. For a first look, use
+- Stacks capture the whole page, like every step (since 0.21.0; before, the
+  first screen only). Vision checks see what is below the fold, and so does a
+  reviewer. A typical long page then stores a screenshot of about 600 KB
+  (about 70 KB for the first screen) and a thumbnail of about 180 KB (about
+  25 KB), because the thumbnail is as tall as the page.
+- To keep a stack small, set on the step:
+  - `capture: { full_page: false }`: a sharp first screen of each page;
+  - `capture: { screenshot: thumb, full_page: false }`: the smallest, a
+    640 px thumbnail of the first screen. Recommend it for big monitors
+    (SEO crawls, catalogues of thousands of pages) whose checks read the
+    page and do not need the picture below the fold.
+  - `capture: { screenshot: thumb }` alone keeps a thumbnail of the whole
+    page, still several times the first screen.
+- Walking takes about 1–2 s per page, a little more for long pages captured
+  whole. For a first look, use
   `kaloko walk --limit 50`. Re-walk chosen pages with
   `--items clanek-7,clanek-12` (matches the id or the path; picked from
   every matching page, also outside this week's rotation, and such a
   partial walk leaves the rotation state alone).
 - A shared run holds at most 20000 files and 300 MB as stored (HTML and JSON
   travel gzip-compressed, about a tenth of their size on disk). Each page
-  stores about four files, so a stack of a few thousand first-screen pages
-  fits. For SEO crawls where the checks read the page, not the picture, set
-  `capture: { screenshot: thumb }`: only the 640 px thumbnail is kept. Split
-  larger catalogues by section (`match`) or walk a rotating sample
-  (`rotate: week`).
+  stores about four files. Whole pages fill 300 MB at roughly 400 pages;
+  `screenshot: thumb, full_page: false` lets the 20000 files be the limit
+  (about 5000 pages). Split larger catalogues by section (`match`) or walk a
+  rotating sample (`rotate: week`). When a whole-page stack fills most of a
+  run near the limit, `kaloko share` names it and suggests these settings.
 
 ## Reading the results
 

package/src/cli.mjs

@@ -33,7 +33,7 @@ import { detectProject, initScenario, skeletonScenario, scenarioHeader, formatDe
 import { checkSlug, checkName, setConfigKeys, createProject, renameProject, syncProject, shareProjectNote } from './lib/projects-cli.mjs'
 import { recheckCaptures } from './lib/recheck.mjs'
 import { runFindings, findingsText } from './lib/findings.mjs'
-import { resolveItems, fieldsOf, DEFAULT_LIMIT, timeLeft, rotationState, rotationCoverage, sourceLocaleOf, variantReader } from './lib/stack.mjs'
+import { resolveItems, fieldsOf, DEFAULT_LIMIT, timeLeft, rotationState, rotationCoverage, sourceLocaleOf, stackReader, readsOutside } from './lib/stack.mjs'
 import { walkSiteChecks, siteChecksOn } from './lib/site.mjs'
 import { DEFAULTS as JEV_DEFAULTS } from './lib/jev.mjs'
 import { expandPacks } from './lib/packs.mjs'
@@ -48,6 +48,7 @@ import { login } from './lib/login.mjs'
 import { initNextSteps, loginNextSteps } from './lib/onboarding.mjs'
 import { specCmd, driftCmd } from './lib/spec-cli.mjs'
 import { hashFiles, missingHashes, uploadRunFiles, PROGRESS_FILE, fmtBytes } from './lib/upload.mjs'
+import { biggestSteps, stackSizeAdvice } from './lib/run-size.mjs'
 import { affectedFor, carryCaptures } from './lib/affected.mjs'
 import { formatAffected, selectedIds } from './lib/affected-core.mjs'
 import { healStep, formatHeal } from './lib/heal.mjs'
@@ -818,8 +819,9 @@ async function walkStack ({ node, locale, page, env, run, scenario, scenarioDir,
   const sourceLocale = node.items?.variants ? sourceLocaleOf(node, scenario, locale) : locale
   const counter = state && node.items.variants && node.items.rotate === 'run' ? await rotationState(statePath('variants'), run.meta.id) : null
   const round = counter ? counter.run : state?.run || 0
-  // locale variants read the source pages for their hreflang with the environment's access (variantReader)
-  const reader = node.items?.variants ? await variantReader({ page, gate, origins: envOrigins(env, scenario, locale) }) : null
+  // sitemaps, robots.txt and the hreflang of source pages are read with the environment's access (stackReader): its
+  // gate and the walk's cookies on its own origins only
+  const reader = readsOutside(node.items) ? await stackReader({ page, gate, origins: envOrigins(env, scenario, locale) }) : null
   let resolved
   try { resolved = await resolveItems({ node, base: baseUrl(env, locale), sourceBase: baseUrl(env, sourceLocale), page, scenarioDir, env, locale, scenario, limit: Number(args.limit) || undefined, rotation: { run: round }, checked: state?.checked || null, only: list(args.items), cache, readPage: reader?.read || null }) } finally { await reader?.dispose() }
   const { items, total, truncated, all, rotation, partial, variants, resolve_failed: unresolved, resolve_error: unresolvedWhy } = resolved
@@ -1177,14 +1179,6 @@ async function projectsWrite (sub, args) {
   if (sub !== 'sync' && slug === config.project && name && config.project_name && name !== config.project_name) say(`Note: kaloko.config.yml still says project_name: ${config.project_name}; change it there too, or kaloko projects sync would rename the project back.`)
 }
 
-/** The steps that take most of a run, as stored: " Largest steps: products 212 MB (1 390 files), …" or ''. */
-function biggestSteps (entries, n = 3) {
-  const by = new Map()
-  for (const e of entries) { const k = e.path.match(/^steps\/([^/]+)\//)?.[1]; if (!k) continue; const x = by.get(k) || { bytes: 0, files: 0 }; x.bytes += e.stored ?? e.bytes; x.files++; by.set(k, x) }
-  const top = [...by].sort((a, b) => b[1].bytes - a[1].bytes).slice(0, n)
-  return top.length ? ` Largest steps: ${top.map(([k, x]) => `${k} ${fmtBytes(x.bytes)} (${x.files} files)`).join(', ')}.` : ''
-}
-
 const commands = {
   async init (args) {
     const cwd = process.cwd()
@@ -1610,7 +1604,7 @@ const commands = {
     const pick = list(args.steps)
     // stacks with locale variants read their source list and each page's hreflang once per walk, for every locale
     const stackCache = new Map()
-    const stackGate = await gateForRequests(env, { root }) // the gate for requests outside the browser: link checks, hreflang of stack pages
+    const stackGate = await gateForRequests(env, { root }) // the gate for requests outside the browser: link checks, stack sitemaps and hreflang
     const locales = list(args.locales) || run.meta.scope.locales
     const scope = run.meta.scope
     // browser engines and device profiles (batch E): one pass each, capturing only its own keys; a run without
@@ -2244,7 +2238,10 @@ const commands = {
     const resumed = (await readJson(join(run.dir, PROGRESS_FILE), null).catch(() => null))?.run === manifest.id
     try {
       if (!u) throw new Error('usage unavailable')
-      if (total > u.limits.max_run_bytes) throw new UserError(`This run is ${fmtBytes(total)}; the limit per run is ${fmtBytes(u.limits.max_run_bytes)}.${biggestSteps(entries)} Capture fewer viewports or locales; for a stack, lower items.limit, walk a rotating sample (items.rotate: week) or keep thumbnails only (capture: { screenshot: thumb }). An admin can also ask for a higher limit.`)
+      if (total > u.limits.max_run_bytes) throw new UserError(`This run is ${fmtBytes(total)}; the limit per run is ${fmtBytes(u.limits.max_run_bytes)}.${biggestSteps(entries)}${stackSizeAdvice({ entries, scenario, limit: u.limits.max_run_bytes, over: true })} Capture fewer viewports or locales; for a stack, lower items.limit, walk a rotating sample (items.rotate: week), keep the first screen only (capture: { full_page: false }) or, smallest, keep thumbnails of it only (capture: { screenshot: thumb, full_page: false }). An admin can also ask for a higher limit.`)
+      // a stack captured whole (the default) that fills most of a run close to the limit: say so before the next walk grows it
+      const near = stackSizeAdvice({ entries, scenario, limit: u.limits.max_run_bytes })
+      if (near) say(near)
       if (u.usage.bytes + fresh > u.limits.quota_bytes) throw new UserError(`Storage quota would be exceeded: ${fmtBytes(u.usage.bytes)} used + ${fmtBytes(fresh)} > ${fmtBytes(u.limits.quota_bytes)}. Delete old runs in Settings or ask for a higher quota.`)
       const recBytes = entries.filter((e) => /\.rec\./.test(e.path)).reduce((n, e) => n + e.bytes, 0)
       say(`${resumed ? 'Resuming the interrupted share: ' : ''}${files.size} files, ${fmtBytes(total)}${recBytes ? ` (recordings ${fmtBytes(recBytes)})` : ''}${missing ? `; ${fmtBytes(fresh)} new, the rest already stored` : ''} (organization uses ${fmtBytes(u.usage.bytes)} of ${fmtBytes(u.limits.quota_bytes)})`)

package/src/lib/capture.mjs

@@ -88,8 +88,9 @@ export function captureOptions (scenario, node) {
     mask: [...new Set(masks.flatMap((m) => m.selectors))],
     maskAuto: masks.some((m) => m.auto),
     annotate: o.annotate === true,
-    // stack pages default to the first screen: hundreds of full-page shots would swamp storage
-    fullPage: o.full_page != null ? o.full_page !== false : node?.kind !== 'stack',
+    // the whole page by default, stack pages too: a review needs what is below the fold (full_page: false for the
+    // first screen only; screenshot: thumb when a crawl of thousands of pages needs no picture)
+    fullPage: o.full_page !== false,
     // screenshot: thumb (#150): only the card thumbnail is stored, as the step's image too (SEO crawls of thousands of
     // pages need the page facts, not a sharp picture of each)
     thumbOnly: o.screenshot === 'thumb'
@@ -138,9 +139,45 @@ export async function settle (page) {
     await page.evaluate(async (overlay) => {
       const open = [...document.querySelectorAll(overlay)].some((el) => { const r = el.getBoundingClientRect(); return r.width > 0 && r.height > 0 })
       if (open) return
-      const step = innerHeight * 0.8
-      for (let y = 0; y < document.documentElement.scrollHeight; y += step) { scrollTo(0, y); await new Promise((r) => setTimeout(r, 60)) }
-      scrollTo(0, 0)
+      // what scrolling changes on the fixed and sticky parts (a header that floats, a search panel that fades in):
+      // the capture waits until they are back as they were at the top, or until they stopped changing for 1 s (a
+      // widget revealed on scroll for good), at most 3 s
+      // fixed, sticky or inside such a part; asked once per element (the observer sees every class change of the page)
+      const floats = new WeakMap()
+      const floating = (el) => {
+        if (!el || el === document.body || el === document.documentElement) return false
+        if (!floats.has(el)) { const p = getComputedStyle(el).position; floats.set(el, p === 'fixed' || p === 'sticky' || floating(el.parentElement)) }
+        return floats.get(el)
+      }
+      const before = new Map()
+      let changed = performance.now()
+      const mo = new MutationObserver((list) => {
+        for (const m of list) {
+          const el = m.target
+          if (!(el === document.documentElement || el === document.body || floating(el))) continue
+          changed = performance.now()
+          const key = `${m.attributeName}`
+          const seen = before.get(el) || before.set(el, {}).get(el)
+          if (!(key in seen)) seen[key] = m.oldValue
+        }
+      })
+      // an empty class or style attribute looks the same as none
+      const same = (name, a, b) => name === 'class' || name === 'style' ? (a || '') === (b || '') : a === b
+      try {
+        mo.observe(document.documentElement, { subtree: true, attributes: true, attributeOldValue: true, attributeFilter: ['class', 'style', 'hidden', 'aria-expanded'] })
+        const step = innerHeight * 0.8
+        for (let y = 0; y < document.documentElement.scrollHeight; y += step) { scrollTo(0, y); await new Promise((r) => setTimeout(r, 60)) }
+        scrollTo(0, 0)
+        const start = performance.now()
+        // the quiet second counts from the return to the top: a header that changed once on the first scroll and goes
+        // back a moment after the page is at the top again is not quiet yet
+        changed = Math.max(changed, start)
+        const back = () => [...before].every(([el, attrs]) => Object.entries(attrs).every(([name, value]) => same(name, el.getAttribute(name), value)))
+        // only what moves on the fixed and sticky parts: a carousel or spinner elsewhere does not hold the capture
+        const moving = () => (document.getAnimations?.() || []).some((a) => a.playState === 'running' && a.effect?.getComputedTiming?.().iterations !== Infinity && floating(a.effect?.target))
+        const quiet = () => performance.now() - changed >= 1000
+        while (performance.now() - start < 3000 && (moving() || !(back() || quiet()))) await new Promise((r) => setTimeout(r, 50))
+      } finally { mo.disconnect() }
     }, OVERLAY)
   } catch {}
   await page.waitForTimeout(250)
@@ -229,18 +266,21 @@ export async function captureStep ({ page, node, locale, viewports, run, console
       // (or 2× for documentation), so the canvas, boxes and diffs work the same for every device
       const dpr = await page.evaluate(() => devicePixelRatio).catch(() => 1)
       let shotScale = opts.scale
+      // a full-page capture resizes the page; resize handlers restart CSS transitions (a panel fading out again), and
+      // the image would catch them halfway. Transitions end at once and finite animations finish for the capture
       if (opts.scale === 2 && engine === 'chromium') {
         const cdp = await page.context().newCDPSession(page)
+        const still = await page.addStyleTag({ content: '*, *::before, *::after { transition: none !important }' }).catch(() => null)
         try {
           const doc = await page.evaluate(() => ({ w: Math.ceil(document.documentElement.scrollWidth), h: Math.ceil(document.documentElement.scrollHeight) }))
           const r = await cdp.send('Page.captureScreenshot', { format: 'png', captureBeyondViewport: opts.fullPage, clip: { x: 0, y: 0, width: opts.fullPage ? doc.w : width, height: opts.fullPage ? doc.h : height, scale: 2 / (dpr || 1) } })
           shot = Buffer.from(r.data, 'base64')
-        } finally { await cdp.detach().catch(() => {}) }
+        } finally { await cdp.detach().catch(() => {}); await still?.evaluate((el) => el.remove()).catch(() => {}) }
       } else {
         // WebKit and Firefox have no 2× capture of their own: 2× when the context renders at 2× or more, else 1×
         // (exactly 2×: a 3× or 2.625× device would give an image the masks and marks are not drawn for)
         if (opts.scale === 2 && dpr !== 2) shotScale = 1
-        shot = await page.screenshot({ fullPage: opts.fullPage, scale: shotScale === 2 ? 'device' : 'css' })
+        shot = await page.screenshot({ fullPage: opts.fullPage, scale: shotScale === 2 ? 'device' : 'css', animations: 'disabled' })
       }
       // secrets the CLI used (gate, accounts) never land in a capture, even if the page echoes them
       await writeFile(html, redact(maskHtml(await page.content(), masking?.attrs)))

package/src/lib/run-size.mjs

@@ -0,0 +1,45 @@
+// What takes the room in a run before `kaloko share` sends it: the largest steps, and the advice for a stack whose
+// pages are captured whole (the default since 0.21.0) when it fills most of the per-run limit.
+import { fmtBytes } from './upload.mjs'
+import { captureOptions } from './capture.mjs'
+
+/** Bytes and files per step, as stored (entries from hashFiles: { path, bytes, stored? }), largest first. */
+export function stepSizes (entries) {
+  const by = new Map()
+  for (const e of entries) {
+    const k = e.path.match(/^steps\/([^/]+)\//)?.[1]
+    if (!k) continue
+    const x = by.get(k) || { bytes: 0, files: 0 }
+    x.bytes += e.stored ?? e.bytes; x.files++
+    by.set(k, x)
+  }
+  return [...by].sort((a, b) => b[1].bytes - a[1].bytes)
+}
+
+/** The steps that take most of a run: " Largest steps: products 212 MB (1 390 files), …" or ''. */
+export function biggestSteps (entries, n = 3) {
+  const top = stepSizes(entries).slice(0, n)
+  return top.length ? ` Largest steps: ${top.map(([k, x]) => `${k} ${fmtBytes(x.bytes)} (${x.files} files)`).join(', ')}.` : ''
+}
+
+/** Stack steps of the scenario whose pages are captured whole and kept sharp (no full_page: false, no thumb). */
+export function wholePageStacks (scenario) {
+  return new Set((scenario?.nodes || []).filter((n) => n.kind === 'stack').filter((n) => { const o = captureOptions(scenario, n); return o.fullPage && !o.thumbOnly }).map((n) => n.id))
+}
+
+const SMALL = 'capture: { full_page: false } keeps the first screen of each page, capture: { screenshot: thumb, full_page: false } is the smallest'
+
+/**
+ * Advice when a whole-page stack makes up most of a run that is near or over the limit, or ''.
+ * `over` words it for the refusal (appended to it); otherwise it is a warning of its own.
+ */
+export function stackSizeAdvice ({ entries, scenario, limit, over = false }) {
+  const total = entries.reduce((n, e) => n + (e.stored ?? e.bytes), 0)
+  if (!limit || total < limit * 0.8) return ''
+  const whole = wholePageStacks(scenario)
+  const hit = stepSizes(entries).find(([k, x]) => whole.has(k) && x.bytes > total / 2)
+  if (!hit) return ''
+  const [id, x] = hit
+  if (over) return ` The stack ${id} is ${fmtBytes(x.bytes)} of it because each page is captured whole.`
+  return `This run is ${fmtBytes(total)} of the ${fmtBytes(limit)} allowed per run, and the stack ${id} is ${fmtBytes(x.bytes)} of it because each page is captured whole. If its pictures below the fold are not needed, ${SMALL}.`
+}

package/src/lib/scenario.mjs

@@ -120,7 +120,6 @@ export function scenarioWarnings (data, { projectRequired = null } = {}) {
   const out = []
   if (!data || !Array.isArray(data.nodes)) return out
   for (const node of data.nodes) {
-    if (node.kind === 'stack' && node.items?.variants) out.push(`node "${node.id}": hosted runs (kaloko schedule add) do not group locale variants yet: there each language reads its own list and missing translations (HC1005) are not reported; kaloko walk and the scheduled GitHub workflow do`)
     if (node.kind === 'stack' && node.items?.variants && (data.locales || []).length < 2) out.push(`node "${node.id}": items.variants: hreflang groups the language versions of a page, but the scenario walks ${(data.locales || []).length ? `one locale (${data.locales[0]})` : 'no locales'}; list the other locales under locales:`)
     if (node.kind === 'stack' && (node.packs || []).includes('errors')) out.push(`node "${node.id}": the errors pack is meant for error pages (404, 500, maintenance); on a stack of normal pages HC601 and HC605 stay not applicable. Remove it, or give the step expect_status when its pages are error pages`)
   }

package/src/lib/stack.mjs

@@ -8,7 +8,7 @@ import { pathToFileURL } from 'node:url'
 import { gunzipSync } from 'node:zlib'
 import { UserError, localized, readJson, writeJson } from './util.mjs'
 import { STACK_DEFAULT_FIELDS, STACK_SEMANTIC_SAMPLE, stackFieldsOf } from './manifest-core.mjs'
-import { variantItems, htmlAlternates, headerAlternates, sitemapAlternates, stripInert } from './variants-core.mjs'
+import { variantItems, sitemapAlternates, pageAlternates, readHeadFrom } from './variants-core.mjs'
 import { DEFAULT_LIMIT, SITEMAP_MAX_BYTES, SITEMAP_MAX_CHILDREN, ROTATE_MODES, itemId, sitemapLocs, robotsSitemaps, sitemapsForLocale, handoffOf, isoWeek, rotateWindow, patternFor as corePatternFor, withLocale, sitemapPages, selectItems } from './stack-core.mjs'
 
 /** How long items.resolve may take for one page (ms) before the page is walked without its edit link. */
@@ -94,7 +94,8 @@ export function rotationCoverage (all, checked) {
  * source page; `checked` and `only` use it). `variants` in the result: { mode, source_locale, locale, pages,
  * missing: [{ id, path, url, reason }], shared: [{ url, ids, paths }] } (`source: true` in the source locale). `cache`
  * (a Map the walk keeps for all its locales) reads the source list and each page's alternates once; `readPage` (a
- * pageReader with the environment's gate and the walk's cookies) reads the pages, else a plain one over `fetchImpl`.
+ * stackReader's read: the environment's gate and the walk's cookies) reads the sitemaps, robots.txt and the pages,
+ * else a plain pageReader over `fetchImpl`.
  */
 export async function resolveItems ({ node, base, page, scenarioDir, env, locale, limit, fetchImpl = fetch, rotation = {}, checked = null, scenario = null, only = null, resolveTimeout = RESOLVE_TIMEOUT, sourceBase = null, cache = null, readPage = null }) {
   const memo = cache || new Map()
@@ -111,7 +112,7 @@ export async function resolveItems ({ node, base, page, scenarioDir, env, locale
   const listLocale = mapped ? sourceLocale : locale
   const listBase = mapped ? (sourceBase || base) : base
   const listSrc = mapped ? withLocale(node.items || {}, listLocale) : src
-  const read = () => sourceEntries({ src: listSrc, locale: listLocale, base: listBase, page, scenarioDir, env, info: infoFor(listLocale), fetchImpl, withAlternates: variants })
+  const read = () => sourceEntries({ src: listSrc, locale: listLocale, base: listBase, page, scenarioDir, env, info: infoFor(listLocale), fetchImpl, readSource: readPage, withAlternates: variants })
   const { raw, alternates } = variants ? await cached(memo, `list|${scenario?.id || ''}|${node.id}|${listLocale}|${listBase}`, read) : await read()
   // 1. and 2.: match, exclude, deduplication, the window (stack-core.mjs, shared with hosted runs); with variants in
   // the source locale, so every locale walks the same window with the same ids
@@ -146,7 +147,7 @@ const urlKey = (u) => { try { return new URL(u).href } catch { return String(u)
  * { raw: [{ path, title?, key?, purpose?, edit_url?, content_ref? }], alternates: Map(url → the sitemap's xhtml:link
  * alternates of that page, read when `withAlternates`) }.
  */
-async function sourceEntries ({ src, locale, base, page, scenarioDir, env, info, fetchImpl, withAlternates = false }) {
+async function sourceEntries ({ src, locale, base, page, scenarioDir, env, info, fetchImpl, readSource = null, withAlternates = false }) {
   const origin = new URL(base).origin
   const abs = (p) => new URL(p, base.endsWith('/') ? base : base + '/').href.replace(/^(https?:\/\/[^/]+)\/\//, '$1/')
   const toUrl = (p) => (/^https?:\/\//.test(p) ? p : new URL(p, origin).href)
@@ -154,11 +155,15 @@ async function sourceEntries ({ src, locale, base, page, scenarioDir, env, info,
   if (src.urls) raw = src.urls.map((p) => ({ path: p }))
   else if (src.file) raw = await listFile(resolve(scenarioDir, src.file))
   else if (src.sitemap || src.robots) {
+    // through the environment's gate: its headers, Basic, client certificates and the walk's cookies on its own
+    // origins only; a sitemap on another host, or a redirect there, gets none of them
+    const reader = readSource || pageReader({ fetchImpl })
     const get = async (u, what = 'items.sitemap') => {
-      const r = await fetchImpl(u, { headers: { 'User-Agent': 'Kaloko', 'Accept-Encoding': 'gzip' } })
-      if (!r.ok) throw new UserError(`${what} ${u}: HTTP ${r.status}`)
-      if (what !== 'items.sitemap') return r.text()
-      const text = sitemapText(Buffer.from(await r.arrayBuffer()), u)
+      let r
+      try { r = await reader(u, { bytes: true }) } catch (e) { throw new UserError(`${what} ${u}: ${String(e?.message || e).split('\n')[0]}`) }
+      if (r.status < 200 || r.status >= 300) throw new UserError(`${what} ${u}: HTTP ${r.status}${r.url && r.url !== u ? ` (after a redirect to ${r.url})` : ''}`)
+      if (what !== 'items.sitemap') return r.body.toString('utf8')
+      const text = sitemapText(r.body, u)
       // with locale variants the xhtml:link alternates of each <url> are kept for pages that name none themselves
       if (withAlternates) for (const [k, v] of sitemapAlternates(text)) alternates.set(urlKey(k), v)
       return text
@@ -206,57 +211,76 @@ export const ALTERNATES_MAX = 1024 * 1024
 
 /**
  * A pageReader over a Playwright request client (APIRequestContext), for gates with client certificates, which fetch
- * cannot present: the client brings the certificates and the cookies; the gate's headers and HTTP Basic go only to
- * `origins`, and redirects are followed by hand (at most 5).
+ * cannot present: the client brings the certificates (Playwright presents each only to its origin); the gate's headers,
+ * HTTP Basic and the walk's cookies (`cookiesFor`) go only to `origins`, and redirects are followed by hand (at most 5).
+ * `want` as in pageReader (a sitemap: `{ bytes: true }`).
  */
-export function contextReader (request, { gate = null, origins = [], timeout = ALTERNATES_TIMEOUT, max = ALTERNATES_MAX } = {}) {
-  return async (url) => {
+export function contextReader (request, { gate = null, origins = [], cookiesFor = null, timeout = ALTERNATES_TIMEOUT, max = ALTERNATES_MAX } = {}) {
+  return async (url, want = {}) => {
+    const limit = want.max || (want.bytes ? SITEMAP_MAX_BYTES : max)
+    // one time limit for a sitemap with all its redirects, as with fetch; a page head has it per request
+    const deadline = want.bytes ? Date.now() + (want.timeout || SITEMAP_TIMEOUT) : null
     let at = url
     for (let hop = 0; hop <= 5; hop++) {
-      const gated = gateHeadersFor(gate, at, origins)
-      const r = await request.get(at, { headers: { 'User-Agent': 'Kaloko', Accept: 'text/html,application/xhtml+xml', ...gated }, timeout, maxRedirects: 0 })
+      const left = deadline ? deadline - Date.now() : (want.timeout || timeout)
+      if (left <= 0) throw new Error(`no answer in ${Math.round((want.timeout || SITEMAP_TIMEOUT) / 1000)} s (timeout)`)
+      const headers = { ...readHeaders(want), ...gateHeadersFor(gate, at, origins) }
+      if (cookiesFor && origins.includes(new URL(at).origin)) { const c = await cookiesFor(at); if (c) headers.Cookie = c }
+      const r = await request.get(at, { headers, timeout: Math.max(1, left), maxRedirects: 0 })
       const h = r.headers()
-      if (r.status() >= 300 && r.status() < 400 && h.location) { await r.dispose().catch(() => {}); at = new URL(h.location, at).href; continue }
-      const html = r.ok() ? (await r.body()).subarray(0, max).toString('utf8') : ''
-      await r.dispose().catch(() => {})
-      return { status: r.status(), url: at, link: h.link || null, html }
+      if (r.status() >= 300 && r.status() < 400 && h.location) { await r.dispose().catch(() => {}); at = nextHop(h.location, at); continue }
+      try {
+        if (!want.bytes) return { status: r.status(), url: at, link: h.link || null, html: r.ok() ? (await r.body()).subarray(0, limit).toString('utf8') : '' }
+        // the client reads the whole body before it answers: the limit is checked on what came
+        const body = r.ok() ? await r.body() : Buffer.alloc(0)
+        if (body.length > limit) throw new Error(`larger than ${Math.round(limit / 1048576)} MB`)
+        return { status: r.status(), url: at, link: h.link || null, body }
+      } finally { await r.dispose().catch(() => {}) }
     }
     throw new Error('more than 5 redirects')
   }
 }
 
-/** Options of the request client variantReader makes for client certificates: the gate's and the walk's cookies. */
-export const variantRequestOptions = (gate, cookies = []) => ({ clientCertificates: gate.clientCertificates, ...(cookies.length ? { storageState: { cookies, origins: [] } } : {}) })
+/**
+ * Options of the request client stackReader makes for client certificates: the certificates only. The walk's cookies
+ * go as a header on the environment's own origins (contextReader), not into the client's jar, which would send a
+ * domain cookie to a sibling host after a redirect.
+ */
+export const variantRequestOptions = (gate) => ({ clientCertificates: gate.clientCertificates })
+
+/** The next address of a redirect: http(s) only. */
+const nextHop = (location, from) => {
+  const u = new URL(location, from)
+  if (u.protocol !== 'https:' && u.protocol !== 'http:') throw new Error(`a redirect to ${u.protocol} is not followed`)
+  return u.href
+}
 
 /**
- * How a stack with locale variants reads its source pages for their hreflang: { read, dispose }. With the gate's
- * headers and HTTP Basic, and the walk's cookies, only on `origins`. A gate with client certificates gets a request
- * client of its own (`factory`: Playwright's `request`), made with the certificates and a copy of the page's cookies:
- * the browser context of a shared browser (connected over CDP) has neither, they live where the browser was started.
+ * How a stack reads its sitemaps, robots.txt and (with locale variants) its source pages for their hreflang:
+ * { read, dispose }. With the gate's headers and HTTP Basic, and the walk's cookies, only on `origins`. A gate with
+ * client certificates gets a request client of its own (`factory`: Playwright's `request`), made with the certificates:
+ * the browser context of a shared browser (connected over CDP) has none, they live where the browser was started.
  * `dispose` closes it after the reads.
  */
-export async function variantReader ({ page = null, gate = null, origins = [], factory = null, fetchImpl = fetch }) {
+export async function stackReader ({ page = null, gate = null, origins = [], factory = null, fetchImpl = fetch }) {
+  const cookiesFor = page ? async (url) => (await page.context().cookies(url).catch(() => [])).map((c) => `${c.name}=${c.value}`).join('; ') : null
   if (gate?.clientCertificates?.length) {
-    const cookies = page ? await page.context().cookies().catch(() => []) : []
     const pw = factory || (await import('playwright')).request
-    const request = await pw.newContext(variantRequestOptions(gate, cookies))
-    return { kind: 'certificates', read: contextReader(request, { gate, origins }), dispose: () => request.dispose().catch(() => {}) }
+    const request = await pw.newContext(variantRequestOptions(gate))
+    return { kind: 'certificates', read: contextReader(request, { gate, origins, cookiesFor }), dispose: () => request.dispose().catch(() => {}) }
   }
-  const cookiesFor = page ? async (url) => (await page.context().cookies(url).catch(() => [])).map((c) => `${c.name}=${c.value}`).join('; ') : null
   return { kind: 'fetch', read: pageReader({ fetchImpl, gate, origins, cookiesFor }), dispose: async () => {} }
 }
+/** The name of before, when only locale variants read through the gate. */
+export const variantReader = stackReader
+
+/** Whether a stack reads anything outside the browser: a sitemap, robots.txt or the hreflang of its pages. */
+export const readsOutside = (items = {}) => !!(items.sitemap || items.robots || items.variants)
 
 /** The head of a page as text: read until </head> (outside comments and scripts) or `max` bytes, then stopped. */
 export async function readHead (r, max = ALTERNATES_MAX) {
   if (!r.body?.getReader) return String(await r.text()).slice(0, max)
-  const reader = r.body.getReader(); const dec = new TextDecoder(); let text = ''; let bytes = 0
-  for (;;) {
-    const { done, value } = await reader.read()
-    if (done) break
-    bytes += value.length; text += dec.decode(value, { stream: true })
-    if (bytes >= max || /<\/head\s*>|<body[\s>]/i.test(stripInert(text))) { try { await reader.cancel() } catch {} break }
-  }
-  return text.slice(0, max)
+  return readHeadFrom(r.body.getReader(), max)
 }
 
 /**
@@ -272,21 +296,49 @@ export function gateHeadersFor (gate, url, origins = []) {
   return h
 }
 
+/** How long reading one sitemap or robots.txt may take (ms); its size is capped by SITEMAP_MAX_BYTES. */
+export const SITEMAP_TIMEOUT = 60000
+
+const readHeaders = (want) => want.bytes
+  ? { 'User-Agent': 'Kaloko', Accept: 'application/xml,text/xml,text/plain;q=0.9,*/*;q=0.8', 'Accept-Encoding': 'gzip' }
+  : { 'User-Agent': 'Kaloko', Accept: 'text/html,application/xhtml+xml' }
+
+/** The whole body as bytes, at most `max` (else an error, the rest is not read). */
+export async function readBytes (r, max = SITEMAP_MAX_BYTES) {
+  const big = () => new Error(`larger than ${Math.round(max / 1048576)} MB`)
+  if (!r.body?.getReader) { const b = Buffer.from(await r.arrayBuffer()); if (b.length > max) throw big(); return b }
+  const reader = r.body.getReader(); const parts = []; let bytes = 0
+  for (;;) {
+    const { done, value } = await reader.read()
+    if (done) break
+    bytes += value.length
+    if (bytes > max) { try { await reader.cancel() } catch {} throw big() }
+    parts.push(value)
+  }
+  return Buffer.concat(parts)
+}
+
 /**
  * Reads a page for its hreflang: { status, url (after redirects), link (the Link header), html (the head) }. The
  * gate (headers, Basic) and the walk's cookies (`cookiesFor(url)` → Cookie header) go only to `origins`; redirects
- * are followed by hand (at most 5) so they never carry them elsewhere; each request has `timeout` ms.
+ * are followed by hand (at most 5) so they never carry them elsewhere; each request has `timeout` ms. `want.bytes`
+ * reads a sitemap or robots.txt instead: the whole body as `body` (bytes, at most SITEMAP_MAX_BYTES, all hops within
+ * SITEMAP_TIMEOUT ms; `want.max` and `want.timeout` change them).
  */
 export function pageReader ({ fetchImpl = fetch, gate = null, origins = [], cookiesFor = null, timeout = ALTERNATES_TIMEOUT, max = ALTERNATES_MAX } = {}) {
-  return async (url) => {
+  return async (url, want = {}) => {
     let at = url
+    const whole = want.bytes ? AbortSignal.timeout(want.timeout || SITEMAP_TIMEOUT) : null
     for (let hop = 0; hop <= 5; hop++) {
-      const headers = { 'User-Agent': 'Kaloko', Accept: 'text/html,application/xhtml+xml', ...gateHeadersFor(gate, at, origins) }
+      const headers = { ...readHeaders(want), ...gateHeadersFor(gate, at, origins) }
       if (cookiesFor && origins.includes(new URL(at).origin)) { const c = await cookiesFor(at); if (c) headers.Cookie = c }
-      const r = await fetchImpl(at, { redirect: 'manual', headers, signal: AbortSignal.timeout(timeout) })
+      const r = await fetchImpl(at, { redirect: 'manual', headers, signal: whole || AbortSignal.timeout(timeout) })
       const loc = r.headers?.get?.('location')
-      if (r.status >= 300 && r.status < 400 && loc) { try { await r.body?.cancel() } catch {} at = new URL(loc, at).href; continue }
-      return { status: r.status, url: at, link: r.headers?.get?.('link') || null, html: r.ok ? await readHead(r, max) : '' }
+      if (r.status >= 300 && r.status < 400 && loc) { try { await r.body?.cancel() } catch {} at = nextHop(loc, at); continue }
+      const link = r.headers?.get?.('link') || null
+      if (!want.bytes) return { status: r.status, url: at, link, html: r.ok ? await readHead(r, max) : '' }
+      if (!r.ok) { try { await r.body?.cancel() } catch {} return { status: r.status, url: at, link, body: Buffer.alloc(0) } }
+      return { status: r.status, url: at, link, body: await readBytes(r, want.max || SITEMAP_MAX_BYTES) }
     }
     throw new Error('more than 5 redirects')
   }
@@ -298,22 +350,7 @@ export function pageReader ({ fetchImpl = fetch, gate = null, origins = [], cook
  * a timeout or a 5xx is tried once more; a result with an error is not kept, so another locale asks again.
  */
 export async function readAlternates (read, url, fallback = null, cache = new Map()) {
-  const got = await cached(cache, `alt|${url}`, async () => {
-    let alternates = []; let error = null; let at = url
-    for (let attempt = 0; attempt < 2; attempt++) {
-      try {
-        const r = await read(url)
-        at = r.url || url
-        if (r.status < 200 || r.status >= 300) { error = `HTTP ${r.status}`; if (r.status >= 500) continue; break }
-        error = null
-        alternates = htmlAlternates(r.html, at)
-        if (!alternates.length) alternates = headerAlternates(r.link, at)
-        break
-      } catch (e) { error = String(e?.message || e).split('\n')[0] }
-    }
-    if (!alternates.length && fallback?.length) return { alternates: fallback, error: null, url: at, from: 'sitemap' }
-    return { alternates, error, url: at }
-  })
+  const got = await cached(cache, `alt|${url}`, () => pageAlternates(read, url, fallback))
   if (got.error) cache.delete(`alt|${url}`)
   return got
 }

package/src/lib/variants-core.mjs

@@ -24,6 +24,53 @@ const unique = (list) => { const seen = new Set(); return list.filter((a) => a &
  */
 export const stripInert = (html) => String(html || '').replace(/<!--[\s\S]*?(?:-->|$)/g, '').replace(/<script\b[\s\S]*?(?:<\/script\s*>|$)/gi, '').replace(/<style\b[\s\S]*?(?:<\/style\s*>|$)/gi, '')
 
+/**
+ * Where the head of a page ends, found as the text arrives: `feed(chunk)` is true once `</head>` or `<body>` shows up
+ * outside comments, scripts and styles (the same rule as stripInert, then the search in htmlAlternates). Each call
+ * scans only the new text and a few characters before it, so a long head read in small chunks stays linear.
+ */
+export function headEnd () {
+  let text = ''; let pos = 0; let mode = null
+  const OPEN = /<!--|<script\b|<style\b|<\/head\s*>|<body[\s>]/gi
+  const CLOSE = { comment: /-->/g, script: /<\/script\s*>/gi, style: /<\/style\s*>/gi }
+  return (chunk) => {
+    text += chunk
+    for (;;) {
+      const re = mode ? CLOSE[mode] : OPEN
+      re.lastIndex = pos
+      const m = re.exec(text)
+      // nothing yet: keep the last characters, a token may be split between two chunks
+      if (!m) { pos = Math.max(pos, text.length - 16); return false }
+      // `<script` at the very end may still turn into `<scripts`: wait for the next character
+      if (!mode && m.index + m[0].length >= text.length && /^<(script|style)/i.test(m[0])) { pos = m.index; return false }
+      pos = m.index + m[0].length
+      if (mode) { mode = null; continue }
+      const t = m[0].toLowerCase()
+      if (t === '<!--') mode = 'comment'
+      else if (t.startsWith('<script')) mode = 'script'
+      else if (t.startsWith('<style')) mode = 'style'
+      else return true
+    }
+  }
+}
+
+/**
+ * The head of a page from a stream reader (`body.getReader()`): read until the head ends (headEnd) or `max` bytes,
+ * then the stream is cancelled. Shared by the CLI (stack.mjs readHead) and hosted runs (hosted-stacks.js).
+ */
+export async function readHeadFrom (reader, max) {
+  const dec = new TextDecoder(); const ended = headEnd(); let text = ''; let bytes = 0
+  for (;;) {
+    const { done, value } = await reader.read()
+    if (done) break
+    bytes += value.byteLength
+    const chunk = dec.decode(value, { stream: true })
+    text += chunk
+    if (bytes >= max || ended(chunk)) { try { await reader.cancel() } catch {} break }
+  }
+  return text.slice(0, max)
+}
+
 /** hreflang alternates in the head of an HTML page: [{ lang (lowercase, en-us), href (absolute) }]. */
 export function htmlAlternates (html, url) {
   const text = stripInert(html)
@@ -49,18 +96,30 @@ export function headerAlternates (link, url) {
   return unique(out)
 }
 
-/** The `<url>` entries of a sitemap with their `xhtml:link` alternates: Map(loc → [{ lang, href }]). */
-export function sitemapAlternates (xml) {
+/**
+ * The `<url>` entries of a sitemap that name alternates, unparsed: Map(loc → the entry's markup). The service keeps
+ * these for one sitemap at a time and parses only the pages it takes (urlAlternates).
+ */
+export function sitemapUrlEntries (xml) {
   const out = new Map()
   for (const m of String(xml || '').matchAll(/<url(?:\s[^>]*)?>([\s\S]*?)<\/url>/gi)) {
+    if (!/<(?:xhtml:)?link\b/i.test(m[1])) continue
     const loc = m[1].match(LOC)?.[1]
-    if (!loc) continue
-    const alts = unique([...m[1].matchAll(/<(?:xhtml:)?link\b[^>]*>/gi)].map((l) => { const a = attrsOf(l[0]); return /(^|\s)alternate(\s|$)/i.test(a.rel || '') ? alternate(a.hreflang, a.href) : null }))
-    if (alts.length) out.set(decode(loc), alts)
+    if (loc) out.set(decode(loc), m[1])
   }
   return out
 }
 
+/** The `xhtml:link` alternates of one `<url>` entry of a sitemap: [{ lang, href }]. */
+export const urlAlternates = (entry) => unique([...String(entry || '').matchAll(/<(?:xhtml:)?link\b[^>]*>/gi)].map((l) => { const a = attrsOf(l[0]); return /(^|\s)alternate(\s|$)/i.test(a.rel || '') ? alternate(a.hreflang, a.href) : null }))
+
+/** The `<url>` entries of a sitemap with their `xhtml:link` alternates: Map(loc → [{ lang, href }]). */
+export function sitemapAlternates (xml) {
+  const out = new Map()
+  for (const [loc, entry] of sitemapUrlEntries(xml)) { const alts = urlAlternates(entry); if (alts.length) out.set(loc, alts) }
+  return out
+}
+
 /**
  * The alternate that stands for `locale`: the same tag (en-us for en_US), else the bare language (en) for a locale
  * with a region, else the first regional tag of the language (sk-sk) for a locale without one. A locale with a script
@@ -84,6 +143,30 @@ export function alternateFor (alts, locale) {
   return parts.find(([, q]) => q?.lang === p.lang && !q.script)?.[0] || parts.find(([, q]) => q?.lang === p.lang)?.[0] || null
 }
 
+/**
+ * The hreflang alternates of one page as `read` (url → { status, url (after redirects), link, html }) gives it: from
+ * its HTML head, else its Link header, else `fallback` (the sitemap's xhtml:link entries for it):
+ * { alternates: [{ lang, href }], error, url, from? }. A network error, a timeout or a 5xx is tried once more, except
+ * an error `noRetry(e)` names (a refused address); an error `rethrow(e)` names (the service's spent round) is not
+ * caught. Shared by the CLI (stack.mjs readAlternates) and hosted runs (worker/src/hosted-stacks.js).
+ */
+export async function pageAlternates (read, url, fallback = null, { rethrow = () => false, noRetry = () => false } = {}) {
+  let alternates = []; let error = null; let at = url
+  for (let attempt = 0; attempt < 2; attempt++) {
+    try {
+      const r = await read(url)
+      at = r.url || url
+      if (r.status < 200 || r.status >= 300) { error = `HTTP ${r.status}`; if (r.status >= 500) continue; break }
+      error = null
+      alternates = htmlAlternates(r.html, at)
+      if (!alternates.length) alternates = headerAlternates(r.link, at)
+      break
+    } catch (e) { if (rethrow(e)) throw e; error = String(e?.message || e).split('\n')[0]; if (noRetry(e)) break }
+  }
+  if (!alternates.length && fallback?.length) return { alternates: fallback, error: null, url: at, from: 'sitemap' }
+  return { alternates, error, url: at }
+}
+
 const hostOf = (u) => { try { return new URL(u).host } catch { return null } }
 const sameLanguage = (a, b) => { const x = localeParts(a); const y = localeParts(b); return !!x && !!y && x.lang === y.lang }
 

package/viewer/index.html

@@ -2136,7 +2136,7 @@ function hlLocal () {
   return { checks, counts, findings, legacy: !!RUN.health_legacy };
 }
 /** The step's health checks that apply here, and how many of them are findings in this capture. */
-function hlStepList (n) { return n.item || n.kind === 'decision' ? [] : (n.health || []).filter((c) => !c.not_applicable); }
+function hlStepList (n) { return n.kind === 'decision' ? [] : (n.health || []).filter((c) => !c.not_applicable); }
 const hlCellSev = (c, v) => hlSev((c.by_locale?.[S.loc]?.[v.name] || (c.counts ? c : {})).verdict, c);
 const hlStepBad = (n, v) => hlStepList(n).filter((c) => hlCellSev(c, v)).length;
 // the service's answer adds what changed since the previous run, the adoptions and whether this person may adopt
@@ -2203,11 +2203,11 @@ function hlRow (n, v, cap, c) {
   const st = cellSt(r);
   const sev = hlSev(r.verdict, c);
   const hasBox = (cap?.boxes?.[c.id] || []).some((b) => b.w > 0 && b.h > 0);
-  const ad = hlAdoption(n.id, c.id);
+  const ad = hlAdoption(n.stack || n.id, c.id);
   // a cross-page check of a stack (#151) compares pages with each other: no single page can adopt it
   const adopt = c.cross_page ? '' : ad ? `<p class="note">${esc(T(ad.applied_at ? 'hlApplied' : 'hlAdopted')(ad.criterion?.id || hlAcId(c.id)))}</p>`
     : hlCanAdopt() ? `<div data-hl-adopt-box="${esc(c.id)}"><button type="button" class="btn" data-hl-adopt="${esc(c.id)}">${T('hlAdopt')}</button></div>`
-      : !APP() ? `<details class="fixhelp"><summary>${T('hlAdopt')}</summary><p>${esc(T('hlAdoptLocal')(hlAcId(c.id)))}</p><code class="ev hl-cmd">kaloko health adopt ${esc(n.id)}:${esc(c.id)} --text "${esc(c.text)}"</code></details>` : '';
+      : !APP() ? `<details class="fixhelp"><summary>${T('hlAdopt')}</summary><p>${esc(T('hlAdoptLocal')(hlAcId(c.id)))}</p><code class="ev hl-cmd">kaloko health adopt ${esc(n.stack || n.id)}:${esc(c.id)} --text "${esc(c.text)}"</code></details>` : '';
   return `<li class="st-${st}" data-hl="${esc(c.id)}" data-where="${esc(hasBox ? 'page' : c.target?.kind || '')}" tabindex="0"><span class="mark">${MARK[st]}</span><span><span class="id">${esc(c.id)}</span>${esc(critText(c))}${sev ? `<span class="hl-sev hl-sev--${sev}">${T('hls_' + sev)}</span>` : ''}${c.pack ? `<span class="acb st-skip" style="margin-left:6px">${esc(c.pack)}</span>` : ''}</span><div class="acbody">${r.evidence ? (humanEvidence(r.evidence) ? `<span class="ev ev--text">${esc(humanEvidence(r.evidence))}</span>` : `<code class="ev">${esc(r.evidence)}</code>`) : r.reason ? `<span class="ev ev--text">${esc(r.reason)}</span>` : ''}${c.urls?.length && sev ? `<p class="note hl-urls">${c.urls.map(esc).join(', ')}</p>` : ''}${isFocusCheck(c) ? focusStrip(n, v, cap) : ''}${helpBlock(c, st)}${adopt}</div></li>`;
 }
 /** The step's Health tab: its health checks, findings first, each with its evidence, help and Adopt. */
@@ -2257,13 +2257,15 @@ document.addEventListener('submit', async (e) => {
   const text = f.querySelector('textarea').value.replace(/\s+/g, ' ').trim();
   if (!text) { f.querySelector('textarea').focus(); return; }
   const btn = f.querySelector('[type=submit]'); btn.disabled = true;
+  // a page of a stack adopts for its stack step
+  const step = n.stack || n.id;
   try {
-    const res = await fetch(`${APP().api}/orgs/${APP().org}/runs/${APP().run}/health/adopt`, { method: 'POST', credentials: 'same-origin', headers: { 'Content-Type': 'application/json' }, body: JSON.stringify({ step: n.id, check: c.id, ...(text !== c.text ? { text } : {}) }) });
+    const res = await fetch(`${APP().api}/orgs/${APP().org}/runs/${APP().run}/health/adopt`, { method: 'POST', credentials: 'same-origin', headers: { 'Content-Type': 'application/json' }, body: JSON.stringify({ step, check: c.id, ...(text !== c.text ? { text } : {}) }) });
     if (res.status === 401) throw sessionLost();
     const data = await res.json().catch(() => ({}));
     if (!res.ok) { toast(data.error || `HTTP ${res.status}`); btn.disabled = false; return; }
     const ad = data.adoption || data;
-    S.hl ||= { adoptions: [] }; S.hl.adoptions = [...(S.hl.adoptions || []).filter((a) => !(a.step === n.id && hlId(a.check) === c.id)), { id: ad.id, step: n.id, check: c.id, text: ad.text || text, criterion: ad.criterion || null, applied_at: ad.applied_at || null }];
+    S.hl ||= { adoptions: [] }; S.hl.adoptions = [...(S.hl.adoptions || []).filter((a) => !(a.step === step && hlId(a.check) === c.id)), { id: ad.id, step, check: c.id, text: ad.text || text, criterion: ad.criterion || null, applied_at: ad.applied_at || null }];
     toast(T('hlAdopted')(ad.criterion?.id || hlAcId(c.id)));
     openLb(S.open, S.lbVp, { keepFrame: true });
   } catch (err) { toast(err.message); btn.disabled = false; }
@@ -2512,9 +2514,20 @@ function ensureItemNode (n, data, it) {
   const crit = (data.criteria || n.criteria).map((c) => {
     const by_locale = {};
     for (const [key, res] of Object.entries(it.results || {})) { const [l, vp] = key.split('|'); const r = res[c.id]; if (r) ((by_locale[l] ||= {})[vp] = { verdict: r[0], evidence: r[1], score: r[2] }); }
-    return { id: c.id, text: c.text, type: c.type, pack: c.pack || null, by_locale, evidence: null };
+    return { id: c.id, text: c.text, type: c.type, pack: c.pack || null, ...(c.level ? { level: c.level } : {}), ...(c.acceptance ? { acceptance: true } : {}), ...(c.health ? { health: true } : {}), by_locale, evidence: null };
   }).filter((c) => Object.keys(c.by_locale).length);
-  const node = { id, item: true, itemId: it.id, stack: n.id, kind: 'screen', title: it.title || it.path, path: it.path, edit_url: it.edit_url || null, content_ref: it.content_ref || null, purpose: n.purpose, captures: it.captures || {}, criteria: crit, by_locale: it.by_locale || {}, status: it.status, attachments: it.attachments || [], semantic: {}, metadata: {}, execution: n.execution, session: n.session };
+  // the page's health checks get the Health tab, as on a step: kaloko marks them (health: true);
+  // runs from before the mark tell them by the pack outside acceptance or by the HC id
+  const marked = (data.criteria || n.criteria).some((c) => 'health' in c);
+  const isHealth = (c) => marked ? !!c.health : hlLegacyCrit(c) || /^HC[0-9]/.test(c.id);
+  const captures = it.captures || {};
+  const health = crit.filter(isHealth).map((c) => {
+    const hid = hlId(c.id); if (hid === c.id) return c;
+    // a run from before the split names the check AC…: its highlight boxes answer to the HC id too
+    for (const vps of Object.values(captures)) for (const cap of Object.values(vps || {})) if (cap?.boxes?.[c.id] && !cap.boxes[hid]) cap.boxes[hid] = cap.boxes[c.id];
+    return { ...c, sid: c.id, id: hid };
+  });
+  const node = { id, item: true, itemId: it.id, stack: n.id, kind: 'screen', title: it.title || it.path, path: it.path, edit_url: it.edit_url || null, content_ref: it.content_ref || null, purpose: n.purpose, captures, criteria: crit.filter((c) => !isHealth(c)), health, by_locale: it.by_locale || {}, status: it.status, attachments: it.attachments || [], semantic: {}, metadata: {}, execution: n.execution, session: n.session };
   ITEM_NODES.set(id, node);
   return node;
 }