kaloko 0.18.0 → 0.19.0

package/src/cli.mjs

@@ -5,7 +5,7 @@ import { pathToFileURL, fileURLToPath } from 'node:url'
 import { AsyncLocalStorage } from 'node:async_hooks'
 import { availableParallelism } from 'node:os'
 import yaml from 'js-yaml'
-import { loadConfig, environment, urlFor, baseUrl, CONFIG_FILE, LEGACY_CONFIG_FILE } from './lib/config.mjs'
+import { loadConfig, environment, urlFor, baseUrl, CONFIG_FILE, LEGACY_CONFIG_FILE, projectRequirement } from './lib/config.mjs'
 import { loadScenario, parseScenario, validateScenario, scenarioWarnings } from './lib/scenario.mjs'
 import { createRun, resolveRun, readCaptures, listRuns } from './lib/run.mjs'
 import { startRefs, normalizePurpose, RUN_PURPOSES } from './lib/run-refs.mjs'
@@ -15,13 +15,13 @@ import { captureStep, captureCrop, captureOptions, acceptDialogs } from './lib/c
 import { observeConsole } from './lib/console.mjs'
 import { mailClient, waitForMessage, recipient, readMessageFile } from './lib/mail.mjs'
 import { newContext, accountFor, gateHeaders, gateOrigins, originMatcher, gateForRequests } from './lib/access.mjs'
-import { configureGate } from './lib/linkcheck.mjs'
+import { configureGate, disposeShared } from './lib/linkcheck.mjs'
 import { rememberEnvironment } from './lib/secrets.mjs'
 import { evaluateRun, semanticPrefetch } from './lib/evaluate.mjs'
 import { buildManifest, writePreview, attentionLines } from './lib/build.mjs'
 import { stabilityCmd, pullDrawn } from './lib/stability.mjs'
 import { signoffCmd, evidenceCmd } from './lib/signoff.mjs'
-import { localized, readJson, UserError, UsageError, globToRegExp, failureStreak, expiryNote, SUPPORT_LINE, SUPPORT_EMAIL, NO_ORG, EXIT, parseDuration } from './lib/util.mjs'
+import { localized, readJson, writeJson, UserError, UsageError, globToRegExp, failureStreak, expiryNote, SUPPORT_LINE, SUPPORT_EMAIL, NO_ORG, EXIT, parseDuration } from './lib/util.mjs'
 import { noteLegacy, legacyHint, resetLegacy, legacyInUse } from './lib/legacy.mjs'
 import { packageRoot } from './lib/schema.mjs'
 import { compareRuns, formatComparison } from './lib/compare.mjs'
@@ -32,14 +32,16 @@ import { pageFacts, draftScenario, scenarioYaml } from './lib/draft.mjs'
 import { detectProject, initScenario, skeletonScenario, scenarioHeader, formatDetection } from './lib/init-detect.mjs'
 import { checkSlug, checkName, setConfigKeys, createProject, renameProject, syncProject, shareProjectNote } from './lib/projects-cli.mjs'
 import { recheckCaptures } from './lib/recheck.mjs'
-import { resolveItems, fieldsOf, DEFAULT_LIMIT, timeLeft } from './lib/stack.mjs'
+import { runFindings, findingsText } from './lib/findings.mjs'
+import { resolveItems, fieldsOf, DEFAULT_LIMIT, timeLeft, rotationState, rotationCoverage } from './lib/stack.mjs'
+import { walkSiteChecks, siteChecksOn } from './lib/site.mjs'
 import { DEFAULTS as JEV_DEFAULTS } from './lib/jev.mjs'
 import { expandPacks } from './lib/packs.mjs'
 import { upsertPrComment } from './lib/pr.mjs'
 import { formatReviewQueue, shareReviewLine } from './lib/review.mjs'
 import { healthCmd } from './lib/health.mjs'
 import { htmlViewAllowed } from './lib/htmlview.mjs'
-import { service, serviceUrl } from './lib/service.mjs'
+import { service, serviceUrl, shareNotFound } from './lib/service.mjs'
 import { parseArgs, helpFor, checkOptions, closest, catalogue, unknownOptions, subcommandsFor, takesWords } from './lib/cli-args.mjs'
 import { issueCmd, scheduleCmd } from './lib/integrations-cli.mjs'
 import { login } from './lib/login.mjs'
@@ -114,6 +116,11 @@ Project
                                     listed); without it no routes are guessed, and --skeleton writes a scenario with
                                     a TODO instead. --base-url reads the pages from the running site (sitemap.xml,
                                     then the links of its home page) instead of the code; only that host is asked
+       [--seo <site> [--env production] [--families 4] [--limit 50] [--packs seo,aeo,a11y,perf] [--out file] [--force]]
+                                    A weekly SEO and answer-engine monitor of a public site: reads robots.txt and its
+                                    sitemaps, finds the languages and the biggest content families (first path segment,
+                                    translated per language) and writes qa/flows/seo-<site>.yml with one rotating stack
+                                    step per family (packs seo and aeo, a11y and perf on request)
   ci github|gitlab [--scenario a.yml,b.yml] [--env preview] [--trigger deployment|pull_request] [--no-affected]
        [--out file] [--print] [--force]
                                     Write a workflow that walks what each pull request affects on its preview
@@ -200,8 +207,11 @@ Run
                                     --devices walks only those; dark mode and reduced motion are set per capture
                                     Semantic questions start as each step is captured; the walk ends with the
                                     verdicts and what failed (--no-evaluate skips that; kaloko evaluate redoes it)
+       [--budget N]                 At most N uncached evaluator calls for the walk and its evaluation (default
+                                    evaluate.budget.calls); semantic criteria past it stay not evaluated
        [--items a,b] [--limit N]    Stack steps (kind: stack, many pages of one template): only pages whose id or
-                                    path contains a, b; at most N pages (default items.limit, 500)
+                                    path contains a, b, from every matching page (also outside this week's rotation,
+                                    which such a walk leaves as it was); at most N pages (default items.limit, 500)
        [--stability N]              Take N screenshots of each capture a moment apart and note regions that move
                                     (clocks, carousels); kaloko stability turns them into ignore suggestions
        [--concurrency N]            Locales at the same time, each in its own browser context (default: by CPU in an
@@ -231,6 +241,8 @@ Run
 
 Results
   evaluate [--json]                 Deterministic checks + semantic evaluator (default JEV) for all captures
+  evaluate --budget N               At most N uncached evaluator calls (default evaluate.budget.calls): steps first,
+                                    then stack pages failing a check or checked longest ago; the rest not evaluated
   evaluate --recheck [--steps a,b]  After a scenario change: re-run deterministic checks on the stored HTML and ask
                                     the semantic questions again on the stored captures (no walk);
                                     console, overflow and performance keep their live result
@@ -304,6 +316,10 @@ Results
                                     Newest capture of every step as steps/<step>/<locale>/<viewport>.<ext> plus
                                     index.json (documentation layer, Business plan); unchanged files are skipped
                                     (also kaloko export --scenario <id>)
+  export findings [<run>] [--format csv|json] [--locale l] [--step id] [--out file]
+                                    The fix list of a local run: every failing criterion and health check per page and
+                                    locale with its evidence, the page's edit_url and content_ref (from the item script
+                                    or items.resolve) and the walk that checks it again (kaloko walk --steps s --items p)
   public [--run id] [--off]         Make a shared run readable without sign-in (admin; testers may refresh an already public scenario)
   stability [--last 4 | --runs a,b] [--same-commit] [--apply] [--json]
                                     Regions that change on every run or move while the page is idle (clocks, ads,
@@ -472,10 +488,10 @@ async function warnRetired (args) {
 }
 
 /** The end of a walk: evaluate (the semantic answers are mostly cached by now) and say what did not pass. */
-async function walkVerdicts ({ prefetch, run, scenario, config, root }) {
+async function walkVerdicts ({ prefetch, run, scenario, config, root, budget = null }) {
   const t0 = Date.now()
   await prefetch.done()
-  const report = await evaluateRun({ run, scenario, config, root })
+  const report = await evaluateRun({ run, scenario, config, root, budget })
   // health checks (advisory) are counted apart; they never make a step fail
   const health = report.findings.filter((f) => f.health); const accept = report.findings.filter((f) => !f.health)
   const fails = accept.filter((f) => f.verdict === 'fail'); const unsure = accept.filter((f) => f.verdict === 'needs_review')
@@ -487,9 +503,17 @@ async function walkVerdicts ({ prefetch, run, scenario, config, root }) {
   if (fails.length) say(`  ✕ ${fails.slice(0, 15).map(withFix).join('\n  ✕ ')}${fails.length > 15 ? `\n  … ${fails.length - 15} more (kaloko build lists them all)` : ''}`)
   if (unsure.length) say(`  ? ${unsure.slice(0, 8).map(line).join('\n  ? ')}${unsure.length > 8 ? `\n  … ${unsure.length - 8} more` : ''}`)
   if (report.skipped.length) say(`  Semantic criteria not evaluated: ${[...new Set(report.skipped.map((x) => x.split(': ').slice(1).join(': ')))].slice(0, 2).join('; ')}`)
+  if (report.budget) say(budgetLine(report))
   if (health.length) say(healthLine(health))
 }
 
+/** One line on the evaluator budget (#158): calls used, and how many captures and uncertain answers it left out. */
+function budgetLine (report) {
+  const b = report.budget
+  const out = report.evaluator?.escalation_skipped ? `, ${report.evaluator.escalation_skipped} uncertain answer(s) not escalated` : ''
+  return `Evaluator budget: ${b.used} of ${b.calls} call(s) used${b.skipped ? `; semantic criteria of ${b.skipped} capture(s) not evaluated (raise evaluate.budget.calls or pass --budget)` : ''}${out}.`
+}
+
 /** One line on the health findings of an evaluation: counts by severity (one per check and step), never a verdict. */
 function healthLine (findings) {
   const worst = new Map()
@@ -784,23 +808,27 @@ async function exportScenario ({ args, config, root, client }) {
  * fields (HTTP status of the main document included). A step script, when given, runs per page with `item`.
  */
 async function walkStack ({ node, locale, page, env, run, scenario, scenarioDir, config, consoleFile, args, keys = null, tag = '' }) {
-  const { items, total, truncated } = await resolveItems({ node, base: baseUrl(env, locale), page, scenarioDir, env, locale, limit: Number(args.limit) || undefined })
-  const only = list(args.items)
-  const chosen = only ? items.filter((it) => only.some((q) => it.id === q || it.path.includes(q))) : items
+  // a rotating sample (items.rotate) remembers between runs how many runs walked it and when each page was checked
+  const state = node.items?.rotate ? await rotationState(join(run.dir, '..', '.cache', 'stack', scenario.id, `${node.id}.${locale}.json`), run.meta.id) : null
+  const { items, total, truncated, all, rotation, resolve_failed: unresolved, resolve_error: unresolvedWhy } = await resolveItems({ node, base: baseUrl(env, locale), page, scenarioDir, env, locale, scenario, limit: Number(args.limit) || undefined, rotation: { run: state?.run || 0 }, checked: state?.checked || null, only: list(args.items) })
+  if (unresolved) say(`! ${node.id} ${locale}${tag}: items.resolve failed for ${unresolved} page(s), walked without an edit link (${unresolvedWhy})`)
+  // --items picked the pages from the whole list (resolveItems only); a partial walk leaves the rotation state alone
+  const chosen = items
   const passKeys = keys || engineKeys(run.meta.scope, 'chromium')
   const bases = baseViewports(run.meta.scope)
   const vpNames = (node.items.viewports || []).filter((v) => bases.includes(v) || run.meta.scope.viewports[v])
   const passBases = baseViewports({ ...run.meta.scope, viewports: Object.fromEntries(passKeys.map((k) => [k, run.meta.scope.viewports[k]])) })
   const viewports = scopeKeys(run.meta.scope, passKeys, vpNames.length ? vpNames : [passBases[0] || bases[0]])
   if (!Object.keys(viewports).length) { say(`▦ ${node.id} ${locale}${tag}: no viewport of this step in this pass`); return [] }
-  say(`▦ ${node.id} ${locale}${tag}: ${chosen.length} page(s)${truncated ? ` — limit ${Number(args.limit) || node.items.limit || DEFAULT_LIMIT} reached (${total}+ found; raise items.limit or pass --limit)` : ''}`)
+  if (rotation) say(`▦ ${node.id} ${locale}${tag}: rotating sample, ${rotation.week ? `week ${rotation.week}` : `run ${rotation.run}`} — pages ${rotation.offset + 1}…${((rotation.offset + items.length - 1) % Math.max(1, total)) + 1} of ${total}, all covered in ${rotation.rounds} ${rotation.week ? 'weeks' : 'runs'}`)
+  say(`▦ ${node.id} ${locale}${tag}: ${chosen.length} page(s)${rotation ? '' : truncated ? ` — limit ${Number(args.limit) || node.items.limit || DEFAULT_LIMIT} reached (${total}+ found; raise items.limit or pass --limit)` : ''}`)
   const fields = fieldsOf(node)
   const script = node.execution?.script ? await import(pathToFileURL(resolve(scenarioDir, node.execution.script)).href) : null
   let status = null
   const onResponse = (r) => { if (r.request().isNavigationRequest() && r.frame() === page.mainFrame()) status = r.status() }
   page.on('response', onResponse)
   const streak = failureStreak(5)
-  const failed = []; let ok = 0
+  const failed = []; let ok = 0; const checkedNow = []
   const started = Date.now(); let told = started
   try {
     for (const [i, item] of chosen.entries()) {
@@ -813,7 +841,7 @@ async function walkStack ({ node, locale, page, env, run, scenario, scenarioDir,
         if (script) await script.default({ page, item, url: () => item.url, locale, env, run: run.meta, capture, acceptDialogs: () => acceptDialogs(page) })
         else await page.goto(item.url, { waitUntil: 'domcontentloaded', timeout: 30000 })
         if (!captured) await capture()
-        ok++; streak.ok('stack')
+        ok++; streak.ok('stack'); checkedNow.push(item.path)
       } catch (error) {
         failed.push(`${item.id} (${item.path}): ${error.message.split('\n')[0]}`)
         if (streak.record('stack', error.message)) say(`  stopping ${node.id} (${locale}${tag}): the last 5 pages failed with the same error`)
@@ -822,6 +850,11 @@ async function walkStack ({ node, locale, page, env, run, scenario, scenarioDir,
       if (i + 1 < chosen.length && ((i + 1) % 25 === 0 || Date.now() - told > 30000)) { told = Date.now(); const left = timeLeft(i + 1, chosen.length, told - started); say(`  ${node.id} ${locale}: ${i + 1}/${chosen.length}${left ? ` · ${left}` : ''}`) }
     }
   } finally { page.off('response', onResponse) }
+  // the rotation of this locale goes into the run (the stack index shows it) and the pages checked into the state
+  if (state && rotation) {
+    const checked = await state.save(checkedNow, all)
+    await writeJson(join(run.dir, 'steps', node.id, `rotation.${locale}.json`), { ...rotation, ...rotationCoverage(all, checked) })
+  }
   say(`${failed.length ? '✗' : '✓'} ${node.id} ${locale}${tag}: ${ok}/${chosen.length} page(s) captured`)
   return failed
 }
@@ -1029,21 +1062,65 @@ export function exportSubcommand (args) {
   const sub = args._[0]
   if (sub === undefined) return null
   // the flags of the other kinds (live and design exports take --scenario too)
-  const clashes = { run: ['live', 'design', 'format', 'scenario'], steps: ['live', 'design', 'format'], docs: ['live', 'design', 'scenario'], live: ['design', 'format'], design: ['live', 'format'] }
+  const clashes = { run: ['live', 'design', 'format', 'scenario'], steps: ['live', 'design', 'format'], findings: ['live', 'design', 'scenario'], docs: ['live', 'design', 'scenario'], live: ['design', 'format'], design: ['live', 'format'] }
   const owner = { live: 'live', design: 'design', format: 'docs', scenario: 'steps' }
   // kaloko export <run-id>: the run named as a word, as people type it
   if (/^[0-9]{4}-[0-9]{2}-[0-9]{2}-[a-z0-9-]+$/.test(sub) && args.run === undefined) { args._.shift(); args.run = sub; return 'run' }
-  if (!(sub in clashes)) throw new UsageError(`kaloko export ${sub}: there is no such export. Use run, steps, docs, live or design (kaloko help export).`)
+  if (!(sub in clashes)) throw new UsageError(`kaloko export ${sub}: there is no such export. Use run, steps, findings, docs, live or design (kaloko help export).`)
   args._.shift()
   const clash = clashes[sub].find((f) => args[f] !== undefined)
   if (clash) throw new UsageError(`kaloko export ${sub} does not take --${clash}; that is kaloko export ${owner[clash]}.`)
   if (sub === 'live') args.live = true
   if (sub === 'design') args.design = typeof args.design === 'string' ? args.design : true
   if (sub === 'docs' && typeof args.format !== 'string') args.format = 'md'
+  // kaloko export findings <run-id>: the run named as a word
+  if (sub === 'findings' && args._[0] && args.run === undefined) args.run = args._.shift()
+  if (sub === 'findings' && args.format !== undefined && !['csv', 'json'].includes(args.format)) throw new UsageError('kaloko export findings takes --format csv or --format json.')
   if (sub === 'steps' && typeof args.scenario !== 'string') throw new UsageError('kaloko export steps needs --scenario <id>: the flow whose steps to export.')
   return sub
 }
 
+/** kaloko init --seo: writes the drafted monitor and says whether the config's environment points at the site. */
+async function writeSeoScenario ({ seo, cwd }) {
+  const { scenarioYaml } = await import('./lib/draft.mjs')
+  const { seoNextSteps } = await import('./lib/seo-init.mjs')
+  await mkdir(dirname(seo.out), { recursive: true })
+  await writeFile(seo.out, `${seo.header}${scenarioYaml(seo.scenario)}`)
+  const file = relative(cwd, seo.out)
+  say(`Drafted ${file}: ${seo.scenario.nodes.length} stack step(s) in ${seo.locales.length} language(s).`)
+  for (const n of seo.notes) say(`  ${n}`)
+  let configNote = null
+  try {
+    const { config } = await loadConfig(cwd)
+    const env = config.environments?.[seo.env]
+    const base = typeof env?.base_url === 'string' ? env.base_url : env?.base_url?.default
+    let host = null; try { host = new URL(base).host } catch {}
+    if (!env) configNote = `! ${CONFIG_FILE} has no environment ${seo.env}. Add it under environments:\n    ${seo.env}:\n      base_url: ${seo.origin}\n      readonly: true`
+    else if (host !== new URL(seo.origin).host) configNote = `! The ${seo.env} environment points at ${base || 'nothing'}, not ${seo.origin}. Set its base_url, or pass --env with the environment of this site.`
+  } catch {}
+  say(seoNextSteps({ file, env: seo.env, origin: seo.origin, configNote }).join('\n'))
+  return { file, steps: seo.scenario.nodes.map((n) => n.id), locales: seo.locales }
+}
+
+/** kaloko export findings: the fix list of a local run, per page and locale (src/lib/findings.mjs). */
+async function exportFindings (args) {
+  const { config, root, run, scenario } = await runContext(args)
+  // read-only: the manifest and the stack indexes are assembled in memory from the captures, nothing is written
+  const indexes = new Map()
+  const manifest = await buildManifest({ run, scenario, config, root, catalog: [], indexes })
+  const step = typeof args.step === 'string' ? args.step : null
+  if (step && !(manifest.nodes || []).some((n) => n.id === step)) throw new UsageError(`No step ${step} in this run. Steps: ${(manifest.nodes || []).filter((n) => n.kind !== 'decision').map((n) => n.id).join(', ')}.`)
+  const rows = await runFindings(manifest, { stackIndex: (n) => indexes.get(n.id) || null, locale: typeof args.locale === 'string' ? args.locale : null, step })
+  const format = typeof args.format === 'string' ? args.format : args.json ? 'json' : 'csv'
+  const text = findingsText(rows, format, { run: manifest.id, scenario: manifest.scenario?.id || null })
+  if (typeof args.out === 'string') {
+    await writeFile(resolve(args.out), text)
+    const pages = new Set(rows.map((r) => `${r.step}|${r.page || ''}|${r.locale}`)).size
+    say(args.json ? JSON.stringify({ file: resolve(args.out), count: rows.length, pages }) : `${rows.length} finding(s) on ${pages} page(s) → ${relative(process.cwd(), resolve(args.out)) || args.out}${rows.some((r) => r.edit_url) ? ' (with edit links)' : ''}`)
+  } else process.stdout.write(format === 'json' ? `${text}\n` : text)
+  return { count: rows.length }
+}
+
 async function affectedSelection (args, { config, root, scenario, scenarioFile }) {
   let client = null
   try { if (!args.offline) client = service(config) } catch {}
@@ -1073,6 +1150,14 @@ async function projectsWrite (sub, args) {
   if (sub !== 'sync' && slug === config.project && name && config.project_name && name !== config.project_name) say(`Note: kaloko.config.yml still says project_name: ${config.project_name}; change it there too, or kaloko projects sync would rename the project back.`)
 }
 
+/** The steps that take most of a run, as stored: " Largest steps: products 212 MB (1 390 files), …" or ''. */
+function biggestSteps (entries, n = 3) {
+  const by = new Map()
+  for (const e of entries) { const k = e.path.match(/^steps\/([^/]+)\//)?.[1]; if (!k) continue; const x = by.get(k) || { bytes: 0, files: 0 }; x.bytes += e.stored ?? e.bytes; x.files++; by.set(k, x) }
+  const top = [...by].sort((a, b) => b[1].bytes - a[1].bytes).slice(0, n)
+  return top.length ? ` Largest steps: ${top.map(([k, x]) => `${k} ${fmtBytes(x.bytes)} (${x.files} files)`).join(', ')}.` : ''
+}
+
 const commands = {
   async init (args) {
     const cwd = process.cwd()
@@ -1088,13 +1173,29 @@ const commands = {
       product: args.product !== undefined ? checkSlug(args.product === true ? '' : String(args.product), '--product slug') : null
     }
     const projectAsked = Object.values(projectKeys).some((v) => v != null)
+    // --seo <site> (#162): the monitor is drafted from the live site before anything is written, so a site that
+    // cannot be reached leaves the project as it was
+    let seo = null
+    if (args.seo !== undefined) {
+      const { seoDraft, SEO_DEFAULTS } = await import('./lib/seo-init.mjs')
+      const num = (k, d, max) => { if (args[k] === undefined) return d; const n = Number(args[k]); if (!Number.isInteger(n) || n < 1 || n > max) throw new UsageError(`--${k} takes a whole number from 1 to ${max}.`); return n }
+      const env = typeof args.env === 'string' ? args.env : 'production'
+      // the name goes into kaloko.config.yml as a key: a plain slug only
+      if (!/^[a-z0-9][a-z0-9_-]{0,39}$/i.test(env)) throw new UsageError(`--env takes an environment name (letters, digits, - and _), like production; got ${env}.`)
+      const draft = await seoDraft(String(args.seo), { families: num('families', SEO_DEFAULTS.families, 12), limit: num('limit', SEO_DEFAULTS.limit, 10000), packs: typeof args.packs === 'string' ? list(args.packs) : SEO_DEFAULTS.packs, env })
+      const out = typeof args.out === 'string' ? resolve(cwd, args.out) : join(cwd, 'qa', 'flows', `seo-${new URL(draft.origin).hostname.replace(/^www\./, '').replace(/[^a-z0-9]+/gi, '-').toLowerCase()}.yml`)
+      if (existsSync(out) && !args.force) throw new UserError(`${relative(cwd, out)} already exists; kept. --force replaces it, --out writes another file.`)
+      const problems = validateScenario(JSON.parse(JSON.stringify(draft.scenario)))
+      if (problems.length) throw new UserError(`The drafted scenario does not validate: ${problems.join('; ')}`)
+      seo = { ...draft, out, env }
+    }
     // what the project is (#26): framework, dev server, locales, main routes; --no-detect keeps the plain template
     // several sites or apps (--site picks one) and pages read from the running site (--base-url, or the dev URL)
     let baseUrl = null
     if (args['base-url'] !== undefined) {
       try { baseUrl = new URL(String(args['base-url'])); if (!/^https?:$/.test(baseUrl.protocol)) throw new Error() } catch { throw new UsageError('--base-url must be the http(s) address the site runs on, e.g. http://localhost:3000.') }
     }
-    const detected = args['no-detect'] ? null : await detectProject(cwd, { site: args.site ?? null, baseUrl: baseUrl?.href || null, probe: true })
+    const detected = args['no-detect'] || seo ? null : await detectProject(cwd, { site: args.site ?? null, baseUrl: baseUrl?.href || null, probe: true })
     if (detected) say(formatDetection(detected))
     if (!exists) {
       let tpl = await readFile(join(packageRoot, 'templates', 'kaloko.config.yml'), 'utf8')
@@ -1107,9 +1208,15 @@ const commands = {
         if (project && !projectKeys.project) tpl = tpl.replace(/^project: .*$/m, `project: ${project}`)
         tpl = tpl.replace('  local:\n    base_url:\n      default: http://localhost:3000', `  local:\n    # dev server: ${detected.command || 'start your app'}${detected.framework ? ` (${detected.framework.name}, found by kaloko init)` : ''}\n    base_url:\n      default: ${detected.url}`)
       }
+      // the monitored site is production (or --env): its pages are only read
+      if (seo) {
+        if (seo.env === 'production') tpl = tpl.replace(/^(  production:\n(?:    #.*\n)*    base_url: ).*$/m, `$1${seo.origin}`)
+        else tpl = tpl.replace(/^environments:\n/m, `environments:\n  ${seo.env}:\n    base_url: ${seo.origin}\n    readonly: true\n`)
+        if (!projectKeys.project) tpl = tpl.replace(/^project: .*$/m, `project: ${new URL(seo.origin).hostname.replace(/^www\./, '').replace(/[^a-z0-9]+/gi, '-').toLowerCase()}`)
+      }
       if (projectAsked) tpl = setConfigKeys(tpl, projectKeys).replace(/^# The name people see on Kaloko.*\n# project_name: .*\n/m, projectKeys.project_name ? '' : '$&')
       await writeFile(file, tpl)
-      say(`Created ${CONFIG_FILE}${detected ? ` (local: ${detected.url})` : ''}. Edit org and the staging and production environments.`)
+      say(`Created ${CONFIG_FILE}${detected ? ` (local: ${detected.url})` : seo ? ` (${seo.env}: ${seo.origin})` : ''}. Edit org${seo ? '' : ' and the staging and production environments'}.`)
     } else if (projectAsked) {
       // an existing config keeps everything else, comments included
       const cfgFile = existsSync(file) ? file : join(cwd, LEGACY_CONFIG_FILE)
@@ -1129,7 +1236,7 @@ const commands = {
     // the visual changelog (docs/DOCS-CHANGELOG.md): asked once; --docs=off|changelog|full for scripts and agents
     let docsMode = typeof args.docs === 'string' ? args.docs : null
     if (docsMode && !['off', 'changelog', 'full'].includes(docsMode)) throw new UsageError('--docs must be off, changelog or full.')
-    if (!docsMode && process.stdin.isTTY && process.stdout.isTTY && !process.env.CI && !process.env.NODE_TEST_CONTEXT) {
+    if (!docsMode && !seo && process.stdin.isTTY && process.stdout.isTTY && !process.env.CI && !process.env.NODE_TEST_CONTEXT) {
       const rl = (await import('node:readline/promises')).createInterface({ input: process.stdin, output: process.stdout })
       const a = (await rl.question('Keep a visual changelog? (recommended) [Y/n] ')).trim().toLowerCase(); rl.close()
       docsMode = a === 'n' || a === 'no' ? 'off' : 'changelog'
@@ -1157,7 +1264,7 @@ const commands = {
         }
         say('Docs folders in docs/ (tutorials, how-to, reference, explanation) with an index and a template in each; kaloko docs publish sends them, kaloko docs check checks them')
       }
-    } else if (!docsMode) say('Visual changelog: kaloko init --docs=changelog adds CHANGELOG.md and the config block (recommended).')
+    } else if (!docsMode && !seo) say('Visual changelog: kaloko init --docs=changelog adds CHANGELOG.md and the config block (recommended).')
     const agent = typeof args.agent === 'string' ? args.agent : 'none'
     if (!['claude', 'codex', 'all', 'none'].includes(agent)) throw new UsageError('--agent must be claude, codex, all or none.')
     const { cp, mkdir, appendFile, rm } = await import('node:fs/promises')
@@ -1189,6 +1296,7 @@ const commands = {
       await cp(join(packageRoot, 'templates', 'steps', 'consent.mjs'), consentStep)
       say('Added qa/flows/steps/consent.mjs (accepts the consent dialog, if the site has one)')
     }
+    if (seo) return writeSeoScenario({ seo, cwd })
     // a first scenario that walks as it is, when the project has none yet
     let first = null
     if (detected && !args['no-scenario']) {
@@ -1228,13 +1336,16 @@ const commands = {
     let failed = 0
     // --json: one entry per file with its problems, warnings and coverage; the lines below go to stderr
     const results = []
+    // the config, when there is one, says whether every scenario must name its project (kaloko share answers 404 else)
+    let projectRequired = null
+    try { const { config, file } = await loadConfig(process.cwd(), args.config); projectRequired = projectRequirement(config, await readFile(file, 'utf8')) } catch {}
     for (const file of args._) {
       let scenario
       try { scenario = parseScenario(await readFile(file, 'utf8'), file) } catch (e) { if (!(e instanceof UserError)) throw e; failed++; say(`✕ ${e.message}`); results.push({ file, ok: false, problems: [e.message], warnings: [] }); continue }
       const problems = validateScenario(scenario)
       if (!problems.length) expandPacks(scenario)
       if (problems.length) { failed++; say(`✕ ${file}\n  - ${problems.join('\n  - ')}`) } else say(`✓ ${file}`)
-      const warnings = problems.length ? [] : scenarioWarnings(scenario)
+      const warnings = problems.length ? [] : scenarioWarnings(scenario, { projectRequired })
       for (const w of warnings) say(`  ! ${w}`)
       const entry = { file, id: scenario.id || null, ok: !problems.length, problems, warnings }
       results.push(entry)
@@ -1264,13 +1375,15 @@ const commands = {
   },
 
   async start (args) {
-    const { config, root } = await context(args)
+    const { config, root, file: configFile } = await context(args)
     if (!args.scenario || !args.env) throw new UsageError('Usage: kaloko start --scenario <file> --env <name>')
     // the records the run serves and why it was walked (#126): checked before anything is created
     const { refs, purpose, problems: refIssues } = startRefs(args, process.env)
     if (refIssues.length) throw new UserError(refIssues.join('\n'))
     const scenarioFile = resolve(args.scenario)
     const scenario = await loadScenario(scenarioFile)
+    // warnings of kaloko validate (errors pack on a stack, a project the config asks for): said, not blocking
+    for (const w of scenarioWarnings(scenario, { projectRequired: projectRequirement(config, await readFile(configFile, 'utf8').catch(() => '')) }).filter((x) => !/reads in one language only/.test(x))) (args.json ? console.error : say)(`! ${w}`)
     const env = environment(config, args.env)
     if (scenario.environments && !scenario.environments.includes(env.name)) throw new UserError(`Scenario ${scenario.id} is not allowed on ${env.name} (allowed: ${scenario.environments.join(', ')}).`)
     if (env.readonly && !scenario.readonly) throw new UserError(`Environment ${env.name} is readonly; the scenario must declare readonly: true.`)
@@ -1464,7 +1577,7 @@ const commands = {
     if (isApp(scenario.platform)) {
       await walkApp({ config, root, run, scenario, env, args })
       // an app walk ends with the verdicts like a web walk (--no-evaluate skips them; kaloko evaluate redoes them)
-      if (!args['no-evaluate']) await walkVerdicts({ prefetch: { done: async () => {} }, run, scenario, config, root })
+      if (!args['no-evaluate']) await walkVerdicts({ prefetch: { done: async () => {} }, run, scenario, config, root, budget: args.budget ?? null })
       return
     }
     const pick = list(args.steps)
@@ -1488,7 +1601,7 @@ const commands = {
     const parallel = concurrency > 1 && locales.length > 1
     const scenarioDir = resolve(root, run.meta.scenario_file, '..')
     // semantic questions of every new capture start right away; the walk ends with the verdicts (--no-evaluate: skip)
-    const prefetch = args['no-evaluate'] ? null : semanticPrefetch({ scenario, config, root, environment: env?.name || null })
+    const prefetch = args['no-evaluate'] ? null : semanticPrefetch({ scenario, config, root, environment: env?.name || null, budget: args.budget ?? null })
     const walked = await readCaptures(run.dir)
     // recordings and traces (batch D, #12): opt-in recordings, traces unless turned off; encoded in an own headless Chrome
     const recSettings = recordSettings({ args, config, scenario })
@@ -1664,7 +1777,14 @@ const commands = {
     if (failed.length) { say(`\nSteps that failed (fix and rerun with --steps):\n  - ${failed.join('\n  - ')}`); process.exitCode = 1 }
     if (proposals.length) say(`\nSelector proposals above: review the patch, then kaloko walk --steps ${[...new Set(proposals)].join(',')} --heal applies it once the step passes.`)
     if (skipped.length) { say(`\nSteps not walked after repeated failures (rerun with --steps once the cause is fixed):\n  - ${skipped.join('\n  - ')}`); process.exitCode = 1 }
-    if (prefetch) await walkVerdicts({ prefetch, run, scenario, config, root })
+    // site checks (#152, health.site): robots.txt, sitemaps, llms.txt and redirects, once per site and run
+    // health.site.environments limits them (e.g. [production]); off production, HC1102 is a warning and HC1103 skipped
+    const siteOpts = scenario.health?.site === true ? {} : scenario.health?.site
+    if (siteOpts && env.base_url && siteChecksOn(siteOpts, env.name)) {
+      say('')
+      await walkSiteChecks({ runDir: run.dir, bases: locales.map((l) => baseUrl(env, l)).filter(Boolean), options: siteOpts, gate: await gateForRequests(env, { root }), production: env.name === 'production', say }).catch((error) => say(`site checks: ${error.message.split('\n')[0]}`))
+    }
+    if (prefetch) await walkVerdicts({ prefetch, run, scenario, config, root, budget: args.budget ?? null })
   },
 
   async affected (args) {
@@ -1979,9 +2099,11 @@ const commands = {
       say(`Rechecked deterministic criteria of ${r.rechecked} capture(s) from the stored HTML (console, overflow and performance keep their live result).`)
       if (r.skipped.length) say(`  skipped: ${r.skipped.join('; ')}`)
     }
-    const report = await evaluateRun({ run, scenario, config, root })
-    const via = report.via.qawalk ? ` via Kaloko${report.hosted?.limit != null ? ` (${report.hosted.questions} of ${report.hosted.limit} questions this month)` : ''}` : report.via['own-key'] ? ' with your key' : ''
+    const report = await evaluateRun({ run, scenario, config, root, budget: args.budget ?? null })
+    const via = report.evaluator ? ` at ${report.evaluator.base_url}` : report.via.qawalk ? ` via Kaloko${report.hosted?.limit != null ? ` (${report.hosted.questions} of ${report.hosted.limit} questions this month)` : ''}` : report.via['own-key'] ? ' with your key' : ''
     say(`Evaluated ${report.evaluated} capture(s); semantic evaluator ${report.model}${via}: ${report.semantic_calls} call(s), ${report.semantic_cached} cached.`)
+    if (report.evaluator?.escalate_to) say(`Uncertain answers escalated to ${report.evaluator.escalate_to}: ${report.evaluator.escalated}${report.evaluator.escalation_skipped ? `; ${report.evaluator.escalation_skipped} not escalated` : ''}.`)
+    if (report.budget) say(budgetLine(report))
     if (report.vision?.asked) say(`Vision evaluator ${report.vision.model}: ${report.vision.asked} capture(s) with flagged or uncertain criteria, ${report.vision.calls} call(s), ${report.vision.cached} cached${report.vision.skipped.length ? `; not asked: ${[...new Set(report.vision.skipped.map((x) => x.split(': ').slice(1).join(': ')))].slice(0, 2).join('; ')}` : ''}.`)
     if (report.triage) say(report.triage.screens ? `Triage (experimental, ${report.triage.model}): ${report.triage.screens} screen(s)${report.triage.frames ? ` and ${report.triage.frames} recording frame(s)` : ''}, ${report.triage.shown} finding(s) to look at${report.triage.findings > report.triage.shown ? ` and ${report.triage.findings - report.triage.shown} less likely` : ''}; they never change a verdict.${report.triage.skipped ? ` Not every request was answered: ${report.triage.skipped}` : ''}` : `Triage (experimental): not run${report.triage.skipped ? `: ${report.triage.skipped}` : ' (no screenshot to look at)'}`)
     if (report.skipped.length) {
@@ -1995,7 +2117,7 @@ const commands = {
     // --json: the counts and every verdict a person or agent has to act on (fail, needs_review)
     if (args.json) {
       const pick = (f) => ({ step: f.node, ...(f.item ? { item: f.item } : {}), locale: f.locale, viewport: f.viewport, criterion: f.id, verdict: f.verdict, why: f.why || null, fix: f.fix || null })
-      say(JSON.stringify({ run: run.meta.id, evaluated: report.evaluated, semantic: { model: report.model, calls: report.semantic_calls, cached: report.semantic_cached, via: report.via.qawalk ? 'kaloko' : report.via['own-key'] ? 'own-key' : null }, vision: report.vision || null, triage: report.triage || null, findings: (report.findings || []).filter((f) => !f.health && (f.verdict === 'fail' || f.verdict === 'needs_review')).map(pick), health: (report.findings || []).filter((f) => f.health).map((f) => ({ ...pick(f), pack: f.pack, severity: f.severity })), skipped: report.skipped, outdated: report.outdated }, null, 2))
+      say(JSON.stringify({ run: run.meta.id, evaluated: report.evaluated, semantic: { model: report.model, calls: report.semantic_calls, cached: report.semantic_cached, via: report.via.qawalk ? 'kaloko' : report.via['own-key'] ? 'own-key' : report.via['own-model'] ? 'own-model' : null, ...(report.evaluator ? { evaluator: report.evaluator } : {}) }, budget: report.budget || null, vision: report.vision || null, triage: report.triage || null, findings: (report.findings || []).filter((f) => !f.health && (f.verdict === 'fail' || f.verdict === 'needs_review')).map(pick), health: (report.findings || []).filter((f) => f.health).map((f) => ({ ...pick(f), pack: f.pack, severity: f.severity })), skipped: report.skipped, outdated: report.outdated }, null, 2))
     }
   },
 
@@ -2086,17 +2208,18 @@ const commands = {
       say(`${tooBig.size} recording(s) larger than ${fmtBytes(u.limits.max_file_bytes)} stay in the local preview: ${[...tooBig].slice(0, 3).join(', ')}${tooBig.size > 3 ? ', …' : ''}. Record them with record.quality: low or a lower max_seconds to share them.`)
     }
     const missing = await missingHashes(client, config.org, entries.map((e) => e.hash))
-    const total = entries.reduce((n, e) => n + e.bytes, 0)
-    const fresh = missing ? [...new Map(entries.filter((e) => missing.has(e.hash)).map((e) => [e.hash, e.bytes])).values()].reduce((n, b) => n + b, 0) : total
+    // sizes as the service stores them (HTML and JSON gzip-compressed): what the per-run limit and the quota count
+    const total = entries.reduce((n, e) => n + (e.stored ?? e.bytes), 0)
+    const fresh = missing ? [...new Map(entries.filter((e) => missing.has(e.hash)).map((e) => [e.hash, e.stored ?? e.bytes])).values()].reduce((n, b) => n + b, 0) : total
     const resumed = (await readJson(join(run.dir, PROGRESS_FILE), null).catch(() => null))?.run === manifest.id
     try {
       if (!u) throw new Error('usage unavailable')
-      if (total > u.limits.max_run_bytes) throw new UserError(`This run is ${fmtBytes(total)}; the limit per run is ${fmtBytes(u.limits.max_run_bytes)}. Capture fewer viewports or locales, or ask an admin for a higher limit.`)
+      if (total > u.limits.max_run_bytes) throw new UserError(`This run is ${fmtBytes(total)}; the limit per run is ${fmtBytes(u.limits.max_run_bytes)}.${biggestSteps(entries)} Capture fewer viewports or locales; for a stack, lower items.limit, walk a rotating sample (items.rotate: week) or keep thumbnails only (capture: { screenshot: thumb }). An admin can also ask for a higher limit.`)
       if (u.usage.bytes + fresh > u.limits.quota_bytes) throw new UserError(`Storage quota would be exceeded: ${fmtBytes(u.usage.bytes)} used + ${fmtBytes(fresh)} > ${fmtBytes(u.limits.quota_bytes)}. Delete old runs in Settings or ask for a higher quota.`)
       const recBytes = entries.filter((e) => /\.rec\./.test(e.path)).reduce((n, e) => n + e.bytes, 0)
       say(`${resumed ? 'Resuming the interrupted share: ' : ''}${files.size} files, ${fmtBytes(total)}${recBytes ? ` (recordings ${fmtBytes(recBytes)})` : ''}${missing ? `; ${fmtBytes(fresh)} new, the rest already stored` : ''} (organization uses ${fmtBytes(u.usage.bytes)} of ${fmtBytes(u.limits.quota_bytes)})`)
     } catch (e) { if (e instanceof UserError && /limit|quota/.test(e.message)) throw e }
-    const created = await client.request('POST', `/orgs/${config.org}/runs?expires_days=${days}`, manifest)
+    const created = await client.request('POST', `/orgs/${config.org}/runs?expires_days=${days}`, manifest).catch(async (e) => { throw e.status === 404 ? await shareNotFound(e, { client, org: config.org, project: run.meta.project }) : e })
     await writeFile(join(run.dir, PROGRESS_FILE), JSON.stringify({ run: manifest.id, started_at: new Date().toISOString() }) + '\n').catch(() => {})
     const up = await uploadRunFiles({ client, org: config.org, runId: manifest.id, dir: run.dir, entries, missing, say, ...(args.json ? { write: (t) => process.stderr.write(t) } : {}) })
     if (up.mode === 'blobs') say(`  ${up.uploaded} new file(s) sent (${fmtBytes(up.bytes)}), ${up.reused} unchanged file(s) reused`)
@@ -2578,7 +2701,7 @@ const commands = {
   },
 
   async export (args) {
-    exportSubcommand(args)
+    if (exportSubcommand(args) === 'findings') return exportFindings(args)
     const { config, root } = await context(args)
     const client = service(config)
     if (args.design) {
@@ -2849,6 +2972,7 @@ export async function main (argv, { bin = 'kaloko' } = {}) {
   try {
     await runCommand(args, json)
   } finally {
+    await disposeShared() // the crawler checks' request context, when a walk or capture made one
     if (json) console.log = stdout
     const hint = legacyHint()
     if (hint) console.error(hint)

package/src/lib/build.mjs

@@ -12,7 +12,11 @@ import { variantOf } from './dimensions.mjs'
 
 export { targetOf, statusOf } from './manifest-core.mjs'
 
-export async function buildManifest ({ run, scenario, config, root, catalog = [] }) {
+/**
+ * The run manifest from the captures. `indexes` (a Map) makes it read-only: stack indexes go into the map by step id
+ * instead of steps/<step>/items.json and step specs are not written (kaloko export findings).
+ */
+export async function buildManifest ({ run, scenario, config, root, catalog = [], indexes = null }) {
   const meta = run.meta
   const entries = await readCaptures(run.dir)
   const captures = latestCaptures(entries)
@@ -25,7 +29,7 @@ export async function buildManifest ({ run, scenario, config, root, catalog = []
   const mailKeys = viewportNames.filter((k) => variantOf(meta.scope, k).browser === 'chromium')
   const nodes = []
   for (const node of scenario.nodes) {
-    if (node.kind === 'stack') { const p = pos.get(node.id); nodes.push(await buildStack({ run, node, base: { id: node.id, kind: node.kind, title: node.title, path: node.path ?? null, purpose: node.purpose ?? '', col: p.col, lane: p.lane, ...(node.refs?.length ? { refs: node.refs } : {}) }, captures, attachments })); continue }
+    if (node.kind === 'stack') { const p = pos.get(node.id); nodes.push(await buildStack({ run, node, base: { id: node.id, kind: node.kind, title: node.title, path: node.path ?? null, purpose: node.purpose ?? '', col: p.col, lane: p.lane, ...(node.refs?.length ? { refs: node.refs } : {}) }, captures, attachments, indexes })); continue }
     const caps = []
     // pages of a stack live in its items index; normal steps take their own captures only
     for (const c of captures.filter((x) => x.node === node.id && !x.item)) {
@@ -34,7 +38,7 @@ export async function buildManifest ({ run, scenario, config, root, catalog = []
       const ev = stored?.source === evaluationSource(node, cap) ? stored : null
       // the step spec travels as its own file (Dev mode on screenshots, `kaloko spec`, drift on the service)
       const spec = cap.spec ? `steps/${c.node}/${c.locale}/${c.viewport}.spec.json` : null
-      if (spec) await writeJson(join(run.dir, spec), cap.spec)
+      if (spec && !indexes) await writeJson(join(run.dir, spec), cap.spec)
       caps.push({ locale: c.locale, viewport: c.viewport, cap, ev, files: { png: cap.files.png, thumb: cap.files.thumb || null, html: cap.files.html || null, tree: cap.files.tree || null, eval: `steps/${c.node}/${c.locale}/${c.viewport}.eval.json`, spec }, ...(cap.live ? { live: cap.live } : {}) })
     }
     // focus shots and details count only when a current capture names them (manifest-core.mjs stepAttachments)
@@ -42,7 +46,10 @@ export async function buildManifest ({ run, scenario, config, root, catalog = []
     await attachMedia(out, { run, entries: media, captures: caps })
     nodes.push(out)
   }
-  const summary = summarize(nodes, scenario.sessions)
+  // site checks (#152): written by the walk once per site (site.json), advisory like the packs
+  const siteFile = await readJson(join(run.dir, 'site.json'), null)
+  const site = siteFile?.origins?.length ? { origins: siteFile.origins } : null
+  const summary = summarize(nodes, scenario.sessions, site)
 
   const allRuns = await listRuns(root, config.output)
   // flaky: a criterion whose verdict differs from another run of this scenario on the same commit and environment
@@ -87,6 +94,7 @@ export async function buildManifest ({ run, scenario, config, root, catalog = []
     summary,
     // "personal data hidden" (#64): the canvas says so, so a blank is not taken for a bug
     ...(runPii(meta, nodes) ? { pii: runPii(meta, nodes) } : {}),
+    ...(site ? { site } : {}),
     coverage: cov,
     history,
     catalog,
@@ -157,7 +165,7 @@ async function runMedia ({ run, entries }) {
  * A stack step: its pages go to steps/<node>/items.json (qawalk.stack.v1: fields, criteria, and per page the path,
  * status, index fields, capture files and compact results); the node keeps counts, a visual sample and the path.
  */
-async function buildStack ({ run, node, base, captures, attachments }) {
+async function buildStack ({ run, node, base, captures, attachments, indexes = null }) {
   const paths = runPaths(run.dir)
   const list = []
   for (const c of captures.filter((x) => x.node === node.id && x.item)) {
@@ -167,8 +175,11 @@ async function buildStack ({ run, node, base, captures, attachments }) {
     // a variant of a design stack (#61) opens live in the lightbox: its page in the step's revision
     list.push({ item: c.item, locale: c.locale, viewport: c.viewport, cap, ev, files: { png: cap.files.png, thumb: cap.files.thumb || null, html: cap.files.html, eval: `steps/${stepIdOf(c)}/${c.locale}/${c.viewport}.eval.json` }, ...(cap.live ? { live: cap.live } : {}) })
   }
-  const { node: out, index, file } = assembleStack({ node, base, environment: run.meta?.environment?.name || null, captures: list, attachments })
-  await writeJson(join(run.dir, file), index)
+  // rotating samples: steps/<node>/rotation.<locale>.json, written by the walk
+  const rotation = {}
+  for (const l of run.meta?.scope?.locales || []) { const r = await readJson(join(run.dir, 'steps', node.id, `rotation.${l}.json`), null); if (r) rotation[l] = r }
+  const { node: out, index, file } = assembleStack({ node, base, environment: run.meta?.environment?.name || null, captures: list, attachments, rotation })
+  if (indexes) indexes.set(node.id, index); else await writeJson(join(run.dir, file), index)
   return out
 }