kaloko 0.18.0 → 0.19.0

package/src/lib/capture.mjs

@@ -89,7 +89,10 @@ export function captureOptions (scenario, node) {
     maskAuto: masks.some((m) => m.auto),
     annotate: o.annotate === true,
     // stack pages default to the first screen: hundreds of full-page shots would swamp storage
-    fullPage: o.full_page != null ? o.full_page !== false : node?.kind !== 'stack'
+    fullPage: o.full_page != null ? o.full_page !== false : node?.kind !== 'stack',
+    // screenshot: thumb (#150): only the card thumbnail is stored, as the step's image too (SEO crawls of thousands of
+    // pages need the page facts, not a sharp picture of each)
+    thumbOnly: o.screenshot === 'thumb'
   }
 }
 
@@ -97,12 +100,14 @@ import { criteriaHash } from './evaluate.mjs'
 import { consoleEntriesFor } from './console.mjs'
 import { redact, redactDeep } from './secrets.mjs'
 import { readFields } from './stack.mjs'
-import { checkLinks, checkHreflang, requester, walkCache, NETWORK_ASSERTS, isolatedRequest, cacheScope } from './linkcheck.mjs'
+import { checkLinks, checkHreflang, checkLlmsTxt, requester, walkCache, NETWORK_ASSERTS, isolatedRequest, anonymousRequest, cacheScope } from './linkcheck.mjs'
+import { checkNojsParity, botAgent, renderedSnapshot } from './crawler.mjs'
 import { keyboardChecks, needsKeyboard, KEYBOARD_ASSERTS } from './keyboard.mjs'
 import { repeatRegions, resolveIgnore } from './stability.mjs'
 import { applyVariant, engineOf, markEngineGaps, variantFields } from './dimensions.mjs'
 import { detailsFor, detailFacts, detailBox, detailName, DETAIL_PADDING } from './detail.mjs'
 import { liveValueChecks } from './assertions.mjs'
+import { localeChecks } from './seo-core.mjs'
 
 /** Probe arguments derived from the node's criteria. */
 export function probeArgs (node, maxChars, design = false) {
@@ -184,6 +189,8 @@ export async function captureStep ({ page, node, locale, viewports, run, console
       const ignore = await resolveIgnore(page, node, locale, viewport)
       // URL, computed styles and read-only expressions (src/lib/assertions.mjs), before the mask and the keyboard walk
       await liveValueChecks({ page, node, result, url: page.url(), tokens })
+      // the locale pack (#159): html lang, the own hreflang, the language of the text and prices against the locale
+      Object.assign(result.checks, localeChecks(node.criteria, { locale, meta: result.meta, url: page.url(), state: result.state }))
       // the console is read before the mask: the page's own reaction to masked text never counts as its error
       const url = page.url()
       const collected = consoleFile ? await readJsonl(consoleFile) : consoleEntriesFor(page)
@@ -252,7 +259,8 @@ export async function captureStep ({ page, node, locale, viewports, run, console
         // the extension follows the bytes (WebKit encodes PNG), so the service accepts the file
         if (enc.imageType === 'image/png') image = paths.step(stepId, locale, viewport, 'png')
         if (enc.thumbType === 'image/png') thumb = paths.step(stepId, locale, viewport, 'thumb.png')
-        await writeFile(image, enc.image)
+        if (opts.thumbOnly) image = thumb
+        else await writeFile(image, enc.image)
         await writeFile(thumb, enc.thumb)
       } catch (error) {
         // never store an unmasked image when masks were asked for
@@ -262,7 +270,7 @@ export async function captureStep ({ page, node, locale, viewports, run, console
       }
       // named details for guides (details:): from the same masked page as the screenshot, with the same mask boxes
       const details = item ? [] : await captureDetails({ page, node, locale, viewport, run, stepId, maskRects, scale: shotScale, engine, dpr })
-      // network checks (internal links, hreflang pairs) run once per page, outside the page, with a walk-wide cache
+      // network checks (internal links, hreflang pairs, llms.txt, the raw HTML) run once per page, outside the page, with a walk-wide cache
       const net = (node.criteria || []).filter((c) => c.check.type === 'deterministic' && NETWORK_ASSERTS.has(c.check.assert))
       if (net.length) {
         netResults ||= await networkChecks(page, net, url, { run, locale })
@@ -321,14 +329,14 @@ export async function captureStep ({ page, node, locale, viewports, run, console
         ...result,
         ...extra,
         http_status: extra.http_status ?? result.http_status ?? null,
-        capture: { format: String(image).split('.').pop(), scale: shotScale, ...(dpr !== 1 ? { dpr } : {}), masked: opts.mask.length, annotations, ...(opts.fullPage ? {} : { full_page: false }) },
+        capture: { format: String(image).split('.').pop(), scale: shotScale, ...(dpr !== 1 ? { dpr } : {}), masked: opts.mask.length, annotations, ...(opts.fullPage ? {} : { full_page: false }), ...(opts.thumbOnly && image === thumb ? { screenshot: 'thumb' } : {}) },
         // what was hidden and where (#64): the canvas marks these places, so a blank is not read as a bug
         ...(mask.active ? { pii: { mode: mask.mode, auto: mask.auto, selectors: mask.selectors.length, kinds: maskedPage?.counts || {}, rects: (maskedPage?.rects || []).slice(0, 200).map(({ x, y, w, h, kind }) => ({ x, y, w, h, kind })) } } : {}),
         ...(ignore.length ? { ignore } : {}),
         ...(unstable ? { unstable } : {}),
         ...(keyboard ? { keyboard } : {}),
         ...(details.length ? { details } : {}),
-        ...(item ? { item: { id: item.id, path: item.path, ...(item.title ? { title: item.title } : {}), ...(item.purpose ? { purpose: item.purpose } : {}) } } : {}),
+        ...(item ? { item: { id: item.id, path: item.path, ...(item.title ? { title: item.title } : {}), ...(item.purpose ? { purpose: item.purpose } : {}), ...(item.edit_url ? { edit_url: item.edit_url } : {}), ...(item.content_ref ? { content_ref: item.content_ref } : {}), ...(item.last_checked !== undefined ? { last_checked: item.last_checked } : {}) } } : {}),
         ...(fields ? { fields: { ...(await page.evaluate(readFields, fields).catch(() => ({}))), ...(fields.some((f) => (f.from || f.id) === 'http') ? Object.fromEntries(fields.filter((f) => (f.from || f.id) === 'http').map((f) => [f.id, extra.http_status ?? result.http_status ?? null])) : {}) } } : {}),
         files: { png: relative(run.dir, image), image: relative(run.dir, image), thumb: thumbWritten ? relative(run.dir, thumb) : null, html: relative(run.dir, html) }
       }
@@ -433,9 +441,11 @@ export async function captureCrop ({ page, node, locale, selector, title, run, p
 }
 
 /**
- * links_ok and hreflang_pairs for the current page: { [criterion id]: { ok, evidence } }. The requests go through an
+ * links_ok, hreflang_pairs, llms_txt and nojs_parity for the current page: { [criterion id]: { ok, evidence } }. The requests go through an
  * isolated request context (linkcheck.mjs), so the walker's cookies, language and session stay as they were.
  * links_ok follows links only on a readonly environment or scenario, or when the check says `follow: true`.
+ * nojs_parity with a bot user agent requests without cookies, as a crawler does, from one anonymous request context
+ * for the walk (crawler.mjs).
  */
 export async function networkChecks (page, criteria, url, { run = null, locale = '', factory = null } = {}) {
   const out = {}
@@ -449,10 +459,11 @@ export async function networkChecks (page, criteria, url, { run = null, locale =
   if (!todo.length) return out
   let request
   let get
+  const env = run?.meta?.environment?.name || ''
   try {
     request = await isolatedRequest(page.context(), { factory })
     const { cookies = [] } = await page.context().storageState().catch(() => ({}))
-    get = requester(request, walkCache, cacheScope({ environment: run?.meta?.environment?.name || '', locale, cookies }))
+    get = requester(request, walkCache, cacheScope({ environment: env, locale, cookies }))
   } catch (error) {
     for (const c of todo) out[c.id] = { ok: null, evidence: `network checks unavailable: ${error.message}` }
     await request?.dispose().catch(() => {})
@@ -465,10 +476,21 @@ export async function networkChecks (page, criteria, url, { run = null, locale =
       canonical: document.querySelector("link[rel='canonical'][href]")?.href || null,
       links: Object.fromEntries(sels.map((sel) => [sel, (() => { try { return [...document.querySelectorAll(sel)].map((a) => a.href).filter(Boolean) } catch { return [] } })()]))
     }), [...new Set(todo.filter((c) => c.check.assert === 'links_ok').map((c) => c.check.selector || 'a[href]'))]).catch(() => null)
+    if (doc && todo.some((c) => c.check.assert === 'nojs_parity')) doc.rendered = await page.evaluate(renderedSnapshot).catch(() => null)
     for (const c of todo) {
       if (!doc) { out[c.id] = { ok: null, evidence: 'page could not be read for network checks' }; continue }
       try {
-        out[c.id] = c.check.assert === 'links_ok'
+        if (c.check.assert === 'nojs_parity') {
+          if (!doc.rendered) { out[c.id] = { ok: null, evidence: 'the rendered page could not be read for nojs_parity' }; continue }
+          const agent = botAgent(c.check.user_agent)
+          // a bot comes without cookies; robots.txt once per origin, bot and walk
+          const anon = agent ? await anonymousRequest({ factory }) : null
+          out[c.id] = await checkNojsParity({ pageUrl: url, rendered: doc.rendered, agent, ratio: c.check.ratio ?? undefined, get: agent ? requester(anon, walkCache, cacheScope({ environment: env })) : get, session: get, robots: agent ? requester(anon, walkCache, 'site-root') : get })
+          continue
+        }
+        out[c.id] = c.check.assert === 'llms_txt'
+          ? await checkLlmsTxt({ pageUrl: url, get: requester(request, walkCache, 'site-root') }) // once per origin and walk, whatever the locale or session
+          : c.check.assert === 'links_ok'
           ? await checkLinks({ links: doc.links[c.check.selector || 'a[href]'] || [], pageUrl: url, get, max: c.check.max ?? undefined, exclude: c.check.pattern || null })
           : await checkHreflang({ alternates: doc.alternates, pageUrl: url, canonical: doc.canonical, lang: doc.lang, get, min: c.check.min ?? 2 })
       } catch (error) { out[c.id] = { ok: false, evidence: `check error: ${error.message}` } }

package/src/lib/config.mjs

@@ -5,6 +5,7 @@ import yaml from 'js-yaml'
 import { schemaErrors } from './schema.mjs'
 import { UserError, localized, globToRegExp } from './util.mjs'
 import { noteLegacy, legacyVariables } from './legacy.mjs'
+import { withEvaluatorProblems } from './local-evaluator.mjs'
 
 export const CONFIG_FILE = 'kaloko.config.yml'
 /** The file name before the rename to Kaloko; still found, so existing projects keep working. */
@@ -51,7 +52,7 @@ export async function loadConfig (start = process.cwd(), explicit) {
   await loadEnvFiles([join(dirname(file), '.env'), ...(userEnv ? [join(homedir(), '.config', 'kaloko', '.env'), join(homedir(), '.config', 'qawalk', '.env')] : [])])
   aliasLegacyEnv()
   const config = yaml.load(await readFile(file, 'utf8'))
-  const problems = schemaErrors('config.v1', config)
+  const problems = withEvaluatorProblems(config, schemaErrors('config.v1', config))
   if (problems.length) throw new UserError(`${file} is invalid:\n  - ${problems.join('\n  - ')}`)
   aliasConfigKeys(config, file)
   // projects started before the rename keep their run folder (it is in their .gitignore)
@@ -76,6 +77,19 @@ export function aliasConfigKeys (config, file = CONFIG_FILE) {
   return config
 }
 
+/**
+ * Whether the config asks every scenario to name its own `project:`: a product lists `projects:` (several projects work
+ * on it, so the product's project is no safe default), or a comment says so ("every scenario must name a project",
+ * "Každý scénář musí určit vlastní projekt"). Returns { reason, projects } or null.
+ */
+export function projectRequirement (config, text = '') {
+  const projects = [...new Set(Object.values(config?.products || {}).flatMap((p) => (Array.isArray(p?.projects) ? p.projects : [])))]
+  const comment = String(text).split(/\r?\n/).map((l) => l.match(/^\s*#(.*)$/)?.[1] || '').find((l) => /(?<!\p{L})(scenari|scénář|flow)/iu.test(l) && /(?<!\p{L})(project|projekt)/iu.test(l) && /(?<!\p{L})(every|each|must|required|každ|mus|povinn)/iu.test(l))
+  if (comment) return { reason: `the config says "${comment.trim()}"`, projects }
+  if (projects.length) return { reason: `the config lists several projects (products: projects: ${projects.join(', ')})`, projects }
+  return null
+}
+
 export function environment (config, name, vars = process.env) {
   const env = config.environments[name]
   if (!env) throw new UserError(`Unknown environment "${name}". Known: ${Object.keys(config.environments).join(', ')}`)

package/src/lib/crawler.mjs

@@ -0,0 +1,242 @@
+// nojs_parity: the page as a crawler that does not run JavaScript gets it. One GET of the page's raw HTML (optionally
+// with a bot user agent, anonymous like a crawler, with the walker's Accept-Language), read with a light parse in Node:
+// title, canonical, meta description, H1, JSON-LD types and the length of the body text. It is compared with the
+// rendered page the walk captured; key content that exists only after JavaScript fails the check. A bot user agent
+// respects robots.txt: a page the bot may not fetch is reported as blocked, not fetched. A bot the site, a WAF or a
+// gate turns away (401/403/429/503, a challenge page, a redirect to another origin) is not applicable: that is not a
+// rendering problem. Requests go through linkcheck.mjs (isolated; the walk-wide cache keeps the parsed snapshot only).
+
+/** Bots by name (`user_agent: gptbot`): the robots.txt product token and the user agent string they send. */
+export const BOTS = {
+  gptbot: { token: 'GPTBot', ua: 'Mozilla/5.0 AppleWebKit/537.36 (KHTML, like Gecko; compatible; GPTBot/1.2; +https://openai.com/gptbot)' },
+  'oai-searchbot': { token: 'OAI-SearchBot', ua: 'Mozilla/5.0 AppleWebKit/537.36 (KHTML, like Gecko; compatible; OAI-SearchBot/1.0; +https://openai.com/searchbot)' },
+  claudebot: { token: 'ClaudeBot', ua: 'Mozilla/5.0 AppleWebKit/537.36 (KHTML, like Gecko; compatible; ClaudeBot/1.0; +claudebot@anthropic.com)' },
+  perplexitybot: { token: 'PerplexityBot', ua: 'Mozilla/5.0 AppleWebKit/537.36 (KHTML, like Gecko; compatible; PerplexityBot/1.0; +https://perplexity.ai/perplexitybot)' },
+  googlebot: { token: 'Googlebot', ua: 'Mozilla/5.0 (compatible; Googlebot/2.1; +http://www.google.com/bot.html)' },
+  bingbot: { token: 'bingbot', ua: 'Mozilla/5.0 (compatible; bingbot/2.0; +http://www.bing.com/bingbot.htm)' }
+}
+/** Body text of the raw HTML below this share of the rendered one fails (`ratio:` on the check). */
+export const NOJS_RATIO = 0.3
+// below this much rendered text the share says little (a login screen, a gallery): only the other signals count
+const MIN_TEXT = 200
+// answers that mean "not you" rather than "this is the page": the site, a WAF or a gate turned the request away
+const BLOCKED = new Set([401, 403, 407, 429, 503])
+const CHALLENGE = /cf-chl-|cf_chl_opt|\/cdn-cgi\/challenge-platform|<title>\s*(?:just a moment|attention required|access denied|checking your browser|ddos-guard)|_incapsula_resource|px-captcha|captcha-delivery\.com|awswaf|sucuri_cloudproxy/i
+
+/** A named bot or a user agent string → { label, token, ua }; nothing → null (the walker's own browser). */
+export function botAgent (spec) {
+  const s = String(spec || '').trim()
+  if (!s) return null
+  const known = BOTS[s.toLowerCase()]
+  if (known) return { label: known.token, ...known }
+  const token = s.match(/compatible;\s*([A-Za-z][\w.-]*?)(?:\/|;|\)|\s|$)/)?.[1] || s.match(/^([A-Za-z][\w.-]*?)(?:\/|\s|$)/)?.[1] || s.slice(0, 40)
+  return { label: token, token, ua: s }
+}
+
+// robots.txt paths compare percent-encoded (RFC 9309): raw UTF-8 in a rule matches the encoded path, %xx in any case
+const encodePath = (p) => String(p).replace(/%[0-9a-f]{2}/gi, (m) => m.toUpperCase()).replace(/[^\x21-\x7e]/gu, (c) => encodeURIComponent(c))
+
+/**
+ * robots.txt for one product token (RFC 9309): the groups naming the token, else `*`; the longest matching rule wins,
+ * Allow on a tie; `*` and a closing `$` in rules. → { allowed, rule } (rule: the deciding line, or null).
+ */
+export function robotsAllows (text, token, path) {
+  const groups = []; let cur = null; let agents = false
+  for (const raw of String(text || '').split(/\r?\n/)) {
+    const m = raw.replace(/#.*/, '').trim().match(/^([a-z-]+)\s*:\s*(.*)$/i)
+    if (!m) continue
+    const key = m[1].toLowerCase(); const val = m[2].trim()
+    if (key === 'user-agent') { if (!agents) groups.push(cur = { agents: [], rules: [] }); cur.agents.push(val.toLowerCase()); agents = true; continue }
+    agents = false
+    if (cur && (key === 'allow' || key === 'disallow') && val) cur.rules.push({ allow: key === 'allow', path: encodePath(val), line: `${key === 'allow' ? 'Allow' : 'Disallow'}: ${val}` })
+  }
+  const t = String(token || '').toLowerCase()
+  const own = groups.filter((g) => g.agents.includes(t))
+  const rules = (own.length ? own : groups.filter((g) => g.agents.includes('*'))).flatMap((g) => g.rules)
+  const target = encodePath(path)
+  let best = null
+  for (const r of rules) {
+    const re = new RegExp('^' + r.path.replace(/\$$/, '').replace(/[.+?^${}()|[\]\\]/g, '\\$&').replace(/\*/g, '.*') + (r.path.endsWith('$') ? '$' : ''))
+    if (!re.test(target)) continue
+    if (!best || r.path.length > best.path.length || (r.path.length === best.path.length && r.allow)) best = r
+  }
+  return { allowed: !best || best.allow, rule: best?.line ?? null }
+}
+
+/** schema.org @type values of JSON-LD texts (graphs, lists, nested values), in order; blocks that do not parse are skipped. */
+export function jsonldTypes (texts = []) {
+  const types = []
+  const walk = (x, depth) => {
+    if (!x || typeof x !== 'object' || depth > 6) return
+    if (Array.isArray(x)) { for (const y of x) walk(y, depth + 1); return }
+    for (const t of [].concat(x['@type'] || [])) { const n = typeof t === 'string' ? t.replace(/^https?:\/\/schema\.org\//, '') : null; if (n && !types.includes(n)) types.push(n) }
+    for (const [k, v] of Object.entries(x)) if (k !== '@context' && v && typeof v === 'object') walk(v, depth + 1)
+  }
+  for (const t of texts) { try { walk(JSON.parse(t), 0) } catch {} }
+  return types
+}
+
+// HTML named entities common in text (titles with Yoast separators, typography, currency, Latin letters); letters
+// with diacritics (&eacute; &ccaron; &Scaron; &uring;…) are composed from the name below
+const NAMED = {
+  amp: '&', lt: '<', gt: '>', quot: '"', apos: "'", nbsp: ' ', ensp: ' ', emsp: ' ', thinsp: ' ', shy: '', zwnj: '', zwj: '', lrm: '', rlm: '',
+  ndash: '–', mdash: '—', minus: '−', hellip: '…', middot: '·', bull: '•', sdot: '⋅', lsquo: '‘', rsquo: '’', sbquo: '‚', ldquo: '“', rdquo: '”', bdquo: '„',
+  laquo: '«', raquo: '»', lsaquo: '‹', rsaquo: '›', prime: '′', Prime: '″', dagger: '†', Dagger: '‡', permil: '‰', sect: '§', para: '¶', verbar: '|', vert: '|',
+  copy: '©', reg: '®', trade: '™', deg: '°', plusmn: '±', times: '×', divide: '÷', frasl: '⁄', micro: 'µ', iexcl: '¡', iquest: '¿', ordf: 'ª', ordm: 'º',
+  sup1: '¹', sup2: '²', sup3: '³', frac12: '½', frac14: '¼', frac34: '¾', euro: '€', pound: '£', yen: '¥', cent: '¢', curren: '¤', dollar: '$', percnt: '%',
+  larr: '←', rarr: '→', uarr: '↑', darr: '↓', harr: '↔', rArr: '⇒', lArr: '⇐', check: '✓', star: '☆', starf: '★', hearts: '♥', loz: '◊',
+  szlig: 'ß', aelig: 'æ', AElig: 'Æ', oslash: 'ø', Oslash: 'Ø', eth: 'ð', ETH: 'Ð', thorn: 'þ', THORN: 'Þ', oelig: 'œ', OElig: 'Œ', dstrok: 'đ', Dstrok: 'Đ',
+  lstrok: 'ł', Lstrok: 'Ł', imath: 'ı', excl: '!', num: '#', lpar: '(', rpar: ')', ast: '*', plus: '+', comma: ',', period: '.', sol: '/', colon: ':', semi: ';',
+  equals: '=', quest: '?', commat: '@', lsqb: '[', rsqb: ']', bsol: '\\', lowbar: '_', grave: '`', lcub: '{', rcub: '}', tilde: '˜', circ: 'ˆ', NewLine: '\n', Tab: '\t'
+}
+const MARKS = { acute: '́', grave: '̀', circ: '̂', uml: '̈', tilde: '̃', ring: '̊', cedil: '̧', caron: '̌', macr: '̄', ogon: '̨', dot: '̇', breve: '̆', dblac: '̋' }
+// entities a browser also reads without the closing semicolon
+const LEGACY = new Set(['amp', 'lt', 'gt', 'quot', 'nbsp', 'copy', 'reg', 'AMP', 'LT', 'GT', 'QUOT'])
+const named = (name) => {
+  if (NAMED[name] != null) return NAMED[name]
+  if (/^(AMP|LT|GT|QUOT)$/.test(name)) return NAMED[name.toLowerCase()]
+  const m = name.match(/^([a-zA-Z])(acute|grave|circ|uml|tilde|ring|cedil|caron|macr|ogon|dot|breve|dblac)$/)
+  const c = m && (m[1] + MARKS[m[2]]).normalize('NFC')
+  return c && c.length === 1 ? c : null
+}
+/** Decodes HTML character references: named ones common in text, decimal and hexadecimal. */
+export function decodeEntities (s) {
+  return String(s ?? '').replace(/&(#[xX][0-9a-fA-F]+|#\d+|[a-zA-Z][a-zA-Z0-9]*)(;?)/g, (m, e, semi) => {
+    if (e[0] === '#') { const n = e[1] === 'x' || e[1] === 'X' ? parseInt(e.slice(2), 16) : +e.slice(1); try { return n > 0 && n <= 0x10ffff ? String.fromCodePoint(n === 160 ? 32 : n) : m } catch { return m } }
+    if (!semi && !LEGACY.has(e)) return m
+    return named(e) ?? m
+  })
+}
+/** Text as people read it: no invisible characters, every run of whitespace (no-break spaces too) one space. */
+export const normText = (s) => String(s ?? '').replace(/[­​-‍⁠]/g, '').replace(/\s+/g, ' ').trim()
+const textOf = (html) => normText(decodeEntities(String(html).replace(/<[^>]*>/g, ' ')))
+const attrOf = (tag, name) => tag.match(new RegExp(`\\s${name}\\s*=\\s*(?:"([^"]*)"|'([^']*)'|([^\\s>]+))`, 'i'))?.slice(1).find((v) => v != null)
+const SKIP = /<(script|style|noscript|template|svg)\b[\s\S]*?<\/\1\s*>/gi
+// an element cut off by the size limit (or never closed) swallows the rest of the document
+const UNCLOSED = /<(script|style|noscript|template|svg)\b[\s\S]*$/i
+
+/**
+ * What a crawler without JavaScript reads in raw HTML: { title, canonical, description, h1, lang, types, text,
+ * challenge, truncated }. text is the length of the body text without scripts, styles, noscript, templates and SVG
+ * (hidden elements count, as they do in the rendered page: streamed parts such as Next.js `<div hidden id="S:0">`).
+ */
+export function rawSnapshot (html, base, truncated = false) {
+  const doc = String(html || '').replace(/<!--[\s\S]*?(?:-->|$)/g, '')
+  const jsonld = [...doc.matchAll(/<script\b([^>]*)>([\s\S]*?)<\/script\s*>/gi)].filter((m) => /application\/ld\+json/i.test(attrOf(m[1], 'type') || '')).map((m) => m[2])
+  const bare = doc.replace(SKIP, ' ').replace(UNCLOSED, ' ')
+  const href = (bare.match(/<base\b[^>]*>/i) || [])[0]
+  try { if (href && attrOf(href, 'href')) base = new URL(decodeEntities(attrOf(href, 'href')), base).href } catch {}
+  const title = textOf(bare.match(/<title\b[^>]*>([\s\S]*?)<\/title\s*>/i)?.[1] || '') || null
+  let canonical = null; let description = null
+  for (const tag of bare.match(/<link\b[^>]*>/gi) || []) if (!canonical && /(^|\s)canonical(\s|$)/i.test(attrOf(tag, 'rel') || '') && attrOf(tag, 'href')) { try { canonical = new URL(decodeEntities(attrOf(tag, 'href')), base).href } catch {} }
+  for (const tag of bare.match(/<meta\b[^>]*>/gi) || []) if (description == null && /^description$/i.test(attrOf(tag, 'name') || '')) description = textOf(attrOf(tag, 'content') || '') || null
+  const h1 = textOf(bare.match(/<h1\b[^>]*>([\s\S]*?)<\/h1\s*>/i)?.[1] || '') || null
+  const lang = attrOf(bare.match(/<html\b[^>]*>/i)?.[0] || '', 'lang') || null
+  const noHead = bare.replace(/<head\b[\s\S]*?<\/head\s*>/i, ' ')
+  const body = noHead.match(/<body\b[^>]*>([\s\S]*)/i)?.[1] ?? noHead.replace(/<title\b[\s\S]*?<\/title\s*>/gi, ' ')
+  return { title, canonical, description, h1, lang, types: jsonldTypes(jsonld), text: textOf(body).length, challenge: CHALLENGE.test(doc.slice(0, 30000)), truncated: !!truncated }
+}
+
+/** The same snapshot of the rendered page, plus the browser's user agent and languages; runs in the page (self-contained). */
+export function renderedSnapshot () {
+  const norm = (s) => String(s ?? '').replace(/[­​-‍⁠]/g, '').replace(/\s+/g, ' ').trim()
+  const body = document.body?.cloneNode(true)
+  for (const el of body ? body.querySelectorAll('script, style, noscript, template, svg') : []) el.remove()
+  return {
+    title: norm(document.title) || null,
+    canonical: document.querySelector("link[rel~='canonical'][href]")?.href || null,
+    description: norm(document.querySelector("meta[name='description' i]")?.getAttribute('content')) || null,
+    h1: norm(document.querySelector('h1')?.textContent) || null,
+    lang: document.documentElement.getAttribute('lang') || null,
+    jsonld: [...document.querySelectorAll('script[type="application/ld+json"]')].map((s) => s.textContent),
+    text: body ? norm(body.textContent).length : 0,
+    ua: navigator.userAgent,
+    languages: [...(navigator.languages || [])]
+  }
+}
+
+/** Accept-Language for the browser's languages: cs-CZ,cs;q=0.9,en;q=0.8. */
+export function acceptLanguage (languages = []) {
+  return languages.slice(0, 6).map((l, i) => (i ? `${l};q=${(1 - i / 10).toFixed(1)}` : l)).join(',') || null
+}
+
+const cut = (s, n = 60) => (s.length > n ? s.slice(0, n - 1) + '…' : s)
+const sameUrl = (a, b) => { try { const x = new URL(a); const y = new URL(b); x.hash = y.hash = ''; return x.href.replace(/\/$/, '') === y.href.replace(/\/$/, '') } catch { return a === b } }
+const primary = (l) => String(l || '').toLowerCase().split(/[-_]/)[0]
+
+/**
+ * Raw against rendered: { ok, problems, found }. Fails when the rendered page has a title, canonical, meta description,
+ * H1 or JSON-LD type the raw HTML lacks, when title or canonical differ, or when the raw body text is below `ratio`
+ * of the rendered one (rendered text of 200 characters or more). A raw body cut at the size limit does not judge JSON-LD.
+ */
+export function compareSnapshots (raw, rendered, { ratio = NOJS_RATIO } = {}) {
+  const problems = []; const found = []
+  const rt = normText(rendered.title); const wt = normText(raw.title)
+  if (rt) {
+    const other = raw.lang && rendered.lang && primary(raw.lang) !== primary(rendered.lang) ? ` (the raw HTML is lang="${raw.lang}", the walked page lang="${rendered.lang}")` : ''
+    if (!wt) problems.push(`no title (rendered: "${cut(rt)}")`)
+    else if (wt !== rt) problems.push(`title "${cut(wt)}", rendered "${cut(rt)}"${other}`)
+    else found.push('title')
+  }
+  if (rendered.canonical) {
+    if (!raw.canonical) problems.push(`no canonical (rendered: ${rendered.canonical})`)
+    else if (!sameUrl(raw.canonical, rendered.canonical)) problems.push(`canonical ${raw.canonical}, rendered ${rendered.canonical}`)
+    else found.push('canonical')
+  }
+  if (normText(rendered.description)) { if (normText(raw.description)) found.push('meta description'); else problems.push('no meta description') }
+  if (normText(rendered.h1)) { if (normText(raw.h1)) found.push('H1'); else problems.push(`no H1 (rendered: "${cut(normText(rendered.h1))}")`) }
+  const types = rendered.types || []
+  const late = types.filter((t) => !raw.types.includes(t))
+  if (late.length && raw.truncated) found.push(`JSON-LD not compared: the raw HTML was cut at 500,000 characters (${late.slice(0, 6).join(', ')} not before the cut)`)
+  else if (late.length) problems.push(`JSON-LD ${late.slice(0, 6).join(', ')} only after JavaScript`)
+  else if (types.length) found.push(`JSON-LD (${types.slice(0, 6).join(', ')})`)
+  const share = rendered.text ? raw.text / rendered.text : 1
+  const text = `body text ${raw.text} of ${rendered.text} characters (${Math.round(Math.min(share, 9.99) * 100)} %)`
+  if (rendered.text >= MIN_TEXT && share < ratio) problems.push(`${text}, at least ${Math.round(ratio * 100)} % expected`)
+  else found.push(text)
+  return { ok: !problems.length, problems, found }
+}
+
+const turnedAway = (r, pageUrl) => {
+  if (BLOCKED.has(r.status)) return `HTTP ${r.status}`
+  try { if (r.url && new URL(r.url).origin !== new URL(pageUrl).origin) return `a redirect to ${new URL(r.url).origin} (a sign-in or a gate)` } catch {}
+  if (r.status === 200 && r.parsed?.challenge) return 'a challenge page instead of the page'
+  return null
+}
+
+/**
+ * nojs_parity: { pageUrl, rendered (renderedSnapshot), agent (botAgent or null), get(url, opts) (anonymous for a bot),
+ * session(url, opts) (the walker's cookies, to tell a gate from a bot block), robots(url, opts), ratio } → { ok, evidence, na? }.
+ * One request for the page (the parsed snapshot is cached for the walk); with a bot, robots.txt once per origin first.
+ */
+export async function checkNojsParity ({ pageUrl, rendered, agent = null, get, session = null, robots = get, ratio = NOJS_RATIO }) {
+  const who = agent ? agent.label : 'the browser'
+  const ua = agent?.ua || rendered?.ua || null
+  const lang = acceptLanguage(rendered?.languages)
+  const u = new URL(pageUrl)
+  if (agent) {
+    const r = await robots(new URL('/robots.txt', pageUrl).href, { html: true, ua })
+    if (!r || r.error || r.status >= 500) return { ok: null, na: true, evidence: `robots.txt ${r?.error ? `did not answer (${r.error})` : `answered HTTP ${r?.status}`}; crawlers then fetch nothing, so ${agent.token} was not sent` }
+    if (r.status === 200) {
+      const verdict = robotsAllows(r.html, agent.token, u.pathname + u.search)
+      if (!verdict.allowed) return { ok: null, na: true, evidence: `blocked by robots.txt for ${agent.token} (${verdict.rule}); not fetched` }
+    }
+  }
+  const r = await get(pageUrl, { ua, lang, parse: rawSnapshot })
+  if (!r || r.error) return { ok: null, na: true, evidence: `${who} without JavaScript got no answer (${r?.error || 'no response'}); not a rendering problem` }
+  const away = turnedAway(r, pageUrl)
+  if (away) {
+    // who turned it away: the bot's user agent, the missing cookies (a sign-in or a cookie gate), or everyone
+    const ok = (x) => x && !x.error && x.status === 200 && !turnedAway(x, pageUrl)
+    let why = 'blocked by the site, a WAF or a gate'
+    if (agent && ok(await get(pageUrl, { ua: rendered?.ua, lang, parse: rawSnapshot }))) why = `the ${agent.token} user agent is blocked by the site or a WAF (the browser user agent gets the page)`
+    else if (agent && session && ok(await session(pageUrl, { ua: rendered?.ua, lang, parse: rawSnapshot }))) why = 'the page needs the walk\'s cookies (a sign-in or a cookie gate), which a crawler does not have'
+    return { ok: null, na: true, evidence: `${who} without JavaScript gets ${away}: ${why}; not a rendering problem` }
+  }
+  if (r.status !== 200) return { ok: false, evidence: `${who} without JavaScript gets HTTP ${r.status}${r.url && !sameUrl(r.url, pageUrl) ? ` (from ${r.url})` : ''}` }
+  const { ok, problems, found } = compareSnapshots(r.parsed, { ...rendered, types: jsonldTypes(rendered?.jsonld || []) }, { ratio })
+  const moved = r.url && !sameUrl(r.url, pageUrl) ? ` (redirected to ${r.url})` : ''
+  return ok
+    ? { ok: true, evidence: `${who} without JavaScript${moved}: ${found.join(', ')}` }
+    : { ok: false, evidence: `${who} without JavaScript${moved}: ${problems.join('; ')}${found.length ? `; present: ${found.join(', ')}` : ''}` }
+}

package/src/lib/draft.mjs

@@ -47,6 +47,8 @@ export function draftScenario (pages, { id, title } = {}) {
   nodes.push({
     id: nf, kind: 'screen', title: 'Page not found', path: '/no-such-page-{random}',
     purpose: 'A visitor who follows a broken link gets a real 404 with a way back to the site.',
+    // expect_status marks it as an error page: a soft 404 (HTTP 200) fails HC601 instead of being not applicable
+    expect_status: 404,
     packs: ['errors'],
     execution: { strategy: 'playwright', script: 'steps/open.mjs', instructions: 'Open an address that does not exist ({random} makes it unique).' },
     criteria: []