kaloko 0.4.11 → 0.5.0

package/src/lib/assertions.mjs

@@ -0,0 +1,300 @@
+// Value assertions: the page URL after the step (url_matches, url_equals), computed CSS of elements
+// (style_equals, style_matches) and a read-only expression in the page (js_equals). They exist so a step never has
+// to compute a value itself and leave a data-qa marker for a criterion to find.
+//
+// The page records the raw values (the probe for styles, page.evaluate for expressions, page.url() for the URL);
+// the decision is made here, in Node, from those records. The same decision runs at capture, on
+// `kaloko evaluate` (token values from the run's token set) and on `kaloko evaluate --recheck`, which reuses the
+// recorded values when the criterion still reads the same thing.
+import { normColor, flattenTokens } from './tokens.mjs'
+
+export const URL_ASSERTS = new Set(['url_matches', 'url_equals'])
+export const STYLE_ASSERTS = new Set(['style_equals', 'style_matches'])
+export const VALUE_ASSERTS = new Set([...URL_ASSERTS, ...STYLE_ASSERTS, 'js_equals'])
+export const isColorProperty = (prop) => /(^|-)color$|^fill$|^stroke$/.test(String(prop || '').trim().toLowerCase())
+
+const regexOf = (check) => check.pattern != null ? new RegExp(check.pattern, 'u' + [...new Set(String(check.flags || '').replace(/[^ims]/g, ''))].join('')) : null
+const OPS = { '<': (a, b) => a < b, '<=': (a, b) => a <= b, '>': (a, b) => a > b, '>=': (a, b) => a >= b, '==': (a, b) => a === b }
+
+// ---- URL ----
+
+/** The part of a URL a check reads: href (default), path, query, hash, host or origin. */
+export function urlPart (url, part = 'href') {
+  let u
+  try { u = new URL(url) } catch { return url == null ? null : String(url) }
+  switch (part) {
+    case 'path': return u.pathname
+    case 'query': return u.search
+    case 'hash': return u.hash
+    case 'host': return u.host
+    case 'origin': return u.origin
+    default: return u.href
+  }
+}
+
+/** url_matches / url_equals against the page URL after the step. */
+export function urlCheck (check, url) {
+  if (!url || url === 'about:blank') return { ok: null, evidence: 'the capture has no page URL' }
+  if (check.assert === 'url_matches') {
+    const part = check.part || 'href'
+    const v = urlPart(url, part)
+    const ok = regexOf(check).test(v)
+    return { ok, evidence: `URL ${url}${part !== 'href' ? ` (${part} ${v || '""'})` : ''} ${ok ? 'matches' : 'does not match'} /${check.pattern}/` }
+  }
+  // url_equals: an absolute value compares the whole URL, a path compares path + query (+ hash when given)
+  const want = String(check.value ?? '')
+  let u
+  try { u = new URL(url) } catch { return { ok: false, evidence: `URL ${url} cannot be read` } }
+  const absolute = /^[a-z][\w+.-]*:\/\//i.test(want)
+  const strip = (s) => (check.ignore_query ? s.replace(/[?#].*$/, '') : s)
+  const actual = absolute ? strip(u.href) : strip(u.pathname + u.search + (want.includes('#') ? u.hash : ''))
+  const ok = actual === strip(want)
+  return { ok, evidence: `URL ${url}${absolute ? '' : ` (path ${actual})`} ${ok ? '=' : '≠'} ${want}${check.ignore_query ? ' (query ignored)' : ''}` }
+}
+
+// ---- computed styles ----
+
+const fmt = (n) => String(Math.round(n * 1000) / 1000)
+/** A CSS value as a list of numbers in px (px, rem, unitless), or null when any part is not a length or number. */
+export function lengths (value, rootPx = 16) {
+  const parts = String(value ?? '').trim().split(/\s+/).filter(Boolean)
+  if (!parts.length) return null
+  const out = []
+  for (const p of parts) {
+    const m = p.match(/^(-?(?:\d+\.?\d*|\.\d+)(?:e-?\d+)?)(px|rem)?$/i)
+    if (!m) return null
+    out.push(Number(m[1]) * (m[2]?.toLowerCase() === 'rem' ? rootPx : 1))
+  }
+  return out
+}
+/** Canonical colour: #rrggbb, or #rrggbbaa with transparency (the token module's form). */
+export const canonColor = (v) => normColor(typeof v === 'string' ? v.trim() : v)
+const plain = (v) => String(v ?? '').trim().replace(/\s+/g, ' ').replace(/["']/g, '').toLowerCase()
+
+/**
+ * Compares one computed value with the expected one. Colours compare in canonical hex, lengths in px (rem uses
+ * the page's root font size) within `tolerance`, the rest as text without quotes and case.
+ * Returns { ok, how } where how names the canonical form used.
+ */
+export function compareStyle ({ property, actual, actualColor = null, expected, expectedColor = null, rootPx = 16, tolerance = 0, op = null }) {
+  if (isColorProperty(property)) {
+    const a = canonColor(actualColor || actual); const e = canonColor(expectedColor || expected)
+    if (a && e) return { ok: a === e, how: 'color', a, e }
+  }
+  const a = lengths(actual, rootPx); const e = lengths(expected, rootPx)
+  if (a && e) {
+    if (op && op !== '==') return { ok: a.length === 1 && e.length === 1 && OPS[op](a[0], e[0]), how: 'length', a, e }
+    return { ok: a.length === e.length && a.every((x, i) => Math.abs(x - e[i]) <= (tolerance || 0) + 1e-6), how: 'length', a, e }
+  }
+  if (op && op !== '==') return { ok: false, how: 'text', note: 'not a number' }
+  return { ok: plain(actual) === plain(expected), how: 'text' }
+}
+
+/** The CSS value of a DTCG token by dotted name, from a token set (or resolver) in a mode, or null. */
+export function tokenValue (tokens, name, mode = null) {
+  if (!tokens || !name) return null
+  try {
+    const t = flattenTokens(tokens, mode).find((x) => x.name === name)
+    return t && (typeof t.value === 'string' || typeof t.value === 'number') ? String(t.value) : null
+  } catch { return null }
+}
+
+/**
+ * style_equals / style_matches from the computed values the probe recorded:
+ * rec = { selector, property, values: [{ el, value, color? }], root_px, token_var?, expected_color? }.
+ */
+export function styleCheck (check, rec, { tokens = null, mode = null } = {}) {
+  if (!rec || !Array.isArray(rec.values)) return { ok: null, evidence: 'no computed values recorded; capture the step again' }
+  const prop = check.property
+  if (!rec.values.length) return { ok: false, evidence: `no ${check.selector}` }
+  const show = (x) => x.value || '""'
+  const all = check.all === true
+  let results
+  let label
+  if (check.assert === 'style_matches') {
+    const re = regexOf(check)
+    results = rec.values.map((x) => ({ x, ok: re.test(x.value) }))
+    label = `/${check.pattern}/`
+  } else {
+    let expected = check.value != null ? String(check.value) : null
+    let expectedColor = check.value != null ? rec.expected_color || null : null
+    if (check.token) {
+      const fromSet = tokenValue(tokens, check.token, mode)
+      const fromVar = rec.token_var?.value || null
+      expected = fromSet ?? fromVar
+      expectedColor = fromSet ? null : rec.token_var?.color || null
+      if (expected == null) return { ok: null, evidence: `token ${check.token} is not in the token set${rec.token_var === null ? ` and the page has no --${check.token.replace(/\./g, '-')}` : ''}; the check runs on kaloko evaluate with the project's tokens` }
+    }
+    if (expected == null) return { ok: null, evidence: `${check.assert} needs value or token` }
+    const shownExpected = isColorProperty(prop) && canonColor(expectedColor || expected) ? canonColor(expectedColor || expected) : expected
+    label = check.token ? `${check.token} (${shownExpected})` : (shownExpected !== expected ? `${expected} (${shownExpected})` : expected)
+    results = rec.values.map((x) => ({ x, ...compareStyle({ property: prop, actual: x.value, actualColor: x.color, expected, expectedColor, rootPx: rec.root_px || 16, tolerance: Number(check.tolerance) || 0, op: check.op || null }) }))
+  }
+  const passed = results.filter((r) => r.ok)
+  const ok = all ? passed.length === results.length : passed.length > 0
+  const rel = check.assert === 'style_matches' ? ['matches', 'does not match'] : check.op && check.op !== '==' ? [check.op, `not ${check.op}`] : ['=', '≠']
+  const tol = check.tolerance ? ` ± ${fmt(Number(check.tolerance))}px` : ''
+  const recorded = rec.recorded ? ' (computed values recorded at capture)' : ''
+  if (results.length === 1) {
+    const r = results[0]
+    return { ok, evidence: `${prop} is ${show(r.x)} ${r.ok ? rel[0] : rel[1]} ${label}${tol}${recorded}` }
+  }
+  const shown = all ? (results.find((r) => !r.ok) || results[0]) : (results.find((r) => r.ok) || results[0])
+  return { ok, evidence: `${prop}: ${passed.length} of ${results.length} × ${check.selector} ${rel[0]} ${label}${tol}${all ? ' (all must)' : ''}; ${shown.x.el} is ${show(shown.x)}${recorded}` }
+}
+
+// ---- read-only expressions ----
+
+const CALLS = new Set(('querySelector querySelectorAll getElementById getElementsByClassName getElementsByTagName getAttribute hasAttribute ' +
+  'getItem key getComputedStyle getPropertyValue getBoundingClientRect checkVisibility matches closest contains includes startsWith ' +
+  'endsWith indexOf lastIndexOf trim trimStart trimEnd toLowerCase toUpperCase slice substring split join at some every filter map ' +
+  'find findIndex findLast reduce flat flatMap keys values entries from isArray parse stringify has get getAll toString toFixed ' +
+  'Number String Boolean parseInt parseFloat isNaN isFinite test match item max min round abs floor ceil ' +
+  'getEntriesByType getEntriesByName URL URLSearchParams').split(/\s+/))
+const FORBIDDEN = /\b(delete|new|function|class|import|await|async|yield|eval|Function|fetch|XMLHttpRequest|WebSocket|sendBeacon|setTimeout|setInterval|requestAnimationFrame|while|for|do|with|debugger|constructor|__proto__|prototype|defineProperty|assign|document\.write|open|close|click|submit|focus|blur|dispatchEvent|remove|append|prepend|insertAdjacentHTML|innerHTML|outerHTML|setItem|removeItem|clear|setAttribute|removeAttribute|toggleAttribute|push|pop|shift|unshift|splice|sort|reverse|fill|copyWithin|reload|replaceState|pushState|go|back|forward)\s*\(/
+
+/**
+ * Why an expression is not allowed for js_equals, or null. The guard keeps expressions read-only by convention:
+ * one expression, no assignment, no ++/--, no templates, and only calls from a list of reading methods. It is not a
+ * sandbox: a getter in the page can still do anything, so read data the page exposes, nothing that acts.
+ */
+export function jsGuard (expression) {
+  const src = String(expression ?? '').trim()
+  if (!src) return 'empty expression'
+  if (src.length > 500) return 'longer than 500 characters'
+  if (src.includes('`')) return 'template strings are not allowed; use quotes'
+  // `new URL(…)` and `new URLSearchParams(…)` read a URL; every other `new` is refused below
+  const code = src.replace(/'(?:\\.|[^'\\])*'|"(?:\\.|[^"\\])*"/g, '""').replace(/\bnew\s+(URL|URLSearchParams)\s*\(/g, '$1(')
+  if (/['"]/.test(code.replace(/""/g, ''))) return 'unclosed string'
+  if (/;/.test(code)) return 'one expression only (no ;)'
+  if (/(^|[^=!<>])=(?![=>])/.test(code)) return 'assignment is not allowed'
+  if (/\+\+|--/.test(code)) return '++ and -- are not allowed'
+  const bad = code.match(FORBIDDEN)
+  if (bad) return `${bad[1]}() is not allowed`
+  if (/\b(delete|new|function|class|import|await|async|yield|eval|while|for|do|with|debugger|constructor|__proto__|prototype)\b/.test(code)) return `${code.match(/\b(delete|new|function|class|import|await|async|yield|eval|while|for|do|with|debugger|constructor|__proto__|prototype)\b/)[1]} is not allowed`
+  if (/[)\]]\s*(\?\.)?\s*\(/.test(code)) return 'calling the result of a call or index is not allowed'
+  for (const m of code.matchAll(/([A-Za-z_$][\w$]*)\s*(?:\?\.)?\s*\(/g)) if (!CALLS.has(m[1])) return `${m[1]}() is not on the list of reading calls`
+  return null
+}
+
+export const JS_TIMEOUT = 2000
+
+/** Evaluates the js_equals expressions of `checks` in the page: { [id]: { expression, result | error } }. */
+export async function jsValues (page, checks, timeout = JS_TIMEOUT) {
+  const out = {}
+  for (const { id, check } of checks) {
+    const expression = String(check.expression ?? '')
+    const problem = jsGuard(expression)
+    if (problem) { out[id] = { expression, error: `not allowed: ${problem}`, refused: true }; continue }
+    // Playwright evaluates a string as an expression through the debugger protocol, so a page's CSP does not
+    // block it; promises are awaited up to the timeout
+    const wrapped = `(async () => {
+      const timer = new Promise((_, no) => setTimeout(() => no(new Error('did not finish within ${timeout} ms')), ${timeout}))
+      const v = await Promise.race([Promise.resolve().then(() => (${expression})), timer])
+      const type = v === null ? 'null' : Array.isArray(v) ? 'array' : v instanceof Element ? 'element' : typeof v
+      let value = v
+      if (type === 'element') value = v.tagName.toLowerCase() + (v.id ? '#' + v.id : '')
+      else if (type === 'undefined') value = null
+      else if (type === 'object' || type === 'array') { try { value = JSON.parse(JSON.stringify(v)) } catch { value = String(v) } }
+      else if (type === 'function' || type === 'symbol' || type === 'bigint') value = String(v)
+      return { type, value }
+    })()`
+    try {
+      let timer
+      const result = await Promise.race([
+        page.evaluate(wrapped),
+        new Promise((_, no) => { timer = setTimeout(() => no(new Error(`did not finish within ${timeout} ms`)), timeout + 1000) })
+      ]).finally(() => clearTimeout(timer))
+      const text = JSON.stringify(result.value)
+      out[id] = { expression, result: text && text.length > 2000 ? { type: result.type, value: `${text.slice(0, 2000)}…` } : result }
+    } catch (error) {
+      out[id] = { expression, error: String(error.message || error).split('\n')[0].replace(/^page\.evaluate: /, '').slice(0, 300) }
+    }
+  }
+  return out
+}
+
+const showValue = (v) => (typeof v === 'string' ? JSON.stringify(v) : JSON.stringify(v) ?? 'undefined')
+/** js_equals from a recorded result: value (equality), op (numeric comparison), pattern (regex) or truthy. */
+export function jsCheck (check, rec) {
+  if (!rec) return { ok: null, evidence: 'no expression result recorded; capture the step again' }
+  const expr = rec.expression.length > 80 ? `${rec.expression.slice(0, 77)}…` : rec.expression
+  if (rec.error) return { ok: rec.refused ? null : false, evidence: `${expr}: ${rec.error}` }
+  const { type, value } = rec.result || {}
+  const recorded = rec.recorded ? ' (result recorded at capture)' : ''
+  let ok
+  let how
+  if (check.pattern != null) {
+    const s = typeof value === 'string' ? value : JSON.stringify(value)
+    ok = regexOf(check).test(s ?? ''); how = `${ok ? 'matches' : 'does not match'} /${check.pattern}/`
+  } else if (check.op && check.op !== '==') {
+    const a = Number(value); const b = Number(check.value)
+    ok = value !== null && value !== '' && Number.isFinite(a) && Number.isFinite(b) && OPS[check.op](a, b); how = `${ok ? check.op : `not ${check.op}`} ${check.value}`
+  } else if (check.value !== undefined) {
+    const want = check.value
+    if (want === null) ok = value === null
+    else if (typeof want === 'number') ok = value !== null && value !== '' && Number(value) === want
+    else if (typeof want === 'boolean') ok = value === want
+    else ok = value !== null && (typeof value === 'object' ? JSON.stringify(value) : String(value)) === want
+    how = `${ok ? '=' : '≠'} ${showValue(want)}`
+  } else {
+    ok = !!value && !(Array.isArray(value) && !value.length); how = ok ? 'is truthy' : 'is not truthy'
+  }
+  return { ok, evidence: `${expr} → ${showValue(value)}${type && !['string', 'number', 'boolean', 'null'].includes(type) ? ` (${type})` : ''} ${how}${recorded}` }
+}
+
+// ---- wiring ----
+
+const valueCriteria = (node) => (node.criteria || []).filter((c) => c.check.type === 'deterministic' && VALUE_ASSERTS.has(c.check.assert))
+
+/**
+ * Decides the value assertions of one capture in the live page: reads the expressions, takes the style records
+ * of the probe and the URL. Fills result.checks and result.values (stored with the capture for --recheck).
+ */
+export async function liveValueChecks ({ page, node, result, url, tokens = null, mode = null }) {
+  const crit = valueCriteria(node)
+  if (!crit.length) return
+  const js = crit.filter((c) => c.check.assert === 'js_equals')
+  const evaluated = js.length && page ? await jsValues(page, js.map((c) => ({ id: c.id, check: c.check }))) : {}
+  result.values = { ...(result.values || {}), ...evaluated }
+  for (const c of crit) {
+    const a = c.check.assert
+    if (URL_ASSERTS.has(a)) result.checks[c.id] = safely(() => urlCheck(c.check, url))
+    else if (STYLE_ASSERTS.has(a)) result.checks[c.id] = safely(() => styleCheck(c.check, result.values?.[c.id], { tokens, mode }))
+    else result.checks[c.id] = safely(() => jsCheck(c.check, result.values?.[c.id]))
+  }
+}
+
+const safely = (fn) => { try { return fn() } catch (error) { return { ok: false, evidence: `check error: ${error.message}` } } }
+
+/** The recorded values still answer this check: same selector and property, or the same expression. */
+export function sameReading (check, rec) {
+  if (!rec) return false
+  if (STYLE_ASSERTS.has(check.assert)) return rec.selector === check.selector && rec.property === check.property
+  if (check.assert === 'js_equals') return rec.expression === String(check.expression ?? '')
+  return false
+}
+
+/**
+ * The value assertions of a capture decided again without the live page (recheck, evaluate): the URL from the
+ * capture, styles and expressions from the values recorded at capture when the criterion reads the same thing.
+ */
+export function storedValueChecks (node, cap, { tokens = null, mode = null, only = null, recorded = true } = {}) {
+  const out = {}
+  for (const c of valueCriteria(node)) {
+    const a = c.check.assert
+    if (only && !only(c)) continue
+    if (URL_ASSERTS.has(a)) { out[c.id] = safely(() => urlCheck(c.check, cap.url)); continue }
+    const rec = cap.values?.[c.id]
+    if (!sameReading(c.check, rec)) {
+      out[c.id] = { ok: null, evidence: STYLE_ASSERTS.has(a) ? 'not evaluated on recheck: computed styles come from the live page and this selector or property was not recorded; capture the step again' : 'not evaluated on recheck: the expression runs in the live page and was not recorded; capture the step again' }
+      continue
+    }
+    const stored = { ...rec, recorded }
+    out[c.id] = safely(() => (STYLE_ASSERTS.has(a) ? styleCheck(c.check, stored, { tokens, mode }) : jsCheck(c.check, stored)))
+  }
+  return out
+}
+
+/** Criteria whose expected value is a token of the token set (decided again on evaluate with the project's set). */
+export const tokenCriteria = (node) => valueCriteria(node).filter((c) => STYLE_ASSERTS.has(c.check.assert) && c.check.token)

package/src/lib/build.mjs

@@ -9,6 +9,7 @@ import { fieldsOf, DEFAULT_SEMANTIC_SAMPLE } from './stack.mjs'
 import { layout } from './scenario.mjs'
 import { assembleNode, summarize, statusOf, worst } from './manifest-core.mjs'
 import { packageRoot } from './schema.mjs'
+import { variantOf } from './dimensions.mjs'
 
 export { targetOf, statusOf } from './manifest-core.mjs'
 
@@ -21,6 +22,8 @@ export async function buildManifest ({ run, scenario, config, root, catalog = []
   const pos = new Map(layout(scenario).map((p) => [p.id, p]))
   const locales = meta.scope.locales
   const viewportNames = Object.keys(meta.scope.viewports)
+  // an e-mail is rendered once, in Chromium: its other engines' keys are not missing captures
+  const mailKeys = viewportNames.filter((k) => variantOf(meta.scope, k).browser === 'chromium')
   const nodes = []
   for (const node of scenario.nodes) {
     if (node.kind === 'stack') { const p = pos.get(node.id); nodes.push(await buildStack({ run, node, base: { id: node.id, kind: node.kind, title: node.title, path: node.path ?? null, purpose: node.purpose ?? '', col: p.col, lane: p.lane }, captures, attachments })); continue }
@@ -37,7 +40,7 @@ export async function buildManifest ({ run, scenario, config, root, catalog = []
     }
     // focus screenshots of the keyboard walk (#19): only those of the current captures, not of earlier walks
     const shots = new Set(caps.flatMap((c) => (c.cap.keyboard?.shots || []).map((s) => s.file)))
-    const out = assembleNode({ node, pos: pos.get(node.id), locales, viewportNames, captures: caps, attachments: attachments.filter((a) => a.node === node.id && (a.kind !== 'focus' || shots.has(a.file))) })
+    const out = assembleNode({ node, pos: pos.get(node.id), locales, viewportNames: node.kind === 'email' ? mailKeys : viewportNames, captures: caps, attachments: attachments.filter((a) => a.node === node.id && (a.kind !== 'focus' || shots.has(a.file))) })
     await attachMedia(out, { run, entries: media, captures: caps })
     nodes.push(out)
   }