kaloko 0.4.1 → 0.4.2

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "kaloko",
-  "version": "0.4.1",
+  "version": "0.4.2",
   "description": "See what your AI agents built: every screen, language and device walked, assessed and approved on one canvas.",
   "keywords": [
     "qa",

package/skills/kaloko/reference/stacks.md

@@ -54,14 +54,22 @@ Item sources:
 - `urls: [/a, /b]` is a short fixed list.
 - `file: qa/data/products.csv` is a list next to the scenario. Accepted
   formats are text with one path per line, CSV (first column), or JSON:
-  `["/a"]` or `[{ path, title, key }]`, where `key` gives a stable id.
+  `["/a"]` or `[{ path, title, key, purpose }]`, where `key` gives a stable id.
 - `sitemap: /sitemap.xml` also follows a sitemap index (up to 50 sitemaps).
 - `crawl: { from: /clanky, links: "a.card", next: "a.next", pages: 20 }`
   collects links from a listing and follows its pagination.
 - `script: qa/steps/items.mjs` uses the default export
-  `({ page, env, locale, url }) => [paths | { path, title, key }]` for
+  `({ page, env, locale, url }) => [paths | { path, title, key, purpose }]` for
   anything else, such as an API or a search.
 
+An item's `purpose` (for example its title and meta description) is what the
+semantic evaluator reads as `screen_purpose` for that page instead of the
+step's generic purpose, so one question asked of forty different pages is
+judged against each page's own intent. A script also decides which pages
+belong to which locale: return only the pages of `locale` when the pages
+are language-specific (`/cs/…` in cs, the rest in en), so no page is judged
+in the wrong language.
+
 Pages are deduplicated. External URLs are allowed (e.g. an old domain).
 `{locale}` in any source string is replaced by the walked locale. This is
 how one sitemap with every language is split:

package/src/cli.mjs

@@ -213,8 +213,8 @@ const list = (v) => (typeof v === 'string' ? v.split(',').map((s) => s.trim()).f
 const say = (...m) => console.log(...m)
 
 async function context (args) {
-  const { config, root } = await loadConfig(process.cwd(), args.config)
-  return { config, root }
+  const { config, root, file } = await loadConfig(process.cwd(), args.config)
+  return { config, root, file }
 }
 
 async function runContext (args) {
@@ -1370,8 +1370,8 @@ const commands = {
   },
 
   async doctor (args) {
-    const { config, root } = await context(args)
-    const r = await doctor({ config, root, fix: !!args.fix })
+    const { config, root, file } = await context(args)
+    const r = await doctor({ config, root, file, fix: !!args.fix })
     say(args.json ? JSON.stringify(r, null, 2) : formatDoctor(r))
     if (!r.ok) process.exitCode = 1
   },

package/src/lib/capture.mjs

@@ -189,7 +189,7 @@ export async function captureStep ({ page, node, locale, viewports, run, console
       ...extra,
       http_status: extra.http_status ?? result.http_status ?? null,
       capture: { format: String(image).split('.').pop(), scale: opts.scale, masked: opts.mask.length, annotations, ...(opts.fullPage ? {} : { full_page: false }) },
-      ...(item ? { item: { id: item.id, path: item.path, ...(item.title ? { title: item.title } : {}) } } : {}),
+      ...(item ? { item: { id: item.id, path: item.path, ...(item.title ? { title: item.title } : {}), ...(item.purpose ? { purpose: item.purpose } : {}) } } : {}),
       ...(fields ? { fields: { ...(await page.evaluate(readFields, fields).catch(() => ({}))), ...(fields.some((f) => (f.from || f.id) === 'http') ? Object.fromEntries(fields.filter((f) => (f.from || f.id) === 'http').map((f) => [f.id, extra.http_status ?? result.http_status ?? null])) : {}) } } : {}),
       files: { png: relative(run.dir, image), image: relative(run.dir, image), thumb: thumbWritten ? relative(run.dir, thumb) : null, html: relative(run.dir, html) }
     }

package/src/lib/doctor.mjs

@@ -32,14 +32,14 @@ async function reach (url, fetchImpl, ms = 6000, headers = {}) {
  * @param {Function} [o.launchChrome] async () => { version } — throws when Chrome is missing
  * @param {object} [o.env] process.env
  */
-export async function doctor ({ config, root, fetchImpl = fetch, launchChrome = defaultLaunch, env = process.env, fix = false }) {
+export async function doctor ({ config, root, file = null, fetchImpl = fetch, launchChrome = defaultLaunch, env = process.env, fix = false }) {
   const checks = []
   const add = (name, ok, detail = '', hint = '') => checks.push({ name, ok, detail, hint })
 
   const major = Number(process.versions.node.split('.')[0])
   add('node', major >= 20, `Node ${process.versions.node}`, major >= 20 ? '' : 'Kaloko needs Node 20 or newer.')
 
-  add('config', true, `${root}/qawalk.config.yml · org ${config.org || '—'} · ${Object.keys(config.environments || {}).length} environments`)
+  add('config', true, `${file || `${root}/kaloko.config.yml`} · org ${config.org || '—'} · ${Object.keys(config.environments || {}).length} environments`)
   if (!config.org) add('org', false, 'no org in config', 'Set org: <slug> (the organization slug from kaloko.app).')
 
   try { const c = await launchChrome(config); add('chrome', true, c.version ? `Chrome ${c.version}` : 'Chrome found') } catch (e) { add('chrome', false, e.message.split('\n')[0].slice(0, 160), config.browser?.channel ? 'Install Google Chrome, or remove browser.channel to use bundled Chromium (npx playwright install chromium).' : 'Run: npx playwright install chromium') }

package/src/lib/jev.mjs

@@ -10,11 +10,13 @@ export const DEFAULTS = { hosted: true, signals: true, concurrency: 6, model: 'j
  * does not count them against the allowance).
  */
 export const SIGNALS = {
-  _purpose: (node) => node.purpose ? 'Does `page` do what `screen_purpose` says this screen must achieve?' : null,
+  _purpose: (node, capture) => purposeOf(node, capture) ? 'Does `page` do what `screen_purpose` says this screen must achieve?' : null,
   _language: (node, capture) => capture.locale ? `Is the visible text of \`page\` mainly in the language with code "${capture.locale}"?` : null,
   _broken: () => 'Does `page` look broken: an error message, placeholder text (lorem ipsum, TODO, {{variables}}), missing content or a blank page?'
 }
 export const SIGNAL_NAMES = { _purpose: 'purpose', _language: 'language', _broken: 'broken' }
+/** What the captured page must achieve: a stack page's own purpose (items: { purpose }) before the step's. */
+export const purposeOf = (node, capture) => capture?.item?.purpose || node.purpose || ''
 
 /** Builds the JEV request for one capture: one `noul` question per semantic criterion, plus the quick signals. */
 export function buildRequest ({ node, capture, criteria, model, signals = false }) {
@@ -37,7 +39,7 @@ export function buildRequest ({ node, capture, criteria, model, signals = false
     }
   }
   const state = {
-    screen_purpose: node.purpose || '',
+    screen_purpose: purposeOf(node, capture),
     url: capture.url,
     locale: capture.locale,
     viewport: capture.viewport,

package/src/lib/stack.mjs

@@ -34,18 +34,18 @@ export function sitemapLocs (xml) {
   return { index: /<sitemapindex[\s>]/i.test(xml), locs }
 }
 
-/** Reads a list file: text (one per line), CSV (first column) or JSON ([path] or [{path|url, title, key}]). */
+/** Reads a list file: text (one per line), CSV (first column) or JSON ([path] or [{path|url, title, key, purpose}]). */
 export async function listFile (file) {
   const text = await readFile(file, 'utf8').catch(() => { throw new UserError(`items.file ${file} cannot be read.`) })
   if (extname(file) === '.json') {
     const data = JSON.parse(text)
-    return (Array.isArray(data) ? data : data.items || []).map((x) => (typeof x === 'string' ? { path: x } : { path: x.path || x.url, title: x.title, key: x.key }))
+    return (Array.isArray(data) ? data : data.items || []).map((x) => (typeof x === 'string' ? { path: x } : { path: x.path || x.url, title: x.title, key: x.key, purpose: x.purpose }))
   }
   return text.split(/\r?\n/).map((l) => l.trim()).filter((l) => l && !l.startsWith('#')).map((l) => ({ path: extname(file) === '.csv' ? l.split(/[,;]/)[0].replace(/^"|"$/g, '') : l })).filter((x) => x.path && !/^(path|url)$/i.test(x.path))
 }
 
 /**
- * The pages of a stack step, deduplicated, filtered (match/exclude), limited: [{ id, path, url, title? }].
+ * The pages of a stack step, deduplicated, filtered (match/exclude), limited: [{ id, path, url, title?, purpose? }].
  * `base` is the environment's URL for the locale; `page` is used for crawl and script sources. `match` and
  * `exclude` are regexes on the path with its query; `{locale}` in any source string is the walked locale.
  */
@@ -79,7 +79,7 @@ export async function resolveItems ({ node, base, page, scenarioDir, env, locale
     const fn = mod.default || mod.items
     if (typeof fn !== 'function') throw new UserError(`items.script ${src.script} must export a default function.`)
     raw = (await fn({ page, env, locale, url: abs })) || []
-    raw = raw.map((x) => (typeof x === 'string' ? { path: x } : { path: x.path || x.url, title: x.title, key: x.key }))
+    raw = raw.map((x) => (typeof x === 'string' ? { path: x } : { path: x.path || x.url, title: x.title, key: x.key, purpose: x.purpose }))
   }
   const match = src.match ? new RegExp(src.match) : null
   const exclude = src.exclude ? new RegExp(src.exclude) : null
@@ -93,7 +93,8 @@ export async function resolveItems ({ node, base, page, scenarioDir, env, locale
     if (exclude && exclude.test(test)) continue
     if (seen.has(path)) continue
     seen.add(path)
-    out.push({ id: r.key ? itemId(String(r.key), taken) : itemId(path, taken), path, url: path.startsWith('http') ? path : toUrl(path), ...(r.title ? { title: r.title } : {}) })
+    // `purpose` of an item (what this one page must achieve) replaces the step's purpose for the evaluator
+    out.push({ id: r.key ? itemId(String(r.key), taken) : itemId(path, taken), path, url: path.startsWith('http') ? path : toUrl(path), ...(r.title ? { title: r.title } : {}), ...(r.purpose ? { purpose: String(r.purpose).slice(0, 500) } : {}) })
     if (out.length >= max) break
   }
   return { items: out, total: seen.size, truncated: out.length >= max && raw.length > out.length }