kaloko 0.3.0

package/src/lib/access.mjs

@@ -0,0 +1,169 @@
+// Access to an environment, in three layers:
+//   gate     — protection in front of the whole environment (HTTP Basic, service-token headers,
+//              client certificate); applied by the CLI to every browser context it creates;
+//   accounts — people who sign in to the application (user, password, TOTP), handed to step
+//              scripts; SSO/MFA sessions come from a storage state a person saved once;
+//   policy   — readonly and blocked paths (config.mjs), enforced in the same place.
+// Secret values are read from the environment only here, and only when needed.
+import { createHmac } from 'node:crypto'
+import { access as exists } from 'node:fs/promises'
+import { join, resolve } from 'node:path'
+import { policyViolation, baseOrigins } from './config.mjs'
+import { secret, remember } from './secrets.mjs'
+import { UserError } from './util.mjs'
+
+/** The environment's gate: `access`, with the older `http_auth` read as `access.basic`. */
+export function gateOf (env) {
+  const gate = { ...(env.access || {}) }
+  if (!gate.basic && env.http_auth?.user_env) gate.basic = env.http_auth
+  return gate
+}
+
+/** Origins that receive the gate credentials: `access.origins`, else the base URL's origins. */
+export function gateOrigins (env) {
+  const list = gateOf(env).origins
+  return list?.length ? list : baseOrigins(env)
+}
+
+/** Matcher for origins; `https://*.example.com` covers every subdomain (not the apex). */
+export function originMatcher (origins) {
+  const rules = origins.map((o) => {
+    const m = String(o).match(/^(https?):\/\/(\*\.)?([^/:]+)(?::(\d+))?\/?$/)
+    if (!m) throw new UserError(`access.origins: "${o}" is not an origin like https://staging.example.com`)
+    return { scheme: m[1], wildcard: !!m[2], host: m[3].toLowerCase(), port: m[4] || (m[1] === 'https' ? '443' : '80') }
+  })
+  return (url) => {
+    let u
+    try { u = new URL(url) } catch { return false }
+    const scheme = u.protocol.slice(0, -1)
+    const port = u.port || (scheme === 'https' ? '443' : '80')
+    const host = u.hostname.toLowerCase()
+    return rules.some((r) => r.scheme === scheme && r.port === port && (r.wildcard ? host.endsWith(`.${r.host}`) : host === r.host))
+  }
+}
+
+/** Resolved gate headers (Authorization for Basic plus `access.headers`). */
+export function gateHeaders (env, vars = process.env) {
+  const gate = gateOf(env)
+  const headers = {}
+  if (gate.basic) {
+    const user = secret(gate.basic.user_env, `HTTP Basic user for ${env.name}`, vars)
+    const pass = secret(gate.basic.password_env, `HTTP Basic password for ${env.name}`, vars)
+    headers.Authorization = 'Basic ' + Buffer.from(`${user}:${pass}`).toString('base64')
+    remember(headers.Authorization.slice(6)) // the pair's base64 is not the password's
+  }
+  for (const [name, spec] of Object.entries(gate.headers || {})) headers[name] = typeof spec === 'string' ? spec : secret(spec.env, `header ${name}`, vars)
+  return headers
+}
+
+/** Playwright `clientCertificates` for the gate's exact origins (mutual TLS). */
+export function clientCertificates (env, root = process.cwd()) {
+  const cert = gateOf(env).client_certificate
+  if (!cert) return []
+  const pem = (fileKey, envKey, what) => {
+    if (cert[fileKey]) return { path: resolve(root, cert[fileKey]) }
+    if (!cert[envKey]) return null
+    const value = secret(cert[envKey], what)
+    return value.includes('-----BEGIN') ? { data: Buffer.from(value) } : { path: resolve(root, value) }
+  }
+  const c = pem('cert_file', 'cert_env', `client certificate for ${env.name}`)
+  const k = pem('key_file', 'key_env', `client key for ${env.name}`)
+  if (!c || !k) throw new UserError(`access.client_certificate for ${env.name} needs a certificate and a key (cert_file/cert_env, key_file/key_env).`)
+  const passphrase = cert.passphrase_env ? secret(cert.passphrase_env, `client key passphrase for ${env.name}`) : undefined
+  return gateOrigins(env).filter((o) => !o.includes('*')).map((origin) => ({
+    origin: new URL(origin).origin,
+    ...(c.path ? { certPath: c.path } : { cert: c.data }),
+    ...(k.path ? { keyPath: k.path } : { key: k.data }),
+    ...(passphrase ? { passphrase } : {})
+  }))
+}
+
+/**
+ * Enforces the environment on a browser context: the readonly policy for every request and the
+ * gate headers for the gate's origins only (third parties never see them). Headers the page set
+ * itself win, so an application's own Authorization header is left alone.
+ */
+export async function prepareContext (context, env, { log } = {}) {
+  const headers = gateHeaders(env)
+  const names = Object.keys(headers)
+  const toGate = names.length ? originMatcher(gateOrigins(env)) : () => false
+  await context.route('**/*', (route) => {
+    const request = route.request()
+    const reason = policyViolation(env, { method: request.method(), url: request.url() })
+    if (reason) {
+      log?.({ type: 'blocked', url: request.url(), text: reason })
+      return route.abort('blockedbyclient')
+    }
+    if (toGate(request.url())) {
+      const current = request.headers()
+      const extra = names.filter((n) => !(n.toLowerCase() in current))
+      if (extra.length) return route.continue({ headers: { ...current, ...Object.fromEntries(extra.map((n) => [n, headers[n]])) } })
+    }
+    return route.continue()
+  })
+  return context
+}
+
+/** Context options for the environment: client certificates and, for an account, its saved session. */
+export async function contextOptions (env, { root, config, account } = {}) {
+  const options = {}
+  const certs = clientCertificates(env, root)
+  if (certs.length) options.clientCertificates = certs
+  // Basic auth also for navigations that arrive through a redirect (#56): route headers do not apply to redirected
+  // requests, so answer the 401 challenge with httpCredentials — only for one exact gate origin, never elsewhere
+  const gate = gateOf(env); const origins = gateOrigins(env)
+  if (gate.basic && origins.length === 1 && !String(origins[0]).includes('*')) {
+    options.httpCredentials = { username: secret(gate.basic.user_env, `HTTP Basic user for ${env.name}`), password: secret(gate.basic.password_env, `HTTP Basic password for ${env.name}`), origin: new URL(origins[0]).origin }
+  }
+  if (account?.storage_state && await exists(account.storage_state).then(() => true, () => false)) options.storageState = account.storage_state
+  return options
+}
+
+/** A new browser context with the environment's gate, policy and (optionally) an account's session. */
+export async function newContext (browser, env, opts = {}) {
+  const context = await browser.newContext(await contextOptions(env, opts))
+  return prepareContext(context, env, opts)
+}
+
+/** Where an account's saved browser session lives (cookies, storage): never inside a run. */
+export function storageStatePath (env, key, { root, config }) {
+  const custom = env.accounts?.[key]?.storage_state
+  return custom ? resolve(root, custom) : join(resolve(root, config.output || 'tmp/kaloko'), '.auth', env.name, `${key}.json`)
+}
+
+/**
+ * Credentials of a scenario session's `account` in this environment. Values are read lazily,
+ * so a script that signs in with a saved session never needs the password.
+ */
+export function accountFor (env, key, ctx) {
+  if (!key) return null
+  const spec = env.accounts?.[key]
+  const account = { key, storage_state: storageStatePath(env, key, ctx), configured: !!spec }
+  if (!spec) return account
+  Object.defineProperties(account, {
+    user: { enumerable: true, get: () => spec.user ?? (spec.user_env ? secret(spec.user_env, `user of account ${key}`) : null) },
+    password: { enumerable: false, get: () => (spec.password_env ? secret(spec.password_env, `password of account ${key}`) : null) }
+  })
+  account.totp = () => {
+    if (!spec.totp_env) throw new UserError(`Account ${key} has no totp_env in environment ${env.name}.`)
+    return totp(secret(spec.totp_env, `TOTP secret of account ${key}`))
+  }
+  return account
+}
+
+/** RFC 6238 time-based one-time password from a base32 secret. */
+export function totp (base32, { step = 30, digits = 6, now = Date.now() } = {}) {
+  const alphabet = 'ABCDEFGHIJKLMNOPQRSTUVWXYZ234567'
+  let bits = ''
+  for (const ch of base32.replace(/[\s=]/g, '').toUpperCase()) {
+    const v = alphabet.indexOf(ch)
+    if (v < 0) throw new UserError('The TOTP secret is not base32.')
+    bits += v.toString(2).padStart(5, '0')
+  }
+  const key = Buffer.from((bits.match(/.{8}/g) || []).map((b) => parseInt(b, 2)))
+  const counter = Buffer.alloc(8)
+  counter.writeBigUInt64BE(BigInt(Math.floor(now / 1000 / step)))
+  const h = createHmac('sha1', key).update(counter).digest()
+  const o = h[h.length - 1] & 0xf
+  return String((h.readUInt32BE(o) & 0x7fffffff) % 10 ** digits).padStart(digits, '0')
+}

package/src/lib/apps/capture.mjs

@@ -0,0 +1,59 @@
+// Capture records of native app screens: a screenshot, the UI tree (qawalk.tree.v1), deterministic checks on the
+// tree and a text state for the semantic evaluator. Same record shape as web captures (schema qawalk.capture.v1)
+// plus `platform`, `device`, `scale` and `files.tree`; `html` is absent. Used by walk (drivers) and by
+// `kaloko capture --image` (screens a person, an agent, Maestro, Appium or XCUITest took).
+import { mkdir, writeFile, rm } from 'node:fs/promises'
+import { dirname, relative } from 'node:path'
+import { runPaths, addCapture } from '../run.mjs'
+import { writeJson } from '../util.mjs'
+import { redactDeep } from '../secrets.mjs'
+import { criteriaHash } from '../evaluate.mjs'
+import { encodeImage } from '../capture.mjs'
+import { treeChecks, outline, treeFile } from './tree.mjs'
+import { pngSize } from './drivers.mjs'
+
+/**
+ * snapshot: { png, tree: { root, format } | null, size: [w, h] logical, scale, device, crash, logErrors }
+ * encoder: a Playwright page used to encode WebP and the thumbnail (optional: PNG only without it).
+ */
+export async function captureApp ({ snapshot, node, locale, viewport, run, platform, encoder = null, maxChars = 90000, extra = {}, format = 'webp' }) {
+  const paths = runPaths(run.dir)
+  const stepId = node.id
+  const png = snapshot.png
+  const px = pngSize(png) || [0, 0]
+  const scale = snapshot.scale || (snapshot.size?.[0] ? px[0] / snapshot.size[0] : 1)
+  const size = snapshot.size || [Math.round(px[0] / scale), Math.round(px[1] / scale)]
+  const vp = viewport || (size[0] < 700 ? 'phone' : size[0] < 1100 ? 'tablet' : 'desktop')
+  const base = paths.step(stepId, locale, vp, 'x')
+  await mkdir(dirname(base), { recursive: true })
+  for (const ext of ['eval.json', 'html']) await rm(paths.step(stepId, locale, vp, ext), { force: true })
+  let image = paths.step(stepId, locale, vp, format === 'png' ? 'png' : 'webp')
+  const thumb = paths.step(stepId, locale, vp, 'thumb.webp')
+  let thumbWritten = false
+  if (encoder) {
+    try { const enc = await encodeImage(encoder, png, { format }); await writeFile(image, enc.image); await writeFile(thumb, enc.thumb); thumbWritten = true } catch { image = paths.step(stepId, locale, vp, 'png'); await writeFile(image, png) }
+  } else { image = paths.step(stepId, locale, vp, 'png'); await writeFile(image, png) }
+  const tree = snapshot.tree
+  const { checks, boxes } = tree ? treeChecks(node.criteria, tree.root, { platform, format: tree.format, crash: snapshot.crash, logErrors: snapshot.logErrors }) : { checks: Object.fromEntries((node.criteria || []).filter((c) => c.check.type === 'deterministic').map((c) => [c.id, c.check.assert === 'app_no_crash' ? { ok: !snapshot.crash, evidence: snapshot.crash || 'the app is running' } : { ok: null, evidence: 'no UI tree for this capture (pass --tree, or install idb for the iOS Simulator)' }])), boxes: {} }
+  const treePath = paths.step(stepId, locale, vp, 'tree.json')
+  if (tree) await writeJson(treePath, redactDeep(treeFile({ root: tree.root, format: tree.format, platform, device: snapshot.device, scale, size })))
+  const state = tree ? outline(tree.root, maxChars) : ''
+  const record = {
+    schema: 'qawalk.capture.v1',
+    node: node.id, locale, viewport: vp, url: extra.url || null,
+    captured_at: new Date().toISOString(),
+    criteria_hash: criteriaHash(node),
+    platform, device: snapshot.device || null, scale,
+    console_errors: snapshot.logErrors || [], blocked_requests: [],
+    state, scoped: {}, checks, boxes, overflow: [], meta: null, metadata: null,
+    size: { width: size[0], height: size[1], viewport: size },
+    ...(snapshot.crash ? { crash: String(snapshot.crash).slice(0, 2000) } : {}),
+    ...extra,
+    capture: { format: String(image).split('.').pop(), scale: 1, masked: 0, annotations: [], source: extra.source || 'driver', tree: tree?.format || null },
+    files: { png: relative(run.dir, image), image: relative(run.dir, image), thumb: thumbWritten ? relative(run.dir, thumb) : null, html: null, tree: tree ? relative(run.dir, treePath) : null }
+  }
+  const file = paths.step(stepId, locale, vp, 'capture.json')
+  await writeJson(file, redactDeep(record))
+  await addCapture(run.dir, { type: 'capture', node: node.id, locale, viewport: vp, file: relative(run.dir, file), at: record.captured_at })
+  return record
+}

package/src/lib/apps/drivers.mjs

@@ -0,0 +1,284 @@
+// Drivers for native app captures: one small interface over adb (Android devices and emulators), simctl (iOS
+// Simulator, with idb for the UI tree and taps when installed), Playwright's Electron support, and macOS windows.
+//   open({ locale, reset, attach })   start the app for a locale (attach: use what is on the screen now)
+//   link(url)                  open a deep link (the step's path)
+//   tap(target) type(text) back() wait(ms|{ text })   interactions (target: selector string or { x, y })
+//   snapshot()                 → { png: Buffer, tree: { root, format } | null, size: [w, h] logical, scale, device, crash, logErrors }
+//   close()
+// Electron drivers also expose `page` (a Playwright page): their captures are web captures with the DOM.
+import { execFile } from 'node:child_process'
+import { promisify } from 'node:util'
+import { mkdtemp, readFile, rm, mkdir, access, writeFile } from 'node:fs/promises'
+import { tmpdir, homedir } from 'node:os'
+import { join, resolve, dirname } from 'node:path'
+import { createHash } from 'node:crypto'
+import { fileURLToPath } from 'node:url'
+import { UserError } from '../util.mjs'
+import { readTree, query } from './tree.mjs'
+
+const run = promisify(execFile)
+const sleep = (ms) => new Promise((r) => setTimeout(r, ms))
+export const PLATFORMS = ['web', 'android', 'ios', 'electron', 'desktop']
+export const isApp = (platform) => !!platform && platform !== 'web'
+
+async function exec (bin, args, opts = {}) {
+  try { return await run(bin, args, { maxBuffer: 64 * 1024 * 1024, encoding: opts.buffer ? 'buffer' : 'utf8', timeout: opts.timeout || 60000, env: { ...process.env, ...(opts.env || {}) } }) } catch (error) {
+    if (error.code === 'ENOENT') throw new UserError(`${bin} is not installed or not on PATH.${opts.hint ? ' ' + opts.hint : ''}`)
+    throw new Error(`${bin} ${args.slice(0, 3).join(' ')}: ${String(error.stderr || error.message).trim().split('\n').pop()}`)
+  }
+}
+const which = async (bin) => { try { await run(process.platform === 'win32' ? 'where' : 'which', [bin]); return true } catch { return false } }
+
+/** PNG width and height from the IHDR chunk. */
+export const pngSize = (buf) => (buf?.length > 24 && buf.readUInt32BE(12) === 0x49484452 ? [buf.readUInt32BE(16), buf.readUInt32BE(20)] : null)
+
+/* ---------- macOS helper (OCR, windows, accessibility), compiled on first use ---------- */
+const here = dirname(fileURLToPath(import.meta.url))
+let helperPath = null
+export async function helper () {
+  if (helperPath) return helperPath
+  if (process.platform !== 'darwin') throw new UserError('Text recognition and desktop window capture need macOS.')
+  const src = join(here, 'helper.swift')
+  const hash = createHash('sha1').update(await readFile(src)).digest('hex').slice(0, 10)
+  const dir = join(process.env.XDG_CACHE_HOME || join(homedir(), '.cache'), 'qawalk')
+  const bin = join(dir, `helper-${hash}`)
+  try { await access(bin); helperPath = bin; return bin } catch {}
+  await mkdir(dir, { recursive: true })
+  await exec('xcrun', ['swiftc', '-O', '-o', bin, src], { timeout: 300000, hint: 'Install the Xcode command line tools (xcode-select --install).' })
+  helperPath = bin
+  return bin
+}
+/** macOS permissions of this terminal for desktop capture; request=true shows the system dialogs. */
+export async function macPermissions (request = false) {
+  const { stdout } = await exec(await helper(), ['permissions', ...(request ? ['request'] : [])], { timeout: 60000 })
+  return JSON.parse(stdout)
+}
+export async function ocr (png, langs) {
+  const dir = await mkdtemp(join(tmpdir(), 'qawalk-ocr-'))
+  try {
+    const f = join(dir, 'shot.png'); await writeFile(f, png)
+    const { stdout } = await exec(await helper(), ['ocr', f, ...(langs?.length ? [langs.join(',')] : [])], { timeout: 120000 })
+    return JSON.parse(stdout)
+  } finally { await rm(dir, { recursive: true, force: true }) }
+}
+const ocrLangs = (locale) => (locale ? [locale.includes('-') ? locale : { cs: 'cs-CZ', en: 'en-US', de: 'de-DE', sk: 'sk-SK', pl: 'pl-PL', fr: 'fr-FR', es: 'es-ES', it: 'it-IT' }[locale] || locale, 'en-US'] : undefined)
+/** OCR lines (pixels) → tree in logical units. */
+const ocrTree = (res, scale) => readTree({ ocr: res.ocr.map((l) => ({ text: l.text, x: Math.round(l.x / scale), y: Math.round(l.y / scale), w: Math.round(l.w / scale), h: Math.round(l.h / scale) })) })
+
+/* ---------- Android (adb) ---------- */
+function android (cfg, ctx) {
+  const serial = cfg.serial || process.env.ANDROID_SERIAL || null
+  const adb = (args, o) => exec(cfg.adb || 'adb', [...(serial ? ['-s', serial] : []), ...args], { hint: 'Install Android platform-tools and start an emulator or connect a device (adb devices).', ...o })
+  const shell = (...a) => adb(['shell', ...a])
+  let density = 1; let device = cfg.device || 'android'
+  const logStart = { t: null }
+  const d = {
+    platform: 'android',
+    async open ({ locale, reset, attach = false } = {}) {
+      const { stdout } = await adb(['get-state']).catch((e) => { throw new UserError(`No Android device: ${e.message}. Start an emulator or connect a device (adb devices).`) })
+      if (!/device/.test(stdout)) throw new UserError('The Android device is not ready (adb get-state).')
+      const dens = (await shell('wm', 'density')).stdout.match(/(\d+)\s*$/m)
+      density = dens ? Number(dens[1]) / 160 : 1
+      if (!cfg.device) device = (await shell('getprop', 'ro.product.model')).stdout.trim() || 'android'
+      if (attach) return // capture what is on the screen now
+      if (cfg.apk && cfg.install !== false) await adb(['install', '-r', '-g', resolve(ctx.root, cfg.apk)], { timeout: 300000 })
+      if (!cfg.package) return
+      if (reset ?? cfg.reset) await shell('pm', 'clear', cfg.package)
+      if (locale) await shell('cmd', 'locale', 'set-app-locales', cfg.package, '--locales', locale).catch(() => {}) // Android 13+; older devices keep the system language
+      await shell('am', 'force-stop', cfg.package)
+      await adb(['logcat', '-c']).catch(() => {})
+      logStart.t = Date.now()
+      if (cfg.activity) await shell('am', 'start', '-W', '-n', `${cfg.package}/${cfg.activity}`)
+      else await shell('monkey', '-p', cfg.package, '-c', 'android.intent.category.LAUNCHER', '1')
+      await sleep(cfg.launch_wait ?? 1500)
+    },
+    async link (url) { await shell('am', 'start', '-W', '-a', 'android.intent.action.VIEW', '-d', `'${String(url).replace(/'/g, "'\\''")}'`, ...(cfg.package ? [cfg.package] : [])); await sleep(cfg.settle ?? 800) },
+    async tap (target) { const p = await point(d, target); await shell('input', 'tap', String(Math.round(p.x * density)), String(Math.round(p.y * density))); await sleep(cfg.settle ?? 600) },
+    async type (text) { await shell('input', 'text', `'${String(text).replace(/ /g, '%s').replace(/'/g, "'\\''")}'`); await sleep(300) },
+    async back () { await shell('input', 'keyevent', '4'); await sleep(cfg.settle ?? 600) },
+    async wait (w) { await waitFor(d, w) },
+    async tree () { const { stdout } = await adb(['exec-out', 'uiautomator', 'dump', '/dev/tty']); const xml = stdout.slice(stdout.indexOf('<'), stdout.lastIndexOf('>') + 1); return readTree(xml, { density }) },
+    async snapshot () {
+      const png = (await adb(['exec-out', 'screencap', '-p'], { buffer: true })).stdout
+      const px = pngSize(png) || [1080, 2400]
+      const tree = await d.tree().catch(() => null)
+      let crash = null; let logErrors = null
+      if (cfg.package) {
+        const log = (await adb(['logcat', '-d', '-v', 'brief', '*:E']).catch(() => ({ stdout: '' }))).stdout
+        const fatal = log.match(/FATAL EXCEPTION[\s\S]{0,600}/)
+        const anr = log.match(/ANR in [^\n]*/)
+        crash = fatal ? fatal[0] : anr ? anr[0] : null
+        logErrors = log.split('\n').filter((l) => /^E\//.test(l) && (l.includes(cfg.package) || /AndroidRuntime/.test(l))).slice(-20)
+      }
+      return { png, tree, size: [Math.round(px[0] / density), Math.round(px[1] / density)], scale: density, device, crash, logErrors }
+    },
+    async close () {}
+  }
+  return d
+}
+
+/* ---------- iOS Simulator (simctl; idb for the tree and taps) ---------- */
+function ios (cfg, ctx) {
+  let udid = cfg.udid || null; let device = cfg.device || 'iPhone'
+  let idb = false; let scale = cfg.scale || 3
+  const simctl = (args, o) => exec('xcrun', ['simctl', ...args], { hint: 'Install Xcode with an iOS Simulator runtime.', ...o })
+  const d = {
+    platform: 'ios',
+    async open ({ locale, reset, attach = false } = {}) {
+      const list = JSON.parse((await simctl(['list', 'devices', 'available', '-j'])).stdout).devices
+      const all = Object.values(list).flat()
+      let dev = udid ? all.find((x) => x.udid === udid) : cfg.device ? all.find((x) => x.name === cfg.device && x.state === 'Booted') || all.find((x) => x.name === cfg.device) : all.find((x) => x.state === 'Booted') || all.find((x) => /iPhone/.test(x.name))
+      if (!dev) throw new UserError(`No iOS simulator${cfg.device ? ` named "${cfg.device}"` : ''} (xcrun simctl list devices).`)
+      udid = dev.udid; device = dev.name
+      if (dev.state !== 'Booted') { await simctl(['boot', udid]); await simctl(['bootstatus', udid, '-b'], { timeout: 300000 }) }
+      if (/iPad|mini/.test(device) && !cfg.scale) scale = 2
+      if (cfg.status_bar !== false) await simctl(['status_bar', udid, 'override', '--time', '9:41', '--batteryState', 'charged', '--batteryLevel', '100', '--cellularBars', '4', '--wifiBars', '3']).catch(() => {})
+      idb = cfg.idb !== false && await which('idb')
+      if (attach) return
+      if (cfg.app && cfg.install !== false) await simctl(['install', udid, resolve(ctx.root, cfg.app)], { timeout: 300000 })
+      if (!cfg.bundle) return
+      await simctl(['terminate', udid, cfg.bundle]).catch(() => {})
+      if (reset ?? cfg.reset) { const c = (await simctl(['get_app_container', udid, cfg.bundle, 'data']).catch(() => ({ stdout: '' }))).stdout.trim(); if (c) await rm(join(c, 'Library', 'Preferences'), { recursive: true, force: true }).catch(() => {}) }
+      const lang = locale ? ['-AppleLanguages', `(${locale})`, '-AppleLocale', locale.replace('-', '_')] : []
+      await simctl(['launch', udid, cfg.bundle, ...lang, ...(cfg.args || [])])
+      await sleep(cfg.launch_wait ?? 1500)
+    },
+    async link (url) { await simctl(['openurl', udid, url]); await sleep(cfg.settle ?? 1000) },
+    async tap (target) {
+      if (!idb) throw new UserError('Tapping on the iOS Simulator needs idb (brew install facebook/fb/idb-companion && pip install fb-idb), or drive the app with your own tool and capture with kaloko capture --image.')
+      const p = await point(d, target); await exec('idb', ['ui', 'tap', '--udid', udid, String(Math.round(p.x)), String(Math.round(p.y))]); await sleep(cfg.settle ?? 600)
+    },
+    async type (text) { if (!idb) throw new UserError('Typing on the iOS Simulator needs idb.'); await exec('idb', ['ui', 'text', '--udid', udid, String(text)]); await sleep(300) },
+    async back () { throw new UserError('iOS has no system back: tap the app\'s own back button.') },
+    async wait (w) { await waitFor(d, w) },
+    async tree () {
+      if (idb) { const { stdout } = await exec('idb', ['ui', 'describe-all', '--udid', udid, '--json']); return readTree(stdout) }
+      return null
+    },
+    async snapshot () {
+      const dir = await mkdtemp(join(tmpdir(), 'qawalk-ios-'))
+      try {
+        const f = join(dir, 's.png'); await simctl(['io', udid, 'screenshot', '--type=png', f])
+        const png = await readFile(f); const px = pngSize(png) || [1179, 2556]
+        let tree = await d.tree().catch(() => null)
+        if (!tree && cfg.ocr !== false && process.platform === 'darwin') tree = ocrTree(await ocr(png, ocrLangs(ctx.locale)), scale)
+        let crash = null
+        if (cfg.bundle) { const { stdout } = await simctl(['spawn', udid, 'launchctl', 'list']).catch(() => ({ stdout: '' })); if (stdout && !stdout.includes(cfg.bundle)) crash = `${cfg.bundle} is not running (crashed or quit)` }
+        return { png, tree, size: [Math.round(px[0] / scale), Math.round(px[1] / scale)], scale, device, crash, logErrors: null }
+      } finally { await rm(dir, { recursive: true, force: true }) }
+    },
+    async close () {}
+  }
+  return d
+}
+
+/* ---------- Electron (Playwright): the window is a page, captures are web captures ---------- */
+function electron (cfg, ctx) {
+  let app = null
+  const d = {
+    platform: 'electron', page: null, web: true,
+    async open ({ locale } = {}) {
+      if (app) await app.close().catch(() => {})
+      const { _electron } = await import('playwright')
+      const args = [...(cfg.main ? [resolve(ctx.root, cfg.main)] : []), ...(cfg.args || []), ...(locale ? [`--lang=${locale}`] : [])]
+      app = await _electron.launch({ ...(cfg.executable ? { executablePath: resolve(ctx.root, cfg.executable) } : {}), args, cwd: ctx.root, env: { ...process.env, ...(locale ? { LANG: `${locale.replace('-', '_')}.UTF-8` } : {}), ...(cfg.env || {}) } })
+      d.page = await app.firstWindow()
+      await d.page.waitForLoadState('domcontentloaded')
+      if (cfg.size) await d.page.setViewportSize({ width: cfg.size[0], height: cfg.size[1] }).catch(() => {})
+      d.app = app
+    },
+    async link (url) { await d.page.goto(url, { waitUntil: 'domcontentloaded' }) },
+    async tap (target) { if (typeof target === 'string') await d.page.locator(target).first().click(); else await d.page.mouse.click(target.x, target.y) },
+    async type (text) { await d.page.keyboard.type(String(text)) },
+    async back () { await d.page.goBack() },
+    async wait (w) { if (typeof w === 'number') await sleep(w); else if (w?.text) await d.page.getByText(w.text).first().waitFor(); else if (w?.selector) await d.page.locator(w.selector).first().waitFor() },
+    async close () { if (app) await app.close().catch(() => {}); app = null }
+  }
+  return d
+}
+
+/* ---------- macOS desktop apps: window screenshot, accessibility tree or OCR ---------- */
+function desktop (cfg, ctx) {
+  if (process.platform !== 'darwin') throw new UserError('Built-in desktop capture supports macOS. On Windows or Linux drive the app with your tool and capture with kaloko capture --image (and --tree).')
+  const name = cfg.name || cfg.bundle
+  if (!name) throw new UserError('environments.<env>.app.desktop needs name (the app as macOS shows it) or bundle.')
+  let win = null
+  const d = {
+    platform: 'desktop',
+    async open ({ reset, attach = false } = {}) {
+      if (cfg.launch !== false && !attach) {
+        if (reset ?? cfg.reset) await exec('osascript', ['-e', `quit app "${name.replace(/"/g, '')}"`]).catch(() => {})
+        await exec('open', [...(cfg.bundle ? ['-b', cfg.bundle] : ['-a', cfg.path ? resolve(ctx.root, cfg.path) : name]), ...(cfg.args?.length ? ['--args', ...cfg.args] : [])])
+        await sleep(cfg.launch_wait ?? 2000)
+      }
+      win = JSON.parse((await exec(await helper(), ['window', name])).stdout)
+    },
+    async link (url) { await exec('open', [...(cfg.bundle ? ['-b', cfg.bundle] : []), url]); await sleep(cfg.settle ?? 1000) },
+    async tap () { throw new UserError('Built-in desktop capture does not click; drive the app with your tool (or AppleScript in a step script) and let Kaloko capture.') },
+    async type () { throw new UserError('Built-in desktop capture does not type; use your tool or AppleScript in a step script.') },
+    async back () { throw new UserError('Desktop apps have no system back.') },
+    async wait (w) { if (typeof w === 'number') await sleep(w); else await waitFor(d, w) },
+    async tree () {
+      const ax = await exec(await helper(), ['ax', String(win.pid)]).then((r) => JSON.parse(r.stdout)).catch(() => null)
+      if (!ax) return null
+      const t = readTree(ax)
+      // window-relative logical coordinates
+      const shift = (n) => { if (n.bounds) n.bounds = { ...n.bounds, x: Math.round(n.bounds.x - win.x), y: Math.round(n.bounds.y - win.y) }; (n.children || []).forEach(shift) }
+      shift(t.root)
+      return t
+    },
+    async snapshot () {
+      win = JSON.parse((await exec(await helper(), ['window', name])).stdout)
+      const dir = await mkdtemp(join(tmpdir(), 'qawalk-mac-'))
+      try {
+        const f = join(dir, 'w.png')
+        await exec('screencapture', ['-x', '-o', `-l${win.id}`, f]).catch(async (e) => {
+          if (!/could not create image/i.test(e.message)) throw e
+          // the first refusal also puts this terminal into the Screen Recording list, so one switch grants it
+          await macPermissions(true).catch(() => {})
+          throw new UserError('macOS did not allow a screenshot of the window. Turn on this terminal (or the agent app) in System Settings → Privacy & Security → Screen Recording, restart it, and walk again. `kaloko doctor` shows the state.')
+        })
+        const png = await readFile(f); const px = pngSize(png) || [win.w, win.h]
+        const scale = Math.max(1, Math.round(px[0] / win.w))
+        let tree = await d.tree()
+        if (!tree && cfg.ocr !== false) tree = ocrTree(await ocr(png, ocrLangs(ctx.locale)), scale)
+        return { png, tree, size: [Math.round(px[0] / scale), Math.round(px[1] / scale)], scale, device: `macOS · ${win.owner}`, crash: null, logErrors: null }
+      } finally { await rm(dir, { recursive: true, force: true }) }
+    },
+    async close () {}
+  }
+  return d
+}
+
+async function point (d, target) {
+  if (target && typeof target === 'object' && 'x' in target) return target
+  const t = await d.tree()
+  if (!t) throw new UserError('No UI tree to find the element in; tap by coordinates { x, y }.')
+  const n = query(t.root, String(target)).find((x) => x.bounds)
+  if (!n) throw new Error(`no visible ${target}`)
+  return { x: n.bounds.x + n.bounds.w / 2, y: n.bounds.y + n.bounds.h / 2 }
+}
+async function waitFor (d, w, timeout = 15000) {
+  if (typeof w === 'number') return sleep(w)
+  const sel = w?.selector || (w?.text ? `[text*="${String(w.text).replace(/"/g, '')}"],[name*="${String(w.text).replace(/"/g, '')}"]` : null)
+  if (!sel) return
+  const end = Date.now() + (w.timeout || timeout)
+  while (Date.now() < end) { const t = await d.tree().catch(() => null); if (t && query(t.root, sel).length) return; await sleep(500) }
+  throw new Error(`timed out waiting for ${sel}`)
+}
+
+/** The driver of a scenario's platform, configured by the environment's `app.<platform>` block. */
+export function driverFor (platform, env, ctx) {
+  const cfg = env.app?.[platform]
+  if (!cfg && platform !== 'desktop') throw new UserError(`Environment ${env.name} has no app.${platform} block in kaloko.config.yml.`)
+  if (platform === 'android') return android(cfg, ctx)
+  if (platform === 'ios') return ios(cfg, ctx)
+  if (platform === 'electron') return electron(cfg, ctx)
+  if (platform === 'desktop') return desktop(cfg || {}, ctx)
+  throw new UserError(`Unknown platform ${platform}.`)
+}
+
+/** Tools available for app platforms (doctor). */
+export async function appTools () {
+  return { adb: await which('adb'), simctl: process.platform === 'darwin' && await which('xcrun'), idb: await which('idb'), swiftc: process.platform === 'darwin' && await which('xcrun') }
+}

package/src/lib/apps/helper.swift

@@ -0,0 +1,102 @@
+// Kaloko macOS helper: text recognition for screenshots without a UI tree (iOS Simulator without idb, any desktop
+// app), window lookup for desktop captures, and the accessibility tree of a running app. Compiled on first use
+// by the CLI (xcrun swiftc) into the user's cache; prints JSON on stdout.
+//   qawalk-helper ocr <image.png> [lang,lang]     → { "ocr": [{ text, x, y, w, h }], "width", "height" } (pixels)
+//   qawalk-helper window <app name or bundle id>  → { id, pid, owner, title, x, y, w, h } of its largest window
+//   qawalk-helper ax <pid>                        → { "ax": tree } (needs Accessibility permission)
+//   qawalk-helper permissions [request]           → { screen, accessibility } granted?; request asks macOS
+//                                                   (the system dialog adds this terminal to both lists)
+import Foundation
+import Vision
+import AppKit
+import ApplicationServices
+
+func out(_ obj: Any) {
+  let data = try! JSONSerialization.data(withJSONObject: obj, options: [])
+  FileHandle.standardOutput.write(data)
+}
+func fail(_ msg: String) -> Never { FileHandle.standardError.write((msg + "\n").data(using: .utf8)!); exit(1) }
+
+let args = CommandLine.arguments
+guard args.count >= 2 else { fail("usage: qawalk-helper ocr|window|ax|permissions <arg>") }
+if args[1] != "permissions" && args.count < 3 { fail("usage: qawalk-helper ocr|window|ax <arg>") }
+
+switch args[1] {
+case "ocr":
+  guard let img = NSImage(contentsOfFile: args[2]), let cg = img.cgImage(forProposedRect: nil, context: nil, hints: nil) else { fail("cannot read image") }
+  let req = VNRecognizeTextRequest()
+  req.recognitionLevel = .accurate
+  req.usesLanguageCorrection = true
+  if args.count > 3 { req.recognitionLanguages = args[3].split(separator: ",").map(String.init) }
+  let handler = VNImageRequestHandler(cgImage: cg, options: [:])
+  do { try handler.perform([req]) } catch { fail("ocr failed: \(error)") }
+  let w = Double(cg.width), h = Double(cg.height)
+  var lines: [[String: Any]] = []
+  for o in req.results ?? [] {
+    guard let c = o.topCandidates(1).first else { continue }
+    let b = o.boundingBox // normalized, origin bottom-left
+    lines.append(["text": c.string, "x": Int(b.minX * w), "y": Int((1 - b.maxY) * h), "w": Int(b.width * w), "h": Int(b.height * h), "confidence": Double(c.confidence)])
+  }
+  lines.sort { (($0["y"] as! Int), ($0["x"] as! Int)) < (($1["y"] as! Int), ($1["x"] as! Int)) }
+  out(["ocr": lines, "width": Int(w), "height": Int(h)])
+
+case "window":
+  let want = args[2].lowercased()
+  let apps = NSWorkspace.shared.runningApplications.filter { ($0.localizedName ?? "").lowercased() == want || ($0.bundleIdentifier ?? "").lowercased() == want }
+  let pids = Set(apps.map { $0.processIdentifier })
+  let list = (CGWindowListCopyWindowInfo([.optionOnScreenOnly, .excludeDesktopElements], kCGNullWindowID) as? [[String: Any]]) ?? []
+  var best: [String: Any]? = nil; var area = 0.0
+  for w in list {
+    let pid = (w[kCGWindowOwnerPID as String] as? Int32) ?? -1
+    let owner = (w[kCGWindowOwnerName as String] as? String) ?? ""
+    guard pids.contains(pid) || owner.lowercased() == want else { continue }
+    guard (w[kCGWindowLayer as String] as? Int) == 0, let b = w[kCGWindowBounds as String] as? [String: Double] else { continue }
+    let a = (b["Width"] ?? 0) * (b["Height"] ?? 0)
+    if a > area { area = a; best = ["id": w[kCGWindowNumber as String] as? Int ?? 0, "pid": Int(pid), "owner": owner, "title": w[kCGWindowName as String] as? String ?? "", "x": b["X"] ?? 0, "y": b["Y"] ?? 0, "w": b["Width"] ?? 0, "h": b["Height"] ?? 0] }
+  }
+  guard let found = best else { fail("no window of \(args[2]) on screen") }
+  out(found)
+
+case "ax":
+  guard AXIsProcessTrusted() else { fail("accessibility: grant this terminal Accessibility access (System Settings → Privacy & Security → Accessibility)") }
+  guard let pid = Int32(args[2]) else { fail("pid expected") }
+  let app = AXUIElementCreateApplication(pid)
+  func attr(_ e: AXUIElement, _ name: String) -> AnyObject? { var v: AnyObject?; return AXUIElementCopyAttributeValue(e, name as CFString, &v) == .success ? v : nil }
+  var count = 0
+  func node(_ e: AXUIElement, _ depth: Int) -> [String: Any] {
+    count += 1
+    var n: [String: Any] = [:]
+    n["role"] = attr(e, kAXRoleAttribute) as? String ?? ""
+    if let t = attr(e, kAXTitleAttribute) as? String, !t.isEmpty { n["title"] = t }
+    if let d = attr(e, kAXDescriptionAttribute) as? String, !d.isEmpty { n["description"] = d }
+    if let v = attr(e, kAXValueAttribute) { if let s = v as? String { n["value"] = s } else if let num = v as? NSNumber { n["value"] = num.stringValue } }
+    if let i = attr(e, "AXIdentifier") as? String, !i.isEmpty { n["identifier"] = i }
+    if let en = attr(e, kAXEnabledAttribute) as? Bool { n["enabled"] = en }
+    if let f = attr(e, kAXFocusedAttribute) as? Bool, f { n["focused"] = true }
+    var pos = CGPoint.zero, size = CGSize.zero
+    if let p = attr(e, kAXPositionAttribute) { AXValueGetValue(p as! AXValue, .cgPoint, &pos) }
+    if let s = attr(e, kAXSizeAttribute) { AXValueGetValue(s as! AXValue, .cgSize, &size) }
+    n["frame"] = ["x": Double(pos.x), "y": Double(pos.y), "w": Double(size.width), "h": Double(size.height)]
+    var names: CFArray?
+    if AXUIElementCopyActionNames(e, &names) == .success, let a = names as? [String], !a.isEmpty { n["actions"] = a }
+    if depth < 40 && count < 4000, let kids = attr(e, kAXChildrenAttribute) as? [AXUIElement] { n["children"] = kids.map { node($0, depth + 1) } }
+    return n
+  }
+  // only real windows: without full access macOS answers window queries with the application element itself
+  let isWindow = { (e: AXUIElement) -> Bool in (attr(e, kAXRoleAttribute) as? String) == "AXWindow" }
+  var windows = ((attr(app, kAXWindowsAttribute) as? [AXUIElement]) ?? []).filter(isWindow)
+  if windows.isEmpty { windows = ((attr(app, kAXChildrenAttribute) as? [AXUIElement]) ?? []).filter(isWindow) }
+  guard let win = windows.first else { fail("accessibility: the app's windows are not readable; give this terminal Accessibility and Screen Recording access and restart it") }
+  out(["ax": node(win, 0)])
+
+case "permissions":
+  let ask = args.count > 2 && args[2] == "request"
+  var screen = CGPreflightScreenCaptureAccess()
+  if !screen && ask { screen = CGRequestScreenCaptureAccess() }
+  let opts = [kAXTrustedCheckOptionPrompt.takeUnretainedValue() as String: ask] as CFDictionary
+  let ax = AXIsProcessTrustedWithOptions(opts)
+  out(["screen": screen, "accessibility": ax])
+
+default:
+  fail("unknown command \(args[1])")
+}