kaloko 0.4.11 → 0.5.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +45 -4
- package/package.json +1 -1
- package/schemas/config.v1.json +42 -2
- package/schemas/run.v1.json +33 -2
- package/schemas/scenario.v1.json +107 -16
- package/skills/kaloko/SKILL.md +12 -4
- package/skills/kaloko/reference/acceptance.md +74 -3
- package/skills/kaloko/reference/apps.md +103 -8
- package/skills/kaloko/reference/ci.md +3 -1
- package/skills/kaloko/reference/collaboration.md +35 -6
- package/skills/kaloko/reference/design.md +39 -0
- package/skills/kaloko/reference/devices.md +109 -0
- package/skills/kaloko/reference/environments.md +6 -0
- package/skills/kaloko/reference/execution.md +15 -0
- package/skills/kaloko/reference/mcp.md +5 -2
- package/src/cli.mjs +366 -192
- package/src/lib/access.mjs +5 -2
- package/src/lib/apps/devices.mjs +187 -0
- package/src/lib/apps/drivers.mjs +11 -3
- package/src/lib/apps/ios-device.mjs +78 -0
- package/src/lib/apps/tree.mjs +9 -1
- package/src/lib/apps/webview.mjs +54 -0
- package/src/lib/apps/windows.mjs +108 -0
- package/src/lib/apps/windows.ps1 +78 -0
- package/src/lib/assertions.mjs +300 -0
- package/src/lib/build.mjs +4 -1
- package/src/lib/capture.mjs +146 -119
- package/src/lib/ci.mjs +8 -4
- package/src/lib/design-cli.mjs +35 -6
- package/src/lib/design.mjs +3 -1
- package/src/lib/devices.mjs +85 -0
- package/src/lib/devspec.mjs +3 -2
- package/src/lib/dimensions.mjs +326 -0
- package/src/lib/doctor.mjs +31 -6
- package/src/lib/drift.mjs +8 -4
- package/src/lib/evaluate.mjs +25 -7
- package/src/lib/import.mjs +2 -1
- package/src/lib/jev.mjs +3 -1
- package/src/lib/manifest-core.mjs +12 -3
- package/src/lib/probe.mjs +60 -0
- package/src/lib/recheck.mjs +6 -1
- package/src/lib/scenario.mjs +3 -0
- package/src/lib/signoff-core.mjs +138 -0
- package/src/lib/signoff.mjs +83 -0
- package/src/lib/stability-core.mjs +4 -2
- package/src/lib/stability.mjs +82 -0
- package/src/lib/walk-dimensions.mjs +83 -0
- package/viewer/index.html +739 -48
package/src/cli.mjs
CHANGED
|
@@ -18,7 +18,8 @@ import { configureGate } from './lib/linkcheck.mjs'
|
|
|
18
18
|
import { rememberEnvironment } from './lib/secrets.mjs'
|
|
19
19
|
import { evaluateRun, semanticPrefetch } from './lib/evaluate.mjs'
|
|
20
20
|
import { buildManifest, writePreview, attentionLines } from './lib/build.mjs'
|
|
21
|
-
import { stabilityCmd } from './lib/stability.mjs'
|
|
21
|
+
import { stabilityCmd, pullDrawn } from './lib/stability.mjs'
|
|
22
|
+
import { signoffCmd } from './lib/signoff.mjs'
|
|
22
23
|
import { localized, readJson, UserError, globToRegExp, failureStreak, expiryNote } from './lib/util.mjs'
|
|
23
24
|
import { packageRoot } from './lib/schema.mjs'
|
|
24
25
|
import { compareRuns, formatComparison } from './lib/compare.mjs'
|
|
@@ -51,9 +52,13 @@ import { exportDesign } from './lib/design-export.mjs'
|
|
|
51
52
|
import { driverFor, isApp, pngSize, ocr as ocrImage } from './lib/apps/drivers.mjs'
|
|
52
53
|
import { captureApp } from './lib/apps/capture.mjs'
|
|
53
54
|
import { readTree, flatten } from './lib/apps/tree.mjs'
|
|
54
|
-
import { recordSettings, recordingSession, recordingFiles, traceFiles, stripRecordings, withoutRecordings } from './lib/recording.mjs'
|
|
55
|
+
import { recordSettings, recordingSession, parseRecord, recordingFiles, traceFiles, stripRecordings, withoutRecordings } from './lib/recording.mjs'
|
|
55
56
|
import { nativeSession } from './lib/apps/recording-native.mjs'
|
|
56
57
|
import { videoCmd } from './lib/video-cli.mjs'
|
|
58
|
+
import { passesOf, enginesOf, contextOptionsOf, launchEngine, pickKeys, variantFields, variantOf, dimensionValues, BROWSERS } from './lib/dimensions.mjs'
|
|
59
|
+
import { startScope, scopeKeys, dimensionLine, baseViewports, engineKeys, appDeviceGroups } from './lib/walk-dimensions.mjs'
|
|
60
|
+
import { deviceConfig, prepareAndroid, readyAndroid, finishAndroid, prepareSimulator, applyAppAppearance } from './lib/apps/devices.mjs'
|
|
61
|
+
import { attachAndroidWebView } from './lib/apps/webview.mjs'
|
|
57
62
|
|
|
58
63
|
|
|
59
64
|
const HELP = `kaloko — walk, assess and approve what your agents built, on one flow canvas
|
|
@@ -91,9 +96,19 @@ Run
|
|
|
91
96
|
start --scenario <file> --env <name> [--locales cs,en] [--viewports desktop,mobile] [--executor <name>]
|
|
92
97
|
[--app-version <v>] (default: the git tag on HEAD; documentation captures carry it)
|
|
93
98
|
Create a run directory and make it current
|
|
99
|
+
[--viewports iphone-15,ipad-landscape,desktop] Viewport names of the scenario or device presets
|
|
100
|
+
(kaloko devices lists them; <preset>-landscape turns one sideways)
|
|
101
|
+
[--browsers chromium,webkit,firefox] [--color-schemes light,dark] [--reduced-motion no-preference,reduce]
|
|
102
|
+
[--forced-colors none,active] [--display-modes browser,standalone]
|
|
103
|
+
More dimensions: every step is captured in each combination (keys like
|
|
104
|
+
mobile.webkit.dark); default from the scenario, then walk: in the config
|
|
105
|
+
[--devices pixel-8-api-34,iphone-15-ios-18] App scenarios: the emulators, simulators or devices to walk on
|
|
94
106
|
browser start|stop|status [--env e] [--headed] One shared browser per environment (port = browser.port + index)
|
|
95
107
|
Shared Chromium with CDP (agents, scripts and people drive it)
|
|
96
|
-
browser install [--with-deps]
|
|
108
|
+
browser install [--with-deps] [--browsers chromium,webkit,firefox]
|
|
109
|
+
Download the engines of kaloko's Playwright (default Chromium; CI:
|
|
110
|
+
KALOKO_BROWSER_CHANNEL=chromium uses it); WebKit and Firefox for cross-browser runs
|
|
111
|
+
devices [--json] Device presets for viewports: (phones, foldables, tablets, laptop, desktop, TV)
|
|
97
112
|
open --step <id> [--locale l] Navigate the current page to the step URL
|
|
98
113
|
session reset New visitor: clear cookies and storage in the shared browser
|
|
99
114
|
capture --step <id> [--locale l] [--url <url>] [--page <substring>] [--viewports a,b]
|
|
@@ -115,10 +130,14 @@ Run
|
|
|
115
130
|
A person signs in once (SSO, MFA) in a visible browser; the session is saved
|
|
116
131
|
for that account and walk reuses it (never uploaded)
|
|
117
132
|
auth list Saved sessions per environment and account, with their age
|
|
118
|
-
walk [--steps a,b] [--locales l] [--keep-going] [--ephemeral]
|
|
133
|
+
walk [--steps a,b] [--locales l] [--keep-going] [--ephemeral] [--browsers webkit]
|
|
119
134
|
Run playwright steps (execution.script) and capture them; a session stops
|
|
120
135
|
after 3 steps in a row fail with the same error unless --keep-going
|
|
121
136
|
One walk at a time per shared browser; --ephemeral walks in an own browser
|
|
137
|
+
A run with several browsers or device presets walks once per engine and device
|
|
138
|
+
profile; --browsers walks only those engines of the run
|
|
139
|
+
[--devices pixel-8-api-34] App runs walk once per device of devices: (emulators start, phones wake up);
|
|
140
|
+
--devices walks only those; dark mode and reduced motion are set per capture
|
|
122
141
|
Semantic questions start as each step is captured; the walk ends with the
|
|
123
142
|
verdicts and what failed (--no-evaluate skips that; kaloko evaluate redoes it)
|
|
124
143
|
[--items a,b] [--limit N] Stack steps (kind: stack, many pages of one template): only pages whose id or
|
|
@@ -199,6 +218,15 @@ Results
|
|
|
199
218
|
Regions that change on every run or move while the page is idle (clocks, ads,
|
|
200
219
|
carousels): ignore suggestions per step in <run>/stability.yml; --apply adds
|
|
201
220
|
them to the scenario (ignore:), and pixel diffs then skip them
|
|
221
|
+
signoff <run-id> [--out dir] [--lang cs|en] [--json]
|
|
222
|
+
The signed acceptance record of an accepted run (Business): JSON and PDF with who
|
|
223
|
+
approved what and when, criteria results and capture hashes, checked against the
|
|
224
|
+
service's public key
|
|
225
|
+
signoff --verify <file.json|file.pdf> [--key file|url] [--json]
|
|
226
|
+
Check a record someone handed over (no token needed); exit 1 when it does not verify
|
|
227
|
+
stability --pull [--scenario id] [--json]
|
|
228
|
+
Regions people drew on screenshots on kaloko.app (step detail → Page → Draw a
|
|
229
|
+
region) go into the scenario's ignore: of their steps; walk them again after
|
|
202
230
|
compare [--run id] [--baseline id | --design] [--json]
|
|
203
231
|
Text diff of two runs: step statuses, criterion verdicts, scores and evidence
|
|
204
232
|
(default: current run against the scenario baseline, else the previous run)
|
|
@@ -239,6 +267,15 @@ Design (environments with kind: design; live HTML steps in a folder)
|
|
|
239
267
|
Common options: --run <id|dir> (or KALOKO_RUN) --config <file> KALOKO_AGENT=<name> labels writes by an agent
|
|
240
268
|
`
|
|
241
269
|
|
|
270
|
+
/** A comment's drawing for the terminal: "· box 120,340 300×80 px on desktop" (CSS px when the size is known). */
|
|
271
|
+
export function regionText (g) {
|
|
272
|
+
if (!g) return ''
|
|
273
|
+
const pct = (v) => `${Math.round(v * 100)}%`
|
|
274
|
+
const where = g.px ? `${g.px[0]},${g.px[1]} ${g.px[2]}×${g.px[3]} px` : `${pct(g.box[0])},${pct(g.box[1])} ${pct(g.box[2])}×${pct(g.box[3])}`
|
|
275
|
+
const kind = g.kind === 'arrow' ? 'arrow over' : g.kind === 'pen' ? 'drawing over' : 'box'
|
|
276
|
+
return ` · ${kind} ${where}${g.viewport ? ` on ${g.viewport}` : ''}`
|
|
277
|
+
}
|
|
278
|
+
|
|
242
279
|
/** validate: scenarios the service still lists as active whose scenario file no longer exists here (#58). */
|
|
243
280
|
async function warnRetired (args) {
|
|
244
281
|
if (args.offline) return
|
|
@@ -353,10 +390,20 @@ async function scenarioIdOf (args, config, root) {
|
|
|
353
390
|
throw new UserError('--scenario <id|file> is required when there is no current run.')
|
|
354
391
|
}
|
|
355
392
|
|
|
393
|
+
/**
|
|
394
|
+
* Keys a capture of the current page takes (kaloko capture): the Chromium keys without a device preset, whose
|
|
395
|
+
* pixel ratio, touch and agent the page does not have; a preset key only when --viewports names it.
|
|
396
|
+
*/
|
|
397
|
+
function manualKeys (run, pick) {
|
|
398
|
+
const scope = run.meta.scope
|
|
399
|
+
if (pick) return scopeViewports(run, pick)
|
|
400
|
+
const keys = engineKeys(scope, 'chromium').filter((k) => !scope.devices?.[variantOf(scope, k).viewport]?.emulation)
|
|
401
|
+
return scopeKeys(scope, keys.length ? keys : engineKeys(scope, 'chromium'), null)
|
|
402
|
+
}
|
|
403
|
+
|
|
404
|
+
/** { key: [w, h] } a capture on a Chromium page takes: every Chromium key of the run, or those of `pick`. */
|
|
356
405
|
function scopeViewports (run, pick) {
|
|
357
|
-
|
|
358
|
-
const names = pick || Object.keys(all)
|
|
359
|
-
return Object.fromEntries(names.map((n) => { if (!all[n]) throw new UserError(`Unknown viewport "${n}".`); return [n, all[n]] }))
|
|
406
|
+
return scopeKeys(run.meta.scope, engineKeys(run.meta.scope, 'chromium'), pick)
|
|
360
407
|
}
|
|
361
408
|
|
|
362
409
|
async function scenarioFiles (config, root) {
|
|
@@ -472,14 +519,17 @@ async function exportScenario ({ args, config, root, client }) {
|
|
|
472
519
|
* Walks a stack step: resolves its pages, then opens and captures each one with the step's criteria and index
|
|
473
520
|
* fields (HTTP status of the main document included). A step script, when given, runs per page with `item`.
|
|
474
521
|
*/
|
|
475
|
-
async function walkStack ({ node, locale, page, env, run, scenario, scenarioDir, config, consoleFile, args }) {
|
|
522
|
+
async function walkStack ({ node, locale, page, env, run, scenario, scenarioDir, config, consoleFile, args, keys = null, tag = '' }) {
|
|
476
523
|
const { items, total, truncated } = await resolveItems({ node, base: baseUrl(env, locale), page, scenarioDir, env, locale, limit: Number(args.limit) || undefined })
|
|
477
524
|
const only = list(args.items)
|
|
478
525
|
const chosen = only ? items.filter((it) => only.some((q) => it.id === q || it.path.includes(q))) : items
|
|
479
|
-
const
|
|
480
|
-
const
|
|
481
|
-
const
|
|
482
|
-
|
|
526
|
+
const passKeys = keys || engineKeys(run.meta.scope, 'chromium')
|
|
527
|
+
const bases = baseViewports(run.meta.scope)
|
|
528
|
+
const vpNames = (node.items.viewports || []).filter((v) => bases.includes(v) || run.meta.scope.viewports[v])
|
|
529
|
+
const passBases = baseViewports({ ...run.meta.scope, viewports: Object.fromEntries(passKeys.map((k) => [k, run.meta.scope.viewports[k]])) })
|
|
530
|
+
const viewports = scopeKeys(run.meta.scope, passKeys, vpNames.length ? vpNames : [passBases[0] || bases[0]])
|
|
531
|
+
if (!Object.keys(viewports).length) { say(`▦ ${node.id} ${locale}${tag}: no viewport of this step in this pass`); return [] }
|
|
532
|
+
say(`▦ ${node.id} ${locale}${tag}: ${chosen.length} page(s)${truncated ? ` — limit ${Number(args.limit) || node.items.limit || DEFAULT_LIMIT} reached (${total}+ found; raise items.limit or pass --limit)` : ''}`)
|
|
483
533
|
const fields = fieldsOf(node)
|
|
484
534
|
const script = node.execution?.script ? await import(pathToFileURL(resolve(scenarioDir, node.execution.script)).href) : null
|
|
485
535
|
let status = null
|
|
@@ -493,7 +543,7 @@ async function walkStack ({ node, locale, page, env, run, scenario, scenarioDir,
|
|
|
493
543
|
status = null
|
|
494
544
|
const since = Date.now()
|
|
495
545
|
let captured = false
|
|
496
|
-
const capture = (opts = {}) => { captured = true; return captureStep({ page, node, locale, viewports: opts.viewports ?
|
|
546
|
+
const capture = (opts = {}) => { captured = true; return captureStep({ page, node, locale, viewports: opts.viewports ? scopeKeys(run.meta.scope, passKeys, opts.viewports) : viewports, run, consoleFile, since, item, fields, maxChars: config.evaluators?.jev?.max_state_chars, options: captureOptions(scenario, node), extra: { http_status: status } }) }
|
|
497
547
|
try {
|
|
498
548
|
if (script) await script.default({ page, item, url: () => item.url, locale, env, run: run.meta, capture, acceptDialogs: () => acceptDialogs(page) })
|
|
499
549
|
else await page.goto(item.url, { waitUntil: 'domcontentloaded', timeout: 30000 })
|
|
@@ -501,12 +551,12 @@ async function walkStack ({ node, locale, page, env, run, scenario, scenarioDir,
|
|
|
501
551
|
ok++; streak.ok('stack')
|
|
502
552
|
} catch (error) {
|
|
503
553
|
failed.push(`${item.id} (${item.path}): ${error.message.split('\n')[0]}`)
|
|
504
|
-
if (streak.record('stack', error.message)) say(` stopping ${node.id} (${locale}): the last 5 pages failed with the same error`)
|
|
554
|
+
if (streak.record('stack', error.message)) say(` stopping ${node.id} (${locale}${tag}): the last 5 pages failed with the same error`)
|
|
505
555
|
}
|
|
506
556
|
if ((i + 1) % 25 === 0 && i + 1 < chosen.length) say(` ${node.id} ${locale}: ${i + 1}/${chosen.length}`)
|
|
507
557
|
}
|
|
508
558
|
} finally { page.off('response', onResponse) }
|
|
509
|
-
say(`${failed.length ? '✗' : '✓'} ${node.id} ${locale}: ${ok}/${chosen.length} page(s) captured`)
|
|
559
|
+
say(`${failed.length ? '✗' : '✓'} ${node.id} ${locale}${tag}: ${ok}/${chosen.length} page(s) captured`)
|
|
510
560
|
return failed
|
|
511
561
|
}
|
|
512
562
|
|
|
@@ -529,14 +579,17 @@ function lazyEncoder (config) {
|
|
|
529
579
|
const appViewport = (run) => Object.keys(run.meta.scope.viewports || {})[0] || null
|
|
530
580
|
|
|
531
581
|
/**
|
|
532
|
-
* Walks an app scenario (platform android, ios, electron, desktop) with the environment's driver: per
|
|
533
|
-
* app starts fresh, each step runs its script (with `app`, the
|
|
582
|
+
* Walks an app scenario (platform android, ios, electron, desktop, windows) with the environment's driver: per
|
|
583
|
+
* device of the run (devices:, batch E) and locale the app starts fresh, each step runs its script (with `app`, the
|
|
584
|
+
* driver) or opens its deep link, then is captured in each of the device's keys (dark mode, reduced motion set on
|
|
585
|
+
* the device). `capture({ webview: true })` (or app.android.webview) captures an Android WebView's page instead.
|
|
534
586
|
*/
|
|
535
587
|
async function walkApp ({ config, root, run, scenario, env, args }) {
|
|
536
588
|
const platform = scenario.platform
|
|
537
589
|
const pick = list(args.steps)
|
|
538
590
|
const locales = list(args.locales) || run.meta.scope.locales
|
|
539
591
|
const scenarioDir = resolve(root, run.meta.scenario_file, '..')
|
|
592
|
+
const scope = run.meta.scope
|
|
540
593
|
const pending = []; const failed = []; const skipped = []
|
|
541
594
|
// WebP and thumbnails are encoded in a headless Chrome page (no native image dependency); PNG without Chrome
|
|
542
595
|
const enc = await launchEphemeral(config).catch(() => null)
|
|
@@ -545,65 +598,111 @@ async function walkApp ({ config, root, run, scenario, env, args }) {
|
|
|
545
598
|
const maxChars = config.evaluators?.jev?.max_state_chars
|
|
546
599
|
// recordings and traces (batch D): Electron windows use the screencast recorder, devices record themselves
|
|
547
600
|
const recSettings = recordSettings({ args, config, scenario })
|
|
601
|
+
const groups = appDeviceGroups(scope, list(args.devices))
|
|
602
|
+
// app captures carry the scheme and motion of their key (no browser: the app draws itself)
|
|
603
|
+
const fieldsOf = (key) => { const { browser: _b, ...f } = variantFields(scope, key); return f }
|
|
548
604
|
try {
|
|
549
|
-
for (const
|
|
550
|
-
const
|
|
551
|
-
|
|
552
|
-
|
|
553
|
-
|
|
554
|
-
say(`${platform} ${locale}: ${driver.web ? 'Electron window' : 'app'} ready`)
|
|
555
|
-
} catch (error) { failed.push(`${platform} (${locale}): ${error.message.split('\n')[0]}`); say(`✗ ${platform} ${locale}: ${error.message.split('\n')[0]}`); await driver.close().catch(() => {}); if (error instanceof UserError) break; continue }
|
|
556
|
-
const rec = driver.web
|
|
557
|
-
? await recordingSession({ page: driver.page, run, scenario, locale, settings: recSettings, viewports: run.meta.scope.viewports, encoder: async () => encoder, say, label: `${platform} ${locale}` }).catch(() => null)
|
|
558
|
-
: await nativeSession({ platform, env, run, scenario, locale, settings: recSettings, driver, encoder: async () => encoder, say }).catch(() => null)
|
|
605
|
+
for (const group of groups) {
|
|
606
|
+
const tag = group.device && groups.length > 1 ? ` · ${group.title}` : ''
|
|
607
|
+
if (groups.length > 1) say(`\n${group.title}: ${group.keys.join(', ')}`)
|
|
608
|
+
// the device: its entry in app.<platform>.devices or what its name says; an emulator starts, a phone wakes up
|
|
609
|
+
let cfg = deviceConfig(platform, env.app?.[platform] || {}, group.device)
|
|
559
610
|
try {
|
|
560
|
-
|
|
561
|
-
|
|
562
|
-
|
|
563
|
-
|
|
564
|
-
|
|
565
|
-
|
|
566
|
-
|
|
567
|
-
|
|
568
|
-
|
|
569
|
-
|
|
570
|
-
|
|
571
|
-
|
|
572
|
-
|
|
573
|
-
|
|
574
|
-
|
|
575
|
-
|
|
576
|
-
|
|
577
|
-
|
|
578
|
-
|
|
579
|
-
|
|
580
|
-
|
|
581
|
-
|
|
582
|
-
|
|
611
|
+
if (platform === 'android') { cfg = await prepareAndroid(cfg, { say }); await readyAndroid(cfg) }
|
|
612
|
+
if (platform === 'ios' && group.device) cfg = await prepareSimulator(cfg)
|
|
613
|
+
} catch (error) { failed.push(`${group.device || platform}: ${error.message.split('\n')[0]}`); say(`✗ ${group.device || platform}: ${error.message.split('\n')[0]}`); continue }
|
|
614
|
+
const denv = { ...env, app: { ...(env.app || {}), [platform]: cfg } }
|
|
615
|
+
// recordings and traces land on this device's keys (a failed step without a capture: its first key)
|
|
616
|
+
const grun = { ...run, meta: { ...run.meta, scope: { ...scope, viewports: scopeKeys(scope, group.keys) } } }
|
|
617
|
+
let look = { key: null, undo: async () => {} } // the system appearance the device is in now
|
|
618
|
+
const appearance = async (driver, key) => {
|
|
619
|
+
const v = scope.variants?.[key] || null
|
|
620
|
+
const want = JSON.stringify([v?.color_scheme || null, v?.reduced_motion || null])
|
|
621
|
+
if (look.key === want) return look.notes || []
|
|
622
|
+
await look.undo()
|
|
623
|
+
const r = await applyAppAppearance(platform, cfg, driver, v)
|
|
624
|
+
look = { key: want, undo: r.undo, notes: r.notes }
|
|
625
|
+
return r.notes
|
|
626
|
+
}
|
|
627
|
+
for (const locale of locales) {
|
|
628
|
+
const driver = driverFor(platform, denv, { root, scenarioDir, locale })
|
|
629
|
+
let webview = null
|
|
630
|
+
if (platform === 'android') driver.webview = async () => (webview ||= await attachAndroidWebView(cfg)).page
|
|
631
|
+
try {
|
|
632
|
+
await driver.open({ locale, reset: true })
|
|
633
|
+
if (driver.web) observeConsole(driver.page)
|
|
634
|
+
say(`${platform} ${locale}${tag}: ${driver.web ? 'Electron window' : 'app'} ready`)
|
|
635
|
+
} catch (error) { failed.push(`${platform} (${locale}${tag}): ${error.message.split('\n')[0]}`); say(`✗ ${platform} ${locale}${tag}: ${error.message.split('\n')[0]}`); await driver.close().catch(() => {}); if (error instanceof UserError) break; continue }
|
|
636
|
+
const rec = driver.web
|
|
637
|
+
? await recordingSession({ page: driver.page, run: grun, scenario, locale, settings: recSettings, viewports: grun.meta.scope.viewports, encoder: async () => encoder, say, label: `${platform} ${locale}${tag}` }).catch(() => null)
|
|
638
|
+
: await nativeSession({ platform, env: denv, run: grun, scenario, locale, settings: recSettings, driver, encoder: async () => encoder, say }).catch(() => null)
|
|
639
|
+
try {
|
|
640
|
+
for (const node of scenario.nodes) {
|
|
641
|
+
if (node.kind === 'decision' || (pick && !pick.includes(node.id))) continue
|
|
642
|
+
if (node.kind === 'stack') { pending.push(`${node.id} (${locale}): stack steps list web pages; not walked on ${platform}`); continue }
|
|
643
|
+
if (node.kind === 'email' && group !== groups[0]) continue // one message, captured on the first device
|
|
644
|
+
if (node.kind === 'email' && node.mail?.to) {
|
|
645
|
+
try { const r = await captureMail({ config, root, run, env, node, locale, to: expandRun(node.mail.to, run.meta.short), subject: localized(node.mail?.subject, locale), timeoutMs: 45000 }); say(`✓ ${node.id} ${locale}: "${r.subject}"`) } catch (error) { failed.push(`${node.id} (${locale}): ${error.message.split('\n')[0]}`); say(`✗ ${node.id} ${locale}: ${error.message.split('\n')[0]}`) }
|
|
646
|
+
continue
|
|
647
|
+
}
|
|
648
|
+
const scripted = ['playwright', 'script'].includes(node.execution?.strategy) && node.execution?.script
|
|
649
|
+
const link = linkFor(env, node, locale)
|
|
650
|
+
if (!scripted && !(link && ['playwright', 'script'].includes(node.execution?.strategy))) { pending.push(`${node.id} (${locale}): ${node.execution?.strategy || 'agent'} — ${(node.execution?.instructions || '').split('\n')[0]}`); continue }
|
|
651
|
+
const streakKey = `${group.device || ''}|${locale}`
|
|
652
|
+
if (streak.stopped(streakKey)) { skipped.push(`${node.id} (${locale}${tag})`); continue }
|
|
653
|
+
const records = []
|
|
654
|
+
try {
|
|
655
|
+
let captured = false
|
|
656
|
+
await rec?.beforeStep(node)
|
|
657
|
+
const capture = async (o = {}) => {
|
|
658
|
+
captured = true
|
|
659
|
+
await rec?.beforeCapture(node)
|
|
660
|
+
// a named viewport: within this device, or any of the run when it has no devices (as before)
|
|
661
|
+
const keys = o.viewport ? pickKeys(scope, group.device ? group.keys : Object.keys(scope.viewports), [o.viewport]) : group.keys
|
|
662
|
+
if (!keys.length) throw new UserError(`Unknown viewport "${o.viewport}" for ${group.device || platform}.`)
|
|
663
|
+
const out = []
|
|
664
|
+
for (const key of keys) {
|
|
665
|
+
if (driver.web) {
|
|
666
|
+
// Electron: a web capture; captureStep emulates the key's scheme and motion itself
|
|
667
|
+
const [w, h] = await driver.page.evaluate(() => [innerWidth, innerHeight])
|
|
668
|
+
out.push(...await captureStep({ page: driver.page, node, locale, options: captureOptions(scenario, node), viewports: { [key]: [w, h] }, run, consoleFile: null, maxChars, extra: { platform, device: 'Electron' } }))
|
|
669
|
+
continue
|
|
670
|
+
}
|
|
671
|
+
const notes = await appearance(driver, key)
|
|
672
|
+
if (o.webview || (cfg.webview && o.webview !== false)) {
|
|
673
|
+
if (!driver.webview) throw new UserError(`WebView captures need an Android app; on ${platform} capture the screen (tree or OCR).`)
|
|
674
|
+
const page = await driver.webview()
|
|
675
|
+
const [w, h] = await page.evaluate(() => [innerWidth, innerHeight])
|
|
676
|
+
out.push(...await captureStep({ page, node, locale, options: captureOptions(scenario, node), viewports: { [key]: [w, h] }, run, consoleFile: null, maxChars, extra: { platform, device: cfg.device || null, webview: true, ...fieldsOf(key) } }))
|
|
677
|
+
continue
|
|
678
|
+
}
|
|
679
|
+
out.push(await captureApp({ snapshot: await driver.snapshot(), node, locale, viewport: key, run, platform, encoder, maxChars, extra: { url: link, ...fieldsOf(key), ...(notes.length ? { notes } : {}) } }))
|
|
680
|
+
}
|
|
681
|
+
records.push(...out)
|
|
682
|
+
return out.length === 1 ? out[0] : out
|
|
583
683
|
}
|
|
584
|
-
|
|
585
|
-
|
|
586
|
-
|
|
684
|
+
if (scripted) {
|
|
685
|
+
const mod = await import(pathToFileURL(resolve(scenarioDir, node.execution.script)).href)
|
|
686
|
+
await mod.default({ app: driver, page: driver.page || null, webview: driver.webview || null, locale, node: { id: node.id, title: localized(node.title, locale), kind: node.kind }, device: group.device, env, run: run.meta, capture, link: (id = node.id) => linkFor(env, stepOf(scenario, id), locale), url: (id = node.id) => linkFor(env, stepOf(scenario, id), locale) })
|
|
687
|
+
} else await driver.link(link)
|
|
688
|
+
if (!captured) await capture()
|
|
689
|
+
await rec?.afterStep(node, driver.web ? { records } : { record: records[0] })
|
|
690
|
+
say(`✓ ${node.id} ${locale}${tag}`)
|
|
691
|
+
streak.ok(streakKey)
|
|
692
|
+
} catch (error) {
|
|
693
|
+
await rec?.afterStep(node, driver.web ? { error, records } : { error, record: records[0] }).catch(() => {})
|
|
694
|
+
failed.push(`${node.id} (${locale}${tag}): ${error.message.split('\n')[0]}`)
|
|
695
|
+
say(`✗ ${node.id} ${locale}${tag}: ${error.message.split('\n')[0]}`)
|
|
696
|
+
if (streak.record(streakKey, error.message)) say(` stopping ${locale}${tag}: the last 3 steps failed with the same error`)
|
|
587
697
|
}
|
|
588
|
-
if (scripted) {
|
|
589
|
-
const mod = await import(pathToFileURL(resolve(scenarioDir, node.execution.script)).href)
|
|
590
|
-
await mod.default({ app: driver, page: driver.page || null, locale, node: { id: node.id, title: localized(node.title, locale), kind: node.kind }, env, run: run.meta, capture, link: (id = node.id) => linkFor(env, stepOf(scenario, id), locale), url: (id = node.id) => linkFor(env, stepOf(scenario, id), locale) })
|
|
591
|
-
} else await driver.link(link)
|
|
592
|
-
if (!captured) await capture()
|
|
593
|
-
await rec?.afterStep(node, driver.web ? { records } : { record: records[0] })
|
|
594
|
-
say(`✓ ${node.id} ${locale}`)
|
|
595
|
-
streak.ok(locale)
|
|
596
|
-
} catch (error) {
|
|
597
|
-
await rec?.afterStep(node, driver.web ? { error, records } : { error, record: records[0] }).catch(() => {})
|
|
598
|
-
failed.push(`${node.id} (${locale}): ${error.message.split('\n')[0]}`)
|
|
599
|
-
say(`✗ ${node.id} ${locale}: ${error.message.split('\n')[0]}`)
|
|
600
|
-
if (streak.record(locale, error.message)) say(` stopping ${locale}: the last 3 steps failed with the same error`)
|
|
601
698
|
}
|
|
699
|
+
} finally {
|
|
700
|
+
for (const n of (await rec?.finish().catch((e) => [`recording: ${e.message}`])) || []) say(` ${n}`)
|
|
701
|
+
await webview?.close(); await driver.close().catch(() => {})
|
|
602
702
|
}
|
|
603
|
-
} finally {
|
|
604
|
-
for (const n of (await rec?.finish().catch((e) => [`recording: ${e.message}`])) || []) say(` ${n}`)
|
|
605
|
-
await driver.close().catch(() => {})
|
|
606
703
|
}
|
|
704
|
+
await look.undo().catch(() => {})
|
|
705
|
+
if (platform === 'android') await finishAndroid(cfg)
|
|
607
706
|
}
|
|
608
707
|
} finally { await enc?.close().catch(() => {}) }
|
|
609
708
|
if (pending.length) say(`\nSteps for an agent or a person (drive the app, then kaloko capture --step <id> --image shot.png [--tree dump.xml|json]):\n - ${[...new Set(pending)].join('\n - ')}`)
|
|
@@ -771,29 +870,40 @@ const commands = {
|
|
|
771
870
|
if (scenario.environments && !scenario.environments.includes(env.name)) throw new UserError(`Scenario ${scenario.id} is not allowed on ${env.name} (allowed: ${scenario.environments.join(', ')}).`)
|
|
772
871
|
if (env.readonly && !scenario.readonly) throw new UserError(`Environment ${env.name} is readonly; the scenario must declare readonly: true.`)
|
|
773
872
|
const locales = list(args.locales) || scenario.locales || ['default']
|
|
774
|
-
const
|
|
775
|
-
|
|
776
|
-
const
|
|
873
|
+
const scope = { ...startScope({ args, scenario, config, env }), ...(args.note ? { note: args.note } : {}) }
|
|
874
|
+
scope.locales = locales
|
|
875
|
+
const vpNames = baseViewports(scope)
|
|
777
876
|
const git = await gitMeta(root)
|
|
778
877
|
const executor = { kind: args.executor ? 'agent' : 'cli', name: args.executor || 'kaloko-cli' }
|
|
779
878
|
const { project, issue } = await projectAndIssue({ cwd: root, config, scenario, branch: git?.branch })
|
|
780
879
|
const appVer = typeof args['app-version'] === 'string' ? args['app-version'].slice(0, 80) : await appVersion(root)
|
|
781
880
|
const { dir, meta } = await createRun({ output: config.output, scenario, scenarioFile, env, scope, git, executor, root, project, issue, appVersion: appVer })
|
|
782
|
-
say(`Run ${meta.id}\n dir: ${relative(process.cwd(), dir)}\n env: ${env.name}${env.readonly ? ' (readonly)' : ''}\n locales: ${locales.join(', ')} viewports: ${vpNames.join(', ')}\n project: ${project.slug}${issue ? ` issue: ${issue.repo ? issue.repo + '#' : '#'}${issue.number}` : ''}`)
|
|
881
|
+
say(`Run ${meta.id}\n dir: ${relative(process.cwd(), dir)}\n env: ${env.name}${env.readonly ? ' (readonly)' : ''}\n locales: ${locales.join(', ')} viewports: ${vpNames.length > 12 ? `${vpNames.slice(0, 12).join(', ')} … (${vpNames.length})` : vpNames.join(', ')}${dimensionLine(scope)}\n project: ${project.slug}${issue ? ` issue: ${issue.repo ? issue.repo + '#' : '#'}${issue.number}` : ''}`)
|
|
783
882
|
const addr = recipient(env.mail, meta.short)
|
|
784
883
|
if (addr) say(` test e-mail: ${addr}`)
|
|
785
884
|
say(` other agents may start runs here too: pin this one with --run ${meta.id} or KALOKO_RUN=${meta.id}`)
|
|
786
885
|
},
|
|
787
886
|
|
|
887
|
+
/** `kaloko devices`: the device presets a scenario's viewports may name. */
|
|
888
|
+
async devices (args) {
|
|
889
|
+
const { DEVICES, devicePreset } = await import('./lib/devices.mjs')
|
|
890
|
+
const all = Object.keys(DEVICES).map((n) => devicePreset(n))
|
|
891
|
+
if (args.json) { say(JSON.stringify(all, null, 2)); return }
|
|
892
|
+
for (const d of all) say(`${d.name.padEnd(26)} ${d.title.padEnd(28)} ${`${d.size[0]}×${d.size[1]}`.padEnd(10)} screen ${`${d.screen[0]}×${d.screen[1]}`.padEnd(10)} ${String(d.dpr).padEnd(6)}× ${d.touch ? 'touch' : ''}`)
|
|
893
|
+
say('\nUse them in viewports: (a list of names, or name: preset), or kaloko start --viewports iphone-15,desktop-hd.\n<preset>-landscape turns a phone, foldable or tablet sideways; Design environments use the whole screen.')
|
|
894
|
+
},
|
|
895
|
+
|
|
788
896
|
async browser (args) {
|
|
789
897
|
const { config, root } = await context(args)
|
|
790
898
|
const action = args._[0] || 'status'
|
|
791
899
|
if (action === 'install') {
|
|
792
|
-
// the
|
|
900
|
+
// the engines of the Playwright version kaloko itself uses (CI: --with-deps adds the system libraries)
|
|
901
|
+
const engines = list(args.browsers) || ['chromium']
|
|
902
|
+
for (const e of engines) if (!BROWSERS.includes(e)) throw new UserError(`Unknown browser engine ${e}. Use: ${BROWSERS.join(', ')}.`)
|
|
793
903
|
const { spawn } = await import('node:child_process')
|
|
794
904
|
const cli = join(dirname(fileURLToPath(import.meta.resolve('playwright/package.json'))), 'cli.js')
|
|
795
|
-
const code = await new Promise((done) => spawn(process.execPath, [cli, 'install', ...(args['with-deps'] ? ['--with-deps'] : []),
|
|
796
|
-
if (code) throw new UserError(`Installing
|
|
905
|
+
const code = await new Promise((done) => spawn(process.execPath, [cli, 'install', ...(args['with-deps'] ? ['--with-deps'] : []), ...engines], { stdio: 'inherit' }).on('close', done))
|
|
906
|
+
if (code) throw new UserError(`Installing ${engines.join(', ')} failed (exit ${code}).`)
|
|
797
907
|
return
|
|
798
908
|
}
|
|
799
909
|
let env = null
|
|
@@ -836,13 +946,17 @@ const commands = {
|
|
|
836
946
|
return
|
|
837
947
|
}
|
|
838
948
|
if (isApp(scenario.platform)) {
|
|
839
|
-
// the app's current screen, through the environment's driver (no navigation)
|
|
840
|
-
const
|
|
949
|
+
// the app's current screen, through the environment's driver (no navigation); --viewport may name a device key
|
|
950
|
+
const key = typeof args.viewport === 'string' ? args.viewport : Object.keys(run.meta.scope.viewports || {})[0]
|
|
951
|
+
const device = run.meta.scope.devices?.[variantOf(run.meta.scope, key).viewport]?.app ? variantOf(run.meta.scope, key).viewport : null
|
|
952
|
+
const cfg = deviceConfig(scenario.platform, env.app?.[scenario.platform] || {}, device)
|
|
953
|
+
const driver = driverFor(scenario.platform, { ...env, app: { ...(env.app || {}), [scenario.platform]: cfg } }, { root, scenarioDir: resolve(root, run.meta.scenario_file, '..'), locale })
|
|
841
954
|
if (driver.web) throw new UserError('For Electron, run kaloko walk --steps <id>, or pass --image.')
|
|
842
955
|
await driver.open({ locale, attach: true })
|
|
843
956
|
const enc = await launchEphemeral(config).catch(() => null)
|
|
844
957
|
try {
|
|
845
|
-
const
|
|
958
|
+
const { browser: _b, ...dims } = variantFields(run.meta.scope, key)
|
|
959
|
+
const r = await captureApp({ snapshot: await driver.snapshot(), node, locale, viewport: key, run, platform: scenario.platform, encoder: enc ? await enc.newPage() : null, maxChars: config.evaluators?.jev?.max_state_chars, extra: dims })
|
|
846
960
|
say(`✓ ${node.id} ${locale} ${r.viewport} ${r.device || scenario.platform} ${r.files.tree ? `UI tree (${r.capture.tree})` : 'no UI tree'}`)
|
|
847
961
|
} finally { await enc?.close().catch(() => {}) }
|
|
848
962
|
return
|
|
@@ -867,7 +981,7 @@ const commands = {
|
|
|
867
981
|
return
|
|
868
982
|
}
|
|
869
983
|
const since = (await readCaptures(run.dir)).filter((c) => c.node === node.id).map((c) => c.at).pop()
|
|
870
|
-
const results = await captureStep({ page, node, locale, options: captureOptions(scenario, node), viewports:
|
|
984
|
+
const results = await captureStep({ page, node, locale, options: captureOptions(scenario, node), viewports: manualKeys(run, list(args.viewports)), run, consoleFile: conn ? consoleLog(root, config.output, env.name) : null, maxChars: config.evaluators?.jev?.max_state_chars, since, htmlView: htmlViewAllowed(env, scenario) ? { scenarioId: scenario.id } : null, stability: Number(args.stability) || 0 })
|
|
871
985
|
for (const r of results) {
|
|
872
986
|
const failed = Object.entries(r.checks).filter(([, v]) => v.ok === false).map(([k]) => k)
|
|
873
987
|
say(`✓ ${node.id} ${locale} ${r.viewport} ${r.size.width}×${r.size.height}px ${failed.length ? `failing checks: ${failed.join(', ')}` : 'checks ok'}${r.blocked_requests.length ? ` blocked: ${r.blocked_requests.length}` : ''}`)
|
|
@@ -933,17 +1047,23 @@ const commands = {
|
|
|
933
1047
|
if (isApp(scenario.platform)) return walkApp({ config, root, run, scenario, env, args })
|
|
934
1048
|
const pick = list(args.steps)
|
|
935
1049
|
const locales = list(args.locales) || run.meta.scope.locales
|
|
1050
|
+
const scope = run.meta.scope
|
|
1051
|
+
// browser engines and device profiles (batch E): one pass each, capturing only its own keys; a run without
|
|
1052
|
+
// dimensions has a single pass, the walk of before
|
|
1053
|
+
const only = args.browsers ? dimensionValues('browsers', args.browsers) : null
|
|
1054
|
+
const passes = passesOf(scope, { only })
|
|
1055
|
+
if (!passes.length) throw new UserError(`This run has no captures in ${only.join(', ')} (its browsers: ${enginesOf(scope).join(', ')}).`)
|
|
1056
|
+
const needChromium = passes.some((p) => p.engine === 'chromium')
|
|
936
1057
|
// --ephemeral: an own browser (own cookies and tabs) even when the shared one runs, so runs can walk in parallel
|
|
937
1058
|
configureGate(await gateForRequests(env, { root }))
|
|
938
|
-
const conn = args.ephemeral ? null : await connectBrowser({ root, config, env })
|
|
1059
|
+
const conn = args.ephemeral || !needChromium ? null : await connectBrowser({ root, config, env })
|
|
939
1060
|
const releaseLock = conn ? await acquireWalkLock({ root, config, env, run: run.meta.id }).catch(async (e) => { await conn.browser.close(); throw e }) : null
|
|
940
|
-
const ephemeral = conn ? null : await launchEphemeral(config)
|
|
941
|
-
const
|
|
1061
|
+
const ephemeral = conn || !needChromium ? null : await launchEphemeral(config)
|
|
1062
|
+
const chromium = conn ? conn.browser : ephemeral
|
|
942
1063
|
// locales are independent: with --concurrency > 1 each walks in its own browser context at the same time (#27);
|
|
943
1064
|
// one locale at a time keeps the shared browser's own context, as before
|
|
944
1065
|
const concurrency = walkConcurrency(args, config, locales.length, !!conn)
|
|
945
1066
|
const parallel = concurrency > 1 && locales.length > 1
|
|
946
|
-
const ctx = parallel ? null : conn ? conn.context : await newContext(ephemeral, env, { root, config })
|
|
947
1067
|
const scenarioDir = resolve(root, run.meta.scenario_file, '..')
|
|
948
1068
|
// semantic questions of every new capture start right away; the walk ends with the verdicts (--no-evaluate: skip)
|
|
949
1069
|
const prefetch = args['no-evaluate'] ? null : semanticPrefetch({ scenario, config, root })
|
|
@@ -952,127 +1072,156 @@ const commands = {
|
|
|
952
1072
|
const recSettings = recordSettings({ args, config, scenario })
|
|
953
1073
|
const encoder = lazyEncoder(config)
|
|
954
1074
|
const sessions = scenario.sessions || null
|
|
955
|
-
// named sessions: one isolated browser context per person and locale (cookies, storage, sign-in), created lazily
|
|
956
|
-
const sessionContexts = new Map()
|
|
957
1075
|
const ownContexts = []
|
|
1076
|
+
const engines = new Map() // WebKit and Firefox, launched for their passes
|
|
958
1077
|
// `account` stays the scenario's key; `credentials` resolves it in this environment (config accounts)
|
|
959
1078
|
const sessionInfo = (key) => key && sessions?.[key] ? { key, title: localized(sessions[key].title, locales[0]), mailbox: sessions[key].mailbox ? expandRun(sessions[key].mailbox, run.meta.short) : null, account: sessions[key].account || null, credentials: accountFor(env, sessions[key].account, { root, config }) } : null
|
|
960
|
-
const contextFor = async (key, locale) => {
|
|
961
|
-
if (!key || !sessions?.[key]) return null
|
|
962
|
-
const id = `${key}|${locale}`
|
|
963
|
-
if (!sessionContexts.has(id)) {
|
|
964
|
-
// every context gets the environment's gate and readonly policy; an account's saved sign-in is loaded
|
|
965
|
-
const account = accountFor(env, sessions[key].account, { root, config })
|
|
966
|
-
const c = await newContext(browser, env, { root, config, account })
|
|
967
|
-
const p = await c.newPage()
|
|
968
|
-
observeConsole(p)
|
|
969
|
-
sessionContexts.set(id, { context: c, page: p })
|
|
970
|
-
const saved = account && await access(account.storage_state).then(() => true, () => false)
|
|
971
|
-
say(`session ${key} (${locale}): new browser context${saved ? ' with the saved sign-in' : ''}`)
|
|
972
|
-
}
|
|
973
|
-
return sessionContexts.get(id)
|
|
974
|
-
}
|
|
975
1079
|
// stop a session after the same failure repeats (shared precondition such as a failed sign-in);
|
|
976
1080
|
// --keep-going walks every step regardless
|
|
977
1081
|
const streak = failureStreak(args['keep-going'] ? 0 : 3)
|
|
978
1082
|
const proposals = []
|
|
979
|
-
|
|
980
|
-
const
|
|
981
|
-
|
|
982
|
-
|
|
983
|
-
|
|
984
|
-
|
|
985
|
-
|
|
986
|
-
|
|
987
|
-
|
|
988
|
-
|
|
989
|
-
|
|
990
|
-
|
|
991
|
-
|
|
992
|
-
|
|
993
|
-
|
|
994
|
-
|
|
995
|
-
|
|
996
|
-
|
|
997
|
-
|
|
998
|
-
if (
|
|
999
|
-
|
|
1000
|
-
|
|
1001
|
-
|
|
1002
|
-
|
|
1003
|
-
|
|
1004
|
-
|
|
1005
|
-
if (
|
|
1006
|
-
|
|
1007
|
-
|
|
1008
|
-
|
|
1009
|
-
|
|
1010
|
-
|
|
1011
|
-
|
|
1012
|
-
|
|
1013
|
-
|
|
1014
|
-
|
|
1015
|
-
|
|
1016
|
-
|
|
1017
|
-
try {
|
|
1018
|
-
const to = expandRun(node.mail?.to || sessionInfo(node.session).mailbox, run.meta.short)
|
|
1019
|
-
const results = await captureMail({ config, root, run, env, node, locale, to, subject: localized(node.mail?.subject, locale), timeoutMs: 45000 })
|
|
1020
|
-
say(`✓ ${node.id} ${locale}: "${results.subject}"`)
|
|
1021
|
-
} catch (error) {
|
|
1022
|
-
failed.push(`${node.id} (${locale}): ${error.message.split('\n')[0]}`)
|
|
1023
|
-
say(`✗ ${node.id} ${locale}: ${error.message.split('\n')[0]}`)
|
|
1024
|
-
}
|
|
1025
|
-
continue
|
|
1083
|
+
const results = []
|
|
1084
|
+
const recSkipped = new Set() // engines whose passes are not recorded (once per engine)
|
|
1085
|
+
try {
|
|
1086
|
+
for (const [passIndex, pass] of passes.entries()) {
|
|
1087
|
+
const first = passIndex === 0
|
|
1088
|
+
const plain = pass.engine === 'chromium' && !pass.emulation
|
|
1089
|
+
let browser = chromium
|
|
1090
|
+
if (pass.engine !== 'chromium') {
|
|
1091
|
+
if (!engines.has(pass.engine)) engines.set(pass.engine, await launchEngine(pass.engine))
|
|
1092
|
+
browser = engines.get(pass.engine)
|
|
1093
|
+
}
|
|
1094
|
+
const extra = contextOptionsOf(pass)
|
|
1095
|
+
const tag = passes.length > 1 && pass.label ? ` · ${pass.label}` : ''
|
|
1096
|
+
if (passes.length > 1) say(`\n${pass.label || 'chromium'}: ${pass.keys.join(', ')}`)
|
|
1097
|
+
// recordings and traces (batch D) in Chromium passes; a flow recording is one per locale, made in the first
|
|
1098
|
+
let passRec = recSettings
|
|
1099
|
+
if (pass.engine !== 'chromium') {
|
|
1100
|
+
passRec = null
|
|
1101
|
+
const wanted = !!recSettings.record || scenario.nodes.some((n) => parseRecord(n.record ?? n.capture?.record))
|
|
1102
|
+
if (wanted && !recSkipped.has(pass.engine)) { recSkipped.add(pass.engine); say(`Recording and traces skipped in ${pass.engine}: they need Chromium (the Chromium keys of this run are recorded).`) }
|
|
1103
|
+
} else if (passes.slice(0, passIndex).some((p) => p.engine === 'chromium') && recSettings.record?.mode === 'flow') passRec = { ...recSettings, record: null }
|
|
1104
|
+
const ctx = parallel ? null : plain && conn ? conn.context : await newContext(browser, env, { root, config, extra })
|
|
1105
|
+
if (ctx && ctx !== conn?.context) ownContexts.push(ctx)
|
|
1106
|
+
// named sessions: one isolated browser context per person and locale (cookies, storage, sign-in), created lazily
|
|
1107
|
+
const sessionContexts = new Map()
|
|
1108
|
+
const contextFor = async (key, locale) => {
|
|
1109
|
+
if (!key || !sessions?.[key]) return null
|
|
1110
|
+
const id = `${key}|${locale}`
|
|
1111
|
+
if (!sessionContexts.has(id)) {
|
|
1112
|
+
// every context gets the environment's gate and readonly policy; an account's saved sign-in is loaded
|
|
1113
|
+
const account = accountFor(env, sessions[key].account, { root, config })
|
|
1114
|
+
const c = await newContext(browser, env, { root, config, account, extra })
|
|
1115
|
+
ownContexts.push(c)
|
|
1116
|
+
const p = await c.newPage()
|
|
1117
|
+
observeConsole(p)
|
|
1118
|
+
sessionContexts.set(id, { context: c, page: p })
|
|
1119
|
+
const saved = account && await access(account.storage_state).then(() => true, () => false)
|
|
1120
|
+
say(`session ${key} (${locale}${tag}): new browser context${saved ? ' with the saved sign-in' : ''}`)
|
|
1026
1121
|
}
|
|
1027
|
-
|
|
1028
|
-
|
|
1029
|
-
|
|
1030
|
-
|
|
1031
|
-
|
|
1032
|
-
|
|
1033
|
-
|
|
1034
|
-
|
|
1035
|
-
|
|
1036
|
-
|
|
1037
|
-
|
|
1038
|
-
|
|
1039
|
-
|
|
1040
|
-
|
|
1041
|
-
|
|
1042
|
-
|
|
1043
|
-
|
|
1122
|
+
return sessionContexts.get(id)
|
|
1123
|
+
}
|
|
1124
|
+
// one locale: its own tab (and, in parallel, its own context); returns what it could not walk
|
|
1125
|
+
const walkLocale = async (locale) => {
|
|
1126
|
+
const pending = []; const failed = []; const skipped = []
|
|
1127
|
+
let lctx = ctx
|
|
1128
|
+
if (parallel) { lctx = await newContext(browser, env, { root, config, extra }); ownContexts.push(lctx) }
|
|
1129
|
+
// the walk owns its tab; agents and people keep theirs in the same shared browser
|
|
1130
|
+
const page = await lctx.newPage()
|
|
1131
|
+
const shared = conn && !parallel && plain // the shared browser's own context logs its console to a file
|
|
1132
|
+
if (!shared) observeConsole(page)
|
|
1133
|
+
// one recorder per page (the walk's tab and each named session's), started on its first step;
|
|
1134
|
+
// recordings and traces need CDP: Chromium passes only
|
|
1135
|
+
const recorders = new Map()
|
|
1136
|
+
const recorderFor = async (p) => {
|
|
1137
|
+
if (!passRec) return null
|
|
1138
|
+
if (!recorders.has(p)) recorders.set(p, recordingSession({ page: p, run, scenario, locale, settings: passRec, viewports: scopeKeys(scope, pass.keys), encoder: encoder.page, say, label: `${locale}${tag}` }).catch(() => null))
|
|
1139
|
+
return recorders.get(p)
|
|
1044
1140
|
}
|
|
1045
1141
|
try {
|
|
1046
|
-
|
|
1047
|
-
say(
|
|
1048
|
-
|
|
1049
|
-
|
|
1050
|
-
|
|
1051
|
-
|
|
1052
|
-
|
|
1053
|
-
|
|
1054
|
-
|
|
1055
|
-
|
|
1142
|
+
// every locale starts as a new visitor on its first walk in the run (an own context is new already)
|
|
1143
|
+
if (shared && !walked.some((c) => c.locale === locale)) { await resetSession(lctx, page, baseUrl(env, locale)); say(`session reset (new visitor, ${locale}${tag})`) }
|
|
1144
|
+
for (const node of scenario.nodes) {
|
|
1145
|
+
if (node.kind === 'decision' || (pick && !pick.includes(node.id))) continue
|
|
1146
|
+
if (node.kind !== 'stack' && node.path && !urlFor(env, node, locale)) continue // step does not exist in this locale
|
|
1147
|
+
if (node.session && sessions?.[node.session]?.blocked_on?.includes(env.name)) { pending.push(`${node.id} (${locale}${tag}): session ${node.session} is blocked on ${env.name}`); continue }
|
|
1148
|
+
const sess = await contextFor(node.session, locale)
|
|
1149
|
+
const sPage = sess ? sess.page : page
|
|
1150
|
+
const sCtx = sess ? sess.context : lctx
|
|
1151
|
+
const consoleFile = shared && !sess ? consoleLog(root, config.output, env.name) : null
|
|
1152
|
+
if (node.kind === 'stack') {
|
|
1153
|
+
try {
|
|
1154
|
+
const bad = await walkStack({ node, locale, page: sPage, env, run, scenario, scenarioDir, config, consoleFile, args, keys: pass.keys, tag })
|
|
1155
|
+
for (const b of bad) failed.push(`${node.id} (${locale}${tag}) ${b}`)
|
|
1156
|
+
} catch (error) {
|
|
1157
|
+
failed.push(`${node.id} (${locale}${tag}): ${error.message.split('\n')[0]}`)
|
|
1158
|
+
say(`✗ ${node.id} ${locale}${tag}: ${error.message.split('\n')[0]}`)
|
|
1159
|
+
}
|
|
1160
|
+
continue
|
|
1161
|
+
}
|
|
1162
|
+
if (node.kind === 'email' && !first) continue // one message, captured in the first pass
|
|
1163
|
+
if (node.kind === 'email' && (node.mail?.to || sessionInfo(node.session)?.mailbox)) {
|
|
1164
|
+
// an e-mail node with a recipient is walked like a step: wait for the message and capture it
|
|
1165
|
+
try {
|
|
1166
|
+
const to = expandRun(node.mail?.to || sessionInfo(node.session).mailbox, run.meta.short)
|
|
1167
|
+
const results = await captureMail({ config, root, run, env, node, locale, to, subject: localized(node.mail?.subject, locale), timeoutMs: 45000 })
|
|
1168
|
+
say(`✓ ${node.id} ${locale}${tag}: "${results.subject}"`)
|
|
1169
|
+
} catch (error) {
|
|
1170
|
+
failed.push(`${node.id} (${locale}${tag}): ${error.message.split('\n')[0]}`)
|
|
1171
|
+
say(`✗ ${node.id} ${locale}${tag}: ${error.message.split('\n')[0]}`)
|
|
1172
|
+
}
|
|
1173
|
+
continue
|
|
1174
|
+
}
|
|
1175
|
+
if (!['playwright', 'script'].includes(node.execution.strategy) || !node.execution.script) { pending.push(`${node.id} (${locale}${tag}): ${node.execution.strategy} — ${node.execution.instructions.split('\n')[0]}`); continue }
|
|
1176
|
+
const streakKey = `${pass.engine}|${pass.emulation?.key || ''}|${node.session || ''}|${locale}`
|
|
1177
|
+
if (streak.stopped(streakKey)) { skipped.push(`${node.id} (${locale}${tag})`); continue }
|
|
1178
|
+
const scriptFile = resolve(scenarioDir, node.execution.script)
|
|
1179
|
+
// one invocation of the step script; the automatic capture runs unless the script captured the step itself:
|
|
1180
|
+
// a crop is an attachment (#49), and a capture from an earlier walk must not stand in for this one (#50)
|
|
1181
|
+
const rec = await recorderFor(sPage)
|
|
1182
|
+
const runStep = async (mod) => {
|
|
1183
|
+
let captured = false
|
|
1184
|
+
rec?.beforeStep(node)
|
|
1185
|
+
const records = []
|
|
1186
|
+
const capture = (opts = {}) => { captured = true; rec?.beforeCapture(node); return captureStep({ page: sPage, node, locale, options: captureOptions(scenario, node), viewports: scopeKeys(run.meta.scope, pass.keys, opts.viewports || null), run, htmlView: htmlViewAllowed(env, scenario) ? { scenarioId: scenario.id } : null, consoleFile, maxChars: config.evaluators?.jev?.max_state_chars, extra: node.session ? { session: node.session } : {}, stability: Number(args.stability) || 0 }).then((r) => { prefetch?.add(r, run.dir); records.push(...r); return r }) }
|
|
1187
|
+
try {
|
|
1188
|
+
await mod.default({ page: sPage, context: sCtx, locale, node: { id: node.id, title: localized(node.title, locale), kind: node.kind }, session: sessionInfo(node.session), url: (id = node.id) => urlFor(env, stepOf(scenario, id), locale), env, run: run.meta, capture, crop: (selector, title, o = {}) => captureCrop({ page: sPage, node, locale, selector, title, run, padding: Number(o.padding) || 0 }), acceptDialogs: () => acceptDialogs(sPage) })
|
|
1189
|
+
if (!captured) await capture()
|
|
1190
|
+
} catch (error) { await rec?.afterStep(node, { error }); throw error }
|
|
1191
|
+
await rec?.afterStep(node, { records })
|
|
1192
|
+
}
|
|
1193
|
+
try {
|
|
1194
|
+
await runStep(await import(pathToFileURL(scriptFile).href))
|
|
1195
|
+
say(`✓ ${node.id} ${locale}${tag}`)
|
|
1196
|
+
streak.ok(streakKey)
|
|
1197
|
+
} catch (error) {
|
|
1198
|
+
// a selector the page no longer has: propose one from the page (#13); --heal applies it once the step passes
|
|
1199
|
+
const healed = args['no-heal'] ? null : await healStep({ error, page: sPage, scriptFile, root, output: config.output, run, node, locale, apply: !!args.heal, rerun: runStep }).catch(() => null)
|
|
1200
|
+
if (healed?.applied) {
|
|
1201
|
+
say(`✓ ${node.id} ${locale}${tag} (healed)\n${formatHeal(healed, { stepArg: node.id })}`)
|
|
1202
|
+
streak.ok(streakKey)
|
|
1203
|
+
continue
|
|
1204
|
+
}
|
|
1205
|
+
failed.push(`${node.id} (${locale}${tag}): ${error.message.split('\n')[0]}`)
|
|
1206
|
+
say(`✗ ${node.id} ${locale}${tag}: ${error.message.split('\n')[0]}`)
|
|
1207
|
+
if (healed) { say(formatHeal(healed, { stepArg: node.id })); if (healed.patch) proposals.push(node.id) }
|
|
1208
|
+
if (streak.record(streakKey, error.message)) say(` stopping ${node.session ? `session ${node.session}` : 'this walk'} (${locale}${tag}): the last 3 steps failed with the same error; fix the shared cause (sign-in, environment) and rerun, or pass --keep-going`)
|
|
1209
|
+
}
|
|
1056
1210
|
}
|
|
1057
|
-
|
|
1058
|
-
|
|
1059
|
-
|
|
1060
|
-
if (streak.record(streakKey, error.message)) say(` stopping ${node.session ? `session ${node.session}` : 'this walk'} (${locale}): the last 3 steps failed with the same error; fix the shared cause (sign-in, environment) and rerun, or pass --keep-going`)
|
|
1211
|
+
} finally {
|
|
1212
|
+
for (const r of recorders.values()) { const notes = await (await r)?.finish().catch((e) => [`recording: ${e.message}`]); for (const n of notes || []) say(` ${n}`) }
|
|
1213
|
+
await page.close().catch(() => {})
|
|
1061
1214
|
}
|
|
1215
|
+
return { pending, failed, skipped }
|
|
1062
1216
|
}
|
|
1063
|
-
|
|
1064
|
-
|
|
1065
|
-
|
|
1217
|
+
if (parallel) say(`Walking ${locales.length} locales, ${concurrency} at a time (each in its own browser context); output in locale order.`)
|
|
1218
|
+
results.push(...await inOrder(locales, parallel ? concurrency : 1, (locale) => walkLocale(locale).catch((error) => ({ pending: [], failed: [`${locale}${tag}: ${error.message.split('\n')[0]}`], skipped: [] }))))
|
|
1219
|
+
// the next pass has its own contexts: close this one's now
|
|
1220
|
+
for (const c of ownContexts.splice(0)) await c.close().catch(() => {})
|
|
1066
1221
|
}
|
|
1067
|
-
return { pending, failed, skipped }
|
|
1068
|
-
}
|
|
1069
|
-
let results
|
|
1070
|
-
try {
|
|
1071
|
-
if (parallel) say(`Walking ${locales.length} locales, ${concurrency} at a time (each in its own browser context); output in locale order.`)
|
|
1072
|
-
results = await inOrder(locales, parallel ? concurrency : 1, (locale) => walkLocale(locale).catch((error) => ({ pending: [], failed: [`${locale}: ${error.message.split('\n')[0]}`], skipped: [] })))
|
|
1073
1222
|
} finally {
|
|
1074
|
-
for (const { context } of sessionContexts.values()) await context.close().catch(() => {})
|
|
1075
1223
|
for (const c of ownContexts) await c.close().catch(() => {})
|
|
1224
|
+
for (const b of engines.values()) await b.close().catch(() => {})
|
|
1076
1225
|
if (conn) await conn.browser.close() // over CDP this only disconnects; the shared browser keeps running
|
|
1077
1226
|
if (ephemeral) await ephemeral.close()
|
|
1078
1227
|
await encoder.close()
|
|
@@ -1211,17 +1360,20 @@ const commands = {
|
|
|
1211
1360
|
say(`Added the environment ${envName} to ${relative(process.cwd(), file) || CONFIG_FILE}: base_url \${KALOKO_PREVIEW_URL}, set by kaloko ci preview-url in CI.`)
|
|
1212
1361
|
} else say(`Add to environments: in ${CONFIG_FILE}:\n ${envName}:\n base_url: \${KALOKO_PREVIEW_URL}`)
|
|
1213
1362
|
}
|
|
1363
|
+
// the engines the job installs: Chromium and whatever the scenarios and walk.browsers ask for (batch E)
|
|
1364
|
+
const engines = new Set(['chromium', ...(config.walk?.browsers || [])])
|
|
1214
1365
|
for (const f of scenarios) {
|
|
1215
1366
|
let sc = null
|
|
1216
1367
|
try { sc = yaml.load(await readFile(resolve(root, f), 'utf8')) } catch { throw new UserError(`${f} cannot be read.`) }
|
|
1217
1368
|
if (sc?.environments && !sc.environments.includes(envName)) say(`! ${f} allows environments [${sc.environments.join(', ')}]: add ${envName}, or kaloko start refuses it in CI.`)
|
|
1369
|
+
for (const b of sc?.browsers || []) engines.add(b)
|
|
1218
1370
|
}
|
|
1219
1371
|
let pkg = null
|
|
1220
1372
|
try { pkg = JSON.parse(await readFile(join(root, 'package.json'), 'utf8')) } catch {}
|
|
1221
1373
|
const pm = existsSync(join(root, 'pnpm-lock.yaml')) ? 'pnpm' : existsSync(join(root, 'yarn.lock')) ? 'yarn' : existsSync(join(root, 'bun.lockb')) || existsSync(join(root, 'bun.lock')) ? 'bun' : 'npm'
|
|
1222
1374
|
const dependency = !!(pkg && ({ ...pkg.dependencies, ...pkg.devDependencies }).kaloko)
|
|
1223
1375
|
const version = JSON.parse(await readFile(join(packageRoot, 'package.json'), 'utf8')).version
|
|
1224
|
-
const opts = { scenarios, env: envName, affected: !args['no-affected'], pm, dependency, version: `^${version}`, trigger: args.trigger === 'pull_request' ? 'pull_request' : 'deployment' }
|
|
1376
|
+
const opts = { scenarios, env: envName, affected: !args['no-affected'], pm, dependency, version: `^${version}`, trigger: args.trigger === 'pull_request' ? 'pull_request' : 'deployment', browsers: [...engines] }
|
|
1225
1377
|
const content = action === 'github' ? githubWorkflow(opts) : gitlabJob(opts)
|
|
1226
1378
|
if (args.print) { say(content); return }
|
|
1227
1379
|
const out = resolve(root, typeof args.out === 'string' ? args.out : action === 'github' ? '.github/workflows/kaloko.yml' : 'kaloko.gitlab-ci.yml')
|
|
@@ -1352,6 +1504,8 @@ const commands = {
|
|
|
1352
1504
|
|
|
1353
1505
|
async stability (args) {
|
|
1354
1506
|
const { config, root, run, scenario } = await runContext(args)
|
|
1507
|
+
// regions drawn by hand on the canvas (kaloko.app) into the scenario's ignore:
|
|
1508
|
+
if (args.pull) { await pullDrawn({ root, config, run, args, say, client: service(config) }); return }
|
|
1355
1509
|
await stabilityCmd({ root, config, run, scenario, args, say })
|
|
1356
1510
|
},
|
|
1357
1511
|
|
|
@@ -1552,7 +1706,8 @@ const commands = {
|
|
|
1552
1706
|
say(`Feedback for ${data.run.id} (${data.scenario}) — run verdict: ${data.verdict?.verdict || 'pending'}${data.verdict?.by ? ` by ${data.verdict.by}` : ''}${data.verdict?.note ? `: ${data.verdict.note}` : ''}${data.verdict?.awaiting?.length ? `\n awaiting approval from: ${data.verdict.awaiting.join(', ')}` : ''}${data.approved.length ? `\n approved: ${data.approved.join(', ')}` : ''}`)
|
|
1553
1707
|
for (const c of data.criterion_verdicts || []) say(` ✕ ${c.node_id} ${c.criterion_id}${c.locale ? ` [${c.locale}]` : ''} — ${c.email}: ${c.note}`)
|
|
1554
1708
|
const thread = (r, where) => {
|
|
1555
|
-
say(` ${r.status === 'rejected' ? '✕' : '•'} [${r.id}] ${where}${r.locale ? ` [${r.locale}]` : ''}${r.pin ? ` · pinned at ${Math.round(r.pin.x * 100)}%,${Math.round(r.pin.y * 100)}%${r.pin.viewport ? ` (${r.pin.viewport})` : ''}` : ''}${r.anchor?.selector ? ` · on ${r.anchor.selector}` : ''} — ${r.email}${r.agent ? ` via ${r.agent}` : ''}: ${r.note}`)
|
|
1709
|
+
say(` ${r.status === 'rejected' ? '✕' : '•'} [${r.id}] ${where}${r.locale ? ` [${r.locale}]` : ''}${r.pin ? ` · pinned at ${Math.round(r.pin.x * 100)}%,${Math.round(r.pin.y * 100)}%${r.pin.viewport ? ` (${r.pin.viewport})` : ''}` : ''}${r.anchor?.selector ? ` · on ${r.anchor.selector}` : ''}${regionText(r.region)} — ${r.email}${r.agent ? ` via ${r.agent}` : ''}: ${r.note}`)
|
|
1710
|
+
for (const f of r.attachments || []) say(` attached ${f.name} (${f.type}): ${f.url || f.id}`)
|
|
1556
1711
|
for (const x of r.replies || []) say(` ↳ ${x.email}${x.agent ? ` via ${x.agent}` : ''}: ${x.deleted ? '(deleted)' : x.note}`)
|
|
1557
1712
|
}
|
|
1558
1713
|
const flow = data.flow_comments || []
|
|
@@ -1575,6 +1730,22 @@ const commands = {
|
|
|
1575
1730
|
* accepted run, returned, commented), the approvals carried over unchanged steps, and the review-mode link. For
|
|
1576
1731
|
* agents: report it to the person; never approve on their behalf.
|
|
1577
1732
|
*/
|
|
1733
|
+
/**
|
|
1734
|
+
* The signed acceptance record of an accepted run: JSON and PDF downloaded and checked against the service's public
|
|
1735
|
+
* key; --verify checks a record someone handed over (no token, no project needed).
|
|
1736
|
+
*/
|
|
1737
|
+
async signoff (args) {
|
|
1738
|
+
if (typeof args.verify === 'string') {
|
|
1739
|
+
const config = await context(args).then((c) => c.config).catch(() => ({}))
|
|
1740
|
+
return signoffCmd({ config, args, say })
|
|
1741
|
+
}
|
|
1742
|
+
const { config, root } = await context(args)
|
|
1743
|
+
let runId = args._[0] || (typeof args.run === 'string' ? args.run : null)
|
|
1744
|
+
if (!runId) { try { const r = await resolveRun({ root, output: config.output }); if (r.meta.shared) runId = r.meta.id } catch {} }
|
|
1745
|
+
if (!runId) throw new UserError('Usage: kaloko signoff <run-id> [--out dir] | kaloko signoff --verify <file.json|file.pdf>')
|
|
1746
|
+
return signoffCmd({ config, args, say, client: service(config), runId })
|
|
1747
|
+
},
|
|
1748
|
+
|
|
1578
1749
|
async review (args) {
|
|
1579
1750
|
const { config, root } = await context(args)
|
|
1580
1751
|
const client = service(config)
|
|
@@ -1846,7 +2017,10 @@ const commands = {
|
|
|
1846
2017
|
|
|
1847
2018
|
async doctor (args) {
|
|
1848
2019
|
const { config, root, file } = await context(args)
|
|
1849
|
-
|
|
2020
|
+
// engines the scenarios ask for (browsers: [webkit, firefox]), so doctor checks they are installed
|
|
2021
|
+
const engines = []
|
|
2022
|
+
for (const f of await scenarioFiles(config, root).catch(() => [])) { try { engines.push(...((yaml.load(await readFile(f, 'utf8')) || {}).browsers || [])) } catch {} }
|
|
2023
|
+
const r = await doctor({ config, root, file, fix: !!args.fix, engines })
|
|
1850
2024
|
say(args.json ? JSON.stringify(r, null, 2) : formatDoctor(r))
|
|
1851
2025
|
if (!r.ok) process.exitCode = 1
|
|
1852
2026
|
},
|