kaloko 0.4.10 → 0.5.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +64 -4
- package/package.json +1 -1
- package/schemas/config.v1.json +70 -2
- package/schemas/run.v1.json +33 -2
- package/schemas/scenario.v1.json +130 -17
- package/skills/kaloko/SKILL.md +18 -5
- package/skills/kaloko/reference/acceptance.md +70 -1
- package/skills/kaloko/reference/apps.md +103 -8
- package/skills/kaloko/reference/ci.md +3 -1
- package/skills/kaloko/reference/design.md +39 -0
- package/skills/kaloko/reference/devices.md +109 -0
- package/skills/kaloko/reference/environments.md +6 -0
- package/skills/kaloko/reference/execution.md +6 -0
- package/skills/kaloko/reference/mcp.md +1 -0
- package/skills/kaloko/reference/recording.md +95 -0
- package/src/cli.mjs +381 -167
- package/src/lib/access.mjs +5 -2
- package/src/lib/anim.mjs +125 -0
- package/src/lib/apps/devices.mjs +187 -0
- package/src/lib/apps/drivers.mjs +11 -3
- package/src/lib/apps/ios-device.mjs +78 -0
- package/src/lib/apps/recording-native.mjs +212 -0
- package/src/lib/apps/tree.mjs +9 -1
- package/src/lib/apps/webview.mjs +54 -0
- package/src/lib/apps/windows.mjs +108 -0
- package/src/lib/apps/windows.ps1 +78 -0
- package/src/lib/assertions.mjs +300 -0
- package/src/lib/build.mjs +63 -4
- package/src/lib/capture.mjs +146 -119
- package/src/lib/ci.mjs +8 -4
- package/src/lib/design-cli.mjs +35 -6
- package/src/lib/design.mjs +3 -1
- package/src/lib/devices.mjs +85 -0
- package/src/lib/devspec.mjs +3 -2
- package/src/lib/dimensions.mjs +326 -0
- package/src/lib/doctor.mjs +35 -6
- package/src/lib/drift.mjs +8 -4
- package/src/lib/evaluate.mjs +25 -7
- package/src/lib/import.mjs +3 -2
- package/src/lib/jev.mjs +3 -1
- package/src/lib/manifest-core.mjs +12 -3
- package/src/lib/probe.mjs +60 -0
- package/src/lib/recheck.mjs +6 -1
- package/src/lib/recording.mjs +373 -0
- package/src/lib/scenario.mjs +3 -0
- package/src/lib/stability-core.mjs +4 -2
- package/src/lib/trace.mjs +109 -0
- package/src/lib/video-cli.mjs +209 -0
- package/src/lib/video.mjs +427 -0
- package/src/lib/walk-dimensions.mjs +83 -0
- package/src/lib/webm.mjs +144 -0
- package/viewer/index.html +467 -25
package/src/cli.mjs
CHANGED
|
@@ -51,6 +51,13 @@ import { exportDesign } from './lib/design-export.mjs'
|
|
|
51
51
|
import { driverFor, isApp, pngSize, ocr as ocrImage } from './lib/apps/drivers.mjs'
|
|
52
52
|
import { captureApp } from './lib/apps/capture.mjs'
|
|
53
53
|
import { readTree, flatten } from './lib/apps/tree.mjs'
|
|
54
|
+
import { recordSettings, recordingSession, parseRecord, recordingFiles, traceFiles, stripRecordings, withoutRecordings } from './lib/recording.mjs'
|
|
55
|
+
import { nativeSession } from './lib/apps/recording-native.mjs'
|
|
56
|
+
import { videoCmd } from './lib/video-cli.mjs'
|
|
57
|
+
import { passesOf, enginesOf, contextOptionsOf, launchEngine, pickKeys, variantFields, variantOf, dimensionValues, BROWSERS } from './lib/dimensions.mjs'
|
|
58
|
+
import { startScope, scopeKeys, dimensionLine, baseViewports, engineKeys, appDeviceGroups } from './lib/walk-dimensions.mjs'
|
|
59
|
+
import { deviceConfig, prepareAndroid, readyAndroid, finishAndroid, prepareSimulator, applyAppAppearance } from './lib/apps/devices.mjs'
|
|
60
|
+
import { attachAndroidWebView } from './lib/apps/webview.mjs'
|
|
54
61
|
|
|
55
62
|
|
|
56
63
|
const HELP = `kaloko — walk, assess and approve what your agents built, on one flow canvas
|
|
@@ -88,9 +95,19 @@ Run
|
|
|
88
95
|
start --scenario <file> --env <name> [--locales cs,en] [--viewports desktop,mobile] [--executor <name>]
|
|
89
96
|
[--app-version <v>] (default: the git tag on HEAD; documentation captures carry it)
|
|
90
97
|
Create a run directory and make it current
|
|
98
|
+
[--viewports iphone-15,ipad-landscape,desktop] Viewport names of the scenario or device presets
|
|
99
|
+
(kaloko devices lists them; <preset>-landscape turns one sideways)
|
|
100
|
+
[--browsers chromium,webkit,firefox] [--color-schemes light,dark] [--reduced-motion no-preference,reduce]
|
|
101
|
+
[--forced-colors none,active] [--display-modes browser,standalone]
|
|
102
|
+
More dimensions: every step is captured in each combination (keys like
|
|
103
|
+
mobile.webkit.dark); default from the scenario, then walk: in the config
|
|
104
|
+
[--devices pixel-8-api-34,iphone-15-ios-18] App scenarios: the emulators, simulators or devices to walk on
|
|
91
105
|
browser start|stop|status [--env e] [--headed] One shared browser per environment (port = browser.port + index)
|
|
92
106
|
Shared Chromium with CDP (agents, scripts and people drive it)
|
|
93
|
-
browser install [--with-deps]
|
|
107
|
+
browser install [--with-deps] [--browsers chromium,webkit,firefox]
|
|
108
|
+
Download the engines of kaloko's Playwright (default Chromium; CI:
|
|
109
|
+
KALOKO_BROWSER_CHANNEL=chromium uses it); WebKit and Firefox for cross-browser runs
|
|
110
|
+
devices [--json] Device presets for viewports: (phones, foldables, tablets, laptop, desktop, TV)
|
|
94
111
|
open --step <id> [--locale l] Navigate the current page to the step URL
|
|
95
112
|
session reset New visitor: clear cookies and storage in the shared browser
|
|
96
113
|
capture --step <id> [--locale l] [--url <url>] [--page <substring>] [--viewports a,b]
|
|
@@ -112,10 +129,14 @@ Run
|
|
|
112
129
|
A person signs in once (SSO, MFA) in a visible browser; the session is saved
|
|
113
130
|
for that account and walk reuses it (never uploaded)
|
|
114
131
|
auth list Saved sessions per environment and account, with their age
|
|
115
|
-
walk [--steps a,b] [--locales l] [--keep-going] [--ephemeral]
|
|
132
|
+
walk [--steps a,b] [--locales l] [--keep-going] [--ephemeral] [--browsers webkit]
|
|
116
133
|
Run playwright steps (execution.script) and capture them; a session stops
|
|
117
134
|
after 3 steps in a row fail with the same error unless --keep-going
|
|
118
135
|
One walk at a time per shared browser; --ephemeral walks in an own browser
|
|
136
|
+
A run with several browsers or device presets walks once per engine and device
|
|
137
|
+
profile; --browsers walks only those engines of the run
|
|
138
|
+
[--devices pixel-8-api-34] App runs walk once per device of devices: (emulators start, phones wake up);
|
|
139
|
+
--devices walks only those; dark mode and reduced motion are set per capture
|
|
119
140
|
Semantic questions start as each step is captured; the walk ends with the
|
|
120
141
|
verdicts and what failed (--no-evaluate skips that; kaloko evaluate redoes it)
|
|
121
142
|
[--items a,b] [--limit N] Stack steps (kind: stack, many pages of one template): only pages whose id or
|
|
@@ -130,6 +151,11 @@ Run
|
|
|
130
151
|
steps' captures from the previous local run
|
|
131
152
|
[--heal | --no-heal] A selector the page lost gets a proposal from the page (test id, role and name,
|
|
132
153
|
text) with a patch; --heal applies it to the step script once the step passes
|
|
154
|
+
[--record [step|flow|failed]] [--record-quality low|medium|high] [--no-trace]
|
|
155
|
+
Record each step from the previous screen to its capture (step), the whole walk
|
|
156
|
+
per locale with a chapter per step (flow), or only steps that fail (failed);
|
|
157
|
+
also capture.record in the config or scenario, record: true on a step.
|
|
158
|
+
A trace (actions, requests, console) is kept per step unless --no-trace
|
|
133
159
|
affected [--scenario file | --all] [--base ref] [--files a,b] [--json] [--check] [--offline]
|
|
134
160
|
Which steps changed files reach, and why: framework route files against step paths
|
|
135
161
|
and URLs captured earlier, imports, global files, i18n messages, step scripts and
|
|
@@ -147,6 +173,11 @@ Results
|
|
|
147
173
|
the semantic questions again on the stored captures (no walk);
|
|
148
174
|
console, overflow and performance keep their live result
|
|
149
175
|
build Write run.json and the local preview (index.html)
|
|
176
|
+
video export [--format mp4|webm|gif] [--locale l] [--steps a,b] [--lang en|cs] [--out file]
|
|
177
|
+
The run's recordings in flow order with a title card per step (title, verdict);
|
|
178
|
+
shared with the run on Business and up
|
|
179
|
+
video list | trace --step <id> | tools
|
|
180
|
+
Recordings and traces of the run; a step's trace as a timeline; ffmpeg, adb, simctl
|
|
150
181
|
preview Build and print the preview path
|
|
151
182
|
share [--expires 30] [--pr] Upload the run to kaloko.app (KALOKO_TOKEN, tester role); files travel by content
|
|
152
183
|
hash, so unchanged captures are not sent again and a broken-off share resumes;
|
|
@@ -340,10 +371,20 @@ async function scenarioIdOf (args, config, root) {
|
|
|
340
371
|
throw new UserError('--scenario <id|file> is required when there is no current run.')
|
|
341
372
|
}
|
|
342
373
|
|
|
374
|
+
/**
|
|
375
|
+
* Keys a capture of the current page takes (kaloko capture): the Chromium keys without a device preset, whose
|
|
376
|
+
* pixel ratio, touch and agent the page does not have; a preset key only when --viewports names it.
|
|
377
|
+
*/
|
|
378
|
+
function manualKeys (run, pick) {
|
|
379
|
+
const scope = run.meta.scope
|
|
380
|
+
if (pick) return scopeViewports(run, pick)
|
|
381
|
+
const keys = engineKeys(scope, 'chromium').filter((k) => !scope.devices?.[variantOf(scope, k).viewport]?.emulation)
|
|
382
|
+
return scopeKeys(scope, keys.length ? keys : engineKeys(scope, 'chromium'), null)
|
|
383
|
+
}
|
|
384
|
+
|
|
385
|
+
/** { key: [w, h] } a capture on a Chromium page takes: every Chromium key of the run, or those of `pick`. */
|
|
343
386
|
function scopeViewports (run, pick) {
|
|
344
|
-
|
|
345
|
-
const names = pick || Object.keys(all)
|
|
346
|
-
return Object.fromEntries(names.map((n) => { if (!all[n]) throw new UserError(`Unknown viewport "${n}".`); return [n, all[n]] }))
|
|
387
|
+
return scopeKeys(run.meta.scope, engineKeys(run.meta.scope, 'chromium'), pick)
|
|
347
388
|
}
|
|
348
389
|
|
|
349
390
|
async function scenarioFiles (config, root) {
|
|
@@ -459,14 +500,17 @@ async function exportScenario ({ args, config, root, client }) {
|
|
|
459
500
|
* Walks a stack step: resolves its pages, then opens and captures each one with the step's criteria and index
|
|
460
501
|
* fields (HTTP status of the main document included). A step script, when given, runs per page with `item`.
|
|
461
502
|
*/
|
|
462
|
-
async function walkStack ({ node, locale, page, env, run, scenario, scenarioDir, config, consoleFile, args }) {
|
|
503
|
+
async function walkStack ({ node, locale, page, env, run, scenario, scenarioDir, config, consoleFile, args, keys = null, tag = '' }) {
|
|
463
504
|
const { items, total, truncated } = await resolveItems({ node, base: baseUrl(env, locale), page, scenarioDir, env, locale, limit: Number(args.limit) || undefined })
|
|
464
505
|
const only = list(args.items)
|
|
465
506
|
const chosen = only ? items.filter((it) => only.some((q) => it.id === q || it.path.includes(q))) : items
|
|
466
|
-
const
|
|
467
|
-
const
|
|
468
|
-
const
|
|
469
|
-
|
|
507
|
+
const passKeys = keys || engineKeys(run.meta.scope, 'chromium')
|
|
508
|
+
const bases = baseViewports(run.meta.scope)
|
|
509
|
+
const vpNames = (node.items.viewports || []).filter((v) => bases.includes(v) || run.meta.scope.viewports[v])
|
|
510
|
+
const passBases = baseViewports({ ...run.meta.scope, viewports: Object.fromEntries(passKeys.map((k) => [k, run.meta.scope.viewports[k]])) })
|
|
511
|
+
const viewports = scopeKeys(run.meta.scope, passKeys, vpNames.length ? vpNames : [passBases[0] || bases[0]])
|
|
512
|
+
if (!Object.keys(viewports).length) { say(`▦ ${node.id} ${locale}${tag}: no viewport of this step in this pass`); return [] }
|
|
513
|
+
say(`▦ ${node.id} ${locale}${tag}: ${chosen.length} page(s)${truncated ? ` — limit ${Number(args.limit) || node.items.limit || DEFAULT_LIMIT} reached (${total}+ found; raise items.limit or pass --limit)` : ''}`)
|
|
470
514
|
const fields = fieldsOf(node)
|
|
471
515
|
const script = node.execution?.script ? await import(pathToFileURL(resolve(scenarioDir, node.execution.script)).href) : null
|
|
472
516
|
let status = null
|
|
@@ -480,7 +524,7 @@ async function walkStack ({ node, locale, page, env, run, scenario, scenarioDir,
|
|
|
480
524
|
status = null
|
|
481
525
|
const since = Date.now()
|
|
482
526
|
let captured = false
|
|
483
|
-
const capture = (opts = {}) => { captured = true; return captureStep({ page, node, locale, viewports: opts.viewports ?
|
|
527
|
+
const capture = (opts = {}) => { captured = true; return captureStep({ page, node, locale, viewports: opts.viewports ? scopeKeys(run.meta.scope, passKeys, opts.viewports) : viewports, run, consoleFile, since, item, fields, maxChars: config.evaluators?.jev?.max_state_chars, options: captureOptions(scenario, node), extra: { http_status: status } }) }
|
|
484
528
|
try {
|
|
485
529
|
if (script) await script.default({ page, item, url: () => item.url, locale, env, run: run.meta, capture, acceptDialogs: () => acceptDialogs(page) })
|
|
486
530
|
else await page.goto(item.url, { waitUntil: 'domcontentloaded', timeout: 30000 })
|
|
@@ -488,12 +532,12 @@ async function walkStack ({ node, locale, page, env, run, scenario, scenarioDir,
|
|
|
488
532
|
ok++; streak.ok('stack')
|
|
489
533
|
} catch (error) {
|
|
490
534
|
failed.push(`${item.id} (${item.path}): ${error.message.split('\n')[0]}`)
|
|
491
|
-
if (streak.record('stack', error.message)) say(` stopping ${node.id} (${locale}): the last 5 pages failed with the same error`)
|
|
535
|
+
if (streak.record('stack', error.message)) say(` stopping ${node.id} (${locale}${tag}): the last 5 pages failed with the same error`)
|
|
492
536
|
}
|
|
493
537
|
if ((i + 1) % 25 === 0 && i + 1 < chosen.length) say(` ${node.id} ${locale}: ${i + 1}/${chosen.length}`)
|
|
494
538
|
}
|
|
495
539
|
} finally { page.off('response', onResponse) }
|
|
496
|
-
say(`${failed.length ? '✗' : '✓'} ${node.id} ${locale}: ${ok}/${chosen.length} page(s) captured`)
|
|
540
|
+
say(`${failed.length ? '✗' : '✓'} ${node.id} ${locale}${tag}: ${ok}/${chosen.length} page(s) captured`)
|
|
497
541
|
return failed
|
|
498
542
|
}
|
|
499
543
|
|
|
@@ -505,67 +549,141 @@ function linkFor (env, node, locale) {
|
|
|
505
549
|
if (/^[a-z][a-z0-9+.-]*:/i.test(p)) return p
|
|
506
550
|
return env.base_url ? urlFor(env, { path: p }, locale) : null
|
|
507
551
|
}
|
|
552
|
+
/** A headless Chrome of its own for encoding recordings, started on first use (one per walk). */
|
|
553
|
+
function lazyEncoder (config) {
|
|
554
|
+
let started = null
|
|
555
|
+
return {
|
|
556
|
+
page: () => (started ||= launchEphemeral(config).then(async (b) => ({ b, p: await b.newPage() })).catch(() => ({ b: null, p: null }))).then((x) => x.p),
|
|
557
|
+
close: async () => { if (started) await (await started).b?.close().catch(() => {}) }
|
|
558
|
+
}
|
|
559
|
+
}
|
|
508
560
|
const appViewport = (run) => Object.keys(run.meta.scope.viewports || {})[0] || null
|
|
509
561
|
|
|
510
562
|
/**
|
|
511
|
-
* Walks an app scenario (platform android, ios, electron, desktop) with the environment's driver: per
|
|
512
|
-
* app starts fresh, each step runs its script (with `app`, the
|
|
563
|
+
* Walks an app scenario (platform android, ios, electron, desktop, windows) with the environment's driver: per
|
|
564
|
+
* device of the run (devices:, batch E) and locale the app starts fresh, each step runs its script (with `app`, the
|
|
565
|
+
* driver) or opens its deep link, then is captured in each of the device's keys (dark mode, reduced motion set on
|
|
566
|
+
* the device). `capture({ webview: true })` (or app.android.webview) captures an Android WebView's page instead.
|
|
513
567
|
*/
|
|
514
568
|
async function walkApp ({ config, root, run, scenario, env, args }) {
|
|
515
569
|
const platform = scenario.platform
|
|
516
570
|
const pick = list(args.steps)
|
|
517
571
|
const locales = list(args.locales) || run.meta.scope.locales
|
|
518
572
|
const scenarioDir = resolve(root, run.meta.scenario_file, '..')
|
|
573
|
+
const scope = run.meta.scope
|
|
519
574
|
const pending = []; const failed = []; const skipped = []
|
|
520
575
|
// WebP and thumbnails are encoded in a headless Chrome page (no native image dependency); PNG without Chrome
|
|
521
576
|
const enc = await launchEphemeral(config).catch(() => null)
|
|
522
577
|
const encoder = enc ? await enc.newPage().catch(() => null) : null
|
|
523
578
|
const streak = failureStreak(args['keep-going'] ? 0 : 3)
|
|
524
579
|
const maxChars = config.evaluators?.jev?.max_state_chars
|
|
580
|
+
// recordings and traces (batch D): Electron windows use the screencast recorder, devices record themselves
|
|
581
|
+
const recSettings = recordSettings({ args, config, scenario })
|
|
582
|
+
const groups = appDeviceGroups(scope, list(args.devices))
|
|
583
|
+
// app captures carry the scheme and motion of their key (no browser: the app draws itself)
|
|
584
|
+
const fieldsOf = (key) => { const { browser: _b, ...f } = variantFields(scope, key); return f }
|
|
525
585
|
try {
|
|
526
|
-
for (const
|
|
527
|
-
const
|
|
586
|
+
for (const group of groups) {
|
|
587
|
+
const tag = group.device && groups.length > 1 ? ` · ${group.title}` : ''
|
|
588
|
+
if (groups.length > 1) say(`\n${group.title}: ${group.keys.join(', ')}`)
|
|
589
|
+
// the device: its entry in app.<platform>.devices or what its name says; an emulator starts, a phone wakes up
|
|
590
|
+
let cfg = deviceConfig(platform, env.app?.[platform] || {}, group.device)
|
|
528
591
|
try {
|
|
529
|
-
|
|
530
|
-
if (
|
|
531
|
-
|
|
532
|
-
|
|
533
|
-
|
|
534
|
-
|
|
535
|
-
|
|
536
|
-
|
|
537
|
-
|
|
538
|
-
|
|
539
|
-
|
|
540
|
-
|
|
541
|
-
|
|
542
|
-
|
|
543
|
-
|
|
544
|
-
|
|
545
|
-
|
|
546
|
-
|
|
547
|
-
|
|
548
|
-
|
|
549
|
-
|
|
550
|
-
|
|
551
|
-
|
|
592
|
+
if (platform === 'android') { cfg = await prepareAndroid(cfg, { say }); await readyAndroid(cfg) }
|
|
593
|
+
if (platform === 'ios' && group.device) cfg = await prepareSimulator(cfg)
|
|
594
|
+
} catch (error) { failed.push(`${group.device || platform}: ${error.message.split('\n')[0]}`); say(`✗ ${group.device || platform}: ${error.message.split('\n')[0]}`); continue }
|
|
595
|
+
const denv = { ...env, app: { ...(env.app || {}), [platform]: cfg } }
|
|
596
|
+
// recordings and traces land on this device's keys (a failed step without a capture: its first key)
|
|
597
|
+
const grun = { ...run, meta: { ...run.meta, scope: { ...scope, viewports: scopeKeys(scope, group.keys) } } }
|
|
598
|
+
let look = { key: null, undo: async () => {} } // the system appearance the device is in now
|
|
599
|
+
const appearance = async (driver, key) => {
|
|
600
|
+
const v = scope.variants?.[key] || null
|
|
601
|
+
const want = JSON.stringify([v?.color_scheme || null, v?.reduced_motion || null])
|
|
602
|
+
if (look.key === want) return look.notes || []
|
|
603
|
+
await look.undo()
|
|
604
|
+
const r = await applyAppAppearance(platform, cfg, driver, v)
|
|
605
|
+
look = { key: want, undo: r.undo, notes: r.notes }
|
|
606
|
+
return r.notes
|
|
607
|
+
}
|
|
608
|
+
for (const locale of locales) {
|
|
609
|
+
const driver = driverFor(platform, denv, { root, scenarioDir, locale })
|
|
610
|
+
let webview = null
|
|
611
|
+
if (platform === 'android') driver.webview = async () => (webview ||= await attachAndroidWebView(cfg)).page
|
|
612
|
+
try {
|
|
613
|
+
await driver.open({ locale, reset: true })
|
|
614
|
+
if (driver.web) observeConsole(driver.page)
|
|
615
|
+
say(`${platform} ${locale}${tag}: ${driver.web ? 'Electron window' : 'app'} ready`)
|
|
616
|
+
} catch (error) { failed.push(`${platform} (${locale}${tag}): ${error.message.split('\n')[0]}`); say(`✗ ${platform} ${locale}${tag}: ${error.message.split('\n')[0]}`); await driver.close().catch(() => {}); if (error instanceof UserError) break; continue }
|
|
617
|
+
const rec = driver.web
|
|
618
|
+
? await recordingSession({ page: driver.page, run: grun, scenario, locale, settings: recSettings, viewports: grun.meta.scope.viewports, encoder: async () => encoder, say, label: `${platform} ${locale}${tag}` }).catch(() => null)
|
|
619
|
+
: await nativeSession({ platform, env: denv, run: grun, scenario, locale, settings: recSettings, driver, encoder: async () => encoder, say }).catch(() => null)
|
|
620
|
+
try {
|
|
621
|
+
for (const node of scenario.nodes) {
|
|
622
|
+
if (node.kind === 'decision' || (pick && !pick.includes(node.id))) continue
|
|
623
|
+
if (node.kind === 'stack') { pending.push(`${node.id} (${locale}): stack steps list web pages; not walked on ${platform}`); continue }
|
|
624
|
+
if (node.kind === 'email' && group !== groups[0]) continue // one message, captured on the first device
|
|
625
|
+
if (node.kind === 'email' && node.mail?.to) {
|
|
626
|
+
try { const r = await captureMail({ config, root, run, env, node, locale, to: expandRun(node.mail.to, run.meta.short), subject: localized(node.mail?.subject, locale), timeoutMs: 45000 }); say(`✓ ${node.id} ${locale}: "${r.subject}"`) } catch (error) { failed.push(`${node.id} (${locale}): ${error.message.split('\n')[0]}`); say(`✗ ${node.id} ${locale}: ${error.message.split('\n')[0]}`) }
|
|
627
|
+
continue
|
|
628
|
+
}
|
|
629
|
+
const scripted = ['playwright', 'script'].includes(node.execution?.strategy) && node.execution?.script
|
|
630
|
+
const link = linkFor(env, node, locale)
|
|
631
|
+
if (!scripted && !(link && ['playwright', 'script'].includes(node.execution?.strategy))) { pending.push(`${node.id} (${locale}): ${node.execution?.strategy || 'agent'} — ${(node.execution?.instructions || '').split('\n')[0]}`); continue }
|
|
632
|
+
const streakKey = `${group.device || ''}|${locale}`
|
|
633
|
+
if (streak.stopped(streakKey)) { skipped.push(`${node.id} (${locale}${tag})`); continue }
|
|
634
|
+
const records = []
|
|
635
|
+
try {
|
|
636
|
+
let captured = false
|
|
637
|
+
await rec?.beforeStep(node)
|
|
638
|
+
const capture = async (o = {}) => {
|
|
639
|
+
captured = true
|
|
640
|
+
await rec?.beforeCapture(node)
|
|
641
|
+
// a named viewport: within this device, or any of the run when it has no devices (as before)
|
|
642
|
+
const keys = o.viewport ? pickKeys(scope, group.device ? group.keys : Object.keys(scope.viewports), [o.viewport]) : group.keys
|
|
643
|
+
if (!keys.length) throw new UserError(`Unknown viewport "${o.viewport}" for ${group.device || platform}.`)
|
|
644
|
+
const out = []
|
|
645
|
+
for (const key of keys) {
|
|
646
|
+
if (driver.web) {
|
|
647
|
+
// Electron: a web capture; captureStep emulates the key's scheme and motion itself
|
|
648
|
+
const [w, h] = await driver.page.evaluate(() => [innerWidth, innerHeight])
|
|
649
|
+
out.push(...await captureStep({ page: driver.page, node, locale, options: captureOptions(scenario, node), viewports: { [key]: [w, h] }, run, consoleFile: null, maxChars, extra: { platform, device: 'Electron' } }))
|
|
650
|
+
continue
|
|
651
|
+
}
|
|
652
|
+
const notes = await appearance(driver, key)
|
|
653
|
+
if (o.webview || (cfg.webview && o.webview !== false)) {
|
|
654
|
+
if (!driver.webview) throw new UserError(`WebView captures need an Android app; on ${platform} capture the screen (tree or OCR).`)
|
|
655
|
+
const page = await driver.webview()
|
|
656
|
+
const [w, h] = await page.evaluate(() => [innerWidth, innerHeight])
|
|
657
|
+
out.push(...await captureStep({ page, node, locale, options: captureOptions(scenario, node), viewports: { [key]: [w, h] }, run, consoleFile: null, maxChars, extra: { platform, device: cfg.device || null, webview: true, ...fieldsOf(key) } }))
|
|
658
|
+
continue
|
|
659
|
+
}
|
|
660
|
+
out.push(await captureApp({ snapshot: await driver.snapshot(), node, locale, viewport: key, run, platform, encoder, maxChars, extra: { url: link, ...fieldsOf(key), ...(notes.length ? { notes } : {}) } }))
|
|
661
|
+
}
|
|
662
|
+
records.push(...out)
|
|
663
|
+
return out.length === 1 ? out[0] : out
|
|
552
664
|
}
|
|
553
|
-
|
|
665
|
+
if (scripted) {
|
|
666
|
+
const mod = await import(pathToFileURL(resolve(scenarioDir, node.execution.script)).href)
|
|
667
|
+
await mod.default({ app: driver, page: driver.page || null, webview: driver.webview || null, locale, node: { id: node.id, title: localized(node.title, locale), kind: node.kind }, device: group.device, env, run: run.meta, capture, link: (id = node.id) => linkFor(env, stepOf(scenario, id), locale), url: (id = node.id) => linkFor(env, stepOf(scenario, id), locale) })
|
|
668
|
+
} else await driver.link(link)
|
|
669
|
+
if (!captured) await capture()
|
|
670
|
+
await rec?.afterStep(node, driver.web ? { records } : { record: records[0] })
|
|
671
|
+
say(`✓ ${node.id} ${locale}${tag}`)
|
|
672
|
+
streak.ok(streakKey)
|
|
673
|
+
} catch (error) {
|
|
674
|
+
await rec?.afterStep(node, driver.web ? { error, records } : { error, record: records[0] }).catch(() => {})
|
|
675
|
+
failed.push(`${node.id} (${locale}${tag}): ${error.message.split('\n')[0]}`)
|
|
676
|
+
say(`✗ ${node.id} ${locale}${tag}: ${error.message.split('\n')[0]}`)
|
|
677
|
+
if (streak.record(streakKey, error.message)) say(` stopping ${locale}${tag}: the last 3 steps failed with the same error`)
|
|
554
678
|
}
|
|
555
|
-
if (scripted) {
|
|
556
|
-
const mod = await import(pathToFileURL(resolve(scenarioDir, node.execution.script)).href)
|
|
557
|
-
await mod.default({ app: driver, page: driver.page || null, locale, node: { id: node.id, title: localized(node.title, locale), kind: node.kind }, env, run: run.meta, capture, link: (id = node.id) => linkFor(env, stepOf(scenario, id), locale), url: (id = node.id) => linkFor(env, stepOf(scenario, id), locale) })
|
|
558
|
-
} else await driver.link(link)
|
|
559
|
-
if (!captured) await capture()
|
|
560
|
-
say(`✓ ${node.id} ${locale}`)
|
|
561
|
-
streak.ok(locale)
|
|
562
|
-
} catch (error) {
|
|
563
|
-
failed.push(`${node.id} (${locale}): ${error.message.split('\n')[0]}`)
|
|
564
|
-
say(`✗ ${node.id} ${locale}: ${error.message.split('\n')[0]}`)
|
|
565
|
-
if (streak.record(locale, error.message)) say(` stopping ${locale}: the last 3 steps failed with the same error`)
|
|
566
679
|
}
|
|
680
|
+
} finally {
|
|
681
|
+
for (const n of (await rec?.finish().catch((e) => [`recording: ${e.message}`])) || []) say(` ${n}`)
|
|
682
|
+
await webview?.close(); await driver.close().catch(() => {})
|
|
567
683
|
}
|
|
568
|
-
}
|
|
684
|
+
}
|
|
685
|
+
await look.undo().catch(() => {})
|
|
686
|
+
if (platform === 'android') await finishAndroid(cfg)
|
|
569
687
|
}
|
|
570
688
|
} finally { await enc?.close().catch(() => {}) }
|
|
571
689
|
if (pending.length) say(`\nSteps for an agent or a person (drive the app, then kaloko capture --step <id> --image shot.png [--tree dump.xml|json]):\n - ${[...new Set(pending)].join('\n - ')}`)
|
|
@@ -733,29 +851,40 @@ const commands = {
|
|
|
733
851
|
if (scenario.environments && !scenario.environments.includes(env.name)) throw new UserError(`Scenario ${scenario.id} is not allowed on ${env.name} (allowed: ${scenario.environments.join(', ')}).`)
|
|
734
852
|
if (env.readonly && !scenario.readonly) throw new UserError(`Environment ${env.name} is readonly; the scenario must declare readonly: true.`)
|
|
735
853
|
const locales = list(args.locales) || scenario.locales || ['default']
|
|
736
|
-
const
|
|
737
|
-
|
|
738
|
-
const
|
|
854
|
+
const scope = { ...startScope({ args, scenario, config, env }), ...(args.note ? { note: args.note } : {}) }
|
|
855
|
+
scope.locales = locales
|
|
856
|
+
const vpNames = baseViewports(scope)
|
|
739
857
|
const git = await gitMeta(root)
|
|
740
858
|
const executor = { kind: args.executor ? 'agent' : 'cli', name: args.executor || 'kaloko-cli' }
|
|
741
859
|
const { project, issue } = await projectAndIssue({ cwd: root, config, scenario, branch: git?.branch })
|
|
742
860
|
const appVer = typeof args['app-version'] === 'string' ? args['app-version'].slice(0, 80) : await appVersion(root)
|
|
743
861
|
const { dir, meta } = await createRun({ output: config.output, scenario, scenarioFile, env, scope, git, executor, root, project, issue, appVersion: appVer })
|
|
744
|
-
say(`Run ${meta.id}\n dir: ${relative(process.cwd(), dir)}\n env: ${env.name}${env.readonly ? ' (readonly)' : ''}\n locales: ${locales.join(', ')} viewports: ${vpNames.join(', ')}\n project: ${project.slug}${issue ? ` issue: ${issue.repo ? issue.repo + '#' : '#'}${issue.number}` : ''}`)
|
|
862
|
+
say(`Run ${meta.id}\n dir: ${relative(process.cwd(), dir)}\n env: ${env.name}${env.readonly ? ' (readonly)' : ''}\n locales: ${locales.join(', ')} viewports: ${vpNames.length > 12 ? `${vpNames.slice(0, 12).join(', ')} … (${vpNames.length})` : vpNames.join(', ')}${dimensionLine(scope)}\n project: ${project.slug}${issue ? ` issue: ${issue.repo ? issue.repo + '#' : '#'}${issue.number}` : ''}`)
|
|
745
863
|
const addr = recipient(env.mail, meta.short)
|
|
746
864
|
if (addr) say(` test e-mail: ${addr}`)
|
|
747
865
|
say(` other agents may start runs here too: pin this one with --run ${meta.id} or KALOKO_RUN=${meta.id}`)
|
|
748
866
|
},
|
|
749
867
|
|
|
868
|
+
/** `kaloko devices`: the device presets a scenario's viewports may name. */
|
|
869
|
+
async devices (args) {
|
|
870
|
+
const { DEVICES, devicePreset } = await import('./lib/devices.mjs')
|
|
871
|
+
const all = Object.keys(DEVICES).map((n) => devicePreset(n))
|
|
872
|
+
if (args.json) { say(JSON.stringify(all, null, 2)); return }
|
|
873
|
+
for (const d of all) say(`${d.name.padEnd(26)} ${d.title.padEnd(28)} ${`${d.size[0]}×${d.size[1]}`.padEnd(10)} screen ${`${d.screen[0]}×${d.screen[1]}`.padEnd(10)} ${String(d.dpr).padEnd(6)}× ${d.touch ? 'touch' : ''}`)
|
|
874
|
+
say('\nUse them in viewports: (a list of names, or name: preset), or kaloko start --viewports iphone-15,desktop-hd.\n<preset>-landscape turns a phone, foldable or tablet sideways; Design environments use the whole screen.')
|
|
875
|
+
},
|
|
876
|
+
|
|
750
877
|
async browser (args) {
|
|
751
878
|
const { config, root } = await context(args)
|
|
752
879
|
const action = args._[0] || 'status'
|
|
753
880
|
if (action === 'install') {
|
|
754
|
-
// the
|
|
881
|
+
// the engines of the Playwright version kaloko itself uses (CI: --with-deps adds the system libraries)
|
|
882
|
+
const engines = list(args.browsers) || ['chromium']
|
|
883
|
+
for (const e of engines) if (!BROWSERS.includes(e)) throw new UserError(`Unknown browser engine ${e}. Use: ${BROWSERS.join(', ')}.`)
|
|
755
884
|
const { spawn } = await import('node:child_process')
|
|
756
885
|
const cli = join(dirname(fileURLToPath(import.meta.resolve('playwright/package.json'))), 'cli.js')
|
|
757
|
-
const code = await new Promise((done) => spawn(process.execPath, [cli, 'install', ...(args['with-deps'] ? ['--with-deps'] : []),
|
|
758
|
-
if (code) throw new UserError(`Installing
|
|
886
|
+
const code = await new Promise((done) => spawn(process.execPath, [cli, 'install', ...(args['with-deps'] ? ['--with-deps'] : []), ...engines], { stdio: 'inherit' }).on('close', done))
|
|
887
|
+
if (code) throw new UserError(`Installing ${engines.join(', ')} failed (exit ${code}).`)
|
|
759
888
|
return
|
|
760
889
|
}
|
|
761
890
|
let env = null
|
|
@@ -798,13 +927,17 @@ const commands = {
|
|
|
798
927
|
return
|
|
799
928
|
}
|
|
800
929
|
if (isApp(scenario.platform)) {
|
|
801
|
-
// the app's current screen, through the environment's driver (no navigation)
|
|
802
|
-
const
|
|
930
|
+
// the app's current screen, through the environment's driver (no navigation); --viewport may name a device key
|
|
931
|
+
const key = typeof args.viewport === 'string' ? args.viewport : Object.keys(run.meta.scope.viewports || {})[0]
|
|
932
|
+
const device = run.meta.scope.devices?.[variantOf(run.meta.scope, key).viewport]?.app ? variantOf(run.meta.scope, key).viewport : null
|
|
933
|
+
const cfg = deviceConfig(scenario.platform, env.app?.[scenario.platform] || {}, device)
|
|
934
|
+
const driver = driverFor(scenario.platform, { ...env, app: { ...(env.app || {}), [scenario.platform]: cfg } }, { root, scenarioDir: resolve(root, run.meta.scenario_file, '..'), locale })
|
|
803
935
|
if (driver.web) throw new UserError('For Electron, run kaloko walk --steps <id>, or pass --image.')
|
|
804
936
|
await driver.open({ locale, attach: true })
|
|
805
937
|
const enc = await launchEphemeral(config).catch(() => null)
|
|
806
938
|
try {
|
|
807
|
-
const
|
|
939
|
+
const { browser: _b, ...dims } = variantFields(run.meta.scope, key)
|
|
940
|
+
const r = await captureApp({ snapshot: await driver.snapshot(), node, locale, viewport: key, run, platform: scenario.platform, encoder: enc ? await enc.newPage() : null, maxChars: config.evaluators?.jev?.max_state_chars, extra: dims })
|
|
808
941
|
say(`✓ ${node.id} ${locale} ${r.viewport} ${r.device || scenario.platform} ${r.files.tree ? `UI tree (${r.capture.tree})` : 'no UI tree'}`)
|
|
809
942
|
} finally { await enc?.close().catch(() => {}) }
|
|
810
943
|
return
|
|
@@ -829,7 +962,7 @@ const commands = {
|
|
|
829
962
|
return
|
|
830
963
|
}
|
|
831
964
|
const since = (await readCaptures(run.dir)).filter((c) => c.node === node.id).map((c) => c.at).pop()
|
|
832
|
-
const results = await captureStep({ page, node, locale, options: captureOptions(scenario, node), viewports:
|
|
965
|
+
const results = await captureStep({ page, node, locale, options: captureOptions(scenario, node), viewports: manualKeys(run, list(args.viewports)), run, consoleFile: conn ? consoleLog(root, config.output, env.name) : null, maxChars: config.evaluators?.jev?.max_state_chars, since, htmlView: htmlViewAllowed(env, scenario) ? { scenarioId: scenario.id } : null, stability: Number(args.stability) || 0 })
|
|
833
966
|
for (const r of results) {
|
|
834
967
|
const failed = Object.entries(r.checks).filter(([, v]) => v.ok === false).map(([k]) => k)
|
|
835
968
|
say(`✓ ${node.id} ${locale} ${r.viewport} ${r.size.width}×${r.size.height}px ${failed.length ? `failing checks: ${failed.join(', ')}` : 'checks ok'}${r.blocked_requests.length ? ` blocked: ${r.blocked_requests.length}` : ''}`)
|
|
@@ -861,6 +994,11 @@ const commands = {
|
|
|
861
994
|
say(`✓ ${node.id} ${locale}: "${results.subject}" → ${results.count} viewport(s)`)
|
|
862
995
|
},
|
|
863
996
|
|
|
997
|
+
async video (args) {
|
|
998
|
+
const { config, root } = await context(args)
|
|
999
|
+
await videoCmd({ config, root, args, say })
|
|
1000
|
+
},
|
|
1001
|
+
|
|
864
1002
|
async walk (args) {
|
|
865
1003
|
const { config, root, run, scenario, env } = await runContext(args)
|
|
866
1004
|
if (args.affected) {
|
|
@@ -890,130 +1028,184 @@ const commands = {
|
|
|
890
1028
|
if (isApp(scenario.platform)) return walkApp({ config, root, run, scenario, env, args })
|
|
891
1029
|
const pick = list(args.steps)
|
|
892
1030
|
const locales = list(args.locales) || run.meta.scope.locales
|
|
1031
|
+
const scope = run.meta.scope
|
|
1032
|
+
// browser engines and device profiles (batch E): one pass each, capturing only its own keys; a run without
|
|
1033
|
+
// dimensions has a single pass, the walk of before
|
|
1034
|
+
const only = args.browsers ? dimensionValues('browsers', args.browsers) : null
|
|
1035
|
+
const passes = passesOf(scope, { only })
|
|
1036
|
+
if (!passes.length) throw new UserError(`This run has no captures in ${only.join(', ')} (its browsers: ${enginesOf(scope).join(', ')}).`)
|
|
1037
|
+
const needChromium = passes.some((p) => p.engine === 'chromium')
|
|
893
1038
|
// --ephemeral: an own browser (own cookies and tabs) even when the shared one runs, so runs can walk in parallel
|
|
894
1039
|
configureGate(await gateForRequests(env, { root }))
|
|
895
|
-
const conn = args.ephemeral ? null : await connectBrowser({ root, config, env })
|
|
1040
|
+
const conn = args.ephemeral || !needChromium ? null : await connectBrowser({ root, config, env })
|
|
896
1041
|
const releaseLock = conn ? await acquireWalkLock({ root, config, env, run: run.meta.id }).catch(async (e) => { await conn.browser.close(); throw e }) : null
|
|
897
|
-
const ephemeral = conn ? null : await launchEphemeral(config)
|
|
898
|
-
const
|
|
1042
|
+
const ephemeral = conn || !needChromium ? null : await launchEphemeral(config)
|
|
1043
|
+
const chromium = conn ? conn.browser : ephemeral
|
|
899
1044
|
// locales are independent: with --concurrency > 1 each walks in its own browser context at the same time (#27);
|
|
900
1045
|
// one locale at a time keeps the shared browser's own context, as before
|
|
901
1046
|
const concurrency = walkConcurrency(args, config, locales.length, !!conn)
|
|
902
1047
|
const parallel = concurrency > 1 && locales.length > 1
|
|
903
|
-
const ctx = parallel ? null : conn ? conn.context : await newContext(ephemeral, env, { root, config })
|
|
904
1048
|
const scenarioDir = resolve(root, run.meta.scenario_file, '..')
|
|
905
1049
|
// semantic questions of every new capture start right away; the walk ends with the verdicts (--no-evaluate: skip)
|
|
906
1050
|
const prefetch = args['no-evaluate'] ? null : semanticPrefetch({ scenario, config, root })
|
|
907
1051
|
const walked = await readCaptures(run.dir)
|
|
1052
|
+
// recordings and traces (batch D, #12): opt-in recordings, traces unless turned off; encoded in an own headless Chrome
|
|
1053
|
+
const recSettings = recordSettings({ args, config, scenario })
|
|
1054
|
+
const encoder = lazyEncoder(config)
|
|
908
1055
|
const sessions = scenario.sessions || null
|
|
909
|
-
// named sessions: one isolated browser context per person and locale (cookies, storage, sign-in), created lazily
|
|
910
|
-
const sessionContexts = new Map()
|
|
911
1056
|
const ownContexts = []
|
|
1057
|
+
const engines = new Map() // WebKit and Firefox, launched for their passes
|
|
912
1058
|
// `account` stays the scenario's key; `credentials` resolves it in this environment (config accounts)
|
|
913
1059
|
const sessionInfo = (key) => key && sessions?.[key] ? { key, title: localized(sessions[key].title, locales[0]), mailbox: sessions[key].mailbox ? expandRun(sessions[key].mailbox, run.meta.short) : null, account: sessions[key].account || null, credentials: accountFor(env, sessions[key].account, { root, config }) } : null
|
|
914
|
-
const contextFor = async (key, locale) => {
|
|
915
|
-
if (!key || !sessions?.[key]) return null
|
|
916
|
-
const id = `${key}|${locale}`
|
|
917
|
-
if (!sessionContexts.has(id)) {
|
|
918
|
-
// every context gets the environment's gate and readonly policy; an account's saved sign-in is loaded
|
|
919
|
-
const account = accountFor(env, sessions[key].account, { root, config })
|
|
920
|
-
const c = await newContext(browser, env, { root, config, account })
|
|
921
|
-
const p = await c.newPage()
|
|
922
|
-
observeConsole(p)
|
|
923
|
-
sessionContexts.set(id, { context: c, page: p })
|
|
924
|
-
const saved = account && await access(account.storage_state).then(() => true, () => false)
|
|
925
|
-
say(`session ${key} (${locale}): new browser context${saved ? ' with the saved sign-in' : ''}`)
|
|
926
|
-
}
|
|
927
|
-
return sessionContexts.get(id)
|
|
928
|
-
}
|
|
929
1060
|
// stop a session after the same failure repeats (shared precondition such as a failed sign-in);
|
|
930
1061
|
// --keep-going walks every step regardless
|
|
931
1062
|
const streak = failureStreak(args['keep-going'] ? 0 : 3)
|
|
932
1063
|
const proposals = []
|
|
933
|
-
|
|
934
|
-
const
|
|
935
|
-
|
|
936
|
-
|
|
937
|
-
|
|
938
|
-
|
|
939
|
-
|
|
940
|
-
|
|
941
|
-
|
|
942
|
-
|
|
943
|
-
|
|
944
|
-
|
|
945
|
-
|
|
946
|
-
|
|
947
|
-
|
|
948
|
-
|
|
949
|
-
|
|
950
|
-
|
|
951
|
-
const
|
|
952
|
-
|
|
953
|
-
|
|
954
|
-
|
|
955
|
-
|
|
956
|
-
|
|
957
|
-
|
|
958
|
-
|
|
959
|
-
|
|
960
|
-
|
|
961
|
-
|
|
962
|
-
|
|
963
|
-
|
|
964
|
-
|
|
965
|
-
|
|
966
|
-
|
|
967
|
-
|
|
968
|
-
|
|
969
|
-
|
|
970
|
-
|
|
971
|
-
say(`✗ ${node.id} ${locale}: ${error.message.split('\n')[0]}`)
|
|
972
|
-
}
|
|
973
|
-
continue
|
|
1064
|
+
const results = []
|
|
1065
|
+
const recSkipped = new Set() // engines whose passes are not recorded (once per engine)
|
|
1066
|
+
try {
|
|
1067
|
+
for (const [passIndex, pass] of passes.entries()) {
|
|
1068
|
+
const first = passIndex === 0
|
|
1069
|
+
const plain = pass.engine === 'chromium' && !pass.emulation
|
|
1070
|
+
let browser = chromium
|
|
1071
|
+
if (pass.engine !== 'chromium') {
|
|
1072
|
+
if (!engines.has(pass.engine)) engines.set(pass.engine, await launchEngine(pass.engine))
|
|
1073
|
+
browser = engines.get(pass.engine)
|
|
1074
|
+
}
|
|
1075
|
+
const extra = contextOptionsOf(pass)
|
|
1076
|
+
const tag = passes.length > 1 && pass.label ? ` · ${pass.label}` : ''
|
|
1077
|
+
if (passes.length > 1) say(`\n${pass.label || 'chromium'}: ${pass.keys.join(', ')}`)
|
|
1078
|
+
// recordings and traces (batch D) in Chromium passes; a flow recording is one per locale, made in the first
|
|
1079
|
+
let passRec = recSettings
|
|
1080
|
+
if (pass.engine !== 'chromium') {
|
|
1081
|
+
passRec = null
|
|
1082
|
+
const wanted = !!recSettings.record || scenario.nodes.some((n) => parseRecord(n.record ?? n.capture?.record))
|
|
1083
|
+
if (wanted && !recSkipped.has(pass.engine)) { recSkipped.add(pass.engine); say(`Recording and traces skipped in ${pass.engine}: they need Chromium (the Chromium keys of this run are recorded).`) }
|
|
1084
|
+
} else if (passes.slice(0, passIndex).some((p) => p.engine === 'chromium') && recSettings.record?.mode === 'flow') passRec = { ...recSettings, record: null }
|
|
1085
|
+
const ctx = parallel ? null : plain && conn ? conn.context : await newContext(browser, env, { root, config, extra })
|
|
1086
|
+
if (ctx && ctx !== conn?.context) ownContexts.push(ctx)
|
|
1087
|
+
// named sessions: one isolated browser context per person and locale (cookies, storage, sign-in), created lazily
|
|
1088
|
+
const sessionContexts = new Map()
|
|
1089
|
+
const contextFor = async (key, locale) => {
|
|
1090
|
+
if (!key || !sessions?.[key]) return null
|
|
1091
|
+
const id = `${key}|${locale}`
|
|
1092
|
+
if (!sessionContexts.has(id)) {
|
|
1093
|
+
// every context gets the environment's gate and readonly policy; an account's saved sign-in is loaded
|
|
1094
|
+
const account = accountFor(env, sessions[key].account, { root, config })
|
|
1095
|
+
const c = await newContext(browser, env, { root, config, account, extra })
|
|
1096
|
+
ownContexts.push(c)
|
|
1097
|
+
const p = await c.newPage()
|
|
1098
|
+
observeConsole(p)
|
|
1099
|
+
sessionContexts.set(id, { context: c, page: p })
|
|
1100
|
+
const saved = account && await access(account.storage_state).then(() => true, () => false)
|
|
1101
|
+
say(`session ${key} (${locale}${tag}): new browser context${saved ? ' with the saved sign-in' : ''}`)
|
|
974
1102
|
}
|
|
975
|
-
|
|
976
|
-
|
|
977
|
-
|
|
978
|
-
|
|
979
|
-
|
|
980
|
-
|
|
981
|
-
|
|
982
|
-
|
|
983
|
-
|
|
984
|
-
|
|
985
|
-
|
|
1103
|
+
return sessionContexts.get(id)
|
|
1104
|
+
}
|
|
1105
|
+
// one locale: its own tab (and, in parallel, its own context); returns what it could not walk
|
|
1106
|
+
const walkLocale = async (locale) => {
|
|
1107
|
+
const pending = []; const failed = []; const skipped = []
|
|
1108
|
+
let lctx = ctx
|
|
1109
|
+
if (parallel) { lctx = await newContext(browser, env, { root, config, extra }); ownContexts.push(lctx) }
|
|
1110
|
+
// the walk owns its tab; agents and people keep theirs in the same shared browser
|
|
1111
|
+
const page = await lctx.newPage()
|
|
1112
|
+
const shared = conn && !parallel && plain // the shared browser's own context logs its console to a file
|
|
1113
|
+
if (!shared) observeConsole(page)
|
|
1114
|
+
// one recorder per page (the walk's tab and each named session's), started on its first step;
|
|
1115
|
+
// recordings and traces need CDP: Chromium passes only
|
|
1116
|
+
const recorders = new Map()
|
|
1117
|
+
const recorderFor = async (p) => {
|
|
1118
|
+
if (!passRec) return null
|
|
1119
|
+
if (!recorders.has(p)) recorders.set(p, recordingSession({ page: p, run, scenario, locale, settings: passRec, viewports: scopeKeys(scope, pass.keys), encoder: encoder.page, say, label: `${locale}${tag}` }).catch(() => null))
|
|
1120
|
+
return recorders.get(p)
|
|
986
1121
|
}
|
|
987
1122
|
try {
|
|
988
|
-
|
|
989
|
-
say(
|
|
990
|
-
|
|
991
|
-
|
|
992
|
-
|
|
993
|
-
|
|
994
|
-
|
|
995
|
-
|
|
996
|
-
|
|
997
|
-
|
|
1123
|
+
// every locale starts as a new visitor on its first walk in the run (an own context is new already)
|
|
1124
|
+
if (shared && !walked.some((c) => c.locale === locale)) { await resetSession(lctx, page, baseUrl(env, locale)); say(`session reset (new visitor, ${locale}${tag})`) }
|
|
1125
|
+
for (const node of scenario.nodes) {
|
|
1126
|
+
if (node.kind === 'decision' || (pick && !pick.includes(node.id))) continue
|
|
1127
|
+
if (node.kind !== 'stack' && node.path && !urlFor(env, node, locale)) continue // step does not exist in this locale
|
|
1128
|
+
if (node.session && sessions?.[node.session]?.blocked_on?.includes(env.name)) { pending.push(`${node.id} (${locale}${tag}): session ${node.session} is blocked on ${env.name}`); continue }
|
|
1129
|
+
const sess = await contextFor(node.session, locale)
|
|
1130
|
+
const sPage = sess ? sess.page : page
|
|
1131
|
+
const sCtx = sess ? sess.context : lctx
|
|
1132
|
+
const consoleFile = shared && !sess ? consoleLog(root, config.output, env.name) : null
|
|
1133
|
+
if (node.kind === 'stack') {
|
|
1134
|
+
try {
|
|
1135
|
+
const bad = await walkStack({ node, locale, page: sPage, env, run, scenario, scenarioDir, config, consoleFile, args, keys: pass.keys, tag })
|
|
1136
|
+
for (const b of bad) failed.push(`${node.id} (${locale}${tag}) ${b}`)
|
|
1137
|
+
} catch (error) {
|
|
1138
|
+
failed.push(`${node.id} (${locale}${tag}): ${error.message.split('\n')[0]}`)
|
|
1139
|
+
say(`✗ ${node.id} ${locale}${tag}: ${error.message.split('\n')[0]}`)
|
|
1140
|
+
}
|
|
1141
|
+
continue
|
|
1142
|
+
}
|
|
1143
|
+
if (node.kind === 'email' && !first) continue // one message, captured in the first pass
|
|
1144
|
+
if (node.kind === 'email' && (node.mail?.to || sessionInfo(node.session)?.mailbox)) {
|
|
1145
|
+
// an e-mail node with a recipient is walked like a step: wait for the message and capture it
|
|
1146
|
+
try {
|
|
1147
|
+
const to = expandRun(node.mail?.to || sessionInfo(node.session).mailbox, run.meta.short)
|
|
1148
|
+
const results = await captureMail({ config, root, run, env, node, locale, to, subject: localized(node.mail?.subject, locale), timeoutMs: 45000 })
|
|
1149
|
+
say(`✓ ${node.id} ${locale}${tag}: "${results.subject}"`)
|
|
1150
|
+
} catch (error) {
|
|
1151
|
+
failed.push(`${node.id} (${locale}${tag}): ${error.message.split('\n')[0]}`)
|
|
1152
|
+
say(`✗ ${node.id} ${locale}${tag}: ${error.message.split('\n')[0]}`)
|
|
1153
|
+
}
|
|
1154
|
+
continue
|
|
1155
|
+
}
|
|
1156
|
+
if (!['playwright', 'script'].includes(node.execution.strategy) || !node.execution.script) { pending.push(`${node.id} (${locale}${tag}): ${node.execution.strategy} — ${node.execution.instructions.split('\n')[0]}`); continue }
|
|
1157
|
+
const streakKey = `${pass.engine}|${pass.emulation?.key || ''}|${node.session || ''}|${locale}`
|
|
1158
|
+
if (streak.stopped(streakKey)) { skipped.push(`${node.id} (${locale}${tag})`); continue }
|
|
1159
|
+
const scriptFile = resolve(scenarioDir, node.execution.script)
|
|
1160
|
+
// one invocation of the step script; the automatic capture runs unless the script captured the step itself:
|
|
1161
|
+
// a crop is an attachment (#49), and a capture from an earlier walk must not stand in for this one (#50)
|
|
1162
|
+
const rec = await recorderFor(sPage)
|
|
1163
|
+
const runStep = async (mod) => {
|
|
1164
|
+
let captured = false
|
|
1165
|
+
rec?.beforeStep(node)
|
|
1166
|
+
const records = []
|
|
1167
|
+
const capture = (opts = {}) => { captured = true; rec?.beforeCapture(node); return captureStep({ page: sPage, node, locale, options: captureOptions(scenario, node), viewports: scopeKeys(run.meta.scope, pass.keys, opts.viewports || null), run, htmlView: htmlViewAllowed(env, scenario) ? { scenarioId: scenario.id } : null, consoleFile, maxChars: config.evaluators?.jev?.max_state_chars, extra: node.session ? { session: node.session } : {}, stability: Number(args.stability) || 0 }).then((r) => { prefetch?.add(r, run.dir); records.push(...r); return r }) }
|
|
1168
|
+
try {
|
|
1169
|
+
await mod.default({ page: sPage, context: sCtx, locale, node: { id: node.id, title: localized(node.title, locale), kind: node.kind }, session: sessionInfo(node.session), url: (id = node.id) => urlFor(env, stepOf(scenario, id), locale), env, run: run.meta, capture, crop: (selector, title, o = {}) => captureCrop({ page: sPage, node, locale, selector, title, run, padding: Number(o.padding) || 0 }), acceptDialogs: () => acceptDialogs(sPage) })
|
|
1170
|
+
if (!captured) await capture()
|
|
1171
|
+
} catch (error) { await rec?.afterStep(node, { error }); throw error }
|
|
1172
|
+
await rec?.afterStep(node, { records })
|
|
1173
|
+
}
|
|
1174
|
+
try {
|
|
1175
|
+
await runStep(await import(pathToFileURL(scriptFile).href))
|
|
1176
|
+
say(`✓ ${node.id} ${locale}${tag}`)
|
|
1177
|
+
streak.ok(streakKey)
|
|
1178
|
+
} catch (error) {
|
|
1179
|
+
// a selector the page no longer has: propose one from the page (#13); --heal applies it once the step passes
|
|
1180
|
+
const healed = args['no-heal'] ? null : await healStep({ error, page: sPage, scriptFile, root, output: config.output, run, node, locale, apply: !!args.heal, rerun: runStep }).catch(() => null)
|
|
1181
|
+
if (healed?.applied) {
|
|
1182
|
+
say(`✓ ${node.id} ${locale}${tag} (healed)\n${formatHeal(healed, { stepArg: node.id })}`)
|
|
1183
|
+
streak.ok(streakKey)
|
|
1184
|
+
continue
|
|
1185
|
+
}
|
|
1186
|
+
failed.push(`${node.id} (${locale}${tag}): ${error.message.split('\n')[0]}`)
|
|
1187
|
+
say(`✗ ${node.id} ${locale}${tag}: ${error.message.split('\n')[0]}`)
|
|
1188
|
+
if (healed) { say(formatHeal(healed, { stepArg: node.id })); if (healed.patch) proposals.push(node.id) }
|
|
1189
|
+
if (streak.record(streakKey, error.message)) say(` stopping ${node.session ? `session ${node.session}` : 'this walk'} (${locale}${tag}): the last 3 steps failed with the same error; fix the shared cause (sign-in, environment) and rerun, or pass --keep-going`)
|
|
1190
|
+
}
|
|
998
1191
|
}
|
|
999
|
-
|
|
1000
|
-
|
|
1001
|
-
|
|
1002
|
-
if (streak.record(streakKey, error.message)) say(` stopping ${node.session ? `session ${node.session}` : 'this walk'} (${locale}): the last 3 steps failed with the same error; fix the shared cause (sign-in, environment) and rerun, or pass --keep-going`)
|
|
1192
|
+
} finally {
|
|
1193
|
+
for (const r of recorders.values()) { const notes = await (await r)?.finish().catch((e) => [`recording: ${e.message}`]); for (const n of notes || []) say(` ${n}`) }
|
|
1194
|
+
await page.close().catch(() => {})
|
|
1003
1195
|
}
|
|
1196
|
+
return { pending, failed, skipped }
|
|
1004
1197
|
}
|
|
1005
|
-
|
|
1006
|
-
|
|
1007
|
-
|
|
1008
|
-
|
|
1009
|
-
|
|
1010
|
-
if (parallel) say(`Walking ${locales.length} locales, ${concurrency} at a time (each in its own browser context); output in locale order.`)
|
|
1011
|
-
results = await inOrder(locales, parallel ? concurrency : 1, (locale) => walkLocale(locale).catch((error) => ({ pending: [], failed: [`${locale}: ${error.message.split('\n')[0]}`], skipped: [] })))
|
|
1198
|
+
if (parallel) say(`Walking ${locales.length} locales, ${concurrency} at a time (each in its own browser context); output in locale order.`)
|
|
1199
|
+
results.push(...await inOrder(locales, parallel ? concurrency : 1, (locale) => walkLocale(locale).catch((error) => ({ pending: [], failed: [`${locale}${tag}: ${error.message.split('\n')[0]}`], skipped: [] }))))
|
|
1200
|
+
// the next pass has its own contexts: close this one's now
|
|
1201
|
+
for (const c of ownContexts.splice(0)) await c.close().catch(() => {})
|
|
1202
|
+
}
|
|
1012
1203
|
} finally {
|
|
1013
|
-
for (const { context } of sessionContexts.values()) await context.close().catch(() => {})
|
|
1014
1204
|
for (const c of ownContexts) await c.close().catch(() => {})
|
|
1205
|
+
for (const b of engines.values()) await b.close().catch(() => {})
|
|
1015
1206
|
if (conn) await conn.browser.close() // over CDP this only disconnects; the shared browser keeps running
|
|
1016
1207
|
if (ephemeral) await ephemeral.close()
|
|
1208
|
+
await encoder.close()
|
|
1017
1209
|
await releaseLock?.()
|
|
1018
1210
|
}
|
|
1019
1211
|
const pending = results.flatMap((r) => r.pending); const failed = results.flatMap((r) => r.failed); const skipped = results.flatMap((r) => r.skipped)
|
|
@@ -1107,7 +1299,7 @@ const commands = {
|
|
|
1107
1299
|
const kept = await keepVideos(run, source.videos)
|
|
1108
1300
|
say(`\nImported ${captured} screen(s)${attached ? `, ${attached} more as attachments` : ''} into ${run.meta.id}.${encoder ? '' : ' (No Chrome: images kept as they are, without thumbnails.)'}`)
|
|
1109
1301
|
if (unmatched.length) say(`Not matched to a step (${unmatched.length}): name the step in steps.json ("step": "<id>"), or pass --map "${unmatched[0]}=<step id>". Steps: ${scenario.nodes.filter((n) => n.kind !== 'decision').map((n) => n.id).join(', ')}`)
|
|
1110
|
-
if (kept.length) say(`Recordings kept in the run folder, not
|
|
1302
|
+
if (kept.length) say(`Recordings kept in the run folder, not attached to a step and not uploaded: ${kept.join(', ')}. kaloko walk --record records steps for the canvas.`)
|
|
1111
1303
|
say('Next: kaloko evaluate, then kaloko preview (local canvas) or kaloko share. Imported screens have no page to check: their deterministic criteria wait for a walk, semantic ones for a person.')
|
|
1112
1304
|
},
|
|
1113
1305
|
|
|
@@ -1149,17 +1341,20 @@ const commands = {
|
|
|
1149
1341
|
say(`Added the environment ${envName} to ${relative(process.cwd(), file) || CONFIG_FILE}: base_url \${KALOKO_PREVIEW_URL}, set by kaloko ci preview-url in CI.`)
|
|
1150
1342
|
} else say(`Add to environments: in ${CONFIG_FILE}:\n ${envName}:\n base_url: \${KALOKO_PREVIEW_URL}`)
|
|
1151
1343
|
}
|
|
1344
|
+
// the engines the job installs: Chromium and whatever the scenarios and walk.browsers ask for (batch E)
|
|
1345
|
+
const engines = new Set(['chromium', ...(config.walk?.browsers || [])])
|
|
1152
1346
|
for (const f of scenarios) {
|
|
1153
1347
|
let sc = null
|
|
1154
1348
|
try { sc = yaml.load(await readFile(resolve(root, f), 'utf8')) } catch { throw new UserError(`${f} cannot be read.`) }
|
|
1155
1349
|
if (sc?.environments && !sc.environments.includes(envName)) say(`! ${f} allows environments [${sc.environments.join(', ')}]: add ${envName}, or kaloko start refuses it in CI.`)
|
|
1350
|
+
for (const b of sc?.browsers || []) engines.add(b)
|
|
1156
1351
|
}
|
|
1157
1352
|
let pkg = null
|
|
1158
1353
|
try { pkg = JSON.parse(await readFile(join(root, 'package.json'), 'utf8')) } catch {}
|
|
1159
1354
|
const pm = existsSync(join(root, 'pnpm-lock.yaml')) ? 'pnpm' : existsSync(join(root, 'yarn.lock')) ? 'yarn' : existsSync(join(root, 'bun.lockb')) || existsSync(join(root, 'bun.lock')) ? 'bun' : 'npm'
|
|
1160
1355
|
const dependency = !!(pkg && ({ ...pkg.dependencies, ...pkg.devDependencies }).kaloko)
|
|
1161
1356
|
const version = JSON.parse(await readFile(join(packageRoot, 'package.json'), 'utf8')).version
|
|
1162
|
-
const opts = { scenarios, env: envName, affected: !args['no-affected'], pm, dependency, version: `^${version}`, trigger: args.trigger === 'pull_request' ? 'pull_request' : 'deployment' }
|
|
1357
|
+
const opts = { scenarios, env: envName, affected: !args['no-affected'], pm, dependency, version: `^${version}`, trigger: args.trigger === 'pull_request' ? 'pull_request' : 'deployment', browsers: [...engines] }
|
|
1163
1358
|
const content = action === 'github' ? githubWorkflow(opts) : gitlabJob(opts)
|
|
1164
1359
|
if (args.print) { say(content); return }
|
|
1165
1360
|
const out = resolve(root, typeof args.out === 'string' ? args.out : action === 'github' ? '.github/workflows/kaloko.yml' : 'kaloko.gitlab-ci.yml')
|
|
@@ -1311,9 +1506,16 @@ const commands = {
|
|
|
1311
1506
|
await writeFile(join(run.dir, 'meta.json'), JSON.stringify(meta, null, 2) + '\n')
|
|
1312
1507
|
return
|
|
1313
1508
|
}
|
|
1314
|
-
|
|
1509
|
+
let manifest = await buildManifest({ run, scenario, config, root, catalog: [] })
|
|
1315
1510
|
await writePreview({ run, manifest })
|
|
1316
|
-
|
|
1511
|
+
// the plan and storage before anything is sent; recordings travel on plans with the recordings feature only
|
|
1512
|
+
const u = await client.request('GET', `/orgs/${config.org}/usage`).catch(() => null)
|
|
1513
|
+
const recs = recordingFiles(manifest)
|
|
1514
|
+
if (recs.length && u?.limits?.features && !u.limits.features.recordings) {
|
|
1515
|
+
manifest = stripRecordings(manifest)
|
|
1516
|
+
say(`Recordings stay in the local preview (${recs.length} file(s)): sharing them is part of the Business plan and up, and this organization is on ${u.limits.plan_name}. Screenshots and traces are shared as usual.`)
|
|
1517
|
+
}
|
|
1518
|
+
const files = new Set([...traceFiles(manifest), ...recordingFiles(manifest)])
|
|
1317
1519
|
for (const n of manifest.nodes) {
|
|
1318
1520
|
for (const byVp of Object.values(n.captures || {})) for (const c of Object.values(byVp)) [c.image || c.png, c.thumb, c.html, c.eval, c.tree, c.spec].filter(Boolean).forEach((p) => files.add(p))
|
|
1319
1521
|
for (const a of n.attachments || []) files.add(a.file)
|
|
@@ -1326,16 +1528,25 @@ const commands = {
|
|
|
1326
1528
|
}
|
|
1327
1529
|
const days = Number(args.expires) || config.share?.expires_days || 30
|
|
1328
1530
|
// by content hash (#28): files the organization already stores (earlier runs, an interrupted share) are not sent
|
|
1329
|
-
|
|
1531
|
+
let entries = await hashFiles(run.dir, [...files])
|
|
1532
|
+
// a recording above the service's file limit stays local (with its poster and preview), the rest is shared
|
|
1533
|
+
const tooBig = new Set(entries.filter((e) => /\.rec\./.test(e.path) && u?.limits?.max_file_bytes && e.bytes > u.limits.max_file_bytes).map((e) => e.path))
|
|
1534
|
+
if (tooBig.size) {
|
|
1535
|
+
manifest = withoutRecordings(manifest, tooBig)
|
|
1536
|
+
const keep = new Set([...traceFiles(manifest), ...recordingFiles(manifest)])
|
|
1537
|
+
entries = entries.filter((e) => !/\.rec\./.test(e.path) || keep.has(e.path))
|
|
1538
|
+
say(`${tooBig.size} recording(s) larger than ${fmtBytes(u.limits.max_file_bytes)} stay in the local preview: ${[...tooBig].slice(0, 3).join(', ')}${tooBig.size > 3 ? ', …' : ''}. Record them with record.quality: low or a lower max_seconds to share them.`)
|
|
1539
|
+
}
|
|
1330
1540
|
const missing = await missingHashes(client, config.org, entries.map((e) => e.hash))
|
|
1331
1541
|
const total = entries.reduce((n, e) => n + e.bytes, 0)
|
|
1332
1542
|
const fresh = missing ? [...new Map(entries.filter((e) => missing.has(e.hash)).map((e) => [e.hash, e.bytes])).values()].reduce((n, b) => n + b, 0) : total
|
|
1333
1543
|
const resumed = (await readJson(join(run.dir, PROGRESS_FILE), null).catch(() => null))?.run === manifest.id
|
|
1334
1544
|
try {
|
|
1335
|
-
|
|
1545
|
+
if (!u) throw new Error('usage unavailable')
|
|
1336
1546
|
if (total > u.limits.max_run_bytes) throw new UserError(`This run is ${fmtBytes(total)}; the limit per run is ${fmtBytes(u.limits.max_run_bytes)}. Capture fewer viewports or locales, or ask an admin for a higher limit.`)
|
|
1337
1547
|
if (u.usage.bytes + fresh > u.limits.quota_bytes) throw new UserError(`Storage quota would be exceeded: ${fmtBytes(u.usage.bytes)} used + ${fmtBytes(fresh)} > ${fmtBytes(u.limits.quota_bytes)}. Delete old runs in Settings or ask for a higher quota.`)
|
|
1338
|
-
|
|
1548
|
+
const recBytes = entries.filter((e) => /\.rec\./.test(e.path)).reduce((n, e) => n + e.bytes, 0)
|
|
1549
|
+
say(`${resumed ? 'Resuming the interrupted share: ' : ''}${files.size} files, ${fmtBytes(total)}${recBytes ? ` (recordings ${fmtBytes(recBytes)})` : ''}${missing ? `; ${fmtBytes(fresh)} new, the rest already stored` : ''} (organization uses ${fmtBytes(u.usage.bytes)} of ${fmtBytes(u.limits.quota_bytes)})`)
|
|
1339
1550
|
} catch (e) { if (e instanceof UserError && /limit|quota/.test(e.message)) throw e }
|
|
1340
1551
|
const created = await client.request('POST', `/orgs/${config.org}/runs?expires_days=${days}`, manifest)
|
|
1341
1552
|
await writeFile(join(run.dir, PROGRESS_FILE), JSON.stringify({ run: manifest.id, started_at: new Date().toISOString() }) + '\n').catch(() => {})
|
|
@@ -1768,7 +1979,10 @@ const commands = {
|
|
|
1768
1979
|
|
|
1769
1980
|
async doctor (args) {
|
|
1770
1981
|
const { config, root, file } = await context(args)
|
|
1771
|
-
|
|
1982
|
+
// engines the scenarios ask for (browsers: [webkit, firefox]), so doctor checks they are installed
|
|
1983
|
+
const engines = []
|
|
1984
|
+
for (const f of await scenarioFiles(config, root).catch(() => [])) { try { engines.push(...((yaml.load(await readFile(f, 'utf8')) || {}).browsers || [])) } catch {} }
|
|
1985
|
+
const r = await doctor({ config, root, file, fix: !!args.fix, engines })
|
|
1772
1986
|
say(args.json ? JSON.stringify(r, null, 2) : formatDoctor(r))
|
|
1773
1987
|
if (!r.ok) process.exitCode = 1
|
|
1774
1988
|
},
|