kaloko 0.3.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +119 -0
- package/bin/kaloko.mjs +4 -0
- package/bin/qawalk.mjs +4 -0
- package/package.json +42 -0
- package/schemas/bundle.v1.json +19 -0
- package/schemas/config.v1.json +268 -0
- package/schemas/revision.v1.json +36 -0
- package/schemas/run.v1.json +549 -0
- package/schemas/scenario.v1.json +668 -0
- package/schemas/tokens.v1.json +102 -0
- package/skills/kaloko/SKILL.md +94 -0
- package/skills/kaloko/reference/acceptance.md +208 -0
- package/skills/kaloko/reference/apps.md +161 -0
- package/skills/kaloko/reference/collaboration.md +218 -0
- package/skills/kaloko/reference/design.md +143 -0
- package/skills/kaloko/reference/documentation.md +72 -0
- package/skills/kaloko/reference/environments.md +140 -0
- package/skills/kaloko/reference/evaluation.md +62 -0
- package/skills/kaloko/reference/execution.md +75 -0
- package/skills/kaloko/reference/mcp.md +69 -0
- package/skills/kaloko/reference/stacks.md +122 -0
- package/src/cli.mjs +1344 -0
- package/src/lib/access.mjs +169 -0
- package/src/lib/apps/capture.mjs +59 -0
- package/src/lib/apps/drivers.mjs +284 -0
- package/src/lib/apps/helper.swift +102 -0
- package/src/lib/apps/tree.mjs +222 -0
- package/src/lib/browser-daemon.mjs +41 -0
- package/src/lib/browser.mjs +161 -0
- package/src/lib/build.mjs +149 -0
- package/src/lib/calibrate.mjs +97 -0
- package/src/lib/capture.mjs +233 -0
- package/src/lib/compare.mjs +88 -0
- package/src/lib/config.mjs +117 -0
- package/src/lib/console.mjs +13 -0
- package/src/lib/coverage.mjs +75 -0
- package/src/lib/design-cli.mjs +440 -0
- package/src/lib/design-core.mjs +163 -0
- package/src/lib/design.mjs +164 -0
- package/src/lib/doctor.mjs +149 -0
- package/src/lib/draft.mjs +76 -0
- package/src/lib/drift.mjs +145 -0
- package/src/lib/evaluate.mjs +222 -0
- package/src/lib/git.mjs +70 -0
- package/src/lib/htmlview.mjs +130 -0
- package/src/lib/jev.mjs +95 -0
- package/src/lib/mail.mjs +182 -0
- package/src/lib/manifest-core.mjs +127 -0
- package/src/lib/pack-help.mjs +55 -0
- package/src/lib/packs.mjs +121 -0
- package/src/lib/pr.mjs +78 -0
- package/src/lib/probe.mjs +379 -0
- package/src/lib/recheck.mjs +70 -0
- package/src/lib/run.mjs +85 -0
- package/src/lib/scenario.mjs +98 -0
- package/src/lib/schema.mjs +25 -0
- package/src/lib/secrets.mjs +59 -0
- package/src/lib/service.mjs +52 -0
- package/src/lib/stack.mjs +130 -0
- package/src/lib/tokens.mjs +352 -0
- package/src/lib/util.mjs +97 -0
- package/templates/ACCEPTANCE.md +55 -0
- package/templates/cs/ACCEPTANCE.md +55 -0
- package/templates/kaloko.config.yml +76 -0
- package/templates/mail/letter-opener.mjs +47 -0
- package/templates/scenario.yml +119 -0
- package/templates/steps/consent.mjs +33 -0
- package/viewer/index.html +2772 -0
|
@@ -0,0 +1,102 @@
|
|
|
1
|
+
{
|
|
2
|
+
"$schema": "https://json-schema.org/draft/2020-12/schema",
|
|
3
|
+
"$id": "https://qawalk.com/schemas/tokens.v1.json",
|
|
4
|
+
"title": "qawalk.tokens.v1",
|
|
5
|
+
"description": "A project's token set in the W3C Design Tokens Community Group format (#42): nested groups, tokens with $value and optional $type (string values or the 2025.10 objects for colors, dimensions, durations and composite types), aliases as {group.token} or {\"$ref\": \"#/pointer\"}, $extends, $root and $deprecated; or a DTCG resolver (sets, modifiers, resolutionOrder) with its sources inlined. Top-level groups named like a mode (light, dark, high-contrast, or listed in $extensions[\"qawalk.modes\"]) overlay the rest in that mode. Stored versioned per project; tokens.json in the design folder is the same format.",
|
|
6
|
+
"$defs": {
|
|
7
|
+
"group": {
|
|
8
|
+
"type": "object",
|
|
9
|
+
"properties": {
|
|
10
|
+
"$type": {
|
|
11
|
+
"type": "string"
|
|
12
|
+
},
|
|
13
|
+
"$description": {
|
|
14
|
+
"type": "string"
|
|
15
|
+
}
|
|
16
|
+
},
|
|
17
|
+
"additionalProperties": {
|
|
18
|
+
"anyOf": [
|
|
19
|
+
{
|
|
20
|
+
"$ref": "#/$defs/token"
|
|
21
|
+
},
|
|
22
|
+
{
|
|
23
|
+
"$ref": "#/$defs/group"
|
|
24
|
+
}
|
|
25
|
+
]
|
|
26
|
+
}
|
|
27
|
+
},
|
|
28
|
+
"token": {
|
|
29
|
+
"type": "object",
|
|
30
|
+
"required": [
|
|
31
|
+
"$value"
|
|
32
|
+
],
|
|
33
|
+
"properties": {
|
|
34
|
+
"$value": {},
|
|
35
|
+
"$type": {
|
|
36
|
+
"type": "string"
|
|
37
|
+
},
|
|
38
|
+
"$description": {
|
|
39
|
+
"type": "string"
|
|
40
|
+
}
|
|
41
|
+
}
|
|
42
|
+
},
|
|
43
|
+
"set": {
|
|
44
|
+
"type": "object",
|
|
45
|
+
"properties": {
|
|
46
|
+
"$extensions": {
|
|
47
|
+
"type": "object",
|
|
48
|
+
"properties": {
|
|
49
|
+
"qawalk.modes": {
|
|
50
|
+
"type": "array",
|
|
51
|
+
"items": {
|
|
52
|
+
"type": "string"
|
|
53
|
+
}
|
|
54
|
+
}
|
|
55
|
+
}
|
|
56
|
+
}
|
|
57
|
+
},
|
|
58
|
+
"additionalProperties": {
|
|
59
|
+
"$ref": "#/$defs/group"
|
|
60
|
+
}
|
|
61
|
+
},
|
|
62
|
+
"resolver": {
|
|
63
|
+
"type": "object",
|
|
64
|
+
"required": [
|
|
65
|
+
"resolutionOrder"
|
|
66
|
+
],
|
|
67
|
+
"properties": {
|
|
68
|
+
"version": {
|
|
69
|
+
"type": "string"
|
|
70
|
+
},
|
|
71
|
+
"sets": {
|
|
72
|
+
"type": "object"
|
|
73
|
+
},
|
|
74
|
+
"modifiers": {
|
|
75
|
+
"type": "object"
|
|
76
|
+
},
|
|
77
|
+
"resolutionOrder": {
|
|
78
|
+
"type": "array"
|
|
79
|
+
}
|
|
80
|
+
}
|
|
81
|
+
}
|
|
82
|
+
},
|
|
83
|
+
"oneOf": [
|
|
84
|
+
{
|
|
85
|
+
"$ref": "#/$defs/resolver"
|
|
86
|
+
},
|
|
87
|
+
{
|
|
88
|
+
"allOf": [
|
|
89
|
+
{
|
|
90
|
+
"not": {
|
|
91
|
+
"required": [
|
|
92
|
+
"resolutionOrder"
|
|
93
|
+
]
|
|
94
|
+
}
|
|
95
|
+
},
|
|
96
|
+
{
|
|
97
|
+
"$ref": "#/$defs/set"
|
|
98
|
+
}
|
|
99
|
+
]
|
|
100
|
+
}
|
|
101
|
+
]
|
|
102
|
+
}
|
|
@@ -0,0 +1,94 @@
|
|
|
1
|
+
---
|
|
2
|
+
name: kaloko
|
|
3
|
+
description: >
|
|
4
|
+
Use when a task or flow needs an acceptance or regression walkthrough with
|
|
5
|
+
evidence: writing or updating ACCEPTANCE.md or qa/scenario.yml, running a
|
|
6
|
+
scenario on local, review, staging or production (readonly), capturing
|
|
7
|
+
screens and e-mails, evaluating criteria, or building and sharing the
|
|
8
|
+
Kaloko canvas, for web pages as well as Android, iOS and desktop apps,
|
|
9
|
+
including checking many pages of one template at once (magazine articles,
|
|
10
|
+
product pages, an SEO page catalogue). Also when someone asks what a
|
|
11
|
+
reviewer rejected on a canvas, what is waiting in the Kaloko inbox, or how
|
|
12
|
+
to report a run in an issue, and when designing screens of a flow in a
|
|
13
|
+
design environment (live HTML steps, drafts, versions, the design
|
|
14
|
+
reference, checking implementation against it).
|
|
15
|
+
---
|
|
16
|
+
|
|
17
|
+
# Kaloko
|
|
18
|
+
|
|
19
|
+
Kaloko turns an acceptance plan into a run bundle (`run.json` + screenshots,
|
|
20
|
+
HTML, evaluations) and renders it as a flow canvas. You fill it with content;
|
|
21
|
+
the canvas, storage and collaboration are the service.
|
|
22
|
+
|
|
23
|
+
## Done means
|
|
24
|
+
|
|
25
|
+
- The scenario validates (`kaloko validate`).
|
|
26
|
+
- Every step in the requested scope has a capture, or the summary says why not.
|
|
27
|
+
- The captures are evaluated (`kaloko walk` ends with the verdicts; after
|
|
28
|
+
`kaloko capture` run `kaloko evaluate`) and `kaloko build` ran; you report the summary line,
|
|
29
|
+
failing criteria with evidence, and the preview path (or shared URL).
|
|
30
|
+
- When the run was shared, its URL and the summary are in the issue or PR
|
|
31
|
+
the work belongs to, and you read `kaloko feedback` and `kaloko inbox`
|
|
32
|
+
before continuing on that task.
|
|
33
|
+
- Changes to intent are written back to `ACCEPTANCE.md` (change log row).
|
|
34
|
+
- On a design environment: the folder follows the bundle rules, `kaloko walk`
|
|
35
|
+
and `kaloko sync` ran, `kaloko status` shows no conflict, and you read the
|
|
36
|
+
draft's feedback; a version is published only when the person asked.
|
|
37
|
+
|
|
38
|
+
## You may do without asking
|
|
39
|
+
|
|
40
|
+
Start the shared browser, walk a scenario on a non-production environment,
|
|
41
|
+
create test data there (users, roles, carts), capture screens and e-mails,
|
|
42
|
+
evaluate, build the preview, and fix scenario mechanics (selectors, scripts)
|
|
43
|
+
that do not change what is being accepted. On a design environment: change
|
|
44
|
+
the design folder, walk, and sync the draft (including taking others' steps
|
|
45
|
+
with `--theirs`). Move to the trash runs you uploaded in this session that
|
|
46
|
+
were superseded before anyone looked at them (debug rounds: no comments, no
|
|
47
|
+
verdict), and run `kaloko prune` without `--yes` to see what a clean-up would do.
|
|
48
|
+
|
|
49
|
+
## Ask first
|
|
50
|
+
|
|
51
|
+
Changing acceptance criteria or scope, any production run, uploading with
|
|
52
|
+
`kaloko share` (on a design environment: publishing a version), setting a
|
|
53
|
+
design reference or baseline, merging a design branch, keeping your version of
|
|
54
|
+
a step in a sync conflict (`--mine`), turning on `html_view`, a real clean-up
|
|
55
|
+
(`kaloko prune --yes`), trashing someone else's run, archiving a scenario, and
|
|
56
|
+
anything the environment profile blocks. Never trash an accepted run, a
|
|
57
|
+
baseline, a run with review comments, or a run linked from an open PR or issue
|
|
58
|
+
comment. Ask the person
|
|
59
|
+
for secret values (passwords, tokens, gate credentials) and for sign-ins only
|
|
60
|
+
they can do; never write secrets into the config, a scenario or a script.
|
|
61
|
+
|
|
62
|
+
## Where to look
|
|
63
|
+
|
|
64
|
+
- Writing or updating the plan and scenario: [reference/acceptance.md](reference/acceptance.md)
|
|
65
|
+
- Android, iOS (Simulator), Electron and macOS apps, or screens captured by
|
|
66
|
+
another tool (Maestro, Appium, XCUITest) with their UI tree:
|
|
67
|
+
[reference/apps.md](reference/apps.md)
|
|
68
|
+
- Many pages of one template in one step (articles, products, an SEO
|
|
69
|
+
catalogue from a sitemap or a listing), with an index on the canvas:
|
|
70
|
+
[reference/stacks.md](reference/stacks.md)
|
|
71
|
+
- Choosing playwright, agent or manual per step, and driving the shared
|
|
72
|
+
browser with your capabilities: [reference/execution.md](reference/execution.md)
|
|
73
|
+
- Evaluation, JEV thresholds and reading results: [reference/evaluation.md](reference/evaluation.md)
|
|
74
|
+
- Environments, access (gates, accounts, sign-in), e-mails, readonly and
|
|
75
|
+
production rules, and what is yours to arrange for the project's stack:
|
|
76
|
+
[reference/environments.md](reference/environments.md)
|
|
77
|
+
- Sharing, reviewer feedback, mentions, inbox, verdicts and baselines, and
|
|
78
|
+
cleaning up runs (trash, prune, keep, archive):
|
|
79
|
+
[reference/collaboration.md](reference/collaboration.md)
|
|
80
|
+
- Reading runs, feedback and the inbox over MCP instead of the CLI:
|
|
81
|
+
[reference/mcp.md](reference/mcp.md)
|
|
82
|
+
- Refreshing a manual, course or product documentation from the newest
|
|
83
|
+
captures (latest URLs, changed steps, webhooks, export):
|
|
84
|
+
[reference/documentation.md](reference/documentation.md)
|
|
85
|
+
- Designing in a design environment (bundle, sync and publish, comments,
|
|
86
|
+
references, handoff, tokens, the design pack, canvas edits, branches, HTML
|
|
87
|
+
view, bringing your own design skill): [reference/design.md](reference/design.md)
|
|
88
|
+
- Command reference: `kaloko help`; when something does not work (Chrome,
|
|
89
|
+
token, environment URL, evaluator key), run `kaloko doctor` first and fix
|
|
90
|
+
what it marks ✕.
|
|
91
|
+
|
|
92
|
+
Project configuration is `kaloko.config.yml` in the repository root.
|
|
93
|
+
Project skills may add conventions (paths, test data, locales); follow them
|
|
94
|
+
where they are more specific.
|
|
@@ -0,0 +1,208 @@
|
|
|
1
|
+
# Acceptance plan and scenario
|
|
2
|
+
|
|
3
|
+
Two files per task, next to the task documentation:
|
|
4
|
+
|
|
5
|
+
- `ACCEPTANCE.md` — for people. Goal, flow (Mermaid), steps with purpose and
|
|
6
|
+
criteria `AC<n>`, variants, locales, environments, execution per step,
|
|
7
|
+
change log. Template: `templates/ACCEPTANCE.md` (Czech: `templates/cs/`).
|
|
8
|
+
- `qa/scenario.yml` — for the runner. Schema `schemas/scenario.v1.json`,
|
|
9
|
+
example `templates/scenario.yml`.
|
|
10
|
+
|
|
11
|
+
Long-lived flows that outlive tasks (checkout, SEO on production) live in
|
|
12
|
+
`qa/flows/*.yml` with `kind: flow`.
|
|
13
|
+
|
|
14
|
+
## Creating or updating
|
|
15
|
+
|
|
16
|
+
Read the task documentation and recent commits first. The plan states intent;
|
|
17
|
+
the scenario is its executable form. When sources disagree or intent is
|
|
18
|
+
unclear, ask one focused question instead of guessing.
|
|
19
|
+
|
|
20
|
+
When intent changes, update `ACCEPTANCE.md` first, add a change-log row with
|
|
21
|
+
date and commit, bump `version` in the scenario, then adjust the scenario.
|
|
22
|
+
|
|
23
|
+
For public pages of a site (landings, marketing, documentation) start from a
|
|
24
|
+
draft: `npx kaloko draft https://example.com/ https://example.com/pricing`
|
|
25
|
+
fetches the pages and writes a read-only production scenario and its plan
|
|
26
|
+
(title, H1, description as purpose; packs seo, a11y, perf, console). Then
|
|
27
|
+
sharpen each purpose and add the criteria the page must really meet.
|
|
28
|
+
|
|
29
|
+
## Check packs
|
|
30
|
+
|
|
31
|
+
`packs: [seo]` or `packs: [seo, usability]` on a node adds a fixed set of
|
|
32
|
+
criteria (ids `AC1xx` for `seo`, `AC2xx` for `usability`) at load time:
|
|
33
|
+
`seo` checks one H1, title and description length, canonical, hreflang for
|
|
34
|
+
every locale, Open Graph, indexing, viewport, `html lang`, overflow and
|
|
35
|
+
console; `usability` asks the evaluator about purpose, one primary action
|
|
36
|
+
and link texts and checks unlabeled inputs. A node's own criterion with the
|
|
37
|
+
same id wins. Pack criteria are marked `pack` on the canvas and do not count
|
|
38
|
+
as drift in `--coverage`; the plan may still mention them by id. Use packs
|
|
39
|
+
for public pages and landings; task flows keep their own criteria.
|
|
40
|
+
|
|
41
|
+
- `a11y` (`AC3xx`): images with alt, links and buttons with a name, labelled
|
|
42
|
+
fields, heading levels not skipped, WCAG AA text contrast, a `main`
|
|
43
|
+
landmark, unique ids. A subset of WCAG 2.2 AA checked in the page; it does
|
|
44
|
+
not replace an audit with a screen reader.
|
|
45
|
+
- `perf` (`AC4xx`): LCP ≤ 2.5 s, CLS ≤ 0.1, TTFB ≤ 800 ms, page weight ≤ 3 MB,
|
|
46
|
+
measured in this walk's browser since the page load (lab values: the
|
|
47
|
+
machine and network of the walk matter; compare runs, not absolutes).
|
|
48
|
+
Override a limit with an own criterion of the same id, e.g.
|
|
49
|
+
`{ id: AC401, text: LCP within 4 s, check: { type: deterministic, assert: perf_lcp, max: 4000 } }`.
|
|
50
|
+
- `console` (`AC501`): the evaluator reads the step's console errors with its
|
|
51
|
+
purpose and decides whether they touch what the step accepts (pass: only
|
|
52
|
+
third-party noise; needs review: unsure; fail: relevant). Without errors it
|
|
53
|
+
passes without a call. Keep `no_console_errors` where any error must fail.
|
|
54
|
+
|
|
55
|
+
- `errors` (`AC6xx`) is for error pages. It checks:
|
|
56
|
+
- a real 4xx/5xx status (a "soft 404" answering 200 fails);
|
|
57
|
+
- a link to the home page and navigation or search;
|
|
58
|
+
- no stack trace, SQL error or framework debug page;
|
|
59
|
+
- a plain-language message with a way forward (semantic);
|
|
60
|
+
- no console errors. The document's own error status is not counted.
|
|
61
|
+
|
|
62
|
+
## Error pages
|
|
63
|
+
|
|
64
|
+
Every site has them and they are easy to forget. Give each one its own
|
|
65
|
+
step with `packs: [errors]`:
|
|
66
|
+
|
|
67
|
+
- **404:** use `path: /no-such-page-{random}`. `{random}` becomes a new
|
|
68
|
+
token on every walk, so no cache or an accidentally created page makes it
|
|
69
|
+
pass. `kaloko draft` adds this step on its own.
|
|
70
|
+
- **Exact status:** override AC601, for example
|
|
71
|
+
`{ id: AC601, text: Answers 410, check: { type: deterministic, assert: http_status, value: 410 } }`.
|
|
72
|
+
`http_status` also takes `min`/`max`; the default is 200–299.
|
|
73
|
+
- **Every step:** each capture records the document's HTTP status. The
|
|
74
|
+
canvas shows 3xx–5xx on the node, so a normal step that suddenly answers
|
|
75
|
+
500 stands out. On a stack, `http_status` with `min: 200, max: 299` finds
|
|
76
|
+
broken pages among hundreds.
|
|
77
|
+
- **5xx and maintenance pages:** you cannot trigger them on production.
|
|
78
|
+
Use a test route the project provides (for example `/500` on a review
|
|
79
|
+
app), the static error page the server serves (`/500.html` answers 200,
|
|
80
|
+
so override AC601 with `value: 200`), or a local environment where the
|
|
81
|
+
person agrees to stop a dependency. Say which one you used.
|
|
82
|
+
- **Access errors:** 401/403 for a signed-out visitor on a protected
|
|
83
|
+
address, and an expired link (410). Walk these with the right session.
|
|
84
|
+
|
|
85
|
+
## Many pages of one template
|
|
86
|
+
|
|
87
|
+
When the plan says "every article / product / landing page must …", do not
|
|
88
|
+
write one step per page. Use one `kind: stack` step whose `items` come from
|
|
89
|
+
a sitemap, a listing, a file or a script, with the criteria every page must
|
|
90
|
+
meet. The canvas shows it as a stack with an index. The plan keeps one
|
|
91
|
+
`### <step-id>` section for it, naming the source and the limit. See
|
|
92
|
+
[stacks.md](stacks.md).
|
|
93
|
+
|
|
94
|
+
## Capture quality
|
|
95
|
+
|
|
96
|
+
The default capture is WebP at 1× — right for daily acceptance, small and fast on the canvas. For
|
|
97
|
+
documentation, manuals and training material set print quality on the scenario or on single steps:
|
|
98
|
+
|
|
99
|
+
```yaml
|
|
100
|
+
capture:
|
|
101
|
+
format: png # lossless
|
|
102
|
+
scale: 2 # 2× device pixels, crisp in a PDF
|
|
103
|
+
mask: ["[data-qa=email]", ".customer-name"] # solid box over personal data (image only; the HTML keeps it)
|
|
104
|
+
annotate: true # numbered marks on the elements of criteria with `highlight`, in criteria order
|
|
105
|
+
```
|
|
106
|
+
|
|
107
|
+
A node's `capture:` overrides the scenario's. Crops for a manual take padding in step scripts:
|
|
108
|
+
`await crop('#checkout-summary', 'Order summary', { padding: 16 })`. Print quality costs storage (a
|
|
109
|
+
2× PNG is several times a 1× WebP), so keep it for documentation scenarios, not for every run. The
|
|
110
|
+
canvas still loads a small WebP thumbnail; the lightbox and `latest.image` serve the full file.
|
|
111
|
+
Masks hide pixels, not the captured HTML: do not use production personal data in documentation runs.
|
|
112
|
+
|
|
113
|
+
## Keeping plan and scenario aligned
|
|
114
|
+
|
|
115
|
+
`kaloko validate <scenario.yml> --coverage` compares `AC<n>` lines under each
|
|
116
|
+
`### <step-id>` heading of the plan (`source`) with the scenario: planned
|
|
117
|
+
criteria without a check fail the command, checks without a planned criterion
|
|
118
|
+
are listed. The canvas shows the same card ("Plan coverage") and marks
|
|
119
|
+
criteria whose verdict differs between runs on the same commit as flaky;
|
|
120
|
+
treat a flaky criterion as a selector or timing problem to fix in the
|
|
121
|
+
scenario, not as a verdict.
|
|
122
|
+
|
|
123
|
+
## Naming and linkage
|
|
124
|
+
|
|
125
|
+
When the work has a repository and an issue number, make them visible:
|
|
126
|
+
|
|
127
|
+
- scenario `id` carries the ticket (`TASK-1273-club-activation`, `WEB-12-checkout`); the CLI derives the issue number from it or from the branch (`feat/1273-…`) and links it to the repository's origin (`acme/shop#1273`);
|
|
128
|
+
- set `issue: { number, repo?, url? }` explicitly when the id or branch cannot carry it (for example a Plane item);
|
|
129
|
+
- the project is the repository name unless `project:` in `kaloko.config.yml` says otherwise; a single flow may sort itself elsewhere with `project: <slug>` in its scenario (optional). Projects are records of the organization: `kaloko projects` or MCP `list_projects` show the slugs to use; admins create them in Settings → Projects or assign a scenario in the overview, which wins over the scenario file;
|
|
130
|
+
- the canvas header, the organization overview and `kaloko feedback` show `repo#number · title`, so name the scenario `title` for humans, not for the ticket.
|
|
131
|
+
|
|
132
|
+
## Session and consent
|
|
133
|
+
|
|
134
|
+
Default, unless the plan says otherwise: a run starts as a new visitor
|
|
135
|
+
(`walk` clears cookies and storage before the first step; agents call
|
|
136
|
+
`kaloko session reset`). When the site shows a consent dialog:
|
|
137
|
+
|
|
138
|
+
1. the first screen is captured with the dialog still open — it is part of
|
|
139
|
+
what a new visitor sees, with one criterion that it appears;
|
|
140
|
+
2. the next step accepts it (`steps/consent.mjs`, installed by `kaloko init`,
|
|
141
|
+
handles common consent libraries and captures the same screen without the
|
|
142
|
+
dialog; criterion: the dialog is gone);
|
|
143
|
+
3. every later screen runs without the dialog. Do not repeat consent
|
|
144
|
+
criteria on them — a returning dialog is visible on the canvas anyway.
|
|
145
|
+
|
|
146
|
+
Skip the consent step when the site has no consent dialog. Change the order
|
|
147
|
+
in `ACCEPTANCE.md` when the plan tests consent itself (rejecting, settings,
|
|
148
|
+
per-category choices) or needs another session state (signed-in user,
|
|
149
|
+
returning visitor). The plan is the source of truth; this is only the default.
|
|
150
|
+
|
|
151
|
+
## Writing criteria
|
|
152
|
+
|
|
153
|
+
Each criterion gets exactly one check:
|
|
154
|
+
|
|
155
|
+
- `deterministic` for anything countable or exact: presence, counts, text,
|
|
156
|
+
prices, attributes, overflow, console errors, title/meta/canonical.
|
|
157
|
+
- `semantic` for judgement: clarity, separation of consents, whether copy
|
|
158
|
+
fits the purpose. Write the question in English, yes = met, one condition
|
|
159
|
+
per question. Add `true`/`false` for edge cases and `scope` (CSS selector)
|
|
160
|
+
to send only the relevant region.
|
|
161
|
+
- `manual` when neither works; a reviewer decides on the canvas.
|
|
162
|
+
|
|
163
|
+
A semantic question with two conditions ("says that the draft is live **and**
|
|
164
|
+
shared, so everyone sees changes", "says how to publish **and** what
|
|
165
|
+
publishing saves") lands in the middle of the score range and stays
|
|
166
|
+
`needs_review` even when the page is clear, because the evaluator weighs the
|
|
167
|
+
part it is less sure about. Split it into one criterion per condition, scope
|
|
168
|
+
it to the element that should say it (`scope: ".draft-panel"`, not `body`),
|
|
169
|
+
and put the expected wording into `true`:
|
|
170
|
+
|
|
171
|
+
```yaml
|
|
172
|
+
- { id: AC6, text: Says the draft is live, check: { type: semantic, question: "Does the draft panel say that changes appear live for everyone who has the draft open?", scope: ".draft-panel", true: "Yes: it states that others see changes as they happen." } }
|
|
173
|
+
- { id: AC7, text: Says how to publish, check: { type: semantic, question: "Does the draft panel say how to publish a new version?", scope: ".draft-panel" } }
|
|
174
|
+
- { id: AC8, text: Says what publishing saves, check: { type: semantic, question: "Does the draft panel say what publishing a version saves?", scope: ".draft-panel" } }
|
|
175
|
+
```
|
|
176
|
+
|
|
177
|
+
When the exact wording is fixed and matters, a `text_matches` check is better
|
|
178
|
+
than a question.
|
|
179
|
+
|
|
180
|
+
Numbers, dates and counts never go to `semantic`. Add `highlight` when the
|
|
181
|
+
check selector is not the element a reviewer should see.
|
|
182
|
+
|
|
183
|
+
Text checks (`text_matches`, `text_absent`, `title_matches`) run a JavaScript
|
|
184
|
+
regex on the element's `innerText`, which follows CSS `text-transform`: a
|
|
185
|
+
button styled uppercase reads `VYTVOŘIT`. Add `flags: i` to ignore case
|
|
186
|
+
(also `m`, `s`) instead of writing `[Vv][Yy]…`. Keep patterns specific: a
|
|
187
|
+
white-label check for `vyvolej` also matches the Czech verb "Vyvolej si";
|
|
188
|
+
anchor it (`vyvolej\.to`).
|
|
189
|
+
|
|
190
|
+
Describe every relevant variant as nodes and `alt` edges, even when a given
|
|
191
|
+
run covers only part of it. Decisions are `kind: decision` nodes.
|
|
192
|
+
|
|
193
|
+
## People in the flow
|
|
194
|
+
|
|
195
|
+
When a flow involves more than one person (a guest who buys, a member who
|
|
196
|
+
shares a link, the friend who arrives through it, an operator in the console),
|
|
197
|
+
list them in `ACCEPTANCE.md` under "People in the flow": one line per person
|
|
198
|
+
with who they are, what test data they need, and what they must not see.
|
|
199
|
+
In `scenario.yml` declare them as `sessions` and give every non-decision node
|
|
200
|
+
a `session`. `kaloko walk` keeps one browser context per session and locale,
|
|
201
|
+
so nobody signs anybody out; step scripts receive `session` (`key`, `title`,
|
|
202
|
+
`mailbox`, `account`) and `session.credentials` from
|
|
203
|
+
`environments.<env>.accounts.<account>` (see environments.md).
|
|
204
|
+
E-mail nodes default to the session's `mailbox`. `blocked_on: [production]`
|
|
205
|
+
skips a session's nodes there. The canvas shows a chip with the person on
|
|
206
|
+
every step, groups the step list by person and draws hand-offs between
|
|
207
|
+
people as dashed arrows. Sessions run one after another, never in parallel,
|
|
208
|
+
and never share cookies.
|
|
@@ -0,0 +1,161 @@
|
|
|
1
|
+
# Mobile and desktop apps
|
|
2
|
+
|
|
3
|
+
Kaloko walks native apps the way it walks web pages. Each step becomes a
|
|
4
|
+
screenshot, a UI tree, checked criteria and a place on the canvas. Only the
|
|
5
|
+
driver differs, and it is chosen by the scenario's `platform`.
|
|
6
|
+
|
|
7
|
+
| platform | Driven by | UI tree | Interactions |
|
|
8
|
+
|---|---|---|---|
|
|
9
|
+
| `android` | `adb` (emulator or USB device) | uiautomator, in dp | deep links, tap by selector or point, type, back |
|
|
10
|
+
| `ios` | `xcrun simctl` (iOS Simulator) | `idb` when installed, otherwise text by OCR (macOS Vision) | deep links; taps and typing need `idb` |
|
|
11
|
+
| `electron` | Playwright Electron | the full DOM: every web check applies | everything Playwright does (`page`) |
|
|
12
|
+
| `desktop` | macOS window of a named app | macOS accessibility tree, otherwise OCR | none built in: use your tool, AppleScript or a person |
|
|
13
|
+
|
|
14
|
+
Anything else, including a real iPhone, Windows apps, Flutter/React Native
|
|
15
|
+
under Detox, Maestro flows or XCUITest, is captured by import:
|
|
16
|
+
|
|
17
|
+
```bash
|
|
18
|
+
npx kaloko capture --step checkout --image shot.png --tree hierarchy.xml --device "Pixel 8" --density 2.625
|
|
19
|
+
```
|
|
20
|
+
|
|
21
|
+
`--tree` accepts:
|
|
22
|
+
|
|
23
|
+
- uiautomator or Appium Android XML;
|
|
24
|
+
- Appium iOS XML (`XCUIElementType…`);
|
|
25
|
+
- `idb ui describe-all --json`;
|
|
26
|
+
- Maestro `hierarchy` JSON;
|
|
27
|
+
- Kaloko's own `qawalk.tree.v1`.
|
|
28
|
+
|
|
29
|
+
`--ocr` reads the text from the image instead (macOS). `--density` or
|
|
30
|
+
`--scale` converts device pixels to dp or points. Without either, Kaloko
|
|
31
|
+
infers the scale from the tree.
|
|
32
|
+
|
|
33
|
+
## Setting it up
|
|
34
|
+
|
|
35
|
+
The config names the app per environment; the scenario only says the
|
|
36
|
+
platform:
|
|
37
|
+
|
|
38
|
+
```yaml
|
|
39
|
+
# kaloko.config.yml
|
|
40
|
+
environments:
|
|
41
|
+
staging:
|
|
42
|
+
app:
|
|
43
|
+
android: { package: com.acme.shop.staging, apk: build/app-staging.apk, reset: true }
|
|
44
|
+
ios: { bundle: com.acme.shop, app: build/Shop.app, device: iPhone 17 }
|
|
45
|
+
desktop:
|
|
46
|
+
app:
|
|
47
|
+
electron: { main: dist/main.js } # or executable: path to the packaged app
|
|
48
|
+
# desktop: { name: Notes } # a macOS app by its window owner name
|
|
49
|
+
```
|
|
50
|
+
|
|
51
|
+
```yaml
|
|
52
|
+
# qa/flows/shop-android.yml
|
|
53
|
+
schema: qawalk.scenario.v1
|
|
54
|
+
id: SHOP-android
|
|
55
|
+
platform: android
|
|
56
|
+
locales: [cs, en]
|
|
57
|
+
viewports: { phone: [412, 915] } # a label and the nominal size; the device decides the real one
|
|
58
|
+
nodes:
|
|
59
|
+
- id: product
|
|
60
|
+
kind: screen
|
|
61
|
+
title: Product detail
|
|
62
|
+
path: shop://product/42 # deep link; https://… universal links work too
|
|
63
|
+
packs: [app]
|
|
64
|
+
execution: { strategy: script, instructions: Open the product by deep link. }
|
|
65
|
+
criteria:
|
|
66
|
+
- { id: AC1, text: Price in CZK, check: { type: deterministic, assert: text_matches, selector: '[id$=price]', pattern: '^[\d ]+ Kč$' } }
|
|
67
|
+
- { id: AC2, text: Add to cart, check: { type: deterministic, assert: exists, selector: 'button[text="Do košíku"]' } }
|
|
68
|
+
- id: cart
|
|
69
|
+
kind: screen
|
|
70
|
+
title: Cart
|
|
71
|
+
execution: { strategy: script, script: steps/add-to-cart.mjs, instructions: Tap Add to cart, then the cart icon. }
|
|
72
|
+
criteria: [...]
|
|
73
|
+
```
|
|
74
|
+
|
|
75
|
+
**Stepping through the app**
|
|
76
|
+
|
|
77
|
+
- Screens of an app scenario do not need a `path`. A step script reaches
|
|
78
|
+
them, or a person or agent does and then runs `kaloko capture`.
|
|
79
|
+
- Every locale starts the app fresh. `reset: true` clears the app data, so
|
|
80
|
+
each locale starts as a new user.
|
|
81
|
+
- The app language is set per locale: Android 13+ app locales, iOS
|
|
82
|
+
`-AppleLanguages`, Electron `--lang`.
|
|
83
|
+
|
|
84
|
+
**Step scripts**
|
|
85
|
+
|
|
86
|
+
A script gets `{ app, page, locale, env, run, capture, link }`:
|
|
87
|
+
|
|
88
|
+
```js
|
|
89
|
+
// steps/add-to-cart.mjs
|
|
90
|
+
export default async ({ app, link, capture }) => {
|
|
91
|
+
await app.link(link('product')) // the product step's deep link
|
|
92
|
+
await app.tap('button[text="Do košíku"]')
|
|
93
|
+
await app.wait({ text: 'V košíku' })
|
|
94
|
+
await app.tap('[id$=cart]')
|
|
95
|
+
await capture()
|
|
96
|
+
}
|
|
97
|
+
```
|
|
98
|
+
|
|
99
|
+
- `app.tap`, `app.wait` and `app.type` take app selectors. `app.tap({ x, y })`
|
|
100
|
+
taps by coordinates, in dp or points.
|
|
101
|
+
- On Electron, selectors are CSS and `page` is the Playwright page, so
|
|
102
|
+
any Playwright code works there.
|
|
103
|
+
|
|
104
|
+
## Selectors and checks on the UI tree
|
|
105
|
+
|
|
106
|
+
Selectors are CSS-like, over roles and attributes:
|
|
107
|
+
|
|
108
|
+
- `button[text="Sign in"]`
|
|
109
|
+
- `[id$=login]` (Android resource-id or iOS accessibility id)
|
|
110
|
+
- `text[text~="^\d+ Kč$"]` (regex)
|
|
111
|
+
- `[name*=close i]` (case-insensitive)
|
|
112
|
+
|
|
113
|
+
Operators are `=`, `^=`, `$=`, `*=` and `~=`, and commas separate
|
|
114
|
+
alternatives. The roles are button, text, textbox, image, checkbox, switch,
|
|
115
|
+
radio, slider, combobox, list, cell, tab, tablist, toolbar, link, alert,
|
|
116
|
+
dialog, webview and group.
|
|
117
|
+
|
|
118
|
+
- **Checks that work on apps:** `exists`, `absent`, `count`,
|
|
119
|
+
`text_matches`, `text_absent`, `attr_equals`, `a11y_names`, and
|
|
120
|
+
`no_console_errors` (errors from the Android app log).
|
|
121
|
+
- **App-only checks:**
|
|
122
|
+
- `app_labels`: every control has a label;
|
|
123
|
+
- `app_touch_targets`: 48 dp on Android, 44 pt on iOS, or `min`;
|
|
124
|
+
- `app_no_truncation`: texts ending in "…";
|
|
125
|
+
- `app_no_crash`: FATAL EXCEPTION or ANR in logcat, a "keeps stopping"
|
|
126
|
+
dialog, or the iOS app no longer running.
|
|
127
|
+
- **Pack `app` (AC7xx):** those four checks, plus a semantic question about
|
|
128
|
+
the screen's purpose and next action.
|
|
129
|
+
- **Web-only checks** (`no_overflow`, `http_status`, meta, `a11y_contrast`,
|
|
130
|
+
`perf_*`): these are "not evaluated" on app captures. Do not put them on
|
|
131
|
+
app steps.
|
|
132
|
+
- **OCR trees:** these have text only, so role checks (labels, touch
|
|
133
|
+
targets) are not evaluated there. Say so in the report and install `idb`
|
|
134
|
+
when those checks matter.
|
|
135
|
+
|
|
136
|
+
Semantic criteria read the tree as an outline (roles and texts), so write
|
|
137
|
+
questions about what is on the screen, not about the markup.
|
|
138
|
+
|
|
139
|
+
## Reading the canvas
|
|
140
|
+
|
|
141
|
+
- Phone captures sit in a device frame, and desktop captures in a window
|
|
142
|
+
frame.
|
|
143
|
+
- The lightbox has a **UI tree** panel: search it, and click an element to
|
|
144
|
+
outline it on the screenshot. Criteria highlights use the same boxes.
|
|
145
|
+
- The run card shows the platform and the device.
|
|
146
|
+
- `kaloko evaluate --recheck` re-runs the checks on the stored tree, so no
|
|
147
|
+
device is needed after a criteria fix.
|
|
148
|
+
|
|
149
|
+
## Rules
|
|
150
|
+
|
|
151
|
+
- **Devices:** never install on a person's own phone or sign in with
|
|
152
|
+
real accounts without asking. Emulators and simulators are yours to
|
|
153
|
+
start, reset and install builds on.
|
|
154
|
+
- **Production apps:** production apps (store builds) are read-only in
|
|
155
|
+
spirit. Walk them only with deep links and screens that change no data,
|
|
156
|
+
unless the person agrees.
|
|
157
|
+
- **Sign-in:** ask the person for credentials, or ask them to sign in once
|
|
158
|
+
on the emulator. Never write credentials into a script.
|
|
159
|
+
- **macOS permissions:** desktop capture needs Screen Recording, and the
|
|
160
|
+
accessibility tree needs Accessibility. If macOS refuses, tell the person
|
|
161
|
+
which permission to grant to which app. Do not work around it.
|