kaloko 0.14.0 → 0.16.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +6 -2
- package/package.json +1 -1
- package/schemas/config.v1.json +3 -1
- package/schemas/evidence.v1.json +257 -0
- package/schemas/run.v1.json +71 -1
- package/schemas/scenario.v1.json +52 -1
- package/skills/kaloko/SKILL.md +35 -6
- package/skills/kaloko/reference/acceptance.md +41 -30
- package/skills/kaloko/reference/apps.md +1 -1
- package/skills/kaloko/reference/design.md +2 -1
- package/skills/kaloko/reference/docs.md +24 -0
- package/skills/kaloko/reference/evaluation.md +42 -1
- package/skills/kaloko/reference/evidence.md +82 -0
- package/skills/kaloko/reference/first-run.md +37 -4
- package/skills/kaloko/reference/integrations.md +6 -1
- package/skills/kaloko/reference/mcp.md +5 -1
- package/skills/kaloko/reference/stacks.md +1 -1
- package/src/cli.mjs +172 -21
- package/src/lib/build.mjs +12 -4
- package/src/lib/capture.mjs +57 -4
- package/src/lib/changelog-core.mjs +12 -6
- package/src/lib/cli-args.mjs +7 -3
- package/src/lib/detail.mjs +102 -0
- package/src/lib/docs-export.mjs +2 -1
- package/src/lib/docs.mjs +14 -9
- package/src/lib/draft.mjs +1 -1
- package/src/lib/evaluate.mjs +7 -2
- package/src/lib/evidence-core.mjs +300 -0
- package/src/lib/git.mjs +1 -1
- package/src/lib/health-core.mjs +205 -0
- package/src/lib/health.mjs +219 -0
- package/src/lib/init-detect.mjs +336 -17
- package/src/lib/integrations-cli.mjs +3 -1
- package/src/lib/jev.mjs +4 -2
- package/src/lib/manifest-core.mjs +58 -12
- package/src/lib/pack-help.mjs +53 -51
- package/src/lib/packs.mjs +67 -55
- package/src/lib/pr.mjs +2 -0
- package/src/lib/projects-cli.mjs +88 -0
- package/src/lib/run.mjs +13 -1
- package/src/lib/scenario.mjs +8 -0
- package/src/lib/signoff.mjs +86 -6
- package/src/lib/util.mjs +6 -2
- package/templates/kaloko.config.yml +2 -0
- package/templates/scenario.yml +1 -1
- package/viewer/index.html +608 -63
package/README.md
CHANGED
|
@@ -51,7 +51,7 @@ For example:
|
|
|
51
51
|
|
|
52
52
|
> Draft a Kaloko scenario for https://example.com (home, pricing, contact)
|
|
53
53
|
> with the seo, a11y and perf packs, run it on production read-only and
|
|
54
|
-
> summarise
|
|
54
|
+
> summarise its health.
|
|
55
55
|
|
|
56
56
|
## Run a scenario by hand (or in CI)
|
|
57
57
|
|
|
@@ -229,7 +229,11 @@ show (overlap, cut-off text, a raw translation key, an endless spinner);
|
|
|
229
229
|
reviewers turn the right findings into comments with one click, and findings
|
|
230
230
|
never change a verdict. Triage counts against the hosted-evaluation allowance,
|
|
231
231
|
so it needs the Team plan or higher.
|
|
232
|
-
|
|
232
|
+
Check packs (`seo`, `a11y`, `perf`, `usability`, `console`) report the app's
|
|
233
|
+
health apart from acceptance: they never fail a step that meets its brief,
|
|
234
|
+
`kaloko health` lists what is new since the previous run, and
|
|
235
|
+
`kaloko health adopt` makes one check an acceptance criterion when the team
|
|
236
|
+
wants it to decide. The `a11y` pack walks the page with Tab (order, visible focus, traps, skip
|
|
233
237
|
link) and reads names from the accessibility tree. Pixel diffs skip the
|
|
234
238
|
regions a step ignores; `kaloko walk --stability 3` and `kaloko stability`
|
|
235
239
|
find the clocks, ads and carousels worth ignoring, and `kaloko stability --pull`
|
package/package.json
CHANGED
package/schemas/config.v1.json
CHANGED
|
@@ -8,7 +8,9 @@
|
|
|
8
8
|
"properties": {
|
|
9
9
|
"schema": { "const": "qawalk.config.v1" },
|
|
10
10
|
"org": { "type": "string", "description": "Organization slug on kaloko.app." },
|
|
11
|
-
"project": { "type": "string" },
|
|
11
|
+
"project": { "type": "string", "description": "Slug of the Kaloko project runs of this repository belong to (a scenario's project: wins)." },
|
|
12
|
+
"project_name": { "type": "string", "maxLength": 80, "description": "The name people see for that project on Kaloko (ECO Podcasty). A share creates a missing project with it; kaloko projects sync renames an existing one." },
|
|
13
|
+
"product": { "type": "string", "pattern": "^[a-z0-9][a-z0-9._-]{0,79}$", "description": "Slug of a product on Kaloko the project is part of; kaloko projects sync adds the project to it." },
|
|
12
14
|
"products": {
|
|
13
15
|
"type": "object",
|
|
14
16
|
"description": "Optional: what is released and versioned as a whole and has its own visual changelog (docs/DOCS-CHANGELOG.md). Without it the project is the product. Keys are product slugs.",
|
|
@@ -0,0 +1,257 @@
|
|
|
1
|
+
{
|
|
2
|
+
"$schema": "https://json-schema.org/draft/2020-12/schema",
|
|
3
|
+
"$id": "https://qawalk.com/schemas/evidence.v1.json",
|
|
4
|
+
"title": "Kaloko evidence (qawalk.evidence.v1)",
|
|
5
|
+
"description": "What a downstream generator reads from Kaloko to write its own documentation: the evidence of a run (kind run), of a release (kind release), the trace matrix on its own (kind trace) and the MANIFEST.json of an evidence package (kind package). The contract is additive: new fields may appear in v1, existing fields keep their meaning and type; a breaking change makes qawalk.evidence.v2. Kaloko never formats output for a particular QMS. Produced by GET /api/v1/orgs/<org>/runs/<id>/evidence, …/evidence.zip, …/trace.json, GET /api/v1/orgs/<org>/evidence?product=&version=, MCP get_evidence and kaloko evidence.",
|
|
6
|
+
"type": "object",
|
|
7
|
+
"required": ["schema", "kind"],
|
|
8
|
+
"properties": {
|
|
9
|
+
"schema": { "const": "qawalk.evidence.v1" },
|
|
10
|
+
"kind": { "enum": ["run", "release", "trace", "package"] }
|
|
11
|
+
},
|
|
12
|
+
"allOf": [
|
|
13
|
+
{ "if": { "properties": { "kind": { "const": "run" } } }, "then": { "$ref": "#/$defs/run" } },
|
|
14
|
+
{ "if": { "properties": { "kind": { "const": "release" } } }, "then": { "$ref": "#/$defs/release" } },
|
|
15
|
+
{ "if": { "properties": { "kind": { "const": "trace" } } }, "then": { "$ref": "#/$defs/trace" } },
|
|
16
|
+
{ "if": { "properties": { "kind": { "const": "package" } } }, "then": { "$ref": "#/$defs/package" } }
|
|
17
|
+
],
|
|
18
|
+
"$defs": {
|
|
19
|
+
"time": { "type": ["string", "null"], "description": "ISO 8601 time in UTC (…Z), from the service's clock." },
|
|
20
|
+
"sha256": { "type": ["string", "null"], "pattern": "^[0-9a-f]{64}$", "description": "SHA-256, lowercase hex." },
|
|
21
|
+
"refs": { "type": "array", "items": { "type": "string" }, "description": "Requirement references as written in the scenario (refs:), free text, never interpreted." },
|
|
22
|
+
"run": {
|
|
23
|
+
"type": "object",
|
|
24
|
+
"required": ["generated_at", "acceptance", "signing", "record", "run", "scenario", "steps", "signatures", "trace"],
|
|
25
|
+
"properties": {
|
|
26
|
+
"generated_at": { "$ref": "#/$defs/time", "description": "When this document was made. It is not signed itself; record.sha256 names the signed record it was made from." },
|
|
27
|
+
"issuer": {
|
|
28
|
+
"type": "object",
|
|
29
|
+
"properties": {
|
|
30
|
+
"service": { "type": ["string", "null"], "description": "The Kaloko service that holds the run, e.g. https://kaloko.app." },
|
|
31
|
+
"commit": { "type": ["string", "null"], "description": "The service's deployed version (git commit), when known." },
|
|
32
|
+
"public_keys": { "type": ["string", "null"], "description": "URL of the public keys the record and the package manifest verify against (/.well-known/kaloko-signoff.json)." }
|
|
33
|
+
}
|
|
34
|
+
},
|
|
35
|
+
"org": { "type": ["object", "null"], "properties": { "slug": { "type": "string" }, "name": { "type": ["string", "null"] } } },
|
|
36
|
+
"acceptance": { "enum": ["none", "accepted", "returned"], "description": "The run's acceptance in Kaloko: accepted, returned, or none (nobody decided the run, or the decision was reset). A generator writes from this, never from the presence of other fields." },
|
|
37
|
+
"signing": { "enum": ["in_kaloko", "none"], "description": "in_kaloko: at least one electronic signature made in Kaloko stands (see signatures). none: no signature in Kaloko; if the approval is signed, it is signed in another system (the client's eQMS) and that system holds it." },
|
|
38
|
+
"record": {
|
|
39
|
+
"type": "object",
|
|
40
|
+
"description": "The signed record this evidence was derived from (record/kaloko-signoff.json in the package).",
|
|
41
|
+
"properties": {
|
|
42
|
+
"schema": { "type": ["string", "null"], "description": "kaloko.signoff.v1." },
|
|
43
|
+
"kind": { "enum": ["acceptance", "execution"], "description": "acceptance: the stored record of an accepted run (byte-identical on every download while its verdict and signatures stay). execution: a run that is not accepted; signed when made, not stored." },
|
|
44
|
+
"sha256": { "$ref": "#/$defs/sha256", "description": "SHA-256 of the record's canonical JSON without its signature: what kaloko signoff --verify prints as the record hash." },
|
|
45
|
+
"signed": { "type": "boolean", "description": "The record carries the service's Ed25519 signature." },
|
|
46
|
+
"kid": { "type": ["string", "null"], "description": "Id of the signing key." },
|
|
47
|
+
"issued_at": { "$ref": "#/$defs/time" },
|
|
48
|
+
"exported_by": { "type": ["string", "null"], "description": "Who downloaded the record first (e-mail)." }
|
|
49
|
+
}
|
|
50
|
+
},
|
|
51
|
+
"run": {
|
|
52
|
+
"type": "object",
|
|
53
|
+
"properties": {
|
|
54
|
+
"id": { "type": "string" },
|
|
55
|
+
"url": { "type": ["string", "null"], "description": "The run on the canvas." },
|
|
56
|
+
"project": { "type": ["string", "null"] },
|
|
57
|
+
"started_at": { "$ref": "#/$defs/time", "description": "When the walk started (the walking machine's clock)." },
|
|
58
|
+
"finished_at": { "$ref": "#/$defs/time", "description": "When the manifest was built (the walking machine's clock)." },
|
|
59
|
+
"version": { "type": ["string", "null"], "description": "Design version, for runs of design environments." },
|
|
60
|
+
"app_version": { "type": ["string", "null"], "description": "Version of the application under test, when the run names one." }
|
|
61
|
+
}
|
|
62
|
+
},
|
|
63
|
+
"scenario": {
|
|
64
|
+
"type": "object",
|
|
65
|
+
"properties": {
|
|
66
|
+
"id": { "type": "string" },
|
|
67
|
+
"title": { "type": ["string", "null"] },
|
|
68
|
+
"version": { "type": ["integer", "string", "null"], "description": "The scenario's own version: field, when set." },
|
|
69
|
+
"file": { "type": ["string", "null"], "description": "Path of the scenario file in the repository." },
|
|
70
|
+
"sha256": { "$ref": "#/$defs/sha256", "description": "SHA-256 of the scenario file as walked (CLI 0.15+; null before)." },
|
|
71
|
+
"refs": { "$ref": "#/$defs/refs" }
|
|
72
|
+
}
|
|
73
|
+
},
|
|
74
|
+
"environment": { "type": ["object", "null"], "description": "{ name, url }: the environment and its base URL." },
|
|
75
|
+
"git": { "type": ["object", "null"], "description": "{ commit, branch, repo } of the code the run walked, when known." },
|
|
76
|
+
"tools": {
|
|
77
|
+
"type": ["object", "null"],
|
|
78
|
+
"description": "What walked and evaluated the run.",
|
|
79
|
+
"properties": {
|
|
80
|
+
"cli": { "type": ["object", "null"], "description": "{ name, version } of the Kaloko CLI (null version: walked before 0.15)." },
|
|
81
|
+
"node": { "type": ["string", "null"] },
|
|
82
|
+
"os": { "type": ["string", "null"] },
|
|
83
|
+
"browsers": { "type": "array", "items": { "type": "string" }, "description": "Browser engines and versions seen in the captures, e.g. \"chromium 131.0.6778.33\"." },
|
|
84
|
+
"executor": { "type": ["object", "null"], "description": "{ kind: cli | agent, name }." },
|
|
85
|
+
"service": { "type": ["object", "null"], "description": "{ url, commit } of the service that signed the record." }
|
|
86
|
+
}
|
|
87
|
+
},
|
|
88
|
+
"manifest": { "type": ["object", "null"], "description": "{ path: run.json, sha256, bytes }: the run manifest as stored. Every signature is bound to this hash." },
|
|
89
|
+
"verdict": { "type": ["object", "null"], "description": "{ verdict: pending | accepted | returned, by, via: session | token | agent, at, note, country, counts_as_person }. via session is a person in the app; token and agent are never a person's decision." },
|
|
90
|
+
"approvals": { "type": ["object", "null"], "description": "{ required: [e-mail], people: [{ email, at }], tokens_and_agents: [{ email, at, via, counts_as_person: false }] }." },
|
|
91
|
+
"summary": { "type": ["object", "null"] },
|
|
92
|
+
"ai": { "type": ["object", "null"], "description": "{ advisory: true, models: [model id], statement }: AI verdicts are advisory; the decisions that count are a person's." },
|
|
93
|
+
"advisory": { "type": ["object", "null"], "description": "Results the run carries besides its acceptance criteria (e.g. a health section), copied as they are. They never change acceptance." },
|
|
94
|
+
"steps": {
|
|
95
|
+
"type": "array",
|
|
96
|
+
"items": {
|
|
97
|
+
"type": "object",
|
|
98
|
+
"properties": {
|
|
99
|
+
"id": { "type": "string" },
|
|
100
|
+
"kind": { "type": "string" },
|
|
101
|
+
"title": { "type": ["string", "null"] },
|
|
102
|
+
"refs": { "$ref": "#/$defs/refs" },
|
|
103
|
+
"status": { "enum": ["pass", "partial", "fail", "skip"] },
|
|
104
|
+
"decision": { "type": ["object", "null"], "description": "The step's current decision: { id, state: approved | returned, by, at, via, agent?, counts_as_person, note?, attachments?: [{ id, name, type, bytes, sha256, by, at, path: decisions/<step>/<file> in the evidence package }], pin?, shape? }." },
|
|
105
|
+
"decisions": { "type": "array", "description": "Every decision on the step, oldest first, shaped like decision; superseded ones too (resolved_at, deleted: true without content)." },
|
|
106
|
+
"carried": { "type": ["object", "null"], "description": "An approval carried over from an earlier accepted run: { from_run, compared_run, kind, approved_by, approved_at, approved_via }." },
|
|
107
|
+
"criteria": {
|
|
108
|
+
"type": "array",
|
|
109
|
+
"items": {
|
|
110
|
+
"type": "object",
|
|
111
|
+
"properties": {
|
|
112
|
+
"id": { "type": "string" },
|
|
113
|
+
"text": { "type": ["string", "null"], "description": "The criterion as walked (from the scenario at the time of the run)." },
|
|
114
|
+
"type": { "type": ["string", "null"], "description": "deterministic, semantic or manual." },
|
|
115
|
+
"pack": { "type": "string", "description": "The pack the criterion came from, when it did." },
|
|
116
|
+
"refs": { "$ref": "#/$defs/refs" },
|
|
117
|
+
"verdict": { "type": "string", "description": "pass, fail, needs_review, not_evaluated or not_reached; the worst over locales and viewports after people's verdicts." },
|
|
118
|
+
"assessed_by": { "enum": ["person", "ai", "deterministic", "none"] },
|
|
119
|
+
"models": { "type": "array", "items": { "type": "string" }, "description": "AI models that assessed it." },
|
|
120
|
+
"advisory": { "type": "boolean", "description": "true: the result is an AI's and no person decided this criterion." },
|
|
121
|
+
"results": {
|
|
122
|
+
"type": "array",
|
|
123
|
+
"items": {
|
|
124
|
+
"type": "object",
|
|
125
|
+
"properties": {
|
|
126
|
+
"locale": { "type": "string" },
|
|
127
|
+
"viewport": { "type": "string" },
|
|
128
|
+
"verdict": { "type": "string" },
|
|
129
|
+
"score": { "type": "number", "description": "The AI's probability that the criterion holds (0–1)." },
|
|
130
|
+
"assessed_by": { "enum": ["person", "ai", "deterministic", "none"] },
|
|
131
|
+
"assessor": { "type": ["string", "null"], "description": "The person's e-mail, or the AI model id." },
|
|
132
|
+
"advisory": { "type": "boolean" }
|
|
133
|
+
}
|
|
134
|
+
}
|
|
135
|
+
},
|
|
136
|
+
"reviewer": { "type": "array", "description": "Criterion verdicts set by people: { email, verdict, note, locale, at }." }
|
|
137
|
+
}
|
|
138
|
+
}
|
|
139
|
+
},
|
|
140
|
+
"captures": {
|
|
141
|
+
"type": "array",
|
|
142
|
+
"items": { "type": "object", "properties": { "locale": { "type": "string" }, "viewport": { "type": "string" }, "path": { "type": ["string", "null"], "description": "Path in the run; the package stores it under captures/<path>." }, "sha256": { "$ref": "#/$defs/sha256" } } }
|
|
143
|
+
}
|
|
144
|
+
}
|
|
145
|
+
}
|
|
146
|
+
},
|
|
147
|
+
"signatures": {
|
|
148
|
+
"type": "array",
|
|
149
|
+
"description": "Electronic signatures made in Kaloko, oldest first, withdrawn ones included. Empty unless the organization signs in Kaloko.",
|
|
150
|
+
"items": {
|
|
151
|
+
"type": "object",
|
|
152
|
+
"properties": {
|
|
153
|
+
"id": { "type": "string" },
|
|
154
|
+
"scope": { "enum": ["run", "step"] },
|
|
155
|
+
"step": { "type": ["string", "null"] },
|
|
156
|
+
"review": { "type": ["string", "null"], "description": "The step approval it signs (comment id)." },
|
|
157
|
+
"decision": { "type": "string", "description": "accepted (run) or approved (step)." },
|
|
158
|
+
"meaning": { "type": "string", "description": "The meaning the signer chose from the organization's list, as the label they saw." },
|
|
159
|
+
"meaning_id": { "type": ["string", "null"], "description": "subject_v 2: the meaning's id (reviewed, approved, verified, or the organization's own)." },
|
|
160
|
+
"lang": { "type": ["string", "null"], "description": "subject_v 2: the language the meaning's label was shown in (en, cs)." },
|
|
161
|
+
"subject_v": { "type": "integer", "description": "2 for signatures made since 7 Oct 2026; absent (1) before." },
|
|
162
|
+
"decision_sha256": { "type": ["string", "null"], "description": "subject_v 2: SHA-256 of the canonical decision { note, attachments [{ name, bytes, sha256 }], pin, shape } (a run: its note)." },
|
|
163
|
+
"name": { "type": "string", "description": "Printed name of the signer." },
|
|
164
|
+
"email": { "type": "string" },
|
|
165
|
+
"signed_at": { "$ref": "#/$defs/time" },
|
|
166
|
+
"time_zone": { "type": "string", "description": "The signer's IANA time zone." },
|
|
167
|
+
"local_time": { "type": "string", "description": "The signing time in the signer's zone with its UTC offset." },
|
|
168
|
+
"auth": { "type": "object", "description": "{ session: how the session signed in, second_step_at: when the second step was confirmed, second_step_by: kaloko | idp }." },
|
|
169
|
+
"verdict_at": { "$ref": "#/$defs/time" },
|
|
170
|
+
"manifest_sha256": { "$ref": "#/$defs/sha256" },
|
|
171
|
+
"subject_sha256": { "$ref": "#/$defs/sha256", "description": "SHA-256 of the canonical JSON { v: 1, run, step, decision, meaning, email, name, signed_at, manifest_sha256 } (subject_v 2 adds meaning_id, lang and decision_sha256): what was signed. kaloko signoff --verify recomputes it." },
|
|
172
|
+
"withdrawn": { "type": ["object", "null"], "description": "{ at, by, reason } once the decision changed; the signature is never rewritten otherwise." }
|
|
173
|
+
}
|
|
174
|
+
}
|
|
175
|
+
},
|
|
176
|
+
"decision_changes": { "type": "array", "description": "Withdrawn signatures: { at, by, reason, signature, scope, step, signed_by, meaning }." },
|
|
177
|
+
"audit": {
|
|
178
|
+
"type": "array",
|
|
179
|
+
"description": "The audit trail of the run, oldest first: rows of the audit log about the run (source audit_log) and its decisions from comments and criterion verdicts (source decisions).",
|
|
180
|
+
"items": { "type": "object", "properties": { "at": { "$ref": "#/$defs/time" }, "actor": { "type": ["string", "null"] }, "action": { "type": ["string", "null"] }, "target": { "type": ["string", "null"] }, "via": { "type": ["string", "null"] }, "agent": { "type": ["string", "null"] }, "country": { "type": ["string", "null"] }, "source": { "enum": ["audit_log", "decisions"] }, "detail": {} } }
|
|
181
|
+
},
|
|
182
|
+
"trace": { "type": "array", "items": { "$ref": "#/$defs/traceRow" } }
|
|
183
|
+
}
|
|
184
|
+
},
|
|
185
|
+
"traceRow": {
|
|
186
|
+
"type": "object",
|
|
187
|
+
"description": "One row of the trace matrix: a requirement, a criterion, a step in one locale and viewport, its screenshot, the result and the decisions. A criterion's requirements are its own refs, its step's and its scenario's (requirement_level); a criterion without any gets one row with requirement null. The CSV has these columns in this order.",
|
|
188
|
+
"properties": {
|
|
189
|
+
"requirement": { "type": ["string", "null"] },
|
|
190
|
+
"requirement_level": { "enum": ["criterion", "step", "scenario", null] },
|
|
191
|
+
"run_id": { "type": ["string", "null"] },
|
|
192
|
+
"scenario_id": { "type": ["string", "null"] },
|
|
193
|
+
"scenario_version": { "type": ["integer", "string", "null"] },
|
|
194
|
+
"step_id": { "type": "string" },
|
|
195
|
+
"step_title": { "type": ["string", "null"] },
|
|
196
|
+
"criterion_id": { "type": "string" },
|
|
197
|
+
"criterion_text": { "type": ["string", "null"] },
|
|
198
|
+
"criterion_type": { "type": ["string", "null"] },
|
|
199
|
+
"locale": { "type": ["string", "null"] },
|
|
200
|
+
"viewport": { "type": ["string", "null"] },
|
|
201
|
+
"evidence_file": { "type": ["string", "null"] },
|
|
202
|
+
"evidence_sha256": { "$ref": "#/$defs/sha256" },
|
|
203
|
+
"result": { "type": ["string", "null"] },
|
|
204
|
+
"assessed_by": { "type": ["string", "null"] },
|
|
205
|
+
"assessor": { "type": ["string", "null"] },
|
|
206
|
+
"advisory": { "type": "boolean" },
|
|
207
|
+
"step_decision": { "type": ["string", "null"] },
|
|
208
|
+
"step_decided_by": { "type": ["string", "null"] },
|
|
209
|
+
"step_decided_at": { "$ref": "#/$defs/time" },
|
|
210
|
+
"step_decided_via": { "type": ["string", "null"] },
|
|
211
|
+
"step_signature_meaning": { "type": ["string", "null"] },
|
|
212
|
+
"step_signer_name": { "type": ["string", "null"] },
|
|
213
|
+
"step_signed_at": { "$ref": "#/$defs/time" },
|
|
214
|
+
"step_signer_time_zone": { "type": ["string", "null"] },
|
|
215
|
+
"step_comment": { "type": ["string", "null"] },
|
|
216
|
+
"run_verdict": { "type": ["string", "null"] },
|
|
217
|
+
"run_decided_by": { "type": ["string", "null"] },
|
|
218
|
+
"run_decided_at": { "$ref": "#/$defs/time" },
|
|
219
|
+
"run_decided_via": { "type": ["string", "null"] },
|
|
220
|
+
"run_signature_meaning": { "type": ["string", "null"] },
|
|
221
|
+
"run_signer_name": { "type": ["string", "null"] },
|
|
222
|
+
"run_signed_at": { "$ref": "#/$defs/time" },
|
|
223
|
+
"run_signer_time_zone": { "type": ["string", "null"] }
|
|
224
|
+
}
|
|
225
|
+
},
|
|
226
|
+
"release": {
|
|
227
|
+
"type": "object",
|
|
228
|
+
"required": ["release", "runs", "trace"],
|
|
229
|
+
"properties": {
|
|
230
|
+
"generated_at": { "$ref": "#/$defs/time" },
|
|
231
|
+
"release": { "type": "object", "description": "{ product, version, released_on, tag, commit, runs: [run id] }." },
|
|
232
|
+
"runs": { "type": "array", "items": { "$ref": "#/$defs/run" }, "description": "The evidence of each run of the release the caller can read." },
|
|
233
|
+
"missing": { "type": "array", "items": { "type": "object", "properties": { "run": { "type": "string" }, "reason": { "type": "string" } } }, "description": "Runs of the release without evidence here, with the reason." },
|
|
234
|
+
"trace": { "type": "array", "items": { "$ref": "#/$defs/traceRow" } }
|
|
235
|
+
}
|
|
236
|
+
},
|
|
237
|
+
"trace": {
|
|
238
|
+
"type": "object",
|
|
239
|
+
"required": ["rows"],
|
|
240
|
+
"properties": { "run": { "type": "string" }, "record_sha256": { "$ref": "#/$defs/sha256" }, "rows": { "type": "array", "items": { "$ref": "#/$defs/traceRow" } } }
|
|
241
|
+
},
|
|
242
|
+
"package": {
|
|
243
|
+
"type": "object",
|
|
244
|
+
"required": ["files", "signature"],
|
|
245
|
+
"description": "MANIFEST.json of an evidence package, signed like a record (Ed25519 over its canonical JSON without signature).",
|
|
246
|
+
"properties": {
|
|
247
|
+
"created_at": { "$ref": "#/$defs/time" },
|
|
248
|
+
"issuer": { "type": "object", "description": "{ service, kid, public_keys }." },
|
|
249
|
+
"run": { "type": "string" },
|
|
250
|
+
"record_sha256": { "$ref": "#/$defs/sha256" },
|
|
251
|
+
"files": { "type": "array", "items": { "type": "object", "properties": { "path": { "type": "string" }, "sha256": { "$ref": "#/$defs/sha256" }, "bytes": { "type": "integer" } } } },
|
|
252
|
+
"missing": { "type": "array", "items": { "type": "string" }, "description": "Screenshots the record names that could not be read when the package was made." },
|
|
253
|
+
"signature": { "type": "object", "description": "{ alg: Ed25519, kid, value, canonical: kaloko-c14n-v1 }." }
|
|
254
|
+
}
|
|
255
|
+
}
|
|
256
|
+
}
|
|
257
|
+
}
|
package/schemas/run.v1.json
CHANGED
|
@@ -50,6 +50,15 @@
|
|
|
50
50
|
"file": {
|
|
51
51
|
"type": "string"
|
|
52
52
|
},
|
|
53
|
+
"sha256": {
|
|
54
|
+
"type": "string",
|
|
55
|
+
"description": "SHA-256 of the scenario file as it was walked (kaloko 0.15+)."
|
|
56
|
+
},
|
|
57
|
+
"refs": {
|
|
58
|
+
"type": "array",
|
|
59
|
+
"items": { "type": "string" },
|
|
60
|
+
"description": "Requirement references of the scenario (scenario refs:)."
|
|
61
|
+
},
|
|
53
62
|
"source": {
|
|
54
63
|
"type": [
|
|
55
64
|
"string",
|
|
@@ -132,6 +141,16 @@
|
|
|
132
141
|
"null"
|
|
133
142
|
]
|
|
134
143
|
},
|
|
144
|
+
"tools": {
|
|
145
|
+
"type": "object",
|
|
146
|
+
"description": "What walked the run (kaloko 0.15+): cli {name, version}, node (the Node.js version), os (platform and architecture), browsers (engine versions seen in the captures).",
|
|
147
|
+
"properties": {
|
|
148
|
+
"cli": { "type": "object" },
|
|
149
|
+
"node": { "type": ["string", "null"] },
|
|
150
|
+
"os": { "type": ["string", "null"] },
|
|
151
|
+
"browsers": { "type": "array", "items": { "type": "string" } }
|
|
152
|
+
}
|
|
153
|
+
},
|
|
135
154
|
"started_at": {
|
|
136
155
|
"type": "string"
|
|
137
156
|
},
|
|
@@ -227,6 +246,17 @@
|
|
|
227
246
|
},
|
|
228
247
|
"criteria": {
|
|
229
248
|
"type": "object"
|
|
249
|
+
},
|
|
250
|
+
"health": {
|
|
251
|
+
"type": "object",
|
|
252
|
+
"description": "Health counts (advisory, CS Kondice): checks evaluated and findings by severity (error, warning; info = the evaluator was unsure), also per pack. Never part of pass, partial or fail.",
|
|
253
|
+
"properties": {
|
|
254
|
+
"checks": { "type": "integer" },
|
|
255
|
+
"error": { "type": "integer" },
|
|
256
|
+
"warning": { "type": "integer" },
|
|
257
|
+
"info": { "type": "integer" },
|
|
258
|
+
"packs": { "type": "object" }
|
|
259
|
+
}
|
|
230
260
|
}
|
|
231
261
|
}
|
|
232
262
|
},
|
|
@@ -251,6 +281,10 @@
|
|
|
251
281
|
"slug": {
|
|
252
282
|
"type": "string"
|
|
253
283
|
},
|
|
284
|
+
"name": {
|
|
285
|
+
"type": "string",
|
|
286
|
+
"description": "The project's name from project_name: in kaloko.config.yml; a project that a share creates gets it."
|
|
287
|
+
},
|
|
254
288
|
"repo": {
|
|
255
289
|
"type": [
|
|
256
290
|
"string",
|
|
@@ -417,6 +451,11 @@
|
|
|
417
451
|
"purpose": {
|
|
418
452
|
"type": "string"
|
|
419
453
|
},
|
|
454
|
+
"refs": {
|
|
455
|
+
"type": "array",
|
|
456
|
+
"items": { "type": "string" },
|
|
457
|
+
"description": "Requirement references of the step (refs: in the scenario)."
|
|
458
|
+
},
|
|
420
459
|
"col": {
|
|
421
460
|
"type": "integer"
|
|
422
461
|
},
|
|
@@ -535,6 +574,11 @@
|
|
|
535
574
|
"verdict": {
|
|
536
575
|
"$ref": "#/$defs/verdict"
|
|
537
576
|
},
|
|
577
|
+
"refs": {
|
|
578
|
+
"type": "array",
|
|
579
|
+
"items": { "type": "string" },
|
|
580
|
+
"description": "Requirement references of the criterion (refs: in the scenario)."
|
|
581
|
+
},
|
|
538
582
|
"by_locale": {
|
|
539
583
|
"type": "object"
|
|
540
584
|
},
|
|
@@ -559,7 +603,33 @@
|
|
|
559
603
|
},
|
|
560
604
|
"check": {
|
|
561
605
|
"type": "object"
|
|
562
|
-
}
|
|
606
|
+
},
|
|
607
|
+
"pack": {
|
|
608
|
+
"type": ["string", "null"],
|
|
609
|
+
"description": "The check pack the criterion comes from. In criteria a pack criterion counts for acceptance only with acceptance: true; runs stored before health was split off have pack criteria here without it (readers show them as health; their stored verdicts stay)."
|
|
610
|
+
},
|
|
611
|
+
"acceptance": { "type": "boolean" },
|
|
612
|
+
"from": { "type": "string", "description": "The health check an adopted criterion came from (e.g. HC302)." }
|
|
613
|
+
}
|
|
614
|
+
}
|
|
615
|
+
},
|
|
616
|
+
"health": {
|
|
617
|
+
"type": "array",
|
|
618
|
+
"description": "Health checks (CS Kondice): criteria of check packs outside acceptance, shaped like criteria plus level (error or warning when the check fails). Advisory: they never decide the step status or the run verdict. Absent when the step has none.",
|
|
619
|
+
"items": {
|
|
620
|
+
"type": "object",
|
|
621
|
+
"required": ["id", "text", "verdict"],
|
|
622
|
+
"properties": {
|
|
623
|
+
"id": { "type": "string", "description": "Health check id, HC<n> (HC302). Runs written before 7 Oct 2026 store AC<n>; readers map it to HC<n>." },
|
|
624
|
+
"text": { "type": "string" },
|
|
625
|
+
"pack": { "type": ["string", "null"] },
|
|
626
|
+
"level": { "enum": ["error", "warning"] },
|
|
627
|
+
"verdict": { "$ref": "#/$defs/verdict" },
|
|
628
|
+
"by_locale": { "type": "object" },
|
|
629
|
+
"counts": { "type": "object", "description": "Stack steps: pages per verdict." },
|
|
630
|
+
"evidence": { "type": ["string", "null"] },
|
|
631
|
+
"check": { "type": "object" },
|
|
632
|
+
"help": { "type": "object" }
|
|
563
633
|
}
|
|
564
634
|
}
|
|
565
635
|
},
|
package/schemas/scenario.v1.json
CHANGED
|
@@ -89,12 +89,23 @@
|
|
|
89
89
|
"enum": ["landing", "work", "list", "settings", "form", "doc"],
|
|
90
90
|
"description": "Default page type of every step (a step may set its own). landing: a public or marketing page; work: an app screen where people work (overview, canvas, inbox); list: a list or table of items; settings: settings or billing; form: a form to fill in; doc: documentation or long text. The usability pack picks its questions by type. Not set: landing for steps with the seo pack or scenarios allowed on production, work otherwise."
|
|
91
91
|
},
|
|
92
|
+
"health": {
|
|
93
|
+
"type": "object",
|
|
94
|
+
"additionalProperties": false,
|
|
95
|
+
"description": "Health (CS Kondice): the checks of packs are advisory, shown apart from acceptance and never part of a verdict. acceptance lists the packs whose checks still count for acceptance in this scenario, as they did before health was split off (the design pack always does). To make one check count, adopt it: kaloko health adopt <step>:<check> (e.g. home:HC103) adds the criterion AC103 with from: HC103.",
|
|
96
|
+
"properties": {
|
|
97
|
+
"acceptance": { "type": "array", "uniqueItems": true, "items": { "enum": ["seo", "usability", "a11y", "perf", "console", "errors", "app", "design"] } }
|
|
98
|
+
}
|
|
99
|
+
},
|
|
92
100
|
"tags": {
|
|
93
101
|
"type": "array",
|
|
94
102
|
"items": {
|
|
95
103
|
"type": "string"
|
|
96
104
|
}
|
|
97
105
|
},
|
|
106
|
+
"refs": {
|
|
107
|
+
"$ref": "#/$defs/refs"
|
|
108
|
+
},
|
|
98
109
|
"nodes": {
|
|
99
110
|
"type": "array",
|
|
100
111
|
"minItems": 1,
|
|
@@ -193,6 +204,13 @@
|
|
|
193
204
|
}
|
|
194
205
|
},
|
|
195
206
|
"$defs": {
|
|
207
|
+
"refs": {
|
|
208
|
+
"description": "Requirement references this part verifies, as your QMS or tracker writes them (URS-12, FS-3.4, JIRA-1234). Free text; Kaloko carries them into the run, the acceptance record and the trace matrix (qawalk.evidence.v1) and never interprets them.",
|
|
209
|
+
"type": "array",
|
|
210
|
+
"maxItems": 50,
|
|
211
|
+
"uniqueItems": true,
|
|
212
|
+
"items": { "type": "string", "minLength": 1, "maxLength": 80, "pattern": "^\\S(.*\\S)?$" }
|
|
213
|
+
},
|
|
196
214
|
"record": {
|
|
197
215
|
"description": "Recordings of walked steps (opt-in): true or step records each step from the previous screen to its capture; flow records the whole walk per locale with a chapter per step; failed keeps only steps that fail; false or off turns it off. Uploaded on Business and Enterprise.",
|
|
198
216
|
"oneOf": [
|
|
@@ -277,6 +295,9 @@
|
|
|
277
295
|
"purpose": {
|
|
278
296
|
"type": "string"
|
|
279
297
|
},
|
|
298
|
+
"refs": {
|
|
299
|
+
"$ref": "#/$defs/refs"
|
|
300
|
+
},
|
|
280
301
|
"lane": {
|
|
281
302
|
"type": "integer",
|
|
282
303
|
"minimum": 0
|
|
@@ -341,7 +362,7 @@
|
|
|
341
362
|
"design"
|
|
342
363
|
]
|
|
343
364
|
},
|
|
344
|
-
"description": "Check packs whose
|
|
365
|
+
"description": "Check packs whose checks are added to this node. Health checks (advisory): HC1xx seo, HC2xx usability, HC3xx a11y, HC4xx perf, HC5xx console, HC6xx errors, HC7xx app. AC8xx design: checks against the design reference, always acceptance. A pack listed in health.acceptance adds its checks as acceptance criteria with the same number (HC302 → AC302)."
|
|
345
366
|
},
|
|
346
367
|
"capture": {
|
|
347
368
|
"$ref": "#/$defs/capture"
|
|
@@ -352,6 +373,24 @@
|
|
|
352
373
|
"description": "Regions left out of pixel comparisons (clocks, ads, carousels): a selector, or a rectangle in page pixels of one viewport. kaloko stability suggests them.",
|
|
353
374
|
"items": { "$ref": "#/$defs/ignoreRegion" }
|
|
354
375
|
},
|
|
376
|
+
"details": {
|
|
377
|
+
"type": "array",
|
|
378
|
+
"maxItems": 20,
|
|
379
|
+
"description": "Named details for user guides and docs: one element in the part of the page around it (its section or card), outlined, captured in every locale and viewport. Docs show one with .",
|
|
380
|
+
"items": {
|
|
381
|
+
"type": "object",
|
|
382
|
+
"required": ["id", "name", "selector"],
|
|
383
|
+
"additionalProperties": false,
|
|
384
|
+
"properties": {
|
|
385
|
+
"id": { "type": "string", "pattern": "^[A-Za-z0-9][A-Za-z0-9_-]{0,59}$", "description": "Stable id, used in docs references (#add-to-cart) and file names." },
|
|
386
|
+
"name": { "$ref": "#/$defs/localized", "description": "What the reader looks at, e.g. \"Add to cart button\"; one text or one per locale." },
|
|
387
|
+
"selector": { "type": "string", "minLength": 1, "description": "CSS selector of the element; the first match is outlined." },
|
|
388
|
+
"context": { "type": "string", "minLength": 1, "description": "How much of the page around it: section (the nearest section or landmark), card (the nearest card, list item or framed box), page (one screen around it) or a CSS selector of an ancestor. Default: the nearest section or card at least 480 × 240 px, at most one screen." },
|
|
389
|
+
"padding": { "type": "integer", "minimum": 0, "maximum": 200, "description": "Pixels of page around the context (default 16)." },
|
|
390
|
+
"states": { "type": "array", "minItems": 1, "uniqueItems": true, "items": { "type": "string", "pattern": "^[A-Za-z0-9][A-Za-z0-9_-]{0,39}$" }, "description": "Step states to take it in (default: every state the step is captured in)." }
|
|
391
|
+
}
|
|
392
|
+
}
|
|
393
|
+
},
|
|
355
394
|
"items": {
|
|
356
395
|
"type": "object",
|
|
357
396
|
"additionalProperties": false,
|
|
@@ -532,6 +571,9 @@
|
|
|
532
571
|
"type": "string",
|
|
533
572
|
"description": "CSS selector to outline in the screenshot."
|
|
534
573
|
},
|
|
574
|
+
"refs": {
|
|
575
|
+
"$ref": "#/$defs/refs"
|
|
576
|
+
},
|
|
535
577
|
"check": {
|
|
536
578
|
"oneOf": [
|
|
537
579
|
{
|
|
@@ -549,6 +591,15 @@
|
|
|
549
591
|
"type": "string",
|
|
550
592
|
"description": "Set by the CLI on criteria that came from a pack."
|
|
551
593
|
},
|
|
594
|
+
"from": {
|
|
595
|
+
"type": "string",
|
|
596
|
+
"pattern": "^(HC|AC)[0-9]+$",
|
|
597
|
+
"description": "The health check this criterion was adopted from (kaloko health adopt), e.g. HC302. The step's pack then leaves that check out of its health, so it is not shown twice. AC302 (written before the HC ids) reads as HC302."
|
|
598
|
+
},
|
|
599
|
+
"acceptance": {
|
|
600
|
+
"type": "boolean",
|
|
601
|
+
"description": "Set by the CLI on criteria of a pack that counts for acceptance (design, or health.acceptance). Without it a pack criterion is a health check: advisory, never part of a verdict."
|
|
602
|
+
},
|
|
552
603
|
"environments": {
|
|
553
604
|
"type": "array",
|
|
554
605
|
"minItems": 1,
|