@miller-tech/uap 1.170.1 → 1.172.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/.tsbuildinfo +1 -1
- package/dist/bin/cli.js +26 -0
- package/dist/bin/cli.js.map +1 -1
- package/dist/browser/web-browser.d.ts +17 -0
- package/dist/browser/web-browser.d.ts.map +1 -1
- package/dist/browser/web-browser.js +36 -0
- package/dist/browser/web-browser.js.map +1 -1
- package/dist/cli/interaction.d.ts +29 -0
- package/dist/cli/interaction.d.ts.map +1 -0
- package/dist/cli/interaction.js +217 -0
- package/dist/cli/interaction.js.map +1 -0
- package/dist/cli/verify.d.ts +6 -0
- package/dist/cli/verify.d.ts.map +1 -1
- package/dist/cli/verify.js +112 -7
- package/dist/cli/verify.js.map +1 -1
- package/dist/delivery/fidelity.d.ts +16 -0
- package/dist/delivery/fidelity.d.ts.map +1 -1
- package/dist/delivery/fidelity.js +7 -0
- package/dist/delivery/fidelity.js.map +1 -1
- package/dist/delivery/interaction/driver.d.ts +60 -0
- package/dist/delivery/interaction/driver.d.ts.map +1 -0
- package/dist/delivery/interaction/driver.js +18 -0
- package/dist/delivery/interaction/driver.js.map +1 -0
- package/dist/delivery/interaction/manifest.d.ts +84 -0
- package/dist/delivery/interaction/manifest.d.ts.map +1 -0
- package/dist/delivery/interaction/manifest.js +241 -0
- package/dist/delivery/interaction/manifest.js.map +1 -0
- package/dist/delivery/interaction/mine.d.ts +54 -0
- package/dist/delivery/interaction/mine.d.ts.map +1 -0
- package/dist/delivery/interaction/mine.js +218 -0
- package/dist/delivery/interaction/mine.js.map +1 -0
- package/dist/delivery/interaction/runner.d.ts +51 -0
- package/dist/delivery/interaction/runner.d.ts.map +1 -0
- package/dist/delivery/interaction/runner.js +277 -0
- package/dist/delivery/interaction/runner.js.map +1 -0
- package/dist/delivery/interaction/types.d.ts +223 -0
- package/dist/delivery/interaction/types.d.ts.map +1 -0
- package/dist/delivery/interaction/types.js +19 -0
- package/dist/delivery/interaction/types.js.map +1 -0
- package/dist/delivery/interaction/verdict.d.ts +26 -0
- package/dist/delivery/interaction/verdict.d.ts.map +1 -0
- package/dist/delivery/interaction/verdict.js +139 -0
- package/dist/delivery/interaction/verdict.js.map +1 -0
- package/dist/delivery/interaction/watchdog.d.ts +82 -0
- package/dist/delivery/interaction/watchdog.d.ts.map +1 -0
- package/dist/delivery/interaction/watchdog.js +210 -0
- package/dist/delivery/interaction/watchdog.js.map +1 -0
- package/dist/delivery/interaction/web-driver.d.ts +102 -0
- package/dist/delivery/interaction/web-driver.d.ts.map +1 -0
- package/dist/delivery/interaction/web-driver.js +306 -0
- package/dist/delivery/interaction/web-driver.js.map +1 -0
- package/dist/delivery/interaction-gate.d.ts +45 -0
- package/dist/delivery/interaction-gate.d.ts.map +1 -0
- package/dist/delivery/interaction-gate.js +248 -0
- package/dist/delivery/interaction-gate.js.map +1 -0
- package/dist/delivery/mission-acceptance.d.ts +2 -0
- package/dist/delivery/mission-acceptance.d.ts.map +1 -1
- package/dist/delivery/mission-acceptance.js +23 -0
- package/dist/delivery/mission-acceptance.js.map +1 -1
- package/dist/delivery/vision-judge.d.ts +21 -0
- package/dist/delivery/vision-judge.d.ts.map +1 -1
- package/dist/delivery/vision-judge.js +96 -0
- package/dist/delivery/vision-judge.js.map +1 -1
- package/dist/types/config.d.ts +20 -0
- package/dist/types/config.d.ts.map +1 -1
- package/dist/types/config.js +7 -0
- package/dist/types/config.js.map +1 -1
- package/package.json +1 -1
- package/src/policies/enforcers/__pycache__/_common.cpython-312.pyc +0 -0
- package/templates/hooks/__pycache__/deliver_autoroute.cpython-312.pyc +0 -0
- package/tools/agents/scripts/__pycache__/toolcall_path_normalizer.cpython-312.pyc +0 -0
|
@@ -0,0 +1,218 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Manifest mining — turn a requirements document into probes.
|
|
3
|
+
*
|
|
4
|
+
* Two halves, deliberately:
|
|
5
|
+
*
|
|
6
|
+
* - a DETERMINISTIC baseline that needs no model and always produces something
|
|
7
|
+
* real (it loads the artifact, drives its start interaction, and asserts the
|
|
8
|
+
* loop is alive and error-free). Tonight's fatal defect would have been
|
|
9
|
+
* caught by the baseline alone.
|
|
10
|
+
* - a MODEL pass that reads the requirements and proposes artifact-specific
|
|
11
|
+
* probes. Its output is validated and any probe naming an observable the
|
|
12
|
+
* artifact does not expose is dropped rather than shipped, because a probe
|
|
13
|
+
* that cannot observe anything reports failures the agent then "fixes" in
|
|
14
|
+
* working code.
|
|
15
|
+
*
|
|
16
|
+
* The model is an accelerator, never a dependency: with no model configured the
|
|
17
|
+
* gate still runs, still gates, and still says plainly which requirements it
|
|
18
|
+
* could not cover.
|
|
19
|
+
*/
|
|
20
|
+
import { hashSpec, validateManifest } from './manifest.js';
|
|
21
|
+
/** Split a requirements document into candidate requirement lines. */
|
|
22
|
+
export function extractRequirements(specText) {
|
|
23
|
+
const lines = specText
|
|
24
|
+
.split('\n')
|
|
25
|
+
.map((l) => l.trim())
|
|
26
|
+
// Bullets and numbered items are where requirements actually live; prose
|
|
27
|
+
// paragraphs are usually framing.
|
|
28
|
+
.filter((l) => /^[-*•]\s+|^\d+[.)]\s+/.test(l))
|
|
29
|
+
.map((l) => l.replace(/^[-*•]\s+|^\d+[.)]\s+/, '').trim())
|
|
30
|
+
.filter((l) => l.length >= 12 && l.length <= 300);
|
|
31
|
+
const seen = new Set();
|
|
32
|
+
const out = [];
|
|
33
|
+
for (const text of lines) {
|
|
34
|
+
const key = text.toLowerCase();
|
|
35
|
+
if (seen.has(key))
|
|
36
|
+
continue;
|
|
37
|
+
seen.add(key);
|
|
38
|
+
out.push({ id: `R${out.length + 1}`, text, source: 'requirements' });
|
|
39
|
+
}
|
|
40
|
+
return out;
|
|
41
|
+
}
|
|
42
|
+
/**
|
|
43
|
+
* Probes that apply to any web artifact, with no knowledge of what it does.
|
|
44
|
+
* This is the floor: it cannot prove features work, but it does prove the thing
|
|
45
|
+
* loads, responds to a click, and is still running afterwards — which is
|
|
46
|
+
* precisely the class the visual gate reports as a pass.
|
|
47
|
+
*/
|
|
48
|
+
export function baselineWebProbes(requirementIds = []) {
|
|
49
|
+
return [
|
|
50
|
+
{
|
|
51
|
+
id: 'baseline-loads',
|
|
52
|
+
requirementIds,
|
|
53
|
+
mode: 'core',
|
|
54
|
+
description: 'the artifact loads and reports no runtime errors',
|
|
55
|
+
steps: [{ do: 'wait', ms: 1500 }],
|
|
56
|
+
asserts: [
|
|
57
|
+
{ expect: 'truthy', expr: 'document.body !== null', label: 'the document rendered' },
|
|
58
|
+
{ expect: 'noErrors' },
|
|
59
|
+
],
|
|
60
|
+
},
|
|
61
|
+
{
|
|
62
|
+
id: 'baseline-survives-interaction',
|
|
63
|
+
requirementIds,
|
|
64
|
+
mode: 'core',
|
|
65
|
+
description: 'the artifact survives a click and keyboard input without dying',
|
|
66
|
+
steps: [
|
|
67
|
+
{ do: 'click', x: 640, y: 400 },
|
|
68
|
+
{ do: 'wait', ms: 600 },
|
|
69
|
+
{ do: 'key', key: 'Escape' },
|
|
70
|
+
{ do: 'wait', ms: 400 },
|
|
71
|
+
{ do: 'click', x: 640, y: 400 },
|
|
72
|
+
{ do: 'wait', ms: 2000 },
|
|
73
|
+
],
|
|
74
|
+
// The watchdog supplies the "is the loop still ticking" invariant; this
|
|
75
|
+
// probe's job is to make sure something actually drove the artifact first.
|
|
76
|
+
asserts: [{ expect: 'noErrors' }],
|
|
77
|
+
},
|
|
78
|
+
];
|
|
79
|
+
}
|
|
80
|
+
const MINE_PROMPT = `You are deriving an INTERACTION TEST MANIFEST for a built artifact.
|
|
81
|
+
|
|
82
|
+
You will be given (1) the requirements the artifact was built from and (2) a
|
|
83
|
+
listing of its source. Produce probes that drive the artifact with real input
|
|
84
|
+
and assert against its OWN runtime state.
|
|
85
|
+
|
|
86
|
+
Return ONLY a JSON object of this shape, with no prose and no code fence:
|
|
87
|
+
|
|
88
|
+
{
|
|
89
|
+
"watch": ["<expression yielding a number>", ...],
|
|
90
|
+
"probes": [
|
|
91
|
+
{
|
|
92
|
+
"id": "kebab-case-id",
|
|
93
|
+
"requirementIds": ["R1"],
|
|
94
|
+
"mode": "core" | "soak" | "accelerated",
|
|
95
|
+
"description": "what this proves, in one line",
|
|
96
|
+
"steps": [
|
|
97
|
+
{"do":"click","x":640,"y":400}, {"do":"wait","ms":800},
|
|
98
|
+
{"do":"move","x":640,"y":560}, {"do":"down"}, {"do":"up"},
|
|
99
|
+
{"do":"key","key":"Escape"},
|
|
100
|
+
{"do":"repeat","times":20,"steps":[...]},
|
|
101
|
+
{"do":"inject","expr":"<mutates state — ACCELERATED PROBES ONLY>"}
|
|
102
|
+
],
|
|
103
|
+
"asserts": [
|
|
104
|
+
{"expect":"equals","expr":"<expr>","value":"<v>"},
|
|
105
|
+
{"expect":"gte","expr":"<expr>","value":1},
|
|
106
|
+
{"expect":"increases","expr":"<expr>","overMs":12000,"by":1},
|
|
107
|
+
{"expect":"changes","expr":"<expr>","overMs":8000},
|
|
108
|
+
{"expect":"noErrors"}
|
|
109
|
+
]
|
|
110
|
+
}
|
|
111
|
+
]
|
|
112
|
+
}
|
|
113
|
+
|
|
114
|
+
RULES
|
|
115
|
+
- Every expression must name state the artifact ACTUALLY exposes. Read the
|
|
116
|
+
source listing and use the real identifiers. A module that only exports
|
|
117
|
+
functions does not expose its internal arrays — do not invent members.
|
|
118
|
+
- Top-level \`const\`/\`let\` in a classic script are NOT properties of window;
|
|
119
|
+
reference them bare (\`gameState\`), not as \`window.gameState\`.
|
|
120
|
+
- NEVER inject the state you then assert. Injection exists only to REACH a
|
|
121
|
+
hard-to-reach path in an "accelerated" probe; the assertion must still be
|
|
122
|
+
about what the artifact does next.
|
|
123
|
+
- Prefer assertions that would FAIL on a plausible bug: "score increases after
|
|
124
|
+
sustained fire" catches a broken kill path; "score is a number" catches nothing.
|
|
125
|
+
- Use "soak" mode for anything proving sustained operation (many levels, long
|
|
126
|
+
sessions). Use "core" for the primary promised behaviours.
|
|
127
|
+
`;
|
|
128
|
+
/** Strip a fence if the model added one despite instructions. */
|
|
129
|
+
export function parseMinedJson(raw) {
|
|
130
|
+
const text = raw.trim().replace(/^```(?:json)?\s*/i, '').replace(/```\s*$/, '');
|
|
131
|
+
const start = text.indexOf('{');
|
|
132
|
+
const end = text.lastIndexOf('}');
|
|
133
|
+
if (start < 0 || end <= start)
|
|
134
|
+
return null;
|
|
135
|
+
try {
|
|
136
|
+
return JSON.parse(text.slice(start, end + 1));
|
|
137
|
+
}
|
|
138
|
+
catch {
|
|
139
|
+
return null;
|
|
140
|
+
}
|
|
141
|
+
}
|
|
142
|
+
/**
|
|
143
|
+
* Build a manifest. Always returns one: the deterministic baseline is included
|
|
144
|
+
* even when the model is absent or returns junk, so the gate never silently
|
|
145
|
+
* degrades into "nothing to check".
|
|
146
|
+
*/
|
|
147
|
+
export async function mineManifest(opts) {
|
|
148
|
+
const requirements = extractRequirements(opts.specText);
|
|
149
|
+
const allIds = requirements.map((r) => r.id);
|
|
150
|
+
// Baseline probes carry NO requirement ids. They prove the artifact loads and
|
|
151
|
+
// survives input — nothing about any specific promise. Linking them to R1
|
|
152
|
+
// would report that requirement as covered when nothing tested it, which is
|
|
153
|
+
// exactly the false assurance the coverage ledger exists to prevent.
|
|
154
|
+
const probes = [...baselineWebProbes()];
|
|
155
|
+
let watch = [];
|
|
156
|
+
if (opts.executor) {
|
|
157
|
+
try {
|
|
158
|
+
const prompt = `${MINE_PROMPT}\n\nREQUIREMENTS:\n${requirements
|
|
159
|
+
.map((r) => `${r.id}: ${r.text}`)
|
|
160
|
+
.join('\n')}\n\nSOURCE:\n${(opts.sourceDigest ?? '').slice(0, 24_000)}\n`;
|
|
161
|
+
const parsed = parseMinedJson(await opts.executor(prompt));
|
|
162
|
+
if (parsed) {
|
|
163
|
+
if (Array.isArray(parsed.watch))
|
|
164
|
+
watch = parsed.watch.filter((w) => typeof w === 'string');
|
|
165
|
+
for (const p of parsed.probes ?? []) {
|
|
166
|
+
// Keep only probes that reference requirements we actually extracted,
|
|
167
|
+
// so a hallucinated requirement id cannot smuggle in an unanchored probe.
|
|
168
|
+
const ids = (p.requirementIds ?? []).filter((id) => allIds.includes(id));
|
|
169
|
+
if (ids.length === 0)
|
|
170
|
+
continue;
|
|
171
|
+
probes.push({ ...p, requirementIds: ids });
|
|
172
|
+
}
|
|
173
|
+
}
|
|
174
|
+
}
|
|
175
|
+
catch {
|
|
176
|
+
// A mining failure must not disable the gate — the baseline still runs.
|
|
177
|
+
}
|
|
178
|
+
}
|
|
179
|
+
const manifest = {
|
|
180
|
+
version: 1,
|
|
181
|
+
kind: opts.kind,
|
|
182
|
+
entry: opts.entry,
|
|
183
|
+
specHash: hashSpec(opts.specText),
|
|
184
|
+
generatedAt: new Date().toISOString(),
|
|
185
|
+
requirements,
|
|
186
|
+
probes,
|
|
187
|
+
...(watch.length > 0 ? { watch } : {}),
|
|
188
|
+
};
|
|
189
|
+
// Drop anything structurally invalid rather than shipping a manifest the
|
|
190
|
+
// runner will reject wholesale (one bad model probe must not lose the rest).
|
|
191
|
+
// Then RE-VALIDATE: a manifest that is still invalid after filtering would be
|
|
192
|
+
// written to disk, fail to load forever, and make the gate report "no
|
|
193
|
+
// manifest" — i.e. pass — on every subsequent run.
|
|
194
|
+
const dropInvalidProbes = (m) => {
|
|
195
|
+
const problems = validateManifest(m);
|
|
196
|
+
if (problems.length === 0)
|
|
197
|
+
return [];
|
|
198
|
+
const bad = new Set(problems
|
|
199
|
+
.map((p) => /^probe (\S+):/.exec(p)?.[1])
|
|
200
|
+
.filter((id) => Boolean(id)));
|
|
201
|
+
if (bad.size > 0)
|
|
202
|
+
m.probes = m.probes.filter((p) => !bad.has(p.id));
|
|
203
|
+
return validateManifest(m);
|
|
204
|
+
};
|
|
205
|
+
const remaining = dropInvalidProbes(manifest);
|
|
206
|
+
if (remaining.length > 0) {
|
|
207
|
+
// Whatever is left is not probe-scoped (a bad `watch` entry, a bad kind).
|
|
208
|
+
// Fall back to the baseline rather than persisting something unloadable.
|
|
209
|
+
manifest.probes = baselineWebProbes();
|
|
210
|
+
delete manifest.watch;
|
|
211
|
+
const stillBad = validateManifest(manifest);
|
|
212
|
+
if (stillBad.length > 0) {
|
|
213
|
+
throw new Error(`mined manifest is invalid and could not be repaired: ${stillBad.join('; ')}`);
|
|
214
|
+
}
|
|
215
|
+
}
|
|
216
|
+
return manifest;
|
|
217
|
+
}
|
|
218
|
+
//# sourceMappingURL=mine.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"mine.js","sourceRoot":"","sources":["../../../src/delivery/interaction/mine.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;GAkBG;AAGH,OAAO,EAAE,QAAQ,EAAE,gBAAgB,EAAE,MAAM,eAAe,CAAC;AAQ3D,sEAAsE;AACtE,MAAM,UAAU,mBAAmB,CAAC,QAAgB;IAClD,MAAM,KAAK,GAAG,QAAQ;SACnB,KAAK,CAAC,IAAI,CAAC;SACX,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC;QACrB,yEAAyE;QACzE,kCAAkC;SACjC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,uBAAuB,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;SAC9C,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,OAAO,CAAC,uBAAuB,EAAE,EAAE,CAAC,CAAC,IAAI,EAAE,CAAC;SACzD,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,MAAM,IAAI,EAAE,IAAI,CAAC,CAAC,MAAM,IAAI,GAAG,CAAC,CAAC;IAEpD,MAAM,IAAI,GAAG,IAAI,GAAG,EAAU,CAAC;IAC/B,MAAM,GAAG,GAAkB,EAAE,CAAC;IAC9B,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE,CAAC;QACzB,MAAM,GAAG,GAAG,IAAI,CAAC,WAAW,EAAE,CAAC;QAC/B,IAAI,IAAI,CAAC,GAAG,CAAC,GAAG,CAAC;YAAE,SAAS;QAC5B,IAAI,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;QACd,GAAG,CAAC,IAAI,CAAC,EAAE,EAAE,EAAE,IAAI,GAAG,CAAC,MAAM,GAAG,CAAC,EAAE,EAAE,IAAI,EAAE,MAAM,EAAE,cAAc,EAAE,CAAC,CAAC;IACvE,CAAC;IACD,OAAO,GAAG,CAAC;AACb,CAAC;AAED;;;;;GAKG;AACH,MAAM,UAAU,iBAAiB,CAAC,iBAA2B,EAAE;IAC7D,OAAO;QACL;YACE,EAAE,EAAE,gBAAgB;YACpB,cAAc;YACd,IAAI,EAAE,MAAM;YACZ,WAAW,EAAE,kDAAkD;YAC/D,KAAK,EAAE,CAAC,EAAE,EAAE,EAAE,MAAM,EAAE,EAAE,EAAE,IAAI,EAAE,CAAC;YACjC,OAAO,EAAE;gBACP,EAAE,MAAM,EAAE,QAAQ,EAAE,IAAI,EAAE,wBAAwB,EAAE,KAAK,EAAE,uBAAuB,EAAE;gBACpF,EAAE,MAAM,EAAE,UAAU,EAAE;aACvB;SACF;QACD;YACE,EAAE,EAAE,+BAA+B;YACnC,cAAc;YACd,IAAI,EAAE,MAAM;YACZ,WAAW,EAAE,gEAAgE;YAC7E,KAAK,EAAE;gBACL,EAAE,EAAE,EAAE,OAAO,EAAE,CAAC,EAAE,GAAG,EAAE,CAAC,EAAE,GAAG,EAAE;gBAC/B,EAAE,EAAE,EAAE,MAAM,EAAE,EAAE,EAAE,GAAG,EAAE;gBACvB,EAAE,EAAE,EAAE,KAAK,EAAE,GAAG,EAAE,QAAQ,EAAE;gBAC5B,EAAE,EAAE,EAAE,MAAM,EAAE,EAAE,EAAE,GAAG,EAAE;gBACvB,EAAE,EAAE,EAAE,OAAO,EAAE,CAAC,EAAE,GAAG,EAAE,CAAC,EAAE,GAAG,EAAE;gBAC/B,EAAE,EAAE,EAAE,MAAM,EAAE,EAAE,EAAE,IAAI,EAAE;aACzB;YACD,wEAAwE;YACxE,2EAA2E;YAC3E,OAAO,EAAE,CAAC,EAAE,MAAM,EAAE,UAAU,EAAE,CAAC;SAClC;KACF,CAAC;AACJ,CAAC;AAED,MAAM,WAAW,GAAG;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;CA+CnB,CAAC;AAkBF,iEAAiE;AACjE,MAAM,UAAU,cAAc,CAAC,GAAW;IACxC,MAAM,IAAI,GAAG,GAAG,CAAC,IAAI,EAAE,CAAC,OAAO,CAAC,mBAAmB,EAAE,EAAE,CAAC,CAAC,OAAO,CAAC,SAAS,EAAE,EAAE,CAAC,CAAC;IAChF,MAAM,KAAK,GAAG,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC;IAChC,MAAM,GAAG,GAAG,IAAI,CAAC,WAAW,CAAC,GAAG,CAAC,CAAC;IAClC,IAAI,KAAK,GAAG,CAAC,IAAI,GAAG,IAAI,KAAK;QAAE,OAAO,IAAI,CAAC;IAC3C,IAAI,CAAC;QACH,OAAO,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,KAAK,CAAC,KAAK,EAAE,GAAG,GAAG,CAAC,CAAC,CAAsB,CAAC;IACrE,CAAC;IAAC,MAAM,CAAC;QACP,OAAO,IAAI,CAAC;IACd,CAAC;AACH,CAAC;AAED;;;;GAIG;AACH,MAAM,CAAC,KAAK,UAAU,YAAY,CAAC,IAAiB;IAClD,MAAM,YAAY,GAAG,mBAAmB,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;IACxD,MAAM,MAAM,GAAG,YAAY,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC;IAC7C,8EAA8E;IAC9E,0EAA0E;IAC1E,4EAA4E;IAC5E,qEAAqE;IACrE,MAAM,MAAM,GAAY,CAAC,GAAG,iBAAiB,EAAE,CAAC,CAAC;IACjD,IAAI,KAAK,GAAa,EAAE,CAAC;IAEzB,IAAI,IAAI,CAAC,QAAQ,EAAE,CAAC;QAClB,IAAI,CAAC;YACH,MAAM,MAAM,GAAG,GAAG,WAAW,sBAAsB,YAAY;iBAC5D,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,GAAG,CAAC,CAAC,EAAE,KAAK,CAAC,CAAC,IAAI,EAAE,CAAC;iBAChC,IAAI,CAAC,IAAI,CAAC,gBAAgB,CAAC,IAAI,CAAC,YAAY,IAAI,EAAE,CAAC,CAAC,KAAK,CAAC,CAAC,EAAE,MAAM,CAAC,IAAI,CAAC;YAC5E,MAAM,MAAM,GAAG,cAAc,CAAC,MAAM,IAAI,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAC,CAAC;YAC3D,IAAI,MAAM,EAAE,CAAC;gBACX,IAAI,KAAK,CAAC,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC;oBAAE,KAAK,GAAG,MAAM,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,OAAO,CAAC,KAAK,QAAQ,CAAC,CAAC;gBAC3F,KAAK,MAAM,CAAC,IAAI,MAAM,CAAC,MAAM,IAAI,EAAE,EAAE,CAAC;oBACpC,sEAAsE;oBACtE,0EAA0E;oBAC1E,MAAM,GAAG,GAAG,CAAC,CAAC,CAAC,cAAc,IAAI,EAAE,CAAC,CAAC,MAAM,CAAC,CAAC,EAAE,EAAE,EAAE,CAAC,MAAM,CAAC,QAAQ,CAAC,EAAE,CAAC,CAAC,CAAC;oBACzE,IAAI,GAAG,CAAC,MAAM,KAAK,CAAC;wBAAE,SAAS;oBAC/B,MAAM,CAAC,IAAI,CAAC,EAAE,GAAG,CAAC,EAAE,cAAc,EAAE,GAAG,EAAE,CAAC,CAAC;gBAC7C,CAAC;YACH,CAAC;QACH,CAAC;QAAC,MAAM,CAAC;YACP,wEAAwE;QAC1E,CAAC;IACH,CAAC;IAED,MAAM,QAAQ,GAAwB;QACpC,OAAO,EAAE,CAAC;QACV,IAAI,EAAE,IAAI,CAAC,IAAI;QACf,KAAK,EAAE,IAAI,CAAC,KAAK;QACjB,QAAQ,EAAE,QAAQ,CAAC,IAAI,CAAC,QAAQ,CAAC;QACjC,WAAW,EAAE,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE;QACrC,YAAY;QACZ,MAAM;QACN,GAAG,CAAC,KAAK,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,EAAE,KAAK,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;KACvC,CAAC;IAEF,yEAAyE;IACzE,6EAA6E;IAC7E,8EAA8E;IAC9E,sEAAsE;IACtE,mDAAmD;IACnD,MAAM,iBAAiB,GAAG,CAAC,CAAsB,EAAY,EAAE;QAC7D,MAAM,QAAQ,GAAG,gBAAgB,CAAC,CAAC,CAAC,CAAC;QACrC,IAAI,QAAQ,CAAC,MAAM,KAAK,CAAC;YAAE,OAAO,EAAE,CAAC;QACrC,MAAM,GAAG,GAAG,IAAI,GAAG,CACjB,QAAQ;aACL,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,eAAe,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC;aACxC,MAAM,CAAC,CAAC,EAAE,EAAgB,EAAE,CAAC,OAAO,CAAC,EAAE,CAAC,CAAC,CAC7C,CAAC;QACF,IAAI,GAAG,CAAC,IAAI,GAAG,CAAC;YAAE,CAAC,CAAC,MAAM,GAAG,CAAC,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC;QACpE,OAAO,gBAAgB,CAAC,CAAC,CAAC,CAAC;IAC7B,CAAC,CAAC;IACF,MAAM,SAAS,GAAG,iBAAiB,CAAC,QAAQ,CAAC,CAAC;IAC9C,IAAI,SAAS,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QACzB,0EAA0E;QAC1E,yEAAyE;QACzE,QAAQ,CAAC,MAAM,GAAG,iBAAiB,EAAE,CAAC;QACtC,OAAO,QAAQ,CAAC,KAAK,CAAC;QACtB,MAAM,QAAQ,GAAG,gBAAgB,CAAC,QAAQ,CAAC,CAAC;QAC5C,IAAI,QAAQ,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YACxB,MAAM,IAAI,KAAK,CAAC,wDAAwD,QAAQ,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;QACjG,CAAC;IACH,CAAC;IACD,OAAO,QAAQ,CAAC;AAClB,CAAC"}
|
|
@@ -0,0 +1,51 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Probe runner — drives one probe and judges its assertions.
|
|
3
|
+
*
|
|
4
|
+
* Time-based assertions (`increases`, `changes`) capture their baseline BEFORE
|
|
5
|
+
* the probe's input runs and read again after. That ordering is what makes
|
|
6
|
+
* "hold fire for 20 seconds → the score rose" expressible: sampling only after
|
|
7
|
+
* the input stops would measure a game that is no longer being played.
|
|
8
|
+
*/
|
|
9
|
+
import type { InteractionDriver } from './driver.js';
|
|
10
|
+
import type { Assertion, AssertionResult, Probe, ProbeResult, Step } from './types.js';
|
|
11
|
+
export declare const DEFAULT_PROBE_TIMEOUT_MS = 120000;
|
|
12
|
+
/** Guard against a manifest asking for an effectively infinite loop. */
|
|
13
|
+
export declare const MAX_REPEAT = 10000;
|
|
14
|
+
/**
|
|
15
|
+
* Expand `repeat` blocks HERE rather than in each driver.
|
|
16
|
+
*
|
|
17
|
+
* When expansion lived in the web driver, every other adapter silently dropped
|
|
18
|
+
* repeat blocks: the probe drove no input at all and then still reported a pass
|
|
19
|
+
* whenever its assertion happened to hold anyway. A gate that quietly stops
|
|
20
|
+
* driving input is worse than no gate, because it reports success.
|
|
21
|
+
*/
|
|
22
|
+
export declare function dispatchStep(driver: InteractionDriver, step: Step): Promise<void>;
|
|
23
|
+
/** True when the assertion needs a reading taken before the input runs. */
|
|
24
|
+
export declare function needsBaseline(a: Assertion): a is Extract<Assertion, {
|
|
25
|
+
expect: 'increases' | 'changes';
|
|
26
|
+
}>;
|
|
27
|
+
/**
|
|
28
|
+
* Pure judgement for one assertion given its observations — the interesting
|
|
29
|
+
* logic, unit-testable without a browser.
|
|
30
|
+
*/
|
|
31
|
+
export declare function judgeAssertion(a: Assertion, index: number, observed: {
|
|
32
|
+
value?: unknown;
|
|
33
|
+
baseline?: unknown;
|
|
34
|
+
errors?: string[];
|
|
35
|
+
/** Set when the observation expression did not resolve. */
|
|
36
|
+
unresolved?: string;
|
|
37
|
+
}): AssertionResult;
|
|
38
|
+
export interface RunProbeOptions {
|
|
39
|
+
timeoutMs?: number;
|
|
40
|
+
/** Written to when the probe wants an evidence screenshot. */
|
|
41
|
+
evidencePath?: string;
|
|
42
|
+
}
|
|
43
|
+
/** Marks a probe that blew its wall-clock bound rather than failing on merit. */
|
|
44
|
+
export declare const PROBE_TIMEOUT_MARKER = "probe exceeded its time budget";
|
|
45
|
+
/**
|
|
46
|
+
* Run one probe end to end. Never throws: a driver failure becomes a FAILING
|
|
47
|
+
* result rather than an exception, because an interaction gate that crashes is
|
|
48
|
+
* indistinguishable from one that passed.
|
|
49
|
+
*/
|
|
50
|
+
export declare function runProbe(driver: InteractionDriver, probe: Probe, opts?: RunProbeOptions): Promise<ProbeResult>;
|
|
51
|
+
//# sourceMappingURL=runner.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"runner.d.ts","sourceRoot":"","sources":["../../../src/delivery/interaction/runner.ts"],"names":[],"mappings":"AAAA;;;;;;;GAOG;AAGH,OAAO,KAAK,EAAE,iBAAiB,EAAc,MAAM,aAAa,CAAC;AAEjE,OAAO,KAAK,EAAE,SAAS,EAAE,eAAe,EAAE,KAAK,EAAE,WAAW,EAAE,IAAI,EAAE,MAAM,YAAY,CAAC;AAEvF,eAAO,MAAM,wBAAwB,SAAU,CAAC;AAEhD,wEAAwE;AACxE,eAAO,MAAM,UAAU,QAAS,CAAC;AAEjC;;;;;;;GAOG;AACH,wBAAsB,YAAY,CAAC,MAAM,EAAE,iBAAiB,EAAE,IAAI,EAAE,IAAI,GAAG,OAAO,CAAC,IAAI,CAAC,CASvF;AAmCD,2EAA2E;AAC3E,wBAAgB,aAAa,CAAC,CAAC,EAAE,SAAS,GAAG,CAAC,IAAI,OAAO,CAAC,SAAS,EAAE;IAAE,MAAM,EAAE,WAAW,GAAG,SAAS,CAAA;CAAE,CAAC,CAExG;AAED;;;GAGG;AACH,wBAAgB,cAAc,CAC5B,CAAC,EAAE,SAAS,EACZ,KAAK,EAAE,MAAM,EACb,QAAQ,EAAE;IACR,KAAK,CAAC,EAAE,OAAO,CAAC;IAChB,QAAQ,CAAC,EAAE,OAAO,CAAC;IACnB,MAAM,CAAC,EAAE,MAAM,EAAE,CAAC;IAClB,2DAA2D;IAC3D,UAAU,CAAC,EAAE,MAAM,CAAC;CACrB,GACA,eAAe,CAqFjB;AAED,MAAM,WAAW,eAAe;IAC9B,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,8DAA8D;IAC9D,YAAY,CAAC,EAAE,MAAM,CAAC;CACvB;AAED,iFAAiF;AACjF,eAAO,MAAM,oBAAoB,mCAAmC,CAAC;AA2BrE;;;;GAIG;AACH,wBAAsB,QAAQ,CAC5B,MAAM,EAAE,iBAAiB,EACzB,KAAK,EAAE,KAAK,EACZ,IAAI,GAAE,eAAoB,GACzB,OAAO,CAAC,WAAW,CAAC,CA8FtB"}
|
|
@@ -0,0 +1,277 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Probe runner — drives one probe and judges its assertions.
|
|
3
|
+
*
|
|
4
|
+
* Time-based assertions (`increases`, `changes`) capture their baseline BEFORE
|
|
5
|
+
* the probe's input runs and read again after. That ordering is what makes
|
|
6
|
+
* "hold fire for 20 seconds → the score rose" expressible: sampling only after
|
|
7
|
+
* the input stops would measure a game that is no longer being played.
|
|
8
|
+
*/
|
|
9
|
+
import { flattenSteps } from './manifest.js';
|
|
10
|
+
import { delay } from './driver.js';
|
|
11
|
+
export const DEFAULT_PROBE_TIMEOUT_MS = 120_000;
|
|
12
|
+
/** Guard against a manifest asking for an effectively infinite loop. */
|
|
13
|
+
export const MAX_REPEAT = 10_000;
|
|
14
|
+
/**
|
|
15
|
+
* Expand `repeat` blocks HERE rather than in each driver.
|
|
16
|
+
*
|
|
17
|
+
* When expansion lived in the web driver, every other adapter silently dropped
|
|
18
|
+
* repeat blocks: the probe drove no input at all and then still reported a pass
|
|
19
|
+
* whenever its assertion happened to hold anyway. A gate that quietly stops
|
|
20
|
+
* driving input is worse than no gate, because it reports success.
|
|
21
|
+
*/
|
|
22
|
+
export async function dispatchStep(driver, step) {
|
|
23
|
+
if (step.do === 'repeat') {
|
|
24
|
+
const times = Math.max(0, Math.min(step.times, MAX_REPEAT));
|
|
25
|
+
for (let i = 0; i < times; i++) {
|
|
26
|
+
for (const inner of step.steps)
|
|
27
|
+
await dispatchStep(driver, inner);
|
|
28
|
+
}
|
|
29
|
+
return;
|
|
30
|
+
}
|
|
31
|
+
await driver.runStep(step);
|
|
32
|
+
}
|
|
33
|
+
function label(a, index) {
|
|
34
|
+
if ('label' in a && a.label)
|
|
35
|
+
return a.label;
|
|
36
|
+
switch (a.expect) {
|
|
37
|
+
case 'noErrors':
|
|
38
|
+
return 'no runtime errors';
|
|
39
|
+
case 'truthy':
|
|
40
|
+
return `${a.expr} is truthy`;
|
|
41
|
+
case 'equals':
|
|
42
|
+
return `${a.expr} === ${JSON.stringify(a.value)}`;
|
|
43
|
+
case 'gte':
|
|
44
|
+
return `${a.expr} >= ${a.value}`;
|
|
45
|
+
case 'lte':
|
|
46
|
+
return `${a.expr} <= ${a.value}`;
|
|
47
|
+
case 'increases':
|
|
48
|
+
return `${a.expr} increases${a.by ? ` by >= ${a.by}` : ''}`;
|
|
49
|
+
case 'changes':
|
|
50
|
+
return `${a.expr} changes`;
|
|
51
|
+
default:
|
|
52
|
+
return `assertion #${index + 1}`;
|
|
53
|
+
}
|
|
54
|
+
}
|
|
55
|
+
function show(v) {
|
|
56
|
+
if (v === undefined)
|
|
57
|
+
return 'undefined';
|
|
58
|
+
if (typeof v === 'number' || typeof v === 'boolean' || v === null)
|
|
59
|
+
return String(v);
|
|
60
|
+
try {
|
|
61
|
+
const s = JSON.stringify(v);
|
|
62
|
+
return s.length > 120 ? `${s.slice(0, 117)}...` : s;
|
|
63
|
+
}
|
|
64
|
+
catch {
|
|
65
|
+
return String(v);
|
|
66
|
+
}
|
|
67
|
+
}
|
|
68
|
+
/** True when the assertion needs a reading taken before the input runs. */
|
|
69
|
+
export function needsBaseline(a) {
|
|
70
|
+
return a.expect === 'increases' || a.expect === 'changes';
|
|
71
|
+
}
|
|
72
|
+
/**
|
|
73
|
+
* Pure judgement for one assertion given its observations — the interesting
|
|
74
|
+
* logic, unit-testable without a browser.
|
|
75
|
+
*/
|
|
76
|
+
export function judgeAssertion(a, index, observed) {
|
|
77
|
+
const name = label(a, index);
|
|
78
|
+
// A probe that observes something the artifact does not expose has told us
|
|
79
|
+
// nothing about the artifact. Report it as a broken probe so the fix lands on
|
|
80
|
+
// the manifest instead of on working code.
|
|
81
|
+
if (observed.unresolved && a.expect !== 'noErrors') {
|
|
82
|
+
return {
|
|
83
|
+
label: name,
|
|
84
|
+
passed: false,
|
|
85
|
+
unresolved: true,
|
|
86
|
+
expected: 'an observable the artifact exposes',
|
|
87
|
+
observed: observed.unresolved,
|
|
88
|
+
};
|
|
89
|
+
}
|
|
90
|
+
switch (a.expect) {
|
|
91
|
+
case 'noErrors': {
|
|
92
|
+
const errs = observed.errors ?? [];
|
|
93
|
+
return {
|
|
94
|
+
label: name,
|
|
95
|
+
passed: errs.length === 0,
|
|
96
|
+
expected: 'no runtime errors',
|
|
97
|
+
observed: errs.length === 0 ? 'none' : errs.slice(0, 3).join(' | '),
|
|
98
|
+
};
|
|
99
|
+
}
|
|
100
|
+
case 'truthy':
|
|
101
|
+
return {
|
|
102
|
+
label: name,
|
|
103
|
+
passed: Boolean(observed.value),
|
|
104
|
+
expected: 'truthy',
|
|
105
|
+
observed: show(observed.value),
|
|
106
|
+
};
|
|
107
|
+
case 'equals':
|
|
108
|
+
return {
|
|
109
|
+
label: name,
|
|
110
|
+
passed: JSON.stringify(observed.value) === JSON.stringify(a.value),
|
|
111
|
+
expected: show(a.value),
|
|
112
|
+
observed: show(observed.value),
|
|
113
|
+
};
|
|
114
|
+
case 'gte': {
|
|
115
|
+
const v = Number(observed.value);
|
|
116
|
+
return {
|
|
117
|
+
label: name,
|
|
118
|
+
passed: Number.isFinite(v) && v >= a.value,
|
|
119
|
+
expected: `>= ${a.value}`,
|
|
120
|
+
observed: show(observed.value),
|
|
121
|
+
};
|
|
122
|
+
}
|
|
123
|
+
case 'lte': {
|
|
124
|
+
const v = Number(observed.value);
|
|
125
|
+
return {
|
|
126
|
+
label: name,
|
|
127
|
+
passed: Number.isFinite(v) && v <= a.value,
|
|
128
|
+
expected: `<= ${a.value}`,
|
|
129
|
+
observed: show(observed.value),
|
|
130
|
+
};
|
|
131
|
+
}
|
|
132
|
+
case 'increases': {
|
|
133
|
+
const start = Number(observed.baseline);
|
|
134
|
+
const end = Number(observed.value);
|
|
135
|
+
const by = a.by ?? 1;
|
|
136
|
+
const ok = Number.isFinite(start) && Number.isFinite(end) && end - start >= by;
|
|
137
|
+
return {
|
|
138
|
+
label: name,
|
|
139
|
+
passed: ok,
|
|
140
|
+
expected: `rise of >= ${by} within ${a.overMs}ms`,
|
|
141
|
+
observed: `${show(observed.baseline)} → ${show(observed.value)}`,
|
|
142
|
+
};
|
|
143
|
+
}
|
|
144
|
+
case 'changes': {
|
|
145
|
+
const ok = JSON.stringify(observed.baseline) !== JSON.stringify(observed.value);
|
|
146
|
+
return {
|
|
147
|
+
label: name,
|
|
148
|
+
passed: ok,
|
|
149
|
+
expected: `different value within ${a.overMs}ms`,
|
|
150
|
+
observed: `${show(observed.baseline)} → ${show(observed.value)}`,
|
|
151
|
+
};
|
|
152
|
+
}
|
|
153
|
+
default:
|
|
154
|
+
return {
|
|
155
|
+
label: name,
|
|
156
|
+
passed: false,
|
|
157
|
+
expected: 'a known assertion kind',
|
|
158
|
+
observed: `unsupported: ${String(a.expect)}`,
|
|
159
|
+
};
|
|
160
|
+
}
|
|
161
|
+
}
|
|
162
|
+
/** Marks a probe that blew its wall-clock bound rather than failing on merit. */
|
|
163
|
+
export const PROBE_TIMEOUT_MARKER = 'probe exceeded its time budget';
|
|
164
|
+
class ProbeTimeout extends Error {
|
|
165
|
+
}
|
|
166
|
+
/**
|
|
167
|
+
* Bound a probe's wall clock. `repeat` blocks are legal up to MAX_REPEAT
|
|
168
|
+
* iterations, so a single probe can otherwise run for hours — and on the Stop
|
|
169
|
+
* hook that means an outer `timeout` kills the process mid-probe, skipping the
|
|
170
|
+
* `finally` that reaps the browser. An unbounded gate is a session-wedging
|
|
171
|
+
* hazard, not just a slow one.
|
|
172
|
+
*/
|
|
173
|
+
async function withTimeout(work, ms) {
|
|
174
|
+
let timer;
|
|
175
|
+
try {
|
|
176
|
+
return await Promise.race([
|
|
177
|
+
work,
|
|
178
|
+
new Promise((_, reject) => {
|
|
179
|
+
timer = setTimeout(() => reject(new ProbeTimeout(PROBE_TIMEOUT_MARKER)), ms);
|
|
180
|
+
// Never hold the event loop open just for the watchdog timer.
|
|
181
|
+
timer.unref?.();
|
|
182
|
+
}),
|
|
183
|
+
]);
|
|
184
|
+
}
|
|
185
|
+
finally {
|
|
186
|
+
if (timer)
|
|
187
|
+
clearTimeout(timer);
|
|
188
|
+
}
|
|
189
|
+
}
|
|
190
|
+
/**
|
|
191
|
+
* Run one probe end to end. Never throws: a driver failure becomes a FAILING
|
|
192
|
+
* result rather than an exception, because an interaction gate that crashes is
|
|
193
|
+
* indistinguishable from one that passed.
|
|
194
|
+
*/
|
|
195
|
+
export async function runProbe(driver, probe, opts = {}) {
|
|
196
|
+
const started = Date.now();
|
|
197
|
+
const base = {
|
|
198
|
+
probeId: probe.id,
|
|
199
|
+
description: probe.description,
|
|
200
|
+
mode: probe.mode,
|
|
201
|
+
requirementIds: probe.requirementIds ?? [],
|
|
202
|
+
};
|
|
203
|
+
// Defence in depth: manifest validation already rejects this, but a manifest
|
|
204
|
+
// could be hand-edited between validation and execution.
|
|
205
|
+
if (probe.mode !== 'accelerated' && flattenSteps(probe.steps).some((s) => s.do === 'inject')) {
|
|
206
|
+
return {
|
|
207
|
+
...base,
|
|
208
|
+
passed: false,
|
|
209
|
+
assertions: [
|
|
210
|
+
{
|
|
211
|
+
label: 'probe integrity',
|
|
212
|
+
passed: false,
|
|
213
|
+
expected: 'no state injection outside accelerated probes',
|
|
214
|
+
observed: 'probe injects the state it then asserts',
|
|
215
|
+
},
|
|
216
|
+
],
|
|
217
|
+
errors: [],
|
|
218
|
+
durationMs: Date.now() - started,
|
|
219
|
+
};
|
|
220
|
+
}
|
|
221
|
+
const errorsBefore = new Set(driver.errors());
|
|
222
|
+
const baselines = new Map();
|
|
223
|
+
const assertions = [];
|
|
224
|
+
// Prefer the detailed read so an unresolvable observable is reported as a
|
|
225
|
+
// broken probe; drivers that only implement `read` degrade to the old
|
|
226
|
+
// behaviour rather than failing.
|
|
227
|
+
const readDetailed = async (expr) => driver.readDetailed ? driver.readDetailed(expr) : { ok: true, value: await driver.read(expr) };
|
|
228
|
+
try {
|
|
229
|
+
// Baselines first — before any input runs.
|
|
230
|
+
for (let i = 0; i < probe.asserts.length; i++) {
|
|
231
|
+
const a = probe.asserts[i];
|
|
232
|
+
if (needsBaseline(a))
|
|
233
|
+
baselines.set(i, (await readDetailed(a.expr)).value);
|
|
234
|
+
}
|
|
235
|
+
const budget = probe.timeoutMs ?? opts.timeoutMs ?? DEFAULT_PROBE_TIMEOUT_MS;
|
|
236
|
+
await withTimeout((async () => {
|
|
237
|
+
for (const step of probe.steps)
|
|
238
|
+
await dispatchStep(driver, step);
|
|
239
|
+
})(), budget);
|
|
240
|
+
// Honour each time-based assertion's observation window even when the input
|
|
241
|
+
// finished sooner.
|
|
242
|
+
const longestWindow = probe.asserts.reduce((max, a) => (needsBaseline(a) ? Math.max(max, a.overMs) : max), 0);
|
|
243
|
+
const elapsed = Date.now() - started;
|
|
244
|
+
if (longestWindow > elapsed)
|
|
245
|
+
await delay(longestWindow - elapsed);
|
|
246
|
+
const newErrors = driver.errors().filter((e) => !errorsBefore.has(e));
|
|
247
|
+
for (let i = 0; i < probe.asserts.length; i++) {
|
|
248
|
+
const a = probe.asserts[i];
|
|
249
|
+
const read = a.expect === 'noErrors' ? { ok: true } : await readDetailed(a.expr);
|
|
250
|
+
assertions.push(judgeAssertion(a, i, {
|
|
251
|
+
value: read.value,
|
|
252
|
+
baseline: baselines.get(i),
|
|
253
|
+
errors: newErrors,
|
|
254
|
+
...(read.ok ? {} : { unresolved: read.error ?? 'expression did not resolve' }),
|
|
255
|
+
}));
|
|
256
|
+
}
|
|
257
|
+
if (opts.evidencePath && driver.capture)
|
|
258
|
+
await driver.capture(opts.evidencePath);
|
|
259
|
+
return {
|
|
260
|
+
...base,
|
|
261
|
+
passed: assertions.every((r) => r.passed),
|
|
262
|
+
assertions,
|
|
263
|
+
errors: newErrors,
|
|
264
|
+
durationMs: Date.now() - started,
|
|
265
|
+
};
|
|
266
|
+
}
|
|
267
|
+
catch (e) {
|
|
268
|
+
return {
|
|
269
|
+
...base,
|
|
270
|
+
passed: false,
|
|
271
|
+
assertions,
|
|
272
|
+
errors: [`probe execution failed: ${String(e).slice(0, 300)}`],
|
|
273
|
+
durationMs: Date.now() - started,
|
|
274
|
+
};
|
|
275
|
+
}
|
|
276
|
+
}
|
|
277
|
+
//# sourceMappingURL=runner.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"runner.js","sourceRoot":"","sources":["../../../src/delivery/interaction/runner.ts"],"names":[],"mappings":"AAAA;;;;;;;GAOG;AAEH,OAAO,EAAE,YAAY,EAAE,MAAM,eAAe,CAAC;AAE7C,OAAO,EAAE,KAAK,EAAE,MAAM,aAAa,CAAC;AAGpC,MAAM,CAAC,MAAM,wBAAwB,GAAG,OAAO,CAAC;AAEhD,wEAAwE;AACxE,MAAM,CAAC,MAAM,UAAU,GAAG,MAAM,CAAC;AAEjC;;;;;;;GAOG;AACH,MAAM,CAAC,KAAK,UAAU,YAAY,CAAC,MAAyB,EAAE,IAAU;IACtE,IAAI,IAAI,CAAC,EAAE,KAAK,QAAQ,EAAE,CAAC;QACzB,MAAM,KAAK,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,KAAK,EAAE,UAAU,CAAC,CAAC,CAAC;QAC5D,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,KAAK,EAAE,CAAC,EAAE,EAAE,CAAC;YAC/B,KAAK,MAAM,KAAK,IAAI,IAAI,CAAC,KAAK;gBAAE,MAAM,YAAY,CAAC,MAAM,EAAE,KAAK,CAAC,CAAC;QACpE,CAAC;QACD,OAAO;IACT,CAAC;IACD,MAAM,MAAM,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC;AAC7B,CAAC;AAED,SAAS,KAAK,CAAC,CAAY,EAAE,KAAa;IACxC,IAAI,OAAO,IAAI,CAAC,IAAI,CAAC,CAAC,KAAK;QAAE,OAAO,CAAC,CAAC,KAAK,CAAC;IAC5C,QAAQ,CAAC,CAAC,MAAM,EAAE,CAAC;QACjB,KAAK,UAAU;YACb,OAAO,mBAAmB,CAAC;QAC7B,KAAK,QAAQ;YACX,OAAO,GAAG,CAAC,CAAC,IAAI,YAAY,CAAC;QAC/B,KAAK,QAAQ;YACX,OAAO,GAAG,CAAC,CAAC,IAAI,QAAQ,IAAI,CAAC,SAAS,CAAC,CAAC,CAAC,KAAK,CAAC,EAAE,CAAC;QACpD,KAAK,KAAK;YACR,OAAO,GAAG,CAAC,CAAC,IAAI,OAAO,CAAC,CAAC,KAAK,EAAE,CAAC;QACnC,KAAK,KAAK;YACR,OAAO,GAAG,CAAC,CAAC,IAAI,OAAO,CAAC,CAAC,KAAK,EAAE,CAAC;QACnC,KAAK,WAAW;YACd,OAAO,GAAG,CAAC,CAAC,IAAI,aAAa,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,UAAU,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC;QAC9D,KAAK,SAAS;YACZ,OAAO,GAAG,CAAC,CAAC,IAAI,UAAU,CAAC;QAC7B;YACE,OAAO,cAAc,KAAK,GAAG,CAAC,EAAE,CAAC;IACrC,CAAC;AACH,CAAC;AAED,SAAS,IAAI,CAAC,CAAU;IACtB,IAAI,CAAC,KAAK,SAAS;QAAE,OAAO,WAAW,CAAC;IACxC,IAAI,OAAO,CAAC,KAAK,QAAQ,IAAI,OAAO,CAAC,KAAK,SAAS,IAAI,CAAC,KAAK,IAAI;QAAE,OAAO,MAAM,CAAC,CAAC,CAAC,CAAC;IACpF,IAAI,CAAC;QACH,MAAM,CAAC,GAAG,IAAI,CAAC,SAAS,CAAC,CAAC,CAAC,CAAC;QAC5B,OAAO,CAAC,CAAC,MAAM,GAAG,GAAG,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,KAAK,CAAC,CAAC,EAAE,GAAG,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC;IACtD,CAAC;IAAC,MAAM,CAAC;QACP,OAAO,MAAM,CAAC,CAAC,CAAC,CAAC;IACnB,CAAC;AACH,CAAC;AAED,2EAA2E;AAC3E,MAAM,UAAU,aAAa,CAAC,CAAY;IACxC,OAAO,CAAC,CAAC,MAAM,KAAK,WAAW,IAAI,CAAC,CAAC,MAAM,KAAK,SAAS,CAAC;AAC5D,CAAC;AAED;;;GAGG;AACH,MAAM,UAAU,cAAc,CAC5B,CAAY,EACZ,KAAa,EACb,QAMC;IAED,MAAM,IAAI,GAAG,KAAK,CAAC,CAAC,EAAE,KAAK,CAAC,CAAC;IAC7B,2EAA2E;IAC3E,8EAA8E;IAC9E,2CAA2C;IAC3C,IAAI,QAAQ,CAAC,UAAU,IAAI,CAAC,CAAC,MAAM,KAAK,UAAU,EAAE,CAAC;QACnD,OAAO;YACL,KAAK,EAAE,IAAI;YACX,MAAM,EAAE,KAAK;YACb,UAAU,EAAE,IAAI;YAChB,QAAQ,EAAE,oCAAoC;YAC9C,QAAQ,EAAE,QAAQ,CAAC,UAAU;SAC9B,CAAC;IACJ,CAAC;IACD,QAAQ,CAAC,CAAC,MAAM,EAAE,CAAC;QACjB,KAAK,UAAU,CAAC,CAAC,CAAC;YAChB,MAAM,IAAI,GAAG,QAAQ,CAAC,MAAM,IAAI,EAAE,CAAC;YACnC,OAAO;gBACL,KAAK,EAAE,IAAI;gBACX,MAAM,EAAE,IAAI,CAAC,MAAM,KAAK,CAAC;gBACzB,QAAQ,EAAE,mBAAmB;gBAC7B,QAAQ,EAAE,IAAI,CAAC,MAAM,KAAK,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC;aACpE,CAAC;QACJ,CAAC;QACD,KAAK,QAAQ;YACX,OAAO;gBACL,KAAK,EAAE,IAAI;gBACX,MAAM,EAAE,OAAO,CAAC,QAAQ,CAAC,KAAK,CAAC;gBAC/B,QAAQ,EAAE,QAAQ;gBAClB,QAAQ,EAAE,IAAI,CAAC,QAAQ,CAAC,KAAK,CAAC;aAC/B,CAAC;QACJ,KAAK,QAAQ;YACX,OAAO;gBACL,KAAK,EAAE,IAAI;gBACX,MAAM,EAAE,IAAI,CAAC,SAAS,CAAC,QAAQ,CAAC,KAAK,CAAC,KAAK,IAAI,CAAC,SAAS,CAAC,CAAC,CAAC,KAAK,CAAC;gBAClE,QAAQ,EAAE,IAAI,CAAC,CAAC,CAAC,KAAK,CAAC;gBACvB,QAAQ,EAAE,IAAI,CAAC,QAAQ,CAAC,KAAK,CAAC;aAC/B,CAAC;QACJ,KAAK,KAAK,CAAC,CAAC,CAAC;YACX,MAAM,CAAC,GAAG,MAAM,CAAC,QAAQ,CAAC,KAAK,CAAC,CAAC;YACjC,OAAO;gBACL,KAAK,EAAE,IAAI;gBACX,MAAM,EAAE,MAAM,CAAC,QAAQ,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,KAAK;gBAC1C,QAAQ,EAAE,MAAM,CAAC,CAAC,KAAK,EAAE;gBACzB,QAAQ,EAAE,IAAI,CAAC,QAAQ,CAAC,KAAK,CAAC;aAC/B,CAAC;QACJ,CAAC;QACD,KAAK,KAAK,CAAC,CAAC,CAAC;YACX,MAAM,CAAC,GAAG,MAAM,CAAC,QAAQ,CAAC,KAAK,CAAC,CAAC;YACjC,OAAO;gBACL,KAAK,EAAE,IAAI;gBACX,MAAM,EAAE,MAAM,CAAC,QAAQ,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,KAAK;gBAC1C,QAAQ,EAAE,MAAM,CAAC,CAAC,KAAK,EAAE;gBACzB,QAAQ,EAAE,IAAI,CAAC,QAAQ,CAAC,KAAK,CAAC;aAC/B,CAAC;QACJ,CAAC;QACD,KAAK,WAAW,CAAC,CAAC,CAAC;YACjB,MAAM,KAAK,GAAG,MAAM,CAAC,QAAQ,CAAC,QAAQ,CAAC,CAAC;YACxC,MAAM,GAAG,GAAG,MAAM,CAAC,QAAQ,CAAC,KAAK,CAAC,CAAC;YACnC,MAAM,EAAE,GAAG,CAAC,CAAC,EAAE,IAAI,CAAC,CAAC;YACrB,MAAM,EAAE,GAAG,MAAM,CAAC,QAAQ,CAAC,KAAK,CAAC,IAAI,MAAM,CAAC,QAAQ,CAAC,GAAG,CAAC,IAAI,GAAG,GAAG,KAAK,IAAI,EAAE,CAAC;YAC/E,OAAO;gBACL,KAAK,EAAE,IAAI;gBACX,MAAM,EAAE,EAAE;gBACV,QAAQ,EAAE,cAAc,EAAE,WAAW,CAAC,CAAC,MAAM,IAAI;gBACjD,QAAQ,EAAE,GAAG,IAAI,CAAC,QAAQ,CAAC,QAAQ,CAAC,MAAM,IAAI,CAAC,QAAQ,CAAC,KAAK,CAAC,EAAE;aACjE,CAAC;QACJ,CAAC;QACD,KAAK,SAAS,CAAC,CAAC,CAAC;YACf,MAAM,EAAE,GAAG,IAAI,CAAC,SAAS,CAAC,QAAQ,CAAC,QAAQ,CAAC,KAAK,IAAI,CAAC,SAAS,CAAC,QAAQ,CAAC,KAAK,CAAC,CAAC;YAChF,OAAO;gBACL,KAAK,EAAE,IAAI;gBACX,MAAM,EAAE,EAAE;gBACV,QAAQ,EAAE,0BAA0B,CAAC,CAAC,MAAM,IAAI;gBAChD,QAAQ,EAAE,GAAG,IAAI,CAAC,QAAQ,CAAC,QAAQ,CAAC,MAAM,IAAI,CAAC,QAAQ,CAAC,KAAK,CAAC,EAAE;aACjE,CAAC;QACJ,CAAC;QACD;YACE,OAAO;gBACL,KAAK,EAAE,IAAI;gBACX,MAAM,EAAE,KAAK;gBACb,QAAQ,EAAE,wBAAwB;gBAClC,QAAQ,EAAE,gBAAgB,MAAM,CAAE,CAAyB,CAAC,MAAM,CAAC,EAAE;aACtE,CAAC;IACN,CAAC;AACH,CAAC;AAQD,iFAAiF;AACjF,MAAM,CAAC,MAAM,oBAAoB,GAAG,gCAAgC,CAAC;AAErE,MAAM,YAAa,SAAQ,KAAK;CAAG;AAEnC;;;;;;GAMG;AACH,KAAK,UAAU,WAAW,CAAI,IAAgB,EAAE,EAAU;IACxD,IAAI,KAAiC,CAAC;IACtC,IAAI,CAAC;QACH,OAAO,MAAM,OAAO,CAAC,IAAI,CAAC;YACxB,IAAI;YACJ,IAAI,OAAO,CAAQ,CAAC,CAAC,EAAE,MAAM,EAAE,EAAE;gBAC/B,KAAK,GAAG,UAAU,CAAC,GAAG,EAAE,CAAC,MAAM,CAAC,IAAI,YAAY,CAAC,oBAAoB,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC;gBAC7E,8DAA8D;gBAC9D,KAAK,CAAC,KAAK,EAAE,EAAE,CAAC;YAClB,CAAC,CAAC;SACH,CAAC,CAAC;IACL,CAAC;YAAS,CAAC;QACT,IAAI,KAAK;YAAE,YAAY,CAAC,KAAK,CAAC,CAAC;IACjC,CAAC;AACH,CAAC;AAED;;;;GAIG;AACH,MAAM,CAAC,KAAK,UAAU,QAAQ,CAC5B,MAAyB,EACzB,KAAY,EACZ,OAAwB,EAAE;IAE1B,MAAM,OAAO,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;IAC3B,MAAM,IAAI,GAAyE;QACjF,OAAO,EAAE,KAAK,CAAC,EAAE;QACjB,WAAW,EAAE,KAAK,CAAC,WAAW;QAC9B,IAAI,EAAE,KAAK,CAAC,IAAI;QAChB,cAAc,EAAE,KAAK,CAAC,cAAc,IAAI,EAAE;KAC3C,CAAC;IAEF,6EAA6E;IAC7E,yDAAyD;IACzD,IAAI,KAAK,CAAC,IAAI,KAAK,aAAa,IAAI,YAAY,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,EAAE,KAAK,QAAQ,CAAC,EAAE,CAAC;QAC7F,OAAO;YACL,GAAG,IAAI;YACP,MAAM,EAAE,KAAK;YACb,UAAU,EAAE;gBACV;oBACE,KAAK,EAAE,iBAAiB;oBACxB,MAAM,EAAE,KAAK;oBACb,QAAQ,EAAE,+CAA+C;oBACzD,QAAQ,EAAE,yCAAyC;iBACpD;aACF;YACD,MAAM,EAAE,EAAE;YACV,UAAU,EAAE,IAAI,CAAC,GAAG,EAAE,GAAG,OAAO;SACjC,CAAC;IACJ,CAAC;IAED,MAAM,YAAY,GAAG,IAAI,GAAG,CAAC,MAAM,CAAC,MAAM,EAAE,CAAC,CAAC;IAC9C,MAAM,SAAS,GAAG,IAAI,GAAG,EAAmB,CAAC;IAC7C,MAAM,UAAU,GAAsB,EAAE,CAAC;IAEzC,0EAA0E;IAC1E,sEAAsE;IACtE,iCAAiC;IACjC,MAAM,YAAY,GAAG,KAAK,EAAE,IAAY,EAAuB,EAAE,CAC/D,MAAM,CAAC,YAAY,CAAC,CAAC,CAAC,MAAM,CAAC,YAAY,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,EAAE,EAAE,EAAE,IAAI,EAAE,KAAK,EAAE,MAAM,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC;IAEjG,IAAI,CAAC;QACH,2CAA2C;QAC3C,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,KAAK,CAAC,OAAO,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;YAC9C,MAAM,CAAC,GAAG,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC;YAC3B,IAAI,aAAa,CAAC,CAAC,CAAC;gBAAE,SAAS,CAAC,GAAG,CAAC,CAAC,EAAE,CAAC,MAAM,YAAY,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC;QAC7E,CAAC;QAED,MAAM,MAAM,GAAG,KAAK,CAAC,SAAS,IAAI,IAAI,CAAC,SAAS,IAAI,wBAAwB,CAAC;QAC7E,MAAM,WAAW,CACf,CAAC,KAAK,IAAI,EAAE;YACV,KAAK,MAAM,IAAI,IAAI,KAAK,CAAC,KAAK;gBAAE,MAAM,YAAY,CAAC,MAAM,EAAE,IAAI,CAAC,CAAC;QACnE,CAAC,CAAC,EAAE,EACJ,MAAM,CACP,CAAC;QAEF,4EAA4E;QAC5E,mBAAmB;QACnB,MAAM,aAAa,GAAG,KAAK,CAAC,OAAO,CAAC,MAAM,CACxC,CAAC,GAAG,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,aAAa,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,GAAG,EAAE,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,EAC9D,CAAC,CACF,CAAC;QACF,MAAM,OAAO,GAAG,IAAI,CAAC,GAAG,EAAE,GAAG,OAAO,CAAC;QACrC,IAAI,aAAa,GAAG,OAAO;YAAE,MAAM,KAAK,CAAC,aAAa,GAAG,OAAO,CAAC,CAAC;QAElE,MAAM,SAAS,GAAG,MAAM,CAAC,MAAM,EAAE,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,YAAY,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC;QACtE,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,KAAK,CAAC,OAAO,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;YAC9C,MAAM,CAAC,GAAG,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC;YAC3B,MAAM,IAAI,GAAG,CAAC,CAAC,MAAM,KAAK,UAAU,CAAC,CAAC,CAAC,EAAE,EAAE,EAAE,IAAI,EAAE,CAAC,CAAC,CAAC,MAAM,YAAY,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC;YACjF,UAAU,CAAC,IAAI,CACb,cAAc,CAAC,CAAC,EAAE,CAAC,EAAE;gBACnB,KAAK,EAAE,IAAI,CAAC,KAAK;gBACjB,QAAQ,EAAE,SAAS,CAAC,GAAG,CAAC,CAAC,CAAC;gBAC1B,MAAM,EAAE,SAAS;gBACjB,GAAG,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,UAAU,EAAE,IAAI,CAAC,KAAK,IAAI,4BAA4B,EAAE,CAAC;aAC/E,CAAC,CACH,CAAC;QACJ,CAAC;QAED,IAAI,IAAI,CAAC,YAAY,IAAI,MAAM,CAAC,OAAO;YAAE,MAAM,MAAM,CAAC,OAAO,CAAC,IAAI,CAAC,YAAY,CAAC,CAAC;QAEjF,OAAO;YACL,GAAG,IAAI;YACP,MAAM,EAAE,UAAU,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,MAAM,CAAC;YACzC,UAAU;YACV,MAAM,EAAE,SAAS;YACjB,UAAU,EAAE,IAAI,CAAC,GAAG,EAAE,GAAG,OAAO;SACjC,CAAC;IACJ,CAAC;IAAC,OAAO,CAAC,EAAE,CAAC;QACX,OAAO;YACL,GAAG,IAAI;YACP,MAAM,EAAE,KAAK;YACb,UAAU;YACV,MAAM,EAAE,CAAC,2BAA2B,MAAM,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,EAAE,GAAG,CAAC,EAAE,CAAC;YAC9D,UAAU,EAAE,IAAI,CAAC,GAAG,EAAE,GAAG,OAAO;SACjC,CAAC;IACJ,CAAC;AACH,CAAC"}
|