@itsthelore/proofkeeper 2026.6.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +201 -0
- package/NOTICE +10 -0
- package/README.md +207 -0
- package/dist/agent/adapters/claude.d.ts +93 -0
- package/dist/agent/adapters/claude.d.ts.map +1 -0
- package/dist/agent/adapters/claude.js +96 -0
- package/dist/agent/adapters/claude.js.map +1 -0
- package/dist/agent/drive.d.ts +53 -0
- package/dist/agent/drive.d.ts.map +1 -0
- package/dist/agent/drive.js +194 -0
- package/dist/agent/drive.js.map +1 -0
- package/dist/agent/loop.d.ts +40 -0
- package/dist/agent/loop.d.ts.map +1 -0
- package/dist/agent/loop.js +29 -0
- package/dist/agent/loop.js.map +1 -0
- package/dist/agent/model.d.ts +43 -0
- package/dist/agent/model.d.ts.map +1 -0
- package/dist/agent/model.js +10 -0
- package/dist/agent/model.js.map +1 -0
- package/dist/agent/observe.d.ts +48 -0
- package/dist/agent/observe.d.ts.map +1 -0
- package/dist/agent/observe.js +65 -0
- package/dist/agent/observe.js.map +1 -0
- package/dist/agent/tools.d.ts +74 -0
- package/dist/agent/tools.d.ts.map +1 -0
- package/dist/agent/tools.js +257 -0
- package/dist/agent/tools.js.map +1 -0
- package/dist/cli.d.ts +61 -0
- package/dist/cli.d.ts.map +1 -0
- package/dist/cli.js +648 -0
- package/dist/cli.js.map +1 -0
- package/dist/compiler/actions.d.ts +101 -0
- package/dist/compiler/actions.d.ts.map +1 -0
- package/dist/compiler/actions.js +13 -0
- package/dist/compiler/actions.js.map +1 -0
- package/dist/compiler/compiler.d.ts +25 -0
- package/dist/compiler/compiler.d.ts.map +1 -0
- package/dist/compiler/compiler.js +42 -0
- package/dist/compiler/compiler.js.map +1 -0
- package/dist/compiler/emit.d.ts +21 -0
- package/dist/compiler/emit.d.ts.map +1 -0
- package/dist/compiler/emit.js +164 -0
- package/dist/compiler/emit.js.map +1 -0
- package/dist/compiler/http.d.ts +30 -0
- package/dist/compiler/http.d.ts.map +1 -0
- package/dist/compiler/http.js +30 -0
- package/dist/compiler/http.js.map +1 -0
- package/dist/compiler/recorder.d.ts +62 -0
- package/dist/compiler/recorder.d.ts.map +1 -0
- package/dist/compiler/recorder.js +148 -0
- package/dist/compiler/recorder.js.map +1 -0
- package/dist/compiler/summary.d.ts +11 -0
- package/dist/compiler/summary.d.ts.map +1 -0
- package/dist/compiler/summary.js +56 -0
- package/dist/compiler/summary.js.map +1 -0
- package/dist/compiler/terminal.d.ts +42 -0
- package/dist/compiler/terminal.d.ts.map +1 -0
- package/dist/compiler/terminal.js +47 -0
- package/dist/compiler/terminal.js.map +1 -0
- package/dist/compiler/types.d.ts +25 -0
- package/dist/compiler/types.d.ts.map +1 -0
- package/dist/compiler/types.js +10 -0
- package/dist/compiler/types.js.map +1 -0
- package/dist/coverage/graph.d.ts +55 -0
- package/dist/coverage/graph.d.ts.map +1 -0
- package/dist/coverage/graph.js +87 -0
- package/dist/coverage/graph.js.map +1 -0
- package/dist/coverage/model.d.ts +36 -0
- package/dist/coverage/model.d.ts.map +1 -0
- package/dist/coverage/model.js +57 -0
- package/dist/coverage/model.js.map +1 -0
- package/dist/coverage/report.d.ts +27 -0
- package/dist/coverage/report.d.ts.map +1 -0
- package/dist/coverage/report.js +45 -0
- package/dist/coverage/report.js.map +1 -0
- package/dist/coverage/source.d.ts +23 -0
- package/dist/coverage/source.d.ts.map +1 -0
- package/dist/coverage/source.js +48 -0
- package/dist/coverage/source.js.map +1 -0
- package/dist/fidelity/gate.d.ts +34 -0
- package/dist/fidelity/gate.d.ts.map +1 -0
- package/dist/fidelity/gate.js +38 -0
- package/dist/fidelity/gate.js.map +1 -0
- package/dist/index.d.ts +69 -0
- package/dist/index.d.ts.map +1 -0
- package/dist/index.js +49 -0
- package/dist/index.js.map +1 -0
- package/dist/learning/store.d.ts +44 -0
- package/dist/learning/store.d.ts.map +1 -0
- package/dist/learning/store.js +64 -0
- package/dist/learning/store.js.map +1 -0
- package/dist/qa/concurrency.d.ts +7 -0
- package/dist/qa/concurrency.d.ts.map +1 -0
- package/dist/qa/concurrency.js +21 -0
- package/dist/qa/concurrency.js.map +1 -0
- package/dist/qa/run-qa.d.ts +87 -0
- package/dist/qa/run-qa.d.ts.map +1 -0
- package/dist/qa/run-qa.js +106 -0
- package/dist/qa/run-qa.js.map +1 -0
- package/dist/qa/run-scoped.d.ts +82 -0
- package/dist/qa/run-scoped.d.ts.map +1 -0
- package/dist/qa/run-scoped.js +96 -0
- package/dist/qa/run-scoped.js.map +1 -0
- package/dist/runner/playwright-report.d.ts +52 -0
- package/dist/runner/playwright-report.d.ts.map +1 -0
- package/dist/runner/playwright-report.js +90 -0
- package/dist/runner/playwright-report.js.map +1 -0
- package/dist/runner/playwright-runner.d.ts +38 -0
- package/dist/runner/playwright-runner.d.ts.map +1 -0
- package/dist/runner/playwright-runner.js +73 -0
- package/dist/runner/playwright-runner.js.map +1 -0
- package/dist/runner/types.d.ts +45 -0
- package/dist/runner/types.d.ts.map +1 -0
- package/dist/runner/types.js +10 -0
- package/dist/runner/types.js.map +1 -0
- package/dist/scaffold/scaffold.d.ts +22 -0
- package/dist/scaffold/scaffold.d.ts.map +1 -0
- package/dist/scaffold/scaffold.js +34 -0
- package/dist/scaffold/scaffold.js.map +1 -0
- package/dist/scope/config.d.ts +89 -0
- package/dist/scope/config.d.ts.map +1 -0
- package/dist/scope/config.js +172 -0
- package/dist/scope/config.js.map +1 -0
- package/dist/scope/diff-scope.d.ts +31 -0
- package/dist/scope/diff-scope.d.ts.map +1 -0
- package/dist/scope/diff-scope.js +42 -0
- package/dist/scope/diff-scope.js.map +1 -0
- package/dist/scope/glob.d.ts +17 -0
- package/dist/scope/glob.d.ts.map +1 -0
- package/dist/scope/glob.js +50 -0
- package/dist/scope/glob.js.map +1 -0
- package/dist/writeback/comment.d.ts +103 -0
- package/dist/writeback/comment.d.ts.map +1 -0
- package/dist/writeback/comment.js +150 -0
- package/dist/writeback/comment.js.map +1 -0
- package/dist/writeback/gateways/github-rest.d.ts +66 -0
- package/dist/writeback/gateways/github-rest.d.ts.map +1 -0
- package/dist/writeback/gateways/github-rest.js +107 -0
- package/dist/writeback/gateways/github-rest.js.map +1 -0
- package/dist/writeback/merge.d.ts +27 -0
- package/dist/writeback/merge.d.ts.map +1 -0
- package/dist/writeback/merge.js +89 -0
- package/dist/writeback/merge.js.map +1 -0
- package/dist/writeback/proposal.d.ts +52 -0
- package/dist/writeback/proposal.d.ts.map +1 -0
- package/dist/writeback/proposal.js +79 -0
- package/dist/writeback/proposal.js.map +1 -0
- package/dist/writeback/proposer.d.ts +94 -0
- package/dist/writeback/proposer.d.ts.map +1 -0
- package/dist/writeback/proposer.js +79 -0
- package/dist/writeback/proposer.js.map +1 -0
- package/dist/writeback/verified-by.d.ts +56 -0
- package/dist/writeback/verified-by.d.ts.map +1 -0
- package/dist/writeback/verified-by.js +60 -0
- package/dist/writeback/verified-by.js.map +1 -0
- package/package.json +62 -0
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"run-qa.d.ts","sourceRoot":"","sources":["../../src/qa/run-qa.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;GAcG;AAEH,OAAO,KAAK,EAAE,KAAK,EAAE,MAAM,sBAAsB,CAAC;AAClD,OAAO,EAAmB,KAAK,kBAAkB,EAAE,MAAM,sBAAsB,CAAC;AAChF,OAAO,KAAK,EAAE,QAAQ,EAAE,MAAM,sBAAsB,CAAC;AAErD,OAAO,KAAK,EAAE,MAAM,EAAE,SAAS,EAAE,MAAM,oBAAoB,CAAC;AAC5D,OAAO,EAAgB,KAAK,eAAe,EAAE,MAAM,kBAAkB,CAAC;AACtE,OAAO,KAAK,EAAE,YAAY,EAAE,WAAW,EAAE,MAAM,mBAAmB,CAAC;AAEnE,OAAO,KAAK,EAAE,iBAAiB,EAAE,eAAe,EAAE,MAAM,0BAA0B,CAAC;AACnF,OAAO,KAAK,EAAE,aAAa,EAAE,MAAM,sBAAsB,CAAC;AAE1D;;;;;;GAMG;AACH,wBAAgB,gBAAgB,CAAC,KAAK,EAAE,KAAK,EAAE,YAAY,CAAC,EAAE,MAAM,GAAG,kBAAkB,CAcxF;AAED,mFAAmF;AACnF,wBAAgB,WAAW,CAAC,UAAU,EAAE,kBAAkB,GAAG,MAAM,CAKlE;AAED,MAAM,WAAW,MAAM;IACrB;;;OAGG;IACH,KAAK,CAAC,OAAO,EAAE,YAAY,GAAG,OAAO,CAAC,WAAW,CAAC,CAAC;IACnD,QAAQ,EAAE,QAAQ,CAAC;IACnB,MAAM,EAAE,MAAM,CAAC;IACf,gFAAgF;IAChF,QAAQ,CAAC,EAAE,iBAAiB,CAAC;IAC7B,kFAAkF;IAClF,QAAQ,CAAC,EAAE,aAAa,CAAC;CAC1B;AAED,MAAM,WAAW,SAAS;IACxB,gEAAgE;IAChE,KAAK,EAAE,KAAK,CAAC;IACb,oEAAoE;IACpE,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,wDAAwD;IACxD,QAAQ,EAAE,MAAM,CAAC;IACjB,2DAA2D;IAC3D,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,6DAA6D;IAC7D,MAAM,EAAE,SAAS,CAAC;IAClB,yDAAyD;IACzD,CAAC,EAAE,MAAM,CAAC;IACV,2CAA2C;IAC3C,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,oEAAoE;IACpE,IAAI,CAAC,EAAE,OAAO,CAAC;IACf,gFAAgF;IAChF,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,4EAA4E;IAC5E,OAAO,CAAC,EAAE;QAAE,UAAU,EAAE,MAAM,CAAC;QAAC,UAAU,CAAC,EAAE,MAAM,CAAA;KAAE,CAAC;CACvD;AAED,MAAM,WAAW,QAAQ;IACvB,UAAU,EAAE,kBAAkB,CAAC;IAC/B,KAAK,EAAE,WAAW,CAAC;IACnB,IAAI,EAAE,eAAe,CAAC;IACtB,2DAA2D;IAC3D,QAAQ,EAAE,OAAO,CAAC;IAClB,kFAAkF;IAClF,SAAS,CAAC,EAAE,eAAe,CAAC;CAC7B;AAED;;;;GAIG;AACH,wBAAsB,KAAK,CAAC,IAAI,EAAE,MAAM,EAAE,OAAO,EAAE,SAAS,GAAG,OAAO,CAAC,QAAQ,CAAC,CA+D/E"}
|
|
@@ -0,0 +1,106 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* The QA loop behind one command — Proofkeeper's autonomous-QA spine.
|
|
3
|
+
*
|
|
4
|
+
* It wires the already-real pieces into a single autonomous pass: pick an
|
|
5
|
+
* unverified capability from the coverage read-model (Initiative 1), drive the
|
|
6
|
+
* product to record a session (Initiative 2), compile it and gate it on
|
|
7
|
+
* fidelity, then run the accepted test (Initiatives 3–4), and — only if it is
|
|
8
|
+
* stable and a proposer is wired — propose the `## Verified By` write-back as a
|
|
9
|
+
* human-reviewed pull request (Initiative 5).
|
|
10
|
+
*
|
|
11
|
+
* The drive is injected as a {@link QaDeps.drive} seam, not constructed here: a
|
|
12
|
+
* browser-backed driver supplies it at the CLI, a scripted double supplies it in
|
|
13
|
+
* tests. That keeps this orchestrator pure of any browser or model dependency
|
|
14
|
+
* and unit-testable end to end.
|
|
15
|
+
*/
|
|
16
|
+
import { computeCoverage } from "../coverage/model.js";
|
|
17
|
+
import { summarizeSession } from "../compiler/summary.js";
|
|
18
|
+
import { runAgentLoop } from "../agent/loop.js";
|
|
19
|
+
import { linksFromResults } from "../writeback/proposal.js";
|
|
20
|
+
/**
|
|
21
|
+
* Pick the capability to verify: the named one if given (verified or not — a
|
|
22
|
+
* re-verify is allowed), otherwise the first unverified capability.
|
|
23
|
+
*
|
|
24
|
+
* @throws when a named capability is absent, or when nothing is unverified and
|
|
25
|
+
* no capability was named.
|
|
26
|
+
*/
|
|
27
|
+
export function selectCapability(graph, capabilityId) {
|
|
28
|
+
const report = computeCoverage(graph);
|
|
29
|
+
if (capabilityId) {
|
|
30
|
+
const found = [...report.verified, ...report.unverified].find((c) => c.id === capabilityId);
|
|
31
|
+
if (!found) {
|
|
32
|
+
throw new Error(`capability '${capabilityId}' is not a requirement node in the graph`);
|
|
33
|
+
}
|
|
34
|
+
return found;
|
|
35
|
+
}
|
|
36
|
+
const next = report.unverified[0];
|
|
37
|
+
if (!next) {
|
|
38
|
+
throw new Error("every capability is already verified — nothing to drive");
|
|
39
|
+
}
|
|
40
|
+
return next;
|
|
41
|
+
}
|
|
42
|
+
/** A default goal derived from the capability, used when the caller gives none. */
|
|
43
|
+
export function defaultGoal(capability) {
|
|
44
|
+
return (`Drive the product to verify the capability "${capability.title}" (${capability.id}). ` +
|
|
45
|
+
"Exercise its primary end-to-end flow and assert every observable outcome.");
|
|
46
|
+
}
|
|
47
|
+
/**
|
|
48
|
+
* Run the full QA loop for one capability: select → drive → compile → fidelity →
|
|
49
|
+
* run → (optional) propose. Returns the outcome; never throws on an unstable
|
|
50
|
+
* test (that is a verdict, surfaced in {@link QaResult.verified}).
|
|
51
|
+
*/
|
|
52
|
+
export async function runQa(deps, options) {
|
|
53
|
+
const capability = selectCapability(options.graph, options.capabilityId);
|
|
54
|
+
const baseGoal = options.goal ?? defaultGoal(capability);
|
|
55
|
+
const goal = options.goalContext ? `${baseGoal} ${options.goalContext}` : baseGoal;
|
|
56
|
+
// Feed reasons from earlier failed attempts into the drive (failure-learning).
|
|
57
|
+
const prior = deps.learning ? await deps.learning.priorFailures(capability.id) : [];
|
|
58
|
+
const driveOptions = {
|
|
59
|
+
capabilityId: capability.id,
|
|
60
|
+
title: `verify ${capability.title}`,
|
|
61
|
+
startUrl: options.startUrl,
|
|
62
|
+
goal,
|
|
63
|
+
...(options.maxSteps !== undefined ? { maxSteps: options.maxSteps } : {}),
|
|
64
|
+
...(prior.length > 0 ? { priorFailures: prior.map((f) => f.reason) } : {}),
|
|
65
|
+
...(options.plan ? { plan: true } : {}),
|
|
66
|
+
};
|
|
67
|
+
const drive = await deps.drive(driveOptions);
|
|
68
|
+
const loop = await runAgentLoop({ compiler: deps.compiler, runner: deps.runner }, {
|
|
69
|
+
session: drive.session,
|
|
70
|
+
fidelity: { n: options.n, target: options.target },
|
|
71
|
+
runTargets: [options.target],
|
|
72
|
+
});
|
|
73
|
+
const verified = loop.verdict.stable;
|
|
74
|
+
const result = { capability, drive, loop, verified };
|
|
75
|
+
// Propose the write-back only for a stable test (ADR-065: a human reviews it).
|
|
76
|
+
// The proposal carries the readable step summary so a reviewer can read the
|
|
77
|
+
// driven flow without opening the trace.
|
|
78
|
+
if (verified && deps.proposer && options.propose) {
|
|
79
|
+
result.writeBack = await deps.proposer.propose({
|
|
80
|
+
capabilityId: capability.id,
|
|
81
|
+
targetPath: options.propose.targetPath,
|
|
82
|
+
links: linksFromResults(loop.candidate, loop.runResults),
|
|
83
|
+
steps: summarizeSession(drive.session),
|
|
84
|
+
...(drive.session.plan !== undefined ? { plan: drive.session.plan } : {}),
|
|
85
|
+
...(options.propose.baseBranch !== undefined ? { baseBranch: options.propose.baseBranch } : {}),
|
|
86
|
+
fidelity: {
|
|
87
|
+
attempts: loop.verdict.attempts,
|
|
88
|
+
passed: loop.verdict.passed,
|
|
89
|
+
stable: loop.verdict.stable,
|
|
90
|
+
},
|
|
91
|
+
});
|
|
92
|
+
}
|
|
93
|
+
// Remember a failure so the next attempt avoids it (failure-learning).
|
|
94
|
+
if (deps.learning && (!verified || !drive.finished)) {
|
|
95
|
+
await deps.learning.recordFailure({
|
|
96
|
+
capabilityId: capability.id,
|
|
97
|
+
goal,
|
|
98
|
+
reason: !drive.finished
|
|
99
|
+
? `drive did not finish within the step budget (${drive.steps} steps)`
|
|
100
|
+
: `compiled test was unstable: ${loop.verdict.passed}/${loop.verdict.attempts} re-runs green`,
|
|
101
|
+
steps: drive.steps,
|
|
102
|
+
});
|
|
103
|
+
}
|
|
104
|
+
return result;
|
|
105
|
+
}
|
|
106
|
+
//# sourceMappingURL=run-qa.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"run-qa.js","sourceRoot":"","sources":["../../src/qa/run-qa.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;GAcG;AAGH,OAAO,EAAE,eAAe,EAA2B,MAAM,sBAAsB,CAAC;AAEhF,OAAO,EAAE,gBAAgB,EAAE,MAAM,wBAAwB,CAAC;AAE1D,OAAO,EAAE,YAAY,EAAwB,MAAM,kBAAkB,CAAC;AAEtE,OAAO,EAAE,gBAAgB,EAAE,MAAM,0BAA0B,CAAC;AAI5D;;;;;;GAMG;AACH,MAAM,UAAU,gBAAgB,CAAC,KAAY,EAAE,YAAqB;IAClE,MAAM,MAAM,GAAG,eAAe,CAAC,KAAK,CAAC,CAAC;IACtC,IAAI,YAAY,EAAE,CAAC;QACjB,MAAM,KAAK,GAAG,CAAC,GAAG,MAAM,CAAC,QAAQ,EAAE,GAAG,MAAM,CAAC,UAAU,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,EAAE,KAAK,YAAY,CAAC,CAAC;QAC5F,IAAI,CAAC,KAAK,EAAE,CAAC;YACX,MAAM,IAAI,KAAK,CAAC,eAAe,YAAY,0CAA0C,CAAC,CAAC;QACzF,CAAC;QACD,OAAO,KAAK,CAAC;IACf,CAAC;IACD,MAAM,IAAI,GAAG,MAAM,CAAC,UAAU,CAAC,CAAC,CAAC,CAAC;IAClC,IAAI,CAAC,IAAI,EAAE,CAAC;QACV,MAAM,IAAI,KAAK,CAAC,yDAAyD,CAAC,CAAC;IAC7E,CAAC;IACD,OAAO,IAAI,CAAC;AACd,CAAC;AAED,mFAAmF;AACnF,MAAM,UAAU,WAAW,CAAC,UAA8B;IACxD,OAAO,CACL,+CAA+C,UAAU,CAAC,KAAK,MAAM,UAAU,CAAC,EAAE,KAAK;QACvF,2EAA2E,CAC5E,CAAC;AACJ,CAAC;AAiDD;;;;GAIG;AACH,MAAM,CAAC,KAAK,UAAU,KAAK,CAAC,IAAY,EAAE,OAAkB;IAC1D,MAAM,UAAU,GAAG,gBAAgB,CAAC,OAAO,CAAC,KAAK,EAAE,OAAO,CAAC,YAAY,CAAC,CAAC;IACzE,MAAM,QAAQ,GAAG,OAAO,CAAC,IAAI,IAAI,WAAW,CAAC,UAAU,CAAC,CAAC;IACzD,MAAM,IAAI,GAAG,OAAO,CAAC,WAAW,CAAC,CAAC,CAAC,GAAG,QAAQ,IAAI,OAAO,CAAC,WAAW,EAAE,CAAC,CAAC,CAAC,QAAQ,CAAC;IAEnF,+EAA+E;IAC/E,MAAM,KAAK,GAAG,IAAI,CAAC,QAAQ,CAAC,CAAC,CAAC,MAAM,IAAI,CAAC,QAAQ,CAAC,aAAa,CAAC,UAAU,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC;IAEpF,MAAM,YAAY,GAAiB;QACjC,YAAY,EAAE,UAAU,CAAC,EAAE;QAC3B,KAAK,EAAE,UAAU,UAAU,CAAC,KAAK,EAAE;QACnC,QAAQ,EAAE,OAAO,CAAC,QAAQ;QAC1B,IAAI;QACJ,GAAG,CAAC,OAAO,CAAC,QAAQ,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,QAAQ,EAAE,OAAO,CAAC,QAAQ,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;QACzE,GAAG,CAAC,KAAK,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,EAAE,aAAa,EAAE,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,MAAM,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;QAC1E,GAAG,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,IAAI,EAAE,IAAI,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;KACxC,CAAC;IACF,MAAM,KAAK,GAAG,MAAM,IAAI,CAAC,KAAK,CAAC,YAAY,CAAC,CAAC;IAE7C,MAAM,IAAI,GAAG,MAAM,YAAY,CAC7B,EAAE,QAAQ,EAAE,IAAI,CAAC,QAAQ,EAAE,MAAM,EAAE,IAAI,CAAC,MAAM,EAAE,EAChD;QACE,OAAO,EAAE,KAAK,CAAC,OAAO;QACtB,QAAQ,EAAE,EAAE,CAAC,EAAE,OAAO,CAAC,CAAC,EAAE,MAAM,EAAE,OAAO,CAAC,MAAM,EAAE;QAClD,UAAU,EAAE,CAAC,OAAO,CAAC,MAAM,CAAC;KAC7B,CACF,CAAC;IAEF,MAAM,QAAQ,GAAG,IAAI,CAAC,OAAO,CAAC,MAAM,CAAC;IACrC,MAAM,MAAM,GAAa,EAAE,UAAU,EAAE,KAAK,EAAE,IAAI,EAAE,QAAQ,EAAE,CAAC;IAE/D,+EAA+E;IAC/E,4EAA4E;IAC5E,yCAAyC;IACzC,IAAI,QAAQ,IAAI,IAAI,CAAC,QAAQ,IAAI,OAAO,CAAC,OAAO,EAAE,CAAC;QACjD,MAAM,CAAC,SAAS,GAAG,MAAM,IAAI,CAAC,QAAQ,CAAC,OAAO,CAAC;YAC7C,YAAY,EAAE,UAAU,CAAC,EAAE;YAC3B,UAAU,EAAE,OAAO,CAAC,OAAO,CAAC,UAAU;YACtC,KAAK,EAAE,gBAAgB,CAAC,IAAI,CAAC,SAAS,EAAE,IAAI,CAAC,UAAU,CAAC;YACxD,KAAK,EAAE,gBAAgB,CAAC,KAAK,CAAC,OAAO,CAAC;YACtC,GAAG,CAAC,KAAK,CAAC,OAAO,CAAC,IAAI,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,IAAI,EAAE,KAAK,CAAC,OAAO,CAAC,IAAI,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;YACzE,GAAG,CAAC,OAAO,CAAC,OAAO,CAAC,UAAU,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,UAAU,EAAE,OAAO,CAAC,OAAO,CAAC,UAAU,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;YAC/F,QAAQ,EAAE;gBACR,QAAQ,EAAE,IAAI,CAAC,OAAO,CAAC,QAAQ;gBAC/B,MAAM,EAAE,IAAI,CAAC,OAAO,CAAC,MAAM;gBAC3B,MAAM,EAAE,IAAI,CAAC,OAAO,CAAC,MAAM;aAC5B;SACF,CAAC,CAAC;IACL,CAAC;IAED,uEAAuE;IACvE,IAAI,IAAI,CAAC,QAAQ,IAAI,CAAC,CAAC,QAAQ,IAAI,CAAC,KAAK,CAAC,QAAQ,CAAC,EAAE,CAAC;QACpD,MAAM,IAAI,CAAC,QAAQ,CAAC,aAAa,CAAC;YAChC,YAAY,EAAE,UAAU,CAAC,EAAE;YAC3B,IAAI;YACJ,MAAM,EAAE,CAAC,KAAK,CAAC,QAAQ;gBACrB,CAAC,CAAC,gDAAgD,KAAK,CAAC,KAAK,SAAS;gBACtE,CAAC,CAAC,+BAA+B,IAAI,CAAC,OAAO,CAAC,MAAM,IAAI,IAAI,CAAC,OAAO,CAAC,QAAQ,gBAAgB;YAC/F,KAAK,EAAE,KAAK,CAAC,KAAK;SACnB,CAAC,CAAC;IACL,CAAC;IAED,OAAO,MAAM,CAAC;AAChB,CAAC"}
|
|
@@ -0,0 +1,82 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* PR-triggered QA — drive every unverified capability a change touched.
|
|
3
|
+
*
|
|
4
|
+
* Built on the Phase-1 {@link runQa} loop: scope the change to capabilities
|
|
5
|
+
* ({@link scopeCapabilities}), then drive each one that still lacks a verifying
|
|
6
|
+
* test, proposing a write-back when the capability declares its corpus artifact.
|
|
7
|
+
* Capabilities are driven **concurrently** with a bounded pool, each isolated in
|
|
8
|
+
* its own browser context, compiler output directory, and runner output
|
|
9
|
+
* directory (Ranger's context-isolated sub-agent lesson). Results stay in scoped
|
|
10
|
+
* order. The caller posts the summary to the feature pull request.
|
|
11
|
+
*/
|
|
12
|
+
import { type QaDeps, type QaResult } from "./run-qa.js";
|
|
13
|
+
import type { Graph } from "../coverage/graph.js";
|
|
14
|
+
import type { Compiler } from "../compiler/types.js";
|
|
15
|
+
import type { Runner } from "../runner/types.js";
|
|
16
|
+
import type { WriteBackProposer } from "../writeback/proposer.js";
|
|
17
|
+
import type { LearningStore } from "../learning/store.js";
|
|
18
|
+
import { type ProofkeeperConfig } from "../scope/config.js";
|
|
19
|
+
import { type ScopedCapability, type ScopeResult } from "../scope/diff-scope.js";
|
|
20
|
+
/** Default capabilities driven at once — conservative to bound browser/runner load. */
|
|
21
|
+
export declare const DEFAULT_SCOPED_CONCURRENCY = 3;
|
|
22
|
+
/**
|
|
23
|
+
* Dependencies for scoped QA. The compiler and runner are minted **per
|
|
24
|
+
* capability** so concurrent drives write to isolated directories; the drive
|
|
25
|
+
* seam already isolates the browser context per call.
|
|
26
|
+
*/
|
|
27
|
+
export interface ScopedQaDeps {
|
|
28
|
+
drive: QaDeps["drive"];
|
|
29
|
+
/** Mint a compiler whose output directory is isolated to this capability. */
|
|
30
|
+
makeCompiler(capabilityId: string): Compiler;
|
|
31
|
+
/** Mint a runner whose output directory is isolated to this capability. */
|
|
32
|
+
makeRunner(capabilityId: string): Runner;
|
|
33
|
+
proposer?: WriteBackProposer;
|
|
34
|
+
learning?: LearningStore;
|
|
35
|
+
}
|
|
36
|
+
export interface ScopedQaOptions {
|
|
37
|
+
graph: Graph;
|
|
38
|
+
config: ProofkeeperConfig;
|
|
39
|
+
/** The files the pull request changed. */
|
|
40
|
+
changedPaths: string[];
|
|
41
|
+
/** Run-target name; each capability runs against its own start URL. */
|
|
42
|
+
targetName: string;
|
|
43
|
+
/** Start URL used when a capability config declares none. */
|
|
44
|
+
defaultUrl?: string;
|
|
45
|
+
/** Fidelity re-run count. */
|
|
46
|
+
n: number;
|
|
47
|
+
maxSteps?: number;
|
|
48
|
+
/** When set, the drive emits a Markdown test plan before acting. */
|
|
49
|
+
plan?: boolean;
|
|
50
|
+
/** Max capabilities driven at once. Defaults to {@link DEFAULT_SCOPED_CONCURRENCY}. */
|
|
51
|
+
concurrency?: number;
|
|
52
|
+
/** When set, propose a write-back for capabilities that declare an `artifact`. */
|
|
53
|
+
propose?: {
|
|
54
|
+
baseBranch?: string;
|
|
55
|
+
};
|
|
56
|
+
}
|
|
57
|
+
export interface ScopedCapabilityResult {
|
|
58
|
+
capability: ScopedCapability;
|
|
59
|
+
/** Present when the capability was driven. */
|
|
60
|
+
result?: QaResult;
|
|
61
|
+
/** Present when the capability could not be driven (e.g. no start URL). */
|
|
62
|
+
error?: string;
|
|
63
|
+
}
|
|
64
|
+
export interface ScopedQaResult {
|
|
65
|
+
scope: ScopeResult;
|
|
66
|
+
/** One entry per unverified scoped capability, in scoped order. */
|
|
67
|
+
driven: ScopedCapabilityResult[];
|
|
68
|
+
}
|
|
69
|
+
/** A capability's recorded failure reasons, for the suggest-in-report strategy. */
|
|
70
|
+
export interface FailureSuggestion {
|
|
71
|
+
id: string;
|
|
72
|
+
title: string;
|
|
73
|
+
reasons: string[];
|
|
74
|
+
}
|
|
75
|
+
/**
|
|
76
|
+
* Collect recorded failure reasons for each driven capability that failed or
|
|
77
|
+
* could not be driven, from the learning store — the `suggest_in_report`
|
|
78
|
+
* failure-learning strategy's source. Pure over the store (no drive).
|
|
79
|
+
*/
|
|
80
|
+
export declare function collectFailureSuggestions(result: ScopedQaResult, learning: LearningStore): Promise<FailureSuggestion[]>;
|
|
81
|
+
export declare function runScopedQa(deps: ScopedQaDeps, options: ScopedQaOptions): Promise<ScopedQaResult>;
|
|
82
|
+
//# sourceMappingURL=run-scoped.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"run-scoped.d.ts","sourceRoot":"","sources":["../../src/qa/run-scoped.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;GAUG;AAEH,OAAO,EAAS,KAAK,MAAM,EAAE,KAAK,QAAQ,EAAE,MAAM,aAAa,CAAC;AAEhE,OAAO,KAAK,EAAE,KAAK,EAAE,MAAM,sBAAsB,CAAC;AAClD,OAAO,KAAK,EAAE,QAAQ,EAAE,MAAM,sBAAsB,CAAC;AACrD,OAAO,KAAK,EAAE,MAAM,EAAE,MAAM,oBAAoB,CAAC;AACjD,OAAO,KAAK,EAAE,iBAAiB,EAAE,MAAM,0BAA0B,CAAC;AAClE,OAAO,KAAK,EAAE,aAAa,EAAE,MAAM,sBAAsB,CAAC;AAC1D,OAAO,EAA8C,KAAK,iBAAiB,EAAE,MAAM,oBAAoB,CAAC;AACxG,OAAO,EAAqB,KAAK,gBAAgB,EAAE,KAAK,WAAW,EAAE,MAAM,wBAAwB,CAAC;AAEpG,uFAAuF;AACvF,eAAO,MAAM,0BAA0B,IAAI,CAAC;AAE5C;;;;GAIG;AACH,MAAM,WAAW,YAAY;IAC3B,KAAK,EAAE,MAAM,CAAC,OAAO,CAAC,CAAC;IACvB,6EAA6E;IAC7E,YAAY,CAAC,YAAY,EAAE,MAAM,GAAG,QAAQ,CAAC;IAC7C,2EAA2E;IAC3E,UAAU,CAAC,YAAY,EAAE,MAAM,GAAG,MAAM,CAAC;IACzC,QAAQ,CAAC,EAAE,iBAAiB,CAAC;IAC7B,QAAQ,CAAC,EAAE,aAAa,CAAC;CAC1B;AAED,MAAM,WAAW,eAAe;IAC9B,KAAK,EAAE,KAAK,CAAC;IACb,MAAM,EAAE,iBAAiB,CAAC;IAC1B,0CAA0C;IAC1C,YAAY,EAAE,MAAM,EAAE,CAAC;IACvB,uEAAuE;IACvE,UAAU,EAAE,MAAM,CAAC;IACnB,6DAA6D;IAC7D,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,6BAA6B;IAC7B,CAAC,EAAE,MAAM,CAAC;IACV,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,oEAAoE;IACpE,IAAI,CAAC,EAAE,OAAO,CAAC;IACf,uFAAuF;IACvF,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,kFAAkF;IAClF,OAAO,CAAC,EAAE;QAAE,UAAU,CAAC,EAAE,MAAM,CAAA;KAAE,CAAC;CACnC;AAED,MAAM,WAAW,sBAAsB;IACrC,UAAU,EAAE,gBAAgB,CAAC;IAC7B,8CAA8C;IAC9C,MAAM,CAAC,EAAE,QAAQ,CAAC;IAClB,2EAA2E;IAC3E,KAAK,CAAC,EAAE,MAAM,CAAC;CAChB;AAED,MAAM,WAAW,cAAc;IAC7B,KAAK,EAAE,WAAW,CAAC;IACnB,mEAAmE;IACnE,MAAM,EAAE,sBAAsB,EAAE,CAAC;CAClC;AAED,mFAAmF;AACnF,MAAM,WAAW,iBAAiB;IAChC,EAAE,EAAE,MAAM,CAAC;IACX,KAAK,EAAE,MAAM,CAAC;IACd,OAAO,EAAE,MAAM,EAAE,CAAC;CACnB;AAED;;;;GAIG;AACH,wBAAsB,yBAAyB,CAC7C,MAAM,EAAE,cAAc,EACtB,QAAQ,EAAE,aAAa,GACtB,OAAO,CAAC,iBAAiB,EAAE,CAAC,CAW9B;AAED,wBAAsB,WAAW,CAAC,IAAI,EAAE,YAAY,EAAE,OAAO,EAAE,eAAe,GAAG,OAAO,CAAC,cAAc,CAAC,CAmEvG"}
|
|
@@ -0,0 +1,96 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* PR-triggered QA — drive every unverified capability a change touched.
|
|
3
|
+
*
|
|
4
|
+
* Built on the Phase-1 {@link runQa} loop: scope the change to capabilities
|
|
5
|
+
* ({@link scopeCapabilities}), then drive each one that still lacks a verifying
|
|
6
|
+
* test, proposing a write-back when the capability declares its corpus artifact.
|
|
7
|
+
* Capabilities are driven **concurrently** with a bounded pool, each isolated in
|
|
8
|
+
* its own browser context, compiler output directory, and runner output
|
|
9
|
+
* directory (Ranger's context-isolated sub-agent lesson). Results stay in scoped
|
|
10
|
+
* order. The caller posts the summary to the feature pull request.
|
|
11
|
+
*/
|
|
12
|
+
import { runQa } from "./run-qa.js";
|
|
13
|
+
import { mapPool } from "./concurrency.js";
|
|
14
|
+
import { resolveTarget, authContext, personaContext } from "../scope/config.js";
|
|
15
|
+
import { scopeCapabilities } from "../scope/diff-scope.js";
|
|
16
|
+
/** Default capabilities driven at once — conservative to bound browser/runner load. */
|
|
17
|
+
export const DEFAULT_SCOPED_CONCURRENCY = 3;
|
|
18
|
+
/**
|
|
19
|
+
* Collect recorded failure reasons for each driven capability that failed or
|
|
20
|
+
* could not be driven, from the learning store — the `suggest_in_report`
|
|
21
|
+
* failure-learning strategy's source. Pure over the store (no drive).
|
|
22
|
+
*/
|
|
23
|
+
export async function collectFailureSuggestions(result, learning) {
|
|
24
|
+
const suggestions = [];
|
|
25
|
+
for (const driven of result.driven) {
|
|
26
|
+
const failed = driven.error !== undefined || driven.result?.verified === false;
|
|
27
|
+
if (!failed)
|
|
28
|
+
continue;
|
|
29
|
+
const prior = await learning.priorFailures(driven.capability.id);
|
|
30
|
+
if (prior.length > 0) {
|
|
31
|
+
suggestions.push({ id: driven.capability.id, title: driven.capability.title, reasons: prior.map((f) => f.reason) });
|
|
32
|
+
}
|
|
33
|
+
}
|
|
34
|
+
return suggestions;
|
|
35
|
+
}
|
|
36
|
+
export async function runScopedQa(deps, options) {
|
|
37
|
+
const scope = scopeCapabilities(options.changedPaths, options.config, options.graph);
|
|
38
|
+
const driven = await mapPool(scope.toVerify, options.concurrency ?? DEFAULT_SCOPED_CONCURRENCY, async (cap) => {
|
|
39
|
+
// Resolve the target: explicit url, else a named environment, else the default.
|
|
40
|
+
const target = resolveTarget(options.config, cap.config, {
|
|
41
|
+
...(options.defaultUrl !== undefined ? { fallbackUrl: options.defaultUrl } : {}),
|
|
42
|
+
defaultName: options.targetName,
|
|
43
|
+
});
|
|
44
|
+
if (target === undefined) {
|
|
45
|
+
return { capability: cap, error: "no start URL — set config.url, an environment, or pass --url" };
|
|
46
|
+
}
|
|
47
|
+
// Thread persona, environment restrictions, and auth context into the goal.
|
|
48
|
+
const contextParts = [];
|
|
49
|
+
let persona;
|
|
50
|
+
try {
|
|
51
|
+
persona = personaContext(options.config, cap.config);
|
|
52
|
+
}
|
|
53
|
+
catch (err) {
|
|
54
|
+
return { capability: cap, error: err.message };
|
|
55
|
+
}
|
|
56
|
+
if (persona)
|
|
57
|
+
contextParts.push(persona);
|
|
58
|
+
if (target.restrictions.length > 0) {
|
|
59
|
+
contextParts.push(`Environment restrictions: ${target.restrictions.join("; ")}. Respect them strictly.`);
|
|
60
|
+
}
|
|
61
|
+
const auth = authContext(options.config);
|
|
62
|
+
if (auth)
|
|
63
|
+
contextParts.push(auth);
|
|
64
|
+
const goalContext = contextParts.length > 0 ? contextParts.join(" ") : undefined;
|
|
65
|
+
const propose = options.propose && cap.config.artifact !== undefined
|
|
66
|
+
? {
|
|
67
|
+
targetPath: cap.config.artifact,
|
|
68
|
+
...(options.propose.baseBranch !== undefined ? { baseBranch: options.propose.baseBranch } : {}),
|
|
69
|
+
}
|
|
70
|
+
: undefined;
|
|
71
|
+
// Per-capability isolated compiler + runner; the drive seam isolates the browser.
|
|
72
|
+
const capDeps = {
|
|
73
|
+
drive: deps.drive,
|
|
74
|
+
compiler: deps.makeCompiler(cap.id),
|
|
75
|
+
runner: deps.makeRunner(cap.id),
|
|
76
|
+
...(deps.proposer ? { proposer: deps.proposer } : {}),
|
|
77
|
+
...(deps.learning ? { learning: deps.learning } : {}),
|
|
78
|
+
};
|
|
79
|
+
const result = await runQa(capDeps, {
|
|
80
|
+
graph: options.graph,
|
|
81
|
+
capabilityId: cap.id,
|
|
82
|
+
startUrl: target.url,
|
|
83
|
+
...(cap.config.goal !== undefined ? { goal: cap.config.goal } : {}),
|
|
84
|
+
...(goalContext !== undefined ? { goalContext } : {}),
|
|
85
|
+
// Each capability runs against its resolved environment URL.
|
|
86
|
+
target: { name: target.name, baseURL: target.url },
|
|
87
|
+
n: options.n,
|
|
88
|
+
...(options.maxSteps !== undefined ? { maxSteps: options.maxSteps } : {}),
|
|
89
|
+
...(options.plan ? { plan: true } : {}),
|
|
90
|
+
...(propose ? { propose } : {}),
|
|
91
|
+
});
|
|
92
|
+
return { capability: cap, result };
|
|
93
|
+
});
|
|
94
|
+
return { scope, driven };
|
|
95
|
+
}
|
|
96
|
+
//# sourceMappingURL=run-scoped.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"run-scoped.js","sourceRoot":"","sources":["../../src/qa/run-scoped.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;GAUG;AAEH,OAAO,EAAE,KAAK,EAA8B,MAAM,aAAa,CAAC;AAChE,OAAO,EAAE,OAAO,EAAE,MAAM,kBAAkB,CAAC;AAM3C,OAAO,EAAE,aAAa,EAAE,WAAW,EAAE,cAAc,EAA0B,MAAM,oBAAoB,CAAC;AACxG,OAAO,EAAE,iBAAiB,EAA2C,MAAM,wBAAwB,CAAC;AAEpG,uFAAuF;AACvF,MAAM,CAAC,MAAM,0BAA0B,GAAG,CAAC,CAAC;AA0D5C;;;;GAIG;AACH,MAAM,CAAC,KAAK,UAAU,yBAAyB,CAC7C,MAAsB,EACtB,QAAuB;IAEvB,MAAM,WAAW,GAAwB,EAAE,CAAC;IAC5C,KAAK,MAAM,MAAM,IAAI,MAAM,CAAC,MAAM,EAAE,CAAC;QACnC,MAAM,MAAM,GAAG,MAAM,CAAC,KAAK,KAAK,SAAS,IAAI,MAAM,CAAC,MAAM,EAAE,QAAQ,KAAK,KAAK,CAAC;QAC/E,IAAI,CAAC,MAAM;YAAE,SAAS;QACtB,MAAM,KAAK,GAAG,MAAM,QAAQ,CAAC,aAAa,CAAC,MAAM,CAAC,UAAU,CAAC,EAAE,CAAC,CAAC;QACjE,IAAI,KAAK,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YACrB,WAAW,CAAC,IAAI,CAAC,EAAE,EAAE,EAAE,MAAM,CAAC,UAAU,CAAC,EAAE,EAAE,KAAK,EAAE,MAAM,CAAC,UAAU,CAAC,KAAK,EAAE,OAAO,EAAE,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,MAAM,CAAC,EAAE,CAAC,CAAC;QACtH,CAAC;IACH,CAAC;IACD,OAAO,WAAW,CAAC;AACrB,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,WAAW,CAAC,IAAkB,EAAE,OAAwB;IAC5E,MAAM,KAAK,GAAG,iBAAiB,CAAC,OAAO,CAAC,YAAY,EAAE,OAAO,CAAC,MAAM,EAAE,OAAO,CAAC,KAAK,CAAC,CAAC;IAErF,MAAM,MAAM,GAAG,MAAM,OAAO,CAC1B,KAAK,CAAC,QAAQ,EACd,OAAO,CAAC,WAAW,IAAI,0BAA0B,EACjD,KAAK,EAAE,GAAG,EAAmC,EAAE;QAC7C,gFAAgF;QAChF,MAAM,MAAM,GAAG,aAAa,CAAC,OAAO,CAAC,MAAM,EAAE,GAAG,CAAC,MAAM,EAAE;YACvD,GAAG,CAAC,OAAO,CAAC,UAAU,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,WAAW,EAAE,OAAO,CAAC,UAAU,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;YAChF,WAAW,EAAE,OAAO,CAAC,UAAU;SAChC,CAAC,CAAC;QACH,IAAI,MAAM,KAAK,SAAS,EAAE,CAAC;YACzB,OAAO,EAAE,UAAU,EAAE,GAAG,EAAE,KAAK,EAAE,8DAA8D,EAAE,CAAC;QACpG,CAAC;QAED,4EAA4E;QAC5E,MAAM,YAAY,GAAa,EAAE,CAAC;QAClC,IAAI,OAA2B,CAAC;QAChC,IAAI,CAAC;YACH,OAAO,GAAG,cAAc,CAAC,OAAO,CAAC,MAAM,EAAE,GAAG,CAAC,MAAM,CAAC,CAAC;QACvD,CAAC;QAAC,OAAO,GAAG,EAAE,CAAC;YACb,OAAO,EAAE,UAAU,EAAE,GAAG,EAAE,KAAK,EAAG,GAAa,CAAC,OAAO,EAAE,CAAC;QAC5D,CAAC;QACD,IAAI,OAAO;YAAE,YAAY,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;QACxC,IAAI,MAAM,CAAC,YAAY,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YACnC,YAAY,CAAC,IAAI,CAAC,6BAA6B,MAAM,CAAC,YAAY,CAAC,IAAI,CAAC,IAAI,CAAC,0BAA0B,CAAC,CAAC;QAC3G,CAAC;QACD,MAAM,IAAI,GAAG,WAAW,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC;QACzC,IAAI,IAAI;YAAE,YAAY,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QAClC,MAAM,WAAW,GAAG,YAAY,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,YAAY,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,SAAS,CAAC;QAEjF,MAAM,OAAO,GACX,OAAO,CAAC,OAAO,IAAI,GAAG,CAAC,MAAM,CAAC,QAAQ,KAAK,SAAS;YAClD,CAAC,CAAC;gBACE,UAAU,EAAE,GAAG,CAAC,MAAM,CAAC,QAAQ;gBAC/B,GAAG,CAAC,OAAO,CAAC,OAAO,CAAC,UAAU,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,UAAU,EAAE,OAAO,CAAC,OAAO,CAAC,UAAU,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;aAChG;YACH,CAAC,CAAC,SAAS,CAAC;QAEhB,kFAAkF;QAClF,MAAM,OAAO,GAAW;YACtB,KAAK,EAAE,IAAI,CAAC,KAAK;YACjB,QAAQ,EAAE,IAAI,CAAC,YAAY,CAAC,GAAG,CAAC,EAAE,CAAC;YACnC,MAAM,EAAE,IAAI,CAAC,UAAU,CAAC,GAAG,CAAC,EAAE,CAAC;YAC/B,GAAG,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC,CAAC,EAAE,QAAQ,EAAE,IAAI,CAAC,QAAQ,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;YACrD,GAAG,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC,CAAC,EAAE,QAAQ,EAAE,IAAI,CAAC,QAAQ,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;SACtD,CAAC;QAEF,MAAM,MAAM,GAAG,MAAM,KAAK,CAAC,OAAO,EAAE;YAClC,KAAK,EAAE,OAAO,CAAC,KAAK;YACpB,YAAY,EAAE,GAAG,CAAC,EAAE;YACpB,QAAQ,EAAE,MAAM,CAAC,GAAG;YACpB,GAAG,CAAC,GAAG,CAAC,MAAM,CAAC,IAAI,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,IAAI,EAAE,GAAG,CAAC,MAAM,CAAC,IAAI,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;YACnE,GAAG,CAAC,WAAW,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,WAAW,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;YACrD,6DAA6D;YAC7D,MAAM,EAAE,EAAE,IAAI,EAAE,MAAM,CAAC,IAAI,EAAE,OAAO,EAAE,MAAM,CAAC,GAAG,EAAE;YAClD,CAAC,EAAE,OAAO,CAAC,CAAC;YACZ,GAAG,CAAC,OAAO,CAAC,QAAQ,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,QAAQ,EAAE,OAAO,CAAC,QAAQ,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;YACzE,GAAG,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,IAAI,EAAE,IAAI,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;YACvC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,OAAO,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;SAChC,CAAC,CAAC;QACH,OAAO,EAAE,UAAU,EAAE,GAAG,EAAE,MAAM,EAAE,CAAC;IACrC,CAAC,CACF,CAAC;IAEF,OAAO,EAAE,KAAK,EAAE,MAAM,EAAE,CAAC;AAC3B,CAAC"}
|
|
@@ -0,0 +1,52 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Parse Playwright's JSON reporter output into Proofkeeper {@link RunResult}s.
|
|
3
|
+
*
|
|
4
|
+
* Kept pure and separate from the runner so the report→result mapping is
|
|
5
|
+
* unit-testable with a fixture, no browser required. The runner (which does
|
|
6
|
+
* the actual `npx playwright test` shell-out) feeds raw report JSON in here.
|
|
7
|
+
*
|
|
8
|
+
* We model only the fields we read and tolerate the rest — Playwright's report
|
|
9
|
+
* is large and versioned, and we never want a new field to break parsing.
|
|
10
|
+
*/
|
|
11
|
+
import type { RunResult } from "./types.js";
|
|
12
|
+
/** A single attempt's result inside the Playwright report. */
|
|
13
|
+
interface PwResult {
|
|
14
|
+
status?: string;
|
|
15
|
+
duration?: number;
|
|
16
|
+
attachments?: {
|
|
17
|
+
name?: string;
|
|
18
|
+
path?: string;
|
|
19
|
+
contentType?: string;
|
|
20
|
+
}[];
|
|
21
|
+
}
|
|
22
|
+
interface PwTest {
|
|
23
|
+
results?: PwResult[];
|
|
24
|
+
}
|
|
25
|
+
interface PwSpec {
|
|
26
|
+
title?: string;
|
|
27
|
+
tests?: PwTest[];
|
|
28
|
+
}
|
|
29
|
+
interface PwSuite {
|
|
30
|
+
specs?: PwSpec[];
|
|
31
|
+
suites?: PwSuite[];
|
|
32
|
+
}
|
|
33
|
+
interface PwReport {
|
|
34
|
+
suites?: PwSuite[];
|
|
35
|
+
}
|
|
36
|
+
/** Raised when reporter output is not parseable JSON. */
|
|
37
|
+
export declare class ReportParseError extends Error {
|
|
38
|
+
constructor(message: string);
|
|
39
|
+
}
|
|
40
|
+
/**
|
|
41
|
+
* Reduce a parsed report to a single {@link RunResult} for one (test, target).
|
|
42
|
+
*
|
|
43
|
+
* A spec file may contain several `test(...)` blocks; we aggregate them: the
|
|
44
|
+
* run passed iff every result passed, the duration is the sum, and the trace is
|
|
45
|
+
* the first trace attachment found. An empty report (no results) is a failure —
|
|
46
|
+
* a spec that ran nothing did not verify anything.
|
|
47
|
+
*/
|
|
48
|
+
export declare function reduceReport(report: PwReport, testId: string, target: string): RunResult;
|
|
49
|
+
/** Parse reporter stdout (JSON) and reduce it to a {@link RunResult}. */
|
|
50
|
+
export declare function parseReport(stdout: string, testId: string, target: string): RunResult;
|
|
51
|
+
export {};
|
|
52
|
+
//# sourceMappingURL=playwright-report.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"playwright-report.d.ts","sourceRoot":"","sources":["../../src/runner/playwright-report.ts"],"names":[],"mappings":"AAAA;;;;;;;;;GASG;AAEH,OAAO,KAAK,EAAE,SAAS,EAAa,MAAM,YAAY,CAAC;AAEvD,8DAA8D;AAC9D,UAAU,QAAQ;IAChB,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,WAAW,CAAC,EAAE;QAAE,IAAI,CAAC,EAAE,MAAM,CAAC;QAAC,IAAI,CAAC,EAAE,MAAM,CAAC;QAAC,WAAW,CAAC,EAAE,MAAM,CAAA;KAAE,EAAE,CAAC;CACxE;AAED,UAAU,MAAM;IACd,OAAO,CAAC,EAAE,QAAQ,EAAE,CAAC;CACtB;AAED,UAAU,MAAM;IACd,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,KAAK,CAAC,EAAE,MAAM,EAAE,CAAC;CAClB;AAED,UAAU,OAAO;IACf,KAAK,CAAC,EAAE,MAAM,EAAE,CAAC;IACjB,MAAM,CAAC,EAAE,OAAO,EAAE,CAAC;CACpB;AAED,UAAU,QAAQ;IAChB,MAAM,CAAC,EAAE,OAAO,EAAE,CAAC;CACpB;AAED,yDAAyD;AACzD,qBAAa,gBAAiB,SAAQ,KAAK;gBAC7B,OAAO,EAAE,MAAM;CAI5B;AAuCD;;;;;;;GAOG;AACH,wBAAgB,YAAY,CAAC,MAAM,EAAE,QAAQ,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,GAAG,SAAS,CAoBxF;AAED,yEAAyE;AACzE,wBAAgB,WAAW,CAAC,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,GAAG,SAAS,CAUrF"}
|
|
@@ -0,0 +1,90 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Parse Playwright's JSON reporter output into Proofkeeper {@link RunResult}s.
|
|
3
|
+
*
|
|
4
|
+
* Kept pure and separate from the runner so the report→result mapping is
|
|
5
|
+
* unit-testable with a fixture, no browser required. The runner (which does
|
|
6
|
+
* the actual `npx playwright test` shell-out) feeds raw report JSON in here.
|
|
7
|
+
*
|
|
8
|
+
* We model only the fields we read and tolerate the rest — Playwright's report
|
|
9
|
+
* is large and versioned, and we never want a new field to break parsing.
|
|
10
|
+
*/
|
|
11
|
+
/** Raised when reporter output is not parseable JSON. */
|
|
12
|
+
export class ReportParseError extends Error {
|
|
13
|
+
constructor(message) {
|
|
14
|
+
super(message);
|
|
15
|
+
this.name = "ReportParseError";
|
|
16
|
+
}
|
|
17
|
+
}
|
|
18
|
+
function mapStatus(raw) {
|
|
19
|
+
switch (raw) {
|
|
20
|
+
case "passed":
|
|
21
|
+
return "passed";
|
|
22
|
+
case "skipped":
|
|
23
|
+
return "skipped";
|
|
24
|
+
case "timedOut":
|
|
25
|
+
return "timedout";
|
|
26
|
+
default:
|
|
27
|
+
// failed, interrupted, or anything unknown is a non-pass.
|
|
28
|
+
return "failed";
|
|
29
|
+
}
|
|
30
|
+
}
|
|
31
|
+
/** Depth-first walk of the nested suite tree, yielding every result. */
|
|
32
|
+
function collectResults(suites) {
|
|
33
|
+
const out = [];
|
|
34
|
+
for (const suite of suites ?? []) {
|
|
35
|
+
for (const spec of suite.specs ?? []) {
|
|
36
|
+
for (const test of spec.tests ?? []) {
|
|
37
|
+
out.push(...(test.results ?? []));
|
|
38
|
+
}
|
|
39
|
+
}
|
|
40
|
+
out.push(...collectResults(suite.suites));
|
|
41
|
+
}
|
|
42
|
+
return out;
|
|
43
|
+
}
|
|
44
|
+
function firstTracePath(results) {
|
|
45
|
+
for (const result of results) {
|
|
46
|
+
for (const attachment of result.attachments ?? []) {
|
|
47
|
+
if (attachment.name === "trace" && attachment.path)
|
|
48
|
+
return attachment.path;
|
|
49
|
+
}
|
|
50
|
+
}
|
|
51
|
+
return undefined;
|
|
52
|
+
}
|
|
53
|
+
/**
|
|
54
|
+
* Reduce a parsed report to a single {@link RunResult} for one (test, target).
|
|
55
|
+
*
|
|
56
|
+
* A spec file may contain several `test(...)` blocks; we aggregate them: the
|
|
57
|
+
* run passed iff every result passed, the duration is the sum, and the trace is
|
|
58
|
+
* the first trace attachment found. An empty report (no results) is a failure —
|
|
59
|
+
* a spec that ran nothing did not verify anything.
|
|
60
|
+
*/
|
|
61
|
+
export function reduceReport(report, testId, target) {
|
|
62
|
+
const results = collectResults(report.suites);
|
|
63
|
+
if (results.length === 0) {
|
|
64
|
+
return { testId, target, status: "failed", durationMs: 0 };
|
|
65
|
+
}
|
|
66
|
+
const statuses = results.map((r) => mapStatus(r.status));
|
|
67
|
+
const durationMs = results.reduce((sum, r) => sum + (r.duration ?? 0), 0);
|
|
68
|
+
const allPassed = statuses.every((s) => s === "passed");
|
|
69
|
+
// Surface the most informative non-pass status when not all passed.
|
|
70
|
+
const status = allPassed
|
|
71
|
+
? "passed"
|
|
72
|
+
: (statuses.find((s) => s === "timedout") ?? statuses.find((s) => s === "failed") ?? "failed");
|
|
73
|
+
const result = { testId, target, status, durationMs };
|
|
74
|
+
const tracePath = firstTracePath(results);
|
|
75
|
+
if (tracePath)
|
|
76
|
+
result.tracePath = tracePath;
|
|
77
|
+
return result;
|
|
78
|
+
}
|
|
79
|
+
/** Parse reporter stdout (JSON) and reduce it to a {@link RunResult}. */
|
|
80
|
+
export function parseReport(stdout, testId, target) {
|
|
81
|
+
let report;
|
|
82
|
+
try {
|
|
83
|
+
report = JSON.parse(stdout);
|
|
84
|
+
}
|
|
85
|
+
catch (err) {
|
|
86
|
+
throw new ReportParseError(`could not parse Playwright JSON report: ${err.message}`);
|
|
87
|
+
}
|
|
88
|
+
return reduceReport(report, testId, target);
|
|
89
|
+
}
|
|
90
|
+
//# sourceMappingURL=playwright-report.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"playwright-report.js","sourceRoot":"","sources":["../../src/runner/playwright-report.ts"],"names":[],"mappings":"AAAA;;;;;;;;;GASG;AA6BH,yDAAyD;AACzD,MAAM,OAAO,gBAAiB,SAAQ,KAAK;IACzC,YAAY,OAAe;QACzB,KAAK,CAAC,OAAO,CAAC,CAAC;QACf,IAAI,CAAC,IAAI,GAAG,kBAAkB,CAAC;IACjC,CAAC;CACF;AAED,SAAS,SAAS,CAAC,GAAuB;IACxC,QAAQ,GAAG,EAAE,CAAC;QACZ,KAAK,QAAQ;YACX,OAAO,QAAQ,CAAC;QAClB,KAAK,SAAS;YACZ,OAAO,SAAS,CAAC;QACnB,KAAK,UAAU;YACb,OAAO,UAAU,CAAC;QACpB;YACE,0DAA0D;YAC1D,OAAO,QAAQ,CAAC;IACpB,CAAC;AACH,CAAC;AAED,wEAAwE;AACxE,SAAS,cAAc,CAAC,MAA6B;IACnD,MAAM,GAAG,GAAe,EAAE,CAAC;IAC3B,KAAK,MAAM,KAAK,IAAI,MAAM,IAAI,EAAE,EAAE,CAAC;QACjC,KAAK,MAAM,IAAI,IAAI,KAAK,CAAC,KAAK,IAAI,EAAE,EAAE,CAAC;YACrC,KAAK,MAAM,IAAI,IAAI,IAAI,CAAC,KAAK,IAAI,EAAE,EAAE,CAAC;gBACpC,GAAG,CAAC,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,OAAO,IAAI,EAAE,CAAC,CAAC,CAAC;YACpC,CAAC;QACH,CAAC;QACD,GAAG,CAAC,IAAI,CAAC,GAAG,cAAc,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC;IAC5C,CAAC;IACD,OAAO,GAAG,CAAC;AACb,CAAC;AAED,SAAS,cAAc,CAAC,OAAmB;IACzC,KAAK,MAAM,MAAM,IAAI,OAAO,EAAE,CAAC;QAC7B,KAAK,MAAM,UAAU,IAAI,MAAM,CAAC,WAAW,IAAI,EAAE,EAAE,CAAC;YAClD,IAAI,UAAU,CAAC,IAAI,KAAK,OAAO,IAAI,UAAU,CAAC,IAAI;gBAAE,OAAO,UAAU,CAAC,IAAI,CAAC;QAC7E,CAAC;IACH,CAAC;IACD,OAAO,SAAS,CAAC;AACnB,CAAC;AAED;;;;;;;GAOG;AACH,MAAM,UAAU,YAAY,CAAC,MAAgB,EAAE,MAAc,EAAE,MAAc;IAC3E,MAAM,OAAO,GAAG,cAAc,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC;IAE9C,IAAI,OAAO,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QACzB,OAAO,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,QAAQ,EAAE,UAAU,EAAE,CAAC,EAAE,CAAC;IAC7D,CAAC;IAED,MAAM,QAAQ,GAAG,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,SAAS,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC;IACzD,MAAM,UAAU,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,CAAC,EAAE,EAAE,CAAC,GAAG,GAAG,CAAC,CAAC,CAAC,QAAQ,IAAI,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC;IAC1E,MAAM,SAAS,GAAG,QAAQ,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,KAAK,QAAQ,CAAC,CAAC;IAExD,oEAAoE;IACpE,MAAM,MAAM,GAAc,SAAS;QACjC,CAAC,CAAC,QAAQ;QACV,CAAC,CAAC,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,KAAK,UAAU,CAAC,IAAI,QAAQ,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,KAAK,QAAQ,CAAC,IAAI,QAAQ,CAAC,CAAC;IAEjG,MAAM,MAAM,GAAc,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,UAAU,EAAE,CAAC;IACjE,MAAM,SAAS,GAAG,cAAc,CAAC,OAAO,CAAC,CAAC;IAC1C,IAAI,SAAS;QAAE,MAAM,CAAC,SAAS,GAAG,SAAS,CAAC;IAC5C,OAAO,MAAM,CAAC;AAChB,CAAC;AAED,yEAAyE;AACzE,MAAM,UAAU,WAAW,CAAC,MAAc,EAAE,MAAc,EAAE,MAAc;IACxE,IAAI,MAAgB,CAAC;IACrB,IAAI,CAAC;QACH,MAAM,GAAG,IAAI,CAAC,KAAK,CAAC,MAAM,CAAa,CAAC;IAC1C,CAAC;IAAC,OAAO,GAAG,EAAE,CAAC;QACb,MAAM,IAAI,gBAAgB,CACxB,2CAA4C,GAAa,CAAC,OAAO,EAAE,CACpE,CAAC;IACJ,CAAC;IACD,OAAO,YAAY,CAAC,MAAM,EAAE,MAAM,EAAE,MAAM,CAAC,CAAC;AAC9C,CAAC"}
|
|
@@ -0,0 +1,38 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* The local Playwright runner — Proofkeeper Initiative 4.
|
|
3
|
+
*
|
|
4
|
+
* The open-source local runner. It shells out to the Playwright CLI
|
|
5
|
+
* (`npx playwright test`) so the test runtime stays entirely in this product,
|
|
6
|
+
* never in the Lore engine, and parses Playwright's JSON reporter into typed
|
|
7
|
+
* {@link RunResult}s — real status, duration, and the actual trace path, not
|
|
8
|
+
* values inferred from an exit code.
|
|
9
|
+
*
|
|
10
|
+
* The full cross-target / cross-OS matrix is still minimal here (targets run
|
|
11
|
+
* sequentially) and grows in later versions; the hosted VM-fabric runner is a
|
|
12
|
+
* separate implementation of the same {@link Runner} interface.
|
|
13
|
+
*/
|
|
14
|
+
import type { CompiledTest, RunOptions, RunResult, Runner } from "./types.js";
|
|
15
|
+
export interface PlaywrightRunnerOptions {
|
|
16
|
+
/** Working directory the Playwright project lives in. Defaults to cwd. */
|
|
17
|
+
cwd?: string;
|
|
18
|
+
/**
|
|
19
|
+
* Output directory for test results and traces. Isolating this per runner
|
|
20
|
+
* keeps concurrent runs (across targets, or parallel suites) from clobbering
|
|
21
|
+
* each other's `test-results/`. Defaults to Playwright's config value.
|
|
22
|
+
*/
|
|
23
|
+
outputDir?: string;
|
|
24
|
+
/** Override the Playwright invocation (advanced/testing). */
|
|
25
|
+
command?: {
|
|
26
|
+
bin: string;
|
|
27
|
+
baseArgs: string[];
|
|
28
|
+
};
|
|
29
|
+
}
|
|
30
|
+
export declare class PlaywrightRunner implements Runner {
|
|
31
|
+
private readonly cwd;
|
|
32
|
+
private readonly outputDir;
|
|
33
|
+
private readonly command;
|
|
34
|
+
constructor(options?: PlaywrightRunnerOptions);
|
|
35
|
+
run(suite: CompiledTest[], options: RunOptions): Promise<RunResult[]>;
|
|
36
|
+
private runOne;
|
|
37
|
+
}
|
|
38
|
+
//# sourceMappingURL=playwright-runner.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"playwright-runner.d.ts","sourceRoot":"","sources":["../../src/runner/playwright-runner.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;GAYG;AAMH,OAAO,KAAK,EAAE,YAAY,EAAE,UAAU,EAAE,SAAS,EAAE,MAAM,EAAE,MAAM,YAAY,CAAC;AAU9E,MAAM,WAAW,uBAAuB;IACtC,0EAA0E;IAC1E,GAAG,CAAC,EAAE,MAAM,CAAC;IACb;;;;OAIG;IACH,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,6DAA6D;IAC7D,OAAO,CAAC,EAAE;QAAE,GAAG,EAAE,MAAM,CAAC;QAAC,QAAQ,EAAE,MAAM,EAAE,CAAA;KAAE,CAAC;CAC/C;AAED,qBAAa,gBAAiB,YAAW,MAAM;IAC7C,OAAO,CAAC,QAAQ,CAAC,GAAG,CAAS;IAC7B,OAAO,CAAC,QAAQ,CAAC,SAAS,CAAqB;IAC/C,OAAO,CAAC,QAAQ,CAAC,OAAO,CAAsC;gBAElD,OAAO,GAAE,uBAA4B;IAM3C,GAAG,CAAC,KAAK,EAAE,YAAY,EAAE,EAAE,OAAO,EAAE,UAAU,GAAG,OAAO,CAAC,SAAS,EAAE,CAAC;YAU7D,MAAM;CA2CrB"}
|
|
@@ -0,0 +1,73 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* The local Playwright runner — Proofkeeper Initiative 4.
|
|
3
|
+
*
|
|
4
|
+
* The open-source local runner. It shells out to the Playwright CLI
|
|
5
|
+
* (`npx playwright test`) so the test runtime stays entirely in this product,
|
|
6
|
+
* never in the Lore engine, and parses Playwright's JSON reporter into typed
|
|
7
|
+
* {@link RunResult}s — real status, duration, and the actual trace path, not
|
|
8
|
+
* values inferred from an exit code.
|
|
9
|
+
*
|
|
10
|
+
* The full cross-target / cross-OS matrix is still minimal here (targets run
|
|
11
|
+
* sequentially) and grows in later versions; the hosted VM-fabric runner is a
|
|
12
|
+
* separate implementation of the same {@link Runner} interface.
|
|
13
|
+
*/
|
|
14
|
+
import { execFile } from "node:child_process";
|
|
15
|
+
import { promisify } from "node:util";
|
|
16
|
+
import { parseReport } from "./playwright-report.js";
|
|
17
|
+
const execFileAsync = promisify(execFile);
|
|
18
|
+
export class PlaywrightRunner {
|
|
19
|
+
cwd;
|
|
20
|
+
outputDir;
|
|
21
|
+
command;
|
|
22
|
+
constructor(options = {}) {
|
|
23
|
+
this.cwd = options.cwd ?? process.cwd();
|
|
24
|
+
this.outputDir = options.outputDir;
|
|
25
|
+
this.command = options.command ?? { bin: "npx", baseArgs: ["playwright", "test"] };
|
|
26
|
+
}
|
|
27
|
+
async run(suite, options) {
|
|
28
|
+
const results = [];
|
|
29
|
+
for (const target of options.targets) {
|
|
30
|
+
for (const test of suite) {
|
|
31
|
+
results.push(await this.runOne(test, target.name, target.baseURL, options.parallelism));
|
|
32
|
+
}
|
|
33
|
+
}
|
|
34
|
+
return results;
|
|
35
|
+
}
|
|
36
|
+
async runOne(test, targetName, baseURL, workers) {
|
|
37
|
+
const args = [
|
|
38
|
+
...this.command.baseArgs,
|
|
39
|
+
test.specPath,
|
|
40
|
+
"--reporter=json",
|
|
41
|
+
"--trace=on",
|
|
42
|
+
...(this.outputDir ? [`--output=${this.outputDir}`] : []),
|
|
43
|
+
...(workers ? [`--workers=${workers}`] : []),
|
|
44
|
+
];
|
|
45
|
+
const env = { ...process.env, PROOFKEEPER_BASE_URL: baseURL };
|
|
46
|
+
// Playwright exits non-zero when tests fail, but still writes the JSON
|
|
47
|
+
// report to stdout. Capture stdout in both cases and let the report — not
|
|
48
|
+
// the exit code — decide the verdict.
|
|
49
|
+
let stdout;
|
|
50
|
+
try {
|
|
51
|
+
({ stdout } = await execFileAsync(this.command.bin, args, {
|
|
52
|
+
cwd: this.cwd,
|
|
53
|
+
env,
|
|
54
|
+
maxBuffer: 256 * 1024 * 1024,
|
|
55
|
+
}));
|
|
56
|
+
}
|
|
57
|
+
catch (err) {
|
|
58
|
+
const execErr = err;
|
|
59
|
+
if (typeof execErr.stdout === "string" && execErr.stdout.trim().startsWith("{")) {
|
|
60
|
+
stdout = execErr.stdout;
|
|
61
|
+
}
|
|
62
|
+
else {
|
|
63
|
+
// The CLI failed before producing a report (e.g. Playwright not
|
|
64
|
+
// installed, browsers missing). That is a runner error, not a test
|
|
65
|
+
// verdict — surface it.
|
|
66
|
+
throw new Error(`playwright run failed for '${test.specPath}': ${execErr.message}` +
|
|
67
|
+
(execErr.stderr ? `\n${execErr.stderr}` : ""));
|
|
68
|
+
}
|
|
69
|
+
}
|
|
70
|
+
return parseReport(stdout, test.id, targetName);
|
|
71
|
+
}
|
|
72
|
+
}
|
|
73
|
+
//# sourceMappingURL=playwright-runner.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"playwright-runner.js","sourceRoot":"","sources":["../../src/runner/playwright-runner.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;GAYG;AAEH,OAAO,EAAE,QAAQ,EAAE,MAAM,oBAAoB,CAAC;AAC9C,OAAO,EAAE,SAAS,EAAE,MAAM,WAAW,CAAC;AAEtC,OAAO,EAAE,WAAW,EAAE,MAAM,wBAAwB,CAAC;AAGrD,MAAM,aAAa,GAAG,SAAS,CAAC,QAAQ,CAAC,CAAC;AAqB1C,MAAM,OAAO,gBAAgB;IACV,GAAG,CAAS;IACZ,SAAS,CAAqB;IAC9B,OAAO,CAAsC;IAE9D,YAAY,UAAmC,EAAE;QAC/C,IAAI,CAAC,GAAG,GAAG,OAAO,CAAC,GAAG,IAAI,OAAO,CAAC,GAAG,EAAE,CAAC;QACxC,IAAI,CAAC,SAAS,GAAG,OAAO,CAAC,SAAS,CAAC;QACnC,IAAI,CAAC,OAAO,GAAG,OAAO,CAAC,OAAO,IAAI,EAAE,GAAG,EAAE,KAAK,EAAE,QAAQ,EAAE,CAAC,YAAY,EAAE,MAAM,CAAC,EAAE,CAAC;IACrF,CAAC;IAED,KAAK,CAAC,GAAG,CAAC,KAAqB,EAAE,OAAmB;QAClD,MAAM,OAAO,GAAgB,EAAE,CAAC;QAChC,KAAK,MAAM,MAAM,IAAI,OAAO,CAAC,OAAO,EAAE,CAAC;YACrC,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE,CAAC;gBACzB,OAAO,CAAC,IAAI,CAAC,MAAM,IAAI,CAAC,MAAM,CAAC,IAAI,EAAE,MAAM,CAAC,IAAI,EAAE,MAAM,CAAC,OAAO,EAAE,OAAO,CAAC,WAAW,CAAC,CAAC,CAAC;YAC1F,CAAC;QACH,CAAC;QACD,OAAO,OAAO,CAAC;IACjB,CAAC;IAEO,KAAK,CAAC,MAAM,CAClB,IAAkB,EAClB,UAAkB,EAClB,OAAe,EACf,OAAgB;QAEhB,MAAM,IAAI,GAAG;YACX,GAAG,IAAI,CAAC,OAAO,CAAC,QAAQ;YACxB,IAAI,CAAC,QAAQ;YACb,iBAAiB;YACjB,YAAY;YACZ,GAAG,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC,CAAC,CAAC,YAAY,IAAI,CAAC,SAAS,EAAE,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC;YACzD,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,aAAa,OAAO,EAAE,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC;SAC7C,CAAC;QACF,MAAM,GAAG,GAAG,EAAE,GAAG,OAAO,CAAC,GAAG,EAAE,oBAAoB,EAAE,OAAO,EAAE,CAAC;QAE9D,uEAAuE;QACvE,0EAA0E;QAC1E,sCAAsC;QACtC,IAAI,MAAc,CAAC;QACnB,IAAI,CAAC;YACH,CAAC,EAAE,MAAM,EAAE,GAAG,MAAM,aAAa,CAAC,IAAI,CAAC,OAAO,CAAC,GAAG,EAAE,IAAI,EAAE;gBACxD,GAAG,EAAE,IAAI,CAAC,GAAG;gBACb,GAAG;gBACH,SAAS,EAAE,GAAG,GAAG,IAAI,GAAG,IAAI;aAC7B,CAAC,CAAC,CAAC;QACN,CAAC;QAAC,OAAO,GAAG,EAAE,CAAC;YACb,MAAM,OAAO,GAAG,GAAgB,CAAC;YACjC,IAAI,OAAO,OAAO,CAAC,MAAM,KAAK,QAAQ,IAAI,OAAO,CAAC,MAAM,CAAC,IAAI,EAAE,CAAC,UAAU,CAAC,GAAG,CAAC,EAAE,CAAC;gBAChF,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC;YAC1B,CAAC;iBAAM,CAAC;gBACN,gEAAgE;gBAChE,mEAAmE;gBACnE,wBAAwB;gBACxB,MAAM,IAAI,KAAK,CACb,8BAA8B,IAAI,CAAC,QAAQ,MAAM,OAAO,CAAC,OAAO,EAAE;oBAChE,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,KAAK,OAAO,CAAC,MAAM,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC,CAChD,CAAC;YACJ,CAAC;QACH,CAAC;QAED,OAAO,WAAW,CAAC,MAAM,EAAE,IAAI,CAAC,EAAE,EAAE,UAAU,CAAC,CAAC;IAClD,CAAC;CACF"}
|
|
@@ -0,0 +1,45 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* The runner interface — Proofkeeper Initiative 4.
|
|
3
|
+
*
|
|
4
|
+
* A runner executes a compiled suite fast and in parallel across targets and
|
|
5
|
+
* operating systems, emitting a replayable trace per result. The interface is
|
|
6
|
+
* pluggable on purpose: the open-source local runner and the (out-of-scope for
|
|
7
|
+
* v0.0.1) hosted VM-fabric runner are two implementations of the same shape.
|
|
8
|
+
*/
|
|
9
|
+
/** A compiled, durable end-to-end test the runner can execute. */
|
|
10
|
+
export interface CompiledTest {
|
|
11
|
+
/** Stable identifier, used to correlate a result with a capability. */
|
|
12
|
+
id: string;
|
|
13
|
+
/** Path to the Playwright spec file (Proofkeeper-owned content). */
|
|
14
|
+
specPath: string;
|
|
15
|
+
/** Optional human title. */
|
|
16
|
+
title?: string;
|
|
17
|
+
}
|
|
18
|
+
/** Where to run: a named environment with a base URL (e.g. dev, prod). */
|
|
19
|
+
export interface RunTarget {
|
|
20
|
+
name: string;
|
|
21
|
+
baseURL: string;
|
|
22
|
+
}
|
|
23
|
+
export interface RunOptions {
|
|
24
|
+
targets: RunTarget[];
|
|
25
|
+
/** Max concurrent test executions. Defaults are implementation-defined. */
|
|
26
|
+
parallelism?: number;
|
|
27
|
+
}
|
|
28
|
+
export type RunStatus = "passed" | "failed" | "timedout" | "skipped";
|
|
29
|
+
/** The outcome of one test against one target, with its replayable trace. */
|
|
30
|
+
export interface RunResult {
|
|
31
|
+
testId: string;
|
|
32
|
+
target: string;
|
|
33
|
+
status: RunStatus;
|
|
34
|
+
durationMs: number;
|
|
35
|
+
/** Path to the replayable trace artifact, when one was produced. */
|
|
36
|
+
tracePath?: string;
|
|
37
|
+
}
|
|
38
|
+
/**
|
|
39
|
+
* Runs compiled suites and returns one {@link RunResult} per (test, target).
|
|
40
|
+
* Implementations own all runtime and content (browsers, runs, traces).
|
|
41
|
+
*/
|
|
42
|
+
export interface Runner {
|
|
43
|
+
run(suite: CompiledTest[], options: RunOptions): Promise<RunResult[]>;
|
|
44
|
+
}
|
|
45
|
+
//# sourceMappingURL=types.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"types.d.ts","sourceRoot":"","sources":["../../src/runner/types.ts"],"names":[],"mappings":"AAAA;;;;;;;GAOG;AAEH,kEAAkE;AAClE,MAAM,WAAW,YAAY;IAC3B,uEAAuE;IACvE,EAAE,EAAE,MAAM,CAAC;IACX,oEAAoE;IACpE,QAAQ,EAAE,MAAM,CAAC;IACjB,4BAA4B;IAC5B,KAAK,CAAC,EAAE,MAAM,CAAC;CAChB;AAED,0EAA0E;AAC1E,MAAM,WAAW,SAAS;IACxB,IAAI,EAAE,MAAM,CAAC;IACb,OAAO,EAAE,MAAM,CAAC;CACjB;AAED,MAAM,WAAW,UAAU;IACzB,OAAO,EAAE,SAAS,EAAE,CAAC;IACrB,2EAA2E;IAC3E,WAAW,CAAC,EAAE,MAAM,CAAC;CACtB;AAED,MAAM,MAAM,SAAS,GAAG,QAAQ,GAAG,QAAQ,GAAG,UAAU,GAAG,SAAS,CAAC;AAErE,6EAA6E;AAC7E,MAAM,WAAW,SAAS;IACxB,MAAM,EAAE,MAAM,CAAC;IACf,MAAM,EAAE,MAAM,CAAC;IACf,MAAM,EAAE,SAAS,CAAC;IAClB,UAAU,EAAE,MAAM,CAAC;IACnB,oEAAoE;IACpE,SAAS,CAAC,EAAE,MAAM,CAAC;CACpB;AAED;;;GAGG;AACH,MAAM,WAAW,MAAM;IACrB,GAAG,CAAC,KAAK,EAAE,YAAY,EAAE,EAAE,OAAO,EAAE,UAAU,GAAG,OAAO,CAAC,SAAS,EAAE,CAAC,CAAC;CACvE"}
|