flakelab 0.1.0 → 0.1.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +225 -38
- package/dist/analysis/blob-report.d.ts +15 -0
- package/dist/analysis/blob-report.js +137 -0
- package/dist/analysis/blob-report.js.map +1 -0
- package/dist/analysis/cli.d.ts +7 -0
- package/dist/analysis/cli.js +28 -0
- package/dist/analysis/cli.js.map +1 -0
- package/dist/analysis/schema.d.ts +175 -0
- package/dist/analysis/schema.js +34 -0
- package/dist/analysis/schema.js.map +1 -0
- package/dist/analysis/zip-archive.d.ts +6 -0
- package/dist/analysis/zip-archive.js +193 -0
- package/dist/analysis/zip-archive.js.map +1 -0
- package/dist/artifacts/paths.d.ts +3 -0
- package/dist/artifacts/paths.js +34 -0
- package/dist/artifacts/paths.js.map +1 -0
- package/dist/artifacts/retention.d.ts +1 -0
- package/dist/artifacts/retention.js +80 -0
- package/dist/artifacts/retention.js.map +1 -0
- package/dist/bisect/engine.js +0 -1
- package/dist/bisect/engine.js.map +1 -1
- package/dist/bisect/schema.d.ts +5 -6
- package/dist/bisect/schema.js +1 -2
- package/dist/bisect/schema.js.map +1 -1
- package/dist/bisect/solari-evaluator.js +9 -1
- package/dist/bisect/solari-evaluator.js.map +1 -1
- package/dist/ci/job-summary.d.ts +2 -0
- package/dist/ci/job-summary.js +120 -2
- package/dist/ci/job-summary.js.map +1 -1
- package/dist/ci/select-command.js +1 -1
- package/dist/ci/select-command.js.map +1 -1
- package/dist/ci/summary-command.js +12 -7
- package/dist/ci/summary-command.js.map +1 -1
- package/dist/cli-arguments.d.ts +44 -0
- package/dist/cli-arguments.js +356 -0
- package/dist/cli-arguments.js.map +1 -0
- package/dist/cli-command-help.d.ts +11 -0
- package/dist/cli-command-help.js +211 -0
- package/dist/cli-command-help.js.map +1 -0
- package/dist/cli-help.d.ts +5 -2
- package/dist/cli-help.js +122 -45
- package/dist/cli-help.js.map +1 -1
- package/dist/cli-option-definitions.d.ts +202 -0
- package/dist/cli-option-definitions.js +59 -0
- package/dist/cli-option-definitions.js.map +1 -0
- package/dist/cli.js +100 -100
- package/dist/cli.js.map +1 -1
- package/dist/commands/analyze-summary.d.ts +5 -0
- package/dist/commands/analyze-summary.js +89 -0
- package/dist/commands/analyze-summary.js.map +1 -0
- package/dist/commands/analyze.d.ts +6 -0
- package/dist/commands/analyze.js +128 -0
- package/dist/commands/analyze.js.map +1 -0
- package/dist/commands/bisect.d.ts +2 -2
- package/dist/commands/bisect.js +32 -3
- package/dist/commands/bisect.js.map +1 -1
- package/dist/commands/diagnose.d.ts +3 -2
- package/dist/commands/diagnose.js +355 -23
- package/dist/commands/diagnose.js.map +1 -1
- package/dist/commands/discover.d.ts +18 -2
- package/dist/commands/discover.js +215 -25
- package/dist/commands/discover.js.map +1 -1
- package/dist/commands/doctor.d.ts +2 -0
- package/dist/commands/doctor.js +66 -25
- package/dist/commands/doctor.js.map +1 -1
- package/dist/commands/investigate.d.ts +3 -2
- package/dist/commands/investigate.js +22 -3
- package/dist/commands/investigate.js.map +1 -1
- package/dist/commands/options.d.ts +123 -16
- package/dist/commands/options.js +24 -2
- package/dist/commands/options.js.map +1 -1
- package/dist/commands/prove.d.ts +2 -2
- package/dist/commands/prove.js +38 -1
- package/dist/commands/prove.js.map +1 -1
- package/dist/commands/repair.d.ts +8 -2
- package/dist/commands/repair.js +32 -9
- package/dist/commands/repair.js.map +1 -1
- package/dist/commands/replay.d.ts +2 -2
- package/dist/commands/replay.js +15 -2
- package/dist/commands/replay.js.map +1 -1
- package/dist/commands/report.d.ts +2 -2
- package/dist/commands/report.js +35 -10
- package/dist/commands/report.js.map +1 -1
- package/dist/commands/scan-summary.d.ts +7 -0
- package/dist/commands/scan-summary.js +175 -0
- package/dist/commands/scan-summary.js.map +1 -0
- package/dist/commands/scan.d.ts +9 -12
- package/dist/commands/scan.js +66 -62
- package/dist/commands/scan.js.map +1 -1
- package/dist/core/plan.js +1 -1
- package/dist/core/plan.js.map +1 -1
- package/dist/diagnosis/checkpoint.d.ts +6 -0
- package/dist/diagnosis/checkpoint.js +46 -0
- package/dist/diagnosis/checkpoint.js.map +1 -0
- package/dist/diagnosis/cli.d.ts +7 -0
- package/dist/diagnosis/cli.js +84 -0
- package/dist/diagnosis/cli.js.map +1 -0
- package/dist/diagnosis/recommendation.d.ts +13 -0
- package/dist/diagnosis/recommendation.js +95 -0
- package/dist/diagnosis/recommendation.js.map +1 -0
- package/dist/diagnosis/run-state.d.ts +52 -0
- package/dist/diagnosis/run-state.js +173 -0
- package/dist/diagnosis/run-state.js.map +1 -0
- package/dist/diagnosis/schema.d.ts +207 -0
- package/dist/diagnosis/schema.js +108 -0
- package/dist/diagnosis/schema.js.map +1 -0
- package/dist/diagnosis/solari-handoff.d.ts +9 -0
- package/dist/diagnosis/solari-handoff.js +40 -0
- package/dist/diagnosis/solari-handoff.js.map +1 -0
- package/dist/diagnosis/summary.d.ts +3 -0
- package/dist/diagnosis/summary.js +115 -0
- package/dist/diagnosis/summary.js.map +1 -0
- package/dist/discovery/auth-cookie.d.ts +19 -0
- package/dist/discovery/auth-cookie.js +26 -0
- package/dist/discovery/auth-cookie.js.map +1 -0
- package/dist/discovery/evaluate.d.ts +35 -2
- package/dist/discovery/evaluate.js +136 -30
- package/dist/discovery/evaluate.js.map +1 -1
- package/dist/discovery/event-loop.d.ts +25 -0
- package/dist/discovery/event-loop.js +84 -0
- package/dist/discovery/event-loop.js.map +1 -0
- package/dist/discovery/minimize.d.ts +61 -1
- package/dist/discovery/minimize.js +179 -7
- package/dist/discovery/minimize.js.map +1 -1
- package/dist/discovery/resource-loading.d.ts +25 -0
- package/dist/discovery/resource-loading.js +84 -0
- package/dist/discovery/resource-loading.js.map +1 -0
- package/dist/discovery/runner-environment.d.ts +26 -0
- package/dist/discovery/runner-environment.js +68 -0
- package/dist/discovery/runner-environment.js.map +1 -0
- package/dist/discovery/startup-event.d.ts +25 -0
- package/dist/discovery/startup-event.js +84 -0
- package/dist/discovery/startup-event.js.map +1 -0
- package/dist/discovery/storage-state.d.ts +26 -0
- package/dist/discovery/storage-state.js +86 -0
- package/dist/discovery/storage-state.js.map +1 -0
- package/dist/discovery/temporal-environment.d.ts +31 -0
- package/dist/discovery/temporal-environment.js +43 -0
- package/dist/discovery/temporal-environment.js.map +1 -0
- package/dist/discovery/visual-environment.d.ts +28 -0
- package/dist/discovery/visual-environment.js +42 -0
- package/dist/discovery/visual-environment.js.map +1 -0
- package/dist/domain/schema.d.ts +840 -20
- package/dist/domain/schema.js +216 -6
- package/dist/domain/schema.js.map +1 -1
- package/dist/faults/browser-context.d.ts +16 -0
- package/dist/faults/browser-context.js +60 -0
- package/dist/faults/browser-context.js.map +1 -0
- package/dist/faults/cookie-header.d.ts +5 -0
- package/dist/faults/cookie-header.js +25 -0
- package/dist/faults/cookie-header.js.map +1 -0
- package/dist/faults/document-bootstrap.d.ts +5 -0
- package/dist/faults/document-bootstrap.js +141 -0
- package/dist/faults/document-bootstrap.js.map +1 -0
- package/dist/faults/install.d.ts +1 -1
- package/dist/faults/install.js +199 -6
- package/dist/faults/install.js.map +1 -1
- package/dist/faults/pattern.d.ts +1 -0
- package/dist/faults/pattern.js +24 -0
- package/dist/faults/pattern.js.map +1 -0
- package/dist/investigator/agent.js +39 -17
- package/dist/investigator/agent.js.map +1 -1
- package/dist/investigator/ledger.d.ts +1 -1
- package/dist/investigator/ledger.js +2 -2
- package/dist/investigator/ledger.js.map +1 -1
- package/dist/investigator/safe-source.d.ts +1 -0
- package/dist/investigator/safe-source.js +12 -3
- package/dist/investigator/safe-source.js.map +1 -1
- package/dist/investigator/schema.d.ts +293 -2
- package/dist/investigator/schema.js +100 -2
- package/dist/investigator/schema.js.map +1 -1
- package/dist/proof/cli.d.ts +7 -0
- package/dist/proof/cli.js +108 -0
- package/dist/proof/cli.js.map +1 -0
- package/dist/proof/preflight.d.ts +4 -0
- package/dist/proof/preflight.js +9 -0
- package/dist/proof/preflight.js.map +1 -0
- package/dist/repair/generator.d.ts +12 -1
- package/dist/repair/generator.js +30 -6
- package/dist/repair/generator.js.map +1 -1
- package/dist/repair/remote-proof-runner.js +6 -10
- package/dist/repair/remote-proof-runner.js.map +1 -1
- package/dist/repair/schema.d.ts +77 -2
- package/dist/repair/schema.js +23 -2
- package/dist/repair/schema.js.map +1 -1
- package/dist/repair/solari-validator.d.ts +2 -0
- package/dist/repair/solari-validator.js +4 -6
- package/dist/repair/solari-validator.js.map +1 -1
- package/dist/repair/summary.d.ts +10 -0
- package/dist/repair/summary.js +58 -0
- package/dist/repair/summary.js.map +1 -0
- package/dist/repair/validator.d.ts +1 -0
- package/dist/repair/validator.js +53 -12
- package/dist/repair/validator.js.map +1 -1
- package/dist/report/bundle.js +16 -37
- package/dist/report/bundle.js.map +1 -1
- package/dist/report/model.js +131 -12
- package/dist/report/model.js.map +1 -1
- package/dist/report/open.d.ts +9 -0
- package/dist/report/open.js +23 -0
- package/dist/report/open.js.map +1 -1
- package/dist/report/report-client.css +2 -0
- package/dist/report/report-client.js +58 -0
- package/dist/report/schema.d.ts +115 -5
- package/dist/report/schema.js +24 -5
- package/dist/report/schema.js.map +1 -1
- package/dist/reproducer/schema.d.ts +89 -4
- package/dist/reproducer/schema.js +3 -4
- package/dist/reproducer/schema.js.map +1 -1
- package/dist/runner/execution-fault.d.ts +10 -0
- package/dist/runner/execution-fault.js +24 -0
- package/dist/runner/execution-fault.js.map +1 -0
- package/dist/runner/fault-proxy.d.ts +7 -0
- package/dist/runner/fault-proxy.js +366 -0
- package/dist/runner/fault-proxy.js.map +1 -0
- package/dist/runner/local.js +2 -2
- package/dist/runner/local.js.map +1 -1
- package/dist/runner/native-scan.d.ts +25 -0
- package/dist/runner/native-scan.js +306 -0
- package/dist/runner/native-scan.js.map +1 -0
- package/dist/runner/playwright-executor.d.ts +3 -0
- package/dist/runner/playwright-executor.js +125 -66
- package/dist/runner/playwright-executor.js.map +1 -1
- package/dist/runner/process-tree.d.ts +7 -0
- package/dist/runner/process-tree.js +62 -0
- package/dist/runner/process-tree.js.map +1 -0
- package/dist/runner/project-bridge.d.ts +7 -0
- package/dist/runner/project-bridge.js +83 -0
- package/dist/runner/project-bridge.js.map +1 -0
- package/dist/runner/trial-reporter.d.ts +7 -0
- package/dist/runner/trial-reporter.js +47 -0
- package/dist/runner/trial-reporter.js.map +1 -0
- package/dist/scan/failure-clusters.d.ts +24 -0
- package/dist/scan/failure-clusters.js +74 -0
- package/dist/scan/failure-clusters.js.map +1 -0
- package/dist/scan/schema.d.ts +170 -0
- package/dist/scan/schema.js +71 -0
- package/dist/scan/schema.js.map +1 -0
- package/dist/solari/executor.d.ts +5 -1
- package/dist/solari/executor.js +25 -17
- package/dist/solari/executor.js.map +1 -1
- package/dist/solari/run-demo.d.ts +13 -1
- package/dist/solari/run-demo.js +35 -16
- package/dist/solari/run-demo.js.map +1 -1
- package/dist/solari/usage.d.ts +8 -2
- package/dist/solari/usage.js +14 -4
- package/dist/solari/usage.js.map +1 -1
- package/dist/ui/boundary.d.ts +17 -0
- package/dist/ui/boundary.js +18 -0
- package/dist/ui/boundary.js.map +1 -0
- package/dist/ui/console.d.ts +4 -0
- package/dist/ui/console.js +11 -0
- package/dist/ui/console.js.map +1 -0
- package/dist/ui/document.d.ts +40 -0
- package/dist/ui/document.js +120 -0
- package/dist/ui/document.js.map +1 -0
- package/dist/ui/format.d.ts +8 -0
- package/dist/ui/format.js +32 -0
- package/dist/ui/format.js.map +1 -0
- package/dist/ui/progress.d.ts +17 -1
- package/dist/ui/progress.js +51 -8
- package/dist/ui/progress.js.map +1 -1
- package/dist/ui/status.d.ts +7 -0
- package/dist/ui/status.js +36 -0
- package/dist/ui/status.js.map +1 -0
- package/dist/ui/style.d.ts +8 -0
- package/dist/ui/style.js +22 -0
- package/dist/ui/style.js.map +1 -0
- package/dist/ui/text.d.ts +17 -0
- package/dist/ui/text.js +123 -0
- package/dist/ui/text.js.map +1 -0
- package/dist/ui/theme.d.ts +23 -0
- package/dist/ui/theme.js +77 -0
- package/dist/ui/theme.js.map +1 -0
- package/dist/ui/trial-progress.d.ts +13 -0
- package/dist/ui/trial-progress.js +42 -0
- package/dist/ui/trial-progress.js.map +1 -0
- package/package.json +14 -19
- package/dist/faults/network-delay.d.ts +0 -5
- package/dist/faults/network-delay.js +0 -34
- package/dist/faults/network-delay.js.map +0 -1
- package/dist/faults/request-failure.d.ts +0 -5
- package/dist/faults/request-failure.js +0 -24
- package/dist/faults/request-failure.js.map +0 -1
- package/dist/playwright.d.ts +0 -3
- package/dist/playwright.js +0 -21
- package/dist/playwright.js.map +0 -1
- package/dist/report/browser-entry.d.ts +0 -2
- package/dist/report/browser-entry.js +0 -15
- package/dist/report/browser-entry.js.map +0 -1
- package/dist/report/components/artifacts.d.ts +0 -4
- package/dist/report/components/artifacts.js +0 -6
- package/dist/report/components/artifacts.js.map +0 -1
- package/dist/report/components/experiments.d.ts +0 -4
- package/dist/report/components/experiments.js +0 -7
- package/dist/report/components/experiments.js.map +0 -1
- package/dist/report/components/hypotheses.d.ts +0 -4
- package/dist/report/components/hypotheses.js +0 -26
- package/dist/report/components/hypotheses.js.map +0 -1
- package/dist/report/components/indicators.d.ts +0 -32
- package/dist/report/components/indicators.js +0 -42
- package/dist/report/components/indicators.js.map +0 -1
- package/dist/report/components/layout.d.ts +0 -29
- package/dist/report/components/layout.js +0 -20
- package/dist/report/components/layout.js.map +0 -1
- package/dist/report/components/proof.d.ts +0 -4
- package/dist/report/components/proof.js +0 -34
- package/dist/report/components/proof.js.map +0 -1
- package/dist/report/components/reproduction.d.ts +0 -4
- package/dist/report/components/reproduction.js +0 -19
- package/dist/report/components/reproduction.js.map +0 -1
- package/dist/report/components/summary.d.ts +0 -4
- package/dist/report/components/summary.js +0 -12
- package/dist/report/components/summary.js.map +0 -1
- package/dist/report/components/verdict.d.ts +0 -4
- package/dist/report/components/verdict.js +0 -27
- package/dist/report/components/verdict.js.map +0 -1
- package/dist/report/report-app.d.ts +0 -6
- package/dist/report/report-app.js +0 -16
- package/dist/report/report-app.js.map +0 -1
- package/dist/report/report-details.css +0 -182
- package/dist/report/report.css +0 -404
|
@@ -0,0 +1,175 @@
|
|
|
1
|
+
import { TerminalDocument } from "../ui/document.js";
|
|
2
|
+
import { formatCount, formatRate, shellArgument } from "../ui/format.js";
|
|
3
|
+
import { PLAIN_THEME } from "../ui/theme.js";
|
|
4
|
+
const SUMMARY_TEST_LIMIT = 20;
|
|
5
|
+
const STATUS_TONES = {
|
|
6
|
+
errored: "inconclusive",
|
|
7
|
+
"failed-every-run": "failure",
|
|
8
|
+
inconclusive: "inconclusive",
|
|
9
|
+
"mixed-outcomes": "warning",
|
|
10
|
+
"no-failure-observed": "success",
|
|
11
|
+
skipped: "muted",
|
|
12
|
+
};
|
|
13
|
+
const FINDINGS = {
|
|
14
|
+
"failed-every-run": "At least one test failed in every measured run; this looks like a regular test failure.",
|
|
15
|
+
inconclusive: "The scan was inconclusive because Playwright or one or more tests could not run.",
|
|
16
|
+
"mixed-outcomes": "Mixed outcomes observed: at least one test both passed and failed.",
|
|
17
|
+
"no-failure-observed": "No failure was observed in this bounded scan.",
|
|
18
|
+
};
|
|
19
|
+
const TEST_STATUS_PRIORITY = {
|
|
20
|
+
errored: 0,
|
|
21
|
+
"mixed-outcomes": 1,
|
|
22
|
+
"failed-every-run": 2,
|
|
23
|
+
skipped: 3,
|
|
24
|
+
"no-failure-observed": 4,
|
|
25
|
+
};
|
|
26
|
+
export function finding(status) {
|
|
27
|
+
return FINDINGS[status];
|
|
28
|
+
}
|
|
29
|
+
export function failureEvidence(failed, passed, lowerBound80, upperBound80) {
|
|
30
|
+
const measured = failed + passed;
|
|
31
|
+
if (measured === 0) {
|
|
32
|
+
return "no measured pass/fail trials";
|
|
33
|
+
}
|
|
34
|
+
if (failed === 0) {
|
|
35
|
+
return `0/${measured} failures observed · 80% upper bound ${formatRate(upperBound80)}`;
|
|
36
|
+
}
|
|
37
|
+
if (passed === 0) {
|
|
38
|
+
return `${failed}/${measured} failures observed · 80% lower bound ${formatRate(lowerBound80)}`;
|
|
39
|
+
}
|
|
40
|
+
return `${failed}/${measured} failures observed · 80% interval ${formatRate(lowerBound80)}–${formatRate(upperBound80)}`;
|
|
41
|
+
}
|
|
42
|
+
function additionalOutcomes(counts) {
|
|
43
|
+
const outcomes = [];
|
|
44
|
+
if (counts.skipped > 0) {
|
|
45
|
+
outcomes.push(`${counts.skipped} skipped`);
|
|
46
|
+
}
|
|
47
|
+
if (counts.errors > 0) {
|
|
48
|
+
outcomes.push(`${counts.errors} errors`);
|
|
49
|
+
}
|
|
50
|
+
return outcomes.length > 0 ? ` · ${outcomes.join(" · ")}` : "";
|
|
51
|
+
}
|
|
52
|
+
function failureModes(test) {
|
|
53
|
+
if (test.failureClusters.length === 0) {
|
|
54
|
+
return "";
|
|
55
|
+
}
|
|
56
|
+
const modeCount = test.failureClusters.length + test.omittedFailureModes;
|
|
57
|
+
const label = test.multipleFailureModes ? "distinct failure modes" : "failure mode";
|
|
58
|
+
return ` · ${modeCount} ${label}`;
|
|
59
|
+
}
|
|
60
|
+
function compareSummaryOrder(left, right) {
|
|
61
|
+
const priority = TEST_STATUS_PRIORITY[left.status] - TEST_STATUS_PRIORITY[right.status];
|
|
62
|
+
if (priority !== 0) {
|
|
63
|
+
return priority;
|
|
64
|
+
}
|
|
65
|
+
return testIdentity(left).localeCompare(testIdentity(right));
|
|
66
|
+
}
|
|
67
|
+
function testIdentity(test) {
|
|
68
|
+
const { column, file, line, project, titlePath } = test.identity;
|
|
69
|
+
return `${file}:${line}:${column}:${project}:${titlePath.join(":")}`;
|
|
70
|
+
}
|
|
71
|
+
function isProblematic(test) {
|
|
72
|
+
return test.status !== "no-failure-observed" && test.status !== "skipped";
|
|
73
|
+
}
|
|
74
|
+
function appendTest(document, test) {
|
|
75
|
+
const { column, file, line, project, titlePath } = test.identity;
|
|
76
|
+
document.entry(STATUS_TONES[test.status], `${test.status} · ${project}`);
|
|
77
|
+
document.detail(`${file}:${line}:${column} › ${titlePath.join(" › ")}`);
|
|
78
|
+
const evidence = failureEvidence(test.counts.failed, test.counts.passed, test.lowerBound80, test.upperBound80);
|
|
79
|
+
document.detail(`${evidence}${failureModes(test)}${additionalOutcomes(test.counts)}`);
|
|
80
|
+
}
|
|
81
|
+
function appendCollapsedTests(document, tests, ordered, artifactPath) {
|
|
82
|
+
const problematic = ordered.filter(isProblematic);
|
|
83
|
+
const displayed = problematic.slice(0, SUMMARY_TEST_LIMIT);
|
|
84
|
+
document.section(`Problematic tests (showing ${displayed.length} of ${problematic.length})`);
|
|
85
|
+
if (displayed.length === 0) {
|
|
86
|
+
document.entry("muted", "None observed");
|
|
87
|
+
}
|
|
88
|
+
for (const test of displayed) {
|
|
89
|
+
appendTest(document, test);
|
|
90
|
+
}
|
|
91
|
+
const omitted = problematic.length - displayed.length;
|
|
92
|
+
if (omitted > 0) {
|
|
93
|
+
document.entry("muted", `${omitted} additional problematic tests omitted from this summary.`);
|
|
94
|
+
}
|
|
95
|
+
const clean = tests.filter((test) => test.status === "no-failure-observed").length;
|
|
96
|
+
const skipped = tests.filter((test) => test.status === "skipped").length;
|
|
97
|
+
document.entry("muted", `${clean} tests with no failure observed · ${skipped} skipped`);
|
|
98
|
+
document.entry("muted", `Complete per-test evidence is in ${artifactPath}; use --verbose or --json to print it.`);
|
|
99
|
+
}
|
|
100
|
+
function appendTests(document, tests, artifactPath) {
|
|
101
|
+
const ordered = [...tests].sort(compareSummaryOrder);
|
|
102
|
+
if (ordered.length > SUMMARY_TEST_LIMIT) {
|
|
103
|
+
appendCollapsedTests(document, tests, ordered, artifactPath);
|
|
104
|
+
return;
|
|
105
|
+
}
|
|
106
|
+
document.section("Tests");
|
|
107
|
+
for (const test of ordered) {
|
|
108
|
+
appendTest(document, test);
|
|
109
|
+
}
|
|
110
|
+
}
|
|
111
|
+
function nextCommand(scan) {
|
|
112
|
+
const target = shellArgument(scan.target);
|
|
113
|
+
if (scan.status === "no-failure-observed") {
|
|
114
|
+
return `flakelab scan ${target} --runs 20`;
|
|
115
|
+
}
|
|
116
|
+
if (scan.status === "mixed-outcomes") {
|
|
117
|
+
return `flakelab diagnose ${target} --discover`;
|
|
118
|
+
}
|
|
119
|
+
if (scan.status === "inconclusive") {
|
|
120
|
+
return "flakelab doctor";
|
|
121
|
+
}
|
|
122
|
+
return undefined;
|
|
123
|
+
}
|
|
124
|
+
function overviewRows(scan) {
|
|
125
|
+
const counts = scan.totals;
|
|
126
|
+
return [
|
|
127
|
+
{ label: "Target", value: scan.target },
|
|
128
|
+
{
|
|
129
|
+
label: "Scan",
|
|
130
|
+
value: `${formatCount(scan.runs, "run")} · ${formatCount(scan.workers, "worker")}`
|
|
131
|
+
+ ` · ${formatCount(scan.tests.length, "test")}`,
|
|
132
|
+
},
|
|
133
|
+
{
|
|
134
|
+
label: "Outcomes",
|
|
135
|
+
value: `${counts.passed} passed · ${counts.failed} failed · ${counts.skipped} skipped · ${counts.errors} errors`,
|
|
136
|
+
},
|
|
137
|
+
{
|
|
138
|
+
label: "Confidence",
|
|
139
|
+
value: failureEvidence(counts.failed, counts.passed, counts.lowerBound80, counts.upperBound80),
|
|
140
|
+
},
|
|
141
|
+
];
|
|
142
|
+
}
|
|
143
|
+
export function formatScanSummary(scan, artifactPath, theme = PLAIN_THEME) {
|
|
144
|
+
const document = new TerminalDocument(theme);
|
|
145
|
+
document.heading("stability scan");
|
|
146
|
+
document.verdict(STATUS_TONES[scan.status], scan.status, finding(scan.status));
|
|
147
|
+
document.blank().rows(overviewRows(scan));
|
|
148
|
+
appendTests(document, scan.tests, artifactPath);
|
|
149
|
+
if (scan.runnerErrors.length > 0) {
|
|
150
|
+
document.section("Runner errors");
|
|
151
|
+
for (const error of scan.runnerErrors) {
|
|
152
|
+
document.entry("failure", error);
|
|
153
|
+
}
|
|
154
|
+
}
|
|
155
|
+
document.section("Evidence");
|
|
156
|
+
const evidence = [{ label: "Scan artifact", value: artifactPath }];
|
|
157
|
+
if (scan.playwrightOutputDirectory) {
|
|
158
|
+
evidence.push({ label: "Playwright artifacts", value: scan.playwrightOutputDirectory });
|
|
159
|
+
}
|
|
160
|
+
document.rows(evidence);
|
|
161
|
+
const next = nextCommand(scan);
|
|
162
|
+
if (next) {
|
|
163
|
+
document.section("Next").command(next);
|
|
164
|
+
}
|
|
165
|
+
return document.render();
|
|
166
|
+
}
|
|
167
|
+
export function formatScanOutput(scanResult, values, artifactPath, theme = PLAIN_THEME) {
|
|
168
|
+
const evidence = JSON.stringify(scanResult, null, 2);
|
|
169
|
+
if (values.json) {
|
|
170
|
+
return evidence;
|
|
171
|
+
}
|
|
172
|
+
const summary = formatScanSummary(scanResult, artifactPath, theme);
|
|
173
|
+
return values.verbose ? `${summary}\n\n${evidence}` : summary;
|
|
174
|
+
}
|
|
175
|
+
//# sourceMappingURL=scan-summary.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"scan-summary.js","sourceRoot":"","sources":["../../src/commands/scan-summary.ts"],"names":[],"mappings":"AAEA,OAAO,EAAE,gBAAgB,EAAE,MAAM,mBAAmB,CAAA;AACpD,OAAO,EAAE,WAAW,EAAE,UAAU,EAAE,aAAa,EAAE,MAAM,iBAAiB,CAAA;AAExE,OAAO,EAAE,WAAW,EAAE,MAAM,gBAAgB,CAAA;AAI5C,MAAM,kBAAkB,GAAG,EAAE,CAAA;AAE7B,MAAM,YAAY,GAA2D;IAC3E,OAAO,EAAE,cAAc;IACvB,kBAAkB,EAAE,SAAS;IAC7B,YAAY,EAAE,cAAc;IAC5B,gBAAgB,EAAE,SAAS;IAC3B,qBAAqB,EAAE,SAAS;IAChC,OAAO,EAAE,OAAO;CACjB,CAAA;AAED,MAAM,QAAQ,GAA+B;IAC3C,kBAAkB,EAAE,yFAAyF;IAC7G,YAAY,EAAE,kFAAkF;IAChG,gBAAgB,EAAE,oEAAoE;IACtF,qBAAqB,EAAE,+CAA+C;CACvE,CAAA;AAED,MAAM,oBAAoB,GAA6C;IACrE,OAAO,EAAE,CAAC;IACV,gBAAgB,EAAE,CAAC;IACnB,kBAAkB,EAAE,CAAC;IACrB,OAAO,EAAE,CAAC;IACV,qBAAqB,EAAE,CAAC;CACzB,CAAA;AAED,MAAM,UAAU,OAAO,CAAC,MAAkB;IACxC,OAAO,QAAQ,CAAC,MAAM,CAAC,CAAA;AACzB,CAAC;AAED,MAAM,UAAU,eAAe,CAC7B,MAAc,EACd,MAAc,EACd,YAAoB,EACpB,YAAoB;IAEpB,MAAM,QAAQ,GAAG,MAAM,GAAG,MAAM,CAAA;IAChC,IAAI,QAAQ,KAAK,CAAC,EAAE,CAAC;QACnB,OAAO,8BAA8B,CAAA;IACvC,CAAC;IACD,IAAI,MAAM,KAAK,CAAC,EAAE,CAAC;QACjB,OAAO,KAAK,QAAQ,wCAAwC,UAAU,CAAC,YAAY,CAAC,EAAE,CAAA;IACxF,CAAC;IACD,IAAI,MAAM,KAAK,CAAC,EAAE,CAAC;QACjB,OAAO,GAAG,MAAM,IAAI,QAAQ,wCAAwC,UAAU,CAAC,YAAY,CAAC,EAAE,CAAA;IAChG,CAAC;IACD,OAAO,GAAG,MAAM,IAAI,QAAQ,qCAAqC,UAAU,CAAC,YAAY,CAAC,IAAI,UAAU,CAAC,YAAY,CAAC,EAAE,CAAA;AACzH,CAAC;AAED,SAAS,kBAAkB,CAAC,MAAkB;IAC5C,MAAM,QAAQ,GAAa,EAAE,CAAA;IAC7B,IAAI,MAAM,CAAC,OAAO,GAAG,CAAC,EAAE,CAAC;QACvB,QAAQ,CAAC,IAAI,CAAC,GAAG,MAAM,CAAC,OAAO,UAAU,CAAC,CAAA;IAC5C,CAAC;IACD,IAAI,MAAM,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QACtB,QAAQ,CAAC,IAAI,CAAC,GAAG,MAAM,CAAC,MAAM,SAAS,CAAC,CAAA;IAC1C,CAAC;IACD,OAAO,QAAQ,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,MAAM,QAAQ,CAAC,IAAI,CAAC,KAAK,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,CAAA;AAChE,CAAC;AAED,SAAS,YAAY,CAAC,IAAoB;IACxC,IAAI,IAAI,CAAC,eAAe,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QACtC,OAAO,EAAE,CAAA;IACX,CAAC;IACD,MAAM,SAAS,GAAG,IAAI,CAAC,eAAe,CAAC,MAAM,GAAG,IAAI,CAAC,mBAAmB,CAAA;IACxE,MAAM,KAAK,GAAG,IAAI,CAAC,oBAAoB,CAAC,CAAC,CAAC,wBAAwB,CAAC,CAAC,CAAC,cAAc,CAAA;IACnF,OAAO,MAAM,SAAS,IAAI,KAAK,EAAE,CAAA;AACnC,CAAC;AAED,SAAS,mBAAmB,CAAC,IAAoB,EAAE,KAAqB;IACtE,MAAM,QAAQ,GAAG,oBAAoB,CAAC,IAAI,CAAC,MAAM,CAAC,GAAG,oBAAoB,CAAC,KAAK,CAAC,MAAM,CAAC,CAAA;IACvF,IAAI,QAAQ,KAAK,CAAC,EAAE,CAAC;QACnB,OAAO,QAAQ,CAAA;IACjB,CAAC;IACD,OAAO,YAAY,CAAC,IAAI,CAAC,CAAC,aAAa,CAAC,YAAY,CAAC,KAAK,CAAC,CAAC,CAAA;AAC9D,CAAC;AAED,SAAS,YAAY,CAAC,IAAoB;IACxC,MAAM,EAAE,MAAM,EAAE,IAAI,EAAE,IAAI,EAAE,OAAO,EAAE,SAAS,EAAE,GAAG,IAAI,CAAC,QAAQ,CAAA;IAChE,OAAO,GAAG,IAAI,IAAI,IAAI,IAAI,MAAM,IAAI,OAAO,IAAI,SAAS,CAAC,IAAI,CAAC,GAAG,CAAC,EAAE,CAAA;AACtE,CAAC;AAED,SAAS,aAAa,CAAC,IAAoB;IACzC,OAAO,IAAI,CAAC,MAAM,KAAK,qBAAqB,IAAI,IAAI,CAAC,MAAM,KAAK,SAAS,CAAA;AAC3E,CAAC;AAED,SAAS,UAAU,CAAC,QAA0B,EAAE,IAAoB;IAClE,MAAM,EAAE,MAAM,EAAE,IAAI,EAAE,IAAI,EAAE,OAAO,EAAE,SAAS,EAAE,GAAG,IAAI,CAAC,QAAQ,CAAA;IAChE,QAAQ,CAAC,KAAK,CAAC,YAAY,CAAC,IAAI,CAAC,MAAM,CAAC,EAAE,GAAG,IAAI,CAAC,MAAM,MAAM,OAAO,EAAE,CAAC,CAAA;IACxE,QAAQ,CAAC,MAAM,CAAC,GAAG,IAAI,IAAI,IAAI,IAAI,MAAM,MAAM,SAAS,CAAC,IAAI,CAAC,KAAK,CAAC,EAAE,CAAC,CAAA;IACvE,MAAM,QAAQ,GAAG,eAAe,CAC9B,IAAI,CAAC,MAAM,CAAC,MAAM,EAClB,IAAI,CAAC,MAAM,CAAC,MAAM,EAClB,IAAI,CAAC,YAAY,EACjB,IAAI,CAAC,YAAY,CAClB,CAAA;IACD,QAAQ,CAAC,MAAM,CAAC,GAAG,QAAQ,GAAG,YAAY,CAAC,IAAI,CAAC,GAAG,kBAAkB,CAAC,IAAI,CAAC,MAAM,CAAC,EAAE,CAAC,CAAA;AACvF,CAAC;AAED,SAAS,oBAAoB,CAC3B,QAA0B,EAC1B,KAAuB,EACvB,OAAyB,EACzB,YAAoB;IAEpB,MAAM,WAAW,GAAG,OAAO,CAAC,MAAM,CAAC,aAAa,CAAC,CAAA;IACjD,MAAM,SAAS,GAAG,WAAW,CAAC,KAAK,CAAC,CAAC,EAAE,kBAAkB,CAAC,CAAA;IAC1D,QAAQ,CAAC,OAAO,CAAC,8BAA8B,SAAS,CAAC,MAAM,OAAO,WAAW,CAAC,MAAM,GAAG,CAAC,CAAA;IAC5F,IAAI,SAAS,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QAC3B,QAAQ,CAAC,KAAK,CAAC,OAAO,EAAE,eAAe,CAAC,CAAA;IAC1C,CAAC;IACD,KAAK,MAAM,IAAI,IAAI,SAAS,EAAE,CAAC;QAC7B,UAAU,CAAC,QAAQ,EAAE,IAAI,CAAC,CAAA;IAC5B,CAAC;IACD,MAAM,OAAO,GAAG,WAAW,CAAC,MAAM,GAAG,SAAS,CAAC,MAAM,CAAA;IACrD,IAAI,OAAO,GAAG,CAAC,EAAE,CAAC;QAChB,QAAQ,CAAC,KAAK,CAAC,OAAO,EAAE,GAAG,OAAO,0DAA0D,CAAC,CAAA;IAC/F,CAAC;IACD,MAAM,KAAK,GAAG,KAAK,CAAC,MAAM,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,MAAM,KAAK,qBAAqB,CAAC,CAAC,MAAM,CAAA;IAClF,MAAM,OAAO,GAAG,KAAK,CAAC,MAAM,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,MAAM,KAAK,SAAS,CAAC,CAAC,MAAM,CAAA;IACxE,QAAQ,CAAC,KAAK,CAAC,OAAO,EAAE,GAAG,KAAK,qCAAqC,OAAO,UAAU,CAAC,CAAA;IACvF,QAAQ,CAAC,KAAK,CACZ,OAAO,EACP,oCAAoC,YAAY,wCAAwC,CACzF,CAAA;AACH,CAAC;AAED,SAAS,WAAW,CAClB,QAA0B,EAC1B,KAAuB,EACvB,YAAoB;IAEpB,MAAM,OAAO,GAAG,CAAC,GAAG,KAAK,CAAC,CAAC,IAAI,CAAC,mBAAmB,CAAC,CAAA;IACpD,IAAI,OAAO,CAAC,MAAM,GAAG,kBAAkB,EAAE,CAAC;QACxC,oBAAoB,CAAC,QAAQ,EAAE,KAAK,EAAE,OAAO,EAAE,YAAY,CAAC,CAAA;QAC5D,OAAM;IACR,CAAC;IACD,QAAQ,CAAC,OAAO,CAAC,OAAO,CAAC,CAAA;IACzB,KAAK,MAAM,IAAI,IAAI,OAAO,EAAE,CAAC;QAC3B,UAAU,CAAC,QAAQ,EAAE,IAAI,CAAC,CAAA;IAC5B,CAAC;AACH,CAAC;AAED,SAAS,WAAW,CAAC,IAAkB;IACrC,MAAM,MAAM,GAAG,aAAa,CAAC,IAAI,CAAC,MAAM,CAAC,CAAA;IACzC,IAAI,IAAI,CAAC,MAAM,KAAK,qBAAqB,EAAE,CAAC;QAC1C,OAAO,iBAAiB,MAAM,YAAY,CAAA;IAC5C,CAAC;IACD,IAAI,IAAI,CAAC,MAAM,KAAK,gBAAgB,EAAE,CAAC;QACrC,OAAO,qBAAqB,MAAM,aAAa,CAAA;IACjD,CAAC;IACD,IAAI,IAAI,CAAC,MAAM,KAAK,cAAc,EAAE,CAAC;QACnC,OAAO,iBAAiB,CAAA;IAC1B,CAAC;IACD,OAAO,SAAS,CAAA;AAClB,CAAC;AAED,SAAS,YAAY,CAAC,IAAkB;IACtC,MAAM,MAAM,GAAG,IAAI,CAAC,MAAM,CAAA;IAC1B,OAAO;QACL,EAAE,KAAK,EAAE,QAAQ,EAAE,KAAK,EAAE,IAAI,CAAC,MAAM,EAAE;QACvC;YACE,KAAK,EAAE,MAAM;YACb,KAAK,EAAE,GAAG,WAAW,CAAC,IAAI,CAAC,IAAI,EAAE,KAAK,CAAC,MAAM,WAAW,CAAC,IAAI,CAAC,OAAO,EAAE,QAAQ,CAAC,EAAE;kBAC9E,MAAM,WAAW,CAAC,IAAI,CAAC,KAAK,CAAC,MAAM,EAAE,MAAM,CAAC,EAAE;SACnD;QACD;YACE,KAAK,EAAE,UAAU;YACjB,KAAK,EAAE,GAAG,MAAM,CAAC,MAAM,aAAa,MAAM,CAAC,MAAM,aAAa,MAAM,CAAC,OAAO,cAAc,MAAM,CAAC,MAAM,SAAS;SACjH;QACD;YACE,KAAK,EAAE,YAAY;YACnB,KAAK,EAAE,eAAe,CACpB,MAAM,CAAC,MAAM,EACb,MAAM,CAAC,MAAM,EACb,MAAM,CAAC,YAAY,EACnB,MAAM,CAAC,YAAY,CACpB;SACF;KACF,CAAA;AACH,CAAC;AAED,MAAM,UAAU,iBAAiB,CAC/B,IAAkB,EAClB,YAAoB,EACpB,QAAuB,WAAW;IAElC,MAAM,QAAQ,GAAG,IAAI,gBAAgB,CAAC,KAAK,CAAC,CAAA;IAC5C,QAAQ,CAAC,OAAO,CAAC,gBAAgB,CAAC,CAAA;IAClC,QAAQ,CAAC,OAAO,CAAC,YAAY,CAAC,IAAI,CAAC,MAAM,CAAC,EAAE,IAAI,CAAC,MAAM,EAAE,OAAO,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC,CAAA;IAC9E,QAAQ,CAAC,KAAK,EAAE,CAAC,IAAI,CAAC,YAAY,CAAC,IAAI,CAAC,CAAC,CAAA;IACzC,WAAW,CAAC,QAAQ,EAAE,IAAI,CAAC,KAAK,EAAE,YAAY,CAAC,CAAA;IAC/C,IAAI,IAAI,CAAC,YAAY,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QACjC,QAAQ,CAAC,OAAO,CAAC,eAAe,CAAC,CAAA;QACjC,KAAK,MAAM,KAAK,IAAI,IAAI,CAAC,YAAY,EAAE,CAAC;YACtC,QAAQ,CAAC,KAAK,CAAC,SAAS,EAAE,KAAK,CAAC,CAAA;QAClC,CAAC;IACH,CAAC;IACD,QAAQ,CAAC,OAAO,CAAC,UAAU,CAAC,CAAA;IAC5B,MAAM,QAAQ,GAAkB,CAAC,EAAE,KAAK,EAAE,eAAe,EAAE,KAAK,EAAE,YAAY,EAAE,CAAC,CAAA;IACjF,IAAI,IAAI,CAAC,yBAAyB,EAAE,CAAC;QACnC,QAAQ,CAAC,IAAI,CAAC,EAAE,KAAK,EAAE,sBAAsB,EAAE,KAAK,EAAE,IAAI,CAAC,yBAAyB,EAAE,CAAC,CAAA;IACzF,CAAC;IACD,QAAQ,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAA;IACvB,MAAM,IAAI,GAAG,WAAW,CAAC,IAAI,CAAC,CAAA;IAC9B,IAAI,IAAI,EAAE,CAAC;QACT,QAAQ,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC,OAAO,CAAC,IAAI,CAAC,CAAA;IACxC,CAAC;IACD,OAAO,QAAQ,CAAC,MAAM,EAAE,CAAA;AAC1B,CAAC;AAED,MAAM,UAAU,gBAAgB,CAC9B,UAAwB,EACxB,MAA6C,EAC7C,YAAoB,EACpB,QAAuB,WAAW;IAElC,MAAM,QAAQ,GAAG,IAAI,CAAC,SAAS,CAAC,UAAU,EAAE,IAAI,EAAE,CAAC,CAAC,CAAA;IACpD,IAAI,MAAM,CAAC,IAAI,EAAE,CAAC;QAChB,OAAO,QAAQ,CAAA;IACjB,CAAC;IACD,MAAM,OAAO,GAAG,iBAAiB,CAAC,UAAU,EAAE,YAAY,EAAE,KAAK,CAAC,CAAA;IAClE,OAAO,MAAM,CAAC,OAAO,CAAC,CAAC,CAAC,GAAG,OAAO,OAAO,QAAQ,EAAE,CAAC,CAAC,CAAC,OAAO,CAAA;AAC/D,CAAC"}
|
package/dist/commands/scan.d.ts
CHANGED
|
@@ -1,12 +1,9 @@
|
|
|
1
|
-
import type {
|
|
2
|
-
import type {
|
|
3
|
-
export type ScanStatus
|
|
4
|
-
export
|
|
5
|
-
|
|
6
|
-
|
|
7
|
-
|
|
8
|
-
|
|
9
|
-
|
|
10
|
-
export declare function classifyScan(result: ExperimentResult): ScanStatus;
|
|
11
|
-
export declare function formatScanSummary(scan: ScanResult): string;
|
|
12
|
-
export declare function scan(target: string, values: CliValues): Promise<ScanResult>;
|
|
1
|
+
import type { ScanArtifact, ScanStatus, ScanTestResult, ScanTotals } from "../scan/schema.js";
|
|
2
|
+
import type { ScanOptions } from "./options.js";
|
|
3
|
+
export type { ScanStatus } from "../scan/schema.js";
|
|
4
|
+
export type ScanResult = ScanArtifact;
|
|
5
|
+
export { failureEvidence, finding, formatScanOutput, formatScanSummary, } from "./scan-summary.js";
|
|
6
|
+
export declare function classifyScan(tests: ScanTestResult[], runnerErrors: string[]): ScanStatus;
|
|
7
|
+
export declare function scanExitCode(status: ScanStatus): 0 | 1 | 2;
|
|
8
|
+
export declare function summarizeScanTests(tests: ScanTestResult[]): ScanTotals;
|
|
9
|
+
export declare function scan(target: string, values: ScanOptions): Promise<ScanResult>;
|
package/dist/commands/scan.js
CHANGED
|
@@ -1,86 +1,90 @@
|
|
|
1
1
|
import { mkdir, writeFile } from "node:fs/promises";
|
|
2
2
|
import { dirname, resolve } from "node:path";
|
|
3
|
-
import {
|
|
4
|
-
import {
|
|
3
|
+
import { portableProjectPath } from "../artifacts/paths.js";
|
|
4
|
+
import { failureConfidence, runNativePlaywrightScan } from "../runner/native-scan.js";
|
|
5
|
+
import { scanArtifactSchema } from "../scan/schema.js";
|
|
5
6
|
import { ProgressReporter } from "../ui/progress.js";
|
|
7
|
+
import { formatCount } from "../ui/format.js";
|
|
8
|
+
import { writeStdout } from "../ui/console.js";
|
|
9
|
+
import { stdoutTheme } from "../ui/theme.js";
|
|
10
|
+
import { formatScanOutput } from "./scan-summary.js";
|
|
6
11
|
import { integerOption, withInterruption } from "./options.js";
|
|
7
|
-
export
|
|
8
|
-
|
|
12
|
+
export { failureEvidence, finding, formatScanOutput, formatScanSummary, } from "./scan-summary.js";
|
|
13
|
+
const SCAN_EXIT_CODES = {
|
|
14
|
+
"failed-every-run": 1,
|
|
15
|
+
inconclusive: 2,
|
|
16
|
+
"mixed-outcomes": 1,
|
|
17
|
+
"no-failure-observed": 0,
|
|
18
|
+
};
|
|
19
|
+
function sumCounts(tests) {
|
|
20
|
+
return tests.reduce((total, test) => ({
|
|
21
|
+
errors: total.errors + test.counts.errors,
|
|
22
|
+
failed: total.failed + test.counts.failed,
|
|
23
|
+
passed: total.passed + test.counts.passed,
|
|
24
|
+
skipped: total.skipped + test.counts.skipped,
|
|
25
|
+
}), { errors: 0, failed: 0, passed: 0, skipped: 0 });
|
|
26
|
+
}
|
|
27
|
+
export function classifyScan(tests, runnerErrors) {
|
|
28
|
+
if (runnerErrors.length > 0 || tests.length === 0 || tests.every((test) => test.status === "skipped")) {
|
|
29
|
+
return "inconclusive";
|
|
30
|
+
}
|
|
31
|
+
if (tests.some((test) => test.status === "errored")) {
|
|
9
32
|
return "inconclusive";
|
|
10
33
|
}
|
|
11
|
-
if (
|
|
12
|
-
return "
|
|
34
|
+
if (tests.some((test) => test.status === "mixed-outcomes")) {
|
|
35
|
+
return "mixed-outcomes";
|
|
13
36
|
}
|
|
14
|
-
if (
|
|
15
|
-
return "
|
|
37
|
+
if (tests.some((test) => test.status === "failed-every-run")) {
|
|
38
|
+
return "failed-every-run";
|
|
16
39
|
}
|
|
17
|
-
return "
|
|
40
|
+
return "no-failure-observed";
|
|
18
41
|
}
|
|
19
|
-
function
|
|
20
|
-
return
|
|
42
|
+
export function scanExitCode(status) {
|
|
43
|
+
return SCAN_EXIT_CODES[status];
|
|
21
44
|
}
|
|
22
|
-
function
|
|
23
|
-
const
|
|
24
|
-
|
|
25
|
-
|
|
26
|
-
|
|
27
|
-
|
|
45
|
+
export function summarizeScanTests(tests) {
|
|
46
|
+
const counts = sumCounts(tests);
|
|
47
|
+
const measuredTrials = counts.passed + counts.failed;
|
|
48
|
+
return {
|
|
49
|
+
...counts,
|
|
50
|
+
executions: measuredTrials + counts.skipped + counts.errors,
|
|
51
|
+
failureRate: measuredTrials === 0 ? 0 : counts.failed / measuredTrials,
|
|
52
|
+
lowerBound80: failureConfidence(counts.failed, measuredTrials).lowerBound80,
|
|
53
|
+
upperBound80: failureConfidence(counts.failed, measuredTrials).upperBound80,
|
|
28
54
|
};
|
|
29
|
-
return findings[status];
|
|
30
|
-
}
|
|
31
|
-
export function formatScanSummary(scan) {
|
|
32
|
-
const lines = [
|
|
33
|
-
"",
|
|
34
|
-
"FlakeLab stability scan",
|
|
35
|
-
` Target ${scan.target}`,
|
|
36
|
-
` Runs ${scan.result.trials}`,
|
|
37
|
-
` Passed ${scan.result.passed}`,
|
|
38
|
-
` Failed ${scan.result.failed}`,
|
|
39
|
-
` Failure rate ${percentage(scan.result.failureRate)}`,
|
|
40
|
-
"",
|
|
41
|
-
finding(scan.status),
|
|
42
|
-
];
|
|
43
|
-
if (scan.result.dominantFailureReason) {
|
|
44
|
-
const reason = scan.result.dominantFailureReason
|
|
45
|
-
.split("\n")
|
|
46
|
-
.map((line) => ` ${line}`);
|
|
47
|
-
lines.push("", "Most common failure", ...reason);
|
|
48
|
-
}
|
|
49
|
-
lines.push("", `Evidence saved to ${scan.artifactPath}`);
|
|
50
|
-
if (scan.status === "no-failure-observed") {
|
|
51
|
-
lines.push("Run again with --runs 20 for a stronger scan.");
|
|
52
|
-
}
|
|
53
|
-
return lines.join("\n");
|
|
54
55
|
}
|
|
55
56
|
export async function scan(target, values) {
|
|
56
57
|
const projectRoot = process.cwd();
|
|
57
58
|
const artifactPath = resolve(projectRoot, values.artifacts, "scan.json");
|
|
59
|
+
const portableArtifactPath = portableProjectPath(projectRoot, artifactPath);
|
|
58
60
|
const progress = new ProgressReporter();
|
|
59
61
|
const runs = integerOption(values.runs, "runs");
|
|
60
|
-
|
|
61
|
-
|
|
62
|
-
|
|
63
|
-
|
|
64
|
-
|
|
62
|
+
const workers = integerOption(values.concurrency, "concurrency");
|
|
63
|
+
progress.start("stability scan", `${formatCount(runs, "native Playwright run")} · ${formatCount(workers, "worker")}`);
|
|
64
|
+
const nativeResult = await withInterruption(async (signal) => runNativePlaywrightScan(projectRoot, target, {
|
|
65
|
+
artifactDirectory: values.artifacts,
|
|
66
|
+
runs,
|
|
65
67
|
signal,
|
|
66
|
-
|
|
68
|
+
workers,
|
|
67
69
|
}));
|
|
68
|
-
|
|
69
|
-
|
|
70
|
-
|
|
71
|
-
|
|
72
|
-
|
|
73
|
-
|
|
74
|
-
|
|
70
|
+
const status = classifyScan(nativeResult.tests, nativeResult.runnerErrors);
|
|
71
|
+
progress.done(`${status} · ${formatCount(nativeResult.tests.length, "test")} classified`
|
|
72
|
+
+ ` · ${formatCount(nativeResult.runnerErrors.length, "runner error")}`);
|
|
73
|
+
const scanResult = scanArtifactSchema.parse({
|
|
74
|
+
generatedAt: new Date().toISOString(),
|
|
75
|
+
playwrightOutputDirectory: nativeResult.playwrightOutputDirectory,
|
|
76
|
+
runs,
|
|
77
|
+
runnerErrors: nativeResult.runnerErrors,
|
|
78
|
+
status,
|
|
79
|
+
target: portableProjectPath(projectRoot, target),
|
|
80
|
+
tests: nativeResult.tests,
|
|
81
|
+
totals: summarizeScanTests(nativeResult.tests),
|
|
82
|
+
workers,
|
|
83
|
+
});
|
|
75
84
|
await mkdir(dirname(artifactPath), { recursive: true });
|
|
76
85
|
await writeFile(artifactPath, `${JSON.stringify(scanResult, null, 2)}\n`, "utf8");
|
|
77
|
-
|
|
78
|
-
|
|
79
|
-
process.stdout.write(`\n${JSON.stringify(scanResult, null, 2)}\n`);
|
|
80
|
-
}
|
|
81
|
-
if (scanResult.status !== "no-failure-observed") {
|
|
82
|
-
process.exitCode = 1;
|
|
83
|
-
}
|
|
86
|
+
writeStdout(formatScanOutput(scanResult, values, portableArtifactPath, stdoutTheme()));
|
|
87
|
+
process.exitCode = scanExitCode(scanResult.status);
|
|
84
88
|
return scanResult;
|
|
85
89
|
}
|
|
86
90
|
//# sourceMappingURL=scan.js.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"scan.js","sourceRoot":"","sources":["../../src/commands/scan.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,KAAK,EAAE,SAAS,EAAE,MAAM,kBAAkB,CAAA;AACnD,OAAO,EAAE,OAAO,EAAE,OAAO,EAAE,MAAM,WAAW,CAAA;
|
|
1
|
+
{"version":3,"file":"scan.js","sourceRoot":"","sources":["../../src/commands/scan.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,KAAK,EAAE,SAAS,EAAE,MAAM,kBAAkB,CAAA;AACnD,OAAO,EAAE,OAAO,EAAE,OAAO,EAAE,MAAM,WAAW,CAAA;AAE5C,OAAO,EAAE,mBAAmB,EAAE,MAAM,uBAAuB,CAAA;AAC3D,OAAO,EAAE,iBAAiB,EAAE,uBAAuB,EAAE,MAAM,0BAA0B,CAAA;AAQrF,OAAO,EAAE,kBAAkB,EAAE,MAAM,mBAAmB,CAAA;AACtD,OAAO,EAAE,gBAAgB,EAAE,MAAM,mBAAmB,CAAA;AACpD,OAAO,EAAE,WAAW,EAAE,MAAM,iBAAiB,CAAA;AAC7C,OAAO,EAAE,WAAW,EAAE,MAAM,kBAAkB,CAAA;AAC9C,OAAO,EAAE,WAAW,EAAE,MAAM,gBAAgB,CAAA;AAC5C,OAAO,EAAE,gBAAgB,EAAE,MAAM,mBAAmB,CAAA;AAEpD,OAAO,EAAE,aAAa,EAAE,gBAAgB,EAAE,MAAM,cAAc,CAAA;AAK9D,OAAO,EACL,eAAe,EACf,OAAO,EACP,gBAAgB,EAChB,iBAAiB,GAClB,MAAM,mBAAmB,CAAA;AAE1B,MAAM,eAAe,GAAkC;IACrD,kBAAkB,EAAE,CAAC;IACrB,YAAY,EAAE,CAAC;IACf,gBAAgB,EAAE,CAAC;IACnB,qBAAqB,EAAE,CAAC;CACzB,CAAA;AAED,SAAS,SAAS,CAAC,KAAuB;IACxC,OAAO,KAAK,CAAC,MAAM,CAAa,CAAC,KAAK,EAAE,IAAI,EAAE,EAAE,CAAC,CAAC;QAChD,MAAM,EAAE,KAAK,CAAC,MAAM,GAAG,IAAI,CAAC,MAAM,CAAC,MAAM;QACzC,MAAM,EAAE,KAAK,CAAC,MAAM,GAAG,IAAI,CAAC,MAAM,CAAC,MAAM;QACzC,MAAM,EAAE,KAAK,CAAC,MAAM,GAAG,IAAI,CAAC,MAAM,CAAC,MAAM;QACzC,OAAO,EAAE,KAAK,CAAC,OAAO,GAAG,IAAI,CAAC,MAAM,CAAC,OAAO;KAC7C,CAAC,EAAE,EAAE,MAAM,EAAE,CAAC,EAAE,MAAM,EAAE,CAAC,EAAE,MAAM,EAAE,CAAC,EAAE,OAAO,EAAE,CAAC,EAAE,CAAC,CAAA;AACtD,CAAC;AAED,MAAM,UAAU,YAAY,CAAC,KAAuB,EAAE,YAAsB;IAC1E,IAAI,YAAY,CAAC,MAAM,GAAG,CAAC,IAAI,KAAK,CAAC,MAAM,KAAK,CAAC,IAAI,KAAK,CAAC,KAAK,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,MAAM,KAAK,SAAS,CAAC,EAAE,CAAC;QACtG,OAAO,cAAc,CAAA;IACvB,CAAC;IACD,IAAI,KAAK,CAAC,IAAI,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,MAAM,KAAK,SAAS,CAAC,EAAE,CAAC;QACpD,OAAO,cAAc,CAAA;IACvB,CAAC;IACD,IAAI,KAAK,CAAC,IAAI,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,MAAM,KAAK,gBAAgB,CAAC,EAAE,CAAC;QAC3D,OAAO,gBAAgB,CAAA;IACzB,CAAC;IACD,IAAI,KAAK,CAAC,IAAI,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,MAAM,KAAK,kBAAkB,CAAC,EAAE,CAAC;QAC7D,OAAO,kBAAkB,CAAA;IAC3B,CAAC;IACD,OAAO,qBAAqB,CAAA;AAC9B,CAAC;AAED,MAAM,UAAU,YAAY,CAAC,MAAkB;IAC7C,OAAO,eAAe,CAAC,MAAM,CAAC,CAAA;AAChC,CAAC;AAED,MAAM,UAAU,kBAAkB,CAAC,KAAuB;IACxD,MAAM,MAAM,GAAG,SAAS,CAAC,KAAK,CAAC,CAAA;IAC/B,MAAM,cAAc,GAAG,MAAM,CAAC,MAAM,GAAG,MAAM,CAAC,MAAM,CAAA;IACpD,OAAO;QACL,GAAG,MAAM;QACT,UAAU,EAAE,cAAc,GAAG,MAAM,CAAC,OAAO,GAAG,MAAM,CAAC,MAAM;QAC3D,WAAW,EAAE,cAAc,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,MAAM,GAAG,cAAc;QACtE,YAAY,EAAE,iBAAiB,CAAC,MAAM,CAAC,MAAM,EAAE,cAAc,CAAC,CAAC,YAAY;QAC3E,YAAY,EAAE,iBAAiB,CAAC,MAAM,CAAC,MAAM,EAAE,cAAc,CAAC,CAAC,YAAY;KAC5E,CAAA;AACH,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,IAAI,CAAC,MAAc,EAAE,MAAmB;IAC5D,MAAM,WAAW,GAAG,OAAO,CAAC,GAAG,EAAE,CAAA;IACjC,MAAM,YAAY,GAAG,OAAO,CAAC,WAAW,EAAE,MAAM,CAAC,SAAS,EAAE,WAAW,CAAC,CAAA;IACxE,MAAM,oBAAoB,GAAG,mBAAmB,CAAC,WAAW,EAAE,YAAY,CAAC,CAAA;IAC3E,MAAM,QAAQ,GAAG,IAAI,gBAAgB,EAAE,CAAA;IACvC,MAAM,IAAI,GAAG,aAAa,CAAC,MAAM,CAAC,IAAI,EAAE,MAAM,CAAC,CAAA;IAC/C,MAAM,OAAO,GAAG,aAAa,CAAC,MAAM,CAAC,WAAW,EAAE,aAAa,CAAC,CAAA;IAChE,QAAQ,CAAC,KAAK,CACZ,gBAAgB,EAChB,GAAG,WAAW,CAAC,IAAI,EAAE,uBAAuB,CAAC,MAAM,WAAW,CAAC,OAAO,EAAE,QAAQ,CAAC,EAAE,CACpF,CAAA;IACD,MAAM,YAAY,GAAG,MAAM,gBAAgB,CAAC,KAAK,EAAE,MAAM,EAAE,EAAE,CAAC,uBAAuB,CACnF,WAAW,EACX,MAAM,EACN;QACE,iBAAiB,EAAE,MAAM,CAAC,SAAS;QACnC,IAAI;QACJ,MAAM;QACN,OAAO;KACR,CACF,CAAC,CAAA;IACF,MAAM,MAAM,GAAG,YAAY,CAAC,YAAY,CAAC,KAAK,EAAE,YAAY,CAAC,YAAY,CAAC,CAAA;IAC1E,QAAQ,CAAC,IAAI,CACX,GAAG,MAAM,MAAM,WAAW,CAAC,YAAY,CAAC,KAAK,CAAC,MAAM,EAAE,MAAM,CAAC,aAAa;UACxE,MAAM,WAAW,CAAC,YAAY,CAAC,YAAY,CAAC,MAAM,EAAE,cAAc,CAAC,EAAE,CACxE,CAAA;IACD,MAAM,UAAU,GAAG,kBAAkB,CAAC,KAAK,CAAC;QAC1C,WAAW,EAAE,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE;QACrC,yBAAyB,EAAE,YAAY,CAAC,yBAAyB;QACjE,IAAI;QACJ,YAAY,EAAE,YAAY,CAAC,YAAY;QACvC,MAAM;QACN,MAAM,EAAE,mBAAmB,CAAC,WAAW,EAAE,MAAM,CAAC;QAChD,KAAK,EAAE,YAAY,CAAC,KAAK;QACzB,MAAM,EAAE,kBAAkB,CAAC,YAAY,CAAC,KAAK,CAAC;QAC9C,OAAO;KACR,CAAC,CAAA;IACF,MAAM,KAAK,CAAC,OAAO,CAAC,YAAY,CAAC,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAA;IACvD,MAAM,SAAS,CAAC,YAAY,EAAE,GAAG,IAAI,CAAC,SAAS,CAAC,UAAU,EAAE,IAAI,EAAE,CAAC,CAAC,IAAI,EAAE,MAAM,CAAC,CAAA;IACjF,WAAW,CAAC,gBAAgB,CAAC,UAAU,EAAE,MAAM,EAAE,oBAAoB,EAAE,WAAW,EAAE,CAAC,CAAC,CAAA;IACtF,OAAO,CAAC,QAAQ,GAAG,YAAY,CAAC,UAAU,CAAC,MAAM,CAAC,CAAA;IAClD,OAAO,UAAU,CAAA;AACnB,CAAC"}
|
package/dist/core/plan.js
CHANGED
package/dist/core/plan.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"plan.js","sourceRoot":"","sources":["../../src/core/plan.ts"],"names":[],"mappings":"AAEA,MAAM,YAAY,GAAG,aAAa,CAAA;AAClC,MAAM,eAAe,GAAG,aAAa,CAAA;AAErC,MAAM,UAAU,eAAe,CAAC,IAAY,EAAE,KAAa;IACzD,OAAO,CAAC,IAAI,GAAG,KAAK,GAAG,eAAe,CAAC,GAAG,YAAY,CAAA;AACxD,CAAC;AAED,MAAM,UAAU,UAAU,CAAC,OAAmB;IAC5C,OAAO,KAAK,CAAC,IAAI,CAAC,EAAE,MAAM,EAAE,OAAO,CAAC,IAAI,EAAE,EAAE,CAAC,CAAC,EAAE,KAAK,EAAE,EAAE;QACvD,MAAM,IAAI,GAAG,eAAe,CAAC,OAAO,CAAC,IAAI,EAAE,KAAK,CAAC,CAAA;QACjD,OAAO;YACL,OAAO,EAAE,SAAS,MAAM,CAAC,KAAK,GAAG,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC,EAAE,GAAG,CAAC,IAAI,IAAI,EAAE;YAC9D,KAAK;YACL,IAAI;YACJ,
|
|
1
|
+
{"version":3,"file":"plan.js","sourceRoot":"","sources":["../../src/core/plan.ts"],"names":[],"mappings":"AAEA,MAAM,YAAY,GAAG,aAAa,CAAA;AAClC,MAAM,eAAe,GAAG,aAAa,CAAA;AAErC,MAAM,UAAU,eAAe,CAAC,IAAY,EAAE,KAAa;IACzD,OAAO,CAAC,IAAI,GAAG,KAAK,GAAG,eAAe,CAAC,GAAG,YAAY,CAAA;AACxD,CAAC;AAED,MAAM,UAAU,UAAU,CAAC,OAAmB;IAC5C,OAAO,KAAK,CAAC,IAAI,CAAC,EAAE,MAAM,EAAE,OAAO,CAAC,IAAI,EAAE,EAAE,CAAC,CAAC,EAAE,KAAK,EAAE,EAAE;QACvD,MAAM,IAAI,GAAG,eAAe,CAAC,OAAO,CAAC,IAAI,EAAE,KAAK,CAAC,CAAA;QACjD,OAAO;YACL,OAAO,EAAE,SAAS,MAAM,CAAC,KAAK,GAAG,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC,EAAE,GAAG,CAAC,IAAI,IAAI,EAAE;YAC9D,KAAK;YACL,IAAI;YACJ,MAAM,EAAE,KAAK,GAAG,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE;SAC9C,CAAA;IACH,CAAC,CAAC,CAAA;AACJ,CAAC"}
|
|
@@ -0,0 +1,6 @@
|
|
|
1
|
+
import type { DiagnosisArtifact } from "./schema.js";
|
|
2
|
+
export type DiagnosisPhase = "complete" | "discover" | "investigate" | "repair";
|
|
3
|
+
export declare function diagnosisInputHash(input: DiagnosisArtifact["input"]): string;
|
|
4
|
+
export declare function nextDiagnosisPhase(checkpoint: DiagnosisArtifact): DiagnosisPhase;
|
|
5
|
+
export declare function readDiagnosisCheckpoint(path: string): Promise<DiagnosisArtifact>;
|
|
6
|
+
export declare function writeDiagnosisCheckpoint(path: string, checkpoint: DiagnosisArtifact): Promise<void>;
|
|
@@ -0,0 +1,46 @@
|
|
|
1
|
+
import { createHash, randomUUID } from "node:crypto";
|
|
2
|
+
import { mkdir, readFile, rename, rm, writeFile } from "node:fs/promises";
|
|
3
|
+
import { dirname } from "node:path";
|
|
4
|
+
import { diagnosisArtifactSchema, diagnosisInputSchema } from "./schema.js";
|
|
5
|
+
export function diagnosisInputHash(input) {
|
|
6
|
+
const canonicalInput = diagnosisInputSchema.parse(input);
|
|
7
|
+
return createHash("sha256").update(JSON.stringify(canonicalInput)).digest("hex");
|
|
8
|
+
}
|
|
9
|
+
export function nextDiagnosisPhase(checkpoint) {
|
|
10
|
+
const workflow = checkpoint.input.options;
|
|
11
|
+
const phases = {
|
|
12
|
+
investigated: workflow.repair ? "repair" : "complete",
|
|
13
|
+
observed: workflow.discover || workflow.investigate || workflow.repair
|
|
14
|
+
? "discover"
|
|
15
|
+
: "complete",
|
|
16
|
+
"repair-proven": "complete",
|
|
17
|
+
"repair-rejected": "complete",
|
|
18
|
+
"reproducer-created": workflow.investigate || workflow.repair
|
|
19
|
+
? "investigate"
|
|
20
|
+
: "complete",
|
|
21
|
+
};
|
|
22
|
+
return phases[checkpoint.stage];
|
|
23
|
+
}
|
|
24
|
+
export async function readDiagnosisCheckpoint(path) {
|
|
25
|
+
const checkpoint = diagnosisArtifactSchema.parse(JSON.parse(await readFile(path, "utf8")));
|
|
26
|
+
if (diagnosisInputHash(checkpoint.input) !== checkpoint.inputHash) {
|
|
27
|
+
throw new Error("Diagnosis checkpoint inputs do not match its integrity hash");
|
|
28
|
+
}
|
|
29
|
+
return checkpoint;
|
|
30
|
+
}
|
|
31
|
+
export async function writeDiagnosisCheckpoint(path, checkpoint) {
|
|
32
|
+
const validated = diagnosisArtifactSchema.parse(checkpoint);
|
|
33
|
+
const temporaryPath = `${path}.${randomUUID()}.tmp`;
|
|
34
|
+
await mkdir(dirname(path), { recursive: true });
|
|
35
|
+
try {
|
|
36
|
+
await writeFile(temporaryPath, `${JSON.stringify(validated, null, 2)}\n`, {
|
|
37
|
+
encoding: "utf8",
|
|
38
|
+
flag: "wx",
|
|
39
|
+
});
|
|
40
|
+
await rename(temporaryPath, path);
|
|
41
|
+
}
|
|
42
|
+
finally {
|
|
43
|
+
await rm(temporaryPath, { force: true });
|
|
44
|
+
}
|
|
45
|
+
}
|
|
46
|
+
//# sourceMappingURL=checkpoint.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"checkpoint.js","sourceRoot":"","sources":["../../src/diagnosis/checkpoint.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,UAAU,EAAE,UAAU,EAAE,MAAM,aAAa,CAAA;AACpD,OAAO,EAAE,KAAK,EAAE,QAAQ,EAAE,MAAM,EAAE,EAAE,EAAE,SAAS,EAAE,MAAM,kBAAkB,CAAA;AACzE,OAAO,EAAE,OAAO,EAAE,MAAM,WAAW,CAAA;AAGnC,OAAO,EAAE,uBAAuB,EAAE,oBAAoB,EAAE,MAAM,aAAa,CAAA;AAI3E,MAAM,UAAU,kBAAkB,CAAC,KAAiC;IAClE,MAAM,cAAc,GAAG,oBAAoB,CAAC,KAAK,CAAC,KAAK,CAAC,CAAA;IACxD,OAAO,UAAU,CAAC,QAAQ,CAAC,CAAC,MAAM,CAAC,IAAI,CAAC,SAAS,CAAC,cAAc,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,CAAA;AAClF,CAAC;AAED,MAAM,UAAU,kBAAkB,CAAC,UAA6B;IAC9D,MAAM,QAAQ,GAAG,UAAU,CAAC,KAAK,CAAC,OAAO,CAAA;IACzC,MAAM,MAAM,GAAuD;QACjE,YAAY,EAAE,QAAQ,CAAC,MAAM,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC,UAAU;QACrD,QAAQ,EAAE,QAAQ,CAAC,QAAQ,IAAI,QAAQ,CAAC,WAAW,IAAI,QAAQ,CAAC,MAAM;YACpE,CAAC,CAAC,UAAU;YACZ,CAAC,CAAC,UAAU;QACd,eAAe,EAAE,UAAU;QAC3B,iBAAiB,EAAE,UAAU;QAC7B,oBAAoB,EAAE,QAAQ,CAAC,WAAW,IAAI,QAAQ,CAAC,MAAM;YAC3D,CAAC,CAAC,aAAa;YACf,CAAC,CAAC,UAAU;KACf,CAAA;IACD,OAAO,MAAM,CAAC,UAAU,CAAC,KAAK,CAAC,CAAA;AACjC,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,uBAAuB,CAAC,IAAY;IACxD,MAAM,UAAU,GAAG,uBAAuB,CAAC,KAAK,CAAC,IAAI,CAAC,KAAK,CAAC,MAAM,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAC,CAAC,CAAC,CAAA;IAC1F,IAAI,kBAAkB,CAAC,UAAU,CAAC,KAAK,CAAC,KAAK,UAAU,CAAC,SAAS,EAAE,CAAC;QAClE,MAAM,IAAI,KAAK,CAAC,6DAA6D,CAAC,CAAA;IAChF,CAAC;IACD,OAAO,UAAU,CAAA;AACnB,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,wBAAwB,CAC5C,IAAY,EACZ,UAA6B;IAE7B,MAAM,SAAS,GAAG,uBAAuB,CAAC,KAAK,CAAC,UAAU,CAAC,CAAA;IAC3D,MAAM,aAAa,GAAG,GAAG,IAAI,IAAI,UAAU,EAAE,MAAM,CAAA;IACnD,MAAM,KAAK,CAAC,OAAO,CAAC,IAAI,CAAC,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAA;IAC/C,IAAI,CAAC;QACH,MAAM,SAAS,CAAC,aAAa,EAAE,GAAG,IAAI,CAAC,SAAS,CAAC,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC,IAAI,EAAE;YACxE,QAAQ,EAAE,MAAM;YAChB,IAAI,EAAE,IAAI;SACX,CAAC,CAAA;QACF,MAAM,MAAM,CAAC,aAAa,EAAE,IAAI,CAAC,CAAA;IACnC,CAAC;YAAS,CAAC;QACT,MAAM,EAAE,CAAC,aAAa,EAAE,EAAE,KAAK,EAAE,IAAI,EAAE,CAAC,CAAA;IAC1C,CAAC;AACH,CAAC"}
|
|
@@ -0,0 +1,84 @@
|
|
|
1
|
+
import { parseArgs } from "node:util";
|
|
2
|
+
function withDefault(value, fallback) {
|
|
3
|
+
return value === undefined ? fallback : value;
|
|
4
|
+
}
|
|
5
|
+
export function parseDiagnoseArguments(args) {
|
|
6
|
+
const parsed = parseArgs({
|
|
7
|
+
args,
|
|
8
|
+
allowPositionals: true,
|
|
9
|
+
strict: true,
|
|
10
|
+
options: {
|
|
11
|
+
artifacts: { type: "string", default: ".flakelab/runs" },
|
|
12
|
+
baseline: { type: "string" },
|
|
13
|
+
concurrency: { type: "string", default: "1" },
|
|
14
|
+
discover: { type: "boolean", default: false },
|
|
15
|
+
evidence: { type: "string", default: "flakelab.investigation.json" },
|
|
16
|
+
html: { type: "string", default: "flakelab.report.html" },
|
|
17
|
+
investigate: { type: "boolean", default: false },
|
|
18
|
+
"max-cost": { type: "string", default: "0.25" },
|
|
19
|
+
"max-delay": { type: "string", default: "250" },
|
|
20
|
+
"max-experiments": { type: "string", default: "3" },
|
|
21
|
+
"max-seconds": { type: "string", default: "90" },
|
|
22
|
+
"max-steps": { type: "string", default: "3" },
|
|
23
|
+
"max-trials": { type: "string", default: "12" },
|
|
24
|
+
"min-rate": { type: "string", default: "0.7" },
|
|
25
|
+
model: { type: "string", default: "qwen/qwen3.8-27b" },
|
|
26
|
+
open: { type: "boolean", default: false },
|
|
27
|
+
patch: { type: "string", default: "candidate.diff" },
|
|
28
|
+
pattern: { type: "string", default: "**/api/checkout" },
|
|
29
|
+
proof: { type: "string", default: "flakelab.proof.json" },
|
|
30
|
+
"prompt-credentials": { type: "boolean", default: false },
|
|
31
|
+
repair: { type: "boolean", default: false },
|
|
32
|
+
report: { type: "string" },
|
|
33
|
+
reproducer: { type: "string", default: "flakelab.repro.yaml" },
|
|
34
|
+
runs: { type: "string", default: "4" },
|
|
35
|
+
seed: { type: "string", default: "1" },
|
|
36
|
+
source: { type: "string", multiple: true, default: [] },
|
|
37
|
+
trials: { type: "string", default: "4" },
|
|
38
|
+
},
|
|
39
|
+
});
|
|
40
|
+
if (parsed.positionals.length > 1) {
|
|
41
|
+
throw new Error("diagnose accepts at most one test target");
|
|
42
|
+
}
|
|
43
|
+
const target = parsed.positionals[0];
|
|
44
|
+
if (!target && !parsed.values.report) {
|
|
45
|
+
throw new Error("diagnose requires a test target or --report <blob-report>");
|
|
46
|
+
}
|
|
47
|
+
if (!target && (parsed.values.discover || parsed.values.investigate || parsed.values.repair)) {
|
|
48
|
+
throw new Error("diagnose needs an explicit test target before running new experiments");
|
|
49
|
+
}
|
|
50
|
+
return {
|
|
51
|
+
command: "diagnose",
|
|
52
|
+
...(target ? { target } : {}),
|
|
53
|
+
options: {
|
|
54
|
+
artifacts: withDefault(parsed.values.artifacts, ".flakelab/runs"),
|
|
55
|
+
baseline: parsed.values.baseline,
|
|
56
|
+
concurrency: withDefault(parsed.values.concurrency, "1"),
|
|
57
|
+
discover: withDefault(parsed.values.discover, false),
|
|
58
|
+
evidence: withDefault(parsed.values.evidence, "flakelab.investigation.json"),
|
|
59
|
+
html: withDefault(parsed.values.html, "flakelab.report.html"),
|
|
60
|
+
investigate: withDefault(parsed.values.investigate, false),
|
|
61
|
+
"max-cost": withDefault(parsed.values["max-cost"], "0.25"),
|
|
62
|
+
"max-delay": withDefault(parsed.values["max-delay"], "250"),
|
|
63
|
+
"max-experiments": withDefault(parsed.values["max-experiments"], "3"),
|
|
64
|
+
"max-seconds": withDefault(parsed.values["max-seconds"], "90"),
|
|
65
|
+
"max-steps": withDefault(parsed.values["max-steps"], "3"),
|
|
66
|
+
"max-trials": withDefault(parsed.values["max-trials"], "12"),
|
|
67
|
+
"min-rate": withDefault(parsed.values["min-rate"], "0.7"),
|
|
68
|
+
model: withDefault(parsed.values.model, "qwen/qwen3.8-27b"),
|
|
69
|
+
open: withDefault(parsed.values.open, false),
|
|
70
|
+
patch: withDefault(parsed.values.patch, "candidate.diff"),
|
|
71
|
+
pattern: withDefault(parsed.values.pattern, "**/api/checkout"),
|
|
72
|
+
proof: withDefault(parsed.values.proof, "flakelab.proof.json"),
|
|
73
|
+
"prompt-credentials": withDefault(parsed.values["prompt-credentials"], false),
|
|
74
|
+
repair: withDefault(parsed.values.repair, false),
|
|
75
|
+
report: parsed.values.report,
|
|
76
|
+
reproducer: withDefault(parsed.values.reproducer, "flakelab.repro.yaml"),
|
|
77
|
+
runs: withDefault(parsed.values.runs, "4"),
|
|
78
|
+
seed: withDefault(parsed.values.seed, "1"),
|
|
79
|
+
source: parsed.values.source ?? [],
|
|
80
|
+
trials: withDefault(parsed.values.trials, "4"),
|
|
81
|
+
},
|
|
82
|
+
};
|
|
83
|
+
}
|
|
84
|
+
//# sourceMappingURL=cli.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"cli.js","sourceRoot":"","sources":["../../src/diagnosis/cli.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,SAAS,EAAE,MAAM,WAAW,CAAA;AAUrC,SAAS,WAAW,CAAI,KAAoB,EAAE,QAAW;IACvD,OAAO,KAAK,KAAK,SAAS,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC,KAAK,CAAA;AAC/C,CAAC;AAED,MAAM,UAAU,sBAAsB,CAAC,IAAc;IACnD,MAAM,MAAM,GAAG,SAAS,CAAC;QACvB,IAAI;QACJ,gBAAgB,EAAE,IAAI;QACtB,MAAM,EAAE,IAAI;QACZ,OAAO,EAAE;YACP,SAAS,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE,OAAO,EAAE,gBAAgB,EAAE;YACxD,QAAQ,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE;YAC5B,WAAW,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE,OAAO,EAAE,GAAG,EAAE;YAC7C,QAAQ,EAAE,EAAE,IAAI,EAAE,SAAS,EAAE,OAAO,EAAE,KAAK,EAAE;YAC7C,QAAQ,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE,OAAO,EAAE,6BAA6B,EAAE;YACpE,IAAI,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE,OAAO,EAAE,sBAAsB,EAAE;YACzD,WAAW,EAAE,EAAE,IAAI,EAAE,SAAS,EAAE,OAAO,EAAE,KAAK,EAAE;YAChD,UAAU,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE,OAAO,EAAE,MAAM,EAAE;YAC/C,WAAW,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE,OAAO,EAAE,KAAK,EAAE;YAC/C,iBAAiB,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE,OAAO,EAAE,GAAG,EAAE;YACnD,aAAa,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE,OAAO,EAAE,IAAI,EAAE;YAChD,WAAW,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE,OAAO,EAAE,GAAG,EAAE;YAC7C,YAAY,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE,OAAO,EAAE,IAAI,EAAE;YAC/C,UAAU,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE,OAAO,EAAE,KAAK,EAAE;YAC9C,KAAK,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE,OAAO,EAAE,kBAAkB,EAAE;YACtD,IAAI,EAAE,EAAE,IAAI,EAAE,SAAS,EAAE,OAAO,EAAE,KAAK,EAAE;YACzC,KAAK,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE,OAAO,EAAE,gBAAgB,EAAE;YACpD,OAAO,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE,OAAO,EAAE,iBAAiB,EAAE;YACvD,KAAK,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE,OAAO,EAAE,qBAAqB,EAAE;YACzD,oBAAoB,EAAE,EAAE,IAAI,EAAE,SAAS,EAAE,OAAO,EAAE,KAAK,EAAE;YACzD,MAAM,EAAE,EAAE,IAAI,EAAE,SAAS,EAAE,OAAO,EAAE,KAAK,EAAE;YAC3C,MAAM,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE;YAC1B,UAAU,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE,OAAO,EAAE,qBAAqB,EAAE;YAC9D,IAAI,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE,OAAO,EAAE,GAAG,EAAE;YACtC,IAAI,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE,OAAO,EAAE,GAAG,EAAE;YACtC,MAAM,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE,QAAQ,EAAE,IAAI,EAAE,OAAO,EAAE,EAAE,EAAE;YACvD,MAAM,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE,OAAO,EAAE,GAAG,EAAE;SACzC;KACF,CAAC,CAAA;IACF,IAAI,MAAM,CAAC,WAAW,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QAClC,MAAM,IAAI,KAAK,CAAC,0CAA0C,CAAC,CAAA;IAC7D,CAAC;IACD,MAAM,MAAM,GAAG,MAAM,CAAC,WAAW,CAAC,CAAC,CAAC,CAAA;IACpC,IAAI,CAAC,MAAM,IAAI,CAAC,MAAM,CAAC,MAAM,CAAC,MAAM,EAAE,CAAC;QACrC,MAAM,IAAI,KAAK,CAAC,2DAA2D,CAAC,CAAA;IAC9E,CAAC;IACD,IAAI,CAAC,MAAM,IAAI,CAAC,MAAM,CAAC,MAAM,CAAC,QAAQ,IAAI,MAAM,CAAC,MAAM,CAAC,WAAW,IAAI,MAAM,CAAC,MAAM,CAAC,MAAM,CAAC,EAAE,CAAC;QAC7F,MAAM,IAAI,KAAK,CAAC,uEAAuE,CAAC,CAAA;IAC1F,CAAC;IACD,OAAO;QACL,OAAO,EAAE,UAAU;QACnB,GAAG,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,MAAM,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;QAC7B,OAAO,EAAE;YACP,SAAS,EAAE,WAAW,CAAC,MAAM,CAAC,MAAM,CAAC,SAAS,EAAE,gBAAgB,CAAC;YACjE,QAAQ,EAAE,MAAM,CAAC,MAAM,CAAC,QAAQ;YAChC,WAAW,EAAE,WAAW,CAAC,MAAM,CAAC,MAAM,CAAC,WAAW,EAAE,GAAG,CAAC;YACxD,QAAQ,EAAE,WAAW,CAAC,MAAM,CAAC,MAAM,CAAC,QAAQ,EAAE,KAAK,CAAC;YACpD,QAAQ,EAAE,WAAW,CAAC,MAAM,CAAC,MAAM,CAAC,QAAQ,EAAE,6BAA6B,CAAC;YAC5E,IAAI,EAAE,WAAW,CAAC,MAAM,CAAC,MAAM,CAAC,IAAI,EAAE,sBAAsB,CAAC;YAC7D,WAAW,EAAE,WAAW,CAAC,MAAM,CAAC,MAAM,CAAC,WAAW,EAAE,KAAK,CAAC;YAC1D,UAAU,EAAE,WAAW,CAAC,MAAM,CAAC,MAAM,CAAC,UAAU,CAAC,EAAE,MAAM,CAAC;YAC1D,WAAW,EAAE,WAAW,CAAC,MAAM,CAAC,MAAM,CAAC,WAAW,CAAC,EAAE,KAAK,CAAC;YAC3D,iBAAiB,EAAE,WAAW,CAAC,MAAM,CAAC,MAAM,CAAC,iBAAiB,CAAC,EAAE,GAAG,CAAC;YACrE,aAAa,EAAE,WAAW,CAAC,MAAM,CAAC,MAAM,CAAC,aAAa,CAAC,EAAE,IAAI,CAAC;YAC9D,WAAW,EAAE,WAAW,CAAC,MAAM,CAAC,MAAM,CAAC,WAAW,CAAC,EAAE,GAAG,CAAC;YACzD,YAAY,EAAE,WAAW,CAAC,MAAM,CAAC,MAAM,CAAC,YAAY,CAAC,EAAE,IAAI,CAAC;YAC5D,UAAU,EAAE,WAAW,CAAC,MAAM,CAAC,MAAM,CAAC,UAAU,CAAC,EAAE,KAAK,CAAC;YACzD,KAAK,EAAE,WAAW,CAAC,MAAM,CAAC,MAAM,CAAC,KAAK,EAAE,kBAAkB,CAAC;YAC3D,IAAI,EAAE,WAAW,CAAC,MAAM,CAAC,MAAM,CAAC,IAAI,EAAE,KAAK,CAAC;YAC5C,KAAK,EAAE,WAAW,CAAC,MAAM,CAAC,MAAM,CAAC,KAAK,EAAE,gBAAgB,CAAC;YACzD,OAAO,EAAE,WAAW,CAAC,MAAM,CAAC,MAAM,CAAC,OAAO,EAAE,iBAAiB,CAAC;YAC9D,KAAK,EAAE,WAAW,CAAC,MAAM,CAAC,MAAM,CAAC,KAAK,EAAE,qBAAqB,CAAC;YAC9D,oBAAoB,EAAE,WAAW,CAAC,MAAM,CAAC,MAAM,CAAC,oBAAoB,CAAC,EAAE,KAAK,CAAC;YAC7E,MAAM,EAAE,WAAW,CAAC,MAAM,CAAC,MAAM,CAAC,MAAM,EAAE,KAAK,CAAC;YAChD,MAAM,EAAE,MAAM,CAAC,MAAM,CAAC,MAAM;YAC5B,UAAU,EAAE,WAAW,CAAC,MAAM,CAAC,MAAM,CAAC,UAAU,EAAE,qBAAqB,CAAC;YACxE,IAAI,EAAE,WAAW,CAAC,MAAM,CAAC,MAAM,CAAC,IAAI,EAAE,GAAG,CAAC;YAC1C,IAAI,EAAE,WAAW,CAAC,MAAM,CAAC,MAAM,CAAC,IAAI,EAAE,GAAG,CAAC;YAC1C,MAAM,EAAE,MAAM,CAAC,MAAM,CAAC,MAAM,IAAI,EAAE;YAClC,MAAM,EAAE,WAAW,CAAC,MAAM,CAAC,MAAM,CAAC,MAAM,EAAE,GAAG,CAAC;SAC/C;KACF,CAAA;AACH,CAAC"}
|
|
@@ -0,0 +1,13 @@
|
|
|
1
|
+
import type { DiagnoseOptions } from "../commands/options.js";
|
|
2
|
+
import type { ScanStatus } from "../scan/schema.js";
|
|
3
|
+
import type { DiagnosisRecommendation, DiagnosisStage } from "./schema.js";
|
|
4
|
+
interface RecommendationInput {
|
|
5
|
+
elapsedMilliseconds: number;
|
|
6
|
+
observedRuns: number;
|
|
7
|
+
stage: DiagnosisStage;
|
|
8
|
+
status: ScanStatus;
|
|
9
|
+
target?: string;
|
|
10
|
+
values: DiagnoseOptions;
|
|
11
|
+
}
|
|
12
|
+
export declare function buildDiagnosisRecommendation(input: RecommendationInput): DiagnosisRecommendation;
|
|
13
|
+
export {};
|