@specific.dev/spectest 0.56.2 → 0.58.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/browser-coverage.d.ts +20 -0
- package/dist/browser-coverage.js +185 -0
- package/dist/browser.js +36 -0
- package/dist/daemon.js +200 -11
- package/dist/harness/browser-coverage.d.ts +95 -0
- package/dist/harness/browser-coverage.js +257 -0
- package/dist/harness/coverage.d.ts +60 -0
- package/dist/harness/coverage.js +122 -0
- package/dist/index.d.ts +55 -0
- package/dist/index.js +32 -0
- package/package.json +1 -1
- package/src/browser-coverage.ts +209 -0
- package/src/browser.ts +36 -0
- package/src/daemon.ts +230 -11
- package/src/harness/browser-coverage.test.ts +94 -0
- package/src/harness/browser-coverage.ts +310 -0
- package/src/harness/coverage.test.ts +88 -0
- package/src/harness/coverage.ts +173 -0
- package/src/index.ts +81 -0
|
@@ -0,0 +1,209 @@
|
|
|
1
|
+
// Browser coverage — the collector.
|
|
2
|
+
//
|
|
3
|
+
// One collector per CDP session (i.e. per page view). `attach` turns on
|
|
4
|
+
// V8 precise coverage and tracks parsed scripts; `harvest` takes the
|
|
5
|
+
// counters (which RESETS them — V8 reports nothing for a script that ran
|
|
6
|
+
// nothing since the last take, so the counts are summed here across
|
|
7
|
+
// takes), maps each script the browser loaded from a covered service back
|
|
8
|
+
// to its original sources through the served source map, and folds the
|
|
9
|
+
// result into a per-service total in module memory — which forks with the
|
|
10
|
+
// daemon, so a child test inherits its ancestors' browser coverage exactly
|
|
11
|
+
// as a container's counters are inherited.
|
|
12
|
+
//
|
|
13
|
+
// Harvest runs before every browser step (a click that navigates
|
|
14
|
+
// cross-site replaces the renderer, and the old counters with it) and at
|
|
15
|
+
// every coverage capture. `browserCoverageReports` renders the totals as
|
|
16
|
+
// one lcov per service for the `case-coverage` bundle.
|
|
17
|
+
//
|
|
18
|
+
// Configured by the daemon at /load with the host → service table; a
|
|
19
|
+
// project with no `coverage: { browser: true }` service never attaches
|
|
20
|
+
// anything, so it costs nothing.
|
|
21
|
+
|
|
22
|
+
import { createHash } from "node:crypto";
|
|
23
|
+
import type { CDPSession } from "playwright-core";
|
|
24
|
+
import {
|
|
25
|
+
type DecodedSourceMap,
|
|
26
|
+
type FileCounts,
|
|
27
|
+
decodeSourceMap,
|
|
28
|
+
lcovDocument,
|
|
29
|
+
mapToOriginal,
|
|
30
|
+
mergeFileCounts,
|
|
31
|
+
rangesToLineCounts,
|
|
32
|
+
type V8FunctionCoverage,
|
|
33
|
+
} from "./harness/browser-coverage.js";
|
|
34
|
+
import { rawFetch } from "./harness/raw-fetch.js";
|
|
35
|
+
|
|
36
|
+
export interface BrowserCoverageConfig {
|
|
37
|
+
/** Which covered service (if any) serves scripts from this host. */
|
|
38
|
+
serviceForHost(host: string): string | undefined;
|
|
39
|
+
}
|
|
40
|
+
|
|
41
|
+
let CONFIG: BrowserCoverageConfig | null = null;
|
|
42
|
+
|
|
43
|
+
/** Set (or clear, with `null`) at /load. */
|
|
44
|
+
export function configureBrowserCoverage(config: BrowserCoverageConfig | null): void {
|
|
45
|
+
CONFIG = config;
|
|
46
|
+
}
|
|
47
|
+
|
|
48
|
+
export function browserCoverageActive(): boolean {
|
|
49
|
+
return CONFIG !== null;
|
|
50
|
+
}
|
|
51
|
+
|
|
52
|
+
/** service → file → line → count, summed over every harvest. */
|
|
53
|
+
const TOTALS = new Map<string, FileCounts>();
|
|
54
|
+
|
|
55
|
+
interface ScriptInfo {
|
|
56
|
+
url: string;
|
|
57
|
+
sourceMapURL?: string;
|
|
58
|
+
source?: string;
|
|
59
|
+
/** `undefined` = not looked up yet; `null` = none / unusable. */
|
|
60
|
+
map?: DecodedSourceMap | null;
|
|
61
|
+
}
|
|
62
|
+
|
|
63
|
+
interface Collector {
|
|
64
|
+
cdp: CDPSession;
|
|
65
|
+
scripts: Map<string, ScriptInfo>;
|
|
66
|
+
inFlight: Promise<void> | null;
|
|
67
|
+
}
|
|
68
|
+
|
|
69
|
+
const COLLECTORS = new WeakMap<CDPSession, Collector>();
|
|
70
|
+
|
|
71
|
+
/** Decoded maps by URL (or content hash for inline `data:` maps). */
|
|
72
|
+
const SOURCE_MAPS = new Map<string, DecodedSourceMap | null>();
|
|
73
|
+
|
|
74
|
+
const SOURCE_MAP_FETCH_TIMEOUT_MS = 10_000;
|
|
75
|
+
|
|
76
|
+
/** Turn coverage on for a page's CDP session. Idempotent; a no-op when no
|
|
77
|
+
* service opted in. */
|
|
78
|
+
export async function attachBrowserCoverage(cdp: CDPSession): Promise<void> {
|
|
79
|
+
if (!CONFIG || COLLECTORS.has(cdp)) return;
|
|
80
|
+
const collector: Collector = { cdp, scripts: new Map(), inFlight: null };
|
|
81
|
+
COLLECTORS.set(cdp, collector);
|
|
82
|
+
cdp.on("Debugger.scriptParsed", (ev: { scriptId: string; url: string; sourceMapURL?: string }) => {
|
|
83
|
+
if (!ev.url) return;
|
|
84
|
+
collector.scripts.set(ev.scriptId, {
|
|
85
|
+
url: ev.url,
|
|
86
|
+
sourceMapURL: ev.sourceMapURL || undefined,
|
|
87
|
+
});
|
|
88
|
+
});
|
|
89
|
+
await cdp.send("Debugger.enable");
|
|
90
|
+
await cdp.send("Profiler.enable");
|
|
91
|
+
await cdp.send("Profiler.startPreciseCoverage", { callCount: true, detailed: true });
|
|
92
|
+
}
|
|
93
|
+
|
|
94
|
+
/**
|
|
95
|
+
* Take the session's counters and fold them into the totals. Concurrent
|
|
96
|
+
* calls coalesce onto the in-flight one. Errors (a page mid-navigation,
|
|
97
|
+
* a closed view) are swallowed: the counters then stay in V8 for the next
|
|
98
|
+
* take, nothing is lost.
|
|
99
|
+
*/
|
|
100
|
+
export async function harvestBrowserCoverage(cdp: CDPSession): Promise<void> {
|
|
101
|
+
const c = COLLECTORS.get(cdp);
|
|
102
|
+
if (!c || !CONFIG) return;
|
|
103
|
+
if (c.inFlight) return c.inFlight;
|
|
104
|
+
c.inFlight = harvestInner(c, CONFIG).catch(() => {}).finally(() => {
|
|
105
|
+
c.inFlight = null;
|
|
106
|
+
});
|
|
107
|
+
return c.inFlight;
|
|
108
|
+
}
|
|
109
|
+
|
|
110
|
+
async function harvestInner(c: Collector, config: BrowserCoverageConfig): Promise<void> {
|
|
111
|
+
const { result } = (await c.cdp.send("Profiler.takePreciseCoverage")) as {
|
|
112
|
+
result: { scriptId: string; url: string; functions: V8FunctionCoverage[] }[];
|
|
113
|
+
};
|
|
114
|
+
for (const script of result) {
|
|
115
|
+
const info = c.scripts.get(script.scriptId) ?? (script.url ? { url: script.url } : undefined);
|
|
116
|
+
if (!info) continue;
|
|
117
|
+
let url: URL;
|
|
118
|
+
try {
|
|
119
|
+
url = new URL(info.url);
|
|
120
|
+
} catch {
|
|
121
|
+
continue;
|
|
122
|
+
}
|
|
123
|
+
if (url.protocol !== "http:" && url.protocol !== "https:") continue;
|
|
124
|
+
const service = config.serviceForHost(url.hostname);
|
|
125
|
+
if (!service) continue;
|
|
126
|
+
if (info.source === undefined) {
|
|
127
|
+
try {
|
|
128
|
+
const r = (await c.cdp.send("Debugger.getScriptSource", {
|
|
129
|
+
scriptId: script.scriptId,
|
|
130
|
+
})) as { scriptSource: string };
|
|
131
|
+
info.source = r.scriptSource;
|
|
132
|
+
} catch {
|
|
133
|
+
continue; // script gone with its document
|
|
134
|
+
}
|
|
135
|
+
c.scripts.set(script.scriptId, info);
|
|
136
|
+
}
|
|
137
|
+
const generated = rangesToLineCounts(info.source, script.functions);
|
|
138
|
+
if (generated.size === 0) continue;
|
|
139
|
+
if (info.map === undefined) info.map = await sourceMapFor(info, url);
|
|
140
|
+
const files: FileCounts = info.map
|
|
141
|
+
? mapToOriginal(info.map, generated)
|
|
142
|
+
: new Map([[url.pathname, generated]]);
|
|
143
|
+
let totals = TOTALS.get(service);
|
|
144
|
+
if (!totals) {
|
|
145
|
+
totals = new Map();
|
|
146
|
+
TOTALS.set(service, totals);
|
|
147
|
+
}
|
|
148
|
+
mergeFileCounts(totals, files);
|
|
149
|
+
}
|
|
150
|
+
}
|
|
151
|
+
|
|
152
|
+
/** Resolve, fetch (or inline-decode) and decode a script's source map. */
|
|
153
|
+
async function sourceMapFor(info: ScriptInfo, scriptUrl: URL): Promise<DecodedSourceMap | null> {
|
|
154
|
+
let ref = info.sourceMapURL;
|
|
155
|
+
if (!ref && info.source) {
|
|
156
|
+
const m = /\/\/[#@]\s*sourceMappingURL=(\S+)\s*$/.exec(info.source.slice(-4096));
|
|
157
|
+
if (m) ref = m[1];
|
|
158
|
+
}
|
|
159
|
+
if (!ref) return null;
|
|
160
|
+
if (ref.startsWith("data:")) {
|
|
161
|
+
const key = "data:" + createHash("sha1").update(ref).digest("hex");
|
|
162
|
+
const cached = SOURCE_MAPS.get(key);
|
|
163
|
+
if (cached !== undefined) return cached;
|
|
164
|
+
const map = decodeSourceMap(parseDataUrl(ref));
|
|
165
|
+
SOURCE_MAPS.set(key, map);
|
|
166
|
+
return map;
|
|
167
|
+
}
|
|
168
|
+
let mapUrl: string;
|
|
169
|
+
try {
|
|
170
|
+
mapUrl = new URL(ref, scriptUrl).toString();
|
|
171
|
+
} catch {
|
|
172
|
+
return null;
|
|
173
|
+
}
|
|
174
|
+
const cached = SOURCE_MAPS.get(mapUrl);
|
|
175
|
+
if (cached !== undefined) return cached;
|
|
176
|
+
let map: DecodedSourceMap | null = null;
|
|
177
|
+
try {
|
|
178
|
+
const res = await rawFetch(mapUrl, { signal: AbortSignal.timeout(SOURCE_MAP_FETCH_TIMEOUT_MS) });
|
|
179
|
+
if (res.ok) map = decodeSourceMap(await res.json());
|
|
180
|
+
} catch {
|
|
181
|
+
map = null;
|
|
182
|
+
}
|
|
183
|
+
SOURCE_MAPS.set(mapUrl, map);
|
|
184
|
+
return map;
|
|
185
|
+
}
|
|
186
|
+
|
|
187
|
+
function parseDataUrl(ref: string): unknown {
|
|
188
|
+
const comma = ref.indexOf(",");
|
|
189
|
+
if (comma < 0) return null;
|
|
190
|
+
const meta = ref.slice(5, comma);
|
|
191
|
+
const payload = ref.slice(comma + 1);
|
|
192
|
+
try {
|
|
193
|
+
const text = /;base64$/i.test(meta)
|
|
194
|
+
? Buffer.from(payload, "base64").toString("utf8")
|
|
195
|
+
: decodeURIComponent(payload);
|
|
196
|
+
return JSON.parse(text);
|
|
197
|
+
} catch {
|
|
198
|
+
return null;
|
|
199
|
+
}
|
|
200
|
+
}
|
|
201
|
+
|
|
202
|
+
/** One lcov document per service that has any browser coverage. */
|
|
203
|
+
export function browserCoverageReports(): Map<string, string> {
|
|
204
|
+
const out = new Map<string, string>();
|
|
205
|
+
for (const [service, files] of TOTALS) {
|
|
206
|
+
if (files.size > 0) out.set(service, lcovDocument(files));
|
|
207
|
+
}
|
|
208
|
+
return out;
|
|
209
|
+
}
|
package/src/browser.ts
CHANGED
|
@@ -38,6 +38,11 @@ import { recordBrowser, reserveBackdated, reserveEvent, truncateUtf8 } from "./r
|
|
|
38
38
|
import { wrap } from "./inspect.js";
|
|
39
39
|
import type { Wrapped } from "./inspect.js";
|
|
40
40
|
import { describeUrlPattern, matchesUrl } from "./url-match.js";
|
|
41
|
+
import {
|
|
42
|
+
attachBrowserCoverage,
|
|
43
|
+
browserCoverageActive,
|
|
44
|
+
harvestBrowserCoverage,
|
|
45
|
+
} from "./browser-coverage.js";
|
|
41
46
|
import type { UrlPattern } from "./url-match.js";
|
|
42
47
|
export type { UrlPattern } from "./url-match.js";
|
|
43
48
|
import {
|
|
@@ -1223,6 +1228,13 @@ async function spawnPage(context: BrowserContext): Promise<SpawnedPage> {
|
|
|
1223
1228
|
console.warn("[spectest] failed to install rrweb recorder:", err);
|
|
1224
1229
|
}
|
|
1225
1230
|
}
|
|
1231
|
+
// Frontend coverage (opt-in): turn on V8 precise coverage for this page.
|
|
1232
|
+
// A no-op unless a service declared `coverage: { browser: true }`.
|
|
1233
|
+
try {
|
|
1234
|
+
await attachBrowserCoverage(cdp);
|
|
1235
|
+
} catch {
|
|
1236
|
+
/* coverage is best-effort; a page without it still works */
|
|
1237
|
+
}
|
|
1226
1238
|
return { page, cdp, recordingInstalled };
|
|
1227
1239
|
}
|
|
1228
1240
|
|
|
@@ -1558,6 +1570,13 @@ async function rebuildView(holder: ViewHolder): Promise<void> {
|
|
|
1558
1570
|
} catch {
|
|
1559
1571
|
/* page may already be gone */
|
|
1560
1572
|
}
|
|
1573
|
+
// The old renderer's coverage counters die with its page; harvest what
|
|
1574
|
+
// it had before the swap (rebuild is triggered on a fresh navigation).
|
|
1575
|
+
try {
|
|
1576
|
+
await harvestBrowserCoverage(holder.cdp);
|
|
1577
|
+
} catch {
|
|
1578
|
+
/* best-effort */
|
|
1579
|
+
}
|
|
1561
1580
|
const fresh = await spawnPage(holder.context);
|
|
1562
1581
|
holder.page = fresh.page;
|
|
1563
1582
|
holder.cdp = fresh.cdp;
|
|
@@ -1660,6 +1679,16 @@ function buildBackend(
|
|
|
1660
1679
|
durationMs: endT - t,
|
|
1661
1680
|
}, resv);
|
|
1662
1681
|
await drain(action);
|
|
1682
|
+
// Fold this page's V8 counters into the per-service totals now — a
|
|
1683
|
+
// navigation to another origin replaces the renderer and its
|
|
1684
|
+
// counters, so harvesting per op is what keeps them.
|
|
1685
|
+
if (browserCoverageActive()) {
|
|
1686
|
+
try {
|
|
1687
|
+
await harvestBrowserCoverage(holder.cdp);
|
|
1688
|
+
} catch {
|
|
1689
|
+
/* best-effort */
|
|
1690
|
+
}
|
|
1691
|
+
}
|
|
1663
1692
|
// Reads (`opts.wrap`) return user-visible JS values someone is likely to
|
|
1664
1693
|
// assert on — provenance-wrap so `expect(...)` nests under this step.
|
|
1665
1694
|
// Actions return void/internals; leave them raw to avoid Proxy surprises.
|
|
@@ -1688,6 +1717,13 @@ function buildBackend(
|
|
|
1688
1717
|
if (recordingEnded) return;
|
|
1689
1718
|
// Final drain before we stop writing to this recorder.
|
|
1690
1719
|
await drain("close");
|
|
1720
|
+
if (browserCoverageActive()) {
|
|
1721
|
+
try {
|
|
1722
|
+
await harvestBrowserCoverage(holder.cdp);
|
|
1723
|
+
} catch {
|
|
1724
|
+
/* best-effort */
|
|
1725
|
+
}
|
|
1726
|
+
}
|
|
1691
1727
|
recordingEnded = true;
|
|
1692
1728
|
}
|
|
1693
1729
|
|
package/src/daemon.ts
CHANGED
|
@@ -32,8 +32,26 @@ import {
|
|
|
32
32
|
dnsName as makeDnsDecl,
|
|
33
33
|
isWildcard,
|
|
34
34
|
proxy as makeProxyDecl,
|
|
35
|
+
validateCoverage,
|
|
36
|
+
coverageUsesBrowser,
|
|
37
|
+
coverageUsesContainer,
|
|
35
38
|
} from "./index.js";
|
|
36
39
|
import type { DnsTarget, LoweredIngress } from "./index.js";
|
|
40
|
+
import {
|
|
41
|
+
COVERAGE_CONTAINER_DIR,
|
|
42
|
+
coverageBundleRef,
|
|
43
|
+
coverageHostDir,
|
|
44
|
+
encodeCoverageBundle,
|
|
45
|
+
readCoverageDir,
|
|
46
|
+
type CoverageBundleRef,
|
|
47
|
+
type CoverageReport,
|
|
48
|
+
type ServiceCoverageCapture,
|
|
49
|
+
} from "./harness/coverage.js";
|
|
50
|
+
import {
|
|
51
|
+
browserCoverageReports,
|
|
52
|
+
configureBrowserCoverage,
|
|
53
|
+
} from "./browser-coverage.js";
|
|
54
|
+
import { serviceForHost as hostToService } from "./harness/browser-coverage.js";
|
|
37
55
|
import { acquirePersistentBrowser, mobileKey } from "./browser.js";
|
|
38
56
|
import { isMobileApp, openPersistentMobile } from "./mobile.js";
|
|
39
57
|
// Pure ingress hostname matching, ported out of this file (see
|
|
@@ -756,6 +774,26 @@ async function ensureVolumes(svc: NamedService): Promise<string[]> {
|
|
|
756
774
|
return flags;
|
|
757
775
|
}
|
|
758
776
|
|
|
777
|
+
/**
|
|
778
|
+
* The coverage directory for an opted-in service: one host dir per service
|
|
779
|
+
* under the workspace (so the delta-restore teardown wipes it), mounted
|
|
780
|
+
* writable at `/spectest/coverage/`. World-writable because the container
|
|
781
|
+
* may run as any user and spectest creates the dir as root. The reports in
|
|
782
|
+
* it ride snapshots and forks like everything else on the rootfs, which is
|
|
783
|
+
* what makes each read cumulative along the test's ancestor chain.
|
|
784
|
+
*/
|
|
785
|
+
async function ensureCoverage(svc: NamedService): Promise<string[]> {
|
|
786
|
+
if (svc.coverage === undefined) return [];
|
|
787
|
+
validateCoverage(svc.name, svc.coverage);
|
|
788
|
+
// A browser-only opt-in has no container process to write reports, so it
|
|
789
|
+
// gets no mount — its coverage comes from the guest browser.
|
|
790
|
+
if (!coverageUsesContainer(svc.coverage)) return [];
|
|
791
|
+
const host = coverageHostDir(WORKSPACE, svc.name);
|
|
792
|
+
await fs.mkdir(host, { recursive: true });
|
|
793
|
+
await fs.chmod(host, 0o777);
|
|
794
|
+
return [`--volume=${host}:${COVERAGE_CONTAINER_DIR}`];
|
|
795
|
+
}
|
|
796
|
+
|
|
759
797
|
/** A `user`/`group` pair resolved to the numeric ids the kernel wants.
|
|
760
798
|
* `-1` is chown(2)'s "leave this one alone". */
|
|
761
799
|
interface OwnerIds {
|
|
@@ -1775,6 +1813,7 @@ function buildIngress(project: Project): void {
|
|
|
1775
1813
|
// generic decl set the daemon executes. The special-casing lives in the
|
|
1776
1814
|
// SDK's lowerIngress, not here.
|
|
1777
1815
|
LOWERED = lowerIngress(project);
|
|
1816
|
+
configureCoverageAttribution(project);
|
|
1778
1817
|
if (!project.fakes) return;
|
|
1779
1818
|
for (const [name, def] of Object.entries(project.fakes)) {
|
|
1780
1819
|
FAKES.set(name, {
|
|
@@ -2780,6 +2819,7 @@ async function startRuntimeService(spec: RuntimeServiceSpec): Promise<RuntimeSer
|
|
|
2780
2819
|
...(await ensureVolumes(svc)),
|
|
2781
2820
|
...(await ensureFiles(svc, tag)),
|
|
2782
2821
|
...(await ensureCertificates(svc, tag)),
|
|
2822
|
+
...(await ensureCoverage(svc)),
|
|
2783
2823
|
];
|
|
2784
2824
|
await runContainer(svc, tag, flags, aliases);
|
|
2785
2825
|
await waitForReady(svc);
|
|
@@ -3228,6 +3268,7 @@ async function bootstrapInner(): Promise<BootstrapTimings> {
|
|
|
3228
3268
|
...(await ensureVolumes(svc)),
|
|
3229
3269
|
...(await ensureFiles(svc, tag)),
|
|
3230
3270
|
...(await ensureCertificates(svc, tag)),
|
|
3271
|
+
...(await ensureCoverage(svc)),
|
|
3231
3272
|
];
|
|
3232
3273
|
const tRun = Date.now();
|
|
3233
3274
|
progressService(svc.name, { status: "starting", detail: undefined });
|
|
@@ -3369,6 +3410,16 @@ interface RunResult {
|
|
|
3369
3410
|
* failing case's own delta. See {captureServiceLogDeltas}.
|
|
3370
3411
|
*/
|
|
3371
3412
|
serviceLogDeltas: ServiceLogDelta[];
|
|
3413
|
+
/**
|
|
3414
|
+
* Per-service coverage for the services that opted in (`coverage` on
|
|
3415
|
+
* the service): their lcov reports, verbatim, read after this case.
|
|
3416
|
+
* Cumulative along the ancestor chain by construction. Like
|
|
3417
|
+
* `browserSessions` this never rides the `run` reply — the handler
|
|
3418
|
+
* parks a gzipped bundle and replies with a ref the control plane pulls
|
|
3419
|
+
* through `coverageChunk`. An entry with `error` fails the case. See
|
|
3420
|
+
* {captureServiceCoverage}.
|
|
3421
|
+
*/
|
|
3422
|
+
coverageCaptures: ServiceCoverageCapture[];
|
|
3372
3423
|
error?: { message: string; stack?: string };
|
|
3373
3424
|
}
|
|
3374
3425
|
|
|
@@ -3391,19 +3442,42 @@ interface RunResult {
|
|
|
3391
3442
|
// bundle in guest RAM — entries older than the last few are always
|
|
3392
3443
|
// already-fetched leftovers frozen into some snapshot.
|
|
3393
3444
|
const REPLAY_BUNDLES = new Map<string, Buffer>();
|
|
3394
|
-
|
|
3395
|
-
|
|
3396
|
-
|
|
3445
|
+
/** Coverage bundles, same side-channel and same lifetime rules; pulled
|
|
3446
|
+
* through `coverageChunk`. The env-setup baseline parks under the
|
|
3447
|
+
* control plane's synthetic `__setup__` case id. */
|
|
3448
|
+
const COVERAGE_BUNDLES = new Map<string, Buffer>();
|
|
3449
|
+
const PARKED_BUNDLES_MAX = 8;
|
|
3450
|
+
/** The control plane's synthetic case id for env-setup captures
|
|
3451
|
+
* (`storage::SETUP_CASE_ID`). Not a real case id — those are kebab slugs. */
|
|
3452
|
+
const SETUP_CASE_ID = "__setup__";
|
|
3453
|
+
|
|
3454
|
+
function stashBundle(map: Map<string, Buffer>, caseId: string, gz: Buffer): void {
|
|
3397
3455
|
// Re-insert to refresh recency (Map iterates in insertion order).
|
|
3398
|
-
|
|
3399
|
-
|
|
3400
|
-
while (
|
|
3401
|
-
const oldest =
|
|
3456
|
+
map.delete(caseId);
|
|
3457
|
+
map.set(caseId, gz);
|
|
3458
|
+
while (map.size > PARKED_BUNDLES_MAX) {
|
|
3459
|
+
const oldest = map.keys().next().value;
|
|
3402
3460
|
if (oldest === undefined) break;
|
|
3403
|
-
|
|
3461
|
+
map.delete(oldest);
|
|
3404
3462
|
}
|
|
3405
3463
|
}
|
|
3406
3464
|
|
|
3465
|
+
function stashReplayBundle(caseId: string, gz: Buffer): void {
|
|
3466
|
+
stashBundle(REPLAY_BUNDLES, caseId, gz);
|
|
3467
|
+
}
|
|
3468
|
+
|
|
3469
|
+
/** Encode + park a case's coverage captures; returns the ref for the reply.
|
|
3470
|
+
* `undefined` when no service opted in, so uncovered projects ship nothing. */
|
|
3471
|
+
function parkCoverageBundle(
|
|
3472
|
+
caseId: string,
|
|
3473
|
+
captures: ServiceCoverageCapture[],
|
|
3474
|
+
): CoverageBundleRef | undefined {
|
|
3475
|
+
if (captures.length === 0) return undefined;
|
|
3476
|
+
const gz = encodeCoverageBundle(caseId, captures);
|
|
3477
|
+
stashBundle(COVERAGE_BUNDLES, caseId, gz);
|
|
3478
|
+
return coverageBundleRef(gz, captures);
|
|
3479
|
+
}
|
|
3480
|
+
|
|
3407
3481
|
// ────────────────────────────────────────────────────────────────────────
|
|
3408
3482
|
// Cumulative service logs (per-case deltas → S3, reconstructed on the web)
|
|
3409
3483
|
// ────────────────────────────────────────────────────────────────────────
|
|
@@ -3526,6 +3600,101 @@ async function captureServiceLogDeltas(): Promise<ServiceLogDelta[]> {
|
|
|
3526
3600
|
);
|
|
3527
3601
|
}
|
|
3528
3602
|
|
|
3603
|
+
// ────────────────────────────────────────────────────────────────────────
|
|
3604
|
+
// Coverage capture (opt-in per service; see harness/coverage.ts)
|
|
3605
|
+
// ────────────────────────────────────────────────────────────────────────
|
|
3606
|
+
//
|
|
3607
|
+
// Read after bring-up and after every test, on every case. The reports are
|
|
3608
|
+
// cumulative from process start and live on the rootfs, so what a child
|
|
3609
|
+
// fork reads already includes everything its ancestors ran — no markers,
|
|
3610
|
+
// no deltas (unlike the service logs above). A `command` runs in the
|
|
3611
|
+
// container first so the service writes a fresh report. The reports ship
|
|
3612
|
+
// verbatim; the server derives what it needs from the stored bytes.
|
|
3613
|
+
|
|
3614
|
+
const COVERAGE_COMMAND_TIMEOUT_MS = 60_000;
|
|
3615
|
+
|
|
3616
|
+
/** Set the browser-coverage host→service attribution from the loaded
|
|
3617
|
+
* project: only services that opted in to `coverage: { browser: true }`
|
|
3618
|
+
* are attributed, so a script from an uncovered service is ignored. Null
|
|
3619
|
+
* when no service opted in — coverage then attaches nothing. Called on
|
|
3620
|
+
* every /load. */
|
|
3621
|
+
function configureCoverageAttribution(project: Project): void {
|
|
3622
|
+
const env = project.environment;
|
|
3623
|
+
const browserServices = new Set<string>();
|
|
3624
|
+
for (const svc of namedServices(env)) {
|
|
3625
|
+
if (coverageUsesBrowser(svc.coverage)) browserServices.add(svc.name);
|
|
3626
|
+
}
|
|
3627
|
+
if (browserServices.size === 0) {
|
|
3628
|
+
configureBrowserCoverage(null);
|
|
3629
|
+
return;
|
|
3630
|
+
}
|
|
3631
|
+
const all = namedServices(env).map((s) => s.name);
|
|
3632
|
+
const table = {
|
|
3633
|
+
proxies: LOWERED.proxies,
|
|
3634
|
+
aliasesByService: LOWERED.aliasesByService,
|
|
3635
|
+
wildcards: LOWERED.wildcards,
|
|
3636
|
+
};
|
|
3637
|
+
configureBrowserCoverage({
|
|
3638
|
+
serviceForHost(host: string): string | undefined {
|
|
3639
|
+
const svc = hostToService(host, all, table);
|
|
3640
|
+
return svc && browserServices.has(svc) ? svc : undefined;
|
|
3641
|
+
},
|
|
3642
|
+
});
|
|
3643
|
+
}
|
|
3644
|
+
|
|
3645
|
+
async function captureServiceCoverage(): Promise<ServiceCoverageCapture[]> {
|
|
3646
|
+
const l = loaded;
|
|
3647
|
+
if (!l) return [];
|
|
3648
|
+
const byName = new Map<string, NamedService>();
|
|
3649
|
+
for (const s of namedServices(l.project.environment)) byName.set(s.name, s);
|
|
3650
|
+
for (const [name, s] of RUNTIME_SERVICES) byName.set(name, s);
|
|
3651
|
+
const services = [...byName.values()].filter((s) => s.coverage !== undefined);
|
|
3652
|
+
// Browser coverage is per-service lcov, accumulated in module memory
|
|
3653
|
+
// (browser-coverage.ts) across every harvest; read it once here.
|
|
3654
|
+
const browser = browserCoverageReports();
|
|
3655
|
+
return Promise.all(
|
|
3656
|
+
services.map(async (svc): Promise<ServiceCoverageCapture> => {
|
|
3657
|
+
const cov = svc.coverage;
|
|
3658
|
+
const reports: CoverageReport[] = [];
|
|
3659
|
+
// Container reports (a `command` runs first to flush a fresh dump).
|
|
3660
|
+
if (coverageUsesContainer(cov)) {
|
|
3661
|
+
if (typeof cov === "object" && cov !== null && cov.command) {
|
|
3662
|
+
const r = await docker(
|
|
3663
|
+
["exec", svc.name, "sh", "-c", cov.command],
|
|
3664
|
+
COVERAGE_COMMAND_TIMEOUT_MS,
|
|
3665
|
+
);
|
|
3666
|
+
if (r.code !== 0) {
|
|
3667
|
+
const tail = capMiddle((r.stderr || r.stdout).trim(), 2000).value;
|
|
3668
|
+
return {
|
|
3669
|
+
service: svc.name,
|
|
3670
|
+
reports: [],
|
|
3671
|
+
error: `coverage command for service "${svc.name}" failed (exit ${r.code}): ${JSON.stringify(cov.command)}${tail ? `\n${tail}` : ""}`,
|
|
3672
|
+
};
|
|
3673
|
+
}
|
|
3674
|
+
}
|
|
3675
|
+
const dir = await readCoverageDir(svc.name, coverageHostDir(WORKSPACE, svc.name));
|
|
3676
|
+
// A service that ALSO does browser coverage may legitimately have
|
|
3677
|
+
// an empty container dir for a test — don't fail if the browser
|
|
3678
|
+
// side has something. Otherwise the container contract stands.
|
|
3679
|
+
if (dir.error && !(coverageUsesBrowser(cov) && browser.has(svc.name))) {
|
|
3680
|
+
return dir;
|
|
3681
|
+
}
|
|
3682
|
+
reports.push(...dir.reports);
|
|
3683
|
+
}
|
|
3684
|
+
// Browser report for this service, when any script it served ran.
|
|
3685
|
+
const lcov = browser.get(svc.name);
|
|
3686
|
+
if (lcov) reports.push({ name: "browser.lcov", content: lcov });
|
|
3687
|
+
if (reports.length === 0 && coverageUsesBrowser(cov) && !coverageUsesContainer(cov)) {
|
|
3688
|
+
// Browser-only service, no browser activity yet this branch. Not an
|
|
3689
|
+
// error — a test needn't open a browser; the report appears once
|
|
3690
|
+
// one does (and forks to descendants from there).
|
|
3691
|
+
return { service: svc.name, reports: [] };
|
|
3692
|
+
}
|
|
3693
|
+
return { service: svc.name, reports };
|
|
3694
|
+
}),
|
|
3695
|
+
);
|
|
3696
|
+
}
|
|
3697
|
+
|
|
3529
3698
|
/** Per-Browser rrweb session shipped to the control plane. */
|
|
3530
3699
|
interface BrowserSessionRecord {
|
|
3531
3700
|
sessionId: string;
|
|
@@ -5020,6 +5189,32 @@ async function runOne(testCase: TestCase<unknown>): Promise<RunResult> {
|
|
|
5020
5189
|
// eslint-disable-next-line no-console
|
|
5021
5190
|
console.warn("[service-logs] delta capture failed:", err);
|
|
5022
5191
|
}
|
|
5192
|
+
// Coverage for the opted-in services, read after the clock stops like
|
|
5193
|
+
// the log deltas. Unlike them it is NOT best-effort: a service that opted
|
|
5194
|
+
// in and produced nothing readable fails the case, so a hole in the
|
|
5195
|
+
// coverage data is visible where it happened rather than silently
|
|
5196
|
+
// widening every later subset run.
|
|
5197
|
+
let coverageCaptures: ServiceCoverageCapture[] = [];
|
|
5198
|
+
try {
|
|
5199
|
+
coverageCaptures = await captureServiceCoverage();
|
|
5200
|
+
} catch (err) {
|
|
5201
|
+
coverageCaptures = [
|
|
5202
|
+
{
|
|
5203
|
+
service: "*",
|
|
5204
|
+
reports: [],
|
|
5205
|
+
error: `coverage capture failed: ${err instanceof Error ? err.message : String(err)}`,
|
|
5206
|
+
},
|
|
5207
|
+
];
|
|
5208
|
+
}
|
|
5209
|
+
const coverageErrors = coverageCaptures
|
|
5210
|
+
.filter((c) => c.error)
|
|
5211
|
+
.map((c) => c.error as string);
|
|
5212
|
+
if (coverageErrors.length > 0 && outcome.status === "passed") {
|
|
5213
|
+
outcome = {
|
|
5214
|
+
status: "failed",
|
|
5215
|
+
error: { message: coverageErrors.join("\n") },
|
|
5216
|
+
};
|
|
5217
|
+
}
|
|
5023
5218
|
const events = stopRecording();
|
|
5024
5219
|
// Drop sessions whose linking event didn't survive — an exec/terminal
|
|
5025
5220
|
// inside a failed `ctx.poll` iteration has its events removed by
|
|
@@ -5038,6 +5233,7 @@ async function runOne(testCase: TestCase<unknown>): Promise<RunResult> {
|
|
|
5038
5233
|
browserSessions: sessions.map((s) => s.record),
|
|
5039
5234
|
terminalSessions: terminalSessions.filter((s) => referencedSessions.has(s.sessionId)),
|
|
5040
5235
|
serviceLogDeltas,
|
|
5236
|
+
coverageCaptures,
|
|
5041
5237
|
error: outcome.error,
|
|
5042
5238
|
};
|
|
5043
5239
|
}
|
|
@@ -5940,7 +6136,19 @@ export function harnessMethods(state: RouteState): MethodTable {
|
|
|
5940
6136
|
// eslint-disable-next-line no-console
|
|
5941
6137
|
console.warn("[service-logs] baseline capture failed:", err);
|
|
5942
6138
|
}
|
|
5943
|
-
|
|
6139
|
+
// The coverage baseline rides the same call: what the opted-in
|
|
6140
|
+
// services ran during bring-up + project setup, parked under the
|
|
6141
|
+
// synthetic `__setup__` case for `coverageChunk`. A capture error is
|
|
6142
|
+
// reported in the ref, not thrown — there is no case to fail here,
|
|
6143
|
+
// the control plane logs it.
|
|
6144
|
+
let coverage: CoverageBundleRef | undefined;
|
|
6145
|
+
try {
|
|
6146
|
+
coverage = parkCoverageBundle(SETUP_CASE_ID, await captureServiceCoverage());
|
|
6147
|
+
} catch (err) {
|
|
6148
|
+
// eslint-disable-next-line no-console
|
|
6149
|
+
console.warn("[coverage] baseline capture failed:", err);
|
|
6150
|
+
}
|
|
6151
|
+
return { serviceLogDeltas, coverage };
|
|
5944
6152
|
},
|
|
5945
6153
|
|
|
5946
6154
|
run: async (params) => {
|
|
@@ -5954,7 +6162,10 @@ export function harnessMethods(state: RouteState): MethodTable {
|
|
|
5954
6162
|
// (with a warning) rather than failing the case — and never falls
|
|
5955
6163
|
// back to inlining, which is exactly the oversized reply this path
|
|
5956
6164
|
// exists to avoid.
|
|
5957
|
-
const { browserSessions, ...wire } = result;
|
|
6165
|
+
const { browserSessions, coverageCaptures, ...wire } = result;
|
|
6166
|
+
// Coverage reports take the same side channel: park, reply with a
|
|
6167
|
+
// ref, let the control plane pull through `coverageChunk`.
|
|
6168
|
+
const coverage = parkCoverageBundle(caseId, coverageCaptures);
|
|
5958
6169
|
let replay: { bytes: number } | undefined;
|
|
5959
6170
|
if (browserSessions.length > 0) {
|
|
5960
6171
|
try {
|
|
@@ -5966,7 +6177,7 @@ export function harnessMethods(state: RouteState): MethodTable {
|
|
|
5966
6177
|
console.warn("[replay] bundle encode failed; dropping recordings:", err);
|
|
5967
6178
|
}
|
|
5968
6179
|
}
|
|
5969
|
-
return { ...wire, replay };
|
|
6180
|
+
return { ...wire, replay, coverage };
|
|
5970
6181
|
});
|
|
5971
6182
|
},
|
|
5972
6183
|
|
|
@@ -5978,6 +6189,14 @@ export function harnessMethods(state: RouteState): MethodTable {
|
|
|
5978
6189
|
if (!gz) throw notFound(`no replay bundle for case: ${caseId}`);
|
|
5979
6190
|
return replayChunk(gz, params.offset as number | undefined, params.limit as number | undefined);
|
|
5980
6191
|
},
|
|
6192
|
+
|
|
6193
|
+
// One chunk of a parked coverage bundle (same shape as replayChunk).
|
|
6194
|
+
coverageChunk: async (params) => {
|
|
6195
|
+
const caseId = requireString(params, "caseId", "caseId");
|
|
6196
|
+
const gz = COVERAGE_BUNDLES.get(caseId);
|
|
6197
|
+
if (!gz) throw notFound(`no coverage bundle for case: ${caseId}`);
|
|
6198
|
+
return replayChunk(gz, params.offset as number | undefined, params.limit as number | undefined);
|
|
6199
|
+
},
|
|
5981
6200
|
};
|
|
5982
6201
|
}
|
|
5983
6202
|
|