@specific.dev/spectest 0.58.0 → 0.59.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/coverage.d.ts +142 -0
- package/dist/coverage.js +314 -0
- package/dist/daemon.js +124 -71
- package/dist/harness/coverage.d.ts +23 -4
- package/dist/harness/coverage.js +50 -14
- package/dist/index.d.ts +18 -46
- package/dist/index.js +6 -32
- package/package.json +6 -1
- package/src/coverage.test.ts +233 -0
- package/src/coverage.ts +401 -0
- package/src/daemon.ts +136 -75
- package/src/harness/coverage.test.ts +46 -1
- package/src/harness/coverage.ts +53 -13
- package/src/index.ts +36 -71
package/src/daemon.ts
CHANGED
|
@@ -32,9 +32,6 @@ import {
|
|
|
32
32
|
dnsName as makeDnsDecl,
|
|
33
33
|
isWildcard,
|
|
34
34
|
proxy as makeProxyDecl,
|
|
35
|
-
validateCoverage,
|
|
36
|
-
coverageUsesBrowser,
|
|
37
|
-
coverageUsesContainer,
|
|
38
35
|
} from "./index.js";
|
|
39
36
|
import type { DnsTarget, LoweredIngress } from "./index.js";
|
|
40
37
|
import {
|
|
@@ -42,15 +39,20 @@ import {
|
|
|
42
39
|
coverageBundleRef,
|
|
43
40
|
coverageHostDir,
|
|
44
41
|
encodeCoverageBundle,
|
|
42
|
+
isEmptyReport,
|
|
45
43
|
readCoverageDir,
|
|
46
44
|
type CoverageBundleRef,
|
|
47
45
|
type CoverageReport,
|
|
48
46
|
type ServiceCoverageCapture,
|
|
49
47
|
} from "./harness/coverage.js";
|
|
48
|
+
import { configureBrowserCoverage } from "./browser-coverage.js";
|
|
50
49
|
import {
|
|
51
|
-
|
|
52
|
-
|
|
53
|
-
|
|
50
|
+
applyCoverageAdapters,
|
|
51
|
+
coverageAdapters,
|
|
52
|
+
validateCoverage,
|
|
53
|
+
type CoverageAdapter,
|
|
54
|
+
type CoverageCaptureContext,
|
|
55
|
+
} from "./coverage.js";
|
|
54
56
|
import { serviceForHost as hostToService } from "./harness/browser-coverage.js";
|
|
55
57
|
import { acquirePersistentBrowser, mobileKey } from "./browser.js";
|
|
56
58
|
import { isMobileApp, openPersistentMobile } from "./mobile.js";
|
|
@@ -785,9 +787,6 @@ async function ensureVolumes(svc: NamedService): Promise<string[]> {
|
|
|
785
787
|
async function ensureCoverage(svc: NamedService): Promise<string[]> {
|
|
786
788
|
if (svc.coverage === undefined) return [];
|
|
787
789
|
validateCoverage(svc.name, svc.coverage);
|
|
788
|
-
// A browser-only opt-in has no container process to write reports, so it
|
|
789
|
-
// gets no mount — its coverage comes from the guest browser.
|
|
790
|
-
if (!coverageUsesContainer(svc.coverage)) return [];
|
|
791
790
|
const host = coverageHostDir(WORKSPACE, svc.name);
|
|
792
791
|
await fs.mkdir(host, { recursive: true });
|
|
793
792
|
await fs.chmod(host, 0o777);
|
|
@@ -1813,7 +1812,11 @@ function buildIngress(project: Project): void {
|
|
|
1813
1812
|
// generic decl set the daemon executes. The special-casing lives in the
|
|
1814
1813
|
// SDK's lowerIngress, not here.
|
|
1815
1814
|
LOWERED = lowerIngress(project);
|
|
1816
|
-
|
|
1815
|
+
BROWSER_COVERAGE_SERVICES.clear();
|
|
1816
|
+
void runCoverageLoadHooks(namedServices(project.environment)).catch((err) => {
|
|
1817
|
+
// eslint-disable-next-line no-console
|
|
1818
|
+
console.warn("[coverage] load hook failed:", err);
|
|
1819
|
+
});
|
|
1817
1820
|
if (!project.fakes) return;
|
|
1818
1821
|
for (const [name, def] of Object.entries(project.fakes)) {
|
|
1819
1822
|
FAKES.set(name, {
|
|
@@ -2797,7 +2800,9 @@ const RUNTIME_SERVICES = new Map<string, NamedService>();
|
|
|
2797
2800
|
// the orchestration helpers want a NamedService, which is the same shape.
|
|
2798
2801
|
function specToNamedService(spec: RuntimeServiceSpec): NamedService {
|
|
2799
2802
|
const { name, ...rest } = spec;
|
|
2800
|
-
|
|
2803
|
+
// A runtime spec skipped `defineEnvironment`, so its coverage adapters
|
|
2804
|
+
// get their config-time pass here.
|
|
2805
|
+
return { name, ...applyCoverageAdapters(name, rest as ServiceConfig) };
|
|
2801
2806
|
}
|
|
2802
2807
|
|
|
2803
2808
|
/** Implementation behind `ctx.startService` / a fake's `ctx.startService`.
|
|
@@ -2811,6 +2816,7 @@ async function startRuntimeService(spec: RuntimeServiceSpec): Promise<RuntimeSer
|
|
|
2811
2816
|
const t0 = Date.now();
|
|
2812
2817
|
const resv = reserveEvent();
|
|
2813
2818
|
const svc = specToNamedService(spec);
|
|
2819
|
+
await runCoverageLoadHooks([svc]);
|
|
2814
2820
|
const aliases = (spec.hostnames ?? []).map((h) => h.toLowerCase());
|
|
2815
2821
|
const imageRef = svc.image.type === "registry" ? svc.image.reference : "(dockerfile)";
|
|
2816
2822
|
try {
|
|
@@ -3601,34 +3607,53 @@ async function captureServiceLogDeltas(): Promise<ServiceLogDelta[]> {
|
|
|
3601
3607
|
}
|
|
3602
3608
|
|
|
3603
3609
|
// ────────────────────────────────────────────────────────────────────────
|
|
3604
|
-
// Coverage capture (opt-in per service; see harness/coverage.ts)
|
|
3610
|
+
// Coverage capture (opt-in per service; see coverage.ts + harness/coverage.ts)
|
|
3605
3611
|
// ────────────────────────────────────────────────────────────────────────
|
|
3606
3612
|
//
|
|
3607
|
-
// Read after bring-up and after every test, on every case.
|
|
3608
|
-
//
|
|
3609
|
-
//
|
|
3610
|
-
//
|
|
3611
|
-
//
|
|
3612
|
-
//
|
|
3613
|
-
|
|
3614
|
-
|
|
3615
|
-
|
|
3616
|
-
|
|
3617
|
-
|
|
3618
|
-
|
|
3619
|
-
|
|
3620
|
-
|
|
3621
|
-
|
|
3622
|
-
|
|
3623
|
-
|
|
3624
|
-
|
|
3625
|
-
|
|
3613
|
+
// Read after bring-up and after every test, on every case. A service's
|
|
3614
|
+
// adapters run first (each makes its report appear in the directory —
|
|
3615
|
+
// `nodeCoverage` asks the hook for a dump, `browserCoverage` writes the
|
|
3616
|
+
// harness's totals, `commandCoverage` runs the command), then the
|
|
3617
|
+
// directory is read the same way for every service, `coverage: true`
|
|
3618
|
+
// included. The reports live on the rootfs, so what a child fork reads
|
|
3619
|
+
// already includes everything its ancestors ran — no markers, no deltas
|
|
3620
|
+
// (unlike the service logs above). The reports ship verbatim; the server
|
|
3621
|
+
// derives what it needs from the stored bytes.
|
|
3622
|
+
|
|
3623
|
+
/** Budget for one service's adapters at one capture, all of them. */
|
|
3624
|
+
const COVERAGE_CAPTURE_TIMEOUT_MS = 60_000;
|
|
3625
|
+
|
|
3626
|
+
/** Services whose adapters asked for browser scripts at /load
|
|
3627
|
+
* (`CoverageLoadContext.collectBrowserScripts`). Cleared on every /load;
|
|
3628
|
+
* a runtime service's adapters add to it when it starts. */
|
|
3629
|
+
const BROWSER_COVERAGE_SERVICES = new Set<string>();
|
|
3630
|
+
|
|
3631
|
+
/** Run every adapter's `load` hook for `services`, then rebuild the
|
|
3632
|
+
* browser-coverage attribution from whoever asked. The attribution is a
|
|
3633
|
+
* host → service resolver over the lowered ingress, restricted to those
|
|
3634
|
+
* services, so a script from an uncovered service is ignored; null when
|
|
3635
|
+
* nobody asked, which costs nothing. */
|
|
3636
|
+
async function runCoverageLoadHooks(services: NamedService[]): Promise<void> {
|
|
3637
|
+
for (const svc of services) {
|
|
3638
|
+
for (const a of coverageAdapters(svc.coverage)) {
|
|
3639
|
+
if (!a.load) continue;
|
|
3640
|
+
await a.load({
|
|
3641
|
+
service: svc.name,
|
|
3642
|
+
collectBrowserScripts: () => {
|
|
3643
|
+
BROWSER_COVERAGE_SERVICES.add(svc.name);
|
|
3644
|
+
},
|
|
3645
|
+
});
|
|
3646
|
+
}
|
|
3626
3647
|
}
|
|
3627
|
-
|
|
3648
|
+
rebuildBrowserCoverageResolver();
|
|
3649
|
+
}
|
|
3650
|
+
|
|
3651
|
+
function rebuildBrowserCoverageResolver(): void {
|
|
3652
|
+
if (BROWSER_COVERAGE_SERVICES.size === 0 || !loaded) {
|
|
3628
3653
|
configureBrowserCoverage(null);
|
|
3629
3654
|
return;
|
|
3630
3655
|
}
|
|
3631
|
-
const all = namedServices(
|
|
3656
|
+
const all = [...namedServices(loaded.project.environment).map((s) => s.name), ...RUNTIME_SERVICES.keys()];
|
|
3632
3657
|
const table = {
|
|
3633
3658
|
proxies: LOWERED.proxies,
|
|
3634
3659
|
aliasesByService: LOWERED.aliasesByService,
|
|
@@ -3637,11 +3662,72 @@ function configureCoverageAttribution(project: Project): void {
|
|
|
3637
3662
|
configureBrowserCoverage({
|
|
3638
3663
|
serviceForHost(host: string): string | undefined {
|
|
3639
3664
|
const svc = hostToService(host, all, table);
|
|
3640
|
-
return svc &&
|
|
3665
|
+
return svc && BROWSER_COVERAGE_SERVICES.has(svc) ? svc : undefined;
|
|
3641
3666
|
},
|
|
3642
3667
|
});
|
|
3643
3668
|
}
|
|
3644
3669
|
|
|
3670
|
+
/** The context one adapter's `capture` runs against. */
|
|
3671
|
+
function coverageCaptureContext(svc: NamedService, signal: AbortSignal): CoverageCaptureContext {
|
|
3672
|
+
const reportDir = coverageHostDir(WORKSPACE, svc.name);
|
|
3673
|
+
return {
|
|
3674
|
+
service: svc.name,
|
|
3675
|
+
reportDir,
|
|
3676
|
+
signal,
|
|
3677
|
+
async exec(command: string) {
|
|
3678
|
+
const r = await docker(["exec", svc.name, "sh", "-c", command], COVERAGE_CAPTURE_TIMEOUT_MS);
|
|
3679
|
+
if (r.code !== 0) {
|
|
3680
|
+
const tail = capMiddle((r.stderr || r.stdout).trim(), 2000).value;
|
|
3681
|
+
throw new Error(`${JSON.stringify(command)} exited ${r.code}${tail ? `\n${tail}` : ""}`);
|
|
3682
|
+
}
|
|
3683
|
+
return { stdout: r.stdout, stderr: r.stderr };
|
|
3684
|
+
},
|
|
3685
|
+
async writeReport(name: string, content: string) {
|
|
3686
|
+
const file = path.join(reportDir, name);
|
|
3687
|
+
if (path.relative(reportDir, file).startsWith("..")) {
|
|
3688
|
+
throw new Error(`writeReport: ${JSON.stringify(name)} leaves the coverage directory`);
|
|
3689
|
+
}
|
|
3690
|
+
await fs.mkdir(path.dirname(file), { recursive: true });
|
|
3691
|
+
const tmp = path.join(path.dirname(file), `.${path.basename(file)}.tmp`);
|
|
3692
|
+
await fs.writeFile(tmp, content);
|
|
3693
|
+
await fs.rename(tmp, file);
|
|
3694
|
+
},
|
|
3695
|
+
};
|
|
3696
|
+
}
|
|
3697
|
+
|
|
3698
|
+
/** Run one service's adapters in order under one budget. Returns the
|
|
3699
|
+
* error to fail the capture with, naming the adapter. */
|
|
3700
|
+
async function runCoverageAdapters(svc: NamedService): Promise<string | undefined> {
|
|
3701
|
+
const adapters = coverageAdapters(svc.coverage).filter((a) => a.capture);
|
|
3702
|
+
if (adapters.length === 0) return undefined;
|
|
3703
|
+
const ac = new AbortController();
|
|
3704
|
+
const timer = setTimeout(() => ac.abort(), COVERAGE_CAPTURE_TIMEOUT_MS);
|
|
3705
|
+
const ctx = coverageCaptureContext(svc, ac.signal);
|
|
3706
|
+
try {
|
|
3707
|
+
for (const a of adapters) {
|
|
3708
|
+
try {
|
|
3709
|
+
await Promise.race([
|
|
3710
|
+
(a as Required<Pick<CoverageAdapter, "capture">>).capture(ctx),
|
|
3711
|
+
new Promise<never>((_, reject) => {
|
|
3712
|
+
ac.signal.addEventListener(
|
|
3713
|
+
"abort",
|
|
3714
|
+
() => reject(new Error(`no report within ${COVERAGE_CAPTURE_TIMEOUT_MS / 1000}s`)),
|
|
3715
|
+
{ once: true },
|
|
3716
|
+
);
|
|
3717
|
+
}),
|
|
3718
|
+
]);
|
|
3719
|
+
} catch (err) {
|
|
3720
|
+
return `coverage adapter "${a.name}" on service "${svc.name}" failed: ${
|
|
3721
|
+
err instanceof Error ? err.message : String(err)
|
|
3722
|
+
}`;
|
|
3723
|
+
}
|
|
3724
|
+
}
|
|
3725
|
+
return undefined;
|
|
3726
|
+
} finally {
|
|
3727
|
+
clearTimeout(timer);
|
|
3728
|
+
}
|
|
3729
|
+
}
|
|
3730
|
+
|
|
3645
3731
|
async function captureServiceCoverage(): Promise<ServiceCoverageCapture[]> {
|
|
3646
3732
|
const l = loaded;
|
|
3647
3733
|
if (!l) return [];
|
|
@@ -3649,48 +3735,11 @@ async function captureServiceCoverage(): Promise<ServiceCoverageCapture[]> {
|
|
|
3649
3735
|
for (const s of namedServices(l.project.environment)) byName.set(s.name, s);
|
|
3650
3736
|
for (const [name, s] of RUNTIME_SERVICES) byName.set(name, s);
|
|
3651
3737
|
const services = [...byName.values()].filter((s) => s.coverage !== undefined);
|
|
3652
|
-
// Browser coverage is per-service lcov, accumulated in module memory
|
|
3653
|
-
// (browser-coverage.ts) across every harvest; read it once here.
|
|
3654
|
-
const browser = browserCoverageReports();
|
|
3655
3738
|
return Promise.all(
|
|
3656
3739
|
services.map(async (svc): Promise<ServiceCoverageCapture> => {
|
|
3657
|
-
const
|
|
3658
|
-
|
|
3659
|
-
|
|
3660
|
-
if (coverageUsesContainer(cov)) {
|
|
3661
|
-
if (typeof cov === "object" && cov !== null && cov.command) {
|
|
3662
|
-
const r = await docker(
|
|
3663
|
-
["exec", svc.name, "sh", "-c", cov.command],
|
|
3664
|
-
COVERAGE_COMMAND_TIMEOUT_MS,
|
|
3665
|
-
);
|
|
3666
|
-
if (r.code !== 0) {
|
|
3667
|
-
const tail = capMiddle((r.stderr || r.stdout).trim(), 2000).value;
|
|
3668
|
-
return {
|
|
3669
|
-
service: svc.name,
|
|
3670
|
-
reports: [],
|
|
3671
|
-
error: `coverage command for service "${svc.name}" failed (exit ${r.code}): ${JSON.stringify(cov.command)}${tail ? `\n${tail}` : ""}`,
|
|
3672
|
-
};
|
|
3673
|
-
}
|
|
3674
|
-
}
|
|
3675
|
-
const dir = await readCoverageDir(svc.name, coverageHostDir(WORKSPACE, svc.name));
|
|
3676
|
-
// A service that ALSO does browser coverage may legitimately have
|
|
3677
|
-
// an empty container dir for a test — don't fail if the browser
|
|
3678
|
-
// side has something. Otherwise the container contract stands.
|
|
3679
|
-
if (dir.error && !(coverageUsesBrowser(cov) && browser.has(svc.name))) {
|
|
3680
|
-
return dir;
|
|
3681
|
-
}
|
|
3682
|
-
reports.push(...dir.reports);
|
|
3683
|
-
}
|
|
3684
|
-
// Browser report for this service, when any script it served ran.
|
|
3685
|
-
const lcov = browser.get(svc.name);
|
|
3686
|
-
if (lcov) reports.push({ name: "browser.lcov", content: lcov });
|
|
3687
|
-
if (reports.length === 0 && coverageUsesBrowser(cov) && !coverageUsesContainer(cov)) {
|
|
3688
|
-
// Browser-only service, no browser activity yet this branch. Not an
|
|
3689
|
-
// error — a test needn't open a browser; the report appears once
|
|
3690
|
-
// one does (and forks to descendants from there).
|
|
3691
|
-
return { service: svc.name, reports: [] };
|
|
3692
|
-
}
|
|
3693
|
-
return { service: svc.name, reports };
|
|
3740
|
+
const error = await runCoverageAdapters(svc);
|
|
3741
|
+
if (error) return { service: svc.name, reports: [], error };
|
|
3742
|
+
return readCoverageDir(svc.name, coverageHostDir(WORKSPACE, svc.name));
|
|
3694
3743
|
}),
|
|
3695
3744
|
);
|
|
3696
3745
|
}
|
|
@@ -6143,7 +6192,19 @@ export function harnessMethods(state: RouteState): MethodTable {
|
|
|
6143
6192
|
// the control plane logs it.
|
|
6144
6193
|
let coverage: CoverageBundleRef | undefined;
|
|
6145
6194
|
try {
|
|
6146
|
-
|
|
6195
|
+
const captures = await captureServiceCoverage();
|
|
6196
|
+
for (const c of captures) {
|
|
6197
|
+
// After bring-up the code that started the service has run, so a
|
|
6198
|
+
// report set that names no file at all is a misconfigured tool —
|
|
6199
|
+
// visible in the boot log, never a failure (there is no case).
|
|
6200
|
+
if (!c.error && c.reports.length > 0 && c.reports.every((r) => isEmptyReport(r.content))) {
|
|
6201
|
+
// eslint-disable-next-line no-console
|
|
6202
|
+
console.warn(
|
|
6203
|
+
`[coverage] service "${c.service}": every report is empty after bring-up — is coverage enabled at process start?`,
|
|
6204
|
+
);
|
|
6205
|
+
}
|
|
6206
|
+
}
|
|
6207
|
+
coverage = parkCoverageBundle(SETUP_CASE_ID, captures);
|
|
6147
6208
|
} catch (err) {
|
|
6148
6209
|
// eslint-disable-next-line no-console
|
|
6149
6210
|
console.warn("[coverage] baseline capture failed:", err);
|
|
@@ -8,6 +8,10 @@ import {
|
|
|
8
8
|
coverageHostDir,
|
|
9
9
|
encodeCoverageBundle,
|
|
10
10
|
hasLcovSourceFile,
|
|
11
|
+
isEmptyReport,
|
|
12
|
+
isLcovShaped,
|
|
13
|
+
isV8CoverageJson,
|
|
14
|
+
isWellFormedReport,
|
|
11
15
|
listReportFiles,
|
|
12
16
|
readCoverageDir,
|
|
13
17
|
} from "./coverage";
|
|
@@ -24,7 +28,48 @@ describe("hasLcovSourceFile", () => {
|
|
|
24
28
|
});
|
|
25
29
|
});
|
|
26
30
|
|
|
31
|
+
describe("report shapes", () => {
|
|
32
|
+
test("lcov: every non-blank line a record; an SF-less report is well-formed and empty", () => {
|
|
33
|
+
expect(isLcovShaped("TN:\nSF:/a.js\nDA:1,1\nend_of_record\n")).toBe(true);
|
|
34
|
+
expect(isLcovShaped("TN:\n")).toBe(true);
|
|
35
|
+
expect(isEmptyReport("TN:\n")).toBe(true);
|
|
36
|
+
expect(isEmptyReport("TN:\nSF:/a.js\n")).toBe(false);
|
|
37
|
+
expect(isLcovShaped("")).toBe(false);
|
|
38
|
+
expect(isLcovShaped("hello world\n")).toBe(false);
|
|
39
|
+
expect(isLcovShaped("SF:/a.js\ngarbage\n")).toBe(false);
|
|
40
|
+
});
|
|
41
|
+
test("V8 JSON: shape check only; an empty result is empty", () => {
|
|
42
|
+
const v8 = '{"result":[{"scriptId":"1","url":"file:///app/a.js","functions":[]}]}';
|
|
43
|
+
expect(isV8CoverageJson(v8)).toBe(true);
|
|
44
|
+
expect(isEmptyReport(v8)).toBe(false);
|
|
45
|
+
expect(isEmptyReport('{"result":[]}')).toBe(true);
|
|
46
|
+
expect(isV8CoverageJson("{}")).toBe(false);
|
|
47
|
+
expect(isWellFormedReport(v8)).toBe(true);
|
|
48
|
+
expect(isWellFormedReport("TN:\n")).toBe(true);
|
|
49
|
+
expect(isWellFormedReport("nope")).toBe(false);
|
|
50
|
+
});
|
|
51
|
+
});
|
|
52
|
+
|
|
27
53
|
describe("readCoverageDir", () => {
|
|
54
|
+
test("a V8 coverage JSON file satisfies the contract", async () => {
|
|
55
|
+
const dir = await fs.mkdtemp(path.join(os.tmpdir(), "spectest-cov-"));
|
|
56
|
+
await fs.writeFile(
|
|
57
|
+
path.join(dir, "coverage-1-2-3.json"),
|
|
58
|
+
'{"result":[{"scriptId":"1","url":"file:///app/a.js","functions":[]}]}',
|
|
59
|
+
);
|
|
60
|
+
const r = await readCoverageDir("web", dir);
|
|
61
|
+
expect(r.error).toBeUndefined();
|
|
62
|
+
expect(r.reports.length).toBe(1);
|
|
63
|
+
});
|
|
64
|
+
test("a malformed file next to a good one is an error naming the file", async () => {
|
|
65
|
+
const dir = await fs.mkdtemp(path.join(os.tmpdir(), "spectest-cov-"));
|
|
66
|
+
await fs.writeFile(path.join(dir, "lcov.info"), "TN:\nSF:/a.js\nend_of_record\n");
|
|
67
|
+
await fs.writeFile(path.join(dir, "junk.txt"), "not a report\n");
|
|
68
|
+
const r = await readCoverageDir("web", dir);
|
|
69
|
+
expect(r.error).toContain("junk.txt");
|
|
70
|
+
expect(r.reports.length).toBe(2);
|
|
71
|
+
});
|
|
72
|
+
|
|
28
73
|
test("ships every report verbatim, nested, with relative names", async () => {
|
|
29
74
|
const dir = await tmp();
|
|
30
75
|
await fs.mkdir(path.join(dir, "sub"));
|
|
@@ -56,7 +101,7 @@ describe("readCoverageDir", () => {
|
|
|
56
101
|
await fs.writeFile(path.join(dir, "coverage.json"), "{}");
|
|
57
102
|
const r = await readCoverageDir("web", dir);
|
|
58
103
|
expect(r.reports.length).toBe(1);
|
|
59
|
-
expect(r.error).toContain("
|
|
104
|
+
expect(r.error).toContain("none is an lcov report or a V8 coverage JSON document");
|
|
60
105
|
});
|
|
61
106
|
|
|
62
107
|
test("listReportFiles skips dotfiles", async () => {
|
package/src/harness/coverage.ts
CHANGED
|
@@ -2,12 +2,15 @@
|
|
|
2
2
|
//
|
|
3
3
|
// A service that opts in (`coverage: true | { command }`) gets a writable
|
|
4
4
|
// directory bind-mounted at `/spectest/coverage/` in its container. The
|
|
5
|
-
// service's own tool
|
|
5
|
+
// service's own tool, or its coverage adapters, write reports there —
|
|
6
|
+
// lcov, or the V8 coverage JSON that `NODE_V8_COVERAGE` writes. The daemon reads the
|
|
6
7
|
// directory from the VM side (no `docker exec` to fetch) after bring-up
|
|
7
8
|
// and after each test, and ships the reports **verbatim** to the control
|
|
8
9
|
// plane as a gzipped bundle. Nothing is interpreted here beyond the one
|
|
9
|
-
// contract the harness must fail a case on: at least one report,
|
|
10
|
-
//
|
|
10
|
+
// contract the harness must fail a case on: at least one report, every
|
|
11
|
+
// one well-formed. Counters may reset between dumps and a dump may be a
|
|
12
|
+
// new file: the directory forks with the environment, so it accumulates
|
|
13
|
+
// along the ancestor chain by itself. The server derives whatever it
|
|
11
14
|
// needs (today the set of source files; later, finer things) from the
|
|
12
15
|
// stored bytes, so a change in that derivation is a re-read of stored
|
|
13
16
|
// runs, never a hole in the history.
|
|
@@ -22,7 +25,7 @@ import { gzipSync } from "node:zlib";
|
|
|
22
25
|
export const COVERAGE_CONTAINER_DIR = "/spectest/coverage";
|
|
23
26
|
|
|
24
27
|
/** Raw bytes of reports one service may ship per capture. Reports are
|
|
25
|
-
*
|
|
28
|
+
* bounded by code size times chain depth, so anything past this is a
|
|
26
29
|
* runaway (a tool writing a new file per request), not coverage. */
|
|
27
30
|
export const COVERAGE_MAX_BYTES_PER_SERVICE = 64 * 1024 * 1024;
|
|
28
31
|
|
|
@@ -76,6 +79,37 @@ export function hasLcovSourceFile(text: string): boolean {
|
|
|
76
79
|
return /^SF:\S/m.test(text);
|
|
77
80
|
}
|
|
78
81
|
|
|
82
|
+
/** True when `text` is lcov-shaped: at least one line, every non-blank
|
|
83
|
+
* line an `XX:` record or `end_of_record`. An lcov with no `SF:` record
|
|
84
|
+
* at all (`TN:` alone) is well-formed — it is a report that says
|
|
85
|
+
* nothing ran. */
|
|
86
|
+
export function isLcovShaped(text: string): boolean {
|
|
87
|
+
const lines = text.split(/\r?\n/).filter((l) => l.trim().length > 0);
|
|
88
|
+
return lines.length > 0 && lines.every((l) => /^[A-Z]{2,5}:/.test(l) || l.trim() === "end_of_record");
|
|
89
|
+
}
|
|
90
|
+
|
|
91
|
+
/**
|
|
92
|
+
* True when `text` looks like a V8 coverage JSON document — what
|
|
93
|
+
* `NODE_V8_COVERAGE` / `v8.takeCoverage()` write: an object with a
|
|
94
|
+
* `result` array of scripts. A cheap shape check, not a parse (a report
|
|
95
|
+
* can be megabytes); the control plane parses for real.
|
|
96
|
+
*/
|
|
97
|
+
export function isV8CoverageJson(text: string): boolean {
|
|
98
|
+
return /^\s*\{/.test(text) && /"result"\s*:\s*\[/.test(text);
|
|
99
|
+
}
|
|
100
|
+
|
|
101
|
+
/** True when `text` is a report spectest stores: lcov or V8 JSON. */
|
|
102
|
+
export function isWellFormedReport(text: string): boolean {
|
|
103
|
+
return isLcovShaped(text) || isV8CoverageJson(text);
|
|
104
|
+
}
|
|
105
|
+
|
|
106
|
+
/** True when a well-formed report names no source file at all (an lcov
|
|
107
|
+
* with no `SF:`, a V8 document with an empty `result`). */
|
|
108
|
+
export function isEmptyReport(text: string): boolean {
|
|
109
|
+
if (isV8CoverageJson(text)) return /"result"\s*:\s*\[\s*\]/.test(text);
|
|
110
|
+
return !hasLcovSourceFile(text);
|
|
111
|
+
}
|
|
112
|
+
|
|
79
113
|
/** Every regular file under `dir`, recursively, as paths relative to it.
|
|
80
114
|
* Hidden files are skipped (a tool's own bookkeeping — `.seq`, `.lock`,
|
|
81
115
|
* a tmp file mid-write — is not a report). */
|
|
@@ -100,9 +134,11 @@ export async function listReportFiles(dir: string): Promise<string[]> {
|
|
|
100
134
|
}
|
|
101
135
|
|
|
102
136
|
/**
|
|
103
|
-
* Read every report under `dir`, verbatim. An empty directory, or
|
|
104
|
-
*
|
|
105
|
-
* contract is that an opted-in service always has a
|
|
137
|
+
* Read every report under `dir`, verbatim. An empty directory, or a file
|
|
138
|
+
* that is neither lcov nor V8 JSON, is an error rather than an empty
|
|
139
|
+
* capture: the contract is that an opted-in service always has a
|
|
140
|
+
* well-formed report to read. A report that names no file is fine — it
|
|
141
|
+
* says nothing ran.
|
|
106
142
|
*/
|
|
107
143
|
export async function readCoverageDir(
|
|
108
144
|
service: string,
|
|
@@ -113,12 +149,13 @@ export async function readCoverageDir(
|
|
|
113
149
|
return {
|
|
114
150
|
service,
|
|
115
151
|
reports: [],
|
|
116
|
-
error: `service "${service}" opted in to coverage but ${COVERAGE_CONTAINER_DIR}/ holds no report — the service (or its coverage command) must write an lcov file there`,
|
|
152
|
+
error: `service "${service}" opted in to coverage but ${COVERAGE_CONTAINER_DIR}/ holds no report — the service (or its coverage command) must write an lcov or V8 coverage JSON file there`,
|
|
117
153
|
};
|
|
118
154
|
}
|
|
119
155
|
const reports: CoverageReport[] = [];
|
|
120
156
|
let total = 0;
|
|
121
|
-
let
|
|
157
|
+
let anyReport = false;
|
|
158
|
+
const malformed: string[] = [];
|
|
122
159
|
for (const name of names) {
|
|
123
160
|
let content: string;
|
|
124
161
|
try {
|
|
@@ -131,17 +168,20 @@ export async function readCoverageDir(
|
|
|
131
168
|
return {
|
|
132
169
|
service,
|
|
133
170
|
reports: [],
|
|
134
|
-
error: `service "${service}" has more than ${COVERAGE_MAX_BYTES_PER_SERVICE >> 20} MiB of reports in ${COVERAGE_CONTAINER_DIR}/ —
|
|
171
|
+
error: `service "${service}" has more than ${COVERAGE_MAX_BYTES_PER_SERVICE >> 20} MiB of reports in ${COVERAGE_CONTAINER_DIR}/ — that is a runaway, not coverage`,
|
|
135
172
|
};
|
|
136
173
|
}
|
|
137
|
-
if (
|
|
174
|
+
if (isWellFormedReport(content)) anyReport = true;
|
|
175
|
+
else malformed.push(name);
|
|
138
176
|
reports.push({ name, content });
|
|
139
177
|
}
|
|
140
|
-
if (!
|
|
178
|
+
if (!anyReport || malformed.length > 0) {
|
|
141
179
|
return {
|
|
142
180
|
service,
|
|
143
181
|
reports,
|
|
144
|
-
error: `service "${service}" wrote ${reports.length} file(s) to ${COVERAGE_CONTAINER_DIR}/ but
|
|
182
|
+
error: `service "${service}" wrote ${reports.length} file(s) to ${COVERAGE_CONTAINER_DIR}/ but ${
|
|
183
|
+
anyReport ? `${malformed.slice(0, 5).join(", ")} ${malformed.length === 1 ? "is" : "are"} neither` : "none is"
|
|
184
|
+
} an lcov report or a V8 coverage JSON document — only those two formats are read`,
|
|
145
185
|
};
|
|
146
186
|
}
|
|
147
187
|
return { service, reports };
|
package/src/index.ts
CHANGED
|
@@ -132,6 +132,19 @@ export type { Terminal, TerminalOpts, TerminalResult } from "./terminal.js";
|
|
|
132
132
|
|
|
133
133
|
import type { Terminal, TerminalOpts, TerminalResult } from "./terminal.js";
|
|
134
134
|
|
|
135
|
+
// Coverage adapter types. The adapters themselves (`node()`, `browser()`,
|
|
136
|
+
// `command()`, `defineAdapter`) are imported from
|
|
137
|
+
// `@specific.dev/spectest/coverage` — see `coverage.ts` and
|
|
138
|
+
// `spectest docs /services/coverage`.
|
|
139
|
+
export {
|
|
140
|
+
type CoverageAdapter,
|
|
141
|
+
type CoverageCaptureContext,
|
|
142
|
+
type CoverageConfigureInfo,
|
|
143
|
+
type CoverageLoadContext,
|
|
144
|
+
type ServiceCoverage,
|
|
145
|
+
} from "./coverage.js";
|
|
146
|
+
import { applyCoverageAdapters, validateCoverage, type ServiceCoverage } from "./coverage.js";
|
|
147
|
+
|
|
135
148
|
// Low-level ingress primitives + the framework lowering that the friendly
|
|
136
149
|
// `tls` / `hostnames` fields and `defineFake(...)` are built on. See
|
|
137
150
|
// `ingress.ts`.
|
|
@@ -256,20 +269,24 @@ export interface ServiceConfig {
|
|
|
256
269
|
* Opt this service in to **code coverage collection** (experimental).
|
|
257
270
|
*
|
|
258
271
|
* The service gets a writable directory bind-mounted at
|
|
259
|
-
* `/spectest/coverage/`.
|
|
260
|
-
*
|
|
261
|
-
*
|
|
262
|
-
* the container, then reads every file in the directory and records the
|
|
263
|
-
* set of source files that ran. Only the `SF:` records are read.
|
|
272
|
+
* `/spectest/coverage/`. After bring-up and after every test, spectest
|
|
273
|
+
* reads every file in it — **lcov** or **V8 coverage JSON**, verbatim —
|
|
274
|
+
* and stores them with the test; the control plane derives what ran.
|
|
264
275
|
*
|
|
265
|
-
* - `true`: the
|
|
266
|
-
*
|
|
267
|
-
* - `{
|
|
268
|
-
*
|
|
276
|
+
* - `true`: the program writes its own reports there (an exit-time dump
|
|
277
|
+
* for a short-lived process, a dump on a timer, its own hook).
|
|
278
|
+
* - `{ adapters: [...] }`: spectest gets the reports out. From
|
|
279
|
+
* `@specific.dev/spectest/coverage`: `node()` (V8 JSON from a node
|
|
280
|
+
* service, through a hook spectest mounts), `browser()` (the frontend
|
|
281
|
+
* this service serves, collected from the guest browser),
|
|
282
|
+
* `command(cmd)` (run a command in the container first), or your own
|
|
283
|
+
* via `defineAdapter`.
|
|
269
284
|
*
|
|
270
|
-
* The directory must hold at least one
|
|
271
|
-
* an empty directory or a
|
|
272
|
-
* error, so a hole in the coverage data is never silent.
|
|
285
|
+
* The directory must hold at least one well-formed report each time it
|
|
286
|
+
* is read; an empty directory or a failing adapter fails the test with a
|
|
287
|
+
* clear error, so a hole in the coverage data is never silent. Reports
|
|
288
|
+
* accumulate along the test's ancestor chain by themselves (the
|
|
289
|
+
* directory forks with the environment). See
|
|
273
290
|
* `spectest docs /services/coverage`.
|
|
274
291
|
*/
|
|
275
292
|
coverage?: ServiceCoverage;
|
|
@@ -1038,31 +1055,8 @@ export type ServiceImage =
|
|
|
1038
1055
|
exclude?: readonly string[];
|
|
1039
1056
|
};
|
|
1040
1057
|
|
|
1041
|
-
|
|
1042
|
-
|
|
1043
|
-
* `true` when the service writes its own reports; `{ command }` when
|
|
1044
|
-
* spectest must run a command in the container to produce one.
|
|
1045
|
-
*/
|
|
1046
|
-
export type ServiceCoverage =
|
|
1047
|
-
| true
|
|
1048
|
-
| {
|
|
1049
|
-
/** Run this (via `sh -c`) in the container before the coverage
|
|
1050
|
-
* directory is read, so the service writes a fresh lcov report. */
|
|
1051
|
-
command?: string;
|
|
1052
|
-
/**
|
|
1053
|
-
* Collect **frontend** coverage for the app this service serves.
|
|
1054
|
-
* The code runs in the guest browser — spectest's own process — so
|
|
1055
|
-
* the user cannot write a report for it: spectest turns on V8
|
|
1056
|
-
* coverage for every script `ctx.browser()`/`ctx.mobile()` loads
|
|
1057
|
-
* from this service's origin, maps it back to original sources
|
|
1058
|
-
* through the served source maps, and adds one lcov report to this
|
|
1059
|
-
* service's capture. Scripts a source map can't resolve are
|
|
1060
|
-
* attributed to their served path (which the repo mapping then
|
|
1061
|
-
* treats as unknown — the safe answer). Needs the app to serve
|
|
1062
|
-
* source maps to the test environment.
|
|
1063
|
-
*/
|
|
1064
|
-
browser?: boolean;
|
|
1065
|
-
};
|
|
1058
|
+
// Coverage adapters live in `coverage.ts`; the type is re-declared here
|
|
1059
|
+
// through the import below so `ServiceConfig.coverage` reads in one place.
|
|
1066
1060
|
|
|
1067
1061
|
export interface VolumeMount {
|
|
1068
1062
|
/**
|
|
@@ -1283,39 +1277,6 @@ export interface FakeContext {
|
|
|
1283
1277
|
certificate(hostnames: readonly string[]): Promise<CertificateMaterial>;
|
|
1284
1278
|
}
|
|
1285
1279
|
|
|
1286
|
-
/**
|
|
1287
|
-
* Check one service's `coverage` field. Exported so the daemon applies the
|
|
1288
|
-
* same rule to a runtime `startService` spec.
|
|
1289
|
-
*/
|
|
1290
|
-
export function validateCoverage(service: string, cov: unknown): void {
|
|
1291
|
-
if (cov === undefined || cov === true) return;
|
|
1292
|
-
if (typeof cov === "object" && cov !== null) {
|
|
1293
|
-
const o = cov as { command?: unknown; browser?: unknown };
|
|
1294
|
-
const cmdOk =
|
|
1295
|
-
o.command === undefined ||
|
|
1296
|
-
(typeof o.command === "string" && o.command.trim().length > 0);
|
|
1297
|
-
const browserOk = o.browser === undefined || typeof o.browser === "boolean";
|
|
1298
|
-
if (cmdOk && browserOk && (o.command !== undefined || o.browser === true)) return;
|
|
1299
|
-
}
|
|
1300
|
-
throw new Error(
|
|
1301
|
-
`service "${service}" has an invalid \`coverage\` value — use \`true\`, \`{ command: "<shell command>" }\`, or \`{ browser: true }\``,
|
|
1302
|
-
);
|
|
1303
|
-
}
|
|
1304
|
-
|
|
1305
|
-
/** Whether a `coverage` value collects reports from the container's own
|
|
1306
|
-
* filesystem (`/spectest/coverage/`). True for `true`, `{ command }`, and
|
|
1307
|
-
* any object that isn't browser-only. */
|
|
1308
|
-
export function coverageUsesContainer(cov: ServiceCoverage | undefined): boolean {
|
|
1309
|
-
if (cov === undefined) return false;
|
|
1310
|
-
if (cov === true) return true;
|
|
1311
|
-
return cov.command !== undefined || cov.browser !== true;
|
|
1312
|
-
}
|
|
1313
|
-
|
|
1314
|
-
/** Whether a `coverage` value collects frontend (browser) coverage. */
|
|
1315
|
-
export function coverageUsesBrowser(cov: ServiceCoverage | undefined): boolean {
|
|
1316
|
-
return typeof cov === "object" && cov.browser === true;
|
|
1317
|
-
}
|
|
1318
|
-
|
|
1319
1280
|
function validateEnvironmentConfig<S extends ServicesMap>(
|
|
1320
1281
|
config: EnvironmentConfig<S>,
|
|
1321
1282
|
): void {
|
|
@@ -2236,9 +2197,13 @@ export function defineEnvironment<
|
|
|
2236
2197
|
// { name, services, timeoutSecs } — with plain, fully-expanded services —
|
|
2237
2198
|
// is stored as `config` and shipped to the control plane.
|
|
2238
2199
|
const { fakes, services: rawServices, ...rest } = input;
|
|
2200
|
+
const expanded = expandServiceGroups(rawServices);
|
|
2201
|
+
for (const [key, svc] of Object.entries(expanded)) {
|
|
2202
|
+
expanded[key] = applyCoverageAdapters(key, svc);
|
|
2203
|
+
}
|
|
2239
2204
|
const config: EnvironmentConfig<S> = {
|
|
2240
2205
|
...rest,
|
|
2241
|
-
services:
|
|
2206
|
+
services: expanded as S,
|
|
2242
2207
|
};
|
|
2243
2208
|
validateEnvironmentConfig(config);
|
|
2244
2209
|
if (fakes) validateFakes(config, fakes);
|