@specific.dev/spectest 0.58.0 → 0.59.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/coverage.d.ts +142 -0
- package/dist/coverage.js +314 -0
- package/dist/daemon.js +124 -71
- package/dist/harness/coverage.d.ts +23 -4
- package/dist/harness/coverage.js +50 -14
- package/dist/index.d.ts +18 -46
- package/dist/index.js +6 -32
- package/package.json +6 -1
- package/src/coverage.test.ts +233 -0
- package/src/coverage.ts +401 -0
- package/src/daemon.ts +136 -75
- package/src/harness/coverage.test.ts +46 -1
- package/src/harness/coverage.ts +53 -13
- package/src/index.ts +36 -71
package/dist/daemon.js
CHANGED
|
@@ -21,9 +21,10 @@ import { existsSync, promises as fs, readFileSync } from "node:fs";
|
|
|
21
21
|
import net from "node:net";
|
|
22
22
|
import path from "node:path";
|
|
23
23
|
import { pathToFileURL } from "node:url";
|
|
24
|
-
import { assert, expect, expectRaw, lowerIngress, dnsName as makeDnsDecl, isWildcard, proxy as makeProxyDecl,
|
|
25
|
-
import { COVERAGE_CONTAINER_DIR, coverageBundleRef, coverageHostDir, encodeCoverageBundle, readCoverageDir, } from "./harness/coverage.js";
|
|
26
|
-
import {
|
|
24
|
+
import { assert, expect, expectRaw, lowerIngress, dnsName as makeDnsDecl, isWildcard, proxy as makeProxyDecl, } from "./index.js";
|
|
25
|
+
import { COVERAGE_CONTAINER_DIR, coverageBundleRef, coverageHostDir, encodeCoverageBundle, isEmptyReport, readCoverageDir, } from "./harness/coverage.js";
|
|
26
|
+
import { configureBrowserCoverage } from "./browser-coverage.js";
|
|
27
|
+
import { applyCoverageAdapters, coverageAdapters, validateCoverage, } from "./coverage.js";
|
|
27
28
|
import { serviceForHost as hostToService } from "./harness/browser-coverage.js";
|
|
28
29
|
import { acquirePersistentBrowser, mobileKey } from "./browser.js";
|
|
29
30
|
import { isMobileApp, openPersistentMobile } from "./mobile.js";
|
|
@@ -525,10 +526,6 @@ async function ensureCoverage(svc) {
|
|
|
525
526
|
if (svc.coverage === undefined)
|
|
526
527
|
return [];
|
|
527
528
|
validateCoverage(svc.name, svc.coverage);
|
|
528
|
-
// A browser-only opt-in has no container process to write reports, so it
|
|
529
|
-
// gets no mount — its coverage comes from the guest browser.
|
|
530
|
-
if (!coverageUsesContainer(svc.coverage))
|
|
531
|
-
return [];
|
|
532
529
|
const host = coverageHostDir(WORKSPACE, svc.name);
|
|
533
530
|
await fs.mkdir(host, { recursive: true });
|
|
534
531
|
await fs.chmod(host, 0o777);
|
|
@@ -1428,7 +1425,11 @@ function buildIngress(project) {
|
|
|
1428
1425
|
// generic decl set the daemon executes. The special-casing lives in the
|
|
1429
1426
|
// SDK's lowerIngress, not here.
|
|
1430
1427
|
LOWERED = lowerIngress(project);
|
|
1431
|
-
|
|
1428
|
+
BROWSER_COVERAGE_SERVICES.clear();
|
|
1429
|
+
void runCoverageLoadHooks(namedServices(project.environment)).catch((err) => {
|
|
1430
|
+
// eslint-disable-next-line no-console
|
|
1431
|
+
console.warn("[coverage] load hook failed:", err);
|
|
1432
|
+
});
|
|
1432
1433
|
if (!project.fakes)
|
|
1433
1434
|
return;
|
|
1434
1435
|
for (const [name, def] of Object.entries(project.fakes)) {
|
|
@@ -2313,7 +2314,9 @@ const RUNTIME_SERVICES = new Map();
|
|
|
2313
2314
|
// the orchestration helpers want a NamedService, which is the same shape.
|
|
2314
2315
|
function specToNamedService(spec) {
|
|
2315
2316
|
const { name, ...rest } = spec;
|
|
2316
|
-
|
|
2317
|
+
// A runtime spec skipped `defineEnvironment`, so its coverage adapters
|
|
2318
|
+
// get their config-time pass here.
|
|
2319
|
+
return { name, ...applyCoverageAdapters(name, rest) };
|
|
2317
2320
|
}
|
|
2318
2321
|
/** Implementation behind `ctx.startService` / a fake's `ctx.startService`.
|
|
2319
2322
|
* Prepares the image (pulling on first use through the host cache), runs
|
|
@@ -2326,6 +2329,7 @@ async function startRuntimeService(spec) {
|
|
|
2326
2329
|
const t0 = Date.now();
|
|
2327
2330
|
const resv = reserveEvent();
|
|
2328
2331
|
const svc = specToNamedService(spec);
|
|
2332
|
+
await runCoverageLoadHooks([svc]);
|
|
2329
2333
|
const aliases = (spec.hostnames ?? []).map((h) => h.toLowerCase());
|
|
2330
2334
|
const imageRef = svc.image.type === "registry" ? svc.image.reference : "(dockerfile)";
|
|
2331
2335
|
try {
|
|
@@ -2977,33 +2981,50 @@ async function captureServiceLogDeltas() {
|
|
|
2977
2981
|
}));
|
|
2978
2982
|
}
|
|
2979
2983
|
// ────────────────────────────────────────────────────────────────────────
|
|
2980
|
-
// Coverage capture (opt-in per service; see harness/coverage.ts)
|
|
2984
|
+
// Coverage capture (opt-in per service; see coverage.ts + harness/coverage.ts)
|
|
2981
2985
|
// ────────────────────────────────────────────────────────────────────────
|
|
2982
2986
|
//
|
|
2983
|
-
// Read after bring-up and after every test, on every case.
|
|
2984
|
-
//
|
|
2985
|
-
//
|
|
2986
|
-
//
|
|
2987
|
-
//
|
|
2988
|
-
//
|
|
2989
|
-
|
|
2990
|
-
|
|
2991
|
-
|
|
2992
|
-
|
|
2993
|
-
|
|
2994
|
-
|
|
2995
|
-
|
|
2996
|
-
|
|
2997
|
-
|
|
2998
|
-
|
|
2999
|
-
|
|
3000
|
-
|
|
3001
|
-
|
|
3002
|
-
|
|
2987
|
+
// Read after bring-up and after every test, on every case. A service's
|
|
2988
|
+
// adapters run first (each makes its report appear in the directory —
|
|
2989
|
+
// `nodeCoverage` asks the hook for a dump, `browserCoverage` writes the
|
|
2990
|
+
// harness's totals, `commandCoverage` runs the command), then the
|
|
2991
|
+
// directory is read the same way for every service, `coverage: true`
|
|
2992
|
+
// included. The reports live on the rootfs, so what a child fork reads
|
|
2993
|
+
// already includes everything its ancestors ran — no markers, no deltas
|
|
2994
|
+
// (unlike the service logs above). The reports ship verbatim; the server
|
|
2995
|
+
// derives what it needs from the stored bytes.
|
|
2996
|
+
/** Budget for one service's adapters at one capture, all of them. */
|
|
2997
|
+
const COVERAGE_CAPTURE_TIMEOUT_MS = 60_000;
|
|
2998
|
+
/** Services whose adapters asked for browser scripts at /load
|
|
2999
|
+
* (`CoverageLoadContext.collectBrowserScripts`). Cleared on every /load;
|
|
3000
|
+
* a runtime service's adapters add to it when it starts. */
|
|
3001
|
+
const BROWSER_COVERAGE_SERVICES = new Set();
|
|
3002
|
+
/** Run every adapter's `load` hook for `services`, then rebuild the
|
|
3003
|
+
* browser-coverage attribution from whoever asked. The attribution is a
|
|
3004
|
+
* host → service resolver over the lowered ingress, restricted to those
|
|
3005
|
+
* services, so a script from an uncovered service is ignored; null when
|
|
3006
|
+
* nobody asked, which costs nothing. */
|
|
3007
|
+
async function runCoverageLoadHooks(services) {
|
|
3008
|
+
for (const svc of services) {
|
|
3009
|
+
for (const a of coverageAdapters(svc.coverage)) {
|
|
3010
|
+
if (!a.load)
|
|
3011
|
+
continue;
|
|
3012
|
+
await a.load({
|
|
3013
|
+
service: svc.name,
|
|
3014
|
+
collectBrowserScripts: () => {
|
|
3015
|
+
BROWSER_COVERAGE_SERVICES.add(svc.name);
|
|
3016
|
+
},
|
|
3017
|
+
});
|
|
3018
|
+
}
|
|
3019
|
+
}
|
|
3020
|
+
rebuildBrowserCoverageResolver();
|
|
3021
|
+
}
|
|
3022
|
+
function rebuildBrowserCoverageResolver() {
|
|
3023
|
+
if (BROWSER_COVERAGE_SERVICES.size === 0 || !loaded) {
|
|
3003
3024
|
configureBrowserCoverage(null);
|
|
3004
3025
|
return;
|
|
3005
3026
|
}
|
|
3006
|
-
const all = namedServices(
|
|
3027
|
+
const all = [...namedServices(loaded.project.environment).map((s) => s.name), ...RUNTIME_SERVICES.keys()];
|
|
3007
3028
|
const table = {
|
|
3008
3029
|
proxies: LOWERED.proxies,
|
|
3009
3030
|
aliasesByService: LOWERED.aliasesByService,
|
|
@@ -3012,10 +3033,66 @@ function configureCoverageAttribution(project) {
|
|
|
3012
3033
|
configureBrowserCoverage({
|
|
3013
3034
|
serviceForHost(host) {
|
|
3014
3035
|
const svc = hostToService(host, all, table);
|
|
3015
|
-
return svc &&
|
|
3036
|
+
return svc && BROWSER_COVERAGE_SERVICES.has(svc) ? svc : undefined;
|
|
3016
3037
|
},
|
|
3017
3038
|
});
|
|
3018
3039
|
}
|
|
3040
|
+
/** The context one adapter's `capture` runs against. */
|
|
3041
|
+
function coverageCaptureContext(svc, signal) {
|
|
3042
|
+
const reportDir = coverageHostDir(WORKSPACE, svc.name);
|
|
3043
|
+
return {
|
|
3044
|
+
service: svc.name,
|
|
3045
|
+
reportDir,
|
|
3046
|
+
signal,
|
|
3047
|
+
async exec(command) {
|
|
3048
|
+
const r = await docker(["exec", svc.name, "sh", "-c", command], COVERAGE_CAPTURE_TIMEOUT_MS);
|
|
3049
|
+
if (r.code !== 0) {
|
|
3050
|
+
const tail = capMiddle((r.stderr || r.stdout).trim(), 2000).value;
|
|
3051
|
+
throw new Error(`${JSON.stringify(command)} exited ${r.code}${tail ? `\n${tail}` : ""}`);
|
|
3052
|
+
}
|
|
3053
|
+
return { stdout: r.stdout, stderr: r.stderr };
|
|
3054
|
+
},
|
|
3055
|
+
async writeReport(name, content) {
|
|
3056
|
+
const file = path.join(reportDir, name);
|
|
3057
|
+
if (path.relative(reportDir, file).startsWith("..")) {
|
|
3058
|
+
throw new Error(`writeReport: ${JSON.stringify(name)} leaves the coverage directory`);
|
|
3059
|
+
}
|
|
3060
|
+
await fs.mkdir(path.dirname(file), { recursive: true });
|
|
3061
|
+
const tmp = path.join(path.dirname(file), `.${path.basename(file)}.tmp`);
|
|
3062
|
+
await fs.writeFile(tmp, content);
|
|
3063
|
+
await fs.rename(tmp, file);
|
|
3064
|
+
},
|
|
3065
|
+
};
|
|
3066
|
+
}
|
|
3067
|
+
/** Run one service's adapters in order under one budget. Returns the
|
|
3068
|
+
* error to fail the capture with, naming the adapter. */
|
|
3069
|
+
async function runCoverageAdapters(svc) {
|
|
3070
|
+
const adapters = coverageAdapters(svc.coverage).filter((a) => a.capture);
|
|
3071
|
+
if (adapters.length === 0)
|
|
3072
|
+
return undefined;
|
|
3073
|
+
const ac = new AbortController();
|
|
3074
|
+
const timer = setTimeout(() => ac.abort(), COVERAGE_CAPTURE_TIMEOUT_MS);
|
|
3075
|
+
const ctx = coverageCaptureContext(svc, ac.signal);
|
|
3076
|
+
try {
|
|
3077
|
+
for (const a of adapters) {
|
|
3078
|
+
try {
|
|
3079
|
+
await Promise.race([
|
|
3080
|
+
a.capture(ctx),
|
|
3081
|
+
new Promise((_, reject) => {
|
|
3082
|
+
ac.signal.addEventListener("abort", () => reject(new Error(`no report within ${COVERAGE_CAPTURE_TIMEOUT_MS / 1000}s`)), { once: true });
|
|
3083
|
+
}),
|
|
3084
|
+
]);
|
|
3085
|
+
}
|
|
3086
|
+
catch (err) {
|
|
3087
|
+
return `coverage adapter "${a.name}" on service "${svc.name}" failed: ${err instanceof Error ? err.message : String(err)}`;
|
|
3088
|
+
}
|
|
3089
|
+
}
|
|
3090
|
+
return undefined;
|
|
3091
|
+
}
|
|
3092
|
+
finally {
|
|
3093
|
+
clearTimeout(timer);
|
|
3094
|
+
}
|
|
3095
|
+
}
|
|
3019
3096
|
async function captureServiceCoverage() {
|
|
3020
3097
|
const l = loaded;
|
|
3021
3098
|
if (!l)
|
|
@@ -3026,45 +3103,11 @@ async function captureServiceCoverage() {
|
|
|
3026
3103
|
for (const [name, s] of RUNTIME_SERVICES)
|
|
3027
3104
|
byName.set(name, s);
|
|
3028
3105
|
const services = [...byName.values()].filter((s) => s.coverage !== undefined);
|
|
3029
|
-
// Browser coverage is per-service lcov, accumulated in module memory
|
|
3030
|
-
// (browser-coverage.ts) across every harvest; read it once here.
|
|
3031
|
-
const browser = browserCoverageReports();
|
|
3032
3106
|
return Promise.all(services.map(async (svc) => {
|
|
3033
|
-
const
|
|
3034
|
-
|
|
3035
|
-
|
|
3036
|
-
|
|
3037
|
-
if (typeof cov === "object" && cov !== null && cov.command) {
|
|
3038
|
-
const r = await docker(["exec", svc.name, "sh", "-c", cov.command], COVERAGE_COMMAND_TIMEOUT_MS);
|
|
3039
|
-
if (r.code !== 0) {
|
|
3040
|
-
const tail = capMiddle((r.stderr || r.stdout).trim(), 2000).value;
|
|
3041
|
-
return {
|
|
3042
|
-
service: svc.name,
|
|
3043
|
-
reports: [],
|
|
3044
|
-
error: `coverage command for service "${svc.name}" failed (exit ${r.code}): ${JSON.stringify(cov.command)}${tail ? `\n${tail}` : ""}`,
|
|
3045
|
-
};
|
|
3046
|
-
}
|
|
3047
|
-
}
|
|
3048
|
-
const dir = await readCoverageDir(svc.name, coverageHostDir(WORKSPACE, svc.name));
|
|
3049
|
-
// A service that ALSO does browser coverage may legitimately have
|
|
3050
|
-
// an empty container dir for a test — don't fail if the browser
|
|
3051
|
-
// side has something. Otherwise the container contract stands.
|
|
3052
|
-
if (dir.error && !(coverageUsesBrowser(cov) && browser.has(svc.name))) {
|
|
3053
|
-
return dir;
|
|
3054
|
-
}
|
|
3055
|
-
reports.push(...dir.reports);
|
|
3056
|
-
}
|
|
3057
|
-
// Browser report for this service, when any script it served ran.
|
|
3058
|
-
const lcov = browser.get(svc.name);
|
|
3059
|
-
if (lcov)
|
|
3060
|
-
reports.push({ name: "browser.lcov", content: lcov });
|
|
3061
|
-
if (reports.length === 0 && coverageUsesBrowser(cov) && !coverageUsesContainer(cov)) {
|
|
3062
|
-
// Browser-only service, no browser activity yet this branch. Not an
|
|
3063
|
-
// error — a test needn't open a browser; the report appears once
|
|
3064
|
-
// one does (and forks to descendants from there).
|
|
3065
|
-
return { service: svc.name, reports: [] };
|
|
3066
|
-
}
|
|
3067
|
-
return { service: svc.name, reports };
|
|
3107
|
+
const error = await runCoverageAdapters(svc);
|
|
3108
|
+
if (error)
|
|
3109
|
+
return { service: svc.name, reports: [], error };
|
|
3110
|
+
return readCoverageDir(svc.name, coverageHostDir(WORKSPACE, svc.name));
|
|
3068
3111
|
}));
|
|
3069
3112
|
}
|
|
3070
3113
|
/**
|
|
@@ -5070,7 +5113,17 @@ export function harnessMethods(state) {
|
|
|
5070
5113
|
// the control plane logs it.
|
|
5071
5114
|
let coverage;
|
|
5072
5115
|
try {
|
|
5073
|
-
|
|
5116
|
+
const captures = await captureServiceCoverage();
|
|
5117
|
+
for (const c of captures) {
|
|
5118
|
+
// After bring-up the code that started the service has run, so a
|
|
5119
|
+
// report set that names no file at all is a misconfigured tool —
|
|
5120
|
+
// visible in the boot log, never a failure (there is no case).
|
|
5121
|
+
if (!c.error && c.reports.length > 0 && c.reports.every((r) => isEmptyReport(r.content))) {
|
|
5122
|
+
// eslint-disable-next-line no-console
|
|
5123
|
+
console.warn(`[coverage] service "${c.service}": every report is empty after bring-up — is coverage enabled at process start?`);
|
|
5124
|
+
}
|
|
5125
|
+
}
|
|
5126
|
+
coverage = parkCoverageBundle(SETUP_CASE_ID, captures);
|
|
5074
5127
|
}
|
|
5075
5128
|
catch (err) {
|
|
5076
5129
|
// eslint-disable-next-line no-console
|
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
/** Where the directory is mounted inside an opted-in container. */
|
|
2
2
|
export declare const COVERAGE_CONTAINER_DIR = "/spectest/coverage";
|
|
3
3
|
/** Raw bytes of reports one service may ship per capture. Reports are
|
|
4
|
-
*
|
|
4
|
+
* bounded by code size times chain depth, so anything past this is a
|
|
5
5
|
* runaway (a tool writing a new file per request), not coverage. */
|
|
6
6
|
export declare const COVERAGE_MAX_BYTES_PER_SERVICE: number;
|
|
7
7
|
/** Root of the per-service host directories, under the workspace so the
|
|
@@ -44,14 +44,33 @@ export interface CoverageBundle {
|
|
|
44
44
|
}
|
|
45
45
|
/** True when `text` holds at least one lcov `SF:` record. */
|
|
46
46
|
export declare function hasLcovSourceFile(text: string): boolean;
|
|
47
|
+
/** True when `text` is lcov-shaped: at least one line, every non-blank
|
|
48
|
+
* line an `XX:` record or `end_of_record`. An lcov with no `SF:` record
|
|
49
|
+
* at all (`TN:` alone) is well-formed — it is a report that says
|
|
50
|
+
* nothing ran. */
|
|
51
|
+
export declare function isLcovShaped(text: string): boolean;
|
|
52
|
+
/**
|
|
53
|
+
* True when `text` looks like a V8 coverage JSON document — what
|
|
54
|
+
* `NODE_V8_COVERAGE` / `v8.takeCoverage()` write: an object with a
|
|
55
|
+
* `result` array of scripts. A cheap shape check, not a parse (a report
|
|
56
|
+
* can be megabytes); the control plane parses for real.
|
|
57
|
+
*/
|
|
58
|
+
export declare function isV8CoverageJson(text: string): boolean;
|
|
59
|
+
/** True when `text` is a report spectest stores: lcov or V8 JSON. */
|
|
60
|
+
export declare function isWellFormedReport(text: string): boolean;
|
|
61
|
+
/** True when a well-formed report names no source file at all (an lcov
|
|
62
|
+
* with no `SF:`, a V8 document with an empty `result`). */
|
|
63
|
+
export declare function isEmptyReport(text: string): boolean;
|
|
47
64
|
/** Every regular file under `dir`, recursively, as paths relative to it.
|
|
48
65
|
* Hidden files are skipped (a tool's own bookkeeping — `.seq`, `.lock`,
|
|
49
66
|
* a tmp file mid-write — is not a report). */
|
|
50
67
|
export declare function listReportFiles(dir: string): Promise<string[]>;
|
|
51
68
|
/**
|
|
52
|
-
* Read every report under `dir`, verbatim. An empty directory, or
|
|
53
|
-
*
|
|
54
|
-
* contract is that an opted-in service always has a
|
|
69
|
+
* Read every report under `dir`, verbatim. An empty directory, or a file
|
|
70
|
+
* that is neither lcov nor V8 JSON, is an error rather than an empty
|
|
71
|
+
* capture: the contract is that an opted-in service always has a
|
|
72
|
+
* well-formed report to read. A report that names no file is fine — it
|
|
73
|
+
* says nothing ran.
|
|
55
74
|
*/
|
|
56
75
|
export declare function readCoverageDir(service: string, dir: string): Promise<ServiceCoverageCapture>;
|
|
57
76
|
/** Gzip a case's captures into the bundle `coverageChunk` serves. */
|
package/dist/harness/coverage.js
CHANGED
|
@@ -2,12 +2,15 @@
|
|
|
2
2
|
//
|
|
3
3
|
// A service that opts in (`coverage: true | { command }`) gets a writable
|
|
4
4
|
// directory bind-mounted at `/spectest/coverage/` in its container. The
|
|
5
|
-
// service's own tool
|
|
5
|
+
// service's own tool, or its coverage adapters, write reports there —
|
|
6
|
+
// lcov, or the V8 coverage JSON that `NODE_V8_COVERAGE` writes. The daemon reads the
|
|
6
7
|
// directory from the VM side (no `docker exec` to fetch) after bring-up
|
|
7
8
|
// and after each test, and ships the reports **verbatim** to the control
|
|
8
9
|
// plane as a gzipped bundle. Nothing is interpreted here beyond the one
|
|
9
|
-
// contract the harness must fail a case on: at least one report,
|
|
10
|
-
//
|
|
10
|
+
// contract the harness must fail a case on: at least one report, every
|
|
11
|
+
// one well-formed. Counters may reset between dumps and a dump may be a
|
|
12
|
+
// new file: the directory forks with the environment, so it accumulates
|
|
13
|
+
// along the ancestor chain by itself. The server derives whatever it
|
|
11
14
|
// needs (today the set of source files; later, finer things) from the
|
|
12
15
|
// stored bytes, so a change in that derivation is a re-read of stored
|
|
13
16
|
// runs, never a hole in the history.
|
|
@@ -19,7 +22,7 @@ import { gzipSync } from "node:zlib";
|
|
|
19
22
|
/** Where the directory is mounted inside an opted-in container. */
|
|
20
23
|
export const COVERAGE_CONTAINER_DIR = "/spectest/coverage";
|
|
21
24
|
/** Raw bytes of reports one service may ship per capture. Reports are
|
|
22
|
-
*
|
|
25
|
+
* bounded by code size times chain depth, so anything past this is a
|
|
23
26
|
* runaway (a tool writing a new file per request), not coverage. */
|
|
24
27
|
export const COVERAGE_MAX_BYTES_PER_SERVICE = 64 * 1024 * 1024;
|
|
25
28
|
/** Root of the per-service host directories, under the workspace so the
|
|
@@ -31,6 +34,34 @@ export function coverageHostDir(workspace, service) {
|
|
|
31
34
|
export function hasLcovSourceFile(text) {
|
|
32
35
|
return /^SF:\S/m.test(text);
|
|
33
36
|
}
|
|
37
|
+
/** True when `text` is lcov-shaped: at least one line, every non-blank
|
|
38
|
+
* line an `XX:` record or `end_of_record`. An lcov with no `SF:` record
|
|
39
|
+
* at all (`TN:` alone) is well-formed — it is a report that says
|
|
40
|
+
* nothing ran. */
|
|
41
|
+
export function isLcovShaped(text) {
|
|
42
|
+
const lines = text.split(/\r?\n/).filter((l) => l.trim().length > 0);
|
|
43
|
+
return lines.length > 0 && lines.every((l) => /^[A-Z]{2,5}:/.test(l) || l.trim() === "end_of_record");
|
|
44
|
+
}
|
|
45
|
+
/**
|
|
46
|
+
* True when `text` looks like a V8 coverage JSON document — what
|
|
47
|
+
* `NODE_V8_COVERAGE` / `v8.takeCoverage()` write: an object with a
|
|
48
|
+
* `result` array of scripts. A cheap shape check, not a parse (a report
|
|
49
|
+
* can be megabytes); the control plane parses for real.
|
|
50
|
+
*/
|
|
51
|
+
export function isV8CoverageJson(text) {
|
|
52
|
+
return /^\s*\{/.test(text) && /"result"\s*:\s*\[/.test(text);
|
|
53
|
+
}
|
|
54
|
+
/** True when `text` is a report spectest stores: lcov or V8 JSON. */
|
|
55
|
+
export function isWellFormedReport(text) {
|
|
56
|
+
return isLcovShaped(text) || isV8CoverageJson(text);
|
|
57
|
+
}
|
|
58
|
+
/** True when a well-formed report names no source file at all (an lcov
|
|
59
|
+
* with no `SF:`, a V8 document with an empty `result`). */
|
|
60
|
+
export function isEmptyReport(text) {
|
|
61
|
+
if (isV8CoverageJson(text))
|
|
62
|
+
return /"result"\s*:\s*\[\s*\]/.test(text);
|
|
63
|
+
return !hasLcovSourceFile(text);
|
|
64
|
+
}
|
|
34
65
|
/** Every regular file under `dir`, recursively, as paths relative to it.
|
|
35
66
|
* Hidden files are skipped (a tool's own bookkeeping — `.seq`, `.lock`,
|
|
36
67
|
* a tmp file mid-write — is not a report). */
|
|
@@ -58,9 +89,11 @@ export async function listReportFiles(dir) {
|
|
|
58
89
|
return out.sort();
|
|
59
90
|
}
|
|
60
91
|
/**
|
|
61
|
-
* Read every report under `dir`, verbatim. An empty directory, or
|
|
62
|
-
*
|
|
63
|
-
* contract is that an opted-in service always has a
|
|
92
|
+
* Read every report under `dir`, verbatim. An empty directory, or a file
|
|
93
|
+
* that is neither lcov nor V8 JSON, is an error rather than an empty
|
|
94
|
+
* capture: the contract is that an opted-in service always has a
|
|
95
|
+
* well-formed report to read. A report that names no file is fine — it
|
|
96
|
+
* says nothing ran.
|
|
64
97
|
*/
|
|
65
98
|
export async function readCoverageDir(service, dir) {
|
|
66
99
|
const names = await listReportFiles(dir);
|
|
@@ -68,12 +101,13 @@ export async function readCoverageDir(service, dir) {
|
|
|
68
101
|
return {
|
|
69
102
|
service,
|
|
70
103
|
reports: [],
|
|
71
|
-
error: `service "${service}" opted in to coverage but ${COVERAGE_CONTAINER_DIR}/ holds no report — the service (or its coverage command) must write an lcov file there`,
|
|
104
|
+
error: `service "${service}" opted in to coverage but ${COVERAGE_CONTAINER_DIR}/ holds no report — the service (or its coverage command) must write an lcov or V8 coverage JSON file there`,
|
|
72
105
|
};
|
|
73
106
|
}
|
|
74
107
|
const reports = [];
|
|
75
108
|
let total = 0;
|
|
76
|
-
let
|
|
109
|
+
let anyReport = false;
|
|
110
|
+
const malformed = [];
|
|
77
111
|
for (const name of names) {
|
|
78
112
|
let content;
|
|
79
113
|
try {
|
|
@@ -87,18 +121,20 @@ export async function readCoverageDir(service, dir) {
|
|
|
87
121
|
return {
|
|
88
122
|
service,
|
|
89
123
|
reports: [],
|
|
90
|
-
error: `service "${service}" has more than ${COVERAGE_MAX_BYTES_PER_SERVICE >> 20} MiB of reports in ${COVERAGE_CONTAINER_DIR}/ —
|
|
124
|
+
error: `service "${service}" has more than ${COVERAGE_MAX_BYTES_PER_SERVICE >> 20} MiB of reports in ${COVERAGE_CONTAINER_DIR}/ — that is a runaway, not coverage`,
|
|
91
125
|
};
|
|
92
126
|
}
|
|
93
|
-
if (
|
|
94
|
-
|
|
127
|
+
if (isWellFormedReport(content))
|
|
128
|
+
anyReport = true;
|
|
129
|
+
else
|
|
130
|
+
malformed.push(name);
|
|
95
131
|
reports.push({ name, content });
|
|
96
132
|
}
|
|
97
|
-
if (!
|
|
133
|
+
if (!anyReport || malformed.length > 0) {
|
|
98
134
|
return {
|
|
99
135
|
service,
|
|
100
136
|
reports,
|
|
101
|
-
error: `service "${service}" wrote ${reports.length} file(s) to ${COVERAGE_CONTAINER_DIR}/ but none
|
|
137
|
+
error: `service "${service}" wrote ${reports.length} file(s) to ${COVERAGE_CONTAINER_DIR}/ but ${anyReport ? `${malformed.slice(0, 5).join(", ")} ${malformed.length === 1 ? "is" : "are"} neither` : "none is"} an lcov report or a V8 coverage JSON document — only those two formats are read`,
|
|
102
138
|
};
|
|
103
139
|
}
|
|
104
140
|
return { service, reports };
|
package/dist/index.d.ts
CHANGED
|
@@ -20,6 +20,8 @@ export type { Mobile, MobileApp } from "./mobile.js";
|
|
|
20
20
|
import type { Mobile, MobileApp } from "./mobile.js";
|
|
21
21
|
export type { Terminal, TerminalOpts, TerminalResult } from "./terminal.js";
|
|
22
22
|
import type { Terminal, TerminalOpts, TerminalResult } from "./terminal.js";
|
|
23
|
+
export { type CoverageAdapter, type CoverageCaptureContext, type CoverageConfigureInfo, type CoverageLoadContext, type ServiceCoverage, } from "./coverage.js";
|
|
24
|
+
import { type ServiceCoverage } from "./coverage.js";
|
|
23
25
|
export { certificate, dnsName, proxy, provides, lowerIngress, isWildcard, SELF_SERVICE_TOKEN, } from "./ingress.js";
|
|
24
26
|
export type { CertificateDecl, DnsDecl, ProxyDecl, IngressDecl, DnsTarget, LoweredIngress, } from "./ingress.js";
|
|
25
27
|
import type { DnsTarget } from "./ingress.js";
|
|
@@ -119,20 +121,24 @@ export interface ServiceConfig {
|
|
|
119
121
|
* Opt this service in to **code coverage collection** (experimental).
|
|
120
122
|
*
|
|
121
123
|
* The service gets a writable directory bind-mounted at
|
|
122
|
-
* `/spectest/coverage/`.
|
|
123
|
-
*
|
|
124
|
-
*
|
|
125
|
-
* the container, then reads every file in the directory and records the
|
|
126
|
-
* set of source files that ran. Only the `SF:` records are read.
|
|
124
|
+
* `/spectest/coverage/`. After bring-up and after every test, spectest
|
|
125
|
+
* reads every file in it — **lcov** or **V8 coverage JSON**, verbatim —
|
|
126
|
+
* and stores them with the test; the control plane derives what ran.
|
|
127
127
|
*
|
|
128
|
-
* - `true`: the
|
|
129
|
-
*
|
|
130
|
-
* - `{
|
|
131
|
-
*
|
|
128
|
+
* - `true`: the program writes its own reports there (an exit-time dump
|
|
129
|
+
* for a short-lived process, a dump on a timer, its own hook).
|
|
130
|
+
* - `{ adapters: [...] }`: spectest gets the reports out. From
|
|
131
|
+
* `@specific.dev/spectest/coverage`: `node()` (V8 JSON from a node
|
|
132
|
+
* service, through a hook spectest mounts), `browser()` (the frontend
|
|
133
|
+
* this service serves, collected from the guest browser),
|
|
134
|
+
* `command(cmd)` (run a command in the container first), or your own
|
|
135
|
+
* via `defineAdapter`.
|
|
132
136
|
*
|
|
133
|
-
* The directory must hold at least one
|
|
134
|
-
* an empty directory or a
|
|
135
|
-
* error, so a hole in the coverage data is never silent.
|
|
137
|
+
* The directory must hold at least one well-formed report each time it
|
|
138
|
+
* is read; an empty directory or a failing adapter fails the test with a
|
|
139
|
+
* clear error, so a hole in the coverage data is never silent. Reports
|
|
140
|
+
* accumulate along the test's ancestor chain by themselves (the
|
|
141
|
+
* directory forks with the environment). See
|
|
136
142
|
* `spectest docs /services/coverage`.
|
|
137
143
|
*/
|
|
138
144
|
coverage?: ServiceCoverage;
|
|
@@ -663,29 +669,6 @@ export type ServiceImage = {
|
|
|
663
669
|
/** Extra glob patterns to exclude from the build context. */
|
|
664
670
|
exclude?: readonly string[];
|
|
665
671
|
};
|
|
666
|
-
/**
|
|
667
|
-
* A service's opt-in to coverage collection — see `ServiceConfig.coverage`.
|
|
668
|
-
* `true` when the service writes its own reports; `{ command }` when
|
|
669
|
-
* spectest must run a command in the container to produce one.
|
|
670
|
-
*/
|
|
671
|
-
export type ServiceCoverage = true | {
|
|
672
|
-
/** Run this (via `sh -c`) in the container before the coverage
|
|
673
|
-
* directory is read, so the service writes a fresh lcov report. */
|
|
674
|
-
command?: string;
|
|
675
|
-
/**
|
|
676
|
-
* Collect **frontend** coverage for the app this service serves.
|
|
677
|
-
* The code runs in the guest browser — spectest's own process — so
|
|
678
|
-
* the user cannot write a report for it: spectest turns on V8
|
|
679
|
-
* coverage for every script `ctx.browser()`/`ctx.mobile()` loads
|
|
680
|
-
* from this service's origin, maps it back to original sources
|
|
681
|
-
* through the served source maps, and adds one lcov report to this
|
|
682
|
-
* service's capture. Scripts a source map can't resolve are
|
|
683
|
-
* attributed to their served path (which the repo mapping then
|
|
684
|
-
* treats as unknown — the safe answer). Needs the app to serve
|
|
685
|
-
* source maps to the test environment.
|
|
686
|
-
*/
|
|
687
|
-
browser?: boolean;
|
|
688
|
-
};
|
|
689
672
|
export interface VolumeMount {
|
|
690
673
|
/**
|
|
691
674
|
* Named shared volume. Two services mounting the same `name` share one
|
|
@@ -898,17 +881,6 @@ export interface FakeContext {
|
|
|
898
881
|
* for a TLS-provisioning provider hands back to the app under test. */
|
|
899
882
|
certificate(hostnames: readonly string[]): Promise<CertificateMaterial>;
|
|
900
883
|
}
|
|
901
|
-
/**
|
|
902
|
-
* Check one service's `coverage` field. Exported so the daemon applies the
|
|
903
|
-
* same rule to a runtime `startService` spec.
|
|
904
|
-
*/
|
|
905
|
-
export declare function validateCoverage(service: string, cov: unknown): void;
|
|
906
|
-
/** Whether a `coverage` value collects reports from the container's own
|
|
907
|
-
* filesystem (`/spectest/coverage/`). True for `true`, `{ command }`, and
|
|
908
|
-
* any object that isn't browser-only. */
|
|
909
|
-
export declare function coverageUsesContainer(cov: ServiceCoverage | undefined): boolean;
|
|
910
|
-
/** Whether a `coverage` value collects frontend (browser) coverage. */
|
|
911
|
-
export declare function coverageUsesBrowser(cov: ServiceCoverage | undefined): boolean;
|
|
912
884
|
/**
|
|
913
885
|
* A single test step. Created via `test(...)` or `createTest(services)`.
|
|
914
886
|
* Tests are referenced (not named by string) when one test depends on
|
package/dist/index.js
CHANGED
|
@@ -40,6 +40,7 @@ export { McpHttpError, McpRpcError, McpAuthDeniedError, } from "./mcp.js";
|
|
|
40
40
|
import { isLocator, getLocatorProbe, isBrowserSession, getBrowserProbe, DEFAULT_ACTION_TIMEOUT_MS, } from "./locator.js";
|
|
41
41
|
import { formatWaited, locatorFailureMessage } from "./locator-errors.js";
|
|
42
42
|
import { describeUrlPattern, matchesUrl } from "./url-match.js";
|
|
43
|
+
import { applyCoverageAdapters, validateCoverage } from "./coverage.js";
|
|
43
44
|
// Low-level ingress primitives + the framework lowering that the friendly
|
|
44
45
|
// `tls` / `hostnames` fields and `defineFake(...)` are built on. See
|
|
45
46
|
// `ingress.ts`.
|
|
@@ -215,37 +216,6 @@ function expandServiceGroups(input) {
|
|
|
215
216
|
}
|
|
216
217
|
return out;
|
|
217
218
|
}
|
|
218
|
-
/**
|
|
219
|
-
* Check one service's `coverage` field. Exported so the daemon applies the
|
|
220
|
-
* same rule to a runtime `startService` spec.
|
|
221
|
-
*/
|
|
222
|
-
export function validateCoverage(service, cov) {
|
|
223
|
-
if (cov === undefined || cov === true)
|
|
224
|
-
return;
|
|
225
|
-
if (typeof cov === "object" && cov !== null) {
|
|
226
|
-
const o = cov;
|
|
227
|
-
const cmdOk = o.command === undefined ||
|
|
228
|
-
(typeof o.command === "string" && o.command.trim().length > 0);
|
|
229
|
-
const browserOk = o.browser === undefined || typeof o.browser === "boolean";
|
|
230
|
-
if (cmdOk && browserOk && (o.command !== undefined || o.browser === true))
|
|
231
|
-
return;
|
|
232
|
-
}
|
|
233
|
-
throw new Error(`service "${service}" has an invalid \`coverage\` value — use \`true\`, \`{ command: "<shell command>" }\`, or \`{ browser: true }\``);
|
|
234
|
-
}
|
|
235
|
-
/** Whether a `coverage` value collects reports from the container's own
|
|
236
|
-
* filesystem (`/spectest/coverage/`). True for `true`, `{ command }`, and
|
|
237
|
-
* any object that isn't browser-only. */
|
|
238
|
-
export function coverageUsesContainer(cov) {
|
|
239
|
-
if (cov === undefined)
|
|
240
|
-
return false;
|
|
241
|
-
if (cov === true)
|
|
242
|
-
return true;
|
|
243
|
-
return cov.command !== undefined || cov.browser !== true;
|
|
244
|
-
}
|
|
245
|
-
/** Whether a `coverage` value collects frontend (browser) coverage. */
|
|
246
|
-
export function coverageUsesBrowser(cov) {
|
|
247
|
-
return typeof cov === "object" && cov.browser === true;
|
|
248
|
-
}
|
|
249
219
|
function validateEnvironmentConfig(config) {
|
|
250
220
|
const entries = Object.entries(config.services);
|
|
251
221
|
const serviceNames = new Set(entries.map(([n]) => n));
|
|
@@ -353,9 +323,13 @@ export function defineEnvironment(input) {
|
|
|
353
323
|
// { name, services, timeoutSecs } — with plain, fully-expanded services —
|
|
354
324
|
// is stored as `config` and shipped to the control plane.
|
|
355
325
|
const { fakes, services: rawServices, ...rest } = input;
|
|
326
|
+
const expanded = expandServiceGroups(rawServices);
|
|
327
|
+
for (const [key, svc] of Object.entries(expanded)) {
|
|
328
|
+
expanded[key] = applyCoverageAdapters(key, svc);
|
|
329
|
+
}
|
|
356
330
|
const config = {
|
|
357
331
|
...rest,
|
|
358
|
-
services:
|
|
332
|
+
services: expanded,
|
|
359
333
|
};
|
|
360
334
|
validateEnvironmentConfig(config);
|
|
361
335
|
if (fakes)
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@specific.dev/spectest",
|
|
3
|
-
"version": "0.
|
|
3
|
+
"version": "0.59.1",
|
|
4
4
|
"description": "Spectest SDK for defining test environments in TypeScript.",
|
|
5
5
|
"type": "module",
|
|
6
6
|
"main": "./dist/index.js",
|
|
@@ -16,6 +16,11 @@
|
|
|
16
16
|
"bun": "./src/components/index.ts",
|
|
17
17
|
"default": "./dist/components/index.js"
|
|
18
18
|
},
|
|
19
|
+
"./coverage": {
|
|
20
|
+
"types": "./dist/coverage.d.ts",
|
|
21
|
+
"bun": "./src/coverage.ts",
|
|
22
|
+
"default": "./dist/coverage.js"
|
|
23
|
+
},
|
|
19
24
|
"./browser": {
|
|
20
25
|
"types": "./dist/browser.d.ts",
|
|
21
26
|
"bun": "./src/browser.ts",
|