explainer-video 0.0.0-stage → 0.4.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.claude-plugin/plugin.json +12 -0
- package/LICENSE +21 -0
- package/README.md +152 -2
- package/cordis.patch.yml +7 -0
- package/lib/index.js +35 -0
- package/package.json +56 -4
- package/plugin.json +13 -0
- package/skills/coursework-author/.synced-from +5 -0
- package/skills/coursework-author/SKILL.md +209 -0
- package/skills/craft-codeflow-flows/SKILL.md +116 -0
- package/skills/craft-codeflow-mine/SKILL.md +207 -0
- package/skills/craft-codeflow-plan/SKILL.md +65 -0
- package/skills/craft-codeflow-scene/SKILL.md +168 -0
- package/skills/craft-video-align/SKILL.md +109 -0
- package/skills/craft-video-assemble/SKILL.md +69 -0
- package/skills/craft-video-audio/SKILL.md +59 -0
- package/skills/craft-video-enrich/SKILL.md +46 -0
- package/skills/craft-video-plan/SKILL.md +57 -0
- package/skills/craft-video-publish/SKILL.md +79 -0
- package/skills/craft-video-render/SKILL.md +68 -0
- package/skills/craft-video-scene/SKILL.md +112 -0
- package/skills/craft-video-script/SKILL.md +80 -0
- package/skills/explainer-video-from-codebase/FLOWS.md +577 -0
- package/skills/explainer-video-from-codebase/SKILL.md +659 -0
- package/skills/explainer-video-from-codebase/tools/apply-flows.mjs +531 -0
- package/skills/explainer-video-from-codebase/tools/approve-flows.mjs +56 -0
- package/skills/explainer-video-from-codebase/tools/codemap-md.mjs +116 -0
- package/skills/explainer-video-from-codebase/tools/config/dotenv.mjs +13 -0
- package/skills/explainer-video-from-codebase/tools/config/json.mjs +14 -0
- package/skills/explainer-video-from-codebase/tools/config/redact.mjs +16 -0
- package/skills/explainer-video-from-codebase/tools/config/toml-lite.mjs +13 -0
- package/skills/explainer-video-from-codebase/tools/config/yaml-lite.mjs +18 -0
- package/skills/explainer-video-from-codebase/tools/excerpt-check.mjs +186 -0
- package/skills/explainer-video-from-codebase/tools/excerpt.mjs +116 -0
- package/skills/explainer-video-from-codebase/tools/flow-diagram.mjs +168 -0
- package/skills/explainer-video-from-codebase/tools/flow-plan.mjs +129 -0
- package/skills/explainer-video-from-codebase/tools/flows-md.mjs +87 -0
- package/skills/explainer-video-from-codebase/tools/lang/c.mjs +684 -0
- package/skills/explainer-video-from-codebase/tools/lang/go.mjs +673 -0
- package/skills/explainer-video-from-codebase/tools/lang/python-config.mjs +143 -0
- package/skills/explainer-video-from-codebase/tools/lang/python-deps.mjs +62 -0
- package/skills/explainer-video-from-codebase/tools/lang/python-stdlib.mjs +24 -0
- package/skills/explainer-video-from-codebase/tools/lang/python.mjs +609 -0
- package/skills/explainer-video-from-codebase/tools/lang/typescript.mjs +963 -0
- package/skills/explainer-video-from-codebase/tools/lib/decompose.mjs +102 -0
- package/skills/explainer-video-from-codebase/tools/lib/groups.mjs +77 -0
- package/skills/explainer-video-from-codebase/tools/lib/observe-coverage.mjs +68 -0
- package/skills/explainer-video-from-codebase/tools/lib/observe-v8.mjs +232 -0
- package/skills/explainer-video-from-codebase/tools/lib/observe.mjs +162 -0
- package/skills/explainer-video-from-codebase/tools/lib/redact.mjs +28 -0
- package/skills/explainer-video-from-codebase/tools/lib/sandbox.mjs +97 -0
- package/skills/explainer-video-from-codebase/tools/lib/test-links.mjs +28 -0
- package/skills/explainer-video-from-codebase/tools/lib/toml.mjs +129 -0
- package/skills/explainer-video-from-codebase/tools/lib/trace-c.mjs +95 -0
- package/skills/explainer-video-from-codebase/tools/lib/trace-go.mjs +101 -0
- package/skills/explainer-video-from-codebase/tools/lib/trace-js.mjs +136 -0
- package/skills/explainer-video-from-codebase/tools/lib/tree-sitter.mjs +19 -0
- package/skills/explainer-video-from-codebase/tools/lib/walk.mjs +138 -0
- package/skills/explainer-video-from-codebase/tools/mine.mjs +409 -0
- package/skills/explainer-video-from-codebase/tools/propose.mjs +127 -0
- package/skills/explainer-video-from-codebase/tools/rank.mjs +119 -0
- package/skills/explainer-video-from-codebase/tools/ranking-md.mjs +76 -0
- package/skills/explainer-video-from-codebase/tools/rules/c-handlers.mjs +53 -0
- package/skills/explainer-video-from-codebase/tools/rules/celery-taskiq.mjs +48 -0
- package/skills/explainer-video-from-codebase/tools/rules/click-typer.mjs +6 -0
- package/skills/explainer-video-from-codebase/tools/rules/express.mjs +72 -0
- package/skills/explainer-video-from-codebase/tools/rules/fastapi.mjs +99 -0
- package/skills/explainer-video-from-codebase/tools/rules/go-http.mjs +173 -0
- package/skills/explainer-video-from-codebase/tools/rules/go-test.mjs +108 -0
- package/skills/explainer-video-from-codebase/tools/rules/grpc.mjs +57 -0
- package/skills/explainer-video-from-codebase/tools/rules/gtest.mjs +97 -0
- package/skills/explainer-video-from-codebase/tools/rules/haystack.mjs +55 -0
- package/skills/explainer-video-from-codebase/tools/rules/jest-vitest.mjs +100 -0
- package/skills/explainer-video-from-codebase/tools/rules/js-http-client.mjs +111 -0
- package/skills/explainer-video-from-codebase/tools/rules/nextjs.mjs +53 -0
- package/skills/explainer-video-from-codebase/tools/rules/process-spawn.mjs +49 -0
- package/skills/explainer-video-from-codebase/tools/rules/py-http-client.mjs +26 -0
- package/skills/explainer-video-from-codebase/tools/rules/pytest.mjs +107 -0
- package/skills/explainer-video-from-codebase/tools/rules/react-router.mjs +17 -0
- package/skills/explainer-video-from-codebase/tools/rules/registry.mjs +31 -0
- package/skills/explainer-video-from-codebase/tools/trace/c/fmtrace-sh +14 -0
- package/skills/explainer-video-from-codebase/tools/trace/js/fmtrace-net.cjs +33 -0
- package/skills/explainer-video-from-codebase/tools/trace/js/fmtrace-srcmap.cjs +41 -0
- package/skills/explainer-video-from-codebase/tools/trace/python/fmtrace_plugin.py +87 -0
- package/skills/explainer-video-from-codebase/tools/trace/python/sitecustomize.py +330 -0
- package/skills/explainer-video-from-codebase/tools/trace/sandbox/linux-bwrap.sh +73 -0
- package/skills/explainer-video-from-codebase/tools/trace/sandbox/macos-nonet.sb +41 -0
- package/skills/explainer-video-from-codebase/tools/trace.mjs +577 -0
- package/skills/explainer-video-from-codebase/vendor/tree-sitter/LICENSES/tree-sitter-c.txt +21 -0
- package/skills/explainer-video-from-codebase/vendor/tree-sitter/LICENSES/tree-sitter-cpp.txt +21 -0
- package/skills/explainer-video-from-codebase/vendor/tree-sitter/LICENSES/tree-sitter-go.txt +21 -0
- package/skills/explainer-video-from-codebase/vendor/tree-sitter/LICENSES/tree-sitter-javascript.txt +21 -0
- package/skills/explainer-video-from-codebase/vendor/tree-sitter/LICENSES/tree-sitter-python.txt +21 -0
- package/skills/explainer-video-from-codebase/vendor/tree-sitter/LICENSES/tree-sitter-typescript.txt +21 -0
- package/skills/explainer-video-from-codebase/vendor/tree-sitter/LICENSES/web-tree-sitter.txt +21 -0
- package/skills/explainer-video-from-codebase/vendor/tree-sitter/MANIFEST.json +172 -0
- package/skills/explainer-video-from-codebase/vendor/tree-sitter/tree-sitter-c.wasm +0 -0
- package/skills/explainer-video-from-codebase/vendor/tree-sitter/tree-sitter-cpp.wasm +0 -0
- package/skills/explainer-video-from-codebase/vendor/tree-sitter/tree-sitter-go.wasm +0 -0
- package/skills/explainer-video-from-codebase/vendor/tree-sitter/tree-sitter-javascript.wasm +0 -0
- package/skills/explainer-video-from-codebase/vendor/tree-sitter/tree-sitter-python.wasm +0 -0
- package/skills/explainer-video-from-codebase/vendor/tree-sitter/tree-sitter-tsx.wasm +0 -0
- package/skills/explainer-video-from-codebase/vendor/tree-sitter/tree-sitter-typescript.wasm +0 -0
- package/skills/explainer-video-from-codebase/vendor/tree-sitter/web-tree-sitter.js +4081 -0
- package/skills/explainer-video-from-codebase/vendor/tree-sitter/web-tree-sitter.wasm +0 -0
- package/skills/explainer-video-from-coursework/ARCHITECTURE.md +59 -0
- package/skills/explainer-video-from-coursework/PIPELINE.md +343 -0
- package/skills/explainer-video-from-coursework/ROADMAP.md +725 -0
- package/skills/explainer-video-from-coursework/SERIES.md +232 -0
- package/skills/explainer-video-from-coursework/SETUP.md +105 -0
- package/skills/explainer-video-from-coursework/SKILL.md +858 -0
- package/skills/explainer-video-from-coursework/scenes/_pt-chart.js +12 -0
- package/skills/explainer-video-from-coursework/scenes/_pt-equation.js +10 -0
- package/skills/explainer-video-from-coursework/scenes/_pt-hubspoke.js +7 -0
- package/skills/explainer-video-from-coursework/scenes/_pt-loopflow.js +7 -0
- package/skills/explainer-video-from-coursework/scenes/_pt-sequence.js +12 -0
- package/skills/explainer-video-from-coursework/scenes/_pt-timeline.js +11 -0
- package/skills/explainer-video-from-coursework/src/scene-code.html +127 -0
- package/skills/explainer-video-from-coursework/src/scene-code.js +379 -0
- package/skills/explainer-video-from-coursework/src/scene-core.js +1250 -0
- package/skills/explainer-video-from-coursework/src/scene-diagram.html +50 -0
- package/skills/explainer-video-from-coursework/src/scene-svg.html +50 -0
- package/skills/explainer-video-from-coursework/src/timeline.js +297 -0
- package/skills/explainer-video-from-coursework/src/timing-chart.js +160 -0
- package/skills/explainer-video-from-coursework/tools/align.mjs +419 -0
- package/skills/explainer-video-from-coursework/tools/apply-enrichment.mjs +206 -0
- package/skills/explainer-video-from-coursework/tools/assemble.mjs +223 -0
- package/skills/explainer-video-from-coursework/tools/build-audio.mjs +234 -0
- package/skills/explainer-video-from-coursework/tools/chromium.mjs +73 -0
- package/skills/explainer-video-from-coursework/tools/debug-page.mjs +75 -0
- package/skills/explainer-video-from-coursework/tools/env.mjs +30 -0
- package/skills/explainer-video-from-coursework/tools/frame-dedup.mjs +131 -0
- package/skills/explainer-video-from-coursework/tools/key-source.mjs +151 -0
- package/skills/explainer-video-from-coursework/tools/model.mjs +21 -0
- package/skills/explainer-video-from-coursework/tools/naming.mjs +38 -0
- package/skills/explainer-video-from-coursework/tools/onset-audit.mjs +51 -0
- package/skills/explainer-video-from-coursework/tools/pacing-check.mjs +350 -0
- package/skills/explainer-video-from-coursework/tools/planner.mjs +616 -0
- package/skills/explainer-video-from-coursework/tools/publish.mjs +230 -0
- package/skills/explainer-video-from-coursework/tools/render.mjs +205 -0
- package/skills/explainer-video-from-coursework/tools/resolve-beats.mjs +239 -0
- package/skills/explainer-video-from-coursework/tools/script-check.mjs +174 -0
- package/skills/explainer-video-from-coursework/tools/stamp.mjs +71 -0
- package/skills/explainer-video-from-coursework/tools/timing-report.mjs +299 -0
- package/skills/explainer-video-from-coursework/tools/transcribe-gemini.mjs +96 -0
- package/skills/explainer-video-from-coursework/tools/tts-gemini.mjs +394 -0
- package/skills/explainer-video-from-coursework/tools/verify-audio.mjs +88 -0
- package/skills/explainer-video-from-coursework/tools/workdir.mjs +124 -0
- package/skills/explainer-video-from-coursework/vendor/plotly-basic.min.js +8 -0
- package/skills/explainer-video-from-coursework/vendor/plotly-basic.min.js.LICENSE.txt +16 -0
|
@@ -0,0 +1,102 @@
|
|
|
1
|
+
/* Validated flows → videos (design §4.5): the cost model and the splitting rules, shared by flow-plan.mjs and by
|
|
2
|
+
apply-flows' estimate in the approval table. WPM and EXPANSION are measured on coursework (imported from its
|
|
3
|
+
model.mjs). FRAME, HOP_BASE, WPL, BRANCH and TEST are calibrated on one narrated pilot code video (CALIBRATION:
|
|
4
|
+
8 hops with 96 narrated lines spoken in 609 words, 2 taught branches in 135, 3 cited tests in 134, the frame in
|
|
5
|
+
154; 1032 words in all, 381.6 s of speech). BOUNDARY and OVERVIEW are still estimates: the pilot crossed no
|
|
6
|
+
boundary and made no overview.
|
|
7
|
+
|
|
8
|
+
words(video) = FRAME + Σ_hops (HOP_BASE + WPL × narrated(h)) + Σ_crossings BOUNDARY + Σ_taught branches BRANCH
|
|
9
|
+
+ min(MAX_TESTS, cited tests) × TEST
|
|
10
|
+
narrated(h) = h.lines ? count(h.lines) : min(LINE_CAP, ceil(NARRATE × span lines))
|
|
11
|
+
minutes = words / WPM × EXPANSION
|
|
12
|
+
|
|
13
|
+
Splitting: a hop with `break` starts a new video; hops pack greedily under the cap; on overflow the video splits at
|
|
14
|
+
its latest crossing, else at the overflowing hop (a hop is never split); flows and cores are never glued together;
|
|
15
|
+
a piece under the minimum merges with a neighbour of the same item when the two fit. One overview video opens
|
|
16
|
+
the series. */
|
|
17
|
+
import { WPM, EXPANSION } from '../../../explainer-video-from-coursework/tools/model.mjs';
|
|
18
|
+
|
|
19
|
+
// what the calibration was measured on, in counts (one pilot video, 18 segments, 0 pause prompts)
|
|
20
|
+
export const CALIBRATION = { videos: 1, segments: 18, hops: 8, narratedLines: 96, hopWords: 609, branches: 2, branchWords: 135,
|
|
21
|
+
citedTests: 3, testWords: 134, frameWords: 154, crossings: 0, pausePrompts: 0, words: 1032, speechSeconds: 381.6 };
|
|
22
|
+
|
|
23
|
+
export const MODEL = { wpm: WPM, expansion: EXPANSION, frame: 150, hopBase: 5, wpl: 6, lineCap: 22, narrate: 0.5, boundary: 60, branch: 70,
|
|
24
|
+
prompt: 12, test: 45, maxTests: 3, overview: 1000, calibrated: true, uncalibrated: ['boundary', 'overview'], basis: CALIBRATION };
|
|
25
|
+
|
|
26
|
+
export const narratedLines = h => h.lines ? h.lines.reduce((a, [x, y]) => a + y - x + 1, 0) : Math.min(MODEL.lineCap, Math.ceil(MODEL.narrate * (h.end - h.start + 1)));
|
|
27
|
+
const hopWords = h => MODEL.hopBase + MODEL.wpl * narratedLines(h);
|
|
28
|
+
const seconds = words => Math.round((words / MODEL.wpm) * 60 * MODEL.expansion);
|
|
29
|
+
export const section = h => `${h.file} :: ${h.name} @L${h.start}`;
|
|
30
|
+
|
|
31
|
+
// the words of one video holding `hops` of an item whose taught branches sit on the hops listed in `teach`
|
|
32
|
+
function cost(hops, teach) {
|
|
33
|
+
const at = new Set(hops.map(h => h.index));
|
|
34
|
+
const branches = teach.filter(t => at.has(t.hop)).length, cited = hops.reduce((a, h) => a + h.pinnedBy.length, 0);
|
|
35
|
+
const crossings = hops.filter(h => h.crossing).length;
|
|
36
|
+
const words = MODEL.frame + hops.reduce((a, h) => a + hopWords(h), 0) + crossings * MODEL.boundary + branches * MODEL.branch + Math.min(MODEL.maxTests, cited) * MODEL.test;
|
|
37
|
+
return { words, seconds: seconds(words), crossings, branches, cited: Math.min(MODEL.maxTests, cited) };
|
|
38
|
+
}
|
|
39
|
+
|
|
40
|
+
// one item's hops → pieces under the cap
|
|
41
|
+
function split(item, capSec, minSec) {
|
|
42
|
+
const groups = [];
|
|
43
|
+
for (const h of item.hops) { if (!groups.length || (h.break && h.index !== item.hops[0].index)) groups.push([]); groups.at(-1).push(h); }
|
|
44
|
+
const pieces = [];
|
|
45
|
+
groups.forEach((g, gi) => {
|
|
46
|
+
let cur = [];
|
|
47
|
+
for (const h of g) {
|
|
48
|
+
if (cur.length && cost([...cur, h], item.teach).seconds > capSec) {
|
|
49
|
+
const k = cur.findLastIndex((x, i) => i > 0 && x.crossing);
|
|
50
|
+
if (k > 0) { pieces.push({ hops: cur.slice(0, k), group: gi }); cur = cur.slice(k); }
|
|
51
|
+
if (cur.length && cost([...cur, h], item.teach).seconds > capSec) { pieces.push({ hops: cur, group: gi }); cur = []; }
|
|
52
|
+
}
|
|
53
|
+
cur.push(h);
|
|
54
|
+
}
|
|
55
|
+
pieces.push({ hops: cur, group: gi });
|
|
56
|
+
});
|
|
57
|
+
for (let i = 0; i < pieces.length; i++) {
|
|
58
|
+
if (pieces.length < 2 || cost(pieces[i].hops, item.teach).seconds >= minSec) continue;
|
|
59
|
+
const fits = j => pieces[j] && pieces[j].group === pieces[i].group && cost([...pieces[Math.min(i, j)].hops, ...pieces[Math.max(i, j)].hops], item.teach).seconds <= capSec;
|
|
60
|
+
const j = fits(i - 1) ? i - 1 : fits(i + 1) ? i + 1 : null;
|
|
61
|
+
if (j == null) continue;
|
|
62
|
+
const [a, b] = [Math.min(i, j), Math.max(i, j)];
|
|
63
|
+
pieces.splice(a, 2, { hops: [...pieces[a].hops, ...pieces[b].hops], group: pieces[a].group });
|
|
64
|
+
i = a - 1;
|
|
65
|
+
}
|
|
66
|
+
return pieces;
|
|
67
|
+
}
|
|
68
|
+
|
|
69
|
+
export function planVideos(v, { maxMin = 25, minMin = 2.5 } = {}) {
|
|
70
|
+
const capSec = maxMin * 60, minSec = minMin * 60;
|
|
71
|
+
const items = new Map([...v.flows, ...v.cores].map(x => [x.id, x]));
|
|
72
|
+
const videos = [], coverage = [];
|
|
73
|
+
const overviewSec = seconds(MODEL.overview);
|
|
74
|
+
videos.push({ index: 1, title: 'Series overview', form: 'overview', group: 'overview', flowGroup: null, words: MODEL.overview, seconds: overviewSec, minutes: +(overviewSec / 60).toFixed(1),
|
|
75
|
+
formWhy: 'one system map per series: the processes, the stores, and every flow and core as a route through them', sources: [], artefacts: [],
|
|
76
|
+
evidence: { items: v.series.length },
|
|
77
|
+
sections: v.series.map(s => `${s.id} :: ${s.title}`),
|
|
78
|
+
sectionSpans: v.series.map(s => ({ id: `${s.id} :: ${s.title}`, src: 'flows.validated.json', title: s.title, item: s.id, kind: s.kind, why: s.why })) });
|
|
79
|
+
for (const s of v.series) {
|
|
80
|
+
const item = items.get(s.id), pieces = split(item, capSec, minSec);
|
|
81
|
+
pieces.forEach((p, i) => {
|
|
82
|
+
const c = cost(p.hops, item.teach), index = videos.length + 1;
|
|
83
|
+
for (const h of p.hops) coverage.push({ item: item.id, hop: h.index, symbol: h.symbol, section: section(h), video: index });
|
|
84
|
+
const at = new Set(p.hops.map(h => h.index));
|
|
85
|
+
videos.push({ index, title: pieces.length > 1 ? `${item.title} (part ${i + 1} of ${pieces.length})` : item.title, form: item.kind, group: item.id, flowGroup: item.group,
|
|
86
|
+
words: c.words, seconds: c.seconds, minutes: +(c.seconds / 60).toFixed(1), formWhy: s.why,
|
|
87
|
+
sources: [...new Set(p.hops.map(h => h.file))], artefacts: [...new Set(p.hops.flatMap(h => h.pinnedBy.map(t => t.file)))],
|
|
88
|
+
evidence: { hops: p.hops.length, crossings: c.crossings, branches: c.branches, citedTests: c.cited, part: `${i + 1}/${pieces.length}`, rank: item.rank || null },
|
|
89
|
+
teach: item.teach.filter(t => at.has(t.hop)),
|
|
90
|
+
sections: p.hops.map(section),
|
|
91
|
+
sectionSpans: p.hops.map(h => ({ id: section(h), src: h.file, title: h.name, line: h.start, endLine: h.end, symbol: h.symbol, sha256: h.sha256, item: item.id, hop: h.index,
|
|
92
|
+
lines: h.lines, narratedLines: narratedLines(h), crossing: h.crossing, uses: h.uses, exercises: h.exercises, pinnedBy: h.pinnedBy })) });
|
|
93
|
+
});
|
|
94
|
+
}
|
|
95
|
+
const assigned = new Set(coverage.map(c => `${c.item}#${c.hop}`));
|
|
96
|
+
const orphans = [...items.values()].flatMap(x => x.hops.filter(h => !assigned.has(`${x.id}#${h.index}`)).map(h => ({ item: x.id, hop: h.index, symbol: h.symbol })));
|
|
97
|
+
const capViolations = videos.filter(x => x.seconds > capSec).map(x => ({ index: x.index, minutes: x.minutes }));
|
|
98
|
+
const total = videos.reduce((a, x) => a + x.seconds, 0);
|
|
99
|
+
return { videos, coverage, orphans, capViolations,
|
|
100
|
+
totals: { videos: videos.length, seriesMinutes: +(total / 60).toFixed(1), longestVideoMin: Math.max(...videos.map(x => x.minutes)), capMinutes: maxMin,
|
|
101
|
+
hops: coverage.length, flows: v.flows.length, cores: v.cores.length } };
|
|
102
|
+
}
|
|
@@ -0,0 +1,77 @@
|
|
|
1
|
+
/* Every file lands in exactly one group: prod, test, eval or dev. Ordered rules, first match
|
|
2
|
+
wins; a user override (`--group 'glob=group'`) always wins over them. Path rules decide;
|
|
3
|
+
content signals only reclassify a file that fell through to prod (design rev3 §3.15). */
|
|
4
|
+
|
|
5
|
+
export const GROUPS = ['prod', 'test', 'eval', 'dev'];
|
|
6
|
+
|
|
7
|
+
export const PATH_RULES = [
|
|
8
|
+
{ group: 'test', rule: 'test: a tests/, test/ or __tests__/ dir; test_*.py, *_test.py, conftest.py; *.test|spec.[cm][jt]s(x), setupTests / jest.setup / vitest.setup; *_test.go; test_*.c(c|pp), *_test.c(c|pp), *_unittest.cc',
|
|
9
|
+
re: /(^|\/)(tests?|__tests__)\/|(^|\/)(test_[^/]*|[^/]*_test|conftest)\.py$|\.(test|spec)\.[cm]?[jt]sx?$|(^|\/)(setupTests|setup-tests|jest\.setup|vitest\.setup|test-setup)\.[cm]?[jt]sx?$|_test\.go$|(^|\/)test_[^/]*\.(c|cc|cpp|cxx)$|[^/]_(test|unittest)\.(c|cc|cpp|cxx)$/ },
|
|
10
|
+
{ group: 'dev', rule: 'dev (file name): debug_*.py, diagnose_*.py, *migration_controller*.py, dev.py; debug*.[jt]s, *.dev.[jt]s, build and test tool configs (jest/vitest/vite/webpack/… .config.[jt]s)',
|
|
11
|
+
re: /(^|\/)(debug|diagnose)_[^/]*\.py$|(^|\/)[^/]*migration_controller[^/]*\.py$|(^|\/)dev\.py$|(^|\/)debug[^/]*\.[cm]?[jt]sx?$|\.dev\.[cm]?[jt]sx?$|(^|\/)(jest|vitest|vite|webpack|rollup|babel|eslint|prettier|tailwind|postcss|playwright|cypress)\.config\.[cm]?[jt]s$/ },
|
|
12
|
+
{ group: 'eval', rule: 'eval: an evals/, eval/, *_evals/, evals_*/, evaluation(s)/, benchmark(s)/ or bench/ dir',
|
|
13
|
+
re: /(^|\/)(evals?|[\w-]+_evals?|evals?_[\w-]+|evaluations?|benchmarks?|bench)\// },
|
|
14
|
+
{ group: 'dev', rule: 'dev (dir): migrations/, alembic/, dev_setup/, dev/, scripts/, verification/, debug/',
|
|
15
|
+
re: /(^|\/)(migrations|alembic|dev_setup|dev|scripts|verification|debug)\// },
|
|
16
|
+
];
|
|
17
|
+
export const DEFAULT_RULE = 'prod: everything else';
|
|
18
|
+
|
|
19
|
+
const TEST_IMPORT = /^\s*(import|from)\s+(pytest|unittest)\b/m;
|
|
20
|
+
const TEST_DEF = /^\s*(async\s+)?def\s+test_\w*\s*\(|^\s*class\s+Test\w*\s*[(:]/m;
|
|
21
|
+
const EVAL_IMPORT = /^\s*(import|from)\s+(ragas|deepeval|promptfoo|lm_eval|inspect_ai|mteb|beir|trulens\w*)\b|^\s*from\s+langsmith(\.\w+)*\s+import\s+[^\n]*\bevaluate\b/m;
|
|
22
|
+
const TRAINING = /\bTrainer\(|\.backward\(\)|^\s*(import|from)\s+accelerate\b/m;
|
|
23
|
+
const JS_TEST_IMPORT = /\bfrom\s+["'](vitest|@jest\/globals|node:test|mocha)["']|\brequire\(\s*["'](node:test|mocha|@jest\/globals)["']\s*\)/;
|
|
24
|
+
const JS_TEST_DEF = /^\s*(describe|it|test)(\.\w+)*\s*\(/m;
|
|
25
|
+
// design rev3 §3.15 (eval row, content signals): an eval-framework import puts a file in eval, in every language
|
|
26
|
+
const JS_EVAL_IMPORT = /\bfrom\s+["'](promptfoo|evalite|autoevals|braintrust|langsmith\/evaluation)["']|\brequire\(\s*["'](promptfoo|autoevals|braintrust)["']\s*\)/;
|
|
27
|
+
// a Go file only `go test` compiles is already a _test.go file; a generated or hand-written mock is a test double
|
|
28
|
+
const GO_MOCK = /^\/\/ Code generated by MockGen\b|^\s*mock\.Mock\s*$/m;
|
|
29
|
+
|
|
30
|
+
// a C / C++ file that includes a test framework and defines its tests
|
|
31
|
+
const C_TEST_INCLUDE = /^\s*#\s*include\s*[<"](gtest\/gtest\.h|gmock\/gmock\.h|catch2\/[\w/]+\.hpp|catch\.hpp|unity\.h|cmocka\.h|check\.h|CUnit\/\w+\.h)[>"]/m;
|
|
32
|
+
const C_TEST_DEF = /^\s*(TEST|TEST_F|TEST_P|TYPED_TEST|TEST_CASE|SCENARIO|TEST_CASE_METHOD)\s*\(|^\s*(RUN_TEST|cmocka_unit_test)\s*\(|\bMOCK_METHOD\d*\s*\(/m;
|
|
33
|
+
|
|
34
|
+
export function contentRule(text, lang = 'python') {
|
|
35
|
+
if (lang === 'go') return GO_MOCK.test(text) ? { group: 'test', rule: 'content: a MockGen-generated or testify mock.Mock test double' } : null;
|
|
36
|
+
if (lang === 'c' || lang === 'cpp') return C_TEST_INCLUDE.test(text) && C_TEST_DEF.test(text) ? { group: 'test', rule: 'content: includes gtest/gmock/Catch2/Unity/cmocka/check and defines tests or mocks' } : null;
|
|
37
|
+
if (lang !== 'python') {
|
|
38
|
+
if (JS_TEST_IMPORT.test(text) && JS_TEST_DEF.test(text)) return { group: 'test', rule: 'content: imports vitest/jest/node:test/mocha and defines tests' };
|
|
39
|
+
if (JS_EVAL_IMPORT.test(text)) return { group: 'eval', rule: 'content: imports an eval framework' };
|
|
40
|
+
return null;
|
|
41
|
+
}
|
|
42
|
+
if (TEST_IMPORT.test(text) && TEST_DEF.test(text)) return { group: 'test', rule: 'content: imports pytest/unittest and defines tests' };
|
|
43
|
+
if (EVAL_IMPORT.test(text)) return { group: 'eval', rule: 'content: imports an eval framework' };
|
|
44
|
+
if (TRAINING.test(text)) return { group: 'eval', rule: 'content: training-loop markers' };
|
|
45
|
+
return null;
|
|
46
|
+
}
|
|
47
|
+
|
|
48
|
+
// `**` crosses directories, `*` and `?` do not; a pattern with no slash matches anywhere
|
|
49
|
+
export function globToRegExp(glob) {
|
|
50
|
+
let re = '';
|
|
51
|
+
for (let i = 0; i < glob.length; i++) {
|
|
52
|
+
const c = glob[i];
|
|
53
|
+
if (c === '*' && glob[i + 1] === '*') { re += glob[i + 2] === '/' ? '(?:.*/)?' : '.*'; i += glob[i + 2] === '/' ? 2 : 1; }
|
|
54
|
+
else if (c === '*') re += '[^/]*';
|
|
55
|
+
else if (c === '?') re += '[^/]';
|
|
56
|
+
else re += c.replace(/[.+^${}()|[\]\\]/g, '\\$&');
|
|
57
|
+
}
|
|
58
|
+
return new RegExp((glob.includes('/') ? '^' : '(^|/)') + re + '$');
|
|
59
|
+
}
|
|
60
|
+
|
|
61
|
+
export function parseOverrides(specs) {
|
|
62
|
+
return specs.map(spec => {
|
|
63
|
+
const i = spec.lastIndexOf('=');
|
|
64
|
+
const glob = spec.slice(0, i), group = spec.slice(i + 1);
|
|
65
|
+
if (i < 1 || !GROUPS.includes(group)) throw new Error(`--group '${spec}': expected 'glob=${GROUPS.join('|')}'`);
|
|
66
|
+
return { glob, group, re: globToRegExp(glob) };
|
|
67
|
+
});
|
|
68
|
+
}
|
|
69
|
+
|
|
70
|
+
export function classifyFile(rel, overrides, readText, lang = 'python') {
|
|
71
|
+
const o = overrides.find(x => x.re.test(rel));
|
|
72
|
+
if (o) return { group: o.group, rule: `override: ${o.glob}=${o.group}` };
|
|
73
|
+
const r = PATH_RULES.find(x => x.re.test(rel));
|
|
74
|
+
if (r) return { group: r.group, rule: r.rule };
|
|
75
|
+
const c = readText && contentRule(readText(), lang);
|
|
76
|
+
return c || { group: 'prod', rule: DEFAULT_RULE };
|
|
77
|
+
}
|
|
@@ -0,0 +1,68 @@
|
|
|
1
|
+
/* Map a coverage-based test run (Go's -coverprofile, gcov) onto a code map (design §3.14, §3.17). Coverage says which
|
|
2
|
+
lines ran, not who called them, so no edge is marked observed: a symbol ran when a counted line or block inside it
|
|
3
|
+
ran (the innermost symbol holding it), and a direct test link counts as observed when its target ran in the unit
|
|
4
|
+
its test ran in — the test itself (Go, one run per test) or its test binary (make, `binary`). Nothing is removed or
|
|
5
|
+
demoted: the result is an overlay beside the code map, which is not edited. */
|
|
6
|
+
const byStr = (a, b) => a < b ? -1 : a > b ? 1 : 0;
|
|
7
|
+
|
|
8
|
+
/* counts: Map(repo-relative file -> [{ start, end, count }]) -> Set(symbol id) */
|
|
9
|
+
export function symbolsRan(m, counts) {
|
|
10
|
+
const inFile = new Map();
|
|
11
|
+
for (const s of m.symbols) if (s.kind !== 'class') (inFile.get(s.file) || inFile.set(s.file, []).get(s.file)).push(s);
|
|
12
|
+
const ran = new Set();
|
|
13
|
+
for (const [file, blocks] of counts) {
|
|
14
|
+
const syms = inFile.get(file); if (!syms) continue;
|
|
15
|
+
for (const b of blocks) {
|
|
16
|
+
if (!b.count) continue;
|
|
17
|
+
let best = null;
|
|
18
|
+
for (const s of syms) if (s.start <= b.start && b.start <= s.end && (!best || s.start > best.start || (s.start === best.start && s.end < best.end))) best = s;
|
|
19
|
+
if (best) ran.add(best.id);
|
|
20
|
+
}
|
|
21
|
+
}
|
|
22
|
+
return ran;
|
|
23
|
+
}
|
|
24
|
+
// gcov line counts per source -> the blocks symbolsRan reads
|
|
25
|
+
export const linesToBlocks = lines => [...lines].map(([line, count]) => ({ start: line, end: line, count }));
|
|
26
|
+
|
|
27
|
+
/* units: [{ key, tests: [test ids], ran: Set }], results: Map(test id -> { status, message, nodeid }), granularity:
|
|
28
|
+
'test' | 'binary', ownFile: which code-map files this runner measures */
|
|
29
|
+
export function observeCoverage(m, { units, results, failures, denied, granularity, ownFile, notCollected = [], runtime = {} }) {
|
|
30
|
+
const sym = new Map(m.symbols.map(s => [s.id, s]));
|
|
31
|
+
const grp = id => sym.get(id)?.group || '?';
|
|
32
|
+
const unitOf = new Map(); for (const u of units) for (const t of u.tests) unitOf.set(t, u);
|
|
33
|
+
const outcome = { collected: 0, passed: 0, failed: 0, errors: 0, skipped: 0, xfailed: 0, xpassed: 0, unknown: 0, notRun: 0 };
|
|
34
|
+
for (const r of results.values()) { outcome.collected++; if (r.status === 'not run') outcome.notRun++; else if (outcome[r.status] != null) outcome[r.status]++; }
|
|
35
|
+
outcome.errors += failures.filter(f => f.status === 'error').length;
|
|
36
|
+
const byTest = {};
|
|
37
|
+
let dObs = 0, dAll = 0, pObs = 0, pAll = 0, linked = 0;
|
|
38
|
+
for (const t of (m.tests || []).filter(x => ownFile(x.file))) {
|
|
39
|
+
const r = results.get(t.id), u = unitOf.get(t.id);
|
|
40
|
+
if (!r) continue;
|
|
41
|
+
const direct = [...new Set(t.direct.map(d => d.to).filter(to => sym.has(to) && grp(to) !== 'test' && sym.get(to).kind !== 'class'))];
|
|
42
|
+
const ran = r.status === 'not run' ? null : u?.ran;
|
|
43
|
+
const missing = ran ? direct.filter(to => !ran.has(to)).sort(byStr) : direct;
|
|
44
|
+
if (ran && direct.length) { linked++; dAll += direct.length; dObs += direct.length - missing.length;
|
|
45
|
+
const prod = direct.filter(to => grp(to) === 'prod'); pAll += prod.length; pObs += prod.filter(to => ran.has(to)).length; }
|
|
46
|
+
byTest[t.id] = { file: t.file, nodeids: r.nodeid ? [r.nodeid] : [], status: r.status, unit: u?.key ?? null, direct: { total: direct.length, observed: direct.length - missing.length, missing } };
|
|
47
|
+
}
|
|
48
|
+
let fixO = 0, fixN = 0;
|
|
49
|
+
for (const b of m.fixtureBindings || []) { const r = results.get(b.fn), u = unitOf.get(b.fn); if (!r || r.status === 'not run' || !u) continue; fixN++; if (u.ran.has(b.fixture)) fixO++; }
|
|
50
|
+
const ranAnywhere = new Set(units.flatMap(u => [...u.ran]));
|
|
51
|
+
const fns = m.symbols.filter(s => (s.kind === 'function' || s.kind === 'method') && ownFile(s.file));
|
|
52
|
+
const functionsRan = {};
|
|
53
|
+
for (const g of ['prod', 'eval', 'test', 'dev']) { const f = fns.filter(s => s.group === g); functionsRan[g] = { ran: f.filter(s => ranAnywhere.has(s.id)).length, total: f.length }; }
|
|
54
|
+
const ran = {};
|
|
55
|
+
for (const u of [...units].sort((a, b) => byStr(a.key, b.key))) for (const id of u.ran) (ran[id] ||= []).push(u.key);
|
|
56
|
+
return {
|
|
57
|
+
runtime, granularity, outcome, notCollected, failures, deniedBySandbox: denied,
|
|
58
|
+
summary: {
|
|
59
|
+
functionsRan,
|
|
60
|
+
pairs: 'not recorded: coverage says which code ran under a test, not which function called which',
|
|
61
|
+
directLinksPerTest: { tests: linked, observed: dObs, total: dAll, prod: { observed: pObs, total: pAll }, granularity },
|
|
62
|
+
fixtureLinks: { observed: fixO, total: fixN },
|
|
63
|
+
},
|
|
64
|
+
ran: Object.fromEntries(Object.entries(ran).sort(([a], [b]) => byStr(a, b))),
|
|
65
|
+
byUnit: Object.fromEntries([...units].sort((a, b) => byStr(a.key, b.key)).map(u => [u.key, { tests: [...u.tests].sort(byStr), ran: [...u.ran].sort(byStr) }])),
|
|
66
|
+
byTest, pairs: [], edges: [], noStaticEdge: [], callbacks: [],
|
|
67
|
+
};
|
|
68
|
+
}
|
|
@@ -0,0 +1,232 @@
|
|
|
1
|
+
/* Map a JavaScript / TypeScript test run onto a code map (design §3.14 TS/JS row, §3.17). Each test file ran in its
|
|
2
|
+
own runner process under V8 precise coverage (NODE_V8_COVERAGE), so a function that ran there ran under that test
|
|
3
|
+
file: attribution is per test file, never per test. Coverage records which functions ran, not who called them, so no
|
|
4
|
+
edge is marked observed; a direct test link counts as observed when its target ran in its test's file. It never
|
|
5
|
+
removes or demotes an edge: the result is an overlay beside the code map, which is not edited.
|
|
6
|
+
|
|
7
|
+
A covered function is matched to a code-map symbol through the script's source map when there is one (Node's own,
|
|
8
|
+
or the inline map of a script jest or vitest compiled through node:vm, which trace/js/fmtrace-srcmap.cjs records);
|
|
9
|
+
else by its position when the script V8 ran is the file itself (plain JavaScript), or the file with something
|
|
10
|
+
appended (TypeScript with its types stripped in place, plus a sourceURL comment) and every named function sits on a
|
|
11
|
+
line of the file that names it; and otherwise by name when exactly one symbol of that file has it. A named function
|
|
12
|
+
that matches nothing is counted, never guessed; an anonymous one (a callback inside a symbol) has nothing of its own
|
|
13
|
+
to mark.
|
|
14
|
+
A file whose run ended cleanly but reported no test is listed as not collected (the runner's own config leaves it
|
|
15
|
+
out, or it defines none), with the code map's count of tests in it, never as an error; those tests never ran, so they
|
|
16
|
+
are left out of byTest and the direct-link share. */
|
|
17
|
+
import { fileURLToPath } from 'node:url';
|
|
18
|
+
|
|
19
|
+
const byStr = (a, b) => a < b ? -1 : a > b ? 1 : 0;
|
|
20
|
+
const STATUS_RANK = { error: 4, failed: 3, passed: 1, skipped: 0 };
|
|
21
|
+
|
|
22
|
+
/* ---------------- source maps (v3, `mappings` only) ---------------- */
|
|
23
|
+
const B64 = Object.fromEntries([...'ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/'].map((c, i) => [c, i]));
|
|
24
|
+
export function decodeMappings(mappings) {
|
|
25
|
+
const lines = []; let src = 0, sLine = 0, sCol = 0;
|
|
26
|
+
for (const line of mappings.split(';')) {
|
|
27
|
+
const segs = []; let gCol = 0;
|
|
28
|
+
for (const seg of line.split(',')) {
|
|
29
|
+
if (!seg) continue;
|
|
30
|
+
const v = []; let shift = 0, val = 0;
|
|
31
|
+
for (const ch of seg) { const d = B64[ch]; if (d == null) break; val += (d & 31) << shift; if (d & 32) shift += 5; else { v.push(val & 1 ? -(val >>> 1) : val >>> 1); val = 0; shift = 0; } }
|
|
32
|
+
gCol += v[0];
|
|
33
|
+
if (v.length >= 4) { src += v[1]; sLine += v[2]; sCol += v[3]; segs.push([gCol, src, sLine, sCol]); }
|
|
34
|
+
}
|
|
35
|
+
lines.push(segs);
|
|
36
|
+
}
|
|
37
|
+
return lines;
|
|
38
|
+
}
|
|
39
|
+
// the generated line (0-based) an offset falls on, and that line's start offset
|
|
40
|
+
function generatedLine(entry, offset) {
|
|
41
|
+
let line = 0, start = 0;
|
|
42
|
+
while (line < entry.lineLengths.length && start + entry.lineLengths[line] < offset) { start += entry.lineLengths[line] + 1; line++; }
|
|
43
|
+
return [line, start];
|
|
44
|
+
}
|
|
45
|
+
// a function that closes after every line the map covers is the runner's module wrapper (jest's `Object.<anonymous>`,
|
|
46
|
+
// opened on the file's first line, closed after the inline map), no function of the file
|
|
47
|
+
function wrapsMappedCode(entry, endOffset) {
|
|
48
|
+
if (!entry?.lineLengths || !entry.data?.mappings) return false;
|
|
49
|
+
const decoded = entry.decoded ||= decodeMappings(entry.data.mappings);
|
|
50
|
+
return generatedLine(entry, endOffset)[0] > decoded.findLastIndex(segs => segs.length);
|
|
51
|
+
}
|
|
52
|
+
// the original line (1-based) a generated offset maps to, through a source-map-cache entry
|
|
53
|
+
function sourceMapLine(entry, offset) {
|
|
54
|
+
if (!entry?.lineLengths || !entry.data?.mappings) return null;
|
|
55
|
+
const [line, start] = generatedLine(entry, offset);
|
|
56
|
+
const segs = (entry.decoded ||= decodeMappings(entry.data.mappings))[line] || [];
|
|
57
|
+
const col = offset - start;
|
|
58
|
+
let best = null; for (const s of segs) { if (s[0] <= col) best = s; else break; }
|
|
59
|
+
best ||= segs[0];
|
|
60
|
+
return best ? best[2] + 1 : null;
|
|
61
|
+
}
|
|
62
|
+
|
|
63
|
+
const lineStarts = text => { const out = [0]; for (let i = 0; i < text.length; i++) if (text[i] === '\n') out.push(i + 1); return out; };
|
|
64
|
+
const lineOf = (starts, off) => { let lo = 0, hi = starts.length - 1; while (lo < hi) { const mid = (lo + hi + 1) >> 1; if (starts[mid] <= off) lo = mid; else hi = mid - 1; } return lo + 1; };
|
|
65
|
+
const nameMatches = (s, fn) => { const n = fn.replace(/^(get|set|async) /, ''); return n && (s.name === n || s.name.split('.').pop() === n || (s.kind === 'class' && s.name === n)); };
|
|
66
|
+
|
|
67
|
+
// which code-map symbols ran in one process's coverage dump; `root` is the copy the run used
|
|
68
|
+
export function functionsRan(m, dump, root, readFile) {
|
|
69
|
+
const ran = new Set(); let unmapped = 0, scripts = 0, anonymous = 0;
|
|
70
|
+
const syms = new Map();
|
|
71
|
+
for (const s of m.symbols) if (s.kind !== 'main' && /\.[cm]?[jt]sx?$/.test(s.file)) (syms.get(s.file) || syms.set(s.file, []).get(s.file)).push(s);
|
|
72
|
+
const ctorOf = new Map(m.symbols.filter(s => s.name === 'constructor' && s.cls).map(s => [s.cls, s.id]));
|
|
73
|
+
const smc = dump['source-map-cache'] || {};
|
|
74
|
+
for (const script of dump.result || []) {
|
|
75
|
+
let path; try { path = script.url.startsWith('file://') ? fileURLToPath(script.url) : script.url; } catch { continue; }
|
|
76
|
+
if (!path.startsWith(root + '/')) continue;
|
|
77
|
+
const rel = path.slice(root.length + 1), here = syms.get(rel);
|
|
78
|
+
if (!here) continue;
|
|
79
|
+
scripts++;
|
|
80
|
+
const text = readFile(rel);
|
|
81
|
+
const len = Math.max(0, ...script.functions.filter(f => f.ranges[0]?.startOffset === 0).map(f => f.ranges[0].endOffset));
|
|
82
|
+
const starts = text != null ? lineStarts(text) : null;
|
|
83
|
+
const fns = script.functions.filter(f => f.ranges[0] && !(f.ranges[0].startOffset === 0 && !f.functionName));
|
|
84
|
+
const lineText = off => text.slice(starts[lineOf(starts, off) - 1], starts[lineOf(starts, off)] ?? text.length);
|
|
85
|
+
const identical = text != null && (len === text.length || (len > text.length && fns.every(f => f.ranges[0].endOffset <= text.length)
|
|
86
|
+
&& fns.filter(f => f.functionName).every(f => lineText(f.ranges[0].startOffset).includes(f.functionName.replace(/^(get|set|async) /, '')) || /constructor/.test(lineText(f.ranges[0].startOffset)))));
|
|
87
|
+
const sm = smc[script.url] || smc[path];
|
|
88
|
+
for (const f of fns) {
|
|
89
|
+
const r0 = f.ranges[0];
|
|
90
|
+
if (!r0.count || (sm && wrapsMappedCode(sm, r0.endOffset))) continue;
|
|
91
|
+
const line = sm ? sourceMapLine(sm, r0.startOffset) : identical ? lineOf(starts, r0.startOffset) : null;
|
|
92
|
+
let cand = line != null ? here.filter(s => s.start === line) : [];
|
|
93
|
+
if (cand.length > 1) cand = cand.filter(s => nameMatches(s, f.functionName));
|
|
94
|
+
if (!cand.length && f.functionName && !identical) { const named = here.filter(s => nameMatches(s, f.functionName)); if (named.length === 1) cand = named; }
|
|
95
|
+
// a class's constructor runs under the class's name; a class with none has nothing of its own to mark
|
|
96
|
+
cand = cand.map(s => s.kind === 'class' ? ctorOf.get(s.id) : s.id).filter(Boolean);
|
|
97
|
+
if (cand.length === 1) ran.add(cand[0]); else if (f.functionName) unmapped++; else anonymous++;
|
|
98
|
+
}
|
|
99
|
+
}
|
|
100
|
+
return { ran, unmapped, scripts, anonymous };
|
|
101
|
+
}
|
|
102
|
+
|
|
103
|
+
/* ---------------- outcomes ---------------- */
|
|
104
|
+
const safeTitle = s => s.replace(/:/g, '-').replace(/\s+/g, ' ').trim();
|
|
105
|
+
// a test-id pattern from a code-map test title: `%s`, `%i`, `$x`, `${x}` stand for what `.each` and templates fill in
|
|
106
|
+
const titleRe = q => new RegExp('^' + q.replace(/[.*+?^${}()|[\]\\]/g, '\\$&').replace(/%[sdifjop#]|\\\$\\\{[^}]*\\\}|\\\$\w+/g, '.+?') + '$');
|
|
107
|
+
|
|
108
|
+
// jest's --json and vitest's json reporter (the same shape): per test file, per assertion
|
|
109
|
+
export function parseJestJson(json, root) {
|
|
110
|
+
const tests = [], suiteErrors = [];
|
|
111
|
+
for (const tr of json?.testResults || []) {
|
|
112
|
+
const file = tr.name?.startsWith(root + '/') ? tr.name.slice(root.length + 1) : tr.name;
|
|
113
|
+
const ars = tr.assertionResults || [];
|
|
114
|
+
for (const a of ars) tests.push({ file, titles: [...(a.ancestorTitles || []), a.title], status: a.status === 'passed' ? 'passed' : a.status === 'failed' ? 'failed' : 'skipped',
|
|
115
|
+
message: (a.failureMessages || []).join('\n') });
|
|
116
|
+
if ((tr.status === 'failed' && !ars.some(a => a.status === 'failed')) || (tr.message && !ars.length)) suiteErrors.push({ file, message: tr.message || tr.failureMessage || '' });
|
|
117
|
+
}
|
|
118
|
+
return { tests, suiteErrors };
|
|
119
|
+
}
|
|
120
|
+
|
|
121
|
+
// node --test's TAP: `# Subtest: T` opens a test at an indent, `ok N - T` / `not ok N - T` at that indent closes it, and
|
|
122
|
+
// the YAML block under it says whether it was a suite and carries the error. A suite fails on its own only by a hook
|
|
123
|
+
export function parseTap(text) {
|
|
124
|
+
const tests = [], suiteErrors = [], stack = [];
|
|
125
|
+
const lines = text.split('\n');
|
|
126
|
+
const unescape = t => t.replace(/\\(.)/g, '$1');
|
|
127
|
+
for (let i = 0; i < lines.length; i++) {
|
|
128
|
+
const l = lines[i], ind = l.match(/^ */)[0].length;
|
|
129
|
+
const sub = /^ *# Subtest: (.*)$/.exec(l);
|
|
130
|
+
if (sub) { stack.push({ ind, title: unescape(sub[1]) }); continue; }
|
|
131
|
+
const pt = /^ *(not )?ok \d+ - (.*?)(?: # (SKIP|TODO)\b.*)?$/.exec(l);
|
|
132
|
+
if (!pt) continue;
|
|
133
|
+
while (stack.length && stack.at(-1).ind > ind) stack.pop();
|
|
134
|
+
if (stack.at(-1)?.ind === ind) stack.pop();
|
|
135
|
+
const titles = [...stack.map(x => x.title), unescape(pt[2])];
|
|
136
|
+
const yaml = [];
|
|
137
|
+
if (lines[i + 1]?.trim() === '---') for (let j = i + 2; j < lines.length && lines[j].trim() !== '...'; j++) yaml.push(lines[j]);
|
|
138
|
+
const y = yaml.join('\n'), type = /^\s*type: '(\w+)'/m.exec(y)?.[1], failureType = /^\s*failureType: '(\w+)'/m.exec(y)?.[1];
|
|
139
|
+
const block = /^( *)error: \|-\n([\s\S]*?)(?=\n\1\w+:|(?![\s\S]))/m.exec(y);
|
|
140
|
+
const line = /^\s*error: (?:'(.*)'|"(.*)")$/m.exec(y), code = /^\s*code: '?(EPERM|EACCES|EROFS|ECONN\w+|ENOTFOUND|EAI_AGAIN|ENETUNREACH|EHOSTUNREACH)'?$/m.exec(y)?.[1];
|
|
141
|
+
const message = [...(block ? block[2] : line?.[1] ?? line?.[2] ?? '').split('\n'), ...(code && !(block?.[2] || line?.[0] || '').includes(code) ? [`code: ${code}`] : [])]
|
|
142
|
+
.map(x => x.trim()).filter(Boolean).join('\n');
|
|
143
|
+
if (type === 'suite') { if (pt[1] && failureType !== 'subtestsFailed') suiteErrors.push({ titles, message }); continue; }
|
|
144
|
+
tests.push({ titles, status: pt[3] ? 'skipped' : pt[1] ? 'failed' : 'passed', message });
|
|
145
|
+
}
|
|
146
|
+
return { tests, suiteErrors };
|
|
147
|
+
}
|
|
148
|
+
|
|
149
|
+
const NETWORK = /\b(ECONNREFUSED|ENOTFOUND|EAI_AGAIN|ENETUNREACH|EHOSTUNREACH)\b|connect E\w+|getaddrinfo E\w+|fetch failed/;
|
|
150
|
+
const PERMISSION = /\b(EPERM|EACCES|EROFS)\b/;
|
|
151
|
+
// the attempts a run recorded, else the target its error names (a process or worker the preload did not hook)
|
|
152
|
+
const attempts = (run, message) => run.network.length ? run.network
|
|
153
|
+
: [...new Set([...message.matchAll(/\bconnect E\w+ (\S+?)(?=[\s,)]|$)|\bgetaddrinfo E\w+ (\S+?)(?=[\s,)]|$)/g)].map(x => x[1] ? `connect ${x[1]}` : `getaddrinfo ${x[2]}`))];
|
|
154
|
+
|
|
155
|
+
/* runs: [{ file, outcomes: { tests, suiteErrors }, coverage: [dump], network: [attempt], timedOut, exitCode }] */
|
|
156
|
+
export function observeV8(m, runs, { root, readFile, node }) {
|
|
157
|
+
const sym = new Map(m.symbols.map(s => [s.id, s]));
|
|
158
|
+
const grp = id => sym.get(id)?.group || '?';
|
|
159
|
+
const tests = (m.tests || []).filter(t => /\.[cm]?[jt]sx?$/.test(t.file));
|
|
160
|
+
const testsIn = new Map(); for (const t of tests) (testsIn.get(t.file) || testsIn.set(t.file, []).get(t.file)).push(t);
|
|
161
|
+
const ranIn = new Map(), statusBy = new Map(), nodeidsOf = new Map(), failures = [], denied = [], notCollected = [];
|
|
162
|
+
const outcome = { collected: 0, passed: 0, failed: 0, errors: 0, skipped: 0, xfailed: 0, xpassed: 0 };
|
|
163
|
+
let unmapped = 0, scripts = 0, anonymous = 0;
|
|
164
|
+
for (const run of runs) {
|
|
165
|
+
const ran = new Set();
|
|
166
|
+
for (const d of run.coverage) { const r = functionsRan(m, d, root, readFile); r.ran.forEach(x => ran.add(x)); unmapped += r.unmapped; scripts += r.scripts; anonymous += r.anonymous; }
|
|
167
|
+
ranIn.set(run.file, ran);
|
|
168
|
+
const known = testsIn.get(run.file) || [];
|
|
169
|
+
const idOf = titles => { const q = `${run.file}:${safeTitle(titles.map(safeTitle).join(' > '))}`;
|
|
170
|
+
return known.find(t => t.id === q)?.id || known.find(t => titleRe(t.id).test(q))?.id || null; };
|
|
171
|
+
const reported = run.outcomes.tests.filter(t => !t.file || t.file === run.file);
|
|
172
|
+
for (const t of reported) {
|
|
173
|
+
outcome.collected++;
|
|
174
|
+
outcome[{ passed: 'passed', failed: 'failed', skipped: 'skipped' }[t.status]]++;
|
|
175
|
+
const id = idOf(t.titles), nodeid = `${run.file}::${t.titles.join(' > ')}`, name = id || nodeid;
|
|
176
|
+
(nodeidsOf.get(name) || nodeidsOf.set(name, new Set()).get(name)).add(nodeid);
|
|
177
|
+
if (!statusBy.has(name) || STATUS_RANK[t.status] > STATUS_RANK[statusBy.get(name)]) statusBy.set(name, t.status);
|
|
178
|
+
if (t.status !== 'failed') continue;
|
|
179
|
+
failures.push({ test: id, nodeid, status: 'failed', phase: 'call', message: t.message });
|
|
180
|
+
const network = NETWORK.test(t.message) ? attempts(run, t.message) : [];
|
|
181
|
+
if (network.length || PERMISSION.test(t.message)) denied.push({ test: id, nodeid, status: 'failed', network, permission: !network.length && PERMISSION.test(t.message), message: t.message });
|
|
182
|
+
}
|
|
183
|
+
const errs = [...run.outcomes.suiteErrors];
|
|
184
|
+
if (!reported.length && !errs.length && run.exitCode === 0 && !run.timedOut) notCollected.push({ file: run.file, tests: known.length });
|
|
185
|
+
if (!reported.length && !errs.length && (run.exitCode !== 0 || run.timedOut)) errs.push({ message: run.timedOut ? 'cut off by the time limit' : `the runner exited ${run.exitCode} and reported no test` });
|
|
186
|
+
for (const e of errs) {
|
|
187
|
+
outcome.errors++;
|
|
188
|
+
const nodeid = e.titles ? `${run.file}::${e.titles.join(' > ')}` : run.file;
|
|
189
|
+
failures.push({ test: null, nodeid, status: 'error', phase: e.titles ? 'setup' : 'collect', message: e.message || '' });
|
|
190
|
+
const network = NETWORK.test(e.message || '') ? attempts(run, e.message || '') : [];
|
|
191
|
+
if (network.length || PERMISSION.test(e.message || '')) denied.push({ test: null, nodeid, status: 'error', network, permission: !network.length, message: e.message || '' });
|
|
192
|
+
}
|
|
193
|
+
}
|
|
194
|
+
|
|
195
|
+
/* ---- per test (by its file's run): direct links and hooks that ran ---- */
|
|
196
|
+
const byTest = {};
|
|
197
|
+
let dObs = 0, dAll = 0, pObs = 0, pAll = 0, linked = 0;
|
|
198
|
+
const uncollected = new Set(notCollected.map(n => n.file));
|
|
199
|
+
for (const t of tests) {
|
|
200
|
+
const ran = ranIn.get(t.file);
|
|
201
|
+
if ((!ran && !statusBy.has(t.id)) || uncollected.has(t.file)) continue;
|
|
202
|
+
const direct = [...new Set(t.direct.map(d => d.to).filter(to => sym.has(to) && grp(to) !== 'test' && sym.get(to).kind !== 'class'))];
|
|
203
|
+
const missing = ran ? direct.filter(to => !ran.has(to)).sort(byStr) : direct;
|
|
204
|
+
if (ran && direct.length) { linked++; dAll += direct.length; dObs += direct.length - missing.length;
|
|
205
|
+
const prod = direct.filter(to => grp(to) === 'prod'); pAll += prod.length; pObs += prod.filter(to => ran.has(to)).length; }
|
|
206
|
+
byTest[t.id] = { file: t.file, nodeids: [...(nodeidsOf.get(t.id) || [])].sort(byStr), status: statusBy.get(t.id) || null,
|
|
207
|
+
direct: { total: direct.length, observed: direct.length - missing.length, missing } };
|
|
208
|
+
}
|
|
209
|
+
for (const [name, status] of statusBy) if (!byTest[name]) byTest[name] = { file: name.split('::')[0], nodeids: [...nodeidsOf.get(name)].sort(byStr), status, direct: { total: 0, observed: 0, missing: [] } };
|
|
210
|
+
let fixO = 0, fixN = 0;
|
|
211
|
+
const testFile = new Map(tests.map(t => [t.id, t.file]));
|
|
212
|
+
for (const b of m.fixtureBindings || []) { const ran = ranIn.get(testFile.get(b.fn)); if (!ran) continue; fixN++; if (ran.has(b.fixture)) fixO++; }
|
|
213
|
+
|
|
214
|
+
const ranAnywhere = new Set([...ranIn.values()].flatMap(s => [...s]));
|
|
215
|
+
const fns = m.symbols.filter(s => (s.kind === 'function' || s.kind === 'method') && /\.[cm]?[jt]sx?$/.test(s.file));
|
|
216
|
+
const functionsRanBy = {};
|
|
217
|
+
for (const g of ['prod', 'eval', 'test', 'dev']) { const f = fns.filter(s => s.group === g); functionsRanBy[g] = { ran: f.filter(s => ranAnywhere.has(s.id)).length, total: f.length }; }
|
|
218
|
+
const ran = {};
|
|
219
|
+
for (const [file, set] of [...ranIn].sort(([a], [b]) => byStr(a, b))) for (const id of set) (ran[id] ||= []).push(file);
|
|
220
|
+
const byFile = Object.fromEntries([...ranIn].sort(([a], [b]) => byStr(a, b)).map(([file, set]) => [file, { ran: [...set].sort(byStr) }]));
|
|
221
|
+
return {
|
|
222
|
+
runtime: { node }, granularity: 'test-file', outcome, notCollected: notCollected.sort((a, b) => byStr(a.file, b.file)), failures, deniedBySandbox: denied,
|
|
223
|
+
summary: {
|
|
224
|
+
scripts, functionsRan: functionsRanBy, unmappedFunctions: unmapped, anonymousCallbacks: anonymous,
|
|
225
|
+
pairs: 'not recorded: V8 coverage says which functions ran in a test file, not which called which (an inspector-based tracer would be needed)',
|
|
226
|
+
directLinksPerTest: { tests: linked, observed: dObs, total: dAll, prod: { observed: pObs, total: pAll }, granularity: 'test-file' },
|
|
227
|
+
fixtureLinks: { observed: fixO, total: fixN },
|
|
228
|
+
},
|
|
229
|
+
ran: Object.fromEntries(Object.entries(ran).sort(([a], [b]) => byStr(a, b))), byFile, byTest,
|
|
230
|
+
pairs: [], edges: [], noStaticEdge: [], callbacks: [],
|
|
231
|
+
};
|
|
232
|
+
}
|