@ngockhoale/ukit 3.5.10 → 3.5.13
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +37 -0
- package/manifests/documentation.yaml +11 -0
- package/manifests/platform.full.yaml +39 -2
- package/package.json +1 -1
- package/template_project/.claude/agents/code-reviewer.md +1 -0
- package/template_project/.claude/agents/feature-implementer.md +1 -0
- package/template_project/.claude/hooks/completion-gate.sh +3 -3
- package/template_project/.claude/settings.json +2 -2
- package/template_project/.claude/skills/code-review/SKILL.md +14 -0
- package/template_project/.claude/skills/testing-quality/SKILL.md +31 -1
- package/template_project/.claude/skills/testing-quality/memory-safe-tests.md +133 -0
- package/template_project/.claude/ukit/index/test-batches-adapters.mjs +261 -0
- package/template_project/.claude/ukit/index/test-batches.mjs +829 -0
- package/template_project/.claude/ukit/runtime/output-compression.mjs +81 -0
- package/template_project/.claude/ukit/runtime/stop-coordinator.mjs +4 -4
- package/template_project/.omp/agents/code-reviewer.md +1 -0
- package/template_project/.omp/agents/feature-implementer.md +1 -0
- package/template_project/docs/memory/testing.md +10 -0
|
@@ -0,0 +1,829 @@
|
|
|
1
|
+
#!/usr/bin/env node
|
|
2
|
+
// Memory-safe test execution + test-design audit (UKit testing standard).
|
|
3
|
+
//
|
|
4
|
+
// node .claude/ukit/index/test-batches.mjs plan [opts] discover + print the batch plan (runs nothing)
|
|
5
|
+
// node .claude/ukit/index/test-batches.mjs run [opts] run the FULL suite: sequential fresh-process batches
|
|
6
|
+
// node .claude/ukit/index/test-batches.mjs audit [opts] design-time audit: case budgets, fixture teardown, real paths
|
|
7
|
+
// node .claude/ukit/index/test-batches.mjs detect-leaks --file <test file> [opts] in-process memory growth evidence
|
|
8
|
+
//
|
|
9
|
+
// Contract (never relaxed):
|
|
10
|
+
// * Every discovered test file runs. Memory/time thresholds only FLAG; they never skip, filter or kill tests.
|
|
11
|
+
// * One batch at a time, one fresh OS process per batch, no parallel workers added by this runner.
|
|
12
|
+
// * Batches are whole files (native file paths, never name regexes) so module/file fixtures keep their lifecycle.
|
|
13
|
+
// * Verdict: PASS only when every inventory test is accounted for, no failure/error, every batch completed and no
|
|
14
|
+
// guarded path changed. Resource warnings never turn a pass into a failure; an incomplete run is never a pass.
|
|
15
|
+
// Exit codes: 0 PASS · 1 FAIL (test failures/errors) · 2 INCOMPLETE (runner/infra, unaccounted, deferred) · 3 UNSAFE.
|
|
16
|
+
// No dependencies beyond node: builtins. No network. No background jobs.
|
|
17
|
+
import fs from 'node:fs';
|
|
18
|
+
import os from 'node:os';
|
|
19
|
+
import path from 'node:path';
|
|
20
|
+
import { spawn, spawnSync } from 'node:child_process';
|
|
21
|
+
import { fileURLToPath } from 'node:url';
|
|
22
|
+
import { ADAPTERS, NODE_SAMPLER, PYTEST_PLUGIN } from './test-batches-adapters.mjs';
|
|
23
|
+
|
|
24
|
+
export { ADAPTERS };
|
|
25
|
+
export const DEFAULTS = Object.freeze({
|
|
26
|
+
batchFiles: 10,
|
|
27
|
+
heavyMb: 1024,
|
|
28
|
+
criticalFreeMb: 200,
|
|
29
|
+
lowMemWaitMs: 120_000,
|
|
30
|
+
lowMemPollMs: 5_000,
|
|
31
|
+
timeoutMs: 30 * 60 * 1000,
|
|
32
|
+
targetBatchMb: 1536,
|
|
33
|
+
keepRuns: 5,
|
|
34
|
+
guardMaxEntries: 20_000,
|
|
35
|
+
});
|
|
36
|
+
|
|
37
|
+
// Starting design budgets (cases per file / cases per batch). Defaults, not physical limits:
|
|
38
|
+
// override per project in .ukit/storage/test-batches.json → { "budgets": { "unit": { "file": 50 } } }.
|
|
39
|
+
export const BUDGETS = Object.freeze({
|
|
40
|
+
unit: Object.freeze({ file: 40, batch: 100 }),
|
|
41
|
+
integration: Object.freeze({ file: 20, batch: 40 }),
|
|
42
|
+
'app-heavy': Object.freeze({ file: 10, batch: 20 }),
|
|
43
|
+
e2e: Object.freeze({ file: 10, batch: 10 }),
|
|
44
|
+
});
|
|
45
|
+
|
|
46
|
+
export const EXIT = Object.freeze({ PASS: 0, FAIL: 1, INCOMPLETE: 2, UNSAFE: 3 });
|
|
47
|
+
const MODES = new Set(['plan', 'run', 'audit', 'detect-leaks']);
|
|
48
|
+
const toPosix = (p) => String(p).split(path.sep).join('/').replace(/\\/g, '/');
|
|
49
|
+
|
|
50
|
+
// ---------------------------------------------------------------- args / config
|
|
51
|
+
|
|
52
|
+
export function parseArgs(argv) {
|
|
53
|
+
const out = { mode: 'run', framework: null, root: null, out: null, json: false, guard: [], changed: false,
|
|
54
|
+
batchCases: null, batchFiles: null, heavyMb: null, timeoutMs: null, file: null, python: null,
|
|
55
|
+
resourceWarnings: false, criticalFreeMb: null, extra: [], errors: [] };
|
|
56
|
+
const num = (flag, v) => { const n = Number(v); if (!Number.isFinite(n) || n <= 0) out.errors.push(`${flag} needs a positive number`); return n; };
|
|
57
|
+
for (let i = 0; i < argv.length; i += 1) {
|
|
58
|
+
const a = argv[i];
|
|
59
|
+
const next = () => { i += 1; if (i >= argv.length) { out.errors.push(`${a} needs a value`); return ''; } return argv[i]; };
|
|
60
|
+
if (a === '--') { out.extra = argv.slice(i + 1); break; }
|
|
61
|
+
if (i === 0 && MODES.has(a)) { out.mode = a; continue; }
|
|
62
|
+
switch (a) {
|
|
63
|
+
case '--framework': out.framework = next(); break;
|
|
64
|
+
case '--root': out.root = next(); break;
|
|
65
|
+
case '--out': out.out = next(); break;
|
|
66
|
+
case '--json': out.json = true; break;
|
|
67
|
+
case '--guard': out.guard.push(next()); break;
|
|
68
|
+
case '--changed': out.changed = true; break;
|
|
69
|
+
case '--batch-size': out.batchCases = num(a, next()); break;
|
|
70
|
+
case '--batch-files': out.batchFiles = num(a, next()); break;
|
|
71
|
+
case '--heavy-mb': out.heavyMb = num(a, next()); break;
|
|
72
|
+
case '--timeout-ms': out.timeoutMs = num(a, next()); break;
|
|
73
|
+
case '--critical-free-mb': { const n = Number(next()); if (!Number.isFinite(n) || n < 0) out.errors.push(`${a} needs a number >= 0`); out.criticalFreeMb = n; break; }
|
|
74
|
+
case '--file': out.file = next(); break;
|
|
75
|
+
case '--python': out.python = next(); break;
|
|
76
|
+
case '--resource-warnings': out.resourceWarnings = true; break;
|
|
77
|
+
default: out.errors.push(`unknown argument: ${a}`);
|
|
78
|
+
}
|
|
79
|
+
}
|
|
80
|
+
if (out.framework && !ADAPTERS[out.framework]) out.errors.push(`--framework must be one of: ${Object.keys(ADAPTERS).join(', ')}`);
|
|
81
|
+
if (out.mode === 'detect-leaks' && !out.file) out.errors.push('detect-leaks needs --file <test file>');
|
|
82
|
+
return out;
|
|
83
|
+
}
|
|
84
|
+
|
|
85
|
+
export function loadProjectConfig(root) {
|
|
86
|
+
try {
|
|
87
|
+
const raw = JSON.parse(fs.readFileSync(path.join(root, '.ukit/storage/test-batches.json'), 'utf8'));
|
|
88
|
+
return raw && typeof raw === 'object' ? raw : {};
|
|
89
|
+
} catch { return {}; }
|
|
90
|
+
}
|
|
91
|
+
|
|
92
|
+
export function resolveBudgets(config = {}) {
|
|
93
|
+
const merged = {};
|
|
94
|
+
for (const [cat, b] of Object.entries(BUDGETS)) merged[cat] = { ...b, ...(config.budgets?.[cat] ?? {}) };
|
|
95
|
+
return merged;
|
|
96
|
+
}
|
|
97
|
+
|
|
98
|
+
// ---------------------------------------------------------------- framework detection
|
|
99
|
+
|
|
100
|
+
const exists = (root, rel) => fs.existsSync(path.join(root, rel));
|
|
101
|
+
function readJson(file) { try { return JSON.parse(fs.readFileSync(file, 'utf8')); } catch { return null; } }
|
|
102
|
+
|
|
103
|
+
export function detectFrameworks(root) {
|
|
104
|
+
const found = [];
|
|
105
|
+
const pyproject = exists(root, 'pyproject.toml') ? fs.readFileSync(path.join(root, 'pyproject.toml'), 'utf8') : '';
|
|
106
|
+
const setupCfg = exists(root, 'setup.cfg') ? fs.readFileSync(path.join(root, 'setup.cfg'), 'utf8') : '';
|
|
107
|
+
if (exists(root, 'pytest.ini') || exists(root, 'conftest.py') || /\[tool\.pytest/.test(pyproject)
|
|
108
|
+
|| /\[tool:pytest\]/.test(setupCfg) || (exists(root, 'tox.ini') && /\[pytest\]/.test(fs.readFileSync(path.join(root, 'tox.ini'), 'utf8')))) {
|
|
109
|
+
found.push('pytest');
|
|
110
|
+
}
|
|
111
|
+
const pkg = readJson(path.join(root, 'package.json')) ?? {};
|
|
112
|
+
const deps = { ...(pkg.dependencies ?? {}), ...(pkg.devDependencies ?? {}) };
|
|
113
|
+
const hasConfig = (base) => ['js', 'mjs', 'cjs', 'ts', 'mts', 'cts'].some((ext) => exists(root, `${base}.${ext}`));
|
|
114
|
+
if (deps.vitest || hasConfig('vitest.config')) found.push('vitest');
|
|
115
|
+
if (deps.jest || hasConfig('jest.config') || pkg.jest) found.push('jest');
|
|
116
|
+
return found;
|
|
117
|
+
}
|
|
118
|
+
|
|
119
|
+
export function chooseFramework(root, explicit) {
|
|
120
|
+
if (explicit) return { framework: explicit };
|
|
121
|
+
const found = detectFrameworks(root);
|
|
122
|
+
if (found.length === 1) return { framework: found[0] };
|
|
123
|
+
if (!found.length) return { error: 'no supported test framework detected (pytest, vitest, jest); pass --framework or see the adapter contract in .claude/skills/testing-quality/memory-safe-tests.md' };
|
|
124
|
+
return { error: `several test frameworks detected (${found.join(', ')}); pass --framework <name> and run once per framework` };
|
|
125
|
+
}
|
|
126
|
+
|
|
127
|
+
// ---------------------------------------------------------------- inventory / classification / planning
|
|
128
|
+
|
|
129
|
+
// Category from path conventions only (cheap, deterministic). Projects override with config.categories:
|
|
130
|
+
// { "app-heavy": ["tests/api/"], "e2e": ["tests/browser/"] } — prefix matches win over the heuristic.
|
|
131
|
+
export function classifyFile(file, config = {}) {
|
|
132
|
+
const f = toPosix(file).toLowerCase();
|
|
133
|
+
for (const [cat, prefixes] of Object.entries(config.categories ?? {})) {
|
|
134
|
+
if ((prefixes ?? []).some((p) => f.startsWith(toPosix(p).toLowerCase()))) return cat;
|
|
135
|
+
}
|
|
136
|
+
if (/(^|\/)(e2e|playwright|cypress|browser)(\/|[._-])/.test(f)) return 'e2e';
|
|
137
|
+
if (/(^|\/)(api|routes?|endpoints?|apps?|server|views)(\/|[._-])/.test(f)) return 'app-heavy';
|
|
138
|
+
if (/(^|\/)(integration|it)(\/|[._-])/.test(f)) return 'integration';
|
|
139
|
+
return 'unit';
|
|
140
|
+
}
|
|
141
|
+
|
|
142
|
+
// Deterministic manifest: files sorted, ids in native collection order within each file.
|
|
143
|
+
export function buildInventory(discovered) {
|
|
144
|
+
const files = [...new Set(discovered.files ?? [])].map(toPosix).sort();
|
|
145
|
+
const caseCounts = new Map(files.map((f) => [f, null]));
|
|
146
|
+
const ids = [];
|
|
147
|
+
const duplicates = [];
|
|
148
|
+
if (Array.isArray(discovered.tests)) {
|
|
149
|
+
const seen = new Set();
|
|
150
|
+
for (const f of files) caseCounts.set(f, 0);
|
|
151
|
+
for (const t of discovered.tests) {
|
|
152
|
+
const file = toPosix(t.file);
|
|
153
|
+
if (!caseCounts.has(file)) { caseCounts.set(file, 0); files.push(file); }
|
|
154
|
+
if (seen.has(t.id)) { duplicates.push(t.id); continue; }
|
|
155
|
+
seen.add(t.id);
|
|
156
|
+
ids.push(t.id);
|
|
157
|
+
caseCounts.set(file, caseCounts.get(file) + 1);
|
|
158
|
+
}
|
|
159
|
+
files.sort();
|
|
160
|
+
}
|
|
161
|
+
return { files, caseCounts, ids: Array.isArray(discovered.tests) ? ids : null, duplicates };
|
|
162
|
+
}
|
|
163
|
+
|
|
164
|
+
// History: .ukit/storage/test-batches/history.json → { files: { "<file>": { isolatedPeakMb, ms } } }.
|
|
165
|
+
// isolatedPeakMb is recorded ONLY from single-file batches or detect-leaks — a multi-file batch peak is
|
|
166
|
+
// never attributed to its members.
|
|
167
|
+
export function planBatches(inventory, { batchFiles = DEFAULTS.batchFiles, batchCases = null, budgets = BUDGETS, config = {}, history = {}, targetBatchMb = DEFAULTS.targetBatchMb } = {}) {
|
|
168
|
+
const batches = [];
|
|
169
|
+
let cur = null;
|
|
170
|
+
const close = () => { if (cur && cur.files.length) batches.push(cur); cur = null; };
|
|
171
|
+
for (const file of inventory.files) {
|
|
172
|
+
const category = classifyFile(file, config);
|
|
173
|
+
const cases = inventory.caseCounts.get(file);
|
|
174
|
+
const caseCap = batchCases ?? budgets[category]?.batch ?? BUDGETS.unit.batch;
|
|
175
|
+
const knownMb = history.files?.[file]?.isolatedPeakMb ?? null;
|
|
176
|
+
// Measured-heavy and e2e files run alone: no other file shares their process.
|
|
177
|
+
const solo = category === 'e2e' || (knownMb != null && knownMb >= targetBatchMb / 2);
|
|
178
|
+
if (solo) { close(); batches.push({ files: [file], cases, categories: [category], memMb: knownMb, solo: true }); continue; }
|
|
179
|
+
const fits = cur && cur.files.length < batchFiles
|
|
180
|
+
&& (cases == null || cur.cases == null || cur.cases + cases <= caseCap || cur.files.length === 0)
|
|
181
|
+
&& (knownMb == null || cur.memMb == null || cur.memMb + knownMb <= targetBatchMb);
|
|
182
|
+
if (!fits) { close(); cur = { files: [], cases: cases == null ? null : 0, categories: [], memMb: null, solo: false }; }
|
|
183
|
+
cur.files.push(file);
|
|
184
|
+
if (cur.cases != null && cases != null) cur.cases += cases; else cur.cases = null;
|
|
185
|
+
if (!cur.categories.includes(category)) cur.categories.push(category);
|
|
186
|
+
if (knownMb != null) cur.memMb = (cur.memMb ?? 0) + knownMb;
|
|
187
|
+
}
|
|
188
|
+
close();
|
|
189
|
+
return batches.map((b, i) => ({ index: i + 1, ...b }));
|
|
190
|
+
}
|
|
191
|
+
|
|
192
|
+
// Plan invariant: every inventory file appears in exactly one batch.
|
|
193
|
+
export function verifyPlan(inventory, batches) {
|
|
194
|
+
const seen = new Map();
|
|
195
|
+
for (const b of batches) for (const f of b.files) seen.set(f, (seen.get(f) ?? 0) + 1);
|
|
196
|
+
return {
|
|
197
|
+
missing: inventory.files.filter((f) => !seen.has(f)),
|
|
198
|
+
duplicated: [...seen].filter(([, n]) => n > 1).map(([f]) => f),
|
|
199
|
+
extra: [...seen.keys()].filter((f) => !inventory.files.includes(f)),
|
|
200
|
+
};
|
|
201
|
+
}
|
|
202
|
+
|
|
203
|
+
// ---------------------------------------------------------------- RSS units
|
|
204
|
+
|
|
205
|
+
// `/usr/bin/time -l` (macOS/BSD) prints bytes; GNU `/usr/bin/time -v` prints KiB. Unknown → null, never 0.
|
|
206
|
+
export function parseTimeRss(text, platform = process.platform) {
|
|
207
|
+
const s = String(text ?? '');
|
|
208
|
+
const bsd = s.match(/(\d+)\s+maximum resident set size/);
|
|
209
|
+
if (bsd) return platform === 'darwin' ? Number(bsd[1]) : Number(bsd[1]) * 1024;
|
|
210
|
+
const gnu = s.match(/Maximum resident set size \(kbytes\):\s*(\d+)/);
|
|
211
|
+
if (gnu) return Number(gnu[1]) * 1024;
|
|
212
|
+
return null;
|
|
213
|
+
}
|
|
214
|
+
|
|
215
|
+
export const toMb = (bytes) => (bytes == null || !Number.isFinite(bytes) ? null : Math.round(bytes / 1048576));
|
|
216
|
+
|
|
217
|
+
export function timeWrapper(platform = process.platform, fileExists = fs.existsSync) {
|
|
218
|
+
if (platform === 'darwin' && fileExists('/usr/bin/time')) return { cmd: '/usr/bin/time', flag: '-l' };
|
|
219
|
+
if (platform === 'linux' && fileExists('/usr/bin/time')) return { cmd: '/usr/bin/time', flag: '-v' };
|
|
220
|
+
return null; // Windows / no GNU time: peak RSS comes from the in-process plugin when available, else unknown.
|
|
221
|
+
}
|
|
222
|
+
|
|
223
|
+
// ---------------------------------------------------------------- aggregation / verdict
|
|
224
|
+
|
|
225
|
+
const OUTCOMES = ['passed', 'failed', 'error', 'skipped', 'xfailed', 'xpassed'];
|
|
226
|
+
|
|
227
|
+
export function aggregate({ inventory, batches, batchResults, guard = null, heavyMb = DEFAULTS.heavyMb, framework, version, runId, startedAt, finishedAt, deferred = null }) {
|
|
228
|
+
const counts = Object.fromEntries(OUTCOMES.map((o) => [o, 0]));
|
|
229
|
+
const failedIds = [];
|
|
230
|
+
const outcomeById = new Map();
|
|
231
|
+
const fileErrors = [];
|
|
232
|
+
const heavy = [];
|
|
233
|
+
const warnings = [];
|
|
234
|
+
const completed = [];
|
|
235
|
+
const incomplete = [];
|
|
236
|
+
let maxRss = null;
|
|
237
|
+
for (const br of batchResults) {
|
|
238
|
+
(br.complete ? completed : incomplete).push(br.index);
|
|
239
|
+
if (br.peakRss != null) maxRss = Math.max(maxRss ?? 0, br.peakRss);
|
|
240
|
+
for (const r of br.results ?? []) {
|
|
241
|
+
if (outcomeById.has(r.id)) continue; // a re-reported id never double counts
|
|
242
|
+
outcomeById.set(r.id, r.outcome);
|
|
243
|
+
counts[r.outcome] = (counts[r.outcome] ?? 0) + 1;
|
|
244
|
+
if (r.outcome === 'failed' || r.outcome === 'error') failedIds.push(r.id);
|
|
245
|
+
}
|
|
246
|
+
for (const fe of br.fileErrors ?? []) fileErrors.push({ batch: br.index, ...fe });
|
|
247
|
+
const mb = toMb(br.peakRss);
|
|
248
|
+
if (mb != null && mb >= heavyMb) {
|
|
249
|
+
// Attribution is honest: a single-file batch measures that file; a multi-file batch only the batch.
|
|
250
|
+
heavy.push(br.files.length === 1
|
|
251
|
+
? { file: br.files[0], peakMb: mb, attribution: 'isolated' }
|
|
252
|
+
: { batch: br.index, files: br.files, peakMb: mb, attribution: 'batch — profile files alone (detect-leaks) before blaming one' });
|
|
253
|
+
}
|
|
254
|
+
if (br.liveThreads?.length) warnings.push({ batch: br.index, kind: 'live-threads-at-exit', detail: br.liveThreads });
|
|
255
|
+
if (br.liveChildren?.length) warnings.push({ batch: br.index, kind: 'live-child-processes-at-exit', detail: br.liveChildren });
|
|
256
|
+
}
|
|
257
|
+
const growth = rssGrowthWarnings(batchResults);
|
|
258
|
+
warnings.push(...growth);
|
|
259
|
+
const unaccounted = inventory.ids ? inventory.ids.filter((id) => !outcomeById.has(id)) : [];
|
|
260
|
+
const unaccountedFiles = inventory.ids ? [] : inventory.files.filter((f) => !batchResults.some((b) => b.complete && b.files.includes(f)));
|
|
261
|
+
const total = inventory.ids ? inventory.ids.length : null;
|
|
262
|
+
const notRun = batches.filter((b) => !batchResults.some((r) => r.index === b.index)).map((b) => b.index);
|
|
263
|
+
const unsafe = guard?.changed?.length ? guard.changed : [];
|
|
264
|
+
let verdict;
|
|
265
|
+
if (unsafe.length) verdict = 'UNSAFE';
|
|
266
|
+
else if (incomplete.length || notRun.length || unaccounted.length || unaccountedFiles.length || deferred) verdict = 'INCOMPLETE';
|
|
267
|
+
else if (counts.failed || counts.error || fileErrors.length) verdict = 'FAIL';
|
|
268
|
+
else verdict = 'PASS';
|
|
269
|
+
return {
|
|
270
|
+
runId, framework, frameworkVersion: version ?? null, startedAt, finishedAt,
|
|
271
|
+
durationMs: finishedAt && startedAt ? Date.parse(finishedAt) - Date.parse(startedAt) : null,
|
|
272
|
+
inventory: { files: inventory.files.length, tests: total, idLevel: Boolean(inventory.ids), duplicateIds: inventory.duplicates },
|
|
273
|
+
batches: { planned: batches.length, completed: completed.length, incomplete, notRun },
|
|
274
|
+
counts, reported: outcomeById.size, failedIds, fileErrors, unaccounted, unaccountedFiles,
|
|
275
|
+
maxBatchRssMb: toMb(maxRss), heavy, warnings, isolation: guard ? { guarded: guard.roots, changed: unsafe } : null,
|
|
276
|
+
deferred, verdict, exitCode: EXIT[verdict],
|
|
277
|
+
};
|
|
278
|
+
}
|
|
279
|
+
|
|
280
|
+
// ---------------------------------------------------------------- real-data guard
|
|
281
|
+
|
|
282
|
+
// Snapshot of a guarded tree (real data dirs, auth/system DBs, shared logs): relPath → "size:mtimeMs".
|
|
283
|
+
// A run whose tests change any guarded entry is UNSAFE regardless of test results.
|
|
284
|
+
export function snapshotTree(target, maxEntries = DEFAULTS.guardMaxEntries) {
|
|
285
|
+
const map = new Map();
|
|
286
|
+
let truncated = false;
|
|
287
|
+
const walk = (p, relp) => {
|
|
288
|
+
if (map.size >= maxEntries) { truncated = true; return; }
|
|
289
|
+
let st; try { st = fs.lstatSync(p); } catch { return; }
|
|
290
|
+
if (st.isDirectory()) {
|
|
291
|
+
let names = []; try { names = fs.readdirSync(p).sort(); } catch { return; }
|
|
292
|
+
for (const n of names) walk(path.join(p, n), relp ? `${relp}/${n}` : n);
|
|
293
|
+
} else {
|
|
294
|
+
map.set(relp || '.', `${st.size}:${Math.trunc(st.mtimeMs)}`);
|
|
295
|
+
}
|
|
296
|
+
};
|
|
297
|
+
if (fs.existsSync(target)) walk(target, '');
|
|
298
|
+
return { map, truncated, existed: fs.existsSync(target) };
|
|
299
|
+
}
|
|
300
|
+
|
|
301
|
+
export function diffSnapshots(root, before, after) {
|
|
302
|
+
const changed = [];
|
|
303
|
+
if (before.existed !== after.existed) changed.push(`${root} ${after.existed ? 'created' : 'deleted'}`);
|
|
304
|
+
for (const [k, v] of before.map) if (after.map.get(k) !== v) changed.push(`${root}/${k} ${after.map.has(k) ? 'modified' : 'deleted'}`);
|
|
305
|
+
for (const k of after.map.keys()) if (!before.map.has(k)) changed.push(`${root}/${k} created`);
|
|
306
|
+
return changed;
|
|
307
|
+
}
|
|
308
|
+
|
|
309
|
+
export function createGuard(projectRoot, roots) {
|
|
310
|
+
const abs = [...new Set(roots)].map((r) => path.resolve(projectRoot, r));
|
|
311
|
+
const before = abs.map((r) => snapshotTree(r));
|
|
312
|
+
return {
|
|
313
|
+
roots: abs.map((r) => toPosix(path.relative(projectRoot, r) || '.')),
|
|
314
|
+
truncated: before.some((s) => s.truncated),
|
|
315
|
+
check() {
|
|
316
|
+
const changed = [];
|
|
317
|
+
abs.forEach((r, i) => changed.push(...diffSnapshots(toPosix(path.relative(projectRoot, r) || '.'), before[i], snapshotTree(r))));
|
|
318
|
+
return { roots: this.roots, changed };
|
|
319
|
+
},
|
|
320
|
+
};
|
|
321
|
+
}
|
|
322
|
+
|
|
323
|
+
// ---------------------------------------------------------------- free memory
|
|
324
|
+
|
|
325
|
+
// macOS os.freemem() counts only "free" pages and reads near zero on a healthy machine; reclaimable
|
|
326
|
+
// inactive/speculative/purgeable pages are available too. Unknown → null (treated as "do not defer").
|
|
327
|
+
export function availableMemoryMb({ platform = process.platform, run = spawnSync, freemem = os.freemem } = {}) {
|
|
328
|
+
if (platform === 'darwin') {
|
|
329
|
+
try {
|
|
330
|
+
const res = run('vm_stat', [], { encoding: 'utf8', timeout: 5000 });
|
|
331
|
+
const text = res.stdout ?? '';
|
|
332
|
+
const page = Number((text.match(/page size of (\d+) bytes/) ?? [])[1] ?? 4096);
|
|
333
|
+
const pick = (label) => Number((text.match(new RegExp(`Pages ${label}:\\s+(\\d+)`)) ?? [])[1] ?? 0);
|
|
334
|
+
const pages = pick('free') + pick('inactive') + pick('speculative') + pick('purgeable');
|
|
335
|
+
if (pages > 0) return Math.round((pages * page) / 1048576);
|
|
336
|
+
} catch { /* fall through */ }
|
|
337
|
+
}
|
|
338
|
+
if (platform === 'linux') {
|
|
339
|
+
try {
|
|
340
|
+
const m = fs.readFileSync('/proc/meminfo', 'utf8').match(/MemAvailable:\s+(\d+)\s+kB/);
|
|
341
|
+
if (m) return Math.round(Number(m[1]) / 1024);
|
|
342
|
+
} catch { /* fall through */ }
|
|
343
|
+
}
|
|
344
|
+
const free = freemem();
|
|
345
|
+
return Number.isFinite(free) ? Math.round(free / 1048576) : null;
|
|
346
|
+
}
|
|
347
|
+
|
|
348
|
+
// Bounded wait for memory before starting a batch. Never kills/skips; after the bound the run stops as
|
|
349
|
+
// INCOMPLETE ("deferred") so the gate can never read green while batches remain.
|
|
350
|
+
export async function waitForMemory({ criticalFreeMb, maxWaitMs, pollMs, probe = availableMemoryMb, sleep = (ms) => new Promise((r) => setTimeout(r, ms)), now = Date.now }) {
|
|
351
|
+
if (!criticalFreeMb) return { ok: true, waitedMs: 0 };
|
|
352
|
+
const start = now();
|
|
353
|
+
for (;;) {
|
|
354
|
+
const free = probe();
|
|
355
|
+
if (free == null || free >= criticalFreeMb) return { ok: true, waitedMs: now() - start, freeMb: free };
|
|
356
|
+
if (now() - start >= maxWaitMs) return { ok: false, waitedMs: now() - start, freeMb: free };
|
|
357
|
+
await sleep(pollMs);
|
|
358
|
+
}
|
|
359
|
+
}
|
|
360
|
+
|
|
361
|
+
// ---------------------------------------------------------------- batch execution
|
|
362
|
+
|
|
363
|
+
function spawnBatch({ cmd, args, env, cwd, logFile, timeoutMs, wrap }) {
|
|
364
|
+
return new Promise((resolve) => {
|
|
365
|
+
const finalCmd = wrap ? wrap.cmd : cmd;
|
|
366
|
+
const finalArgs = wrap ? [wrap.flag, cmd, ...args] : args;
|
|
367
|
+
const log = fs.openSync(logFile, 'w');
|
|
368
|
+
let child;
|
|
369
|
+
try {
|
|
370
|
+
child = spawn(finalCmd, finalArgs, { cwd, env: { ...process.env, ...env }, stdio: ['ignore', log, log], detached: process.platform !== 'win32', windowsHide: true });
|
|
371
|
+
} catch (error) { fs.closeSync(log); resolve({ code: null, signal: null, error: error.message, timedOut: false }); return; }
|
|
372
|
+
let timedOut = false;
|
|
373
|
+
const killTree = (sig) => { try { if (process.platform === 'win32') spawnSync('taskkill', ['/pid', String(child.pid), '/T', '/F']); else process.kill(-child.pid, sig); } catch { /* gone */ } };
|
|
374
|
+
const timer = setTimeout(() => { timedOut = true; killTree('SIGTERM'); setTimeout(() => killTree('SIGKILL'), 5000).unref(); }, timeoutMs);
|
|
375
|
+
child.on('error', (error) => { clearTimeout(timer); fs.closeSync(log); resolve({ code: null, signal: null, error: error.message, timedOut }); });
|
|
376
|
+
child.on('close', (code, signal) => {
|
|
377
|
+
clearTimeout(timer);
|
|
378
|
+
fs.closeSync(log);
|
|
379
|
+
// Reap anything the batch left behind (servers, workers) so the next batch starts clean.
|
|
380
|
+
if (process.platform !== 'win32') { try { process.kill(-child.pid, 'SIGKILL'); } catch { /* none left */ } }
|
|
381
|
+
resolve({ code, signal, error: null, timedOut });
|
|
382
|
+
});
|
|
383
|
+
});
|
|
384
|
+
}
|
|
385
|
+
|
|
386
|
+
export async function runBatch(ctx, adapter, batch, { wrap = timeWrapper() } = {}) {
|
|
387
|
+
const dir = path.join(ctx.runDir, `batch-${String(batch.index).padStart(3, '0')}`);
|
|
388
|
+
fs.mkdirSync(dir, { recursive: true });
|
|
389
|
+
const report = path.join(dir, 'report.json');
|
|
390
|
+
const logFile = path.join(dir, 'output.log');
|
|
391
|
+
const tmp = path.join(dir, 'tmp');
|
|
392
|
+
fs.mkdirSync(tmp, { recursive: true });
|
|
393
|
+
const command = adapter.batchCommand(ctx, batch.files, report);
|
|
394
|
+
const t0 = Date.now();
|
|
395
|
+
const proc = await spawnBatch({ ...command, env: { ...command.env, UKIT_TEST_TMPDIR: tmp }, cwd: ctx.root, logFile, timeoutMs: ctx.timeoutMs, wrap });
|
|
396
|
+
const durationMs = Date.now() - t0;
|
|
397
|
+
const log = (() => { try { return fs.readFileSync(logFile, 'utf8'); } catch { return ''; } })();
|
|
398
|
+
const parsed = adapter.parseReport(ctx, batch.files, report);
|
|
399
|
+
const timeRss = wrap ? parseTimeRss(log) : null;
|
|
400
|
+
const perTestPeak = Math.max(0, ...(parsed.results ?? []).map((r) => r.peakRss ?? 0));
|
|
401
|
+
const pluginRss = parsed.session?.peakRss ?? (perTestPeak > 0 ? perTestPeak : null);
|
|
402
|
+
const peakRss = timeRss ?? pluginRss ?? null;
|
|
403
|
+
// Complete = the framework process ended on its own AND wrote a parsable report. Ordinary test failures keep
|
|
404
|
+
// complete=true (non-zero exit); crashes, signals, timeouts and missing reports do not.
|
|
405
|
+
// pytest: 1 = test failures, 5 = nothing collected; 2/3/4 = interrupted/internal/usage error → incomplete.
|
|
406
|
+
const crashed = Boolean(proc.error || proc.timedOut || proc.signal || !parsed.ok
|
|
407
|
+
|| proc.code == null || (proc.code > 1 && proc.code !== 5));
|
|
408
|
+
const result = {
|
|
409
|
+
index: batch.index, files: batch.files, cases: batch.cases, durationMs, exitCode: proc.code, signal: proc.signal,
|
|
410
|
+
timedOut: proc.timedOut, complete: !crashed, error: crashed ? (proc.error ?? (proc.timedOut ? `timed out after ${ctx.timeoutMs} ms` : parsed.error ?? `exit ${proc.code}${proc.signal ? ` signal ${proc.signal}` : ''}`)) : null,
|
|
411
|
+
peakRss, peakRssSource: timeRss != null ? `${wrap.cmd} ${wrap.flag}` : pluginRss != null ? 'in-process ru_maxrss' : 'unknown',
|
|
412
|
+
results: parsed.results ?? [], fileErrors: parsed.fileErrors ?? [],
|
|
413
|
+
liveThreads: parsed.session?.liveThreads ?? [], liveChildren: parsed.session?.liveChildren ?? [],
|
|
414
|
+
logTail: crashed ? log.slice(-2000) : undefined,
|
|
415
|
+
};
|
|
416
|
+
const reportedFiles = new Set(result.results.map((r) => r.file));
|
|
417
|
+
result.msPerTest = result.results.length ? Math.round(durationMs / result.results.length) : null;
|
|
418
|
+
result.silentFiles = batch.files.filter((f) => !reportedFiles.has(f) && !result.fileErrors.some((e) => e.file === f));
|
|
419
|
+
fs.writeFileSync(path.join(dir, 'batch.json'), `${JSON.stringify({ ...result, results: undefined, resultCount: result.results.length }, null, 2)}\n`);
|
|
420
|
+
fs.rmSync(tmp, { recursive: true, force: true });
|
|
421
|
+
return result;
|
|
422
|
+
}
|
|
423
|
+
|
|
424
|
+
// Cross-batch drift: if each later batch starts from a higher floor something outlives its process (e.g. a
|
|
425
|
+
// shared server, cache file, or tmpfs) — per-batch processes should return to a flat baseline.
|
|
426
|
+
export function rssGrowthWarnings(batchResults) {
|
|
427
|
+
const mbs = batchResults.map((b) => toMb(b.peakRss)).filter((v) => v != null);
|
|
428
|
+
if (mbs.length < 4) return [];
|
|
429
|
+
const half = Math.floor(mbs.length / 2);
|
|
430
|
+
const avg = (a) => a.reduce((s, v) => s + v, 0) / a.length;
|
|
431
|
+
const first = avg(mbs.slice(0, half));
|
|
432
|
+
const second = avg(mbs.slice(half));
|
|
433
|
+
return second > first * 1.5 && second - first > 200
|
|
434
|
+
? [{ kind: 'batch-peak-drift', detail: `later batches average ${Math.round(second)} MB vs ${Math.round(first)} MB earlier — check per-test leaks or growing shared fixtures` }]
|
|
435
|
+
: [];
|
|
436
|
+
}
|
|
437
|
+
|
|
438
|
+
// ---------------------------------------------------------------- context / history / retention
|
|
439
|
+
|
|
440
|
+
const stamp = () => new Date().toISOString().replace(/[:.]/g, '-');
|
|
441
|
+
|
|
442
|
+
export function makeContext(args, root) {
|
|
443
|
+
const config = loadProjectConfig(root);
|
|
444
|
+
const storage = path.join(root, '.ukit', 'storage', 'test-batches');
|
|
445
|
+
const runId = `${args.mode}-${stamp()}`;
|
|
446
|
+
const runDir = path.resolve(root, args.out ?? path.join(storage, 'runs', runId));
|
|
447
|
+
fs.mkdirSync(runDir, { recursive: true });
|
|
448
|
+
const pluginDir = path.join(runDir, '.plugin');
|
|
449
|
+
fs.mkdirSync(pluginDir, { recursive: true });
|
|
450
|
+
fs.writeFileSync(path.join(pluginDir, 'ukit_batch_report.py'), PYTEST_PLUGIN);
|
|
451
|
+
const samplerFile = path.join(pluginDir, 'ukit-leak-sampler.cjs');
|
|
452
|
+
fs.writeFileSync(samplerFile, NODE_SAMPLER);
|
|
453
|
+
return {
|
|
454
|
+
root, config, storage, runId, runDir, pluginDir, samplerFile, tmpDir: pluginDir,
|
|
455
|
+
extra: args.extra.length ? args.extra : (config.extraArgs ?? []),
|
|
456
|
+
python: args.python ?? config.python ?? null,
|
|
457
|
+
resourceWarnings: args.resourceWarnings || config.resourceWarnings === true,
|
|
458
|
+
timeoutMs: args.timeoutMs ?? config.timeoutMs ?? DEFAULTS.timeoutMs,
|
|
459
|
+
heavyMb: args.heavyMb ?? config.heavyMb ?? DEFAULTS.heavyMb,
|
|
460
|
+
batchFiles: args.batchFiles ?? config.batchFiles ?? DEFAULTS.batchFiles,
|
|
461
|
+
batchCases: args.batchCases ?? config.batchCases ?? null,
|
|
462
|
+
criticalFreeMb: args.criticalFreeMb ?? config.criticalFreeMb ?? DEFAULTS.criticalFreeMb,
|
|
463
|
+
guard: [...(config.guard ?? []), ...args.guard],
|
|
464
|
+
};
|
|
465
|
+
}
|
|
466
|
+
|
|
467
|
+
export function readHistory(storage) {
|
|
468
|
+
try { const h = JSON.parse(fs.readFileSync(path.join(storage, 'history.json'), 'utf8')); return h && typeof h === 'object' ? h : { files: {} }; } catch { return { files: {} }; }
|
|
469
|
+
}
|
|
470
|
+
|
|
471
|
+
// Only single-file batches give per-file truth; multi-file peaks stay batch-level evidence.
|
|
472
|
+
export function updateHistory(history, batchResults) {
|
|
473
|
+
const next = { files: { ...(history.files ?? {}) }, updatedAt: new Date().toISOString() };
|
|
474
|
+
for (const b of batchResults) {
|
|
475
|
+
if (b.complete && b.files.length === 1 && b.peakRss != null) {
|
|
476
|
+
next.files[b.files[0]] = { isolatedPeakMb: toMb(b.peakRss), ms: b.durationMs, at: next.updatedAt };
|
|
477
|
+
}
|
|
478
|
+
}
|
|
479
|
+
return next;
|
|
480
|
+
}
|
|
481
|
+
|
|
482
|
+
// Keep the newest N run directories; superseded runs (and their logs) are removed so nothing piles up.
|
|
483
|
+
export function pruneRuns(storage, keep = DEFAULTS.keepRuns) {
|
|
484
|
+
const runs = path.join(storage, 'runs');
|
|
485
|
+
let names = [];
|
|
486
|
+
const RUN_RE = /^(?:run|plan|audit|detect-leaks)-(.+)$/;
|
|
487
|
+
const when = (n) => n.match(RUN_RE)[1];
|
|
488
|
+
try { names = fs.readdirSync(runs).filter((n) => RUN_RE.test(n)).sort((a, b) => (when(a) < when(b) ? -1 : when(a) > when(b) ? 1 : 0)); } catch { return []; }
|
|
489
|
+
const drop = names.slice(0, Math.max(0, names.length - keep));
|
|
490
|
+
for (const n of drop) fs.rmSync(path.join(runs, n), { recursive: true, force: true });
|
|
491
|
+
return drop;
|
|
492
|
+
}
|
|
493
|
+
|
|
494
|
+
function discoverAndPlan(ctx, adapter) {
|
|
495
|
+
const discovered = adapter.discover(ctx);
|
|
496
|
+
if (!discovered.ok) return { error: discovered.error, version: discovered.version };
|
|
497
|
+
const inventory = buildInventory(discovered);
|
|
498
|
+
const history = readHistory(ctx.storage);
|
|
499
|
+
const batches = planBatches(inventory, { batchFiles: ctx.batchFiles, batchCases: ctx.batchCases, budgets: resolveBudgets(ctx.config), config: ctx.config, history });
|
|
500
|
+
const check = verifyPlan(inventory, batches);
|
|
501
|
+
if (check.missing.length || check.duplicated.length || check.extra.length) return { error: `internal plan invariant broken: ${JSON.stringify(check)}`, version: discovered.version };
|
|
502
|
+
return { inventory, batches, version: discovered.version };
|
|
503
|
+
}
|
|
504
|
+
|
|
505
|
+
// ---------------------------------------------------------------- modes
|
|
506
|
+
|
|
507
|
+
export async function runSuite(ctx, framework, deps = {}) {
|
|
508
|
+
const adapter = ADAPTERS[framework];
|
|
509
|
+
const startedAt = new Date().toISOString();
|
|
510
|
+
const planned = discoverAndPlan(ctx, adapter);
|
|
511
|
+
if (planned.error) {
|
|
512
|
+
return { runId: ctx.runId, framework, frameworkVersion: planned.version ?? null, verdict: 'INCOMPLETE', exitCode: EXIT.INCOMPLETE, error: planned.error };
|
|
513
|
+
}
|
|
514
|
+
const { inventory, batches } = planned;
|
|
515
|
+
fs.writeFileSync(path.join(ctx.runDir, 'manifest.json'), `${JSON.stringify({ framework, files: inventory.files, ids: inventory.ids, batches: batches.map((b) => ({ index: b.index, files: b.files, cases: b.cases })) }, null, 2)}\n`);
|
|
516
|
+
const guard = ctx.guard.length ? createGuard(ctx.root, ctx.guard) : null;
|
|
517
|
+
const batchResults = [];
|
|
518
|
+
let deferred = null;
|
|
519
|
+
for (const batch of batches) {
|
|
520
|
+
const mem = await (deps.waitForMemory ?? waitForMemory)({ criticalFreeMb: ctx.criticalFreeMb, maxWaitMs: DEFAULTS.lowMemWaitMs, pollMs: DEFAULTS.lowMemPollMs });
|
|
521
|
+
if (!mem.ok) { deferred = `available memory ${mem.freeMb} MB stayed below ${ctx.criticalFreeMb} MB for ${Math.round(mem.waitedMs / 1000)} s before batch ${batch.index}; remaining batches not started — free memory and rerun`; break; }
|
|
522
|
+
const result = await (deps.runBatch ?? runBatch)(ctx, adapter, batch);
|
|
523
|
+
batchResults.push(result);
|
|
524
|
+
deps.onBatch?.(result, batches.length);
|
|
525
|
+
}
|
|
526
|
+
const summary = aggregate({ inventory, batches, batchResults, guard: guard ? guard.check() : null, heavyMb: ctx.heavyMb, framework, version: planned.version, runId: ctx.runId, startedAt, finishedAt: new Date().toISOString(), deferred });
|
|
527
|
+
if (guard?.truncated) summary.warnings.push({ kind: 'guard-truncated', detail: `guard snapshot capped at ${DEFAULTS.guardMaxEntries} entries — narrow --guard paths` });
|
|
528
|
+
summary.batchDetail = batchResults.map((b) => ({ index: b.index, files: b.files.length, cases: b.cases, reported: b.results.length, durationMs: b.durationMs, msPerTest: b.msPerTest, peakMb: toMb(b.peakRss), peakSource: b.peakRssSource, complete: b.complete, error: b.error, silentFiles: b.silentFiles }));
|
|
529
|
+
fs.mkdirSync(ctx.storage, { recursive: true });
|
|
530
|
+
fs.writeFileSync(path.join(ctx.storage, 'history.json'), `${JSON.stringify(updateHistory(readHistory(ctx.storage), batchResults), null, 2)}\n`);
|
|
531
|
+
fs.writeFileSync(path.join(ctx.runDir, 'summary.json'), `${JSON.stringify(summary, null, 2)}\n`);
|
|
532
|
+
return summary;
|
|
533
|
+
}
|
|
534
|
+
|
|
535
|
+
export function planOnly(ctx, framework) {
|
|
536
|
+
const planned = discoverAndPlan(ctx, ADAPTERS[framework]);
|
|
537
|
+
if (planned.error) return { framework, verdict: 'INCOMPLETE', exitCode: EXIT.INCOMPLETE, error: planned.error };
|
|
538
|
+
const { inventory, batches } = planned;
|
|
539
|
+
return {
|
|
540
|
+
framework, frameworkVersion: planned.version, files: inventory.files.length, tests: inventory.ids?.length ?? null,
|
|
541
|
+
duplicateIds: inventory.duplicates, batches: batches.map((b) => ({ index: b.index, files: b.files, cases: b.cases, categories: b.categories, solo: b.solo })),
|
|
542
|
+
verdict: 'PLANNED', exitCode: EXIT.PASS,
|
|
543
|
+
};
|
|
544
|
+
}
|
|
545
|
+
|
|
546
|
+
// ---------------------------------------------------------------- audit (design-time guard)
|
|
547
|
+
|
|
548
|
+
const PY_APP_RE = /\b(Sanic|FastAPI|Flask|Starlette|Quart|Litestar|web\.Application|create_app|make_app|build_app|get_application)\s*\(/;
|
|
549
|
+
const JS_APP_RE = /\b(express\(\)|new\s+Koa\(|Fastify\(|fastify\(|createServer\(|\.listen\(|NestFactory\.create|createApp\(|buildApp\(|chromium\.launch|puppeteer\.launch|new\s+(?:Pool|Client|PrismaClient)\()/;
|
|
550
|
+
const PY_TEARDOWN_RE = /\b(\.close\(|\.cancel\(|\.stop\(|\.shutdown\(|\.dispose\(|\.aclose\(|_app_registry|gc\.collect|\.clear\(\)|unregister|reset_)/;
|
|
551
|
+
const JS_TEARDOWN_RE = /\b(after(Each|All)|onTestFinished)\s*\(/;
|
|
552
|
+
const REAL_PATH_RE = /(['"`])(\/Users\/[^'"`]+|\/home\/[^'"`]+|[A-Za-z]:\\\\[^'"`]+|~\/[^'"`]+)\1/;
|
|
553
|
+
const UNIQUE_APP_NAME_RE = /\b(Sanic|FastAPI|Flask)\s*\(\s*(f['"][^'"]*\{|.*uuid|.*random|.*time\(\)|.*next\()/;
|
|
554
|
+
|
|
555
|
+
// Static estimate when the framework cannot list cases: decorators/params are expanded only when the
|
|
556
|
+
// parametrize list is a literal on one line; otherwise the base count is reported with `estimated: true`.
|
|
557
|
+
export function estimateCases(text, lang) {
|
|
558
|
+
if (lang === 'py') {
|
|
559
|
+
let n = 0;
|
|
560
|
+
const lines = text.split('\n');
|
|
561
|
+
for (let i = 0; i < lines.length; i += 1) {
|
|
562
|
+
if (!/^\s*(async\s+)?def\s+test_/.test(lines[i])) continue;
|
|
563
|
+
let mult = 1;
|
|
564
|
+
for (let j = i - 1; j >= 0 && /^\s*@/.test(lines[j]); j -= 1) {
|
|
565
|
+
const m = lines[j].match(/parametrize\([^,]+,\s*\[(.*)\]\s*\)/);
|
|
566
|
+
if (m) mult *= Math.max(1, splitTopLevel(m[1]).length);
|
|
567
|
+
}
|
|
568
|
+
n += mult;
|
|
569
|
+
}
|
|
570
|
+
return n;
|
|
571
|
+
}
|
|
572
|
+
return (text.match(/^\s*(it|test)(\.each\([^)]*\))?\s*\(/gm) ?? []).length
|
|
573
|
+
+ (text.match(/^\s*(it|test)\.(skip|only|todo|concurrent)\s*\(/gm) ?? []).length;
|
|
574
|
+
}
|
|
575
|
+
|
|
576
|
+
function splitTopLevel(s) {
|
|
577
|
+
const out = []; let depth = 0; let cur = ''; let quote = null;
|
|
578
|
+
for (const ch of s) {
|
|
579
|
+
if (quote) { cur += ch; if (ch === quote) quote = null; continue; }
|
|
580
|
+
if (ch === '"' || ch === "'") { quote = ch; cur += ch; continue; }
|
|
581
|
+
if ('([{'.includes(ch)) depth += 1;
|
|
582
|
+
if (')]}'.includes(ch)) depth -= 1;
|
|
583
|
+
if (ch === ',' && depth === 0) { if (cur.trim()) out.push(cur); cur = ''; continue; }
|
|
584
|
+
cur += ch;
|
|
585
|
+
}
|
|
586
|
+
if (cur.trim()) out.push(cur);
|
|
587
|
+
return out;
|
|
588
|
+
}
|
|
589
|
+
|
|
590
|
+
// Python fixture bodies: decorator @pytest.fixture / @pytest_asyncio.fixture followed by def ... up to next top-level def.
|
|
591
|
+
export function pythonFixtures(text) {
|
|
592
|
+
const lines = text.split('\n');
|
|
593
|
+
const out = [];
|
|
594
|
+
for (let i = 0; i < lines.length; i += 1) {
|
|
595
|
+
if (!/^\s*@(pytest|pytest_asyncio)\.fixture/.test(lines[i])) continue;
|
|
596
|
+
let j = i + 1;
|
|
597
|
+
while (j < lines.length && /^\s*@/.test(lines[j])) j += 1;
|
|
598
|
+
const header = lines[j] ?? '';
|
|
599
|
+
const indent = header.match(/^(\s*)/)[1].length;
|
|
600
|
+
const m = header.match(/def\s+(\w+)/);
|
|
601
|
+
let k = j + 1;
|
|
602
|
+
while (k < lines.length && (lines[k].trim() === '' || lines[k].match(/^(\s*)/)[1].length > indent)) k += 1;
|
|
603
|
+
out.push({ name: m?.[1] ?? '?', line: j + 1, scope: (lines.slice(i, j).join(' ').match(/scope\s*=\s*['"](\w+)['"]/) ?? [])[1] ?? 'function', body: lines.slice(j + 1, k).join('\n') });
|
|
604
|
+
i = k - 1;
|
|
605
|
+
}
|
|
606
|
+
return out;
|
|
607
|
+
}
|
|
608
|
+
|
|
609
|
+
export function auditFileText(file, text, { cases = null, budgets = BUDGETS, config = {} } = {}) {
|
|
610
|
+
const lang = /\.py$/.test(file) ? 'py' : 'js';
|
|
611
|
+
const category = classifyFile(file, config);
|
|
612
|
+
const findings = [];
|
|
613
|
+
const add = (code, line, message) => findings.push({ file, line, code, message });
|
|
614
|
+
const count = cases ?? estimateCases(text, lang);
|
|
615
|
+
const budget = budgets[category]?.file ?? BUDGETS.unit.file;
|
|
616
|
+
if (count > budget) add('file-over-budget', 1, `${count} cases${cases == null ? ' (static estimate)' : ''} > ${category} budget ${budget}/file — split by feature/behaviour into focused files that keep fixture lifecycles intact`);
|
|
617
|
+
if (lang === 'py') {
|
|
618
|
+
for (const fx of pythonFixtures(text)) {
|
|
619
|
+
if (!PY_APP_RE.test(fx.body)) continue;
|
|
620
|
+
const yieldIdx = fx.body.search(/^\s*yield\b/m);
|
|
621
|
+
if (yieldIdx === -1) add('fixture-no-teardown', fx.line, `fixture "${fx.name}" builds an app/client and returns it — use yield + teardown (cancel background tasks, close clients/DB, unregister from global app registries, clear doc/OpenAPI singletons)`);
|
|
622
|
+
else if (!PY_TEARDOWN_RE.test(fx.body.slice(yieldIdx))) add('fixture-weak-teardown', fx.line, `fixture "${fx.name}" yields an app but nothing after yield closes, cancels, unregisters or clears it`);
|
|
623
|
+
if (fx.scope === 'function' && /(Sanic|FastAPI|Flask|create_app|make_app|build_app)\s*\(/.test(fx.body)) add('app-per-test', fx.line, `fixture "${fx.name}" builds a full app for every test — prefer scope="module" when tests do not mutate app state, otherwise make teardown complete`);
|
|
624
|
+
if (UNIQUE_APP_NAME_RE.test(fx.body)) add('registry-growth', fx.line, `fixture "${fx.name}" gives each app a unique name — framework app registries keep every instance alive; unregister it in teardown`);
|
|
625
|
+
}
|
|
626
|
+
} else if (JS_APP_RE.test(text) && /\bbefore(Each|All)\s*\(/.test(text) && !JS_TEARDOWN_RE.test(text)) {
|
|
627
|
+
const line = text.split('\n').findIndex((l) => /\bbefore(Each|All)\s*\(/.test(l)) + 1;
|
|
628
|
+
add('fixture-no-teardown', line, 'setup creates a server/app/browser/DB client but the file has no afterEach/afterAll — close it, stop timers, and await shutdown');
|
|
629
|
+
}
|
|
630
|
+
text.split('\n').forEach((l, i) => {
|
|
631
|
+
if (REAL_PATH_RE.test(l)) add('real-path', i + 1, 'hardcoded absolute user path in a test — use a temp dir (tmp_path / mkdtemp) wired through app config');
|
|
632
|
+
if (lang === 'js' && /\bsetInterval\s*\(/.test(l) && !/clearInterval|\.unref\(\)/.test(text)) add('timer-leak', i + 1, 'setInterval without clearInterval/unref keeps the worker alive and retains closures');
|
|
633
|
+
});
|
|
634
|
+
return { file, category, cases: count, casesSource: cases == null ? 'static-estimate' : 'collected', findings };
|
|
635
|
+
}
|
|
636
|
+
|
|
637
|
+
const SCAN_SKIP = new Set(['node_modules', '.git', '.venv', 'venv', '.tox', '.ukit', '.claude', '.codex', '.omp', 'dist', 'build', '__pycache__', '.cache', 'coverage']);
|
|
638
|
+
const TEST_FILE_RE = /(^|\/)(test_[^/]*\.py|[^/]*_test\.py|conftest\.py|[^/]*\.(test|spec)\.[cm]?[jt]sx?)$/;
|
|
639
|
+
|
|
640
|
+
export function scanTestFiles(root, limit = 20_000) {
|
|
641
|
+
const out = [];
|
|
642
|
+
const walk = (dir, relp) => {
|
|
643
|
+
if (out.length >= limit) return;
|
|
644
|
+
let entries = []; try { entries = fs.readdirSync(dir, { withFileTypes: true }); } catch { return; }
|
|
645
|
+
for (const e of entries) {
|
|
646
|
+
if (e.isSymbolicLink()) continue;
|
|
647
|
+
const r = relp ? `${relp}/${e.name}` : e.name;
|
|
648
|
+
if (e.isDirectory()) { if (!SCAN_SKIP.has(e.name)) walk(path.join(dir, e.name), r); } else if (TEST_FILE_RE.test(r)) out.push(r);
|
|
649
|
+
}
|
|
650
|
+
};
|
|
651
|
+
walk(root, '');
|
|
652
|
+
return out;
|
|
653
|
+
}
|
|
654
|
+
|
|
655
|
+
function gitLines(root, args) {
|
|
656
|
+
const res = spawnSync('git', args, { cwd: root, encoding: 'utf8', timeout: 15000 });
|
|
657
|
+
return res.status === 0 ? res.stdout.split('\n').map((s) => s.trim()).filter(Boolean) : null;
|
|
658
|
+
}
|
|
659
|
+
|
|
660
|
+
// New = untracked or added since HEAD. Only new files are enforced; existing files are legacy (reported).
|
|
661
|
+
export function gitTestFileStatus(root) {
|
|
662
|
+
const added = gitLines(root, ['diff', '--name-only', '--diff-filter=A', 'HEAD']) ?? [];
|
|
663
|
+
const untracked = gitLines(root, ['ls-files', '--others', '--exclude-standard']) ?? [];
|
|
664
|
+
const modified = gitLines(root, ['diff', '--name-only', '--diff-filter=M', 'HEAD']) ?? [];
|
|
665
|
+
return { isGit: gitLines(root, ['rev-parse', '--is-inside-work-tree']) != null, newFiles: new Set([...added, ...untracked].map(toPosix)), modified: new Set(modified.map(toPosix)) };
|
|
666
|
+
}
|
|
667
|
+
|
|
668
|
+
export function auditProject(ctx, framework, { changedOnly = false, discover = true } = {}) {
|
|
669
|
+
let files = [];
|
|
670
|
+
let caseCounts = new Map();
|
|
671
|
+
if (framework && discover) {
|
|
672
|
+
const d = ADAPTERS[framework].discover(ctx);
|
|
673
|
+
if (d.ok) { const inv = buildInventory(d); files = inv.files; caseCounts = inv.caseCounts; }
|
|
674
|
+
else return { verdict: 'INCOMPLETE', exitCode: EXIT.INCOMPLETE, error: d.error };
|
|
675
|
+
}
|
|
676
|
+
// Fixtures live in conftest.py / setup files that collect zero tests — scan the tree for them too.
|
|
677
|
+
files = [...new Set([...files, ...scanTestFiles(ctx.root)])].sort();
|
|
678
|
+
const git = gitTestFileStatus(ctx.root);
|
|
679
|
+
if (changedOnly) files = files.filter((f) => git.newFiles.has(f) || git.modified.has(f));
|
|
680
|
+
const budgets = resolveBudgets(ctx.config);
|
|
681
|
+
const exceptions = ctx.config.exceptions ?? {};
|
|
682
|
+
const reports = [];
|
|
683
|
+
for (const f of files) {
|
|
684
|
+
let text = ''; try { text = fs.readFileSync(path.join(ctx.root, f), 'utf8'); } catch { continue; }
|
|
685
|
+
const r = auditFileText(f, text, { cases: caseCounts.get(f) ?? null, budgets, config: ctx.config });
|
|
686
|
+
r.status = git.newFiles.has(f) ? 'new' : 'legacy';
|
|
687
|
+
r.exception = exceptions[f] ?? null;
|
|
688
|
+
reports.push(r);
|
|
689
|
+
}
|
|
690
|
+
const errors = reports.filter((r) => r.status === 'new' && !r.exception).flatMap((r) => r.findings);
|
|
691
|
+
const warnings = reports.filter((r) => r.status !== 'new' || r.exception).flatMap((r) => r.findings);
|
|
692
|
+
return {
|
|
693
|
+
framework, files: reports.length, errors, warnings,
|
|
694
|
+
budgets, verdict: errors.length ? 'FAIL' : 'PASS', exitCode: errors.length ? EXIT.FAIL : EXIT.PASS,
|
|
695
|
+
note: 'Only NEW test files are enforced; legacy findings are reported for refactoring and never deleted, rewritten or excluded. Exceptions: .ukit/storage/test-batches.json → exceptions { "<file>": "<reason>" }.',
|
|
696
|
+
};
|
|
697
|
+
}
|
|
698
|
+
|
|
699
|
+
// ---------------------------------------------------------------- detect-leaks
|
|
700
|
+
|
|
701
|
+
// Evidence, not root cause. pytest: the file runs twice in ONE process; per-test RSS after teardown is sampled.
|
|
702
|
+
// Pass 1 includes import/cache warm-up; growth that continues through pass 2 is the retention signal.
|
|
703
|
+
// JS: every process samples RSS/heap every 250 ms; a monotonic rise of the worker across the file is the signal.
|
|
704
|
+
export function analyzeSeries(values) {
|
|
705
|
+
const v = values.filter((x) => Number.isFinite(x));
|
|
706
|
+
if (v.length < 4) return { samples: v.length, verdict: 'insufficient-samples' };
|
|
707
|
+
const half = Math.floor(v.length / 2);
|
|
708
|
+
const pass1 = v.slice(0, half);
|
|
709
|
+
const pass2 = v.slice(half);
|
|
710
|
+
const growth1 = pass1[pass1.length - 1] - pass1[0];
|
|
711
|
+
const growth2 = pass2[pass2.length - 1] - pass2[0];
|
|
712
|
+
const perStep2 = growth2 / Math.max(1, pass2.length - 1);
|
|
713
|
+
const mb = (b) => Math.round((b / 1048576) * 10) / 10;
|
|
714
|
+
const verdict = growth2 > 32 * 1048576 && growth2 >= growth1 * 0.5 ? 'likely-unbounded-growth'
|
|
715
|
+
: growth2 > 8 * 1048576 ? 'some-growth-recheck'
|
|
716
|
+
: 'warmup-then-flat';
|
|
717
|
+
return { samples: v.length, pass1GrowthMb: mb(growth1), pass2GrowthMb: mb(growth2), perTestPass2Mb: mb(perStep2), verdict };
|
|
718
|
+
}
|
|
719
|
+
|
|
720
|
+
export async function detectLeaks(ctx, framework, file) {
|
|
721
|
+
const adapter = ADAPTERS[framework];
|
|
722
|
+
if (!adapter.leakCommand) return { verdict: 'UNSUPPORTED', exitCode: EXIT.INCOMPLETE, error: `${framework} adapter has no leak sampler` };
|
|
723
|
+
const samples = path.join(ctx.runDir, 'leak-samples.jsonl');
|
|
724
|
+
const cmd = adapter.leakCommand(ctx, toPosix(file), samples);
|
|
725
|
+
const logFile = path.join(ctx.runDir, 'leak.log');
|
|
726
|
+
const proc = await spawnBatch({ ...cmd, cwd: ctx.root, logFile, timeoutMs: ctx.timeoutMs, wrap: null });
|
|
727
|
+
let rows = [];
|
|
728
|
+
try { rows = fs.readFileSync(samples, 'utf8').split('\n').filter(Boolean).map((l) => JSON.parse(l)); } catch { rows = []; }
|
|
729
|
+
let series; let resourceFindings = [];
|
|
730
|
+
if (framework === 'pytest') {
|
|
731
|
+
const tests = rows.filter((r) => !r.session);
|
|
732
|
+
series = analyzeSeries(tests.map((r) => r.rss ?? r.peakRss));
|
|
733
|
+
series.rssSource = tests.some((r) => r.rss != null) ? 'current RSS (/proc)' : 'peak RSS (ru_maxrss; monotonic by definition — growth shows only new highs)';
|
|
734
|
+
const session = rows.find((r) => r.session);
|
|
735
|
+
if (session?.liveThreads?.length) resourceFindings.push(`threads alive at session end: ${session.liveThreads.join(', ')}`);
|
|
736
|
+
if (session?.liveChildren?.length) resourceFindings.push(`child processes alive at session end: ${session.liveChildren.join(', ')}`);
|
|
737
|
+
} else {
|
|
738
|
+
const byPid = new Map();
|
|
739
|
+
for (const r of rows) { if (!byPid.has(r.pid)) byPid.set(r.pid, []); byPid.get(r.pid).push(r.heap); }
|
|
740
|
+
const worker = [...byPid.entries()].sort((a, b) => b[1].length - a[1].length)[0];
|
|
741
|
+
series = { ...analyzeSeries(worker?.[1] ?? []), rssSource: 'heapUsed of the busiest process (sampled every 250 ms)' };
|
|
742
|
+
}
|
|
743
|
+
const log = (() => { try { return fs.readFileSync(logFile, 'utf8'); } catch { return ''; } })();
|
|
744
|
+
const rw = (log.match(/ResourceWarning: [^\n]+/g) ?? []).slice(0, 10);
|
|
745
|
+
if (rw.length) resourceFindings.push(...rw);
|
|
746
|
+
if (/MaxListenersExceededWarning|open handle|did not exit one second/i.test(log)) resourceFindings.push('open handles reported by the runner (see leak.log)');
|
|
747
|
+
return {
|
|
748
|
+
file: toPosix(file), framework, exitCode: proc.error || proc.timedOut ? EXIT.INCOMPLETE : EXIT.PASS,
|
|
749
|
+
verdict: proc.error || proc.timedOut ? 'INCOMPLETE' : series.verdict, series, resourceFindings,
|
|
750
|
+
note: 'Peak RSS of two fresh processes does not prove a leak; this mode samples inside one process. Confirm a suspect with tracemalloc / --expose-gc heap snapshots before changing code.',
|
|
751
|
+
log: toPosix(path.relative(ctx.root, logFile)),
|
|
752
|
+
};
|
|
753
|
+
}
|
|
754
|
+
|
|
755
|
+
// ---------------------------------------------------------------- rendering / main
|
|
756
|
+
|
|
757
|
+
export function renderSummary(s) {
|
|
758
|
+
if (s.error) return `[test-batches] ${s.verdict}: ${s.error}`;
|
|
759
|
+
const L = [];
|
|
760
|
+
if (s.verdict === 'PLANNED') {
|
|
761
|
+
L.push(`[test-batches] plan ${s.framework}${s.frameworkVersion ? ` ${s.frameworkVersion}` : ''}: ${s.files} files, ${s.tests ?? 'unknown (file-level)'} tests, ${s.batches.length} batches`);
|
|
762
|
+
for (const b of s.batches) L.push(` #${b.index} ${b.files.length} files${b.cases != null ? `, ${b.cases} cases` : ''}${b.solo ? ' (solo)' : ''} [${b.categories.join(',')}]`);
|
|
763
|
+
return L.join('\n');
|
|
764
|
+
}
|
|
765
|
+
if ('series' in s) {
|
|
766
|
+
L.push(`[test-batches] detect-leaks ${s.file}: ${s.verdict}`);
|
|
767
|
+
L.push(` ${JSON.stringify(s.series)}`);
|
|
768
|
+
for (const f of s.resourceFindings) L.push(` resource: ${f}`);
|
|
769
|
+
L.push(` ${s.note}`);
|
|
770
|
+
return L.join('\n');
|
|
771
|
+
}
|
|
772
|
+
if ('errors' in s) {
|
|
773
|
+
L.push(`[test-batches] audit: ${s.verdict} — ${s.files} files, ${s.errors.length} enforced (new files), ${s.warnings.length} advisory (legacy/excepted)`);
|
|
774
|
+
for (const f of s.errors) L.push(` ERROR ${f.file}:${f.line} ${f.code} — ${f.message}`);
|
|
775
|
+
for (const f of s.warnings.slice(0, 40)) L.push(` warn ${f.file}:${f.line} ${f.code} — ${f.message}`);
|
|
776
|
+
if (s.warnings.length > 40) L.push(` … ${s.warnings.length - 40} more advisory findings (--json for all)`);
|
|
777
|
+
return L.join('\n');
|
|
778
|
+
}
|
|
779
|
+
const c = s.counts;
|
|
780
|
+
L.push(`[test-batches] ${s.verdict} ${s.framework}${s.frameworkVersion ? ` ${s.frameworkVersion}` : ''} run ${s.runId}`);
|
|
781
|
+
L.push(` inventory: ${s.inventory.files} files, ${s.inventory.tests ?? 'n/a (file-level)'} tests · reported ${s.reported}`);
|
|
782
|
+
L.push(` batches: ${s.batches.completed}/${s.batches.planned} complete${s.batches.incomplete.length ? ` · incomplete #${s.batches.incomplete.join(',#')}` : ''}${s.batches.notRun.length ? ` · not run #${s.batches.notRun.join(',#')}` : ''}`);
|
|
783
|
+
L.push(` passed ${c.passed} · failed ${c.failed} · error ${c.error} · skipped ${c.skipped} · xfailed ${c.xfailed} · xpassed ${c.xpassed}`);
|
|
784
|
+
L.push(` duration ${Math.round((s.durationMs ?? 0) / 1000)} s · max batch RSS ${s.maxBatchRssMb ?? 'unknown'} MB`);
|
|
785
|
+
for (const id of s.failedIds.slice(0, 50)) L.push(` FAILED ${id}`);
|
|
786
|
+
if (s.failedIds.length > 50) L.push(` … ${s.failedIds.length - 50} more failed ids in summary.json`);
|
|
787
|
+
for (const e of s.fileErrors) L.push(` FILE ERROR ${e.file}: ${e.message.split('\n')[0]}`);
|
|
788
|
+
for (const id of s.unaccounted.slice(0, 20)) L.push(` UNACCOUNTED ${id}`);
|
|
789
|
+
for (const f of s.unaccountedFiles) L.push(` UNACCOUNTED FILE ${f}`);
|
|
790
|
+
for (const h of s.heavy) L.push(` HEAVY ${h.file ?? `batch #${h.batch} (${h.files.length} files)`}: ${h.peakMb} MB (${h.attribution})`);
|
|
791
|
+
for (const w of s.warnings) L.push(` warn ${w.kind}: ${Array.isArray(w.detail) ? w.detail.join(', ') : w.detail}`);
|
|
792
|
+
for (const ch of s.isolation?.changed ?? []) L.push(` UNSAFE guarded path changed: ${ch}`);
|
|
793
|
+
if (s.deferred) L.push(` DEFERRED: ${s.deferred}`);
|
|
794
|
+
return L.join('\n');
|
|
795
|
+
}
|
|
796
|
+
|
|
797
|
+
export async function main(argv = process.argv.slice(2), cwd = process.cwd()) {
|
|
798
|
+
const args = parseArgs(argv);
|
|
799
|
+
if (args.errors.length) { process.stderr.write(`${args.errors.join('\n')}\n`); return EXIT.INCOMPLETE; }
|
|
800
|
+
const root = path.resolve(args.root ?? cwd);
|
|
801
|
+
const ctx = makeContext(args, root);
|
|
802
|
+
const chosen = chooseFramework(root, args.framework ?? ctx.config.framework);
|
|
803
|
+
let result;
|
|
804
|
+
try {
|
|
805
|
+
if (chosen.error && !(args.mode === 'audit')) result = { verdict: 'INCOMPLETE', exitCode: EXIT.INCOMPLETE, error: chosen.error };
|
|
806
|
+
else if (args.mode === 'plan') result = planOnly(ctx, chosen.framework);
|
|
807
|
+
else if (args.mode === 'audit') result = auditProject(ctx, chosen.framework ?? null, { changedOnly: args.changed });
|
|
808
|
+
else if (args.mode === 'detect-leaks') result = await detectLeaks(ctx, chosen.framework, args.file);
|
|
809
|
+
else {
|
|
810
|
+
result = await runSuite(ctx, chosen.framework, {
|
|
811
|
+
onBatch: (b, n) => process.stderr.write(`[test-batches] batch ${b.index}/${n}: ${b.results.length} results, ${toMb(b.peakRss) ?? '?'} MB, ${Math.round(b.durationMs / 1000)} s${b.complete ? '' : ` INCOMPLETE (${b.error})`}\n`),
|
|
812
|
+
});
|
|
813
|
+
result.runDir = toPosix(path.relative(root, ctx.runDir));
|
|
814
|
+
}
|
|
815
|
+
} finally {
|
|
816
|
+
fs.rmSync(ctx.pluginDir, { recursive: true, force: true });
|
|
817
|
+
if (args.mode !== 'run' && !args.out) fs.rmSync(ctx.runDir, { recursive: true, force: true });
|
|
818
|
+
if (!args.out) pruneRuns(ctx.storage);
|
|
819
|
+
}
|
|
820
|
+
process.stdout.write(`${args.json ? JSON.stringify(result, null, 2) : renderSummary(result)}\n`);
|
|
821
|
+
return result.exitCode ?? EXIT.INCOMPLETE;
|
|
822
|
+
}
|
|
823
|
+
|
|
824
|
+
const isEntry = () => {
|
|
825
|
+
try { return fs.realpathSync(process.argv[1]) === fs.realpathSync(fileURLToPath(import.meta.url)); } catch { return false; }
|
|
826
|
+
};
|
|
827
|
+
if (process.argv[1] && isEntry()) {
|
|
828
|
+
main().then((c) => { process.exitCode = c; }, (e) => { process.stderr.write(`${e.stack || e}\n`); process.exitCode = EXIT.INCOMPLETE; });
|
|
829
|
+
}
|