@veda-hq/pariksha 0.5.18 → 0.5.20

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -0,0 +1,252 @@
1
+ // flaky-tests: run the listed test files several times with Vitest and
2
+ // refuse a test that passes in some runs and fails in others.
3
+ //
4
+ // Verdicts (evaluateRuns):
5
+ // - a test passes in some runs and fails in others → flaky, exit 1
6
+ // - every run fails → not flaky, still exit 1
7
+ // - a run fails without collecting a test case → runner failure, exit 1
8
+ // - every run passes → exit 0
9
+ // A file that fails every time is not flaky, but it is not a pass either.
10
+ import { spawnSync } from 'node:child_process';
11
+ import { relative, resolve } from 'node:path';
12
+ import { readVitestJsonReports } from "./vitest-json-report.js";
13
+ /** `npx vitest …`, its output captured. */
14
+ export const npxVitest = (args, cwd) => spawnSync('npx', ['vitest', ...args], { cwd, encoding: 'utf8', stdio: ['ignore', 'pipe', 'pipe'], maxBuffer: 10 * 1024 * 1024 });
15
+ export const DEFAULT_CONFIG = { include: /\.(test|spec)\.(ts|tsx|js|jsx)$/, packageRoots: [], exclude: [] };
16
+ const isRecord = (v) => typeof v === 'object' && v !== null && !Array.isArray(v);
17
+ const isStrings = (v) => Array.isArray(v) && v.every((s) => typeof s === 'string');
18
+ /** The config a product passes; each field replaces its default, and an unknown field is refused. */
19
+ export const parseConfig = (text) => {
20
+ if (text === undefined)
21
+ return DEFAULT_CONFIG;
22
+ const raw = JSON.parse(text);
23
+ if (!isRecord(raw))
24
+ throw new Error('the config is not a JSON object');
25
+ const unknown = Object.keys(raw).filter((k) => !['include', 'packageRoots', 'exclude'].includes(k));
26
+ if (unknown.length > 0)
27
+ throw new Error(`the config has unknown field(s): ${unknown.join(', ')}`);
28
+ const { include, packageRoots, exclude } = raw;
29
+ if (include !== undefined && typeof include !== 'string')
30
+ throw new Error('include is not a regex string');
31
+ if (packageRoots !== undefined && !isStrings(packageRoots))
32
+ throw new Error('packageRoots is not a list of strings');
33
+ if (exclude !== undefined && !isStrings(exclude))
34
+ throw new Error('exclude is not a list of strings');
35
+ return {
36
+ include: include === undefined ? DEFAULT_CONFIG.include : new RegExp(include),
37
+ packageRoots: packageRoots ?? DEFAULT_CONFIG.packageRoots,
38
+ exclude: exclude ?? DEFAULT_CONFIG.exclude,
39
+ };
40
+ };
41
+ /** A run count: an integer from 1 to 1000, or null. */
42
+ export const parseRuns = (text) => {
43
+ if (!/^[+-]?\d+$/.test(text))
44
+ return null;
45
+ const value = Number(text);
46
+ return Number.isSafeInteger(value) && value >= 1 && value <= 1000 ? value : null;
47
+ };
48
+ export const DEFAULT_RUNS = 5;
49
+ /** Keep the tail of the runner output so a huge stack trace stays readable. */
50
+ const MAX_OUTPUT_CHARS = 6000;
51
+ /** A JSON line: the reporter's payload, or a test's own logged object. */
52
+ const isReporterPayload = (line) => line.trim().startsWith('{');
53
+ /** The output that is not the JSON payload: what a human needs when a suite fails to load. */
54
+ export const runnerNoise = (stdout, stderr) => {
55
+ const stdoutNoise = stdout
56
+ .split('\n')
57
+ .filter((l) => l.trim().length > 0 && !isReporterPayload(l))
58
+ .join('\n');
59
+ return [stderr.trim(), stdoutNoise.trim()].filter(Boolean).join('\n');
60
+ };
61
+ /** A root's run that exited non-zero delivering no test case: an unloadable file or a crashed runner. */
62
+ export const isEmptyFailure = (status, collected) => status !== 0 && collected === 0;
63
+ const tail = (text) => (text.length > MAX_OUTPUT_CHARS ? `…(truncated)…\n${text.slice(-MAX_OUTPUT_CHARS)}` : text);
64
+ /** The verdict on the collected per-test stats and per-run outcomes; no I/O. */
65
+ export const evaluateRuns = (results, runOutcomes) => {
66
+ const lines = [];
67
+ const allTests = Array.from(results.values());
68
+ const flakyTests = allTests.filter((r) => r.isFlaky);
69
+ const consistentFailures = allTests.filter((r) => r.failures === r.runs && r.runs > 0);
70
+ const failedRuns = runOutcomes.filter((r) => !r.passed);
71
+ // Judged per root and per run, never in aggregate: a run that collected
72
+ // tests must not launder one that could not.
73
+ const emptyFailedRuns = failedRuns.filter((r) => r.emptyFailure);
74
+ const allRunsFailed = runOutcomes.length > 0 && failedRuns.length === runOutcomes.length;
75
+ const lastFailureOutput = failedRuns
76
+ .map((r) => r.output)
77
+ .filter(Boolean)
78
+ .at(-1);
79
+ const flakyLines = [];
80
+ for (const t of flakyTests) {
81
+ flakyLines.push('', ` ${t.file}`, ` Rate: ${(t.flakinessRate * 100).toFixed(0)}% (${String(t.failures)}/${String(t.runs)} failed)`, ` Test: ${t.testName}`);
82
+ const [first] = t.failureMessages;
83
+ if (first !== undefined)
84
+ flakyLines.push(` Error: ${first.substring(0, 200)}...`);
85
+ }
86
+ // A run that did not execute the file takes the verdict; flaky tests seen in the other runs are still listed.
87
+ if (emptyFailedRuns.length > 0) {
88
+ lines.push('', '❌ Test execution failed: the runner exited non-zero without collecting a single test case.', ` ${String(emptyFailedRuns.length)}/${String(runOutcomes.length)} run(s) failed that way. The file under test is not flaky — it did not run at all.`, '', ' Runner output (last failing run):', '', tail(lastFailureOutput ?? ' (the runner produced no output)'));
89
+ if (flakyTests.length > 0)
90
+ lines.push('', '⚠️ The runs that did execute also found flaky tests:', ...flakyLines);
91
+ return { verdict: 'runner-failure', exitCode: 1, lines };
92
+ }
93
+ if (flakyTests.length > 0) {
94
+ lines.push('', '⚠️ Found flaky tests:', ...flakyLines);
95
+ return { verdict: 'flaky', exitCode: 1, lines };
96
+ }
97
+ if (allRunsFailed || consistentFailures.length > 0) {
98
+ lines.push('', '❌ No flaky tests found, but the tests failed in EVERY run.', ' A deterministic failure is not flakiness — it is a broken test or environment.', '');
99
+ for (const t of consistentFailures) {
100
+ lines.push(` ❌ ${t.file}`, ` Test: ${t.testName}`);
101
+ const [first] = t.failureMessages;
102
+ if (first !== undefined)
103
+ lines.push(` Error: ${first.split('\n')[0].substring(0, 150)}...`);
104
+ lines.push('');
105
+ }
106
+ if (lastFailureOutput)
107
+ lines.push(' Runner output (last failing run):', '', tail(lastFailureOutput));
108
+ return { verdict: 'consistent-failure', exitCode: 1, lines };
109
+ }
110
+ lines.push('', `✅ No flaky tests found across ${String(results.size)} test cases.`);
111
+ return { verdict: 'clean', exitCode: 0, lines };
112
+ };
113
+ /**
114
+ * Fold one run's JSON reporter output into `results`. An unloadable file is a
115
+ * suite with a non-empty `message` and no assertion results, and the reporter
116
+ * prints nothing to stderr, so the message alone is its cause.
117
+ */
118
+ export const parseTestResults = (output, results) => {
119
+ const suiteErrors = [];
120
+ let collected = 0;
121
+ const { reports, errors } = readVitestJsonReports(output);
122
+ for (const report of reports) {
123
+ for (const testResult of report.testResults) {
124
+ const file = testResult.name;
125
+ if (testResult.message)
126
+ suiteErrors.push(`${file}\n ${testResult.message}`);
127
+ for (const test of testResult.assertionResults) {
128
+ const testName = test.title || test.fullName || 'unknown';
129
+ const key = `${file}::${testName}`;
130
+ const stats = results.get(key) ?? { testName, file, runs: 0, passes: 0, failures: 0, isFlaky: false, flakinessRate: 0, failureMessages: [] };
131
+ results.set(key, stats);
132
+ stats.runs++;
133
+ collected++;
134
+ if (test.status === 'passed')
135
+ stats.passes++;
136
+ else if (test.status === 'failed') {
137
+ stats.failures++;
138
+ const [first] = test.failureMessages;
139
+ if (first !== undefined)
140
+ stats.failureMessages.push(first);
141
+ }
142
+ // 'skipped', 'todo' and 'pending' count as collected, not as outcomes.
143
+ if (stats.passes > 0 && stats.failures > 0) {
144
+ stats.isFlaky = true;
145
+ stats.flakinessRate = stats.failures / stats.runs;
146
+ }
147
+ }
148
+ }
149
+ }
150
+ return { suiteErrors, collected, reports: reports.length, failedReports: reports.filter((r) => !r.success).length, reportErrors: errors };
151
+ };
152
+ /**
153
+ * Did this root's run leave a report the detector can trust? A clean exit
154
+ * with no report, a report that broke the contract, or a clean exit over a
155
+ * report saying `success: false` is a harness failure.
156
+ */
157
+ export const harnessFailure = (status, parsed) => {
158
+ if (parsed.reportErrors.length > 0)
159
+ return parsed.reportErrors.join('\n');
160
+ if (status === 0 && parsed.reports === 0)
161
+ return 'vitest exited 0 but printed no JSON report';
162
+ if (status === 0 && parsed.failedReports > 0)
163
+ return 'vitest exited 0 but its JSON report says success: false';
164
+ return null;
165
+ };
166
+ export const readRootRun = (root, status, stdout, results) => {
167
+ // Judged on a scratch map first: an untrusted root's outcomes must not reach the shared results.
168
+ const parsed = parseTestResults(stdout, new Map());
169
+ const harness = harnessFailure(status, parsed);
170
+ if (harness === null)
171
+ parseTestResults(stdout, results);
172
+ return {
173
+ parsed,
174
+ trusted: harness === null,
175
+ emptyFailure: harness !== null || isEmptyFailure(status, parsed.collected),
176
+ errors: harness === null ? '' : `[${root}] untrusted test report: ${harness}\n`,
177
+ };
178
+ };
179
+ /**
180
+ * The files grouped by package root, each relative to its root; '.' (the
181
+ * repository root) first, for files under none. A root holds a file at
182
+ * `<root>/…`, or the path equal to it.
183
+ */
184
+ export const groupByRoot = (config, files) => {
185
+ const byRoot = new Map([['.', []]]);
186
+ for (const file of files) {
187
+ const root = config.packageRoots.find((r) => file.startsWith(`${r}/`) || file === r) ?? '.';
188
+ const list = byRoot.get(root) ?? [];
189
+ byRoot.set(root, list);
190
+ list.push(file);
191
+ }
192
+ return byRoot;
193
+ };
194
+ /**
195
+ * The detector: `runs` passes over the listed files (paths relative to
196
+ * `projectRoot`), each package root's files in one Vitest call from that root.
197
+ * Lines go to `out` as the original printed them.
198
+ */
199
+ export const runFlakyTests = (config, runs, mode, projectRoot, files, runner, out) => {
200
+ out(`🔄 Detecting flaky tests (${String(runs)} runs)...`);
201
+ out(` Mode: ${mode}`);
202
+ if (files.length === 0) {
203
+ out('No files found to analyze.');
204
+ return 0;
205
+ }
206
+ out(` Running on ${String(files.length)} files...`);
207
+ const results = new Map();
208
+ const byRoot = groupByRoot(config, files);
209
+ const runOutcomes = [];
210
+ for (let i = 1; i <= runs; i++) {
211
+ let runSuccess = true;
212
+ let runEmptyFailure = false;
213
+ let groupErrors = '';
214
+ for (const [root, rootFiles] of byRoot) {
215
+ if (rootFiles.length === 0)
216
+ continue;
217
+ const args = ['run', '--reporter=json', '--passWithNoTests', ...config.exclude.flatMap((g) => ['--exclude', g]), ...rootFiles.map((f) => relative(root, f))];
218
+ try {
219
+ const result = runner(args, resolve(projectRoot, root));
220
+ // A runner that never started has no output to judge.
221
+ if (result.error)
222
+ throw result.error;
223
+ // A runner killed before it wrote anything leaves null streams.
224
+ const stdout = result.stdout ?? '';
225
+ const stderr = result.stderr ?? '';
226
+ const rootRun = readRootRun(root, result.status, stdout, results);
227
+ runEmptyFailure ||= rootRun.emptyFailure;
228
+ runSuccess &&= rootRun.trusted;
229
+ groupErrors += rootRun.errors;
230
+ if (result.status !== 0) {
231
+ runSuccess = false;
232
+ const noise = runnerNoise(stdout, stderr);
233
+ groupErrors += `[${root}] vitest exited with status ${String(result.status)}\n`;
234
+ for (const suiteError of rootRun.parsed.suiteErrors)
235
+ groupErrors += `${suiteError}\n`;
236
+ if (noise)
237
+ groupErrors += `${noise}\n`;
238
+ }
239
+ }
240
+ catch (e) {
241
+ runSuccess = false;
242
+ runEmptyFailure = true;
243
+ groupErrors += `[${root}] Spawn Error: ${e.message}\n`;
244
+ }
245
+ }
246
+ out(` Run ${String(i)}/${String(runs)}: ${runSuccess ? '✅ Passed' : '❌ Failed'}`);
247
+ runOutcomes.push({ passed: runSuccess, emptyFailure: runEmptyFailure, output: groupErrors });
248
+ }
249
+ const evaluation = evaluateRuns(results, runOutcomes);
250
+ evaluation.lines.forEach(out);
251
+ return evaluation.exitCode;
252
+ };
@@ -0,0 +1,32 @@
1
+ // python-floor: every tracked Python source runs on the interpreter a repository
2
+ // declares. The check is python/python-floor.py, run under the interpreter the
3
+ // caller names; pariksha never chooses or bundles a Python, and passes the
4
+ // script's stdout, stderr and exit code through unchanged.
5
+ import { spawnSync } from 'node:child_process';
6
+ import { join } from 'node:path';
7
+ /** The packaged script: beside src/ in the repository, beside dist/ in the npm package. */
8
+ export const SCRIPT = join(import.meta.dirname, '..', 'python', 'python-floor.py');
9
+ export const inheritRunner = (python, args) => spawnSync(python, args, { stdio: 'inherit' });
10
+ /** The script's arguments: each option only when given, so its own defaults hold. */
11
+ export const scriptArgs = (options) => [
12
+ SCRIPT,
13
+ '--floor-file',
14
+ options.floorFile,
15
+ ...(options.root === undefined ? [] : ['--root', options.root]),
16
+ ...(options.exclude === undefined ? [] : ['--exclude', options.exclude]),
17
+ ...(options.selector === undefined ? [] : ['--selector', options.selector]),
18
+ ];
19
+ /** The script's exit code; 2, with a reason on `err`, when it could not run or did not exit. */
20
+ export const runPythonFloor = (options, runner, err) => {
21
+ const P = 'pariksha python-floor:';
22
+ const result = runner(options.python, scriptArgs(options));
23
+ if (result.error !== undefined) {
24
+ err(`${P} cannot run ${options.python}: ${result.error.message}; nothing was checked, so this is not a pass`);
25
+ return 2;
26
+ }
27
+ if (result.status === null) {
28
+ err(`${P} ${options.python} was stopped by ${String(result.signal)}; nothing was checked, so this is not a pass`);
29
+ return 2;
30
+ }
31
+ return result.status;
32
+ };
@@ -0,0 +1,129 @@
1
+ // A strict reader for Vitest's JSON reporter output.
2
+ //
3
+ // A gate that reads a test harness's report must not mistake a malformed or
4
+ // inconsistent report for a run: a `testResults` that is not an array, a
5
+ // status it does not know, or totals that disagree with the results they sum
6
+ // read as "fewer tests" rather than "the harness lied", so each is refused.
7
+ // Shape and invariants follow Vitest's JsonReporter (vitest 4); unknown extra
8
+ // fields are allowed, so a reporter that adds data does not break the gate.
9
+ export const TEST_STATUSES = ['passed', 'failed', 'skipped', 'pending', 'todo'];
10
+ export class HarnessReportError extends Error {
11
+ constructor(problem) {
12
+ super(`vitest JSON report rejected: ${problem}`);
13
+ this.name = 'HarnessReportError';
14
+ }
15
+ }
16
+ const isRecord = (value) => typeof value === 'object' && value !== null && !Array.isArray(value);
17
+ const count = (raw, key) => {
18
+ const value = raw[key];
19
+ if (typeof value !== 'number' || !Number.isSafeInteger(value) || value < 0)
20
+ throw new HarnessReportError(`${key} is not a non-negative integer`);
21
+ return value;
22
+ };
23
+ const text = (raw, key, where) => {
24
+ const value = raw[key];
25
+ if (typeof value !== 'string')
26
+ throw new HarnessReportError(`${where}.${key} is not a string`);
27
+ return value;
28
+ };
29
+ const assertionOf = (raw, where) => {
30
+ if (!isRecord(raw))
31
+ throw new HarnessReportError(`${where} is not an object`);
32
+ const status = raw['status'];
33
+ if (!TEST_STATUSES.includes(status))
34
+ throw new HarnessReportError(`${where}.status ${String(JSON.stringify(status))} is not a known test status`);
35
+ const messages = raw['failureMessages'];
36
+ if (!Array.isArray(messages) || !messages.every((m) => typeof m === 'string'))
37
+ throw new HarnessReportError(`${where}.failureMessages is not a string[]`);
38
+ return { title: text(raw, 'title', where), fullName: text(raw, 'fullName', where), status: status, failureMessages: messages };
39
+ };
40
+ const fileOf = (raw, index) => {
41
+ const where = `testResults[${String(index)}]`;
42
+ if (!isRecord(raw))
43
+ throw new HarnessReportError(`${where} is not an object`);
44
+ const name = text(raw, 'name', where);
45
+ if (name.length === 0)
46
+ throw new HarnessReportError(`${where}.name is empty`);
47
+ const status = raw['status'];
48
+ if (status !== undefined && status !== 'passed' && status !== 'failed')
49
+ throw new HarnessReportError(`${where}.status ${JSON.stringify(status)} is not passed or failed`);
50
+ const message = text(raw, 'message', where);
51
+ const results = raw['assertionResults'];
52
+ if (!Array.isArray(results))
53
+ throw new HarnessReportError(`${where}.assertionResults is not an array`);
54
+ const assertionResults = results.map((item, i) => assertionOf(item, `${where}.assertionResults[${String(i)}]`));
55
+ // The converse is not checked: a failed hook fails the file with its error on a nested suite, which this report does not carry.
56
+ if (status === 'passed' && (assertionResults.some((t) => t.status === 'failed') || message.length > 0)) {
57
+ throw new HarnessReportError(`${where} says passed but carries a failed test or a suite error`);
58
+ }
59
+ return { name, ...(status === undefined ? {} : { status }), message, assertionResults };
60
+ };
61
+ /** One JSON reporter object, parsed; throws {@link HarnessReportError}. */
62
+ export const parseVitestJsonReport = (raw) => {
63
+ if (!isRecord(raw))
64
+ throw new HarnessReportError('report is not an object');
65
+ const files = raw['testResults'];
66
+ if (!Array.isArray(files))
67
+ throw new HarnessReportError('testResults is not an array');
68
+ const success = raw['success'];
69
+ if (typeof success !== 'boolean')
70
+ throw new HarnessReportError('success is not a boolean');
71
+ const testResults = files.map(fileOf);
72
+ const tests = testResults.flatMap((f) => f.assertionResults);
73
+ const report = {
74
+ numTotalTests: count(raw, 'numTotalTests'),
75
+ numPassedTests: count(raw, 'numPassedTests'),
76
+ numFailedTests: count(raw, 'numFailedTests'),
77
+ success,
78
+ testResults,
79
+ };
80
+ const failedSuites = count(raw, 'numFailedTestSuites');
81
+ const failed = tests.filter((t) => t.status === 'failed').length;
82
+ const passed = tests.filter((t) => t.status === 'passed').length;
83
+ if (report.numTotalTests !== tests.length)
84
+ throw new HarnessReportError(`numTotalTests ${String(report.numTotalTests)} but ${String(tests.length)} test results`);
85
+ if (report.numFailedTests !== failed || report.numPassedTests !== passed) {
86
+ throw new HarnessReportError(`passed/failed totals ${String(report.numPassedTests)}/${String(report.numFailedTests)} but results say ${String(passed)}/${String(failed)}`);
87
+ }
88
+ const anyFileFailed = testResults.some((f) => f.status === 'failed' || f.message.length > 0);
89
+ if (report.success && (failed > 0 || anyFileFailed || failedSuites > 0))
90
+ throw new HarnessReportError('success is true over a failed run');
91
+ if (!report.success && failed === 0 && !anyFileFailed && failedSuites === 0 && tests.length > 0)
92
+ throw new HarnessReportError('success is false over a clean run');
93
+ return report;
94
+ };
95
+ /** A line that claims to be a report: a JSON object with its own `testResults` key. */
96
+ const claim = (line) => {
97
+ let value;
98
+ try {
99
+ value = JSON.parse(line);
100
+ }
101
+ catch {
102
+ // fail-open: a stdout line that is not JSON is runner noise, not a report.
103
+ return undefined;
104
+ }
105
+ return isRecord(value) && Object.hasOwn(value, 'testResults') ? value : undefined;
106
+ };
107
+ /**
108
+ * The reports in a runner's stdout: every line that claims to be one, parsed
109
+ * strictly. Other JSON lines (a test's own logged object) are ignored; a
110
+ * claim that breaks the contract is returned as an error, never dropped.
111
+ */
112
+ export const readVitestJsonReports = (stdout) => {
113
+ const reports = [];
114
+ const errors = [];
115
+ for (const line of stdout.split('\n')) {
116
+ const raw = claim(line);
117
+ if (raw === undefined)
118
+ continue;
119
+ try {
120
+ reports.push(parseVitestJsonReport(raw));
121
+ }
122
+ catch (error) {
123
+ if (!(error instanceof HarnessReportError))
124
+ throw error;
125
+ errors.push(error.message);
126
+ }
127
+ }
128
+ return { reports, errors };
129
+ };
package/package.json CHANGED
@@ -1,7 +1,7 @@
1
1
  {
2
2
  "name": "@veda-hq/pariksha",
3
- "version": "0.5.18",
4
- "description": "Repository checks: doc drift, rule enforcement, test-quality ratchets. Checks: absolute-paths, aggregate-gate-needs, authority-citations, boundary, codec-registry, coverage-by-type, coverage-exemptions, doc-budgets, doc-references, enforcement-claims, env-gate-switches, export-deletions, fault-injection-registry, masked-exit-codes, non-synced-resources, pnpm-settings, required-settings, shell-gates. Also markdownlint rules for Markdown that renders mangled: @veda-hq/pariksha/markdownlint.",
3
+ "version": "0.5.20",
4
+ "description": "Repository checks: doc drift, rule enforcement, test-quality ratchets. Checks: absolute-paths, aggregate-gate-needs, authority-citations, boundary, codec-registry, coverage-by-type, coverage-exemptions, doc-budgets, doc-references, duplicate-tests, enforcement-claims, env-gate-switches, export-deletions, fault-injection-registry, flaky-tests, masked-exit-codes, non-synced-resources, pnpm-settings, python-floor, required-settings, shell-gates. Also markdownlint rules for Markdown that renders mangled: @veda-hq/pariksha/markdownlint.",
5
5
  "keywords": [
6
6
  "repository checks",
7
7
  "lint",
@@ -36,6 +36,7 @@
36
36
  },
37
37
  "files": [
38
38
  "dist",
39
+ "python",
39
40
  "baselines",
40
41
  "NOTICE"
41
42
  ],