@vitronai/themis 0.1.4 → 0.1.6
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +7 -0
- package/README.md +57 -29
- package/docs/agents-adoption.md +5 -2
- package/docs/api.md +45 -26
- package/docs/migration.md +2 -2
- package/docs/showcases.md +41 -6
- package/docs/vscode-extension.md +12 -12
- package/docs/why-themis.md +5 -3
- package/package.json +17 -4
- package/src/artifact-paths.js +111 -0
- package/src/artifacts.js +107 -39
- package/src/cli.js +156 -18
- package/src/contract-runtime.js +1166 -0
- package/src/environment.js +1 -1
- package/src/expect.js +1 -1
- package/src/generate.js +37 -1209
- package/src/gitignore.js +53 -0
- package/src/init.js +2 -13
- package/src/migrate.js +6 -1
- package/src/module-loader.js +13 -3
- package/src/reporter.js +14 -12
- package/src/runtime.js +2 -2
- package/src/test-utils.js +1 -1
- package/templates/AGENTS.themis.md +3 -0
|
@@ -0,0 +1,111 @@
|
|
|
1
|
+
const path = require('path');
|
|
2
|
+
|
|
3
|
+
const ARTIFACT_DIR = '.themis';
|
|
4
|
+
const RUNS_DIR = ['runs'];
|
|
5
|
+
const DIFFS_DIR = ['diffs'];
|
|
6
|
+
const GENERATE_DIR = ['generate'];
|
|
7
|
+
const REPORTS_DIR = ['reports'];
|
|
8
|
+
const MIGRATION_DIR = ['migration'];
|
|
9
|
+
const BENCHMARKS_DIR = ['benchmarks'];
|
|
10
|
+
const SHOWCASE_COMPARISON_DIR = [...BENCHMARKS_DIR, 'showcase-comparison'];
|
|
11
|
+
const MIGRATION_FIXTURES_DIR = [...MIGRATION_DIR, 'fixtures'];
|
|
12
|
+
|
|
13
|
+
const ARTIFACT_SEGMENTS = Object.freeze({
|
|
14
|
+
lastRun: [...RUNS_DIR, 'last-run.json'],
|
|
15
|
+
failedTests: [...RUNS_DIR, 'failed-tests.json'],
|
|
16
|
+
runDiff: [...DIFFS_DIR, 'run-diff.json'],
|
|
17
|
+
runHistory: [...RUNS_DIR, 'run-history.json'],
|
|
18
|
+
fixHandoff: [...RUNS_DIR, 'fix-handoff.json'],
|
|
19
|
+
contractDiff: [...DIFFS_DIR, 'contract-diff.json'],
|
|
20
|
+
generateMap: [...GENERATE_DIR, 'generate-map.json'],
|
|
21
|
+
generateResult: [...GENERATE_DIR, 'generate-last.json'],
|
|
22
|
+
generateHandoff: [...GENERATE_DIR, 'generate-handoff.json'],
|
|
23
|
+
generateBacklog: [...GENERATE_DIR, 'generate-backlog.json'],
|
|
24
|
+
migrationReport: [...MIGRATION_DIR, 'migration-report.json'],
|
|
25
|
+
htmlReport: [...REPORTS_DIR, 'report.html'],
|
|
26
|
+
benchmarkLast: [...BENCHMARKS_DIR, 'benchmark-last.json'],
|
|
27
|
+
migrationProof: [...BENCHMARKS_DIR, 'migration-proof.json']
|
|
28
|
+
});
|
|
29
|
+
|
|
30
|
+
const LEGACY_ARTIFACT_SEGMENTS = Object.freeze({
|
|
31
|
+
lastRun: ['last-run.json'],
|
|
32
|
+
failedTests: ['failed-tests.json'],
|
|
33
|
+
runDiff: ['run-diff.json'],
|
|
34
|
+
runHistory: ['run-history.json'],
|
|
35
|
+
fixHandoff: ['fix-handoff.json'],
|
|
36
|
+
contractDiff: ['contract-diff.json'],
|
|
37
|
+
generateMap: ['generate-map.json'],
|
|
38
|
+
generateResult: ['generate-last.json'],
|
|
39
|
+
generateHandoff: ['generate-handoff.json'],
|
|
40
|
+
generateBacklog: ['generate-backlog.json'],
|
|
41
|
+
migrationReport: ['migration-report.json'],
|
|
42
|
+
htmlReport: ['report.html'],
|
|
43
|
+
benchmarkLast: ['benchmark-last.json'],
|
|
44
|
+
migrationProof: ['migration-proof.json']
|
|
45
|
+
});
|
|
46
|
+
|
|
47
|
+
function joinRelative(segments) {
|
|
48
|
+
return path.posix.join(ARTIFACT_DIR, ...segments);
|
|
49
|
+
}
|
|
50
|
+
|
|
51
|
+
function joinAbsolute(cwd, segments) {
|
|
52
|
+
return path.join(cwd, ARTIFACT_DIR, ...segments);
|
|
53
|
+
}
|
|
54
|
+
|
|
55
|
+
function buildRelativePathMap(segmentMap) {
|
|
56
|
+
return Object.fromEntries(
|
|
57
|
+
Object.entries(segmentMap).map(([key, segments]) => [key, joinRelative(segments)])
|
|
58
|
+
);
|
|
59
|
+
}
|
|
60
|
+
|
|
61
|
+
const ARTIFACT_RELATIVE_PATHS = Object.freeze(buildRelativePathMap(ARTIFACT_SEGMENTS));
|
|
62
|
+
const LEGACY_ARTIFACT_RELATIVE_PATHS = Object.freeze(buildRelativePathMap(LEGACY_ARTIFACT_SEGMENTS));
|
|
63
|
+
|
|
64
|
+
function resolveArtifactPath(cwd, key) {
|
|
65
|
+
return joinAbsolute(cwd, ARTIFACT_SEGMENTS[key]);
|
|
66
|
+
}
|
|
67
|
+
|
|
68
|
+
function resolveLegacyArtifactPath(cwd, key) {
|
|
69
|
+
return joinAbsolute(cwd, LEGACY_ARTIFACT_SEGMENTS[key]);
|
|
70
|
+
}
|
|
71
|
+
|
|
72
|
+
function resolveArtifactDir(cwd, ...segments) {
|
|
73
|
+
return path.join(cwd, ARTIFACT_DIR, ...segments);
|
|
74
|
+
}
|
|
75
|
+
|
|
76
|
+
function resolveRelativeDir(...segments) {
|
|
77
|
+
return path.posix.join(ARTIFACT_DIR, ...segments);
|
|
78
|
+
}
|
|
79
|
+
|
|
80
|
+
function getArtifactPaths(cwd) {
|
|
81
|
+
return Object.fromEntries(
|
|
82
|
+
Object.keys(ARTIFACT_SEGMENTS).map((key) => [key, resolveArtifactPath(cwd, key)])
|
|
83
|
+
);
|
|
84
|
+
}
|
|
85
|
+
|
|
86
|
+
function getArtifactPathCandidates(cwd, key) {
|
|
87
|
+
const nextPath = resolveArtifactPath(cwd, key);
|
|
88
|
+
const legacyPath = resolveLegacyArtifactPath(cwd, key);
|
|
89
|
+
return nextPath === legacyPath ? [nextPath] : [nextPath, legacyPath];
|
|
90
|
+
}
|
|
91
|
+
|
|
92
|
+
module.exports = {
|
|
93
|
+
ARTIFACT_DIR,
|
|
94
|
+
RUNS_DIR,
|
|
95
|
+
DIFFS_DIR,
|
|
96
|
+
GENERATE_DIR,
|
|
97
|
+
REPORTS_DIR,
|
|
98
|
+
MIGRATION_DIR,
|
|
99
|
+
BENCHMARKS_DIR,
|
|
100
|
+
SHOWCASE_COMPARISON_DIR,
|
|
101
|
+
MIGRATION_FIXTURES_DIR,
|
|
102
|
+
ARTIFACT_SEGMENTS,
|
|
103
|
+
ARTIFACT_RELATIVE_PATHS,
|
|
104
|
+
LEGACY_ARTIFACT_RELATIVE_PATHS,
|
|
105
|
+
resolveArtifactPath,
|
|
106
|
+
resolveLegacyArtifactPath,
|
|
107
|
+
resolveArtifactDir,
|
|
108
|
+
resolveRelativeDir,
|
|
109
|
+
getArtifactPaths,
|
|
110
|
+
getArtifactPathCandidates
|
|
111
|
+
};
|
package/src/artifacts.js
CHANGED
|
@@ -1,31 +1,35 @@
|
|
|
1
1
|
const fs = require('fs');
|
|
2
2
|
const path = require('path');
|
|
3
|
-
|
|
4
|
-
|
|
5
|
-
|
|
6
|
-
|
|
7
|
-
|
|
8
|
-
const RUN_HISTORY_FILE = 'run-history.json';
|
|
9
|
-
const FIX_HANDOFF_FILE = 'fix-handoff.json';
|
|
10
|
-
const CONTRACT_DIFF_FILE = 'contract-diff.json';
|
|
11
|
-
const GENERATE_MAP_FILE = 'generate-map.json';
|
|
12
|
-
const GENERATE_BACKLOG_FILE = 'generate-backlog.json';
|
|
3
|
+
const {
|
|
4
|
+
ARTIFACT_RELATIVE_PATHS,
|
|
5
|
+
getArtifactPathCandidates,
|
|
6
|
+
getArtifactPaths
|
|
7
|
+
} = require('./artifact-paths');
|
|
13
8
|
|
|
14
9
|
function writeRunArtifacts(cwd, result) {
|
|
15
|
-
const
|
|
16
|
-
|
|
10
|
+
const artifactPaths = getArtifactPaths(cwd);
|
|
11
|
+
for (const artifactPath of [
|
|
12
|
+
artifactPaths.lastRun,
|
|
13
|
+
artifactPaths.failedTests,
|
|
14
|
+
artifactPaths.runDiff,
|
|
15
|
+
artifactPaths.runHistory,
|
|
16
|
+
artifactPaths.fixHandoff,
|
|
17
|
+
artifactPaths.contractDiff
|
|
18
|
+
]) {
|
|
19
|
+
fs.mkdirSync(path.dirname(artifactPath), { recursive: true });
|
|
20
|
+
}
|
|
17
21
|
|
|
18
|
-
const runPath =
|
|
19
|
-
const previousRun =
|
|
22
|
+
const runPath = artifactPaths.lastRun;
|
|
23
|
+
const previousRun = readJsonFromCandidates(getArtifactPathCandidates(cwd, 'lastRun'));
|
|
20
24
|
const runId = createRunId(result.meta?.startedAt || new Date().toISOString());
|
|
21
25
|
const comparison = buildRunComparison(previousRun, result);
|
|
22
26
|
const relativePaths = {
|
|
23
|
-
lastRun:
|
|
24
|
-
failedTests:
|
|
25
|
-
runDiff:
|
|
26
|
-
runHistory:
|
|
27
|
-
fixHandoff:
|
|
28
|
-
contractDiff:
|
|
27
|
+
lastRun: ARTIFACT_RELATIVE_PATHS.lastRun,
|
|
28
|
+
failedTests: ARTIFACT_RELATIVE_PATHS.failedTests,
|
|
29
|
+
runDiff: ARTIFACT_RELATIVE_PATHS.runDiff,
|
|
30
|
+
runHistory: ARTIFACT_RELATIVE_PATHS.runHistory,
|
|
31
|
+
fixHandoff: ARTIFACT_RELATIVE_PATHS.fixHandoff,
|
|
32
|
+
contractDiff: ARTIFACT_RELATIVE_PATHS.contractDiff
|
|
29
33
|
};
|
|
30
34
|
|
|
31
35
|
result.artifacts = {
|
|
@@ -35,6 +39,7 @@ function writeRunArtifacts(cwd, result) {
|
|
|
35
39
|
};
|
|
36
40
|
|
|
37
41
|
fs.writeFileSync(runPath, `${stringifyArtifact(result)}\n`, 'utf8');
|
|
42
|
+
removeLegacyArtifact(cwd, 'lastRun');
|
|
38
43
|
|
|
39
44
|
const failedTests = [];
|
|
40
45
|
for (const fileEntry of result.files || []) {
|
|
@@ -61,19 +66,21 @@ function writeRunArtifacts(cwd, result) {
|
|
|
61
66
|
failedTests
|
|
62
67
|
};
|
|
63
68
|
|
|
64
|
-
const failuresPath =
|
|
69
|
+
const failuresPath = artifactPaths.failedTests;
|
|
65
70
|
fs.writeFileSync(failuresPath, `${stringifyArtifact(failuresPayload)}\n`, 'utf8');
|
|
71
|
+
removeLegacyArtifact(cwd, 'failedTests');
|
|
66
72
|
|
|
67
73
|
const diffPayload = {
|
|
68
74
|
schema: 'themis.run.diff.v1',
|
|
69
75
|
runId,
|
|
70
76
|
...comparison
|
|
71
77
|
};
|
|
72
|
-
const diffPath =
|
|
78
|
+
const diffPath = artifactPaths.runDiff;
|
|
73
79
|
fs.writeFileSync(diffPath, `${stringifyArtifact(diffPayload)}\n`, 'utf8');
|
|
80
|
+
removeLegacyArtifact(cwd, 'runDiff');
|
|
74
81
|
|
|
75
|
-
const historyPath =
|
|
76
|
-
const previousHistory =
|
|
82
|
+
const historyPath = artifactPaths.runHistory;
|
|
83
|
+
const previousHistory = readJsonFromCandidates(getArtifactPathCandidates(cwd, 'runHistory'));
|
|
77
84
|
const nextHistory = Array.isArray(previousHistory) ? previousHistory.slice(-19) : [];
|
|
78
85
|
nextHistory.push({
|
|
79
86
|
runId,
|
|
@@ -84,16 +91,18 @@ function writeRunArtifacts(cwd, result) {
|
|
|
84
91
|
comparison
|
|
85
92
|
});
|
|
86
93
|
fs.writeFileSync(historyPath, `${stringifyArtifact(nextHistory)}\n`, 'utf8');
|
|
94
|
+
removeLegacyArtifact(cwd, 'runHistory');
|
|
87
95
|
|
|
88
96
|
const contractDiffPayload = buildContractDiffPayload(result, {
|
|
89
97
|
runId,
|
|
90
98
|
createdAt: new Date().toISOString(),
|
|
91
99
|
relativePaths
|
|
92
100
|
});
|
|
93
|
-
const contractDiffPath =
|
|
101
|
+
const contractDiffPath = artifactPaths.contractDiff;
|
|
94
102
|
fs.writeFileSync(contractDiffPath, `${stringifyArtifact(contractDiffPayload)}\n`, 'utf8');
|
|
103
|
+
removeLegacyArtifact(cwd, 'contractDiff');
|
|
95
104
|
|
|
96
|
-
const fixHandoffPath =
|
|
105
|
+
const fixHandoffPath = artifactPaths.fixHandoff;
|
|
97
106
|
let fixHandoff = null;
|
|
98
107
|
if (failedTests.length > 0) {
|
|
99
108
|
fixHandoff = buildFixHandoffPayload(cwd, result, {
|
|
@@ -102,7 +111,11 @@ function writeRunArtifacts(cwd, result) {
|
|
|
102
111
|
relativePaths
|
|
103
112
|
});
|
|
104
113
|
fs.writeFileSync(fixHandoffPath, `${stringifyArtifact(fixHandoff)}\n`, 'utf8');
|
|
114
|
+
removeLegacyArtifact(cwd, 'fixHandoff');
|
|
115
|
+
} else if (fs.existsSync(fixHandoffPath)) {
|
|
116
|
+
fs.rmSync(fixHandoffPath, { force: true });
|
|
105
117
|
}
|
|
118
|
+
removeLegacyArtifact(cwd, 'fixHandoff');
|
|
106
119
|
|
|
107
120
|
return {
|
|
108
121
|
runPath,
|
|
@@ -119,18 +132,19 @@ function writeRunArtifacts(cwd, result) {
|
|
|
119
132
|
}
|
|
120
133
|
|
|
121
134
|
function readFailedTestsArtifact(cwd) {
|
|
122
|
-
const
|
|
123
|
-
|
|
135
|
+
const candidates = getArtifactPathCandidates(cwd, 'failedTests');
|
|
136
|
+
const existingPath = candidates.find((candidate) => fs.existsSync(candidate));
|
|
137
|
+
if (!existingPath) {
|
|
124
138
|
return null;
|
|
125
139
|
}
|
|
126
140
|
|
|
127
|
-
const raw = fs.readFileSync(
|
|
141
|
+
const raw = fs.readFileSync(existingPath, 'utf8');
|
|
128
142
|
let parsed;
|
|
129
143
|
try {
|
|
130
144
|
parsed = JSON.parse(raw);
|
|
131
145
|
} catch (error) {
|
|
132
146
|
return {
|
|
133
|
-
failuresPath,
|
|
147
|
+
failuresPath: existingPath,
|
|
134
148
|
failedTests: [],
|
|
135
149
|
parseError: String(error?.message || error)
|
|
136
150
|
};
|
|
@@ -138,18 +152,52 @@ function readFailedTestsArtifact(cwd) {
|
|
|
138
152
|
|
|
139
153
|
if (!parsed || !Array.isArray(parsed.failedTests)) {
|
|
140
154
|
return {
|
|
141
|
-
failuresPath,
|
|
155
|
+
failuresPath: existingPath,
|
|
142
156
|
failedTests: [],
|
|
143
157
|
parseError: 'Invalid artifact shape: expected "failedTests" to be an array'
|
|
144
158
|
};
|
|
145
159
|
}
|
|
146
160
|
|
|
147
161
|
return {
|
|
148
|
-
failuresPath,
|
|
162
|
+
failuresPath: existingPath,
|
|
149
163
|
failedTests: parsed.failedTests
|
|
150
164
|
};
|
|
151
165
|
}
|
|
152
166
|
|
|
167
|
+
function readFixHandoffArtifact(cwd) {
|
|
168
|
+
const candidates = getArtifactPathCandidates(cwd, 'fixHandoff');
|
|
169
|
+
const existingPath = candidates.find((candidate) => fs.existsSync(candidate));
|
|
170
|
+
if (!existingPath) {
|
|
171
|
+
return null;
|
|
172
|
+
}
|
|
173
|
+
|
|
174
|
+
const raw = fs.readFileSync(existingPath, 'utf8');
|
|
175
|
+
let parsed;
|
|
176
|
+
try {
|
|
177
|
+
parsed = JSON.parse(raw);
|
|
178
|
+
} catch (error) {
|
|
179
|
+
return {
|
|
180
|
+
fixHandoffPath: existingPath,
|
|
181
|
+
items: [],
|
|
182
|
+
parseError: String(error?.message || error)
|
|
183
|
+
};
|
|
184
|
+
}
|
|
185
|
+
|
|
186
|
+
if (!parsed || !Array.isArray(parsed.items)) {
|
|
187
|
+
return {
|
|
188
|
+
fixHandoffPath: existingPath,
|
|
189
|
+
items: [],
|
|
190
|
+
parseError: 'Invalid artifact shape: expected "items" to be an array'
|
|
191
|
+
};
|
|
192
|
+
}
|
|
193
|
+
|
|
194
|
+
return {
|
|
195
|
+
fixHandoffPath: existingPath,
|
|
196
|
+
items: parsed.items,
|
|
197
|
+
payload: parsed
|
|
198
|
+
};
|
|
199
|
+
}
|
|
200
|
+
|
|
153
201
|
function buildRunComparison(previousRun, result) {
|
|
154
202
|
const currentFailures = collectFailureNames(result);
|
|
155
203
|
const currentFailureSet = new Set(currentFailures);
|
|
@@ -266,14 +314,24 @@ function readJsonIfExists(filePath) {
|
|
|
266
314
|
|
|
267
315
|
try {
|
|
268
316
|
return JSON.parse(fs.readFileSync(filePath, 'utf8'));
|
|
269
|
-
} catch
|
|
317
|
+
} catch {
|
|
270
318
|
return null;
|
|
271
319
|
}
|
|
272
320
|
}
|
|
273
321
|
|
|
322
|
+
function readJsonFromCandidates(filePaths) {
|
|
323
|
+
for (const filePath of filePaths) {
|
|
324
|
+
const parsed = readJsonIfExists(filePath);
|
|
325
|
+
if (parsed) {
|
|
326
|
+
return parsed;
|
|
327
|
+
}
|
|
328
|
+
}
|
|
329
|
+
return null;
|
|
330
|
+
}
|
|
331
|
+
|
|
274
332
|
function buildFixHandoffPayload(cwd, result, context) {
|
|
275
|
-
const generateMap =
|
|
276
|
-
const generateBacklog =
|
|
333
|
+
const generateMap = readJsonFromCandidates(getArtifactPathCandidates(cwd, 'generateMap'));
|
|
334
|
+
const generateBacklog = readJsonFromCandidates(getArtifactPathCandidates(cwd, 'generateBacklog'));
|
|
277
335
|
const mapEntries = Array.isArray(generateMap && generateMap.entries) ? generateMap.entries : [];
|
|
278
336
|
const backlogItems = Array.isArray(generateBacklog && generateBacklog.items) ? generateBacklog.items : [];
|
|
279
337
|
const byGeneratedTest = new Map();
|
|
@@ -341,8 +399,8 @@ function buildFixHandoffPayload(cwd, result, context) {
|
|
|
341
399
|
summary,
|
|
342
400
|
artifacts: {
|
|
343
401
|
failedTests: context.relativePaths.failedTests,
|
|
344
|
-
generateMap:
|
|
345
|
-
generateBacklog:
|
|
402
|
+
generateMap: ARTIFACT_RELATIVE_PATHS.generateMap,
|
|
403
|
+
generateBacklog: ARTIFACT_RELATIVE_PATHS.generateBacklog,
|
|
346
404
|
fixHandoff: context.relativePaths.fixHandoff
|
|
347
405
|
},
|
|
348
406
|
items,
|
|
@@ -423,7 +481,7 @@ function buildFixCandidateFiles(entry, backlogMatch) {
|
|
|
423
481
|
function buildFixNextActions(summary) {
|
|
424
482
|
const actions = [];
|
|
425
483
|
if (summary.generatedFailures > 0) {
|
|
426
|
-
actions.push(
|
|
484
|
+
actions.push(`Review ${ARTIFACT_RELATIVE_PATHS.fixHandoff} and start with source-drift items.`);
|
|
427
485
|
actions.push('Regenerate narrow targets before rerunning the full suite.');
|
|
428
486
|
}
|
|
429
487
|
if (summary.generatedFailures === 0) {
|
|
@@ -436,11 +494,21 @@ function roundDuration(value) {
|
|
|
436
494
|
return Math.round(Number(value || 0) * 100) / 100;
|
|
437
495
|
}
|
|
438
496
|
|
|
497
|
+
function removeLegacyArtifact(cwd, key) {
|
|
498
|
+
const [currentPath, ...legacyPaths] = getArtifactPathCandidates(cwd, key);
|
|
499
|
+
for (const legacyPath of legacyPaths) {
|
|
500
|
+
if (legacyPath !== currentPath && fs.existsSync(legacyPath)) {
|
|
501
|
+
fs.rmSync(legacyPath, { force: true });
|
|
502
|
+
}
|
|
503
|
+
}
|
|
504
|
+
}
|
|
505
|
+
|
|
439
506
|
function stringifyArtifact(value) {
|
|
440
507
|
return JSON.stringify(value);
|
|
441
508
|
}
|
|
442
509
|
|
|
443
510
|
module.exports = {
|
|
444
511
|
writeRunArtifacts,
|
|
445
|
-
readFailedTestsArtifact
|
|
512
|
+
readFailedTestsArtifact,
|
|
513
|
+
readFixHandoffArtifact
|
|
446
514
|
};
|
package/src/cli.js
CHANGED
|
@@ -3,10 +3,11 @@ const { loadConfig } = require('./config');
|
|
|
3
3
|
const { discoverTests } = require('./discovery');
|
|
4
4
|
const { runTests } = require('./runner');
|
|
5
5
|
const { printSpec, printJson, printAgent, printNext, writeHtmlReport } = require('./reporter');
|
|
6
|
+
const { ARTIFACT_RELATIVE_PATHS } = require('./artifact-paths');
|
|
6
7
|
const { runInit } = require('./init');
|
|
7
8
|
const { runMigrate } = require('./migrate');
|
|
8
9
|
const { generateTestsFromSource, writeGenerateArtifacts } = require('./generate');
|
|
9
|
-
const { writeRunArtifacts, readFailedTestsArtifact } = require('./artifacts');
|
|
10
|
+
const { writeRunArtifacts, readFailedTestsArtifact, readFixHandoffArtifact } = require('./artifacts');
|
|
10
11
|
const { buildStabilityReport, hasStabilityBreaches } = require('./stability');
|
|
11
12
|
const { verdictReveal } = require('./verdict');
|
|
12
13
|
const { runWatchMode } = require('./watch');
|
|
@@ -21,7 +22,7 @@ async function main(argv) {
|
|
|
21
22
|
|
|
22
23
|
if (command === 'init') {
|
|
23
24
|
runInit(cwd);
|
|
24
|
-
console.log('Themis initialized.
|
|
25
|
+
console.log('Themis initialized. Next: npx themis generate src && npx themis test');
|
|
25
26
|
return;
|
|
26
27
|
}
|
|
27
28
|
|
|
@@ -77,6 +78,9 @@ async function main(argv) {
|
|
|
77
78
|
if (result.packageUpdated && result.packageJsonPath) {
|
|
78
79
|
console.log(`Scripts: updated ${formatCliPath(cwd, result.packageJsonPath)} with test:themis`);
|
|
79
80
|
}
|
|
81
|
+
if (result.gitignoreUpdated) {
|
|
82
|
+
console.log(`Gitignore: updated ${formatCliPath(cwd, result.gitignorePath)} with .themis/`);
|
|
83
|
+
}
|
|
80
84
|
if (result.rewriteImports) {
|
|
81
85
|
console.log(`Imports: rewrote ${result.rewrittenFiles.length} file(s) to local Themis compatibility imports.`);
|
|
82
86
|
}
|
|
@@ -124,6 +128,62 @@ async function main(argv) {
|
|
|
124
128
|
}
|
|
125
129
|
|
|
126
130
|
async function executeTestRun(cwd, flags) {
|
|
131
|
+
const execution = flags.fix
|
|
132
|
+
? await executeTestFixRun(cwd, flags)
|
|
133
|
+
: await runTestCommand(cwd, flags);
|
|
134
|
+
|
|
135
|
+
await finalizeTestExecution(execution);
|
|
136
|
+
}
|
|
137
|
+
|
|
138
|
+
async function executeTestFixRun(cwd, flags) {
|
|
139
|
+
let fixArtifact = readFixHandoffArtifact(cwd);
|
|
140
|
+
|
|
141
|
+
if (fixArtifact && fixArtifact.parseError) {
|
|
142
|
+
return {
|
|
143
|
+
notices: [
|
|
144
|
+
`Failed to parse fix handoff artifact at ${fixArtifact.fixHandoffPath}: ${fixArtifact.parseError}. Run npx themis test to regenerate it.`
|
|
145
|
+
],
|
|
146
|
+
...(await runTestCommand(cwd, { ...flags, fix: false }))
|
|
147
|
+
};
|
|
148
|
+
}
|
|
149
|
+
|
|
150
|
+
let seededExecution = null;
|
|
151
|
+
if (!fixArtifact || fixArtifact.items.length === 0) {
|
|
152
|
+
seededExecution = await runTestCommand(cwd, { ...flags, fix: false });
|
|
153
|
+
fixArtifact = readFixHandoffArtifact(cwd);
|
|
154
|
+
|
|
155
|
+
if (fixArtifact && fixArtifact.parseError) {
|
|
156
|
+
return {
|
|
157
|
+
...seededExecution,
|
|
158
|
+
notices: [
|
|
159
|
+
`Failed to parse fix handoff artifact at ${fixArtifact.fixHandoffPath}: ${fixArtifact.parseError}.`
|
|
160
|
+
]
|
|
161
|
+
};
|
|
162
|
+
}
|
|
163
|
+
|
|
164
|
+
if (!fixArtifact || fixArtifact.items.length === 0) {
|
|
165
|
+
const notices = seededExecution.result.summary.failed > 0
|
|
166
|
+
? [`No generated-test autofixes were available for this run. Review ${ARTIFACT_RELATIVE_PATHS.fixHandoff} when generated tests fail.`]
|
|
167
|
+
: ['No generated-test repairs were needed.'];
|
|
168
|
+
return {
|
|
169
|
+
...seededExecution,
|
|
170
|
+
notices
|
|
171
|
+
};
|
|
172
|
+
}
|
|
173
|
+
}
|
|
174
|
+
|
|
175
|
+
const fixSummary = applyGeneratedAutofix(cwd, fixArtifact.payload || { items: fixArtifact.items });
|
|
176
|
+
const rerunExecution = await runTestCommand(cwd, { ...flags, fix: false });
|
|
177
|
+
return {
|
|
178
|
+
...rerunExecution,
|
|
179
|
+
notices: [
|
|
180
|
+
`Applied generated-test fixes for ${fixSummary.sources.length} source file(s).`,
|
|
181
|
+
...fixSummary.messages
|
|
182
|
+
]
|
|
183
|
+
};
|
|
184
|
+
}
|
|
185
|
+
|
|
186
|
+
async function runTestCommand(cwd, flags) {
|
|
127
187
|
const config = loadConfig(cwd);
|
|
128
188
|
|
|
129
189
|
if (flags.match) {
|
|
@@ -141,29 +201,29 @@ async function executeTestRun(cwd, flags) {
|
|
|
141
201
|
if (flags.isolation) {
|
|
142
202
|
validateIsolation(flags.isolation);
|
|
143
203
|
}
|
|
144
|
-
printBanner(reporter);
|
|
145
204
|
const maxWorkers = resolveWorkerCount(flags.workers, config.maxWorkers);
|
|
146
205
|
const stabilityRuns = resolveStabilityRuns(flags.stability);
|
|
147
206
|
|
|
148
207
|
let files = discoverTests(cwd, config);
|
|
149
208
|
if (files.length === 0) {
|
|
150
|
-
|
|
151
|
-
return;
|
|
209
|
+
return buildNoResultExecution(reporter, lexicon, cwd, flags, [`No test files found in ${config.testDir}`]);
|
|
152
210
|
}
|
|
153
211
|
|
|
154
212
|
let allowedFullNames = null;
|
|
155
213
|
if (flags.rerunFailed) {
|
|
156
214
|
const artifact = readFailedTestsArtifact(cwd);
|
|
157
215
|
if (artifact && artifact.parseError) {
|
|
158
|
-
|
|
159
|
-
|
|
216
|
+
return buildNoResultExecution(
|
|
217
|
+
reporter,
|
|
218
|
+
lexicon,
|
|
219
|
+
cwd,
|
|
220
|
+
flags,
|
|
221
|
+
[`Failed to parse failed test artifact at ${artifact.failuresPath}: ${artifact.parseError}. Run a full test pass to regenerate it.`]
|
|
160
222
|
);
|
|
161
|
-
return;
|
|
162
223
|
}
|
|
163
224
|
|
|
164
225
|
if (!artifact || artifact.failedTests.length === 0) {
|
|
165
|
-
|
|
166
|
-
return;
|
|
226
|
+
return buildNoResultExecution(reporter, lexicon, cwd, flags, ['No failed test artifact found. Run a failing test first.']);
|
|
167
227
|
}
|
|
168
228
|
|
|
169
229
|
const fileSet = new Set(artifact.failedTests.map((entry) => entry.file));
|
|
@@ -171,8 +231,7 @@ async function executeTestRun(cwd, flags) {
|
|
|
171
231
|
files = files.filter((file) => fileSet.has(file));
|
|
172
232
|
|
|
173
233
|
if (files.length === 0) {
|
|
174
|
-
|
|
175
|
-
return;
|
|
234
|
+
return buildNoResultExecution(reporter, lexicon, cwd, flags, ['No matching files found for failed test artifact.']);
|
|
176
235
|
}
|
|
177
236
|
}
|
|
178
237
|
|
|
@@ -201,20 +260,95 @@ async function executeTestRun(cwd, flags) {
|
|
|
201
260
|
}
|
|
202
261
|
|
|
203
262
|
writeRunArtifacts(cwd, result);
|
|
204
|
-
|
|
263
|
+
return {
|
|
264
|
+
reporter,
|
|
205
265
|
lexicon,
|
|
266
|
+
result,
|
|
206
267
|
cwd,
|
|
207
268
|
htmlOutput: flags.htmlOutput || null
|
|
269
|
+
};
|
|
270
|
+
}
|
|
271
|
+
|
|
272
|
+
async function finalizeTestExecution(execution) {
|
|
273
|
+
printBanner(execution.reporter);
|
|
274
|
+
|
|
275
|
+
if (execution.reporter !== 'json' && execution.reporter !== 'agent') {
|
|
276
|
+
for (const notice of execution.notices || []) {
|
|
277
|
+
console.log(notice);
|
|
278
|
+
}
|
|
279
|
+
}
|
|
280
|
+
|
|
281
|
+
if (!execution.result) {
|
|
282
|
+
return;
|
|
283
|
+
}
|
|
284
|
+
|
|
285
|
+
printResult(execution.reporter, execution.result, {
|
|
286
|
+
lexicon: execution.lexicon,
|
|
287
|
+
cwd: execution.cwd,
|
|
288
|
+
htmlOutput: execution.htmlOutput || null
|
|
208
289
|
});
|
|
209
290
|
|
|
210
|
-
const revealFailed = result.summary.failed > 0 || hasStabilityBreaches(
|
|
211
|
-
await maybeRevealVerdict(reporter, result,
|
|
291
|
+
const revealFailed = execution.result.summary.failed > 0 || hasStabilityBreaches(execution.result.stability);
|
|
292
|
+
await maybeRevealVerdict(execution.reporter, execution.result, execution.result.stability, revealFailed);
|
|
212
293
|
|
|
213
294
|
if (revealFailed) {
|
|
214
295
|
process.exitCode = 1;
|
|
215
296
|
}
|
|
216
297
|
}
|
|
217
298
|
|
|
299
|
+
function buildNoResultExecution(reporter, lexicon, cwd, flags, notices) {
|
|
300
|
+
return {
|
|
301
|
+
reporter,
|
|
302
|
+
lexicon,
|
|
303
|
+
cwd,
|
|
304
|
+
htmlOutput: flags.htmlOutput || null,
|
|
305
|
+
notices,
|
|
306
|
+
result: null
|
|
307
|
+
};
|
|
308
|
+
}
|
|
309
|
+
|
|
310
|
+
function applyGeneratedAutofix(cwd, fixPayload) {
|
|
311
|
+
const bySourceFile = new Map();
|
|
312
|
+
for (const item of fixPayload.items || []) {
|
|
313
|
+
if (!item || !item.sourceFile) {
|
|
314
|
+
continue;
|
|
315
|
+
}
|
|
316
|
+
const current = bySourceFile.get(item.sourceFile);
|
|
317
|
+
if (!current) {
|
|
318
|
+
bySourceFile.set(item.sourceFile, item);
|
|
319
|
+
continue;
|
|
320
|
+
}
|
|
321
|
+
if (current.repairStrategy !== 'tighten-hints' && item.repairStrategy === 'tighten-hints') {
|
|
322
|
+
bySourceFile.set(item.sourceFile, item);
|
|
323
|
+
}
|
|
324
|
+
}
|
|
325
|
+
|
|
326
|
+
const sources = [...bySourceFile.keys()];
|
|
327
|
+
const messages = [];
|
|
328
|
+
|
|
329
|
+
for (const sourceFile of sources) {
|
|
330
|
+
const item = bySourceFile.get(sourceFile);
|
|
331
|
+
const writeHints = item.repairStrategy === 'tighten-hints' || item.category === 'generated-contract-failure';
|
|
332
|
+
const summary = generateTestsFromSource(cwd, {
|
|
333
|
+
targetDir: sourceFile,
|
|
334
|
+
update: true,
|
|
335
|
+
writeHints
|
|
336
|
+
});
|
|
337
|
+
writeGenerateArtifacts(summary, cwd);
|
|
338
|
+
|
|
339
|
+
const createdHintCount = Number(summary.hintFiles?.created?.length || 0);
|
|
340
|
+
const updatedHintCount = Number(summary.hintFiles?.updated?.length || 0);
|
|
341
|
+
if (writeHints && (createdHintCount > 0 || updatedHintCount > 0)) {
|
|
342
|
+
messages.push(`Updated hints for ${sourceFile} (${createdHintCount} created, ${updatedHintCount} updated).`);
|
|
343
|
+
}
|
|
344
|
+
}
|
|
345
|
+
|
|
346
|
+
return {
|
|
347
|
+
sources,
|
|
348
|
+
messages
|
|
349
|
+
};
|
|
350
|
+
}
|
|
351
|
+
|
|
218
352
|
function resolveReporter(flags, config) {
|
|
219
353
|
if (flags.reporter) {
|
|
220
354
|
return flags.reporter;
|
|
@@ -292,6 +426,10 @@ function parseFlags(args) {
|
|
|
292
426
|
flags.updateContracts = true;
|
|
293
427
|
continue;
|
|
294
428
|
}
|
|
429
|
+
if (token === '--fix') {
|
|
430
|
+
flags.fix = true;
|
|
431
|
+
continue;
|
|
432
|
+
}
|
|
295
433
|
if (token === '-w' || token === '--watch') {
|
|
296
434
|
flags.watch = true;
|
|
297
435
|
continue;
|
|
@@ -485,7 +623,7 @@ function parseGenerateFlags(args) {
|
|
|
485
623
|
function validateRegex(pattern) {
|
|
486
624
|
try {
|
|
487
625
|
new RegExp(pattern);
|
|
488
|
-
} catch
|
|
626
|
+
} catch {
|
|
489
627
|
throw new Error(`Invalid --match regex: ${pattern}`);
|
|
490
628
|
}
|
|
491
629
|
}
|
|
@@ -572,12 +710,12 @@ function resolveStabilityRuns(value) {
|
|
|
572
710
|
function printUsage() {
|
|
573
711
|
console.log('Usage: themis <command> [options]');
|
|
574
712
|
console.log('Commands:');
|
|
575
|
-
console.log(' init Create themis.config.json and
|
|
713
|
+
console.log(' init Create themis.config.json and gitignore .themis/ artifacts');
|
|
576
714
|
console.log(' generate [path] Scan source files and generate Themis contract tests');
|
|
577
715
|
console.log(' Options: [--json] [--plan] [--output path] [--files a,b] [--match-source regex] [--match-export regex] [--scenario name] [--min-confidence level] [--require-confidence level] [--include regex] [--exclude regex] [--review] [--update] [--clean] [--changed] [--force] [--strict] [--write-hints] [--fail-on-skips] [--fail-on-conflicts]');
|
|
578
716
|
console.log(' scan [path] Alias for generate');
|
|
579
717
|
console.log(' migrate <jest|vitest> [--rewrite-imports] [--convert] Scaffold an incremental migration bridge for existing suites');
|
|
580
|
-
console.log(' test [--json] [--agent] [--next] [--reporter spec|next|json|agent|html] [--workers N] [--stability N] [--environment node|jsdom] [--isolation worker|in-process] [--cache] [--update-contracts] [-w|--watch] [--html-output path] [--match regex] [--rerun-failed] [--no-memes] [--lexicon classic|themis]');
|
|
718
|
+
console.log(' test [--json] [--agent] [--next] [--reporter spec|next|json|agent|html] [--workers N] [--stability N] [--environment node|jsdom] [--isolation worker|in-process] [--cache] [--update-contracts] [--fix] [-w|--watch] [--html-output path] [--match regex] [--rerun-failed] [--no-memes] [--lexicon classic|themis]');
|
|
581
719
|
}
|
|
582
720
|
|
|
583
721
|
function printGenerateSummary(summary, cwd) {
|