@wix/pathgrade 0.36.0 → 1.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +24 -0
- package/README.md +508 -0
- package/dist/adapter-kit/index.d.ts +17 -0
- package/dist/adapter-kit/index.js +14 -0
- package/dist/adapters/jest/index.d.ts +3 -0
- package/dist/adapters/jest/index.js +3 -0
- package/dist/adapters/jest/invocation-adapter.d.ts +15 -0
- package/dist/adapters/jest/invocation-adapter.js +115 -0
- package/dist/adapters/jest/lifecycle.d.ts +21 -0
- package/dist/adapters/jest/lifecycle.js +62 -0
- package/dist/adapters/jest/metadata.d.ts +6 -0
- package/dist/adapters/jest/metadata.js +35 -0
- package/dist/adapters/jest/reporter.cjs +10 -0
- package/dist/adapters/jest/reporter.d.ts +5 -0
- package/dist/adapters/jest/reporter.js +67 -0
- package/dist/adapters/jest/results.d.ts +31 -0
- package/dist/adapters/jest/results.js +144 -0
- package/dist/adapters/jest/runner-adapter.d.ts +9 -0
- package/dist/adapters/jest/runner-adapter.js +49 -0
- package/dist/adapters/jest/setup.d.ts +1 -0
- package/dist/adapters/jest/setup.js +10 -0
- package/dist/adapters/node-test/index.d.ts +8 -0
- package/dist/adapters/node-test/index.js +106 -0
- package/dist/adapters/node-test/invocation-adapter.d.ts +5 -0
- package/dist/adapters/node-test/invocation-adapter.js +47 -0
- package/dist/adapters/node-test/runner-adapter.d.ts +2 -0
- package/dist/adapters/node-test/runner-adapter.js +116 -0
- package/dist/adapters/vitest/index.d.ts +9 -0
- package/dist/adapters/vitest/index.js +82 -0
- package/dist/adapters/vitest/lifecycle.d.ts +24 -0
- package/dist/adapters/vitest/lifecycle.js +41 -0
- package/dist/adapters/vitest/reporter.d.ts +12 -0
- package/dist/adapters/vitest/reporter.js +75 -0
- package/dist/adapters/vitest/setup.d.ts +1 -0
- package/dist/adapters/vitest/setup.js +3 -0
- package/dist/agents/codex-app-server/agent.js +5 -1
- package/dist/agents/codex-app-server/protocol/index.js +3 -3
- package/dist/agents/codex-app-server/turn-completion.d.ts +1 -0
- package/dist/agents/codex-app-server/turn-completion.js +9 -0
- package/dist/commands/affected.js +22 -15
- package/dist/commands/run-args.d.ts +4 -3
- package/dist/commands/run-args.js +12 -6
- package/dist/commands/run-changed.d.ts +4 -4
- package/dist/commands/run-changed.js +45 -58
- package/dist/config/pathgrade.d.ts +48 -0
- package/dist/config/pathgrade.js +204 -0
- package/dist/pathgrade.d.ts +2 -2
- package/dist/pathgrade.js +35 -34
- package/dist/plugin/index.d.ts +2 -9
- package/dist/plugin/index.js +1 -81
- package/dist/plugin/lifecycle.d.ts +1 -45
- package/dist/plugin/lifecycle.js +1 -146
- package/dist/plugin/reporter.d.ts +1 -37
- package/dist/plugin/reporter.js +1 -280
- package/dist/plugin/setup.d.ts +1 -1
- package/dist/plugin/setup.js +1 -3
- package/dist/providers/copy-filter.d.ts +5 -0
- package/dist/providers/copy-filter.js +9 -0
- package/dist/providers/sandbox.js +2 -2
- package/dist/providers/workspace.d.ts +6 -0
- package/dist/providers/workspace.js +34 -14
- package/dist/reporters/diagnostics.d.ts +2 -40
- package/dist/reporters/diagnostics.js +1 -80
- package/dist/reporters/report-summary.d.ts +6 -0
- package/dist/reporters/report-summary.js +29 -0
- package/dist/reporting/artifacts.d.ts +2 -0
- package/dist/reporting/artifacts.js +20 -0
- package/dist/reporting/core.d.ts +2 -0
- package/dist/reporting/core.js +173 -0
- package/dist/reporting/types.d.ts +57 -0
- package/dist/reporting/types.js +1 -0
- package/dist/reporting/vitest-edge.d.ts +1 -0
- package/dist/reporting/vitest-edge.js +1 -0
- package/dist/runners/adapter-loader.d.ts +17 -0
- package/dist/runners/adapter-loader.js +67 -0
- package/dist/runners/adapter.d.ts +65 -0
- package/dist/runners/adapter.js +1 -0
- package/dist/runners/invocation.d.ts +10 -0
- package/dist/runners/invocation.js +1 -0
- package/dist/runners/lifecycle-hooks.d.ts +2 -0
- package/dist/runners/lifecycle-hooks.js +18 -0
- package/dist/runners/model-builders.d.ts +4 -0
- package/dist/runners/model-builders.js +67 -0
- package/dist/runners/model-validation.d.ts +12 -0
- package/dist/runners/model-validation.js +213 -0
- package/dist/runners/model.d.ts +117 -0
- package/dist/runners/model.js +1 -0
- package/dist/runners/orchestrator.d.ts +30 -0
- package/dist/runners/orchestrator.js +74 -0
- package/dist/runners/report-projection.d.ts +6 -0
- package/dist/runners/report-projection.js +62 -0
- package/dist/runners/selection.d.ts +6 -0
- package/dist/runners/selection.js +10 -0
- package/dist/runners/vitest-adapter.d.ts +8 -0
- package/dist/runners/vitest-adapter.js +108 -0
- package/dist/runners/vitest-invocation.d.ts +10 -0
- package/dist/runners/vitest-invocation.js +44 -0
- package/dist/runners/vitest-lifecycle.d.ts +43 -0
- package/dist/runners/vitest-lifecycle.js +150 -0
- package/dist/sdk/agent.js +15 -16
- package/dist/sdk/case-context.d.ts +20 -0
- package/dist/sdk/case-context.js +31 -0
- package/dist/sdk/diagnostics.d.ts +40 -0
- package/dist/sdk/diagnostics.js +80 -0
- package/dist/sdk/eval-runtime.d.ts +4 -0
- package/dist/sdk/eval-runtime.js +11 -1
- package/dist/sdk/evaluate.js +2 -1
- package/dist/sdk/index.d.ts +2 -0
- package/dist/sdk/index.js +1 -0
- package/dist/sdk/lifecycle.d.ts +40 -0
- package/dist/sdk/lifecycle.js +170 -0
- package/dist/sdk/result-capture.d.ts +18 -0
- package/dist/sdk/result-capture.js +49 -0
- package/dist/sdk/types.d.ts +1 -1
- package/dist/types.d.ts +2 -2
- package/dist/utils/llm-providers/anthropic.js +9 -2
- package/package.json +73 -36
|
@@ -0,0 +1,106 @@
|
|
|
1
|
+
import nodeTest from 'node:test';
|
|
2
|
+
import * as fs from 'node:fs';
|
|
3
|
+
import * as path from 'node:path';
|
|
4
|
+
import { fileURLToPath } from 'node:url';
|
|
5
|
+
import { subscribeToEvalResults } from '../../sdk/result-capture.js';
|
|
6
|
+
import { createRunnerLifecycleHooks } from '../../runners/lifecycle-hooks.js';
|
|
7
|
+
const lifecycle = createRunnerLifecycleHooks();
|
|
8
|
+
let subscribed = false;
|
|
9
|
+
let caseCounter = 0;
|
|
10
|
+
export function test(name, optionsOrFn, maybeFn) {
|
|
11
|
+
installResultCapture();
|
|
12
|
+
const options = typeof optionsOrFn === 'function' ? undefined : optionsOrFn;
|
|
13
|
+
const fn = typeof optionsOrFn === 'function' ? optionsOrFn : maybeFn;
|
|
14
|
+
const sourceRef = callerSourceRef();
|
|
15
|
+
const caseId = `${sourceRef}:${name}:${++caseCounter}`;
|
|
16
|
+
nodeTest(name, options, async (ctx) => {
|
|
17
|
+
const startedAt = performance.now();
|
|
18
|
+
let state = 'passed';
|
|
19
|
+
let thrown;
|
|
20
|
+
try {
|
|
21
|
+
await lifecycle.withCaseContext({
|
|
22
|
+
caseId,
|
|
23
|
+
caseName: name,
|
|
24
|
+
filePath: sourceRef,
|
|
25
|
+
sourceRef,
|
|
26
|
+
runnerNativeId: caseId,
|
|
27
|
+
scope: 'runner-case',
|
|
28
|
+
}, async () => {
|
|
29
|
+
await fn?.(ctx);
|
|
30
|
+
});
|
|
31
|
+
}
|
|
32
|
+
catch (err) {
|
|
33
|
+
state = 'failed';
|
|
34
|
+
thrown = err;
|
|
35
|
+
}
|
|
36
|
+
finally {
|
|
37
|
+
const evaluations = await lifecycle.flushCase(caseId);
|
|
38
|
+
writeCase({
|
|
39
|
+
caseId,
|
|
40
|
+
name,
|
|
41
|
+
state,
|
|
42
|
+
runnerDurationMs: performance.now() - startedAt,
|
|
43
|
+
sourceRef,
|
|
44
|
+
filePath: sourceRef,
|
|
45
|
+
groupName: sourceRef,
|
|
46
|
+
runnerCaseId: caseId,
|
|
47
|
+
evaluations: toReportEvaluations(evaluations),
|
|
48
|
+
});
|
|
49
|
+
}
|
|
50
|
+
if (thrown)
|
|
51
|
+
throw thrown;
|
|
52
|
+
});
|
|
53
|
+
}
|
|
54
|
+
export default test;
|
|
55
|
+
export { createNodeTestAdapter } from './runner-adapter.js';
|
|
56
|
+
export { createNodeTestInvocationAdapter } from './invocation-adapter.js';
|
|
57
|
+
function installResultCapture() {
|
|
58
|
+
if (subscribed)
|
|
59
|
+
return;
|
|
60
|
+
subscribed = true;
|
|
61
|
+
subscribeToEvalResults(event => lifecycle.onResult(event), {
|
|
62
|
+
owner: 'adapter',
|
|
63
|
+
key: 'node-test-lifecycle',
|
|
64
|
+
});
|
|
65
|
+
}
|
|
66
|
+
function toReportEvaluations(evaluations) {
|
|
67
|
+
if (evaluations.length === 0)
|
|
68
|
+
return undefined;
|
|
69
|
+
return evaluations.map(entry => ({
|
|
70
|
+
score: entry.score,
|
|
71
|
+
trial: entry.trial,
|
|
72
|
+
diagnostics: entry.diagnostics,
|
|
73
|
+
}));
|
|
74
|
+
}
|
|
75
|
+
function writeCase(testCase) {
|
|
76
|
+
const resultsPath = process.env.PATHGRADE_NODE_TEST_RESULTS;
|
|
77
|
+
if (!resultsPath)
|
|
78
|
+
return;
|
|
79
|
+
fs.mkdirSync(path.dirname(resultsPath), { recursive: true });
|
|
80
|
+
const existing = fs.existsSync(resultsPath)
|
|
81
|
+
? JSON.parse(fs.readFileSync(resultsPath, 'utf8'))
|
|
82
|
+
: [];
|
|
83
|
+
existing.push(testCase);
|
|
84
|
+
fs.writeFileSync(resultsPath, `${JSON.stringify(existing, null, 2)}\n`);
|
|
85
|
+
}
|
|
86
|
+
function callerSourceRef() {
|
|
87
|
+
const stack = new Error().stack ?? '';
|
|
88
|
+
const cwd = realpath(process.env.PATHGRADE_NODE_TEST_CWD ?? process.cwd());
|
|
89
|
+
for (const line of stack.split('\n')) {
|
|
90
|
+
const match = line.match(/\(?((?:file:\/\/)?[^() ]+\.eval\.[cm]?[jt]s)(?::\d+)?(?::\d+)?\)?/);
|
|
91
|
+
if (!match)
|
|
92
|
+
continue;
|
|
93
|
+
const raw = match[1];
|
|
94
|
+
const abs = realpath(raw.startsWith('file://') ? fileURLToPath(raw) : raw);
|
|
95
|
+
return path.relative(cwd, abs).replaceAll(path.sep, '/');
|
|
96
|
+
}
|
|
97
|
+
return 'node-test';
|
|
98
|
+
}
|
|
99
|
+
function realpath(value) {
|
|
100
|
+
try {
|
|
101
|
+
return fs.realpathSync(value);
|
|
102
|
+
}
|
|
103
|
+
catch {
|
|
104
|
+
return value;
|
|
105
|
+
}
|
|
106
|
+
}
|
|
@@ -0,0 +1,5 @@
|
|
|
1
|
+
import type { ResolvedPathgradeConfig } from '../../config/pathgrade.js';
|
|
2
|
+
import type { RunnerInvocationAdapter } from '../../runners/invocation.js';
|
|
3
|
+
export declare function createNodeTestInvocationAdapter(input: {
|
|
4
|
+
config: ResolvedPathgradeConfig;
|
|
5
|
+
}): RunnerInvocationAdapter;
|
|
@@ -0,0 +1,47 @@
|
|
|
1
|
+
import { readSidecar } from '../../affected/sidecar.js';
|
|
2
|
+
import { getPathgradeDir } from '../../reporters/results-path.js';
|
|
3
|
+
import { printReportSummary } from '../../reporters/report-summary.js';
|
|
4
|
+
import { runWithAdapter } from '../../runners/orchestrator.js';
|
|
5
|
+
import { createNodeTestAdapter } from './runner-adapter.js';
|
|
6
|
+
export function createNodeTestInvocationAdapter(input) {
|
|
7
|
+
return {
|
|
8
|
+
name: 'node-test',
|
|
9
|
+
async run(runInput) {
|
|
10
|
+
const selection = await readSidecar(runInput.cwd, msg => {
|
|
11
|
+
process.stderr.write(`[pathgrade] ${msg}\n`);
|
|
12
|
+
}) ?? undefined;
|
|
13
|
+
const include = runInput.selectedFiles && runInput.selectedFiles.length > 0
|
|
14
|
+
? runInput.selectedFiles
|
|
15
|
+
: input.config.evals.include;
|
|
16
|
+
return await runWithAdapter({
|
|
17
|
+
adapter: createNodeTestAdapter(),
|
|
18
|
+
options: {
|
|
19
|
+
cwd: runInput.cwd,
|
|
20
|
+
discovery: {
|
|
21
|
+
cwd: runInput.cwd,
|
|
22
|
+
include,
|
|
23
|
+
exclude: runInput.selectedFiles ? [] : input.config.evals.exclude,
|
|
24
|
+
},
|
|
25
|
+
runnerArgs: runInput.runnerArgs,
|
|
26
|
+
env: runInput.env,
|
|
27
|
+
artifactRoot: getPathgradeDir(runInput.cwd),
|
|
28
|
+
reporterMode: input.config.reporter ?? 'cli',
|
|
29
|
+
threshold: input.config.ci.threshold,
|
|
30
|
+
selection,
|
|
31
|
+
printSummary: summaries => {
|
|
32
|
+
printReportSummary(summaries, {
|
|
33
|
+
forceVerbose: input.config.diagnostics
|
|
34
|
+
|| runInput.env.PATHGRADE_DIAGNOSTICS === '1',
|
|
35
|
+
});
|
|
36
|
+
},
|
|
37
|
+
log: () => {
|
|
38
|
+
process.stdout.write(`\n Results written to ${getPathgradeDir(runInput.cwd)}\n\n`);
|
|
39
|
+
},
|
|
40
|
+
onThresholdFailure: ({ overallPassRate, threshold }) => {
|
|
41
|
+
process.stdout.write(`\n CI THRESHOLD FAILED avg score ${overallPassRate.toFixed(3)} < threshold ${threshold}\n\n`);
|
|
42
|
+
},
|
|
43
|
+
},
|
|
44
|
+
});
|
|
45
|
+
},
|
|
46
|
+
};
|
|
47
|
+
}
|
|
@@ -0,0 +1,116 @@
|
|
|
1
|
+
import * as path from 'node:path';
|
|
2
|
+
import * as os from 'node:os';
|
|
3
|
+
import { spawn } from 'node:child_process';
|
|
4
|
+
import fs from 'fs-extra';
|
|
5
|
+
import { discoverPathgradeEvalFiles } from '../../evals/discovery.js';
|
|
6
|
+
import { buildNormalizedRunSnapshotFromReportGroups } from '../../runners/model-builders.js';
|
|
7
|
+
export function createNodeTestAdapter() {
|
|
8
|
+
return {
|
|
9
|
+
name: 'node-test',
|
|
10
|
+
async discover(input) {
|
|
11
|
+
const include = input.include ?? ['**/*.eval.ts'];
|
|
12
|
+
const exclude = input.exclude ?? [];
|
|
13
|
+
const files = discoverPathgradeEvalFiles({
|
|
14
|
+
cwd: input.cwd,
|
|
15
|
+
include,
|
|
16
|
+
exclude,
|
|
17
|
+
});
|
|
18
|
+
return {
|
|
19
|
+
units: files.map(file => ({
|
|
20
|
+
id: file,
|
|
21
|
+
displayName: file,
|
|
22
|
+
sourceRef: file,
|
|
23
|
+
native: { file, cwd: input.cwd },
|
|
24
|
+
})),
|
|
25
|
+
};
|
|
26
|
+
},
|
|
27
|
+
async invoke(input) {
|
|
28
|
+
const resultsDir = await fs.mkdtemp(path.join(os.tmpdir(), 'pathgrade-node-test-'));
|
|
29
|
+
const resultsPath = path.join(resultsDir, 'cases.json');
|
|
30
|
+
const cwd = readDiscoveredCwd(input.discovered.units[0]?.native) ?? process.cwd();
|
|
31
|
+
const files = input.discovered.units
|
|
32
|
+
.map(unit => unit.sourceRef)
|
|
33
|
+
.filter((sourceRef) => typeof sourceRef === 'string');
|
|
34
|
+
if (files.length === 0) {
|
|
35
|
+
await fs.writeJson(resultsPath, []);
|
|
36
|
+
return {
|
|
37
|
+
adapterName: this.name,
|
|
38
|
+
status: 'completed',
|
|
39
|
+
exitCode: 0,
|
|
40
|
+
native: { resultsPath },
|
|
41
|
+
};
|
|
42
|
+
}
|
|
43
|
+
const exitCode = await spawnNodeTest({
|
|
44
|
+
testCwd: cwd,
|
|
45
|
+
files,
|
|
46
|
+
argv: input.argv,
|
|
47
|
+
env: {
|
|
48
|
+
...input.env,
|
|
49
|
+
PATHGRADE_NODE_TEST_RESULTS: resultsPath,
|
|
50
|
+
PATHGRADE_NODE_TEST_CWD: cwd,
|
|
51
|
+
},
|
|
52
|
+
});
|
|
53
|
+
return {
|
|
54
|
+
adapterName: this.name,
|
|
55
|
+
status: input.signal?.aborted ? 'cancelled' : (exitCode === 0 ? 'completed' : 'failed'),
|
|
56
|
+
exitCode,
|
|
57
|
+
native: { resultsPath },
|
|
58
|
+
};
|
|
59
|
+
},
|
|
60
|
+
async collectNormalizedRunSnapshot(run) {
|
|
61
|
+
const native = readNodeTestRunNative(run);
|
|
62
|
+
const cases = await readCases(native.resultsPath);
|
|
63
|
+
const groupMap = new Map();
|
|
64
|
+
for (const testCase of cases) {
|
|
65
|
+
const groupName = testCase.groupName ?? testCase.filePath ?? testCase.sourceRef ?? 'node-test';
|
|
66
|
+
if (!groupMap.has(groupName))
|
|
67
|
+
groupMap.set(groupName, []);
|
|
68
|
+
groupMap.get(groupName).push(testCase);
|
|
69
|
+
}
|
|
70
|
+
return buildNormalizedRunSnapshotFromReportGroups(run, Array.from(groupMap.entries()).map(([groupName, groupedCases]) => ({
|
|
71
|
+
groupName,
|
|
72
|
+
cases: groupedCases,
|
|
73
|
+
})));
|
|
74
|
+
},
|
|
75
|
+
};
|
|
76
|
+
}
|
|
77
|
+
async function spawnNodeTest(input) {
|
|
78
|
+
const tsxLoader = import.meta.resolve('tsx');
|
|
79
|
+
const args = [
|
|
80
|
+
'--import',
|
|
81
|
+
tsxLoader,
|
|
82
|
+
'--test',
|
|
83
|
+
...input.files,
|
|
84
|
+
...input.argv,
|
|
85
|
+
];
|
|
86
|
+
return await new Promise(resolve => {
|
|
87
|
+
const child = spawn(process.execPath, args, {
|
|
88
|
+
cwd: input.testCwd,
|
|
89
|
+
env: input.env,
|
|
90
|
+
stdio: 'inherit',
|
|
91
|
+
});
|
|
92
|
+
child.on('close', code => resolve(code ?? 0));
|
|
93
|
+
child.on('error', () => resolve(1));
|
|
94
|
+
});
|
|
95
|
+
}
|
|
96
|
+
function readNodeTestRunNative(run) {
|
|
97
|
+
if (typeof run.native === 'object'
|
|
98
|
+
&& run.native !== null
|
|
99
|
+
&& typeof run.native.resultsPath === 'string') {
|
|
100
|
+
return run.native;
|
|
101
|
+
}
|
|
102
|
+
return { resultsPath: '' };
|
|
103
|
+
}
|
|
104
|
+
async function readCases(resultsPath) {
|
|
105
|
+
if (!resultsPath || !(await fs.pathExists(resultsPath)))
|
|
106
|
+
return [];
|
|
107
|
+
return await fs.readJson(resultsPath);
|
|
108
|
+
}
|
|
109
|
+
function readDiscoveredCwd(native) {
|
|
110
|
+
if (typeof native === 'object'
|
|
111
|
+
&& native !== null
|
|
112
|
+
&& typeof native.cwd === 'string') {
|
|
113
|
+
return native.cwd;
|
|
114
|
+
}
|
|
115
|
+
return undefined;
|
|
116
|
+
}
|
|
@@ -0,0 +1,9 @@
|
|
|
1
|
+
import type { PathgradePluginOptions } from '../../sdk/types.js';
|
|
2
|
+
export type { PathgradePluginOptions };
|
|
3
|
+
/**
|
|
4
|
+
* Vitest plugin for pathgrade evaluations.
|
|
5
|
+
*
|
|
6
|
+
* Configures test file patterns, timeout, auto-dispose lifecycle hooks,
|
|
7
|
+
* and the pathgrade reporter for aggregate statistics.
|
|
8
|
+
*/
|
|
9
|
+
export declare function pathgrade(opts?: PathgradePluginOptions): any;
|
|
@@ -0,0 +1,82 @@
|
|
|
1
|
+
import * as fs from 'fs';
|
|
2
|
+
import * as path from 'path';
|
|
3
|
+
import { PathgradeReporter } from './reporter.js';
|
|
4
|
+
import { discoverPathgradeEvalFiles } from '../../evals/discovery.js';
|
|
5
|
+
import { DEFAULT_EVAL_EXCLUDE, DEFAULT_EVAL_INCLUDE } from '../../config/pathgrade.js';
|
|
6
|
+
function resolveSetupFile() {
|
|
7
|
+
// Prefer .ts (running from source via vitest) over .js (built dist)
|
|
8
|
+
const tsPath = path.resolve(import.meta.dirname, 'setup.ts');
|
|
9
|
+
if (fs.existsSync(tsPath))
|
|
10
|
+
return tsPath;
|
|
11
|
+
return path.resolve(import.meta.dirname, 'setup.js');
|
|
12
|
+
}
|
|
13
|
+
/**
|
|
14
|
+
* Vitest plugin for pathgrade evaluations.
|
|
15
|
+
*
|
|
16
|
+
* Configures test file patterns, timeout, auto-dispose lifecycle hooks,
|
|
17
|
+
* and the pathgrade reporter for aggregate statistics.
|
|
18
|
+
*/
|
|
19
|
+
export function pathgrade(opts) {
|
|
20
|
+
const timeoutSec = opts?.timeout ?? 300;
|
|
21
|
+
// Verbose live-streaming seam: the SDK (createAgent) reads
|
|
22
|
+
// `process.env.PATHGRADE_VERBOSE` to decide whether to emit. Wiring the
|
|
23
|
+
// plugin option through the env var keeps a single source of truth and
|
|
24
|
+
// preserves CLI/env precedence — we only set the env if the user hasn't
|
|
25
|
+
// already set it via their shell or the `pathgrade run --verbose` CLI.
|
|
26
|
+
if (opts?.verbose === true && process.env.PATHGRADE_VERBOSE === undefined) {
|
|
27
|
+
process.env.PATHGRADE_VERBOSE = '1';
|
|
28
|
+
}
|
|
29
|
+
return {
|
|
30
|
+
name: 'pathgrade',
|
|
31
|
+
// Surface the user's options to pre-vitest CLI commands
|
|
32
|
+
// (`pathgrade affected`, `pathgrade run --changed`) so they can
|
|
33
|
+
// read `affected.global` after loading vitest.config.ts.
|
|
34
|
+
// See `src/affected/config.ts`.
|
|
35
|
+
__pathgradeOptions: opts ?? {},
|
|
36
|
+
config() {
|
|
37
|
+
return {
|
|
38
|
+
resolve: {
|
|
39
|
+
alias: resolveLocalAliases(),
|
|
40
|
+
},
|
|
41
|
+
test: {
|
|
42
|
+
include: opts?.include ?? DEFAULT_EVAL_INCLUDE,
|
|
43
|
+
exclude: opts?.exclude ?? DEFAULT_EVAL_EXCLUDE,
|
|
44
|
+
testTimeout: (timeoutSec + 30) * 1000,
|
|
45
|
+
setupFiles: [resolveSetupFile()],
|
|
46
|
+
reporters: [
|
|
47
|
+
'default',
|
|
48
|
+
new PathgradeReporter(opts),
|
|
49
|
+
],
|
|
50
|
+
},
|
|
51
|
+
};
|
|
52
|
+
},
|
|
53
|
+
configResolved(resolved) {
|
|
54
|
+
const root = resolved.root ?? process.cwd();
|
|
55
|
+
const include = resolved.test?.include ?? opts?.include ?? DEFAULT_EVAL_INCLUDE;
|
|
56
|
+
const exclude = resolved.test?.exclude ?? opts?.exclude ?? DEFAULT_EVAL_EXCLUDE;
|
|
57
|
+
const discovered = discoverPathgradeEvalFiles({ cwd: root, include, exclude });
|
|
58
|
+
if (!resolved.test) {
|
|
59
|
+
resolved.test = {};
|
|
60
|
+
}
|
|
61
|
+
resolved.test.include = discovered;
|
|
62
|
+
},
|
|
63
|
+
};
|
|
64
|
+
}
|
|
65
|
+
/**
|
|
66
|
+
* When running from source (not installed via npm), resolve
|
|
67
|
+
* `@wix/pathgrade` imports to the local src/ directory so examples
|
|
68
|
+
* can use canonical package imports while developing locally.
|
|
69
|
+
*/
|
|
70
|
+
function resolveLocalAliases() {
|
|
71
|
+
const sdkSource = path.resolve(import.meta.dirname, '..', '..', 'sdk', 'index.ts');
|
|
72
|
+
if (!fs.existsSync(sdkSource))
|
|
73
|
+
return undefined;
|
|
74
|
+
const src = path.resolve(import.meta.dirname, '..', '..');
|
|
75
|
+
return {
|
|
76
|
+
'@wix/pathgrade/mcp-mock': path.join(src, 'core', 'mcp-mock.ts'),
|
|
77
|
+
'@wix/pathgrade/plugin': path.join(src, 'plugin', 'index.ts'),
|
|
78
|
+
'@wix/pathgrade/plugin/vitest': path.join(src, 'adapters', 'vitest', 'index.ts'),
|
|
79
|
+
'@wix/pathgrade/adapters/vitest': path.join(src, 'adapters', 'vitest', 'index.ts'),
|
|
80
|
+
'@wix/pathgrade': path.join(src, 'sdk', 'index.ts'),
|
|
81
|
+
};
|
|
82
|
+
}
|
|
@@ -0,0 +1,24 @@
|
|
|
1
|
+
import type { Agent, RecordedEvalResult } from '../../sdk/types.js';
|
|
2
|
+
import { type VitestAfterAllFn, type VitestAfterEachFn, type VitestAroundEachFn } from '../../runners/vitest-lifecycle.js';
|
|
3
|
+
declare function trackAgent(agent: Agent): void;
|
|
4
|
+
declare function untrackAgent(agent: Agent): void;
|
|
5
|
+
declare function releaseAgent(agent: Agent): void;
|
|
6
|
+
declare function flush(task: {
|
|
7
|
+
id: string;
|
|
8
|
+
meta: Record<string, unknown>;
|
|
9
|
+
}): Promise<void>;
|
|
10
|
+
declare function onResult(result: RecordedEvalResult, agent: Agent): void;
|
|
11
|
+
declare function flushAll(): Promise<void>;
|
|
12
|
+
declare function reset(): void;
|
|
13
|
+
declare function install(afterEach: VitestAfterEachFn, afterAll?: VitestAfterAllFn, aroundEach?: VitestAroundEachFn): void;
|
|
14
|
+
export declare const lifecycle: {
|
|
15
|
+
trackAgent: typeof trackAgent;
|
|
16
|
+
untrackAgent: typeof untrackAgent;
|
|
17
|
+
releaseAgent: typeof releaseAgent;
|
|
18
|
+
flush: typeof flush;
|
|
19
|
+
flushAll: typeof flushAll;
|
|
20
|
+
onResult: typeof onResult;
|
|
21
|
+
reset: typeof reset;
|
|
22
|
+
install: typeof install;
|
|
23
|
+
};
|
|
24
|
+
export {};
|
|
@@ -0,0 +1,41 @@
|
|
|
1
|
+
import { createVitestLifecycleHooks, installVitestLifecycle, releaseVitestAgent, resetVitestLifecycle, trackVitestAgent, untrackVitestAgent, } from '../../runners/vitest-lifecycle.js';
|
|
2
|
+
let installHandle = null;
|
|
3
|
+
function trackAgent(agent) {
|
|
4
|
+
trackVitestAgent(agent, installHandle !== null);
|
|
5
|
+
}
|
|
6
|
+
function untrackAgent(agent) {
|
|
7
|
+
untrackVitestAgent(agent);
|
|
8
|
+
}
|
|
9
|
+
function releaseAgent(agent) {
|
|
10
|
+
releaseVitestAgent(agent);
|
|
11
|
+
}
|
|
12
|
+
async function flush(task) {
|
|
13
|
+
const results = await createVitestLifecycleHooks().flushCase(task.id);
|
|
14
|
+
if (results.length > 0) {
|
|
15
|
+
task.meta.pathgrade = results;
|
|
16
|
+
}
|
|
17
|
+
}
|
|
18
|
+
function onResult(result, agent) {
|
|
19
|
+
createVitestLifecycleHooks().onResult({ result, agent });
|
|
20
|
+
}
|
|
21
|
+
async function flushAll() {
|
|
22
|
+
await createVitestLifecycleHooks().cleanupRun();
|
|
23
|
+
}
|
|
24
|
+
function reset() {
|
|
25
|
+
resetVitestLifecycle(installHandle);
|
|
26
|
+
installHandle = null;
|
|
27
|
+
}
|
|
28
|
+
function install(afterEach, afterAll, aroundEach) {
|
|
29
|
+
installHandle?.restore();
|
|
30
|
+
installHandle = installVitestLifecycle({ afterEach, afterAll, aroundEach });
|
|
31
|
+
}
|
|
32
|
+
export const lifecycle = {
|
|
33
|
+
trackAgent,
|
|
34
|
+
untrackAgent,
|
|
35
|
+
releaseAgent,
|
|
36
|
+
flush,
|
|
37
|
+
flushAll,
|
|
38
|
+
onResult,
|
|
39
|
+
reset,
|
|
40
|
+
install,
|
|
41
|
+
};
|
|
@@ -0,0 +1,12 @@
|
|
|
1
|
+
import type { Reporter, TestModule } from 'vitest/node';
|
|
2
|
+
import type { PathgradePluginOptions } from '../../sdk/types.js';
|
|
3
|
+
/**
|
|
4
|
+
* Custom vitest reporter that layers pathgrade aggregate statistics
|
|
5
|
+
* on top of vitest's default output.
|
|
6
|
+
*/
|
|
7
|
+
export declare class PathgradeReporter implements Reporter {
|
|
8
|
+
private opts;
|
|
9
|
+
constructor(opts?: PathgradePluginOptions);
|
|
10
|
+
onTestRunEnd(testModules: ReadonlyArray<TestModule>): Promise<void>;
|
|
11
|
+
private openBrowserViewer;
|
|
12
|
+
}
|
|
@@ -0,0 +1,75 @@
|
|
|
1
|
+
import * as path from 'path';
|
|
2
|
+
import { execSync } from 'child_process';
|
|
3
|
+
import { fmt } from '../../utils/cli.js';
|
|
4
|
+
import { getPathgradeDir } from '../../reporters/results-path.js';
|
|
5
|
+
import { readSidecar } from '../../affected/sidecar.js';
|
|
6
|
+
import { printReportSummary } from '../../reporters/report-summary.js';
|
|
7
|
+
import { createVitestAdapter } from '../../runners/vitest-adapter.js';
|
|
8
|
+
import { runWithAdapter } from '../../runners/orchestrator.js';
|
|
9
|
+
import { resolvePathgradeConfig } from '../../config/pathgrade.js';
|
|
10
|
+
/**
|
|
11
|
+
* Custom vitest reporter that layers pathgrade aggregate statistics
|
|
12
|
+
* on top of vitest's default output.
|
|
13
|
+
*/
|
|
14
|
+
export class PathgradeReporter {
|
|
15
|
+
opts;
|
|
16
|
+
constructor(opts) {
|
|
17
|
+
this.opts = opts ?? {};
|
|
18
|
+
}
|
|
19
|
+
async onTestRunEnd(testModules) {
|
|
20
|
+
const cwd = process.cwd();
|
|
21
|
+
const config = await resolvePathgradeConfig({ cwd });
|
|
22
|
+
const mode = this.opts.reporter ?? config.reporter ?? 'cli';
|
|
23
|
+
const outputDir = getPathgradeDir(cwd);
|
|
24
|
+
const adapter = createVitestAdapter({ testModules });
|
|
25
|
+
const exitCode = await runWithAdapter({
|
|
26
|
+
adapter,
|
|
27
|
+
options: {
|
|
28
|
+
cwd,
|
|
29
|
+
discovery: { cwd },
|
|
30
|
+
runnerArgs: [],
|
|
31
|
+
env: process.env,
|
|
32
|
+
artifactRoot: outputDir,
|
|
33
|
+
reporterMode: mode,
|
|
34
|
+
threshold: this.opts.ci?.threshold ?? config.ci.threshold,
|
|
35
|
+
writeEmptyReport: false,
|
|
36
|
+
warn: warning => console.warn(warning),
|
|
37
|
+
log: () => console.log(`\n ${fmt.dim('Results written to')} ${outputDir}\n`),
|
|
38
|
+
loadSelection: async () => (await readSidecar(cwd, msg => {
|
|
39
|
+
console.warn(`[pathgrade] ${msg}`);
|
|
40
|
+
})) ?? undefined,
|
|
41
|
+
printSummary: summaries => {
|
|
42
|
+
printReportSummary(summaries, {
|
|
43
|
+
forceVerbose: this.opts.diagnostics === true
|
|
44
|
+
|| config.diagnostics
|
|
45
|
+
|| process.env.PATHGRADE_DIAGNOSTICS === '1',
|
|
46
|
+
currentTimeoutMs: this.opts.timeout != null ? this.opts.timeout * 1000 : undefined,
|
|
47
|
+
});
|
|
48
|
+
},
|
|
49
|
+
openBrowser: () => this.openBrowserViewer(),
|
|
50
|
+
onThresholdFailure: ({ overallPassRate, threshold }) => {
|
|
51
|
+
const avg = overallPassRate;
|
|
52
|
+
const configuredThreshold = this.opts.ci?.threshold ?? threshold;
|
|
53
|
+
process.exitCode = 1;
|
|
54
|
+
console.log(`\n ${fmt.fail('CI THRESHOLD FAILED')} avg score ${fmt.bold(avg.toFixed(3))} < threshold ${fmt.bold(String(configuredThreshold))}\n`);
|
|
55
|
+
},
|
|
56
|
+
},
|
|
57
|
+
});
|
|
58
|
+
if (exitCode !== 0 && (process.exitCode === undefined || process.exitCode === 0)) {
|
|
59
|
+
process.exitCode = exitCode;
|
|
60
|
+
}
|
|
61
|
+
}
|
|
62
|
+
openBrowserViewer() {
|
|
63
|
+
const viewerPath = path.resolve(import.meta.dirname, '..', '..', 'viewer.html');
|
|
64
|
+
try {
|
|
65
|
+
const openCmd = process.platform === 'darwin' ? 'open'
|
|
66
|
+
: process.platform === 'win32' ? 'start'
|
|
67
|
+
: 'xdg-open';
|
|
68
|
+
execSync(`${openCmd} "${viewerPath}"`, { stdio: 'ignore' });
|
|
69
|
+
console.log(`\n ${fmt.dim('Opened viewer in browser')}\n`);
|
|
70
|
+
}
|
|
71
|
+
catch {
|
|
72
|
+
console.log(`\n ${fmt.dim('Open manually:')} ${viewerPath}\n`);
|
|
73
|
+
}
|
|
74
|
+
}
|
|
75
|
+
}
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
export {};
|
|
@@ -2,11 +2,12 @@ import { BaseAgent, getRuntimeEnv, getWorkspacePath, } from '../../types.js';
|
|
|
2
2
|
import { mountMcpForCodexAppServer } from '../../providers/mcp-runtime-mounting.js';
|
|
3
3
|
import { assertMcpSecretReferencesReady } from '../../providers/mcp-config.js';
|
|
4
4
|
import { buildSummary, enrichSkillEvents, extractSkillNameFromPath, inferCodexExecAction, } from '../../tool-events.js';
|
|
5
|
-
import { requireAskBusForLiveBatches
|
|
5
|
+
import { requireAskBusForLiveBatches } from '../../sdk/ask-bus/bus.js';
|
|
6
6
|
import { toAskUserToolEvent } from '../../sdk/ask-bus/projection.js';
|
|
7
7
|
import { decideMcpToolCall, redactMcpSecrets, } from '../../sdk/mcp-safety.js';
|
|
8
8
|
import { spawnAppServerTransport, } from './transport.js';
|
|
9
9
|
import { normalizeUpstreamQuestion, toWireAnswerMap, } from './wire-translators.js';
|
|
10
|
+
import { extractTurnCompletionFailure } from './turn-completion.js';
|
|
10
11
|
const DEFAULT_MODEL = 'gpt-5.4';
|
|
11
12
|
const TURN_COMPLETED_METHOD = 'turn/completed';
|
|
12
13
|
function recordFromUnknown(value) {
|
|
@@ -308,6 +309,9 @@ export class CodexAppServerAgent extends BaseAgent {
|
|
|
308
309
|
settled = true;
|
|
309
310
|
off();
|
|
310
311
|
closeOff();
|
|
312
|
+
const failure = extractTurnCompletionFailure(n.params);
|
|
313
|
+
if (failure)
|
|
314
|
+
Object.assign(turn, { turnFailed: true, failureMessage: failure });
|
|
311
315
|
resolve();
|
|
312
316
|
}
|
|
313
317
|
});
|
|
@@ -1,8 +1,8 @@
|
|
|
1
1
|
// Curated re-exports for the pathgrade Codex app-server driver and fixture
|
|
2
|
-
// tests. Every re-exported symbol is either
|
|
2
|
+
// tests. Every re-exported symbol is either a curated upstream shape from
|
|
3
3
|
// openai/codex@rust-v0.124.0 (with an upstream-citation header in its file) or
|
|
4
|
-
// a pathgrade-local composition (ClientRequest, ServerRequest, Op) whose
|
|
5
|
-
//
|
|
4
|
+
// a pathgrade-local composition (ClientRequest, ServerRequest, Op) whose header
|
|
5
|
+
// spells out the composition rationale.
|
|
6
6
|
//
|
|
7
7
|
// Refresh policy: to bump the vendored version, update each file's header
|
|
8
8
|
// citation to the new tag, replace the `unknown` aliases with their vendored
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
export declare function extractTurnCompletionFailure(params: unknown): string | undefined;
|
|
@@ -0,0 +1,9 @@
|
|
|
1
|
+
export function extractTurnCompletionFailure(params) {
|
|
2
|
+
const turn = params?.turn;
|
|
3
|
+
const message = turn?.error?.message;
|
|
4
|
+
const details = turn?.error?.additionalDetails;
|
|
5
|
+
if (typeof message === 'string' && message.length > 0) {
|
|
6
|
+
return typeof details === 'string' && details.length > 0 ? `${message} ${details}` : message;
|
|
7
|
+
}
|
|
8
|
+
return turn?.status === 'failed' ? 'turn failed' : undefined;
|
|
9
|
+
}
|
|
@@ -12,19 +12,11 @@
|
|
|
12
12
|
* and `--explain` / `--json` presentation.
|
|
13
13
|
*/
|
|
14
14
|
import * as fs from 'fs';
|
|
15
|
-
import { configDefaults } from 'vitest/config';
|
|
16
15
|
import { selectAffected } from '../affected/select.js';
|
|
17
16
|
import { resolveBaseRef, computeChangedFiles } from '../affected/git.js';
|
|
18
|
-
import { loadAffectedConfig } from '../affected/config.js';
|
|
19
17
|
import { formatExplain, formatJson } from '../affected/format.js';
|
|
20
18
|
import { discoverPathgradeEvalFiles } from '../evals/discovery.js';
|
|
21
|
-
|
|
22
|
-
...configDefaults.exclude,
|
|
23
|
-
'.worktrees/**',
|
|
24
|
-
'worktrees/**',
|
|
25
|
-
'**/node_modules/**',
|
|
26
|
-
'**/fixtures/**',
|
|
27
|
-
];
|
|
19
|
+
import { DEFAULT_EVAL_EXCLUDE, DEFAULT_EVAL_INCLUDE, resolvePathgradeConfig, } from '../config/pathgrade.js';
|
|
28
20
|
export async function runAffected(opts) {
|
|
29
21
|
const { cwd } = opts;
|
|
30
22
|
let changes;
|
|
@@ -44,9 +36,21 @@ export async function runAffected(opts) {
|
|
|
44
36
|
if (changes.baseRefLine) {
|
|
45
37
|
process.stderr.write(`${changes.baseRefLine}\n`);
|
|
46
38
|
}
|
|
47
|
-
|
|
48
|
-
|
|
49
|
-
|
|
39
|
+
let config;
|
|
40
|
+
try {
|
|
41
|
+
config = await resolvePathgradeConfig({
|
|
42
|
+
cwd,
|
|
43
|
+
warn: w => process.stderr.write(`${w}\n`),
|
|
44
|
+
});
|
|
45
|
+
}
|
|
46
|
+
catch (err) {
|
|
47
|
+
process.stderr.write(`pathgrade affected: ${errMsg(err)}\n`);
|
|
48
|
+
return 1;
|
|
49
|
+
}
|
|
50
|
+
const evalFiles = discoverPathgradeEvalFiles({
|
|
51
|
+
cwd,
|
|
52
|
+
include: config.evals.include,
|
|
53
|
+
exclude: config.evals.exclude,
|
|
50
54
|
});
|
|
51
55
|
let result;
|
|
52
56
|
try {
|
|
@@ -55,7 +59,7 @@ export async function runAffected(opts) {
|
|
|
55
59
|
changedFiles: changes.changedFiles,
|
|
56
60
|
repoRoot: cwd,
|
|
57
61
|
baseRef: changes.baseRef,
|
|
58
|
-
global: config.global,
|
|
62
|
+
global: config.affected.global,
|
|
59
63
|
});
|
|
60
64
|
}
|
|
61
65
|
catch (err) {
|
|
@@ -132,6 +136,9 @@ function readChangedFilesList(filePath) {
|
|
|
132
136
|
.map(l => l.trim())
|
|
133
137
|
.filter(l => l.length > 0);
|
|
134
138
|
}
|
|
139
|
+
function errMsg(err) {
|
|
140
|
+
return err instanceof Error ? err.message : String(err);
|
|
141
|
+
}
|
|
135
142
|
/**
|
|
136
143
|
* Discover repo-relative `*.eval.ts` files under `cwd`, honoring the same
|
|
137
144
|
* exclude defaults the vitest plugin uses.
|
|
@@ -139,8 +146,8 @@ function readChangedFilesList(filePath) {
|
|
|
139
146
|
export function discoverEvalFiles(cwd) {
|
|
140
147
|
return discoverPathgradeEvalFiles({
|
|
141
148
|
cwd,
|
|
142
|
-
include:
|
|
143
|
-
exclude:
|
|
149
|
+
include: DEFAULT_EVAL_INCLUDE,
|
|
150
|
+
exclude: DEFAULT_EVAL_EXCLUDE,
|
|
144
151
|
});
|
|
145
152
|
}
|
|
146
153
|
function printSelectedToStdout(result) {
|