@wix/pathgrade 0.36.0 → 1.0.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (117) hide show
  1. package/LICENSE +24 -0
  2. package/README.md +508 -0
  3. package/dist/adapter-kit/index.d.ts +17 -0
  4. package/dist/adapter-kit/index.js +14 -0
  5. package/dist/adapters/jest/index.d.ts +3 -0
  6. package/dist/adapters/jest/index.js +3 -0
  7. package/dist/adapters/jest/invocation-adapter.d.ts +15 -0
  8. package/dist/adapters/jest/invocation-adapter.js +115 -0
  9. package/dist/adapters/jest/lifecycle.d.ts +21 -0
  10. package/dist/adapters/jest/lifecycle.js +62 -0
  11. package/dist/adapters/jest/metadata.d.ts +6 -0
  12. package/dist/adapters/jest/metadata.js +35 -0
  13. package/dist/adapters/jest/reporter.cjs +10 -0
  14. package/dist/adapters/jest/reporter.d.ts +5 -0
  15. package/dist/adapters/jest/reporter.js +67 -0
  16. package/dist/adapters/jest/results.d.ts +31 -0
  17. package/dist/adapters/jest/results.js +144 -0
  18. package/dist/adapters/jest/runner-adapter.d.ts +9 -0
  19. package/dist/adapters/jest/runner-adapter.js +49 -0
  20. package/dist/adapters/jest/setup.d.ts +1 -0
  21. package/dist/adapters/jest/setup.js +10 -0
  22. package/dist/adapters/node-test/index.d.ts +8 -0
  23. package/dist/adapters/node-test/index.js +106 -0
  24. package/dist/adapters/node-test/invocation-adapter.d.ts +5 -0
  25. package/dist/adapters/node-test/invocation-adapter.js +47 -0
  26. package/dist/adapters/node-test/runner-adapter.d.ts +2 -0
  27. package/dist/adapters/node-test/runner-adapter.js +116 -0
  28. package/dist/adapters/vitest/index.d.ts +9 -0
  29. package/dist/adapters/vitest/index.js +82 -0
  30. package/dist/adapters/vitest/lifecycle.d.ts +24 -0
  31. package/dist/adapters/vitest/lifecycle.js +41 -0
  32. package/dist/adapters/vitest/reporter.d.ts +12 -0
  33. package/dist/adapters/vitest/reporter.js +75 -0
  34. package/dist/adapters/vitest/setup.d.ts +1 -0
  35. package/dist/adapters/vitest/setup.js +3 -0
  36. package/dist/agents/codex-app-server/agent.js +5 -1
  37. package/dist/agents/codex-app-server/protocol/index.js +3 -3
  38. package/dist/agents/codex-app-server/turn-completion.d.ts +1 -0
  39. package/dist/agents/codex-app-server/turn-completion.js +9 -0
  40. package/dist/commands/affected.js +22 -15
  41. package/dist/commands/run-args.d.ts +4 -3
  42. package/dist/commands/run-args.js +12 -6
  43. package/dist/commands/run-changed.d.ts +4 -4
  44. package/dist/commands/run-changed.js +45 -58
  45. package/dist/config/pathgrade.d.ts +48 -0
  46. package/dist/config/pathgrade.js +204 -0
  47. package/dist/pathgrade.d.ts +2 -2
  48. package/dist/pathgrade.js +35 -34
  49. package/dist/plugin/index.d.ts +2 -9
  50. package/dist/plugin/index.js +1 -81
  51. package/dist/plugin/lifecycle.d.ts +1 -45
  52. package/dist/plugin/lifecycle.js +1 -146
  53. package/dist/plugin/reporter.d.ts +1 -37
  54. package/dist/plugin/reporter.js +1 -280
  55. package/dist/plugin/setup.d.ts +1 -1
  56. package/dist/plugin/setup.js +1 -3
  57. package/dist/providers/copy-filter.d.ts +5 -0
  58. package/dist/providers/copy-filter.js +9 -0
  59. package/dist/providers/sandbox.js +2 -2
  60. package/dist/providers/workspace.d.ts +6 -0
  61. package/dist/providers/workspace.js +34 -14
  62. package/dist/reporters/diagnostics.d.ts +2 -40
  63. package/dist/reporters/diagnostics.js +1 -80
  64. package/dist/reporters/report-summary.d.ts +6 -0
  65. package/dist/reporters/report-summary.js +29 -0
  66. package/dist/reporting/artifacts.d.ts +2 -0
  67. package/dist/reporting/artifacts.js +20 -0
  68. package/dist/reporting/core.d.ts +2 -0
  69. package/dist/reporting/core.js +173 -0
  70. package/dist/reporting/types.d.ts +57 -0
  71. package/dist/reporting/types.js +1 -0
  72. package/dist/reporting/vitest-edge.d.ts +1 -0
  73. package/dist/reporting/vitest-edge.js +1 -0
  74. package/dist/runners/adapter-loader.d.ts +17 -0
  75. package/dist/runners/adapter-loader.js +67 -0
  76. package/dist/runners/adapter.d.ts +65 -0
  77. package/dist/runners/adapter.js +1 -0
  78. package/dist/runners/invocation.d.ts +10 -0
  79. package/dist/runners/invocation.js +1 -0
  80. package/dist/runners/lifecycle-hooks.d.ts +2 -0
  81. package/dist/runners/lifecycle-hooks.js +18 -0
  82. package/dist/runners/model-builders.d.ts +4 -0
  83. package/dist/runners/model-builders.js +67 -0
  84. package/dist/runners/model-validation.d.ts +12 -0
  85. package/dist/runners/model-validation.js +213 -0
  86. package/dist/runners/model.d.ts +117 -0
  87. package/dist/runners/model.js +1 -0
  88. package/dist/runners/orchestrator.d.ts +30 -0
  89. package/dist/runners/orchestrator.js +74 -0
  90. package/dist/runners/report-projection.d.ts +6 -0
  91. package/dist/runners/report-projection.js +62 -0
  92. package/dist/runners/selection.d.ts +6 -0
  93. package/dist/runners/selection.js +10 -0
  94. package/dist/runners/vitest-adapter.d.ts +8 -0
  95. package/dist/runners/vitest-adapter.js +108 -0
  96. package/dist/runners/vitest-invocation.d.ts +10 -0
  97. package/dist/runners/vitest-invocation.js +44 -0
  98. package/dist/runners/vitest-lifecycle.d.ts +43 -0
  99. package/dist/runners/vitest-lifecycle.js +150 -0
  100. package/dist/sdk/agent.js +15 -16
  101. package/dist/sdk/case-context.d.ts +20 -0
  102. package/dist/sdk/case-context.js +31 -0
  103. package/dist/sdk/diagnostics.d.ts +40 -0
  104. package/dist/sdk/diagnostics.js +80 -0
  105. package/dist/sdk/eval-runtime.d.ts +4 -0
  106. package/dist/sdk/eval-runtime.js +11 -1
  107. package/dist/sdk/evaluate.js +2 -1
  108. package/dist/sdk/index.d.ts +2 -0
  109. package/dist/sdk/index.js +1 -0
  110. package/dist/sdk/lifecycle.d.ts +40 -0
  111. package/dist/sdk/lifecycle.js +170 -0
  112. package/dist/sdk/result-capture.d.ts +18 -0
  113. package/dist/sdk/result-capture.js +49 -0
  114. package/dist/sdk/types.d.ts +1 -1
  115. package/dist/types.d.ts +2 -2
  116. package/dist/utils/llm-providers/anthropic.js +9 -2
  117. package/package.json +73 -36
@@ -0,0 +1,106 @@
1
+ import nodeTest from 'node:test';
2
+ import * as fs from 'node:fs';
3
+ import * as path from 'node:path';
4
+ import { fileURLToPath } from 'node:url';
5
+ import { subscribeToEvalResults } from '../../sdk/result-capture.js';
6
+ import { createRunnerLifecycleHooks } from '../../runners/lifecycle-hooks.js';
7
+ const lifecycle = createRunnerLifecycleHooks();
8
+ let subscribed = false;
9
+ let caseCounter = 0;
10
+ export function test(name, optionsOrFn, maybeFn) {
11
+ installResultCapture();
12
+ const options = typeof optionsOrFn === 'function' ? undefined : optionsOrFn;
13
+ const fn = typeof optionsOrFn === 'function' ? optionsOrFn : maybeFn;
14
+ const sourceRef = callerSourceRef();
15
+ const caseId = `${sourceRef}:${name}:${++caseCounter}`;
16
+ nodeTest(name, options, async (ctx) => {
17
+ const startedAt = performance.now();
18
+ let state = 'passed';
19
+ let thrown;
20
+ try {
21
+ await lifecycle.withCaseContext({
22
+ caseId,
23
+ caseName: name,
24
+ filePath: sourceRef,
25
+ sourceRef,
26
+ runnerNativeId: caseId,
27
+ scope: 'runner-case',
28
+ }, async () => {
29
+ await fn?.(ctx);
30
+ });
31
+ }
32
+ catch (err) {
33
+ state = 'failed';
34
+ thrown = err;
35
+ }
36
+ finally {
37
+ const evaluations = await lifecycle.flushCase(caseId);
38
+ writeCase({
39
+ caseId,
40
+ name,
41
+ state,
42
+ runnerDurationMs: performance.now() - startedAt,
43
+ sourceRef,
44
+ filePath: sourceRef,
45
+ groupName: sourceRef,
46
+ runnerCaseId: caseId,
47
+ evaluations: toReportEvaluations(evaluations),
48
+ });
49
+ }
50
+ if (thrown)
51
+ throw thrown;
52
+ });
53
+ }
54
+ export default test;
55
+ export { createNodeTestAdapter } from './runner-adapter.js';
56
+ export { createNodeTestInvocationAdapter } from './invocation-adapter.js';
57
+ function installResultCapture() {
58
+ if (subscribed)
59
+ return;
60
+ subscribed = true;
61
+ subscribeToEvalResults(event => lifecycle.onResult(event), {
62
+ owner: 'adapter',
63
+ key: 'node-test-lifecycle',
64
+ });
65
+ }
66
+ function toReportEvaluations(evaluations) {
67
+ if (evaluations.length === 0)
68
+ return undefined;
69
+ return evaluations.map(entry => ({
70
+ score: entry.score,
71
+ trial: entry.trial,
72
+ diagnostics: entry.diagnostics,
73
+ }));
74
+ }
75
+ function writeCase(testCase) {
76
+ const resultsPath = process.env.PATHGRADE_NODE_TEST_RESULTS;
77
+ if (!resultsPath)
78
+ return;
79
+ fs.mkdirSync(path.dirname(resultsPath), { recursive: true });
80
+ const existing = fs.existsSync(resultsPath)
81
+ ? JSON.parse(fs.readFileSync(resultsPath, 'utf8'))
82
+ : [];
83
+ existing.push(testCase);
84
+ fs.writeFileSync(resultsPath, `${JSON.stringify(existing, null, 2)}\n`);
85
+ }
86
+ function callerSourceRef() {
87
+ const stack = new Error().stack ?? '';
88
+ const cwd = realpath(process.env.PATHGRADE_NODE_TEST_CWD ?? process.cwd());
89
+ for (const line of stack.split('\n')) {
90
+ const match = line.match(/\(?((?:file:\/\/)?[^() ]+\.eval\.[cm]?[jt]s)(?::\d+)?(?::\d+)?\)?/);
91
+ if (!match)
92
+ continue;
93
+ const raw = match[1];
94
+ const abs = realpath(raw.startsWith('file://') ? fileURLToPath(raw) : raw);
95
+ return path.relative(cwd, abs).replaceAll(path.sep, '/');
96
+ }
97
+ return 'node-test';
98
+ }
99
+ function realpath(value) {
100
+ try {
101
+ return fs.realpathSync(value);
102
+ }
103
+ catch {
104
+ return value;
105
+ }
106
+ }
@@ -0,0 +1,5 @@
1
+ import type { ResolvedPathgradeConfig } from '../../config/pathgrade.js';
2
+ import type { RunnerInvocationAdapter } from '../../runners/invocation.js';
3
+ export declare function createNodeTestInvocationAdapter(input: {
4
+ config: ResolvedPathgradeConfig;
5
+ }): RunnerInvocationAdapter;
@@ -0,0 +1,47 @@
1
+ import { readSidecar } from '../../affected/sidecar.js';
2
+ import { getPathgradeDir } from '../../reporters/results-path.js';
3
+ import { printReportSummary } from '../../reporters/report-summary.js';
4
+ import { runWithAdapter } from '../../runners/orchestrator.js';
5
+ import { createNodeTestAdapter } from './runner-adapter.js';
6
+ export function createNodeTestInvocationAdapter(input) {
7
+ return {
8
+ name: 'node-test',
9
+ async run(runInput) {
10
+ const selection = await readSidecar(runInput.cwd, msg => {
11
+ process.stderr.write(`[pathgrade] ${msg}\n`);
12
+ }) ?? undefined;
13
+ const include = runInput.selectedFiles && runInput.selectedFiles.length > 0
14
+ ? runInput.selectedFiles
15
+ : input.config.evals.include;
16
+ return await runWithAdapter({
17
+ adapter: createNodeTestAdapter(),
18
+ options: {
19
+ cwd: runInput.cwd,
20
+ discovery: {
21
+ cwd: runInput.cwd,
22
+ include,
23
+ exclude: runInput.selectedFiles ? [] : input.config.evals.exclude,
24
+ },
25
+ runnerArgs: runInput.runnerArgs,
26
+ env: runInput.env,
27
+ artifactRoot: getPathgradeDir(runInput.cwd),
28
+ reporterMode: input.config.reporter ?? 'cli',
29
+ threshold: input.config.ci.threshold,
30
+ selection,
31
+ printSummary: summaries => {
32
+ printReportSummary(summaries, {
33
+ forceVerbose: input.config.diagnostics
34
+ || runInput.env.PATHGRADE_DIAGNOSTICS === '1',
35
+ });
36
+ },
37
+ log: () => {
38
+ process.stdout.write(`\n Results written to ${getPathgradeDir(runInput.cwd)}\n\n`);
39
+ },
40
+ onThresholdFailure: ({ overallPassRate, threshold }) => {
41
+ process.stdout.write(`\n CI THRESHOLD FAILED avg score ${overallPassRate.toFixed(3)} < threshold ${threshold}\n\n`);
42
+ },
43
+ },
44
+ });
45
+ },
46
+ };
47
+ }
@@ -0,0 +1,2 @@
1
+ import type { RunnerAdapter } from '../../runners/adapter.js';
2
+ export declare function createNodeTestAdapter(): RunnerAdapter;
@@ -0,0 +1,116 @@
1
+ import * as path from 'node:path';
2
+ import * as os from 'node:os';
3
+ import { spawn } from 'node:child_process';
4
+ import fs from 'fs-extra';
5
+ import { discoverPathgradeEvalFiles } from '../../evals/discovery.js';
6
+ import { buildNormalizedRunSnapshotFromReportGroups } from '../../runners/model-builders.js';
7
+ export function createNodeTestAdapter() {
8
+ return {
9
+ name: 'node-test',
10
+ async discover(input) {
11
+ const include = input.include ?? ['**/*.eval.ts'];
12
+ const exclude = input.exclude ?? [];
13
+ const files = discoverPathgradeEvalFiles({
14
+ cwd: input.cwd,
15
+ include,
16
+ exclude,
17
+ });
18
+ return {
19
+ units: files.map(file => ({
20
+ id: file,
21
+ displayName: file,
22
+ sourceRef: file,
23
+ native: { file, cwd: input.cwd },
24
+ })),
25
+ };
26
+ },
27
+ async invoke(input) {
28
+ const resultsDir = await fs.mkdtemp(path.join(os.tmpdir(), 'pathgrade-node-test-'));
29
+ const resultsPath = path.join(resultsDir, 'cases.json');
30
+ const cwd = readDiscoveredCwd(input.discovered.units[0]?.native) ?? process.cwd();
31
+ const files = input.discovered.units
32
+ .map(unit => unit.sourceRef)
33
+ .filter((sourceRef) => typeof sourceRef === 'string');
34
+ if (files.length === 0) {
35
+ await fs.writeJson(resultsPath, []);
36
+ return {
37
+ adapterName: this.name,
38
+ status: 'completed',
39
+ exitCode: 0,
40
+ native: { resultsPath },
41
+ };
42
+ }
43
+ const exitCode = await spawnNodeTest({
44
+ testCwd: cwd,
45
+ files,
46
+ argv: input.argv,
47
+ env: {
48
+ ...input.env,
49
+ PATHGRADE_NODE_TEST_RESULTS: resultsPath,
50
+ PATHGRADE_NODE_TEST_CWD: cwd,
51
+ },
52
+ });
53
+ return {
54
+ adapterName: this.name,
55
+ status: input.signal?.aborted ? 'cancelled' : (exitCode === 0 ? 'completed' : 'failed'),
56
+ exitCode,
57
+ native: { resultsPath },
58
+ };
59
+ },
60
+ async collectNormalizedRunSnapshot(run) {
61
+ const native = readNodeTestRunNative(run);
62
+ const cases = await readCases(native.resultsPath);
63
+ const groupMap = new Map();
64
+ for (const testCase of cases) {
65
+ const groupName = testCase.groupName ?? testCase.filePath ?? testCase.sourceRef ?? 'node-test';
66
+ if (!groupMap.has(groupName))
67
+ groupMap.set(groupName, []);
68
+ groupMap.get(groupName).push(testCase);
69
+ }
70
+ return buildNormalizedRunSnapshotFromReportGroups(run, Array.from(groupMap.entries()).map(([groupName, groupedCases]) => ({
71
+ groupName,
72
+ cases: groupedCases,
73
+ })));
74
+ },
75
+ };
76
+ }
77
+ async function spawnNodeTest(input) {
78
+ const tsxLoader = import.meta.resolve('tsx');
79
+ const args = [
80
+ '--import',
81
+ tsxLoader,
82
+ '--test',
83
+ ...input.files,
84
+ ...input.argv,
85
+ ];
86
+ return await new Promise(resolve => {
87
+ const child = spawn(process.execPath, args, {
88
+ cwd: input.testCwd,
89
+ env: input.env,
90
+ stdio: 'inherit',
91
+ });
92
+ child.on('close', code => resolve(code ?? 0));
93
+ child.on('error', () => resolve(1));
94
+ });
95
+ }
96
+ function readNodeTestRunNative(run) {
97
+ if (typeof run.native === 'object'
98
+ && run.native !== null
99
+ && typeof run.native.resultsPath === 'string') {
100
+ return run.native;
101
+ }
102
+ return { resultsPath: '' };
103
+ }
104
+ async function readCases(resultsPath) {
105
+ if (!resultsPath || !(await fs.pathExists(resultsPath)))
106
+ return [];
107
+ return await fs.readJson(resultsPath);
108
+ }
109
+ function readDiscoveredCwd(native) {
110
+ if (typeof native === 'object'
111
+ && native !== null
112
+ && typeof native.cwd === 'string') {
113
+ return native.cwd;
114
+ }
115
+ return undefined;
116
+ }
@@ -0,0 +1,9 @@
1
+ import type { PathgradePluginOptions } from '../../sdk/types.js';
2
+ export type { PathgradePluginOptions };
3
+ /**
4
+ * Vitest plugin for pathgrade evaluations.
5
+ *
6
+ * Configures test file patterns, timeout, auto-dispose lifecycle hooks,
7
+ * and the pathgrade reporter for aggregate statistics.
8
+ */
9
+ export declare function pathgrade(opts?: PathgradePluginOptions): any;
@@ -0,0 +1,82 @@
1
+ import * as fs from 'fs';
2
+ import * as path from 'path';
3
+ import { PathgradeReporter } from './reporter.js';
4
+ import { discoverPathgradeEvalFiles } from '../../evals/discovery.js';
5
+ import { DEFAULT_EVAL_EXCLUDE, DEFAULT_EVAL_INCLUDE } from '../../config/pathgrade.js';
6
+ function resolveSetupFile() {
7
+ // Prefer .ts (running from source via vitest) over .js (built dist)
8
+ const tsPath = path.resolve(import.meta.dirname, 'setup.ts');
9
+ if (fs.existsSync(tsPath))
10
+ return tsPath;
11
+ return path.resolve(import.meta.dirname, 'setup.js');
12
+ }
13
+ /**
14
+ * Vitest plugin for pathgrade evaluations.
15
+ *
16
+ * Configures test file patterns, timeout, auto-dispose lifecycle hooks,
17
+ * and the pathgrade reporter for aggregate statistics.
18
+ */
19
+ export function pathgrade(opts) {
20
+ const timeoutSec = opts?.timeout ?? 300;
21
+ // Verbose live-streaming seam: the SDK (createAgent) reads
22
+ // `process.env.PATHGRADE_VERBOSE` to decide whether to emit. Wiring the
23
+ // plugin option through the env var keeps a single source of truth and
24
+ // preserves CLI/env precedence — we only set the env if the user hasn't
25
+ // already set it via their shell or the `pathgrade run --verbose` CLI.
26
+ if (opts?.verbose === true && process.env.PATHGRADE_VERBOSE === undefined) {
27
+ process.env.PATHGRADE_VERBOSE = '1';
28
+ }
29
+ return {
30
+ name: 'pathgrade',
31
+ // Surface the user's options to pre-vitest CLI commands
32
+ // (`pathgrade affected`, `pathgrade run --changed`) so they can
33
+ // read `affected.global` after loading vitest.config.ts.
34
+ // See `src/affected/config.ts`.
35
+ __pathgradeOptions: opts ?? {},
36
+ config() {
37
+ return {
38
+ resolve: {
39
+ alias: resolveLocalAliases(),
40
+ },
41
+ test: {
42
+ include: opts?.include ?? DEFAULT_EVAL_INCLUDE,
43
+ exclude: opts?.exclude ?? DEFAULT_EVAL_EXCLUDE,
44
+ testTimeout: (timeoutSec + 30) * 1000,
45
+ setupFiles: [resolveSetupFile()],
46
+ reporters: [
47
+ 'default',
48
+ new PathgradeReporter(opts),
49
+ ],
50
+ },
51
+ };
52
+ },
53
+ configResolved(resolved) {
54
+ const root = resolved.root ?? process.cwd();
55
+ const include = resolved.test?.include ?? opts?.include ?? DEFAULT_EVAL_INCLUDE;
56
+ const exclude = resolved.test?.exclude ?? opts?.exclude ?? DEFAULT_EVAL_EXCLUDE;
57
+ const discovered = discoverPathgradeEvalFiles({ cwd: root, include, exclude });
58
+ if (!resolved.test) {
59
+ resolved.test = {};
60
+ }
61
+ resolved.test.include = discovered;
62
+ },
63
+ };
64
+ }
65
+ /**
66
+ * When running from source (not installed via npm), resolve
67
+ * `@wix/pathgrade` imports to the local src/ directory so examples
68
+ * can use canonical package imports while developing locally.
69
+ */
70
+ function resolveLocalAliases() {
71
+ const sdkSource = path.resolve(import.meta.dirname, '..', '..', 'sdk', 'index.ts');
72
+ if (!fs.existsSync(sdkSource))
73
+ return undefined;
74
+ const src = path.resolve(import.meta.dirname, '..', '..');
75
+ return {
76
+ '@wix/pathgrade/mcp-mock': path.join(src, 'core', 'mcp-mock.ts'),
77
+ '@wix/pathgrade/plugin': path.join(src, 'plugin', 'index.ts'),
78
+ '@wix/pathgrade/plugin/vitest': path.join(src, 'adapters', 'vitest', 'index.ts'),
79
+ '@wix/pathgrade/adapters/vitest': path.join(src, 'adapters', 'vitest', 'index.ts'),
80
+ '@wix/pathgrade': path.join(src, 'sdk', 'index.ts'),
81
+ };
82
+ }
@@ -0,0 +1,24 @@
1
+ import type { Agent, RecordedEvalResult } from '../../sdk/types.js';
2
+ import { type VitestAfterAllFn, type VitestAfterEachFn, type VitestAroundEachFn } from '../../runners/vitest-lifecycle.js';
3
+ declare function trackAgent(agent: Agent): void;
4
+ declare function untrackAgent(agent: Agent): void;
5
+ declare function releaseAgent(agent: Agent): void;
6
+ declare function flush(task: {
7
+ id: string;
8
+ meta: Record<string, unknown>;
9
+ }): Promise<void>;
10
+ declare function onResult(result: RecordedEvalResult, agent: Agent): void;
11
+ declare function flushAll(): Promise<void>;
12
+ declare function reset(): void;
13
+ declare function install(afterEach: VitestAfterEachFn, afterAll?: VitestAfterAllFn, aroundEach?: VitestAroundEachFn): void;
14
+ export declare const lifecycle: {
15
+ trackAgent: typeof trackAgent;
16
+ untrackAgent: typeof untrackAgent;
17
+ releaseAgent: typeof releaseAgent;
18
+ flush: typeof flush;
19
+ flushAll: typeof flushAll;
20
+ onResult: typeof onResult;
21
+ reset: typeof reset;
22
+ install: typeof install;
23
+ };
24
+ export {};
@@ -0,0 +1,41 @@
1
+ import { createVitestLifecycleHooks, installVitestLifecycle, releaseVitestAgent, resetVitestLifecycle, trackVitestAgent, untrackVitestAgent, } from '../../runners/vitest-lifecycle.js';
2
+ let installHandle = null;
3
+ function trackAgent(agent) {
4
+ trackVitestAgent(agent, installHandle !== null);
5
+ }
6
+ function untrackAgent(agent) {
7
+ untrackVitestAgent(agent);
8
+ }
9
+ function releaseAgent(agent) {
10
+ releaseVitestAgent(agent);
11
+ }
12
+ async function flush(task) {
13
+ const results = await createVitestLifecycleHooks().flushCase(task.id);
14
+ if (results.length > 0) {
15
+ task.meta.pathgrade = results;
16
+ }
17
+ }
18
+ function onResult(result, agent) {
19
+ createVitestLifecycleHooks().onResult({ result, agent });
20
+ }
21
+ async function flushAll() {
22
+ await createVitestLifecycleHooks().cleanupRun();
23
+ }
24
+ function reset() {
25
+ resetVitestLifecycle(installHandle);
26
+ installHandle = null;
27
+ }
28
+ function install(afterEach, afterAll, aroundEach) {
29
+ installHandle?.restore();
30
+ installHandle = installVitestLifecycle({ afterEach, afterAll, aroundEach });
31
+ }
32
+ export const lifecycle = {
33
+ trackAgent,
34
+ untrackAgent,
35
+ releaseAgent,
36
+ flush,
37
+ flushAll,
38
+ onResult,
39
+ reset,
40
+ install,
41
+ };
@@ -0,0 +1,12 @@
1
+ import type { Reporter, TestModule } from 'vitest/node';
2
+ import type { PathgradePluginOptions } from '../../sdk/types.js';
3
+ /**
4
+ * Custom vitest reporter that layers pathgrade aggregate statistics
5
+ * on top of vitest's default output.
6
+ */
7
+ export declare class PathgradeReporter implements Reporter {
8
+ private opts;
9
+ constructor(opts?: PathgradePluginOptions);
10
+ onTestRunEnd(testModules: ReadonlyArray<TestModule>): Promise<void>;
11
+ private openBrowserViewer;
12
+ }
@@ -0,0 +1,75 @@
1
+ import * as path from 'path';
2
+ import { execSync } from 'child_process';
3
+ import { fmt } from '../../utils/cli.js';
4
+ import { getPathgradeDir } from '../../reporters/results-path.js';
5
+ import { readSidecar } from '../../affected/sidecar.js';
6
+ import { printReportSummary } from '../../reporters/report-summary.js';
7
+ import { createVitestAdapter } from '../../runners/vitest-adapter.js';
8
+ import { runWithAdapter } from '../../runners/orchestrator.js';
9
+ import { resolvePathgradeConfig } from '../../config/pathgrade.js';
10
+ /**
11
+ * Custom vitest reporter that layers pathgrade aggregate statistics
12
+ * on top of vitest's default output.
13
+ */
14
+ export class PathgradeReporter {
15
+ opts;
16
+ constructor(opts) {
17
+ this.opts = opts ?? {};
18
+ }
19
+ async onTestRunEnd(testModules) {
20
+ const cwd = process.cwd();
21
+ const config = await resolvePathgradeConfig({ cwd });
22
+ const mode = this.opts.reporter ?? config.reporter ?? 'cli';
23
+ const outputDir = getPathgradeDir(cwd);
24
+ const adapter = createVitestAdapter({ testModules });
25
+ const exitCode = await runWithAdapter({
26
+ adapter,
27
+ options: {
28
+ cwd,
29
+ discovery: { cwd },
30
+ runnerArgs: [],
31
+ env: process.env,
32
+ artifactRoot: outputDir,
33
+ reporterMode: mode,
34
+ threshold: this.opts.ci?.threshold ?? config.ci.threshold,
35
+ writeEmptyReport: false,
36
+ warn: warning => console.warn(warning),
37
+ log: () => console.log(`\n ${fmt.dim('Results written to')} ${outputDir}\n`),
38
+ loadSelection: async () => (await readSidecar(cwd, msg => {
39
+ console.warn(`[pathgrade] ${msg}`);
40
+ })) ?? undefined,
41
+ printSummary: summaries => {
42
+ printReportSummary(summaries, {
43
+ forceVerbose: this.opts.diagnostics === true
44
+ || config.diagnostics
45
+ || process.env.PATHGRADE_DIAGNOSTICS === '1',
46
+ currentTimeoutMs: this.opts.timeout != null ? this.opts.timeout * 1000 : undefined,
47
+ });
48
+ },
49
+ openBrowser: () => this.openBrowserViewer(),
50
+ onThresholdFailure: ({ overallPassRate, threshold }) => {
51
+ const avg = overallPassRate;
52
+ const configuredThreshold = this.opts.ci?.threshold ?? threshold;
53
+ process.exitCode = 1;
54
+ console.log(`\n ${fmt.fail('CI THRESHOLD FAILED')} avg score ${fmt.bold(avg.toFixed(3))} < threshold ${fmt.bold(String(configuredThreshold))}\n`);
55
+ },
56
+ },
57
+ });
58
+ if (exitCode !== 0 && (process.exitCode === undefined || process.exitCode === 0)) {
59
+ process.exitCode = exitCode;
60
+ }
61
+ }
62
+ openBrowserViewer() {
63
+ const viewerPath = path.resolve(import.meta.dirname, '..', '..', 'viewer.html');
64
+ try {
65
+ const openCmd = process.platform === 'darwin' ? 'open'
66
+ : process.platform === 'win32' ? 'start'
67
+ : 'xdg-open';
68
+ execSync(`${openCmd} "${viewerPath}"`, { stdio: 'ignore' });
69
+ console.log(`\n ${fmt.dim('Opened viewer in browser')}\n`);
70
+ }
71
+ catch {
72
+ console.log(`\n ${fmt.dim('Open manually:')} ${viewerPath}\n`);
73
+ }
74
+ }
75
+ }
@@ -0,0 +1 @@
1
+ export {};
@@ -0,0 +1,3 @@
1
+ import { afterAll, afterEach, aroundEach } from 'vitest';
2
+ import { lifecycle } from './lifecycle.js';
3
+ lifecycle.install(afterEach, afterAll, aroundEach);
@@ -2,11 +2,12 @@ import { BaseAgent, getRuntimeEnv, getWorkspacePath, } from '../../types.js';
2
2
  import { mountMcpForCodexAppServer } from '../../providers/mcp-runtime-mounting.js';
3
3
  import { assertMcpSecretReferencesReady } from '../../providers/mcp-config.js';
4
4
  import { buildSummary, enrichSkillEvents, extractSkillNameFromPath, inferCodexExecAction, } from '../../tool-events.js';
5
- import { requireAskBusForLiveBatches, } from '../../sdk/ask-bus/bus.js';
5
+ import { requireAskBusForLiveBatches } from '../../sdk/ask-bus/bus.js';
6
6
  import { toAskUserToolEvent } from '../../sdk/ask-bus/projection.js';
7
7
  import { decideMcpToolCall, redactMcpSecrets, } from '../../sdk/mcp-safety.js';
8
8
  import { spawnAppServerTransport, } from './transport.js';
9
9
  import { normalizeUpstreamQuestion, toWireAnswerMap, } from './wire-translators.js';
10
+ import { extractTurnCompletionFailure } from './turn-completion.js';
10
11
  const DEFAULT_MODEL = 'gpt-5.4';
11
12
  const TURN_COMPLETED_METHOD = 'turn/completed';
12
13
  function recordFromUnknown(value) {
@@ -308,6 +309,9 @@ export class CodexAppServerAgent extends BaseAgent {
308
309
  settled = true;
309
310
  off();
310
311
  closeOff();
312
+ const failure = extractTurnCompletionFailure(n.params);
313
+ if (failure)
314
+ Object.assign(turn, { turnFailed: true, failureMessage: failure });
311
315
  resolve();
312
316
  }
313
317
  });
@@ -1,8 +1,8 @@
1
1
  // Curated re-exports for the pathgrade Codex app-server driver and fixture
2
- // tests. Every re-exported symbol is either vendored verbatim from
2
+ // tests. Every re-exported symbol is either a curated upstream shape from
3
3
  // openai/codex@rust-v0.124.0 (with an upstream-citation header in its file) or
4
- // a pathgrade-local composition (ClientRequest, ServerRequest, Op) whose
5
- // header spells out the composition rationale.
4
+ // a pathgrade-local composition (ClientRequest, ServerRequest, Op) whose header
5
+ // spells out the composition rationale.
6
6
  //
7
7
  // Refresh policy: to bump the vendored version, update each file's header
8
8
  // citation to the new tag, replace the `unknown` aliases with their vendored
@@ -0,0 +1 @@
1
+ export declare function extractTurnCompletionFailure(params: unknown): string | undefined;
@@ -0,0 +1,9 @@
1
+ export function extractTurnCompletionFailure(params) {
2
+ const turn = params?.turn;
3
+ const message = turn?.error?.message;
4
+ const details = turn?.error?.additionalDetails;
5
+ if (typeof message === 'string' && message.length > 0) {
6
+ return typeof details === 'string' && details.length > 0 ? `${message} ${details}` : message;
7
+ }
8
+ return turn?.status === 'failed' ? 'turn failed' : undefined;
9
+ }
@@ -12,19 +12,11 @@
12
12
  * and `--explain` / `--json` presentation.
13
13
  */
14
14
  import * as fs from 'fs';
15
- import { configDefaults } from 'vitest/config';
16
15
  import { selectAffected } from '../affected/select.js';
17
16
  import { resolveBaseRef, computeChangedFiles } from '../affected/git.js';
18
- import { loadAffectedConfig } from '../affected/config.js';
19
17
  import { formatExplain, formatJson } from '../affected/format.js';
20
18
  import { discoverPathgradeEvalFiles } from '../evals/discovery.js';
21
- const DEFAULT_EXCLUDE = [
22
- ...configDefaults.exclude,
23
- '.worktrees/**',
24
- 'worktrees/**',
25
- '**/node_modules/**',
26
- '**/fixtures/**',
27
- ];
19
+ import { DEFAULT_EVAL_EXCLUDE, DEFAULT_EVAL_INCLUDE, resolvePathgradeConfig, } from '../config/pathgrade.js';
28
20
  export async function runAffected(opts) {
29
21
  const { cwd } = opts;
30
22
  let changes;
@@ -44,9 +36,21 @@ export async function runAffected(opts) {
44
36
  if (changes.baseRefLine) {
45
37
  process.stderr.write(`${changes.baseRefLine}\n`);
46
38
  }
47
- const evalFiles = discoverEvalFiles(cwd);
48
- const config = await loadAffectedConfig(cwd, {
49
- onWarning: w => process.stderr.write(`${w}\n`),
39
+ let config;
40
+ try {
41
+ config = await resolvePathgradeConfig({
42
+ cwd,
43
+ warn: w => process.stderr.write(`${w}\n`),
44
+ });
45
+ }
46
+ catch (err) {
47
+ process.stderr.write(`pathgrade affected: ${errMsg(err)}\n`);
48
+ return 1;
49
+ }
50
+ const evalFiles = discoverPathgradeEvalFiles({
51
+ cwd,
52
+ include: config.evals.include,
53
+ exclude: config.evals.exclude,
50
54
  });
51
55
  let result;
52
56
  try {
@@ -55,7 +59,7 @@ export async function runAffected(opts) {
55
59
  changedFiles: changes.changedFiles,
56
60
  repoRoot: cwd,
57
61
  baseRef: changes.baseRef,
58
- global: config.global,
62
+ global: config.affected.global,
59
63
  });
60
64
  }
61
65
  catch (err) {
@@ -132,6 +136,9 @@ function readChangedFilesList(filePath) {
132
136
  .map(l => l.trim())
133
137
  .filter(l => l.length > 0);
134
138
  }
139
+ function errMsg(err) {
140
+ return err instanceof Error ? err.message : String(err);
141
+ }
135
142
  /**
136
143
  * Discover repo-relative `*.eval.ts` files under `cwd`, honoring the same
137
144
  * exclude defaults the vitest plugin uses.
@@ -139,8 +146,8 @@ function readChangedFilesList(filePath) {
139
146
  export function discoverEvalFiles(cwd) {
140
147
  return discoverPathgradeEvalFiles({
141
148
  cwd,
142
- include: ['**/*.eval.ts'],
143
- exclude: DEFAULT_EXCLUDE,
149
+ include: DEFAULT_EVAL_INCLUDE,
150
+ exclude: DEFAULT_EVAL_EXCLUDE,
144
151
  });
145
152
  }
146
153
  function printSelectedToStdout(result) {