@wix/pathgrade 1.0.27 → 1.0.29
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +12 -19
- package/dist/adapters/jest/invocation-adapter.js +5 -1
- package/dist/adapters/jest/reporter.js +4 -1
- package/dist/adapters/jest/results.js +8 -0
- package/dist/adapters/node-test/index.d.ts +5 -0
- package/dist/adapters/node-test/index.js +35 -7
- package/dist/adapters/node-test/invocation-adapter.js +3 -1
- package/dist/adapters/node-test/runner-adapter.js +22 -15
- package/dist/adapters/vitest/reporter.js +4 -1
- package/dist/agents/claude.js +5 -4
- package/dist/agents/codex-app-server/agent.js +1 -51
- package/dist/agents/codex-app-server/turn-notifications.d.ts +6 -0
- package/dist/agents/codex-app-server/turn-notifications.js +51 -0
- package/dist/agents/codex-app-server/turn-state.d.ts +19 -0
- package/dist/agents/codex-app-server/turn-state.js +1 -0
- package/dist/agents/opencode/protocol.d.ts +7 -0
- package/dist/agents/opencode/protocol.js +47 -0
- package/dist/agents/opencode.js +1 -47
- package/dist/analytics/engine.js +5 -2
- package/dist/commands/report.js +16 -2
- package/dist/commands/run-args.d.ts +1 -0
- package/dist/commands/run-args.js +13 -0
- package/dist/commands/run-changed.js +5 -15
- package/dist/config/pathgrade.d.ts +3 -0
- package/dist/config/pathgrade.js +33 -1
- package/dist/internal/direct-mcp-v2/claude-direct-mcp.d.ts +2 -1
- package/dist/internal/direct-mcp-v2/claude-direct-mcp.js +35 -17
- package/dist/internal/direct-mcp-v2/claude-profile.d.ts +33 -7
- package/dist/internal/direct-mcp-v2/claude-profile.js +104 -72
- package/dist/internal/direct-mcp-v2/public-scenario-runtime.js +13 -16
- package/dist/pathgrade.js +10 -2
- package/dist/reporters/cli.js +13 -6
- package/dist/reporters/github-comment.js +12 -9
- package/dist/reporters/loader.d.ts +1 -0
- package/dist/reporters/loader.js +27 -2
- package/dist/reporters/report-summary.js +13 -5
- package/dist/reporting/core.d.ts +1 -0
- package/dist/reporting/core.js +183 -105
- package/dist/reporting/types.d.ts +19 -3
- package/dist/runners/adapter-loader.js +17 -12
- package/dist/runners/direct-reporter-attempts.d.ts +1 -0
- package/dist/runners/direct-reporter-attempts.js +7 -0
- package/dist/runners/invocation.d.ts +2 -0
- package/dist/runners/model-builders.js +1 -0
- package/dist/runners/model-validation.js +31 -0
- package/dist/runners/model.d.ts +4 -0
- package/dist/runners/orchestrator.d.ts +2 -0
- package/dist/runners/orchestrator.js +11 -1
- package/dist/runners/repeated-attempts.d.ts +7 -0
- package/dist/runners/repeated-attempts.js +149 -0
- package/dist/runners/repeated-invocation.d.ts +7 -0
- package/dist/runners/repeated-invocation.js +129 -0
- package/dist/runners/report-projection.js +16 -6
- package/dist/runners/vitest-adapter.js +10 -0
- package/dist/runners/vitest-invocation.js +2 -0
- package/dist/sdk/agent-runtime-options.d.ts +12 -0
- package/dist/sdk/agent-runtime-options.js +67 -0
- package/dist/sdk/agent.js +4 -57
- package/dist/sdk/case-context.js +7 -2
- package/dist/sdk/lifecycle.js +8 -3
- package/dist/sdk/result-capture.js +4 -1
- package/dist/types.d.ts +34 -5
- package/dist/viewer.html +19 -19
- package/package.json +2 -2
package/README.md
CHANGED
|
@@ -420,7 +420,7 @@ provider-specific scenario flags and profile controls are not public.
|
|
|
420
420
|
## CLI
|
|
421
421
|
|
|
422
422
|
```bash
|
|
423
|
-
pathgrade run [--changed] [--since=<ref>] [--changed-files=<path>] [--adapter=<name|path>] [--diagnostics] [--verbose] [--quiet] [-- runner-args]
|
|
423
|
+
pathgrade run [--changed] [--since=<ref>] [--changed-files=<path>] [--adapter=<name|path>] [--attempts=N] [--diagnostics] [--verbose] [--quiet] [-- runner-args]
|
|
424
424
|
pathgrade init [--force]
|
|
425
425
|
pathgrade validate <file.eval.ts>
|
|
426
426
|
pathgrade validate --affected
|
|
@@ -434,6 +434,7 @@ pathgrade report [--results-path=<path>] [--no-comment] [--comment-id=<id>] [--d
|
|
|
434
434
|
Useful details:
|
|
435
435
|
|
|
436
436
|
- `pathgrade run --changed` computes affected evals first, writes selection metadata to `.pathgrade/selection.json`, and only then launches the selected runner adapter. Vitest is the default adapter; Jest is selected with `runner.adapter: 'jest'` or `--adapter=jest`.
|
|
437
|
+
- `pathgrade run --attempts=3` reruns every selected case in three sequential, fresh runner processes. The CLI value overrides `attempts` in config; arguments after `--` still belong to the runner.
|
|
437
438
|
- `pathgrade preview browser` starts a local viewer on `http://localhost:3847`.
|
|
438
439
|
- `pathgrade report` posts or updates a PR comment in GitHub Actions; locally it prints the markdown report and then the numeric pass rate. Provider orchestrators can add a `--details-url`, suppress stale updates with `--expected-head-sha`, and opt into surfaced API failures with `--strict`.
|
|
439
440
|
- `pathgrade validate --affected` is a strict mode for CI: every discovered eval must either live under a `SKILL.md` anchor or export valid `__pathgradeMeta`.
|
|
@@ -445,22 +446,20 @@ Run `pathgrade --help` for the full help text.
|
|
|
445
446
|
```typescript
|
|
446
447
|
// pathgrade.config.ts
|
|
447
448
|
export default {
|
|
448
|
-
|
|
449
|
-
|
|
450
|
-
args: [],
|
|
451
|
-
},
|
|
449
|
+
attempts: 1, // opt in to runner-owned repetitions
|
|
450
|
+
runner: { adapter: 'vitest', args: [] },
|
|
452
451
|
evals: {
|
|
453
452
|
include: ['**/*.eval.ts'], // default
|
|
454
453
|
exclude: ['**/fixtures/**'], // replaces the default exclude list if set
|
|
455
454
|
},
|
|
456
|
-
affected: {
|
|
457
|
-
global: ['package.json', 'yarn.lock'],
|
|
458
|
-
},
|
|
455
|
+
affected: { global: ['package.json', 'yarn.lock'] },
|
|
459
456
|
ci: { threshold: 0.8 },
|
|
460
457
|
};
|
|
461
458
|
```
|
|
462
459
|
|
|
463
|
-
Pathgrade reads `pathgrade.config.*` for CLI and affected-selection behavior. `runner.adapter` and `--adapter=<name|path>` select the runner; `--adapter` wins over config. Built-in adapters
|
|
460
|
+
Pathgrade reads `pathgrade.config.*` for CLI and affected-selection behavior. `runner.adapter` and `--adapter=<name|path>` select the runner; `--adapter` wins over config. `attempts` defaults to `1` and must be a positive integer. Built-in adapters `vitest`, `jest`, and `node-test` support repeated attempts; third-party invocation adapters must advertise `supportsRepeatedAttempts: true` and produce the normalized child snapshot contract.
|
|
461
|
+
|
|
462
|
+
New reports use schema version 2. They preserve every attempt with `case_id`, `attempt_id`, and `attempt_index`; include the display-safe `runner_outcome` that determined binary success; publish `mean_reward` for partial scores; and publish finite-sample pass@k only for complete binary attempts. Runner assertions, runner diagnostics, and native runner references remain in the normalized run model and are not copied into public report artifacts; evaluation diagnostics retain their existing report behavior. A partial reward or incomplete attempt makes pass@k explicitly unavailable. Version-1 reports remain readable, but Pathgrade no longer computes pass@k or pass^k by pooling heterogeneous cases.
|
|
464
463
|
|
|
465
464
|
Third-party runner adapters are supported through `@wix/pathgrade/adapter-kit`. Adapter names resolve as follows:
|
|
466
465
|
|
|
@@ -561,18 +560,15 @@ jobs:
|
|
|
561
560
|
with:
|
|
562
561
|
node-version: '20'
|
|
563
562
|
- run: npm ci
|
|
564
|
-
|
|
565
563
|
- name: Run affected evals
|
|
566
564
|
env:
|
|
567
565
|
ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }}
|
|
568
566
|
run: npx pathgrade run --changed
|
|
569
|
-
|
|
570
567
|
- name: Post PR report
|
|
571
568
|
if: always()
|
|
572
569
|
env:
|
|
573
570
|
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
|
|
574
571
|
run: npx pathgrade report
|
|
575
|
-
|
|
576
572
|
- uses: actions/upload-artifact@v4
|
|
577
573
|
if: always()
|
|
578
574
|
with:
|
|
@@ -589,16 +585,13 @@ See the [User Guide - CI Integration](docs/USER_GUIDE.md#ci-integration) for the
|
|
|
589
585
|
## Links
|
|
590
586
|
|
|
591
587
|
- [User Guide](docs/USER_GUIDE.md) - full API reference and usage patterns
|
|
592
|
-
-
|
|
593
|
-
|
|
594
|
-
|
|
595
|
-
- [tool-judge-demo](examples/tool-judge-demo/) - `judge({ tools })` reading workspace artifacts
|
|
588
|
+
- [start-chat](examples/start-chat/) - multi-turn conversation
|
|
589
|
+
- [sdk-showcase](examples/sdk-showcase/) - advanced SDK features in one suite
|
|
590
|
+
- [tool-judge-demo](examples/tool-judge-demo/) - `judge({ tools })` reading workspace artifacts
|
|
596
591
|
|
|
597
592
|
## Note on AI provider dependencies
|
|
598
593
|
|
|
599
|
-
Pathgrade is released under the MIT license.
|
|
600
|
-
|
|
601
|
-
In particular, `@anthropic-ai/claude-agent-sdk` is governed by Anthropic's Commercial Terms of Service, except where Anthropic specifies a different license for a specific component or dependency.
|
|
594
|
+
Pathgrade is released under the MIT license. It can use third-party AI agent tools and SDKs, including `@anthropic-ai/claude-agent-sdk`, Codex CLI, and `cursor-agent`. Those tools, services, and authentication methods have separate provider terms; in particular, the Claude Agent SDK is governed by Anthropic's Commercial Terms unless Anthropic specifies another license for a component.
|
|
602
595
|
|
|
603
596
|
## License
|
|
604
597
|
|
|
@@ -9,6 +9,8 @@ export function createJestInvocationAdapter(input) {
|
|
|
9
9
|
const spawnJest = input.spawnJest ?? defaultSpawnJest;
|
|
10
10
|
return {
|
|
11
11
|
name: 'jest',
|
|
12
|
+
supportsRepeatedAttempts: true,
|
|
13
|
+
writeEmptyReport: false,
|
|
12
14
|
async run(runInput) {
|
|
13
15
|
const selection = await readSidecarForInvocation(runInput.cwd, runInput.env.PATHGRADE_SELECTION_INVOCATION_ID, msg => {
|
|
14
16
|
process.stderr.write(`[pathgrade] ${msg}\n`);
|
|
@@ -36,7 +38,8 @@ export function createJestInvocationAdapter(input) {
|
|
|
36
38
|
},
|
|
37
39
|
runnerArgs: [],
|
|
38
40
|
env: runInput.env,
|
|
39
|
-
artifactRoot: getPathgradeDir(runInput.cwd),
|
|
41
|
+
artifactRoot: runInput.env.PATHGRADE_ARTIFACT_ROOT ?? getPathgradeDir(runInput.cwd),
|
|
42
|
+
normalizedSnapshotPath: runInput.env.PATHGRADE_NORMALIZED_SNAPSHOT_PATH,
|
|
40
43
|
reporterMode: input.config.reporter ?? 'cli',
|
|
41
44
|
threshold: input.config.ci.threshold,
|
|
42
45
|
selection,
|
|
@@ -69,6 +72,7 @@ export function createJestInvocationAdapter(input) {
|
|
|
69
72
|
'--reporters',
|
|
70
73
|
resolveReporterFile(),
|
|
71
74
|
...runInput.runnerArgs,
|
|
75
|
+
'--testLocationInResults',
|
|
72
76
|
],
|
|
73
77
|
});
|
|
74
78
|
},
|
|
@@ -4,12 +4,14 @@ import { fmt, getPathgradeDir, printReportSummary, readSidecarForInvocation, res
|
|
|
4
4
|
import { getJestLifecycleMetadata } from './lifecycle.js';
|
|
5
5
|
import { readJestMetadata, removeJestMetadata } from './metadata.js';
|
|
6
6
|
import { createJestAdapter } from './runner-adapter.js';
|
|
7
|
+
import { assertDirectReporterAttemptsSupported } from '../../runners/direct-reporter-attempts.js';
|
|
7
8
|
export default class PathgradeJestReporter {
|
|
8
9
|
async onRunComplete(_contexts, results) {
|
|
9
10
|
const cwd = process.cwd();
|
|
10
11
|
const config = await resolvePathgradeConfig({ cwd });
|
|
12
|
+
assertDirectReporterAttemptsSupported(config.attempts, process.env);
|
|
11
13
|
const mode = config.reporter ?? 'cli';
|
|
12
|
-
const outputDir = getPathgradeDir(cwd);
|
|
14
|
+
const outputDir = process.env.PATHGRADE_ARTIFACT_ROOT ?? getPathgradeDir(cwd);
|
|
13
15
|
const metadataByCaseId = new Map([
|
|
14
16
|
...readJestMetadata(),
|
|
15
17
|
...getJestLifecycleMetadata(),
|
|
@@ -25,6 +27,7 @@ export default class PathgradeJestReporter {
|
|
|
25
27
|
runnerArgs: [],
|
|
26
28
|
env: process.env,
|
|
27
29
|
artifactRoot: outputDir,
|
|
30
|
+
normalizedSnapshotPath: process.env.PATHGRADE_NORMALIZED_SNAPSHOT_PATH,
|
|
28
31
|
reporterMode: mode,
|
|
29
32
|
threshold: config.ci.threshold,
|
|
30
33
|
writeEmptyReport: false,
|
|
@@ -34,6 +34,7 @@ export function normalizeJestRunResults(input) {
|
|
|
34
34
|
runId,
|
|
35
35
|
unitId: unitId(fileIndex),
|
|
36
36
|
caseId,
|
|
37
|
+
repeatKey: jestRepeatKey(fileResult.testFilePath, assertion.fullName ?? fullCaseName(assertion), assertion.location),
|
|
37
38
|
assertion,
|
|
38
39
|
filePath: fileResult.testFilePath,
|
|
39
40
|
evaluations: input.metadataByCaseId?.get(caseId) ?? [],
|
|
@@ -71,6 +72,7 @@ function runCase(input) {
|
|
|
71
72
|
const failureMessage = input.assertion.failureMessages?.find(message => message.trim().length > 0);
|
|
72
73
|
return {
|
|
73
74
|
id: input.caseId,
|
|
75
|
+
repeatKey: input.repeatKey,
|
|
74
76
|
runId: input.runId,
|
|
75
77
|
unitId: input.unitId,
|
|
76
78
|
name: fullCaseName(input.assertion),
|
|
@@ -110,6 +112,12 @@ function runCase(input) {
|
|
|
110
112
|
}],
|
|
111
113
|
};
|
|
112
114
|
}
|
|
115
|
+
function jestRepeatKey(filePath, fullName, location) {
|
|
116
|
+
return JSON.stringify([
|
|
117
|
+
'jest', filePath.replaceAll('\\', '/'), fullName,
|
|
118
|
+
location?.line ?? null, location?.column ?? null,
|
|
119
|
+
]);
|
|
120
|
+
}
|
|
113
121
|
function fullCaseName(assertion) {
|
|
114
122
|
const ancestors = assertion.ancestorTitles ?? [];
|
|
115
123
|
return [...ancestors, assertion.title].filter(Boolean).join(' > ');
|
|
@@ -6,3 +6,8 @@ export declare function test(name: string, options: NodeTestOptions, fn: NodeTes
|
|
|
6
6
|
export default test;
|
|
7
7
|
export { createNodeTestAdapter } from './runner-adapter.js';
|
|
8
8
|
export { createNodeTestInvocationAdapter } from './invocation-adapter.js';
|
|
9
|
+
export declare function parseNodeTestCallerLocation(stack: string, cwd: string): {
|
|
10
|
+
sourceRef: string;
|
|
11
|
+
line?: number;
|
|
12
|
+
column?: number;
|
|
13
|
+
} | undefined;
|
|
@@ -11,8 +11,18 @@ export function test(name, optionsOrFn, maybeFn) {
|
|
|
11
11
|
installResultCapture();
|
|
12
12
|
const options = typeof optionsOrFn === 'function' ? undefined : optionsOrFn;
|
|
13
13
|
const fn = typeof optionsOrFn === 'function' ? optionsOrFn : maybeFn;
|
|
14
|
-
const
|
|
14
|
+
const source = callerSource();
|
|
15
|
+
const sourceRef = source.sourceRef;
|
|
15
16
|
const caseId = `${sourceRef}:${name}:${++caseCounter}`;
|
|
17
|
+
const repeatKey = JSON.stringify([
|
|
18
|
+
'node-test', sourceRef, source.line ?? null, source.column ?? null, name,
|
|
19
|
+
]);
|
|
20
|
+
if (options?.skip === true || typeof options?.skip === 'string') {
|
|
21
|
+
writeCase({
|
|
22
|
+
caseId, repeatKey, name, state: 'skipped', runnerDurationMs: 0,
|
|
23
|
+
sourceRef, filePath: sourceRef, groupName: sourceRef, runnerCaseId: caseId,
|
|
24
|
+
});
|
|
25
|
+
}
|
|
16
26
|
nodeTest(name, options, async (ctx) => {
|
|
17
27
|
const startedAt = performance.now();
|
|
18
28
|
let state = 'passed';
|
|
@@ -37,6 +47,7 @@ export function test(name, optionsOrFn, maybeFn) {
|
|
|
37
47
|
const evaluations = await lifecycle.flushCase(caseId);
|
|
38
48
|
writeCase({
|
|
39
49
|
caseId,
|
|
50
|
+
repeatKey,
|
|
40
51
|
name,
|
|
41
52
|
state,
|
|
42
53
|
runnerDurationMs: performance.now() - startedAt,
|
|
@@ -87,18 +98,35 @@ function writeCase(testCase) {
|
|
|
87
98
|
existing.push(testCase);
|
|
88
99
|
fs.writeFileSync(resultsPath, `${JSON.stringify(existing, null, 2)}\n`);
|
|
89
100
|
}
|
|
90
|
-
function
|
|
101
|
+
function callerSource() {
|
|
91
102
|
const stack = new Error().stack ?? '';
|
|
92
103
|
const cwd = realpath(process.env.PATHGRADE_NODE_TEST_CWD ?? process.cwd());
|
|
104
|
+
return parseNodeTestCallerLocation(stack, cwd) ?? { sourceRef: 'node-test' };
|
|
105
|
+
}
|
|
106
|
+
export function parseNodeTestCallerLocation(stack, cwd) {
|
|
93
107
|
for (const line of stack.split('\n')) {
|
|
94
|
-
const
|
|
95
|
-
if (!
|
|
108
|
+
const location = line.match(/:(\d+):(\d+)\)?\s*$/);
|
|
109
|
+
if (!location || location.index === undefined)
|
|
110
|
+
continue;
|
|
111
|
+
const prefix = line.slice(0, location.index);
|
|
112
|
+
const extension = prefix.match(/\.eval\.[cm]?[jt]s$/);
|
|
113
|
+
if (!extension)
|
|
96
114
|
continue;
|
|
97
|
-
const
|
|
115
|
+
const fileUrlStart = prefix.indexOf('file://');
|
|
116
|
+
const cwdStart = prefix.indexOf(cwd);
|
|
117
|
+
const wrapperStart = prefix.indexOf('(');
|
|
118
|
+
const start = fileUrlStart >= 0 ? fileUrlStart
|
|
119
|
+
: cwdStart >= 0 ? cwdStart
|
|
120
|
+
: wrapperStart >= 0 ? wrapperStart + 1
|
|
121
|
+
: prefix.lastIndexOf('at ') + 3;
|
|
122
|
+
const raw = prefix.slice(Math.max(0, start));
|
|
98
123
|
const abs = realpath(raw.startsWith('file://') ? fileURLToPath(raw) : raw);
|
|
99
|
-
return
|
|
124
|
+
return {
|
|
125
|
+
sourceRef: path.relative(cwd, abs).replaceAll(path.sep, '/'),
|
|
126
|
+
line: Number(location[1]),
|
|
127
|
+
column: Number(location[2]),
|
|
128
|
+
};
|
|
100
129
|
}
|
|
101
|
-
return 'node-test';
|
|
102
130
|
}
|
|
103
131
|
function realpath(value) {
|
|
104
132
|
try {
|
|
@@ -6,6 +6,7 @@ import { createNodeTestAdapter } from './runner-adapter.js';
|
|
|
6
6
|
export function createNodeTestInvocationAdapter(input) {
|
|
7
7
|
return {
|
|
8
8
|
name: 'node-test',
|
|
9
|
+
supportsRepeatedAttempts: true,
|
|
9
10
|
async run(runInput) {
|
|
10
11
|
const selection = await readSidecarForInvocation(runInput.cwd, runInput.env.PATHGRADE_SELECTION_INVOCATION_ID, msg => {
|
|
11
12
|
process.stderr.write(`[pathgrade] ${msg}\n`);
|
|
@@ -24,7 +25,8 @@ export function createNodeTestInvocationAdapter(input) {
|
|
|
24
25
|
},
|
|
25
26
|
runnerArgs: runInput.runnerArgs,
|
|
26
27
|
env: runInput.env,
|
|
27
|
-
artifactRoot: getPathgradeDir(runInput.cwd),
|
|
28
|
+
artifactRoot: runInput.env.PATHGRADE_ARTIFACT_ROOT ?? getPathgradeDir(runInput.cwd),
|
|
29
|
+
normalizedSnapshotPath: runInput.env.PATHGRADE_NORMALIZED_SNAPSHOT_PATH,
|
|
28
30
|
reporterMode: input.config.reporter ?? 'cli',
|
|
29
31
|
threshold: input.config.ci.threshold,
|
|
30
32
|
selection,
|
|
@@ -37,7 +37,7 @@ export function createNodeTestAdapter() {
|
|
|
37
37
|
adapterName: this.name,
|
|
38
38
|
status: 'completed',
|
|
39
39
|
exitCode: 0,
|
|
40
|
-
native: { resultsPath },
|
|
40
|
+
native: { resultsPath, resultsDir },
|
|
41
41
|
};
|
|
42
42
|
}
|
|
43
43
|
const exitCode = await spawnNodeTest({
|
|
@@ -54,23 +54,29 @@ export function createNodeTestAdapter() {
|
|
|
54
54
|
adapterName: this.name,
|
|
55
55
|
status: input.signal?.aborted ? 'cancelled' : (exitCode === 0 ? 'completed' : 'failed'),
|
|
56
56
|
exitCode,
|
|
57
|
-
native: { resultsPath },
|
|
57
|
+
native: { resultsPath, resultsDir },
|
|
58
58
|
};
|
|
59
59
|
},
|
|
60
60
|
async collectNormalizedRunSnapshot(run) {
|
|
61
61
|
const native = readNodeTestRunNative(run);
|
|
62
|
-
|
|
63
|
-
|
|
64
|
-
|
|
65
|
-
const
|
|
66
|
-
|
|
67
|
-
groupMap.
|
|
68
|
-
|
|
62
|
+
try {
|
|
63
|
+
const cases = await readCases(native.resultsPath);
|
|
64
|
+
const groupMap = new Map();
|
|
65
|
+
for (const testCase of cases) {
|
|
66
|
+
const groupName = testCase.groupName ?? testCase.filePath ?? testCase.sourceRef ?? 'node-test';
|
|
67
|
+
if (!groupMap.has(groupName))
|
|
68
|
+
groupMap.set(groupName, []);
|
|
69
|
+
groupMap.get(groupName).push(testCase);
|
|
70
|
+
}
|
|
71
|
+
return buildNormalizedRunSnapshotFromReportGroups(run, Array.from(groupMap.entries()).map(([groupName, groupedCases]) => ({
|
|
72
|
+
groupName,
|
|
73
|
+
cases: groupedCases,
|
|
74
|
+
})));
|
|
75
|
+
}
|
|
76
|
+
finally {
|
|
77
|
+
if (native.resultsDir)
|
|
78
|
+
await fs.remove(native.resultsDir);
|
|
69
79
|
}
|
|
70
|
-
return buildNormalizedRunSnapshotFromReportGroups(run, Array.from(groupMap.entries()).map(([groupName, groupedCases]) => ({
|
|
71
|
-
groupName,
|
|
72
|
-
cases: groupedCases,
|
|
73
|
-
})));
|
|
74
80
|
},
|
|
75
81
|
};
|
|
76
82
|
}
|
|
@@ -96,10 +102,11 @@ async function spawnNodeTest(input) {
|
|
|
96
102
|
function readNodeTestRunNative(run) {
|
|
97
103
|
if (typeof run.native === 'object'
|
|
98
104
|
&& run.native !== null
|
|
99
|
-
&& typeof run.native.resultsPath === 'string'
|
|
105
|
+
&& typeof run.native.resultsPath === 'string'
|
|
106
|
+
&& typeof run.native.resultsDir === 'string') {
|
|
100
107
|
return run.native;
|
|
101
108
|
}
|
|
102
|
-
return { resultsPath: '' };
|
|
109
|
+
return { resultsPath: '', resultsDir: '' };
|
|
103
110
|
}
|
|
104
111
|
async function readCases(resultsPath) {
|
|
105
112
|
if (!resultsPath || !(await fs.pathExists(resultsPath)))
|
|
@@ -7,6 +7,7 @@ import { printReportSummary } from '../../reporters/report-summary.js';
|
|
|
7
7
|
import { createVitestAdapter } from '../../runners/vitest-adapter.js';
|
|
8
8
|
import { runWithAdapter } from '../../runners/orchestrator.js';
|
|
9
9
|
import { resolvePathgradeConfig } from '../../config/pathgrade.js';
|
|
10
|
+
import { assertDirectReporterAttemptsSupported } from '../../runners/direct-reporter-attempts.js';
|
|
10
11
|
/**
|
|
11
12
|
* Custom vitest reporter that layers pathgrade aggregate statistics
|
|
12
13
|
* on top of vitest's default output.
|
|
@@ -19,8 +20,9 @@ export class PathgradeReporter {
|
|
|
19
20
|
async onTestRunEnd(testModules) {
|
|
20
21
|
const cwd = process.cwd();
|
|
21
22
|
const config = await resolvePathgradeConfig({ cwd });
|
|
23
|
+
assertDirectReporterAttemptsSupported(config.attempts, process.env);
|
|
22
24
|
const mode = this.opts.reporter ?? config.reporter ?? 'cli';
|
|
23
|
-
const outputDir = getPathgradeDir(cwd);
|
|
25
|
+
const outputDir = process.env.PATHGRADE_ARTIFACT_ROOT ?? getPathgradeDir(cwd);
|
|
24
26
|
const adapter = createVitestAdapter({ testModules });
|
|
25
27
|
const exitCode = await runWithAdapter({
|
|
26
28
|
adapter,
|
|
@@ -30,6 +32,7 @@ export class PathgradeReporter {
|
|
|
30
32
|
runnerArgs: [],
|
|
31
33
|
env: process.env,
|
|
32
34
|
artifactRoot: outputDir,
|
|
35
|
+
normalizedSnapshotPath: process.env.PATHGRADE_NORMALIZED_SNAPSHOT_PATH,
|
|
33
36
|
reporterMode: mode,
|
|
34
37
|
threshold: this.opts.ci?.threshold ?? config.ci.threshold,
|
|
35
38
|
writeEmptyReport: false,
|
package/dist/agents/claude.js
CHANGED
|
@@ -88,10 +88,11 @@ export class ClaudeAgent extends BaseAgent {
|
|
|
88
88
|
hostEnv,
|
|
89
89
|
sandboxProfile: this.deps.sandboxProfile,
|
|
90
90
|
});
|
|
91
|
-
const claudeCodeExecutable =
|
|
92
|
-
|
|
93
|
-
|
|
94
|
-
|
|
91
|
+
const claudeCodeExecutable = sessionOptions?.admittedClaudeCodeExecutable
|
|
92
|
+
?? resolveClaudeCodeExecutable({
|
|
93
|
+
agentOptionsExecutable: this.opts.claudeCodeExecutable,
|
|
94
|
+
envExecutable,
|
|
95
|
+
});
|
|
95
96
|
const mcpMountOptions = {
|
|
96
97
|
workspacePath,
|
|
97
98
|
mcpConfigPath: sessionOptions?.mcpConfigPath,
|
|
@@ -16,58 +16,8 @@ import { projectItemIntoTurn } from './item-projection.js';
|
|
|
16
16
|
import { CodexItemLifecycle, } from './item-lifecycle.js';
|
|
17
17
|
import { createCodexManagedAuth } from './managed-auth.js';
|
|
18
18
|
import { clearTurnAuthorityDeadline, closeUncorrelatedTurnTransport, establishAuthoritativeTurn, isAbortError, isAuthoritativeTurnCompletion, rememberAuthoritativeTurnId, shouldRetireAuthoritativeTurnTransport, } from './turn-authority.js';
|
|
19
|
+
import { dispatchCodexNotification, failTurn } from './turn-notifications.js';
|
|
19
20
|
const TURN_COMPLETED_METHOD = 'turn/completed';
|
|
20
|
-
function projectMcpStartupStatusIntoTurn(params, turn) {
|
|
21
|
-
const name = params.name ?? 'unknown';
|
|
22
|
-
const status = params.status ?? 'unknown';
|
|
23
|
-
const error = typeof params.error === 'string' ? params.error : undefined;
|
|
24
|
-
turn.nonAskToolEvents.push({
|
|
25
|
-
action: 'unknown',
|
|
26
|
-
provider: 'codex',
|
|
27
|
-
providerToolName: 'mcpServer/startupStatus/updated',
|
|
28
|
-
turnNumber: turn.turnNumber,
|
|
29
|
-
arguments: {
|
|
30
|
-
name,
|
|
31
|
-
status,
|
|
32
|
-
...(error ? { error } : {}),
|
|
33
|
-
},
|
|
34
|
-
summary: `MCP server ${name} startup ${status}`,
|
|
35
|
-
confidence: 'high',
|
|
36
|
-
rawSnippet: JSON.stringify(params),
|
|
37
|
-
});
|
|
38
|
-
if (status === 'failed') {
|
|
39
|
-
const message = `MCP server ${name} failed to start${error ? `: ${error}` : ''}`;
|
|
40
|
-
failTurn(turn, message);
|
|
41
|
-
}
|
|
42
|
-
}
|
|
43
|
-
function dispatchCodexNotification(notification, turn, sensitiveValues) {
|
|
44
|
-
if (process.env.PATHGRADE_CODEX_DEBUG) {
|
|
45
|
-
const debugParams = sanitizePersistenceValue(notification.params, sensitiveValues);
|
|
46
|
-
console.error(`[codex app-server] notification method=${notification.method} params=${JSON.stringify(debugParams).slice(0, 300)}`);
|
|
47
|
-
}
|
|
48
|
-
if (notification.method === 'mcpServer/startupStatus/updated') {
|
|
49
|
-
if (turn) {
|
|
50
|
-
projectMcpStartupStatusIntoTurn((notification.params ?? {}), turn);
|
|
51
|
-
}
|
|
52
|
-
return;
|
|
53
|
-
}
|
|
54
|
-
if (!turn)
|
|
55
|
-
return;
|
|
56
|
-
const params = notification.params;
|
|
57
|
-
if (notification.method === 'item/started') {
|
|
58
|
-
if (params)
|
|
59
|
-
turn.itemLifecycle?.receiveStarted(params);
|
|
60
|
-
return;
|
|
61
|
-
}
|
|
62
|
-
if (notification.method === 'item/completed' && params) {
|
|
63
|
-
turn.itemLifecycle?.receiveCompleted(params);
|
|
64
|
-
}
|
|
65
|
-
}
|
|
66
|
-
function failTurn(turn, message) {
|
|
67
|
-
turn.turnFailed = true;
|
|
68
|
-
turn.failureMessage = message;
|
|
69
|
-
turn.signalFailure?.(message);
|
|
70
|
-
}
|
|
71
21
|
function recordFromUnknown(value) {
|
|
72
22
|
return value && typeof value === 'object' && !Array.isArray(value)
|
|
73
23
|
? value
|
|
@@ -0,0 +1,6 @@
|
|
|
1
|
+
import type { ActiveTurnState } from './turn-state.js';
|
|
2
|
+
export declare function dispatchCodexNotification(notification: {
|
|
3
|
+
method: string;
|
|
4
|
+
params: unknown;
|
|
5
|
+
}, turn: ActiveTurnState | null, sensitiveValues: readonly string[]): void;
|
|
6
|
+
export declare function failTurn(turn: ActiveTurnState, message: string): void;
|
|
@@ -0,0 +1,51 @@
|
|
|
1
|
+
import { sanitizePersistenceValue } from '../../tool-event-results.js';
|
|
2
|
+
export function dispatchCodexNotification(notification, turn, sensitiveValues) {
|
|
3
|
+
if (process.env.PATHGRADE_CODEX_DEBUG) {
|
|
4
|
+
const debugParams = sanitizePersistenceValue(notification.params, sensitiveValues);
|
|
5
|
+
console.error(`[codex app-server] notification method=${notification.method} params=${JSON.stringify(debugParams).slice(0, 300)}`);
|
|
6
|
+
}
|
|
7
|
+
if (notification.method === 'mcpServer/startupStatus/updated') {
|
|
8
|
+
if (turn) {
|
|
9
|
+
projectMcpStartupStatusIntoTurn((notification.params ?? {}), turn);
|
|
10
|
+
}
|
|
11
|
+
return;
|
|
12
|
+
}
|
|
13
|
+
if (!turn)
|
|
14
|
+
return;
|
|
15
|
+
const params = notification.params;
|
|
16
|
+
if (notification.method === 'item/started') {
|
|
17
|
+
if (params)
|
|
18
|
+
turn.itemLifecycle?.receiveStarted(params);
|
|
19
|
+
return;
|
|
20
|
+
}
|
|
21
|
+
if (notification.method === 'item/completed' && params) {
|
|
22
|
+
turn.itemLifecycle?.receiveCompleted(params);
|
|
23
|
+
}
|
|
24
|
+
}
|
|
25
|
+
export function failTurn(turn, message) {
|
|
26
|
+
turn.turnFailed = true;
|
|
27
|
+
turn.failureMessage = message;
|
|
28
|
+
turn.signalFailure?.(message);
|
|
29
|
+
}
|
|
30
|
+
function projectMcpStartupStatusIntoTurn(params, turn) {
|
|
31
|
+
const name = params.name ?? 'unknown';
|
|
32
|
+
const status = params.status ?? 'unknown';
|
|
33
|
+
const error = typeof params.error === 'string' ? params.error : undefined;
|
|
34
|
+
turn.nonAskToolEvents.push({
|
|
35
|
+
action: 'unknown',
|
|
36
|
+
provider: 'codex',
|
|
37
|
+
providerToolName: 'mcpServer/startupStatus/updated',
|
|
38
|
+
turnNumber: turn.turnNumber,
|
|
39
|
+
arguments: {
|
|
40
|
+
name,
|
|
41
|
+
status,
|
|
42
|
+
...(error ? { error } : {}),
|
|
43
|
+
},
|
|
44
|
+
summary: `MCP server ${name} startup ${status}`,
|
|
45
|
+
confidence: 'high',
|
|
46
|
+
rawSnippet: JSON.stringify(params),
|
|
47
|
+
});
|
|
48
|
+
if (status === 'failed') {
|
|
49
|
+
failTurn(turn, `MCP server ${name} failed to start${error ? `: ${error}` : ''}`);
|
|
50
|
+
}
|
|
51
|
+
}
|
|
@@ -0,0 +1,19 @@
|
|
|
1
|
+
import type { ToolEvent } from '../../tool-events.js';
|
|
2
|
+
import type { PendingMcpDenial } from './mcp-approval-correlator.js';
|
|
3
|
+
import type { CodexItemLifecycle } from './item-lifecycle.js';
|
|
4
|
+
export interface ActiveTurnState {
|
|
5
|
+
turnNumber: number;
|
|
6
|
+
askBatchIds: string[];
|
|
7
|
+
nonAskToolEvents: ToolEvent[];
|
|
8
|
+
pendingMcpDenials: PendingMcpDenial[];
|
|
9
|
+
itemLifecycle?: CodexItemLifecycle;
|
|
10
|
+
assistantMessageParts: string[];
|
|
11
|
+
turnFailed: boolean;
|
|
12
|
+
failureMessage?: string;
|
|
13
|
+
signalFailure?: (message: string) => void;
|
|
14
|
+
authoritativeThreadId?: string;
|
|
15
|
+
authoritativeTurnId?: string;
|
|
16
|
+
pendingTurnCompletions: unknown[];
|
|
17
|
+
acceptTurnCompletion?: (params: unknown) => void;
|
|
18
|
+
authorityDeadlineTimer?: ReturnType<typeof setTimeout>;
|
|
19
|
+
}
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
export {};
|
|
@@ -0,0 +1,7 @@
|
|
|
1
|
+
import type { ToolAction } from '../../tool-events.js';
|
|
2
|
+
export declare const NATIVE_TOOL_ACTIONS: Readonly<Record<string, ToolAction>>;
|
|
3
|
+
export declare const ALLOWED_EVENT_TYPES: ReadonlySet<string>;
|
|
4
|
+
export declare function protocolRecord(value: unknown, label: string): Record<string, unknown>;
|
|
5
|
+
export declare function protocolFiniteNumber(value: unknown, label: string): number;
|
|
6
|
+
export declare function protocolRequiredString(value: unknown, label: string): string;
|
|
7
|
+
export declare function sanitizedProviderError(event: Record<string, unknown>): Error;
|
|
@@ -0,0 +1,47 @@
|
|
|
1
|
+
export const NATIVE_TOOL_ACTIONS = {
|
|
2
|
+
bash: 'run_shell',
|
|
3
|
+
read: 'read_file',
|
|
4
|
+
write: 'write_file',
|
|
5
|
+
edit: 'edit_file',
|
|
6
|
+
grep: 'search_code',
|
|
7
|
+
glob: 'list_files',
|
|
8
|
+
list: 'list_files',
|
|
9
|
+
skill: 'use_skill',
|
|
10
|
+
todowrite: 'update_todos',
|
|
11
|
+
lsp: 'search_code',
|
|
12
|
+
};
|
|
13
|
+
export const ALLOWED_EVENT_TYPES = new Set([
|
|
14
|
+
'tool_use',
|
|
15
|
+
'step_start',
|
|
16
|
+
'step_finish',
|
|
17
|
+
'text',
|
|
18
|
+
'reasoning',
|
|
19
|
+
'error',
|
|
20
|
+
]);
|
|
21
|
+
export function protocolRecord(value, label) {
|
|
22
|
+
if (!value || typeof value !== 'object' || Array.isArray(value)) {
|
|
23
|
+
throw new Error(`OpenCode protocol error: ${label} must be an object`);
|
|
24
|
+
}
|
|
25
|
+
return value;
|
|
26
|
+
}
|
|
27
|
+
export function protocolFiniteNumber(value, label) {
|
|
28
|
+
if (typeof value !== 'number' || !Number.isFinite(value) || value < 0) {
|
|
29
|
+
throw new Error(`OpenCode protocol error: ${label} must be a nonnegative finite number`);
|
|
30
|
+
}
|
|
31
|
+
return value;
|
|
32
|
+
}
|
|
33
|
+
export function protocolRequiredString(value, label) {
|
|
34
|
+
if (typeof value !== 'string' || !value) {
|
|
35
|
+
throw new Error(`OpenCode protocol error: ${label} must be a nonempty string`);
|
|
36
|
+
}
|
|
37
|
+
return value;
|
|
38
|
+
}
|
|
39
|
+
export function sanitizedProviderError(event) {
|
|
40
|
+
const error = protocolRecord(event.error, 'error');
|
|
41
|
+
const data = error.data && typeof error.data === 'object' && !Array.isArray(error.data)
|
|
42
|
+
? error.data
|
|
43
|
+
: {};
|
|
44
|
+
const status = typeof data.statusCode === 'number' ? ` status=${data.statusCode}` : '';
|
|
45
|
+
const retryable = typeof data.isRetryable === 'boolean' ? ` retryable=${data.isRetryable}` : '';
|
|
46
|
+
return new Error(`OpenCode provider error${status}${retryable}`);
|
|
47
|
+
}
|
package/dist/agents/opencode.js
CHANGED
|
@@ -14,28 +14,9 @@ import { OpenCodeRuntimePolicy, OPENCODE_PERMISSION } from './opencode/runtime-p
|
|
|
14
14
|
import { killOpenCodeProcessGroup, registerOpenCodeProcessGroup, unregisterOpenCodeProcessGroup, } from './opencode/process-groups.js';
|
|
15
15
|
import { assertCleanManagedOpenCodeHost, sha256File } from './opencode/host-safety.js';
|
|
16
16
|
import { getOpenCodeScenarioSession } from './opencode/scenario.js';
|
|
17
|
+
import { ALLOWED_EVENT_TYPES, NATIVE_TOOL_ACTIONS, protocolFiniteNumber as finiteNumber, protocolRecord as record, protocolRequiredString as requiredString, sanitizedProviderError, } from './opencode/protocol.js';
|
|
17
18
|
export { assertCleanManagedOpenCodeHost, managedOpenCodeConfigPaths } from './opencode/host-safety.js';
|
|
18
19
|
const OUTPUT_CAP_BYTES = 16 * 1024 * 1024;
|
|
19
|
-
const NATIVE_TOOL_ACTIONS = {
|
|
20
|
-
bash: 'run_shell',
|
|
21
|
-
read: 'read_file',
|
|
22
|
-
write: 'write_file',
|
|
23
|
-
edit: 'edit_file',
|
|
24
|
-
grep: 'search_code',
|
|
25
|
-
glob: 'list_files',
|
|
26
|
-
list: 'list_files',
|
|
27
|
-
skill: 'use_skill',
|
|
28
|
-
todowrite: 'update_todos',
|
|
29
|
-
lsp: 'search_code',
|
|
30
|
-
};
|
|
31
|
-
const ALLOWED_EVENT_TYPES = new Set([
|
|
32
|
-
'tool_use',
|
|
33
|
-
'step_start',
|
|
34
|
-
'step_finish',
|
|
35
|
-
'text',
|
|
36
|
-
'reasoning',
|
|
37
|
-
'error',
|
|
38
|
-
]);
|
|
39
20
|
export function spawnOpenCode(executable, args, options) {
|
|
40
21
|
return new Promise((resolve, reject) => {
|
|
41
22
|
const stdout = [];
|
|
@@ -124,33 +105,6 @@ export function spawnOpenCode(executable, args, options) {
|
|
|
124
105
|
}
|
|
125
106
|
});
|
|
126
107
|
}
|
|
127
|
-
function record(value, label) {
|
|
128
|
-
if (!value || typeof value !== 'object' || Array.isArray(value)) {
|
|
129
|
-
throw new Error(`OpenCode protocol error: ${label} must be an object`);
|
|
130
|
-
}
|
|
131
|
-
return value;
|
|
132
|
-
}
|
|
133
|
-
function finiteNumber(value, label) {
|
|
134
|
-
if (typeof value !== 'number' || !Number.isFinite(value) || value < 0) {
|
|
135
|
-
throw new Error(`OpenCode protocol error: ${label} must be a nonnegative finite number`);
|
|
136
|
-
}
|
|
137
|
-
return value;
|
|
138
|
-
}
|
|
139
|
-
function requiredString(value, label) {
|
|
140
|
-
if (typeof value !== 'string' || !value) {
|
|
141
|
-
throw new Error(`OpenCode protocol error: ${label} must be a nonempty string`);
|
|
142
|
-
}
|
|
143
|
-
return value;
|
|
144
|
-
}
|
|
145
|
-
function sanitizedProviderError(event) {
|
|
146
|
-
const error = record(event.error, 'error');
|
|
147
|
-
const data = error.data && typeof error.data === 'object' && !Array.isArray(error.data)
|
|
148
|
-
? error.data
|
|
149
|
-
: {};
|
|
150
|
-
const status = typeof data.statusCode === 'number' ? ` status=${data.statusCode}` : '';
|
|
151
|
-
const retryable = typeof data.isRetryable === 'boolean' ? ` retryable=${data.isRetryable}` : '';
|
|
152
|
-
return new Error(`OpenCode provider error${status}${retryable}`);
|
|
153
|
-
}
|
|
154
108
|
export function parseOpenCodeOutput(stdout, processResult, mcpToolNames, sensitiveValues = []) {
|
|
155
109
|
if (processResult.overflow)
|
|
156
110
|
throw new Error('OpenCode output exceeded the 16 MiB limit');
|