@wix/pathgrade 1.0.27 → 1.0.29

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (64) hide show
  1. package/README.md +12 -19
  2. package/dist/adapters/jest/invocation-adapter.js +5 -1
  3. package/dist/adapters/jest/reporter.js +4 -1
  4. package/dist/adapters/jest/results.js +8 -0
  5. package/dist/adapters/node-test/index.d.ts +5 -0
  6. package/dist/adapters/node-test/index.js +35 -7
  7. package/dist/adapters/node-test/invocation-adapter.js +3 -1
  8. package/dist/adapters/node-test/runner-adapter.js +22 -15
  9. package/dist/adapters/vitest/reporter.js +4 -1
  10. package/dist/agents/claude.js +5 -4
  11. package/dist/agents/codex-app-server/agent.js +1 -51
  12. package/dist/agents/codex-app-server/turn-notifications.d.ts +6 -0
  13. package/dist/agents/codex-app-server/turn-notifications.js +51 -0
  14. package/dist/agents/codex-app-server/turn-state.d.ts +19 -0
  15. package/dist/agents/codex-app-server/turn-state.js +1 -0
  16. package/dist/agents/opencode/protocol.d.ts +7 -0
  17. package/dist/agents/opencode/protocol.js +47 -0
  18. package/dist/agents/opencode.js +1 -47
  19. package/dist/analytics/engine.js +5 -2
  20. package/dist/commands/report.js +16 -2
  21. package/dist/commands/run-args.d.ts +1 -0
  22. package/dist/commands/run-args.js +13 -0
  23. package/dist/commands/run-changed.js +5 -15
  24. package/dist/config/pathgrade.d.ts +3 -0
  25. package/dist/config/pathgrade.js +33 -1
  26. package/dist/internal/direct-mcp-v2/claude-direct-mcp.d.ts +2 -1
  27. package/dist/internal/direct-mcp-v2/claude-direct-mcp.js +35 -17
  28. package/dist/internal/direct-mcp-v2/claude-profile.d.ts +33 -7
  29. package/dist/internal/direct-mcp-v2/claude-profile.js +104 -72
  30. package/dist/internal/direct-mcp-v2/public-scenario-runtime.js +13 -16
  31. package/dist/pathgrade.js +10 -2
  32. package/dist/reporters/cli.js +13 -6
  33. package/dist/reporters/github-comment.js +12 -9
  34. package/dist/reporters/loader.d.ts +1 -0
  35. package/dist/reporters/loader.js +27 -2
  36. package/dist/reporters/report-summary.js +13 -5
  37. package/dist/reporting/core.d.ts +1 -0
  38. package/dist/reporting/core.js +183 -105
  39. package/dist/reporting/types.d.ts +19 -3
  40. package/dist/runners/adapter-loader.js +17 -12
  41. package/dist/runners/direct-reporter-attempts.d.ts +1 -0
  42. package/dist/runners/direct-reporter-attempts.js +7 -0
  43. package/dist/runners/invocation.d.ts +2 -0
  44. package/dist/runners/model-builders.js +1 -0
  45. package/dist/runners/model-validation.js +31 -0
  46. package/dist/runners/model.d.ts +4 -0
  47. package/dist/runners/orchestrator.d.ts +2 -0
  48. package/dist/runners/orchestrator.js +11 -1
  49. package/dist/runners/repeated-attempts.d.ts +7 -0
  50. package/dist/runners/repeated-attempts.js +149 -0
  51. package/dist/runners/repeated-invocation.d.ts +7 -0
  52. package/dist/runners/repeated-invocation.js +129 -0
  53. package/dist/runners/report-projection.js +16 -6
  54. package/dist/runners/vitest-adapter.js +10 -0
  55. package/dist/runners/vitest-invocation.js +2 -0
  56. package/dist/sdk/agent-runtime-options.d.ts +12 -0
  57. package/dist/sdk/agent-runtime-options.js +67 -0
  58. package/dist/sdk/agent.js +4 -57
  59. package/dist/sdk/case-context.js +7 -2
  60. package/dist/sdk/lifecycle.js +8 -3
  61. package/dist/sdk/result-capture.js +4 -1
  62. package/dist/types.d.ts +34 -5
  63. package/dist/viewer.html +19 -19
  64. package/package.json +2 -2
package/README.md CHANGED
@@ -420,7 +420,7 @@ provider-specific scenario flags and profile controls are not public.
420
420
  ## CLI
421
421
 
422
422
  ```bash
423
- pathgrade run [--changed] [--since=<ref>] [--changed-files=<path>] [--adapter=<name|path>] [--diagnostics] [--verbose] [--quiet] [-- runner-args]
423
+ pathgrade run [--changed] [--since=<ref>] [--changed-files=<path>] [--adapter=<name|path>] [--attempts=N] [--diagnostics] [--verbose] [--quiet] [-- runner-args]
424
424
  pathgrade init [--force]
425
425
  pathgrade validate <file.eval.ts>
426
426
  pathgrade validate --affected
@@ -434,6 +434,7 @@ pathgrade report [--results-path=<path>] [--no-comment] [--comment-id=<id>] [--d
434
434
  Useful details:
435
435
 
436
436
  - `pathgrade run --changed` computes affected evals first, writes selection metadata to `.pathgrade/selection.json`, and only then launches the selected runner adapter. Vitest is the default adapter; Jest is selected with `runner.adapter: 'jest'` or `--adapter=jest`.
437
+ - `pathgrade run --attempts=3` reruns every selected case in three sequential, fresh runner processes. The CLI value overrides `attempts` in config; arguments after `--` still belong to the runner.
437
438
  - `pathgrade preview browser` starts a local viewer on `http://localhost:3847`.
438
439
  - `pathgrade report` posts or updates a PR comment in GitHub Actions; locally it prints the markdown report and then the numeric pass rate. Provider orchestrators can add a `--details-url`, suppress stale updates with `--expected-head-sha`, and opt into surfaced API failures with `--strict`.
439
440
  - `pathgrade validate --affected` is a strict mode for CI: every discovered eval must either live under a `SKILL.md` anchor or export valid `__pathgradeMeta`.
@@ -445,22 +446,20 @@ Run `pathgrade --help` for the full help text.
445
446
  ```typescript
446
447
  // pathgrade.config.ts
447
448
  export default {
448
- runner: {
449
- adapter: 'vitest',
450
- args: [],
451
- },
449
+ attempts: 1, // opt in to runner-owned repetitions
450
+ runner: { adapter: 'vitest', args: [] },
452
451
  evals: {
453
452
  include: ['**/*.eval.ts'], // default
454
453
  exclude: ['**/fixtures/**'], // replaces the default exclude list if set
455
454
  },
456
- affected: {
457
- global: ['package.json', 'yarn.lock'],
458
- },
455
+ affected: { global: ['package.json', 'yarn.lock'] },
459
456
  ci: { threshold: 0.8 },
460
457
  };
461
458
  ```
462
459
 
463
- Pathgrade reads `pathgrade.config.*` for CLI and affected-selection behavior. `runner.adapter` and `--adapter=<name|path>` select the runner; `--adapter` wins over config. Built-in adapters currently include `vitest`, `jest`, and the narrow `node-test` proof adapter.
460
+ Pathgrade reads `pathgrade.config.*` for CLI and affected-selection behavior. `runner.adapter` and `--adapter=<name|path>` select the runner; `--adapter` wins over config. `attempts` defaults to `1` and must be a positive integer. Built-in adapters `vitest`, `jest`, and `node-test` support repeated attempts; third-party invocation adapters must advertise `supportsRepeatedAttempts: true` and produce the normalized child snapshot contract.
461
+
462
+ New reports use schema version 2. They preserve every attempt with `case_id`, `attempt_id`, and `attempt_index`; include the display-safe `runner_outcome` that determined binary success; publish `mean_reward` for partial scores; and publish finite-sample pass@k only for complete binary attempts. Runner assertions, runner diagnostics, and native runner references remain in the normalized run model and are not copied into public report artifacts; evaluation diagnostics retain their existing report behavior. A partial reward or incomplete attempt makes pass@k explicitly unavailable. Version-1 reports remain readable, but Pathgrade no longer computes pass@k or pass^k by pooling heterogeneous cases.
464
463
 
465
464
  Third-party runner adapters are supported through `@wix/pathgrade/adapter-kit`. Adapter names resolve as follows:
466
465
 
@@ -561,18 +560,15 @@ jobs:
561
560
  with:
562
561
  node-version: '20'
563
562
  - run: npm ci
564
-
565
563
  - name: Run affected evals
566
564
  env:
567
565
  ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }}
568
566
  run: npx pathgrade run --changed
569
-
570
567
  - name: Post PR report
571
568
  if: always()
572
569
  env:
573
570
  GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
574
571
  run: npx pathgrade report
575
-
576
572
  - uses: actions/upload-artifact@v4
577
573
  if: always()
578
574
  with:
@@ -589,16 +585,13 @@ See the [User Guide - CI Integration](docs/USER_GUIDE.md#ci-integration) for the
589
585
  ## Links
590
586
 
591
587
  - [User Guide](docs/USER_GUIDE.md) - full API reference and usage patterns
592
- - Examples:
593
- - [start-chat](examples/start-chat/) - multi-turn conversation
594
- - [sdk-showcase](examples/sdk-showcase/) - advanced SDK features in one suite
595
- - [tool-judge-demo](examples/tool-judge-demo/) - `judge({ tools })` reading workspace artifacts
588
+ - [start-chat](examples/start-chat/) - multi-turn conversation
589
+ - [sdk-showcase](examples/sdk-showcase/) - advanced SDK features in one suite
590
+ - [tool-judge-demo](examples/tool-judge-demo/) - `judge({ tools })` reading workspace artifacts
596
591
 
597
592
  ## Note on AI provider dependencies
598
593
 
599
- Pathgrade is released under the MIT license. Pathgrade can use third-party AI agent tools and SDKs to run evaluations, including `@anthropic-ai/claude-agent-sdk` for Claude, Codex CLI for Codex, and `cursor-agent` for Cursor. These tools, SDKs, hosted services, and related authentication methods are governed by their respective provider terms, which are separate from Pathgrade's MIT license.
600
-
601
- In particular, `@anthropic-ai/claude-agent-sdk` is governed by Anthropic's Commercial Terms of Service, except where Anthropic specifies a different license for a specific component or dependency.
594
+ Pathgrade is released under the MIT license. It can use third-party AI agent tools and SDKs, including `@anthropic-ai/claude-agent-sdk`, Codex CLI, and `cursor-agent`. Those tools, services, and authentication methods have separate provider terms; in particular, the Claude Agent SDK is governed by Anthropic's Commercial Terms unless Anthropic specifies another license for a component.
602
595
 
603
596
  ## License
604
597
 
@@ -9,6 +9,8 @@ export function createJestInvocationAdapter(input) {
9
9
  const spawnJest = input.spawnJest ?? defaultSpawnJest;
10
10
  return {
11
11
  name: 'jest',
12
+ supportsRepeatedAttempts: true,
13
+ writeEmptyReport: false,
12
14
  async run(runInput) {
13
15
  const selection = await readSidecarForInvocation(runInput.cwd, runInput.env.PATHGRADE_SELECTION_INVOCATION_ID, msg => {
14
16
  process.stderr.write(`[pathgrade] ${msg}\n`);
@@ -36,7 +38,8 @@ export function createJestInvocationAdapter(input) {
36
38
  },
37
39
  runnerArgs: [],
38
40
  env: runInput.env,
39
- artifactRoot: getPathgradeDir(runInput.cwd),
41
+ artifactRoot: runInput.env.PATHGRADE_ARTIFACT_ROOT ?? getPathgradeDir(runInput.cwd),
42
+ normalizedSnapshotPath: runInput.env.PATHGRADE_NORMALIZED_SNAPSHOT_PATH,
40
43
  reporterMode: input.config.reporter ?? 'cli',
41
44
  threshold: input.config.ci.threshold,
42
45
  selection,
@@ -69,6 +72,7 @@ export function createJestInvocationAdapter(input) {
69
72
  '--reporters',
70
73
  resolveReporterFile(),
71
74
  ...runInput.runnerArgs,
75
+ '--testLocationInResults',
72
76
  ],
73
77
  });
74
78
  },
@@ -4,12 +4,14 @@ import { fmt, getPathgradeDir, printReportSummary, readSidecarForInvocation, res
4
4
  import { getJestLifecycleMetadata } from './lifecycle.js';
5
5
  import { readJestMetadata, removeJestMetadata } from './metadata.js';
6
6
  import { createJestAdapter } from './runner-adapter.js';
7
+ import { assertDirectReporterAttemptsSupported } from '../../runners/direct-reporter-attempts.js';
7
8
  export default class PathgradeJestReporter {
8
9
  async onRunComplete(_contexts, results) {
9
10
  const cwd = process.cwd();
10
11
  const config = await resolvePathgradeConfig({ cwd });
12
+ assertDirectReporterAttemptsSupported(config.attempts, process.env);
11
13
  const mode = config.reporter ?? 'cli';
12
- const outputDir = getPathgradeDir(cwd);
14
+ const outputDir = process.env.PATHGRADE_ARTIFACT_ROOT ?? getPathgradeDir(cwd);
13
15
  const metadataByCaseId = new Map([
14
16
  ...readJestMetadata(),
15
17
  ...getJestLifecycleMetadata(),
@@ -25,6 +27,7 @@ export default class PathgradeJestReporter {
25
27
  runnerArgs: [],
26
28
  env: process.env,
27
29
  artifactRoot: outputDir,
30
+ normalizedSnapshotPath: process.env.PATHGRADE_NORMALIZED_SNAPSHOT_PATH,
28
31
  reporterMode: mode,
29
32
  threshold: config.ci.threshold,
30
33
  writeEmptyReport: false,
@@ -34,6 +34,7 @@ export function normalizeJestRunResults(input) {
34
34
  runId,
35
35
  unitId: unitId(fileIndex),
36
36
  caseId,
37
+ repeatKey: jestRepeatKey(fileResult.testFilePath, assertion.fullName ?? fullCaseName(assertion), assertion.location),
37
38
  assertion,
38
39
  filePath: fileResult.testFilePath,
39
40
  evaluations: input.metadataByCaseId?.get(caseId) ?? [],
@@ -71,6 +72,7 @@ function runCase(input) {
71
72
  const failureMessage = input.assertion.failureMessages?.find(message => message.trim().length > 0);
72
73
  return {
73
74
  id: input.caseId,
75
+ repeatKey: input.repeatKey,
74
76
  runId: input.runId,
75
77
  unitId: input.unitId,
76
78
  name: fullCaseName(input.assertion),
@@ -110,6 +112,12 @@ function runCase(input) {
110
112
  }],
111
113
  };
112
114
  }
115
+ function jestRepeatKey(filePath, fullName, location) {
116
+ return JSON.stringify([
117
+ 'jest', filePath.replaceAll('\\', '/'), fullName,
118
+ location?.line ?? null, location?.column ?? null,
119
+ ]);
120
+ }
113
121
  function fullCaseName(assertion) {
114
122
  const ancestors = assertion.ancestorTitles ?? [];
115
123
  return [...ancestors, assertion.title].filter(Boolean).join(' > ');
@@ -6,3 +6,8 @@ export declare function test(name: string, options: NodeTestOptions, fn: NodeTes
6
6
  export default test;
7
7
  export { createNodeTestAdapter } from './runner-adapter.js';
8
8
  export { createNodeTestInvocationAdapter } from './invocation-adapter.js';
9
+ export declare function parseNodeTestCallerLocation(stack: string, cwd: string): {
10
+ sourceRef: string;
11
+ line?: number;
12
+ column?: number;
13
+ } | undefined;
@@ -11,8 +11,18 @@ export function test(name, optionsOrFn, maybeFn) {
11
11
  installResultCapture();
12
12
  const options = typeof optionsOrFn === 'function' ? undefined : optionsOrFn;
13
13
  const fn = typeof optionsOrFn === 'function' ? optionsOrFn : maybeFn;
14
- const sourceRef = callerSourceRef();
14
+ const source = callerSource();
15
+ const sourceRef = source.sourceRef;
15
16
  const caseId = `${sourceRef}:${name}:${++caseCounter}`;
17
+ const repeatKey = JSON.stringify([
18
+ 'node-test', sourceRef, source.line ?? null, source.column ?? null, name,
19
+ ]);
20
+ if (options?.skip === true || typeof options?.skip === 'string') {
21
+ writeCase({
22
+ caseId, repeatKey, name, state: 'skipped', runnerDurationMs: 0,
23
+ sourceRef, filePath: sourceRef, groupName: sourceRef, runnerCaseId: caseId,
24
+ });
25
+ }
16
26
  nodeTest(name, options, async (ctx) => {
17
27
  const startedAt = performance.now();
18
28
  let state = 'passed';
@@ -37,6 +47,7 @@ export function test(name, optionsOrFn, maybeFn) {
37
47
  const evaluations = await lifecycle.flushCase(caseId);
38
48
  writeCase({
39
49
  caseId,
50
+ repeatKey,
40
51
  name,
41
52
  state,
42
53
  runnerDurationMs: performance.now() - startedAt,
@@ -87,18 +98,35 @@ function writeCase(testCase) {
87
98
  existing.push(testCase);
88
99
  fs.writeFileSync(resultsPath, `${JSON.stringify(existing, null, 2)}\n`);
89
100
  }
90
- function callerSourceRef() {
101
+ function callerSource() {
91
102
  const stack = new Error().stack ?? '';
92
103
  const cwd = realpath(process.env.PATHGRADE_NODE_TEST_CWD ?? process.cwd());
104
+ return parseNodeTestCallerLocation(stack, cwd) ?? { sourceRef: 'node-test' };
105
+ }
106
+ export function parseNodeTestCallerLocation(stack, cwd) {
93
107
  for (const line of stack.split('\n')) {
94
- const match = line.match(/\(?((?:file:\/\/)?[^() ]+\.eval\.[cm]?[jt]s)(?::\d+)?(?::\d+)?\)?/);
95
- if (!match)
108
+ const location = line.match(/:(\d+):(\d+)\)?\s*$/);
109
+ if (!location || location.index === undefined)
110
+ continue;
111
+ const prefix = line.slice(0, location.index);
112
+ const extension = prefix.match(/\.eval\.[cm]?[jt]s$/);
113
+ if (!extension)
96
114
  continue;
97
- const raw = match[1];
115
+ const fileUrlStart = prefix.indexOf('file://');
116
+ const cwdStart = prefix.indexOf(cwd);
117
+ const wrapperStart = prefix.indexOf('(');
118
+ const start = fileUrlStart >= 0 ? fileUrlStart
119
+ : cwdStart >= 0 ? cwdStart
120
+ : wrapperStart >= 0 ? wrapperStart + 1
121
+ : prefix.lastIndexOf('at ') + 3;
122
+ const raw = prefix.slice(Math.max(0, start));
98
123
  const abs = realpath(raw.startsWith('file://') ? fileURLToPath(raw) : raw);
99
- return path.relative(cwd, abs).replaceAll(path.sep, '/');
124
+ return {
125
+ sourceRef: path.relative(cwd, abs).replaceAll(path.sep, '/'),
126
+ line: Number(location[1]),
127
+ column: Number(location[2]),
128
+ };
100
129
  }
101
- return 'node-test';
102
130
  }
103
131
  function realpath(value) {
104
132
  try {
@@ -6,6 +6,7 @@ import { createNodeTestAdapter } from './runner-adapter.js';
6
6
  export function createNodeTestInvocationAdapter(input) {
7
7
  return {
8
8
  name: 'node-test',
9
+ supportsRepeatedAttempts: true,
9
10
  async run(runInput) {
10
11
  const selection = await readSidecarForInvocation(runInput.cwd, runInput.env.PATHGRADE_SELECTION_INVOCATION_ID, msg => {
11
12
  process.stderr.write(`[pathgrade] ${msg}\n`);
@@ -24,7 +25,8 @@ export function createNodeTestInvocationAdapter(input) {
24
25
  },
25
26
  runnerArgs: runInput.runnerArgs,
26
27
  env: runInput.env,
27
- artifactRoot: getPathgradeDir(runInput.cwd),
28
+ artifactRoot: runInput.env.PATHGRADE_ARTIFACT_ROOT ?? getPathgradeDir(runInput.cwd),
29
+ normalizedSnapshotPath: runInput.env.PATHGRADE_NORMALIZED_SNAPSHOT_PATH,
28
30
  reporterMode: input.config.reporter ?? 'cli',
29
31
  threshold: input.config.ci.threshold,
30
32
  selection,
@@ -37,7 +37,7 @@ export function createNodeTestAdapter() {
37
37
  adapterName: this.name,
38
38
  status: 'completed',
39
39
  exitCode: 0,
40
- native: { resultsPath },
40
+ native: { resultsPath, resultsDir },
41
41
  };
42
42
  }
43
43
  const exitCode = await spawnNodeTest({
@@ -54,23 +54,29 @@ export function createNodeTestAdapter() {
54
54
  adapterName: this.name,
55
55
  status: input.signal?.aborted ? 'cancelled' : (exitCode === 0 ? 'completed' : 'failed'),
56
56
  exitCode,
57
- native: { resultsPath },
57
+ native: { resultsPath, resultsDir },
58
58
  };
59
59
  },
60
60
  async collectNormalizedRunSnapshot(run) {
61
61
  const native = readNodeTestRunNative(run);
62
- const cases = await readCases(native.resultsPath);
63
- const groupMap = new Map();
64
- for (const testCase of cases) {
65
- const groupName = testCase.groupName ?? testCase.filePath ?? testCase.sourceRef ?? 'node-test';
66
- if (!groupMap.has(groupName))
67
- groupMap.set(groupName, []);
68
- groupMap.get(groupName).push(testCase);
62
+ try {
63
+ const cases = await readCases(native.resultsPath);
64
+ const groupMap = new Map();
65
+ for (const testCase of cases) {
66
+ const groupName = testCase.groupName ?? testCase.filePath ?? testCase.sourceRef ?? 'node-test';
67
+ if (!groupMap.has(groupName))
68
+ groupMap.set(groupName, []);
69
+ groupMap.get(groupName).push(testCase);
70
+ }
71
+ return buildNormalizedRunSnapshotFromReportGroups(run, Array.from(groupMap.entries()).map(([groupName, groupedCases]) => ({
72
+ groupName,
73
+ cases: groupedCases,
74
+ })));
75
+ }
76
+ finally {
77
+ if (native.resultsDir)
78
+ await fs.remove(native.resultsDir);
69
79
  }
70
- return buildNormalizedRunSnapshotFromReportGroups(run, Array.from(groupMap.entries()).map(([groupName, groupedCases]) => ({
71
- groupName,
72
- cases: groupedCases,
73
- })));
74
80
  },
75
81
  };
76
82
  }
@@ -96,10 +102,11 @@ async function spawnNodeTest(input) {
96
102
  function readNodeTestRunNative(run) {
97
103
  if (typeof run.native === 'object'
98
104
  && run.native !== null
99
- && typeof run.native.resultsPath === 'string') {
105
+ && typeof run.native.resultsPath === 'string'
106
+ && typeof run.native.resultsDir === 'string') {
100
107
  return run.native;
101
108
  }
102
- return { resultsPath: '' };
109
+ return { resultsPath: '', resultsDir: '' };
103
110
  }
104
111
  async function readCases(resultsPath) {
105
112
  if (!resultsPath || !(await fs.pathExists(resultsPath)))
@@ -7,6 +7,7 @@ import { printReportSummary } from '../../reporters/report-summary.js';
7
7
  import { createVitestAdapter } from '../../runners/vitest-adapter.js';
8
8
  import { runWithAdapter } from '../../runners/orchestrator.js';
9
9
  import { resolvePathgradeConfig } from '../../config/pathgrade.js';
10
+ import { assertDirectReporterAttemptsSupported } from '../../runners/direct-reporter-attempts.js';
10
11
  /**
11
12
  * Custom vitest reporter that layers pathgrade aggregate statistics
12
13
  * on top of vitest's default output.
@@ -19,8 +20,9 @@ export class PathgradeReporter {
19
20
  async onTestRunEnd(testModules) {
20
21
  const cwd = process.cwd();
21
22
  const config = await resolvePathgradeConfig({ cwd });
23
+ assertDirectReporterAttemptsSupported(config.attempts, process.env);
22
24
  const mode = this.opts.reporter ?? config.reporter ?? 'cli';
23
- const outputDir = getPathgradeDir(cwd);
25
+ const outputDir = process.env.PATHGRADE_ARTIFACT_ROOT ?? getPathgradeDir(cwd);
24
26
  const adapter = createVitestAdapter({ testModules });
25
27
  const exitCode = await runWithAdapter({
26
28
  adapter,
@@ -30,6 +32,7 @@ export class PathgradeReporter {
30
32
  runnerArgs: [],
31
33
  env: process.env,
32
34
  artifactRoot: outputDir,
35
+ normalizedSnapshotPath: process.env.PATHGRADE_NORMALIZED_SNAPSHOT_PATH,
33
36
  reporterMode: mode,
34
37
  threshold: this.opts.ci?.threshold ?? config.ci.threshold,
35
38
  writeEmptyReport: false,
@@ -88,10 +88,11 @@ export class ClaudeAgent extends BaseAgent {
88
88
  hostEnv,
89
89
  sandboxProfile: this.deps.sandboxProfile,
90
90
  });
91
- const claudeCodeExecutable = resolveClaudeCodeExecutable({
92
- agentOptionsExecutable: this.opts.claudeCodeExecutable,
93
- envExecutable,
94
- });
91
+ const claudeCodeExecutable = sessionOptions?.admittedClaudeCodeExecutable
92
+ ?? resolveClaudeCodeExecutable({
93
+ agentOptionsExecutable: this.opts.claudeCodeExecutable,
94
+ envExecutable,
95
+ });
95
96
  const mcpMountOptions = {
96
97
  workspacePath,
97
98
  mcpConfigPath: sessionOptions?.mcpConfigPath,
@@ -16,58 +16,8 @@ import { projectItemIntoTurn } from './item-projection.js';
16
16
  import { CodexItemLifecycle, } from './item-lifecycle.js';
17
17
  import { createCodexManagedAuth } from './managed-auth.js';
18
18
  import { clearTurnAuthorityDeadline, closeUncorrelatedTurnTransport, establishAuthoritativeTurn, isAbortError, isAuthoritativeTurnCompletion, rememberAuthoritativeTurnId, shouldRetireAuthoritativeTurnTransport, } from './turn-authority.js';
19
+ import { dispatchCodexNotification, failTurn } from './turn-notifications.js';
19
20
  const TURN_COMPLETED_METHOD = 'turn/completed';
20
- function projectMcpStartupStatusIntoTurn(params, turn) {
21
- const name = params.name ?? 'unknown';
22
- const status = params.status ?? 'unknown';
23
- const error = typeof params.error === 'string' ? params.error : undefined;
24
- turn.nonAskToolEvents.push({
25
- action: 'unknown',
26
- provider: 'codex',
27
- providerToolName: 'mcpServer/startupStatus/updated',
28
- turnNumber: turn.turnNumber,
29
- arguments: {
30
- name,
31
- status,
32
- ...(error ? { error } : {}),
33
- },
34
- summary: `MCP server ${name} startup ${status}`,
35
- confidence: 'high',
36
- rawSnippet: JSON.stringify(params),
37
- });
38
- if (status === 'failed') {
39
- const message = `MCP server ${name} failed to start${error ? `: ${error}` : ''}`;
40
- failTurn(turn, message);
41
- }
42
- }
43
- function dispatchCodexNotification(notification, turn, sensitiveValues) {
44
- if (process.env.PATHGRADE_CODEX_DEBUG) {
45
- const debugParams = sanitizePersistenceValue(notification.params, sensitiveValues);
46
- console.error(`[codex app-server] notification method=${notification.method} params=${JSON.stringify(debugParams).slice(0, 300)}`);
47
- }
48
- if (notification.method === 'mcpServer/startupStatus/updated') {
49
- if (turn) {
50
- projectMcpStartupStatusIntoTurn((notification.params ?? {}), turn);
51
- }
52
- return;
53
- }
54
- if (!turn)
55
- return;
56
- const params = notification.params;
57
- if (notification.method === 'item/started') {
58
- if (params)
59
- turn.itemLifecycle?.receiveStarted(params);
60
- return;
61
- }
62
- if (notification.method === 'item/completed' && params) {
63
- turn.itemLifecycle?.receiveCompleted(params);
64
- }
65
- }
66
- function failTurn(turn, message) {
67
- turn.turnFailed = true;
68
- turn.failureMessage = message;
69
- turn.signalFailure?.(message);
70
- }
71
21
  function recordFromUnknown(value) {
72
22
  return value && typeof value === 'object' && !Array.isArray(value)
73
23
  ? value
@@ -0,0 +1,6 @@
1
+ import type { ActiveTurnState } from './turn-state.js';
2
+ export declare function dispatchCodexNotification(notification: {
3
+ method: string;
4
+ params: unknown;
5
+ }, turn: ActiveTurnState | null, sensitiveValues: readonly string[]): void;
6
+ export declare function failTurn(turn: ActiveTurnState, message: string): void;
@@ -0,0 +1,51 @@
1
+ import { sanitizePersistenceValue } from '../../tool-event-results.js';
2
+ export function dispatchCodexNotification(notification, turn, sensitiveValues) {
3
+ if (process.env.PATHGRADE_CODEX_DEBUG) {
4
+ const debugParams = sanitizePersistenceValue(notification.params, sensitiveValues);
5
+ console.error(`[codex app-server] notification method=${notification.method} params=${JSON.stringify(debugParams).slice(0, 300)}`);
6
+ }
7
+ if (notification.method === 'mcpServer/startupStatus/updated') {
8
+ if (turn) {
9
+ projectMcpStartupStatusIntoTurn((notification.params ?? {}), turn);
10
+ }
11
+ return;
12
+ }
13
+ if (!turn)
14
+ return;
15
+ const params = notification.params;
16
+ if (notification.method === 'item/started') {
17
+ if (params)
18
+ turn.itemLifecycle?.receiveStarted(params);
19
+ return;
20
+ }
21
+ if (notification.method === 'item/completed' && params) {
22
+ turn.itemLifecycle?.receiveCompleted(params);
23
+ }
24
+ }
25
+ export function failTurn(turn, message) {
26
+ turn.turnFailed = true;
27
+ turn.failureMessage = message;
28
+ turn.signalFailure?.(message);
29
+ }
30
+ function projectMcpStartupStatusIntoTurn(params, turn) {
31
+ const name = params.name ?? 'unknown';
32
+ const status = params.status ?? 'unknown';
33
+ const error = typeof params.error === 'string' ? params.error : undefined;
34
+ turn.nonAskToolEvents.push({
35
+ action: 'unknown',
36
+ provider: 'codex',
37
+ providerToolName: 'mcpServer/startupStatus/updated',
38
+ turnNumber: turn.turnNumber,
39
+ arguments: {
40
+ name,
41
+ status,
42
+ ...(error ? { error } : {}),
43
+ },
44
+ summary: `MCP server ${name} startup ${status}`,
45
+ confidence: 'high',
46
+ rawSnippet: JSON.stringify(params),
47
+ });
48
+ if (status === 'failed') {
49
+ failTurn(turn, `MCP server ${name} failed to start${error ? `: ${error}` : ''}`);
50
+ }
51
+ }
@@ -0,0 +1,19 @@
1
+ import type { ToolEvent } from '../../tool-events.js';
2
+ import type { PendingMcpDenial } from './mcp-approval-correlator.js';
3
+ import type { CodexItemLifecycle } from './item-lifecycle.js';
4
+ export interface ActiveTurnState {
5
+ turnNumber: number;
6
+ askBatchIds: string[];
7
+ nonAskToolEvents: ToolEvent[];
8
+ pendingMcpDenials: PendingMcpDenial[];
9
+ itemLifecycle?: CodexItemLifecycle;
10
+ assistantMessageParts: string[];
11
+ turnFailed: boolean;
12
+ failureMessage?: string;
13
+ signalFailure?: (message: string) => void;
14
+ authoritativeThreadId?: string;
15
+ authoritativeTurnId?: string;
16
+ pendingTurnCompletions: unknown[];
17
+ acceptTurnCompletion?: (params: unknown) => void;
18
+ authorityDeadlineTimer?: ReturnType<typeof setTimeout>;
19
+ }
@@ -0,0 +1 @@
1
+ export {};
@@ -0,0 +1,7 @@
1
+ import type { ToolAction } from '../../tool-events.js';
2
+ export declare const NATIVE_TOOL_ACTIONS: Readonly<Record<string, ToolAction>>;
3
+ export declare const ALLOWED_EVENT_TYPES: ReadonlySet<string>;
4
+ export declare function protocolRecord(value: unknown, label: string): Record<string, unknown>;
5
+ export declare function protocolFiniteNumber(value: unknown, label: string): number;
6
+ export declare function protocolRequiredString(value: unknown, label: string): string;
7
+ export declare function sanitizedProviderError(event: Record<string, unknown>): Error;
@@ -0,0 +1,47 @@
1
+ export const NATIVE_TOOL_ACTIONS = {
2
+ bash: 'run_shell',
3
+ read: 'read_file',
4
+ write: 'write_file',
5
+ edit: 'edit_file',
6
+ grep: 'search_code',
7
+ glob: 'list_files',
8
+ list: 'list_files',
9
+ skill: 'use_skill',
10
+ todowrite: 'update_todos',
11
+ lsp: 'search_code',
12
+ };
13
+ export const ALLOWED_EVENT_TYPES = new Set([
14
+ 'tool_use',
15
+ 'step_start',
16
+ 'step_finish',
17
+ 'text',
18
+ 'reasoning',
19
+ 'error',
20
+ ]);
21
+ export function protocolRecord(value, label) {
22
+ if (!value || typeof value !== 'object' || Array.isArray(value)) {
23
+ throw new Error(`OpenCode protocol error: ${label} must be an object`);
24
+ }
25
+ return value;
26
+ }
27
+ export function protocolFiniteNumber(value, label) {
28
+ if (typeof value !== 'number' || !Number.isFinite(value) || value < 0) {
29
+ throw new Error(`OpenCode protocol error: ${label} must be a nonnegative finite number`);
30
+ }
31
+ return value;
32
+ }
33
+ export function protocolRequiredString(value, label) {
34
+ if (typeof value !== 'string' || !value) {
35
+ throw new Error(`OpenCode protocol error: ${label} must be a nonempty string`);
36
+ }
37
+ return value;
38
+ }
39
+ export function sanitizedProviderError(event) {
40
+ const error = protocolRecord(event.error, 'error');
41
+ const data = error.data && typeof error.data === 'object' && !Array.isArray(error.data)
42
+ ? error.data
43
+ : {};
44
+ const status = typeof data.statusCode === 'number' ? ` status=${data.statusCode}` : '';
45
+ const retryable = typeof data.isRetryable === 'boolean' ? ` retryable=${data.isRetryable}` : '';
46
+ return new Error(`OpenCode provider error${status}${retryable}`);
47
+ }
@@ -14,28 +14,9 @@ import { OpenCodeRuntimePolicy, OPENCODE_PERMISSION } from './opencode/runtime-p
14
14
  import { killOpenCodeProcessGroup, registerOpenCodeProcessGroup, unregisterOpenCodeProcessGroup, } from './opencode/process-groups.js';
15
15
  import { assertCleanManagedOpenCodeHost, sha256File } from './opencode/host-safety.js';
16
16
  import { getOpenCodeScenarioSession } from './opencode/scenario.js';
17
+ import { ALLOWED_EVENT_TYPES, NATIVE_TOOL_ACTIONS, protocolFiniteNumber as finiteNumber, protocolRecord as record, protocolRequiredString as requiredString, sanitizedProviderError, } from './opencode/protocol.js';
17
18
  export { assertCleanManagedOpenCodeHost, managedOpenCodeConfigPaths } from './opencode/host-safety.js';
18
19
  const OUTPUT_CAP_BYTES = 16 * 1024 * 1024;
19
- const NATIVE_TOOL_ACTIONS = {
20
- bash: 'run_shell',
21
- read: 'read_file',
22
- write: 'write_file',
23
- edit: 'edit_file',
24
- grep: 'search_code',
25
- glob: 'list_files',
26
- list: 'list_files',
27
- skill: 'use_skill',
28
- todowrite: 'update_todos',
29
- lsp: 'search_code',
30
- };
31
- const ALLOWED_EVENT_TYPES = new Set([
32
- 'tool_use',
33
- 'step_start',
34
- 'step_finish',
35
- 'text',
36
- 'reasoning',
37
- 'error',
38
- ]);
39
20
  export function spawnOpenCode(executable, args, options) {
40
21
  return new Promise((resolve, reject) => {
41
22
  const stdout = [];
@@ -124,33 +105,6 @@ export function spawnOpenCode(executable, args, options) {
124
105
  }
125
106
  });
126
107
  }
127
- function record(value, label) {
128
- if (!value || typeof value !== 'object' || Array.isArray(value)) {
129
- throw new Error(`OpenCode protocol error: ${label} must be an object`);
130
- }
131
- return value;
132
- }
133
- function finiteNumber(value, label) {
134
- if (typeof value !== 'number' || !Number.isFinite(value) || value < 0) {
135
- throw new Error(`OpenCode protocol error: ${label} must be a nonnegative finite number`);
136
- }
137
- return value;
138
- }
139
- function requiredString(value, label) {
140
- if (typeof value !== 'string' || !value) {
141
- throw new Error(`OpenCode protocol error: ${label} must be a nonempty string`);
142
- }
143
- return value;
144
- }
145
- function sanitizedProviderError(event) {
146
- const error = record(event.error, 'error');
147
- const data = error.data && typeof error.data === 'object' && !Array.isArray(error.data)
148
- ? error.data
149
- : {};
150
- const status = typeof data.statusCode === 'number' ? ` status=${data.statusCode}` : '';
151
- const retryable = typeof data.isRetryable === 'boolean' ? ` retryable=${data.isRetryable}` : '';
152
- return new Error(`OpenCode provider error${status}${retryable}`);
153
- }
154
108
  export function parseOpenCodeOutput(stdout, processResult, mcpToolNames, sensitiveValues = []) {
155
109
  if (processResult.overflow)
156
110
  throw new Error('OpenCode output exceeded the 16 MiB limit');