@wix/pathgrade 1.0.0 → 1.0.2

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (38) hide show
  1. package/dist/adapter-kit/index.d.ts +1 -1
  2. package/dist/adapter-kit/index.js +1 -1
  3. package/dist/adapters/jest/invocation-adapter.js +2 -2
  4. package/dist/adapters/jest/reporter.js +2 -2
  5. package/dist/adapters/jest/results.js +6 -0
  6. package/dist/adapters/node-test/index.js +4 -0
  7. package/dist/adapters/node-test/invocation-adapter.js +2 -2
  8. package/dist/adapters/vitest/reporter.js +2 -2
  9. package/dist/affected/sidecar.d.ts +4 -2
  10. package/dist/affected/sidecar.js +11 -3
  11. package/dist/agents/claude/sdk-options.js +6 -0
  12. package/dist/commands/run-changed.js +4 -1
  13. package/dist/providers/credentials.js +1 -1
  14. package/dist/providers/sandbox-lifecycle.d.ts +22 -0
  15. package/dist/providers/sandbox-lifecycle.js +133 -0
  16. package/dist/providers/sandbox.js +2 -1
  17. package/dist/providers/workspace.d.ts +0 -6
  18. package/dist/providers/workspace.js +1 -31
  19. package/dist/reporting/core.js +18 -2
  20. package/dist/reporting/types.d.ts +5 -0
  21. package/dist/runners/model-builders.js +6 -0
  22. package/dist/runners/model.d.ts +4 -0
  23. package/dist/runners/report-projection.js +8 -0
  24. package/dist/runners/vitest-adapter.js +4 -0
  25. package/dist/runners/vitest-invocation.d.ts +1 -0
  26. package/dist/runners/vitest-invocation.js +17 -2
  27. package/dist/sdk/agent-result-log.js +9 -0
  28. package/dist/sdk/agent.js +13 -0
  29. package/dist/sdk/evaluate.d.ts +3 -3
  30. package/dist/sdk/evaluate.js +14 -0
  31. package/dist/sdk/index.d.ts +2 -1
  32. package/dist/sdk/lifecycle.js +12 -2
  33. package/dist/sdk/result-capture.d.ts +2 -0
  34. package/dist/sdk/result-capture.js +9 -1
  35. package/dist/sdk/types.d.ts +22 -0
  36. package/dist/types.d.ts +9 -0
  37. package/package.json +19 -19
  38. package/LICENSE +0 -24
@@ -8,7 +8,7 @@ export { runWithAdapter, type PathgradeRunOptions, type AdapterReporterMode } fr
8
8
  export { createRunnerLifecycleHooks } from '../runners/lifecycle-hooks.js';
9
9
  export { discoverPathgradeEvalFiles } from '../evals/discovery.js';
10
10
  export { DEFAULT_EVAL_EXCLUDE, DEFAULT_EVAL_INCLUDE, defaultPathgradeConfig, resolvePathgradeConfig, type PathgradeConfig, type ResolvedPathgradeConfig, } from '../config/pathgrade.js';
11
- export { readSidecar } from '../affected/sidecar.js';
11
+ export { readSidecar, readSidecarForInvocation } from '../affected/sidecar.js';
12
12
  export { getPathgradeDir } from '../reporters/results-path.js';
13
13
  export { printReportSummary } from '../reporters/report-summary.js';
14
14
  export { fmt } from '../utils/cli.js';
@@ -6,7 +6,7 @@ export { runWithAdapter } from '../runners/orchestrator.js';
6
6
  export { createRunnerLifecycleHooks } from '../runners/lifecycle-hooks.js';
7
7
  export { discoverPathgradeEvalFiles } from '../evals/discovery.js';
8
8
  export { DEFAULT_EVAL_EXCLUDE, DEFAULT_EVAL_INCLUDE, defaultPathgradeConfig, resolvePathgradeConfig, } from '../config/pathgrade.js';
9
- export { readSidecar } from '../affected/sidecar.js';
9
+ export { readSidecar, readSidecarForInvocation } from '../affected/sidecar.js';
10
10
  export { getPathgradeDir } from '../reporters/results-path.js';
11
11
  export { printReportSummary } from '../reporters/report-summary.js';
12
12
  export { fmt } from '../utils/cli.js';
@@ -2,7 +2,7 @@ import * as fs from 'node:fs';
2
2
  import * as path from 'node:path';
3
3
  import { spawn } from 'node:child_process';
4
4
  import { createRequire } from 'node:module';
5
- import { getPathgradeDir, printReportSummary, readSidecar, runWithAdapter, } from '@wix/pathgrade/adapter-kit';
5
+ import { getPathgradeDir, printReportSummary, readSidecarForInvocation, runWithAdapter, } from '@wix/pathgrade/adapter-kit';
6
6
  import { removeJestMetadata, withJestMetadataEnv } from './metadata.js';
7
7
  import { createJestAdapter } from './runner-adapter.js';
8
8
  export function createJestInvocationAdapter(input) {
@@ -10,7 +10,7 @@ export function createJestInvocationAdapter(input) {
10
10
  return {
11
11
  name: 'jest',
12
12
  async run(runInput) {
13
- const selection = await readSidecar(runInput.cwd, msg => {
13
+ const selection = await readSidecarForInvocation(runInput.cwd, runInput.env.PATHGRADE_SELECTION_INVOCATION_ID, msg => {
14
14
  process.stderr.write(`[pathgrade] ${msg}\n`);
15
15
  }) ?? undefined;
16
16
  const include = runInput.selectedFiles && runInput.selectedFiles.length > 0
@@ -1,6 +1,6 @@
1
1
  import * as path from 'node:path';
2
2
  import { execSync } from 'node:child_process';
3
- import { fmt, getPathgradeDir, printReportSummary, readSidecar, resolvePathgradeConfig, runWithAdapter, } from '@wix/pathgrade/adapter-kit';
3
+ import { fmt, getPathgradeDir, printReportSummary, readSidecarForInvocation, resolvePathgradeConfig, runWithAdapter, } from '@wix/pathgrade/adapter-kit';
4
4
  import { getJestLifecycleMetadata } from './lifecycle.js';
5
5
  import { readJestMetadata, removeJestMetadata } from './metadata.js';
6
6
  import { createJestAdapter } from './runner-adapter.js';
@@ -30,7 +30,7 @@ export default class PathgradeJestReporter {
30
30
  writeEmptyReport: false,
31
31
  warn: warning => console.warn(warning),
32
32
  log: () => console.log(`\n ${fmt.dim('Results written to')} ${outputDir}\n`),
33
- loadSelection: async () => (await readSidecar(cwd, msg => {
33
+ loadSelection: async () => (await readSidecarForInvocation(cwd, process.env.PATHGRADE_SELECTION_INVOCATION_ID, msg => {
34
34
  console.warn(`[pathgrade] ${msg}`);
35
35
  })) ?? undefined,
36
36
  printSummary: summaries => {
@@ -61,6 +61,12 @@ function runCase(input) {
61
61
  score: entry.score,
62
62
  ...(entry.trial ? { trial: entry.trial } : {}),
63
63
  ...(entry.diagnostics ? { diagnostics: entry.diagnostics } : {}),
64
+ ...(entry.resultKind ? { resultKind: entry.resultKind } : {}),
65
+ ...(entry.scoringDurationMs !== undefined
66
+ ? { scoringDurationMs: entry.scoringDurationMs }
67
+ : {}),
68
+ ...(entry.recordedAt ? { recordedAt: entry.recordedAt } : {}),
69
+ ...(entry.agent ? { agent: entry.agent } : {}),
64
70
  }));
65
71
  const failureMessage = input.assertion.failureMessages?.find(message => message.trim().length > 0);
66
72
  return {
@@ -70,6 +70,10 @@ function toReportEvaluations(evaluations) {
70
70
  score: entry.score,
71
71
  trial: entry.trial,
72
72
  diagnostics: entry.diagnostics,
73
+ resultKind: entry.resultKind,
74
+ scoringDurationMs: entry.scoringDurationMs,
75
+ recordedAt: entry.recordedAt,
76
+ agent: entry.agent,
73
77
  }));
74
78
  }
75
79
  function writeCase(testCase) {
@@ -1,4 +1,4 @@
1
- import { readSidecar } from '../../affected/sidecar.js';
1
+ import { readSidecarForInvocation } from '../../affected/sidecar.js';
2
2
  import { getPathgradeDir } from '../../reporters/results-path.js';
3
3
  import { printReportSummary } from '../../reporters/report-summary.js';
4
4
  import { runWithAdapter } from '../../runners/orchestrator.js';
@@ -7,7 +7,7 @@ export function createNodeTestInvocationAdapter(input) {
7
7
  return {
8
8
  name: 'node-test',
9
9
  async run(runInput) {
10
- const selection = await readSidecar(runInput.cwd, msg => {
10
+ const selection = await readSidecarForInvocation(runInput.cwd, runInput.env.PATHGRADE_SELECTION_INVOCATION_ID, msg => {
11
11
  process.stderr.write(`[pathgrade] ${msg}\n`);
12
12
  }) ?? undefined;
13
13
  const include = runInput.selectedFiles && runInput.selectedFiles.length > 0
@@ -2,7 +2,7 @@ import * as path from 'path';
2
2
  import { execSync } from 'child_process';
3
3
  import { fmt } from '../../utils/cli.js';
4
4
  import { getPathgradeDir } from '../../reporters/results-path.js';
5
- import { readSidecar } from '../../affected/sidecar.js';
5
+ import { readSidecarForInvocation } from '../../affected/sidecar.js';
6
6
  import { printReportSummary } from '../../reporters/report-summary.js';
7
7
  import { createVitestAdapter } from '../../runners/vitest-adapter.js';
8
8
  import { runWithAdapter } from '../../runners/orchestrator.js';
@@ -35,7 +35,7 @@ export class PathgradeReporter {
35
35
  writeEmptyReport: false,
36
36
  warn: warning => console.warn(warning),
37
37
  log: () => console.log(`\n ${fmt.dim('Results written to')} ${outputDir}\n`),
38
- loadSelection: async () => (await readSidecar(cwd, msg => {
38
+ loadSelection: async () => (await readSidecarForInvocation(cwd, process.env.PATHGRADE_SELECTION_INVOCATION_ID, msg => {
39
39
  console.warn(`[pathgrade] ${msg}`);
40
40
  })) ?? undefined,
41
41
  printSummary: summaries => {
@@ -21,12 +21,14 @@ export declare function getSidecarPath(cwd: string): string;
21
21
  * `.pathgrade/selection.json` (and merged into `results.json` as
22
22
  * `PathgradeReport.selection`).
23
23
  */
24
- export declare function toSelectionReport(result: SelectionResult): PathgradeSelectionReport;
25
- export declare function writeSidecar(cwd: string, result: SelectionResult): Promise<void>;
24
+ export declare function toSelectionReport(result: SelectionResult, invocationId?: string): PathgradeSelectionReport;
25
+ export declare function writeSidecar(cwd: string, result: SelectionResult, invocationId?: string): Promise<void>;
26
26
  /**
27
27
  * Read the sidecar if present. Returns `null` when missing, and `null`
28
28
  * with an optional warning when the file exists but is malformed — the
29
29
  * reporter is expected to tolerate corruption rather than break the run.
30
30
  */
31
31
  export declare function readSidecar(cwd: string, onWarning?: (msg: string) => void): Promise<PathgradeSelectionReport | null>;
32
+ /** Reads selection only when it belongs to the currently spawned runner. */
33
+ export declare function readSidecarForInvocation(cwd: string, invocationId: string | undefined, onWarning?: (msg: string) => void): Promise<PathgradeSelectionReport | null>;
32
34
  export declare function clearSidecar(cwd: string): Promise<void>;
@@ -24,8 +24,9 @@ export function getSidecarPath(cwd) {
24
24
  * `.pathgrade/selection.json` (and merged into `results.json` as
25
25
  * `PathgradeReport.selection`).
26
26
  */
27
- export function toSelectionReport(result) {
27
+ export function toSelectionReport(result, invocationId) {
28
28
  const report = {
29
+ ...(invocationId ? { invocation_id: invocationId } : {}),
29
30
  base_ref: result.baseRef,
30
31
  changed_files_count: result.changedFiles.length,
31
32
  selected: result.selected.map(s => s.file).sort(),
@@ -38,10 +39,10 @@ export function toSelectionReport(result) {
38
39
  report.global_match = result.globalMatch;
39
40
  return report;
40
41
  }
41
- export async function writeSidecar(cwd, result) {
42
+ export async function writeSidecar(cwd, result, invocationId) {
42
43
  const sidecarPath = getSidecarPath(cwd);
43
44
  await fs.ensureDir(path.dirname(sidecarPath));
44
- await fs.writeJSON(sidecarPath, toSelectionReport(result), { spaces: 2 });
45
+ await fs.writeJSON(sidecarPath, toSelectionReport(result, invocationId), { spaces: 2 });
45
46
  }
46
47
  /**
47
48
  * Read the sidecar if present. Returns `null` when missing, and `null`
@@ -66,6 +67,13 @@ export async function readSidecar(cwd, onWarning) {
66
67
  return null;
67
68
  }
68
69
  }
70
+ /** Reads selection only when it belongs to the currently spawned runner. */
71
+ export async function readSidecarForInvocation(cwd, invocationId, onWarning) {
72
+ if (!invocationId)
73
+ return null;
74
+ const report = await readSidecar(cwd, onWarning);
75
+ return report?.invocation_id === invocationId ? report : null;
76
+ }
69
77
  export async function clearSidecar(cwd) {
70
78
  const sidecarPath = getSidecarPath(cwd);
71
79
  try {
@@ -78,6 +78,12 @@ export function buildClaudeSdkOptions(inputs) {
78
78
  continue;
79
79
  env[key] = value;
80
80
  }
81
+ // Claude Code normally exposes claude.ai account-level MCP connectors to
82
+ // local OAuth sessions. Keep PathGrade trials isolated from those ambient
83
+ // connectors unless the caller explicitly opts in through their env.
84
+ if (useLocalOAuth && env.ENABLE_CLAUDEAI_MCP_SERVERS === undefined) {
85
+ env.ENABLE_CLAUDEAI_MCP_SERVERS = 'false';
86
+ }
81
87
  if (!useLocalOAuth) {
82
88
  env.CLAUDE_CONFIG_DIR = path.join(inputs.workspacePath, CLAUDE_CONFIG_SUBDIR);
83
89
  }
@@ -13,6 +13,7 @@
13
13
  * only producer of the file list here.
14
14
  */
15
15
  import * as fs from 'fs';
16
+ import { randomUUID } from 'node:crypto';
16
17
  import { selectAffected } from '../affected/select.js';
17
18
  import { resolveBaseRef, computeChangedFiles } from '../affected/git.js';
18
19
  import { writeSidecar } from '../affected/sidecar.js';
@@ -21,10 +22,12 @@ import { resolvePathgradeConfig } from '../config/pathgrade.js';
21
22
  import { loadRunnerInvocationAdapter } from '../runners/adapter-loader.js';
22
23
  export async function runChanged(opts) {
23
24
  const { cwd, parsed } = opts;
25
+ const selectionInvocationId = randomUUID();
24
26
  const runnerEnv = {
25
27
  ...process.env,
26
28
  ...(parsed.forceDiagnostics ? { PATHGRADE_DIAGNOSTICS: '1' } : {}),
27
29
  ...(parsed.forceVerbose ? { PATHGRADE_VERBOSE: '1' } : {}),
30
+ PATHGRADE_SELECTION_INVOCATION_ID: selectionInvocationId,
28
31
  };
29
32
  const configPath = findVitestConfigArg(parsed.runnerArgs);
30
33
  let config;
@@ -111,7 +114,7 @@ export async function runChanged(opts) {
111
114
  // Persist the sidecar immediately — even on empty selection, so the
112
115
  // reporter (if invoked later by another workflow step) has a coherent
113
116
  // record. A subsequent plain `pathgrade run` clears it.
114
- await writeSidecar(cwd, result);
117
+ await writeSidecar(cwd, result, selectionInvocationId);
115
118
  if (result.selected.length === 0) {
116
119
  if (!parsed.quiet) {
117
120
  process.stderr.write(`pathgrade: no affected evals; nothing to run.\n`);
@@ -95,7 +95,7 @@ async function resolveClaude(userEnv, ports) {
95
95
  return {
96
96
  env: { PATHGRADE_CLAUDE_LOCAL_OAUTH: '1' },
97
97
  setupCommands: [],
98
- copyFromHome: ['.claude.json'],
98
+ copyFromHome: [],
99
99
  linkFromHome: ['Library/Keychains'],
100
100
  };
101
101
  }
@@ -0,0 +1,22 @@
1
+ export declare const SANDBOX_PREFIX = "pathgrade-";
2
+ export declare const SANDBOX_MARKER = ".pathgrade-sandbox.json";
3
+ export declare const STALE_SANDBOX_AGE_MS: number;
4
+ interface RemoveSandboxRootOptions {
5
+ remove?: (target: string) => Promise<void>;
6
+ sleep?: (ms: number) => Promise<void>;
7
+ retryDelaysMs?: readonly number[];
8
+ }
9
+ export declare function removeSandboxRoot(rootDir: string, opts?: RemoveSandboxRootOptions): Promise<void>;
10
+ export interface StaleSandboxCleanupOptions extends RemoveSandboxRootOptions {
11
+ tempDir?: string;
12
+ now?: () => number;
13
+ isProcessAlive?: (pid: number) => boolean;
14
+ }
15
+ /**
16
+ * Removes old, crashed PathGrade sandboxes without examining anything outside
17
+ * the resolved operating-system temp directory. Every filesystem failure is
18
+ * intentionally ignored so a cleanup race never prevents a new trial.
19
+ */
20
+ export declare function cleanupStaleSandboxes(opts?: StaleSandboxCleanupOptions): Promise<void>;
21
+ export declare function createSandboxRoot(): Promise<string>;
22
+ export {};
@@ -0,0 +1,133 @@
1
+ import fs from 'fs-extra';
2
+ import * as os from 'os';
3
+ import * as path from 'path';
4
+ export const SANDBOX_PREFIX = 'pathgrade-';
5
+ export const SANDBOX_MARKER = '.pathgrade-sandbox.json';
6
+ export const STALE_SANDBOX_AGE_MS = 24 * 60 * 60 * 1_000;
7
+ let startupCleanup;
8
+ const SANDBOX_REMOVE_RETRY_DELAYS_MS = [
9
+ 50,
10
+ 100,
11
+ 250,
12
+ 500,
13
+ 1_000,
14
+ 2_000,
15
+ 3_000,
16
+ 5_000,
17
+ ];
18
+ function isRetryableRemoveError(error) {
19
+ const code = error.code;
20
+ return code === 'ENOTEMPTY' || code === 'EBUSY' || code === 'EPERM';
21
+ }
22
+ export async function removeSandboxRoot(rootDir, opts = {}) {
23
+ const remove = opts.remove ?? ((target) => fs.remove(target));
24
+ const sleep = opts.sleep ?? ((ms) => new Promise((resolve) => setTimeout(resolve, ms)));
25
+ const retryDelaysMs = opts.retryDelaysMs ?? SANDBOX_REMOVE_RETRY_DELAYS_MS;
26
+ for (let attempt = 0;; attempt++) {
27
+ try {
28
+ await remove(rootDir);
29
+ return;
30
+ }
31
+ catch (error) {
32
+ if (!isRetryableRemoveError(error) || attempt >= retryDelaysMs.length) {
33
+ throw error;
34
+ }
35
+ await sleep(retryDelaysMs[attempt]);
36
+ }
37
+ }
38
+ }
39
+ function isProcessAlive(pid) {
40
+ try {
41
+ process.kill(pid, 0);
42
+ return true;
43
+ }
44
+ catch (error) {
45
+ // A process we cannot signal is still live. Unknown failures are also
46
+ // preserved: cleanup must fail safe rather than delete a live trial.
47
+ return error.code !== 'ESRCH';
48
+ }
49
+ }
50
+ async function readOwnedSandboxMarker(rootDir) {
51
+ const rootStat = await fs.lstat(rootDir);
52
+ if (!rootStat.isDirectory() || rootStat.isSymbolicLink())
53
+ return undefined;
54
+ const markerPath = path.join(rootDir, SANDBOX_MARKER);
55
+ const markerStat = await fs.lstat(markerPath);
56
+ if (!markerStat.isFile() || markerStat.isSymbolicLink())
57
+ return undefined;
58
+ const marker = JSON.parse(await fs.readFile(markerPath, 'utf8'));
59
+ const pid = marker.pid;
60
+ if (marker.version !== 1 || typeof pid !== 'number' || !Number.isSafeInteger(pid) || pid <= 0)
61
+ return undefined;
62
+ return { version: 1, pid };
63
+ }
64
+ async function isStaleOwnedSandbox(rootDir, now, staleAfterMs, processIsAlive) {
65
+ const marker = await readOwnedSandboxMarker(rootDir);
66
+ if (!marker)
67
+ return false;
68
+ const markerStat = await fs.lstat(path.join(rootDir, SANDBOX_MARKER));
69
+ if (now - markerStat.mtimeMs < staleAfterMs)
70
+ return false;
71
+ return !processIsAlive(marker.pid);
72
+ }
73
+ /**
74
+ * Removes old, crashed PathGrade sandboxes without examining anything outside
75
+ * the resolved operating-system temp directory. Every filesystem failure is
76
+ * intentionally ignored so a cleanup race never prevents a new trial.
77
+ */
78
+ export async function cleanupStaleSandboxes(opts = {}) {
79
+ let tempDir;
80
+ try {
81
+ tempDir = await fs.realpath(opts.tempDir ?? os.tmpdir());
82
+ }
83
+ catch {
84
+ return;
85
+ }
86
+ const now = opts.now ?? Date.now;
87
+ const processIsAlive = opts.isProcessAlive ?? isProcessAlive;
88
+ let entries;
89
+ try {
90
+ entries = await fs.readdir(tempDir, { withFileTypes: true });
91
+ }
92
+ catch {
93
+ return;
94
+ }
95
+ for (const entry of entries) {
96
+ if (!entry.isDirectory() || entry.isSymbolicLink() || !entry.name.startsWith(SANDBOX_PREFIX))
97
+ continue;
98
+ const rootDir = path.resolve(tempDir, entry.name);
99
+ if (path.dirname(rootDir) !== tempDir)
100
+ continue;
101
+ try {
102
+ if (!await isStaleOwnedSandbox(rootDir, now(), STALE_SANDBOX_AGE_MS, processIsAlive))
103
+ continue;
104
+ // Re-check ownership and liveness immediately before removal. This
105
+ // narrows the window where another process can replace a candidate.
106
+ if (!await isStaleOwnedSandbox(rootDir, now(), STALE_SANDBOX_AGE_MS, processIsAlive))
107
+ continue;
108
+ await removeSandboxRoot(rootDir, opts);
109
+ }
110
+ catch {
111
+ // A disappeared directory, a permission change, or a deletion race
112
+ // must never make trial creation fail.
113
+ }
114
+ }
115
+ }
116
+ export async function createSandboxRoot() {
117
+ // A process may create many trial sandboxes. Scan only once at startup so
118
+ // repeated trials do not pay to enumerate the entire operating-system temp
119
+ // directory, while concurrent creators share the same best-effort sweep.
120
+ startupCleanup ??= cleanupStaleSandboxes();
121
+ await startupCleanup;
122
+ const tempDir = await fs.realpath(os.tmpdir());
123
+ const rootDir = await fs.mkdtemp(path.join(tempDir, SANDBOX_PREFIX));
124
+ const marker = { version: 1, pid: process.pid };
125
+ try {
126
+ await fs.writeFile(path.join(rootDir, SANDBOX_MARKER), JSON.stringify(marker), 'utf8');
127
+ return rootDir;
128
+ }
129
+ catch (error) {
130
+ await fs.remove(rootDir).catch(() => { });
131
+ throw error;
132
+ }
133
+ }
@@ -2,11 +2,12 @@ import fs from 'fs-extra';
2
2
  import * as path from 'path';
3
3
  import * as os from 'os';
4
4
  import { DEFAULT_COPY_IGNORE, createCopyFilter, isPortableCopyEntry } from './copy-filter.js';
5
+ import { createSandboxRoot } from './sandbox-lifecycle.js';
5
6
  export const SAFE_HOST_VARS = [
6
7
  'PATH', 'SHELL', 'LANG', 'LC_ALL', 'LC_CTYPE', 'TERM', 'USER', 'LOGNAME',
7
8
  ];
8
9
  export async function createSandbox(spec) {
9
- const rootDir = path.join(os.tmpdir(), `pathgrade-${Math.random().toString(36).substring(7)}`);
10
+ const rootDir = await createSandboxRoot();
10
11
  const workspacePath = path.join(rootDir, 'workspace');
11
12
  const homePath = path.join(rootDir, 'home');
12
13
  const tmpPath = path.join(rootDir, 'tmp');
@@ -11,11 +11,5 @@ export interface Workspace {
11
11
  }): Promise<CommandResult>;
12
12
  dispose(): Promise<void>;
13
13
  }
14
- interface RemoveSandboxRootOptions {
15
- remove?: (target: string) => Promise<void>;
16
- sleep?: (ms: number) => Promise<void>;
17
- retryDelaysMs?: readonly number[];
18
- }
19
- export declare function removeSandboxRoot(rootDir: string, opts?: RemoveSandboxRootOptions): Promise<void>;
20
14
  export declare function linkPathsFromHostHome(pathsToLink: string[], sandboxHomePath: string): Promise<void>;
21
15
  export declare function prepareWorkspace(spec: SandboxConfig): Promise<Workspace>;
@@ -2,41 +2,11 @@ import fs from 'fs-extra';
2
2
  import * as os from 'os';
3
3
  import * as path from 'path';
4
4
  import { createSandbox } from './sandbox.js';
5
+ import { removeSandboxRoot } from './sandbox-lifecycle.js';
5
6
  import { stageMcpConfig } from './mcp-config.js';
6
7
  import { sandboxExec } from './sandbox-exec.js';
7
8
  import { resolveCredentials } from './credentials.js';
8
9
  import { isPortableCopyEntry } from './copy-filter.js';
9
- const SANDBOX_REMOVE_RETRY_DELAYS_MS = [
10
- 50,
11
- 100,
12
- 250,
13
- 500,
14
- 1_000,
15
- 2_000,
16
- 3_000,
17
- 5_000,
18
- ];
19
- function isRetryableRemoveError(error) {
20
- const code = error.code;
21
- return code === 'ENOTEMPTY' || code === 'EBUSY' || code === 'EPERM';
22
- }
23
- export async function removeSandboxRoot(rootDir, opts = {}) {
24
- const remove = opts.remove ?? ((target) => fs.remove(target));
25
- const sleep = opts.sleep ?? ((ms) => new Promise((resolve) => setTimeout(resolve, ms)));
26
- const retryDelaysMs = opts.retryDelaysMs ?? SANDBOX_REMOVE_RETRY_DELAYS_MS;
27
- for (let attempt = 0;; attempt++) {
28
- try {
29
- await remove(rootDir);
30
- return;
31
- }
32
- catch (error) {
33
- if (!isRetryableRemoveError(error) || attempt >= retryDelaysMs.length) {
34
- throw error;
35
- }
36
- await sleep(retryDelaysMs[attempt]);
37
- }
38
- }
39
- }
40
10
  async function copyPathsFromHostHome(pathsToCopy, sandboxHomePath) {
41
11
  const realHome = os.homedir();
42
12
  for (const relPath of pathsToCopy) {
@@ -35,7 +35,9 @@ export function buildPathgradeReport(input) {
35
35
  trace_file: traceFile,
36
36
  });
37
37
  }
38
- const scores = reportableGroups.flatMap(group => group.cases.map(testCase => testCase.score));
38
+ const scores = reportableGroups.flatMap(group => group.cases
39
+ .filter(testCase => testCase.resultKind !== 'synthetic_no_evaluation')
40
+ .map(testCase => testCase.score));
39
41
  const overallPassRate = average(scores);
40
42
  const status = input.threshold != null
41
43
  ? (overallPassRate >= input.threshold ? 'pass' : 'fail')
@@ -69,7 +71,10 @@ function toBuiltCase(testCase) {
69
71
  `empty results for "${testCase.name}" — evaluate() may not have been called`,
70
72
  ]);
71
73
  }
72
- const evaluation = testCase.evaluations[testCase.evaluations.length - 1];
74
+ // A case can own multiple agents. Preserve the existing "last evaluation"
75
+ // rule among real evaluations without letting a later synthetic result from
76
+ // an unevaluated sibling agent erase a legitimate score.
77
+ const evaluation = testCase.evaluations.findLast(candidate => candidate.resultKind !== 'synthetic_no_evaluation') ?? testCase.evaluations[testCase.evaluations.length - 1];
73
78
  const diagnostics = evaluation.diagnostics ?? testCase.diagnostics;
74
79
  return {
75
80
  name: testCase.name,
@@ -78,12 +83,17 @@ function toBuiltCase(testCase) {
78
83
  runnerDurationMs: testCase.runnerDurationMs,
79
84
  diagnostics,
80
85
  reportable: testCase.reportable,
86
+ resultKind: evaluation.resultKind,
81
87
  trial: normalizeTrial({
82
88
  name: testCase.name,
83
89
  score: evaluation.score,
84
90
  runnerDurationMs: testCase.runnerDurationMs,
85
91
  trial: evaluation.trial,
86
92
  diagnostics,
93
+ resultKind: evaluation.resultKind,
94
+ scoringDurationMs: evaluation.scoringDurationMs,
95
+ recordedAt: evaluation.recordedAt,
96
+ agent: evaluation.agent,
87
97
  }),
88
98
  warnings: [],
89
99
  };
@@ -123,6 +133,12 @@ function normalizeTrial(input) {
123
133
  name: input.name,
124
134
  duration_ms: base.duration_ms || input.runnerDurationMs,
125
135
  diagnostics: input.diagnostics ?? base.diagnostics,
136
+ ...(input.resultKind ? { result_kind: input.resultKind } : {}),
137
+ ...(input.scoringDurationMs !== undefined
138
+ ? { scoring_duration_ms: input.scoringDurationMs }
139
+ : {}),
140
+ ...(input.recordedAt ? { recorded_at: input.recordedAt } : {}),
141
+ ...(input.agent ? { agent: input.agent } : {}),
126
142
  ...(skills.length > 0 ? { skills_used: skills } : {}),
127
143
  };
128
144
  }
@@ -1,4 +1,5 @@
1
1
  import type { DiagnosticsReport } from '../sdk/diagnostics.js';
2
+ import type { AgentExecutionMetadata, EvaluationResultKind } from '../sdk/types.js';
2
3
  import type { PathgradeReport, PathgradeSelectionReport, TrialResult } from '../types.js';
3
4
  export type ReportCaseState = 'passed' | 'failed' | 'skipped' | 'pending';
4
5
  export interface ReportRunInput {
@@ -27,6 +28,10 @@ export interface ReportEvaluationInput {
27
28
  score: number;
28
29
  trial?: TrialResult;
29
30
  diagnostics?: DiagnosticsReport;
31
+ resultKind?: EvaluationResultKind;
32
+ scoringDurationMs?: number;
33
+ recordedAt?: string;
34
+ agent?: AgentExecutionMetadata;
30
35
  }
31
36
  export interface PathgradeReportBuildResult {
32
37
  report: PathgradeReport;
@@ -27,6 +27,12 @@ export function buildNormalizedRunSnapshotFromReportGroups(run, groups) {
27
27
  score: evaluation.score,
28
28
  ...(evaluation.trial ? { trial: evaluation.trial } : {}),
29
29
  ...(evaluation.diagnostics ? { diagnostics: evaluation.diagnostics } : {}),
30
+ ...(evaluation.resultKind ? { resultKind: evaluation.resultKind } : {}),
31
+ ...(evaluation.scoringDurationMs !== undefined
32
+ ? { scoringDurationMs: evaluation.scoringDurationMs }
33
+ : {}),
34
+ ...(evaluation.recordedAt ? { recordedAt: evaluation.recordedAt } : {}),
35
+ ...(evaluation.agent ? { agent: evaluation.agent } : {}),
30
36
  }));
31
37
  return {
32
38
  id: caseId,
@@ -84,6 +84,10 @@ export interface EvaluationRecord {
84
84
  diagnostics?: DiagnosticsReport;
85
85
  trial?: TrialResult;
86
86
  nativeReferences?: NativeReference[];
87
+ resultKind?: import('../sdk/types.js').EvaluationResultKind;
88
+ scoringDurationMs?: number;
89
+ recordedAt?: string;
90
+ agent?: import('../sdk/types.js').AgentExecutionMetadata;
87
91
  }
88
92
  export interface AssertionRecord {
89
93
  id: string;
@@ -40,6 +40,10 @@ function projectedEvaluations(runCase) {
40
40
  score: evaluation.score,
41
41
  ...(evaluation.trial ? { trial: evaluation.trial } : {}),
42
42
  ...(evaluation.diagnostics ? { diagnostics: evaluation.diagnostics } : {}),
43
+ ...(evaluation.resultKind ? { resultKind: evaluation.resultKind } : {}),
44
+ ...(evaluation.scoringDurationMs !== undefined ? { scoringDurationMs: evaluation.scoringDurationMs } : {}),
45
+ ...(evaluation.recordedAt ? { recordedAt: evaluation.recordedAt } : {}),
46
+ ...(evaluation.agent ? { agent: evaluation.agent } : {}),
43
47
  })));
44
48
  }
45
49
  if (runCase.scoringPolicy.kind === 'score') {
@@ -52,6 +56,10 @@ function projectedEvaluations(runCase) {
52
56
  score: evaluation.score,
53
57
  ...(evaluation.trial ? { trial: evaluation.trial } : {}),
54
58
  ...(evaluation.diagnostics ? { diagnostics: evaluation.diagnostics } : {}),
59
+ ...(evaluation.resultKind ? { resultKind: evaluation.resultKind } : {}),
60
+ ...(evaluation.scoringDurationMs !== undefined ? { scoringDurationMs: evaluation.scoringDurationMs } : {}),
61
+ ...(evaluation.recordedAt ? { recordedAt: evaluation.recordedAt } : {}),
62
+ ...(evaluation.agent ? { agent: evaluation.agent } : {}),
55
63
  }];
56
64
  }
57
65
  function preferredGroupingLabel(hints) {
@@ -89,6 +89,10 @@ function toReportCaseInput(testCase, groupName) {
89
89
  score: entry.score,
90
90
  trial: entry.trial,
91
91
  diagnostics: entry.diagnostics,
92
+ resultKind: entry.resultKind,
93
+ scoringDurationMs: entry.scoringDurationMs,
94
+ recordedAt: entry.recordedAt,
95
+ agent: entry.agent,
92
96
  })),
93
97
  diagnostics: normalized.diagnostics,
94
98
  };
@@ -8,3 +8,4 @@ export type SpawnVitest = (req: SpawnVitestRequest) => Promise<number> | number;
8
8
  export declare function createVitestInvocationAdapter(input?: {
9
9
  spawnVitest?: SpawnVitest;
10
10
  }): RunnerInvocationAdapter;
11
+ export declare function resolveVitestCli(cwd: string): string;
@@ -1,3 +1,6 @@
1
+ import fs from 'node:fs';
2
+ import { createRequire } from 'node:module';
3
+ import path from 'node:path';
1
4
  export function createVitestInvocationAdapter(input = {}) {
2
5
  const spawnVitest = input.spawnVitest ?? defaultSpawnVitest;
3
6
  return {
@@ -32,13 +35,25 @@ function hasPassWithNoTests(args) {
32
35
  }
33
36
  async function defaultSpawnVitest(req) {
34
37
  const { spawn } = await import('child_process');
38
+ const vitestCli = resolveVitestCli(req.cwd);
35
39
  return await new Promise(resolve => {
36
- const child = spawn('npx', ['vitest', ...req.argv], {
40
+ const child = spawn(process.execPath, [vitestCli, ...req.argv], {
37
41
  stdio: 'inherit',
38
42
  env: req.env,
39
43
  cwd: req.cwd,
40
- shell: true,
41
44
  });
42
45
  child.on('close', code => resolve(code ?? 0));
43
46
  });
44
47
  }
48
+ export function resolveVitestCli(cwd) {
49
+ const requireFromCwd = createRequire(path.join(cwd, 'package.json'));
50
+ const packageJsonPath = requireFromCwd.resolve('vitest/package.json');
51
+ const packageJson = JSON.parse(fs.readFileSync(packageJsonPath, 'utf8'));
52
+ const bin = typeof packageJson.bin === 'string'
53
+ ? packageJson.bin
54
+ : packageJson.bin?.vitest;
55
+ if (!bin) {
56
+ throw new Error(`Unable to resolve the Vitest CLI from ${packageJsonPath}`);
57
+ }
58
+ return path.resolve(path.dirname(packageJsonPath), bin);
59
+ }
@@ -20,6 +20,15 @@ export function buildModelAgentResultLogEntry(params) {
20
20
  // never produce a zero-valued field that could be mistaken for
21
21
  // a free turn.
22
22
  ...(params.turnResult.costUsd !== undefined ? { cost_usd: params.turnResult.costUsd } : {}),
23
+ ...(params.turnResult.cacheCreationInputTokens !== undefined
24
+ ? { cache_creation_input_tokens: params.turnResult.cacheCreationInputTokens }
25
+ : {}),
26
+ ...(params.turnResult.cacheReadInputTokens !== undefined
27
+ ? { cache_read_input_tokens: params.turnResult.cacheReadInputTokens }
28
+ : {}),
29
+ ...(params.turnResult.errorSubtype
30
+ ? { error_subtype: params.turnResult.errorSubtype }
31
+ : {}),
23
32
  ...getOutputMetrics(assistantMessage),
24
33
  };
25
34
  }
package/dist/sdk/agent.js CHANGED
@@ -64,6 +64,19 @@ class AgentImpl {
64
64
  get workspace() {
65
65
  return this.ws.path;
66
66
  }
67
+ get executionMetadata() {
68
+ const interactionMode = this.interactionMode === 'startChat'
69
+ ? 'start_chat'
70
+ : this.interactionMode === 'runConversation'
71
+ ? 'conversation'
72
+ : this.interactionMode ?? undefined;
73
+ return {
74
+ name: this.agentName,
75
+ ...(this.modelOpt ? { requestedModel: this.modelOpt } : {}),
76
+ ...(this.transport ? { transport: this.transport } : {}),
77
+ ...(interactionMode ? { interactionMode } : {}),
78
+ };
79
+ }
67
80
  guardInteraction(mode) {
68
81
  if (this.interactionMode && this.interactionMode !== mode) {
69
82
  throw new Error(`Cannot use ${mode}() after ${this.interactionMode}() — an agent supports only one interaction method`);
@@ -1,12 +1,12 @@
1
- import type { Agent, Scorer, EvalResult, RecordedEvalResult, ScorerResultEntry, EvaluateOptions } from './types.js';
1
+ import type { Agent, Scorer, RecordedEvalResult, ScorerResultEntry, EvaluateOptions } from './types.js';
2
2
  export type OnScorerErrorMode = NonNullable<EvaluateOptions['onScorerError']>;
3
3
  export declare class EvalScorerError extends Error {
4
4
  readonly scorerErrors: ScorerResultEntry[];
5
5
  readonly result: RecordedEvalResult;
6
6
  constructor(result: RecordedEvalResult, scorerErrors: ScorerResultEntry[]);
7
7
  }
8
- type EvaluateFromSnapshot = (snapshotPath: string, scorers: Scorer[], opts?: EvaluateOptions) => Promise<EvalResult>;
9
- type EvaluateFn = ((agent: Agent, scorers: Scorer[], opts?: EvaluateOptions) => Promise<EvalResult>) & {
8
+ type EvaluateFromSnapshot = (snapshotPath: string, scorers: Scorer[], opts?: EvaluateOptions) => Promise<RecordedEvalResult>;
9
+ type EvaluateFn = ((agent: Agent, scorers: Scorer[], opts?: EvaluateOptions) => Promise<RecordedEvalResult>) & {
10
10
  fromSnapshot: EvaluateFromSnapshot;
11
11
  };
12
12
  export declare const evaluate: EvaluateFn;
@@ -54,6 +54,7 @@ function makeEvaluateAgent() {
54
54
  // accumulated agent-turn cost.
55
55
  const beforeCostUsd = trackedLLM.costUsd ?? 0;
56
56
  // measure() returns the delta consumed by this evaluate call.
57
+ const scoringStartedAt = performance.now();
57
58
  const { result: evalResult, tokens: deltaTokenUsage } = trackedLLM.measure
58
59
  ? await trackedLLM.measure(() => evaluateWithContext(ctx, scorers, { ...opts, llm: trackedLLM }))
59
60
  : await (async () => {
@@ -75,6 +76,12 @@ function makeEvaluateAgent() {
75
76
  const recordedResult = {
76
77
  ...evalResult,
77
78
  tokenUsage: deltaTokenUsage,
79
+ resultKind: 'evaluated',
80
+ scoringDurationMs: Math.max(0, performance.now() - scoringStartedAt),
81
+ recordedAt: new Date().toISOString(),
82
+ ...(opts?.evaluationDefinitionKey
83
+ ? { evaluationDefinitionKey: opts.evaluationDefinitionKey }
84
+ : {}),
78
85
  trial: buildTrialResult(agent.log, { ...evalResult, tokenUsage: deltaTokenUsage }, conversationTokens, conversationCost),
79
86
  };
80
87
  emitEvalResult({ result: recordedResult, agent });
@@ -106,9 +113,16 @@ async function fromSnapshot(snapshotPath, scorers, opts) {
106
113
  },
107
114
  artifacts: createSessionArtifacts(snapshot.workspace ?? '', snapshot.toolEvents),
108
115
  };
116
+ const scoringStartedAt = performance.now();
109
117
  const evalResult = await evaluateWithContext(ctx, scorers, { ...opts, llm: trackedLLM });
110
118
  const recordedResult = {
111
119
  ...evalResult,
120
+ resultKind: 'snapshot',
121
+ scoringDurationMs: Math.max(0, performance.now() - scoringStartedAt),
122
+ recordedAt: new Date().toISOString(),
123
+ ...(opts?.evaluationDefinitionKey
124
+ ? { evaluationDefinitionKey: opts.evaluationDefinitionKey }
125
+ : {}),
112
126
  trial: buildTrialResult(snapshot.log, evalResult),
113
127
  };
114
128
  maybeThrowOnScorerErrors(recordedResult, opts?.onScorerError ?? 'skip');
@@ -21,13 +21,14 @@ export { emitEvalResult, resetAllResultObserversForTests, resetUserResultObserve
21
21
  export { getAgentCapabilities } from './types.js';
22
22
  export type { AgentTransport, AgentCapabilities, AgentName, McpRunMode, McpSafetyOptions, McpToolPolicy, McpToolPolicyRule, } from './types.js';
23
23
  export type { AskBus, AskBatch, AskQuestion, AskOption, AskAnswer, AskResolution, AskBatchSnapshot, AskAnswerSnapshot, AskResolutionSnapshot, AskHandle, AskHandler, AskSource, AskLifecycle, AskAnswerSource, Unsubscribe as AskBusUnsubscribe, } from './ask-bus/types.js';
24
- export type { Agent, AgentOptions, Message, Scorer, CheckScorer, ScoreScorer, JudgeScorer, ToolUsageScorer, ScorerContext, EvalResult, ScorerResultEntry, ScorerStatus, ChatSession, ConversationResult, ConverseOptions, UntilPredicate, UntilContext, Reaction, TextReaction, AskUserReaction, AskUserQuestion, AskUserOption, ReactionPreviewEntry, TextReactionPreviewEntry, AskUserReactionPreviewEntry, ReactionPreviewResult, ReactionPreviewTurn, StepScorer, Persona, PersonaConfig, ConversationWindowConfig, TurnDetail, ReactionFiredEntry, PathgradePluginOptions, PathgradeMeta, TurnTiming, TokenUsage, EvaluateOptions, ReactionPreviewStatus, ScoreResult, JudgeInput, CodeJudgeToolName, ToolExpectation, SessionArtifactMatchOptions, SessionArtifactContent, SessionArtifacts, RecordedEvalResult, PathgradeTestMeta, } from './types.js';
24
+ export type { Agent, AgentOptions, Message, Scorer, CheckScorer, ScoreScorer, JudgeScorer, ToolUsageScorer, ScorerContext, EvalResult, ScorerResultEntry, ScorerStatus, ChatSession, ConversationResult, ConverseOptions, UntilPredicate, UntilContext, Reaction, TextReaction, AskUserReaction, AskUserQuestion, AskUserOption, ReactionPreviewEntry, TextReactionPreviewEntry, AskUserReactionPreviewEntry, ReactionPreviewResult, ReactionPreviewTurn, StepScorer, Persona, PersonaConfig, ConversationWindowConfig, TurnDetail, ReactionFiredEntry, PathgradePluginOptions, PathgradeMeta, TurnTiming, TokenUsage, EvaluateOptions, ReactionPreviewStatus, ScoreResult, JudgeInput, CodeJudgeToolName, ToolExpectation, SessionArtifactMatchOptions, SessionArtifactContent, SessionArtifacts, RecordedEvalResult, PathgradeTestMeta, EvaluationResultKind, AgentExecutionMetadata, AgentInteractionMode, } from './types.js';
25
25
  export type { ConversationWindow, ConversationWindowOptions } from './conversation-window.js';
26
26
  export type { JudgePipelineOptions } from './judge-pipeline.js';
27
27
  export type { RunScorerOptions } from './run-scorer.js';
28
28
  export type { OnScorerErrorMode } from './evaluate.js';
29
29
  export type { EvalResultEvent, EvalResultObserver, ResultObserverHandle, ResultObserverOptions, ResultObserverOwner, } from './result-capture.js';
30
30
  export type { RunSnapshot } from './snapshots.js';
31
+ export type { DiagnosticsReport } from './diagnostics.js';
31
32
  export type { ExpectedMcpStartupStatus, ExpectedMcpToolCall, McpStartupStatusEvidence, McpToolCallEvidence, } from './mcp-evidence.js';
32
33
  export type { McpPolicyDenialReason, McpToolCallRequest, McpToolPolicyDecision, } from './mcp-safety.js';
33
34
  export type { ToolEvent } from '../tool-events.js';
@@ -73,6 +73,12 @@ function recordResult(result, agent, attribution) {
73
73
  score: result.score,
74
74
  scorers: result.scorers,
75
75
  trial: result.trial,
76
+ resultKind: result.resultKind ?? 'evaluated',
77
+ ...(result.scoringDurationMs !== undefined
78
+ ? { scoringDurationMs: result.scoringDurationMs }
79
+ : {}),
80
+ ...(result.recordedAt ? { recordedAt: result.recordedAt } : {}),
81
+ ...(agent.executionMetadata ? { agent: agent.executionMetadata } : {}),
76
82
  diagnostics: buildDiagnosticsReport({
77
83
  completionReason,
78
84
  completionDetail: conversationEnd?.completion_detail,
@@ -129,14 +135,18 @@ function synthesizeTrialFromAgent(agent) {
129
135
  return {
130
136
  score: 1,
131
137
  scorers: [],
138
+ resultKind: 'synthetic_no_evaluation',
139
+ ...(agent.executionMetadata ? { agent: agent.executionMetadata } : {}),
132
140
  trial: {
133
141
  trial_id: 0,
134
142
  reward: 1,
135
143
  scorer_results: [],
136
144
  duration_ms: 0,
137
145
  n_commands: nCommands,
138
- input_tokens: tokenUsage?.inputTokens ?? 0,
139
- output_tokens: tokenUsage?.outputTokens ?? 0,
146
+ input_tokens: 0,
147
+ output_tokens: 0,
148
+ conversation_input_tokens: tokenUsage?.inputTokens ?? 0,
149
+ conversation_output_tokens: tokenUsage?.outputTokens ?? 0,
140
150
  session_log: [...agent.log],
141
151
  },
142
152
  diagnostics: buildDiagnosticsReport({
@@ -1,7 +1,9 @@
1
1
  import type { Agent, RecordedEvalResult } from './types.js';
2
+ import { type CaseContext } from './case-context.js';
2
3
  export interface EvalResultEvent {
3
4
  readonly result: RecordedEvalResult;
4
5
  readonly agent: Agent;
6
+ readonly case?: CaseContext;
5
7
  }
6
8
  export type EvalResultObserver = (event: EvalResultEvent) => void;
7
9
  export type ResultObserverOwner = 'user' | 'adapter' | 'test';
@@ -1,3 +1,4 @@
1
+ import { getCurrentCaseContext } from './case-context.js';
1
2
  const observers = new Set();
2
3
  export function subscribeToEvalResults(observer, options = {}) {
3
4
  if (options.owner === 'adapter' && options.key) {
@@ -20,9 +21,16 @@ export function subscribeToEvalResults(observer, options = {}) {
20
21
  };
21
22
  }
22
23
  export function emitEvalResult(event) {
24
+ const currentCase = getCurrentCaseContext();
25
+ const deliveredEvent = event.case || currentCase.status !== 'active'
26
+ ? event
27
+ : { ...event, case: currentCase.context };
28
+ // Deliver against a stable snapshot. Observers are allowed to subscribe or
29
+ // unsubscribe from inside a callback, but those mutations must only affect
30
+ // subsequent events.
23
31
  for (const subscription of [...observers]) {
24
32
  try {
25
- subscription.observer(event);
33
+ subscription.observer(deliveredEvent);
26
34
  }
27
35
  catch {
28
36
  // Result capture should be best-effort for every observer; one
@@ -6,6 +6,15 @@ import type { DiagnosticsReport } from './diagnostics.js';
6
6
  import type { LLMPort } from '../utils/llm-types.js';
7
7
  import type { McpSafetyOptions } from './mcp-safety.js';
8
8
  export type AgentName = 'claude' | 'codex' | 'cursor';
9
+ export type AgentInteractionMode = 'prompt' | 'start_chat' | 'conversation';
10
+ /** Privacy-safe execution dimensions exposed to result observers. */
11
+ export interface AgentExecutionMetadata {
12
+ name: AgentName;
13
+ /** Model override requested by the caller; not necessarily the provider-resolved model. */
14
+ requestedModel?: string;
15
+ transport?: AgentTransport;
16
+ interactionMode?: AgentInteractionMode;
17
+ }
9
18
  export interface AgentOptions {
10
19
  agent?: AgentName;
11
20
  model?: string;
@@ -61,6 +70,8 @@ export interface Agent {
61
70
  readonly messages: Message[];
62
71
  readonly log: LogEntry[];
63
72
  readonly workspace: string;
73
+ /** Provider/configuration metadata only; never contains prompts or paths. */
74
+ readonly executionMetadata?: AgentExecutionMetadata;
64
75
  dispose(): Promise<void>;
65
76
  }
66
77
  export interface ConverseOptions {
@@ -318,6 +329,8 @@ export interface EvaluateOptions {
318
329
  failFast?: boolean;
319
330
  llm?: LLMPort;
320
331
  onScorerError?: 'skip' | 'zero' | 'fail';
332
+ /** Stable identity for this evaluation definition across separate runs. */
333
+ evaluationDefinitionKey?: string;
321
334
  }
322
335
  export interface TokenUsage {
323
336
  inputTokens: number;
@@ -331,7 +344,12 @@ export interface EvalResult {
331
344
  }
332
345
  export interface RecordedEvalResult extends EvalResult {
333
346
  trial?: TrialResult;
347
+ resultKind?: EvaluationResultKind;
348
+ scoringDurationMs?: number;
349
+ recordedAt?: string;
350
+ evaluationDefinitionKey?: string;
334
351
  }
352
+ export type EvaluationResultKind = 'evaluated' | 'synthetic_no_evaluation' | 'snapshot';
335
353
  export interface ScorerResultEntry {
336
354
  name: string;
337
355
  type: 'check' | 'score' | 'judge' | 'tool_usage';
@@ -351,6 +369,10 @@ export interface PathgradeTestMeta {
351
369
  scorers: ScorerResultEntry[];
352
370
  trial?: TrialResult;
353
371
  diagnostics?: DiagnosticsReport;
372
+ resultKind?: EvaluationResultKind;
373
+ scoringDurationMs?: number;
374
+ recordedAt?: string;
375
+ agent?: AgentExecutionMetadata;
354
376
  }
355
377
  export interface AgentCapabilities {
356
378
  mcp: boolean;
package/dist/types.d.ts CHANGED
@@ -109,6 +109,9 @@ export interface LogEntry {
109
109
  * carries a `costUsd` value; absent for agents that do not expose cost.
110
110
  */
111
111
  cost_usd?: number;
112
+ cache_creation_input_tokens?: number;
113
+ cache_read_input_tokens?: number;
114
+ error_subtype?: AgentTurnResult['errorSubtype'];
112
115
  completion_reason?: string;
113
116
  completion_detail?: string;
114
117
  turn_timings?: Array<{
@@ -174,6 +177,10 @@ export interface TrialResult {
174
177
  session_log: LogEntry[];
175
178
  skills_used?: string[];
176
179
  diagnostics?: DiagnosticsReport;
180
+ result_kind?: import('./sdk/types.js').EvaluationResultKind;
181
+ scoring_duration_ms?: number;
182
+ recorded_at?: string;
183
+ agent?: import('./sdk/types.js').AgentExecutionMetadata;
177
184
  conversation?: {
178
185
  turns: ConversationTurn[];
179
186
  total_turns: number;
@@ -213,6 +220,8 @@ export type PathgradeGroupReport = Omit<EvalReport, 'trials'> & {
213
220
  * source of truth — new variants added there propagate here automatically).
214
221
  */
215
222
  export interface PathgradeSelectionReport {
223
+ /** Binds this sidecar to the runner process spawned by `run --changed`. */
224
+ invocation_id?: string;
216
225
  base_ref: string;
217
226
  changed_files_count: number;
218
227
  global_match?: string;
package/package.json CHANGED
@@ -1,15 +1,8 @@
1
1
  {
2
2
  "name": "@wix/pathgrade",
3
- "version": "1.0.0",
3
+ "version": "1.0.2",
4
4
  "packageManager": "yarn@4.12.0",
5
5
  "description": "Evaluate whether AI agents discover and use your skills correctly",
6
- "repository": {
7
- "type": "git",
8
- "url": "git+https://github.com/wix-incubator/pathgrade.git"
9
- },
10
- "installConfig": {
11
- "selfReferences": true
12
- },
13
6
  "exports": {
14
7
  ".": {
15
8
  "types": "./dist/sdk/index.d.ts",
@@ -63,8 +56,7 @@
63
56
  "!dist/**/*.d.ts.map",
64
57
  "bin/",
65
58
  "templates/",
66
- "README.md",
67
- "LICENSE"
59
+ "README.md"
68
60
  ],
69
61
  "scripts": {
70
62
  "lint": "oxlint src tests evals examples scripts --deny-warnings --disable-unicorn-plugin --disable-oxc-plugin --allow no-unused-vars --allow no-underscore-dangle --allow no-shadow --allow preserve-caught-error --allow no-unsafe-optional-chaining --allow no-unreachable --ignore-pattern tests/fixtures --ignore-pattern .worktrees",
@@ -95,11 +87,21 @@
95
87
  "llm",
96
88
  "testing"
97
89
  ],
90
+ "publishConfig": {
91
+ "registry": "https://registry.npmjs.org/",
92
+ "access": "public"
93
+ },
94
+ "wix": {
95
+ "artifact": {
96
+ "groupId": "com.wixpress",
97
+ "artifactId": "pathgrade"
98
+ }
99
+ },
98
100
  "author": "Nadav Lachish",
99
101
  "license": "MIT",
100
102
  "type": "module",
101
103
  "engines": {
102
- "node": ">=20.11.0"
104
+ "node": ">=20.19.0"
103
105
  },
104
106
  "peerDependencies": {
105
107
  "jest": "^30.0.0",
@@ -117,13 +119,13 @@
117
119
  "@types/fs-extra": "^11.0.4",
118
120
  "@types/jest": "^30.0.0",
119
121
  "@types/picomatch": "^4.0.2",
120
- "@vitest/coverage-v8": "4.1.7",
122
+ "@vitest/coverage-v8": "4.1.10",
121
123
  "jest": "^30.0.0",
122
- "vitest": "4.1.7"
124
+ "vitest": "4.1.10"
123
125
  },
124
126
  "dependencies": {
125
- "@anthropic-ai/claude-agent-sdk": "0.2.116",
126
- "@modelcontextprotocol/sdk": "1.29.0",
127
+ "@anthropic-ai/claude-agent-sdk": "0.2.141",
128
+ "@modelcontextprotocol/sdk": "1.30.0",
127
129
  "@types/node": "25.6.0",
128
130
  "fs-extra": "11.3.3",
129
131
  "jiti": "2.6.1",
@@ -132,7 +134,5 @@
132
134
  "typescript": "^5.9.3",
133
135
  "zod": "4.3.6"
134
136
  },
135
- "resolutions": {
136
- "fast-uri@npm:^3.0.1": "npm:3.1.2"
137
- }
138
- }
137
+ "falconPackageHash": "ccc242dcd7240e8c890225f5dd2f45be54f0f3b3426b862b95d847f1"
138
+ }
package/LICENSE DELETED
@@ -1,24 +0,0 @@
1
- The MIT License (MIT)
2
-
3
- Copyright (c) 2025 Nadav Lachish
4
-
5
- Permission is hereby granted, free of charge, to any person obtaining a copy
6
- of this software and associated documentation files (the "Software"), to deal
7
- in the Software without restriction, including without limitation the rights
8
- to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
9
- copies of the Software, and to permit persons to whom the Software is
10
- furnished to do so, subject to the following conditions:
11
-
12
- The above copyright notice and this permission notice shall be included in all
13
- copies or substantial portions of the Software.
14
-
15
- THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16
- IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17
- FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
18
- AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19
- LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
20
- OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
21
- SOFTWARE.
22
-
23
- NOTICE: Pathgrade was originally forked from skillgrade by Minko Gechev,
24
- which is MIT licensed.