@wix/pathgrade 1.0.0 → 1.0.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/adapter-kit/index.d.ts +1 -1
- package/dist/adapter-kit/index.js +1 -1
- package/dist/adapters/jest/invocation-adapter.js +2 -2
- package/dist/adapters/jest/reporter.js +2 -2
- package/dist/adapters/jest/results.js +6 -0
- package/dist/adapters/node-test/index.js +4 -0
- package/dist/adapters/node-test/invocation-adapter.js +2 -2
- package/dist/adapters/vitest/reporter.js +2 -2
- package/dist/affected/sidecar.d.ts +4 -2
- package/dist/affected/sidecar.js +11 -3
- package/dist/agents/claude/sdk-options.js +6 -0
- package/dist/commands/run-changed.js +4 -1
- package/dist/providers/credentials.js +1 -1
- package/dist/providers/sandbox-lifecycle.d.ts +22 -0
- package/dist/providers/sandbox-lifecycle.js +133 -0
- package/dist/providers/sandbox.js +2 -1
- package/dist/providers/workspace.d.ts +0 -6
- package/dist/providers/workspace.js +1 -31
- package/dist/reporting/core.js +18 -2
- package/dist/reporting/types.d.ts +5 -0
- package/dist/runners/model-builders.js +6 -0
- package/dist/runners/model.d.ts +4 -0
- package/dist/runners/report-projection.js +8 -0
- package/dist/runners/vitest-adapter.js +4 -0
- package/dist/runners/vitest-invocation.d.ts +1 -0
- package/dist/runners/vitest-invocation.js +17 -2
- package/dist/sdk/agent-result-log.js +9 -0
- package/dist/sdk/agent.js +13 -0
- package/dist/sdk/evaluate.d.ts +3 -3
- package/dist/sdk/evaluate.js +14 -0
- package/dist/sdk/index.d.ts +2 -1
- package/dist/sdk/lifecycle.js +12 -2
- package/dist/sdk/result-capture.d.ts +2 -0
- package/dist/sdk/result-capture.js +9 -1
- package/dist/sdk/types.d.ts +22 -0
- package/dist/types.d.ts +9 -0
- package/package.json +19 -19
- package/LICENSE +0 -24
|
@@ -8,7 +8,7 @@ export { runWithAdapter, type PathgradeRunOptions, type AdapterReporterMode } fr
|
|
|
8
8
|
export { createRunnerLifecycleHooks } from '../runners/lifecycle-hooks.js';
|
|
9
9
|
export { discoverPathgradeEvalFiles } from '../evals/discovery.js';
|
|
10
10
|
export { DEFAULT_EVAL_EXCLUDE, DEFAULT_EVAL_INCLUDE, defaultPathgradeConfig, resolvePathgradeConfig, type PathgradeConfig, type ResolvedPathgradeConfig, } from '../config/pathgrade.js';
|
|
11
|
-
export { readSidecar } from '../affected/sidecar.js';
|
|
11
|
+
export { readSidecar, readSidecarForInvocation } from '../affected/sidecar.js';
|
|
12
12
|
export { getPathgradeDir } from '../reporters/results-path.js';
|
|
13
13
|
export { printReportSummary } from '../reporters/report-summary.js';
|
|
14
14
|
export { fmt } from '../utils/cli.js';
|
|
@@ -6,7 +6,7 @@ export { runWithAdapter } from '../runners/orchestrator.js';
|
|
|
6
6
|
export { createRunnerLifecycleHooks } from '../runners/lifecycle-hooks.js';
|
|
7
7
|
export { discoverPathgradeEvalFiles } from '../evals/discovery.js';
|
|
8
8
|
export { DEFAULT_EVAL_EXCLUDE, DEFAULT_EVAL_INCLUDE, defaultPathgradeConfig, resolvePathgradeConfig, } from '../config/pathgrade.js';
|
|
9
|
-
export { readSidecar } from '../affected/sidecar.js';
|
|
9
|
+
export { readSidecar, readSidecarForInvocation } from '../affected/sidecar.js';
|
|
10
10
|
export { getPathgradeDir } from '../reporters/results-path.js';
|
|
11
11
|
export { printReportSummary } from '../reporters/report-summary.js';
|
|
12
12
|
export { fmt } from '../utils/cli.js';
|
|
@@ -2,7 +2,7 @@ import * as fs from 'node:fs';
|
|
|
2
2
|
import * as path from 'node:path';
|
|
3
3
|
import { spawn } from 'node:child_process';
|
|
4
4
|
import { createRequire } from 'node:module';
|
|
5
|
-
import { getPathgradeDir, printReportSummary,
|
|
5
|
+
import { getPathgradeDir, printReportSummary, readSidecarForInvocation, runWithAdapter, } from '@wix/pathgrade/adapter-kit';
|
|
6
6
|
import { removeJestMetadata, withJestMetadataEnv } from './metadata.js';
|
|
7
7
|
import { createJestAdapter } from './runner-adapter.js';
|
|
8
8
|
export function createJestInvocationAdapter(input) {
|
|
@@ -10,7 +10,7 @@ export function createJestInvocationAdapter(input) {
|
|
|
10
10
|
return {
|
|
11
11
|
name: 'jest',
|
|
12
12
|
async run(runInput) {
|
|
13
|
-
const selection = await
|
|
13
|
+
const selection = await readSidecarForInvocation(runInput.cwd, runInput.env.PATHGRADE_SELECTION_INVOCATION_ID, msg => {
|
|
14
14
|
process.stderr.write(`[pathgrade] ${msg}\n`);
|
|
15
15
|
}) ?? undefined;
|
|
16
16
|
const include = runInput.selectedFiles && runInput.selectedFiles.length > 0
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
import * as path from 'node:path';
|
|
2
2
|
import { execSync } from 'node:child_process';
|
|
3
|
-
import { fmt, getPathgradeDir, printReportSummary,
|
|
3
|
+
import { fmt, getPathgradeDir, printReportSummary, readSidecarForInvocation, resolvePathgradeConfig, runWithAdapter, } from '@wix/pathgrade/adapter-kit';
|
|
4
4
|
import { getJestLifecycleMetadata } from './lifecycle.js';
|
|
5
5
|
import { readJestMetadata, removeJestMetadata } from './metadata.js';
|
|
6
6
|
import { createJestAdapter } from './runner-adapter.js';
|
|
@@ -30,7 +30,7 @@ export default class PathgradeJestReporter {
|
|
|
30
30
|
writeEmptyReport: false,
|
|
31
31
|
warn: warning => console.warn(warning),
|
|
32
32
|
log: () => console.log(`\n ${fmt.dim('Results written to')} ${outputDir}\n`),
|
|
33
|
-
loadSelection: async () => (await
|
|
33
|
+
loadSelection: async () => (await readSidecarForInvocation(cwd, process.env.PATHGRADE_SELECTION_INVOCATION_ID, msg => {
|
|
34
34
|
console.warn(`[pathgrade] ${msg}`);
|
|
35
35
|
})) ?? undefined,
|
|
36
36
|
printSummary: summaries => {
|
|
@@ -61,6 +61,12 @@ function runCase(input) {
|
|
|
61
61
|
score: entry.score,
|
|
62
62
|
...(entry.trial ? { trial: entry.trial } : {}),
|
|
63
63
|
...(entry.diagnostics ? { diagnostics: entry.diagnostics } : {}),
|
|
64
|
+
...(entry.resultKind ? { resultKind: entry.resultKind } : {}),
|
|
65
|
+
...(entry.scoringDurationMs !== undefined
|
|
66
|
+
? { scoringDurationMs: entry.scoringDurationMs }
|
|
67
|
+
: {}),
|
|
68
|
+
...(entry.recordedAt ? { recordedAt: entry.recordedAt } : {}),
|
|
69
|
+
...(entry.agent ? { agent: entry.agent } : {}),
|
|
64
70
|
}));
|
|
65
71
|
const failureMessage = input.assertion.failureMessages?.find(message => message.trim().length > 0);
|
|
66
72
|
return {
|
|
@@ -70,6 +70,10 @@ function toReportEvaluations(evaluations) {
|
|
|
70
70
|
score: entry.score,
|
|
71
71
|
trial: entry.trial,
|
|
72
72
|
diagnostics: entry.diagnostics,
|
|
73
|
+
resultKind: entry.resultKind,
|
|
74
|
+
scoringDurationMs: entry.scoringDurationMs,
|
|
75
|
+
recordedAt: entry.recordedAt,
|
|
76
|
+
agent: entry.agent,
|
|
73
77
|
}));
|
|
74
78
|
}
|
|
75
79
|
function writeCase(testCase) {
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import {
|
|
1
|
+
import { readSidecarForInvocation } from '../../affected/sidecar.js';
|
|
2
2
|
import { getPathgradeDir } from '../../reporters/results-path.js';
|
|
3
3
|
import { printReportSummary } from '../../reporters/report-summary.js';
|
|
4
4
|
import { runWithAdapter } from '../../runners/orchestrator.js';
|
|
@@ -7,7 +7,7 @@ export function createNodeTestInvocationAdapter(input) {
|
|
|
7
7
|
return {
|
|
8
8
|
name: 'node-test',
|
|
9
9
|
async run(runInput) {
|
|
10
|
-
const selection = await
|
|
10
|
+
const selection = await readSidecarForInvocation(runInput.cwd, runInput.env.PATHGRADE_SELECTION_INVOCATION_ID, msg => {
|
|
11
11
|
process.stderr.write(`[pathgrade] ${msg}\n`);
|
|
12
12
|
}) ?? undefined;
|
|
13
13
|
const include = runInput.selectedFiles && runInput.selectedFiles.length > 0
|
|
@@ -2,7 +2,7 @@ import * as path from 'path';
|
|
|
2
2
|
import { execSync } from 'child_process';
|
|
3
3
|
import { fmt } from '../../utils/cli.js';
|
|
4
4
|
import { getPathgradeDir } from '../../reporters/results-path.js';
|
|
5
|
-
import {
|
|
5
|
+
import { readSidecarForInvocation } from '../../affected/sidecar.js';
|
|
6
6
|
import { printReportSummary } from '../../reporters/report-summary.js';
|
|
7
7
|
import { createVitestAdapter } from '../../runners/vitest-adapter.js';
|
|
8
8
|
import { runWithAdapter } from '../../runners/orchestrator.js';
|
|
@@ -35,7 +35,7 @@ export class PathgradeReporter {
|
|
|
35
35
|
writeEmptyReport: false,
|
|
36
36
|
warn: warning => console.warn(warning),
|
|
37
37
|
log: () => console.log(`\n ${fmt.dim('Results written to')} ${outputDir}\n`),
|
|
38
|
-
loadSelection: async () => (await
|
|
38
|
+
loadSelection: async () => (await readSidecarForInvocation(cwd, process.env.PATHGRADE_SELECTION_INVOCATION_ID, msg => {
|
|
39
39
|
console.warn(`[pathgrade] ${msg}`);
|
|
40
40
|
})) ?? undefined,
|
|
41
41
|
printSummary: summaries => {
|
|
@@ -21,12 +21,14 @@ export declare function getSidecarPath(cwd: string): string;
|
|
|
21
21
|
* `.pathgrade/selection.json` (and merged into `results.json` as
|
|
22
22
|
* `PathgradeReport.selection`).
|
|
23
23
|
*/
|
|
24
|
-
export declare function toSelectionReport(result: SelectionResult): PathgradeSelectionReport;
|
|
25
|
-
export declare function writeSidecar(cwd: string, result: SelectionResult): Promise<void>;
|
|
24
|
+
export declare function toSelectionReport(result: SelectionResult, invocationId?: string): PathgradeSelectionReport;
|
|
25
|
+
export declare function writeSidecar(cwd: string, result: SelectionResult, invocationId?: string): Promise<void>;
|
|
26
26
|
/**
|
|
27
27
|
* Read the sidecar if present. Returns `null` when missing, and `null`
|
|
28
28
|
* with an optional warning when the file exists but is malformed — the
|
|
29
29
|
* reporter is expected to tolerate corruption rather than break the run.
|
|
30
30
|
*/
|
|
31
31
|
export declare function readSidecar(cwd: string, onWarning?: (msg: string) => void): Promise<PathgradeSelectionReport | null>;
|
|
32
|
+
/** Reads selection only when it belongs to the currently spawned runner. */
|
|
33
|
+
export declare function readSidecarForInvocation(cwd: string, invocationId: string | undefined, onWarning?: (msg: string) => void): Promise<PathgradeSelectionReport | null>;
|
|
32
34
|
export declare function clearSidecar(cwd: string): Promise<void>;
|
package/dist/affected/sidecar.js
CHANGED
|
@@ -24,8 +24,9 @@ export function getSidecarPath(cwd) {
|
|
|
24
24
|
* `.pathgrade/selection.json` (and merged into `results.json` as
|
|
25
25
|
* `PathgradeReport.selection`).
|
|
26
26
|
*/
|
|
27
|
-
export function toSelectionReport(result) {
|
|
27
|
+
export function toSelectionReport(result, invocationId) {
|
|
28
28
|
const report = {
|
|
29
|
+
...(invocationId ? { invocation_id: invocationId } : {}),
|
|
29
30
|
base_ref: result.baseRef,
|
|
30
31
|
changed_files_count: result.changedFiles.length,
|
|
31
32
|
selected: result.selected.map(s => s.file).sort(),
|
|
@@ -38,10 +39,10 @@ export function toSelectionReport(result) {
|
|
|
38
39
|
report.global_match = result.globalMatch;
|
|
39
40
|
return report;
|
|
40
41
|
}
|
|
41
|
-
export async function writeSidecar(cwd, result) {
|
|
42
|
+
export async function writeSidecar(cwd, result, invocationId) {
|
|
42
43
|
const sidecarPath = getSidecarPath(cwd);
|
|
43
44
|
await fs.ensureDir(path.dirname(sidecarPath));
|
|
44
|
-
await fs.writeJSON(sidecarPath, toSelectionReport(result), { spaces: 2 });
|
|
45
|
+
await fs.writeJSON(sidecarPath, toSelectionReport(result, invocationId), { spaces: 2 });
|
|
45
46
|
}
|
|
46
47
|
/**
|
|
47
48
|
* Read the sidecar if present. Returns `null` when missing, and `null`
|
|
@@ -66,6 +67,13 @@ export async function readSidecar(cwd, onWarning) {
|
|
|
66
67
|
return null;
|
|
67
68
|
}
|
|
68
69
|
}
|
|
70
|
+
/** Reads selection only when it belongs to the currently spawned runner. */
|
|
71
|
+
export async function readSidecarForInvocation(cwd, invocationId, onWarning) {
|
|
72
|
+
if (!invocationId)
|
|
73
|
+
return null;
|
|
74
|
+
const report = await readSidecar(cwd, onWarning);
|
|
75
|
+
return report?.invocation_id === invocationId ? report : null;
|
|
76
|
+
}
|
|
69
77
|
export async function clearSidecar(cwd) {
|
|
70
78
|
const sidecarPath = getSidecarPath(cwd);
|
|
71
79
|
try {
|
|
@@ -78,6 +78,12 @@ export function buildClaudeSdkOptions(inputs) {
|
|
|
78
78
|
continue;
|
|
79
79
|
env[key] = value;
|
|
80
80
|
}
|
|
81
|
+
// Claude Code normally exposes claude.ai account-level MCP connectors to
|
|
82
|
+
// local OAuth sessions. Keep PathGrade trials isolated from those ambient
|
|
83
|
+
// connectors unless the caller explicitly opts in through their env.
|
|
84
|
+
if (useLocalOAuth && env.ENABLE_CLAUDEAI_MCP_SERVERS === undefined) {
|
|
85
|
+
env.ENABLE_CLAUDEAI_MCP_SERVERS = 'false';
|
|
86
|
+
}
|
|
81
87
|
if (!useLocalOAuth) {
|
|
82
88
|
env.CLAUDE_CONFIG_DIR = path.join(inputs.workspacePath, CLAUDE_CONFIG_SUBDIR);
|
|
83
89
|
}
|
|
@@ -13,6 +13,7 @@
|
|
|
13
13
|
* only producer of the file list here.
|
|
14
14
|
*/
|
|
15
15
|
import * as fs from 'fs';
|
|
16
|
+
import { randomUUID } from 'node:crypto';
|
|
16
17
|
import { selectAffected } from '../affected/select.js';
|
|
17
18
|
import { resolveBaseRef, computeChangedFiles } from '../affected/git.js';
|
|
18
19
|
import { writeSidecar } from '../affected/sidecar.js';
|
|
@@ -21,10 +22,12 @@ import { resolvePathgradeConfig } from '../config/pathgrade.js';
|
|
|
21
22
|
import { loadRunnerInvocationAdapter } from '../runners/adapter-loader.js';
|
|
22
23
|
export async function runChanged(opts) {
|
|
23
24
|
const { cwd, parsed } = opts;
|
|
25
|
+
const selectionInvocationId = randomUUID();
|
|
24
26
|
const runnerEnv = {
|
|
25
27
|
...process.env,
|
|
26
28
|
...(parsed.forceDiagnostics ? { PATHGRADE_DIAGNOSTICS: '1' } : {}),
|
|
27
29
|
...(parsed.forceVerbose ? { PATHGRADE_VERBOSE: '1' } : {}),
|
|
30
|
+
PATHGRADE_SELECTION_INVOCATION_ID: selectionInvocationId,
|
|
28
31
|
};
|
|
29
32
|
const configPath = findVitestConfigArg(parsed.runnerArgs);
|
|
30
33
|
let config;
|
|
@@ -111,7 +114,7 @@ export async function runChanged(opts) {
|
|
|
111
114
|
// Persist the sidecar immediately — even on empty selection, so the
|
|
112
115
|
// reporter (if invoked later by another workflow step) has a coherent
|
|
113
116
|
// record. A subsequent plain `pathgrade run` clears it.
|
|
114
|
-
await writeSidecar(cwd, result);
|
|
117
|
+
await writeSidecar(cwd, result, selectionInvocationId);
|
|
115
118
|
if (result.selected.length === 0) {
|
|
116
119
|
if (!parsed.quiet) {
|
|
117
120
|
process.stderr.write(`pathgrade: no affected evals; nothing to run.\n`);
|
|
@@ -0,0 +1,22 @@
|
|
|
1
|
+
export declare const SANDBOX_PREFIX = "pathgrade-";
|
|
2
|
+
export declare const SANDBOX_MARKER = ".pathgrade-sandbox.json";
|
|
3
|
+
export declare const STALE_SANDBOX_AGE_MS: number;
|
|
4
|
+
interface RemoveSandboxRootOptions {
|
|
5
|
+
remove?: (target: string) => Promise<void>;
|
|
6
|
+
sleep?: (ms: number) => Promise<void>;
|
|
7
|
+
retryDelaysMs?: readonly number[];
|
|
8
|
+
}
|
|
9
|
+
export declare function removeSandboxRoot(rootDir: string, opts?: RemoveSandboxRootOptions): Promise<void>;
|
|
10
|
+
export interface StaleSandboxCleanupOptions extends RemoveSandboxRootOptions {
|
|
11
|
+
tempDir?: string;
|
|
12
|
+
now?: () => number;
|
|
13
|
+
isProcessAlive?: (pid: number) => boolean;
|
|
14
|
+
}
|
|
15
|
+
/**
|
|
16
|
+
* Removes old, crashed PathGrade sandboxes without examining anything outside
|
|
17
|
+
* the resolved operating-system temp directory. Every filesystem failure is
|
|
18
|
+
* intentionally ignored so a cleanup race never prevents a new trial.
|
|
19
|
+
*/
|
|
20
|
+
export declare function cleanupStaleSandboxes(opts?: StaleSandboxCleanupOptions): Promise<void>;
|
|
21
|
+
export declare function createSandboxRoot(): Promise<string>;
|
|
22
|
+
export {};
|
|
@@ -0,0 +1,133 @@
|
|
|
1
|
+
import fs from 'fs-extra';
|
|
2
|
+
import * as os from 'os';
|
|
3
|
+
import * as path from 'path';
|
|
4
|
+
export const SANDBOX_PREFIX = 'pathgrade-';
|
|
5
|
+
export const SANDBOX_MARKER = '.pathgrade-sandbox.json';
|
|
6
|
+
export const STALE_SANDBOX_AGE_MS = 24 * 60 * 60 * 1_000;
|
|
7
|
+
let startupCleanup;
|
|
8
|
+
const SANDBOX_REMOVE_RETRY_DELAYS_MS = [
|
|
9
|
+
50,
|
|
10
|
+
100,
|
|
11
|
+
250,
|
|
12
|
+
500,
|
|
13
|
+
1_000,
|
|
14
|
+
2_000,
|
|
15
|
+
3_000,
|
|
16
|
+
5_000,
|
|
17
|
+
];
|
|
18
|
+
function isRetryableRemoveError(error) {
|
|
19
|
+
const code = error.code;
|
|
20
|
+
return code === 'ENOTEMPTY' || code === 'EBUSY' || code === 'EPERM';
|
|
21
|
+
}
|
|
22
|
+
export async function removeSandboxRoot(rootDir, opts = {}) {
|
|
23
|
+
const remove = opts.remove ?? ((target) => fs.remove(target));
|
|
24
|
+
const sleep = opts.sleep ?? ((ms) => new Promise((resolve) => setTimeout(resolve, ms)));
|
|
25
|
+
const retryDelaysMs = opts.retryDelaysMs ?? SANDBOX_REMOVE_RETRY_DELAYS_MS;
|
|
26
|
+
for (let attempt = 0;; attempt++) {
|
|
27
|
+
try {
|
|
28
|
+
await remove(rootDir);
|
|
29
|
+
return;
|
|
30
|
+
}
|
|
31
|
+
catch (error) {
|
|
32
|
+
if (!isRetryableRemoveError(error) || attempt >= retryDelaysMs.length) {
|
|
33
|
+
throw error;
|
|
34
|
+
}
|
|
35
|
+
await sleep(retryDelaysMs[attempt]);
|
|
36
|
+
}
|
|
37
|
+
}
|
|
38
|
+
}
|
|
39
|
+
function isProcessAlive(pid) {
|
|
40
|
+
try {
|
|
41
|
+
process.kill(pid, 0);
|
|
42
|
+
return true;
|
|
43
|
+
}
|
|
44
|
+
catch (error) {
|
|
45
|
+
// A process we cannot signal is still live. Unknown failures are also
|
|
46
|
+
// preserved: cleanup must fail safe rather than delete a live trial.
|
|
47
|
+
return error.code !== 'ESRCH';
|
|
48
|
+
}
|
|
49
|
+
}
|
|
50
|
+
async function readOwnedSandboxMarker(rootDir) {
|
|
51
|
+
const rootStat = await fs.lstat(rootDir);
|
|
52
|
+
if (!rootStat.isDirectory() || rootStat.isSymbolicLink())
|
|
53
|
+
return undefined;
|
|
54
|
+
const markerPath = path.join(rootDir, SANDBOX_MARKER);
|
|
55
|
+
const markerStat = await fs.lstat(markerPath);
|
|
56
|
+
if (!markerStat.isFile() || markerStat.isSymbolicLink())
|
|
57
|
+
return undefined;
|
|
58
|
+
const marker = JSON.parse(await fs.readFile(markerPath, 'utf8'));
|
|
59
|
+
const pid = marker.pid;
|
|
60
|
+
if (marker.version !== 1 || typeof pid !== 'number' || !Number.isSafeInteger(pid) || pid <= 0)
|
|
61
|
+
return undefined;
|
|
62
|
+
return { version: 1, pid };
|
|
63
|
+
}
|
|
64
|
+
async function isStaleOwnedSandbox(rootDir, now, staleAfterMs, processIsAlive) {
|
|
65
|
+
const marker = await readOwnedSandboxMarker(rootDir);
|
|
66
|
+
if (!marker)
|
|
67
|
+
return false;
|
|
68
|
+
const markerStat = await fs.lstat(path.join(rootDir, SANDBOX_MARKER));
|
|
69
|
+
if (now - markerStat.mtimeMs < staleAfterMs)
|
|
70
|
+
return false;
|
|
71
|
+
return !processIsAlive(marker.pid);
|
|
72
|
+
}
|
|
73
|
+
/**
|
|
74
|
+
* Removes old, crashed PathGrade sandboxes without examining anything outside
|
|
75
|
+
* the resolved operating-system temp directory. Every filesystem failure is
|
|
76
|
+
* intentionally ignored so a cleanup race never prevents a new trial.
|
|
77
|
+
*/
|
|
78
|
+
export async function cleanupStaleSandboxes(opts = {}) {
|
|
79
|
+
let tempDir;
|
|
80
|
+
try {
|
|
81
|
+
tempDir = await fs.realpath(opts.tempDir ?? os.tmpdir());
|
|
82
|
+
}
|
|
83
|
+
catch {
|
|
84
|
+
return;
|
|
85
|
+
}
|
|
86
|
+
const now = opts.now ?? Date.now;
|
|
87
|
+
const processIsAlive = opts.isProcessAlive ?? isProcessAlive;
|
|
88
|
+
let entries;
|
|
89
|
+
try {
|
|
90
|
+
entries = await fs.readdir(tempDir, { withFileTypes: true });
|
|
91
|
+
}
|
|
92
|
+
catch {
|
|
93
|
+
return;
|
|
94
|
+
}
|
|
95
|
+
for (const entry of entries) {
|
|
96
|
+
if (!entry.isDirectory() || entry.isSymbolicLink() || !entry.name.startsWith(SANDBOX_PREFIX))
|
|
97
|
+
continue;
|
|
98
|
+
const rootDir = path.resolve(tempDir, entry.name);
|
|
99
|
+
if (path.dirname(rootDir) !== tempDir)
|
|
100
|
+
continue;
|
|
101
|
+
try {
|
|
102
|
+
if (!await isStaleOwnedSandbox(rootDir, now(), STALE_SANDBOX_AGE_MS, processIsAlive))
|
|
103
|
+
continue;
|
|
104
|
+
// Re-check ownership and liveness immediately before removal. This
|
|
105
|
+
// narrows the window where another process can replace a candidate.
|
|
106
|
+
if (!await isStaleOwnedSandbox(rootDir, now(), STALE_SANDBOX_AGE_MS, processIsAlive))
|
|
107
|
+
continue;
|
|
108
|
+
await removeSandboxRoot(rootDir, opts);
|
|
109
|
+
}
|
|
110
|
+
catch {
|
|
111
|
+
// A disappeared directory, a permission change, or a deletion race
|
|
112
|
+
// must never make trial creation fail.
|
|
113
|
+
}
|
|
114
|
+
}
|
|
115
|
+
}
|
|
116
|
+
export async function createSandboxRoot() {
|
|
117
|
+
// A process may create many trial sandboxes. Scan only once at startup so
|
|
118
|
+
// repeated trials do not pay to enumerate the entire operating-system temp
|
|
119
|
+
// directory, while concurrent creators share the same best-effort sweep.
|
|
120
|
+
startupCleanup ??= cleanupStaleSandboxes();
|
|
121
|
+
await startupCleanup;
|
|
122
|
+
const tempDir = await fs.realpath(os.tmpdir());
|
|
123
|
+
const rootDir = await fs.mkdtemp(path.join(tempDir, SANDBOX_PREFIX));
|
|
124
|
+
const marker = { version: 1, pid: process.pid };
|
|
125
|
+
try {
|
|
126
|
+
await fs.writeFile(path.join(rootDir, SANDBOX_MARKER), JSON.stringify(marker), 'utf8');
|
|
127
|
+
return rootDir;
|
|
128
|
+
}
|
|
129
|
+
catch (error) {
|
|
130
|
+
await fs.remove(rootDir).catch(() => { });
|
|
131
|
+
throw error;
|
|
132
|
+
}
|
|
133
|
+
}
|
|
@@ -2,11 +2,12 @@ import fs from 'fs-extra';
|
|
|
2
2
|
import * as path from 'path';
|
|
3
3
|
import * as os from 'os';
|
|
4
4
|
import { DEFAULT_COPY_IGNORE, createCopyFilter, isPortableCopyEntry } from './copy-filter.js';
|
|
5
|
+
import { createSandboxRoot } from './sandbox-lifecycle.js';
|
|
5
6
|
export const SAFE_HOST_VARS = [
|
|
6
7
|
'PATH', 'SHELL', 'LANG', 'LC_ALL', 'LC_CTYPE', 'TERM', 'USER', 'LOGNAME',
|
|
7
8
|
];
|
|
8
9
|
export async function createSandbox(spec) {
|
|
9
|
-
const rootDir =
|
|
10
|
+
const rootDir = await createSandboxRoot();
|
|
10
11
|
const workspacePath = path.join(rootDir, 'workspace');
|
|
11
12
|
const homePath = path.join(rootDir, 'home');
|
|
12
13
|
const tmpPath = path.join(rootDir, 'tmp');
|
|
@@ -11,11 +11,5 @@ export interface Workspace {
|
|
|
11
11
|
}): Promise<CommandResult>;
|
|
12
12
|
dispose(): Promise<void>;
|
|
13
13
|
}
|
|
14
|
-
interface RemoveSandboxRootOptions {
|
|
15
|
-
remove?: (target: string) => Promise<void>;
|
|
16
|
-
sleep?: (ms: number) => Promise<void>;
|
|
17
|
-
retryDelaysMs?: readonly number[];
|
|
18
|
-
}
|
|
19
|
-
export declare function removeSandboxRoot(rootDir: string, opts?: RemoveSandboxRootOptions): Promise<void>;
|
|
20
14
|
export declare function linkPathsFromHostHome(pathsToLink: string[], sandboxHomePath: string): Promise<void>;
|
|
21
15
|
export declare function prepareWorkspace(spec: SandboxConfig): Promise<Workspace>;
|
|
@@ -2,41 +2,11 @@ import fs from 'fs-extra';
|
|
|
2
2
|
import * as os from 'os';
|
|
3
3
|
import * as path from 'path';
|
|
4
4
|
import { createSandbox } from './sandbox.js';
|
|
5
|
+
import { removeSandboxRoot } from './sandbox-lifecycle.js';
|
|
5
6
|
import { stageMcpConfig } from './mcp-config.js';
|
|
6
7
|
import { sandboxExec } from './sandbox-exec.js';
|
|
7
8
|
import { resolveCredentials } from './credentials.js';
|
|
8
9
|
import { isPortableCopyEntry } from './copy-filter.js';
|
|
9
|
-
const SANDBOX_REMOVE_RETRY_DELAYS_MS = [
|
|
10
|
-
50,
|
|
11
|
-
100,
|
|
12
|
-
250,
|
|
13
|
-
500,
|
|
14
|
-
1_000,
|
|
15
|
-
2_000,
|
|
16
|
-
3_000,
|
|
17
|
-
5_000,
|
|
18
|
-
];
|
|
19
|
-
function isRetryableRemoveError(error) {
|
|
20
|
-
const code = error.code;
|
|
21
|
-
return code === 'ENOTEMPTY' || code === 'EBUSY' || code === 'EPERM';
|
|
22
|
-
}
|
|
23
|
-
export async function removeSandboxRoot(rootDir, opts = {}) {
|
|
24
|
-
const remove = opts.remove ?? ((target) => fs.remove(target));
|
|
25
|
-
const sleep = opts.sleep ?? ((ms) => new Promise((resolve) => setTimeout(resolve, ms)));
|
|
26
|
-
const retryDelaysMs = opts.retryDelaysMs ?? SANDBOX_REMOVE_RETRY_DELAYS_MS;
|
|
27
|
-
for (let attempt = 0;; attempt++) {
|
|
28
|
-
try {
|
|
29
|
-
await remove(rootDir);
|
|
30
|
-
return;
|
|
31
|
-
}
|
|
32
|
-
catch (error) {
|
|
33
|
-
if (!isRetryableRemoveError(error) || attempt >= retryDelaysMs.length) {
|
|
34
|
-
throw error;
|
|
35
|
-
}
|
|
36
|
-
await sleep(retryDelaysMs[attempt]);
|
|
37
|
-
}
|
|
38
|
-
}
|
|
39
|
-
}
|
|
40
10
|
async function copyPathsFromHostHome(pathsToCopy, sandboxHomePath) {
|
|
41
11
|
const realHome = os.homedir();
|
|
42
12
|
for (const relPath of pathsToCopy) {
|
package/dist/reporting/core.js
CHANGED
|
@@ -35,7 +35,9 @@ export function buildPathgradeReport(input) {
|
|
|
35
35
|
trace_file: traceFile,
|
|
36
36
|
});
|
|
37
37
|
}
|
|
38
|
-
const scores = reportableGroups.flatMap(group => group.cases
|
|
38
|
+
const scores = reportableGroups.flatMap(group => group.cases
|
|
39
|
+
.filter(testCase => testCase.resultKind !== 'synthetic_no_evaluation')
|
|
40
|
+
.map(testCase => testCase.score));
|
|
39
41
|
const overallPassRate = average(scores);
|
|
40
42
|
const status = input.threshold != null
|
|
41
43
|
? (overallPassRate >= input.threshold ? 'pass' : 'fail')
|
|
@@ -69,7 +71,10 @@ function toBuiltCase(testCase) {
|
|
|
69
71
|
`empty results for "${testCase.name}" — evaluate() may not have been called`,
|
|
70
72
|
]);
|
|
71
73
|
}
|
|
72
|
-
|
|
74
|
+
// A case can own multiple agents. Preserve the existing "last evaluation"
|
|
75
|
+
// rule among real evaluations without letting a later synthetic result from
|
|
76
|
+
// an unevaluated sibling agent erase a legitimate score.
|
|
77
|
+
const evaluation = testCase.evaluations.findLast(candidate => candidate.resultKind !== 'synthetic_no_evaluation') ?? testCase.evaluations[testCase.evaluations.length - 1];
|
|
73
78
|
const diagnostics = evaluation.diagnostics ?? testCase.diagnostics;
|
|
74
79
|
return {
|
|
75
80
|
name: testCase.name,
|
|
@@ -78,12 +83,17 @@ function toBuiltCase(testCase) {
|
|
|
78
83
|
runnerDurationMs: testCase.runnerDurationMs,
|
|
79
84
|
diagnostics,
|
|
80
85
|
reportable: testCase.reportable,
|
|
86
|
+
resultKind: evaluation.resultKind,
|
|
81
87
|
trial: normalizeTrial({
|
|
82
88
|
name: testCase.name,
|
|
83
89
|
score: evaluation.score,
|
|
84
90
|
runnerDurationMs: testCase.runnerDurationMs,
|
|
85
91
|
trial: evaluation.trial,
|
|
86
92
|
diagnostics,
|
|
93
|
+
resultKind: evaluation.resultKind,
|
|
94
|
+
scoringDurationMs: evaluation.scoringDurationMs,
|
|
95
|
+
recordedAt: evaluation.recordedAt,
|
|
96
|
+
agent: evaluation.agent,
|
|
87
97
|
}),
|
|
88
98
|
warnings: [],
|
|
89
99
|
};
|
|
@@ -123,6 +133,12 @@ function normalizeTrial(input) {
|
|
|
123
133
|
name: input.name,
|
|
124
134
|
duration_ms: base.duration_ms || input.runnerDurationMs,
|
|
125
135
|
diagnostics: input.diagnostics ?? base.diagnostics,
|
|
136
|
+
...(input.resultKind ? { result_kind: input.resultKind } : {}),
|
|
137
|
+
...(input.scoringDurationMs !== undefined
|
|
138
|
+
? { scoring_duration_ms: input.scoringDurationMs }
|
|
139
|
+
: {}),
|
|
140
|
+
...(input.recordedAt ? { recorded_at: input.recordedAt } : {}),
|
|
141
|
+
...(input.agent ? { agent: input.agent } : {}),
|
|
126
142
|
...(skills.length > 0 ? { skills_used: skills } : {}),
|
|
127
143
|
};
|
|
128
144
|
}
|
|
@@ -1,4 +1,5 @@
|
|
|
1
1
|
import type { DiagnosticsReport } from '../sdk/diagnostics.js';
|
|
2
|
+
import type { AgentExecutionMetadata, EvaluationResultKind } from '../sdk/types.js';
|
|
2
3
|
import type { PathgradeReport, PathgradeSelectionReport, TrialResult } from '../types.js';
|
|
3
4
|
export type ReportCaseState = 'passed' | 'failed' | 'skipped' | 'pending';
|
|
4
5
|
export interface ReportRunInput {
|
|
@@ -27,6 +28,10 @@ export interface ReportEvaluationInput {
|
|
|
27
28
|
score: number;
|
|
28
29
|
trial?: TrialResult;
|
|
29
30
|
diagnostics?: DiagnosticsReport;
|
|
31
|
+
resultKind?: EvaluationResultKind;
|
|
32
|
+
scoringDurationMs?: number;
|
|
33
|
+
recordedAt?: string;
|
|
34
|
+
agent?: AgentExecutionMetadata;
|
|
30
35
|
}
|
|
31
36
|
export interface PathgradeReportBuildResult {
|
|
32
37
|
report: PathgradeReport;
|
|
@@ -27,6 +27,12 @@ export function buildNormalizedRunSnapshotFromReportGroups(run, groups) {
|
|
|
27
27
|
score: evaluation.score,
|
|
28
28
|
...(evaluation.trial ? { trial: evaluation.trial } : {}),
|
|
29
29
|
...(evaluation.diagnostics ? { diagnostics: evaluation.diagnostics } : {}),
|
|
30
|
+
...(evaluation.resultKind ? { resultKind: evaluation.resultKind } : {}),
|
|
31
|
+
...(evaluation.scoringDurationMs !== undefined
|
|
32
|
+
? { scoringDurationMs: evaluation.scoringDurationMs }
|
|
33
|
+
: {}),
|
|
34
|
+
...(evaluation.recordedAt ? { recordedAt: evaluation.recordedAt } : {}),
|
|
35
|
+
...(evaluation.agent ? { agent: evaluation.agent } : {}),
|
|
30
36
|
}));
|
|
31
37
|
return {
|
|
32
38
|
id: caseId,
|
package/dist/runners/model.d.ts
CHANGED
|
@@ -84,6 +84,10 @@ export interface EvaluationRecord {
|
|
|
84
84
|
diagnostics?: DiagnosticsReport;
|
|
85
85
|
trial?: TrialResult;
|
|
86
86
|
nativeReferences?: NativeReference[];
|
|
87
|
+
resultKind?: import('../sdk/types.js').EvaluationResultKind;
|
|
88
|
+
scoringDurationMs?: number;
|
|
89
|
+
recordedAt?: string;
|
|
90
|
+
agent?: import('../sdk/types.js').AgentExecutionMetadata;
|
|
87
91
|
}
|
|
88
92
|
export interface AssertionRecord {
|
|
89
93
|
id: string;
|
|
@@ -40,6 +40,10 @@ function projectedEvaluations(runCase) {
|
|
|
40
40
|
score: evaluation.score,
|
|
41
41
|
...(evaluation.trial ? { trial: evaluation.trial } : {}),
|
|
42
42
|
...(evaluation.diagnostics ? { diagnostics: evaluation.diagnostics } : {}),
|
|
43
|
+
...(evaluation.resultKind ? { resultKind: evaluation.resultKind } : {}),
|
|
44
|
+
...(evaluation.scoringDurationMs !== undefined ? { scoringDurationMs: evaluation.scoringDurationMs } : {}),
|
|
45
|
+
...(evaluation.recordedAt ? { recordedAt: evaluation.recordedAt } : {}),
|
|
46
|
+
...(evaluation.agent ? { agent: evaluation.agent } : {}),
|
|
43
47
|
})));
|
|
44
48
|
}
|
|
45
49
|
if (runCase.scoringPolicy.kind === 'score') {
|
|
@@ -52,6 +56,10 @@ function projectedEvaluations(runCase) {
|
|
|
52
56
|
score: evaluation.score,
|
|
53
57
|
...(evaluation.trial ? { trial: evaluation.trial } : {}),
|
|
54
58
|
...(evaluation.diagnostics ? { diagnostics: evaluation.diagnostics } : {}),
|
|
59
|
+
...(evaluation.resultKind ? { resultKind: evaluation.resultKind } : {}),
|
|
60
|
+
...(evaluation.scoringDurationMs !== undefined ? { scoringDurationMs: evaluation.scoringDurationMs } : {}),
|
|
61
|
+
...(evaluation.recordedAt ? { recordedAt: evaluation.recordedAt } : {}),
|
|
62
|
+
...(evaluation.agent ? { agent: evaluation.agent } : {}),
|
|
55
63
|
}];
|
|
56
64
|
}
|
|
57
65
|
function preferredGroupingLabel(hints) {
|
|
@@ -89,6 +89,10 @@ function toReportCaseInput(testCase, groupName) {
|
|
|
89
89
|
score: entry.score,
|
|
90
90
|
trial: entry.trial,
|
|
91
91
|
diagnostics: entry.diagnostics,
|
|
92
|
+
resultKind: entry.resultKind,
|
|
93
|
+
scoringDurationMs: entry.scoringDurationMs,
|
|
94
|
+
recordedAt: entry.recordedAt,
|
|
95
|
+
agent: entry.agent,
|
|
92
96
|
})),
|
|
93
97
|
diagnostics: normalized.diagnostics,
|
|
94
98
|
};
|
|
@@ -1,3 +1,6 @@
|
|
|
1
|
+
import fs from 'node:fs';
|
|
2
|
+
import { createRequire } from 'node:module';
|
|
3
|
+
import path from 'node:path';
|
|
1
4
|
export function createVitestInvocationAdapter(input = {}) {
|
|
2
5
|
const spawnVitest = input.spawnVitest ?? defaultSpawnVitest;
|
|
3
6
|
return {
|
|
@@ -32,13 +35,25 @@ function hasPassWithNoTests(args) {
|
|
|
32
35
|
}
|
|
33
36
|
async function defaultSpawnVitest(req) {
|
|
34
37
|
const { spawn } = await import('child_process');
|
|
38
|
+
const vitestCli = resolveVitestCli(req.cwd);
|
|
35
39
|
return await new Promise(resolve => {
|
|
36
|
-
const child = spawn(
|
|
40
|
+
const child = spawn(process.execPath, [vitestCli, ...req.argv], {
|
|
37
41
|
stdio: 'inherit',
|
|
38
42
|
env: req.env,
|
|
39
43
|
cwd: req.cwd,
|
|
40
|
-
shell: true,
|
|
41
44
|
});
|
|
42
45
|
child.on('close', code => resolve(code ?? 0));
|
|
43
46
|
});
|
|
44
47
|
}
|
|
48
|
+
export function resolveVitestCli(cwd) {
|
|
49
|
+
const requireFromCwd = createRequire(path.join(cwd, 'package.json'));
|
|
50
|
+
const packageJsonPath = requireFromCwd.resolve('vitest/package.json');
|
|
51
|
+
const packageJson = JSON.parse(fs.readFileSync(packageJsonPath, 'utf8'));
|
|
52
|
+
const bin = typeof packageJson.bin === 'string'
|
|
53
|
+
? packageJson.bin
|
|
54
|
+
: packageJson.bin?.vitest;
|
|
55
|
+
if (!bin) {
|
|
56
|
+
throw new Error(`Unable to resolve the Vitest CLI from ${packageJsonPath}`);
|
|
57
|
+
}
|
|
58
|
+
return path.resolve(path.dirname(packageJsonPath), bin);
|
|
59
|
+
}
|
|
@@ -20,6 +20,15 @@ export function buildModelAgentResultLogEntry(params) {
|
|
|
20
20
|
// never produce a zero-valued field that could be mistaken for
|
|
21
21
|
// a free turn.
|
|
22
22
|
...(params.turnResult.costUsd !== undefined ? { cost_usd: params.turnResult.costUsd } : {}),
|
|
23
|
+
...(params.turnResult.cacheCreationInputTokens !== undefined
|
|
24
|
+
? { cache_creation_input_tokens: params.turnResult.cacheCreationInputTokens }
|
|
25
|
+
: {}),
|
|
26
|
+
...(params.turnResult.cacheReadInputTokens !== undefined
|
|
27
|
+
? { cache_read_input_tokens: params.turnResult.cacheReadInputTokens }
|
|
28
|
+
: {}),
|
|
29
|
+
...(params.turnResult.errorSubtype
|
|
30
|
+
? { error_subtype: params.turnResult.errorSubtype }
|
|
31
|
+
: {}),
|
|
23
32
|
...getOutputMetrics(assistantMessage),
|
|
24
33
|
};
|
|
25
34
|
}
|
package/dist/sdk/agent.js
CHANGED
|
@@ -64,6 +64,19 @@ class AgentImpl {
|
|
|
64
64
|
get workspace() {
|
|
65
65
|
return this.ws.path;
|
|
66
66
|
}
|
|
67
|
+
get executionMetadata() {
|
|
68
|
+
const interactionMode = this.interactionMode === 'startChat'
|
|
69
|
+
? 'start_chat'
|
|
70
|
+
: this.interactionMode === 'runConversation'
|
|
71
|
+
? 'conversation'
|
|
72
|
+
: this.interactionMode ?? undefined;
|
|
73
|
+
return {
|
|
74
|
+
name: this.agentName,
|
|
75
|
+
...(this.modelOpt ? { requestedModel: this.modelOpt } : {}),
|
|
76
|
+
...(this.transport ? { transport: this.transport } : {}),
|
|
77
|
+
...(interactionMode ? { interactionMode } : {}),
|
|
78
|
+
};
|
|
79
|
+
}
|
|
67
80
|
guardInteraction(mode) {
|
|
68
81
|
if (this.interactionMode && this.interactionMode !== mode) {
|
|
69
82
|
throw new Error(`Cannot use ${mode}() after ${this.interactionMode}() — an agent supports only one interaction method`);
|
package/dist/sdk/evaluate.d.ts
CHANGED
|
@@ -1,12 +1,12 @@
|
|
|
1
|
-
import type { Agent, Scorer,
|
|
1
|
+
import type { Agent, Scorer, RecordedEvalResult, ScorerResultEntry, EvaluateOptions } from './types.js';
|
|
2
2
|
export type OnScorerErrorMode = NonNullable<EvaluateOptions['onScorerError']>;
|
|
3
3
|
export declare class EvalScorerError extends Error {
|
|
4
4
|
readonly scorerErrors: ScorerResultEntry[];
|
|
5
5
|
readonly result: RecordedEvalResult;
|
|
6
6
|
constructor(result: RecordedEvalResult, scorerErrors: ScorerResultEntry[]);
|
|
7
7
|
}
|
|
8
|
-
type EvaluateFromSnapshot = (snapshotPath: string, scorers: Scorer[], opts?: EvaluateOptions) => Promise<
|
|
9
|
-
type EvaluateFn = ((agent: Agent, scorers: Scorer[], opts?: EvaluateOptions) => Promise<
|
|
8
|
+
type EvaluateFromSnapshot = (snapshotPath: string, scorers: Scorer[], opts?: EvaluateOptions) => Promise<RecordedEvalResult>;
|
|
9
|
+
type EvaluateFn = ((agent: Agent, scorers: Scorer[], opts?: EvaluateOptions) => Promise<RecordedEvalResult>) & {
|
|
10
10
|
fromSnapshot: EvaluateFromSnapshot;
|
|
11
11
|
};
|
|
12
12
|
export declare const evaluate: EvaluateFn;
|
package/dist/sdk/evaluate.js
CHANGED
|
@@ -54,6 +54,7 @@ function makeEvaluateAgent() {
|
|
|
54
54
|
// accumulated agent-turn cost.
|
|
55
55
|
const beforeCostUsd = trackedLLM.costUsd ?? 0;
|
|
56
56
|
// measure() returns the delta consumed by this evaluate call.
|
|
57
|
+
const scoringStartedAt = performance.now();
|
|
57
58
|
const { result: evalResult, tokens: deltaTokenUsage } = trackedLLM.measure
|
|
58
59
|
? await trackedLLM.measure(() => evaluateWithContext(ctx, scorers, { ...opts, llm: trackedLLM }))
|
|
59
60
|
: await (async () => {
|
|
@@ -75,6 +76,12 @@ function makeEvaluateAgent() {
|
|
|
75
76
|
const recordedResult = {
|
|
76
77
|
...evalResult,
|
|
77
78
|
tokenUsage: deltaTokenUsage,
|
|
79
|
+
resultKind: 'evaluated',
|
|
80
|
+
scoringDurationMs: Math.max(0, performance.now() - scoringStartedAt),
|
|
81
|
+
recordedAt: new Date().toISOString(),
|
|
82
|
+
...(opts?.evaluationDefinitionKey
|
|
83
|
+
? { evaluationDefinitionKey: opts.evaluationDefinitionKey }
|
|
84
|
+
: {}),
|
|
78
85
|
trial: buildTrialResult(agent.log, { ...evalResult, tokenUsage: deltaTokenUsage }, conversationTokens, conversationCost),
|
|
79
86
|
};
|
|
80
87
|
emitEvalResult({ result: recordedResult, agent });
|
|
@@ -106,9 +113,16 @@ async function fromSnapshot(snapshotPath, scorers, opts) {
|
|
|
106
113
|
},
|
|
107
114
|
artifacts: createSessionArtifacts(snapshot.workspace ?? '', snapshot.toolEvents),
|
|
108
115
|
};
|
|
116
|
+
const scoringStartedAt = performance.now();
|
|
109
117
|
const evalResult = await evaluateWithContext(ctx, scorers, { ...opts, llm: trackedLLM });
|
|
110
118
|
const recordedResult = {
|
|
111
119
|
...evalResult,
|
|
120
|
+
resultKind: 'snapshot',
|
|
121
|
+
scoringDurationMs: Math.max(0, performance.now() - scoringStartedAt),
|
|
122
|
+
recordedAt: new Date().toISOString(),
|
|
123
|
+
...(opts?.evaluationDefinitionKey
|
|
124
|
+
? { evaluationDefinitionKey: opts.evaluationDefinitionKey }
|
|
125
|
+
: {}),
|
|
112
126
|
trial: buildTrialResult(snapshot.log, evalResult),
|
|
113
127
|
};
|
|
114
128
|
maybeThrowOnScorerErrors(recordedResult, opts?.onScorerError ?? 'skip');
|
package/dist/sdk/index.d.ts
CHANGED
|
@@ -21,13 +21,14 @@ export { emitEvalResult, resetAllResultObserversForTests, resetUserResultObserve
|
|
|
21
21
|
export { getAgentCapabilities } from './types.js';
|
|
22
22
|
export type { AgentTransport, AgentCapabilities, AgentName, McpRunMode, McpSafetyOptions, McpToolPolicy, McpToolPolicyRule, } from './types.js';
|
|
23
23
|
export type { AskBus, AskBatch, AskQuestion, AskOption, AskAnswer, AskResolution, AskBatchSnapshot, AskAnswerSnapshot, AskResolutionSnapshot, AskHandle, AskHandler, AskSource, AskLifecycle, AskAnswerSource, Unsubscribe as AskBusUnsubscribe, } from './ask-bus/types.js';
|
|
24
|
-
export type { Agent, AgentOptions, Message, Scorer, CheckScorer, ScoreScorer, JudgeScorer, ToolUsageScorer, ScorerContext, EvalResult, ScorerResultEntry, ScorerStatus, ChatSession, ConversationResult, ConverseOptions, UntilPredicate, UntilContext, Reaction, TextReaction, AskUserReaction, AskUserQuestion, AskUserOption, ReactionPreviewEntry, TextReactionPreviewEntry, AskUserReactionPreviewEntry, ReactionPreviewResult, ReactionPreviewTurn, StepScorer, Persona, PersonaConfig, ConversationWindowConfig, TurnDetail, ReactionFiredEntry, PathgradePluginOptions, PathgradeMeta, TurnTiming, TokenUsage, EvaluateOptions, ReactionPreviewStatus, ScoreResult, JudgeInput, CodeJudgeToolName, ToolExpectation, SessionArtifactMatchOptions, SessionArtifactContent, SessionArtifacts, RecordedEvalResult, PathgradeTestMeta, } from './types.js';
|
|
24
|
+
export type { Agent, AgentOptions, Message, Scorer, CheckScorer, ScoreScorer, JudgeScorer, ToolUsageScorer, ScorerContext, EvalResult, ScorerResultEntry, ScorerStatus, ChatSession, ConversationResult, ConverseOptions, UntilPredicate, UntilContext, Reaction, TextReaction, AskUserReaction, AskUserQuestion, AskUserOption, ReactionPreviewEntry, TextReactionPreviewEntry, AskUserReactionPreviewEntry, ReactionPreviewResult, ReactionPreviewTurn, StepScorer, Persona, PersonaConfig, ConversationWindowConfig, TurnDetail, ReactionFiredEntry, PathgradePluginOptions, PathgradeMeta, TurnTiming, TokenUsage, EvaluateOptions, ReactionPreviewStatus, ScoreResult, JudgeInput, CodeJudgeToolName, ToolExpectation, SessionArtifactMatchOptions, SessionArtifactContent, SessionArtifacts, RecordedEvalResult, PathgradeTestMeta, EvaluationResultKind, AgentExecutionMetadata, AgentInteractionMode, } from './types.js';
|
|
25
25
|
export type { ConversationWindow, ConversationWindowOptions } from './conversation-window.js';
|
|
26
26
|
export type { JudgePipelineOptions } from './judge-pipeline.js';
|
|
27
27
|
export type { RunScorerOptions } from './run-scorer.js';
|
|
28
28
|
export type { OnScorerErrorMode } from './evaluate.js';
|
|
29
29
|
export type { EvalResultEvent, EvalResultObserver, ResultObserverHandle, ResultObserverOptions, ResultObserverOwner, } from './result-capture.js';
|
|
30
30
|
export type { RunSnapshot } from './snapshots.js';
|
|
31
|
+
export type { DiagnosticsReport } from './diagnostics.js';
|
|
31
32
|
export type { ExpectedMcpStartupStatus, ExpectedMcpToolCall, McpStartupStatusEvidence, McpToolCallEvidence, } from './mcp-evidence.js';
|
|
32
33
|
export type { McpPolicyDenialReason, McpToolCallRequest, McpToolPolicyDecision, } from './mcp-safety.js';
|
|
33
34
|
export type { ToolEvent } from '../tool-events.js';
|
package/dist/sdk/lifecycle.js
CHANGED
|
@@ -73,6 +73,12 @@ function recordResult(result, agent, attribution) {
|
|
|
73
73
|
score: result.score,
|
|
74
74
|
scorers: result.scorers,
|
|
75
75
|
trial: result.trial,
|
|
76
|
+
resultKind: result.resultKind ?? 'evaluated',
|
|
77
|
+
...(result.scoringDurationMs !== undefined
|
|
78
|
+
? { scoringDurationMs: result.scoringDurationMs }
|
|
79
|
+
: {}),
|
|
80
|
+
...(result.recordedAt ? { recordedAt: result.recordedAt } : {}),
|
|
81
|
+
...(agent.executionMetadata ? { agent: agent.executionMetadata } : {}),
|
|
76
82
|
diagnostics: buildDiagnosticsReport({
|
|
77
83
|
completionReason,
|
|
78
84
|
completionDetail: conversationEnd?.completion_detail,
|
|
@@ -129,14 +135,18 @@ function synthesizeTrialFromAgent(agent) {
|
|
|
129
135
|
return {
|
|
130
136
|
score: 1,
|
|
131
137
|
scorers: [],
|
|
138
|
+
resultKind: 'synthetic_no_evaluation',
|
|
139
|
+
...(agent.executionMetadata ? { agent: agent.executionMetadata } : {}),
|
|
132
140
|
trial: {
|
|
133
141
|
trial_id: 0,
|
|
134
142
|
reward: 1,
|
|
135
143
|
scorer_results: [],
|
|
136
144
|
duration_ms: 0,
|
|
137
145
|
n_commands: nCommands,
|
|
138
|
-
input_tokens:
|
|
139
|
-
output_tokens:
|
|
146
|
+
input_tokens: 0,
|
|
147
|
+
output_tokens: 0,
|
|
148
|
+
conversation_input_tokens: tokenUsage?.inputTokens ?? 0,
|
|
149
|
+
conversation_output_tokens: tokenUsage?.outputTokens ?? 0,
|
|
140
150
|
session_log: [...agent.log],
|
|
141
151
|
},
|
|
142
152
|
diagnostics: buildDiagnosticsReport({
|
|
@@ -1,7 +1,9 @@
|
|
|
1
1
|
import type { Agent, RecordedEvalResult } from './types.js';
|
|
2
|
+
import { type CaseContext } from './case-context.js';
|
|
2
3
|
export interface EvalResultEvent {
|
|
3
4
|
readonly result: RecordedEvalResult;
|
|
4
5
|
readonly agent: Agent;
|
|
6
|
+
readonly case?: CaseContext;
|
|
5
7
|
}
|
|
6
8
|
export type EvalResultObserver = (event: EvalResultEvent) => void;
|
|
7
9
|
export type ResultObserverOwner = 'user' | 'adapter' | 'test';
|
|
@@ -1,3 +1,4 @@
|
|
|
1
|
+
import { getCurrentCaseContext } from './case-context.js';
|
|
1
2
|
const observers = new Set();
|
|
2
3
|
export function subscribeToEvalResults(observer, options = {}) {
|
|
3
4
|
if (options.owner === 'adapter' && options.key) {
|
|
@@ -20,9 +21,16 @@ export function subscribeToEvalResults(observer, options = {}) {
|
|
|
20
21
|
};
|
|
21
22
|
}
|
|
22
23
|
export function emitEvalResult(event) {
|
|
24
|
+
const currentCase = getCurrentCaseContext();
|
|
25
|
+
const deliveredEvent = event.case || currentCase.status !== 'active'
|
|
26
|
+
? event
|
|
27
|
+
: { ...event, case: currentCase.context };
|
|
28
|
+
// Deliver against a stable snapshot. Observers are allowed to subscribe or
|
|
29
|
+
// unsubscribe from inside a callback, but those mutations must only affect
|
|
30
|
+
// subsequent events.
|
|
23
31
|
for (const subscription of [...observers]) {
|
|
24
32
|
try {
|
|
25
|
-
subscription.observer(
|
|
33
|
+
subscription.observer(deliveredEvent);
|
|
26
34
|
}
|
|
27
35
|
catch {
|
|
28
36
|
// Result capture should be best-effort for every observer; one
|
package/dist/sdk/types.d.ts
CHANGED
|
@@ -6,6 +6,15 @@ import type { DiagnosticsReport } from './diagnostics.js';
|
|
|
6
6
|
import type { LLMPort } from '../utils/llm-types.js';
|
|
7
7
|
import type { McpSafetyOptions } from './mcp-safety.js';
|
|
8
8
|
export type AgentName = 'claude' | 'codex' | 'cursor';
|
|
9
|
+
export type AgentInteractionMode = 'prompt' | 'start_chat' | 'conversation';
|
|
10
|
+
/** Privacy-safe execution dimensions exposed to result observers. */
|
|
11
|
+
export interface AgentExecutionMetadata {
|
|
12
|
+
name: AgentName;
|
|
13
|
+
/** Model override requested by the caller; not necessarily the provider-resolved model. */
|
|
14
|
+
requestedModel?: string;
|
|
15
|
+
transport?: AgentTransport;
|
|
16
|
+
interactionMode?: AgentInteractionMode;
|
|
17
|
+
}
|
|
9
18
|
export interface AgentOptions {
|
|
10
19
|
agent?: AgentName;
|
|
11
20
|
model?: string;
|
|
@@ -61,6 +70,8 @@ export interface Agent {
|
|
|
61
70
|
readonly messages: Message[];
|
|
62
71
|
readonly log: LogEntry[];
|
|
63
72
|
readonly workspace: string;
|
|
73
|
+
/** Provider/configuration metadata only; never contains prompts or paths. */
|
|
74
|
+
readonly executionMetadata?: AgentExecutionMetadata;
|
|
64
75
|
dispose(): Promise<void>;
|
|
65
76
|
}
|
|
66
77
|
export interface ConverseOptions {
|
|
@@ -318,6 +329,8 @@ export interface EvaluateOptions {
|
|
|
318
329
|
failFast?: boolean;
|
|
319
330
|
llm?: LLMPort;
|
|
320
331
|
onScorerError?: 'skip' | 'zero' | 'fail';
|
|
332
|
+
/** Stable identity for this evaluation definition across separate runs. */
|
|
333
|
+
evaluationDefinitionKey?: string;
|
|
321
334
|
}
|
|
322
335
|
export interface TokenUsage {
|
|
323
336
|
inputTokens: number;
|
|
@@ -331,7 +344,12 @@ export interface EvalResult {
|
|
|
331
344
|
}
|
|
332
345
|
export interface RecordedEvalResult extends EvalResult {
|
|
333
346
|
trial?: TrialResult;
|
|
347
|
+
resultKind?: EvaluationResultKind;
|
|
348
|
+
scoringDurationMs?: number;
|
|
349
|
+
recordedAt?: string;
|
|
350
|
+
evaluationDefinitionKey?: string;
|
|
334
351
|
}
|
|
352
|
+
export type EvaluationResultKind = 'evaluated' | 'synthetic_no_evaluation' | 'snapshot';
|
|
335
353
|
export interface ScorerResultEntry {
|
|
336
354
|
name: string;
|
|
337
355
|
type: 'check' | 'score' | 'judge' | 'tool_usage';
|
|
@@ -351,6 +369,10 @@ export interface PathgradeTestMeta {
|
|
|
351
369
|
scorers: ScorerResultEntry[];
|
|
352
370
|
trial?: TrialResult;
|
|
353
371
|
diagnostics?: DiagnosticsReport;
|
|
372
|
+
resultKind?: EvaluationResultKind;
|
|
373
|
+
scoringDurationMs?: number;
|
|
374
|
+
recordedAt?: string;
|
|
375
|
+
agent?: AgentExecutionMetadata;
|
|
354
376
|
}
|
|
355
377
|
export interface AgentCapabilities {
|
|
356
378
|
mcp: boolean;
|
package/dist/types.d.ts
CHANGED
|
@@ -109,6 +109,9 @@ export interface LogEntry {
|
|
|
109
109
|
* carries a `costUsd` value; absent for agents that do not expose cost.
|
|
110
110
|
*/
|
|
111
111
|
cost_usd?: number;
|
|
112
|
+
cache_creation_input_tokens?: number;
|
|
113
|
+
cache_read_input_tokens?: number;
|
|
114
|
+
error_subtype?: AgentTurnResult['errorSubtype'];
|
|
112
115
|
completion_reason?: string;
|
|
113
116
|
completion_detail?: string;
|
|
114
117
|
turn_timings?: Array<{
|
|
@@ -174,6 +177,10 @@ export interface TrialResult {
|
|
|
174
177
|
session_log: LogEntry[];
|
|
175
178
|
skills_used?: string[];
|
|
176
179
|
diagnostics?: DiagnosticsReport;
|
|
180
|
+
result_kind?: import('./sdk/types.js').EvaluationResultKind;
|
|
181
|
+
scoring_duration_ms?: number;
|
|
182
|
+
recorded_at?: string;
|
|
183
|
+
agent?: import('./sdk/types.js').AgentExecutionMetadata;
|
|
177
184
|
conversation?: {
|
|
178
185
|
turns: ConversationTurn[];
|
|
179
186
|
total_turns: number;
|
|
@@ -213,6 +220,8 @@ export type PathgradeGroupReport = Omit<EvalReport, 'trials'> & {
|
|
|
213
220
|
* source of truth — new variants added there propagate here automatically).
|
|
214
221
|
*/
|
|
215
222
|
export interface PathgradeSelectionReport {
|
|
223
|
+
/** Binds this sidecar to the runner process spawned by `run --changed`. */
|
|
224
|
+
invocation_id?: string;
|
|
216
225
|
base_ref: string;
|
|
217
226
|
changed_files_count: number;
|
|
218
227
|
global_match?: string;
|
package/package.json
CHANGED
|
@@ -1,15 +1,8 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@wix/pathgrade",
|
|
3
|
-
"version": "1.0.
|
|
3
|
+
"version": "1.0.2",
|
|
4
4
|
"packageManager": "yarn@4.12.0",
|
|
5
5
|
"description": "Evaluate whether AI agents discover and use your skills correctly",
|
|
6
|
-
"repository": {
|
|
7
|
-
"type": "git",
|
|
8
|
-
"url": "git+https://github.com/wix-incubator/pathgrade.git"
|
|
9
|
-
},
|
|
10
|
-
"installConfig": {
|
|
11
|
-
"selfReferences": true
|
|
12
|
-
},
|
|
13
6
|
"exports": {
|
|
14
7
|
".": {
|
|
15
8
|
"types": "./dist/sdk/index.d.ts",
|
|
@@ -63,8 +56,7 @@
|
|
|
63
56
|
"!dist/**/*.d.ts.map",
|
|
64
57
|
"bin/",
|
|
65
58
|
"templates/",
|
|
66
|
-
"README.md"
|
|
67
|
-
"LICENSE"
|
|
59
|
+
"README.md"
|
|
68
60
|
],
|
|
69
61
|
"scripts": {
|
|
70
62
|
"lint": "oxlint src tests evals examples scripts --deny-warnings --disable-unicorn-plugin --disable-oxc-plugin --allow no-unused-vars --allow no-underscore-dangle --allow no-shadow --allow preserve-caught-error --allow no-unsafe-optional-chaining --allow no-unreachable --ignore-pattern tests/fixtures --ignore-pattern .worktrees",
|
|
@@ -95,11 +87,21 @@
|
|
|
95
87
|
"llm",
|
|
96
88
|
"testing"
|
|
97
89
|
],
|
|
90
|
+
"publishConfig": {
|
|
91
|
+
"registry": "https://registry.npmjs.org/",
|
|
92
|
+
"access": "public"
|
|
93
|
+
},
|
|
94
|
+
"wix": {
|
|
95
|
+
"artifact": {
|
|
96
|
+
"groupId": "com.wixpress",
|
|
97
|
+
"artifactId": "pathgrade"
|
|
98
|
+
}
|
|
99
|
+
},
|
|
98
100
|
"author": "Nadav Lachish",
|
|
99
101
|
"license": "MIT",
|
|
100
102
|
"type": "module",
|
|
101
103
|
"engines": {
|
|
102
|
-
"node": ">=20.
|
|
104
|
+
"node": ">=20.19.0"
|
|
103
105
|
},
|
|
104
106
|
"peerDependencies": {
|
|
105
107
|
"jest": "^30.0.0",
|
|
@@ -117,13 +119,13 @@
|
|
|
117
119
|
"@types/fs-extra": "^11.0.4",
|
|
118
120
|
"@types/jest": "^30.0.0",
|
|
119
121
|
"@types/picomatch": "^4.0.2",
|
|
120
|
-
"@vitest/coverage-v8": "4.1.
|
|
122
|
+
"@vitest/coverage-v8": "4.1.10",
|
|
121
123
|
"jest": "^30.0.0",
|
|
122
|
-
"vitest": "4.1.
|
|
124
|
+
"vitest": "4.1.10"
|
|
123
125
|
},
|
|
124
126
|
"dependencies": {
|
|
125
|
-
"@anthropic-ai/claude-agent-sdk": "0.2.
|
|
126
|
-
"@modelcontextprotocol/sdk": "1.
|
|
127
|
+
"@anthropic-ai/claude-agent-sdk": "0.2.141",
|
|
128
|
+
"@modelcontextprotocol/sdk": "1.30.0",
|
|
127
129
|
"@types/node": "25.6.0",
|
|
128
130
|
"fs-extra": "11.3.3",
|
|
129
131
|
"jiti": "2.6.1",
|
|
@@ -132,7 +134,5 @@
|
|
|
132
134
|
"typescript": "^5.9.3",
|
|
133
135
|
"zod": "4.3.6"
|
|
134
136
|
},
|
|
135
|
-
"
|
|
136
|
-
|
|
137
|
-
}
|
|
138
|
-
}
|
|
137
|
+
"falconPackageHash": "ccc242dcd7240e8c890225f5dd2f45be54f0f3b3426b862b95d847f1"
|
|
138
|
+
}
|
package/LICENSE
DELETED
|
@@ -1,24 +0,0 @@
|
|
|
1
|
-
The MIT License (MIT)
|
|
2
|
-
|
|
3
|
-
Copyright (c) 2025 Nadav Lachish
|
|
4
|
-
|
|
5
|
-
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
6
|
-
of this software and associated documentation files (the "Software"), to deal
|
|
7
|
-
in the Software without restriction, including without limitation the rights
|
|
8
|
-
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
9
|
-
copies of the Software, and to permit persons to whom the Software is
|
|
10
|
-
furnished to do so, subject to the following conditions:
|
|
11
|
-
|
|
12
|
-
The above copyright notice and this permission notice shall be included in all
|
|
13
|
-
copies or substantial portions of the Software.
|
|
14
|
-
|
|
15
|
-
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
16
|
-
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
17
|
-
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
18
|
-
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
19
|
-
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
20
|
-
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
21
|
-
SOFTWARE.
|
|
22
|
-
|
|
23
|
-
NOTICE: Pathgrade was originally forked from skillgrade by Minko Gechev,
|
|
24
|
-
which is MIT licensed.
|