@wix/pathgrade 0.36.0 → 1.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +24 -0
- package/README.md +508 -0
- package/dist/adapter-kit/index.d.ts +17 -0
- package/dist/adapter-kit/index.js +14 -0
- package/dist/adapters/jest/index.d.ts +3 -0
- package/dist/adapters/jest/index.js +3 -0
- package/dist/adapters/jest/invocation-adapter.d.ts +15 -0
- package/dist/adapters/jest/invocation-adapter.js +115 -0
- package/dist/adapters/jest/lifecycle.d.ts +21 -0
- package/dist/adapters/jest/lifecycle.js +62 -0
- package/dist/adapters/jest/metadata.d.ts +6 -0
- package/dist/adapters/jest/metadata.js +35 -0
- package/dist/adapters/jest/reporter.cjs +10 -0
- package/dist/adapters/jest/reporter.d.ts +5 -0
- package/dist/adapters/jest/reporter.js +67 -0
- package/dist/adapters/jest/results.d.ts +31 -0
- package/dist/adapters/jest/results.js +144 -0
- package/dist/adapters/jest/runner-adapter.d.ts +9 -0
- package/dist/adapters/jest/runner-adapter.js +49 -0
- package/dist/adapters/jest/setup.d.ts +1 -0
- package/dist/adapters/jest/setup.js +10 -0
- package/dist/adapters/node-test/index.d.ts +8 -0
- package/dist/adapters/node-test/index.js +106 -0
- package/dist/adapters/node-test/invocation-adapter.d.ts +5 -0
- package/dist/adapters/node-test/invocation-adapter.js +47 -0
- package/dist/adapters/node-test/runner-adapter.d.ts +2 -0
- package/dist/adapters/node-test/runner-adapter.js +116 -0
- package/dist/adapters/vitest/index.d.ts +9 -0
- package/dist/adapters/vitest/index.js +82 -0
- package/dist/adapters/vitest/lifecycle.d.ts +24 -0
- package/dist/adapters/vitest/lifecycle.js +41 -0
- package/dist/adapters/vitest/reporter.d.ts +12 -0
- package/dist/adapters/vitest/reporter.js +75 -0
- package/dist/adapters/vitest/setup.d.ts +1 -0
- package/dist/adapters/vitest/setup.js +3 -0
- package/dist/agents/codex-app-server/agent.js +5 -1
- package/dist/agents/codex-app-server/protocol/index.js +3 -3
- package/dist/agents/codex-app-server/turn-completion.d.ts +1 -0
- package/dist/agents/codex-app-server/turn-completion.js +9 -0
- package/dist/commands/affected.js +22 -15
- package/dist/commands/run-args.d.ts +4 -3
- package/dist/commands/run-args.js +12 -6
- package/dist/commands/run-changed.d.ts +4 -4
- package/dist/commands/run-changed.js +45 -58
- package/dist/config/pathgrade.d.ts +48 -0
- package/dist/config/pathgrade.js +204 -0
- package/dist/pathgrade.d.ts +2 -2
- package/dist/pathgrade.js +35 -34
- package/dist/plugin/index.d.ts +2 -9
- package/dist/plugin/index.js +1 -81
- package/dist/plugin/lifecycle.d.ts +1 -45
- package/dist/plugin/lifecycle.js +1 -146
- package/dist/plugin/reporter.d.ts +1 -37
- package/dist/plugin/reporter.js +1 -280
- package/dist/plugin/setup.d.ts +1 -1
- package/dist/plugin/setup.js +1 -3
- package/dist/providers/copy-filter.d.ts +5 -0
- package/dist/providers/copy-filter.js +9 -0
- package/dist/providers/sandbox.js +2 -2
- package/dist/providers/workspace.d.ts +6 -0
- package/dist/providers/workspace.js +34 -14
- package/dist/reporters/diagnostics.d.ts +2 -40
- package/dist/reporters/diagnostics.js +1 -80
- package/dist/reporters/report-summary.d.ts +6 -0
- package/dist/reporters/report-summary.js +29 -0
- package/dist/reporting/artifacts.d.ts +2 -0
- package/dist/reporting/artifacts.js +20 -0
- package/dist/reporting/core.d.ts +2 -0
- package/dist/reporting/core.js +173 -0
- package/dist/reporting/types.d.ts +57 -0
- package/dist/reporting/types.js +1 -0
- package/dist/reporting/vitest-edge.d.ts +1 -0
- package/dist/reporting/vitest-edge.js +1 -0
- package/dist/runners/adapter-loader.d.ts +17 -0
- package/dist/runners/adapter-loader.js +67 -0
- package/dist/runners/adapter.d.ts +65 -0
- package/dist/runners/adapter.js +1 -0
- package/dist/runners/invocation.d.ts +10 -0
- package/dist/runners/invocation.js +1 -0
- package/dist/runners/lifecycle-hooks.d.ts +2 -0
- package/dist/runners/lifecycle-hooks.js +18 -0
- package/dist/runners/model-builders.d.ts +4 -0
- package/dist/runners/model-builders.js +67 -0
- package/dist/runners/model-validation.d.ts +12 -0
- package/dist/runners/model-validation.js +213 -0
- package/dist/runners/model.d.ts +117 -0
- package/dist/runners/model.js +1 -0
- package/dist/runners/orchestrator.d.ts +30 -0
- package/dist/runners/orchestrator.js +74 -0
- package/dist/runners/report-projection.d.ts +6 -0
- package/dist/runners/report-projection.js +62 -0
- package/dist/runners/selection.d.ts +6 -0
- package/dist/runners/selection.js +10 -0
- package/dist/runners/vitest-adapter.d.ts +8 -0
- package/dist/runners/vitest-adapter.js +108 -0
- package/dist/runners/vitest-invocation.d.ts +10 -0
- package/dist/runners/vitest-invocation.js +44 -0
- package/dist/runners/vitest-lifecycle.d.ts +43 -0
- package/dist/runners/vitest-lifecycle.js +150 -0
- package/dist/sdk/agent.js +15 -16
- package/dist/sdk/case-context.d.ts +20 -0
- package/dist/sdk/case-context.js +31 -0
- package/dist/sdk/diagnostics.d.ts +40 -0
- package/dist/sdk/diagnostics.js +80 -0
- package/dist/sdk/eval-runtime.d.ts +4 -0
- package/dist/sdk/eval-runtime.js +11 -1
- package/dist/sdk/evaluate.js +2 -1
- package/dist/sdk/index.d.ts +2 -0
- package/dist/sdk/index.js +1 -0
- package/dist/sdk/lifecycle.d.ts +40 -0
- package/dist/sdk/lifecycle.js +170 -0
- package/dist/sdk/result-capture.d.ts +18 -0
- package/dist/sdk/result-capture.js +49 -0
- package/dist/sdk/types.d.ts +1 -1
- package/dist/types.d.ts +2 -2
- package/dist/utils/llm-providers/anthropic.js +9 -2
- package/package.json +73 -36
package/dist/plugin/index.js
CHANGED
|
@@ -1,81 +1 @@
|
|
|
1
|
-
|
|
2
|
-
import * as path from 'path';
|
|
3
|
-
import { configDefaults } from 'vitest/config';
|
|
4
|
-
import { PathgradeReporter } from './reporter.js';
|
|
5
|
-
import { discoverPathgradeEvalFiles } from '../evals/discovery.js';
|
|
6
|
-
const DEFAULT_EXCLUDE = [...configDefaults.exclude, '.worktrees/**', 'worktrees/**'];
|
|
7
|
-
function resolveSetupFile() {
|
|
8
|
-
// Prefer .ts (running from source via vitest) over .js (built dist)
|
|
9
|
-
const tsPath = path.resolve(import.meta.dirname, 'setup.ts');
|
|
10
|
-
if (fs.existsSync(tsPath))
|
|
11
|
-
return tsPath;
|
|
12
|
-
return path.resolve(import.meta.dirname, 'setup.js');
|
|
13
|
-
}
|
|
14
|
-
/**
|
|
15
|
-
* Vitest plugin for pathgrade evaluations.
|
|
16
|
-
*
|
|
17
|
-
* Configures test file patterns, timeout, auto-dispose lifecycle hooks,
|
|
18
|
-
* and the pathgrade reporter for aggregate statistics.
|
|
19
|
-
*/
|
|
20
|
-
export function pathgrade(opts) {
|
|
21
|
-
const timeoutSec = opts?.timeout ?? 300;
|
|
22
|
-
// Verbose live-streaming seam: the SDK (createAgent) reads
|
|
23
|
-
// `process.env.PATHGRADE_VERBOSE` to decide whether to emit. Wiring the
|
|
24
|
-
// plugin option through the env var keeps a single source of truth and
|
|
25
|
-
// preserves CLI/env precedence — we only set the env if the user hasn't
|
|
26
|
-
// already set it via their shell or the `pathgrade run --verbose` CLI.
|
|
27
|
-
if (opts?.verbose === true && process.env.PATHGRADE_VERBOSE === undefined) {
|
|
28
|
-
process.env.PATHGRADE_VERBOSE = '1';
|
|
29
|
-
}
|
|
30
|
-
return {
|
|
31
|
-
name: 'pathgrade',
|
|
32
|
-
// Surface the user's options to pre-vitest CLI commands
|
|
33
|
-
// (`pathgrade affected`, `pathgrade run --changed`) so they can
|
|
34
|
-
// read `affected.global` after loading vitest.config.ts.
|
|
35
|
-
// See `src/affected/config.ts`.
|
|
36
|
-
__pathgradeOptions: opts ?? {},
|
|
37
|
-
config() {
|
|
38
|
-
return {
|
|
39
|
-
resolve: {
|
|
40
|
-
alias: resolveLocalAliases(),
|
|
41
|
-
},
|
|
42
|
-
test: {
|
|
43
|
-
include: opts?.include ?? ['**/*.eval.ts'],
|
|
44
|
-
exclude: opts?.exclude ?? DEFAULT_EXCLUDE,
|
|
45
|
-
testTimeout: (timeoutSec + 30) * 1000,
|
|
46
|
-
setupFiles: [resolveSetupFile()],
|
|
47
|
-
reporters: [
|
|
48
|
-
'default',
|
|
49
|
-
new PathgradeReporter(opts),
|
|
50
|
-
],
|
|
51
|
-
},
|
|
52
|
-
};
|
|
53
|
-
},
|
|
54
|
-
configResolved(resolved) {
|
|
55
|
-
const root = resolved.root ?? process.cwd();
|
|
56
|
-
const include = resolved.test?.include ?? opts?.include ?? ['**/*.eval.ts'];
|
|
57
|
-
const exclude = resolved.test?.exclude ?? opts?.exclude ?? DEFAULT_EXCLUDE;
|
|
58
|
-
const discovered = discoverPathgradeEvalFiles({ cwd: root, include, exclude });
|
|
59
|
-
if (!resolved.test) {
|
|
60
|
-
resolved.test = {};
|
|
61
|
-
}
|
|
62
|
-
resolved.test.include = discovered;
|
|
63
|
-
},
|
|
64
|
-
};
|
|
65
|
-
}
|
|
66
|
-
/**
|
|
67
|
-
* When running from source (not installed via npm), resolve
|
|
68
|
-
* `@wix/pathgrade` imports to the local src/ directory so examples
|
|
69
|
-
* can use canonical package imports while developing locally.
|
|
70
|
-
*/
|
|
71
|
-
function resolveLocalAliases() {
|
|
72
|
-
const sdkSource = path.resolve(import.meta.dirname, '..', 'sdk', 'index.ts');
|
|
73
|
-
if (!fs.existsSync(sdkSource))
|
|
74
|
-
return undefined;
|
|
75
|
-
const src = path.resolve(import.meta.dirname, '..');
|
|
76
|
-
return {
|
|
77
|
-
'@wix/pathgrade/mcp-mock': path.join(src, 'core', 'mcp-mock.ts'),
|
|
78
|
-
'@wix/pathgrade/plugin': path.join(src, 'plugin', 'index.ts'),
|
|
79
|
-
'@wix/pathgrade': path.join(src, 'sdk', 'index.ts'),
|
|
80
|
-
};
|
|
81
|
-
}
|
|
1
|
+
export { pathgrade } from '../adapters/vitest/index.js';
|
|
@@ -1,45 +1 @@
|
|
|
1
|
-
|
|
2
|
-
declare module 'vitest' {
|
|
3
|
-
interface TaskMeta {
|
|
4
|
-
pathgrade?: PathgradeTestMeta[];
|
|
5
|
-
}
|
|
6
|
-
}
|
|
7
|
-
type AfterEachFn = (fn: (ctx: {
|
|
8
|
-
task: {
|
|
9
|
-
id: string;
|
|
10
|
-
meta: Record<string, unknown>;
|
|
11
|
-
};
|
|
12
|
-
}) => Promise<void>) => void;
|
|
13
|
-
type AfterAllFn = (fn: () => Promise<void>) => void;
|
|
14
|
-
declare function trackAgent(agent: Agent): void;
|
|
15
|
-
declare function untrackAgent(agent: Agent): void;
|
|
16
|
-
/**
|
|
17
|
-
* Flush results and dispose agents for the current test. Agents created in
|
|
18
|
-
* this test's scope are disposed after flushing. Shared agents (created in a
|
|
19
|
-
* different scope, e.g. module-level or beforeAll) have their pending results
|
|
20
|
-
* flushed but are kept alive for subsequent tests.
|
|
21
|
-
*/
|
|
22
|
-
declare function flush(task: {
|
|
23
|
-
id: string;
|
|
24
|
-
meta: Pick<import('vitest').TaskMeta, 'pathgrade'>;
|
|
25
|
-
}): Promise<void>;
|
|
26
|
-
declare function onResult(result: RecordedEvalResult, agent: Agent): void;
|
|
27
|
-
/**
|
|
28
|
-
* Dispose any agents still pending at end-of-file. Shared agents (created at
|
|
29
|
-
* module scope or in beforeAll) have a stored task id that won't match any
|
|
30
|
-
* per-test id in afterEach, so afterAll is their only disposal opportunity.
|
|
31
|
-
* Without this, sandboxes leak and `debug: true` folders never get written.
|
|
32
|
-
*/
|
|
33
|
-
declare function flushAll(): Promise<void>;
|
|
34
|
-
declare function reset(): void;
|
|
35
|
-
declare function install(afterEach: AfterEachFn, afterAll?: AfterAllFn): void;
|
|
36
|
-
export declare const lifecycle: {
|
|
37
|
-
trackAgent: typeof trackAgent;
|
|
38
|
-
untrackAgent: typeof untrackAgent;
|
|
39
|
-
flush: typeof flush;
|
|
40
|
-
flushAll: typeof flushAll;
|
|
41
|
-
onResult: typeof onResult;
|
|
42
|
-
reset: typeof reset;
|
|
43
|
-
install: typeof install;
|
|
44
|
-
};
|
|
45
|
-
export {};
|
|
1
|
+
export { lifecycle } from '../adapters/vitest/lifecycle.js';
|
package/dist/plugin/lifecycle.js
CHANGED
|
@@ -1,146 +1 @@
|
|
|
1
|
-
|
|
2
|
-
import { buildDiagnosticsReport } from '../reporters/diagnostics.js';
|
|
3
|
-
// Agent tracking: each test creates its own agent(s) and calls evaluate() on them.
|
|
4
|
-
// Results are keyed by agent reference, which is unique per test.
|
|
5
|
-
const pendingAgents = new Set();
|
|
6
|
-
const agentTaskIds = new WeakMap();
|
|
7
|
-
const agentResults = new WeakMap();
|
|
8
|
-
function currentTaskId() {
|
|
9
|
-
try {
|
|
10
|
-
return globalThis.__vitest_worker__?.current?.id ?? '';
|
|
11
|
-
}
|
|
12
|
-
catch {
|
|
13
|
-
return '';
|
|
14
|
-
}
|
|
15
|
-
}
|
|
16
|
-
function trackAgent(agent) {
|
|
17
|
-
pendingAgents.add(agent);
|
|
18
|
-
const taskId = currentTaskId();
|
|
19
|
-
if (taskId)
|
|
20
|
-
agentTaskIds.set(agent, taskId);
|
|
21
|
-
}
|
|
22
|
-
function untrackAgent(agent) {
|
|
23
|
-
pendingAgents.delete(agent);
|
|
24
|
-
}
|
|
25
|
-
/**
|
|
26
|
-
* Flush results and dispose agents for the current test. Agents created in
|
|
27
|
-
* this test's scope are disposed after flushing. Shared agents (created in a
|
|
28
|
-
* different scope, e.g. module-level or beforeAll) have their pending results
|
|
29
|
-
* flushed but are kept alive for subsequent tests.
|
|
30
|
-
*/
|
|
31
|
-
async function flush(task) {
|
|
32
|
-
const results = [];
|
|
33
|
-
const toDispose = [];
|
|
34
|
-
for (const agent of pendingAgents) {
|
|
35
|
-
const storedId = agentTaskIds.get(agent);
|
|
36
|
-
const belongsToThisTest = !storedId || storedId === task.id;
|
|
37
|
-
const meta = agentResults.get(agent);
|
|
38
|
-
if (meta && meta.length > 0) {
|
|
39
|
-
// Always flush pending results, even for shared agents
|
|
40
|
-
results.push(...meta);
|
|
41
|
-
agentResults.delete(agent);
|
|
42
|
-
}
|
|
43
|
-
if (belongsToThisTest) {
|
|
44
|
-
if (!meta || meta.length === 0) {
|
|
45
|
-
// evaluate() was never called — synthesize a trial from agent data
|
|
46
|
-
// so the reporter still surfaces token usage and command counts.
|
|
47
|
-
const synthTrial = synthesizeTrialFromAgent(agent);
|
|
48
|
-
if (synthTrial) {
|
|
49
|
-
results.push(synthTrial);
|
|
50
|
-
}
|
|
51
|
-
}
|
|
52
|
-
pendingAgents.delete(agent);
|
|
53
|
-
toDispose.push(agent);
|
|
54
|
-
}
|
|
55
|
-
}
|
|
56
|
-
if (results.length > 0) {
|
|
57
|
-
task.meta.pathgrade = results;
|
|
58
|
-
}
|
|
59
|
-
await Promise.all(toDispose.map((a) => a.dispose().catch(() => { })));
|
|
60
|
-
}
|
|
61
|
-
function onResult(result, agent) {
|
|
62
|
-
if (!agentResults.has(agent)) {
|
|
63
|
-
agentResults.set(agent, []);
|
|
64
|
-
}
|
|
65
|
-
const conversationEnd = [...agent.log].reverse().find((entry) => entry.type === 'conversation_end');
|
|
66
|
-
const completionReason = conversationEnd?.completion_reason ?? (agent.log.some((entry) => entry.type === 'agent_result') ? 'completed' : undefined);
|
|
67
|
-
agentResults.get(agent).push({
|
|
68
|
-
score: result.score,
|
|
69
|
-
scorers: result.scorers,
|
|
70
|
-
trial: result.trial,
|
|
71
|
-
diagnostics: buildDiagnosticsReport({
|
|
72
|
-
completionReason,
|
|
73
|
-
completionDetail: conversationEnd?.completion_detail,
|
|
74
|
-
turnDetails: conversationEnd?.turn_details,
|
|
75
|
-
reactionsFired: conversationEnd?.reactions_fired,
|
|
76
|
-
score: result.score,
|
|
77
|
-
scorers: result.scorers,
|
|
78
|
-
log: agent.log,
|
|
79
|
-
}),
|
|
80
|
-
});
|
|
81
|
-
}
|
|
82
|
-
/**
|
|
83
|
-
* When evaluate() is never called, synthesize a minimal trial from the agent's
|
|
84
|
-
* log so the reporter still surfaces command count and token usage.
|
|
85
|
-
*/
|
|
86
|
-
function synthesizeTrialFromAgent(agent) {
|
|
87
|
-
// Only synthesize if the agent actually ran (has log entries)
|
|
88
|
-
if (agent.log.length === 0)
|
|
89
|
-
return null;
|
|
90
|
-
const nCommands = agent.log.filter((e) => e.type === 'command').length;
|
|
91
|
-
const tokenUsage = agent.llm.tokenUsage;
|
|
92
|
-
const conversationEnd = [...agent.log].reverse().find((entry) => entry.type === 'conversation_end');
|
|
93
|
-
const completionReason = conversationEnd?.completion_reason ?? (agent.log.some((entry) => entry.type === 'agent_result') ? 'completed' : undefined);
|
|
94
|
-
return {
|
|
95
|
-
score: 1, // Will be overridden by vitest pass/fail in the reporter
|
|
96
|
-
scorers: [],
|
|
97
|
-
trial: {
|
|
98
|
-
trial_id: 0,
|
|
99
|
-
reward: 1,
|
|
100
|
-
scorer_results: [],
|
|
101
|
-
duration_ms: 0,
|
|
102
|
-
n_commands: nCommands,
|
|
103
|
-
input_tokens: tokenUsage?.inputTokens ?? 0,
|
|
104
|
-
output_tokens: tokenUsage?.outputTokens ?? 0,
|
|
105
|
-
session_log: [...agent.log],
|
|
106
|
-
},
|
|
107
|
-
diagnostics: buildDiagnosticsReport({
|
|
108
|
-
completionReason,
|
|
109
|
-
completionDetail: conversationEnd?.completion_detail,
|
|
110
|
-
turnDetails: conversationEnd?.turn_details,
|
|
111
|
-
reactionsFired: conversationEnd?.reactions_fired,
|
|
112
|
-
score: 1,
|
|
113
|
-
scorers: [],
|
|
114
|
-
log: agent.log,
|
|
115
|
-
}),
|
|
116
|
-
};
|
|
117
|
-
}
|
|
118
|
-
/**
|
|
119
|
-
* Dispose any agents still pending at end-of-file. Shared agents (created at
|
|
120
|
-
* module scope or in beforeAll) have a stored task id that won't match any
|
|
121
|
-
* per-test id in afterEach, so afterAll is their only disposal opportunity.
|
|
122
|
-
* Without this, sandboxes leak and `debug: true` folders never get written.
|
|
123
|
-
*/
|
|
124
|
-
async function flushAll() {
|
|
125
|
-
const toDispose = [...pendingAgents];
|
|
126
|
-
pendingAgents.clear();
|
|
127
|
-
await Promise.all(toDispose.map((a) => a.dispose().catch(() => { })));
|
|
128
|
-
}
|
|
129
|
-
function reset() {
|
|
130
|
-
pendingAgents.clear();
|
|
131
|
-
}
|
|
132
|
-
function install(afterEach, afterAll) {
|
|
133
|
-
setRuntime({ onResult });
|
|
134
|
-
afterEach(async ({ task }) => flush(task));
|
|
135
|
-
if (afterAll)
|
|
136
|
-
afterAll(async () => flushAll());
|
|
137
|
-
}
|
|
138
|
-
export const lifecycle = {
|
|
139
|
-
trackAgent,
|
|
140
|
-
untrackAgent,
|
|
141
|
-
flush,
|
|
142
|
-
flushAll,
|
|
143
|
-
onResult,
|
|
144
|
-
reset,
|
|
145
|
-
install,
|
|
146
|
-
};
|
|
1
|
+
export { lifecycle } from '../adapters/vitest/lifecycle.js';
|
|
@@ -1,37 +1 @@
|
|
|
1
|
-
|
|
2
|
-
import type { PathgradePluginOptions } from '../sdk/types.js';
|
|
3
|
-
/**
|
|
4
|
-
* Custom vitest reporter that layers pathgrade aggregate statistics
|
|
5
|
-
* on top of vitest's default output.
|
|
6
|
-
*/
|
|
7
|
-
export declare class PathgradeReporter implements Reporter {
|
|
8
|
-
private opts;
|
|
9
|
-
constructor(opts?: PathgradePluginOptions);
|
|
10
|
-
onTestRunEnd(testModules: ReadonlyArray<TestModule>): Promise<void>;
|
|
11
|
-
private collectGroups;
|
|
12
|
-
/**
|
|
13
|
-
* Group key is the parent suite's full name (file + describe hierarchy).
|
|
14
|
-
* Tests without a describe wrapper are grouped by module path.
|
|
15
|
-
*/
|
|
16
|
-
private getGroupKey;
|
|
17
|
-
private toTestEntry;
|
|
18
|
-
private isReportableEntry;
|
|
19
|
-
private printCliSummary;
|
|
20
|
-
private writeJsonResults;
|
|
21
|
-
private buildEvalReport;
|
|
22
|
-
private toTrialResult;
|
|
23
|
-
private slug;
|
|
24
|
-
private openBrowserViewer;
|
|
25
|
-
private computePassRate;
|
|
26
|
-
/**
|
|
27
|
-
* pass@k = 1 - (1 - p)^k
|
|
28
|
-
* Probability of at least 1 success in k trials.
|
|
29
|
-
*/
|
|
30
|
-
private computePassAtK;
|
|
31
|
-
/**
|
|
32
|
-
* pass^k = p^k
|
|
33
|
-
* Probability of all k trials succeeding.
|
|
34
|
-
*/
|
|
35
|
-
private computePassPowK;
|
|
36
|
-
private computeAvgDuration;
|
|
37
|
-
}
|
|
1
|
+
export { PathgradeReporter } from '../adapters/vitest/reporter.js';
|
package/dist/plugin/reporter.js
CHANGED
|
@@ -1,280 +1 @@
|
|
|
1
|
-
|
|
2
|
-
import { execSync } from 'child_process';
|
|
3
|
-
import { extractSkillsFromLog } from '../tool-events.js';
|
|
4
|
-
import { fmt } from '../utils/cli.js';
|
|
5
|
-
import { getPathgradeDir } from '../reporters/results-path.js';
|
|
6
|
-
import fs from 'fs-extra';
|
|
7
|
-
import { formatDiagnostics, formatDiagnosticsSummary } from '../reporters/diagnostics.js';
|
|
8
|
-
import { readSidecar } from '../affected/sidecar.js';
|
|
9
|
-
/**
|
|
10
|
-
* Custom vitest reporter that layers pathgrade aggregate statistics
|
|
11
|
-
* on top of vitest's default output.
|
|
12
|
-
*/
|
|
13
|
-
export class PathgradeReporter {
|
|
14
|
-
opts;
|
|
15
|
-
constructor(opts) {
|
|
16
|
-
this.opts = opts ?? {};
|
|
17
|
-
}
|
|
18
|
-
async onTestRunEnd(testModules) {
|
|
19
|
-
const groups = this.collectGroups(testModules);
|
|
20
|
-
if (groups.length === 0)
|
|
21
|
-
return;
|
|
22
|
-
const mode = this.opts.reporter ?? 'cli';
|
|
23
|
-
if (mode === 'cli' || mode === 'browser') {
|
|
24
|
-
this.printCliSummary(groups);
|
|
25
|
-
}
|
|
26
|
-
await this.writeJsonResults(groups);
|
|
27
|
-
if (mode === 'browser') {
|
|
28
|
-
this.openBrowserViewer(groups);
|
|
29
|
-
}
|
|
30
|
-
// CI threshold check
|
|
31
|
-
if (this.opts.ci?.threshold != null) {
|
|
32
|
-
const allScores = groups.flatMap((g) => g.tests.map((t) => t.score));
|
|
33
|
-
if (allScores.length > 0) {
|
|
34
|
-
const avg = allScores.reduce((a, b) => a + b, 0) / allScores.length;
|
|
35
|
-
if (avg < this.opts.ci.threshold) {
|
|
36
|
-
console.log(`\n ${fmt.fail('CI THRESHOLD FAILED')} avg score ${fmt.bold(avg.toFixed(3))} < threshold ${fmt.bold(String(this.opts.ci.threshold))}\n`);
|
|
37
|
-
process.exitCode = 1;
|
|
38
|
-
}
|
|
39
|
-
}
|
|
40
|
-
}
|
|
41
|
-
}
|
|
42
|
-
collectGroups(testModules) {
|
|
43
|
-
const groupMap = new Map();
|
|
44
|
-
for (const mod of testModules) {
|
|
45
|
-
for (const testCase of mod.children.allTests()) {
|
|
46
|
-
const groupKey = this.getGroupKey(testCase);
|
|
47
|
-
const entry = this.toTestEntry(testCase);
|
|
48
|
-
if (!this.isReportableEntry(entry))
|
|
49
|
-
continue;
|
|
50
|
-
if (!groupMap.has(groupKey)) {
|
|
51
|
-
groupMap.set(groupKey, []);
|
|
52
|
-
}
|
|
53
|
-
groupMap.get(groupKey).push(entry);
|
|
54
|
-
}
|
|
55
|
-
}
|
|
56
|
-
return Array.from(groupMap.entries()).map(([groupName, tests]) => ({
|
|
57
|
-
groupName,
|
|
58
|
-
tests,
|
|
59
|
-
}));
|
|
60
|
-
}
|
|
61
|
-
/**
|
|
62
|
-
* Group key is the parent suite's full name (file + describe hierarchy).
|
|
63
|
-
* Tests without a describe wrapper are grouped by module path.
|
|
64
|
-
*/
|
|
65
|
-
getGroupKey(testCase) {
|
|
66
|
-
const parent = testCase.parent;
|
|
67
|
-
const modulePath = testCase.module.relativeModuleId;
|
|
68
|
-
if (parent.type === 'suite') {
|
|
69
|
-
return `${modulePath} > ${parent.fullName}`;
|
|
70
|
-
}
|
|
71
|
-
return modulePath;
|
|
72
|
-
}
|
|
73
|
-
toTestEntry(testCase) {
|
|
74
|
-
const meta = testCase.meta();
|
|
75
|
-
const pathgradeMeta = meta.pathgrade;
|
|
76
|
-
const diag = testCase.diagnostic();
|
|
77
|
-
const result = testCase.result();
|
|
78
|
-
let score = 0;
|
|
79
|
-
let trial;
|
|
80
|
-
let diagnostics;
|
|
81
|
-
if (pathgradeMeta === undefined) {
|
|
82
|
-
// Test didn't use pathgrade — infer from test pass/fail
|
|
83
|
-
score = result.state === 'passed' ? 1 : 0;
|
|
84
|
-
}
|
|
85
|
-
else if (pathgradeMeta.length === 0) {
|
|
86
|
-
// Anomaly: pathgrade was set up but evaluate() was never called
|
|
87
|
-
console.warn(` [pathgrade] warning: empty results for "${testCase.name}" — evaluate() may not have been called`);
|
|
88
|
-
score = result.state === 'passed' ? 1 : 0;
|
|
89
|
-
}
|
|
90
|
-
else {
|
|
91
|
-
// Use the last eval result's score
|
|
92
|
-
score = pathgradeMeta[pathgradeMeta.length - 1].score;
|
|
93
|
-
trial = pathgradeMeta[pathgradeMeta.length - 1].trial;
|
|
94
|
-
diagnostics = pathgradeMeta[pathgradeMeta.length - 1].diagnostics;
|
|
95
|
-
}
|
|
96
|
-
return {
|
|
97
|
-
name: testCase.name,
|
|
98
|
-
score,
|
|
99
|
-
durationMs: diag?.duration ?? 0,
|
|
100
|
-
state: result.state === 'pending' ? 'pending' : result.state,
|
|
101
|
-
trial,
|
|
102
|
-
diagnostics,
|
|
103
|
-
};
|
|
104
|
-
}
|
|
105
|
-
isReportableEntry(entry) {
|
|
106
|
-
return entry.state !== 'skipped' && entry.state !== 'pending';
|
|
107
|
-
}
|
|
108
|
-
printCliSummary(groups) {
|
|
109
|
-
console.log(`\n${fmt.bold('── pathgrade summary ')}${fmt.dim('─'.repeat(40))}\n`);
|
|
110
|
-
const forceVerbose = this.opts.diagnostics === true || process.env.PATHGRADE_DIAGNOSTICS === '1';
|
|
111
|
-
const currentTimeoutMs = this.opts.timeout != null ? this.opts.timeout * 1000 : undefined;
|
|
112
|
-
for (const group of groups) {
|
|
113
|
-
const k = group.tests.length;
|
|
114
|
-
const passRate = this.computePassRate(group.tests);
|
|
115
|
-
const passAtK = this.computePassAtK(passRate, k);
|
|
116
|
-
const passPowK = this.computePassPowK(passRate, k);
|
|
117
|
-
const avgDuration = this.computeAvgDuration(group.tests);
|
|
118
|
-
const prColor = passRate >= 0.5 ? fmt.green : fmt.red;
|
|
119
|
-
console.log(` ${fmt.bold(group.groupName)}`);
|
|
120
|
-
console.log(` ${fmt.dim('pass rate'.padEnd(12))} ${prColor((passRate * 100).toFixed(1) + '%')}`);
|
|
121
|
-
console.log(` ${fmt.dim(`pass@${k}`.padEnd(12))} ${(passAtK * 100).toFixed(1)}%`);
|
|
122
|
-
console.log(` ${fmt.dim(`pass^${k}`.padEnd(12))} ${(passPowK * 100).toFixed(1)}%`);
|
|
123
|
-
console.log(` ${fmt.dim('avg time'.padEnd(12))} ${(avgDuration / 1000).toFixed(1)}s`);
|
|
124
|
-
console.log(` ${fmt.dim('trials'.padEnd(12))} ${k}`);
|
|
125
|
-
console.log();
|
|
126
|
-
for (const test of group.tests) {
|
|
127
|
-
if (!test.diagnostics)
|
|
128
|
-
continue;
|
|
129
|
-
const shouldPrintFull = forceVerbose || test.state !== 'passed' || test.diagnostics.completionReason === 'timeout' || test.diagnostics.completionReason === 'agent_crashed';
|
|
130
|
-
console.log(` ${fmt.bold(test.name)}`);
|
|
131
|
-
const formatted = shouldPrintFull
|
|
132
|
-
? formatDiagnostics(test.diagnostics, { verbose: true, currentTimeoutMs })
|
|
133
|
-
: formatDiagnosticsSummary(test.diagnostics);
|
|
134
|
-
for (const line of formatted.split('\n')) {
|
|
135
|
-
console.log(` ${line}`);
|
|
136
|
-
}
|
|
137
|
-
console.log();
|
|
138
|
-
}
|
|
139
|
-
}
|
|
140
|
-
}
|
|
141
|
-
async writeJsonResults(groups) {
|
|
142
|
-
const outputDir = getPathgradeDir(process.cwd());
|
|
143
|
-
const tracesDir = path.join(outputDir, 'traces');
|
|
144
|
-
await fs.ensureDir(tracesDir);
|
|
145
|
-
// Auto-create .gitignore if it doesn't exist
|
|
146
|
-
const gitignorePath = path.join(outputDir, '.gitignore');
|
|
147
|
-
if (!(await fs.pathExists(gitignorePath))) {
|
|
148
|
-
await fs.writeFile(gitignorePath, '*\n');
|
|
149
|
-
}
|
|
150
|
-
const allScores = groups.flatMap((g) => g.tests.map((t) => t.score));
|
|
151
|
-
const overallPassRate = allScores.length > 0
|
|
152
|
-
? allScores.reduce((a, b) => a + b, 0) / allScores.length
|
|
153
|
-
: 0;
|
|
154
|
-
const threshold = this.opts.ci?.threshold;
|
|
155
|
-
const status = threshold != null
|
|
156
|
-
? (overallPassRate >= threshold ? 'pass' : 'fail')
|
|
157
|
-
: (groups.every(g => g.tests.every(t => t.state === 'passed')) ? 'pass' : 'fail');
|
|
158
|
-
const consolidatedGroups = [];
|
|
159
|
-
for (const group of groups) {
|
|
160
|
-
const report = this.buildEvalReport(group);
|
|
161
|
-
const slug = this.slug(group.groupName);
|
|
162
|
-
const traceFile = `traces/${slug}.json`;
|
|
163
|
-
// Write full trace data (with session_log and conversation)
|
|
164
|
-
await fs.writeJson(path.join(outputDir, traceFile), report.trials, { spaces: 2 });
|
|
165
|
-
// Strip session_log and conversation from trials in consolidated report
|
|
166
|
-
const strippedTrials = report.trials.map(t => {
|
|
167
|
-
const { session_log, conversation, ...rest } = t;
|
|
168
|
-
return rest;
|
|
169
|
-
});
|
|
170
|
-
const { trials: _omitted, ...rest } = report;
|
|
171
|
-
consolidatedGroups.push({
|
|
172
|
-
...rest,
|
|
173
|
-
trials: strippedTrials,
|
|
174
|
-
trace_file: traceFile,
|
|
175
|
-
});
|
|
176
|
-
}
|
|
177
|
-
// Merge selection metadata from the CLI-written sidecar, if any.
|
|
178
|
-
// Missing sidecar → plain run; malformed → warn and proceed without it.
|
|
179
|
-
const selection = await readSidecar(process.cwd(), msg => {
|
|
180
|
-
console.warn(`[pathgrade] ${msg}`);
|
|
181
|
-
});
|
|
182
|
-
const consolidated = {
|
|
183
|
-
version: 1,
|
|
184
|
-
timestamp: new Date().toISOString(),
|
|
185
|
-
...(threshold != null ? { threshold } : {}),
|
|
186
|
-
overall_pass_rate: overallPassRate,
|
|
187
|
-
status,
|
|
188
|
-
groups: consolidatedGroups,
|
|
189
|
-
...(selection ? { selection } : {}),
|
|
190
|
-
};
|
|
191
|
-
await fs.writeJson(path.join(outputDir, 'results.json'), consolidated, { spaces: 2 });
|
|
192
|
-
console.log(`\n ${fmt.dim('Results written to')} ${outputDir}\n`);
|
|
193
|
-
}
|
|
194
|
-
buildEvalReport(group) {
|
|
195
|
-
const trials = group.tests.map((test, index) => this.toTrialResult(test, index));
|
|
196
|
-
const passRate = this.computePassRate(group.tests);
|
|
197
|
-
// Aggregate skills_used across all trials (deduplicated)
|
|
198
|
-
const allSkills = new Set();
|
|
199
|
-
for (const trial of trials) {
|
|
200
|
-
for (const skill of trial.skills_used ?? []) {
|
|
201
|
-
allSkills.add(skill);
|
|
202
|
-
}
|
|
203
|
-
}
|
|
204
|
-
return {
|
|
205
|
-
task: group.groupName,
|
|
206
|
-
pass_rate: passRate,
|
|
207
|
-
pass_at_k: this.computePassAtK(passRate, trials.length),
|
|
208
|
-
pass_pow_k: this.computePassPowK(passRate, trials.length),
|
|
209
|
-
trials,
|
|
210
|
-
skills_used: [...allSkills],
|
|
211
|
-
};
|
|
212
|
-
}
|
|
213
|
-
toTrialResult(test, index) {
|
|
214
|
-
const base = test.trial ?? {
|
|
215
|
-
trial_id: index + 1,
|
|
216
|
-
reward: test.score,
|
|
217
|
-
scorer_results: [],
|
|
218
|
-
duration_ms: test.durationMs,
|
|
219
|
-
n_commands: 0,
|
|
220
|
-
input_tokens: 0,
|
|
221
|
-
output_tokens: 0,
|
|
222
|
-
session_log: [],
|
|
223
|
-
};
|
|
224
|
-
// Extract skills_used from session_log tool events if not already present
|
|
225
|
-
const skills = base.skills_used ?? extractSkillsFromLog(base.session_log);
|
|
226
|
-
return {
|
|
227
|
-
...base,
|
|
228
|
-
trial_id: index + 1,
|
|
229
|
-
name: test.name,
|
|
230
|
-
duration_ms: base.duration_ms || test.durationMs,
|
|
231
|
-
diagnostics: test.diagnostics ?? base.diagnostics,
|
|
232
|
-
...(skills.length > 0 ? { skills_used: skills } : {}),
|
|
233
|
-
};
|
|
234
|
-
}
|
|
235
|
-
slug(value) {
|
|
236
|
-
return value
|
|
237
|
-
.toLowerCase()
|
|
238
|
-
.replace(/[^a-z0-9]+/g, '-')
|
|
239
|
-
.replace(/^-+|-+$/g, '') || 'report';
|
|
240
|
-
}
|
|
241
|
-
openBrowserViewer(_groups) {
|
|
242
|
-
// Write results JSON first, then open the viewer
|
|
243
|
-
const viewerPath = path.resolve(import.meta.dirname, '..', 'viewer.html');
|
|
244
|
-
try {
|
|
245
|
-
const openCmd = process.platform === 'darwin' ? 'open'
|
|
246
|
-
: process.platform === 'win32' ? 'start'
|
|
247
|
-
: 'xdg-open';
|
|
248
|
-
execSync(`${openCmd} "${viewerPath}"`, { stdio: 'ignore' });
|
|
249
|
-
console.log(`\n ${fmt.dim('Opened viewer in browser')}\n`);
|
|
250
|
-
}
|
|
251
|
-
catch {
|
|
252
|
-
console.log(`\n ${fmt.dim('Open manually:')} ${viewerPath}\n`);
|
|
253
|
-
}
|
|
254
|
-
}
|
|
255
|
-
computePassRate(tests) {
|
|
256
|
-
if (tests.length === 0)
|
|
257
|
-
return 0;
|
|
258
|
-
const passed = tests.filter((t) => t.state === 'passed').length;
|
|
259
|
-
return passed / tests.length;
|
|
260
|
-
}
|
|
261
|
-
/**
|
|
262
|
-
* pass@k = 1 - (1 - p)^k
|
|
263
|
-
* Probability of at least 1 success in k trials.
|
|
264
|
-
*/
|
|
265
|
-
computePassAtK(passRate, k) {
|
|
266
|
-
return 1 - Math.pow(1 - passRate, k);
|
|
267
|
-
}
|
|
268
|
-
/**
|
|
269
|
-
* pass^k = p^k
|
|
270
|
-
* Probability of all k trials succeeding.
|
|
271
|
-
*/
|
|
272
|
-
computePassPowK(passRate, k) {
|
|
273
|
-
return Math.pow(passRate, k);
|
|
274
|
-
}
|
|
275
|
-
computeAvgDuration(tests) {
|
|
276
|
-
if (tests.length === 0)
|
|
277
|
-
return 0;
|
|
278
|
-
return tests.reduce((sum, t) => sum + t.durationMs, 0) / tests.length;
|
|
279
|
-
}
|
|
280
|
-
}
|
|
1
|
+
export { PathgradeReporter } from '../adapters/vitest/reporter.js';
|
package/dist/plugin/setup.d.ts
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
|
|
1
|
+
import '../adapters/vitest/setup.js';
|
package/dist/plugin/setup.js
CHANGED
|
@@ -16,3 +16,8 @@ export declare const DEFAULT_COPY_IGNORE: readonly string[];
|
|
|
16
16
|
* Pass `[]` to disable filtering. Invalid glob patterns are silently ignored.
|
|
17
17
|
*/
|
|
18
18
|
export declare function createCopyFilter(ignorePatterns: readonly string[]): (src: string) => boolean;
|
|
19
|
+
/**
|
|
20
|
+
* fs-extra cannot copy sockets, FIFOs, or device files. Host config caches can
|
|
21
|
+
* contain transient sockets, so filter them out before copy() reaches them.
|
|
22
|
+
*/
|
|
23
|
+
export declare function isPortableCopyEntry(src: string): boolean;
|
|
@@ -5,6 +5,7 @@
|
|
|
5
5
|
* when copying workspace/skill directories into the sandbox.
|
|
6
6
|
*/
|
|
7
7
|
import * as path from 'path';
|
|
8
|
+
import * as fs from 'fs';
|
|
8
9
|
import picomatch from 'picomatch';
|
|
9
10
|
/**
|
|
10
11
|
* Default ignore patterns applied when copying workspace and skill directories.
|
|
@@ -64,3 +65,11 @@ export function createCopyFilter(ignorePatterns) {
|
|
|
64
65
|
return true;
|
|
65
66
|
};
|
|
66
67
|
}
|
|
68
|
+
/**
|
|
69
|
+
* fs-extra cannot copy sockets, FIFOs, or device files. Host config caches can
|
|
70
|
+
* contain transient sockets, so filter them out before copy() reaches them.
|
|
71
|
+
*/
|
|
72
|
+
export function isPortableCopyEntry(src) {
|
|
73
|
+
const stat = fs.lstatSync(src);
|
|
74
|
+
return stat.isFile() || stat.isDirectory() || stat.isSymbolicLink();
|
|
75
|
+
}
|
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
import fs from 'fs-extra';
|
|
2
2
|
import * as path from 'path';
|
|
3
3
|
import * as os from 'os';
|
|
4
|
-
import { DEFAULT_COPY_IGNORE, createCopyFilter } from './copy-filter.js';
|
|
4
|
+
import { DEFAULT_COPY_IGNORE, createCopyFilter, isPortableCopyEntry } from './copy-filter.js';
|
|
5
5
|
export const SAFE_HOST_VARS = [
|
|
6
6
|
'PATH', 'SHELL', 'LANG', 'LC_ALL', 'LC_CTYPE', 'TERM', 'USER', 'LOGNAME',
|
|
7
7
|
];
|
|
@@ -56,7 +56,7 @@ export async function createSandbox(spec) {
|
|
|
56
56
|
if (await fs.pathExists(srcPath)) {
|
|
57
57
|
const destPath = path.join(homePath, relPath);
|
|
58
58
|
await fs.ensureDir(path.dirname(destPath));
|
|
59
|
-
await fs.copy(srcPath, destPath);
|
|
59
|
+
await fs.copy(srcPath, destPath, { filter: isPortableCopyEntry });
|
|
60
60
|
}
|
|
61
61
|
}
|
|
62
62
|
}
|
|
@@ -11,5 +11,11 @@ export interface Workspace {
|
|
|
11
11
|
}): Promise<CommandResult>;
|
|
12
12
|
dispose(): Promise<void>;
|
|
13
13
|
}
|
|
14
|
+
interface RemoveSandboxRootOptions {
|
|
15
|
+
remove?: (target: string) => Promise<void>;
|
|
16
|
+
sleep?: (ms: number) => Promise<void>;
|
|
17
|
+
retryDelaysMs?: readonly number[];
|
|
18
|
+
}
|
|
19
|
+
export declare function removeSandboxRoot(rootDir: string, opts?: RemoveSandboxRootOptions): Promise<void>;
|
|
14
20
|
export declare function linkPathsFromHostHome(pathsToLink: string[], sandboxHomePath: string): Promise<void>;
|
|
15
21
|
export declare function prepareWorkspace(spec: SandboxConfig): Promise<Workspace>;
|