@wix/pathgrade 0.36.0 → 1.0.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (117) hide show
  1. package/LICENSE +24 -0
  2. package/README.md +508 -0
  3. package/dist/adapter-kit/index.d.ts +17 -0
  4. package/dist/adapter-kit/index.js +14 -0
  5. package/dist/adapters/jest/index.d.ts +3 -0
  6. package/dist/adapters/jest/index.js +3 -0
  7. package/dist/adapters/jest/invocation-adapter.d.ts +15 -0
  8. package/dist/adapters/jest/invocation-adapter.js +115 -0
  9. package/dist/adapters/jest/lifecycle.d.ts +21 -0
  10. package/dist/adapters/jest/lifecycle.js +62 -0
  11. package/dist/adapters/jest/metadata.d.ts +6 -0
  12. package/dist/adapters/jest/metadata.js +35 -0
  13. package/dist/adapters/jest/reporter.cjs +10 -0
  14. package/dist/adapters/jest/reporter.d.ts +5 -0
  15. package/dist/adapters/jest/reporter.js +67 -0
  16. package/dist/adapters/jest/results.d.ts +31 -0
  17. package/dist/adapters/jest/results.js +144 -0
  18. package/dist/adapters/jest/runner-adapter.d.ts +9 -0
  19. package/dist/adapters/jest/runner-adapter.js +49 -0
  20. package/dist/adapters/jest/setup.d.ts +1 -0
  21. package/dist/adapters/jest/setup.js +10 -0
  22. package/dist/adapters/node-test/index.d.ts +8 -0
  23. package/dist/adapters/node-test/index.js +106 -0
  24. package/dist/adapters/node-test/invocation-adapter.d.ts +5 -0
  25. package/dist/adapters/node-test/invocation-adapter.js +47 -0
  26. package/dist/adapters/node-test/runner-adapter.d.ts +2 -0
  27. package/dist/adapters/node-test/runner-adapter.js +116 -0
  28. package/dist/adapters/vitest/index.d.ts +9 -0
  29. package/dist/adapters/vitest/index.js +82 -0
  30. package/dist/adapters/vitest/lifecycle.d.ts +24 -0
  31. package/dist/adapters/vitest/lifecycle.js +41 -0
  32. package/dist/adapters/vitest/reporter.d.ts +12 -0
  33. package/dist/adapters/vitest/reporter.js +75 -0
  34. package/dist/adapters/vitest/setup.d.ts +1 -0
  35. package/dist/adapters/vitest/setup.js +3 -0
  36. package/dist/agents/codex-app-server/agent.js +5 -1
  37. package/dist/agents/codex-app-server/protocol/index.js +3 -3
  38. package/dist/agents/codex-app-server/turn-completion.d.ts +1 -0
  39. package/dist/agents/codex-app-server/turn-completion.js +9 -0
  40. package/dist/commands/affected.js +22 -15
  41. package/dist/commands/run-args.d.ts +4 -3
  42. package/dist/commands/run-args.js +12 -6
  43. package/dist/commands/run-changed.d.ts +4 -4
  44. package/dist/commands/run-changed.js +45 -58
  45. package/dist/config/pathgrade.d.ts +48 -0
  46. package/dist/config/pathgrade.js +204 -0
  47. package/dist/pathgrade.d.ts +2 -2
  48. package/dist/pathgrade.js +35 -34
  49. package/dist/plugin/index.d.ts +2 -9
  50. package/dist/plugin/index.js +1 -81
  51. package/dist/plugin/lifecycle.d.ts +1 -45
  52. package/dist/plugin/lifecycle.js +1 -146
  53. package/dist/plugin/reporter.d.ts +1 -37
  54. package/dist/plugin/reporter.js +1 -280
  55. package/dist/plugin/setup.d.ts +1 -1
  56. package/dist/plugin/setup.js +1 -3
  57. package/dist/providers/copy-filter.d.ts +5 -0
  58. package/dist/providers/copy-filter.js +9 -0
  59. package/dist/providers/sandbox.js +2 -2
  60. package/dist/providers/workspace.d.ts +6 -0
  61. package/dist/providers/workspace.js +34 -14
  62. package/dist/reporters/diagnostics.d.ts +2 -40
  63. package/dist/reporters/diagnostics.js +1 -80
  64. package/dist/reporters/report-summary.d.ts +6 -0
  65. package/dist/reporters/report-summary.js +29 -0
  66. package/dist/reporting/artifacts.d.ts +2 -0
  67. package/dist/reporting/artifacts.js +20 -0
  68. package/dist/reporting/core.d.ts +2 -0
  69. package/dist/reporting/core.js +173 -0
  70. package/dist/reporting/types.d.ts +57 -0
  71. package/dist/reporting/types.js +1 -0
  72. package/dist/reporting/vitest-edge.d.ts +1 -0
  73. package/dist/reporting/vitest-edge.js +1 -0
  74. package/dist/runners/adapter-loader.d.ts +17 -0
  75. package/dist/runners/adapter-loader.js +67 -0
  76. package/dist/runners/adapter.d.ts +65 -0
  77. package/dist/runners/adapter.js +1 -0
  78. package/dist/runners/invocation.d.ts +10 -0
  79. package/dist/runners/invocation.js +1 -0
  80. package/dist/runners/lifecycle-hooks.d.ts +2 -0
  81. package/dist/runners/lifecycle-hooks.js +18 -0
  82. package/dist/runners/model-builders.d.ts +4 -0
  83. package/dist/runners/model-builders.js +67 -0
  84. package/dist/runners/model-validation.d.ts +12 -0
  85. package/dist/runners/model-validation.js +213 -0
  86. package/dist/runners/model.d.ts +117 -0
  87. package/dist/runners/model.js +1 -0
  88. package/dist/runners/orchestrator.d.ts +30 -0
  89. package/dist/runners/orchestrator.js +74 -0
  90. package/dist/runners/report-projection.d.ts +6 -0
  91. package/dist/runners/report-projection.js +62 -0
  92. package/dist/runners/selection.d.ts +6 -0
  93. package/dist/runners/selection.js +10 -0
  94. package/dist/runners/vitest-adapter.d.ts +8 -0
  95. package/dist/runners/vitest-adapter.js +108 -0
  96. package/dist/runners/vitest-invocation.d.ts +10 -0
  97. package/dist/runners/vitest-invocation.js +44 -0
  98. package/dist/runners/vitest-lifecycle.d.ts +43 -0
  99. package/dist/runners/vitest-lifecycle.js +150 -0
  100. package/dist/sdk/agent.js +15 -16
  101. package/dist/sdk/case-context.d.ts +20 -0
  102. package/dist/sdk/case-context.js +31 -0
  103. package/dist/sdk/diagnostics.d.ts +40 -0
  104. package/dist/sdk/diagnostics.js +80 -0
  105. package/dist/sdk/eval-runtime.d.ts +4 -0
  106. package/dist/sdk/eval-runtime.js +11 -1
  107. package/dist/sdk/evaluate.js +2 -1
  108. package/dist/sdk/index.d.ts +2 -0
  109. package/dist/sdk/index.js +1 -0
  110. package/dist/sdk/lifecycle.d.ts +40 -0
  111. package/dist/sdk/lifecycle.js +170 -0
  112. package/dist/sdk/result-capture.d.ts +18 -0
  113. package/dist/sdk/result-capture.js +49 -0
  114. package/dist/sdk/types.d.ts +1 -1
  115. package/dist/types.d.ts +2 -2
  116. package/dist/utils/llm-providers/anthropic.js +9 -2
  117. package/package.json +73 -36
@@ -1,81 +1 @@
1
- import * as fs from 'fs';
2
- import * as path from 'path';
3
- import { configDefaults } from 'vitest/config';
4
- import { PathgradeReporter } from './reporter.js';
5
- import { discoverPathgradeEvalFiles } from '../evals/discovery.js';
6
- const DEFAULT_EXCLUDE = [...configDefaults.exclude, '.worktrees/**', 'worktrees/**'];
7
- function resolveSetupFile() {
8
- // Prefer .ts (running from source via vitest) over .js (built dist)
9
- const tsPath = path.resolve(import.meta.dirname, 'setup.ts');
10
- if (fs.existsSync(tsPath))
11
- return tsPath;
12
- return path.resolve(import.meta.dirname, 'setup.js');
13
- }
14
- /**
15
- * Vitest plugin for pathgrade evaluations.
16
- *
17
- * Configures test file patterns, timeout, auto-dispose lifecycle hooks,
18
- * and the pathgrade reporter for aggregate statistics.
19
- */
20
- export function pathgrade(opts) {
21
- const timeoutSec = opts?.timeout ?? 300;
22
- // Verbose live-streaming seam: the SDK (createAgent) reads
23
- // `process.env.PATHGRADE_VERBOSE` to decide whether to emit. Wiring the
24
- // plugin option through the env var keeps a single source of truth and
25
- // preserves CLI/env precedence — we only set the env if the user hasn't
26
- // already set it via their shell or the `pathgrade run --verbose` CLI.
27
- if (opts?.verbose === true && process.env.PATHGRADE_VERBOSE === undefined) {
28
- process.env.PATHGRADE_VERBOSE = '1';
29
- }
30
- return {
31
- name: 'pathgrade',
32
- // Surface the user's options to pre-vitest CLI commands
33
- // (`pathgrade affected`, `pathgrade run --changed`) so they can
34
- // read `affected.global` after loading vitest.config.ts.
35
- // See `src/affected/config.ts`.
36
- __pathgradeOptions: opts ?? {},
37
- config() {
38
- return {
39
- resolve: {
40
- alias: resolveLocalAliases(),
41
- },
42
- test: {
43
- include: opts?.include ?? ['**/*.eval.ts'],
44
- exclude: opts?.exclude ?? DEFAULT_EXCLUDE,
45
- testTimeout: (timeoutSec + 30) * 1000,
46
- setupFiles: [resolveSetupFile()],
47
- reporters: [
48
- 'default',
49
- new PathgradeReporter(opts),
50
- ],
51
- },
52
- };
53
- },
54
- configResolved(resolved) {
55
- const root = resolved.root ?? process.cwd();
56
- const include = resolved.test?.include ?? opts?.include ?? ['**/*.eval.ts'];
57
- const exclude = resolved.test?.exclude ?? opts?.exclude ?? DEFAULT_EXCLUDE;
58
- const discovered = discoverPathgradeEvalFiles({ cwd: root, include, exclude });
59
- if (!resolved.test) {
60
- resolved.test = {};
61
- }
62
- resolved.test.include = discovered;
63
- },
64
- };
65
- }
66
- /**
67
- * When running from source (not installed via npm), resolve
68
- * `@wix/pathgrade` imports to the local src/ directory so examples
69
- * can use canonical package imports while developing locally.
70
- */
71
- function resolveLocalAliases() {
72
- const sdkSource = path.resolve(import.meta.dirname, '..', 'sdk', 'index.ts');
73
- if (!fs.existsSync(sdkSource))
74
- return undefined;
75
- const src = path.resolve(import.meta.dirname, '..');
76
- return {
77
- '@wix/pathgrade/mcp-mock': path.join(src, 'core', 'mcp-mock.ts'),
78
- '@wix/pathgrade/plugin': path.join(src, 'plugin', 'index.ts'),
79
- '@wix/pathgrade': path.join(src, 'sdk', 'index.ts'),
80
- };
81
- }
1
+ export { pathgrade } from '../adapters/vitest/index.js';
@@ -1,45 +1 @@
1
- import type { Agent, PathgradeTestMeta, RecordedEvalResult } from '../sdk/types.js';
2
- declare module 'vitest' {
3
- interface TaskMeta {
4
- pathgrade?: PathgradeTestMeta[];
5
- }
6
- }
7
- type AfterEachFn = (fn: (ctx: {
8
- task: {
9
- id: string;
10
- meta: Record<string, unknown>;
11
- };
12
- }) => Promise<void>) => void;
13
- type AfterAllFn = (fn: () => Promise<void>) => void;
14
- declare function trackAgent(agent: Agent): void;
15
- declare function untrackAgent(agent: Agent): void;
16
- /**
17
- * Flush results and dispose agents for the current test. Agents created in
18
- * this test's scope are disposed after flushing. Shared agents (created in a
19
- * different scope, e.g. module-level or beforeAll) have their pending results
20
- * flushed but are kept alive for subsequent tests.
21
- */
22
- declare function flush(task: {
23
- id: string;
24
- meta: Pick<import('vitest').TaskMeta, 'pathgrade'>;
25
- }): Promise<void>;
26
- declare function onResult(result: RecordedEvalResult, agent: Agent): void;
27
- /**
28
- * Dispose any agents still pending at end-of-file. Shared agents (created at
29
- * module scope or in beforeAll) have a stored task id that won't match any
30
- * per-test id in afterEach, so afterAll is their only disposal opportunity.
31
- * Without this, sandboxes leak and `debug: true` folders never get written.
32
- */
33
- declare function flushAll(): Promise<void>;
34
- declare function reset(): void;
35
- declare function install(afterEach: AfterEachFn, afterAll?: AfterAllFn): void;
36
- export declare const lifecycle: {
37
- trackAgent: typeof trackAgent;
38
- untrackAgent: typeof untrackAgent;
39
- flush: typeof flush;
40
- flushAll: typeof flushAll;
41
- onResult: typeof onResult;
42
- reset: typeof reset;
43
- install: typeof install;
44
- };
45
- export {};
1
+ export { lifecycle } from '../adapters/vitest/lifecycle.js';
@@ -1,146 +1 @@
1
- import { setRuntime } from '../sdk/eval-runtime.js';
2
- import { buildDiagnosticsReport } from '../reporters/diagnostics.js';
3
- // Agent tracking: each test creates its own agent(s) and calls evaluate() on them.
4
- // Results are keyed by agent reference, which is unique per test.
5
- const pendingAgents = new Set();
6
- const agentTaskIds = new WeakMap();
7
- const agentResults = new WeakMap();
8
- function currentTaskId() {
9
- try {
10
- return globalThis.__vitest_worker__?.current?.id ?? '';
11
- }
12
- catch {
13
- return '';
14
- }
15
- }
16
- function trackAgent(agent) {
17
- pendingAgents.add(agent);
18
- const taskId = currentTaskId();
19
- if (taskId)
20
- agentTaskIds.set(agent, taskId);
21
- }
22
- function untrackAgent(agent) {
23
- pendingAgents.delete(agent);
24
- }
25
- /**
26
- * Flush results and dispose agents for the current test. Agents created in
27
- * this test's scope are disposed after flushing. Shared agents (created in a
28
- * different scope, e.g. module-level or beforeAll) have their pending results
29
- * flushed but are kept alive for subsequent tests.
30
- */
31
- async function flush(task) {
32
- const results = [];
33
- const toDispose = [];
34
- for (const agent of pendingAgents) {
35
- const storedId = agentTaskIds.get(agent);
36
- const belongsToThisTest = !storedId || storedId === task.id;
37
- const meta = agentResults.get(agent);
38
- if (meta && meta.length > 0) {
39
- // Always flush pending results, even for shared agents
40
- results.push(...meta);
41
- agentResults.delete(agent);
42
- }
43
- if (belongsToThisTest) {
44
- if (!meta || meta.length === 0) {
45
- // evaluate() was never called — synthesize a trial from agent data
46
- // so the reporter still surfaces token usage and command counts.
47
- const synthTrial = synthesizeTrialFromAgent(agent);
48
- if (synthTrial) {
49
- results.push(synthTrial);
50
- }
51
- }
52
- pendingAgents.delete(agent);
53
- toDispose.push(agent);
54
- }
55
- }
56
- if (results.length > 0) {
57
- task.meta.pathgrade = results;
58
- }
59
- await Promise.all(toDispose.map((a) => a.dispose().catch(() => { })));
60
- }
61
- function onResult(result, agent) {
62
- if (!agentResults.has(agent)) {
63
- agentResults.set(agent, []);
64
- }
65
- const conversationEnd = [...agent.log].reverse().find((entry) => entry.type === 'conversation_end');
66
- const completionReason = conversationEnd?.completion_reason ?? (agent.log.some((entry) => entry.type === 'agent_result') ? 'completed' : undefined);
67
- agentResults.get(agent).push({
68
- score: result.score,
69
- scorers: result.scorers,
70
- trial: result.trial,
71
- diagnostics: buildDiagnosticsReport({
72
- completionReason,
73
- completionDetail: conversationEnd?.completion_detail,
74
- turnDetails: conversationEnd?.turn_details,
75
- reactionsFired: conversationEnd?.reactions_fired,
76
- score: result.score,
77
- scorers: result.scorers,
78
- log: agent.log,
79
- }),
80
- });
81
- }
82
- /**
83
- * When evaluate() is never called, synthesize a minimal trial from the agent's
84
- * log so the reporter still surfaces command count and token usage.
85
- */
86
- function synthesizeTrialFromAgent(agent) {
87
- // Only synthesize if the agent actually ran (has log entries)
88
- if (agent.log.length === 0)
89
- return null;
90
- const nCommands = agent.log.filter((e) => e.type === 'command').length;
91
- const tokenUsage = agent.llm.tokenUsage;
92
- const conversationEnd = [...agent.log].reverse().find((entry) => entry.type === 'conversation_end');
93
- const completionReason = conversationEnd?.completion_reason ?? (agent.log.some((entry) => entry.type === 'agent_result') ? 'completed' : undefined);
94
- return {
95
- score: 1, // Will be overridden by vitest pass/fail in the reporter
96
- scorers: [],
97
- trial: {
98
- trial_id: 0,
99
- reward: 1,
100
- scorer_results: [],
101
- duration_ms: 0,
102
- n_commands: nCommands,
103
- input_tokens: tokenUsage?.inputTokens ?? 0,
104
- output_tokens: tokenUsage?.outputTokens ?? 0,
105
- session_log: [...agent.log],
106
- },
107
- diagnostics: buildDiagnosticsReport({
108
- completionReason,
109
- completionDetail: conversationEnd?.completion_detail,
110
- turnDetails: conversationEnd?.turn_details,
111
- reactionsFired: conversationEnd?.reactions_fired,
112
- score: 1,
113
- scorers: [],
114
- log: agent.log,
115
- }),
116
- };
117
- }
118
- /**
119
- * Dispose any agents still pending at end-of-file. Shared agents (created at
120
- * module scope or in beforeAll) have a stored task id that won't match any
121
- * per-test id in afterEach, so afterAll is their only disposal opportunity.
122
- * Without this, sandboxes leak and `debug: true` folders never get written.
123
- */
124
- async function flushAll() {
125
- const toDispose = [...pendingAgents];
126
- pendingAgents.clear();
127
- await Promise.all(toDispose.map((a) => a.dispose().catch(() => { })));
128
- }
129
- function reset() {
130
- pendingAgents.clear();
131
- }
132
- function install(afterEach, afterAll) {
133
- setRuntime({ onResult });
134
- afterEach(async ({ task }) => flush(task));
135
- if (afterAll)
136
- afterAll(async () => flushAll());
137
- }
138
- export const lifecycle = {
139
- trackAgent,
140
- untrackAgent,
141
- flush,
142
- flushAll,
143
- onResult,
144
- reset,
145
- install,
146
- };
1
+ export { lifecycle } from '../adapters/vitest/lifecycle.js';
@@ -1,37 +1 @@
1
- import type { Reporter, TestModule } from 'vitest/node';
2
- import type { PathgradePluginOptions } from '../sdk/types.js';
3
- /**
4
- * Custom vitest reporter that layers pathgrade aggregate statistics
5
- * on top of vitest's default output.
6
- */
7
- export declare class PathgradeReporter implements Reporter {
8
- private opts;
9
- constructor(opts?: PathgradePluginOptions);
10
- onTestRunEnd(testModules: ReadonlyArray<TestModule>): Promise<void>;
11
- private collectGroups;
12
- /**
13
- * Group key is the parent suite's full name (file + describe hierarchy).
14
- * Tests without a describe wrapper are grouped by module path.
15
- */
16
- private getGroupKey;
17
- private toTestEntry;
18
- private isReportableEntry;
19
- private printCliSummary;
20
- private writeJsonResults;
21
- private buildEvalReport;
22
- private toTrialResult;
23
- private slug;
24
- private openBrowserViewer;
25
- private computePassRate;
26
- /**
27
- * pass@k = 1 - (1 - p)^k
28
- * Probability of at least 1 success in k trials.
29
- */
30
- private computePassAtK;
31
- /**
32
- * pass^k = p^k
33
- * Probability of all k trials succeeding.
34
- */
35
- private computePassPowK;
36
- private computeAvgDuration;
37
- }
1
+ export { PathgradeReporter } from '../adapters/vitest/reporter.js';
@@ -1,280 +1 @@
1
- import * as path from 'path';
2
- import { execSync } from 'child_process';
3
- import { extractSkillsFromLog } from '../tool-events.js';
4
- import { fmt } from '../utils/cli.js';
5
- import { getPathgradeDir } from '../reporters/results-path.js';
6
- import fs from 'fs-extra';
7
- import { formatDiagnostics, formatDiagnosticsSummary } from '../reporters/diagnostics.js';
8
- import { readSidecar } from '../affected/sidecar.js';
9
- /**
10
- * Custom vitest reporter that layers pathgrade aggregate statistics
11
- * on top of vitest's default output.
12
- */
13
- export class PathgradeReporter {
14
- opts;
15
- constructor(opts) {
16
- this.opts = opts ?? {};
17
- }
18
- async onTestRunEnd(testModules) {
19
- const groups = this.collectGroups(testModules);
20
- if (groups.length === 0)
21
- return;
22
- const mode = this.opts.reporter ?? 'cli';
23
- if (mode === 'cli' || mode === 'browser') {
24
- this.printCliSummary(groups);
25
- }
26
- await this.writeJsonResults(groups);
27
- if (mode === 'browser') {
28
- this.openBrowserViewer(groups);
29
- }
30
- // CI threshold check
31
- if (this.opts.ci?.threshold != null) {
32
- const allScores = groups.flatMap((g) => g.tests.map((t) => t.score));
33
- if (allScores.length > 0) {
34
- const avg = allScores.reduce((a, b) => a + b, 0) / allScores.length;
35
- if (avg < this.opts.ci.threshold) {
36
- console.log(`\n ${fmt.fail('CI THRESHOLD FAILED')} avg score ${fmt.bold(avg.toFixed(3))} < threshold ${fmt.bold(String(this.opts.ci.threshold))}\n`);
37
- process.exitCode = 1;
38
- }
39
- }
40
- }
41
- }
42
- collectGroups(testModules) {
43
- const groupMap = new Map();
44
- for (const mod of testModules) {
45
- for (const testCase of mod.children.allTests()) {
46
- const groupKey = this.getGroupKey(testCase);
47
- const entry = this.toTestEntry(testCase);
48
- if (!this.isReportableEntry(entry))
49
- continue;
50
- if (!groupMap.has(groupKey)) {
51
- groupMap.set(groupKey, []);
52
- }
53
- groupMap.get(groupKey).push(entry);
54
- }
55
- }
56
- return Array.from(groupMap.entries()).map(([groupName, tests]) => ({
57
- groupName,
58
- tests,
59
- }));
60
- }
61
- /**
62
- * Group key is the parent suite's full name (file + describe hierarchy).
63
- * Tests without a describe wrapper are grouped by module path.
64
- */
65
- getGroupKey(testCase) {
66
- const parent = testCase.parent;
67
- const modulePath = testCase.module.relativeModuleId;
68
- if (parent.type === 'suite') {
69
- return `${modulePath} > ${parent.fullName}`;
70
- }
71
- return modulePath;
72
- }
73
- toTestEntry(testCase) {
74
- const meta = testCase.meta();
75
- const pathgradeMeta = meta.pathgrade;
76
- const diag = testCase.diagnostic();
77
- const result = testCase.result();
78
- let score = 0;
79
- let trial;
80
- let diagnostics;
81
- if (pathgradeMeta === undefined) {
82
- // Test didn't use pathgrade — infer from test pass/fail
83
- score = result.state === 'passed' ? 1 : 0;
84
- }
85
- else if (pathgradeMeta.length === 0) {
86
- // Anomaly: pathgrade was set up but evaluate() was never called
87
- console.warn(` [pathgrade] warning: empty results for "${testCase.name}" — evaluate() may not have been called`);
88
- score = result.state === 'passed' ? 1 : 0;
89
- }
90
- else {
91
- // Use the last eval result's score
92
- score = pathgradeMeta[pathgradeMeta.length - 1].score;
93
- trial = pathgradeMeta[pathgradeMeta.length - 1].trial;
94
- diagnostics = pathgradeMeta[pathgradeMeta.length - 1].diagnostics;
95
- }
96
- return {
97
- name: testCase.name,
98
- score,
99
- durationMs: diag?.duration ?? 0,
100
- state: result.state === 'pending' ? 'pending' : result.state,
101
- trial,
102
- diagnostics,
103
- };
104
- }
105
- isReportableEntry(entry) {
106
- return entry.state !== 'skipped' && entry.state !== 'pending';
107
- }
108
- printCliSummary(groups) {
109
- console.log(`\n${fmt.bold('── pathgrade summary ')}${fmt.dim('─'.repeat(40))}\n`);
110
- const forceVerbose = this.opts.diagnostics === true || process.env.PATHGRADE_DIAGNOSTICS === '1';
111
- const currentTimeoutMs = this.opts.timeout != null ? this.opts.timeout * 1000 : undefined;
112
- for (const group of groups) {
113
- const k = group.tests.length;
114
- const passRate = this.computePassRate(group.tests);
115
- const passAtK = this.computePassAtK(passRate, k);
116
- const passPowK = this.computePassPowK(passRate, k);
117
- const avgDuration = this.computeAvgDuration(group.tests);
118
- const prColor = passRate >= 0.5 ? fmt.green : fmt.red;
119
- console.log(` ${fmt.bold(group.groupName)}`);
120
- console.log(` ${fmt.dim('pass rate'.padEnd(12))} ${prColor((passRate * 100).toFixed(1) + '%')}`);
121
- console.log(` ${fmt.dim(`pass@${k}`.padEnd(12))} ${(passAtK * 100).toFixed(1)}%`);
122
- console.log(` ${fmt.dim(`pass^${k}`.padEnd(12))} ${(passPowK * 100).toFixed(1)}%`);
123
- console.log(` ${fmt.dim('avg time'.padEnd(12))} ${(avgDuration / 1000).toFixed(1)}s`);
124
- console.log(` ${fmt.dim('trials'.padEnd(12))} ${k}`);
125
- console.log();
126
- for (const test of group.tests) {
127
- if (!test.diagnostics)
128
- continue;
129
- const shouldPrintFull = forceVerbose || test.state !== 'passed' || test.diagnostics.completionReason === 'timeout' || test.diagnostics.completionReason === 'agent_crashed';
130
- console.log(` ${fmt.bold(test.name)}`);
131
- const formatted = shouldPrintFull
132
- ? formatDiagnostics(test.diagnostics, { verbose: true, currentTimeoutMs })
133
- : formatDiagnosticsSummary(test.diagnostics);
134
- for (const line of formatted.split('\n')) {
135
- console.log(` ${line}`);
136
- }
137
- console.log();
138
- }
139
- }
140
- }
141
- async writeJsonResults(groups) {
142
- const outputDir = getPathgradeDir(process.cwd());
143
- const tracesDir = path.join(outputDir, 'traces');
144
- await fs.ensureDir(tracesDir);
145
- // Auto-create .gitignore if it doesn't exist
146
- const gitignorePath = path.join(outputDir, '.gitignore');
147
- if (!(await fs.pathExists(gitignorePath))) {
148
- await fs.writeFile(gitignorePath, '*\n');
149
- }
150
- const allScores = groups.flatMap((g) => g.tests.map((t) => t.score));
151
- const overallPassRate = allScores.length > 0
152
- ? allScores.reduce((a, b) => a + b, 0) / allScores.length
153
- : 0;
154
- const threshold = this.opts.ci?.threshold;
155
- const status = threshold != null
156
- ? (overallPassRate >= threshold ? 'pass' : 'fail')
157
- : (groups.every(g => g.tests.every(t => t.state === 'passed')) ? 'pass' : 'fail');
158
- const consolidatedGroups = [];
159
- for (const group of groups) {
160
- const report = this.buildEvalReport(group);
161
- const slug = this.slug(group.groupName);
162
- const traceFile = `traces/${slug}.json`;
163
- // Write full trace data (with session_log and conversation)
164
- await fs.writeJson(path.join(outputDir, traceFile), report.trials, { spaces: 2 });
165
- // Strip session_log and conversation from trials in consolidated report
166
- const strippedTrials = report.trials.map(t => {
167
- const { session_log, conversation, ...rest } = t;
168
- return rest;
169
- });
170
- const { trials: _omitted, ...rest } = report;
171
- consolidatedGroups.push({
172
- ...rest,
173
- trials: strippedTrials,
174
- trace_file: traceFile,
175
- });
176
- }
177
- // Merge selection metadata from the CLI-written sidecar, if any.
178
- // Missing sidecar → plain run; malformed → warn and proceed without it.
179
- const selection = await readSidecar(process.cwd(), msg => {
180
- console.warn(`[pathgrade] ${msg}`);
181
- });
182
- const consolidated = {
183
- version: 1,
184
- timestamp: new Date().toISOString(),
185
- ...(threshold != null ? { threshold } : {}),
186
- overall_pass_rate: overallPassRate,
187
- status,
188
- groups: consolidatedGroups,
189
- ...(selection ? { selection } : {}),
190
- };
191
- await fs.writeJson(path.join(outputDir, 'results.json'), consolidated, { spaces: 2 });
192
- console.log(`\n ${fmt.dim('Results written to')} ${outputDir}\n`);
193
- }
194
- buildEvalReport(group) {
195
- const trials = group.tests.map((test, index) => this.toTrialResult(test, index));
196
- const passRate = this.computePassRate(group.tests);
197
- // Aggregate skills_used across all trials (deduplicated)
198
- const allSkills = new Set();
199
- for (const trial of trials) {
200
- for (const skill of trial.skills_used ?? []) {
201
- allSkills.add(skill);
202
- }
203
- }
204
- return {
205
- task: group.groupName,
206
- pass_rate: passRate,
207
- pass_at_k: this.computePassAtK(passRate, trials.length),
208
- pass_pow_k: this.computePassPowK(passRate, trials.length),
209
- trials,
210
- skills_used: [...allSkills],
211
- };
212
- }
213
- toTrialResult(test, index) {
214
- const base = test.trial ?? {
215
- trial_id: index + 1,
216
- reward: test.score,
217
- scorer_results: [],
218
- duration_ms: test.durationMs,
219
- n_commands: 0,
220
- input_tokens: 0,
221
- output_tokens: 0,
222
- session_log: [],
223
- };
224
- // Extract skills_used from session_log tool events if not already present
225
- const skills = base.skills_used ?? extractSkillsFromLog(base.session_log);
226
- return {
227
- ...base,
228
- trial_id: index + 1,
229
- name: test.name,
230
- duration_ms: base.duration_ms || test.durationMs,
231
- diagnostics: test.diagnostics ?? base.diagnostics,
232
- ...(skills.length > 0 ? { skills_used: skills } : {}),
233
- };
234
- }
235
- slug(value) {
236
- return value
237
- .toLowerCase()
238
- .replace(/[^a-z0-9]+/g, '-')
239
- .replace(/^-+|-+$/g, '') || 'report';
240
- }
241
- openBrowserViewer(_groups) {
242
- // Write results JSON first, then open the viewer
243
- const viewerPath = path.resolve(import.meta.dirname, '..', 'viewer.html');
244
- try {
245
- const openCmd = process.platform === 'darwin' ? 'open'
246
- : process.platform === 'win32' ? 'start'
247
- : 'xdg-open';
248
- execSync(`${openCmd} "${viewerPath}"`, { stdio: 'ignore' });
249
- console.log(`\n ${fmt.dim('Opened viewer in browser')}\n`);
250
- }
251
- catch {
252
- console.log(`\n ${fmt.dim('Open manually:')} ${viewerPath}\n`);
253
- }
254
- }
255
- computePassRate(tests) {
256
- if (tests.length === 0)
257
- return 0;
258
- const passed = tests.filter((t) => t.state === 'passed').length;
259
- return passed / tests.length;
260
- }
261
- /**
262
- * pass@k = 1 - (1 - p)^k
263
- * Probability of at least 1 success in k trials.
264
- */
265
- computePassAtK(passRate, k) {
266
- return 1 - Math.pow(1 - passRate, k);
267
- }
268
- /**
269
- * pass^k = p^k
270
- * Probability of all k trials succeeding.
271
- */
272
- computePassPowK(passRate, k) {
273
- return Math.pow(passRate, k);
274
- }
275
- computeAvgDuration(tests) {
276
- if (tests.length === 0)
277
- return 0;
278
- return tests.reduce((sum, t) => sum + t.durationMs, 0) / tests.length;
279
- }
280
- }
1
+ export { PathgradeReporter } from '../adapters/vitest/reporter.js';
@@ -1 +1 @@
1
- export {};
1
+ import '../adapters/vitest/setup.js';
@@ -1,3 +1 @@
1
- import { afterAll, afterEach } from 'vitest';
2
- import { lifecycle } from './lifecycle.js';
3
- lifecycle.install(afterEach, afterAll);
1
+ import '../adapters/vitest/setup.js';
@@ -16,3 +16,8 @@ export declare const DEFAULT_COPY_IGNORE: readonly string[];
16
16
  * Pass `[]` to disable filtering. Invalid glob patterns are silently ignored.
17
17
  */
18
18
  export declare function createCopyFilter(ignorePatterns: readonly string[]): (src: string) => boolean;
19
+ /**
20
+ * fs-extra cannot copy sockets, FIFOs, or device files. Host config caches can
21
+ * contain transient sockets, so filter them out before copy() reaches them.
22
+ */
23
+ export declare function isPortableCopyEntry(src: string): boolean;
@@ -5,6 +5,7 @@
5
5
  * when copying workspace/skill directories into the sandbox.
6
6
  */
7
7
  import * as path from 'path';
8
+ import * as fs from 'fs';
8
9
  import picomatch from 'picomatch';
9
10
  /**
10
11
  * Default ignore patterns applied when copying workspace and skill directories.
@@ -64,3 +65,11 @@ export function createCopyFilter(ignorePatterns) {
64
65
  return true;
65
66
  };
66
67
  }
68
+ /**
69
+ * fs-extra cannot copy sockets, FIFOs, or device files. Host config caches can
70
+ * contain transient sockets, so filter them out before copy() reaches them.
71
+ */
72
+ export function isPortableCopyEntry(src) {
73
+ const stat = fs.lstatSync(src);
74
+ return stat.isFile() || stat.isDirectory() || stat.isSymbolicLink();
75
+ }
@@ -1,7 +1,7 @@
1
1
  import fs from 'fs-extra';
2
2
  import * as path from 'path';
3
3
  import * as os from 'os';
4
- import { DEFAULT_COPY_IGNORE, createCopyFilter } from './copy-filter.js';
4
+ import { DEFAULT_COPY_IGNORE, createCopyFilter, isPortableCopyEntry } from './copy-filter.js';
5
5
  export const SAFE_HOST_VARS = [
6
6
  'PATH', 'SHELL', 'LANG', 'LC_ALL', 'LC_CTYPE', 'TERM', 'USER', 'LOGNAME',
7
7
  ];
@@ -56,7 +56,7 @@ export async function createSandbox(spec) {
56
56
  if (await fs.pathExists(srcPath)) {
57
57
  const destPath = path.join(homePath, relPath);
58
58
  await fs.ensureDir(path.dirname(destPath));
59
- await fs.copy(srcPath, destPath);
59
+ await fs.copy(srcPath, destPath, { filter: isPortableCopyEntry });
60
60
  }
61
61
  }
62
62
  }
@@ -11,5 +11,11 @@ export interface Workspace {
11
11
  }): Promise<CommandResult>;
12
12
  dispose(): Promise<void>;
13
13
  }
14
+ interface RemoveSandboxRootOptions {
15
+ remove?: (target: string) => Promise<void>;
16
+ sleep?: (ms: number) => Promise<void>;
17
+ retryDelaysMs?: readonly number[];
18
+ }
19
+ export declare function removeSandboxRoot(rootDir: string, opts?: RemoveSandboxRootOptions): Promise<void>;
14
20
  export declare function linkPathsFromHostHome(pathsToLink: string[], sandboxHomePath: string): Promise<void>;
15
21
  export declare function prepareWorkspace(spec: SandboxConfig): Promise<Workspace>;