@wix/pathgrade 0.29.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/bin/pathgrade.js +2 -0
- package/dist/affected/anchor.d.ts +18 -0
- package/dist/affected/anchor.js +42 -0
- package/dist/affected/config.d.ts +25 -0
- package/dist/affected/config.js +98 -0
- package/dist/affected/format.d.ts +45 -0
- package/dist/affected/format.js +87 -0
- package/dist/affected/git.d.ts +33 -0
- package/dist/affected/git.js +68 -0
- package/dist/affected/glob.d.ts +17 -0
- package/dist/affected/glob.js +25 -0
- package/dist/affected/meta.d.ts +26 -0
- package/dist/affected/meta.js +158 -0
- package/dist/affected/select.d.ts +30 -0
- package/dist/affected/select.js +101 -0
- package/dist/affected/sidecar.d.ts +32 -0
- package/dist/affected/sidecar.js +86 -0
- package/dist/affected/types.d.ts +43 -0
- package/dist/affected/types.js +7 -0
- package/dist/agents/claude/ask-user-answer-store.d.ts +28 -0
- package/dist/agents/claude/ask-user-answer-store.js +29 -0
- package/dist/agents/claude/ask-user-bridge.d.ts +56 -0
- package/dist/agents/claude/ask-user-bridge.js +162 -0
- package/dist/agents/claude/sdk-message-projector.d.ts +43 -0
- package/dist/agents/claude/sdk-message-projector.js +208 -0
- package/dist/agents/claude/sdk-options.d.ts +52 -0
- package/dist/agents/claude/sdk-options.js +79 -0
- package/dist/agents/claude.d.ts +56 -0
- package/dist/agents/claude.js +143 -0
- package/dist/agents/codex-app-server/agent.d.ts +35 -0
- package/dist/agents/codex-app-server/agent.js +388 -0
- package/dist/agents/codex-app-server/fixtures/app-server-client.d.ts +19 -0
- package/dist/agents/codex-app-server/fixtures/app-server-client.js +104 -0
- package/dist/agents/codex-app-server/fixtures/run-gate.d.ts +25 -0
- package/dist/agents/codex-app-server/fixtures/run-gate.js +48 -0
- package/dist/agents/codex-app-server/protocol/ClientRequest.d.ts +27 -0
- package/dist/agents/codex-app-server/protocol/ClientRequest.js +7 -0
- package/dist/agents/codex-app-server/protocol/DynamicToolCallParams.d.ts +9 -0
- package/dist/agents/codex-app-server/protocol/DynamicToolCallParams.js +4 -0
- package/dist/agents/codex-app-server/protocol/GrantedPermissionProfile.d.ts +6 -0
- package/dist/agents/codex-app-server/protocol/GrantedPermissionProfile.js +7 -0
- package/dist/agents/codex-app-server/protocol/McpElicitationRequestParams.d.ts +23 -0
- package/dist/agents/codex-app-server/protocol/McpElicitationRequestParams.js +8 -0
- package/dist/agents/codex-app-server/protocol/Op.d.ts +11 -0
- package/dist/agents/codex-app-server/protocol/Op.js +5 -0
- package/dist/agents/codex-app-server/protocol/PermissionsRequestApprovalParams.d.ts +10 -0
- package/dist/agents/codex-app-server/protocol/PermissionsRequestApprovalParams.js +6 -0
- package/dist/agents/codex-app-server/protocol/PermissionsRequestApprovalResponse.d.ts +15 -0
- package/dist/agents/codex-app-server/protocol/PermissionsRequestApprovalResponse.js +4 -0
- package/dist/agents/codex-app-server/protocol/SandboxMode.d.ts +1 -0
- package/dist/agents/codex-app-server/protocol/SandboxMode.js +4 -0
- package/dist/agents/codex-app-server/protocol/ServerRequest.d.ts +42 -0
- package/dist/agents/codex-app-server/protocol/ServerRequest.js +7 -0
- package/dist/agents/codex-app-server/protocol/ThreadStartParams.d.ts +45 -0
- package/dist/agents/codex-app-server/protocol/ThreadStartParams.js +10 -0
- package/dist/agents/codex-app-server/protocol/ToolRequestUserInputAnswer.d.ts +6 -0
- package/dist/agents/codex-app-server/protocol/ToolRequestUserInputAnswer.js +4 -0
- package/dist/agents/codex-app-server/protocol/ToolRequestUserInputOption.d.ts +7 -0
- package/dist/agents/codex-app-server/protocol/ToolRequestUserInputOption.js +4 -0
- package/dist/agents/codex-app-server/protocol/ToolRequestUserInputParams.d.ts +10 -0
- package/dist/agents/codex-app-server/protocol/ToolRequestUserInputParams.js +4 -0
- package/dist/agents/codex-app-server/protocol/ToolRequestUserInputQuestion.d.ts +12 -0
- package/dist/agents/codex-app-server/protocol/ToolRequestUserInputQuestion.js +4 -0
- package/dist/agents/codex-app-server/protocol/ToolRequestUserInputResponse.d.ts +9 -0
- package/dist/agents/codex-app-server/protocol/ToolRequestUserInputResponse.js +4 -0
- package/dist/agents/codex-app-server/protocol/TurnCompletedNotification.d.ts +5 -0
- package/dist/agents/codex-app-server/protocol/TurnCompletedNotification.js +4 -0
- package/dist/agents/codex-app-server/protocol/index.d.ts +16 -0
- package/dist/agents/codex-app-server/protocol/index.js +11 -0
- package/dist/agents/codex-app-server/transport.d.ts +84 -0
- package/dist/agents/codex-app-server/transport.js +246 -0
- package/dist/agents/codex-app-server/wire-translators.d.ts +19 -0
- package/dist/agents/codex-app-server/wire-translators.js +52 -0
- package/dist/agents/codex.d.ts +7 -0
- package/dist/agents/codex.js +164 -0
- package/dist/agents/cursor.d.ts +39 -0
- package/dist/agents/cursor.js +233 -0
- package/dist/agents/registry.d.ts +17 -0
- package/dist/agents/registry.js +27 -0
- package/dist/agents/transcript-agent.d.ts +14 -0
- package/dist/agents/transcript-agent.js +59 -0
- package/dist/analytics/engine.d.ts +18 -0
- package/dist/analytics/engine.js +66 -0
- package/dist/commands/affected.d.ts +31 -0
- package/dist/commands/affected.js +150 -0
- package/dist/commands/analyze.d.ts +21 -0
- package/dist/commands/analyze.js +167 -0
- package/dist/commands/init.d.ts +3 -0
- package/dist/commands/init.js +242 -0
- package/dist/commands/preview-reactions.d.ts +10 -0
- package/dist/commands/preview-reactions.js +88 -0
- package/dist/commands/preview.d.ts +5 -0
- package/dist/commands/preview.js +17 -0
- package/dist/commands/report.d.ts +39 -0
- package/dist/commands/report.js +108 -0
- package/dist/commands/run-args.d.ts +22 -0
- package/dist/commands/run-args.js +69 -0
- package/dist/commands/run-changed.d.ts +26 -0
- package/dist/commands/run-changed.js +138 -0
- package/dist/commands/validate.d.ts +13 -0
- package/dist/commands/validate.js +233 -0
- package/dist/core/mcp-mock.d.ts +3 -0
- package/dist/core/mcp-mock.js +23 -0
- package/dist/core/mcp-mock.types.d.ts +15 -0
- package/dist/core/mcp-mock.types.js +1 -0
- package/dist/core/skills.d.ts +15 -0
- package/dist/core/skills.js +76 -0
- package/dist/evals/discovery.d.ts +7 -0
- package/dist/evals/discovery.js +69 -0
- package/dist/mcp-mock-server.d.ts +1 -0
- package/dist/mcp-mock-server.js +116 -0
- package/dist/pathgrade.d.ts +10 -0
- package/dist/pathgrade.js +243 -0
- package/dist/plugin/index.d.ts +9 -0
- package/dist/plugin/index.js +81 -0
- package/dist/plugin/lifecycle.d.ts +45 -0
- package/dist/plugin/lifecycle.js +146 -0
- package/dist/plugin/reporter.d.ts +36 -0
- package/dist/plugin/reporter.js +275 -0
- package/dist/plugin/setup.d.ts +1 -0
- package/dist/plugin/setup.js +3 -0
- package/dist/providers/copy-filter.d.ts +18 -0
- package/dist/providers/copy-filter.js +66 -0
- package/dist/providers/credentials.d.ts +36 -0
- package/dist/providers/credentials.js +202 -0
- package/dist/providers/mcp-config.d.ts +34 -0
- package/dist/providers/mcp-config.js +55 -0
- package/dist/providers/sandbox-exec.d.ts +8 -0
- package/dist/providers/sandbox-exec.js +82 -0
- package/dist/providers/sandbox.d.ts +22 -0
- package/dist/providers/sandbox.js +158 -0
- package/dist/providers/sandboxed-claude-spawn.d.ts +24 -0
- package/dist/providers/sandboxed-claude-spawn.js +57 -0
- package/dist/providers/workspace.d.ts +15 -0
- package/dist/providers/workspace.js +73 -0
- package/dist/reporters/browser.d.ts +2 -0
- package/dist/reporters/browser.js +85 -0
- package/dist/reporters/cli.d.ts +2 -0
- package/dist/reporters/cli.js +131 -0
- package/dist/reporters/diagnostics.d.ts +46 -0
- package/dist/reporters/diagnostics.js +147 -0
- package/dist/reporters/github-comment.d.ts +67 -0
- package/dist/reporters/github-comment.js +252 -0
- package/dist/reporters/loader.d.ts +8 -0
- package/dist/reporters/loader.js +36 -0
- package/dist/reporters/results-path.d.ts +2 -0
- package/dist/reporters/results-path.js +10 -0
- package/dist/reporters/verbose-emitter.d.ts +69 -0
- package/dist/reporters/verbose-emitter.js +103 -0
- package/dist/sdk/agent-crash.d.ts +19 -0
- package/dist/sdk/agent-crash.js +20 -0
- package/dist/sdk/agent-resolution.d.ts +12 -0
- package/dist/sdk/agent-resolution.js +21 -0
- package/dist/sdk/agent-result-log.d.ts +19 -0
- package/dist/sdk/agent-result-log.js +45 -0
- package/dist/sdk/agent.d.ts +4 -0
- package/dist/sdk/agent.js +354 -0
- package/dist/sdk/ask-bus/bus.d.ts +21 -0
- package/dist/sdk/ask-bus/bus.js +129 -0
- package/dist/sdk/ask-bus/handler.d.ts +31 -0
- package/dist/sdk/ask-bus/handler.js +244 -0
- package/dist/sdk/ask-bus/parsers.d.ts +9 -0
- package/dist/sdk/ask-bus/parsers.js +30 -0
- package/dist/sdk/ask-bus/projection.d.ts +26 -0
- package/dist/sdk/ask-bus/projection.js +45 -0
- package/dist/sdk/ask-bus/types.d.ts +69 -0
- package/dist/sdk/ask-bus/types.js +1 -0
- package/dist/sdk/chat.d.ts +29 -0
- package/dist/sdk/chat.js +93 -0
- package/dist/sdk/conversation-window.d.ts +14 -0
- package/dist/sdk/conversation-window.js +38 -0
- package/dist/sdk/converse.d.ts +42 -0
- package/dist/sdk/converse.js +329 -0
- package/dist/sdk/eval-runtime.d.ts +10 -0
- package/dist/sdk/eval-runtime.js +17 -0
- package/dist/sdk/evaluate.d.ts +13 -0
- package/dist/sdk/evaluate.js +325 -0
- package/dist/sdk/index.d.ts +31 -0
- package/dist/sdk/index.js +20 -0
- package/dist/sdk/judge-pipeline.d.ts +7 -0
- package/dist/sdk/judge-pipeline.js +178 -0
- package/dist/sdk/judge-prompt-builder.d.ts +7 -0
- package/dist/sdk/judge-prompt-builder.js +76 -0
- package/dist/sdk/judge-tool-runner.d.ts +19 -0
- package/dist/sdk/judge-tool-runner.js +51 -0
- package/dist/sdk/judge-tool-session.d.ts +47 -0
- package/dist/sdk/judge-tool-session.js +213 -0
- package/dist/sdk/judge-tools.d.ts +29 -0
- package/dist/sdk/judge-tools.js +283 -0
- package/dist/sdk/managed-session.d.ts +50 -0
- package/dist/sdk/managed-session.js +110 -0
- package/dist/sdk/persona.d.ts +5 -0
- package/dist/sdk/persona.js +33 -0
- package/dist/sdk/reaction-loader.d.ts +4 -0
- package/dist/sdk/reaction-loader.js +119 -0
- package/dist/sdk/reaction-preview.d.ts +6 -0
- package/dist/sdk/reaction-preview.js +94 -0
- package/dist/sdk/run-scorer.d.ts +10 -0
- package/dist/sdk/run-scorer.js +124 -0
- package/dist/sdk/runtime-policy.d.ts +21 -0
- package/dist/sdk/runtime-policy.js +89 -0
- package/dist/sdk/scorer-utils.d.ts +6 -0
- package/dist/sdk/scorer-utils.js +39 -0
- package/dist/sdk/scorers.d.ts +64 -0
- package/dist/sdk/scorers.js +80 -0
- package/dist/sdk/snapshots.d.ts +41 -0
- package/dist/sdk/snapshots.js +148 -0
- package/dist/sdk/types.d.ts +416 -0
- package/dist/sdk/types.js +17 -0
- package/dist/sdk/visible-turn.d.ts +4 -0
- package/dist/sdk/visible-turn.js +23 -0
- package/dist/tool-events.d.ts +45 -0
- package/dist/tool-events.js +167 -0
- package/dist/types.d.ts +372 -0
- package/dist/types.js +37 -0
- package/dist/utils/cli.d.ts +55 -0
- package/dist/utils/cli.js +131 -0
- package/dist/utils/env.d.ts +8 -0
- package/dist/utils/env.js +26 -0
- package/dist/utils/llm-mocks.d.ts +43 -0
- package/dist/utils/llm-mocks.js +108 -0
- package/dist/utils/llm-providers/anthropic.d.ts +2 -0
- package/dist/utils/llm-providers/anthropic.js +132 -0
- package/dist/utils/llm-providers/cli.d.ts +20 -0
- package/dist/utils/llm-providers/cli.js +162 -0
- package/dist/utils/llm-providers/openai.d.ts +2 -0
- package/dist/utils/llm-providers/openai.js +56 -0
- package/dist/utils/llm-types.d.ts +107 -0
- package/dist/utils/llm-types.js +1 -0
- package/dist/utils/llm.d.ts +36 -0
- package/dist/utils/llm.js +209 -0
- package/dist/utils/shutdown.d.ts +15 -0
- package/dist/utils/shutdown.js +41 -0
- package/dist/utils/timeout.d.ts +8 -0
- package/dist/utils/timeout.js +32 -0
- package/dist/viewer.html +1178 -0
- package/package.json +88 -0
- package/templates/eval.ts.template +28 -0
|
@@ -0,0 +1,275 @@
|
|
|
1
|
+
import * as path from 'path';
|
|
2
|
+
import { execSync } from 'child_process';
|
|
3
|
+
import { extractSkillsFromLog } from '../tool-events.js';
|
|
4
|
+
import { fmt } from '../utils/cli.js';
|
|
5
|
+
import { getPathgradeDir } from '../reporters/results-path.js';
|
|
6
|
+
import fs from 'fs-extra';
|
|
7
|
+
import { formatDiagnostics, formatDiagnosticsSummary } from '../reporters/diagnostics.js';
|
|
8
|
+
import { readSidecar } from '../affected/sidecar.js';
|
|
9
|
+
/**
|
|
10
|
+
* Custom vitest reporter that layers pathgrade aggregate statistics
|
|
11
|
+
* on top of vitest's default output.
|
|
12
|
+
*/
|
|
13
|
+
export class PathgradeReporter {
|
|
14
|
+
opts;
|
|
15
|
+
constructor(opts) {
|
|
16
|
+
this.opts = opts ?? {};
|
|
17
|
+
}
|
|
18
|
+
async onTestRunEnd(testModules) {
|
|
19
|
+
const groups = this.collectGroups(testModules);
|
|
20
|
+
if (groups.length === 0)
|
|
21
|
+
return;
|
|
22
|
+
const mode = this.opts.reporter ?? 'cli';
|
|
23
|
+
if (mode === 'cli' || mode === 'browser') {
|
|
24
|
+
this.printCliSummary(groups);
|
|
25
|
+
}
|
|
26
|
+
await this.writeJsonResults(groups);
|
|
27
|
+
if (mode === 'browser') {
|
|
28
|
+
this.openBrowserViewer(groups);
|
|
29
|
+
}
|
|
30
|
+
// CI threshold check
|
|
31
|
+
if (this.opts.ci?.threshold != null) {
|
|
32
|
+
const allScores = groups.flatMap((g) => g.tests.map((t) => t.score));
|
|
33
|
+
if (allScores.length > 0) {
|
|
34
|
+
const avg = allScores.reduce((a, b) => a + b, 0) / allScores.length;
|
|
35
|
+
if (avg < this.opts.ci.threshold) {
|
|
36
|
+
console.log(`\n ${fmt.fail('CI THRESHOLD FAILED')} avg score ${fmt.bold(avg.toFixed(3))} < threshold ${fmt.bold(String(this.opts.ci.threshold))}\n`);
|
|
37
|
+
process.exitCode = 1;
|
|
38
|
+
}
|
|
39
|
+
}
|
|
40
|
+
}
|
|
41
|
+
}
|
|
42
|
+
collectGroups(testModules) {
|
|
43
|
+
const groupMap = new Map();
|
|
44
|
+
for (const mod of testModules) {
|
|
45
|
+
for (const testCase of mod.children.allTests()) {
|
|
46
|
+
const groupKey = this.getGroupKey(testCase);
|
|
47
|
+
const entry = this.toTestEntry(testCase);
|
|
48
|
+
if (!groupMap.has(groupKey)) {
|
|
49
|
+
groupMap.set(groupKey, []);
|
|
50
|
+
}
|
|
51
|
+
groupMap.get(groupKey).push(entry);
|
|
52
|
+
}
|
|
53
|
+
}
|
|
54
|
+
return Array.from(groupMap.entries()).map(([groupName, tests]) => ({
|
|
55
|
+
groupName,
|
|
56
|
+
tests,
|
|
57
|
+
}));
|
|
58
|
+
}
|
|
59
|
+
/**
|
|
60
|
+
* Group key is the parent suite's full name (file + describe hierarchy).
|
|
61
|
+
* Tests without a describe wrapper are grouped by module path.
|
|
62
|
+
*/
|
|
63
|
+
getGroupKey(testCase) {
|
|
64
|
+
const parent = testCase.parent;
|
|
65
|
+
const modulePath = testCase.module.relativeModuleId;
|
|
66
|
+
if (parent.type === 'suite') {
|
|
67
|
+
return `${modulePath} > ${parent.fullName}`;
|
|
68
|
+
}
|
|
69
|
+
return modulePath;
|
|
70
|
+
}
|
|
71
|
+
toTestEntry(testCase) {
|
|
72
|
+
const meta = testCase.meta();
|
|
73
|
+
const pathgradeMeta = meta.pathgrade;
|
|
74
|
+
const diag = testCase.diagnostic();
|
|
75
|
+
const result = testCase.result();
|
|
76
|
+
let score = 0;
|
|
77
|
+
let trial;
|
|
78
|
+
let diagnostics;
|
|
79
|
+
if (pathgradeMeta === undefined) {
|
|
80
|
+
// Test didn't use pathgrade — infer from test pass/fail
|
|
81
|
+
score = result.state === 'passed' ? 1 : 0;
|
|
82
|
+
}
|
|
83
|
+
else if (pathgradeMeta.length === 0) {
|
|
84
|
+
// Anomaly: pathgrade was set up but evaluate() was never called
|
|
85
|
+
console.warn(` [pathgrade] warning: empty results for "${testCase.name}" — evaluate() may not have been called`);
|
|
86
|
+
score = result.state === 'passed' ? 1 : 0;
|
|
87
|
+
}
|
|
88
|
+
else {
|
|
89
|
+
// Use the last eval result's score
|
|
90
|
+
score = pathgradeMeta[pathgradeMeta.length - 1].score;
|
|
91
|
+
trial = pathgradeMeta[pathgradeMeta.length - 1].trial;
|
|
92
|
+
diagnostics = pathgradeMeta[pathgradeMeta.length - 1].diagnostics;
|
|
93
|
+
}
|
|
94
|
+
return {
|
|
95
|
+
name: testCase.name,
|
|
96
|
+
score,
|
|
97
|
+
durationMs: diag?.duration ?? 0,
|
|
98
|
+
state: result.state === 'pending' ? 'pending' : result.state,
|
|
99
|
+
trial,
|
|
100
|
+
diagnostics,
|
|
101
|
+
};
|
|
102
|
+
}
|
|
103
|
+
printCliSummary(groups) {
|
|
104
|
+
console.log(`\n${fmt.bold('── pathgrade summary ')}${fmt.dim('─'.repeat(40))}\n`);
|
|
105
|
+
const forceVerbose = this.opts.diagnostics === true || process.env.PATHGRADE_DIAGNOSTICS === '1';
|
|
106
|
+
const currentTimeoutMs = this.opts.timeout != null ? this.opts.timeout * 1000 : undefined;
|
|
107
|
+
for (const group of groups) {
|
|
108
|
+
const k = group.tests.length;
|
|
109
|
+
const passRate = this.computePassRate(group.tests);
|
|
110
|
+
const passAtK = this.computePassAtK(passRate, k);
|
|
111
|
+
const passPowK = this.computePassPowK(passRate, k);
|
|
112
|
+
const avgDuration = this.computeAvgDuration(group.tests);
|
|
113
|
+
const prColor = passRate >= 0.5 ? fmt.green : fmt.red;
|
|
114
|
+
console.log(` ${fmt.bold(group.groupName)}`);
|
|
115
|
+
console.log(` ${fmt.dim('pass rate'.padEnd(12))} ${prColor((passRate * 100).toFixed(1) + '%')}`);
|
|
116
|
+
console.log(` ${fmt.dim(`pass@${k}`.padEnd(12))} ${(passAtK * 100).toFixed(1)}%`);
|
|
117
|
+
console.log(` ${fmt.dim(`pass^${k}`.padEnd(12))} ${(passPowK * 100).toFixed(1)}%`);
|
|
118
|
+
console.log(` ${fmt.dim('avg time'.padEnd(12))} ${(avgDuration / 1000).toFixed(1)}s`);
|
|
119
|
+
console.log(` ${fmt.dim('trials'.padEnd(12))} ${k}`);
|
|
120
|
+
console.log();
|
|
121
|
+
for (const test of group.tests) {
|
|
122
|
+
if (!test.diagnostics)
|
|
123
|
+
continue;
|
|
124
|
+
const shouldPrintFull = forceVerbose || test.state !== 'passed' || test.diagnostics.completionReason === 'timeout' || test.diagnostics.completionReason === 'agent_crashed';
|
|
125
|
+
console.log(` ${fmt.bold(test.name)}`);
|
|
126
|
+
const formatted = shouldPrintFull
|
|
127
|
+
? formatDiagnostics(test.diagnostics, { verbose: true, currentTimeoutMs })
|
|
128
|
+
: formatDiagnosticsSummary(test.diagnostics);
|
|
129
|
+
for (const line of formatted.split('\n')) {
|
|
130
|
+
console.log(` ${line}`);
|
|
131
|
+
}
|
|
132
|
+
console.log();
|
|
133
|
+
}
|
|
134
|
+
}
|
|
135
|
+
}
|
|
136
|
+
async writeJsonResults(groups) {
|
|
137
|
+
const outputDir = getPathgradeDir(process.cwd());
|
|
138
|
+
const tracesDir = path.join(outputDir, 'traces');
|
|
139
|
+
await fs.ensureDir(tracesDir);
|
|
140
|
+
// Auto-create .gitignore if it doesn't exist
|
|
141
|
+
const gitignorePath = path.join(outputDir, '.gitignore');
|
|
142
|
+
if (!(await fs.pathExists(gitignorePath))) {
|
|
143
|
+
await fs.writeFile(gitignorePath, '*\n');
|
|
144
|
+
}
|
|
145
|
+
const allScores = groups.flatMap((g) => g.tests.map((t) => t.score));
|
|
146
|
+
const overallPassRate = allScores.length > 0
|
|
147
|
+
? allScores.reduce((a, b) => a + b, 0) / allScores.length
|
|
148
|
+
: 0;
|
|
149
|
+
const threshold = this.opts.ci?.threshold;
|
|
150
|
+
const status = threshold != null
|
|
151
|
+
? (overallPassRate >= threshold ? 'pass' : 'fail')
|
|
152
|
+
: (groups.every(g => g.tests.every(t => t.state === 'passed')) ? 'pass' : 'fail');
|
|
153
|
+
const consolidatedGroups = [];
|
|
154
|
+
for (const group of groups) {
|
|
155
|
+
const report = this.buildEvalReport(group);
|
|
156
|
+
const slug = this.slug(group.groupName);
|
|
157
|
+
const traceFile = `traces/${slug}.json`;
|
|
158
|
+
// Write full trace data (with session_log and conversation)
|
|
159
|
+
await fs.writeJson(path.join(outputDir, traceFile), report.trials, { spaces: 2 });
|
|
160
|
+
// Strip session_log and conversation from trials in consolidated report
|
|
161
|
+
const strippedTrials = report.trials.map(t => {
|
|
162
|
+
const { session_log, conversation, ...rest } = t;
|
|
163
|
+
return rest;
|
|
164
|
+
});
|
|
165
|
+
const { trials: _omitted, ...rest } = report;
|
|
166
|
+
consolidatedGroups.push({
|
|
167
|
+
...rest,
|
|
168
|
+
trials: strippedTrials,
|
|
169
|
+
trace_file: traceFile,
|
|
170
|
+
});
|
|
171
|
+
}
|
|
172
|
+
// Merge selection metadata from the CLI-written sidecar, if any.
|
|
173
|
+
// Missing sidecar → plain run; malformed → warn and proceed without it.
|
|
174
|
+
const selection = await readSidecar(process.cwd(), msg => {
|
|
175
|
+
console.warn(`[pathgrade] ${msg}`);
|
|
176
|
+
});
|
|
177
|
+
const consolidated = {
|
|
178
|
+
version: 1,
|
|
179
|
+
timestamp: new Date().toISOString(),
|
|
180
|
+
...(threshold != null ? { threshold } : {}),
|
|
181
|
+
overall_pass_rate: overallPassRate,
|
|
182
|
+
status,
|
|
183
|
+
groups: consolidatedGroups,
|
|
184
|
+
...(selection ? { selection } : {}),
|
|
185
|
+
};
|
|
186
|
+
await fs.writeJson(path.join(outputDir, 'results.json'), consolidated, { spaces: 2 });
|
|
187
|
+
console.log(`\n ${fmt.dim('Results written to')} ${outputDir}\n`);
|
|
188
|
+
}
|
|
189
|
+
buildEvalReport(group) {
|
|
190
|
+
const trials = group.tests.map((test, index) => this.toTrialResult(test, index));
|
|
191
|
+
const passRate = this.computePassRate(group.tests);
|
|
192
|
+
// Aggregate skills_used across all trials (deduplicated)
|
|
193
|
+
const allSkills = new Set();
|
|
194
|
+
for (const trial of trials) {
|
|
195
|
+
for (const skill of trial.skills_used ?? []) {
|
|
196
|
+
allSkills.add(skill);
|
|
197
|
+
}
|
|
198
|
+
}
|
|
199
|
+
return {
|
|
200
|
+
task: group.groupName,
|
|
201
|
+
pass_rate: passRate,
|
|
202
|
+
pass_at_k: this.computePassAtK(passRate, trials.length),
|
|
203
|
+
pass_pow_k: this.computePassPowK(passRate, trials.length),
|
|
204
|
+
trials,
|
|
205
|
+
skills_used: [...allSkills],
|
|
206
|
+
};
|
|
207
|
+
}
|
|
208
|
+
toTrialResult(test, index) {
|
|
209
|
+
const base = test.trial ?? {
|
|
210
|
+
trial_id: index + 1,
|
|
211
|
+
reward: test.score,
|
|
212
|
+
scorer_results: [],
|
|
213
|
+
duration_ms: test.durationMs,
|
|
214
|
+
n_commands: 0,
|
|
215
|
+
input_tokens: 0,
|
|
216
|
+
output_tokens: 0,
|
|
217
|
+
session_log: [],
|
|
218
|
+
};
|
|
219
|
+
// Extract skills_used from session_log tool events if not already present
|
|
220
|
+
const skills = base.skills_used ?? extractSkillsFromLog(base.session_log);
|
|
221
|
+
return {
|
|
222
|
+
...base,
|
|
223
|
+
trial_id: index + 1,
|
|
224
|
+
name: test.name,
|
|
225
|
+
duration_ms: base.duration_ms || test.durationMs,
|
|
226
|
+
diagnostics: test.diagnostics ?? base.diagnostics,
|
|
227
|
+
...(skills.length > 0 ? { skills_used: skills } : {}),
|
|
228
|
+
};
|
|
229
|
+
}
|
|
230
|
+
slug(value) {
|
|
231
|
+
return value
|
|
232
|
+
.toLowerCase()
|
|
233
|
+
.replace(/[^a-z0-9]+/g, '-')
|
|
234
|
+
.replace(/^-+|-+$/g, '') || 'report';
|
|
235
|
+
}
|
|
236
|
+
openBrowserViewer(_groups) {
|
|
237
|
+
// Write results JSON first, then open the viewer
|
|
238
|
+
const viewerPath = path.resolve(import.meta.dirname, '..', 'viewer.html');
|
|
239
|
+
try {
|
|
240
|
+
const openCmd = process.platform === 'darwin' ? 'open'
|
|
241
|
+
: process.platform === 'win32' ? 'start'
|
|
242
|
+
: 'xdg-open';
|
|
243
|
+
execSync(`${openCmd} "${viewerPath}"`, { stdio: 'ignore' });
|
|
244
|
+
console.log(`\n ${fmt.dim('Opened viewer in browser')}\n`);
|
|
245
|
+
}
|
|
246
|
+
catch {
|
|
247
|
+
console.log(`\n ${fmt.dim('Open manually:')} ${viewerPath}\n`);
|
|
248
|
+
}
|
|
249
|
+
}
|
|
250
|
+
computePassRate(tests) {
|
|
251
|
+
if (tests.length === 0)
|
|
252
|
+
return 0;
|
|
253
|
+
const passed = tests.filter((t) => t.state === 'passed').length;
|
|
254
|
+
return passed / tests.length;
|
|
255
|
+
}
|
|
256
|
+
/**
|
|
257
|
+
* pass@k = 1 - (1 - p)^k
|
|
258
|
+
* Probability of at least 1 success in k trials.
|
|
259
|
+
*/
|
|
260
|
+
computePassAtK(passRate, k) {
|
|
261
|
+
return 1 - Math.pow(1 - passRate, k);
|
|
262
|
+
}
|
|
263
|
+
/**
|
|
264
|
+
* pass^k = p^k
|
|
265
|
+
* Probability of all k trials succeeding.
|
|
266
|
+
*/
|
|
267
|
+
computePassPowK(passRate, k) {
|
|
268
|
+
return Math.pow(passRate, k);
|
|
269
|
+
}
|
|
270
|
+
computeAvgDuration(tests) {
|
|
271
|
+
if (tests.length === 0)
|
|
272
|
+
return 0;
|
|
273
|
+
return tests.reduce((sum, t) => sum + t.durationMs, 0) / tests.length;
|
|
274
|
+
}
|
|
275
|
+
}
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
export {};
|
|
@@ -0,0 +1,18 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Default ignore patterns applied when copying workspace and skill directories.
|
|
3
|
+
* Matches against the basename (filename or directory name) of each entry.
|
|
4
|
+
*
|
|
5
|
+
* - Plain names (no wildcards) match exact directory/file names at any depth.
|
|
6
|
+
* - Glob patterns (with wildcards) match filenames at any depth.
|
|
7
|
+
*/
|
|
8
|
+
export declare const DEFAULT_COPY_IGNORE: readonly string[];
|
|
9
|
+
/**
|
|
10
|
+
* Creates a filter function suitable for use with `fs.copy()`.
|
|
11
|
+
*
|
|
12
|
+
* The filter matches ignore patterns against the **basename** of each path.
|
|
13
|
+
* Returns `true` to include the file/directory, `false` to exclude it.
|
|
14
|
+
*
|
|
15
|
+
* @param ignorePatterns - Array of patterns to filter. Replaces defaults entirely.
|
|
16
|
+
* Pass `[]` to disable filtering. Invalid glob patterns are silently ignored.
|
|
17
|
+
*/
|
|
18
|
+
export declare function createCopyFilter(ignorePatterns: readonly string[]): (src: string) => boolean;
|
|
@@ -0,0 +1,66 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Copy filter for workspace and skill directory copies.
|
|
3
|
+
*
|
|
4
|
+
* Filters out junk directories and files (node_modules, .git, dist, etc.)
|
|
5
|
+
* when copying workspace/skill directories into the sandbox.
|
|
6
|
+
*/
|
|
7
|
+
import * as path from 'path';
|
|
8
|
+
import picomatch from 'picomatch';
|
|
9
|
+
/**
|
|
10
|
+
* Default ignore patterns applied when copying workspace and skill directories.
|
|
11
|
+
* Matches against the basename (filename or directory name) of each entry.
|
|
12
|
+
*
|
|
13
|
+
* - Plain names (no wildcards) match exact directory/file names at any depth.
|
|
14
|
+
* - Glob patterns (with wildcards) match filenames at any depth.
|
|
15
|
+
*/
|
|
16
|
+
export const DEFAULT_COPY_IGNORE = Object.freeze([
|
|
17
|
+
'node_modules',
|
|
18
|
+
'.git',
|
|
19
|
+
'dist',
|
|
20
|
+
'.DS_Store',
|
|
21
|
+
'__pycache__',
|
|
22
|
+
'npm-debug.log*',
|
|
23
|
+
'yarn-debug.log*',
|
|
24
|
+
'yarn-error.log*',
|
|
25
|
+
]);
|
|
26
|
+
/**
|
|
27
|
+
* Creates a filter function suitable for use with `fs.copy()`.
|
|
28
|
+
*
|
|
29
|
+
* The filter matches ignore patterns against the **basename** of each path.
|
|
30
|
+
* Returns `true` to include the file/directory, `false` to exclude it.
|
|
31
|
+
*
|
|
32
|
+
* @param ignorePatterns - Array of patterns to filter. Replaces defaults entirely.
|
|
33
|
+
* Pass `[]` to disable filtering. Invalid glob patterns are silently ignored.
|
|
34
|
+
*/
|
|
35
|
+
export function createCopyFilter(ignorePatterns) {
|
|
36
|
+
if (ignorePatterns.length === 0) {
|
|
37
|
+
return () => true;
|
|
38
|
+
}
|
|
39
|
+
// Build matchers from the patterns, silently skipping invalid ones
|
|
40
|
+
const matchers = [];
|
|
41
|
+
for (const pattern of ignorePatterns) {
|
|
42
|
+
try {
|
|
43
|
+
matchers.push(picomatch(pattern));
|
|
44
|
+
}
|
|
45
|
+
catch {
|
|
46
|
+
// Invalid glob pattern: silently ignore per PRD
|
|
47
|
+
}
|
|
48
|
+
}
|
|
49
|
+
if (matchers.length === 0) {
|
|
50
|
+
return () => true;
|
|
51
|
+
}
|
|
52
|
+
return (src) => {
|
|
53
|
+
const basename = path.basename(src);
|
|
54
|
+
// Don't filter if basename is empty (shouldn't happen, but be safe)
|
|
55
|
+
if (!basename) {
|
|
56
|
+
return true;
|
|
57
|
+
}
|
|
58
|
+
// Check if any matcher matches the basename
|
|
59
|
+
for (const matcher of matchers) {
|
|
60
|
+
if (matcher(basename)) {
|
|
61
|
+
return false;
|
|
62
|
+
}
|
|
63
|
+
}
|
|
64
|
+
return true;
|
|
65
|
+
};
|
|
66
|
+
}
|
|
@@ -0,0 +1,36 @@
|
|
|
1
|
+
import type { AgentName } from '../sdk/types.js';
|
|
2
|
+
export interface CredentialPorts {
|
|
3
|
+
/** Read a host environment variable. */
|
|
4
|
+
hostEnv(key: string): string | undefined;
|
|
5
|
+
/** Host platform. */
|
|
6
|
+
platform: NodeJS.Platform;
|
|
7
|
+
/** Host home directory. */
|
|
8
|
+
homedir: string;
|
|
9
|
+
/** Read a macOS Keychain credential. Returns undefined on failure. */
|
|
10
|
+
readKeychainToken(service: string): Promise<string | undefined>;
|
|
11
|
+
/**
|
|
12
|
+
* Check whether a macOS Keychain entry exists for the given service/account
|
|
13
|
+
* without decrypting it (no user prompt, no value returned). Returns false
|
|
14
|
+
* on non-darwin or when the entry is absent.
|
|
15
|
+
*/
|
|
16
|
+
keychainEntryExists(service: string, account: string): Promise<boolean>;
|
|
17
|
+
/** Check if a path exists on the host filesystem. */
|
|
18
|
+
fileExists(absolutePath: string): Promise<boolean>;
|
|
19
|
+
}
|
|
20
|
+
export interface CredentialResult {
|
|
21
|
+
/** Env vars to merge into sandbox env. */
|
|
22
|
+
env: Record<string, string>;
|
|
23
|
+
/** Shell commands to run inside sandbox before first agent turn. */
|
|
24
|
+
setupCommands: string[];
|
|
25
|
+
/** Relative paths from host HOME to copy into sandbox HOME. */
|
|
26
|
+
copyFromHome: string[];
|
|
27
|
+
/**
|
|
28
|
+
* Relative paths from host HOME to symlink into sandbox HOME. Used when
|
|
29
|
+
* the target would be too large to copy or must stay in sync with the
|
|
30
|
+
* host (e.g. macOS `Library/Keychains`). Optional; defaults to none.
|
|
31
|
+
*/
|
|
32
|
+
linkFromHome?: string[];
|
|
33
|
+
}
|
|
34
|
+
/** Default ports using real process.env, Keychain, and filesystem. */
|
|
35
|
+
export declare function defaultPorts(): CredentialPorts;
|
|
36
|
+
export declare function resolveCredentials(agent: AgentName, userEnv: Record<string, string>, ports?: CredentialPorts): Promise<CredentialResult>;
|
|
@@ -0,0 +1,202 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Credential resolution — determines what auth env vars an agent needs.
|
|
3
|
+
*
|
|
4
|
+
* This module owns the single question: "given user intent and ambient
|
|
5
|
+
* credentials, what auth environment should the agent get?"
|
|
6
|
+
*
|
|
7
|
+
* Precedence (highest to lowest):
|
|
8
|
+
* 1. User-provided env vars (explicit intent — never overridden)
|
|
9
|
+
* 2. Host env vars (forwarded when compatible)
|
|
10
|
+
* 3. macOS Keychain OAuth (Claude only, skipped when explicit intent detected)
|
|
11
|
+
*
|
|
12
|
+
* All external dependencies (Keychain, host env, filesystem) are injectable
|
|
13
|
+
* via CredentialPorts for testability.
|
|
14
|
+
*/
|
|
15
|
+
import { execSync, execFileSync } from 'child_process';
|
|
16
|
+
import * as os from 'os';
|
|
17
|
+
import * as path from 'path';
|
|
18
|
+
import fs from 'fs-extra';
|
|
19
|
+
const EMPTY = { env: {}, setupCommands: [], copyFromHome: [] };
|
|
20
|
+
/** Default ports using real process.env, Keychain, and filesystem. */
|
|
21
|
+
export function defaultPorts() {
|
|
22
|
+
return {
|
|
23
|
+
hostEnv: (key) => process.env[key],
|
|
24
|
+
platform: process.platform,
|
|
25
|
+
homedir: os.homedir(),
|
|
26
|
+
async readKeychainToken(service) {
|
|
27
|
+
try {
|
|
28
|
+
const raw = execSync(`security find-generic-password -s "${service}" -w 2>/dev/null`, { encoding: 'utf8', timeout: 5000 }).trim();
|
|
29
|
+
const parsed = JSON.parse(raw);
|
|
30
|
+
return parsed?.claudeAiOauth?.accessToken || undefined;
|
|
31
|
+
}
|
|
32
|
+
catch {
|
|
33
|
+
return undefined;
|
|
34
|
+
}
|
|
35
|
+
},
|
|
36
|
+
async keychainEntryExists(service, account) {
|
|
37
|
+
if (process.platform !== 'darwin')
|
|
38
|
+
return false;
|
|
39
|
+
try {
|
|
40
|
+
// execFileSync with argv avoids shell interpretation entirely —
|
|
41
|
+
// no injection surface even if service/account become untrusted.
|
|
42
|
+
execFileSync('security', ['find-generic-password', '-s', service, '-a', account], { timeout: 5000, stdio: 'ignore' });
|
|
43
|
+
return true;
|
|
44
|
+
}
|
|
45
|
+
catch {
|
|
46
|
+
return false;
|
|
47
|
+
}
|
|
48
|
+
},
|
|
49
|
+
async fileExists(absolutePath) {
|
|
50
|
+
return fs.pathExists(absolutePath);
|
|
51
|
+
},
|
|
52
|
+
};
|
|
53
|
+
}
|
|
54
|
+
export async function resolveCredentials(agent, userEnv, ports) {
|
|
55
|
+
const p = ports ?? defaultPorts();
|
|
56
|
+
switch (agent) {
|
|
57
|
+
case 'claude':
|
|
58
|
+
return resolveClaude(userEnv, p);
|
|
59
|
+
case 'codex':
|
|
60
|
+
return resolveCodex(userEnv, p);
|
|
61
|
+
case 'cursor':
|
|
62
|
+
return resolveCursor(userEnv, p);
|
|
63
|
+
default:
|
|
64
|
+
return EMPTY;
|
|
65
|
+
}
|
|
66
|
+
}
|
|
67
|
+
async function resolveClaude(userEnv, ports) {
|
|
68
|
+
// User explicitly provided API key — trust it, nothing to add
|
|
69
|
+
if (userEnv.ANTHROPIC_API_KEY) {
|
|
70
|
+
return EMPTY;
|
|
71
|
+
}
|
|
72
|
+
// User provided a custom base URL → they're targeting a proxy.
|
|
73
|
+
// Keychain OAuth only works with direct Anthropic, so skip it
|
|
74
|
+
// and forward the host's API key instead.
|
|
75
|
+
if (userEnv.ANTHROPIC_BASE_URL) {
|
|
76
|
+
const hostKey = ports.hostEnv('ANTHROPIC_API_KEY');
|
|
77
|
+
if (!hostKey) {
|
|
78
|
+
throw new Error('ANTHROPIC_BASE_URL is set but no ANTHROPIC_API_KEY found. ' +
|
|
79
|
+
'Provide ANTHROPIC_API_KEY in env or set it in your host environment.');
|
|
80
|
+
}
|
|
81
|
+
return { env: { ANTHROPIC_API_KEY: hostKey }, setupCommands: [], copyFromHome: [] };
|
|
82
|
+
}
|
|
83
|
+
// No explicit Anthropic intent — try auto-resolution.
|
|
84
|
+
// Host-level proxy intent (ANTHROPIC_BASE_URL set on host env) makes the
|
|
85
|
+
// Keychain OAuth token unusable — it's scoped to direct Anthropic, not to
|
|
86
|
+
// a proxy. Skip the Keychain branch so the host-forward fallback wins.
|
|
87
|
+
const hostSignalsProxy = !!ports.hostEnv('ANTHROPIC_BASE_URL');
|
|
88
|
+
// On macOS, prefer Keychain OAuth (direct Anthropic token).
|
|
89
|
+
if (ports.platform === 'darwin' && !hostSignalsProxy) {
|
|
90
|
+
const token = await ports.readKeychainToken('Claude Code-credentials');
|
|
91
|
+
if (token) {
|
|
92
|
+
return { env: { ANTHROPIC_API_KEY: token }, setupCommands: [], copyFromHome: [] };
|
|
93
|
+
}
|
|
94
|
+
}
|
|
95
|
+
// Fallback: forward host API key and base URL.
|
|
96
|
+
const env = {};
|
|
97
|
+
const hostKey = ports.hostEnv('ANTHROPIC_API_KEY');
|
|
98
|
+
const hostBaseUrl = ports.hostEnv('ANTHROPIC_BASE_URL');
|
|
99
|
+
if (hostKey)
|
|
100
|
+
env.ANTHROPIC_API_KEY = hostKey;
|
|
101
|
+
if (hostBaseUrl)
|
|
102
|
+
env.ANTHROPIC_BASE_URL = hostBaseUrl;
|
|
103
|
+
return { env, setupCommands: [], copyFromHome: [] };
|
|
104
|
+
}
|
|
105
|
+
async function resolveCursor(userEnv, ports) {
|
|
106
|
+
// User explicitly provided API key — trust it, nothing to add
|
|
107
|
+
if (userEnv.CURSOR_API_KEY) {
|
|
108
|
+
return EMPTY;
|
|
109
|
+
}
|
|
110
|
+
// User provided a custom base URL → they're targeting a proxy.
|
|
111
|
+
// Forward the host's API key; error if missing.
|
|
112
|
+
if (userEnv.CURSOR_API_BASE_URL) {
|
|
113
|
+
const hostKey = ports.hostEnv('CURSOR_API_KEY');
|
|
114
|
+
if (!hostKey) {
|
|
115
|
+
throw new Error('CURSOR_API_BASE_URL is set but no CURSOR_API_KEY found. ' +
|
|
116
|
+
'Provide CURSOR_API_KEY in env or set it in your host environment.');
|
|
117
|
+
}
|
|
118
|
+
return { env: { CURSOR_API_KEY: hostKey }, setupCommands: [], copyFromHome: [] };
|
|
119
|
+
}
|
|
120
|
+
// Forward host key + base URL when user provided neither. OAuth via
|
|
121
|
+
// `cursor-agent login` remains the fallback when neither is set.
|
|
122
|
+
const env = {};
|
|
123
|
+
const hostKey = ports.hostEnv('CURSOR_API_KEY');
|
|
124
|
+
const hostBaseUrl = ports.hostEnv('CURSOR_API_BASE_URL');
|
|
125
|
+
if (hostKey)
|
|
126
|
+
env.CURSOR_API_KEY = hostKey;
|
|
127
|
+
if (hostBaseUrl)
|
|
128
|
+
env.CURSOR_API_BASE_URL = hostBaseUrl;
|
|
129
|
+
// On macOS, `cursor-agent login` stores OAuth tokens in the host's login
|
|
130
|
+
// keychain at `~/Library/Keychains`. The sandbox runs with an isolated
|
|
131
|
+
// HOME so the CLI looks in the sandbox's (empty) Library/Keychains and
|
|
132
|
+
// reports "Authentication required". Symlink the host keychain dir in
|
|
133
|
+
// so the token is visible. Only meaningful when no explicit API key is
|
|
134
|
+
// set — otherwise cursor-agent uses the env var and ignores the keychain.
|
|
135
|
+
const linkFromHome = [];
|
|
136
|
+
if (!hostKey) {
|
|
137
|
+
if (ports.platform === 'darwin') {
|
|
138
|
+
// cursor-agent stores OAuth tokens under the `cursor-access-token`
|
|
139
|
+
// service / `cursor-user` account. Probe before sandbox creation:
|
|
140
|
+
// if the user hasn't run `cursor-agent login`, surface that now
|
|
141
|
+
// rather than 17s into the first turn.
|
|
142
|
+
const loggedIn = await ports.keychainEntryExists('cursor-access-token', 'cursor-user');
|
|
143
|
+
if (!loggedIn) {
|
|
144
|
+
throw new Error('Cursor authentication required but no OAuth tokens were found in the login keychain. ' +
|
|
145
|
+
'Run `cursor-agent login` first, or set CURSOR_API_KEY in your environment.');
|
|
146
|
+
}
|
|
147
|
+
linkFromHome.push('Library/Keychains');
|
|
148
|
+
}
|
|
149
|
+
else {
|
|
150
|
+
// No API key and no keychain-equivalent we know how to wire up.
|
|
151
|
+
throw new Error('Cursor authentication required but CURSOR_API_KEY is not set. ' +
|
|
152
|
+
'Auto-detection via `cursor-agent login` is only supported on macOS. ' +
|
|
153
|
+
'On other platforms, generate an API key from the Cursor dashboard ' +
|
|
154
|
+
'and set CURSOR_API_KEY in your environment.');
|
|
155
|
+
}
|
|
156
|
+
}
|
|
157
|
+
return { env, setupCommands: [], copyFromHome: [], linkFromHome };
|
|
158
|
+
}
|
|
159
|
+
async function resolveCodex(userEnv, ports) {
|
|
160
|
+
const env = {};
|
|
161
|
+
const setupCommands = [];
|
|
162
|
+
const copyFromHome = [];
|
|
163
|
+
const hasUserKey = !!userEnv.OPENAI_API_KEY;
|
|
164
|
+
const hasUserBaseUrl = !!userEnv.OPENAI_BASE_URL;
|
|
165
|
+
// User provided a custom base URL → they're targeting a proxy.
|
|
166
|
+
// Forward the host key; error if missing.
|
|
167
|
+
if (hasUserBaseUrl && !hasUserKey) {
|
|
168
|
+
const hostKey = ports.hostEnv('OPENAI_API_KEY');
|
|
169
|
+
if (!hostKey) {
|
|
170
|
+
throw new Error('OPENAI_BASE_URL is set but no OPENAI_API_KEY found. ' +
|
|
171
|
+
'Provide OPENAI_API_KEY in env or set it in your host environment.');
|
|
172
|
+
}
|
|
173
|
+
env.OPENAI_API_KEY = hostKey;
|
|
174
|
+
}
|
|
175
|
+
// Forward host keys when user didn't provide them
|
|
176
|
+
if (!hasUserKey && !hasUserBaseUrl) {
|
|
177
|
+
const hostKey = ports.hostEnv('OPENAI_API_KEY');
|
|
178
|
+
const hostBaseUrl = ports.hostEnv('OPENAI_BASE_URL');
|
|
179
|
+
if (hostKey)
|
|
180
|
+
env.OPENAI_API_KEY = hostKey;
|
|
181
|
+
if (hostBaseUrl)
|
|
182
|
+
env.OPENAI_BASE_URL = hostBaseUrl;
|
|
183
|
+
}
|
|
184
|
+
// Determine if we have a key (user-provided or host-forwarded)
|
|
185
|
+
const hasKey = hasUserKey || !!env.OPENAI_API_KEY;
|
|
186
|
+
const hasBaseUrl = hasUserBaseUrl || !!env.OPENAI_BASE_URL;
|
|
187
|
+
if (hasKey) {
|
|
188
|
+
// Proxied OpenAI-compatible endpoints use a custom Codex provider that
|
|
189
|
+
// reads OPENAI_API_KEY directly from env, so no auth cache/login is needed.
|
|
190
|
+
if (!hasBaseUrl) {
|
|
191
|
+
setupCommands.push('if [ ! -d "$HOME/.codex" ] && [ -n "${OPENAI_API_KEY:-}" ]; then printenv OPENAI_API_KEY | codex login --with-api-key >/dev/null 2>&1; fi');
|
|
192
|
+
}
|
|
193
|
+
}
|
|
194
|
+
else {
|
|
195
|
+
// No key — check for cached auth.json
|
|
196
|
+
const authCachePath = path.join(ports.homedir, '.codex', 'auth.json');
|
|
197
|
+
if (await ports.fileExists(authCachePath)) {
|
|
198
|
+
copyFromHome.push('.codex/auth.json');
|
|
199
|
+
}
|
|
200
|
+
}
|
|
201
|
+
return { env, setupCommands, copyFromHome };
|
|
202
|
+
}
|
|
@@ -0,0 +1,34 @@
|
|
|
1
|
+
import type { MockMcpServerDescriptor } from '../core/mcp-mock.types.js';
|
|
2
|
+
/**
|
|
3
|
+
* Stdio-server shape pathgrade writes to `.pathgrade-mcp.json`. Lines up
|
|
4
|
+
* directly with the SDK's `McpStdioServerConfig` (`sdk.d.ts:1005`) — the
|
|
5
|
+
* optional `type: 'stdio'` discriminator is omitted because the JSON
|
|
6
|
+
* `writeMcpConfig` produces does not include it.
|
|
7
|
+
*/
|
|
8
|
+
export interface McpStdioEntry {
|
|
9
|
+
command: string;
|
|
10
|
+
args?: string[];
|
|
11
|
+
env?: Record<string, string>;
|
|
12
|
+
}
|
|
13
|
+
/** Object form of MCP server entries the SDK driver passes to `Options.mcpServers`. */
|
|
14
|
+
export type McpServersObject = Record<string, McpStdioEntry>;
|
|
15
|
+
export type McpSpec = {
|
|
16
|
+
configFile: string;
|
|
17
|
+
} | {
|
|
18
|
+
mock: MockMcpServerDescriptor | MockMcpServerDescriptor[];
|
|
19
|
+
};
|
|
20
|
+
export interface McpConfigResult {
|
|
21
|
+
mcpConfigPath: string | undefined;
|
|
22
|
+
}
|
|
23
|
+
export declare function writeMcpConfig(workspacePath: string, mcp: McpSpec | undefined): Promise<McpConfigResult>;
|
|
24
|
+
/**
|
|
25
|
+
* Read the JSON `writeMcpConfig` writes into `<workspace>/.pathgrade-mcp.json`
|
|
26
|
+
* and return the inner `mcpServers` object the Claude SDK driver hands to
|
|
27
|
+
* `Options.mcpServers`. Returns `undefined` when the file is absent (the
|
|
28
|
+
* "no MCP" path through the driver — the fixture didn't declare an `mcp` spec).
|
|
29
|
+
*
|
|
30
|
+
* The on-disk path is preserved because the Cursor agent driver still
|
|
31
|
+
* consumes it via `mcpConfigPath` (`src/agents/cursor.ts`) — Claude moves to
|
|
32
|
+
* the object form, Cursor stays on the file.
|
|
33
|
+
*/
|
|
34
|
+
export declare function loadMcpServersForSdk(workspacePath: string): Promise<McpServersObject | undefined>;
|