@wix/pathgrade 1.0.27 → 1.0.29
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +12 -19
- package/dist/adapters/jest/invocation-adapter.js +5 -1
- package/dist/adapters/jest/reporter.js +4 -1
- package/dist/adapters/jest/results.js +8 -0
- package/dist/adapters/node-test/index.d.ts +5 -0
- package/dist/adapters/node-test/index.js +35 -7
- package/dist/adapters/node-test/invocation-adapter.js +3 -1
- package/dist/adapters/node-test/runner-adapter.js +22 -15
- package/dist/adapters/vitest/reporter.js +4 -1
- package/dist/agents/claude.js +5 -4
- package/dist/agents/codex-app-server/agent.js +1 -51
- package/dist/agents/codex-app-server/turn-notifications.d.ts +6 -0
- package/dist/agents/codex-app-server/turn-notifications.js +51 -0
- package/dist/agents/codex-app-server/turn-state.d.ts +19 -0
- package/dist/agents/codex-app-server/turn-state.js +1 -0
- package/dist/agents/opencode/protocol.d.ts +7 -0
- package/dist/agents/opencode/protocol.js +47 -0
- package/dist/agents/opencode.js +1 -47
- package/dist/analytics/engine.js +5 -2
- package/dist/commands/report.js +16 -2
- package/dist/commands/run-args.d.ts +1 -0
- package/dist/commands/run-args.js +13 -0
- package/dist/commands/run-changed.js +5 -15
- package/dist/config/pathgrade.d.ts +3 -0
- package/dist/config/pathgrade.js +33 -1
- package/dist/internal/direct-mcp-v2/claude-direct-mcp.d.ts +2 -1
- package/dist/internal/direct-mcp-v2/claude-direct-mcp.js +35 -17
- package/dist/internal/direct-mcp-v2/claude-profile.d.ts +33 -7
- package/dist/internal/direct-mcp-v2/claude-profile.js +104 -72
- package/dist/internal/direct-mcp-v2/public-scenario-runtime.js +13 -16
- package/dist/pathgrade.js +10 -2
- package/dist/reporters/cli.js +13 -6
- package/dist/reporters/github-comment.js +12 -9
- package/dist/reporters/loader.d.ts +1 -0
- package/dist/reporters/loader.js +27 -2
- package/dist/reporters/report-summary.js +13 -5
- package/dist/reporting/core.d.ts +1 -0
- package/dist/reporting/core.js +183 -105
- package/dist/reporting/types.d.ts +19 -3
- package/dist/runners/adapter-loader.js +17 -12
- package/dist/runners/direct-reporter-attempts.d.ts +1 -0
- package/dist/runners/direct-reporter-attempts.js +7 -0
- package/dist/runners/invocation.d.ts +2 -0
- package/dist/runners/model-builders.js +1 -0
- package/dist/runners/model-validation.js +31 -0
- package/dist/runners/model.d.ts +4 -0
- package/dist/runners/orchestrator.d.ts +2 -0
- package/dist/runners/orchestrator.js +11 -1
- package/dist/runners/repeated-attempts.d.ts +7 -0
- package/dist/runners/repeated-attempts.js +149 -0
- package/dist/runners/repeated-invocation.d.ts +7 -0
- package/dist/runners/repeated-invocation.js +129 -0
- package/dist/runners/report-projection.js +16 -6
- package/dist/runners/vitest-adapter.js +10 -0
- package/dist/runners/vitest-invocation.js +2 -0
- package/dist/sdk/agent-runtime-options.d.ts +12 -0
- package/dist/sdk/agent-runtime-options.js +67 -0
- package/dist/sdk/agent.js +4 -57
- package/dist/sdk/case-context.js +7 -2
- package/dist/sdk/lifecycle.js +8 -3
- package/dist/sdk/result-capture.js +4 -1
- package/dist/types.d.ts +34 -5
- package/dist/viewer.html +19 -19
- package/package.json +2 -2
package/dist/analytics/engine.js
CHANGED
|
@@ -40,10 +40,10 @@ export class AnalyticsEngine {
|
|
|
40
40
|
for (const [task, data] of Object.entries(taskGroups)) {
|
|
41
41
|
const allReports = [...data.withSkill, ...data.withoutSkill];
|
|
42
42
|
const avgWith = data.withSkill.length > 0
|
|
43
|
-
? data.withSkill.reduce((a, b) => a + b
|
|
43
|
+
? data.withSkill.reduce((a, b) => a + reportMeanReward(b), 0) / data.withSkill.length
|
|
44
44
|
: 0;
|
|
45
45
|
const avgWithout = data.withoutSkill.length > 0
|
|
46
|
-
? data.withoutSkill.reduce((a, b) => a + b
|
|
46
|
+
? data.withoutSkill.reduce((a, b) => a + reportMeanReward(b), 0) / data.withoutSkill.length
|
|
47
47
|
: 0;
|
|
48
48
|
const allTrials = allReports.flatMap(r => r.trials);
|
|
49
49
|
const avgDurationMs = allTrials.length > 0
|
|
@@ -64,3 +64,6 @@ export class AnalyticsEngine {
|
|
|
64
64
|
return stats;
|
|
65
65
|
}
|
|
66
66
|
}
|
|
67
|
+
function reportMeanReward(report) {
|
|
68
|
+
return report.mean_reward ?? report.pass_rate ?? 0;
|
|
69
|
+
}
|
package/dist/commands/report.js
CHANGED
|
@@ -43,10 +43,24 @@ function isPathgradeReport(value) {
|
|
|
43
43
|
if (!value || typeof value !== 'object')
|
|
44
44
|
return false;
|
|
45
45
|
const v = value;
|
|
46
|
-
return (v.version === 1 &&
|
|
46
|
+
return ((v.version === 1 || v.version === 2) &&
|
|
47
47
|
typeof v.overall_pass_rate === 'number' &&
|
|
48
48
|
(v.status === 'pass' || v.status === 'fail') &&
|
|
49
|
-
Array.isArray(v.groups)
|
|
49
|
+
Array.isArray(v.groups) &&
|
|
50
|
+
(v.version === 1 || (typeof v.overall_mean_reward === 'number'
|
|
51
|
+
&& hasValidAttemptCounts(v)
|
|
52
|
+
&& v.groups.every(group => typeof group.mean_reward === 'number'))));
|
|
53
|
+
}
|
|
54
|
+
function hasValidAttemptCounts(report) {
|
|
55
|
+
const requested = report.attempts_requested;
|
|
56
|
+
const completed = report.attempts_completed;
|
|
57
|
+
return typeof requested === 'number'
|
|
58
|
+
&& Number.isSafeInteger(requested)
|
|
59
|
+
&& requested >= 1
|
|
60
|
+
&& typeof completed === 'number'
|
|
61
|
+
&& Number.isSafeInteger(completed)
|
|
62
|
+
&& completed >= 0
|
|
63
|
+
&& completed <= requested;
|
|
50
64
|
}
|
|
51
65
|
async function loadReport(resolvedPath) {
|
|
52
66
|
if (!(await fs.pathExists(resolvedPath))) {
|
|
@@ -14,6 +14,7 @@ export function parsePathgradeRunArgs(args) {
|
|
|
14
14
|
let since;
|
|
15
15
|
let changedFilesPath;
|
|
16
16
|
let adapterName;
|
|
17
|
+
let attempts;
|
|
17
18
|
let passthrough = false;
|
|
18
19
|
for (const arg of args) {
|
|
19
20
|
if (passthrough) {
|
|
@@ -44,6 +45,10 @@ export function parsePathgradeRunArgs(args) {
|
|
|
44
45
|
adapterName = arg.slice('--adapter='.length);
|
|
45
46
|
continue;
|
|
46
47
|
}
|
|
48
|
+
if (arg.startsWith('--attempts=')) {
|
|
49
|
+
attempts = parseAttempts(arg.slice('--attempts='.length));
|
|
50
|
+
continue;
|
|
51
|
+
}
|
|
47
52
|
if (arg.startsWith('--since=')) {
|
|
48
53
|
since = arg.slice('--since='.length);
|
|
49
54
|
continue;
|
|
@@ -68,8 +73,16 @@ export function parsePathgradeRunArgs(args) {
|
|
|
68
73
|
changed,
|
|
69
74
|
quiet,
|
|
70
75
|
adapterName,
|
|
76
|
+
attempts,
|
|
71
77
|
since,
|
|
72
78
|
changedFilesPath,
|
|
73
79
|
};
|
|
74
80
|
return warnings.length > 0 ? { ...base, warnings } : base;
|
|
75
81
|
}
|
|
82
|
+
function parseAttempts(value) {
|
|
83
|
+
const attempts = Number(value);
|
|
84
|
+
if (!Number.isSafeInteger(attempts) || attempts < 1) {
|
|
85
|
+
throw new Error('pathgrade: --attempts must be an integer greater than or equal to 1');
|
|
86
|
+
}
|
|
87
|
+
return attempts;
|
|
88
|
+
}
|
|
@@ -27,13 +27,13 @@ export async function runChanged(opts) {
|
|
|
27
27
|
const runnerEnv = buildRunnerEnv(parsed, {
|
|
28
28
|
PATHGRADE_SELECTION_INVOCATION_ID: selectionInvocationId,
|
|
29
29
|
});
|
|
30
|
-
const configPath = findVitestConfigArg(parsed.runnerArgs);
|
|
31
30
|
let config;
|
|
32
31
|
let runnerInvocation;
|
|
33
32
|
try {
|
|
34
33
|
config = await resolvePathgradeConfig({
|
|
35
34
|
cwd,
|
|
36
|
-
|
|
35
|
+
cli: parsed.attempts === undefined ? undefined : { attempts: parsed.attempts },
|
|
36
|
+
runnerArgs: parsed.runnerArgs,
|
|
37
37
|
warn: w => {
|
|
38
38
|
if (!parsed.quiet)
|
|
39
39
|
process.stderr.write(`${w}\n`);
|
|
@@ -107,6 +107,7 @@ export async function runChanged(opts) {
|
|
|
107
107
|
totalEvals: evalFiles.length,
|
|
108
108
|
changedCount: changedFiles.length,
|
|
109
109
|
result,
|
|
110
|
+
attempts: config.attempts,
|
|
110
111
|
});
|
|
111
112
|
}
|
|
112
113
|
// Persist the sidecar immediately — even on empty selection, so the
|
|
@@ -128,7 +129,7 @@ export async function runChanged(opts) {
|
|
|
128
129
|
cwd,
|
|
129
130
|
runnerArgs,
|
|
130
131
|
selectedFiles,
|
|
131
|
-
env: runnerEnv,
|
|
132
|
+
env: { ...runnerEnv, PATHGRADE_ATTEMPT_COUNT: String(config.attempts) },
|
|
132
133
|
});
|
|
133
134
|
}
|
|
134
135
|
function printRunStartSummary(input) {
|
|
@@ -140,6 +141,7 @@ function printRunStartSummary(input) {
|
|
|
140
141
|
const globalLabel = result.globalMatch ? `\`${result.globalMatch}\`` : 'none';
|
|
141
142
|
process.stderr.write(` global matches: ${globalLabel}\n`);
|
|
142
143
|
process.stderr.write(` selected: ${result.selected.length} / ${totalEvals} evals\n`);
|
|
144
|
+
process.stderr.write(` attempts: ${input.attempts} per selected case\n`);
|
|
143
145
|
for (const entry of result.selected) {
|
|
144
146
|
process.stderr.write(` ${entry.file}\n`);
|
|
145
147
|
}
|
|
@@ -158,15 +160,3 @@ function readChangedFilesList(filePath) {
|
|
|
158
160
|
function errMsg(err) {
|
|
159
161
|
return err instanceof Error ? err.message : String(err);
|
|
160
162
|
}
|
|
161
|
-
function findVitestConfigArg(args) {
|
|
162
|
-
for (let i = 0; i < args.length; i++) {
|
|
163
|
-
const arg = args[i];
|
|
164
|
-
if (arg === '--config' || arg === '-c')
|
|
165
|
-
return args[i + 1];
|
|
166
|
-
if (arg.startsWith('--config='))
|
|
167
|
-
return arg.slice('--config='.length);
|
|
168
|
-
if (arg.startsWith('-c='))
|
|
169
|
-
return arg.slice('-c='.length);
|
|
170
|
-
}
|
|
171
|
-
return undefined;
|
|
172
|
-
}
|
|
@@ -1,4 +1,5 @@
|
|
|
1
1
|
export interface PathgradeConfig {
|
|
2
|
+
attempts?: number;
|
|
2
3
|
runner?: {
|
|
3
4
|
adapter?: string;
|
|
4
5
|
args?: string[];
|
|
@@ -18,6 +19,7 @@ export interface PathgradeConfig {
|
|
|
18
19
|
};
|
|
19
20
|
}
|
|
20
21
|
export interface ResolvedPathgradeConfig {
|
|
22
|
+
attempts: number;
|
|
21
23
|
runner: {
|
|
22
24
|
adapter: string;
|
|
23
25
|
args: string[];
|
|
@@ -44,5 +46,6 @@ export declare function resolvePathgradeConfig(input: {
|
|
|
44
46
|
cli?: PathgradeConfig;
|
|
45
47
|
configPath?: string;
|
|
46
48
|
legacyVitestConfigPath?: string;
|
|
49
|
+
runnerArgs?: readonly string[];
|
|
47
50
|
warn?: (message: string) => void;
|
|
48
51
|
}): Promise<ResolvedPathgradeConfig>;
|
package/dist/config/pathgrade.js
CHANGED
|
@@ -17,6 +17,7 @@ const PATHGRADE_CONFIG_CANDIDATES = [
|
|
|
17
17
|
];
|
|
18
18
|
export function defaultPathgradeConfig() {
|
|
19
19
|
return {
|
|
20
|
+
attempts: 1,
|
|
20
21
|
runner: {
|
|
21
22
|
adapter: 'vitest',
|
|
22
23
|
args: [],
|
|
@@ -35,13 +36,30 @@ export function defaultPathgradeConfig() {
|
|
|
35
36
|
}
|
|
36
37
|
export async function resolvePathgradeConfig(input) {
|
|
37
38
|
const fileConfig = await loadPathgradeConfigFile(input.cwd, input.configPath);
|
|
38
|
-
const legacyConfig = await loadLegacyVitestConfig(input.cwd, input.legacyVitestConfigPath
|
|
39
|
+
const legacyConfig = await loadLegacyVitestConfig(input.cwd, input.legacyVitestConfigPath ?? findVitestConfigArg([
|
|
40
|
+
...(fileConfig?.runner?.args ?? []),
|
|
41
|
+
...(input.runnerArgs ?? []),
|
|
42
|
+
]), input.warn);
|
|
39
43
|
return mergePathgradeConfig(mergePathgradeConfig(mergePathgradeConfig(defaultPathgradeConfig(), legacyConfig), fileConfig), input.cli);
|
|
40
44
|
}
|
|
45
|
+
function findVitestConfigArg(args) {
|
|
46
|
+
let configPath;
|
|
47
|
+
for (let i = 0; i < args.length; i++) {
|
|
48
|
+
const arg = args[i];
|
|
49
|
+
if (arg === '--config' || arg === '-c')
|
|
50
|
+
configPath = args[i + 1];
|
|
51
|
+
else if (arg.startsWith('--config='))
|
|
52
|
+
configPath = arg.slice('--config='.length);
|
|
53
|
+
else if (arg.startsWith('-c='))
|
|
54
|
+
configPath = arg.slice('-c='.length);
|
|
55
|
+
}
|
|
56
|
+
return configPath;
|
|
57
|
+
}
|
|
41
58
|
function mergePathgradeConfig(base, override) {
|
|
42
59
|
if (!override)
|
|
43
60
|
return base;
|
|
44
61
|
return {
|
|
62
|
+
attempts: override.attempts ?? base.attempts,
|
|
45
63
|
runner: {
|
|
46
64
|
adapter: override.runner?.adapter ?? base.runner.adapter,
|
|
47
65
|
args: override.runner?.args ?? base.runner.args,
|
|
@@ -91,6 +109,7 @@ function validatePathgradeConfig(value, label) {
|
|
|
91
109
|
throw invalidConfig(label, 'default export must be an object');
|
|
92
110
|
}
|
|
93
111
|
validateOptionalObject(value.runner, label, 'runner');
|
|
112
|
+
validateOptionalPositiveInteger(value.attempts, label, 'attempts');
|
|
94
113
|
const runner = asOptionalObject(value.runner);
|
|
95
114
|
validateOptionalString(runner?.adapter, label, 'runner.adapter');
|
|
96
115
|
validateOptionalStringArray(runner?.args, label, 'runner.args');
|
|
@@ -139,6 +158,11 @@ function validateOptionalNumber(value, label, field) {
|
|
|
139
158
|
throw invalidConfig(label, `${field} must be a number`);
|
|
140
159
|
}
|
|
141
160
|
}
|
|
161
|
+
function validateOptionalPositiveInteger(value, label, field) {
|
|
162
|
+
if (value !== undefined && (!Number.isSafeInteger(value) || Number(value) < 1)) {
|
|
163
|
+
throw invalidConfig(label, `${field} must be an integer greater than or equal to 1`);
|
|
164
|
+
}
|
|
165
|
+
}
|
|
142
166
|
function invalidConfig(label, reason) {
|
|
143
167
|
return new InvalidPathgradeConfigError(`pathgrade: invalid ${label}: ${reason}`);
|
|
144
168
|
}
|
|
@@ -179,6 +203,14 @@ async function loadLegacyVitestConfig(cwd, configPath, warn = () => { }) {
|
|
|
179
203
|
if (!isObject(opts))
|
|
180
204
|
return undefined;
|
|
181
205
|
return {
|
|
206
|
+
...(opts.reporter === 'cli' || opts.reporter === 'browser' || opts.reporter === 'json'
|
|
207
|
+
? { reporter: opts.reporter }
|
|
208
|
+
: {}),
|
|
209
|
+
...(typeof opts.diagnostics === 'boolean' ? { diagnostics: opts.diagnostics } : {}),
|
|
210
|
+
...(typeof opts.verbose === 'boolean' ? { verbose: opts.verbose } : {}),
|
|
211
|
+
...(isObject(opts.ci) && typeof opts.ci.threshold === 'number'
|
|
212
|
+
? { ci: { threshold: opts.ci.threshold } }
|
|
213
|
+
: {}),
|
|
182
214
|
evals: {
|
|
183
215
|
...(Array.isArray(opts.include) ? { include: opts.include } : {}),
|
|
184
216
|
...(Array.isArray(opts.exclude) ? { exclude: opts.exclude } : {}),
|
|
@@ -2,7 +2,7 @@ import { type McpSdkServerConfigWithInstance, type Options as ClaudeSdkOptions,
|
|
|
2
2
|
import type { CallToolResult } from '@modelcontextprotocol/sdk/types.js';
|
|
3
3
|
import { type CleanupReport } from './cleanup.js';
|
|
4
4
|
import { type MaterializedProjection } from './materialize.js';
|
|
5
|
-
import type { EvidenceEnvelope, ScenarioArtifact, ScenarioMachineV2 } from './types.js';
|
|
5
|
+
import type { AdmittedRuntimeProfile, EvidenceEnvelope, ScenarioArtifact, ScenarioMachineV2 } from './types.js';
|
|
6
6
|
export declare const CLAUDE_DIRECT_MCP_FLAG = "PATHGRADE_INTERNAL_CLAUDE_DIRECT_MCP_V2";
|
|
7
7
|
export type ClaudeDirectMcpChecks = {
|
|
8
8
|
runtimeAdmitted: boolean;
|
|
@@ -19,6 +19,7 @@ export type ClaudeDirectMcpChecks = {
|
|
|
19
19
|
export type ClaudeDirectMcpTrialResult = {
|
|
20
20
|
selected: true;
|
|
21
21
|
profileFingerprint: string;
|
|
22
|
+
runtimeProfile: AdmittedRuntimeProfile;
|
|
22
23
|
artifact: ScenarioArtifact;
|
|
23
24
|
evidence: readonly EvidenceEnvelope[];
|
|
24
25
|
checks: ClaudeDirectMcpChecks;
|
|
@@ -11,19 +11,35 @@ import { compileScenario } from './compiler.js';
|
|
|
11
11
|
import { IdempotentTrialCleaner, persistTrialArtifact, TrialExecutionError, TrialResourceRegistry, scanBearerSurfaces } from './cleanup.js';
|
|
12
12
|
import { PrivateReplayVault, replayPrivateEvidence, verifyEvidenceChain } from './evidence.js';
|
|
13
13
|
import { ScenarioHostReducer } from './host-reducer.js';
|
|
14
|
+
import { sha256Canonical } from './json.js';
|
|
14
15
|
import { materializeProjection } from './materialize.js';
|
|
15
16
|
import { projectEndpoints } from './projection.js';
|
|
16
|
-
import {
|
|
17
|
+
import { admitRuntimeLock } from './runtime-lock.js';
|
|
17
18
|
import { createInvocationBarrier, startScenarioHttpHost } from './scenario-http-host.js';
|
|
18
|
-
import {
|
|
19
|
+
import { observeClaudeRuntimeControlPlane, resolveClaudeScenarioExecutable, selectClaudeRuntimeTarget, } from './claude-profile.js';
|
|
19
20
|
export const CLAUDE_DIRECT_MCP_FLAG = 'PATHGRADE_INTERNAL_CLAUDE_DIRECT_MCP_V2';
|
|
20
21
|
const disabledProfiles = new Map();
|
|
22
|
+
function runtimeLockFingerprint(lock) {
|
|
23
|
+
return sha256Canonical(lock);
|
|
24
|
+
}
|
|
21
25
|
export function isClaudeDirectMcpSelected(environment = process.env) {
|
|
22
|
-
|
|
23
|
-
|
|
26
|
+
if (environment[CLAUDE_DIRECT_MCP_FLAG] !== '1')
|
|
27
|
+
return false;
|
|
28
|
+
try {
|
|
29
|
+
const fingerprint = selectClaudeRuntimeTarget().lock;
|
|
30
|
+
return !disabledProfiles.has(runtimeLockFingerprint(fingerprint));
|
|
31
|
+
}
|
|
32
|
+
catch {
|
|
33
|
+
return false;
|
|
34
|
+
}
|
|
24
35
|
}
|
|
25
36
|
export function claudeDirectMcpDisableReason() {
|
|
26
|
-
|
|
37
|
+
try {
|
|
38
|
+
return disabledProfiles.get(runtimeLockFingerprint(selectClaudeRuntimeTarget().lock));
|
|
39
|
+
}
|
|
40
|
+
catch {
|
|
41
|
+
return undefined;
|
|
42
|
+
}
|
|
27
43
|
}
|
|
28
44
|
/** Test-only reset for proving the fail-closed process latch. */
|
|
29
45
|
// oxlint-disable-next-line inhuman/no-empty-wrappers -- This test seam exposes Map.clear without exposing mutable production state.
|
|
@@ -82,17 +98,18 @@ export function createClaudeParentBridges(options) {
|
|
|
82
98
|
}
|
|
83
99
|
export async function runClaudeDirectMcpTrial(options) {
|
|
84
100
|
const environment = options.environment ?? process.env;
|
|
85
|
-
const
|
|
101
|
+
const target = selectClaudeRuntimeTarget();
|
|
102
|
+
const fingerprint = runtimeLockFingerprint(target.lock);
|
|
86
103
|
if (!isClaudeDirectMcpSelected(environment)) {
|
|
87
|
-
const reason = disabledProfiles.get(
|
|
104
|
+
const reason = disabledProfiles.get(fingerprint);
|
|
88
105
|
throw new Error(reason ? `Claude direct MCP disabled for admitted profile: ${reason}` : `${CLAUDE_DIRECT_MCP_FLAG} is disabled`);
|
|
89
106
|
}
|
|
90
107
|
try {
|
|
91
|
-
return await executeTrial({ ...options, environment });
|
|
108
|
+
return await executeTrial({ ...options, environment, target });
|
|
92
109
|
}
|
|
93
110
|
catch (error) {
|
|
94
111
|
const reason = error instanceof Error ? error.message : String(error);
|
|
95
|
-
disabledProfiles.set(
|
|
112
|
+
disabledProfiles.set(fingerprint, reason.slice(0, 500));
|
|
96
113
|
throw error;
|
|
97
114
|
}
|
|
98
115
|
}
|
|
@@ -179,21 +196,21 @@ async function executeTrial(options) {
|
|
|
179
196
|
zeroBearer: () => { host.bearer.fill(0); macKey.fill(0); replayVault.dispose(); },
|
|
180
197
|
});
|
|
181
198
|
try {
|
|
182
|
-
const admission = await
|
|
199
|
+
const admission = await admitRuntimeLock(options.target.lock, {
|
|
183
200
|
provider: 'claude',
|
|
184
201
|
platform: process.platform,
|
|
185
202
|
architecture: process.arch,
|
|
203
|
+
resolvedExecutable: await resolveClaudeScenarioExecutable(options.target),
|
|
186
204
|
environment: runtimeEnv,
|
|
187
205
|
forbiddenSecrets: [host.bearer.toString(), `Bearer ${host.bearer.toString()}`],
|
|
188
206
|
initializeControlPlane: async () => {
|
|
189
207
|
await options.initializeControlPlane?.();
|
|
190
|
-
|
|
191
|
-
|
|
192
|
-
|
|
208
|
+
return observeClaudeRuntimeControlPlane({
|
|
209
|
+
createSdkMcpServer,
|
|
210
|
+
query: options.query ?? sdkQuery,
|
|
211
|
+
});
|
|
193
212
|
},
|
|
194
213
|
});
|
|
195
|
-
if (!admission.ok)
|
|
196
|
-
throw new Error(`RUNTIME_NOT_ADMITTED: ${admission.reason}`);
|
|
197
214
|
reducer.markAdmitted();
|
|
198
215
|
const projected = projectEndpoints(host.endpoints, admission.profile);
|
|
199
216
|
if (!Array.isArray(projected))
|
|
@@ -212,7 +229,7 @@ async function executeTrial(options) {
|
|
|
212
229
|
}
|
|
213
230
|
reducer.markReady();
|
|
214
231
|
const query = options.query ?? sdkQuery;
|
|
215
|
-
const queryOptions = { query, runtimeEnv, workspacePath, model: options.model, profilePath:
|
|
232
|
+
const queryOptions = { query, runtimeEnv, workspacePath, model: options.model, profilePath: admission.commandPath, mcpServers: bridges, spawnClaudeCodeProcess };
|
|
216
233
|
reducer.beginTurn('direct-call', 'claude');
|
|
217
234
|
const normal = await runQuery(queryOptions, `Invoke mcp__demo__${effectfulCase.pureToolId} exactly once with ${JSON.stringify(effectfulCase.pureArgs)}, then invoke mcp__demo__${effectfulCase.effectfulToolId} exactly once with ${JSON.stringify(effectfulCase.errorArgs)}. Wait for both results and do not use any other tool.`);
|
|
218
235
|
if (normal.status !== 'success')
|
|
@@ -262,7 +279,8 @@ async function executeTrial(options) {
|
|
|
262
279
|
throw new Error(`CLEANUP_FAILED: ${JSON.stringify(cleanup.failures)}`);
|
|
263
280
|
return {
|
|
264
281
|
selected: true,
|
|
265
|
-
profileFingerprint:
|
|
282
|
+
profileFingerprint: admission.identity.lockFingerprint,
|
|
283
|
+
runtimeProfile: admission.profile,
|
|
266
284
|
artifact: compiled.artifact,
|
|
267
285
|
evidence: reducer.ledger.events(),
|
|
268
286
|
checks: { runtimeAdmitted: true, secretFreeProject: true, directCall: true, structuredToolError: true, fencedCancellation: true, providerTerminal: true, hostQuiescent: hostQuiescentBeforeCleanup, postCancelReuse: true, privateReplay: true, cleanup: true },
|
|
@@ -1,9 +1,35 @@
|
|
|
1
|
-
import type { RuntimeLock } from './runtime-lock.js';
|
|
2
|
-
import type { JsonObject
|
|
1
|
+
import type { RuntimeLock, RuntimeLockControlPlane } from './runtime-lock.js';
|
|
2
|
+
import type { JsonObject } from './types.js';
|
|
3
3
|
export declare const CLAUDE_DIRECT_MCP_NORMALIZED_HANDSHAKE: JsonObject;
|
|
4
|
-
|
|
5
|
-
|
|
6
|
-
|
|
4
|
+
export declare const CLAUDE_DIRECT_MCP_CAPABILITIES: {
|
|
5
|
+
readonly streamableHttp: true;
|
|
6
|
+
readonly controlPlaneHeaders: true;
|
|
7
|
+
readonly parentOwnedSdkMcp: true;
|
|
8
|
+
readonly cancellation: "abort-controller";
|
|
9
|
+
};
|
|
10
|
+
/** Code-owned relocatable identity lock captured by RFC 0006. */
|
|
7
11
|
export declare const CLAUDE_DIRECT_MCP_DARWIN_ARM64_LOCK: RuntimeLock;
|
|
8
|
-
/**
|
|
9
|
-
export declare
|
|
12
|
+
/** Provisional Linux x64 glibc lock; CI re-verifies every artifact before admission. */
|
|
13
|
+
export declare const CLAUDE_DIRECT_MCP_LINUX_X64_GLIBC_LOCK: RuntimeLock;
|
|
14
|
+
export type ClaudeRuntimeTarget = {
|
|
15
|
+
id: 'darwin-arm64' | 'linux-x64-glibc';
|
|
16
|
+
platform: 'darwin' | 'linux';
|
|
17
|
+
architecture: 'arm64' | 'x64';
|
|
18
|
+
libc?: 'glibc';
|
|
19
|
+
packageName: string;
|
|
20
|
+
lock: RuntimeLock;
|
|
21
|
+
};
|
|
22
|
+
export declare const CLAUDE_RUNTIME_TARGETS: readonly ClaudeRuntimeTarget[];
|
|
23
|
+
export type ClaudeRuntimeEnvironment = {
|
|
24
|
+
platform: string;
|
|
25
|
+
architecture: string;
|
|
26
|
+
libc?: string;
|
|
27
|
+
};
|
|
28
|
+
export declare function detectClaudeRuntimeEnvironment(): ClaudeRuntimeEnvironment;
|
|
29
|
+
export declare function selectClaudeRuntimeTarget(environment?: ClaudeRuntimeEnvironment): ClaudeRuntimeTarget;
|
|
30
|
+
/** Resolve the selected optional-package binary through Node package resolution. */
|
|
31
|
+
export declare function resolveClaudeScenarioExecutable(target?: ClaudeRuntimeTarget): Promise<string>;
|
|
32
|
+
export declare function observeClaudeRuntimeControlPlane(options: {
|
|
33
|
+
createSdkMcpServer: unknown;
|
|
34
|
+
query: unknown;
|
|
35
|
+
}): RuntimeLockControlPlane;
|
|
@@ -1,5 +1,5 @@
|
|
|
1
|
-
import { access, realpath } from 'node:fs/promises';
|
|
2
1
|
import { constants } from 'node:fs';
|
|
2
|
+
import { access, realpath } from 'node:fs/promises';
|
|
3
3
|
import { createRequire } from 'node:module';
|
|
4
4
|
import { dirname, join } from 'node:path';
|
|
5
5
|
export const CLAUDE_DIRECT_MCP_NORMALIZED_HANDSHAKE = {
|
|
@@ -13,85 +13,117 @@ export const CLAUDE_DIRECT_MCP_NORMALIZED_HANDSHAKE = {
|
|
|
13
13
|
cancellation: 'abort-controller',
|
|
14
14
|
sdkMethods: ['createSdkMcpServer', 'query'],
|
|
15
15
|
};
|
|
16
|
-
|
|
17
|
-
|
|
18
|
-
|
|
19
|
-
|
|
20
|
-
|
|
21
|
-
|
|
22
|
-
|
|
23
|
-
|
|
24
|
-
|
|
16
|
+
export const CLAUDE_DIRECT_MCP_CAPABILITIES = {
|
|
17
|
+
streamableHttp: true,
|
|
18
|
+
controlPlaneHeaders: true,
|
|
19
|
+
parentOwnedSdkMcp: true,
|
|
20
|
+
cancellation: 'abort-controller',
|
|
21
|
+
};
|
|
22
|
+
const SDK_SHA256 = '48bde6aeabf7e71ad5528bf52c8feb1642c21f505ea2495c70f39db7df226d97';
|
|
23
|
+
const ZOD_SHA256 = '2a3455cecff4f7021c92d0a3e2e5fc170c0448dc5cc261160127e08f4888a9c2';
|
|
24
|
+
const NORMALIZED_HANDSHAKE_SHA256 = '990b779b75145c8d7bcc8cfebcb5b9c1aae30eb1436393b38e7e997941ecc162';
|
|
25
|
+
const CLAUDE_VERSION = '2.1.141 (Claude Code)';
|
|
26
|
+
const COMMAND_ARTIFACT = 'claude-code-bundled-2.1.141';
|
|
27
|
+
function claudeLock(options) {
|
|
28
|
+
return {
|
|
29
|
+
format: 'pathgrade-direct-mcp-runtime-lock-v1',
|
|
30
|
+
provider: 'claude',
|
|
31
|
+
platform: options.platform,
|
|
32
|
+
architecture: options.architecture,
|
|
33
|
+
controlPlane: 'native-sdk-parent-bridge',
|
|
34
|
+
version: CLAUDE_VERSION,
|
|
35
|
+
commandArtifact: COMMAND_ARTIFACT,
|
|
25
36
|
args: [],
|
|
26
37
|
versionArgs: ['--version'],
|
|
27
|
-
|
|
28
|
-
|
|
29
|
-
|
|
30
|
-
|
|
31
|
-
|
|
32
|
-
|
|
33
|
-
|
|
34
|
-
|
|
35
|
-
|
|
36
|
-
|
|
37
|
-
|
|
38
|
-
sha256: '31ac95bb19a33b1d0cddd3f3ff594bf8bfd2be5051cd2af7867109641cab705e',
|
|
39
|
-
},
|
|
40
|
-
{
|
|
41
|
-
label: 'zod-4.3.6-commonjs-entry',
|
|
42
|
-
absolutePath: '/Users/nadavlac/projects/agent-evals/node_modules/zod/index.cjs',
|
|
43
|
-
sha256: '2a3455cecff4f7021c92d0a3e2e5fc170c0448dc5cc261160127e08f4888a9c2',
|
|
44
|
-
},
|
|
45
|
-
],
|
|
46
|
-
requiredCapabilities: {
|
|
47
|
-
streamableHttp: true,
|
|
48
|
-
controlPlaneHeaders: true,
|
|
49
|
-
parentOwnedSdkMcp: true,
|
|
50
|
-
cancellation: 'abort-controller',
|
|
51
|
-
},
|
|
52
|
-
profileFingerprint: '463d1485d70e542cf2a356cae007469a7d94d0e848a26d9e9b7470dbc5262104',
|
|
53
|
-
runtimeCapabilityFingerprint: '990b779b75145c8d7bcc8cfebcb5b9c1aae30eb1436393b38e7e997941ecc162',
|
|
54
|
-
};
|
|
55
|
-
/** Code-owned relocatable identity lock for the admitted public Claude SDK slice. */
|
|
56
|
-
export const CLAUDE_DIRECT_MCP_DARWIN_ARM64_LOCK = {
|
|
57
|
-
format: 'pathgrade-direct-mcp-runtime-lock-v1',
|
|
58
|
-
provider: 'claude',
|
|
38
|
+
artifacts: [
|
|
39
|
+
{ id: 'claude-agent-sdk-0.2.141', relativePath: '@anthropic-ai/claude-agent-sdk/sdk.mjs', sha256: SDK_SHA256 },
|
|
40
|
+
{ id: COMMAND_ARTIFACT, relativePath: `${options.packageName}/claude`, sha256: options.commandSha256 },
|
|
41
|
+
{ id: 'zod-4.3.6-commonjs-entry', relativePath: 'zod/index.cjs', sha256: ZOD_SHA256 },
|
|
42
|
+
],
|
|
43
|
+
capabilities: CLAUDE_DIRECT_MCP_CAPABILITIES,
|
|
44
|
+
normalizedHandshakeSha256: NORMALIZED_HANDSHAKE_SHA256,
|
|
45
|
+
};
|
|
46
|
+
}
|
|
47
|
+
/** Code-owned relocatable identity lock captured by RFC 0006. */
|
|
48
|
+
export const CLAUDE_DIRECT_MCP_DARWIN_ARM64_LOCK = claudeLock({
|
|
59
49
|
platform: 'darwin',
|
|
60
50
|
architecture: 'arm64',
|
|
61
|
-
|
|
62
|
-
|
|
63
|
-
|
|
64
|
-
|
|
65
|
-
|
|
66
|
-
|
|
67
|
-
|
|
68
|
-
|
|
69
|
-
|
|
70
|
-
|
|
71
|
-
|
|
72
|
-
|
|
73
|
-
|
|
74
|
-
|
|
75
|
-
|
|
76
|
-
|
|
77
|
-
|
|
78
|
-
|
|
79
|
-
|
|
80
|
-
|
|
81
|
-
|
|
82
|
-
|
|
83
|
-
|
|
84
|
-
|
|
85
|
-
|
|
86
|
-
|
|
87
|
-
|
|
88
|
-
|
|
89
|
-
|
|
51
|
+
packageName: '@anthropic-ai/claude-agent-sdk-darwin-arm64',
|
|
52
|
+
commandSha256: '31ac95bb19a33b1d0cddd3f3ff594bf8bfd2be5051cd2af7867109641cab705e',
|
|
53
|
+
});
|
|
54
|
+
/** Provisional Linux x64 glibc lock; CI re-verifies every artifact before admission. */
|
|
55
|
+
export const CLAUDE_DIRECT_MCP_LINUX_X64_GLIBC_LOCK = claudeLock({
|
|
56
|
+
platform: 'linux',
|
|
57
|
+
architecture: 'x64',
|
|
58
|
+
packageName: '@anthropic-ai/claude-agent-sdk-linux-x64',
|
|
59
|
+
commandSha256: '832be26e8f15b2ae99e520a22b034fc4bfad1cb5b84de6b706487072c56bb42e',
|
|
60
|
+
});
|
|
61
|
+
export const CLAUDE_RUNTIME_TARGETS = [
|
|
62
|
+
{
|
|
63
|
+
id: 'darwin-arm64',
|
|
64
|
+
platform: 'darwin',
|
|
65
|
+
architecture: 'arm64',
|
|
66
|
+
packageName: '@anthropic-ai/claude-agent-sdk-darwin-arm64',
|
|
67
|
+
lock: CLAUDE_DIRECT_MCP_DARWIN_ARM64_LOCK,
|
|
68
|
+
},
|
|
69
|
+
{
|
|
70
|
+
id: 'linux-x64-glibc',
|
|
71
|
+
platform: 'linux',
|
|
72
|
+
architecture: 'x64',
|
|
73
|
+
libc: 'glibc',
|
|
74
|
+
packageName: '@anthropic-ai/claude-agent-sdk-linux-x64',
|
|
75
|
+
lock: CLAUDE_DIRECT_MCP_LINUX_X64_GLIBC_LOCK,
|
|
76
|
+
},
|
|
77
|
+
];
|
|
78
|
+
export function detectClaudeRuntimeEnvironment() {
|
|
79
|
+
if (process.platform !== 'linux')
|
|
80
|
+
return { platform: process.platform, architecture: process.arch };
|
|
81
|
+
const report = process.report?.getReport();
|
|
82
|
+
const glibc = report?.header?.glibcVersionRuntime;
|
|
83
|
+
return {
|
|
84
|
+
platform: process.platform,
|
|
85
|
+
architecture: process.arch,
|
|
86
|
+
libc: typeof glibc === 'string' && glibc.length > 0 ? 'glibc' : 'unknown',
|
|
87
|
+
};
|
|
88
|
+
}
|
|
89
|
+
export function selectClaudeRuntimeTarget(environment = detectClaudeRuntimeEnvironment()) {
|
|
90
|
+
const target = CLAUDE_RUNTIME_TARGETS.find((candidate) => (candidate.platform === environment.platform
|
|
91
|
+
&& candidate.architecture === environment.architecture
|
|
92
|
+
&& candidate.libc === environment.libc));
|
|
93
|
+
if (!target) {
|
|
94
|
+
const identity = [environment.platform, environment.architecture, environment.libc].filter(Boolean).join('-');
|
|
95
|
+
throw new Error(`RUNTIME_NOT_ADMITTED: unsupported Claude runtime target: ${identity}`);
|
|
96
|
+
}
|
|
97
|
+
return target;
|
|
98
|
+
}
|
|
99
|
+
/** Resolve the selected optional-package binary through Node package resolution. */
|
|
100
|
+
export async function resolveClaudeScenarioExecutable(target = selectClaudeRuntimeTarget()) {
|
|
101
|
+
let packageJson;
|
|
102
|
+
try {
|
|
103
|
+
packageJson = createRequire(import.meta.url).resolve(`${target.packageName}/package.json`);
|
|
104
|
+
}
|
|
105
|
+
catch {
|
|
106
|
+
throw new Error(`RUNTIME_NOT_ADMITTED: missing Claude runtime package: ${target.packageName}`);
|
|
107
|
+
}
|
|
108
|
+
const candidate = join(dirname(packageJson), 'claude');
|
|
90
109
|
try {
|
|
91
110
|
await access(candidate, constants.X_OK);
|
|
92
111
|
return realpath(candidate);
|
|
93
112
|
}
|
|
94
113
|
catch {
|
|
95
|
-
throw new Error(
|
|
114
|
+
throw new Error(`RUNTIME_NOT_ADMITTED: Claude runtime executable is unavailable: ${target.id}`);
|
|
115
|
+
}
|
|
116
|
+
}
|
|
117
|
+
export function observeClaudeRuntimeControlPlane(options) {
|
|
118
|
+
if (typeof options.createSdkMcpServer !== 'function' || typeof options.query !== 'function') {
|
|
119
|
+
throw new Error('Claude SDK parent bridge methods unavailable');
|
|
120
|
+
}
|
|
121
|
+
if (typeof AbortController !== 'function' || typeof AbortController.prototype.abort !== 'function') {
|
|
122
|
+
throw new Error('Claude AbortController cancellation unavailable');
|
|
96
123
|
}
|
|
124
|
+
return {
|
|
125
|
+
normalizedHandshake: CLAUDE_DIRECT_MCP_NORMALIZED_HANDSHAKE,
|
|
126
|
+
capabilities: CLAUDE_DIRECT_MCP_CAPABILITIES,
|
|
127
|
+
close: async () => undefined,
|
|
128
|
+
};
|
|
97
129
|
}
|