@wix/pathgrade 1.0.13 → 1.0.15
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +30 -0
- package/dist/adapters/jest/results.js +1 -1
- package/dist/adapters/node-test/index.js +1 -1
- package/dist/agents/claude/sdk-message-projector.js +3 -2
- package/dist/agents/claude/tool-permission-bridge.d.ts +4 -0
- package/dist/agents/claude/tool-permission-bridge.js +68 -1
- package/dist/agents/claude.d.ts +2 -0
- package/dist/agents/claude.js +55 -10
- package/dist/agents/codex-app-server/agent.js +94 -76
- package/dist/agents/codex-app-server/mcp-approval-correlator.d.ts +55 -0
- package/dist/agents/codex-app-server/mcp-approval-correlator.js +299 -0
- package/dist/core/canonical-json.d.ts +2 -0
- package/dist/core/canonical-json.js +51 -0
- package/dist/core/generated-mcp-protocol.d.ts +19 -0
- package/dist/core/generated-mcp-protocol.js +26 -0
- package/dist/core/mcp-mock.d.ts +1 -1
- package/dist/core/mcp-mock.js +24 -0
- package/dist/core/mcp-mock.types.d.ts +6 -0
- package/dist/core/mcp-schema-profile.d.ts +10 -0
- package/dist/core/mcp-schema-profile.js +91 -0
- package/dist/mcp-mock-server.js +29 -13
- package/dist/providers/mcp-config.js +4 -2
- package/dist/providers/sandbox.js +6 -0
- package/dist/providers/scripted-mcp-mock-host.d.ts +39 -0
- package/dist/providers/scripted-mcp-mock-host.js +368 -0
- package/dist/reporters/cli.js +4 -3
- package/dist/reporters/github-comment.js +1 -1
- package/dist/reporters/report-summary.js +1 -0
- package/dist/reporting/core.js +19 -5
- package/dist/reporting/types.d.ts +2 -1
- package/dist/runners/model-builders.js +1 -1
- package/dist/runners/model-validation.js +4 -1
- package/dist/runners/model.d.ts +1 -1
- package/dist/runners/report-projection.js +2 -2
- package/dist/runners/vitest-adapter.js +1 -1
- package/dist/sdk/agent.js +64 -19
- package/dist/sdk/diagnostics.d.ts +1 -0
- package/dist/sdk/diagnostics.js +6 -3
- package/dist/sdk/index.d.ts +4 -3
- package/dist/sdk/index.js +1 -1
- package/dist/sdk/lifecycle.js +3 -3
- package/dist/sdk/managed-session.d.ts +3 -0
- package/dist/sdk/managed-session.js +71 -26
- package/dist/sdk/mcp-event-input.d.ts +3 -0
- package/dist/sdk/mcp-event-input.js +8 -0
- package/dist/sdk/mcp-evidence.d.ts +39 -0
- package/dist/sdk/mcp-evidence.js +71 -12
- package/dist/sdk/mcp-mock-approvals.d.ts +40 -0
- package/dist/sdk/mcp-mock-approvals.js +235 -0
- package/dist/sdk/scripted-mcp-events.d.ts +24 -0
- package/dist/sdk/scripted-mcp-events.js +30 -0
- package/dist/sdk/types.d.ts +6 -2
- package/dist/tool-events.d.ts +16 -1
- package/dist/types.d.ts +3 -1
- package/dist/viewer.html +4 -4
- package/package.json +3 -2
package/dist/sdk/agent.js
CHANGED
|
@@ -17,6 +17,7 @@ import fs from 'fs-extra';
|
|
|
17
17
|
import * as path from 'path';
|
|
18
18
|
import { cleanDebugRuns, DEFAULT_DEBUG_RETAIN_RUNS, prepareManagedDebugRun, } from '../providers/debug-runs.js';
|
|
19
19
|
import { collectOpenCodeMcpToolNames, validateOpenCodeDeclaration } from '../agents/opencode/contract.js';
|
|
20
|
+
import { compileMcpMockApprovalSession, } from './mcp-mock-approvals.js';
|
|
20
21
|
/**
|
|
21
22
|
* Test-only injection point: override the sink used by the next emitter
|
|
22
23
|
* built inside `createAgent`. Pass `null` to restore the default (stderr).
|
|
@@ -47,21 +48,23 @@ class AgentImpl {
|
|
|
47
48
|
opencodeExecutable;
|
|
48
49
|
opencodeMcpToolNames;
|
|
49
50
|
activeChatSession;
|
|
50
|
-
|
|
51
|
-
|
|
52
|
-
this.
|
|
53
|
-
this.
|
|
54
|
-
this.
|
|
55
|
-
this.
|
|
56
|
-
this.
|
|
57
|
-
this.
|
|
58
|
-
this.
|
|
59
|
-
this.
|
|
60
|
-
this.
|
|
61
|
-
this.
|
|
62
|
-
this.
|
|
63
|
-
this.
|
|
64
|
-
this.
|
|
51
|
+
scriptedMcp;
|
|
52
|
+
constructor(opts) {
|
|
53
|
+
this.ws = opts.workspace;
|
|
54
|
+
this.agentName = opts.agentName;
|
|
55
|
+
this.llm = opts.llm;
|
|
56
|
+
this.timeoutSetting = opts.timeoutSetting;
|
|
57
|
+
this.modelOpt = opts.model;
|
|
58
|
+
this.conversationWindowOpt = opts.conversationWindow;
|
|
59
|
+
this.debugOpt = opts.debug;
|
|
60
|
+
this.debugName = opts.debugName;
|
|
61
|
+
this.debugBaseDir = opts.debugBaseDir;
|
|
62
|
+
this.verbose = opts.verbose;
|
|
63
|
+
this.transport = opts.transport;
|
|
64
|
+
this.mcpSafety = opts.mcpSafety;
|
|
65
|
+
this.opencodeExecutable = opts.opencodeExecutable;
|
|
66
|
+
this.opencodeMcpToolNames = opts.opencodeMcpToolNames;
|
|
67
|
+
this.scriptedMcp = opts.scriptedMcp;
|
|
65
68
|
}
|
|
66
69
|
get messages() {
|
|
67
70
|
return this._messages;
|
|
@@ -106,6 +109,7 @@ class AgentImpl {
|
|
|
106
109
|
...(this.mcpSafety !== undefined ? { mcpSafety: this.mcpSafety } : {}),
|
|
107
110
|
...(this.opencodeExecutable !== undefined ? { opencodeExecutable: this.opencodeExecutable } : {}),
|
|
108
111
|
...(this.opencodeMcpToolNames !== undefined ? { opencodeMcpToolNames: this.opencodeMcpToolNames } : {}),
|
|
112
|
+
...(this.scriptedMcp !== undefined ? { scriptedMcp: this.scriptedMcp } : {}),
|
|
109
113
|
});
|
|
110
114
|
}
|
|
111
115
|
resolveTimeoutSec(mode, maxTurns) {
|
|
@@ -270,7 +274,12 @@ class AgentImpl {
|
|
|
270
274
|
// scorer judge calls accumulate on the same tracker.
|
|
271
275
|
const agent = this;
|
|
272
276
|
const runStepScorers = async (scorers) => {
|
|
273
|
-
|
|
277
|
+
const result = await evaluate(agent, scorers, { llm: this.llm });
|
|
278
|
+
const score = result.score;
|
|
279
|
+
if (score === undefined) {
|
|
280
|
+
throw new Error('Step scorer evaluation did not produce a score');
|
|
281
|
+
}
|
|
282
|
+
return { ...result, score };
|
|
274
283
|
};
|
|
275
284
|
try {
|
|
276
285
|
const result = await runConversation(opts, {
|
|
@@ -391,14 +400,34 @@ export async function createAgent(opts) {
|
|
|
391
400
|
const transport = agentName === 'codex'
|
|
392
401
|
? resolveCodexTransport(opts, process.env)
|
|
393
402
|
: undefined;
|
|
403
|
+
let scriptedMcp;
|
|
404
|
+
if (opts.mcpMockApprovalRules !== undefined) {
|
|
405
|
+
if (opts.mcpMock === undefined)
|
|
406
|
+
throw new Error('mcpMockApprovalRules requires mcpMock');
|
|
407
|
+
if (opts.mcpConfigFile !== undefined)
|
|
408
|
+
throw new Error('mcpMockApprovalRules cannot be combined with mcpConfigFile');
|
|
409
|
+
if (opts.mcpSafety?.runMode !== undefined && opts.mcpSafety.runMode !== 'mock') {
|
|
410
|
+
throw new Error('mcpMockApprovalRules requires mcpSafety.runMode to be absent or mock');
|
|
411
|
+
}
|
|
412
|
+
if (agentName !== 'claude' && !(agentName === 'codex' && transport === 'app-server')) {
|
|
413
|
+
throw new Error('mcpMockApprovalRules supports Claude or Codex with transport app-server only');
|
|
414
|
+
}
|
|
415
|
+
scriptedMcp = compileMcpMockApprovalSession({
|
|
416
|
+
mcpMock: opts.mcpMock,
|
|
417
|
+
rules: opts.mcpMockApprovalRules,
|
|
418
|
+
provider: agentName,
|
|
419
|
+
});
|
|
420
|
+
}
|
|
394
421
|
const timeoutSetting = opts.timeout ?? 300;
|
|
395
422
|
// Capture runner context now; adapters own installation and restoration.
|
|
396
423
|
const testCtx = opts.debug ? resolveCaseDebugContext() : { name: '', dir: '' };
|
|
397
|
-
const { timeout: _, mcpMock, mcpConfigFile, agent: __, debug: ___, transport: _____, mcpSafety: ______, opencodeExecutable, ...rest } = opts;
|
|
424
|
+
const { timeout: _, mcpMock, mcpConfigFile, mcpMockApprovalRules: ________, agent: __, debug: ___, transport: _____, mcpSafety: ______, opencodeExecutable, ...rest } = opts;
|
|
398
425
|
const workspace = await prepareWorkspace({
|
|
399
426
|
...rest,
|
|
400
427
|
agent: agentName,
|
|
401
|
-
mcp:
|
|
428
|
+
mcp: scriptedMcp
|
|
429
|
+
? undefined
|
|
430
|
+
: mcpConfigFile ? { configFile: mcpConfigFile } : mcpMock ? { mock: mcpMock } : undefined,
|
|
402
431
|
});
|
|
403
432
|
// Create agent LLM once, using the fully-resolved sandbox env (includes
|
|
404
433
|
// keychain OAuth tokens, API keys, safe host vars).
|
|
@@ -412,7 +441,23 @@ export async function createAgent(opts) {
|
|
|
412
441
|
sink: verboseSinkOverride ?? undefined,
|
|
413
442
|
testName: testCtx.name || undefined,
|
|
414
443
|
});
|
|
415
|
-
const agent = new AgentImpl(
|
|
444
|
+
const agent = new AgentImpl({
|
|
445
|
+
workspace,
|
|
446
|
+
agentName,
|
|
447
|
+
llm,
|
|
448
|
+
timeoutSetting,
|
|
449
|
+
conversationWindow: opts.conversationWindow,
|
|
450
|
+
model: opts.model,
|
|
451
|
+
debug: opts.debug,
|
|
452
|
+
debugName,
|
|
453
|
+
debugBaseDir,
|
|
454
|
+
verbose,
|
|
455
|
+
transport,
|
|
456
|
+
mcpSafety: opts.mcpSafety,
|
|
457
|
+
opencodeExecutable,
|
|
458
|
+
opencodeMcpToolNames: agentName === 'opencode' ? collectOpenCodeMcpToolNames(mcpMock) : undefined,
|
|
459
|
+
scriptedMcp,
|
|
460
|
+
});
|
|
416
461
|
lifecycleCore.registerAgent(agent);
|
|
417
462
|
return agent;
|
|
418
463
|
}
|
|
@@ -35,6 +35,7 @@ export interface BuildDiagnosticsReportInput {
|
|
|
35
35
|
turnDetails?: TurnDetail[];
|
|
36
36
|
reactionsFired?: ReactionFiredEntry[];
|
|
37
37
|
scorers?: DiagnosticsScorer[];
|
|
38
|
+
warnings?: string[];
|
|
38
39
|
log: LogEntry[];
|
|
39
40
|
}
|
|
40
41
|
export declare function buildDiagnosticsReport(input: BuildDiagnosticsReportInput): DiagnosticsReport;
|
package/dist/sdk/diagnostics.js
CHANGED
|
@@ -10,9 +10,12 @@ export function buildDiagnosticsReport(input) {
|
|
|
10
10
|
...detail,
|
|
11
11
|
apiRetries: retriesByTurn.get(detail.turn) ?? 0,
|
|
12
12
|
}));
|
|
13
|
-
const warnings =
|
|
14
|
-
|
|
15
|
-
|
|
13
|
+
const warnings = [
|
|
14
|
+
...(input.warnings ?? []),
|
|
15
|
+
...turnDetails
|
|
16
|
+
.filter((detail) => detail.outputLines > 500)
|
|
17
|
+
.map((detail) => `Turn ${detail.turn}: output exceeded 500 lines (${detail.outputLines} lines)`),
|
|
18
|
+
];
|
|
16
19
|
return {
|
|
17
20
|
turns: turnDetails.length,
|
|
18
21
|
totalDurationMs: turnDetails.reduce((sum, detail) => sum + detail.durationMs, 0),
|
package/dist/sdk/index.d.ts
CHANGED
|
@@ -2,7 +2,8 @@ export { createAgent } from './agent.js';
|
|
|
2
2
|
export { resolveAgentName, resolveCodexTransport, InvalidTransportEnvError, } from './agent-resolution.js';
|
|
3
3
|
export { AgentCrashError } from './agent-crash.js';
|
|
4
4
|
export { check, score, judge, toolUsage } from './scorers.js';
|
|
5
|
-
export { getMcpToolCall, isMcpToolCall, findMcpToolCalls, getMcpStartupStatus, isMcpStartupStatus, } from './mcp-evidence.js';
|
|
5
|
+
export { getMcpToolCall, isMcpToolCall, findMcpToolCalls, getMcpStartupStatus, isMcpStartupStatus, getMcpApproval, isMcpApproval, findMcpApprovals, getMcpInvocation, wasMcpToolInvoked, } from './mcp-evidence.js';
|
|
6
|
+
export type { McpMockApprovalRule, McpMockJsonValue } from './mcp-mock-approvals.js';
|
|
6
7
|
export { decideMcpToolCall, redactMcpSecrets, } from './mcp-safety.js';
|
|
7
8
|
export { evaluate, EvalScorerError } from './evaluate.js';
|
|
8
9
|
export { RUN_SNAPSHOT_VERSION, buildRunSnapshot, loadRunSnapshot, SnapshotParseError, SnapshotVersionError, WorkspaceMissingError, } from './snapshots.js';
|
|
@@ -29,9 +30,9 @@ export type { OnScorerErrorMode } from './evaluate.js';
|
|
|
29
30
|
export type { EvalResultEvent, EvalResultObserver, ResultObserverHandle, ResultObserverOptions, ResultObserverOwner, } from './result-capture.js';
|
|
30
31
|
export type { RunSnapshot } from './snapshots.js';
|
|
31
32
|
export type { DiagnosticsReport } from './diagnostics.js';
|
|
32
|
-
export type { ExpectedMcpStartupStatus, ExpectedMcpToolCall, McpStartupStatusEvidence, McpToolCallEvidence, } from './mcp-evidence.js';
|
|
33
|
+
export type { ExpectedMcpStartupStatus, ExpectedMcpToolCall, McpStartupStatusEvidence, McpToolCallEvidence, McpApprovalEvidence, ExpectedMcpApproval, McpInvocationResult, } from './mcp-evidence.js';
|
|
33
34
|
export type { McpPolicyDenialReason, McpToolCallRequest, McpToolPolicyDecision, } from './mcp-safety.js';
|
|
34
|
-
export type { ToolEvent } from '../tool-events.js';
|
|
35
|
+
export type { ToolEvent, McpToolCallClassification } from '../tool-events.js';
|
|
35
36
|
export type { LLMPort, EvalRuntime } from './eval-runtime.js';
|
|
36
37
|
export { createLLMClient, ProviderNotSupportedError } from '../utils/llm.js';
|
|
37
38
|
export type { CreateLLMClientOptions, LLMProviderAdapter, TokenUsage as LLMTokenUsage } from '../utils/llm.js';
|
package/dist/sdk/index.js
CHANGED
|
@@ -3,7 +3,7 @@ export { createAgent } from './agent.js';
|
|
|
3
3
|
export { resolveAgentName, resolveCodexTransport, InvalidTransportEnvError, } from './agent-resolution.js';
|
|
4
4
|
export { AgentCrashError } from './agent-crash.js';
|
|
5
5
|
export { check, score, judge, toolUsage } from './scorers.js';
|
|
6
|
-
export { getMcpToolCall, isMcpToolCall, findMcpToolCalls, getMcpStartupStatus, isMcpStartupStatus, } from './mcp-evidence.js';
|
|
6
|
+
export { getMcpToolCall, isMcpToolCall, findMcpToolCalls, getMcpStartupStatus, isMcpStartupStatus, getMcpApproval, isMcpApproval, findMcpApprovals, getMcpInvocation, wasMcpToolInvoked, } from './mcp-evidence.js';
|
|
7
7
|
export { decideMcpToolCall, redactMcpSecrets, } from './mcp-safety.js';
|
|
8
8
|
export { evaluate, EvalScorerError } from './evaluate.js';
|
|
9
9
|
export { RUN_SNAPSHOT_VERSION, buildRunSnapshot, loadRunSnapshot, SnapshotParseError, SnapshotVersionError, WorkspaceMissingError, } from './snapshots.js';
|
package/dist/sdk/lifecycle.js
CHANGED
|
@@ -136,13 +136,12 @@ function synthesizeTrialFromAgent(agent) {
|
|
|
136
136
|
const conversationEnd = [...agent.log].reverse().find((entry) => entry.type === 'conversation_end');
|
|
137
137
|
const completionReason = conversationEnd?.completion_reason ?? (agent.log.some((entry) => entry.type === 'agent_result') ? 'completed' : undefined);
|
|
138
138
|
return {
|
|
139
|
-
score:
|
|
139
|
+
score: undefined,
|
|
140
140
|
scorers: [],
|
|
141
141
|
resultKind: 'synthetic_no_evaluation',
|
|
142
142
|
...(agent.executionMetadata ? { agent: agent.executionMetadata } : {}),
|
|
143
143
|
trial: {
|
|
144
144
|
trial_id: 0,
|
|
145
|
-
reward: 1,
|
|
146
145
|
scorer_results: [],
|
|
147
146
|
duration_ms: 0,
|
|
148
147
|
n_commands: nCommands,
|
|
@@ -157,8 +156,9 @@ function synthesizeTrialFromAgent(agent) {
|
|
|
157
156
|
completionDetail: conversationEnd?.completion_detail,
|
|
158
157
|
turnDetails: conversationEnd?.turn_details,
|
|
159
158
|
reactionsFired: conversationEnd?.reactions_fired,
|
|
160
|
-
score:
|
|
159
|
+
score: undefined,
|
|
161
160
|
scorers: [],
|
|
161
|
+
warnings: ['evaluate() was not called; no evaluation score is available'],
|
|
162
162
|
log: agent.log,
|
|
163
163
|
}),
|
|
164
164
|
};
|
|
@@ -4,6 +4,7 @@ import type { AgentName, AgentTransport, Message } from './types.js';
|
|
|
4
4
|
import type { McpSafetyOptions } from './mcp-safety.js';
|
|
5
5
|
import type { LLMPort } from '../utils/llm-types.js';
|
|
6
6
|
import type { AskBus } from './ask-bus/types.js';
|
|
7
|
+
import type { CompiledMcpMockSession } from './mcp-mock-approvals.js';
|
|
7
8
|
export interface ManagedSessionDeps {
|
|
8
9
|
ws: Workspace;
|
|
9
10
|
agentName: AgentName;
|
|
@@ -33,6 +34,8 @@ export interface ManagedSessionDeps {
|
|
|
33
34
|
mcpSafety?: McpSafetyOptions;
|
|
34
35
|
opencodeExecutable?: string;
|
|
35
36
|
opencodeMcpToolNames?: string[];
|
|
37
|
+
/** Trusted pre-workspace compiled generated-MCP declaration. */
|
|
38
|
+
scriptedMcp?: CompiledMcpMockSession;
|
|
36
39
|
}
|
|
37
40
|
export interface ManagedSession {
|
|
38
41
|
/** Full lifecycle: log start/result, push messages, check exit code. */
|
|
@@ -6,6 +6,7 @@ import { buildToolEventLogEntry } from './tool-event-log.js';
|
|
|
6
6
|
import { planRuntimePolicies } from './runtime-policy.js';
|
|
7
7
|
import { getVisibleAssistantMessage } from './visible-turn.js';
|
|
8
8
|
import { createAskBus } from './ask-bus/bus.js';
|
|
9
|
+
import { startScriptedMcpMockHost, } from '../providers/scripted-mcp-mock-host.js';
|
|
9
10
|
export function createManagedSession(deps) {
|
|
10
11
|
const { ws, agentName, timeoutSec, messages, log, model, conversationWindow, llm } = deps;
|
|
11
12
|
const agentTimeoutMs = timeoutSec * 1000;
|
|
@@ -33,6 +34,10 @@ export function createManagedSession(deps) {
|
|
|
33
34
|
let session = null;
|
|
34
35
|
let setupDone = false;
|
|
35
36
|
let currentSignal;
|
|
37
|
+
let scriptedHost;
|
|
38
|
+
let activeTurn = false;
|
|
39
|
+
let nextTurnNumber = 0;
|
|
40
|
+
let disposePromise;
|
|
36
41
|
const runCommand = async (cmd) => {
|
|
37
42
|
const result = await ws.exec(cmd, { signal: currentSignal });
|
|
38
43
|
log.push({
|
|
@@ -46,34 +51,66 @@ export function createManagedSession(deps) {
|
|
|
46
51
|
return result;
|
|
47
52
|
};
|
|
48
53
|
const executeTurn = async (message) => {
|
|
54
|
+
if (activeTurn)
|
|
55
|
+
throw new Error('A Pathgrade agent turn is already active');
|
|
49
56
|
const remaining = deadlineMs - Date.now();
|
|
50
57
|
if (remaining <= 0)
|
|
51
58
|
throw new Error(`${label} timed out`);
|
|
52
|
-
|
|
53
|
-
|
|
54
|
-
|
|
55
|
-
|
|
56
|
-
if (!
|
|
57
|
-
|
|
58
|
-
|
|
59
|
+
activeTurn = true;
|
|
60
|
+
try {
|
|
61
|
+
return await withAbortTimeout(async (signal) => {
|
|
62
|
+
currentSignal = signal;
|
|
63
|
+
if (!session) {
|
|
64
|
+
// Run auth setup commands before first agent turn
|
|
65
|
+
if (!setupDone) {
|
|
66
|
+
for (const cmd of ws.setupCommands) {
|
|
67
|
+
await ws.exec(cmd, { signal });
|
|
68
|
+
signal.throwIfAborted();
|
|
69
|
+
}
|
|
70
|
+
setupDone = true;
|
|
71
|
+
}
|
|
72
|
+
// Carry the workspace's resolved env (from `prepareWorkspace
|
|
73
|
+
// → resolveCredentials`) into the runtime handle so drivers
|
|
74
|
+
// that auth through `Options.env` (notably the Claude SDK
|
|
75
|
+
// driver) can lift Anthropic keys out. Drivers that only
|
|
76
|
+
// need the workspace path keep using `getWorkspacePath`.
|
|
77
|
+
const runtime = {
|
|
78
|
+
handle: ws.path,
|
|
79
|
+
workspacePath: ws.path,
|
|
80
|
+
env: ws.env,
|
|
81
|
+
};
|
|
82
|
+
if (deps.scriptedMcp) {
|
|
83
|
+
scriptedHost = await startScriptedMcpMockHost(deps.scriptedMcp);
|
|
84
|
+
sessionOptions.scriptedMcpHost = scriptedHost;
|
|
85
|
+
}
|
|
86
|
+
try {
|
|
87
|
+
session = await createAgentSession(agent, runtime, runCommand, sessionOptions);
|
|
88
|
+
}
|
|
89
|
+
catch (error) {
|
|
90
|
+
await scriptedHost?.dispose();
|
|
91
|
+
scriptedHost = undefined;
|
|
92
|
+
throw error;
|
|
59
93
|
}
|
|
60
|
-
setupDone = true;
|
|
61
94
|
}
|
|
62
|
-
|
|
63
|
-
|
|
64
|
-
|
|
65
|
-
|
|
66
|
-
|
|
67
|
-
|
|
68
|
-
|
|
69
|
-
|
|
70
|
-
|
|
71
|
-
|
|
72
|
-
|
|
73
|
-
|
|
74
|
-
|
|
75
|
-
|
|
76
|
-
}
|
|
95
|
+
const turnNumber = ++nextTurnNumber;
|
|
96
|
+
scriptedHost?.beginTurn(turnNumber);
|
|
97
|
+
const result = turnNumber === 1
|
|
98
|
+
? await session.start({ message })
|
|
99
|
+
: await session.reply({ message });
|
|
100
|
+
if (scriptedHost) {
|
|
101
|
+
const settled = scriptedHost.settleEvents(result.toolEvents);
|
|
102
|
+
result.toolEvents = settled.events;
|
|
103
|
+
if (settled.error) {
|
|
104
|
+
return { ...result, exitCode: 1, rawOutput: settled.error.message };
|
|
105
|
+
}
|
|
106
|
+
}
|
|
107
|
+
return result;
|
|
108
|
+
}, remaining, label);
|
|
109
|
+
}
|
|
110
|
+
finally {
|
|
111
|
+
activeTurn = false;
|
|
112
|
+
currentSignal = undefined;
|
|
113
|
+
}
|
|
77
114
|
};
|
|
78
115
|
return {
|
|
79
116
|
async send(message) {
|
|
@@ -108,9 +145,17 @@ export function createManagedSession(deps) {
|
|
|
108
145
|
return Math.max(0, deadlineMs - Date.now());
|
|
109
146
|
},
|
|
110
147
|
async dispose() {
|
|
111
|
-
if (
|
|
112
|
-
return;
|
|
113
|
-
|
|
148
|
+
if (disposePromise)
|
|
149
|
+
return disposePromise;
|
|
150
|
+
disposePromise = (async () => {
|
|
151
|
+
try {
|
|
152
|
+
await session?.dispose?.();
|
|
153
|
+
}
|
|
154
|
+
finally {
|
|
155
|
+
await scriptedHost?.dispose();
|
|
156
|
+
}
|
|
157
|
+
})();
|
|
158
|
+
return disposePromise;
|
|
114
159
|
},
|
|
115
160
|
askBus,
|
|
116
161
|
};
|
|
@@ -1,4 +1,38 @@
|
|
|
1
1
|
import type { ToolEvent } from '../tool-events.js';
|
|
2
|
+
export type McpInvocationResult = {
|
|
3
|
+
serverName: string;
|
|
4
|
+
toolName: string;
|
|
5
|
+
} & ({
|
|
6
|
+
invocation: 'confirmed';
|
|
7
|
+
outcome: 'completed' | 'tool_error';
|
|
8
|
+
source: 'typed_receipt';
|
|
9
|
+
} | {
|
|
10
|
+
invocation: 'unknown';
|
|
11
|
+
outcome: 'protocol_error' | 'unknown';
|
|
12
|
+
source: 'typed_receipt';
|
|
13
|
+
} | {
|
|
14
|
+
invocation: 'not_invoked';
|
|
15
|
+
outcome: 'user_denied' | 'policy_denied' | 'protocol_error';
|
|
16
|
+
source: 'typed_enforcement';
|
|
17
|
+
} | {
|
|
18
|
+
invocation: 'not_invoked';
|
|
19
|
+
outcome: 'user_denied' | 'policy_denied' | 'unknown';
|
|
20
|
+
source: 'legacy_denial';
|
|
21
|
+
});
|
|
22
|
+
export interface McpApprovalEvidence {
|
|
23
|
+
serverName?: string;
|
|
24
|
+
toolName?: string;
|
|
25
|
+
decision?: 'approve' | 'deny';
|
|
26
|
+
outcome?: 'matched' | 'unmatched' | 'protocol_error';
|
|
27
|
+
arguments: Record<string, unknown>;
|
|
28
|
+
event: ToolEvent;
|
|
29
|
+
}
|
|
30
|
+
export interface ExpectedMcpApproval {
|
|
31
|
+
serverName?: string;
|
|
32
|
+
toolName?: string;
|
|
33
|
+
decision?: 'approve' | 'deny';
|
|
34
|
+
outcome?: 'matched' | 'unmatched' | 'protocol_error';
|
|
35
|
+
}
|
|
2
36
|
export interface McpToolCallEvidence {
|
|
3
37
|
serverName: string;
|
|
4
38
|
toolName: string;
|
|
@@ -25,5 +59,10 @@ export interface ExpectedMcpStartupStatus {
|
|
|
25
59
|
export declare function getMcpToolCall(event: ToolEvent): McpToolCallEvidence | undefined;
|
|
26
60
|
export declare function isMcpToolCall(event: ToolEvent, expected?: ExpectedMcpToolCall): boolean;
|
|
27
61
|
export declare function findMcpToolCalls(events: readonly ToolEvent[], expected?: ExpectedMcpToolCall): McpToolCallEvidence[];
|
|
62
|
+
export declare function getMcpApproval(event: ToolEvent): McpApprovalEvidence | undefined;
|
|
63
|
+
export declare function isMcpApproval(event: ToolEvent, expected?: ExpectedMcpApproval): boolean;
|
|
64
|
+
export declare function findMcpApprovals(events: readonly ToolEvent[], expected?: ExpectedMcpApproval): McpApprovalEvidence[];
|
|
65
|
+
export declare function getMcpInvocation(event: ToolEvent): McpInvocationResult | undefined;
|
|
66
|
+
export declare function wasMcpToolInvoked(event: ToolEvent): boolean | undefined;
|
|
28
67
|
export declare function getMcpStartupStatus(event: ToolEvent): McpStartupStatusEvidence | undefined;
|
|
29
68
|
export declare function isMcpStartupStatus(event: ToolEvent, expected?: ExpectedMcpStartupStatus): boolean;
|
package/dist/sdk/mcp-evidence.js
CHANGED
|
@@ -17,24 +17,83 @@ export function getMcpToolCall(event) {
|
|
|
17
17
|
}
|
|
18
18
|
export function isMcpToolCall(event, expected = {}) {
|
|
19
19
|
const call = getMcpToolCall(event);
|
|
20
|
-
|
|
21
|
-
return false;
|
|
22
|
-
if (expected.serverName !== undefined && call.serverName !== expected.serverName)
|
|
23
|
-
return false;
|
|
24
|
-
if (expected.toolName !== undefined && call.toolName !== expected.toolName)
|
|
25
|
-
return false;
|
|
26
|
-
if (expected.status !== undefined && call.status !== expected.status)
|
|
27
|
-
return false;
|
|
28
|
-
if (expected.argumentsContaining && !containsArguments(call.arguments, expected.argumentsContaining)) {
|
|
29
|
-
return false;
|
|
30
|
-
}
|
|
31
|
-
return true;
|
|
20
|
+
return call !== undefined && matchesMcpToolCall(call, expected);
|
|
32
21
|
}
|
|
33
22
|
export function findMcpToolCalls(events, expected = {}) {
|
|
34
23
|
return events
|
|
35
24
|
.map((event) => getMcpToolCall(event))
|
|
36
25
|
.filter((call) => call !== undefined && matchesMcpToolCall(call, expected));
|
|
37
26
|
}
|
|
27
|
+
export function getMcpApproval(event) {
|
|
28
|
+
if (event.action !== 'mcp_approval')
|
|
29
|
+
return undefined;
|
|
30
|
+
const args = event.arguments ?? {};
|
|
31
|
+
const serverName = typeof args.server === 'string' ? args.server : undefined;
|
|
32
|
+
const toolName = typeof args.tool === 'string' ? args.tool : undefined;
|
|
33
|
+
const decision = args.decision === 'approve' || args.decision === 'deny' ? args.decision : undefined;
|
|
34
|
+
const outcome = args.outcome === 'matched' || args.outcome === 'unmatched' || args.outcome === 'protocol_error'
|
|
35
|
+
? args.outcome : undefined;
|
|
36
|
+
return { ...(serverName ? { serverName } : {}), ...(toolName ? { toolName } : {}),
|
|
37
|
+
...(decision ? { decision } : {}), ...(outcome ? { outcome } : {}), arguments: args, event };
|
|
38
|
+
}
|
|
39
|
+
export function isMcpApproval(event, expected = {}) {
|
|
40
|
+
const approval = getMcpApproval(event);
|
|
41
|
+
return approval !== undefined && matchesMcpApproval(approval, expected);
|
|
42
|
+
}
|
|
43
|
+
function matchesMcpApproval(approval, expected) {
|
|
44
|
+
return (expected.serverName === undefined || approval.serverName === expected.serverName)
|
|
45
|
+
&& (expected.toolName === undefined || approval.toolName === expected.toolName)
|
|
46
|
+
&& (expected.decision === undefined || approval.decision === expected.decision)
|
|
47
|
+
&& (expected.outcome === undefined || approval.outcome === expected.outcome);
|
|
48
|
+
}
|
|
49
|
+
export function findMcpApprovals(events, expected = {}) {
|
|
50
|
+
return events
|
|
51
|
+
.map((event) => getMcpApproval(event))
|
|
52
|
+
.filter((approval) => approval !== undefined && matchesMcpApproval(approval, expected));
|
|
53
|
+
}
|
|
54
|
+
export function getMcpInvocation(event) {
|
|
55
|
+
if (event.action !== 'mcp_tool_call')
|
|
56
|
+
return undefined;
|
|
57
|
+
const typed = event.mcp;
|
|
58
|
+
if (typed !== undefined) {
|
|
59
|
+
if (!validTypedMcp(typed))
|
|
60
|
+
return undefined;
|
|
61
|
+
return { ...typed, source: typed.invocation === 'not_invoked' ? 'typed_enforcement' : 'typed_receipt' };
|
|
62
|
+
}
|
|
63
|
+
const args = event.arguments ?? {};
|
|
64
|
+
const serverName = typeof args.server === 'string' && args.server ? args.server : undefined;
|
|
65
|
+
const toolName = typeof args.tool === 'string' && args.tool ? args.tool : undefined;
|
|
66
|
+
if (!serverName || !toolName)
|
|
67
|
+
return undefined;
|
|
68
|
+
const status = args.status;
|
|
69
|
+
if (status === 'user_denied' || status === 'policy_denied') {
|
|
70
|
+
return { serverName, toolName, invocation: 'not_invoked', outcome: status, source: 'legacy_denial' };
|
|
71
|
+
}
|
|
72
|
+
if (status === 'declined' || status === 'rejected') {
|
|
73
|
+
return { serverName, toolName, invocation: 'not_invoked', outcome: 'unknown', source: 'legacy_denial' };
|
|
74
|
+
}
|
|
75
|
+
return undefined;
|
|
76
|
+
}
|
|
77
|
+
export function wasMcpToolInvoked(event) {
|
|
78
|
+
const result = getMcpInvocation(event);
|
|
79
|
+
if (!result)
|
|
80
|
+
return undefined;
|
|
81
|
+
if (result.invocation === 'confirmed')
|
|
82
|
+
return true;
|
|
83
|
+
if (result.invocation === 'not_invoked')
|
|
84
|
+
return false;
|
|
85
|
+
return undefined;
|
|
86
|
+
}
|
|
87
|
+
function validTypedMcp(value) {
|
|
88
|
+
if (!value.serverName || !value.toolName)
|
|
89
|
+
return false;
|
|
90
|
+
if (value.invocation === 'confirmed')
|
|
91
|
+
return value.outcome === 'completed' || value.outcome === 'tool_error';
|
|
92
|
+
if (value.invocation === 'not_invoked') {
|
|
93
|
+
return value.outcome === 'user_denied' || value.outcome === 'policy_denied' || value.outcome === 'protocol_error';
|
|
94
|
+
}
|
|
95
|
+
return value.invocation === 'unknown' && (value.outcome === 'protocol_error' || value.outcome === 'unknown');
|
|
96
|
+
}
|
|
38
97
|
export function getMcpStartupStatus(event) {
|
|
39
98
|
if (event.providerToolName !== 'mcpServer/startupStatus/updated')
|
|
40
99
|
return undefined;
|
|
@@ -0,0 +1,40 @@
|
|
|
1
|
+
import type { MockMcpServerDescriptor } from '../core/mcp-mock.types.js';
|
|
2
|
+
export type McpMockJsonValue = null | boolean | number | string | McpMockJsonValue[] | {
|
|
3
|
+
[key: string]: McpMockJsonValue;
|
|
4
|
+
};
|
|
5
|
+
export interface McpMockApprovalRule {
|
|
6
|
+
serverName: string;
|
|
7
|
+
toolName: string;
|
|
8
|
+
argumentsContaining?: Record<string, McpMockJsonValue>;
|
|
9
|
+
decision: 'approve' | 'deny';
|
|
10
|
+
label?: string;
|
|
11
|
+
}
|
|
12
|
+
export interface CompiledMcpTool {
|
|
13
|
+
name: string;
|
|
14
|
+
description?: string;
|
|
15
|
+
inputSchema: Record<string, unknown>;
|
|
16
|
+
readOnlyHint?: boolean;
|
|
17
|
+
cases: readonly Readonly<{
|
|
18
|
+
when?: string;
|
|
19
|
+
response: McpMockJsonValue;
|
|
20
|
+
}>[];
|
|
21
|
+
}
|
|
22
|
+
export interface CompiledMcpServer {
|
|
23
|
+
name: string;
|
|
24
|
+
instructions?: string;
|
|
25
|
+
tools: ReadonlyMap<string, CompiledMcpTool>;
|
|
26
|
+
}
|
|
27
|
+
export interface CompiledMcpMockSession {
|
|
28
|
+
servers: ReadonlyMap<string, CompiledMcpServer>;
|
|
29
|
+
rules: readonly Readonly<McpMockApprovalRule>[];
|
|
30
|
+
}
|
|
31
|
+
export type ScriptedMcpProvider = 'codex' | 'claude';
|
|
32
|
+
export declare function compileMcpMockApprovalSession(opts: {
|
|
33
|
+
mcpMock: MockMcpServerDescriptor | MockMcpServerDescriptor[];
|
|
34
|
+
rules: readonly McpMockApprovalRule[];
|
|
35
|
+
provider: ScriptedMcpProvider;
|
|
36
|
+
}): CompiledMcpMockSession;
|
|
37
|
+
export declare function matchesMcpApprovalArguments(actual: Record<string, McpMockJsonValue>, expected: Record<string, McpMockJsonValue> | undefined): boolean;
|
|
38
|
+
export declare function canonicalizeMcpArguments(value: McpMockJsonValue): string;
|
|
39
|
+
export declare function createMcpArgumentDigest(key: Buffer, value: McpMockJsonValue): string;
|
|
40
|
+
export declare function createMcpReceiptKey(): Buffer;
|