@wix/pathgrade 1.0.25 → 1.0.26

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (75) hide show
  1. package/README.md +42 -0
  2. package/dist/agents/claude/sdk-options.d.ts +1 -2
  3. package/dist/agents/claude.js +2 -1
  4. package/dist/agents/codex-app-server/agent.js +83 -46
  5. package/dist/agents/codex-app-server/scenario-mount.d.ts +40 -0
  6. package/dist/agents/codex-app-server/scenario-mount.js +238 -0
  7. package/dist/agents/cursor-scenario.d.ts +20 -0
  8. package/dist/agents/cursor-scenario.js +96 -0
  9. package/dist/agents/cursor.js +20 -0
  10. package/dist/agents/opencode/scenario.d.ts +25 -0
  11. package/dist/agents/opencode/scenario.js +195 -0
  12. package/dist/agents/opencode.js +50 -0
  13. package/dist/internal/direct-mcp-v2/acp-author-projector.d.ts +20 -0
  14. package/dist/internal/direct-mcp-v2/acp-author-projector.js +147 -0
  15. package/dist/internal/direct-mcp-v2/acp-author-session.d.ts +39 -0
  16. package/dist/internal/direct-mcp-v2/acp-author-session.js +201 -0
  17. package/dist/internal/direct-mcp-v2/acp-client.d.ts +70 -0
  18. package/dist/internal/direct-mcp-v2/acp-client.js +310 -0
  19. package/dist/internal/direct-mcp-v2/acp-direct-mcp.d.ts +38 -0
  20. package/dist/internal/direct-mcp-v2/acp-direct-mcp.js +326 -0
  21. package/dist/internal/direct-mcp-v2/claude-direct-mcp.d.ts +68 -0
  22. package/dist/internal/direct-mcp-v2/claude-direct-mcp.js +456 -0
  23. package/dist/internal/direct-mcp-v2/claude-profile.d.ts +9 -0
  24. package/dist/internal/direct-mcp-v2/claude-profile.js +97 -0
  25. package/dist/internal/direct-mcp-v2/cleanup.d.ts +62 -0
  26. package/dist/internal/direct-mcp-v2/cleanup.js +182 -0
  27. package/dist/internal/direct-mcp-v2/codex-direct-mcp.d.ts +41 -0
  28. package/dist/internal/direct-mcp-v2/codex-direct-mcp.js +436 -0
  29. package/dist/internal/direct-mcp-v2/codex-profile.d.ts +12 -0
  30. package/dist/internal/direct-mcp-v2/codex-profile.js +81 -0
  31. package/dist/internal/direct-mcp-v2/compiler.d.ts +12 -0
  32. package/dist/internal/direct-mcp-v2/compiler.js +449 -0
  33. package/dist/internal/direct-mcp-v2/cursor-direct-mcp.d.ts +14 -0
  34. package/dist/internal/direct-mcp-v2/cursor-direct-mcp.js +60 -0
  35. package/dist/internal/direct-mcp-v2/cursor-profile.d.ts +38 -0
  36. package/dist/internal/direct-mcp-v2/cursor-profile.js +79 -0
  37. package/dist/internal/direct-mcp-v2/evidence.d.ts +50 -0
  38. package/dist/internal/direct-mcp-v2/evidence.js +147 -0
  39. package/dist/internal/direct-mcp-v2/host-reducer.d.ts +61 -0
  40. package/dist/internal/direct-mcp-v2/host-reducer.js +214 -0
  41. package/dist/internal/direct-mcp-v2/json.d.ts +9 -0
  42. package/dist/internal/direct-mcp-v2/json.js +220 -0
  43. package/dist/internal/direct-mcp-v2/materialize.d.ts +39 -0
  44. package/dist/internal/direct-mcp-v2/materialize.js +22 -0
  45. package/dist/internal/direct-mcp-v2/opencode-direct-mcp.d.ts +14 -0
  46. package/dist/internal/direct-mcp-v2/opencode-direct-mcp.js +71 -0
  47. package/dist/internal/direct-mcp-v2/opencode-profile.d.ts +8 -0
  48. package/dist/internal/direct-mcp-v2/opencode-profile.js +52 -0
  49. package/dist/internal/direct-mcp-v2/projection.d.ts +40 -0
  50. package/dist/internal/direct-mcp-v2/projection.js +46 -0
  51. package/dist/internal/direct-mcp-v2/public-scenario-runtime.d.ts +51 -0
  52. package/dist/internal/direct-mcp-v2/public-scenario-runtime.js +353 -0
  53. package/dist/internal/direct-mcp-v2/runtime-lock.d.ts +51 -0
  54. package/dist/internal/direct-mcp-v2/runtime-lock.js +203 -0
  55. package/dist/internal/direct-mcp-v2/runtime-profile.d.ts +28 -0
  56. package/dist/internal/direct-mcp-v2/runtime-profile.js +163 -0
  57. package/dist/internal/direct-mcp-v2/scenario-http-host.d.ts +24 -0
  58. package/dist/internal/direct-mcp-v2/scenario-http-host.js +254 -0
  59. package/dist/internal/direct-mcp-v2/types.d.ts +233 -0
  60. package/dist/internal/direct-mcp-v2/types.js +1 -0
  61. package/dist/sdk/agent.js +49 -2
  62. package/dist/sdk/evaluate.js +6 -3
  63. package/dist/sdk/index.d.ts +4 -0
  64. package/dist/sdk/index.js +2 -0
  65. package/dist/sdk/managed-session.d.ts +6 -0
  66. package/dist/sdk/managed-session.js +157 -50
  67. package/dist/sdk/scenario-evidence.d.ts +27 -0
  68. package/dist/sdk/scenario-evidence.js +94 -0
  69. package/dist/sdk/scenario-machine-v2.d.ts +9 -0
  70. package/dist/sdk/scenario-machine-v2.js +37 -0
  71. package/dist/sdk/snapshots.d.ts +3 -0
  72. package/dist/sdk/snapshots.js +2 -0
  73. package/dist/sdk/types.d.ts +9 -1
  74. package/dist/types.d.ts +6 -0
  75. package/package.json +2 -2
package/dist/sdk/agent.js CHANGED
@@ -19,6 +19,10 @@ import { cleanDebugRuns, DEFAULT_DEBUG_RETAIN_RUNS, prepareManagedDebugRun, } fr
19
19
  import { collectOpenCodeMcpToolNames, validateOpenCodeDeclaration } from '../agents/opencode/contract.js';
20
20
  import { collectSensitiveEnvValues } from '../tool-event-results.js';
21
21
  import { compileMcpMockApprovalSession, } from './mcp-mock-approvals.js';
22
+ import { compileScenario } from './scenario-machine-v2.js';
23
+ import { startPublicCodexScenarioRuntime } from '../agents/codex-app-server/scenario-mount.js';
24
+ import { createPublicOpenCodeScenarioRuntimeFactory } from '../agents/opencode/scenario.js';
25
+ import { createPublicCursorScenarioRuntimeFactory } from '../agents/cursor-scenario.js';
22
26
  import { createAskUserHandler } from './ask-bus/handler.js';
23
27
  /**
24
28
  * Test-only injection point: override the sink used by the next emitter
@@ -52,6 +56,9 @@ class AgentImpl {
52
56
  activeChatSession;
53
57
  scriptedMcp;
54
58
  sensitiveValues;
59
+ scenarioArtifact;
60
+ scenarioRuntimeFactory;
61
+ _scenarioEvidence = [];
55
62
  constructor(opts) {
56
63
  this.ws = opts.workspace;
57
64
  this.agentName = opts.agentName;
@@ -69,6 +76,8 @@ class AgentImpl {
69
76
  this.opencodeMcpToolNames = opts.opencodeMcpToolNames;
70
77
  this.scriptedMcp = opts.scriptedMcp;
71
78
  this.sensitiveValues = opts.sensitiveValues;
79
+ this.scenarioArtifact = opts.scenarioArtifact;
80
+ this.scenarioRuntimeFactory = opts.scenarioRuntimeFactory;
72
81
  }
73
82
  get messages() {
74
83
  return this._messages;
@@ -79,6 +88,9 @@ class AgentImpl {
79
88
  get workspace() {
80
89
  return this.ws.path;
81
90
  }
91
+ get scenarioEvidence() {
92
+ return structuredClone(this._scenarioEvidence);
93
+ }
82
94
  get executionMetadata() {
83
95
  const interactionMode = this.interactionMode === 'startChat'
84
96
  ? 'start_chat'
@@ -88,7 +100,9 @@ class AgentImpl {
88
100
  return {
89
101
  name: this.agentName,
90
102
  ...(this.modelOpt ? { requestedModel: this.modelOpt } : {}),
91
- transport: resolveExecutionTransport(this.agentName, this.transport),
103
+ transport: this.scenarioArtifact && (this.agentName === 'opencode' || this.agentName === 'cursor')
104
+ ? 'acp'
105
+ : resolveExecutionTransport(this.agentName, this.transport),
92
106
  ...(interactionMode ? { interactionMode } : {}),
93
107
  };
94
108
  }
@@ -114,6 +128,9 @@ class AgentImpl {
114
128
  ...(this.opencodeExecutable !== undefined ? { opencodeExecutable: this.opencodeExecutable } : {}),
115
129
  ...(this.opencodeMcpToolNames !== undefined ? { opencodeMcpToolNames: this.opencodeMcpToolNames } : {}),
116
130
  ...(this.scriptedMcp !== undefined ? { scriptedMcp: this.scriptedMcp } : {}),
131
+ ...(this.scenarioArtifact !== undefined ? { scenarioArtifact: this.scenarioArtifact } : {}),
132
+ ...(this.scenarioRuntimeFactory !== undefined ? { scenarioRuntimeFactory: this.scenarioRuntimeFactory } : {}),
133
+ onScenarioEvidence: (evidence) => { this._scenarioEvidence = structuredClone(evidence); },
117
134
  sensitiveValues: this.sensitiveValues,
118
135
  });
119
136
  }
@@ -362,6 +379,7 @@ class AgentImpl {
362
379
  conversationResult: this.lastConversationResult,
363
380
  workspace: dest,
364
381
  sensitiveValues: this.sensitiveValues,
382
+ scenarioEvidence: this._scenarioEvidence,
365
383
  });
366
384
  await fs.writeJSON(path.join(dest, 'run-snapshot.json'), snapshot, { spaces: 2 });
367
385
  }
@@ -409,10 +427,37 @@ export async function createAgent(opts) {
409
427
  }
410
428
  }
411
429
  const agentName = resolveAgentName(opts, process.env);
430
+ let scenarioArtifact;
431
+ if (opts.mcpScenario !== undefined) {
432
+ if (opts.agent === undefined)
433
+ throw new Error('mcpScenario requires an explicit agent');
434
+ if (opts.mcpMock !== undefined || opts.mcpMockApprovalRules !== undefined || opts.mcpConfigFile !== undefined || opts.mcpSafety !== undefined) {
435
+ throw new Error('mcpScenario cannot be combined with legacy or live MCP options');
436
+ }
437
+ const compiled = compileScenario(opts.mcpScenario);
438
+ if (!compiled.ok)
439
+ throw new Error(`Scenario compilation failed: ${JSON.stringify(compiled.diagnostics)}`);
440
+ scenarioArtifact = compiled.artifact;
441
+ }
412
442
  validateOpenCodeDeclaration(agentName, opts);
413
443
  const transport = agentName === 'codex'
414
444
  ? resolveCodexTransport(opts, process.env)
415
445
  : undefined;
446
+ if (scenarioArtifact && agentName === 'codex' && transport !== 'app-server') {
447
+ throw new Error('mcpScenario requires Codex transport app-server; Codex exec is unsupported');
448
+ }
449
+ const scenarioRuntimeFactory = !scenarioArtifact
450
+ ? undefined
451
+ : agentName === 'codex'
452
+ ? startPublicCodexScenarioRuntime
453
+ : agentName === 'opencode'
454
+ ? createPublicOpenCodeScenarioRuntimeFactory({
455
+ opencodeExecutable: opts.opencodeExecutable,
456
+ requestedModel: opts.model,
457
+ })
458
+ : agentName === 'cursor'
459
+ ? createPublicCursorScenarioRuntimeFactory({ requestedModel: opts.model })
460
+ : undefined;
416
461
  let scriptedMcp;
417
462
  if (opts.mcpMockApprovalRules !== undefined) {
418
463
  if (opts.mcpMock === undefined)
@@ -434,7 +479,7 @@ export async function createAgent(opts) {
434
479
  const timeoutSetting = opts.timeout ?? 300;
435
480
  // Capture runner context now; adapters own installation and restoration.
436
481
  const testCtx = opts.debug ? resolveCaseDebugContext() : { name: '', dir: '' };
437
- const { timeout: _, mcpMock, mcpConfigFile, mcpMockApprovalRules: ________, agent: __, debug: ___, transport: _____, mcpSafety: ______, opencodeExecutable, ...rest } = opts;
482
+ const { timeout: _, mcpMock, mcpScenario: _________, mcpConfigFile, mcpMockApprovalRules: ________, agent: __, debug: ___, transport: _____, mcpSafety: ______, opencodeExecutable, ...rest } = opts;
438
483
  const workspace = await prepareWorkspace({
439
484
  ...rest,
440
485
  agent: agentName,
@@ -471,6 +516,8 @@ export async function createAgent(opts) {
471
516
  opencodeExecutable,
472
517
  opencodeMcpToolNames: agentName === 'opencode' ? collectOpenCodeMcpToolNames(mcpMock) : undefined,
473
518
  scriptedMcp,
519
+ scenarioArtifact,
520
+ scenarioRuntimeFactory,
474
521
  sensitiveValues: workspace.sensitiveValues ?? collectSensitiveEnvValues(workspace.env),
475
522
  });
476
523
  lifecycleCore.registerAgent(agent);
@@ -42,6 +42,7 @@ function makeEvaluateAgent() {
42
42
  toolEvents,
43
43
  runCommand: (cmd) => agent.exec(cmd),
44
44
  artifacts: createSessionArtifacts(agent.workspace, toolEvents),
45
+ scenarioEvidence: agent.scenarioEvidence,
45
46
  };
46
47
  const trackedLLM = opts?.llm ?? agent.llm;
47
48
  // Snapshot conversation tokens BEFORE running scorers, for first-eval attribution.
@@ -80,7 +81,7 @@ function makeEvaluateAgent() {
80
81
  ...(opts?.evaluationDefinitionKey
81
82
  ? { evaluationDefinitionKey: opts.evaluationDefinitionKey }
82
83
  : {}),
83
- trial: buildTrialResult(agent.log, { ...evalResult, tokenUsage: deltaTokenUsage }, conversationTokens, conversationCost),
84
+ trial: buildTrialResult(agent.log, { ...evalResult, tokenUsage: deltaTokenUsage }, agent.scenarioEvidence, conversationTokens, conversationCost),
84
85
  };
85
86
  emitEvalResult({ result: recordedResult, agent });
86
87
  maybeThrowOnScorerErrors(recordedResult, opts?.onScorerError ?? 'skip');
@@ -110,6 +111,7 @@ async function fromSnapshot(snapshotPath, scorers, opts) {
110
111
  return sandboxExec(cmd, { cwd: workspace, env: getProcessEnv() });
111
112
  },
112
113
  artifacts: createSessionArtifacts(snapshot.workspace ?? '', snapshot.toolEvents),
114
+ scenarioEvidence: snapshot.scenarioEvidence ?? [],
113
115
  };
114
116
  const scoringStartedAt = performance.now();
115
117
  const evalResult = await evaluateWithContext(ctx, scorers, { ...opts, llm: trackedLLM });
@@ -121,7 +123,7 @@ async function fromSnapshot(snapshotPath, scorers, opts) {
121
123
  ...(opts?.evaluationDefinitionKey
122
124
  ? { evaluationDefinitionKey: opts.evaluationDefinitionKey }
123
125
  : {}),
124
- trial: buildTrialResult(snapshot.log, evalResult),
126
+ trial: buildTrialResult(snapshot.log, evalResult, snapshot.scenarioEvidence),
125
127
  };
126
128
  maybeThrowOnScorerErrors(recordedResult, opts?.onScorerError ?? 'skip');
127
129
  return recordedResult;
@@ -265,7 +267,7 @@ function matchesArtifactPattern(artifactPath, pattern) {
265
267
  return artifactPath.includes(pattern);
266
268
  return pattern.test(artifactPath);
267
269
  }
268
- function buildTrialResult(log, result, conversationTokens, conversationCost) {
270
+ function buildTrialResult(log, result, scenarioEvidence, conversationTokens, conversationCost) {
269
271
  const nCommands = log.filter((entry) => entry.type === 'command').length;
270
272
  const skills = extractSkillsFromLog(log);
271
273
  return {
@@ -285,6 +287,7 @@ function buildTrialResult(log, result, conversationTokens, conversationCost) {
285
287
  // field.
286
288
  ...conversationCost,
287
289
  session_log: [...log],
290
+ ...(scenarioEvidence && scenarioEvidence.length > 0 ? { scenario_evidence: structuredClone(scenarioEvidence) } : {}),
288
291
  ...(skills.length > 0 ? { skills_used: skills } : {}),
289
292
  };
290
293
  }
@@ -2,8 +2,12 @@ export { createAgent } from './agent.js';
2
2
  export { resolveAgentName, resolveCodexTransport, InvalidTransportEnvError, } from './agent-resolution.js';
3
3
  export { AgentCrashError } from './agent-crash.js';
4
4
  export { check, score, judge, toolUsage } from './scorers.js';
5
+ export { compileScenario, compileScenarioText } from './scenario-machine-v2.js';
6
+ export { findScenarioCaseMatches, getFinalScenarioState, getScenarioStateTimeline, ScenarioEvidenceError, wasScenarioCaseMatched, } from './scenario-evidence.js';
5
7
  export { getMcpToolCall, isMcpToolCall, findMcpToolCalls, getMcpStartupStatus, isMcpStartupStatus, getMcpApproval, isMcpApproval, findMcpApprovals, getMcpInvocation, wasMcpToolInvoked, } from './mcp-evidence.js';
6
8
  export type { McpMockApprovalRule, McpMockJsonValue } from './mcp-mock-approvals.js';
9
+ export type { CompileScenarioResult, EvidenceEnvelope, JsonObject, JsonPrimitive, JsonValue, MatchExpr, MatchPredicate, McpContent, ScenarioArtifact, ScenarioCase, ScenarioDiagnostic, ScenarioEvidence, ScenarioMachineV2, ScenarioRejection, ScenarioSuccess, ScenarioToolError, } from './scenario-machine-v2.js';
10
+ export type { ExpectedScenarioCaseMatch, ScenarioCaseMatchEvidence, ScenarioStateTransitionEvidence, } from './scenario-evidence.js';
7
11
  export { decideMcpToolCall, redactMcpSecrets, } from './mcp-safety.js';
8
12
  export { evaluate, EvalScorerError } from './evaluate.js';
9
13
  export { RUN_SNAPSHOT_VERSION, buildRunSnapshot, loadRunSnapshot, SnapshotParseError, SnapshotVersionError, WorkspaceMissingError, } from './snapshots.js';
package/dist/sdk/index.js CHANGED
@@ -3,6 +3,8 @@ export { createAgent } from './agent.js';
3
3
  export { resolveAgentName, resolveCodexTransport, InvalidTransportEnvError, } from './agent-resolution.js';
4
4
  export { AgentCrashError } from './agent-crash.js';
5
5
  export { check, score, judge, toolUsage } from './scorers.js';
6
+ export { compileScenario, compileScenarioText } from './scenario-machine-v2.js';
7
+ export { findScenarioCaseMatches, getFinalScenarioState, getScenarioStateTimeline, ScenarioEvidenceError, wasScenarioCaseMatched, } from './scenario-evidence.js';
6
8
  export { getMcpToolCall, isMcpToolCall, findMcpToolCalls, getMcpStartupStatus, isMcpStartupStatus, getMcpApproval, isMcpApproval, findMcpApprovals, getMcpInvocation, wasMcpToolInvoked, } from './mcp-evidence.js';
7
9
  export { decideMcpToolCall, redactMcpSecrets, } from './mcp-safety.js';
8
10
  export { evaluate, EvalScorerError } from './evaluate.js';
@@ -5,6 +5,8 @@ import type { McpSafetyOptions } from './mcp-safety.js';
5
5
  import type { LLMPort } from '../utils/llm-types.js';
6
6
  import type { AskBus } from './ask-bus/types.js';
7
7
  import type { CompiledMcpMockSession } from './mcp-mock-approvals.js';
8
+ import { type PublicScenarioRuntimeFactory } from '../internal/direct-mcp-v2/public-scenario-runtime.js';
9
+ import type { EvidenceEnvelope, ScenarioArtifact } from '../internal/direct-mcp-v2/types.js';
8
10
  export interface ManagedSessionDeps {
9
11
  ws: Workspace;
10
12
  agentName: AgentName;
@@ -36,6 +38,10 @@ export interface ManagedSessionDeps {
36
38
  opencodeMcpToolNames?: string[];
37
39
  /** Trusted pre-workspace compiled generated-MCP declaration. */
38
40
  scriptedMcp?: CompiledMcpMockSession;
41
+ scenarioArtifact?: ScenarioArtifact;
42
+ /** Provider adapter entrypoint; defaults to the preserved Claude bridge. */
43
+ scenarioRuntimeFactory?: PublicScenarioRuntimeFactory;
44
+ onScenarioEvidence?: (evidence: readonly EvidenceEnvelope[]) => void;
39
45
  /** Runtime-only values that must not enter session logs or persisted results. */
40
46
  sensitiveValues?: readonly string[];
41
47
  }
@@ -10,6 +10,8 @@ import { attachTurnResultSensitiveValues, cloneTurnResultWithSensitiveValues, ge
10
10
  import { attachToolEventSensitiveValues, getToolEventSensitiveValues } from './tool-event-secrets.js';
11
11
  import { sanitizePersistenceValue } from '../tool-event-results.js';
12
12
  import { startScriptedMcpMockHost, } from '../providers/scripted-mcp-mock-host.js';
13
+ import { startPublicClaudeScenarioRuntime, } from '../internal/direct-mcp-v2/public-scenario-runtime.js';
14
+ import { TrialExecutionError } from '../internal/direct-mcp-v2/cleanup.js';
13
15
  export function createManagedSession(deps) {
14
16
  const { ws, agentName, timeoutSec, messages, log, model, conversationWindow, llm } = deps;
15
17
  const agentTimeoutMs = timeoutSec * 1000;
@@ -40,9 +42,12 @@ export function createManagedSession(deps) {
40
42
  let setupDone = false;
41
43
  let currentSignal;
42
44
  let scriptedHost;
45
+ let scenarioRuntime;
43
46
  let activeTurn = false;
44
47
  let nextTurnNumber = 0;
45
48
  let disposePromise;
49
+ let primaryTurnError;
50
+ let inFlightTurn;
46
51
  const runCommand = async (cmd) => {
47
52
  const result = await ws.exec(cmd, { signal: currentSignal });
48
53
  log.push(sanitizePersistenceValue({
@@ -62,67 +67,145 @@ export function createManagedSession(deps) {
62
67
  if (remaining <= 0)
63
68
  throw new Error(`${label} timed out`);
64
69
  activeTurn = true;
70
+ let finishTurnJoin;
71
+ const turnJoin = new Promise((resolve) => { finishTurnJoin = resolve; });
72
+ inFlightTurn = turnJoin;
65
73
  try {
66
74
  return await withAbortTimeout(async (signal) => {
67
- currentSignal = signal;
68
- if (!session) {
69
- // Run auth setup commands before first agent turn
70
- if (!setupDone) {
71
- for (const cmd of ws.setupCommands) {
72
- await ws.exec(cmd, { signal });
73
- signal.throwIfAborted();
75
+ try {
76
+ currentSignal = signal;
77
+ if (!session) {
78
+ // Run auth setup commands before first agent turn
79
+ if (!setupDone) {
80
+ for (const cmd of ws.setupCommands) {
81
+ await ws.exec(cmd, { signal });
82
+ signal.throwIfAborted();
83
+ }
84
+ setupDone = true;
85
+ }
86
+ // Carry the workspace's resolved env (from `prepareWorkspace
87
+ // → resolveCredentials`) into the runtime handle so drivers
88
+ // that auth through `Options.env` (notably the Claude SDK
89
+ // driver) can lift Anthropic keys out. Drivers that only
90
+ // need the workspace path keep using `getWorkspacePath`.
91
+ const runtime = {
92
+ handle: ws.path,
93
+ workspacePath: ws.path,
94
+ env: ws.env,
95
+ };
96
+ if (deps.scriptedMcp) {
97
+ scriptedHost = await startScriptedMcpMockHost(deps.scriptedMcp);
98
+ sessionOptions.scriptedMcpHost = scriptedHost;
99
+ }
100
+ if (deps.scenarioArtifact) {
101
+ const startScenarioRuntime = deps.scenarioRuntimeFactory ?? startPublicClaudeScenarioRuntime;
102
+ scenarioRuntime = await startScenarioRuntime({
103
+ artifact: deps.scenarioArtifact,
104
+ workspacePath: ws.path,
105
+ environment: ws.env,
106
+ onEvidence: deps.onScenarioEvidence,
107
+ });
108
+ try {
109
+ scenarioRuntime.configureSession(sessionOptions);
110
+ }
111
+ catch (error) {
112
+ try {
113
+ await scenarioRuntime.dispose();
114
+ }
115
+ catch (cleanupError) {
116
+ throw new AggregateError([error, cleanupError], 'Scenario provider projection failed and coordinator cleanup also failed');
117
+ }
118
+ scenarioRuntime = undefined;
119
+ throw error;
120
+ }
121
+ }
122
+ try {
123
+ session = await createAgentSession(agent, runtime, runCommand, sessionOptions);
124
+ }
125
+ catch (error) {
126
+ try {
127
+ await scriptedHost?.dispose();
128
+ await scenarioRuntime?.dispose();
129
+ }
130
+ catch (cleanupError) {
131
+ throw new AggregateError([error, cleanupError], 'Agent session setup failed and scenario cleanup also failed');
132
+ }
133
+ scriptedHost = undefined;
134
+ scenarioRuntime = undefined;
135
+ throw error;
74
136
  }
75
- setupDone = true;
76
- }
77
- // Carry the workspace's resolved env (from `prepareWorkspace
78
- // → resolveCredentials`) into the runtime handle so drivers
79
- // that auth through `Options.env` (notably the Claude SDK
80
- // driver) can lift Anthropic keys out. Drivers that only
81
- // need the workspace path keep using `getWorkspacePath`.
82
- const runtime = {
83
- handle: ws.path,
84
- workspacePath: ws.path,
85
- env: ws.env,
86
- };
87
- if (deps.scriptedMcp) {
88
- scriptedHost = await startScriptedMcpMockHost(deps.scriptedMcp);
89
- sessionOptions.scriptedMcpHost = scriptedHost;
90
137
  }
138
+ const turnNumber = ++nextTurnNumber;
139
+ scriptedHost?.beginTurn(turnNumber);
140
+ scenarioRuntime?.beginTurn(`turn-${turnNumber}`, signal);
141
+ let result;
91
142
  try {
92
- session = await createAgentSession(agent, runtime, runCommand, sessionOptions);
143
+ result = turnNumber === 1
144
+ ? await session.start({ message })
145
+ : await session.reply({ message });
93
146
  }
94
147
  catch (error) {
95
- await scriptedHost?.dispose();
96
- scriptedHost = undefined;
148
+ try {
149
+ await scenarioRuntime?.failTurn(error);
150
+ }
151
+ catch (lifecycleError) {
152
+ if (lifecycleError instanceof TrialExecutionError) {
153
+ primaryTurnError = lifecycleError;
154
+ throw lifecycleError;
155
+ }
156
+ const combined = new AggregateError([error, lifecycleError], 'Provider turn failed and scenario settlement also failed');
157
+ primaryTurnError = combined;
158
+ throw combined;
159
+ }
160
+ primaryTurnError = error;
97
161
  throw error;
98
162
  }
99
- }
100
- const turnNumber = ++nextTurnNumber;
101
- scriptedHost?.beginTurn(turnNumber);
102
- let result = turnNumber === 1
103
- ? await session.start({ message })
104
- : await session.reply({ message });
105
- if (scriptedHost) {
106
- const settled = scriptedHost.settleEvents(result.toolEvents);
107
- result.toolEvents = settled.events;
108
- if (settled.error) {
109
- result = cloneTurnResultWithSensitiveValues(result, {
110
- exitCode: 1,
111
- rawOutput: settled.error.message,
112
- });
163
+ if (result.exitCode !== 0 || result.timedOut) {
164
+ const providerTerminalError = new Error(`Provider terminal: exit ${result.exitCode}`);
165
+ primaryTurnError = providerTerminalError;
166
+ try {
167
+ await scenarioRuntime?.failTurn(providerTerminalError);
168
+ }
169
+ catch (lifecycleError) {
170
+ if (lifecycleError instanceof TrialExecutionError) {
171
+ primaryTurnError = lifecycleError;
172
+ throw lifecycleError;
173
+ }
174
+ const combined = new AggregateError([providerTerminalError, lifecycleError], 'Provider terminal failed and scenario settlement also failed');
175
+ primaryTurnError = combined;
176
+ throw combined;
177
+ }
178
+ }
179
+ else {
180
+ await scenarioRuntime?.settleTurn();
181
+ }
182
+ if (scriptedHost) {
183
+ const settled = scriptedHost.settleEvents(result.toolEvents);
184
+ result.toolEvents = settled.events;
185
+ if (settled.error) {
186
+ result = cloneTurnResultWithSensitiveValues(result, {
187
+ exitCode: 1,
188
+ rawOutput: settled.error.message,
189
+ });
190
+ }
191
+ }
192
+ const sensitiveValues = [...new Set([
193
+ ...(deps.sensitiveValues ?? []),
194
+ ...getTurnResultSensitiveValues(result),
195
+ ])];
196
+ for (const event of result.toolEvents) {
197
+ attachToolEventSensitiveValues(event, [...new Set([
198
+ ...sensitiveValues,
199
+ ...getToolEventSensitiveValues(event),
200
+ ])]);
113
201
  }
202
+ return attachTurnResultSensitiveValues(result, sensitiveValues);
114
203
  }
115
- const sensitiveValues = [...new Set([
116
- ...(deps.sensitiveValues ?? []),
117
- ...getTurnResultSensitiveValues(result),
118
- ])];
119
- for (const event of result.toolEvents) {
120
- attachToolEventSensitiveValues(event, [...new Set([
121
- ...sensitiveValues,
122
- ...getToolEventSensitiveValues(event),
123
- ])]);
204
+ finally {
205
+ finishTurnJoin();
206
+ if (inFlightTurn === turnJoin)
207
+ inFlightTurn = undefined;
124
208
  }
125
- return attachTurnResultSensitiveValues(result, sensitiveValues);
126
209
  }, remaining, label);
127
210
  }
128
211
  finally {
@@ -166,12 +249,36 @@ export function createManagedSession(deps) {
166
249
  if (disposePromise)
167
250
  return disposePromise;
168
251
  disposePromise = (async () => {
252
+ const cleanupErrors = [];
169
253
  try {
170
254
  await session?.dispose?.();
171
255
  }
172
- finally {
256
+ catch (error) {
257
+ cleanupErrors.push(error);
258
+ }
259
+ if (inFlightTurn) {
260
+ try {
261
+ await withAbortTimeout(async () => inFlightTurn, Math.min(agentTimeoutMs, 2_000), 'Agent turn cleanup');
262
+ }
263
+ catch (error) {
264
+ cleanupErrors.push(error);
265
+ }
266
+ }
267
+ try {
173
268
  await scriptedHost?.dispose();
174
269
  }
270
+ catch (error) {
271
+ cleanupErrors.push(error);
272
+ }
273
+ try {
274
+ await scenarioRuntime?.dispose();
275
+ }
276
+ catch (error) {
277
+ cleanupErrors.push(error);
278
+ }
279
+ if (cleanupErrors.length > 0) {
280
+ throw new AggregateError(primaryTurnError === undefined ? cleanupErrors : [primaryTurnError, ...cleanupErrors], primaryTurnError === undefined ? 'Agent session cleanup failed' : 'Provider turn failed and cleanup also failed');
281
+ }
175
282
  })();
176
283
  return disposePromise;
177
284
  },
@@ -0,0 +1,27 @@
1
+ import type { EvidenceEnvelope } from '../internal/direct-mcp-v2/types.js';
2
+ export type ExpectedScenarioCaseMatch = {
3
+ caseId?: string;
4
+ serverId?: string;
5
+ toolId?: string;
6
+ turnId?: string;
7
+ };
8
+ export type ScenarioCaseMatchEvidence = {
9
+ trialId: string;
10
+ turnId: string;
11
+ invocationId: string;
12
+ serverId: string;
13
+ toolId: string;
14
+ caseId: string;
15
+ stateBefore: string;
16
+ };
17
+ export type ScenarioStateTransitionEvidence = ScenarioCaseMatchEvidence & {
18
+ stateAfter: string;
19
+ committedEpoch: number;
20
+ };
21
+ export declare class ScenarioEvidenceError extends Error {
22
+ constructor(message: string);
23
+ }
24
+ export declare function findScenarioCaseMatches(evidence: readonly EvidenceEnvelope[], expected?: ExpectedScenarioCaseMatch): ScenarioCaseMatchEvidence[];
25
+ export declare function wasScenarioCaseMatched(evidence: readonly EvidenceEnvelope[], expected: ExpectedScenarioCaseMatch): boolean;
26
+ export declare function getScenarioStateTimeline(evidence: readonly EvidenceEnvelope[]): ScenarioStateTransitionEvidence[];
27
+ export declare function getFinalScenarioState(evidence: readonly EvidenceEnvelope[]): string;
@@ -0,0 +1,94 @@
1
+ import { verifyEvidenceChain } from '../internal/direct-mcp-v2/evidence.js';
2
+ export class ScenarioEvidenceError extends Error {
3
+ constructor(message) {
4
+ super(message);
5
+ this.name = 'ScenarioEvidenceError';
6
+ }
7
+ }
8
+ export function findScenarioCaseMatches(evidence, expected = {}) {
9
+ const invocations = canonicalInvocations(evidence);
10
+ const matches = [];
11
+ for (const envelope of evidence) {
12
+ const decision = envelope.payload;
13
+ if (decision.type !== 'mcp.decision' || decision.outcome !== 'matched' || !decision.caseId)
14
+ continue;
15
+ const invocation = invocations.get(decision.invocationId);
16
+ if (!invocation)
17
+ throw new ScenarioEvidenceError(`matched decision references unknown invocation ${decision.invocationId}`);
18
+ const match = { ...invocation, caseId: decision.caseId };
19
+ if (matchesExpectedCase(match, expected))
20
+ matches.push(match);
21
+ }
22
+ return matches;
23
+ }
24
+ export function wasScenarioCaseMatched(evidence, expected) {
25
+ return findScenarioCaseMatches(evidence, expected).length > 0;
26
+ }
27
+ export function getScenarioStateTimeline(evidence) {
28
+ const matches = new Map(findScenarioCaseMatches(evidence).map((entry) => [entry.invocationId, entry]));
29
+ const timeline = [];
30
+ for (const envelope of evidence) {
31
+ const commit = envelope.payload;
32
+ if (commit.type !== 'mcp.commit')
33
+ continue;
34
+ const match = matches.get(commit.invocationId);
35
+ if (!match)
36
+ throw new ScenarioEvidenceError(`commit references unmatched invocation ${commit.invocationId}`);
37
+ if (match.stateBefore !== commit.stateBefore) {
38
+ throw new ScenarioEvidenceError(`commit state disagrees with invocation ${commit.invocationId}`);
39
+ }
40
+ timeline.push({
41
+ ...match,
42
+ stateAfter: commit.stateAfter,
43
+ committedEpoch: commit.committedEpoch,
44
+ });
45
+ }
46
+ return timeline;
47
+ }
48
+ export function getFinalScenarioState(evidence) {
49
+ assertCanonicalEvidence(evidence);
50
+ const payload = evidence.at(-1)?.payload;
51
+ if (payload?.type === 'turn.finalized' && payload.state !== undefined)
52
+ return payload.state;
53
+ throw new ScenarioEvidenceError('canonical evidence does not end with a finalized scenario state');
54
+ }
55
+ function canonicalInvocations(evidence) {
56
+ assertCanonicalEvidence(evidence);
57
+ const invocations = new Map();
58
+ for (const envelope of evidence) {
59
+ const invocation = envelope.payload;
60
+ if (invocation.type !== 'mcp.invocation')
61
+ continue;
62
+ if (invocations.has(invocation.invocationId)) {
63
+ throw new ScenarioEvidenceError(`duplicate invocation ${invocation.invocationId}`);
64
+ }
65
+ invocations.set(invocation.invocationId, {
66
+ trialId: envelope.trialId,
67
+ turnId: envelope.turnId,
68
+ invocationId: invocation.invocationId,
69
+ serverId: invocation.serverId,
70
+ toolId: invocation.toolId,
71
+ stateBefore: invocation.stateBefore,
72
+ });
73
+ }
74
+ return invocations;
75
+ }
76
+ function assertCanonicalEvidence(evidence) {
77
+ const first = evidence[0];
78
+ if (!first)
79
+ throw new ScenarioEvidenceError('canonical evidence is empty');
80
+ const chain = verifyEvidenceChain(evidence, first.artifactDigest);
81
+ if (!chain.ok)
82
+ throw new ScenarioEvidenceError(`canonical evidence chain failed at index ${chain.index}: ${chain.reason}`);
83
+ for (const [index, envelope] of evidence.entries()) {
84
+ if (envelope.trialId !== first.trialId) {
85
+ throw new ScenarioEvidenceError(`canonical evidence changes trialId at index ${index}`);
86
+ }
87
+ }
88
+ }
89
+ function matchesExpectedCase(match, expected) {
90
+ return (expected.caseId === undefined || match.caseId === expected.caseId)
91
+ && (expected.serverId === undefined || match.serverId === expected.serverId)
92
+ && (expected.toolId === undefined || match.toolId === expected.toolId)
93
+ && (expected.turnId === undefined || match.turnId === expected.turnId);
94
+ }
@@ -0,0 +1,9 @@
1
+ import type { CompileScenarioResult } from '../internal/direct-mcp-v2/compiler.js';
2
+ import type { ScenarioMachineV2 as InternalScenarioMachineV2 } from '../internal/direct-mcp-v2/types.js';
3
+ /** Compile the public V2 source contract, whose concurrency safety policy is fixed by the version. */
4
+ export declare function compileScenario(source: unknown): CompileScenarioResult;
5
+ /** Compile strict JSON text using the public V2 source contract. */
6
+ export declare function compileScenarioText(text: string): CompileScenarioResult;
7
+ export type { CompileScenarioResult } from '../internal/direct-mcp-v2/compiler.js';
8
+ export type ScenarioMachineV2 = Pick<InternalScenarioMachineV2, 'version' | 'scenarioId' | 'initialState' | 'servers' | 'states'>;
9
+ export type { EvidenceEnvelope, JsonObject, JsonPrimitive, JsonValue, MatchExpr, MatchPredicate, McpContent, ScenarioArtifact, ScenarioCase, ScenarioDiagnostic, ScenarioEvidence, ScenarioRejection, ScenarioSuccess, ScenarioToolError, } from '../internal/direct-mcp-v2/types.js';
@@ -0,0 +1,37 @@
1
+ import { compileScenario as compileInternalScenario } from '../internal/direct-mcp-v2/compiler.js';
2
+ import { canonicalJson, parseStrictJson } from '../internal/direct-mcp-v2/json.js';
3
+ const V2_CONCURRENCY_POLICY = 'poison-effectful-overlap';
4
+ function object(value) {
5
+ return value !== null && typeof value === 'object' && !Array.isArray(value);
6
+ }
7
+ /** Compile the public V2 source contract, whose concurrency safety policy is fixed by the version. */
8
+ export function compileScenario(source) {
9
+ try {
10
+ canonicalJson(source);
11
+ }
12
+ catch (error) {
13
+ return { ok: false, diagnostics: [{ code: 'INVALID_JSON_VALUE', path: '$', message: error instanceof Error ? error.message : String(error) }] };
14
+ }
15
+ if (!object(source))
16
+ return compileInternalScenario(source);
17
+ const suppliedConcurrency = Object.prototype.hasOwnProperty.call(source, 'concurrency');
18
+ const compiled = compileInternalScenario({ ...source, concurrency: V2_CONCURRENCY_POLICY });
19
+ if (!suppliedConcurrency)
20
+ return compiled;
21
+ const diagnostic = { code: 'UNKNOWN_FIELD', path: '$/concurrency', message: 'unknown field concurrency' };
22
+ if (compiled.ok)
23
+ return { ok: false, diagnostics: [diagnostic] };
24
+ return {
25
+ ok: false,
26
+ diagnostics: [...compiled.diagnostics, diagnostic].sort((left, right) => canonicalJson(left).localeCompare(canonicalJson(right), 'en')),
27
+ };
28
+ }
29
+ /** Compile strict JSON text using the public V2 source contract. */
30
+ export function compileScenarioText(text) {
31
+ try {
32
+ return compileScenario(parseStrictJson(text));
33
+ }
34
+ catch (error) {
35
+ return { ok: false, diagnostics: [{ code: 'INVALID_JSON_TEXT', path: '$', message: error instanceof Error ? error.message : String(error) }] };
36
+ }
37
+ }