@wix/pathgrade 1.0.38 → 1.0.39
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +1 -1
- package/dist/agents/claude/sdk-message-projector.js +4 -5
- package/dist/agents/claude/tool-permission-bridge.js +2 -1
- package/dist/agents/codex-app-server/item-projection.js +2 -2
- package/dist/agents/codex-app-server/mcp-approval-correlator.js +3 -2
- package/dist/agents/opencode.js +4 -5
- package/dist/commands/report.js +5 -25
- package/dist/internal/direct-mcp-v2/acp-author-projector.js +20 -7
- package/dist/reporters/cli.js +20 -1
- package/dist/reporters/github-comment.js +18 -7
- package/dist/reporters/loader.d.ts +4 -0
- package/dist/reporters/loader.js +26 -9
- package/dist/reporting/core.js +17 -7
- package/dist/reporting/report-parser.js +57 -2
- package/dist/reporting/types.d.ts +2 -1
- package/dist/runners/orchestrator.js +3 -4
- package/dist/runners/repeated-invocation.js +2 -5
- package/dist/runners/report-projection.js +1 -0
- package/dist/sdk/agent.js +10 -1
- package/dist/sdk/evaluate.js +58 -3
- package/dist/sdk/judge-prompt-builder.js +11 -7
- package/dist/sdk/mcp-event-input.d.ts +1 -0
- package/dist/sdk/mcp-event-input.js +3 -0
- package/dist/sdk/mcp-evidence.js +16 -3
- package/dist/sdk/mcp-safety.js +2 -2
- package/dist/sdk/scripted-mcp-events.js +3 -2
- package/dist/sdk/tool-event-log.js +18 -3
- package/dist/sdk/tool-event-secrets.d.ts +4 -0
- package/dist/sdk/tool-event-secrets.js +33 -2
- package/dist/sdk/types.d.ts +2 -0
- package/dist/tool-event-results.d.ts +3 -0
- package/dist/tool-event-results.js +153 -23
- package/dist/types.d.ts +9 -5
- package/package.json +2 -2
package/dist/sdk/agent.js
CHANGED
|
@@ -20,6 +20,8 @@ import { collectOpenCodeMcpToolNames } from '../agents/opencode/contract.js';
|
|
|
20
20
|
import { collectSensitiveEnvValues } from '../tool-event-results.js';
|
|
21
21
|
import { createAskUserHandler } from './ask-bus/handler.js';
|
|
22
22
|
import { resolveAgentRuntimeOptions } from './agent-runtime-options.js';
|
|
23
|
+
import { extractToolEventsFromLog } from '../tool-events.js';
|
|
24
|
+
import { clearToolEventRuntimeMetadata } from './tool-event-secrets.js';
|
|
23
25
|
/**
|
|
24
26
|
* Test-only injection point: override the sink used by the next emitter
|
|
25
27
|
* built inside `createAgent`. Pass `null` to restore the default (stderr).
|
|
@@ -392,7 +394,14 @@ class AgentImpl {
|
|
|
392
394
|
}
|
|
393
395
|
}
|
|
394
396
|
finally {
|
|
395
|
-
|
|
397
|
+
try {
|
|
398
|
+
await this.ws.dispose();
|
|
399
|
+
}
|
|
400
|
+
finally {
|
|
401
|
+
for (const event of extractToolEventsFromLog(this._log)) {
|
|
402
|
+
clearToolEventRuntimeMetadata(event);
|
|
403
|
+
}
|
|
404
|
+
}
|
|
396
405
|
}
|
|
397
406
|
}
|
|
398
407
|
}
|
package/dist/sdk/evaluate.js
CHANGED
|
@@ -7,6 +7,9 @@ import { runScorer } from './run-scorer.js';
|
|
|
7
7
|
import { createLLMClient } from '../utils/llm.js';
|
|
8
8
|
import { sandboxExec } from '../providers/sandbox-exec.js';
|
|
9
9
|
import { buildTranscript, loadRunSnapshot, WorkspaceMissingError } from './snapshots.js';
|
|
10
|
+
import { sanitizePersistenceValue } from '../tool-event-results.js';
|
|
11
|
+
import { getOriginalMcpInput } from './mcp-event-input.js';
|
|
12
|
+
import { cloneToolEventWithRuntimeMetadata, collectToolEventSensitiveValues, } from './tool-event-secrets.js';
|
|
10
13
|
import fs from 'fs-extra';
|
|
11
14
|
import path from 'path';
|
|
12
15
|
import { summarizeFlow } from './agent-flow.js';
|
|
@@ -154,6 +157,9 @@ export function evaluateStepScorers(agent, scorers, opts) {
|
|
|
154
157
|
return evaluateAgent(agent, scorers, opts, false);
|
|
155
158
|
}
|
|
156
159
|
async function fromSnapshot(snapshotPath, scorers, opts) {
|
|
160
|
+
if (opts?.deterministicToolEvidence === 'live') {
|
|
161
|
+
throw new TypeError("evaluate.fromSnapshot() does not support deterministicToolEvidence: 'live'");
|
|
162
|
+
}
|
|
157
163
|
const snapshot = await loadRunSnapshot(snapshotPath);
|
|
158
164
|
const trackedLLM = opts?.llm ?? createLLMClient({ adapters: [{
|
|
159
165
|
name: 'runtime', isAvailable: async () => true,
|
|
@@ -201,6 +207,9 @@ async function evaluateWithContext(ctx, scorers, opts) {
|
|
|
201
207
|
call: (prompt, callOpts) => getRuntime().llm.call(prompt, callOpts),
|
|
202
208
|
}] });
|
|
203
209
|
const onScorerError = opts?.onScorerError ?? 'skip';
|
|
210
|
+
const deterministicCtx = opts?.deterministicToolEvidence === 'live'
|
|
211
|
+
? createLiveDeterministicContext(ctx)
|
|
212
|
+
: ctx;
|
|
204
213
|
const phase1 = [];
|
|
205
214
|
const phase2 = [];
|
|
206
215
|
const phase3 = [];
|
|
@@ -219,7 +228,7 @@ async function evaluateWithContext(ctx, scorers, opts) {
|
|
|
219
228
|
}
|
|
220
229
|
}
|
|
221
230
|
const results = [];
|
|
222
|
-
const phase1Results = await Promise.all(phase1.map((g) => runScorer(g,
|
|
231
|
+
const phase1Results = await Promise.all(phase1.map((g) => runScorer(g, deterministicCtx)));
|
|
223
232
|
results.push(...phase1Results);
|
|
224
233
|
const anyCheckFailed = failFast && phase1Results.some((r) => r.type === 'check' && r.status !== 'error' && r.score === 0);
|
|
225
234
|
if (anyCheckFailed) {
|
|
@@ -230,7 +239,7 @@ async function evaluateWithContext(ctx, scorers, opts) {
|
|
|
230
239
|
else {
|
|
231
240
|
const judgeResults = await runJudgePipeline(phase2, ctx, { llm: trackedLLM });
|
|
232
241
|
results.push(...judgeResults);
|
|
233
|
-
const phase3Results = await Promise.all(phase3.map((g) => runScorer(g,
|
|
242
|
+
const phase3Results = await Promise.all(phase3.map((g) => runScorer(g, deterministicCtx)));
|
|
234
243
|
results.push(...phase3Results);
|
|
235
244
|
}
|
|
236
245
|
const tokenUsage = trackedLLM.tokenUsage ?? { inputTokens: 0, outputTokens: 0 };
|
|
@@ -240,12 +249,58 @@ async function evaluateWithContext(ctx, scorers, opts) {
|
|
|
240
249
|
const totalWeight = scoringResults.reduce((sum, r) => sum + r.weight, 0);
|
|
241
250
|
const weightedSum = scoringResults.reduce((sum, r) => sum + r.score * r.weight, 0);
|
|
242
251
|
const score = totalWeight > 0 ? weightedSum / totalWeight : 0;
|
|
243
|
-
return {
|
|
252
|
+
return sanitizePersistenceValue({
|
|
244
253
|
score,
|
|
245
254
|
scorers: results,
|
|
246
255
|
tokenUsage,
|
|
256
|
+
}, collectToolEventSensitiveValues(ctx.toolEvents));
|
|
257
|
+
}
|
|
258
|
+
function createLiveDeterministicContext(ctx) {
|
|
259
|
+
return {
|
|
260
|
+
...ctx,
|
|
261
|
+
toolEvents: ctx.toolEvents.map((event) => {
|
|
262
|
+
const originalInput = event.action === 'mcp_tool_call'
|
|
263
|
+
? getOriginalMcpInput(event)
|
|
264
|
+
: undefined;
|
|
265
|
+
const mcp = normalizedMcpClassification(event);
|
|
266
|
+
return originalInput
|
|
267
|
+
? cloneToolEventWithRuntimeMetadata(event, { arguments: {
|
|
268
|
+
...event.arguments,
|
|
269
|
+
server: event.arguments?.server,
|
|
270
|
+
tool: event.arguments?.tool,
|
|
271
|
+
status: event.arguments?.status,
|
|
272
|
+
...structuredClone(originalInput),
|
|
273
|
+
}, ...(mcp ? { mcp } : {}) })
|
|
274
|
+
: event;
|
|
275
|
+
}),
|
|
247
276
|
};
|
|
248
277
|
}
|
|
278
|
+
function normalizedMcpClassification(event) {
|
|
279
|
+
if (event.mcp)
|
|
280
|
+
return event.mcp;
|
|
281
|
+
const args = event.arguments ?? {};
|
|
282
|
+
const separator = event.providerToolName.indexOf('.');
|
|
283
|
+
const serverName = typeof args.server === 'string' ? args.server
|
|
284
|
+
: separator > 0 ? event.providerToolName.slice(0, separator) : undefined;
|
|
285
|
+
const toolName = typeof args.tool === 'string' ? args.tool
|
|
286
|
+
: separator > 0 && separator < event.providerToolName.length - 1
|
|
287
|
+
? event.providerToolName.slice(separator + 1) : undefined;
|
|
288
|
+
const status = event.status ?? args.status;
|
|
289
|
+
if (!serverName || !toolName)
|
|
290
|
+
return undefined;
|
|
291
|
+
if (status === 'completed')
|
|
292
|
+
return { serverName, toolName, invocation: 'confirmed', outcome: 'completed' };
|
|
293
|
+
if (status === 'error' || status === 'failed') {
|
|
294
|
+
return { serverName, toolName, invocation: 'confirmed', outcome: 'tool_error' };
|
|
295
|
+
}
|
|
296
|
+
if (status === 'user_denied' || status === 'policy_denied') {
|
|
297
|
+
return { serverName, toolName, invocation: 'not_invoked', outcome: status };
|
|
298
|
+
}
|
|
299
|
+
if (status === 'protocol_error') {
|
|
300
|
+
return { serverName, toolName, invocation: 'unknown', outcome: 'protocol_error' };
|
|
301
|
+
}
|
|
302
|
+
return undefined;
|
|
303
|
+
}
|
|
249
304
|
function maybeThrowOnScorerErrors(result, mode) {
|
|
250
305
|
if (mode !== 'fail')
|
|
251
306
|
return;
|
|
@@ -1,13 +1,15 @@
|
|
|
1
1
|
import { sanitizePersistenceValue } from '../tool-event-results.js';
|
|
2
|
-
import {
|
|
2
|
+
import { collectToolEventSensitiveValues, } from './tool-event-secrets.js';
|
|
3
3
|
export function buildJudgePrompt(scorer, ctx, input) {
|
|
4
4
|
const sections = [];
|
|
5
|
-
|
|
5
|
+
const sensitiveValues = collectToolEventSensitiveValues(ctx.toolEvents);
|
|
6
|
+
sections.push(`## Session Transcript\n${sanitizePersistenceValue(ctx.transcript, sensitiveValues)}`);
|
|
6
7
|
if (scorer.includeToolEvents && ctx.toolEvents.length > 0) {
|
|
7
8
|
sections.push(`## Tool Events\n${formatToolEvents(ctx)}`);
|
|
8
9
|
}
|
|
9
10
|
if (input) {
|
|
10
|
-
|
|
11
|
+
const sanitizedInput = sanitizePersistenceValue(input, sensitiveValues);
|
|
12
|
+
for (const [key, value] of Object.entries(sanitizedInput)) {
|
|
11
13
|
const body = typeof value === 'string' ? value : JSON.stringify(value, null, 2);
|
|
12
14
|
sections.push(`## ${key}\n${body}`);
|
|
13
15
|
}
|
|
@@ -22,10 +24,10 @@ ${scorer.rubric}
|
|
|
22
24
|
Respond with ONLY a JSON object: {"score": <number>, "details": "<brief explanation>"}`;
|
|
23
25
|
}
|
|
24
26
|
function formatToolEvents(ctx) {
|
|
27
|
+
const sensitiveValues = collectToolEventSensitiveValues(ctx.toolEvents);
|
|
25
28
|
return ctx.toolEvents
|
|
26
29
|
.map((event) => {
|
|
27
30
|
const turn = event.turnNumber ? `turn ${event.turnNumber}` : 'instruction';
|
|
28
|
-
const sensitiveValues = getToolEventSensitiveValues(event);
|
|
29
31
|
const details = [
|
|
30
32
|
event.status ? ` status: ${event.status}` : undefined,
|
|
31
33
|
event.mcp ? ` mcp: ${JSON.stringify(event.mcp)}` : undefined,
|
|
@@ -41,7 +43,8 @@ function formatToolEvents(ctx) {
|
|
|
41
43
|
}
|
|
42
44
|
export function buildBatchedJudgePrompt(judges, ctx, inputs) {
|
|
43
45
|
const sections = [];
|
|
44
|
-
|
|
46
|
+
const sensitiveValues = collectToolEventSensitiveValues(ctx.toolEvents);
|
|
47
|
+
sections.push(`## Session Transcript\n${sanitizePersistenceValue(ctx.transcript, sensitiveValues)}`);
|
|
45
48
|
if (judges.some((j) => j.includeToolEvents) && ctx.toolEvents.length > 0) {
|
|
46
49
|
sections.push(`## Tool Events\n${formatToolEvents(ctx)}`);
|
|
47
50
|
}
|
|
@@ -49,7 +52,8 @@ export function buildBatchedJudgePrompt(judges, ctx, inputs) {
|
|
|
49
52
|
const parts = [`### Rubric ${i + 1}: "${j.name}"\n${j.rubric}`];
|
|
50
53
|
const input = inputs[i];
|
|
51
54
|
if (input) {
|
|
52
|
-
|
|
55
|
+
const sanitizedInput = sanitizePersistenceValue(input, sensitiveValues);
|
|
56
|
+
for (const [key, value] of Object.entries(sanitizedInput)) {
|
|
53
57
|
const body = typeof value === 'string' ? value : JSON.stringify(value, null, 2);
|
|
54
58
|
parts.push(`#### ${key}\n${body}`);
|
|
55
59
|
}
|
|
@@ -77,7 +81,7 @@ export function buildToolUseJudgePrompt(scorer, ctx) {
|
|
|
77
81
|
].join('\n');
|
|
78
82
|
const parts = [];
|
|
79
83
|
parts.push('## Session Transcript');
|
|
80
|
-
parts.push(ctx.transcript);
|
|
84
|
+
parts.push(sanitizePersistenceValue(ctx.transcript, collectToolEventSensitiveValues(ctx.toolEvents)));
|
|
81
85
|
if (scorer.includeToolEvents && ctx.toolEvents.length > 0) {
|
|
82
86
|
parts.push('## Tool Events');
|
|
83
87
|
parts.push(formatToolEvents(ctx));
|
|
@@ -1,3 +1,4 @@
|
|
|
1
1
|
import type { ToolEvent } from '../tool-events.js';
|
|
2
2
|
export declare function attachOriginalMcpInput(event: ToolEvent, input: Record<string, unknown>): ToolEvent;
|
|
3
3
|
export declare function getOriginalMcpInput(event: ToolEvent): Record<string, unknown> | undefined;
|
|
4
|
+
export declare function clearOriginalMcpInput(event: ToolEvent): void;
|
package/dist/sdk/mcp-evidence.js
CHANGED
|
@@ -2,9 +2,12 @@ export function getMcpToolCall(event) {
|
|
|
2
2
|
if (event.action !== 'mcp_tool_call')
|
|
3
3
|
return undefined;
|
|
4
4
|
const args = event.arguments ?? {};
|
|
5
|
-
const serverName =
|
|
6
|
-
|
|
7
|
-
const
|
|
5
|
+
const serverName = event.mcp?.serverName
|
|
6
|
+
?? (typeof args.server === 'string' ? args.server : undefined);
|
|
7
|
+
const toolName = event.mcp?.toolName
|
|
8
|
+
?? (typeof args.tool === 'string' ? args.tool : undefined);
|
|
9
|
+
const argumentStatus = typeof args.status === 'string' ? args.status : undefined;
|
|
10
|
+
const status = mcpClassificationStatus(event.mcp, argumentStatus) ?? argumentStatus;
|
|
8
11
|
if (!serverName || !toolName || !status)
|
|
9
12
|
return undefined;
|
|
10
13
|
return {
|
|
@@ -15,6 +18,16 @@ export function getMcpToolCall(event) {
|
|
|
15
18
|
event,
|
|
16
19
|
};
|
|
17
20
|
}
|
|
21
|
+
function mcpClassificationStatus(mcp, argumentStatus) {
|
|
22
|
+
if (!mcp)
|
|
23
|
+
return undefined;
|
|
24
|
+
if (mcp.invocation === 'confirmed') {
|
|
25
|
+
if (mcp.outcome === 'completed')
|
|
26
|
+
return 'completed';
|
|
27
|
+
return argumentStatus === 'failed' || argumentStatus === 'error' ? argumentStatus : 'error';
|
|
28
|
+
}
|
|
29
|
+
return mcp.outcome;
|
|
30
|
+
}
|
|
18
31
|
export function isMcpToolCall(event, expected = {}) {
|
|
19
32
|
const call = getMcpToolCall(event);
|
|
20
33
|
return call !== undefined && matchesMcpToolCall(call, expected);
|
package/dist/sdk/mcp-safety.js
CHANGED
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import {
|
|
1
|
+
import { sanitizeUntrustedPersistenceValue } from '../tool-event-results.js';
|
|
2
2
|
export function decideMcpToolCall(options, request) {
|
|
3
3
|
const runMode = options?.runMode ?? 'mock';
|
|
4
4
|
if (runMode === 'mock')
|
|
@@ -31,7 +31,7 @@ export function decideMcpToolCall(options, request) {
|
|
|
31
31
|
return { action: 'allow' };
|
|
32
32
|
}
|
|
33
33
|
export function redactMcpSecrets(value) {
|
|
34
|
-
return
|
|
34
|
+
return sanitizeUntrustedPersistenceValue(value);
|
|
35
35
|
}
|
|
36
36
|
function ruleMatches(rule, request) {
|
|
37
37
|
if (rule.serverName !== undefined && rule.serverName !== request.serverName)
|
|
@@ -1,4 +1,5 @@
|
|
|
1
1
|
import { redactMcpSecrets } from './mcp-safety.js';
|
|
2
|
+
import { attachLiveMcpInput } from './tool-event-secrets.js';
|
|
2
3
|
export function buildScriptedMcpApprovalEvent(opts) {
|
|
3
4
|
const providerToolName = `${opts.serverName}.${opts.toolName}`;
|
|
4
5
|
return {
|
|
@@ -18,7 +19,7 @@ export function buildScriptedMcpApprovalEvent(opts) {
|
|
|
18
19
|
export function buildScriptedMcpDeniedCallEvent(opts) {
|
|
19
20
|
const providerToolName = `${opts.serverName}.${opts.toolName}`;
|
|
20
21
|
const args = redactMcpSecrets(opts.args);
|
|
21
|
-
return {
|
|
22
|
+
return attachLiveMcpInput({
|
|
22
23
|
action: 'mcp_tool_call', provider: opts.provider, providerToolName,
|
|
23
24
|
toolUseId: opts.toolUseId, turnNumber: opts.turnNumber, status: 'error',
|
|
24
25
|
mcp: { serverName: opts.serverName, toolName: opts.toolName, invocation: 'not_invoked', outcome: opts.outcome },
|
|
@@ -26,5 +27,5 @@ export function buildScriptedMcpDeniedCallEvent(opts) {
|
|
|
26
27
|
summary: `MCP tool ${providerToolName} ${opts.outcome}`,
|
|
27
28
|
confidence: 'high',
|
|
28
29
|
rawSnippet: JSON.stringify({ status: opts.outcome }),
|
|
29
|
-
};
|
|
30
|
+
}, opts.args);
|
|
30
31
|
}
|
|
@@ -1,7 +1,22 @@
|
|
|
1
|
-
import { sanitizePersistenceValue } from '../tool-event-results.js';
|
|
2
|
-
import { getToolEventSensitiveValues } from './tool-event-secrets.js';
|
|
1
|
+
import { collectStructuredSensitiveValues, isSensitiveValueScanLimitError, sanitizePersistenceValue, } from '../tool-event-results.js';
|
|
2
|
+
import { attachToolEventSensitiveValues, cloneToolEventWithRuntimeMetadata, getToolEventSensitiveValues, redactToolEventPayload, } from './tool-event-secrets.js';
|
|
3
3
|
export function buildToolEventLogEntry(toolEvent, fallbackTimestamp) {
|
|
4
|
-
|
|
4
|
+
let event = toolEvent;
|
|
5
|
+
let discoveredValues;
|
|
6
|
+
try {
|
|
7
|
+
discoveredValues = collectStructuredSensitiveValues(event);
|
|
8
|
+
}
|
|
9
|
+
catch (error) {
|
|
10
|
+
if (!isSensitiveValueScanLimitError(error))
|
|
11
|
+
throw error;
|
|
12
|
+
event = redactToolEventPayload(event);
|
|
13
|
+
discoveredValues = [];
|
|
14
|
+
}
|
|
15
|
+
const sensitiveValues = [...new Set([
|
|
16
|
+
...getToolEventSensitiveValues(toolEvent),
|
|
17
|
+
...discoveredValues,
|
|
18
|
+
])];
|
|
19
|
+
const persistedEvent = attachToolEventSensitiveValues(cloneToolEventWithRuntimeMetadata(event, sanitizePersistenceValue(event, sensitiveValues)), sensitiveValues);
|
|
5
20
|
return {
|
|
6
21
|
type: 'tool_event',
|
|
7
22
|
timestamp: persistedEvent.startedAt ?? fallbackTimestamp,
|
|
@@ -1,4 +1,8 @@
|
|
|
1
1
|
import type { ToolEvent } from '../tool-events.js';
|
|
2
2
|
export declare function attachToolEventSensitiveValues(event: ToolEvent, sensitiveValues: readonly string[]): ToolEvent;
|
|
3
3
|
export declare function getToolEventSensitiveValues(event: ToolEvent): readonly string[];
|
|
4
|
+
export declare function attachLiveMcpInput(event: ToolEvent, input: Record<string, unknown>): ToolEvent;
|
|
5
|
+
export declare function redactToolEventPayload(event: ToolEvent): ToolEvent;
|
|
6
|
+
export declare function collectToolEventSensitiveValues(events: readonly ToolEvent[]): string[];
|
|
7
|
+
export declare function clearToolEventRuntimeMetadata(event: ToolEvent): void;
|
|
4
8
|
export declare function cloneToolEventWithRuntimeMetadata(source: ToolEvent, overrides: Partial<ToolEvent>): ToolEvent;
|
|
@@ -1,12 +1,43 @@
|
|
|
1
|
-
import {
|
|
1
|
+
import { collectStructuredSensitiveValues, isSensitiveValueScanLimitError } from '../tool-event-results.js';
|
|
2
|
+
import { attachOriginalMcpInput, clearOriginalMcpInput, getOriginalMcpInput, } from './mcp-event-input.js';
|
|
2
3
|
const sensitiveValuesByEvent = new WeakMap();
|
|
3
4
|
export function attachToolEventSensitiveValues(event, sensitiveValues) {
|
|
4
|
-
sensitiveValuesByEvent.set(event, [...
|
|
5
|
+
sensitiveValuesByEvent.set(event, [...new Set([
|
|
6
|
+
...getToolEventSensitiveValues(event),
|
|
7
|
+
...sensitiveValues,
|
|
8
|
+
])]);
|
|
5
9
|
return event;
|
|
6
10
|
}
|
|
7
11
|
export function getToolEventSensitiveValues(event) {
|
|
8
12
|
return sensitiveValuesByEvent.get(event) ?? [];
|
|
9
13
|
}
|
|
14
|
+
export function attachLiveMcpInput(event, input) {
|
|
15
|
+
let sensitiveValues;
|
|
16
|
+
try {
|
|
17
|
+
sensitiveValues = collectStructuredSensitiveValues(input);
|
|
18
|
+
}
|
|
19
|
+
catch (error) {
|
|
20
|
+
if (!isSensitiveValueScanLimitError(error))
|
|
21
|
+
throw error;
|
|
22
|
+
return redactToolEventPayload(event);
|
|
23
|
+
}
|
|
24
|
+
return attachToolEventSensitiveValues(attachOriginalMcpInput(event, input), sensitiveValues);
|
|
25
|
+
}
|
|
26
|
+
export function redactToolEventPayload(event) {
|
|
27
|
+
return {
|
|
28
|
+
...event,
|
|
29
|
+
arguments: { redacted: true },
|
|
30
|
+
summary: `${event.action} via ${event.providerToolName}`,
|
|
31
|
+
rawSnippet: '<redacted>',
|
|
32
|
+
};
|
|
33
|
+
}
|
|
34
|
+
export function collectToolEventSensitiveValues(events) {
|
|
35
|
+
return [...new Set(events.flatMap((event) => getToolEventSensitiveValues(event)))];
|
|
36
|
+
}
|
|
37
|
+
export function clearToolEventRuntimeMetadata(event) {
|
|
38
|
+
sensitiveValuesByEvent.delete(event);
|
|
39
|
+
clearOriginalMcpInput(event);
|
|
40
|
+
}
|
|
10
41
|
export function cloneToolEventWithRuntimeMetadata(source, overrides) {
|
|
11
42
|
const clone = attachToolEventSensitiveValues({ ...source, ...overrides }, getToolEventSensitiveValues(source));
|
|
12
43
|
const originalInput = getOriginalMcpInput(source);
|
package/dist/sdk/types.d.ts
CHANGED
|
@@ -360,6 +360,8 @@ export interface EvaluateOptions {
|
|
|
360
360
|
failFast?: boolean;
|
|
361
361
|
llm?: LLMPort;
|
|
362
362
|
onScorerError?: 'skip' | 'zero' | 'fail';
|
|
363
|
+
/** Evidence visible to deterministic scorers. Live mode is unavailable for snapshot replay. */
|
|
364
|
+
deterministicToolEvidence?: 'persisted' | 'live';
|
|
363
365
|
/** Stable identity for this evaluation definition across separate runs. */
|
|
364
366
|
evaluationDefinitionKey?: string;
|
|
365
367
|
}
|
|
@@ -7,4 +7,7 @@ export declare function collectSensitiveEnvValues(env?: Readonly<Record<string,
|
|
|
7
7
|
* is important for summaries, snippets, traces, and provider error text.
|
|
8
8
|
*/
|
|
9
9
|
export declare function sanitizePersistenceValue<T>(source: T, explicitSensitiveValues?: readonly string[]): T;
|
|
10
|
+
export declare function sanitizeUntrustedPersistenceValue<T>(source: T, explicitSensitiveValues?: readonly string[]): T;
|
|
11
|
+
export declare function isSensitiveValueScanLimitError(error: unknown): boolean;
|
|
10
12
|
export declare function sanitizeToolEventResult(source: Readonly<ToolEventResult>, sensitiveValues: readonly string[]): ToolEventResult;
|
|
13
|
+
export declare function collectStructuredSensitiveValues(source: unknown): string[];
|
|
@@ -24,6 +24,10 @@ const SECRET_KEY_NAMES = [
|
|
|
24
24
|
'connectionstring',
|
|
25
25
|
'credentials',
|
|
26
26
|
'credential',
|
|
27
|
+
'grant',
|
|
28
|
+
'capability',
|
|
29
|
+
'capabilityurl',
|
|
30
|
+
'cursor',
|
|
27
31
|
'svsession',
|
|
28
32
|
'smsession',
|
|
29
33
|
'wixsession',
|
|
@@ -32,6 +36,10 @@ const SECRET_KEY_NAMES = [
|
|
|
32
36
|
const EXACT_ONLY_SECRET_KEY_NAMES = new Set([
|
|
33
37
|
'auth',
|
|
34
38
|
'token',
|
|
39
|
+
'grant',
|
|
40
|
+
'capability',
|
|
41
|
+
'capabilityurl',
|
|
42
|
+
'cursor',
|
|
35
43
|
'svsession',
|
|
36
44
|
'smsession',
|
|
37
45
|
'wixsession',
|
|
@@ -39,6 +47,9 @@ const EXACT_ONLY_SECRET_KEY_NAMES = new Set([
|
|
|
39
47
|
]);
|
|
40
48
|
const NON_SECRET_ENVIRONMENT_KEY_NAMES = new Set(['tokencount', 'tokenizersparallelism', 'tokenusage']);
|
|
41
49
|
const BOUNDARY_ONLY_SENSITIVE_VALUE_MAX_LENGTH = 1;
|
|
50
|
+
const MAX_SANITIZE_DEPTH = 64;
|
|
51
|
+
const MAX_SENSITIVE_SCAN_NODES = 100_000;
|
|
52
|
+
const MAX_SENSITIVE_JSON_CHARS = 1024 * 1024;
|
|
42
53
|
export function collectSensitiveEnvValues(env) {
|
|
43
54
|
return [...new Set(Object.entries(env ?? {})
|
|
44
55
|
.filter((entry) => typeof entry[1] === 'string')
|
|
@@ -60,15 +71,36 @@ export function sanitizePersistenceValue(source, explicitSensitiveValues = []) {
|
|
|
60
71
|
])].sort((a, b) => b.length - a.length);
|
|
61
72
|
return sanitizeValue(source, '', sensitiveValues, new Set(explicitValues));
|
|
62
73
|
}
|
|
74
|
+
export function sanitizeUntrustedPersistenceValue(source, explicitSensitiveValues = []) {
|
|
75
|
+
try {
|
|
76
|
+
return sanitizePersistenceValue(source, explicitSensitiveValues);
|
|
77
|
+
}
|
|
78
|
+
catch (error) {
|
|
79
|
+
if (!isSensitiveValueScanLimitError(error))
|
|
80
|
+
throw error;
|
|
81
|
+
if (Array.isArray(source))
|
|
82
|
+
return [];
|
|
83
|
+
if (isRecord(source))
|
|
84
|
+
return { redacted: true };
|
|
85
|
+
return '<redacted>';
|
|
86
|
+
}
|
|
87
|
+
}
|
|
88
|
+
export function isSensitiveValueScanLimitError(error) {
|
|
89
|
+
return error instanceof Error && error.message.startsWith('Sensitive-value scan exceeded');
|
|
90
|
+
}
|
|
63
91
|
export function sanitizeToolEventResult(source, sensitiveValues) {
|
|
64
92
|
const result = {};
|
|
65
93
|
let truncated = source.truncated === true;
|
|
66
94
|
const addBounded = (key, value) => {
|
|
67
95
|
if (typeof value !== 'string')
|
|
68
96
|
return;
|
|
69
|
-
const
|
|
70
|
-
|
|
71
|
-
|
|
97
|
+
const explicitValues = [...new Set(sensitiveValues.filter((entry) => entry.length > 0))];
|
|
98
|
+
const overlap = explicitValues.reduce((max, entry) => Math.max(max, entry.length), 0);
|
|
99
|
+
const prefix = value.slice(0, TOOL_RESULT_MAX_CHARS + Math.max(overlap, 1));
|
|
100
|
+
const bounded = boundText(redactSensitiveValues(prefix, explicitValues, new Set(explicitValues)));
|
|
101
|
+
const sanitized = boundText(sanitizePersistenceValue(bounded.value, sensitiveValues));
|
|
102
|
+
result[key] = sanitized.value;
|
|
103
|
+
truncated ||= value.length > prefix.length || bounded.truncated || sanitized.truncated;
|
|
72
104
|
};
|
|
73
105
|
addBounded('content', source.content);
|
|
74
106
|
addBounded('stdout', source.stdout);
|
|
@@ -96,56 +128,93 @@ function redactSensitiveValues(value, sensitiveValues, explicitSensitiveValues)
|
|
|
96
128
|
}
|
|
97
129
|
return redacted;
|
|
98
130
|
}
|
|
99
|
-
function collectStructuredSensitiveValues(source) {
|
|
131
|
+
export function collectStructuredSensitiveValues(source) {
|
|
100
132
|
const values = new Set();
|
|
101
|
-
|
|
102
|
-
|
|
103
|
-
|
|
104
|
-
|
|
105
|
-
|
|
133
|
+
let scannedNodes = 0;
|
|
134
|
+
let parsedJsonChars = 0;
|
|
135
|
+
const pending = [
|
|
136
|
+
{ value: source, key: '' },
|
|
137
|
+
];
|
|
138
|
+
while (pending.length > 0) {
|
|
139
|
+
if (++scannedNodes > MAX_SENSITIVE_SCAN_NODES) {
|
|
140
|
+
throw new Error('Sensitive-value scan exceeded the input node limit');
|
|
141
|
+
}
|
|
142
|
+
const { value, key } = pending.pop();
|
|
143
|
+
if (typeof value === 'string') {
|
|
144
|
+
if (value.length > 0 && !isNumericTokenTelemetry(key, value) && isSecretKey(key)) {
|
|
145
|
+
values.add(value);
|
|
146
|
+
}
|
|
147
|
+
for (const match of value.matchAll(/[?&](?:api[_-]?key|access[_-]?token|refresh[_-]?token|client[_-]?secret|token|secret|auth|password|passwd|session(?:id)?|cookie)=([^&#\s]+)/gi)) {
|
|
148
|
+
if (match[1])
|
|
149
|
+
values.add(match[1]);
|
|
150
|
+
}
|
|
151
|
+
for (const sensitiveValue of collectCredentialShapeSensitiveValues(value)) {
|
|
152
|
+
values.add(sensitiveValue);
|
|
153
|
+
}
|
|
154
|
+
const trimmed = value.trim();
|
|
155
|
+
const looksStructured = (trimmed.startsWith('{') && trimmed.endsWith('}'))
|
|
156
|
+
|| (trimmed.startsWith('[') && trimmed.endsWith(']'));
|
|
157
|
+
if (looksStructured && (parsedJsonChars += value.length) > MAX_SENSITIVE_JSON_CHARS) {
|
|
158
|
+
throw new Error('Sensitive-value scan exceeded the nested JSON limit');
|
|
159
|
+
}
|
|
160
|
+
const structured = looksStructured ? parseStructuredJson(value) : undefined;
|
|
161
|
+
if (structured !== undefined)
|
|
162
|
+
pending.push({ value: structured, key });
|
|
163
|
+
continue;
|
|
164
|
+
}
|
|
165
|
+
if ((typeof value === 'number' || typeof value === 'boolean' || typeof value === 'bigint')
|
|
166
|
+
&& !isNumericTokenTelemetry(key, value) && isSecretKey(key)) {
|
|
167
|
+
values.add(String(value));
|
|
168
|
+
continue;
|
|
106
169
|
}
|
|
107
170
|
if (Array.isArray(value)) {
|
|
108
171
|
for (const entry of value)
|
|
109
|
-
|
|
110
|
-
|
|
172
|
+
pending.push({ value: entry, key });
|
|
173
|
+
continue;
|
|
111
174
|
}
|
|
112
175
|
if (isRecord(value)) {
|
|
113
|
-
for (const [entryKey, entryValue] of Object.entries(value))
|
|
114
|
-
|
|
176
|
+
for (const [entryKey, entryValue] of Object.entries(value)) {
|
|
177
|
+
pending.push({ value: entryValue, key: entryKey });
|
|
178
|
+
}
|
|
115
179
|
}
|
|
116
|
-
}
|
|
117
|
-
visit(source, '');
|
|
180
|
+
}
|
|
118
181
|
return [...values];
|
|
119
182
|
}
|
|
120
|
-
function sanitizeValue(value, key, sensitiveValues, explicitSensitiveValues) {
|
|
183
|
+
function sanitizeValue(value, key, sensitiveValues, explicitSensitiveValues, depth = 0) {
|
|
121
184
|
if (!isNumericTokenTelemetry(key, value) && isSecretKey(key))
|
|
122
185
|
return '<redacted>';
|
|
123
186
|
if (typeof value === 'string') {
|
|
124
|
-
return redactCredentialShapes(redactSensitiveValues(value, sensitiveValues, explicitSensitiveValues), sensitiveValues, explicitSensitiveValues);
|
|
187
|
+
return redactCredentialShapes(redactSensitiveValues(value, sensitiveValues, explicitSensitiveValues), sensitiveValues, explicitSensitiveValues, depth);
|
|
125
188
|
}
|
|
189
|
+
if (depth >= MAX_SANITIZE_DEPTH && (Array.isArray(value) || isRecord(value)))
|
|
190
|
+
return '<redacted>';
|
|
126
191
|
if (Array.isArray(value)) {
|
|
127
|
-
return value.map((entry) => sanitizeValue(entry, key, sensitiveValues, explicitSensitiveValues));
|
|
192
|
+
return value.map((entry) => sanitizeValue(entry, key, sensitiveValues, explicitSensitiveValues, depth + 1));
|
|
128
193
|
}
|
|
129
194
|
if (isRecord(value)) {
|
|
130
195
|
return Object.fromEntries(Object.entries(value).map(([entryKey, entryValue]) => [
|
|
131
196
|
entryKey,
|
|
132
|
-
sanitizeValue(entryValue, entryKey, sensitiveValues, explicitSensitiveValues),
|
|
197
|
+
sanitizeValue(entryValue, entryKey, sensitiveValues, explicitSensitiveValues, depth + 1),
|
|
133
198
|
]));
|
|
134
199
|
}
|
|
135
200
|
return value;
|
|
136
201
|
}
|
|
137
|
-
function redactCredentialShapes(value, sensitiveValues, explicitSensitiveValues) {
|
|
202
|
+
function redactCredentialShapes(value, sensitiveValues, explicitSensitiveValues, depth) {
|
|
138
203
|
let redacted = value;
|
|
204
|
+
const credentialShapeValues = collectCredentialShapeSensitiveValues(redacted);
|
|
205
|
+
redacted = redactSensitiveValues(redacted, credentialShapeValues, new Set(credentialShapeValues));
|
|
139
206
|
const structured = parseStructuredJson(redacted);
|
|
140
207
|
if (structured !== undefined) {
|
|
141
208
|
const nestedSensitiveValues = [...new Set([
|
|
142
209
|
...sensitiveValues,
|
|
143
210
|
...collectStructuredSensitiveValues(structured),
|
|
144
211
|
])].sort((a, b) => b.length - a.length);
|
|
145
|
-
redacted = JSON.stringify(sanitizeValue(structured, '', nestedSensitiveValues, explicitSensitiveValues));
|
|
212
|
+
redacted = JSON.stringify(sanitizeValue(structured, '', nestedSensitiveValues, explicitSensitiveValues, depth + 1));
|
|
146
213
|
}
|
|
147
214
|
if (redacted.includes('PRIVATE KEY-----')) {
|
|
148
|
-
|
|
215
|
+
for (const block of findPrivateKeyBlocks(redacted).reverse()) {
|
|
216
|
+
redacted = `${redacted.slice(0, block.start)}<redacted>${redacted.slice(block.end)}`;
|
|
217
|
+
}
|
|
149
218
|
}
|
|
150
219
|
if (/\b(?:bearer|basic)\s/i.test(redacted)) {
|
|
151
220
|
redacted = redacted.replace(/(\b(?:bearer|basic)\s+)[^\s,;"']+/gi, '$1<redacted>');
|
|
@@ -154,10 +223,71 @@ function redactCredentialShapes(value, sensitiveValues, explicitSensitiveValues)
|
|
|
154
223
|
redacted = redacted.replace(/([a-z][a-z0-9+.-]*:\/\/[^\s:/@]+:)[^\s/@]+(@)/gi, '$1<redacted>$2');
|
|
155
224
|
}
|
|
156
225
|
if (redacted.includes('=') || redacted.includes(':')) {
|
|
157
|
-
redacted = redacted.replace(/(\
|
|
226
|
+
redacted = redacted.replace(/((?:^|[\s{[(,;])["']?(?:api[_-]?key|access[_-]?token|refresh[_-]?token|client[_-]?secret|token|secret|auth|password|passwd|session(?:id)?|cookie)["']?\s*[=:]\s*["']?)[^\s,;"'}]+/gi, '$1<redacted>');
|
|
227
|
+
redacted = redacted.replace(/((?:^|[\s{[(,;])["'](?:grant|capability|capabilityurl|cursor)["']\s*:\s*["']?)[^\s,;"'}]+/gi, '$1<redacted>');
|
|
158
228
|
}
|
|
159
229
|
return redacted;
|
|
160
230
|
}
|
|
231
|
+
function collectCredentialShapeSensitiveValues(value) {
|
|
232
|
+
const values = new Set();
|
|
233
|
+
for (const match of value.matchAll(/https?:\/\/[^\s,;"']+\/(?:cap|capability)\/[^\s,;"']+/gi)) {
|
|
234
|
+
values.add(match[0]);
|
|
235
|
+
}
|
|
236
|
+
if (value.includes('PRIVATE KEY-----')) {
|
|
237
|
+
for (const block of findPrivateKeyBlocks(value)) {
|
|
238
|
+
values.add(value.slice(block.start, block.end));
|
|
239
|
+
if (block.body)
|
|
240
|
+
values.add(block.body);
|
|
241
|
+
}
|
|
242
|
+
}
|
|
243
|
+
if (/\b(?:bearer|basic)\s/i.test(value)) {
|
|
244
|
+
for (const match of value.matchAll(/\b(?:bearer|basic)\s+([^\s,;"']+)/gi)) {
|
|
245
|
+
if (match[1])
|
|
246
|
+
values.add(match[1]);
|
|
247
|
+
}
|
|
248
|
+
}
|
|
249
|
+
if (value.includes('://')) {
|
|
250
|
+
for (const match of value.matchAll(/[a-z][a-z0-9+.-]*:\/\/[^\s:/@]+:([^\s/@]+)@/gi)) {
|
|
251
|
+
if (match[1])
|
|
252
|
+
values.add(match[1]);
|
|
253
|
+
}
|
|
254
|
+
}
|
|
255
|
+
if (value.includes('=') || value.includes(':')) {
|
|
256
|
+
for (const match of value.matchAll(/(?:^|[\s{[(,;])["']?(?:api[_-]?key|access[_-]?token|refresh[_-]?token|client[_-]?secret|token|secret|auth|password|passwd|session(?:id)?|cookie)["']?\s*[=:]\s*["']?([^\s,;"'}]+)/gi)) {
|
|
257
|
+
if (match[1])
|
|
258
|
+
values.add(match[1]);
|
|
259
|
+
}
|
|
260
|
+
for (const match of value.matchAll(/(?:^|[\s{[(,;])["'](?:grant|capability|capabilityurl|cursor)["']\s*:\s*["']?([^\s,;"'}]+)/gi)) {
|
|
261
|
+
if (match[1])
|
|
262
|
+
values.add(match[1]);
|
|
263
|
+
}
|
|
264
|
+
}
|
|
265
|
+
return [...values].sort((a, b) => b.length - a.length);
|
|
266
|
+
}
|
|
267
|
+
function findPrivateKeyBlocks(value) {
|
|
268
|
+
const blocks = [];
|
|
269
|
+
let open;
|
|
270
|
+
let cursor = 0;
|
|
271
|
+
while (cursor < value.length) {
|
|
272
|
+
const start = value.indexOf('-----', cursor);
|
|
273
|
+
if (start < 0)
|
|
274
|
+
break;
|
|
275
|
+
const end = value.indexOf('-----', start + 5);
|
|
276
|
+
if (end < 0)
|
|
277
|
+
break;
|
|
278
|
+
const markerEnd = end + 5;
|
|
279
|
+
const marker = value.slice(start, markerEnd);
|
|
280
|
+
if (!open && marker.startsWith('-----BEGIN ') && marker.endsWith('PRIVATE KEY-----')) {
|
|
281
|
+
open = { start, bodyStart: markerEnd };
|
|
282
|
+
}
|
|
283
|
+
else if (open && marker.startsWith('-----END ') && marker.endsWith('PRIVATE KEY-----')) {
|
|
284
|
+
blocks.push({ start: open.start, end: markerEnd, body: value.slice(open.bodyStart, start).trim() });
|
|
285
|
+
open = undefined;
|
|
286
|
+
}
|
|
287
|
+
cursor = markerEnd;
|
|
288
|
+
}
|
|
289
|
+
return blocks;
|
|
290
|
+
}
|
|
161
291
|
function isSecretKey(key) {
|
|
162
292
|
const normalized = key.replace(/[^a-z0-9]/gi, '').toLowerCase();
|
|
163
293
|
if (normalized === 'tokens')
|