@namzu/sdk 26.1.0 → 27.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +55 -0
- package/dist/compaction/manager.d.ts +22 -0
- package/dist/compaction/manager.d.ts.map +1 -1
- package/dist/compaction/manager.js +31 -3
- package/dist/compaction/manager.js.map +1 -1
- package/dist/connector/mcp/adapter.d.ts +7 -1
- package/dist/connector/mcp/adapter.d.ts.map +1 -1
- package/dist/connector/mcp/adapter.js +12 -1
- package/dist/connector/mcp/adapter.js.map +1 -1
- package/dist/connector/mcp/prompt-adapter.d.ts +10 -4
- package/dist/connector/mcp/prompt-adapter.d.ts.map +1 -1
- package/dist/connector/mcp/prompt-adapter.js +10 -4
- package/dist/connector/mcp/prompt-adapter.js.map +1 -1
- package/dist/eval/experiment.d.ts +0 -6
- package/dist/eval/experiment.d.ts.map +1 -1
- package/dist/eval/experiment.js +28 -0
- package/dist/eval/experiment.js.map +1 -1
- package/dist/eval/index.d.ts +2 -0
- package/dist/eval/index.d.ts.map +1 -1
- package/dist/eval/index.js +1 -0
- package/dist/eval/index.js.map +1 -1
- package/dist/eval/types.d.ts +17 -0
- package/dist/eval/types.d.ts.map +1 -1
- package/dist/eval/uncertainty.d.ts +64 -0
- package/dist/eval/uncertainty.d.ts.map +1 -0
- package/dist/eval/uncertainty.js +92 -0
- package/dist/eval/uncertainty.js.map +1 -0
- package/dist/public-runtime.d.ts +1 -0
- package/dist/public-runtime.d.ts.map +1 -1
- package/dist/public-runtime.js +1 -0
- package/dist/public-runtime.js.map +1 -1
- package/dist/registry/tool/execute.d.ts.map +1 -1
- package/dist/registry/tool/execute.js +2 -1
- package/dist/registry/tool/execute.js.map +1 -1
- package/dist/tools/trusted-read-only.d.ts +42 -0
- package/dist/tools/trusted-read-only.d.ts.map +1 -0
- package/dist/tools/trusted-read-only.js +51 -0
- package/dist/tools/trusted-read-only.js.map +1 -0
- package/dist/tools/untrusted-envelope.d.ts +22 -2
- package/dist/tools/untrusted-envelope.d.ts.map +1 -1
- package/dist/tools/untrusted-envelope.js +22 -2
- package/dist/tools/untrusted-envelope.js.map +1 -1
- package/dist/types/tool/index.d.ts +28 -0
- package/dist/types/tool/index.d.ts.map +1 -1
- package/dist/types/tool/index.js.map +1 -1
- package/dist/verification/rules.d.ts.map +1 -1
- package/dist/verification/rules.js +5 -1
- package/dist/verification/rules.js.map +1 -1
- package/package.json +1 -1
- package/src/compaction/manager.ts +39 -3
- package/src/connector/mcp/adapter.ts +11 -0
- package/src/connector/mcp/prompt-adapter.ts +10 -4
- package/src/eval/experiment.ts +30 -0
- package/src/eval/index.ts +2 -0
- package/src/eval/types.ts +17 -0
- package/src/eval/uncertainty.ts +124 -0
- package/src/public-runtime.ts +1 -0
- package/src/registry/tool/execute.ts +2 -1
- package/src/tools/trusted-read-only.ts +52 -0
- package/src/tools/untrusted-envelope.ts +22 -2
- package/src/types/tool/index.ts +30 -0
- package/src/verification/rules.ts +5 -1
|
@@ -303,6 +303,34 @@ export interface ToolDefinition<TInput = unknown> {
|
|
|
303
303
|
isReadOnly?(input: TInput): boolean;
|
|
304
304
|
isDestructive?(input: TInput): boolean;
|
|
305
305
|
isConcurrencySafe?(input: TInput): boolean;
|
|
306
|
+
/**
|
|
307
|
+
* Where this tool came from, when it did not come from here.
|
|
308
|
+
*
|
|
309
|
+
* Absent means host-defined: this process, code the operator installed,
|
|
310
|
+
* no untrusted party in the chain. Present means a connected server
|
|
311
|
+
* supplied both the tool and its own description of what the tool does
|
|
312
|
+
* — including whether it is read-only, which three separate gates were
|
|
313
|
+
* treating as a fact rather than as the hint the wire calls it.
|
|
314
|
+
*
|
|
315
|
+
* See {@link isTrustedReadOnly}. This field exists so a gate can tell
|
|
316
|
+
* the two apart; `isReadOnly` keeps reporting faithfully what the
|
|
317
|
+
* server said, because the outbound re-export and the destructive
|
|
318
|
+
* label shown to a human both need the server's own answer.
|
|
319
|
+
*/
|
|
320
|
+
provenance?: ToolProvenance;
|
|
321
|
+
}
|
|
322
|
+
export interface ToolProvenance {
|
|
323
|
+
/** The connected server this tool came from, named as configured. */
|
|
324
|
+
readonly server: string;
|
|
325
|
+
/**
|
|
326
|
+
* The operator marked this server's read-only claims as trustworthy.
|
|
327
|
+
*
|
|
328
|
+
* Per server, never global: one switch meaning "trust annotations"
|
|
329
|
+
* hands every connected server the same reach, which is the hole it
|
|
330
|
+
* would be closing. Default false — an unmarked server's claim raises
|
|
331
|
+
* the requirement and never lowers it.
|
|
332
|
+
*/
|
|
333
|
+
readonly readOnlyHintTrusted: boolean;
|
|
306
334
|
}
|
|
307
335
|
export type ToolPermission = 'file_read' | 'file_write' | 'shell_execute' | 'network_access' | 'env_access';
|
|
308
336
|
export interface LLMToolSchema {
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../../src/types/tool/index.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,CAAC,EAAE,MAAM,KAAK,CAAA;AAC5B,OAAO,KAAK,EAAE,MAAM,EAAE,MAAM,uBAAuB,CAAA;AACnD,OAAO,KAAK,EAAE,KAAK,EAAE,MAAM,iBAAiB,CAAA;AAC5C,OAAO,KAAK,EAAE,eAAe,EAAE,MAAM,wBAAwB,CAAA;AAC7D,OAAO,KAAK,EAAE,cAAc,EAAE,MAAM,wBAAwB,CAAA;AAC5D,OAAO,KAAK,EAAE,OAAO,EAAE,MAAM,qBAAqB,CAAA;AAElD,MAAM,WAAW,eAAe;IAC/B,cAAc,CAAC,KAAK,EAAE,MAAM,GAAG,cAAc,EAAE,CAAA;IAC/C,QAAQ,CAAC,KAAK,EAAE,MAAM,EAAE,GAAG,IAAI,CAAA;IAC/B,eAAe,CAAC,IAAI,EAAE,MAAM,GAAG,gBAAgB,CAAA;CAC/C;AAED;;;;;;GAMG;AACH,MAAM,WAAW,eAAe;IAC/B;;;;OAIG;IACH,UAAU,CAAC,GAAG,EAAE,MAAM,EAAE,OAAO,CAAC,EAAE,MAAM,GAAG,IAAI,CAAA;IAC/C,OAAO,CAAC,GAAG,EAAE,MAAM,GAAG,OAAO,CAAA;IAC7B;;;;;;;;;OASG;IACH,WAAW,CAAC,CAAC,GAAG,EAAE,MAAM,GAAG,MAAM,GAAG,SAAS,CAAA;CAC7C;AAED,MAAM,WAAW,eAAe;IAC/B,QAAQ,CAAC,EAAE,EAAE,MAAM,CAAA;IACnB,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAA;IACtB,QAAQ,CAAC,WAAW,CAAC,EAAE,MAAM,CAAA;CAC7B;AAED,MAAM,WAAW,gBAAgB;IAChC;;;;;;;OAOG;IACH,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAA;IACrB,sDAAsD;IACtD,QAAQ,CAAC,MAAM,EAAE,MAAM,CAAA;IACvB,yEAAyE;IACzE,QAAQ,CAAC,MAAM,CAAC,EAAE,MAAM,CAAA;IACxB,QAAQ,CAAC,OAAO,CAAC,EAAE,SAAS,eAAe,EAAE,CAAA;IAC7C,QAAQ,CAAC,WAAW,CAAC,EAAE,OAAO,CAAA;IAC9B,4EAA4E;IAC5E,QAAQ,CAAC,aAAa,CAAC,EAAE,OAAO,CAAA;CAChC;AAED;;;;;;;;GAQG;AACH,MAAM,MAAM,gBAAgB,GACzB;IACA,QAAQ,CAAC,MAAM,EAAE,UAAU,CAAA;IAC3B,QAAQ,CAAC,iBAAiB,EAAE,SAAS,MAAM,EAAE,CAAA;IAC7C,QAAQ,CAAC,IAAI,CAAC,EAAE,MAAM,CAAA;CACrB,GACD;IAAE,QAAQ,CAAC,MAAM,EAAE,YAAY,CAAC;IAAC,QAAQ,CAAC,MAAM,EAAE,MAAM,CAAA;CAAE,GAC1D;IAAE,QAAQ,CAAC,MAAM,EAAE,SAAS,CAAA;CAAE,CAAA;AAEjC,MAAM,MAAM,gBAAgB,GAAG,CAAC,OAAO,EAAE,gBAAgB,KAAK,OAAO,CAAC,gBAAgB,CAAC,CAAA;AAEvF,MAAM,WAAW,WAAW;IAC3B,KAAK,EAAE,KAAK,CAAA;IACZ,gBAAgB,EAAE,MAAM,CAAA;IACxB,WAAW,EAAE,WAAW,CAAA;IACxB,GAAG,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAA;IAC3B,GAAG,EAAE,CAAC,KAAK,EAAE,MAAM,GAAG,MAAM,GAAG,OAAO,EAAE,OAAO,EAAE,MAAM,KAAK,IAAI,CAAA;IAChE,iBAAiB,CAAC,EAAE;QACnB,IAAI,EAAE,cAAc,CAAA;QACpB,KAAK,EAAE,MAAM,CAAA;QACb,gBAAgB,EAAE,MAAM,CAAA;KACxB,CAAA;IAED,eAAe,CAAC,EAAE,eAAe,CAAA;IAEjC,YAAY,CAAC,EAAE,eAAe,CAAA;IAC9B;;;;;;;;;;;;OAYG;IACH,YAAY,CAAC,EAAE,SAAS,MAAM,EAAE,CAAA;IAChC,OAAO,CAAC,EAAE,OAAO,CAAA;IACjB,eAAe,CAAC,EAAE,eAAe,CAAA;IAEjC;;;;;;;OAOG;IACH,SAAS,CAAC,EAAE,MAAM,CAAA;IAElB;;;;;;;;;;;;;;;;;;;;;;OAsBG;IACH,YAAY,CAAC,EAAE,gBAAgB,CAAA;IAE/B;;;;;;;;;;;OAWG;IACH,UAAU,CAAC,EAAE,OAAO,oBAAoB,EAAE,IAAI,CAAA;IAE9C;;;;;;;;;;;;;;;OAeG;IACH,MAAM,CAAC,EAAE,CAAC,OAAO,EAAE,MAAM,EAAE,QAAQ,CAAC,EAAE,MAAM,KAAK,IAAI,CAAA;CACrD;AAED,MAAM,WAAW,UAAU;IAC1B,OAAO,EAAE,OAAO,CAAA;IAChB,MAAM,EAAE,MAAM,CAAA;IACd,IAAI,CAAC,EAAE,OAAO,CAAA;IACd,KAAK,CAAC,EAAE,MAAM,CAAA;IACd;;;;;;;;;OASG;IACH,OAAO,CAAC,EAAE,OAAO,qBAAqB,EAAE,iBAAiB,CAAA;IAEzD;;;;;;;;;;;OAWG;IACH,SAAS,CAAC,EAAE,OAAO,CAAA;CACnB;AAED,MAAM,WAAW,cAAc,CAAC,MAAM,GAAG,OAAO;IAC/C,IAAI,EAAE,MAAM,CAAA;IACZ,WAAW,EAAE,MAAM,CAAA;IACnB,WAAW,EAAE,CAAC,CAAC,OAAO,CAAC,MAAM,EAAE,CAAC,CAAC,UAAU,EAAE,OAAO,CAAC,CAAA;IACrD;;;;;;;OAOG;IACH,gBAAgB,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAA;IAC1C;;;OAGG;IACH,iBAAiB,CAAC,EAAE,OAAO,CAAA;IAC3B;;;;OAIG;IACH,mBAAmB,CAAC,EAAE,MAAM,CAAA;IAE5B;;;;;;;;;;;;;OAaG;IACH,YAAY,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAA;IACtC,OAAO,CAAC,KAAK,EAAE,MAAM,EAAE,OAAO,EAAE,WAAW,GAAG,OAAO,CAAC,UAAU,CAAC,CAAA;IACjE,IAAI,CAAC,EAAE,MAAM,CAAA;IACb,WAAW,CAAC,EAAE,cAAc,EAAE,CAAA;IAC9B,QAAQ,CAAC,EAAE,YAAY,GAAG,OAAO,GAAG,SAAS,GAAG,UAAU,GAAG,QAAQ,CAAA;IAErE;;;;;;;;;;;OAWG;IACH,SAAS,CAAC,EAAE,MAAM,CAAA;IAElB;;;;;;;;;;;;;OAaG;IACH,UAAU,CAAC,EAAE,MAAM,CAAA;IAEnB;;;;;;;;;;;;;;;;;;;;;;;;OAwBG;IACH,QAAQ,CAAC,EAAE,OAAO,CAAA;IAElB,UAAU,CAAC,CAAC,KAAK,EAAE,MAAM,GAAG,OAAO,CAAA;IACnC,aAAa,CAAC,CAAC,KAAK,EAAE,MAAM,GAAG,OAAO,CAAA;IACtC,iBAAiB,CAAC,CAAC,KAAK,EAAE,MAAM,GAAG,OAAO,CAAA;
|
|
1
|
+
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../../src/types/tool/index.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,CAAC,EAAE,MAAM,KAAK,CAAA;AAC5B,OAAO,KAAK,EAAE,MAAM,EAAE,MAAM,uBAAuB,CAAA;AACnD,OAAO,KAAK,EAAE,KAAK,EAAE,MAAM,iBAAiB,CAAA;AAC5C,OAAO,KAAK,EAAE,eAAe,EAAE,MAAM,wBAAwB,CAAA;AAC7D,OAAO,KAAK,EAAE,cAAc,EAAE,MAAM,wBAAwB,CAAA;AAC5D,OAAO,KAAK,EAAE,OAAO,EAAE,MAAM,qBAAqB,CAAA;AAElD,MAAM,WAAW,eAAe;IAC/B,cAAc,CAAC,KAAK,EAAE,MAAM,GAAG,cAAc,EAAE,CAAA;IAC/C,QAAQ,CAAC,KAAK,EAAE,MAAM,EAAE,GAAG,IAAI,CAAA;IAC/B,eAAe,CAAC,IAAI,EAAE,MAAM,GAAG,gBAAgB,CAAA;CAC/C;AAED;;;;;;GAMG;AACH,MAAM,WAAW,eAAe;IAC/B;;;;OAIG;IACH,UAAU,CAAC,GAAG,EAAE,MAAM,EAAE,OAAO,CAAC,EAAE,MAAM,GAAG,IAAI,CAAA;IAC/C,OAAO,CAAC,GAAG,EAAE,MAAM,GAAG,OAAO,CAAA;IAC7B;;;;;;;;;OASG;IACH,WAAW,CAAC,CAAC,GAAG,EAAE,MAAM,GAAG,MAAM,GAAG,SAAS,CAAA;CAC7C;AAED,MAAM,WAAW,eAAe;IAC/B,QAAQ,CAAC,EAAE,EAAE,MAAM,CAAA;IACnB,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAA;IACtB,QAAQ,CAAC,WAAW,CAAC,EAAE,MAAM,CAAA;CAC7B;AAED,MAAM,WAAW,gBAAgB;IAChC;;;;;;;OAOG;IACH,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAA;IACrB,sDAAsD;IACtD,QAAQ,CAAC,MAAM,EAAE,MAAM,CAAA;IACvB,yEAAyE;IACzE,QAAQ,CAAC,MAAM,CAAC,EAAE,MAAM,CAAA;IACxB,QAAQ,CAAC,OAAO,CAAC,EAAE,SAAS,eAAe,EAAE,CAAA;IAC7C,QAAQ,CAAC,WAAW,CAAC,EAAE,OAAO,CAAA;IAC9B,4EAA4E;IAC5E,QAAQ,CAAC,aAAa,CAAC,EAAE,OAAO,CAAA;CAChC;AAED;;;;;;;;GAQG;AACH,MAAM,MAAM,gBAAgB,GACzB;IACA,QAAQ,CAAC,MAAM,EAAE,UAAU,CAAA;IAC3B,QAAQ,CAAC,iBAAiB,EAAE,SAAS,MAAM,EAAE,CAAA;IAC7C,QAAQ,CAAC,IAAI,CAAC,EAAE,MAAM,CAAA;CACrB,GACD;IAAE,QAAQ,CAAC,MAAM,EAAE,YAAY,CAAC;IAAC,QAAQ,CAAC,MAAM,EAAE,MAAM,CAAA;CAAE,GAC1D;IAAE,QAAQ,CAAC,MAAM,EAAE,SAAS,CAAA;CAAE,CAAA;AAEjC,MAAM,MAAM,gBAAgB,GAAG,CAAC,OAAO,EAAE,gBAAgB,KAAK,OAAO,CAAC,gBAAgB,CAAC,CAAA;AAEvF,MAAM,WAAW,WAAW;IAC3B,KAAK,EAAE,KAAK,CAAA;IACZ,gBAAgB,EAAE,MAAM,CAAA;IACxB,WAAW,EAAE,WAAW,CAAA;IACxB,GAAG,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAA;IAC3B,GAAG,EAAE,CAAC,KAAK,EAAE,MAAM,GAAG,MAAM,GAAG,OAAO,EAAE,OAAO,EAAE,MAAM,KAAK,IAAI,CAAA;IAChE,iBAAiB,CAAC,EAAE;QACnB,IAAI,EAAE,cAAc,CAAA;QACpB,KAAK,EAAE,MAAM,CAAA;QACb,gBAAgB,EAAE,MAAM,CAAA;KACxB,CAAA;IAED,eAAe,CAAC,EAAE,eAAe,CAAA;IAEjC,YAAY,CAAC,EAAE,eAAe,CAAA;IAC9B;;;;;;;;;;;;OAYG;IACH,YAAY,CAAC,EAAE,SAAS,MAAM,EAAE,CAAA;IAChC,OAAO,CAAC,EAAE,OAAO,CAAA;IACjB,eAAe,CAAC,EAAE,eAAe,CAAA;IAEjC;;;;;;;OAOG;IACH,SAAS,CAAC,EAAE,MAAM,CAAA;IAElB;;;;;;;;;;;;;;;;;;;;;;OAsBG;IACH,YAAY,CAAC,EAAE,gBAAgB,CAAA;IAE/B;;;;;;;;;;;OAWG;IACH,UAAU,CAAC,EAAE,OAAO,oBAAoB,EAAE,IAAI,CAAA;IAE9C;;;;;;;;;;;;;;;OAeG;IACH,MAAM,CAAC,EAAE,CAAC,OAAO,EAAE,MAAM,EAAE,QAAQ,CAAC,EAAE,MAAM,KAAK,IAAI,CAAA;CACrD;AAED,MAAM,WAAW,UAAU;IAC1B,OAAO,EAAE,OAAO,CAAA;IAChB,MAAM,EAAE,MAAM,CAAA;IACd,IAAI,CAAC,EAAE,OAAO,CAAA;IACd,KAAK,CAAC,EAAE,MAAM,CAAA;IACd;;;;;;;;;OASG;IACH,OAAO,CAAC,EAAE,OAAO,qBAAqB,EAAE,iBAAiB,CAAA;IAEzD;;;;;;;;;;;OAWG;IACH,SAAS,CAAC,EAAE,OAAO,CAAA;CACnB;AAED,MAAM,WAAW,cAAc,CAAC,MAAM,GAAG,OAAO;IAC/C,IAAI,EAAE,MAAM,CAAA;IACZ,WAAW,EAAE,MAAM,CAAA;IACnB,WAAW,EAAE,CAAC,CAAC,OAAO,CAAC,MAAM,EAAE,CAAC,CAAC,UAAU,EAAE,OAAO,CAAC,CAAA;IACrD;;;;;;;OAOG;IACH,gBAAgB,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAA;IAC1C;;;OAGG;IACH,iBAAiB,CAAC,EAAE,OAAO,CAAA;IAC3B;;;;OAIG;IACH,mBAAmB,CAAC,EAAE,MAAM,CAAA;IAE5B;;;;;;;;;;;;;OAaG;IACH,YAAY,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAA;IACtC,OAAO,CAAC,KAAK,EAAE,MAAM,EAAE,OAAO,EAAE,WAAW,GAAG,OAAO,CAAC,UAAU,CAAC,CAAA;IACjE,IAAI,CAAC,EAAE,MAAM,CAAA;IACb,WAAW,CAAC,EAAE,cAAc,EAAE,CAAA;IAC9B,QAAQ,CAAC,EAAE,YAAY,GAAG,OAAO,GAAG,SAAS,GAAG,UAAU,GAAG,QAAQ,CAAA;IAErE;;;;;;;;;;;OAWG;IACH,SAAS,CAAC,EAAE,MAAM,CAAA;IAElB;;;;;;;;;;;;;OAaG;IACH,UAAU,CAAC,EAAE,MAAM,CAAA;IAEnB;;;;;;;;;;;;;;;;;;;;;;;;OAwBG;IACH,QAAQ,CAAC,EAAE,OAAO,CAAA;IAElB,UAAU,CAAC,CAAC,KAAK,EAAE,MAAM,GAAG,OAAO,CAAA;IACnC,aAAa,CAAC,CAAC,KAAK,EAAE,MAAM,GAAG,OAAO,CAAA;IACtC,iBAAiB,CAAC,CAAC,KAAK,EAAE,MAAM,GAAG,OAAO,CAAA;IAE1C;;;;;;;;;;;;;OAaG;IACH,UAAU,CAAC,EAAE,cAAc,CAAA;CAC3B;AAED,MAAM,WAAW,cAAc;IAC9B,qEAAqE;IACrE,QAAQ,CAAC,MAAM,EAAE,MAAM,CAAA;IACvB;;;;;;;OAOG;IACH,QAAQ,CAAC,mBAAmB,EAAE,OAAO,CAAA;CACrC;AAED,MAAM,MAAM,cAAc,GACvB,WAAW,GACX,YAAY,GACZ,eAAe,GACf,gBAAgB,GAChB,YAAY,CAAA;AAEf,MAAM,WAAW,aAAa;IAC7B,IAAI,EAAE,UAAU,CAAA;IAChB,QAAQ,EAAE;QACT,IAAI,EAAE,MAAM,CAAA;QACZ,WAAW,EAAE,MAAM,CAAA;QACnB,UAAU,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAA;KACnC,CAAA;CACD;AAED,MAAM,MAAM,gBAAgB,GAAG,UAAU,GAAG,QAAQ,GAAG,WAAW,CAAA;AAElE,MAAM,MAAM,eAAe,GAAG,CAAC,MAAM,EAAE,CAAC,CAAC,OAAO,KAAK,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAA;AAE5E,MAAM,WAAW,kBAAkB;IAClC,EAAE,EAAE,MAAM,CAAA;IACV,KAAK,EAAE,MAAM,CAAA;IACb,QAAQ,EAAE,MAAM,CAAA;IAChB,WAAW,CAAC,EAAE,MAAM,CAAA;CACpB;AAED,MAAM,WAAW,cAAc;IAC9B,KAAK,EAAE,kBAAkB,EAAE,CAAA;IAC3B,gBAAgB,CAAC,EAAE,CAAC,KAAK,EAAE,kBAAkB,EAAE,KAAK,MAAM,CAAA;IAC1D,kBAAkB,CAAC,EAAE,OAAO,CAAA;CAC5B;AAED,MAAM,WAAW,kBAAkB;IAClC,MAAM,CAAC,EAAE,MAAM,CAAA;IACf,UAAU,CAAC,EAAE,cAAc,CAAA;CAC3B;AAED,MAAM,WAAW,mBAAoB,SAAQ,UAAU;IACtD,gBAAgB,CAAC,EAAE,OAAO,CAAA;IAC1B,iBAAiB,CAAC,EAAE,MAAM,CAAA;CAC1B;AAED;;;GAGG;AACH,MAAM,WAAW,oBAAoB;IACpC,QAAQ,CAAC,EAAE,EAAE,MAAM,EAAE,IAAI,EAAE,cAAc,GAAG,IAAI,CAAA;IAChD,QAAQ,CAAC,IAAI,EAAE,cAAc,EAAE,YAAY,CAAC,EAAE,gBAAgB,GAAG,IAAI,CAAA;IACrE,QAAQ,CAAC,KAAK,EAAE,cAAc,EAAE,EAAE,YAAY,CAAC,EAAE,gBAAgB,GAAG,IAAI,CAAA;IAExE,UAAU,CAAC,EAAE,EAAE,MAAM,GAAG,OAAO,CAAA;IAC/B,KAAK,IAAI,IAAI,CAAA;IAEb,GAAG,CAAC,IAAI,EAAE,MAAM,GAAG,cAAc,GAAG,SAAS,CAAA;IAC7C,UAAU,CAAC,IAAI,EAAE,MAAM,GAAG,cAAc,CAAA;IACxC,GAAG,CAAC,IAAI,EAAE,MAAM,GAAG,OAAO,CAAA;IAC1B,MAAM,IAAI,cAAc,EAAE,CAAA;IAC1B,OAAO,IAAI,MAAM,EAAE,CAAA;IACnB,SAAS,IAAI,MAAM,EAAE,CAAA;IAErB,eAAe,CAAC,IAAI,EAAE,MAAM,GAAG,gBAAgB,CAAA;IAC/C,QAAQ,CAAC,KAAK,EAAE,MAAM,EAAE,GAAG,IAAI,CAAA;IAC/B,KAAK,CAAC,KAAK,EAAE,MAAM,EAAE,GAAG,IAAI,CAAA;IAC5B,UAAU,IAAI,IAAI,CAAA;IAClB,YAAY,IAAI,OAAO,CAAA;IACvB,cAAc,CAAC,KAAK,EAAE,MAAM,GAAG,cAAc,EAAE,CAAA;IAC/C,gBAAgB,CAAC,SAAS,CAAC,EAAE,MAAM,EAAE,GAAG,cAAc,EAAE,CAAA;IAExD,OAAO,CAAC,QAAQ,EAAE,MAAM,EAAE,QAAQ,EAAE,OAAO,EAAE,OAAO,EAAE,WAAW,GAAG,OAAO,CAAC,mBAAmB,CAAC,CAAA;IAEhG,IAAI,IAAI,MAAM,CAAA;IAEd,UAAU,CAAC,SAAS,CAAC,EAAE,MAAM,EAAE,GAAG,aAAa,EAAE,CAAA;IACjD,eAAe,CAAC,SAAS,CAAC,EAAE,MAAM,EAAE,GAAG,MAAM,CAAA;IAC7C,cAAc,IAAI,MAAM,GAAG,IAAI,CAAA;IAC/B,WAAW,CAAC,OAAO,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,GAAG,IAAI,CAAA;CAClD;AAED,cAAc,aAAa,CAAA"}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"index.js","sourceRoot":"","sources":["../../../src/types/tool/index.ts"],"names":[],"mappings":"
|
|
1
|
+
{"version":3,"file":"index.js","sourceRoot":"","sources":["../../../src/types/tool/index.ts"],"names":[],"mappings":"AAobA,cAAc,aAAa,CAAA"}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"rules.d.ts","sourceRoot":"","sources":["../../src/verification/rules.ts"],"names":[],"mappings":"
|
|
1
|
+
{"version":3,"file":"rules.d.ts","sourceRoot":"","sources":["../../src/verification/rules.ts"],"names":[],"mappings":"AAEA,OAAO,KAAK,EAAE,cAAc,EAAE,MAAM,wBAAwB,CAAA;AAC5D,OAAO,KAAK,EAAE,YAAY,EAAE,gBAAgB,EAAE,MAAM,gCAAgC,CAAA;AAEpF,wBAAgB,YAAY,CAC3B,IAAI,EAAE,gBAAgB,EACtB,QAAQ,EAAE,MAAM,EAChB,SAAS,EAAE,OAAO,EAClB,OAAO,EAAE,cAAc,GAAG,SAAS,EACnC,eAAe,CAAC,EAAE,MAAM,EACxB,OAAO,CAAC,EAAE,GAAG,CAAC,MAAM,CAAC,GACnB,YAAY,GAAG,IAAI,CAiGrB"}
|
|
@@ -1,8 +1,12 @@
|
|
|
1
1
|
import { DANGEROUS_PATTERNS } from '../constants/tools/index.js';
|
|
2
|
+
import { isTrustedReadOnly } from '../tools/trusted-read-only.js';
|
|
2
3
|
export function evaluateRule(rule, toolName, toolInput, toolDef, compiledPattern, nameSet) {
|
|
3
4
|
switch (rule.type) {
|
|
4
5
|
case 'allow_read_only': {
|
|
5
|
-
|
|
6
|
+
// A server's own claim about its own tool cannot settle this. See
|
|
7
|
+
// `isTrustedReadOnly`: a self-declaration may raise the requirement
|
|
8
|
+
// and never lower it.
|
|
9
|
+
return isTrustedReadOnly(toolDef, toolInput) ? 'allow' : null;
|
|
6
10
|
}
|
|
7
11
|
case 'deny_dangerous_patterns': {
|
|
8
12
|
const serialized = JSON.stringify(toolInput);
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"rules.js","sourceRoot":"","sources":["../../src/verification/rules.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,kBAAkB,EAAE,MAAM,6BAA6B,CAAA;
|
|
1
|
+
{"version":3,"file":"rules.js","sourceRoot":"","sources":["../../src/verification/rules.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,kBAAkB,EAAE,MAAM,6BAA6B,CAAA;AAChE,OAAO,EAAE,iBAAiB,EAAE,MAAM,+BAA+B,CAAA;AAIjE,MAAM,UAAU,YAAY,CAC3B,IAAsB,EACtB,QAAgB,EAChB,SAAkB,EAClB,OAAmC,EACnC,eAAwB,EACxB,OAAqB;IAErB,QAAQ,IAAI,CAAC,IAAI,EAAE,CAAC;QACnB,KAAK,iBAAiB,CAAC,CAAC,CAAC;YACxB,kEAAkE;YAClE,oEAAoE;YACpE,sBAAsB;YACtB,OAAO,iBAAiB,CAAC,OAAO,EAAE,SAAS,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,IAAI,CAAA;QAC9D,CAAC;QAED,KAAK,yBAAyB,CAAC,CAAC,CAAC;YAChC,MAAM,UAAU,GAAG,IAAI,CAAC,SAAS,CAAC,SAAS,CAAC,CAAA;YAC5C,KAAK,MAAM,OAAO,IAAI,kBAAkB,EAAE,CAAC;gBAC1C,IAAI,OAAO,CAAC,IAAI,CAAC,UAAU,CAAC,EAAE,CAAC;oBAC9B,OAAO,MAAM,CAAA;gBACd,CAAC;YACF,CAAC;YACD,OAAO,IAAI,CAAA;QACZ,CAAC;QAED,KAAK,mBAAmB,CAAC,CAAC,CAAC;YAC1B,IAAI,OAAO,EAAE,QAAQ,IAAI,IAAI,CAAC,UAAU,CAAC,QAAQ,CAAC,OAAO,CAAC,QAAQ,CAAC,EAAE,CAAC;gBACrE,OAAO,OAAO,CAAA;YACf,CAAC;YACD,OAAO,IAAI,CAAA;QACZ,CAAC;QAED,KAAK,eAAe,CAAC,CAAC,CAAC;YACtB,OAAO,OAAO,EAAE,GAAG,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,IAAI,CAAA;QAC/C,CAAC;QAED,KAAK,cAAc,CAAC,CAAC,CAAC;YACrB,OAAO,OAAO,EAAE,GAAG,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,IAAI,CAAA;QAC9C,CAAC;QAED,KAAK,gBAAgB,CAAC,CAAC,CAAC;YACvB,IAAI,CAAC,eAAe;gBAAE,OAAO,IAAI,CAAA;YAEjC,IAAI,MAAc,CAAA;YAClB,QAAQ,IAAI,CAAC,MAAM,EAAE,CAAC;gBACrB,KAAK,MAAM;oBACV,MAAM,GAAG,QAAQ,CAAA;oBACjB,MAAK;gBACN,KAAK,MAAM;oBACV,MAAM,GAAG,IAAI,CAAC,SAAS,CAAC,SAAS,CAAC,CAAA;oBAClC,MAAK;gBACN,KAAK,MAAM;oBACV,MAAM,GAAG,GAAG,QAAQ,IAAI,IAAI,CAAC,SAAS,CAAC,SAAS,CAAC,EAAE,CAAA;oBACnD,MAAK;gBACN,OAAO,CAAC,CAAC,CAAC;oBACT,MAAM,WAAW,GAAU,IAAI,CAAC,MAAM,CAAA;oBACtC,MAAM,IAAI,KAAK,CAAC,oCAAoC,WAAqB,EAAE,CAAC,CAAA;gBAC7E,CAAC;YACF,CAAC;YAED,OAAO,eAAe,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC,CAAC,IAAI,CAAA;QAC3D,CAAC;QAED,KAAK,kBAAkB,CAAC,CAAC,CAAC;YACzB,IAAI,CAAC,eAAe;gBAAE,OAAO,IAAI,CAAA;YACjC,IAAI,CAAC,OAAO,EAAE,GAAG,CAAC,QAAQ,CAAC;gBAAE,OAAO,IAAI,CAAA;YAExC,IAAI,OAAO,SAAS,KAAK,QAAQ,IAAI,SAAS,KAAK,IAAI;gBAAE,OAAO,IAAI,CAAA;YACpE,MAAM,KAAK,GAAI,SAAqC,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAA;YAEnE,oEAAoE;YACpE,iEAAiE;YACjE,2DAA2D;YAC3D,EAAE;YACF,iEAAiE;YACjE,mEAAmE;YACnE,oEAAoE;YACpE,kEAAkE;YAClE,kEAAkE;YAClE,oEAAoE;YACpE,MAAM,OAAO,GACZ,OAAO,KAAK,KAAK,QAAQ;gBACxB,CAAC,CAAC,KAAK;gBACP,CAAC,CAAC,OAAO,KAAK,KAAK,QAAQ,IAAI,OAAO,KAAK,KAAK,SAAS;oBACxD,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC;oBACf,CAAC,CAAC,SAAS,CAAA;YACd,IAAI,OAAO,KAAK,SAAS;gBAAE,OAAO,IAAI,CAAA;YAEtC,OAAO,eAAe,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC,CAAC,IAAI,CAAA;QAC5D,CAAC;QAED,KAAK,eAAe,CAAC,CAAC,CAAC;YACtB,IAAI,OAAO,EAAE,IAAI,IAAI,IAAI,CAAC,KAAK,CAAC,QAAQ,CAAC,OAAO,CAAC,IAAI,CAAC,EAAE,CAAC;gBACxD,OAAO,OAAO,CAAA;YACf,CAAC;YACD,OAAO,IAAI,CAAA;QACZ,CAAC;QAED,OAAO,CAAC,CAAC,CAAC;YACT,MAAM,WAAW,GAAU,IAAI,CAAA;YAC/B,MAAM,IAAI,KAAK,CAAC,qCAAsC,WAAgC,CAAC,IAAI,EAAE,CAAC,CAAA;QAC/F,CAAC;IACF,CAAC;AACF,CAAC"}
|
package/package.json
CHANGED
|
@@ -47,8 +47,30 @@ export class WorkingStateManager {
|
|
|
47
47
|
this.pushWithEviction('decisions', this.state.decisions, decision, this.config.maxListSize)
|
|
48
48
|
}
|
|
49
49
|
|
|
50
|
+
/**
|
|
51
|
+
* Failures evict OLDEST-first, unlike every other slot here.
|
|
52
|
+
*
|
|
53
|
+
* `keepFirstEntries` exists because early decisions are load-bearing —
|
|
54
|
+
* the one that set the run's approach outlives twenty-five incidental
|
|
55
|
+
* notes. That reasoning is right for decisions and backwards for
|
|
56
|
+
* failures: the earliest failure is the one the model has most likely
|
|
57
|
+
* already worked around, and the recent one is the thing it reads to
|
|
58
|
+
* decide what to do differently.
|
|
59
|
+
*
|
|
60
|
+
* It also matters more than a preference. Sinha et al.,
|
|
61
|
+
* "The Illusion of Diminishing Returns" (arXiv:2509.09677), inject
|
|
62
|
+
* errors into a model's own history at controlled rates and measure
|
|
63
|
+
* accuracy far later in the run: conditioning a model on its own
|
|
64
|
+
* error-prone history raises the likelihood of further errors, and
|
|
65
|
+
* scaling does not rescue it. So a permanently-protected early failure
|
|
66
|
+
* is not neutral ballast — it is the input that paper measures.
|
|
67
|
+
*
|
|
68
|
+
* Nothing here decided failures should keep their oldest entries; the
|
|
69
|
+
* behaviour was inherited from a shared helper written for a slot where
|
|
70
|
+
* it is correct.
|
|
71
|
+
*/
|
|
50
72
|
addFailure(failure: string): void {
|
|
51
|
-
this.pushWithEviction('failures', this.state.failures, failure, this.config.maxListSize)
|
|
73
|
+
this.pushWithEviction('failures', this.state.failures, failure, this.config.maxListSize, 0)
|
|
52
74
|
}
|
|
53
75
|
|
|
54
76
|
addDiscovery(discovery: string): void {
|
|
@@ -141,9 +163,23 @@ export class WorkingStateManager {
|
|
|
141
163
|
* condenser uses. The eviction is counted so the serializer can say
|
|
142
164
|
* something was dropped rather than presenting a gap as complete.
|
|
143
165
|
*/
|
|
144
|
-
private pushWithEviction(
|
|
166
|
+
private pushWithEviction(
|
|
167
|
+
slot: string,
|
|
168
|
+
list: string[],
|
|
169
|
+
item: string,
|
|
170
|
+
max: number,
|
|
171
|
+
/**
|
|
172
|
+
* Entries to protect at the front. Defaults to the configured
|
|
173
|
+
* `keepFirstEntries`; pass 0 for a slot where the early entries are
|
|
174
|
+
* the ones to lose. See {@link addFailure}.
|
|
175
|
+
*/
|
|
176
|
+
keepFirstOverride?: number,
|
|
177
|
+
): void {
|
|
145
178
|
list.push(item)
|
|
146
|
-
const keepFirst = Math.min(
|
|
179
|
+
const keepFirst = Math.min(
|
|
180
|
+
keepFirstOverride ?? this.config.keepFirstEntries,
|
|
181
|
+
Math.max(0, max - 1),
|
|
182
|
+
)
|
|
147
183
|
while (list.length > max) {
|
|
148
184
|
list.splice(keepFirst, 1)
|
|
149
185
|
this.state.evicted[slot] = (this.state.evicted[slot] ?? 0) + 1
|
|
@@ -419,6 +419,12 @@ export function mcpToolToToolDefinition(
|
|
|
419
419
|
tool: MCPToolDefinition,
|
|
420
420
|
client: MCPClient,
|
|
421
421
|
serverName: string,
|
|
422
|
+
/**
|
|
423
|
+
* The operator marked this server's read-only claims trustworthy.
|
|
424
|
+
* Default false: an unmarked server's claim raises the requirement and
|
|
425
|
+
* never lowers it. See `isTrustedReadOnly`.
|
|
426
|
+
*/
|
|
427
|
+
readOnlyHintTrusted = false,
|
|
422
428
|
): ToolDefinition {
|
|
423
429
|
const inputSchema = mcpJsonSchemaToZod(tool.inputSchema)
|
|
424
430
|
const toolName = `mcp_${serverName}_${tool.name}`
|
|
@@ -436,9 +442,14 @@ export function mcpToolToToolDefinition(
|
|
|
436
442
|
: {}),
|
|
437
443
|
category: 'network',
|
|
438
444
|
permissions: ['network_access'],
|
|
445
|
+
// Reports what the SERVER said, faithfully. The outbound re-export
|
|
446
|
+
// and the destructive label a human is shown both need the server's
|
|
447
|
+
// own answer; whether a gate may act on it is decided separately, by
|
|
448
|
+
// `isTrustedReadOnly` reading `provenance` below.
|
|
439
449
|
isReadOnly: () => tool.annotations?.readOnlyHint ?? false,
|
|
440
450
|
isDestructive: () => tool.annotations?.destructiveHint ?? false,
|
|
441
451
|
isConcurrencySafe: () => true,
|
|
452
|
+
provenance: { server: serverName, readOnlyHintTrusted },
|
|
442
453
|
|
|
443
454
|
async execute(input: unknown, _context: ToolContext): Promise<ToolResult> {
|
|
444
455
|
const result = await client.callTool(tool.name, input as Record<string, unknown>)
|
|
@@ -34,10 +34,16 @@ import type { MCPClient } from './client.js'
|
|
|
34
34
|
/**
|
|
35
35
|
* Marks where a remote party's words begin and end.
|
|
36
36
|
*
|
|
37
|
-
* A prompt is composed by a SERVER. Untrusted content arriving
|
|
38
|
-
*
|
|
39
|
-
*
|
|
40
|
-
*
|
|
37
|
+
* A prompt is composed by a SERVER. Untrusted content arriving this way is
|
|
38
|
+
* the standard prompt-injection surface, and an unlabelled block reads
|
|
39
|
+
* exactly like the agent's own instructions — so this says whose words
|
|
40
|
+
* they are.
|
|
41
|
+
*
|
|
42
|
+
* Marking, not stopping. See `tools/untrusted-envelope.ts` for the
|
|
43
|
+
* measurement: delimiting reports near-zero attack success on a static
|
|
44
|
+
* benchmark and above 95% once the attacker adapts (arXiv:2510.09023).
|
|
45
|
+
* This paragraph used to call it "the mitigation that survives contact",
|
|
46
|
+
* which was the same overstatement in a second file.
|
|
41
47
|
*/
|
|
42
48
|
export function renderPromptMessages(
|
|
43
49
|
serverName: string,
|
package/src/eval/experiment.ts
CHANGED
|
@@ -7,6 +7,8 @@ import type {
|
|
|
7
7
|
Score,
|
|
8
8
|
Scorer,
|
|
9
9
|
} from './types.js'
|
|
10
|
+
import type { ScoreUncertainty } from './uncertainty.js'
|
|
11
|
+
import { describeUncertainty, uncertaintyOf } from './uncertainty.js'
|
|
10
12
|
|
|
11
13
|
export interface ExperimentConfig<TInput = unknown> {
|
|
12
14
|
name: string
|
|
@@ -151,6 +153,10 @@ export async function runExperiment<TInput>(
|
|
|
151
153
|
name: config.name,
|
|
152
154
|
cases: settled,
|
|
153
155
|
mean,
|
|
156
|
+
// Over the same cases the mean is over. Computing spread across a
|
|
157
|
+
// different denominator than the average it qualifies would produce
|
|
158
|
+
// an interval that does not belong to the number beside it.
|
|
159
|
+
uncertainty: uncertaintyOf(scored.map((r) => r.mean)),
|
|
154
160
|
passed: settled.filter((r) => r.status === 'passed').length,
|
|
155
161
|
failed: settled.filter((r) => r.status === 'failed').length,
|
|
156
162
|
inconclusive: settled.filter((r) => r.status === 'inconclusive').length,
|
|
@@ -278,9 +284,33 @@ function meanByScorer(results: readonly CaseResult[]): Record<string, number> {
|
|
|
278
284
|
* Failures print their scorer reasons, because a CI log that says
|
|
279
285
|
* "0.62" is a log that sends someone back to reproduce it by hand.
|
|
280
286
|
*/
|
|
287
|
+
/**
|
|
288
|
+
* Uncertainty for a report that did not carry its own.
|
|
289
|
+
*
|
|
290
|
+
* A suite file is loaded at runtime and may be plain JavaScript, so a
|
|
291
|
+
* report can reach here hand-built — the type cannot stop it. Deriving
|
|
292
|
+
* from the cases it does carry is better than either alternative:
|
|
293
|
+
* printing the mean alone leaves the reader where they started, and
|
|
294
|
+
* refusing to format would turn a missing convenience into a broken
|
|
295
|
+
* command.
|
|
296
|
+
*
|
|
297
|
+
* Uses the same exclusion `runExperiment` uses, so a derived interval and
|
|
298
|
+
* a carried one are the same number rather than two conventions.
|
|
299
|
+
*/
|
|
300
|
+
function derivedUncertainty(report: ExperimentReport): ScoreUncertainty {
|
|
301
|
+
return uncertaintyOf(report.cases.filter((c) => c.status !== 'inconclusive').map((c) => c.mean))
|
|
302
|
+
}
|
|
303
|
+
|
|
281
304
|
export function formatReport(report: ExperimentReport): string {
|
|
282
305
|
const lines: string[] = [
|
|
283
306
|
`${report.name}: ${report.passed}/${report.cases.length} passed (mean ${report.mean.toFixed(2)}) in ${report.durationMs}ms`,
|
|
307
|
+
// On its own line and always printed, including when the interval is
|
|
308
|
+
// undefined. A mean printed alone is the thing that has been
|
|
309
|
+
// over-read: two runs three points apart look like a difference, and
|
|
310
|
+
// at the n a hand-built suite has they are usually the same run
|
|
311
|
+
// twice. Computing the interval and not showing it would leave the
|
|
312
|
+
// reader exactly where they started.
|
|
313
|
+
` ${describeUncertainty(report.mean, report.uncertainty ?? derivedUncertainty(report))}`,
|
|
284
314
|
'',
|
|
285
315
|
]
|
|
286
316
|
|
package/src/eval/index.ts
CHANGED
package/src/eval/types.ts
CHANGED
|
@@ -1,4 +1,5 @@
|
|
|
1
1
|
import type { StepResult } from '../types/run/step.js'
|
|
2
|
+
import type { ScoreUncertainty } from './uncertainty.js'
|
|
2
3
|
|
|
3
4
|
/**
|
|
4
5
|
* One case in a dataset: an input, and what a good run looks like.
|
|
@@ -143,6 +144,22 @@ export interface ExperimentReport {
|
|
|
143
144
|
cases: readonly CaseResult[]
|
|
144
145
|
/** Mean score across the cases that produced one. */
|
|
145
146
|
mean: number
|
|
147
|
+
/**
|
|
148
|
+
* How much of {@link mean} is signal.
|
|
149
|
+
*
|
|
150
|
+
* A mean on its own has been read as a result, and at the n a
|
|
151
|
+
* hand-built suite has it usually is not one: two runs three points
|
|
152
|
+
* apart are normally the same run twice.
|
|
153
|
+
*
|
|
154
|
+
* Optional, and deliberately so after trying it the other way. A suite
|
|
155
|
+
* file is loaded at runtime and may be plain JavaScript, so a required
|
|
156
|
+
* field is not enforced at the boundary that matters — it buys type
|
|
157
|
+
* safety for one kind of consumer and a crash for the other. Producers
|
|
158
|
+
* that go through `runExperiment` always set it; `formatReport` derives
|
|
159
|
+
* it from {@link cases} when a hand-built report does not, so no report
|
|
160
|
+
* is printed without an interval either way.
|
|
161
|
+
*/
|
|
162
|
+
uncertainty?: ScoreUncertainty
|
|
146
163
|
passed: number
|
|
147
164
|
failed: number
|
|
148
165
|
/** Cases where no scorer could produce a judgement. */
|
|
@@ -0,0 +1,124 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* How much of a score is signal.
|
|
3
|
+
*
|
|
4
|
+
* A suite reported a mean and nothing else, so two runs differing by three
|
|
5
|
+
* points read as a difference. At the n a hand-built suite has, that is
|
|
6
|
+
* usually noise, and there was no number on the page that would have said
|
|
7
|
+
* so.
|
|
8
|
+
*
|
|
9
|
+
* Evan Miller, "Adding Error Bars to Evals" (arXiv:2411.00640), is the
|
|
10
|
+
* reference. Two of its results shape what is and is not computed here.
|
|
11
|
+
*/
|
|
12
|
+
|
|
13
|
+
/**
|
|
14
|
+
* Two-sided 95% critical values of Student's t, by degrees of freedom.
|
|
15
|
+
*
|
|
16
|
+
* The normal approximation (1.96) is what most harnesses use and it is
|
|
17
|
+
* wrong in the direction that matters: at n=5 the true multiplier is 2.78,
|
|
18
|
+
* so a normal interval is nearly 30% too narrow exactly where a suite is
|
|
19
|
+
* small enough for that to mislead. Eval suites are small; this table is
|
|
20
|
+
* the difference between an interval that covers and one that flatters.
|
|
21
|
+
*/
|
|
22
|
+
const T_95: readonly number[] = [
|
|
23
|
+
12.706, 4.303, 3.182, 2.776, 2.571, 2.447, 2.365, 2.306, 2.262, 2.228, 2.201, 2.179, 2.16, 2.145,
|
|
24
|
+
2.131, 2.12, 2.11, 2.101, 2.093, 2.086, 2.08, 2.074, 2.069, 2.064, 2.06, 2.056, 2.052, 2.048,
|
|
25
|
+
2.045, 2.042,
|
|
26
|
+
]
|
|
27
|
+
|
|
28
|
+
function critical95(df: number): number {
|
|
29
|
+
if (df < 1) return Number.NaN
|
|
30
|
+
// Beyond 30 the t value is within ~1% of the normal, and pretending
|
|
31
|
+
// otherwise would imply a precision the rest of this does not have.
|
|
32
|
+
return T_95[df - 1] ?? 1.96
|
|
33
|
+
}
|
|
34
|
+
|
|
35
|
+
export interface ScoreUncertainty {
|
|
36
|
+
/** Cases that produced a score. Not the number of cases run. */
|
|
37
|
+
readonly n: number
|
|
38
|
+
/** Sample standard deviation, Bessel-corrected. */
|
|
39
|
+
readonly stdDev: number
|
|
40
|
+
/** Standard error of the mean. */
|
|
41
|
+
readonly stdError: number
|
|
42
|
+
/** Half-width of the 95% interval: the mean plus or minus this. */
|
|
43
|
+
readonly margin95: number
|
|
44
|
+
/**
|
|
45
|
+
* The 95% interval, clamped to the score range.
|
|
46
|
+
*
|
|
47
|
+
* Clamped because a mean of 0.95 with a wide interval otherwise reports
|
|
48
|
+
* an upper bound above 1, which is not a possible score and makes a
|
|
49
|
+
* reader distrust the whole figure. The clamp is cosmetic and the
|
|
50
|
+
* margin above is not — read that one for the width.
|
|
51
|
+
*/
|
|
52
|
+
readonly ci95: readonly [low: number, high: number]
|
|
53
|
+
/**
|
|
54
|
+
* True when there is not enough data for an interval at all.
|
|
55
|
+
*
|
|
56
|
+
* One case has no spread to measure. Reporting `±0` there would be the
|
|
57
|
+
* most confident-looking output the suite can produce, from the least
|
|
58
|
+
* evidence it can have.
|
|
59
|
+
*/
|
|
60
|
+
readonly undefinedInterval: boolean
|
|
61
|
+
}
|
|
62
|
+
|
|
63
|
+
/**
|
|
64
|
+
* Spread of a set of scores, with the interval a reader should apply.
|
|
65
|
+
*
|
|
66
|
+
* **Assumes the cases are independent, and they may not be.** Miller's
|
|
67
|
+
* clustered standard errors run up to 3x the naive figure when cases come
|
|
68
|
+
* in related groups — several cases derived from one scenario, or one
|
|
69
|
+
* document, or one seed. This harness has no grouping key on a case, so
|
|
70
|
+
* there is nothing here to cluster on and this returns the naive figure.
|
|
71
|
+
* Where a suite does build several cases from one source, treat the
|
|
72
|
+
* interval below as a floor rather than as the answer.
|
|
73
|
+
*
|
|
74
|
+
* Stated rather than silently assumed because a too-narrow interval is
|
|
75
|
+
* worse than none: it turns "we cannot tell" into a number that looks
|
|
76
|
+
* like we can.
|
|
77
|
+
*/
|
|
78
|
+
export function uncertaintyOf(scores: readonly number[]): ScoreUncertainty {
|
|
79
|
+
const n = scores.length
|
|
80
|
+
if (n < 2) {
|
|
81
|
+
return {
|
|
82
|
+
n,
|
|
83
|
+
stdDev: 0,
|
|
84
|
+
stdError: 0,
|
|
85
|
+
margin95: Number.NaN,
|
|
86
|
+
ci95: [Number.NaN, Number.NaN],
|
|
87
|
+
undefinedInterval: true,
|
|
88
|
+
}
|
|
89
|
+
}
|
|
90
|
+
|
|
91
|
+
const mean = scores.reduce((a, b) => a + b, 0) / n
|
|
92
|
+
// Bessel-corrected: dividing by n estimates the spread of THESE cases,
|
|
93
|
+
// and the question is about the suite they were drawn from.
|
|
94
|
+
const variance = scores.reduce((acc, s) => acc + (s - mean) ** 2, 0) / (n - 1)
|
|
95
|
+
const stdDev = Math.sqrt(variance)
|
|
96
|
+
const stdError = stdDev / Math.sqrt(n)
|
|
97
|
+
const margin95 = critical95(n - 1) * stdError
|
|
98
|
+
|
|
99
|
+
return {
|
|
100
|
+
n,
|
|
101
|
+
stdDev,
|
|
102
|
+
stdError,
|
|
103
|
+
margin95,
|
|
104
|
+
ci95: [Math.max(0, mean - margin95), Math.min(1, mean + margin95)],
|
|
105
|
+
undefinedInterval: false,
|
|
106
|
+
}
|
|
107
|
+
}
|
|
108
|
+
|
|
109
|
+
/**
|
|
110
|
+
* One line a reader can act on, for a surface that prints a score.
|
|
111
|
+
*
|
|
112
|
+
* Names the interval rather than only the mean, because the mean alone is
|
|
113
|
+
* the thing that has been over-read. An interval spanning most of the
|
|
114
|
+
* scale says the suite cannot currently tell two runs apart, and that is
|
|
115
|
+
* the most useful sentence such a suite can produce.
|
|
116
|
+
*/
|
|
117
|
+
export function describeUncertainty(mean: number, u: ScoreUncertainty): string {
|
|
118
|
+
if (u.undefinedInterval) {
|
|
119
|
+
return u.n === 0
|
|
120
|
+
? 'no scored cases, so no score'
|
|
121
|
+
: `${mean.toFixed(3)} from a single case — no interval, and one case cannot show spread`
|
|
122
|
+
}
|
|
123
|
+
return `${mean.toFixed(3)} ±${u.margin95.toFixed(3)} (95% CI ${u.ci95[0].toFixed(3)}–${u.ci95[1].toFixed(3)}, n=${u.n}); assumes cases are independent`
|
|
124
|
+
}
|
package/src/public-runtime.ts
CHANGED
|
@@ -63,6 +63,7 @@ export {
|
|
|
63
63
|
export { normaliseModelId, resolveModelPricing, VENDOR_RATES } from './pricing/index.js'
|
|
64
64
|
export { toErrorMessage } from './utils/error.js'
|
|
65
65
|
export { configureLogger, getRootLogger, Logger } from './utils/logger.js'
|
|
66
|
+
export { isTrustedReadOnly } from './tools/trusted-read-only.js'
|
|
66
67
|
export { buildToolResultHashes, hashToolResult } from './utils/hash.js'
|
|
67
68
|
export {
|
|
68
69
|
compressShellOutput,
|
|
@@ -3,6 +3,7 @@ import { assertStrictSchema } from '../../provider/strict-schema.js'
|
|
|
3
3
|
import { GENAI, NAMZU, toolSpanName } from '../../telemetry/attributes.js'
|
|
4
4
|
import { recordToolCall } from '../../telemetry/metrics.js'
|
|
5
5
|
import { getTracer } from '../../telemetry/runtime-accessors.js'
|
|
6
|
+
import { isTrustedReadOnly } from '../../tools/trusted-read-only.js'
|
|
6
7
|
import type {
|
|
7
8
|
LLMToolSchema,
|
|
8
9
|
ToolAvailability,
|
|
@@ -487,7 +488,7 @@ Executable tool names, descriptions, and JSON input schemas are attached through
|
|
|
487
488
|
|
|
488
489
|
const mode = context.permissionContext?.mode ?? 'auto'
|
|
489
490
|
if (mode === 'plan') {
|
|
490
|
-
const isReadOnly = tool
|
|
491
|
+
const isReadOnly = isTrustedReadOnly(tool, rawInput)
|
|
491
492
|
if (!isReadOnly) {
|
|
492
493
|
const msg = `plan mode: non-read-only tool "${toolName}" blocked`
|
|
493
494
|
span.setAttributes({
|
|
@@ -0,0 +1,52 @@
|
|
|
1
|
+
import type { ToolDefinition } from '../types/tool/index.js'
|
|
2
|
+
|
|
3
|
+
/**
|
|
4
|
+
* May this tool's read-only claim settle a gate on its own?
|
|
5
|
+
*
|
|
6
|
+
* A connected server declares whether its own tools are read-only, and
|
|
7
|
+
* that declaration decided whether a call was approved without asking. The
|
|
8
|
+
* thing being gated supplied the input to the gate.
|
|
9
|
+
*
|
|
10
|
+
* The wire itself calls these fields HINTS. Three separate consumers read
|
|
11
|
+
* them as facts, so a server setting `readOnlyHint: true` and
|
|
12
|
+
* `destructiveHint: false` controlled the whole predicate for its own
|
|
13
|
+
* tools — the kernel's `allow_read_only` rule, the operator prompt
|
|
14
|
+
* exemption, and the plan-mode pass.
|
|
15
|
+
*
|
|
16
|
+
* The estate floor this repository inherits already decided this: least
|
|
17
|
+
* privilege, default deny, and — on tool results and fetched content —
|
|
18
|
+
* data is not instructions, and untrusted content cannot escalate
|
|
19
|
+
* capabilities. A server's declaration about its own tools is untrusted
|
|
20
|
+
* content by that definition. This is that rule applied, not a new policy.
|
|
21
|
+
*
|
|
22
|
+
* **The asymmetry is the design.** A self-declaration may RAISE the
|
|
23
|
+
* requirement and never LOWER it:
|
|
24
|
+
*
|
|
25
|
+
* - `destructiveHint: true` from a server is believed. A server
|
|
26
|
+
* volunteering that its tool is dangerous moves toward caution, and
|
|
27
|
+
* disbelieving it buys nothing.
|
|
28
|
+
* - `readOnlyHint: true` from a server does not, on its own, settle a
|
|
29
|
+
* call as allowed or skip a prompt. That is the untrusted party
|
|
30
|
+
* opening its own gate.
|
|
31
|
+
*
|
|
32
|
+
* Trust for the second case comes from the operator, per server, and is
|
|
33
|
+
* recorded on the tool as `provenance.readOnlyHintTrusted`. Never a global
|
|
34
|
+
* switch: one flag meaning "trust annotations" hands every connected
|
|
35
|
+
* server the same reach, which is the hole restated.
|
|
36
|
+
*
|
|
37
|
+
* `isReadOnly` itself is left reporting faithfully what the server said.
|
|
38
|
+
* Provenance and policy are different questions, and collapsing them would
|
|
39
|
+
* corrupt the outbound re-export and the prompt's own destructive label in
|
|
40
|
+
* order to fix a gate.
|
|
41
|
+
*/
|
|
42
|
+
export function isTrustedReadOnly(tool: ToolDefinition | undefined, input: unknown): boolean {
|
|
43
|
+
if (!tool?.isReadOnly) return false
|
|
44
|
+
|
|
45
|
+
// No provenance means the tool is host-defined: it came from this
|
|
46
|
+
// process, from code the operator installed, and there is no untrusted
|
|
47
|
+
// party in the chain. Requiring an opt-in for a builtin would break
|
|
48
|
+
// every read-only exemption for no gain in trust.
|
|
49
|
+
if (tool.provenance && !tool.provenance.readOnlyHintTrusted) return false
|
|
50
|
+
|
|
51
|
+
return tool.isReadOnly(input)
|
|
52
|
+
}
|
|
@@ -2,12 +2,32 @@
|
|
|
2
2
|
* Framing for content the agent did not author and must not obey.
|
|
3
3
|
*
|
|
4
4
|
* An unlabelled block of text in a tool result reads exactly like the agent's
|
|
5
|
-
* own instructions.
|
|
6
|
-
* not filtering — it is saying plainly whose words these are and that they are
|
|
5
|
+
* own instructions. This says plainly whose words these are and that they are
|
|
7
6
|
* material rather than direction. That is the floor this estate already
|
|
8
7
|
* states: data is not instructions, and a tool result cannot escalate what an
|
|
9
8
|
* agent may do.
|
|
10
9
|
*
|
|
10
|
+
* **It marks provenance. It refuses nothing, and it does not stop an
|
|
11
|
+
* attacker who is trying.** This paragraph used to claim the framing was
|
|
12
|
+
* "the mitigation that survives contact with a real model", and that is
|
|
13
|
+
* measurably wrong. Nasr et al., "The Attacker Moves Second"
|
|
14
|
+
* (arXiv:2510.09023), broke twelve published defences at above 90% attack
|
|
15
|
+
* success once the attacker adapts; the majority had originally reported
|
|
16
|
+
* near-zero success. Delimiting specifically goes from as low as 1% under
|
|
17
|
+
* a static benchmark to above 95% under adaptive attack.
|
|
18
|
+
*
|
|
19
|
+
* So read every number for a prompt-level defence as static unless it says
|
|
20
|
+
* otherwise, and treat this envelope as raising cost rather than as a
|
|
21
|
+
* boundary. What survives an adapting attacker in the same literature is
|
|
22
|
+
* architectural: AgentDojo (arXiv:2406.13352) found tool isolation and
|
|
23
|
+
* tool filtering the effective mitigations, and this repository's real
|
|
24
|
+
* boundaries are of that kind — the permission gate, the sandbox, the
|
|
25
|
+
* egress proxy deciding by resolved address.
|
|
26
|
+
*
|
|
27
|
+
* The two details below still matter. They are what stops the framing
|
|
28
|
+
* being trivially removable by the content itself, which is a lower bar
|
|
29
|
+
* than stopping an attacker and worth clearing anyway.
|
|
30
|
+
*
|
|
11
31
|
* Two details make the difference between a boundary and a decoration, and
|
|
12
32
|
* both were missing from this repo's first envelope:
|
|
13
33
|
*
|
package/src/types/tool/index.ts
CHANGED
|
@@ -322,6 +322,36 @@ export interface ToolDefinition<TInput = unknown> {
|
|
|
322
322
|
isReadOnly?(input: TInput): boolean
|
|
323
323
|
isDestructive?(input: TInput): boolean
|
|
324
324
|
isConcurrencySafe?(input: TInput): boolean
|
|
325
|
+
|
|
326
|
+
/**
|
|
327
|
+
* Where this tool came from, when it did not come from here.
|
|
328
|
+
*
|
|
329
|
+
* Absent means host-defined: this process, code the operator installed,
|
|
330
|
+
* no untrusted party in the chain. Present means a connected server
|
|
331
|
+
* supplied both the tool and its own description of what the tool does
|
|
332
|
+
* — including whether it is read-only, which three separate gates were
|
|
333
|
+
* treating as a fact rather than as the hint the wire calls it.
|
|
334
|
+
*
|
|
335
|
+
* See {@link isTrustedReadOnly}. This field exists so a gate can tell
|
|
336
|
+
* the two apart; `isReadOnly` keeps reporting faithfully what the
|
|
337
|
+
* server said, because the outbound re-export and the destructive
|
|
338
|
+
* label shown to a human both need the server's own answer.
|
|
339
|
+
*/
|
|
340
|
+
provenance?: ToolProvenance
|
|
341
|
+
}
|
|
342
|
+
|
|
343
|
+
export interface ToolProvenance {
|
|
344
|
+
/** The connected server this tool came from, named as configured. */
|
|
345
|
+
readonly server: string
|
|
346
|
+
/**
|
|
347
|
+
* The operator marked this server's read-only claims as trustworthy.
|
|
348
|
+
*
|
|
349
|
+
* Per server, never global: one switch meaning "trust annotations"
|
|
350
|
+
* hands every connected server the same reach, which is the hole it
|
|
351
|
+
* would be closing. Default false — an unmarked server's claim raises
|
|
352
|
+
* the requirement and never lowers it.
|
|
353
|
+
*/
|
|
354
|
+
readonly readOnlyHintTrusted: boolean
|
|
325
355
|
}
|
|
326
356
|
|
|
327
357
|
export type ToolPermission =
|
|
@@ -1,4 +1,5 @@
|
|
|
1
1
|
import { DANGEROUS_PATTERNS } from '../constants/tools/index.js'
|
|
2
|
+
import { isTrustedReadOnly } from '../tools/trusted-read-only.js'
|
|
2
3
|
import type { ToolDefinition } from '../types/tool/index.js'
|
|
3
4
|
import type { GateDecision, VerificationRule } from '../types/verification/index.js'
|
|
4
5
|
|
|
@@ -12,7 +13,10 @@ export function evaluateRule(
|
|
|
12
13
|
): GateDecision | null {
|
|
13
14
|
switch (rule.type) {
|
|
14
15
|
case 'allow_read_only': {
|
|
15
|
-
|
|
16
|
+
// A server's own claim about its own tool cannot settle this. See
|
|
17
|
+
// `isTrustedReadOnly`: a self-declaration may raise the requirement
|
|
18
|
+
// and never lower it.
|
|
19
|
+
return isTrustedReadOnly(toolDef, toolInput) ? 'allow' : null
|
|
16
20
|
}
|
|
17
21
|
|
|
18
22
|
case 'deny_dangerous_patterns': {
|