agent-threat-rules 2.1.2 → 2.1.5
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/action-executor.d.ts +1 -1
- package/dist/action-executor.d.ts.map +1 -1
- package/dist/action-executor.js +13 -11
- package/dist/action-executor.js.map +1 -1
- package/dist/adapters/default-adapter.d.ts +2 -1
- package/dist/adapters/default-adapter.d.ts.map +1 -1
- package/dist/adapters/default-adapter.js +14 -11
- package/dist/adapters/default-adapter.js.map +1 -1
- package/dist/adapters/stdio-adapter.d.ts +2 -1
- package/dist/adapters/stdio-adapter.d.ts.map +1 -1
- package/dist/adapters/stdio-adapter.js +43 -26
- package/dist/adapters/stdio-adapter.js.map +1 -1
- package/dist/converters/index.d.ts +4 -0
- package/dist/converters/index.d.ts.map +1 -1
- package/dist/converters/index.js +2 -0
- package/dist/converters/index.js.map +1 -1
- package/dist/converters/sage-reverse.d.ts +52 -0
- package/dist/converters/sage-reverse.d.ts.map +1 -0
- package/dist/converters/sage-reverse.js +216 -0
- package/dist/converters/sage-reverse.js.map +1 -0
- package/dist/converters/sage.d.ts +123 -0
- package/dist/converters/sage.d.ts.map +1 -0
- package/dist/converters/sage.js +702 -0
- package/dist/converters/sage.js.map +1 -0
- package/dist/eval/hackaprompt-corpus.d.ts +24 -0
- package/dist/eval/hackaprompt-corpus.d.ts.map +1 -0
- package/dist/eval/hackaprompt-corpus.js +61 -0
- package/dist/eval/hackaprompt-corpus.js.map +1 -0
- package/dist/eval/run-hackaprompt-benchmark.d.ts +19 -0
- package/dist/eval/run-hackaprompt-benchmark.d.ts.map +1 -0
- package/dist/eval/run-hackaprompt-benchmark.js +86 -0
- package/dist/eval/run-hackaprompt-benchmark.js.map +1 -0
- package/dist/types.d.ts +24 -17
- package/dist/types.d.ts.map +1 -1
- package/package.json +9 -1
- package/rules/context-exfiltration/ATR-2026-00449-spring-ai-chatmemory-cross-user-leak.yaml +196 -0
- package/rules/data-poisoning/ATR-2026-00450-spring-ai-prompt-memory-poisoning.yaml +196 -0
- package/rules/privilege-escalation/ATR-2026-00451-litellm-admin-sqli-cisa-kev.yaml +204 -0
- package/rules/prompt-injection/ATR-2026-00442-quoted-exact-output-forcing.yaml +120 -0
- package/rules/prompt-injection/ATR-2026-00443-word-fragment-concat-assembly.yaml +119 -0
- package/rules/prompt-injection/ATR-2026-00444-unicode-obfuscation-in-user-input.yaml +114 -0
- package/rules/prompt-injection/ATR-2026-00445-translation-hijack-with-side-output.yaml +113 -0
- package/rules/prompt-injection/ATR-2026-00446-variable-assignment-payload-injection.yaml +118 -0
- package/rules/prompt-injection/ATR-2026-00447-fictional-generation-containing-target.yaml +113 -0
- package/rules/tool-poisoning/ATR-2026-00448-spring-ai-milvus-filter-injection.yaml +193 -0
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"sage.js","sourceRoot":"","sources":["../../src/converters/sage.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA+BG;AAsDH,8EAA8E;AAE9E;;;;GAIG;AACH,MAAM,YAAY,GAAqC,MAAM,CAAC,MAAM,CAAC;IACpE,kBAAkB,EAAE,kBAAkB;IACtC,gBAAgB,EAAE,eAAe;IACjC,sBAAsB,EAAE,sBAAsB;IAC9C,oBAAoB,EAAE,oBAAoB;IAC1C,sBAAsB,EAAE,sBAAsB;IAC9C,oBAAoB,EAAE,oBAAoB;IAC1C,gBAAgB,EAAE,gBAAgB;IAClC,aAAa,EAAE,aAAa;IAC5B,kBAAkB,EAAE,kBAAkB;IACtC,gBAAgB,EAAE,gBAAgB;CAClC,CAAC,CAAC;AAEH;;GAEG;AACH,MAAM,YAAY,GAAgD,MAAM,CAAC,MAAM,CAAC;IAC/E,QAAQ,EAAE,UAAU;IACpB,IAAI,EAAE,MAAM;IACZ,MAAM,EAAE,QAAQ;IAChB,GAAG,EAAE,KAAK;IACV,aAAa,EAAE,KAAK;CACpB,CAAC,CAAC;AAEH;;;GAGG;AACH,MAAM,eAAe,GAAyC,MAAM,CAAC,MAAM,CAAC;IAC3E,KAAK,EAAE,CAAC;IACR,gBAAgB,EAAE,CAAC;IACnB,GAAG,EAAE,CAAC;CACN,CAAC,CAAC;AAEH;;;GAGG;AACH,SAAS,SAAS,CAAC,GAAc;IAChC,QAAQ,GAAG,EAAE,CAAC;QACb,KAAK,aAAa,CAAC;QACnB,KAAK,cAAc,CAAC;QACpB,KAAK,YAAY,CAAC;QAClB,KAAK,oBAAoB,CAAC;QAC1B,KAAK,YAAY;YAChB,OAAO,OAAO,CAAC;QAChB,KAAK,UAAU;YACd,OAAO,kBAAkB,CAAC;QAC3B,KAAK,OAAO,CAAC;QACb,KAAK,UAAU,CAAC;QAChB,KAAK,QAAQ;YACZ,OAAO,KAAK,CAAC;QACd,KAAK,eAAe,CAAC;QACrB,KAAK,oBAAoB;YACxB,OAAO,IAAI,CAAC;QACb,OAAO,CAAC,CAAC,CAAC;YACT,mEAAmE;YACnE,+DAA+D;YAC/D,MAAM,WAAW,GAAU,GAAG,CAAC;YAC/B,KAAK,WAAW,CAAC;YACjB,OAAO,IAAI,CAAC;QACb,CAAC;IACF,CAAC;AACF,CAAC;AAED;;;;;GAKG;AACH,SAAS,QAAQ,CAAC,QAAgB;IACjC,QAAQ,QAAQ,EAAE,CAAC;QAClB,KAAK,KAAK;YACT,OAAO,KAAK,CAAC;QACd,KAAK,YAAY,CAAC;QAClB,KAAK,cAAc,CAAC;QACpB,KAAK,SAAS,CAAC;QACf,KAAK,eAAe,CAAC;QACrB,KAAK,WAAW,CAAC;QACjB,KAAK,WAAW,CAAC;QACjB,KAAK,kBAAkB,CAAC;QACxB,KAAK,eAAe;YACnB,OAAO,SAAS,CAAC;QAClB;YACC,OAAO,SAAS,CAAC;IACnB,CAAC;AACF,CAAC;AAED;;;GAGG;AACH,SAAS,WAAW,CAAC,WAAmB;IACvC,OAAO,YAAY,CAAC,WAAW,CAAC,IAAI,WAAW,CAAC,OAAO,CAAC,IAAI,EAAE,GAAG,CAAC,CAAC;AACpE,CAAC;AAED,8EAA8E;AAE9E;;;;;;GAMG;AACH,SAAS,kBAAkB,CAAC,OAAe;IAK1C,MAAM,gBAAgB,GAAG,OAAO,CAAC,KAAK,CAAC,oBAAoB,CAAC,CAAC;IAC7D,IAAI,CAAC,gBAAgB,EAAE,CAAC;QACvB,OAAO,EAAE,OAAO,EAAE,eAAe,EAAE,KAAK,EAAE,gBAAgB,EAAE,EAAE,EAAE,CAAC;IAClE,CAAC;IACD,MAAM,KAAK,GAAG,gBAAgB,CAAC,CAAC,CAAE,CAAC;IACnC,MAAM,QAAQ,GAAG,OAAO,CAAC,KAAK,CAAC,gBAAgB,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC;IAC3D,MAAM,EAAE,GAAG,KAAK,CAAC,QAAQ,CAAC,GAAG,CAAC,CAAC;IAC/B,MAAM,WAAW,GAAG,KAAK,CAAC,OAAO,CAAC,IAAI,EAAE,EAAE,CAAC,CAAC;IAC5C,OAAO;QACN,OAAO,EAAE,QAAQ;QACjB,eAAe,EAAE,EAAE;QACnB,gBAAgB,EAAE,WAAW;KAC7B,CAAC;AACH,CAAC;AAED;;;;GAIG;AACH,SAAS,qBAAqB,CAAC,OAAe,EAAE,eAAwB;IACvE,IAAI,CAAC;QACJ,IAAI,MAAM,CAAC,OAAO,EAAE,eAAe,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC;QAChD,OAAO,IAAI,CAAC;IACb,CAAC;IAAC,OAAO,CAAC,EAAE,CAAC;QACZ,OAAO,CAAC,YAAY,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC;IACnD,CAAC;AACF,CAAC;AAED;;;;GAIG;AACH,SAAS,kBAAkB,CAAC,QAA2B;IACtD,IAAI,QAAQ,CAAC,MAAM,KAAK,CAAC;QAAE,OAAO,QAAQ,CAAC,CAAC,CAAE,CAAC;IAC/C,OAAO,QAAQ,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;AAClD,CAAC;AAED,8EAA8E;AAE9E;;;;GAIG;AACH,SAAS,cAAc,CAAC,GAAY;IACnC,IAAI,OAAO,GAAG,CAAC,UAAU,KAAK,QAAQ,EAAE,CAAC;QACxC,oCAAoC;QACpC,MAAM,UAAU,GAAG,GAAG,CAAC,UAAU,GAAG,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,UAAU,GAAG,GAAG,CAAC,CAAC,CAAC,GAAG,CAAC,UAAU,CAAC;QAC9E,OAAO,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,MAAM,CAAC,UAAU,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;IAChE,CAAC;IACD,MAAM,GAAG,GAAG,GAAG,CAAC,IAAI,CAAC,UAAU,CAAC;IAChC,IAAI,GAAG,KAAK,MAAM;QAAE,OAAO,GAAG,CAAC;IAC/B,IAAI,GAAG,KAAK,QAAQ;QAAE,OAAO,IAAI,CAAC;IAClC,IAAI,GAAG,KAAK,KAAK;QAAE,OAAO,GAAG,CAAC;IAC9B,OAAO,GAAG,CAAC;AACZ,CAAC;AAED,8EAA8E;AAE9E;;;;GAIG;AACH,SAAS,UAAU,CAAC,UAAgC;IAInD,MAAM,OAAO,GAAgB,EAAE,CAAC;IAChC,IAAI,IAAI,GAAe,KAAK,CAAC;IAC7B,IAAI,YAAY,GAAG,CAAC,CAAC;IACrB,KAAK,MAAM,CAAC,IAAI,UAAU,EAAE,CAAC;QAC5B,MAAM,MAAM,GAAG,SAAS,CAAC,CAAC,CAAC,CAAC;QAC5B,IAAI,MAAM,KAAK,IAAI,EAAE,CAAC;YACrB,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;YAChB,SAAS;QACV,CAAC;QACD,MAAM,QAAQ,GAAG,eAAe,CAAC,MAAM,CAAC,CAAC;QACzC,IAAI,QAAQ,GAAG,YAAY,EAAE,CAAC;YAC7B,IAAI,GAAG,MAAM,CAAC;YACd,YAAY,GAAG,QAAQ,CAAC;QACzB,CAAC;IACF,CAAC;IACD,OAAO,EAAE,MAAM,EAAE,IAAI,EAAE,cAAc,EAAE,OAAO,EAAE,CAAC;AAClD,CAAC;AAED,6EAA6E;AAE7E;;GAEG;AACH,MAAM,eAAe,GAAqC,MAAM,CAAC,MAAM,CAAC;IACvE,gBAAgB,EAAE,IAAI;IACtB,aAAa,EAAE,KAAK;IACpB,oBAAoB,EAAE,KAAK;IAC3B,kBAAkB,EAAE,KAAK;IACzB,oBAAoB,EAAE,KAAK;IAC3B,kBAAkB,EAAE,KAAK;IACzB,cAAc,EAAE,KAAK;IACrB,WAAW,EAAE,KAAK;IAClB,gBAAgB,EAAE,KAAK;IACvB,cAAc,EAAE,KAAK;IACrB,YAAY,EAAE,KAAK;CACnB,CAAC,CAAC;AAEH;;;;;;;;;GASG;AACH,MAAM,OAAO,eAAe;IACnB,QAAQ,GAAG,IAAI,GAAG,EAAkB,CAAC;IAE7C;;;;OAIG;IACH,YAAY,kBAAoD,EAAE;QACjE,KAAK,MAAM,CAAC,GAAG,EAAE,MAAM,CAAC,IAAI,MAAM,CAAC,OAAO,CAAC,eAAe,CAAC,EAAE,CAAC;YAC7D,IAAI,CAAC,QAAQ,CAAC,GAAG,CAAC,GAAG,EAAE,MAAM,CAAC,CAAC;QAChC,CAAC;IACF,CAAC;IAED,IAAI,CAAC,YAAoB,EAAE,oBAAmC;QAC7D,MAAM,MAAM,GAAG,eAAe,CAAC,YAAY,CAAC,IAAI,KAAK,CAAC;QACtD,MAAM,OAAO,GAAG,CAAC,IAAI,CAAC,QAAQ,CAAC,GAAG,CAAC,YAAY,CAAC,IAAI,CAAC,CAAC,GAAG,CAAC,CAAC;QAC3D,IAAI,CAAC,QAAQ,CAAC,GAAG,CAAC,YAAY,EAAE,OAAO,CAAC,CAAC;QACzC,MAAM,WAAW,GAAG,MAAM,CAAC,OAAO,CAAC,CAAC,QAAQ,CAAC,CAAC,EAAE,GAAG,CAAC,CAAC;QACrD,MAAM,MAAM,GAAG,oBAAoB,CAAC,CAAC,CAAC,IAAI,oBAAoB,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;QACtE,OAAO,OAAO,MAAM,IAAI,WAAW,GAAG,MAAM,EAAE,CAAC;IAChD,CAAC;CACD;AAED,8EAA8E;AAE9E;;;;;;;;;;;GAWG;AACH,SAAS,4BAA4B,CACpC,KAAa,EACb,UAAwC,EACxC,QAA6B;IAE7B,MAAM,MAAM,GAAG,IAAI,GAAG,EAAoC,CAAC;IAC3D,KAAK,MAAM,IAAI,IAAI,UAAU,EAAE,CAAC;QAC/B,IAAI,CAAC,CAAC,OAAO,IAAI,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,KAAK,EAAE,CAAC;YACvC,QAAQ,CAAC,IAAI,CAAC;gBACb,MAAM,EAAE,KAAK;gBACb,IAAI,EAAE,yBAAyB;gBAC/B,MAAM,EAAE,0CAA0C;aAClD,CAAC,CAAC;YACH,SAAS;QACV,CAAC;QACD,IAAI,IAAI,CAAC,QAAQ,KAAK,OAAO,EAAE,CAAC;YAC/B,QAAQ,CAAC,IAAI,CAAC;gBACb,MAAM,EAAE,KAAK;gBACb,IAAI,EAAE,oBAAoB;gBAC1B,MAAM,EAAE,YAAY,IAAI,CAAC,QAAQ,qCAAqC;aACtE,CAAC,CAAC;YACH,SAAS;QACV,CAAC;QACD,IAAI,CAAC,IAAI,CAAC,KAAK;YAAE,SAAS;QAC1B,MAAM,WAAW,GAAG,QAAQ,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;QACzC,MAAM,QAAQ,GAAG,MAAM,CAAC,GAAG,CAAC,WAAW,CAAC,IAAI,EAAE,CAAC;QAC/C,QAAQ,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QACpB,MAAM,CAAC,GAAG,CAAC,WAAW,EAAE,QAAQ,CAAC,CAAC;IACnC,CAAC;IACD,OAAO,MAAM,CAAC;AACf,CAAC;AAED;;;;GAIG;AACH,SAAS,eAAe,CACvB,GAAY,EACZ,WAAwB,EACxB,UAAwC,EACxC,oBAAmC,EACnC,YAAwB,EACxB,WAA4B,EAC5B,QAA6B;IAI7B,MAAM,SAAS,GAAgB,EAAE,CAAC;IAClC,IAAI,kBAAkB,GAAG,KAAK,CAAC;IAC/B,IAAI,gBAAgB,GAAG,KAAK,CAAC;IAE7B,KAAK,MAAM,IAAI,IAAI,UAAU,EAAE,CAAC;QAC/B,MAAM,EAAE,OAAO,EAAE,eAAe,EAAE,gBAAgB,EAAE,GAAG,kBAAkB,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;QACtF,IAAI,gBAAgB,EAAE,CAAC;YACtB,QAAQ,CAAC,IAAI,CAAC;gBACb,MAAM,EAAE,GAAG,CAAC,EAAE;gBACd,IAAI,EAAE,oBAAoB;gBAC1B,MAAM,EAAE,6BAA6B,gBAAgB,yDAAyD;aAC9G,CAAC,CAAC;QACJ,CAAC;QACD,MAAM,YAAY,GAAG,qBAAqB,CAAC,OAAO,EAAE,eAAe,CAAC,CAAC;QACrE,IAAI,YAAY,EAAE,CAAC;YAClB,QAAQ,CAAC,IAAI,CAAC;gBACb,MAAM,EAAE,GAAG,CAAC,EAAE;gBACd,IAAI,EAAE,oBAAoB;gBAC1B,MAAM,EAAE,2CAA2C,YAAY,EAAE;aACjE,CAAC,CAAC;YACH,SAAS;QACV,CAAC;QACD,SAAS,CAAC,IAAI,CAAC,EAAE,KAAK,EAAE,OAAO,EAAE,eAAe,EAAE,CAAC,CAAC;QACpD,IAAI,eAAe,EAAE,CAAC;YACrB,kBAAkB,GAAG,IAAI,CAAC;QAC3B,CAAC;aAAM,CAAC;YACP,gBAAgB,GAAG,IAAI,CAAC;QACzB,CAAC;IACF,CAAC;IAED,IAAI,SAAS,CAAC,MAAM,KAAK,CAAC;QAAE,OAAO,IAAI,CAAC;IAExC,0EAA0E;IAC1E,0EAA0E;IAC1E,yEAAyE;IACzE,0EAA0E;IAC1E,uEAAuE;IACvE,IAAI,kBAAkB,IAAI,gBAAgB,EAAE,CAAC;QAC5C,QAAQ,CAAC,IAAI,CAAC;YACb,MAAM,EAAE,GAAG,CAAC,EAAE;YACd,IAAI,EAAE,oBAAoB;YAC1B,MAAM,EACL,6GAA6G;SAC9G,CAAC,CAAC;IACJ,CAAC;IACD,MAAM,mBAAmB,GAAG,kBAAkB,CAAC;IAE/C,MAAM,aAAa,GAAG,kBAAkB,CAAC,SAAS,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC;IAExE,2EAA2E;IAC3E,4BAA4B;IAC5B,MAAM,YAAY,GAAG,qBAAqB,CAAC,aAAa,EAAE,mBAAmB,CAAC,CAAC;IAC/E,IAAI,YAAY,EAAE,CAAC;QAClB,QAAQ,CAAC,IAAI,CAAC;YACb,MAAM,EAAE,GAAG,CAAC,EAAE;YACd,IAAI,EAAE,oBAAoB;YAC1B,MAAM,EAAE,kDAAkD,YAAY,EAAE;SACxE,CAAC,CAAC;QACH,OAAO,IAAI,CAAC;IACb,CAAC;IAED,MAAM,YAAY,GAAG,WAAW,CAAC,GAAG,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;IACpD,MAAM,MAAM,GAAG,WAAW,CAAC,IAAI,CAAC,YAAY,EAAE,oBAAoB,CAAC,CAAC;IACpE,MAAM,OAAO,GAAG,WAAW,CAAC;IAC5B,MAAM,KAAK,GAAG,GAAG,CAAC,KAAK,CAAC,MAAM,GAAG,GAAG,CAAC,CAAC,CAAC,GAAG,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC,EAAE,EAAE,CAAC,GAAG,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,KAAK,CAAC;IAElF,MAAM,WAAW,GAAG,2EAA2E,GAAG,CAAC,IAAI,CAAC,QAAQ,IAAI,GAAG,CAAC,EAAE,OAAO,CAAC;IAElI,yEAAyE;IACzE,qEAAqE;IACrE,iEAAiE;IACjE,MAAM,oBAAoB,GAAG,cAAc,CAAC,GAAG,CAAC,CAAC;IACjD,MAAM,gBAAgB,GACrB,YAAY,KAAK,OAAO,IAAI,oBAAoB,GAAG,IAAI;QACtD,CAAC,CAAC,kBAAkB;QACpB,CAAC,CAAC,YAAY,CAAC;IAEjB,OAAO;QACN,EAAE,EAAE,MAAM;QACV,QAAQ,EAAE,YAAY;QACtB,QAAQ,EAAE,YAAY,CAAC,GAAG,CAAC,QAAQ,CAAC;QACpC,UAAU,EAAE,oBAAoB;QAChC,MAAM,EAAE,gBAAgB;QACxB,OAAO,EAAE,aAAa;QACtB,QAAQ,EAAE,OAAO;QACjB,KAAK;QACL,UAAU,EAAE,IAAI;QAChB,OAAO,EAAE,GAAG,CAAC,MAAM,KAAK,OAAO;QAC/B,gBAAgB,EAAE,mBAAmB;QACrC,QAAQ,EAAE,GAAG,CAAC,EAAE;QAChB,YAAY,EAAE,WAAW;QACzB,gBAAgB,EAAE,KAAK;QACvB,OAAO,EAAE,qBAAqB,CAAC,GAAG,CAAC,WAAW,CAAC;KAC/C,CAAC;AACH,CAAC;AAED;;;;GAIG;AACH,SAAS,qBAAqB,CAAC,WAA+B;IAC7D,IAAI,CAAC,WAAW;QAAE,OAAO,EAAE,CAAC;IAC5B,qEAAqE;IACrE,MAAM,aAAa,GAAG,WAAW;SAC/B,OAAO,CAAC,MAAM,EAAE,GAAG,CAAC;SACpB,IAAI,EAAE;SACN,KAAK,CAAC,6BAA6B,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC;IAC5C,IAAI,aAAa;QAAE,OAAO,aAAa,CAAC;IACxC,4BAA4B;IAC5B,MAAM,OAAO,GAAG,WAAW,CAAC,OAAO,CAAC,MAAM,EAAE,GAAG,CAAC,CAAC,IAAI,EAAE,CAAC,KAAK,CAAC,CAAC,EAAE,GAAG,CAAC,CAAC;IACtE,OAAO,OAAO,CAAC,MAAM,KAAK,GAAG,CAAC,CAAC,CAAC,OAAO,GAAG,KAAK,CAAC,CAAC,CAAC,OAAO,CAAC;AAC3D,CAAC;AAED;;;;;;;;;;;;;;GAcG;AACH,MAAM,UAAU,SAAS,CACxB,GAAY,EACZ,cAA+B,IAAI,eAAe,EAAE;IAEpD,MAAM,QAAQ,GAAwB,EAAE,CAAC;IAEzC,yDAAyD;IACzD,IAAI,GAAG,CAAC,cAAc,KAAK,UAAU,EAAE,CAAC;QACvC,QAAQ,CAAC,IAAI,CAAC;YACb,MAAM,EAAE,GAAG,CAAC,EAAE;YACd,IAAI,EAAE,uBAAuB;YAC7B,MAAM,EAAE,2EAA2E;SACnF,CAAC,CAAC;QACH,OAAO,EAAE,KAAK,EAAE,EAAE,EAAE,QAAQ,EAAE,CAAC;IAChC,CAAC;IAED,wBAAwB;IACxB,IAAI,GAAG,CAAC,MAAM,KAAK,YAAY,EAAE,CAAC;QACjC,QAAQ,CAAC,IAAI,CAAC;YACb,MAAM,EAAE,GAAG,CAAC,EAAE;YACd,IAAI,EAAE,oBAAoB;YAC1B,MAAM,EAAE,gCAAgC,GAAG,CAAC,WAAW,IAAI,MAAM,GAAG;SACpE,CAAC,CAAC;QACH,OAAO,EAAE,KAAK,EAAE,EAAE,EAAE,QAAQ,EAAE,CAAC;IAChC,CAAC;IAED,uDAAuD;IACvD,MAAM,EAAE,MAAM,EAAE,YAAY,EAAE,cAAc,EAAE,GAAG,UAAU,CAAC,GAAG,CAAC,QAAQ,CAAC,OAAO,CAAC,CAAC;IAClF,KAAK,MAAM,OAAO,IAAI,cAAc,EAAE,CAAC;QACtC,QAAQ,CAAC,IAAI,CAAC;YACb,MAAM,EAAE,GAAG,CAAC,EAAE;YACd,IAAI,EAAE,4BAA4B;YAClC,MAAM,EAAE,UAAU,OAAO,yBAAyB;SAClD,CAAC,CAAC;IACJ,CAAC;IAED,qBAAqB;IACrB,MAAM,aAAa,GAAG,GAAG,CAAC,SAAS,CAAC,UAAU,CAAC;IAC/C,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,aAAa,CAAC,EAAE,CAAC;QACnC,+EAA+E;QAC/E,QAAQ,CAAC,IAAI,CAAC;YACb,MAAM,EAAE,GAAG,CAAC,EAAE;YACd,IAAI,EAAE,2BAA2B;YACjC,MAAM,EAAE,8CAA8C;SACtD,CAAC,CAAC;QACH,OAAO,EAAE,KAAK,EAAE,EAAE,EAAE,QAAQ,EAAE,CAAC;IAChC,CAAC;IAED,MAAM,MAAM,GAAG,4BAA4B,CAAC,GAAG,CAAC,EAAE,EAAE,aAAa,EAAE,QAAQ,CAAC,CAAC;IAC7E,IAAI,MAAM,CAAC,IAAI,KAAK,CAAC,EAAE,CAAC;QACvB,QAAQ,CAAC,IAAI,CAAC;YACb,MAAM,EAAE,GAAG,CAAC,EAAE;YACd,IAAI,EAAE,2BAA2B;YACjC,MAAM,EAAE,+CAA+C;SACvD,CAAC,CAAC;QACH,OAAO,EAAE,KAAK,EAAE,EAAE,EAAE,QAAQ,EAAE,CAAC;IAChC,CAAC;IAED,0EAA0E;IAC1E,2EAA2E;IAC3E,8BAA8B;IAC9B,MAAM,cAAc,GAAG,KAAK,CAAC,IAAI,CAAC,MAAM,CAAC,OAAO,EAAE,CAAC,CAAC;IACpD,MAAM,cAAc,GAAG,cAAc,CAAC,MAAM,GAAG,CAAC,CAAC;IACjD,MAAM,SAAS,GAAe,EAAE,CAAC;IACjC,MAAM,wBAAwB,GAAG,GAAG,CAAC;IACrC,KAAK,MAAM,CAAC,WAAW,EAAE,UAAU,CAAC,IAAI,cAAc,EAAE,CAAC;QACxD,MAAM,aAAa,GAAG,cAAc,CAAC,CAAC,CAAC,WAAW,CAAC,WAAW,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC;QAEvE,iDAAiD;QACjD,MAAM,iBAAiB,GAAG,UAAU;aAClC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,kBAAkB,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,OAAO,CAAC;aAC/C,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC;QAC9B,MAAM,iBAAiB,GAAG,kBAAkB,CAAC,iBAAiB,CAAC,CAAC;QAEhE,IAAI,iBAAiB,CAAC,MAAM,IAAI,wBAAwB,IAAI,UAAU,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YACrF,mDAAmD;YACnD,MAAM,IAAI,GAAG,eAAe,CAC3B,GAAG,EACH,WAAW,EACX,UAAU,EACV,aAAa,EACb,YAAY,EACZ,WAAW,EACX,QAAQ,CACR,CAAC;YACF,IAAI,IAAI;gBAAE,SAAS,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QAChC,CAAC;aAAM,CAAC;YACP,6DAA6D;YAC7D,kEAAkE;YAClE,6DAA6D;YAC7D,mCAAmC;YACnC,QAAQ,CAAC,IAAI,CAAC;gBACb,MAAM,EAAE,GAAG,CAAC,EAAE;gBACd,IAAI,EAAE,iBAAiB;gBACvB,MAAM,EAAE,2BAA2B,iBAAiB,CAAC,MAAM,YAAY,wBAAwB,qBAAqB,UAAU,CAAC,MAAM,kCAAkC;aACvK,CAAC,CAAC;YACH,MAAM,oBAAoB,GAAG,WAAW,CAAC,GAAG,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;YAC5D,MAAM,MAAM,GAAG,WAAW,CAAC,IAAI,CAAC,oBAAoB,EAAE,aAAa,CAAC,CAAC;YACrE,UAAU,CAAC,OAAO,CAAC,CAAC,IAAI,EAAE,CAAC,EAAE,EAAE;gBAC9B,MAAM,YAAY,GAAG,eAAe,CAAC,CAAC,CAAC,CAAC;gBACxC,MAAM,OAAO,GAAG,iBAAiB,CAChC,GAAG,EACH,WAAW,EACX,IAAI,EACJ,GAAG,MAAM,GAAG,YAAY,EAAE,EAC1B,YAAY,EACZ,QAAQ,CACR,CAAC;gBACF,IAAI,OAAO;oBAAE,SAAS,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;YACtC,CAAC,CAAC,CAAC;QACJ,CAAC;IACF,CAAC;IAED,IAAI,GAAG,CAAC,MAAM,KAAK,OAAO,EAAE,CAAC;QAC5B,QAAQ,CAAC,IAAI,CAAC;YACb,MAAM,EAAE,GAAG,CAAC,EAAE;YACd,IAAI,EAAE,sBAAsB;YAC5B,MAAM,EAAE,gEAAgE;SACxE,CAAC,CAAC;IACJ,CAAC;IAED,OAAO,EAAE,KAAK,EAAE,SAAS,EAAE,QAAQ,EAAE,CAAC;AACvC,CAAC;AAED;;;;;GAKG;AACH,SAAS,iBAAiB,CACzB,GAAY,EACZ,WAAwB,EACxB,SAA4B,EAC5B,EAAU,EACV,YAAwB,EACxB,QAA6B;IAE7B,MAAM,EAAE,OAAO,EAAE,eAAe,EAAE,gBAAgB,EAAE,GAAG,kBAAkB,CAAC,SAAS,CAAC,KAAK,CAAC,CAAC;IAC3F,IAAI,gBAAgB,EAAE,CAAC;QACtB,QAAQ,CAAC,IAAI,CAAC;YACb,MAAM,EAAE,GAAG,CAAC,EAAE;YACd,IAAI,EAAE,oBAAoB;YAC1B,MAAM,EAAE,6BAA6B,gBAAgB,YAAY;SACjE,CAAC,CAAC;IACJ,CAAC;IACD,MAAM,YAAY,GAAG,qBAAqB,CAAC,OAAO,EAAE,eAAe,CAAC,CAAC;IACrE,IAAI,YAAY,EAAE,CAAC;QAClB,QAAQ,CAAC,IAAI,CAAC;YACb,MAAM,EAAE,GAAG,CAAC,EAAE;YACd,IAAI,EAAE,oBAAoB;YAC1B,MAAM,EAAE,uCAAuC,YAAY,EAAE;SAC7D,CAAC,CAAC;QACH,OAAO,IAAI,CAAC;IACb,CAAC;IAED,MAAM,YAAY,GAAG,WAAW,CAAC,GAAG,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;IACpD,MAAM,KAAK,GAAG,GAAG,CAAC,KAAK,CAAC,MAAM,GAAG,GAAG,CAAC,CAAC,CAAC,GAAG,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC,EAAE,EAAE,CAAC,GAAG,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,KAAK,CAAC;IAClF,MAAM,WAAW,GAAG,2EAA2E,GAAG,CAAC,IAAI,CAAC,QAAQ,IAAI,GAAG,CAAC,EAAE,OAAO,CAAC;IAClI,MAAM,oBAAoB,GAAG,cAAc,CAAC,GAAG,CAAC,CAAC;IACjD,MAAM,gBAAgB,GACrB,YAAY,KAAK,OAAO,IAAI,oBAAoB,GAAG,IAAI;QACtD,CAAC,CAAC,kBAAkB;QACpB,CAAC,CAAC,YAAY,CAAC;IAEjB,wEAAwE;IACxE,0EAA0E;IAC1E,yEAAyE;IACzE,MAAM,cAAc,GACnB,CAAC,SAAS,CAAC,WAAW,IAAI,SAAS,CAAC,WAAW,CAAC,MAAM,GAAG,EAAE;QAC1D,CAAC,CAAC,SAAS,CAAC,WAAW;QACvB,CAAC,CAAC,qBAAqB,CAAC,GAAG,CAAC,WAAW,CAAC,CAAC,IAAI,EAAE,CAAC;IAElD,OAAO;QACN,EAAE;QACF,QAAQ,EAAE,YAAY;QACtB,QAAQ,EAAE,YAAY,CAAC,GAAG,CAAC,QAAQ,CAAC;QACpC,UAAU,EAAE,oBAAoB;QAChC,MAAM,EAAE,gBAAgB;QACxB,OAAO;QACP,QAAQ,EAAE,WAAW;QACrB,KAAK;QACL,UAAU,EAAE,IAAI;QAChB,OAAO,EAAE,GAAG,CAAC,MAAM,KAAK,OAAO;QAC/B,gBAAgB,EAAE,eAAe;QACjC,QAAQ,EAAE,GAAG,CAAC,EAAE;QAChB,YAAY,EAAE,WAAW;QACzB,gBAAgB,EAAE,KAAK;QACvB,OAAO,EAAE,cAAc;KACvB,CAAC;AACH,CAAC;AAED;;;;GAIG;AACH,SAAS,eAAe,CAAC,CAAS;IACjC,IAAI,CAAC,GAAG,EAAE;QAAE,OAAO,MAAM,CAAC,YAAY,CAAC,EAAE,GAAG,CAAC,CAAC,CAAC,CAAC,OAAO;IACvD,wCAAwC;IACxC,MAAM,KAAK,GAAG,IAAI,CAAC,KAAK,CAAC,CAAC,GAAG,EAAE,CAAC,GAAG,CAAC,CAAC;IACrC,MAAM,MAAM,GAAG,CAAC,GAAG,EAAE,CAAC;IACtB,OAAO,MAAM,CAAC,YAAY,CAAC,EAAE,GAAG,KAAK,CAAC,GAAG,MAAM,CAAC,YAAY,CAAC,EAAE,GAAG,MAAM,CAAC,CAAC;AAC3E,CAAC;AAED;;;;GAIG;AACH,SAAS,WAAW,CAAC,OAAoB;IACxC,QAAQ,OAAO,EAAE,CAAC;QACjB,KAAK,SAAS;YACb,OAAO,IAAI,CAAC;QACb,KAAK,KAAK;YACT,OAAO,KAAK,CAAC;QACd,KAAK,SAAS;YACb,OAAO,KAAK,CAAC;QACd,KAAK,WAAW;YACf,OAAO,IAAI,CAAC;QACb,KAAK,QAAQ;YACZ,OAAO,KAAK,CAAC;IACf,CAAC;AACF,CAAC;AAED;;;;;;;;;;GAUG;AACH,MAAM,UAAU,cAAc,CAC7B,QAA4B,EAC5B,kBAAoD,EAAE;IAEtD,MAAM,SAAS,GAAG,IAAI,eAAe,CAAC,eAAe,CAAC,CAAC;IACvD,MAAM,KAAK,GAAe,EAAE,CAAC;IAC7B,MAAM,QAAQ,GAAwB,EAAE,CAAC;IACzC,KAAK,MAAM,GAAG,IAAI,QAAQ,EAAE,CAAC;QAC5B,MAAM,MAAM,GAAG,SAAS,CAAC,GAAG,EAAE,SAAS,CAAC,CAAC;QACzC,KAAK,CAAC,IAAI,CAAC,GAAG,MAAM,CAAC,KAAK,CAAC,CAAC;QAC5B,QAAQ,CAAC,IAAI,CAAC,GAAG,MAAM,CAAC,QAAQ,CAAC,CAAC;IACnC,CAAC;IACD,OAAO,EAAE,KAAK,EAAE,QAAQ,EAAE,CAAC;AAC5B,CAAC;AAED,8EAA8E;AAE9E;;;;;;;;GAQG;AACH,MAAM,UAAU,eAAe,CAAC,KAA0B;IACzD,OAAO,KAAK,CAAC,GAAG,CAAC,aAAa,CAAC,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;AAC9C,CAAC;AAED,SAAS,aAAa,CAAC,IAAc;IACpC,MAAM,KAAK,GAAa,EAAE,CAAC;IAC3B,uEAAuE;IACvE,8BAA8B;IAC9B,IAAI,IAAI,CAAC,OAAO,EAAE,CAAC;QAClB,KAAK,CAAC,IAAI,CAAC,gBAAgB,IAAI,CAAC,OAAO,EAAE,CAAC,CAAC;IAC5C,CAAC;IACD,KAAK,CAAC,IAAI,CAAC,UAAU,IAAI,CAAC,EAAE,GAAG,CAAC,CAAC;IACjC,KAAK,CAAC,IAAI,CAAC,eAAe,IAAI,CAAC,QAAQ,EAAE,CAAC,CAAC;IAC3C,KAAK,CAAC,IAAI,CAAC,eAAe,IAAI,CAAC,QAAQ,EAAE,CAAC,CAAC;IAC3C,KAAK,CAAC,IAAI,CAAC,iBAAiB,IAAI,CAAC,UAAU,EAAE,CAAC,CAAC;IAC/C,KAAK,CAAC,IAAI,CAAC,aAAa,IAAI,CAAC,MAAM,EAAE,CAAC,CAAC;IACvC,KAAK,CAAC,IAAI,CAAC,cAAc,UAAU,CAAC,IAAI,CAAC,OAAO,CAAC,EAAE,CAAC,CAAC;IACrD,IAAI,KAAK,CAAC,OAAO,CAAC,IAAI,CAAC,QAAQ,CAAC,EAAE,CAAC;QAClC,KAAK,CAAC,IAAI,CAAC,gBAAgB,IAAI,CAAC,QAAQ,CAAC,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;IACzD,CAAC;SAAM,CAAC;QACP,KAAK,CAAC,IAAI,CAAC,eAAe,IAAI,CAAC,QAAQ,EAAE,CAAC,CAAC;IAC5C,CAAC;IACD,IAAI,IAAI,CAAC,gBAAgB,EAAE,CAAC;QAC3B,KAAK,CAAC,IAAI,CAAC,0BAA0B,CAAC,CAAC;IACxC,CAAC;IACD,KAAK,CAAC,IAAI,CAAC,YAAY,UAAU,CAAC,IAAI,CAAC,KAAK,CAAC,EAAE,CAAC,CAAC;IACjD,KAAK,CAAC,IAAI,CAAC,oBAAoB,CAAC,CAAC;IACjC,KAAK,CAAC,IAAI,CAAC,cAAc,IAAI,CAAC,OAAO,EAAE,CAAC,CAAC;IACzC,IAAI,IAAI,CAAC,QAAQ,EAAE,CAAC;QACnB,MAAM,OAAO,GAAG,IAAI,CAAC,gBAAgB,IAAI,KAAK,CAAC;QAC/C,MAAM,GAAG,GAAG,IAAI,CAAC,YAAY,IAAI,EAAE,CAAC;QACpC,KAAK,CAAC,IAAI,CAAC,iBAAiB,IAAI,CAAC,QAAQ,KAAK,OAAO,KAAK,GAAG,CAAC,CAAC,CAAC,IAAI,GAAG,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,OAAO,EAAE,CAAC,CAAC;IAC9F,CAAC;IACD,OAAO,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;AACzB,CAAC;AAED;;;;;GAKG;AACH,SAAS,UAAU,CAAC,CAAS;IAC5B,0EAA0E;IAC1E,yEAAyE;IACzE,qEAAqE;IACrE,mEAAmE;IACnE,OAAO,IAAI,CAAC,SAAS,CAAC,CAAC,CAAC,CAAC;AAC1B,CAAC"}
|
|
@@ -0,0 +1,24 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* HackAPrompt Corpus Loader
|
|
3
|
+
*
|
|
4
|
+
* Reads the HackAPrompt-format sample JSON (text/category/label/source/language)
|
|
5
|
+
* produced by scripts/hackaprompt-to-corpus.py and converts each row into the
|
|
6
|
+
* CorpusSample shape used by the ATR eval harness.
|
|
7
|
+
*
|
|
8
|
+
* HackAPrompt is an all-adversarial corpus: every sample is an attempt to
|
|
9
|
+
* subvert the system prompt. We therefore only measure recall against this
|
|
10
|
+
* dataset; precision/FP rate is undefined here. For combined precision+recall
|
|
11
|
+
* use this corpus alongside a benign source (PINT, real-traffic).
|
|
12
|
+
*
|
|
13
|
+
* @module agent-threat-rules/eval/hackaprompt-corpus
|
|
14
|
+
*/
|
|
15
|
+
import type { CorpusSample } from './corpus.js';
|
|
16
|
+
export declare function loadHackaPromptCorpus(dataPath: string): readonly CorpusSample[];
|
|
17
|
+
export declare function getHackaPromptCorpusStats(corpus: readonly CorpusSample[]): {
|
|
18
|
+
total: number;
|
|
19
|
+
attacks: number;
|
|
20
|
+
benign: number;
|
|
21
|
+
byCategory: Record<string, number>;
|
|
22
|
+
byDifficulty: Record<string, number>;
|
|
23
|
+
};
|
|
24
|
+
//# sourceMappingURL=hackaprompt-corpus.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"hackaprompt-corpus.d.ts","sourceRoot":"","sources":["../../src/eval/hackaprompt-corpus.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;GAaG;AAGH,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,aAAa,CAAC;AAsBhD,wBAAgB,qBAAqB,CAAC,QAAQ,EAAE,MAAM,GAAG,SAAS,YAAY,EAAE,CAsB/E;AAED,wBAAgB,yBAAyB,CAAC,MAAM,EAAE,SAAS,YAAY,EAAE;;;;gBAKnD,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC;kBACpB,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC;EAS7C"}
|
|
@@ -0,0 +1,61 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* HackAPrompt Corpus Loader
|
|
3
|
+
*
|
|
4
|
+
* Reads the HackAPrompt-format sample JSON (text/category/label/source/language)
|
|
5
|
+
* produced by scripts/hackaprompt-to-corpus.py and converts each row into the
|
|
6
|
+
* CorpusSample shape used by the ATR eval harness.
|
|
7
|
+
*
|
|
8
|
+
* HackAPrompt is an all-adversarial corpus: every sample is an attempt to
|
|
9
|
+
* subvert the system prompt. We therefore only measure recall against this
|
|
10
|
+
* dataset; precision/FP rate is undefined here. For combined precision+recall
|
|
11
|
+
* use this corpus alongside a benign source (PINT, real-traffic).
|
|
12
|
+
*
|
|
13
|
+
* @module agent-threat-rules/eval/hackaprompt-corpus
|
|
14
|
+
*/
|
|
15
|
+
import { readFileSync } from 'node:fs';
|
|
16
|
+
function assignDifficulty(level) {
|
|
17
|
+
if (level <= 2)
|
|
18
|
+
return 'easy';
|
|
19
|
+
if (level <= 6)
|
|
20
|
+
return 'medium';
|
|
21
|
+
return 'hard';
|
|
22
|
+
}
|
|
23
|
+
export function loadHackaPromptCorpus(dataPath) {
|
|
24
|
+
const raw = JSON.parse(readFileSync(dataPath, 'utf-8'));
|
|
25
|
+
return raw.map((sample) => {
|
|
26
|
+
const level = sample.metadata?.level ?? 5;
|
|
27
|
+
return {
|
|
28
|
+
id: sample.id,
|
|
29
|
+
text: sample.text,
|
|
30
|
+
category: sample.category,
|
|
31
|
+
expectedDetection: sample.label,
|
|
32
|
+
eventType: 'llm_input',
|
|
33
|
+
tier: 'any',
|
|
34
|
+
difficulty: assignDifficulty(level),
|
|
35
|
+
fields: {
|
|
36
|
+
text: sample.text,
|
|
37
|
+
prompt: sample.text,
|
|
38
|
+
user_input: sample.text,
|
|
39
|
+
},
|
|
40
|
+
};
|
|
41
|
+
});
|
|
42
|
+
}
|
|
43
|
+
export function getHackaPromptCorpusStats(corpus) {
|
|
44
|
+
const stats = {
|
|
45
|
+
total: corpus.length,
|
|
46
|
+
attacks: 0,
|
|
47
|
+
benign: 0,
|
|
48
|
+
byCategory: {},
|
|
49
|
+
byDifficulty: {},
|
|
50
|
+
};
|
|
51
|
+
for (const s of corpus) {
|
|
52
|
+
if (s.expectedDetection)
|
|
53
|
+
stats.attacks++;
|
|
54
|
+
else
|
|
55
|
+
stats.benign++;
|
|
56
|
+
stats.byCategory[s.category] = (stats.byCategory[s.category] ?? 0) + 1;
|
|
57
|
+
stats.byDifficulty[s.difficulty] = (stats.byDifficulty[s.difficulty] ?? 0) + 1;
|
|
58
|
+
}
|
|
59
|
+
return stats;
|
|
60
|
+
}
|
|
61
|
+
//# sourceMappingURL=hackaprompt-corpus.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"hackaprompt-corpus.js","sourceRoot":"","sources":["../../src/eval/hackaprompt-corpus.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;GAaG;AAEH,OAAO,EAAE,YAAY,EAAE,MAAM,SAAS,CAAC;AAiBvC,SAAS,gBAAgB,CAAC,KAAa;IACrC,IAAI,KAAK,IAAI,CAAC;QAAE,OAAO,MAAM,CAAC;IAC9B,IAAI,KAAK,IAAI,CAAC;QAAE,OAAO,QAAQ,CAAC;IAChC,OAAO,MAAM,CAAC;AAChB,CAAC;AAED,MAAM,UAAU,qBAAqB,CAAC,QAAgB;IACpD,MAAM,GAAG,GAAoC,IAAI,CAAC,KAAK,CACrD,YAAY,CAAC,QAAQ,EAAE,OAAO,CAAC,CAChC,CAAC;IAEF,OAAO,GAAG,CAAC,GAAG,CAAC,CAAC,MAAM,EAAgB,EAAE;QACtC,MAAM,KAAK,GAAG,MAAM,CAAC,QAAQ,EAAE,KAAK,IAAI,CAAC,CAAC;QAC1C,OAAO;YACL,EAAE,EAAE,MAAM,CAAC,EAAE;YACb,IAAI,EAAE,MAAM,CAAC,IAAI;YACjB,QAAQ,EAAE,MAAM,CAAC,QAAQ;YACzB,iBAAiB,EAAE,MAAM,CAAC,KAAK;YAC/B,SAAS,EAAE,WAAW;YACtB,IAAI,EAAE,KAAK;YACX,UAAU,EAAE,gBAAgB,CAAC,KAAK,CAAC;YACnC,MAAM,EAAE;gBACN,IAAI,EAAE,MAAM,CAAC,IAAI;gBACjB,MAAM,EAAE,MAAM,CAAC,IAAI;gBACnB,UAAU,EAAE,MAAM,CAAC,IAAI;aACxB;SACF,CAAC;IACJ,CAAC,CAAC,CAAC;AACL,CAAC;AAED,MAAM,UAAU,yBAAyB,CAAC,MAA+B;IACvE,MAAM,KAAK,GAAG;QACZ,KAAK,EAAE,MAAM,CAAC,MAAM;QACpB,OAAO,EAAE,CAAC;QACV,MAAM,EAAE,CAAC;QACT,UAAU,EAAE,EAA4B;QACxC,YAAY,EAAE,EAA4B;KAC3C,CAAC;IACF,KAAK,MAAM,CAAC,IAAI,MAAM,EAAE,CAAC;QACvB,IAAI,CAAC,CAAC,iBAAiB;YAAE,KAAK,CAAC,OAAO,EAAE,CAAC;;YACpC,KAAK,CAAC,MAAM,EAAE,CAAC;QACpB,KAAK,CAAC,UAAU,CAAC,CAAC,CAAC,QAAQ,CAAC,GAAG,CAAC,KAAK,CAAC,UAAU,CAAC,CAAC,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAC,GAAG,CAAC,CAAC;QACvE,KAAK,CAAC,YAAY,CAAC,CAAC,CAAC,UAAU,CAAC,GAAG,CAAC,KAAK,CAAC,YAAY,CAAC,CAAC,CAAC,UAAU,CAAC,IAAI,CAAC,CAAC,GAAG,CAAC,CAAC;IACjF,CAAC;IACD,OAAO,KAAK,CAAC;AACf,CAAC"}
|
|
@@ -0,0 +1,19 @@
|
|
|
1
|
+
#!/usr/bin/env npx tsx
|
|
2
|
+
/**
|
|
3
|
+
* HackAPrompt Benchmark Runner
|
|
4
|
+
*
|
|
5
|
+
* Loads a sample from the HackAPrompt 600K+ adversarial prompt dataset
|
|
6
|
+
* (produced by scripts/hackaprompt-to-corpus.py) and runs it through the
|
|
7
|
+
* ATR evaluation harness to measure recall against real prompt-hacking
|
|
8
|
+
* attempts collected at competition scale.
|
|
9
|
+
*
|
|
10
|
+
* Usage:
|
|
11
|
+
* npx tsx src/eval/run-hackaprompt-benchmark.ts
|
|
12
|
+
*
|
|
13
|
+
* HackAPrompt is an all-adversarial corpus, so we measure recall, latency,
|
|
14
|
+
* and tier breakdown. Precision/FP rate require a benign companion source.
|
|
15
|
+
*
|
|
16
|
+
* @module agent-threat-rules/eval/run-hackaprompt-benchmark
|
|
17
|
+
*/
|
|
18
|
+
export {};
|
|
19
|
+
//# sourceMappingURL=run-hackaprompt-benchmark.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"run-hackaprompt-benchmark.d.ts","sourceRoot":"","sources":["../../src/eval/run-hackaprompt-benchmark.ts"],"names":[],"mappings":";AACA;;;;;;;;;;;;;;;GAeG"}
|
|
@@ -0,0 +1,86 @@
|
|
|
1
|
+
#!/usr/bin/env npx tsx
|
|
2
|
+
/**
|
|
3
|
+
* HackAPrompt Benchmark Runner
|
|
4
|
+
*
|
|
5
|
+
* Loads a sample from the HackAPrompt 600K+ adversarial prompt dataset
|
|
6
|
+
* (produced by scripts/hackaprompt-to-corpus.py) and runs it through the
|
|
7
|
+
* ATR evaluation harness to measure recall against real prompt-hacking
|
|
8
|
+
* attempts collected at competition scale.
|
|
9
|
+
*
|
|
10
|
+
* Usage:
|
|
11
|
+
* npx tsx src/eval/run-hackaprompt-benchmark.ts
|
|
12
|
+
*
|
|
13
|
+
* HackAPrompt is an all-adversarial corpus, so we measure recall, latency,
|
|
14
|
+
* and tier breakdown. Precision/FP rate require a benign companion source.
|
|
15
|
+
*
|
|
16
|
+
* @module agent-threat-rules/eval/run-hackaprompt-benchmark
|
|
17
|
+
*/
|
|
18
|
+
import { resolve, join } from 'node:path';
|
|
19
|
+
import { loadHackaPromptCorpus, getHackaPromptCorpusStats } from './hackaprompt-corpus.js';
|
|
20
|
+
import { runEval } from './eval-harness.js';
|
|
21
|
+
function formatPercent(n) {
|
|
22
|
+
return `${(n * 100).toFixed(1)}%`;
|
|
23
|
+
}
|
|
24
|
+
function formatMs(n) {
|
|
25
|
+
return `${n.toFixed(2)}ms`;
|
|
26
|
+
}
|
|
27
|
+
async function main() {
|
|
28
|
+
const base = resolve(join(import.meta.dirname ?? '.', '..', '..'));
|
|
29
|
+
const rulesDir = join(base, 'rules');
|
|
30
|
+
const dataPath = join(base, 'data', 'hackaprompt', 'hackaprompt-corpus.json');
|
|
31
|
+
const outputPath = join(base, 'data', 'hackaprompt', 'hackaprompt-eval-report.json');
|
|
32
|
+
console.log('\n=== HackAPrompt Benchmark -- ATR Evaluation ===\n');
|
|
33
|
+
console.log(`Corpus: ${dataPath}`);
|
|
34
|
+
console.log(`Rules: ${rulesDir}\n`);
|
|
35
|
+
const corpus = loadHackaPromptCorpus(dataPath);
|
|
36
|
+
const stats = getHackaPromptCorpusStats(corpus);
|
|
37
|
+
console.log(`Loaded ${stats.total} samples (${stats.attacks} attacks, ${stats.benign} benign)`);
|
|
38
|
+
console.log(`Categories: ${Object.entries(stats.byCategory).map(([k, v]) => `${k}=${v}`).join(', ')}`);
|
|
39
|
+
console.log(`Difficulty: ${Object.entries(stats.byDifficulty).map(([k, v]) => `${k}=${v}`).join(', ')}`);
|
|
40
|
+
// HackAPrompt is an all-adversarial corpus from a public global competition.
|
|
41
|
+
// It contains heavy paraphrasing, role-play, multilingual, and creative
|
|
42
|
+
// attacks. Recall is the headline number. FP rate is meaningless here
|
|
43
|
+
// because there are no benign samples in the corpus.
|
|
44
|
+
const hackaPromptThresholds = {
|
|
45
|
+
minRecall: 0.10,
|
|
46
|
+
maxFpRate: 1.0,
|
|
47
|
+
minF1: 0.0,
|
|
48
|
+
maxP95LatencyMs: 200,
|
|
49
|
+
};
|
|
50
|
+
const { report, tiersUsed, ruleQuality } = await runEval({
|
|
51
|
+
rulesDir,
|
|
52
|
+
corpus,
|
|
53
|
+
thresholds: hackaPromptThresholds,
|
|
54
|
+
outputPath,
|
|
55
|
+
});
|
|
56
|
+
console.log(`\nTiers: ${tiersUsed.join(' + ')}`);
|
|
57
|
+
console.log(`\n--- Overall ---`);
|
|
58
|
+
console.log(` Recall: ${formatPercent(report.overall.recall)}`);
|
|
59
|
+
console.log(` Precision: ${formatPercent(report.overall.precision)} (N/A meaning - no benign samples)`);
|
|
60
|
+
console.log(` Confusion: TP=${report.overall.confusion.tp} FN=${report.overall.confusion.fn}`);
|
|
61
|
+
console.log(`\n--- Latency ---`);
|
|
62
|
+
console.log(` P50: ${formatMs(report.latency.p50)}`);
|
|
63
|
+
console.log(` P95: ${formatMs(report.latency.p95)}`);
|
|
64
|
+
console.log(` P99: ${formatMs(report.latency.p99)}`);
|
|
65
|
+
console.log(` Mean: ${formatMs(report.latency.mean)}`);
|
|
66
|
+
console.log(`\n--- By Category ---`);
|
|
67
|
+
for (const cat of report.byCategory) {
|
|
68
|
+
const m = cat.metrics;
|
|
69
|
+
const tp = m.confusion.tp;
|
|
70
|
+
const fn = m.confusion.fn;
|
|
71
|
+
const recall = tp + fn === 0 ? 0 : tp / (tp + fn);
|
|
72
|
+
console.log(` ${cat.category}: recall=${formatPercent(recall)} (TP=${tp} FN=${fn})`);
|
|
73
|
+
}
|
|
74
|
+
console.log(`\n--- Top Firing Rules (top 15) ---`);
|
|
75
|
+
const fired = ruleQuality?.topRules ?? [];
|
|
76
|
+
for (const r of fired.slice(0, 15)) {
|
|
77
|
+
console.log(` ${r.ruleId}: matches=${r.matchCount} TP=${r.tpCount} FP=${r.fpCount}`);
|
|
78
|
+
}
|
|
79
|
+
console.log(`\nReport saved to: ${outputPath}`);
|
|
80
|
+
console.log('Done.');
|
|
81
|
+
}
|
|
82
|
+
main().catch((err) => {
|
|
83
|
+
console.error('Error:', err);
|
|
84
|
+
process.exit(1);
|
|
85
|
+
});
|
|
86
|
+
//# sourceMappingURL=run-hackaprompt-benchmark.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"run-hackaprompt-benchmark.js","sourceRoot":"","sources":["../../src/eval/run-hackaprompt-benchmark.ts"],"names":[],"mappings":";AACA;;;;;;;;;;;;;;;GAeG;AAEH,OAAO,EAAE,OAAO,EAAE,IAAI,EAAE,MAAM,WAAW,CAAC;AAC1C,OAAO,EAAE,qBAAqB,EAAE,yBAAyB,EAAE,MAAM,yBAAyB,CAAC;AAC3F,OAAO,EAAE,OAAO,EAAE,MAAM,mBAAmB,CAAC;AAE5C,SAAS,aAAa,CAAC,CAAS;IAC9B,OAAO,GAAG,CAAC,CAAC,GAAG,GAAG,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,GAAG,CAAC;AACpC,CAAC;AAED,SAAS,QAAQ,CAAC,CAAS;IACzB,OAAO,GAAG,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,IAAI,CAAC;AAC7B,CAAC;AAED,KAAK,UAAU,IAAI;IACjB,MAAM,IAAI,GAAG,OAAO,CAAC,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,OAAO,IAAI,GAAG,EAAE,IAAI,EAAE,IAAI,CAAC,CAAC,CAAC;IACnE,MAAM,QAAQ,GAAG,IAAI,CAAC,IAAI,EAAE,OAAO,CAAC,CAAC;IACrC,MAAM,QAAQ,GAAG,IAAI,CAAC,IAAI,EAAE,MAAM,EAAE,aAAa,EAAE,yBAAyB,CAAC,CAAC;IAC9E,MAAM,UAAU,GAAG,IAAI,CAAC,IAAI,EAAE,MAAM,EAAE,aAAa,EAAE,8BAA8B,CAAC,CAAC;IAErF,OAAO,CAAC,GAAG,CAAC,qDAAqD,CAAC,CAAC;IACnE,OAAO,CAAC,GAAG,CAAC,WAAW,QAAQ,EAAE,CAAC,CAAC;IACnC,OAAO,CAAC,GAAG,CAAC,WAAW,QAAQ,IAAI,CAAC,CAAC;IAErC,MAAM,MAAM,GAAG,qBAAqB,CAAC,QAAQ,CAAC,CAAC;IAC/C,MAAM,KAAK,GAAG,yBAAyB,CAAC,MAAM,CAAC,CAAC;IAEhD,OAAO,CAAC,GAAG,CAAC,UAAU,KAAK,CAAC,KAAK,aAAa,KAAK,CAAC,OAAO,aAAa,KAAK,CAAC,MAAM,UAAU,CAAC,CAAC;IAChG,OAAO,CAAC,GAAG,CAAC,eAAe,MAAM,CAAC,OAAO,CAAC,KAAK,CAAC,UAAU,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,EAAE,EAAE,CAAC,GAAG,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;IACvG,OAAO,CAAC,GAAG,CAAC,eAAe,MAAM,CAAC,OAAO,CAAC,KAAK,CAAC,YAAY,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,EAAE,EAAE,CAAC,GAAG,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;IAEzG,6EAA6E;IAC7E,wEAAwE;IACxE,sEAAsE;IACtE,qDAAqD;IACrD,MAAM,qBAAqB,GAAG;QAC5B,SAAS,EAAE,IAAI;QACf,SAAS,EAAE,GAAG;QACd,KAAK,EAAE,GAAG;QACV,eAAe,EAAE,GAAG;KACrB,CAAC;IAEF,MAAM,EAAE,MAAM,EAAE,SAAS,EAAE,WAAW,EAAE,GAAG,MAAM,OAAO,CAAC;QACvD,QAAQ;QACR,MAAM;QACN,UAAU,EAAE,qBAAqB;QACjC,UAAU;KACX,CAAC,CAAC;IAEH,OAAO,CAAC,GAAG,CAAC,YAAY,SAAS,CAAC,IAAI,CAAC,KAAK,CAAC,EAAE,CAAC,CAAC;IACjD,OAAO,CAAC,GAAG,CAAC,mBAAmB,CAAC,CAAC;IACjC,OAAO,CAAC,GAAG,CAAC,iBAAiB,aAAa,CAAC,MAAM,CAAC,OAAO,CAAC,MAAM,CAAC,EAAE,CAAC,CAAC;IACrE,OAAO,CAAC,GAAG,CAAC,iBAAiB,aAAa,CAAC,MAAM,CAAC,OAAO,CAAC,SAAS,CAAC,oCAAoC,CAAC,CAAC;IAC1G,OAAO,CAAC,GAAG,CAAC,oBAAoB,MAAM,CAAC,OAAO,CAAC,SAAS,CAAC,EAAE,OAAO,MAAM,CAAC,OAAO,CAAC,SAAS,CAAC,EAAE,EAAE,CAAC,CAAC;IAEjG,OAAO,CAAC,GAAG,CAAC,mBAAmB,CAAC,CAAC;IACjC,OAAO,CAAC,GAAG,CAAC,WAAW,QAAQ,CAAC,MAAM,CAAC,OAAO,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC;IACvD,OAAO,CAAC,GAAG,CAAC,WAAW,QAAQ,CAAC,MAAM,CAAC,OAAO,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC;IACvD,OAAO,CAAC,GAAG,CAAC,WAAW,QAAQ,CAAC,MAAM,CAAC,OAAO,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC;IACvD,OAAO,CAAC,GAAG,CAAC,WAAW,QAAQ,CAAC,MAAM,CAAC,OAAO,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;IAExD,OAAO,CAAC,GAAG,CAAC,uBAAuB,CAAC,CAAC;IACrC,KAAK,MAAM,GAAG,IAAI,MAAM,CAAC,UAAU,EAAE,CAAC;QACpC,MAAM,CAAC,GAAG,GAAG,CAAC,OAAO,CAAC;QACtB,MAAM,EAAE,GAAG,CAAC,CAAC,SAAS,CAAC,EAAE,CAAC;QAC1B,MAAM,EAAE,GAAG,CAAC,CAAC,SAAS,CAAC,EAAE,CAAC;QAC1B,MAAM,MAAM,GAAG,EAAE,GAAG,EAAE,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,EAAE,GAAG,CAAC,EAAE,GAAG,EAAE,CAAC,CAAC;QAClD,OAAO,CAAC,GAAG,CAAC,KAAK,GAAG,CAAC,QAAQ,YAAY,aAAa,CAAC,MAAM,CAAC,QAAQ,EAAE,OAAO,EAAE,GAAG,CAAC,CAAC;IACxF,CAAC;IAED,OAAO,CAAC,GAAG,CAAC,qCAAqC,CAAC,CAAC;IACnD,MAAM,KAAK,GAAG,WAAW,EAAE,QAAQ,IAAI,EAAE,CAAC;IAC1C,KAAK,MAAM,CAAC,IAAI,KAAK,CAAC,KAAK,CAAC,CAAC,EAAE,EAAE,CAAC,EAAE,CAAC;QACnC,OAAO,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC,MAAM,aAAa,CAAC,CAAC,UAAU,OAAO,CAAC,CAAC,OAAO,OAAO,CAAC,CAAC,OAAO,EAAE,CAAC,CAAC;IACxF,CAAC;IAED,OAAO,CAAC,GAAG,CAAC,sBAAsB,UAAU,EAAE,CAAC,CAAC;IAChD,OAAO,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC;AACvB,CAAC;AAED,IAAI,EAAE,CAAC,KAAK,CAAC,CAAC,GAAG,EAAE,EAAE;IACnB,OAAO,CAAC,KAAK,CAAC,QAAQ,EAAE,GAAG,CAAC,CAAC;IAC7B,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;AAClB,CAAC,CAAC,CAAC"}
|
package/dist/types.d.ts
CHANGED
|
@@ -2,14 +2,14 @@
|
|
|
2
2
|
* ATR (Agent Threat Rules) type definitions
|
|
3
3
|
* @module agent-threat-rules/types
|
|
4
4
|
*/
|
|
5
|
-
export type ATRStatus =
|
|
6
|
-
export type ATRSeverity =
|
|
7
|
-
export type ATRCategory =
|
|
8
|
-
export type ATRConfidence =
|
|
9
|
-
export type ATRSourceType =
|
|
10
|
-
export type ATRMatchType =
|
|
11
|
-
export type ATROperator =
|
|
12
|
-
export type ATRAction =
|
|
5
|
+
export type ATRStatus = "draft" | "experimental" | "stable" | "deprecated";
|
|
6
|
+
export type ATRSeverity = "critical" | "high" | "medium" | "low" | "informational";
|
|
7
|
+
export type ATRCategory = "prompt-injection" | "tool-poisoning" | "context-exfiltration" | "agent-manipulation" | "privilege-escalation" | "excessive-autonomy" | "data-poisoning" | "model-abuse" | "skill-compromise";
|
|
8
|
+
export type ATRConfidence = "high" | "medium" | "low";
|
|
9
|
+
export type ATRSourceType = "llm_io" | "tool_call" | "mcp_exchange" | "agent_behavior" | "multi_agent_comm" | "context_window" | "memory_access" | "skill_lifecycle" | "skill_permission" | "skill_chain";
|
|
10
|
+
export type ATRMatchType = "contains" | "regex" | "exact" | "starts_with";
|
|
11
|
+
export type ATROperator = "gt" | "lt" | "eq" | "gte" | "lte" | "deviation_from_baseline";
|
|
12
|
+
export type ATRAction = "block_input" | "block_output" | "block_tool" | "quarantine_session" | "reset_context" | "alert" | "shadow" | "snapshot" | "escalate" | "reduce_permissions" | "kill_agent";
|
|
13
13
|
export interface ATRReferences {
|
|
14
14
|
owasp_llm?: string[];
|
|
15
15
|
owasp_agentic?: string[];
|
|
@@ -17,7 +17,7 @@ export interface ATRReferences {
|
|
|
17
17
|
mitre_attack?: string[];
|
|
18
18
|
cve?: string[];
|
|
19
19
|
}
|
|
20
|
-
export type ATRScanTarget =
|
|
20
|
+
export type ATRScanTarget = "mcp" | "skill" | "both" | "runtime";
|
|
21
21
|
export interface ATRTags {
|
|
22
22
|
category: ATRCategory;
|
|
23
23
|
subcategory?: string;
|
|
@@ -80,7 +80,7 @@ export interface ATRTestCase {
|
|
|
80
80
|
agent_output?: string;
|
|
81
81
|
tool_name?: string;
|
|
82
82
|
tool_args?: string;
|
|
83
|
-
expected:
|
|
83
|
+
expected: "trigger" | "no_trigger" | "triggered" | "not_triggered";
|
|
84
84
|
}
|
|
85
85
|
export interface ATRTestCases {
|
|
86
86
|
true_positives: ATRTestCase[];
|
|
@@ -122,12 +122,12 @@ export interface ATRRule {
|
|
|
122
122
|
}
|
|
123
123
|
export interface ATREvasionTest {
|
|
124
124
|
input: string;
|
|
125
|
-
expected:
|
|
125
|
+
expected: "triggered" | "not_triggered";
|
|
126
126
|
bypass_technique: string;
|
|
127
127
|
notes?: string;
|
|
128
128
|
}
|
|
129
129
|
/** Event types that the ATR engine can evaluate */
|
|
130
|
-
export type AgentEventType =
|
|
130
|
+
export type AgentEventType = "llm_input" | "llm_output" | "tool_call" | "tool_response" | "agent_behavior" | "multi_agent_message" | "mcp_exchange";
|
|
131
131
|
/** An agent event to evaluate against ATR rules */
|
|
132
132
|
export interface AgentEvent {
|
|
133
133
|
type: AgentEventType;
|
|
@@ -146,10 +146,10 @@ export interface AgentEvent {
|
|
|
146
146
|
metadata?: Record<string, unknown>;
|
|
147
147
|
/** Scan context: when 'skill', all rules fire regardless of agent_source.type,
|
|
148
148
|
* with cross-context confidence downweighting for MCP-only rules. */
|
|
149
|
-
scanContext?:
|
|
149
|
+
scanContext?: "mcp" | "skill";
|
|
150
150
|
}
|
|
151
151
|
/** Result when an ATR rule matches an event */
|
|
152
|
-
export type ScanContextType =
|
|
152
|
+
export type ScanContextType = "native" | "cross-context";
|
|
153
153
|
export interface ATRMatch {
|
|
154
154
|
rule: ATRRule;
|
|
155
155
|
matchedConditions: string[];
|
|
@@ -161,7 +161,7 @@ export interface ATRMatch {
|
|
|
161
161
|
scan_context: ScanContextType;
|
|
162
162
|
}
|
|
163
163
|
/** Verdict outcome from evaluating matched rules */
|
|
164
|
-
export type VerdictOutcome =
|
|
164
|
+
export type VerdictOutcome = "allow" | "ask" | "deny";
|
|
165
165
|
/** Verdict returned after evaluating an event against all rules */
|
|
166
166
|
export interface ATRVerdict {
|
|
167
167
|
readonly outcome: VerdictOutcome;
|
|
@@ -197,6 +197,13 @@ export interface PlatformAdapter {
|
|
|
197
197
|
quarantineSession(ctx: ExecutionContext): Promise<ActionResult>;
|
|
198
198
|
resetContext(ctx: ExecutionContext): Promise<ActionResult>;
|
|
199
199
|
alert(ctx: ExecutionContext): Promise<ActionResult>;
|
|
200
|
+
/**
|
|
201
|
+
* Log the match for later audit without surfacing it to the user.
|
|
202
|
+
* Used as the safe default for newly auto-generated rules
|
|
203
|
+
* (CVE collector, probe pipeline, TC crystallisation) until they
|
|
204
|
+
* accumulate FP-free production observation.
|
|
205
|
+
*/
|
|
206
|
+
shadow(ctx: ExecutionContext): Promise<ActionResult>;
|
|
200
207
|
snapshot(ctx: ExecutionContext): Promise<ActionResult>;
|
|
201
208
|
escalate(ctx: ExecutionContext): Promise<ActionResult>;
|
|
202
209
|
reducePermissions(ctx: ExecutionContext): Promise<ActionResult>;
|
|
@@ -204,7 +211,7 @@ export interface PlatformAdapter {
|
|
|
204
211
|
}
|
|
205
212
|
/** Hook input from Claude Code / agent host */
|
|
206
213
|
export interface HookInput {
|
|
207
|
-
readonly hook:
|
|
214
|
+
readonly hook: "PreToolUse" | "PostToolUse";
|
|
208
215
|
readonly tool_name: string;
|
|
209
216
|
readonly tool_input: Readonly<Record<string, unknown>>;
|
|
210
217
|
readonly session_id?: string;
|
|
@@ -218,7 +225,7 @@ export interface HookOutput {
|
|
|
218
225
|
readonly matched_rules?: readonly string[];
|
|
219
226
|
}
|
|
220
227
|
/** Scan type: MCP runtime event scan vs SKILL.md static file scan */
|
|
221
|
-
export type ScanType =
|
|
228
|
+
export type ScanType = "mcp" | "skill";
|
|
222
229
|
/** Unified scan result produced by both evaluate() and scanSkill() paths */
|
|
223
230
|
export interface ScanResult {
|
|
224
231
|
readonly scan_type: ScanType;
|
package/dist/types.d.ts.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"types.d.ts","sourceRoot":"","sources":["../src/types.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAEH,MAAM,MAAM,SAAS,GAAG,OAAO,GAAG,cAAc,GAAG,QAAQ,GAAG,YAAY,CAAC;AAE3E,MAAM,MAAM,WAAW,
|
|
1
|
+
{"version":3,"file":"types.d.ts","sourceRoot":"","sources":["../src/types.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAEH,MAAM,MAAM,SAAS,GAAG,OAAO,GAAG,cAAc,GAAG,QAAQ,GAAG,YAAY,CAAC;AAE3E,MAAM,MAAM,WAAW,GACnB,UAAU,GACV,MAAM,GACN,QAAQ,GACR,KAAK,GACL,eAAe,CAAC;AAEpB,MAAM,MAAM,WAAW,GACnB,kBAAkB,GAClB,gBAAgB,GAChB,sBAAsB,GACtB,oBAAoB,GACpB,sBAAsB,GACtB,oBAAoB,GACpB,gBAAgB,GAChB,aAAa,GACb,kBAAkB,CAAC;AAEvB,MAAM,MAAM,aAAa,GAAG,MAAM,GAAG,QAAQ,GAAG,KAAK,CAAC;AAEtD,MAAM,MAAM,aAAa,GACrB,QAAQ,GACR,WAAW,GACX,cAAc,GACd,gBAAgB,GAChB,kBAAkB,GAClB,gBAAgB,GAChB,eAAe,GACf,iBAAiB,GACjB,kBAAkB,GAClB,aAAa,CAAC;AAElB,MAAM,MAAM,YAAY,GAAG,UAAU,GAAG,OAAO,GAAG,OAAO,GAAG,aAAa,CAAC;AAE1E,MAAM,MAAM,WAAW,GACnB,IAAI,GACJ,IAAI,GACJ,IAAI,GACJ,KAAK,GACL,KAAK,GACL,yBAAyB,CAAC;AAE9B,MAAM,MAAM,SAAS,GACjB,aAAa,GACb,cAAc,GACd,YAAY,GACZ,oBAAoB,GACpB,eAAe,GACf,OAAO,GACP,QAAQ,GACR,UAAU,GACV,UAAU,GACV,oBAAoB,GACpB,YAAY,CAAC;AAEjB,MAAM,WAAW,aAAa;IAC5B,SAAS,CAAC,EAAE,MAAM,EAAE,CAAC;IACrB,aAAa,CAAC,EAAE,MAAM,EAAE,CAAC;IACzB,WAAW,CAAC,EAAE,MAAM,EAAE,CAAC;IACvB,YAAY,CAAC,EAAE,MAAM,EAAE,CAAC;IACxB,GAAG,CAAC,EAAE,MAAM,EAAE,CAAC;CAChB;AAED,MAAM,MAAM,aAAa,GAAG,KAAK,GAAG,OAAO,GAAG,MAAM,GAAG,SAAS,CAAC;AAEjE,MAAM,WAAW,OAAO;IACtB,QAAQ,EAAE,WAAW,CAAC;IACtB,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,UAAU,CAAC,EAAE,aAAa,CAAC;IAC3B,WAAW,CAAC,EAAE,aAAa,CAAC;CAC7B;AAED,MAAM,WAAW,cAAc;IAC7B,IAAI,EAAE,aAAa,CAAC;IACpB,SAAS,CAAC,EAAE,MAAM,EAAE,CAAC;IACrB,QAAQ,CAAC,EAAE,MAAM,EAAE,CAAC;CACrB;AAED,MAAM,WAAW,mBAAmB;IAClC,KAAK,EAAE,MAAM,CAAC;IACd,QAAQ,EAAE,MAAM,EAAE,CAAC;IACnB,UAAU,EAAE,YAAY,CAAC;IACzB,cAAc,CAAC,EAAE,OAAO,CAAC;CAC1B;AAED,MAAM,WAAW,sBAAsB;IACrC,MAAM,EAAE,MAAM,CAAC;IACf,QAAQ,EAAE,WAAW,CAAC;IACtB,SAAS,EAAE,MAAM,CAAC;IAClB,MAAM,CAAC,EAAE,MAAM,CAAC;CACjB;AAED,MAAM,WAAW,eAAe;IAC9B,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,QAAQ,CAAC,EAAE,MAAM,EAAE,CAAC;IACpB,UAAU,CAAC,EAAE,YAAY,CAAC;IAC1B,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,QAAQ,CAAC,EAAE,WAAW,CAAC;IACvB,SAAS,CAAC,EAAE,MAAM,CAAC;CACpB;AAED,MAAM,WAAW,oBAAoB;IACnC,OAAO,EAAE,OAAO,CAAC;IACjB,MAAM,EAAE,MAAM,CAAC;IACf,KAAK,EAAE,eAAe,EAAE,CAAC;CAC1B;AAED,0EAA0E;AAC1E,MAAM,WAAW,iBAAiB;IAChC,KAAK,EAAE,MAAM,CAAC;IACd,QAAQ,EAAE,MAAM,CAAC;IACjB,KAAK,EAAE,MAAM,CAAC;IACd,WAAW,CAAC,EAAE,MAAM,CAAC;CACtB;AAED,+CAA+C;AAC/C,MAAM,MAAM,aAAa,GACrB,iBAAiB,EAAE,GACnB,MAAM,CACJ,MAAM,EACN,mBAAmB,GAAG,sBAAsB,GAAG,oBAAoB,CACpE,CAAC;AAEN,MAAM,WAAW,YAAY;IAC3B,UAAU,EAAE,aAAa,CAAC;IAC1B,kGAAkG;IAClG,SAAS,EAAE,MAAM,CAAC;IAClB,eAAe,CAAC,EAAE,MAAM,EAAE,CAAC;CAC5B;AAED,MAAM,WAAW,WAAW;IAC1B,OAAO,EAAE,SAAS,EAAE,CAAC;IACrB,uBAAuB,CAAC,EAAE,MAAM,CAAC;IACjC,gBAAgB,CAAC,EAAE,MAAM,CAAC;CAC3B;AAED,MAAM,WAAW,WAAW;IAC1B,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,aAAa,CAAC,EAAE,MAAM,CAAC;IACvB,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,QAAQ,EAAE,SAAS,GAAG,YAAY,GAAG,WAAW,GAAG,eAAe,CAAC;CACpE;AAED,MAAM,WAAW,YAAY;IAC3B,cAAc,EAAE,WAAW,EAAE,CAAC;IAC9B,cAAc,EAAE,WAAW,EAAE,CAAC;CAC/B;AAED,MAAM,WAAW,OAAO;IACtB,KAAK,EAAE,MAAM,CAAC;IACd,EAAE,EAAE,MAAM,CAAC;IACX,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,MAAM,EAAE,SAAS,CAAC;IAClB,WAAW,EAAE,MAAM,CAAC;IACpB,MAAM,EAAE,MAAM,CAAC;IACf,IAAI,EAAE,MAAM,CAAC;IACb,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,cAAc,CAAC,EAAE,MAAM,CAAC;IACxB,cAAc,CAAC,EAAE,MAAM,CAAC;IACxB,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,QAAQ,EAAE,WAAW,CAAC;IACtB,UAAU,CAAC,EAAE,aAAa,CAAC;IAC3B,IAAI,EAAE,OAAO,CAAC;IACd,YAAY,EAAE,cAAc,CAAC;IAC7B,SAAS,EAAE,YAAY,CAAC;IACxB,QAAQ,EAAE,WAAW,CAAC;IACtB,UAAU,CAAC,EAAE,YAAY,CAAC;IAC1B,wDAAwD;IACxD,aAAa,CAAC,EAAE,cAAc,EAAE,CAAC;IACjC,iGAAiG;IACjG,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,4DAA4D;IAC5D,cAAc,CAAC,EAAE,MAAM,CAAC;IACxB,uDAAuD;IACvD,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,mEAAmE;IACnE,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,oEAAoE;IACpE,iBAAiB,CAAC,EAAE,MAAM,CAAC;IAC3B,2DAA2D;IAC3D,WAAW,CAAC,EAAE,MAAM,CAAC;CACtB;AAED,MAAM,WAAW,cAAc;IAC7B,KAAK,EAAE,MAAM,CAAC;IACd,QAAQ,EAAE,WAAW,GAAG,eAAe,CAAC;IACxC,gBAAgB,EAAE,MAAM,CAAC;IACzB,KAAK,CAAC,EAAE,MAAM,CAAC;CAChB;AAED,mDAAmD;AACnD,MAAM,MAAM,cAAc,GACtB,WAAW,GACX,YAAY,GACZ,WAAW,GACX,eAAe,GACf,gBAAgB,GAChB,qBAAqB,GACrB,cAAc,CAAC;AAEnB,mDAAmD;AACnD,MAAM,WAAW,UAAU;IACzB,IAAI,EAAE,cAAc,CAAC;IACrB,SAAS,EAAE,MAAM,CAAC;IAClB,kCAAkC;IAClC,OAAO,EAAE,MAAM,CAAC;IAChB,iDAAiD;IACjD,MAAM,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;IAChC,uDAAuD;IACvD,OAAO,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;IACjC,yCAAyC;IACzC,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,8BAA8B;IAC9B,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,0BAA0B;IAC1B,QAAQ,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;IACnC;0EACsE;IACtE,WAAW,CAAC,EAAE,KAAK,GAAG,OAAO,CAAC;CAC/B;AAED,+CAA+C;AAC/C,MAAM,MAAM,eAAe,GAAG,QAAQ,GAAG,eAAe,CAAC;AAEzD,MAAM,WAAW,QAAQ;IACvB,IAAI,EAAE,OAAO,CAAC;IACd,iBAAiB,EAAE,MAAM,EAAE,CAAC;IAC5B,eAAe,EAAE,MAAM,EAAE,CAAC;IAC1B,UAAU,EAAE,MAAM,CAAC;IACnB,SAAS,EAAE,MAAM,CAAC;IAClB;kFAC8E;IAC9E,YAAY,EAAE,eAAe,CAAC;CAC/B;AAED,oDAAoD;AACpD,MAAM,MAAM,cAAc,GAAG,OAAO,GAAG,KAAK,GAAG,MAAM,CAAC;AAEtD,mEAAmE;AACnE,MAAM,WAAW,UAAU;IACzB,QAAQ,CAAC,OAAO,EAAE,cAAc,CAAC;IACjC,QAAQ,CAAC,MAAM,EAAE,MAAM,CAAC;IACxB,QAAQ,CAAC,UAAU,EAAE,MAAM,CAAC;IAC5B,QAAQ,CAAC,eAAe,EAAE,WAAW,GAAG,IAAI,CAAC;IAC7C,QAAQ,CAAC,iBAAiB,EAAE,MAAM,CAAC;IACnC,QAAQ,CAAC,OAAO,EAAE,SAAS,SAAS,EAAE,CAAC;IACvC,QAAQ,CAAC,OAAO,EAAE,SAAS,QAAQ,EAAE,CAAC;IACtC,QAAQ,CAAC,SAAS,EAAE,MAAM,CAAC;CAC5B;AAED,0CAA0C;AAC1C,MAAM,WAAW,YAAY;IAC3B,QAAQ,CAAC,MAAM,EAAE,SAAS,CAAC;IAC3B,QAAQ,CAAC,OAAO,EAAE,OAAO,CAAC;IAC1B,QAAQ,CAAC,OAAO,EAAE,MAAM,CAAC;IACzB,QAAQ,CAAC,SAAS,EAAE,MAAM,CAAC;CAC5B;AAED,mEAAmE;AACnE,MAAM,WAAW,gBAAgB;IAC/B,QAAQ,CAAC,KAAK,EAAE,UAAU,CAAC;IAC3B,QAAQ,CAAC,OAAO,EAAE,SAAS,QAAQ,EAAE,CAAC;IACtC,QAAQ,CAAC,OAAO,EAAE,UAAU,CAAC;IAC7B,QAAQ,CAAC,SAAS,CAAC,EAAE,MAAM,CAAC;IAC5B,QAAQ,CAAC,QAAQ,CAAC,EAAE,QAAQ,CAAC,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC,CAAC;CACvD;AAED,0DAA0D;AAC1D,MAAM,WAAW,eAAe;IAC9B,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAC;IACtB,UAAU,CAAC,GAAG,EAAE,gBAAgB,GAAG,OAAO,CAAC,YAAY,CAAC,CAAC;IACzD,WAAW,CAAC,GAAG,EAAE,gBAAgB,GAAG,OAAO,CAAC,YAAY,CAAC,CAAC;IAC1D,SAAS,CAAC,GAAG,EAAE,gBAAgB,GAAG,OAAO,CAAC,YAAY,CAAC,CAAC;IACxD,iBAAiB,CAAC,GAAG,EAAE,gBAAgB,GAAG,OAAO,CAAC,YAAY,CAAC,CAAC;IAChE,YAAY,CAAC,GAAG,EAAE,gBAAgB,GAAG,OAAO,CAAC,YAAY,CAAC,CAAC;IAC3D,KAAK,CAAC,GAAG,EAAE,gBAAgB,GAAG,OAAO,CAAC,YAAY,CAAC,CAAC;IACpD;;;;;OAKG;IACH,MAAM,CAAC,GAAG,EAAE,gBAAgB,GAAG,OAAO,CAAC,YAAY,CAAC,CAAC;IACrD,QAAQ,CAAC,GAAG,EAAE,gBAAgB,GAAG,OAAO,CAAC,YAAY,CAAC,CAAC;IACvD,QAAQ,CAAC,GAAG,EAAE,gBAAgB,GAAG,OAAO,CAAC,YAAY,CAAC,CAAC;IACvD,iBAAiB,CAAC,GAAG,EAAE,gBAAgB,GAAG,OAAO,CAAC,YAAY,CAAC,CAAC;IAChE,SAAS,CAAC,GAAG,EAAE,gBAAgB,GAAG,OAAO,CAAC,YAAY,CAAC,CAAC;CACzD;AAED,+CAA+C;AAC/C,MAAM,WAAW,SAAS;IACxB,QAAQ,CAAC,IAAI,EAAE,YAAY,GAAG,aAAa,CAAC;IAC5C,QAAQ,CAAC,SAAS,EAAE,MAAM,CAAC;IAC3B,QAAQ,CAAC,UAAU,EAAE,QAAQ,CAAC,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC,CAAC;IACvD,QAAQ,CAAC,UAAU,CAAC,EAAE,MAAM,CAAC;IAC7B,QAAQ,CAAC,SAAS,CAAC,EAAE,MAAM,CAAC;CAC7B;AAED,8CAA8C;AAC9C,MAAM,WAAW,UAAU;IACzB,QAAQ,CAAC,QAAQ,EAAE,cAAc,CAAC;IAClC,QAAQ,CAAC,MAAM,CAAC,EAAE,MAAM,CAAC;IACzB,QAAQ,CAAC,OAAO,CAAC,EAAE,MAAM,CAAC;IAC1B,QAAQ,CAAC,aAAa,CAAC,EAAE,SAAS,MAAM,EAAE,CAAC;CAC5C;AAED,qEAAqE;AACrE,MAAM,MAAM,QAAQ,GAAG,KAAK,GAAG,OAAO,CAAC;AAEvC,4EAA4E;AAC5E,MAAM,WAAW,UAAU;IACzB,QAAQ,CAAC,SAAS,EAAE,QAAQ,CAAC;IAC7B,QAAQ,CAAC,YAAY,EAAE,MAAM,CAAC;IAC9B,QAAQ,CAAC,UAAU,CAAC,EAAE,MAAM,CAAC;IAC7B,QAAQ,CAAC,SAAS,EAAE,MAAM,CAAC;IAC3B,QAAQ,CAAC,YAAY,EAAE,MAAM,CAAC;IAC9B,QAAQ,CAAC,OAAO,EAAE,SAAS,QAAQ,EAAE,CAAC;IACtC,QAAQ,CAAC,YAAY,EAAE,MAAM,CAAC;CAC/B"}
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "agent-threat-rules",
|
|
3
|
-
"version": "2.1.
|
|
3
|
+
"version": "2.1.5",
|
|
4
4
|
"type": "module",
|
|
5
5
|
"description": "Open detection standard -- like Sigma, but for AI agents. 311 rules for prompt injection, tool poisoning, context exfiltration, and MCP attacks. Shipped in Cisco AI Defense. 97.1% recall on NVIDIA garak.",
|
|
6
6
|
"main": "./dist/index.js",
|
|
@@ -22,6 +22,14 @@
|
|
|
22
22
|
"import": "./dist/mcp-server.js",
|
|
23
23
|
"types": "./dist/mcp-server.d.ts"
|
|
24
24
|
},
|
|
25
|
+
"./converters/sage": {
|
|
26
|
+
"import": "./dist/converters/sage.js",
|
|
27
|
+
"types": "./dist/converters/sage.d.ts"
|
|
28
|
+
},
|
|
29
|
+
"./converters/sage-reverse": {
|
|
30
|
+
"import": "./dist/converters/sage-reverse.js",
|
|
31
|
+
"types": "./dist/converters/sage-reverse.d.ts"
|
|
32
|
+
},
|
|
25
33
|
"./rules": "./rules",
|
|
26
34
|
"./spec": "./spec/atr-schema.yaml"
|
|
27
35
|
},
|
|
@@ -0,0 +1,196 @@
|
|
|
1
|
+
title: "Spring AI ChatMemory Cross-User Memory Leakage (CVE-2026-41712)"
|
|
2
|
+
id: ATR-2026-00449
|
|
3
|
+
rule_version: 1
|
|
4
|
+
status: experimental
|
|
5
|
+
description: >
|
|
6
|
+
Detects exploitation of CVE-2026-41712 (High), cross-user memory
|
|
7
|
+
leakage in Spring AI's PromptChatMemoryAdvisor. The vulnerable
|
|
8
|
+
configuration uses a shared ChatMemory without per-session
|
|
9
|
+
conversation_id discipline — when two requests land on the same
|
|
10
|
+
advisor without distinct conversation IDs, memory written by User A
|
|
11
|
+
is retrieved and injected into User B's prompt context as if it were
|
|
12
|
+
B's own conversation history. This produces a confidentiality
|
|
13
|
+
failure: A's private content (prior turns, tool outputs,
|
|
14
|
+
conversation IDs, role tags) leaks into B's session and is processed
|
|
15
|
+
by the LLM on B's behalf. This rule detects the LLM-output / context
|
|
16
|
+
payload patterns that signal cross-user memory bleed: mismatched
|
|
17
|
+
conversation IDs inside advisor-injected context, role/user-tag
|
|
18
|
+
markers from another session appearing in the active session's
|
|
19
|
+
retrieved memory, or memory-fetch log lines whose user-id field
|
|
20
|
+
does not match the active session. CWE-359 (Privacy Disclosure),
|
|
21
|
+
CWE-201 (Insertion of Sensitive Information Into Sent Data).
|
|
22
|
+
Patches in Spring AI >= 1.0.0; this rule detects exploitation against
|
|
23
|
+
unpatched deployments and provides defence-in-depth post-patch by
|
|
24
|
+
catching the leakage pattern regardless of upstream patch state.
|
|
25
|
+
author: "ATR Community"
|
|
26
|
+
date: "2026/05/12"
|
|
27
|
+
schema_version: "0.1"
|
|
28
|
+
detection_tier: pattern
|
|
29
|
+
maturity: test
|
|
30
|
+
severity: high
|
|
31
|
+
|
|
32
|
+
references:
|
|
33
|
+
owasp_llm:
|
|
34
|
+
- "LLM02:2025 - Sensitive Information Disclosure"
|
|
35
|
+
- "LLM08:2025 - Vector and Embedding Weaknesses"
|
|
36
|
+
owasp_agentic:
|
|
37
|
+
- "ASI04:2026 - Memory and Knowledge Base Poisoning"
|
|
38
|
+
- "ASI09:2026 - Identity Spoofing and Impersonation"
|
|
39
|
+
mitre_atlas:
|
|
40
|
+
- "AML.T0024 - Exfiltration via ML Inference API"
|
|
41
|
+
- "AML.T0057 - LLM Data Leakage"
|
|
42
|
+
mitre_attack:
|
|
43
|
+
- "T1530 - Data from Cloud Storage Object"
|
|
44
|
+
cve:
|
|
45
|
+
- "CVE-2026-41712"
|
|
46
|
+
|
|
47
|
+
metadata_provenance:
|
|
48
|
+
mitre_atlas: human-reviewed
|
|
49
|
+
owasp_llm: human-reviewed
|
|
50
|
+
owasp_agentic: human-reviewed
|
|
51
|
+
cve: human-reviewed
|
|
52
|
+
|
|
53
|
+
compliance:
|
|
54
|
+
eu_ai_act:
|
|
55
|
+
- article: "15"
|
|
56
|
+
context: "CVE-2026-41712 lets one user's chat memory leak into another user's session via Spring AI PromptChatMemoryAdvisor; Article 15 cybersecurity requirements mandate that high-risk AI systems enforce per-session memory isolation."
|
|
57
|
+
strength: primary
|
|
58
|
+
- article: "9"
|
|
59
|
+
context: "Article 9 risk management must enumerate cross-user memory leakage as a high-risk class — the conversation-memory layer is typically treated as low-risk infrastructure but actually carries private dialogue content."
|
|
60
|
+
strength: primary
|
|
61
|
+
nist_ai_rmf:
|
|
62
|
+
- subcategory: "MP.5.1"
|
|
63
|
+
context: "Adversarial conditions that cause one user's chat memory to be retrieved into another user's prompt must be tracked as a primary confidentiality failure mode for advisor-based architectures."
|
|
64
|
+
strength: primary
|
|
65
|
+
- subcategory: "MG.2.3"
|
|
66
|
+
context: "Risk treatment plans under MG.2.3 must require per-conversation_id memory partitioning in any code path that uses a shared ChatMemory across multiple users."
|
|
67
|
+
strength: primary
|
|
68
|
+
iso_42001:
|
|
69
|
+
- clause: "8.6"
|
|
70
|
+
context: "Operational controls under clause 8.6 must require that conversation_id is distinct per session and that advisor pipelines reject memory retrieval whose conversation_id does not match the active session."
|
|
71
|
+
strength: primary
|
|
72
|
+
|
|
73
|
+
tags:
|
|
74
|
+
category: context-exfiltration
|
|
75
|
+
subcategory: shared-memory-cross-user-leak
|
|
76
|
+
scan_target: both
|
|
77
|
+
confidence: high
|
|
78
|
+
|
|
79
|
+
agent_source:
|
|
80
|
+
type: llm_io
|
|
81
|
+
framework:
|
|
82
|
+
- spring-ai
|
|
83
|
+
- any
|
|
84
|
+
provider:
|
|
85
|
+
- any
|
|
86
|
+
|
|
87
|
+
detection:
|
|
88
|
+
condition: any
|
|
89
|
+
false_positives:
|
|
90
|
+
- "Legitimate documentation or changelog text discussing CVE-2026-41712 patch notes."
|
|
91
|
+
- "Static analysis tooling output documenting cross-user memory leakage patterns for defensive purposes."
|
|
92
|
+
- "Patched Spring AI PromptChatMemoryAdvisor deployments that enforce per-conversation_id partitioning."
|
|
93
|
+
- "Multi-tenant test fixtures that intentionally include foreign conversation IDs for QA purposes."
|
|
94
|
+
conditions:
|
|
95
|
+
- field: content
|
|
96
|
+
operator: regex
|
|
97
|
+
value: '(?i)\bPromptChatMemoryAdvisor\b[^\n]{0,200}\b(?:shared|global|singleton|static)[A-Za-z_]*ChatMemory\b'
|
|
98
|
+
description: "PromptChatMemoryAdvisor wired with a shared / singleton / static ChatMemory bean (matches both `shared ChatMemory` and `sharedChatMemory` camelCase) — the architectural precondition for CVE-2026-41712"
|
|
99
|
+
|
|
100
|
+
- field: content
|
|
101
|
+
operator: regex
|
|
102
|
+
value: '(?i)conversation_id\s*[:=]\s*["\x27]?(?:default|null|none)\b'
|
|
103
|
+
description: "ChatMemory used with a placeholder / default / null conversation_id — disables per-session partitioning, enabling cross-user bleed"
|
|
104
|
+
|
|
105
|
+
- field: content
|
|
106
|
+
operator: regex
|
|
107
|
+
value: '(?i)\b(?:retrieved|loaded|fetched)\s+memory\s+conversation_id\s*=\s*["\x27]?[a-z0-9_-]{4,}["\x27]?[^\n]{0,100}(?:current|active)\s+(?:session|user)[^\n]{0,40}\b(?:is|=)\s+["\x27]?[a-z0-9_-]{4,}["\x27]?'
|
|
108
|
+
description: "Memory-fetch log entry where retrieved conversation_id does not match the active session — direct leakage evidence"
|
|
109
|
+
|
|
110
|
+
- field: content
|
|
111
|
+
operator: regex
|
|
112
|
+
value: '(?i)\[(?:user|usr)[_\s]?\d+\][^\[\n]{0,200}\[(?:user|usr)[_\s]?\d+\]'
|
|
113
|
+
description: "Two distinct [user_N] tags appearing in the same advisor-injected context window — different users sharing a memory window"
|
|
114
|
+
|
|
115
|
+
- field: content
|
|
116
|
+
operator: regex
|
|
117
|
+
value: '(?i)\b(?:MessageWindow|InMemory|Cassandra|JDBC)?ChatMemory\b\.?(?:builder\s*\(\s*\))?[^.\n]{0,40}\.(?:withoutId|withoutConversationId|noConversationId|defaultConversationId)\s*\('
|
|
118
|
+
description: "ChatMemory builder constructed without a conversation_id discriminator (withoutId / noConversationId / defaultConversationId) — buggy advisor configuration"
|
|
119
|
+
|
|
120
|
+
- field: content
|
|
121
|
+
operator: regex
|
|
122
|
+
value: '(?i)previous\s+conversation[^.\n]{0,80}\b(?:from|of|with)\s+(?:another|different|other)\s+user\b'
|
|
123
|
+
description: "Advisor-injected text explicitly attributing prior turns to another user — direct evidence of leakage"
|
|
124
|
+
|
|
125
|
+
- field: content
|
|
126
|
+
operator: regex
|
|
127
|
+
value: '(?i)active\s+(?:user|session)\s+id\s+["\x27][a-z0-9_-]{4,}["\x27][^.\n]{0,40}\bmemory\s+owner\s+id\s+["\x27][a-z0-9_-]{4,}["\x27]'
|
|
128
|
+
description: "Audit / observability line where active-session id and memory-owner id are emitted as different quoted values — explicit cross-user retrieval"
|
|
129
|
+
|
|
130
|
+
- field: content
|
|
131
|
+
operator: regex
|
|
132
|
+
value: '(?i)PromptChatMemoryAdvisor\b[^\n]{0,200}\.\s*tenant\s*\(\s*["\x27](?:wildcard|any|\*)["\x27]\s*\)'
|
|
133
|
+
description: "Advisor configured with wildcard tenant scope (.tenant(\"*\")) — defeats per-tenant isolation"
|
|
134
|
+
|
|
135
|
+
response:
|
|
136
|
+
actions:
|
|
137
|
+
- block_output
|
|
138
|
+
- alert
|
|
139
|
+
- escalate
|
|
140
|
+
notify:
|
|
141
|
+
- security_team
|
|
142
|
+
|
|
143
|
+
remediation: >
|
|
144
|
+
Upgrade Spring AI to >= 1.0.0. Ensure every PromptChatMemoryAdvisor is
|
|
145
|
+
constructed with per-session conversation_id discipline — derive
|
|
146
|
+
conversation_id from the authenticated user / session id, never share
|
|
147
|
+
a singleton ChatMemory across users without partitioning. Reject
|
|
148
|
+
retrievals whose conversation_id does not match the active session.
|
|
149
|
+
Add observability that emits user_id and memory_owner_id together so
|
|
150
|
+
cross-user retrievals are visible at log time.
|
|
151
|
+
|
|
152
|
+
test_cases:
|
|
153
|
+
true_positives:
|
|
154
|
+
- input: '@Bean public PromptChatMemoryAdvisor advisor() { return PromptChatMemoryAdvisor.builder(sharedChatMemory).build(); }'
|
|
155
|
+
expected: triggered
|
|
156
|
+
description: "Advisor built with a shared ChatMemory bean — CVE-2026-41712 precondition"
|
|
157
|
+
- input: 'conversation_id: "default"'
|
|
158
|
+
expected: triggered
|
|
159
|
+
description: "Placeholder conversation_id disabling per-session partitioning"
|
|
160
|
+
- input: '[INFO] retrieved memory conversation_id=conv_abc1 for current session user is sess_xyz9'
|
|
161
|
+
expected: triggered
|
|
162
|
+
description: "Memory-fetch log proving retrieved conv_id != active session id"
|
|
163
|
+
- input: '[user_42] What is my address? [user_99] My API key is sk-...'
|
|
164
|
+
expected: triggered
|
|
165
|
+
description: "Two distinct user tags in the same advisor-injected context"
|
|
166
|
+
- input: 'MessageWindowChatMemory.builder().withoutId().build()'
|
|
167
|
+
expected: triggered
|
|
168
|
+
description: "Memory builder constructed without a conversation_id discriminator"
|
|
169
|
+
- input: 'Previous conversation from another user: "I need help drafting an email to my CFO about Q3 revenue."'
|
|
170
|
+
expected: triggered
|
|
171
|
+
description: "Advisor-injected text explicitly attributing prior turns to another user"
|
|
172
|
+
- input: 'active user id "alice42" memory owner id "bob77"'
|
|
173
|
+
expected: triggered
|
|
174
|
+
description: "Audit line showing active-session id and memory-owner id mismatch"
|
|
175
|
+
- input: 'PromptChatMemoryAdvisor.builder(memory).tenant("*").build()'
|
|
176
|
+
expected: triggered
|
|
177
|
+
description: "Advisor configured with wildcard tenant scope"
|
|
178
|
+
true_negatives:
|
|
179
|
+
- input: "The Spring AI 1.0.0 changelog notes that PromptChatMemoryAdvisor now requires explicit per-session conversation_id (CVE-2026-41712)."
|
|
180
|
+
expected: not_triggered
|
|
181
|
+
description: "Documentation discussing the patch should not trigger"
|
|
182
|
+
- input: "Best practice: derive conversation_id from the authenticated session id and never share ChatMemory across users."
|
|
183
|
+
expected: not_triggered
|
|
184
|
+
description: "Defensive guidance discussing the mitigation"
|
|
185
|
+
- input: 'PromptChatMemoryAdvisor.builder(memory).conversationId(session.getId()).build()'
|
|
186
|
+
expected: not_triggered
|
|
187
|
+
description: "Properly partitioned advisor using authenticated session id"
|
|
188
|
+
- input: "Reviewing this rule against CVE-2026-41712 to ensure regex coverage is correct."
|
|
189
|
+
expected: not_triggered
|
|
190
|
+
description: "Discussion of CVE in defensive context"
|
|
191
|
+
- input: 'conversation_id: "sess_abc123"'
|
|
192
|
+
expected: not_triggered
|
|
193
|
+
description: "Realistic per-session conversation_id, not a placeholder"
|
|
194
|
+
- input: "The chat memory window stores the last 10 turns per conversation, indexed by conversation_id."
|
|
195
|
+
expected: not_triggered
|
|
196
|
+
description: "Benign documentation of correct memory semantics"
|