agent-threat-rules 4.1.5 → 4.2.0

package/dist/eval/eval-harness.js.map

@@ -1 +1 @@
-{"version":3,"file":"eval-harness.js","sourceRoot":"","sources":["../../src/eval/eval-harness.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;GAYG;AAEH,OAAO,EAAE,OAAO,EAAE,IAAI,EAAE,MAAM,WAAW,CAAC;AAC1C,OAAO,EAAE,UAAU,EAAE,YAAY,EAAE,aAAa,EAAE,MAAM,SAAS,CAAC;AAClE,OAAO,EAAE,8BAA8B,EAAE,MAAM,qCAAqC,CAAC;AACrF,OAAO,EAAE,SAAS,EAAE,MAAM,cAAc,CAAC;AACzC,OAAO,EAAE,YAAY,EAAE,mBAAmB,EAAE,MAAM,oBAAoB,CAAC;AACvE,OAAO,EAAE,eAAe,EAAE,MAAM,yBAAyB,CAAC;AAG1D,OAAO,EAAE,WAAW,EAAE,cAAc,EAAE,MAAM,aAAa,CAAC;AAE1D,OAAO,EAAE,iBAAiB,EAAE,eAAe,EAAE,MAAM,cAAc,CAAC;AAElE,OAAO,EAAE,kBAAkB,EAAE,MAAM,mBAAmB,CAAC;AAmCvD,oFAAoF;AACpF,MAAM,UAAU,qBAAqB;IACnC,MAAM,GAAG,GAAG,OAAO,CAAC,GAAG,CAAC,sBAAsB,CAAC,EAAE,IAAI,EAAE,CAAC,WAAW,EAAE,CAAC;IACtE,IAAI,GAAG,KAAK,QAAQ,IAAI,GAAG,KAAK,MAAM,IAAI,GAAG,KAAK,WAAW;QAAE,OAAO,GAAG,CAAC;IAC1E,OAAO,WAAW,CAAC;AACrB,CAAC;AAmBD;;;;;;;GAOG;AACH,SAAS,aAAa,CAAC,MAAoB;IACzC,OAAO;QACL,IAAI,EAAE,MAAM,CAAC,SAAS;QACtB,OAAO,EAAE,MAAM,CAAC,IAAI;QACpB,SAAS,EAAE,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE;QACnC,MAAM,EAAE,MAAM,CAAC,MAAM;KACtB,CAAC;AACJ,CAAC;AAED;;;;;;;;GAQG;AACH,SAAS,YAAY,CACnB,MAAoB,EACpB,IAAwB;IAExB,MAAM,OAAO,GAAG,aAAa,CAAC,MAAM,CAAC,CAAC;IACtC,IAAI,IAAI,KAAK,QAAQ;QAAE,OAAO,CAAC,OAAO,CAAC,CAAC;IACxC,IAAI,IAAI,KAAK,MAAM;QAAE,OAAO,CAAC,OAAO,EAAE,mBAAmB,CAAC,OAAO,CAAC,CAAC,CAAC;IACpE,OAAO,CAAC,OAAO,EAAE,GAAG,YAAY,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC;AACrE,CAAC;AAED,wEAAwE;AACxE,SAAS,aAAa,CAAC,IAAwB;IAC7C,OAAO,IAAI,KAAK,WAAW,CAAC;AAC9B,CAAC;AAED,uFAAuF;AACvF,SAAS,iBAAiB,CACxB,MAAiB,EACjB,MAAoB,EACpB,IAAwB;IAExB,MAAM,KAAK,GAAG,IAAI,GAAG,EAAkB,CAAC;IACxC,KAAK,MAAM,KAAK,IAAI,YAAY,CAAC,MAAM,EAAE,IAAI,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE,CAAC;QACxD,KAAK,MAAM,CAAC,IAAI,MAAM,CAAC,QAAQ,CAAC,KAAK,CAAC,EAAE,CAAC;YACvC,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,IAAI,CAAC,EAAE,EAAE,IAAI,CAAC,GAAG,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,IAAI,CAAC,EAAE,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC,UAAU,CAAC,CAAC,CAAC;QAC1E,CAAC;IACH,CAAC;IACD,IAAI,aAAa,CAAC,IAAI,CAAC,EAAE,CAAC;QACxB,KAAK,MAAM,CAAC,IAAI,MAAM,CAAC,SAAS,CAAC,MAAM,CAAC,IAAI,CAAC,EAAE,CAAC;YAC9C,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,IAAI,CAAC,EAAE,EAAE,IAAI,CAAC,GAAG,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,IAAI,CAAC,EAAE,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC,UAAU,CAAC,CAAC,CAAC;QAC1E,CAAC;IACH,CAAC;IACD,OAAO,KAAK,CAAC;AACf,CAAC;AAED;;;GAGG;AACH,SAAS,mBAAmB,CAC1B,MAAiB,EACjB,MAAoB,EACpB,IAAwB;IAExB,IAAI,CAAC;QACH,MAAM,KAAK,GAAG,WAAW,CAAC,GAAG,EAAE,CAAC;QAChC,MAAM,MAAM,GAAG,IAAI,GAAG,EAAkB,CAAC;QACzC,KAAK,MAAM,KAAK,IAAI,YAAY,CAAC,MAAM,EAAE,IAAI,CAAC,EAAE,CAAC;YAC/C,KAAK,MAAM,CAAC,IAAI,MAAM,CAAC,QAAQ,CAAC,KAAK,CAAC,EAAE,CAAC;gBACvC,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,IAAI,CAAC,EAAE,EAAE,IAAI,CAAC,GAAG,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,IAAI,CAAC,EAAE,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC,UAAU,CAAC,CAAC,CAAC;YAC5E,CAAC;QACH,CAAC;QACD,IAAI,aAAa,CAAC,IAAI,CAAC,EAAE,CAAC;YACxB,KAAK,MAAM,CAAC,IAAI,MAAM,CAAC,SAAS,CAAC,MAAM,CAAC,IAAI,CAAC,EAAE,CAAC;gBAC9C,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,IAAI,CAAC,EAAE,EAAE,IAAI,CAAC,GAAG,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,IAAI,CAAC,EAAE,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC,UAAU,CAAC,CAAC,CAAC;YAC5E,CAAC;QACH,CAAC;QACD,MAAM,SAAS,GAAG,WAAW,CAAC,GAAG,EAAE,GAAG,KAAK,CAAC;QAE5C,OAAO;YACL,EAAE,EAAE,MAAM,CAAC,EAAE;YACb,QAAQ,EAAE,MAAM,CAAC,QAAQ;YACzB,iBAAiB,EAAE,MAAM,CAAC,iBAAiB;YAC3C,eAAe,EAAE,MAAM,CAAC,IAAI,GAAG,CAAC;YAChC,YAAY,EAAE,CAAC,GAAG,MAAM,CAAC,IAAI,EAAE,CAAC;YAChC,UAAU,EAAE,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,GAAG,MAAM,CAAC,MAAM,EAAE,CAAC;YAC3C,SAAS;YACT,UAAU,EAAE,MAAM,CAAC,UAAU;YAC7B,IAAI,EAAE,MAAM,CAAC,IAAI;SAClB,CAAC;IACJ,CAAC;IAAC,MAAM,CAAC;QACP,OAAO;YACL,EAAE,EAAE,MAAM,CAAC,EAAE;YACb,QAAQ,EAAE,MAAM,CAAC,QAAQ;YACzB,iBAAiB,EAAE,MAAM,CAAC,iBAAiB;YAC3C,eAAe,EAAE,KAAK;YACtB,YAAY,EAAE,EAAE;YAChB,UAAU,EAAE,CAAC;YACb,SAAS,EAAE,CAAC;YACZ,UAAU,EAAE,MAAM,CAAC,UAAU;YAC7B,IAAI,EAAE,MAAM,CAAC,IAAI;SAClB,CAAC;IACJ,CAAC;AACH,CAAC;AAED;;;;;;;;;;;GAWG;AACH,KAAK,UAAU,kBAAkB,CAC/B,MAAiB,EACjB,MAAoB,EACpB,IAAwB;IAExB,IAAI,CAAC;QACH,MAAM,KAAK,GAAG,WAAW,CAAC,GAAG,EAAE,CAAC;QAChC,MAAM,EAAE,OAAO,EAAE,GAAG,MAAM,MAAM,CAAC,mBAAmB,CAAC,aAAa,CAAC,MAAM,CAAC,CAAC,CAAC;QAC5E,MAAM,MAAM,GAAG,IAAI,GAAG,CACpB,OAAO,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,EAAE,EAAE,CAAC,CAAC,UAAU,CAAC,CAAC,CACtD,CAAC;QACF,KAAK,MAAM,CAAC,EAAE,EAAE,UAAU,CAAC,IAAI,iBAAiB,CAAC,MAAM,EAAE,MAAM,EAAE,IAAI,CAAC,EAAE,CAAC;YACvE,MAAM,CAAC,GAAG,CAAC,EAAE,EAAE,IAAI,CAAC,GAAG,CAAC,MAAM,CAAC,GAAG,CAAC,EAAE,CAAC,IAAI,CAAC,EAAE,UAAU,CAAC,CAAC,CAAC;QAC5D,CAAC;QACD,MAAM,SAAS,GAAG,WAAW,CAAC,GAAG,EAAE,GAAG,KAAK,CAAC;QAE5C,OAAO;YACL,EAAE,EAAE,MAAM,CAAC,EAAE;YACb,QAAQ,EAAE,MAAM,CAAC,QAAQ;YACzB,iBAAiB,EAAE,MAAM,CAAC,iBAAiB;YAC3C,eAAe,EAAE,MAAM,CAAC,IAAI,GAAG,CAAC;YAChC,YAAY,EAAE,CAAC,GAAG,MAAM,CAAC,IAAI,EAAE,CAAC;YAChC,UAAU,EAAE,IAAI,CAAC,GAAG,CAAC,OAAO,CAAC,iBAAiB,EAAE,CAAC,EAAE,GAAG,MAAM,CAAC,MAAM,EAAE,CAAC;YACtE,SAAS;YACT,UAAU,EAAE,MAAM,CAAC,UAAU;YAC7B,IAAI,EAAE,MAAM,CAAC,IAAI;SAClB,CAAC;IACJ,CAAC;IAAC,MAAM,CAAC;QACP,OAAO;YACL,EAAE,EAAE,MAAM,CAAC,EAAE;YACb,QAAQ,EAAE,MAAM,CAAC,QAAQ;YACzB,iBAAiB,EAAE,MAAM,CAAC,iBAAiB;YAC3C,eAAe,EAAE,KAAK;YACtB,YAAY,EAAE,EAAE;YAChB,UAAU,EAAE,CAAC;YACb,SAAS,EAAE,CAAC;YACZ,UAAU,EAAE,MAAM,CAAC,UAAU;YAC7B,IAAI,EAAE,MAAM,CAAC,IAAI;SAClB,CAAC;IACJ,CAAC;AACH,CAAC;AAED;;GAEG;AACH,KAAK,UAAU,gBAAgB,CAAC,cAAsB;IACpD,IAAI,CAAC,UAAU,CAAC,cAAc,CAAC;QAAE,OAAO,IAAI,CAAC;IAE7C,IAAI,CAAC;QACH,MAAM,IAAI,GAAG,IAAI,CAAC,KAAK,CAAC,YAAY,CAAC,cAAc,EAAE,OAAO,CAAC,CAAC,CAAC;QAC/D,MAAM,MAAM,GAAG,IAAI,eAAe,CAAC;YACjC,iBAAiB,EAAE,IAAI;YACvB,mBAAmB,EAAE,IAAI;SAC1B,CAAC,CAAC;QACH,MAAM,MAAM,CAAC,UAAU,EAAE,CAAC;QAC1B,OAAO,MAAM,CAAC,WAAW,EAAE,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,IAAI,CAAC;IAC9C,CAAC;IAAC,MAAM,CAAC;QACP,OAAO,IAAI,CAAC;IACd,CAAC;AACH,CAAC;AAED;;;GAGG;AACH,MAAM,CAAC,KAAK,UAAU,OAAO,CAAC,MAAkB;IAQ9C,MAAM,UAAU,GAAG,MAAM,CAAC,UAAU,IAAI,qBAAqB,EAAE,CAAC;IAChE,MAAM,MAAM,GAAG,MAAM,CAAC,MAAM,IAAI,WAAW,CAAC;IAC5C,MAAM,IAAI,GAAG,OAAO,CAAC,MAAM,CAAC,QAAQ,EAAE,IAAI,CAAC,CAAC;IAC5C,MAAM,cAAc,GAAG,MAAM,CAAC,cAAc,IAAI,IAAI,CAAC,IAAI,EAAE,MAAM,EAAE,wBAAwB,CAAC,CAAC;IAE7F,+BAA+B;IAC/B,MAAM,WAAW,GAAG,MAAM,CAAC,eAAe,KAAK,KAAK,CAAC;IACrD,IAAI,eAAe,GAA2B,IAAI,CAAC;IACnD,MAAM,SAAS,GAAa,CAAC,aAAa,CAAC,CAAC;IAE5C,IAAI,WAAW,EAAE,CAAC;QAChB,eAAe,GAAG,MAAM,gBAAgB,CAAC,cAAc,CAAC,CAAC;QACzD,IAAI,eAAe,EAAE,CAAC;YACpB,SAAS,CAAC,IAAI,CAAC,mBAAmB,CAAC,CAAC;QACtC,CAAC;IACH,CAAC;IAED,oBAAoB;IACpB,MAAM,MAAM,GAAG,IAAI,SAAS,CAAC;QAC3B,QAAQ,EAAE,MAAM,CAAC,QAAQ;QACzB,eAAe,EAAE,eAAe,IAAI,SAAS;KAC9C,CAAC,CAAC;IACH,MAAM,SAAS,GAAG,MAAM,MAAM,CAAC,SAAS,EAAE,CAAC;IAE3C,IAAI,SAAS,KAAK,CAAC,EAAE,CAAC;QACpB,MAAM,IAAI,KAAK,CAAC,wBAAwB,MAAM,CAAC,QAAQ,EAAE,CAAC,CAAC;IAC7D,CAAC;IAED,kBAAkB;IAClB,MAAM,OAAO,GAAmB,EAAE,CAAC;IACnC,MAAM,eAAe,GAAG,eAAe,KAAK,IAAI,CAAC;IAEjD,KAAK,MAAM,MAAM,IAAI,MAAM,EAAE,CAAC;QAC5B,MAAM,MAAM,GAAG,eAAe;YAC5B,CAAC,CAAC,MAAM,kBAAkB,CAAC,MAAM,EAAE,MAAM,EAAE,UAAU,CAAC;YACtD,CAAC,CAAC,mBAAmB,CAAC,MAAM,EAAE,MAAM,EAAE,UAAU,CAAC,CAAC;QACpD,OAAO,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;IACvB,CAAC;IAED,2BAA2B;IAC3B,IAAI,eAAe,EAAE,CAAC;QACpB,MAAM,eAAe,CAAC,OAAO,EAAE,CAAC;IAClC,CAAC;IAED,iBAAiB;IACjB,MAAM,MAAM,GAAG,iBAAiB,CAAC,OAAO,CAAC,CAAC;IAC1C,MAAM,UAAU,GAAG,eAAe,CAAC,MAAM,EAAE,MAAM,CAAC,UAAU,CAAC,CAAC;IAC9D,MAAM,WAAW,GAAG,cAAc,EAAE,CAAC;IAErC,2BAA2B;IAC3B,MAAM,aAAa,GAAG,MAAM,CAAC,QAAQ,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC;IACzD,MAAM,WAAW,GAAG,kBAAkB,CAAC,OAAO,EAAE,aAAa,CAAC,CAAC;IAE/D,uCAAuC;IACvC,IAAI,MAAM,CAAC,UAAU,EAAE,CAAC;QACtB,MAAM,MAAM,GAAG;YACb,MAAM;YACN,UAAU;YACV,WAAW;YACX,WAAW;YACX,SAAS;YACT,MAAM,EAAE,WAAW;YACnB,KAAK,EAAE,SAAS;YAChB,UAAU;SACX,CAAC;QACF,aAAa,CAAC,MAAM,CAAC,UAAU,EAAE,IAAI,CAAC,SAAS,CAAC,MAAM,EAAE,IAAI,EAAE,CAAC,CAAC,CAAC,CAAC;IACpE,CAAC;IAED,OAAO,EAAE,MAAM,EAAE,UAAU,EAAE,WAAW,EAAE,SAAS,EAAE,WAAW,EAAE,UAAU,EAAE,CAAC;AACjF,CAAC;AAED,8EAA8E;AAC9E,kBAAkB;AAClB,8EAA8E;AAE9E,SAAS,aAAa,CAAC,CAAS;IAC9B,OAAO,GAAG,CAAC,CAAC,GAAG,GAAG,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,GAAG,CAAC;AACpC,CAAC;AAED,SAAS,QAAQ,CAAC,CAAS;IACzB,OAAO,GAAG,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,IAAI,CAAC;AAC7B,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,UAAU;IAC9B,MAAM,IAAI,GAAG,OAAO,CAAC,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,OAAO,IAAI,GAAG,EAAE,IAAI,EAAE,IAAI,CAAC,CAAC,CAAC;IACnE,MAAM,QAAQ,GAAG,IAAI,CAAC,IAAI,EAAE,OAAO,CAAC,CAAC;IACrC,MAAM,UAAU,GAAG,IAAI,CAAC,IAAI,EAAE,MAAM,EAAE,kBAAkB,CAAC,CAAC;IAE1D,OAAO,CAAC,GAAG,CAAC,oCAAoC,CAAC,CAAC;IAElD,MAAM,EAAE,MAAM,EAAE,UAAU,EAAE,WAAW,EAAE,SAAS,EAAE,WAAW,EAAE,UAAU,EAAE,GAAG,MAAM,OAAO,CAAC;QAC5F,QAAQ;QACR,UAAU;KACX,CAAC,CAAC;IAEH,eAAe;IACf,OAAO,CAAC,GAAG,CAAC,WAAW,WAAW,CAAC,KAAK,aAAa,WAAW,CAAC,OAAO,aAAa,WAAW,CAAC,MAAM,UAAU,CAAC,CAAC;IACnH,OAAO,CAAC,GAAG,CAAC,eAAe,MAAM,CAAC,IAAI,CAAC,WAAW,CAAC,UAAU,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;IAC7E,OAAO,CAAC,GAAG,CAAC,UAAU,SAAS,CAAC,IAAI,CAAC,KAAK,CAAC,EAAE,CAAC,CAAC;IAC/C,OAAO,CAAC,GAAG,CAAC,gBAAgB,UAAU,EAAE,CAAC,CAAC;IAE1C,kBAAkB;IAClB,OAAO,CAAC,GAAG,CAAC,mBAAmB,CAAC,CAAC;IACjC,OAAO,CAAC,GAAG,CAAC,iBAAiB,aAAa,CAAC,MAAM,CAAC,OAAO,CAAC,SAAS,CAAC,EAAE,CAAC,CAAC;IACxE,OAAO,CAAC,GAAG,CAAC,iBAAiB,aAAa,CAAC,MAAM,CAAC,OAAO,CAAC,MAAM,CAAC,EAAE,CAAC,CAAC;IACrE,OAAO,CAAC,GAAG,CAAC,iBAAiB,aAAa,CAAC,MAAM,CAAC,OAAO,CAAC,EAAE,CAAC,EAAE,CAAC,CAAC;IACjE,OAAO,CAAC,GAAG,CAAC,iBAAiB,aAAa,CAAC,MAAM,CAAC,OAAO,CAAC,QAAQ,CAAC,EAAE,CAAC,CAAC;IACvE,OAAO,CAAC,GAAG,CAAC,iBAAiB,aAAa,CAAC,MAAM,CAAC,OAAO,CAAC,MAAM,CAAC,EAAE,CAAC,CAAC;IACrE,OAAO,CAAC,GAAG,CAAC,oBAAoB,MAAM,CAAC,OAAO,CAAC,SAAS,CAAC,EAAE,OAAO,MAAM,CAAC,OAAO,CAAC,SAAS,CAAC,EAAE,OAAO,MAAM,CAAC,OAAO,CAAC,SAAS,CAAC,EAAE,OAAO,MAAM,CAAC,OAAO,CAAC,SAAS,CAAC,EAAE,EAAE,CAAC,CAAC;IAErK,UAAU;IACV,OAAO,CAAC,GAAG,CAAC,mBAAmB,CAAC,CAAC;IACjC,OAAO,CAAC,GAAG,CAAC,WAAW,QAAQ,CAAC,MAAM,CAAC,OAAO,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC;IACvD,OAAO,CAAC,GAAG,CAAC,WAAW,QAAQ,CAAC,MAAM,CAAC,OAAO,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC;IACvD,OAAO,CAAC,GAAG,CAAC,WAAW,QAAQ,CAAC,MAAM,CAAC,OAAO,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC;IACvD,OAAO,CAAC,GAAG,CAAC,WAAW,QAAQ,CAAC,MAAM,CAAC,OAAO,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;IACxD,OAAO,CAAC,GAAG,CAAC,WAAW,QAAQ,CAAC,MAAM,CAAC,OAAO,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC;IAEvD,eAAe;IACf,OAAO,CAAC,GAAG,CAAC,uBAAuB,CAAC,CAAC;IACrC,KAAK,MAAM,GAAG,IAAI,MAAM,CAAC,UAAU,EAAE,CAAC;QACpC,MAAM,MAAM,GAAG,GAAG,CAAC,aAAa,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,aAAa,GAAG,CAAC,aAAa,CAAC,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC;QAChG,MAAM,GAAG,GAAG,GAAG,CAAC,cAAc,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,SAAS,GAAG,CAAC,cAAc,CAAC,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC;QAC3F,OAAO,CAAC,GAAG,CAAC,KAAK,GAAG,CAAC,QAAQ,YAAY,aAAa,CAAC,GAAG,CAAC,OAAO,CAAC,MAAM,CAAC,cAAc,aAAa,CAAC,GAAG,CAAC,OAAO,CAAC,SAAS,CAAC,OAAO,aAAa,CAAC,GAAG,CAAC,OAAO,CAAC,EAAE,CAAC,GAAG,MAAM,GAAG,GAAG,EAAE,CAAC,CAAC;IACrL,CAAC;IAED,iBAAiB;IACjB,OAAO,CAAC,GAAG,CAAC,yBAAyB,CAAC,CAAC;IACvC,KAAK,MAAM,IAAI,IAAI,MAAM,CAAC,YAAY,EAAE,CAAC;QACvC,OAAO,CAAC,GAAG,CAAC,KAAK,IAAI,CAAC,UAAU,YAAY,aAAa,CAAC,IAAI,CAAC,OAAO,CAAC,MAAM,CAAC,cAAc,aAAa,CAAC,IAAI,CAAC,OAAO,CAAC,SAAS,CAAC,OAAO,aAAa,CAAC,IAAI,CAAC,OAAO,CAAC,EAAE,CAAC,EAAE,CAAC,CAAC;IAC5K,CAAC;IAED,iBAAiB;IACjB,IAAI,MAAM,CAAC,aAAa,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QACpC,OAAO,CAAC,GAAG,CAAC,yBAAyB,MAAM,CAAC,aAAa,CAAC,MAAM,OAAO,CAAC,CAAC;QACzE,KAAK,MAAM,CAAC,IAAI,MAAM,CAAC,aAAa,EAAE,CAAC;YACrC,OAAO,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC,EAAE,KAAK,CAAC,CAAC,QAAQ,IAAI,CAAC,CAAC,UAAU,IAAI,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC;QACrE,CAAC;IACH,CAAC;IAED,kBAAkB;IAClB,IAAI,MAAM,CAAC,cAAc,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QACrC,OAAO,CAAC,GAAG,CAAC,0BAA0B,MAAM,CAAC,cAAc,CAAC,MAAM,OAAO,CAAC,CAAC;QAC3E,KAAK,MAAM,EAAE,IAAI,MAAM,CAAC,cAAc,EAAE,CAAC;YACvC,OAAO,CAAC,GAAG,CAAC,MAAM,EAAE,CAAC,EAAE,YAAY,EAAE,CAAC,YAAY,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;QACnE,CAAC;IACH,CAAC;IAED,eAAe;IACf,OAAO,CAAC,GAAG,CAAC,wBAAwB,CAAC,CAAC;IACtC,OAAO,CAAC,GAAG,CAAC,yBAAyB,WAAW,CAAC,mBAAmB,EAAE,CAAC,CAAC;IACxE,OAAO,CAAC,GAAG,CAAC,kBAAkB,WAAW,CAAC,UAAU,EAAE,CAAC,CAAC;IACxD,OAAO,CAAC,GAAG,CAAC,wBAAwB,WAAW,CAAC,eAAe,EAAE,CAAC,CAAC;IAEnE,IAAI,WAAW,CAAC,QAAQ,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QACpC,OAAO,CAAC,GAAG,CAAC,kCAAkC,CAAC,CAAC;QAChD,KAAK,MAAM,IAAI,IAAI,WAAW,CAAC,QAAQ,CAAC,KAAK,CAAC,CAAC,EAAE,EAAE,CAAC,EAAE,CAAC;YACrD,MAAM,SAAS,GAAG,IAAI,CAAC,UAAU,GAAG,CAAC;gBACnC,CAAC,CAAC,aAAa,CAAC,IAAI,CAAC,OAAO,GAAG,IAAI,CAAC,UAAU,CAAC;gBAC/C,CAAC,CAAC,KAAK,CAAC;YACV,OAAO,CAAC,GAAG,CAAC,OAAO,IAAI,CAAC,MAAM,aAAa,IAAI,CAAC,UAAU,OAAO,IAAI,CAAC,OAAO,OAAO,IAAI,CAAC,OAAO,cAAc,SAAS,YAAY,IAAI,CAAC,aAAa,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC;QACtK,CAAC;IACH,CAAC;IAED,IAAI,WAAW,CAAC,SAAS,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QACrC,OAAO,CAAC,GAAG,CAAC,6CAA6C,CAAC,CAAC;QAC3D,KAAK,MAAM,IAAI,IAAI,WAAW,CAAC,SAAS,CAAC,KAAK,CAAC,CAAC,EAAE,EAAE,CAAC,EAAE,CAAC;YACtD,OAAO,CAAC,GAAG,CAAC,OAAO,IAAI,CAAC,MAAM,aAAa,IAAI,CAAC,UAAU,OAAO,IAAI,CAAC,OAAO,OAAO,IAAI,CAAC,OAAO,gBAAgB,IAAI,CAAC,UAAU,CAAC,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;QACjJ,CAAC;IACH,CAAC;IAED,IAAI,WAAW,CAAC,iBAAiB,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QAC7C,OAAO,CAAC,GAAG,CAAC,0BAA0B,WAAW,CAAC,iBAAiB,CAAC,MAAM,IAAI,CAAC,CAAC;QAChF,KAAK,MAAM,EAAE,IAAI,WAAW,CAAC,iBAAiB,CAAC,KAAK,CAAC,CAAC,EAAE,EAAE,CAAC,EAAE,CAAC;YAC5D,OAAO,CAAC,GAAG,CAAC,OAAO,EAAE,EAAE,CAAC,CAAC;QAC3B,CAAC;QACD,IAAI,WAAW,CAAC,iBAAiB,CAAC,MAAM,GAAG,EAAE,EAAE,CAAC;YAC9C,OAAO,CAAC,GAAG,CAAC,eAAe,WAAW,CAAC,iBAAiB,CAAC,MAAM,GAAG,EAAE,OAAO,CAAC,CAAC;QAC/E,CAAC;IACH,CAAC;IAED,mBAAmB;IACnB,OAAO,CAAC,GAAG,CAAC,4BAA4B,CAAC,CAAC;IAC1C,IAAI,UAAU,CAAC,MAAM,EAAE,CAAC;QACtB,OAAO,CAAC,GAAG,CAAC,UAAU,CAAC,CAAC;IAC1B,CAAC;SAAM,CAAC;QACN,OAAO,CAAC,GAAG,CAAC,WAAW,CAAC,CAAC;QACzB,KAAK,MAAM,CAAC,IAAI,UAAU,CAAC,UAAU,EAAE,CAAC;YACtC,OAAO,CAAC,GAAG,CAAC,SAAS,CAAC,EAAE,CAAC,CAAC;QAC5B,CAAC;IACH,CAAC;IAED,OAAO,CAAC,GAAG,CAAC,sBAAsB,UAAU,EAAE,CAAC,CAAC;IAEhD,yEAAyE;IACzE,2EAA2E;IAC3E,MAAM,EAAE,eAAe,EAAE,GAAG,8BAA8B,CAAC,MAAM,EAAE;QACjE,MAAM,EAAE,eAAe;QACvB,cAAc,EAAE,UAAU;QAC1B,KAAK,EACH,2EAA2E;YAC3E,gBAAgB,UAAU,sDAAsD;YAChF,4EAA4E;YAC5E,kFAAkF;KACrF,CAAC,CAAC;IACH,OAAO,CAAC,GAAG,CAAC,gBAAgB,eAAe,EAAE,CAAC,CAAC;IAC/C,OAAO,CAAC,GAAG,CAAC,SAAS,CAAC,CAAC;IAEvB,IAAI,CAAC,UAAU,CAAC,MAAM,EAAE,CAAC;QACvB,OAAO,CAAC,QAAQ,GAAG,CAAC,CAAC;IACvB,CAAC;AACH,CAAC"}
+{"version":3,"file":"eval-harness.js","sourceRoot":"","sources":["../../src/eval/eval-harness.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;GAYG;AAEH,OAAO,EAAE,OAAO,EAAE,IAAI,EAAE,MAAM,WAAW,CAAC;AAC1C,OAAO,EAAE,UAAU,EAAE,YAAY,EAAE,aAAa,EAAE,MAAM,SAAS,CAAC;AAClE,OAAO,EAAE,8BAA8B,EAAE,MAAM,qCAAqC,CAAC;AACrF,OAAO,EAAE,iBAAiB,EAAE,WAAW,EAA0B,MAAM,gCAAgC,CAAC;AACxG,OAAO,EAAE,SAAS,EAAE,MAAM,cAAc,CAAC;AAEzC,OAAO,EAAE,YAAY,EAAE,mBAAmB,EAAE,MAAM,oBAAoB,CAAC;AACvE,OAAO,EAAE,eAAe,EAAE,MAAM,yBAAyB,CAAC;AAG1D,OAAO,EAAE,WAAW,EAAE,cAAc,EAAE,MAAM,aAAa,CAAC;AAE1D,OAAO,EAAE,iBAAiB,EAAE,eAAe,EAAE,MAAM,cAAc,CAAC;AAElE,OAAO,EAAE,kBAAkB,EAAE,MAAM,mBAAmB,CAAC;AAmCvD,oFAAoF;AACpF,MAAM,UAAU,qBAAqB;IACnC,MAAM,GAAG,GAAG,OAAO,CAAC,GAAG,CAAC,sBAAsB,CAAC,EAAE,IAAI,EAAE,CAAC,WAAW,EAAE,CAAC;IACtE,IAAI,GAAG,KAAK,QAAQ,IAAI,GAAG,KAAK,MAAM,IAAI,GAAG,KAAK,WAAW;QAAE,OAAO,GAAG,CAAC;IAC1E,OAAO,WAAW,CAAC;AACrB,CAAC;AA0BD;;;;;;;GAOG;AACH,SAAS,aAAa,CAAC,MAAoB;IACzC,OAAO;QACL,IAAI,EAAE,MAAM,CAAC,SAAS;QACtB,OAAO,EAAE,MAAM,CAAC,IAAI;QACpB,SAAS,EAAE,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE;QACnC,MAAM,EAAE,MAAM,CAAC,MAAM;KACtB,CAAC;AACJ,CAAC;AAED;;;;;;;;GAQG;AACH,SAAS,YAAY,CACnB,MAAoB,EACpB,IAAwB;IAExB,MAAM,OAAO,GAAG,aAAa,CAAC,MAAM,CAAC,CAAC;IACtC,IAAI,IAAI,KAAK,QAAQ;QAAE,OAAO,CAAC,OAAO,CAAC,CAAC;IACxC,IAAI,IAAI,KAAK,MAAM;QAAE,OAAO,CAAC,OAAO,EAAE,mBAAmB,CAAC,OAAO,CAAC,CAAC,CAAC;IACpE,OAAO,CAAC,OAAO,EAAE,GAAG,YAAY,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC;AACrE,CAAC;AAED,wEAAwE;AACxE,SAAS,aAAa,CAAC,IAAwB;IAC7C,OAAO,IAAI,KAAK,WAAW,CAAC;AAC9B,CAAC;AAED,uFAAuF;AACvF,SAAS,iBAAiB,CACxB,MAAiB,EACjB,MAAoB,EACpB,IAAwB;IAExB,MAAM,KAAK,GAAG,IAAI,GAAG,EAAkB,CAAC;IACxC,KAAK,MAAM,KAAK,IAAI,YAAY,CAAC,MAAM,EAAE,IAAI,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE,CAAC;QACxD,KAAK,MAAM,CAAC,IAAI,MAAM,CAAC,QAAQ,CAAC,KAAK,CAAC,EAAE,CAAC;YACvC,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,IAAI,CAAC,EAAE,EAAE,IAAI,CAAC,GAAG,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,IAAI,CAAC,EAAE,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC,UAAU,CAAC,CAAC,CAAC;QAC1E,CAAC;IACH,CAAC;IACD,IAAI,aAAa,CAAC,IAAI,CAAC,EAAE,CAAC;QACxB,KAAK,MAAM,CAAC,IAAI,MAAM,CAAC,SAAS,CAAC,MAAM,CAAC,IAAI,CAAC,EAAE,CAAC;YAC9C,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,IAAI,CAAC,EAAE,EAAE,IAAI,CAAC,GAAG,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,IAAI,CAAC,EAAE,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC,UAAU,CAAC,CAAC,CAAC;QAC1E,CAAC;IACH,CAAC;IACD,OAAO,KAAK,CAAC;AACf,CAAC;AAED;;;GAGG;AACH,SAAS,mBAAmB,CAC1B,MAAiB,EACjB,MAAoB,EACpB,IAAwB;IAExB,IAAI,CAAC;QACH,MAAM,KAAK,GAAG,WAAW,CAAC,GAAG,EAAE,CAAC;QAChC,MAAM,MAAM,GAAG,IAAI,GAAG,EAAkB,CAAC;QACzC,KAAK,MAAM,KAAK,IAAI,YAAY,CAAC,MAAM,EAAE,IAAI,CAAC,EAAE,CAAC;YAC/C,KAAK,MAAM,CAAC,IAAI,MAAM,CAAC,QAAQ,CAAC,KAAK,CAAC,EAAE,CAAC;gBACvC,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,IAAI,CAAC,EAAE,EAAE,IAAI,CAAC,GAAG,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,IAAI,CAAC,EAAE,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC,UAAU,CAAC,CAAC,CAAC;YAC5E,CAAC;QACH,CAAC;QACD,IAAI,aAAa,CAAC,IAAI,CAAC,EAAE,CAAC;YACxB,KAAK,MAAM,CAAC,IAAI,MAAM,CAAC,SAAS,CAAC,MAAM,CAAC,IAAI,CAAC,EAAE,CAAC;gBAC9C,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,IAAI,CAAC,EAAE,EAAE,IAAI,CAAC,GAAG,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,IAAI,CAAC,EAAE,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC,UAAU,CAAC,CAAC,CAAC;YAC5E,CAAC;QACH,CAAC;QACD,MAAM,SAAS,GAAG,WAAW,CAAC,GAAG,EAAE,GAAG,KAAK,CAAC;QAE5C,OAAO;YACL,EAAE,EAAE,MAAM,CAAC,EAAE;YACb,QAAQ,EAAE,MAAM,CAAC,QAAQ;YACzB,iBAAiB,EAAE,MAAM,CAAC,iBAAiB;YAC3C,eAAe,EAAE,MAAM,CAAC,IAAI,GAAG,CAAC;YAChC,YAAY,EAAE,CAAC,GAAG,MAAM,CAAC,IAAI,EAAE,CAAC;YAChC,UAAU,EAAE,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,GAAG,MAAM,CAAC,MAAM,EAAE,CAAC;YAC3C,SAAS;YACT,UAAU,EAAE,MAAM,CAAC,UAAU;YAC7B,IAAI,EAAE,MAAM,CAAC,IAAI;SAClB,CAAC;IACJ,CAAC;IAAC,MAAM,CAAC;QACP,OAAO;YACL,EAAE,EAAE,MAAM,CAAC,EAAE;YACb,QAAQ,EAAE,MAAM,CAAC,QAAQ;YACzB,iBAAiB,EAAE,MAAM,CAAC,iBAAiB;YAC3C,eAAe,EAAE,KAAK;YACtB,YAAY,EAAE,EAAE;YAChB,UAAU,EAAE,CAAC;YACb,SAAS,EAAE,CAAC;YACZ,UAAU,EAAE,MAAM,CAAC,UAAU;YAC7B,IAAI,EAAE,MAAM,CAAC,IAAI;SAClB,CAAC;IACJ,CAAC;AACH,CAAC;AAED;;;;;;;;;;;GAWG;AACH,KAAK,UAAU,kBAAkB,CAC/B,MAAiB,EACjB,MAAoB,EACpB,IAAwB;IAExB,IAAI,CAAC;QACH,MAAM,KAAK,GAAG,WAAW,CAAC,GAAG,EAAE,CAAC;QAChC,MAAM,EAAE,OAAO,EAAE,GAAG,MAAM,MAAM,CAAC,mBAAmB,CAAC,aAAa,CAAC,MAAM,CAAC,CAAC,CAAC;QAC5E,MAAM,MAAM,GAAG,IAAI,GAAG,CACpB,OAAO,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,EAAE,EAAE,CAAC,CAAC,UAAU,CAAC,CAAC,CACtD,CAAC;QACF,KAAK,MAAM,CAAC,EAAE,EAAE,UAAU,CAAC,IAAI,iBAAiB,CAAC,MAAM,EAAE,MAAM,EAAE,IAAI,CAAC,EAAE,CAAC;YACvE,MAAM,CAAC,GAAG,CAAC,EAAE,EAAE,IAAI,CAAC,GAAG,CAAC,MAAM,CAAC,GAAG,CAAC,EAAE,CAAC,IAAI,CAAC,EAAE,UAAU,CAAC,CAAC,CAAC;QAC5D,CAAC;QACD,MAAM,SAAS,GAAG,WAAW,CAAC,GAAG,EAAE,GAAG,KAAK,CAAC;QAE5C,OAAO;YACL,EAAE,EAAE,MAAM,CAAC,EAAE;YACb,QAAQ,EAAE,MAAM,CAAC,QAAQ;YACzB,iBAAiB,EAAE,MAAM,CAAC,iBAAiB;YAC3C,eAAe,EAAE,MAAM,CAAC,IAAI,GAAG,CAAC;YAChC,YAAY,EAAE,CAAC,GAAG,MAAM,CAAC,IAAI,EAAE,CAAC;YAChC,UAAU,EAAE,IAAI,CAAC,GAAG,CAAC,OAAO,CAAC,iBAAiB,EAAE,CAAC,EAAE,GAAG,MAAM,CAAC,MAAM,EAAE,CAAC;YACtE,SAAS;YACT,UAAU,EAAE,MAAM,CAAC,UAAU;YAC7B,IAAI,EAAE,MAAM,CAAC,IAAI;SAClB,CAAC;IACJ,CAAC;IAAC,MAAM,CAAC;QACP,OAAO;YACL,EAAE,EAAE,MAAM,CAAC,EAAE;YACb,QAAQ,EAAE,MAAM,CAAC,QAAQ;YACzB,iBAAiB,EAAE,MAAM,CAAC,iBAAiB;YAC3C,eAAe,EAAE,KAAK;YACtB,YAAY,EAAE,EAAE;YAChB,UAAU,EAAE,CAAC;YACb,SAAS,EAAE,CAAC;YACZ,UAAU,EAAE,MAAM,CAAC,UAAU;YAC7B,IAAI,EAAE,MAAM,CAAC,IAAI;SAClB,CAAC;IACJ,CAAC;AACH,CAAC;AAED,yFAAyF;AACzF,MAAM,CAAC,MAAM,mCAAmC,GAAG,IAAI,CAAC;AAExD;;GAEG;AACH,KAAK,UAAU,gBAAgB,CAAC,cAAsB;IACpD,IAAI,CAAC,UAAU,CAAC,cAAc,CAAC;QAAE,OAAO,IAAI,CAAC;IAE7C,IAAI,CAAC;QACH,MAAM,IAAI,GAAG,IAAI,CAAC,KAAK,CAAC,YAAY,CAAC,cAAc,EAAE,OAAO,CAAC,CAAC,CAAC;QAC/D,MAAM,MAAM,GAAG,IAAI,eAAe,CAAC;YACjC,iBAAiB,EAAE,IAAI;YACvB,mBAAmB,EAAE,mCAAmC;SACzD,CAAC,CAAC;QACH,MAAM,MAAM,CAAC,UAAU,EAAE,CAAC;QAC1B,OAAO,MAAM,CAAC,WAAW,EAAE,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,IAAI,CAAC;IAC9C,CAAC;IAAC,MAAM,CAAC;QACP,OAAO,IAAI,CAAC;IACd,CAAC;AACH,CAAC;AAED;;;GAGG;AACH,MAAM,CAAC,KAAK,UAAU,OAAO,CAAC,MAAkB;IAY9C,MAAM,UAAU,GAAG,MAAM,CAAC,UAAU,IAAI,qBAAqB,EAAE,CAAC;IAChE,MAAM,MAAM,GAAG,MAAM,CAAC,MAAM,IAAI,WAAW,CAAC;IAC5C,MAAM,IAAI,GAAG,OAAO,CAAC,MAAM,CAAC,QAAQ,EAAE,IAAI,CAAC,CAAC;IAC5C,MAAM,cAAc,GAAG,MAAM,CAAC,cAAc,IAAI,IAAI,CAAC,IAAI,EAAE,MAAM,EAAE,wBAAwB,CAAC,CAAC;IAE7F,+BAA+B;IAC/B,MAAM,WAAW,GAAG,MAAM,CAAC,eAAe,KAAK,KAAK,CAAC;IACrD,IAAI,eAAe,GAA2B,IAAI,CAAC;IACnD,MAAM,SAAS,GAAa,CAAC,aAAa,CAAC,CAAC;IAE5C,IAAI,WAAW,EAAE,CAAC;QAChB,eAAe,GAAG,MAAM,gBAAgB,CAAC,cAAc,CAAC,CAAC;QACzD,IAAI,eAAe,EAAE,CAAC;YACpB,SAAS,CAAC,IAAI,CAAC,mBAAmB,CAAC,CAAC;QACtC,CAAC;IACH,CAAC;IAED,oBAAoB;IACpB,MAAM,MAAM,GAAG,IAAI,SAAS,CAAC;QAC3B,QAAQ,EAAE,MAAM,CAAC,QAAQ;QACzB,eAAe,EAAE,eAAe,IAAI,SAAS;QAC7C,IAAI,EAAE,MAAM,CAAC,IAAI;KAClB,CAAC,CAAC;IACH,MAAM,SAAS,GAAG,MAAM,MAAM,CAAC,SAAS,EAAE,CAAC;IAE3C,IAAI,SAAS,KAAK,CAAC,EAAE,CAAC;QACpB,MAAM,IAAI,KAAK,CAAC,wBAAwB,MAAM,CAAC,QAAQ,EAAE,CAAC,CAAC;IAC7D,CAAC;IAED,kBAAkB;IAClB,MAAM,OAAO,GAAmB,EAAE,CAAC;IACnC,MAAM,eAAe,GAAG,eAAe,KAAK,IAAI,CAAC;IAEjD,KAAK,MAAM,MAAM,IAAI,MAAM,EAAE,CAAC;QAC5B,MAAM,MAAM,GAAG,eAAe;YAC5B,CAAC,CAAC,MAAM,kBAAkB,CAAC,MAAM,EAAE,MAAM,EAAE,UAAU,CAAC;YACtD,CAAC,CAAC,mBAAmB,CAAC,MAAM,EAAE,MAAM,EAAE,UAAU,CAAC,CAAC;QACpD,OAAO,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;IACvB,CAAC;IAED,2BAA2B;IAC3B,IAAI,eAAe,EAAE,CAAC;QACpB,MAAM,eAAe,CAAC,OAAO,EAAE,CAAC;IAClC,CAAC;IAED,iBAAiB;IACjB,MAAM,MAAM,GAAG,iBAAiB,CAAC,OAAO,CAAC,CAAC;IAC1C,MAAM,UAAU,GAAG,eAAe,CAAC,MAAM,EAAE,MAAM,CAAC,UAAU,CAAC,CAAC;IAC9D,MAAM,WAAW,GAAG,cAAc,EAAE,CAAC;IAErC,2BAA2B;IAC3B,MAAM,aAAa,GAAG,MAAM,CAAC,QAAQ,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC;IACzD,MAAM,WAAW,GAAG,kBAAkB,CAAC,OAAO,EAAE,aAAa,CAAC,CAAC;IAE/D,uCAAuC;IACvC,IAAI,MAAM,CAAC,UAAU,EAAE,CAAC;QACtB,MAAM,MAAM,GAAG;YACb,MAAM;YACN,UAAU;YACV,WAAW;YACX,WAAW;YACX,SAAS;YACT,MAAM,EAAE,WAAW;YACnB,KAAK,EAAE,SAAS;YAChB,UAAU;SACX,CAAC;QACF,aAAa,CAAC,MAAM,CAAC,UAAU,EAAE,IAAI,CAAC,SAAS,CAAC,MAAM,EAAE,IAAI,EAAE,CAAC,CAAC,CAAC,CAAC;IACpE,CAAC;IAED,MAAM,IAAI,GAAG,MAAM,CAAC,OAAO,EAAE,CAAC;IAC9B,OAAO;QACL,MAAM;QACN,UAAU;QACV,WAAW;QACX,SAAS;QACT,WAAW;QACX,UAAU;QACV,IAAI;QACJ,WAAW,EAAE,WAAW,CAAC,MAAM,CAAC,QAAQ,EAAE,EAAE,IAAI,CAAC;KAClD,CAAC;AACJ,CAAC;AAED;;;;;;;GAOG;AACH,MAAM,UAAU,mBAAmB,CACjC,GAMC,EACD,QAIC;IAED,IAAI,GAAG,CAAC,IAAI,KAAK,SAAS,EAAE,CAAC;QAC3B,MAAM,IAAI,KAAK,CAAC,gDAAgD,GAAG,CAAC,IAAI,oBAAoB,CAAC,CAAC;IAChG,CAAC;IACD,MAAM,OAAO,GAAG;QACd,GAAG,CAAC,MAAM,CAAC,UAAU,KAAK,QAAQ,CAAC,MAAM,CAAC,UAAU,CAAC,CAAC,CAAC,aAAa,CAAC,CAAC,CAAC,IAAI;QAC3E,GAAG,CAAC,UAAU,KAAK,QAAQ,CAAC,UAAU,CAAC,CAAC,CAAC,aAAa,CAAC,CAAC,CAAC,IAAI;QAC7D,GAAG,CAAC,SAAS,CAAC,IAAI,CAAC,GAAG,CAAC,KAAK,QAAQ,CAAC,SAAS,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,IAAI;KAC1E,CAAC,MAAM,CAAC,CAAC,CAAC,EAAe,EAAE,CAAC,CAAC,KAAK,IAAI,CAAC,CAAC;IACzC,IAAI,OAAO,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QACvB,MAAM,IAAI,KAAK,CAAC,qEAAqE,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;IAC7G,CAAC;IACD,MAAM,EAAE,EAAE,EAAE,EAAE,EAAE,GAAG,GAAG,CAAC,MAAM,CAAC,OAAO,CAAC,SAAS,CAAC;IAChD,OAAO,iBAAiB,CAAC,EAAE,QAAQ,EAAE,EAAE,EAAE,OAAO,EAAE,EAAE,GAAG,EAAE,EAAE,WAAW,EAAE,GAAG,CAAC,WAAW,EAAE,CAAC,CAAC;AAC7F,CAAC;AAED,8EAA8E;AAC9E,kBAAkB;AAClB,8EAA8E;AAE9E,SAAS,aAAa,CAAC,CAAS;IAC9B,OAAO,GAAG,CAAC,CAAC,GAAG,GAAG,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,GAAG,CAAC;AACpC,CAAC;AAED,SAAS,QAAQ,CAAC,CAAS;IACzB,OAAO,GAAG,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,IAAI,CAAC;AAC7B,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,UAAU;IAC9B,MAAM,IAAI,GAAG,OAAO,CAAC,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,OAAO,IAAI,GAAG,EAAE,IAAI,EAAE,IAAI,CAAC,CAAC,CAAC;IACnE,MAAM,QAAQ,GAAG,IAAI,CAAC,IAAI,EAAE,OAAO,CAAC,CAAC;IACrC,MAAM,UAAU,GAAG,IAAI,CAAC,IAAI,EAAE,MAAM,EAAE,kBAAkB,CAAC,CAAC;IAE1D,OAAO,CAAC,GAAG,CAAC,oCAAoC,CAAC,CAAC;IAElD,MAAM,EAAE,MAAM,EAAE,UAAU,EAAE,WAAW,EAAE,SAAS,EAAE,WAAW,EAAE,UAAU,EAAE,GAAG,MAAM,OAAO,CAAC;QAC5F,QAAQ;QACR,UAAU;KACX,CAAC,CAAC;IAEH,eAAe;IACf,OAAO,CAAC,GAAG,CAAC,WAAW,WAAW,CAAC,KAAK,aAAa,WAAW,CAAC,OAAO,aAAa,WAAW,CAAC,MAAM,UAAU,CAAC,CAAC;IACnH,OAAO,CAAC,GAAG,CAAC,eAAe,MAAM,CAAC,IAAI,CAAC,WAAW,CAAC,UAAU,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;IAC7E,OAAO,CAAC,GAAG,CAAC,UAAU,SAAS,CAAC,IAAI,CAAC,KAAK,CAAC,EAAE,CAAC,CAAC;IAC/C,OAAO,CAAC,GAAG,CAAC,gBAAgB,UAAU,EAAE,CAAC,CAAC;IAE1C,kBAAkB;IAClB,OAAO,CAAC,GAAG,CAAC,mBAAmB,CAAC,CAAC;IACjC,OAAO,CAAC,GAAG,CAAC,iBAAiB,aAAa,CAAC,MAAM,CAAC,OAAO,CAAC,SAAS,CAAC,EAAE,CAAC,CAAC;IACxE,OAAO,CAAC,GAAG,CAAC,iBAAiB,aAAa,CAAC,MAAM,CAAC,OAAO,CAAC,MAAM,CAAC,EAAE,CAAC,CAAC;IACrE,OAAO,CAAC,GAAG,CAAC,iBAAiB,aAAa,CAAC,MAAM,CAAC,OAAO,CAAC,EAAE,CAAC,EAAE,CAAC,CAAC;IACjE,OAAO,CAAC,GAAG,CAAC,iBAAiB,aAAa,CAAC,MAAM,CAAC,OAAO,CAAC,QAAQ,CAAC,EAAE,CAAC,CAAC;IACvE,OAAO,CAAC,GAAG,CAAC,iBAAiB,aAAa,CAAC,MAAM,CAAC,OAAO,CAAC,MAAM,CAAC,EAAE,CAAC,CAAC;IACrE,OAAO,CAAC,GAAG,CAAC,oBAAoB,MAAM,CAAC,OAAO,CAAC,SAAS,CAAC,EAAE,OAAO,MAAM,CAAC,OAAO,CAAC,SAAS,CAAC,EAAE,OAAO,MAAM,CAAC,OAAO,CAAC,SAAS,CAAC,EAAE,OAAO,MAAM,CAAC,OAAO,CAAC,SAAS,CAAC,EAAE,EAAE,CAAC,CAAC;IAErK,UAAU;IACV,OAAO,CAAC,GAAG,CAAC,mBAAmB,CAAC,CAAC;IACjC,OAAO,CAAC,GAAG,CAAC,WAAW,QAAQ,CAAC,MAAM,CAAC,OAAO,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC;IACvD,OAAO,CAAC,GAAG,CAAC,WAAW,QAAQ,CAAC,MAAM,CAAC,OAAO,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC;IACvD,OAAO,CAAC,GAAG,CAAC,WAAW,QAAQ,CAAC,MAAM,CAAC,OAAO,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC;IACvD,OAAO,CAAC,GAAG,CAAC,WAAW,QAAQ,CAAC,MAAM,CAAC,OAAO,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;IACxD,OAAO,CAAC,GAAG,CAAC,WAAW,QAAQ,CAAC,MAAM,CAAC,OAAO,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC;IAEvD,eAAe;IACf,OAAO,CAAC,GAAG,CAAC,uBAAuB,CAAC,CAAC;IACrC,KAAK,MAAM,GAAG,IAAI,MAAM,CAAC,UAAU,EAAE,CAAC;QACpC,MAAM,MAAM,GAAG,GAAG,CAAC,aAAa,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,aAAa,GAAG,CAAC,aAAa,CAAC,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC;QAChG,MAAM,GAAG,GAAG,GAAG,CAAC,cAAc,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,SAAS,GAAG,CAAC,cAAc,CAAC,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC;QAC3F,OAAO,CAAC,GAAG,CAAC,KAAK,GAAG,CAAC,QAAQ,YAAY,aAAa,CAAC,GAAG,CAAC,OAAO,CAAC,MAAM,CAAC,cAAc,aAAa,CAAC,GAAG,CAAC,OAAO,CAAC,SAAS,CAAC,OAAO,aAAa,CAAC,GAAG,CAAC,OAAO,CAAC,EAAE,CAAC,GAAG,MAAM,GAAG,GAAG,EAAE,CAAC,CAAC;IACrL,CAAC;IAED,iBAAiB;IACjB,OAAO,CAAC,GAAG,CAAC,yBAAyB,CAAC,CAAC;IACvC,KAAK,MAAM,IAAI,IAAI,MAAM,CAAC,YAAY,EAAE,CAAC;QACvC,OAAO,CAAC,GAAG,CAAC,KAAK,IAAI,CAAC,UAAU,YAAY,aAAa,CAAC,IAAI,CAAC,OAAO,CAAC,MAAM,CAAC,cAAc,aAAa,CAAC,IAAI,CAAC,OAAO,CAAC,SAAS,CAAC,OAAO,aAAa,CAAC,IAAI,CAAC,OAAO,CAAC,EAAE,CAAC,EAAE,CAAC,CAAC;IAC5K,CAAC;IAED,iBAAiB;IACjB,IAAI,MAAM,CAAC,aAAa,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QACpC,OAAO,CAAC,GAAG,CAAC,yBAAyB,MAAM,CAAC,aAAa,CAAC,MAAM,OAAO,CAAC,CAAC;QACzE,KAAK,MAAM,CAAC,IAAI,MAAM,CAAC,aAAa,EAAE,CAAC;YACrC,OAAO,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC,EAAE,KAAK,CAAC,CAAC,QAAQ,IAAI,CAAC,CAAC,UAAU,IAAI,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC;QACrE,CAAC;IACH,CAAC;IAED,kBAAkB;IAClB,IAAI,MAAM,CAAC,cAAc,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QACrC,OAAO,CAAC,GAAG,CAAC,0BAA0B,MAAM,CAAC,cAAc,CAAC,MAAM,OAAO,CAAC,CAAC;QAC3E,KAAK,MAAM,EAAE,IAAI,MAAM,CAAC,cAAc,EAAE,CAAC;YACvC,OAAO,CAAC,GAAG,CAAC,MAAM,EAAE,CAAC,EAAE,YAAY,EAAE,CAAC,YAAY,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;QACnE,CAAC;IACH,CAAC;IAED,eAAe;IACf,OAAO,CAAC,GAAG,CAAC,wBAAwB,CAAC,CAAC;IACtC,OAAO,CAAC,GAAG,CAAC,yBAAyB,WAAW,CAAC,mBAAmB,EAAE,CAAC,CAAC;IACxE,OAAO,CAAC,GAAG,CAAC,kBAAkB,WAAW,CAAC,UAAU,EAAE,CAAC,CAAC;IACxD,OAAO,CAAC,GAAG,CAAC,wBAAwB,WAAW,CAAC,eAAe,EAAE,CAAC,CAAC;IAEnE,IAAI,WAAW,CAAC,QAAQ,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QACpC,OAAO,CAAC,GAAG,CAAC,kCAAkC,CAAC,CAAC;QAChD,KAAK,MAAM,IAAI,IAAI,WAAW,CAAC,QAAQ,CAAC,KAAK,CAAC,CAAC,EAAE,EAAE,CAAC,EAAE,CAAC;YACrD,MAAM,SAAS,GAAG,IAAI,CAAC,UAAU,GAAG,CAAC;gBACnC,CAAC,CAAC,aAAa,CAAC,IAAI,CAAC,OAAO,GAAG,IAAI,CAAC,UAAU,CAAC;gBAC/C,CAAC,CAAC,KAAK,CAAC;YACV,OAAO,CAAC,GAAG,CAAC,OAAO,IAAI,CAAC,MAAM,aAAa,IAAI,CAAC,UAAU,OAAO,IAAI,CAAC,OAAO,OAAO,IAAI,CAAC,OAAO,cAAc,SAAS,YAAY,IAAI,CAAC,aAAa,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC;QACtK,CAAC;IACH,CAAC;IAED,IAAI,WAAW,CAAC,SAAS,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QACrC,OAAO,CAAC,GAAG,CAAC,6CAA6C,CAAC,CAAC;QAC3D,KAAK,MAAM,IAAI,IAAI,WAAW,CAAC,SAAS,CAAC,KAAK,CAAC,CAAC,EAAE,EAAE,CAAC,EAAE,CAAC;YACtD,OAAO,CAAC,GAAG,CAAC,OAAO,IAAI,CAAC,MAAM,aAAa,IAAI,CAAC,UAAU,OAAO,IAAI,CAAC,OAAO,OAAO,IAAI,CAAC,OAAO,gBAAgB,IAAI,CAAC,UAAU,CAAC,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;QACjJ,CAAC;IACH,CAAC;IAED,IAAI,WAAW,CAAC,iBAAiB,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QAC7C,OAAO,CAAC,GAAG,CAAC,0BAA0B,WAAW,CAAC,iBAAiB,CAAC,MAAM,IAAI,CAAC,CAAC;QAChF,KAAK,MAAM,EAAE,IAAI,WAAW,CAAC,iBAAiB,CAAC,KAAK,CAAC,CAAC,EAAE,EAAE,CAAC,EAAE,CAAC;YAC5D,OAAO,CAAC,GAAG,CAAC,OAAO,EAAE,EAAE,CAAC,CAAC;QAC3B,CAAC;QACD,IAAI,WAAW,CAAC,iBAAiB,CAAC,MAAM,GAAG,EAAE,EAAE,CAAC;YAC9C,OAAO,CAAC,GAAG,CAAC,eAAe,WAAW,CAAC,iBAAiB,CAAC,MAAM,GAAG,EAAE,OAAO,CAAC,CAAC;QAC/E,CAAC;IACH,CAAC;IAED,mBAAmB;IACnB,OAAO,CAAC,GAAG,CAAC,4BAA4B,CAAC,CAAC;IAC1C,IAAI,UAAU,CAAC,MAAM,EAAE,CAAC;QACtB,OAAO,CAAC,GAAG,CAAC,UAAU,CAAC,CAAC;IAC1B,CAAC;SAAM,CAAC;QACN,OAAO,CAAC,GAAG,CAAC,WAAW,CAAC,CAAC;QACzB,KAAK,MAAM,CAAC,IAAI,UAAU,CAAC,UAAU,EAAE,CAAC;YACtC,OAAO,CAAC,GAAG,CAAC,SAAS,CAAC,EAAE,CAAC,CAAC;QAC5B,CAAC;IACH,CAAC;IAED,OAAO,CAAC,GAAG,CAAC,sBAAsB,UAAU,EAAE,CAAC,CAAC;IAEhD,8EAA8E;IAC9E,8EAA8E;IAC9E,MAAM,WAAW,GAAG,mBAAmB,CAAC,MAAM,OAAO,CAAC,EAAE,QAAQ,EAAE,IAAI,EAAE,SAAS,EAAE,CAAC,EAAE;QACpF,MAAM;QACN,UAAU;QACV,SAAS;KACV,CAAC,CAAC;IACH,OAAO,CAAC,GAAG,CACT,iBAAiB,WAAW,CAAC,QAAQ,IAAI,WAAW,CAAC,OAAO,cAAc,aAAa,CAAC,WAAW,CAAC,MAAM,CAAC,GAAG;QAC5G,IAAI,WAAW,CAAC,aAAa,iBAAiB,CACjD,CAAC;IAEF,yEAAyE;IACzE,2EAA2E;IAC3E,MAAM,EAAE,eAAe,EAAE,GAAG,8BAA8B,CAAC,MAAM,EAAE;QACjE,MAAM,EAAE,eAAe;QACvB,cAAc,EAAE,UAAU;QAC1B,WAAW;QACX,KAAK,EACH,2EAA2E;YAC3E,gBAAgB,UAAU,sDAAsD;YAChF,4EAA4E;YAC5E,kFAAkF;KACrF,CAAC,CAAC;IACH,OAAO,CAAC,GAAG,CAAC,gBAAgB,eAAe,EAAE,CAAC,CAAC;IAC/C,OAAO,CAAC,GAAG,CAAC,SAAS,CAAC,CAAC;IAEvB,IAAI,CAAC,UAAU,CAAC,MAAM,EAAE,CAAC;QACvB,OAAO,CAAC,QAAQ,GAAG,CAAC,CAAC;IACvB,CAAC;AACH,CAAC"}

package/dist/eval/hackaprompt-corpus.d.ts

@@ -13,6 +13,18 @@
  * @module agent-threat-rules/eval/hackaprompt-corpus
  */
 import type { CorpusSample } from './corpus.js';
+/**
+ * The two groups breakdown.by_category reports. HackAPrompt's `correct` field
+ * records whether the submission made its target model produce the level's
+ * target output, i.e. whether the attack succeeded in the competition. It is
+ * not an attack type. The corpus file's own `category` ("jailbreak" /
+ * "prompt-injection") is a label PR #51 derived from that same field; it stays
+ * in the file so the corpus keeps the digest the manifest pins, and is not used.
+ */
+export declare const HACKAPROMPT_OUTCOME_CATEGORY: {
+    readonly succeeded: "competition-outcome:succeeded";
+    readonly failed: "competition-outcome:failed";
+};
 export declare function loadHackaPromptCorpus(dataPath: string): readonly CorpusSample[];
 export declare function getHackaPromptCorpusStats(corpus: readonly CorpusSample[]): {
     total: number;

package/dist/eval/hackaprompt-corpus.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"hackaprompt-corpus.d.ts","sourceRoot":"","sources":["../../src/eval/hackaprompt-corpus.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;GAaG;AAGH,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,aAAa,CAAC;AAsBhD,wBAAgB,qBAAqB,CAAC,QAAQ,EAAE,MAAM,GAAG,SAAS,YAAY,EAAE,CAsB/E;AAED,wBAAgB,yBAAyB,CAAC,MAAM,EAAE,SAAS,YAAY,EAAE;;;;gBAKnD,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC;kBACpB,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC;EAS7C"}
+{"version":3,"file":"hackaprompt-corpus.d.ts","sourceRoot":"","sources":["../../src/eval/hackaprompt-corpus.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;GAaG;AAGH,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,aAAa,CAAC;AAgBhD;;;;;;;GAOG;AACH,eAAO,MAAM,4BAA4B;;;CAG/B,CAAC;AAQX,wBAAgB,qBAAqB,CAAC,QAAQ,EAAE,MAAM,GAAG,SAAS,YAAY,EAAE,CAwB/E;AAED,wBAAgB,yBAAyB,CAAC,MAAM,EAAE,SAAS,YAAY,EAAE;;;;gBAKnD,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC;kBACpB,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC;EAS7C"}

package/dist/eval/hackaprompt-corpus.js

@@ -13,6 +13,18 @@
  * @module agent-threat-rules/eval/hackaprompt-corpus
  */
 import { readFileSync } from 'node:fs';
+/**
+ * The two groups breakdown.by_category reports. HackAPrompt's `correct` field
+ * records whether the submission made its target model produce the level's
+ * target output, i.e. whether the attack succeeded in the competition. It is
+ * not an attack type. The corpus file's own `category` ("jailbreak" /
+ * "prompt-injection") is a label PR #51 derived from that same field; it stays
+ * in the file so the corpus keeps the digest the manifest pins, and is not used.
+ */
+export const HACKAPROMPT_OUTCOME_CATEGORY = {
+    succeeded: 'competition-outcome:succeeded',
+    failed: 'competition-outcome:failed',
+};
 function assignDifficulty(level) {
     if (level <= 2)
         return 'easy';
@@ -27,7 +39,9 @@ export function loadHackaPromptCorpus(dataPath) {
         return {
             id: sample.id,
             text: sample.text,
-            category: sample.category,
+            category: sample.metadata?.correct === true
+                ? HACKAPROMPT_OUTCOME_CATEGORY.succeeded
+                : HACKAPROMPT_OUTCOME_CATEGORY.failed,
             expectedDetection: sample.label,
             eventType: 'llm_input',
             tier: 'any',

package/dist/eval/hackaprompt-corpus.js.map

@@ -1 +1 @@
-{"version":3,"file":"hackaprompt-corpus.js","sourceRoot":"","sources":["../../src/eval/hackaprompt-corpus.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;GAaG;AAEH,OAAO,EAAE,YAAY,EAAE,MAAM,SAAS,CAAC;AAiBvC,SAAS,gBAAgB,CAAC,KAAa;IACrC,IAAI,KAAK,IAAI,CAAC;QAAE,OAAO,MAAM,CAAC;IAC9B,IAAI,KAAK,IAAI,CAAC;QAAE,OAAO,QAAQ,CAAC;IAChC,OAAO,MAAM,CAAC;AAChB,CAAC;AAED,MAAM,UAAU,qBAAqB,CAAC,QAAgB;IACpD,MAAM,GAAG,GAAoC,IAAI,CAAC,KAAK,CACrD,YAAY,CAAC,QAAQ,EAAE,OAAO,CAAC,CAChC,CAAC;IAEF,OAAO,GAAG,CAAC,GAAG,CAAC,CAAC,MAAM,EAAgB,EAAE;QACtC,MAAM,KAAK,GAAG,MAAM,CAAC,QAAQ,EAAE,KAAK,IAAI,CAAC,CAAC;QAC1C,OAAO;YACL,EAAE,EAAE,MAAM,CAAC,EAAE;YACb,IAAI,EAAE,MAAM,CAAC,IAAI;YACjB,QAAQ,EAAE,MAAM,CAAC,QAAQ;YACzB,iBAAiB,EAAE,MAAM,CAAC,KAAK;YAC/B,SAAS,EAAE,WAAW;YACtB,IAAI,EAAE,KAAK;YACX,UAAU,EAAE,gBAAgB,CAAC,KAAK,CAAC;YACnC,MAAM,EAAE;gBACN,IAAI,EAAE,MAAM,CAAC,IAAI;gBACjB,MAAM,EAAE,MAAM,CAAC,IAAI;gBACnB,UAAU,EAAE,MAAM,CAAC,IAAI;aACxB;SACF,CAAC;IACJ,CAAC,CAAC,CAAC;AACL,CAAC;AAED,MAAM,UAAU,yBAAyB,CAAC,MAA+B;IACvE,MAAM,KAAK,GAAG;QACZ,KAAK,EAAE,MAAM,CAAC,MAAM;QACpB,OAAO,EAAE,CAAC;QACV,MAAM,EAAE,CAAC;QACT,UAAU,EAAE,EAA4B;QACxC,YAAY,EAAE,EAA4B;KAC3C,CAAC;IACF,KAAK,MAAM,CAAC,IAAI,MAAM,EAAE,CAAC;QACvB,IAAI,CAAC,CAAC,iBAAiB;YAAE,KAAK,CAAC,OAAO,EAAE,CAAC;;YACpC,KAAK,CAAC,MAAM,EAAE,CAAC;QACpB,KAAK,CAAC,UAAU,CAAC,CAAC,CAAC,QAAQ,CAAC,GAAG,CAAC,KAAK,CAAC,UAAU,CAAC,CAAC,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAC,GAAG,CAAC,CAAC;QACvE,KAAK,CAAC,YAAY,CAAC,CAAC,CAAC,UAAU,CAAC,GAAG,CAAC,KAAK,CAAC,YAAY,CAAC,CAAC,CAAC,UAAU,CAAC,IAAI,CAAC,CAAC,GAAG,CAAC,CAAC;IACjF,CAAC;IACD,OAAO,KAAK,CAAC;AACf,CAAC"}
+{"version":3,"file":"hackaprompt-corpus.js","sourceRoot":"","sources":["../../src/eval/hackaprompt-corpus.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;GAaG;AAEH,OAAO,EAAE,YAAY,EAAE,MAAM,SAAS,CAAC;AAiBvC;;;;;;;GAOG;AACH,MAAM,CAAC,MAAM,4BAA4B,GAAG;IAC1C,SAAS,EAAE,+BAA+B;IAC1C,MAAM,EAAE,4BAA4B;CAC5B,CAAC;AAEX,SAAS,gBAAgB,CAAC,KAAa;IACrC,IAAI,KAAK,IAAI,CAAC;QAAE,OAAO,MAAM,CAAC;IAC9B,IAAI,KAAK,IAAI,CAAC;QAAE,OAAO,QAAQ,CAAC;IAChC,OAAO,MAAM,CAAC;AAChB,CAAC;AAED,MAAM,UAAU,qBAAqB,CAAC,QAAgB;IACpD,MAAM,GAAG,GAAoC,IAAI,CAAC,KAAK,CACrD,YAAY,CAAC,QAAQ,EAAE,OAAO,CAAC,CAChC,CAAC;IAEF,OAAO,GAAG,CAAC,GAAG,CAAC,CAAC,MAAM,EAAgB,EAAE;QACtC,MAAM,KAAK,GAAG,MAAM,CAAC,QAAQ,EAAE,KAAK,IAAI,CAAC,CAAC;QAC1C,OAAO;YACL,EAAE,EAAE,MAAM,CAAC,EAAE;YACb,IAAI,EAAE,MAAM,CAAC,IAAI;YACjB,QAAQ,EAAE,MAAM,CAAC,QAAQ,EAAE,OAAO,KAAK,IAAI;gBACzC,CAAC,CAAC,4BAA4B,CAAC,SAAS;gBACxC,CAAC,CAAC,4BAA4B,CAAC,MAAM;YACvC,iBAAiB,EAAE,MAAM,CAAC,KAAK;YAC/B,SAAS,EAAE,WAAW;YACtB,IAAI,EAAE,KAAK;YACX,UAAU,EAAE,gBAAgB,CAAC,KAAK,CAAC;YACnC,MAAM,EAAE;gBACN,IAAI,EAAE,MAAM,CAAC,IAAI;gBACjB,MAAM,EAAE,MAAM,CAAC,IAAI;gBACnB,UAAU,EAAE,MAAM,CAAC,IAAI;aACxB;SACF,CAAC;IACJ,CAAC,CAAC,CAAC;AACL,CAAC;AAED,MAAM,UAAU,yBAAyB,CAAC,MAA+B;IACvE,MAAM,KAAK,GAAG;QACZ,KAAK,EAAE,MAAM,CAAC,MAAM;QACpB,OAAO,EAAE,CAAC;QACV,MAAM,EAAE,CAAC;QACT,UAAU,EAAE,EAA4B;QACxC,YAAY,EAAE,EAA4B;KAC3C,CAAC;IACF,KAAK,MAAM,CAAC,IAAI,MAAM,EAAE,CAAC;QACvB,IAAI,CAAC,CAAC,iBAAiB;YAAE,KAAK,CAAC,OAAO,EAAE,CAAC;;YACpC,KAAK,CAAC,MAAM,EAAE,CAAC;QACpB,KAAK,CAAC,UAAU,CAAC,CAAC,CAAC,QAAQ,CAAC,GAAG,CAAC,KAAK,CAAC,UAAU,CAAC,CAAC,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAC,GAAG,CAAC,CAAC;QACvE,KAAK,CAAC,YAAY,CAAC,CAAC,CAAC,UAAU,CAAC,GAAG,CAAC,KAAK,CAAC,YAAY,CAAC,CAAC,CAAC,UAAU,CAAC,IAAI,CAAC,CAAC,GAAG,CAAC,CAAC;IACjF,CAAC;IACD,OAAO,KAAK,CAAC;AACf,CAAC"}

package/dist/eval/hackaprompt-provenance.d.ts

@@ -0,0 +1,52 @@
+/**
+ * HackAPrompt corpus provenance and in-sample ablation helpers.
+ *
+ * The HackAPrompt corpus is gated upstream and gitignored here, so the
+ * committed record of WHICH samples were measured is the manifest that
+ * scripts/hackaprompt-to-corpus.py writes beside it: the HuggingFace dataset
+ * id, the pinned dataset revision, the seed, the selected row indices and the
+ * SHA-256 of the corpus file. The runner refuses to measure a corpus whose
+ * digest does not match the manifest, so a measurement file can only ever
+ * describe the sample set its provenance block names.
+ *
+ * Several rules were written from this same corpus (their YAML names
+ * HackAPrompt as the source). Scoring them on it is in-sample, so the runner
+ * also measures the corpus with those rules removed.
+ *
+ * @module agent-threat-rules/eval/hackaprompt-provenance
+ */
+export interface HackaPromptManifest {
+    readonly dataset_id: string;
+    readonly dataset_revision: string;
+    readonly split: string;
+    readonly parquet_sha256: string;
+    readonly rows_total: number;
+    readonly selection: string;
+    readonly seed: number | null;
+    readonly requested: number;
+    readonly dropped_short_or_empty: number;
+    readonly filter: string;
+    readonly samples: number;
+    readonly corpus_sha256: string;
+    readonly sample_ids_sha256: string;
+    readonly row_indices: readonly number[];
+}
+/** Rules ATR-2026-00452..00456, added by e9efefdd on 2026-05-12, mined from this corpus. */
+export declare const HACKAPROMPT_BATCH_0512_RULE_IDS: readonly string[];
+export declare function sha256File(path: string): string;
+export declare function loadManifest(path: string): HackaPromptManifest;
+/** Throws unless the corpus file is byte-identical to the one the manifest describes. */
+export declare function verifyCorpusAgainstManifest(corpusPath: string, manifest: HackaPromptManifest): string;
+/** Rule ids whose YAML text names HackAPrompt anywhere (description, provenance, test cases). */
+export declare function rulesNamingHackaPrompt(rulesDir: string): readonly string[];
+/**
+ * A temporary copy of `rulesDir` without the given rule ids. Returns the copy's
+ * path and the ids actually removed (an id with no file is not silently claimed).
+ */
+export declare function rulesDirWithout(rulesDir: string, excludeIds: readonly string[]): {
+    readonly dir: string;
+    readonly removed: readonly string[];
+};
+/** Repo-relative path, for records that must not carry a machine-local path. */
+export declare function repoRelative(base: string, path: string): string;
+//# sourceMappingURL=hackaprompt-provenance.d.ts.map

package/dist/eval/hackaprompt-provenance.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"hackaprompt-provenance.d.ts","sourceRoot":"","sources":["../../src/eval/hackaprompt-provenance.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;GAgBG;AAOH,MAAM,WAAW,mBAAmB;IAClC,QAAQ,CAAC,UAAU,EAAE,MAAM,CAAC;IAC5B,QAAQ,CAAC,gBAAgB,EAAE,MAAM,CAAC;IAClC,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAC;IACvB,QAAQ,CAAC,cAAc,EAAE,MAAM,CAAC;IAChC,QAAQ,CAAC,UAAU,EAAE,MAAM,CAAC;IAC5B,QAAQ,CAAC,SAAS,EAAE,MAAM,CAAC;IAC3B,QAAQ,CAAC,IAAI,EAAE,MAAM,GAAG,IAAI,CAAC;IAC7B,QAAQ,CAAC,SAAS,EAAE,MAAM,CAAC;IAC3B,QAAQ,CAAC,sBAAsB,EAAE,MAAM,CAAC;IACxC,QAAQ,CAAC,MAAM,EAAE,MAAM,CAAC;IACxB,QAAQ,CAAC,OAAO,EAAE,MAAM,CAAC;IACzB,QAAQ,CAAC,aAAa,EAAE,MAAM,CAAC;IAC/B,QAAQ,CAAC,iBAAiB,EAAE,MAAM,CAAC;IACnC,QAAQ,CAAC,WAAW,EAAE,SAAS,MAAM,EAAE,CAAC;CACzC;AAED,4FAA4F;AAC5F,eAAO,MAAM,+BAA+B,EAAE,SAAS,MAAM,EAM5D,CAAC;AAEF,wBAAgB,UAAU,CAAC,IAAI,EAAE,MAAM,GAAG,MAAM,CAE/C;AAED,wBAAgB,YAAY,CAAC,IAAI,EAAE,MAAM,GAAG,mBAAmB,CAc9D;AAED,yFAAyF;AACzF,wBAAgB,2BAA2B,CAAC,UAAU,EAAE,MAAM,EAAE,QAAQ,EAAE,mBAAmB,GAAG,MAAM,CASrG;AAeD,iGAAiG;AACjG,wBAAgB,sBAAsB,CAAC,QAAQ,EAAE,MAAM,GAAG,SAAS,MAAM,EAAE,CAM1E;AAED;;;GAGG;AACH,wBAAgB,eAAe,CAC7B,QAAQ,EAAE,MAAM,EAChB,UAAU,EAAE,SAAS,MAAM,EAAE,GAC5B;IAAE,QAAQ,CAAC,GAAG,EAAE,MAAM,CAAC;IAAC,QAAQ,CAAC,OAAO,EAAE,SAAS,MAAM,EAAE,CAAA;CAAE,CAqB/D;AAED,gFAAgF;AAChF,wBAAgB,YAAY,CAAC,IAAI,EAAE,MAAM,EAAE,IAAI,EAAE,MAAM,GAAG,MAAM,CAE/D"}

package/dist/eval/hackaprompt-provenance.js

@@ -0,0 +1,108 @@
+/**
+ * HackAPrompt corpus provenance and in-sample ablation helpers.
+ *
+ * The HackAPrompt corpus is gated upstream and gitignored here, so the
+ * committed record of WHICH samples were measured is the manifest that
+ * scripts/hackaprompt-to-corpus.py writes beside it: the HuggingFace dataset
+ * id, the pinned dataset revision, the seed, the selected row indices and the
+ * SHA-256 of the corpus file. The runner refuses to measure a corpus whose
+ * digest does not match the manifest, so a measurement file can only ever
+ * describe the sample set its provenance block names.
+ *
+ * Several rules were written from this same corpus (their YAML names
+ * HackAPrompt as the source). Scoring them on it is in-sample, so the runner
+ * also measures the corpus with those rules removed.
+ *
+ * @module agent-threat-rules/eval/hackaprompt-provenance
+ */
+import { createHash } from 'node:crypto';
+import { cpSync, existsSync, mkdtempSync, readFileSync, readdirSync, statSync } from 'node:fs';
+import { tmpdir } from 'node:os';
+import { join, relative } from 'node:path';
+/** Rules ATR-2026-00452..00456, added by e9efefdd on 2026-05-12, mined from this corpus. */
+export const HACKAPROMPT_BATCH_0512_RULE_IDS = [
+    'ATR-2026-00452',
+    'ATR-2026-00453',
+    'ATR-2026-00454',
+    'ATR-2026-00455',
+    'ATR-2026-00456',
+];
+export function sha256File(path) {
+    return createHash('sha256').update(readFileSync(path)).digest('hex');
+}
+export function loadManifest(path) {
+    if (!existsSync(path)) {
+        throw new Error(`HackAPrompt manifest not found: ${path} (run scripts/hackaprompt-to-corpus.py)`);
+    }
+    const m = JSON.parse(readFileSync(path, 'utf-8'));
+    for (const key of ['dataset_id', 'dataset_revision', 'parquet_sha256', 'corpus_sha256', 'sample_ids_sha256']) {
+        if (typeof m[key] !== 'string' || m[key].length === 0) {
+            throw new Error(`HackAPrompt manifest ${path}: missing ${key}`);
+        }
+    }
+    if (!Array.isArray(m.row_indices) || m.row_indices.length !== m.samples) {
+        throw new Error(`HackAPrompt manifest ${path}: row_indices length != samples`);
+    }
+    return m;
+}
+/** Throws unless the corpus file is byte-identical to the one the manifest describes. */
+export function verifyCorpusAgainstManifest(corpusPath, manifest) {
+    const actual = sha256File(corpusPath);
+    if (actual !== manifest.corpus_sha256) {
+        throw new Error(`HackAPrompt corpus ${corpusPath} has sha256 ${actual}, manifest expects ${manifest.corpus_sha256}. ` +
+            'Regenerate it with scripts/hackaprompt-to-corpus.py at the pinned revision.');
+    }
+    return actual;
+}
+function listYaml(dir) {
+    return readdirSync(dir).flatMap((name) => {
+        const p = join(dir, name);
+        if (statSync(p).isDirectory())
+            return listYaml(p);
+        return /\.ya?ml$/.test(name) ? [p] : [];
+    });
+}
+function ruleIdOf(path) {
+    const m = /^id:\s*["']?([A-Z0-9-]+)["']?\s*$/m.exec(readFileSync(path, 'utf-8'));
+    return m ? m[1] : null;
+}
+/** Rule ids whose YAML text names HackAPrompt anywhere (description, provenance, test cases). */
+export function rulesNamingHackaPrompt(rulesDir) {
+    return listYaml(rulesDir)
+        .filter((p) => /hackaprompt/i.test(readFileSync(p, 'utf-8')))
+        .map(ruleIdOf)
+        .filter((id) => id !== null)
+        .sort();
+}
+/**
+ * A temporary copy of `rulesDir` without the given rule ids. Returns the copy's
+ * path and the ids actually removed (an id with no file is not silently claimed).
+ */
+export function rulesDirWithout(rulesDir, excludeIds) {
+    const exclude = new Set(excludeIds);
+    const removed = [];
+    const dir = join(mkdtempSync(join(tmpdir(), 'atr-hap-ablation-')), 'rules');
+    cpSync(rulesDir, dir, {
+        recursive: true,
+        filter: (src) => {
+            if (!/\.ya?ml$/.test(src) || statSync(src).isDirectory())
+                return true;
+            const id = ruleIdOf(src);
+            if (id !== null && exclude.has(id)) {
+                removed.push(id);
+                return false;
+            }
+            return true;
+        },
+    });
+    const missing = excludeIds.filter((id) => !removed.includes(id));
+    if (missing.length > 0) {
+        throw new Error(`rulesDirWithout: no rule file for ${missing.join(', ')} under ${rulesDir}`);
+    }
+    return { dir, removed: [...removed].sort() };
+}
+/** Repo-relative path, for records that must not carry a machine-local path. */
+export function repoRelative(base, path) {
+    return relative(base, path) || '.';
+}
+//# sourceMappingURL=hackaprompt-provenance.js.map

package/dist/eval/hackaprompt-provenance.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"hackaprompt-provenance.js","sourceRoot":"","sources":["../../src/eval/hackaprompt-provenance.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;GAgBG;AAEH,OAAO,EAAE,UAAU,EAAE,MAAM,aAAa,CAAC;AACzC,OAAO,EAAE,MAAM,EAAE,UAAU,EAAE,WAAW,EAAE,YAAY,EAAE,WAAW,EAAE,QAAQ,EAAE,MAAM,SAAS,CAAC;AAC/F,OAAO,EAAE,MAAM,EAAE,MAAM,SAAS,CAAC;AACjC,OAAO,EAAE,IAAI,EAAE,QAAQ,EAAE,MAAM,WAAW,CAAC;AAmB3C,4FAA4F;AAC5F,MAAM,CAAC,MAAM,+BAA+B,GAAsB;IAChE,gBAAgB;IAChB,gBAAgB;IAChB,gBAAgB;IAChB,gBAAgB;IAChB,gBAAgB;CACjB,CAAC;AAEF,MAAM,UAAU,UAAU,CAAC,IAAY;IACrC,OAAO,UAAU,CAAC,QAAQ,CAAC,CAAC,MAAM,CAAC,YAAY,CAAC,IAAI,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;AACvE,CAAC;AAED,MAAM,UAAU,YAAY,CAAC,IAAY;IACvC,IAAI,CAAC,UAAU,CAAC,IAAI,CAAC,EAAE,CAAC;QACtB,MAAM,IAAI,KAAK,CAAC,mCAAmC,IAAI,yCAAyC,CAAC,CAAC;IACpG,CAAC;IACD,MAAM,CAAC,GAAG,IAAI,CAAC,KAAK,CAAC,YAAY,CAAC,IAAI,EAAE,OAAO,CAAC,CAAwB,CAAC;IACzE,KAAK,MAAM,GAAG,IAAI,CAAC,YAAY,EAAE,kBAAkB,EAAE,gBAAgB,EAAE,eAAe,EAAE,mBAAmB,CAAU,EAAE,CAAC;QACtH,IAAI,OAAO,CAAC,CAAC,GAAG,CAAC,KAAK,QAAQ,IAAI,CAAC,CAAC,GAAG,CAAC,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YACtD,MAAM,IAAI,KAAK,CAAC,wBAAwB,IAAI,aAAa,GAAG,EAAE,CAAC,CAAC;QAClE,CAAC;IACH,CAAC;IACD,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,WAAW,CAAC,IAAI,CAAC,CAAC,WAAW,CAAC,MAAM,KAAK,CAAC,CAAC,OAAO,EAAE,CAAC;QACxE,MAAM,IAAI,KAAK,CAAC,wBAAwB,IAAI,iCAAiC,CAAC,CAAC;IACjF,CAAC;IACD,OAAO,CAAC,CAAC;AACX,CAAC;AAED,yFAAyF;AACzF,MAAM,UAAU,2BAA2B,CAAC,UAAkB,EAAE,QAA6B;IAC3F,MAAM,MAAM,GAAG,UAAU,CAAC,UAAU,CAAC,CAAC;IACtC,IAAI,MAAM,KAAK,QAAQ,CAAC,aAAa,EAAE,CAAC;QACtC,MAAM,IAAI,KAAK,CACb,sBAAsB,UAAU,eAAe,MAAM,sBAAsB,QAAQ,CAAC,aAAa,IAAI;YACnG,6EAA6E,CAChF,CAAC;IACJ,CAAC;IACD,OAAO,MAAM,CAAC;AAChB,CAAC;AAED,SAAS,QAAQ,CAAC,GAAW;IAC3B,OAAO,WAAW,CAAC,GAAG,CAAC,CAAC,OAAO,CAAC,CAAC,IAAI,EAAE,EAAE;QACvC,MAAM,CAAC,GAAG,IAAI,CAAC,GAAG,EAAE,IAAI,CAAC,CAAC;QAC1B,IAAI,QAAQ,CAAC,CAAC,CAAC,CAAC,WAAW,EAAE;YAAE,OAAO,QAAQ,CAAC,CAAC,CAAC,CAAC;QAClD,OAAO,UAAU,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC;IAC1C,CAAC,CAAC,CAAC;AACL,CAAC;AAED,SAAS,QAAQ,CAAC,IAAY;IAC5B,MAAM,CAAC,GAAG,oCAAoC,CAAC,IAAI,CAAC,YAAY,CAAC,IAAI,EAAE,OAAO,CAAC,CAAC,CAAC;IACjF,OAAO,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAE,CAAC,CAAC,CAAC,IAAI,CAAC;AAC1B,CAAC;AAED,iGAAiG;AACjG,MAAM,UAAU,sBAAsB,CAAC,QAAgB;IACrD,OAAO,QAAQ,CAAC,QAAQ,CAAC;SACtB,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,cAAc,CAAC,IAAI,CAAC,YAAY,CAAC,CAAC,EAAE,OAAO,CAAC,CAAC,CAAC;SAC5D,GAAG,CAAC,QAAQ,CAAC;SACb,MAAM,CAAC,CAAC,EAAE,EAAgB,EAAE,CAAC,EAAE,KAAK,IAAI,CAAC;SACzC,IAAI,EAAE,CAAC;AACZ,CAAC;AAED;;;GAGG;AACH,MAAM,UAAU,eAAe,CAC7B,QAAgB,EAChB,UAA6B;IAE7B,MAAM,OAAO,GAAG,IAAI,GAAG,CAAC,UAAU,CAAC,CAAC;IACpC,MAAM,OAAO,GAAa,EAAE,CAAC;IAC7B,MAAM,GAAG,GAAG,IAAI,CAAC,WAAW,CAAC,IAAI,CAAC,MAAM,EAAE,EAAE,mBAAmB,CAAC,CAAC,EAAE,OAAO,CAAC,CAAC;IAC5E,MAAM,CAAC,QAAQ,EAAE,GAAG,EAAE;QACpB,SAAS,EAAE,IAAI;QACf,MAAM,EAAE,CAAC,GAAG,EAAE,EAAE;YACd,IAAI,CAAC,UAAU,CAAC,IAAI,CAAC,GAAG,CAAC,IAAI,QAAQ,CAAC,GAAG,CAAC,CAAC,WAAW,EAAE;gBAAE,OAAO,IAAI,CAAC;YACtE,MAAM,EAAE,GAAG,QAAQ,CAAC,GAAG,CAAC,CAAC;YACzB,IAAI,EAAE,KAAK,IAAI,IAAI,OAAO,CAAC,GAAG,CAAC,EAAE,CAAC,EAAE,CAAC;gBACnC,OAAO,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;gBACjB,OAAO,KAAK,CAAC;YACf,CAAC;YACD,OAAO,IAAI,CAAC;QACd,CAAC;KACF,CAAC,CAAC;IACH,MAAM,OAAO,GAAG,UAAU,CAAC,MAAM,CAAC,CAAC,EAAE,EAAE,EAAE,CAAC,CAAC,OAAO,CAAC,QAAQ,CAAC,EAAE,CAAC,CAAC,CAAC;IACjE,IAAI,OAAO,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QACvB,MAAM,IAAI,KAAK,CAAC,qCAAqC,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC,UAAU,QAAQ,EAAE,CAAC,CAAC;IAC/F,CAAC;IACD,OAAO,EAAE,GAAG,EAAE,OAAO,EAAE,CAAC,GAAG,OAAO,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC;AAC/C,CAAC;AAED,gFAAgF;AAChF,MAAM,UAAU,YAAY,CAAC,IAAY,EAAE,IAAY;IACrD,OAAO,QAAQ,CAAC,IAAI,EAAE,IAAI,CAAC,IAAI,GAAG,CAAC;AACrC,CAAC"}

package/dist/eval/run-hackaprompt-benchmark.d.ts

@@ -2,16 +2,26 @@
 /**
  * HackAPrompt Benchmark Runner
  *
- * Loads a sample from the HackAPrompt 600K+ adversarial prompt dataset
- * (produced by scripts/hackaprompt-to-corpus.py) and runs it through the
- * ATR evaluation harness to measure recall against real prompt-hacking
- * attempts collected at competition scale.
+ * Measures ATR recall on the pinned 4,780-sample HackAPrompt set produced by
+ * scripts/hackaprompt-to-corpus.py (dataset revision, seed and row indices in
+ * data/hackaprompt/hackaprompt-corpus.manifest.json). One command:
  *
- * Usage:
- *   npx tsx src/eval/run-hackaprompt-benchmark.ts
+ *   npm run eval:hackaprompt
  *
- * HackAPrompt is an all-adversarial corpus, so we measure recall, latency,
- * and tier breakdown. Precision/FP rate require a benign companion source.
+ * which regenerates the corpus at the pinned revision (HuggingFace access to
+ * the gated dataset required) and then runs this file. The runner refuses a
+ * corpus whose SHA-256 differs from the manifest.
+ *
+ * Every figure is per lane. The headline (metrics.recall) is the hunt lane,
+ * as for every other source; breakdown.lanes carries hunt, alert and enforce
+ * side by side and breakdown.enforce_lane carries enforce in the shared shape.
+ * Because rules were written from this corpus, the runner also measures it
+ * with those rules removed (breakdown.in_sample_ablation). That ablation is
+ * the reference to cite beside the headline; it approximates an out-of-sample
+ * figure but is not one (see data/hackaprompt/METHOD.md).
+ *
+ * HackAPrompt is an all-adversarial corpus: recall is defined, precision and
+ * FP rate are not (they are recorded as the schema's convention values).
  *
  * @module agent-threat-rules/eval/run-hackaprompt-benchmark
  */

package/dist/eval/run-hackaprompt-benchmark.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"run-hackaprompt-benchmark.d.ts","sourceRoot":"","sources":["../../src/eval/run-hackaprompt-benchmark.ts"],"names":[],"mappings":";AACA;;;;;;;;;;;;;;;GAeG"}
+{"version":3,"file":"run-hackaprompt-benchmark.d.ts","sourceRoot":"","sources":["../../src/eval/run-hackaprompt-benchmark.ts"],"names":[],"mappings":";AACA;;;;;;;;;;;;;;;;;;;;;;;;;GAyBG"}

package/dist/eval/run-hackaprompt-benchmark.js

@@ -2,93 +2,183 @@
 /**
  * HackAPrompt Benchmark Runner
  *
- * Loads a sample from the HackAPrompt 600K+ adversarial prompt dataset
- * (produced by scripts/hackaprompt-to-corpus.py) and runs it through the
- * ATR evaluation harness to measure recall against real prompt-hacking
- * attempts collected at competition scale.
+ * Measures ATR recall on the pinned 4,780-sample HackAPrompt set produced by
+ * scripts/hackaprompt-to-corpus.py (dataset revision, seed and row indices in
+ * data/hackaprompt/hackaprompt-corpus.manifest.json). One command:
  *
- * Usage:
- *   npx tsx src/eval/run-hackaprompt-benchmark.ts
+ *   npm run eval:hackaprompt
  *
- * HackAPrompt is an all-adversarial corpus, so we measure recall, latency,
- * and tier breakdown. Precision/FP rate require a benign companion source.
+ * which regenerates the corpus at the pinned revision (HuggingFace access to
+ * the gated dataset required) and then runs this file. The runner refuses a
+ * corpus whose SHA-256 differs from the manifest.
+ *
+ * Every figure is per lane. The headline (metrics.recall) is the hunt lane,
+ * as for every other source; breakdown.lanes carries hunt, alert and enforce
+ * side by side and breakdown.enforce_lane carries enforce in the shared shape.
+ * Because rules were written from this corpus, the runner also measures it
+ * with those rules removed (breakdown.in_sample_ablation). That ablation is
+ * the reference to cite beside the headline; it approximates an out-of-sample
+ * figure but is not one (see data/hackaprompt/METHOD.md).
+ *
+ * HackAPrompt is an all-adversarial corpus: recall is defined, precision and
+ * FP rate are not (they are recorded as the schema's convention values).
  *
  * @module agent-threat-rules/eval/run-hackaprompt-benchmark
  */
+import { execFileSync } from 'node:child_process';
 import { resolve, join } from 'node:path';
-import { loadHackaPromptCorpus, getHackaPromptCorpusStats } from './hackaprompt-corpus.js';
-import { runEval } from './eval-harness.js';
-import { writeMeasurementFromEvalReport } from '../measurement/from-eval-harness.js';
-function formatPercent(n) {
+import { HACKAPROMPT_OUTCOME_CATEGORY, loadHackaPromptCorpus, getHackaPromptCorpusStats } from './hackaprompt-corpus.js';
+import { embeddingProvenance } from './embedding-provenance.js';
+import { enforceLaneFromEval, runEval } from './eval-harness.js';
+import { HACKAPROMPT_BATCH_0512_RULE_IDS, loadManifest, rulesDirWithout, rulesNamingHackaPrompt, verifyCorpusAgainstManifest, } from './hackaprompt-provenance.js';
+import { measurementInputFromEvalReport } from '../measurement/from-eval-harness.js';
+import { writeMeasurement } from '../measurement/write.js';
+const SOURCE_URL = 'https://huggingface.co/datasets/hackaprompt/hackaprompt-dataset';
+const REPRODUCE = 'git checkout <atr_commit> (the measured tree; its rules/ tree is breakdown.provenance.rules_tree_sha), ' +
+    'then: npm ci && npm run eval:hackaprompt ' +
+    '(= python3 scripts/hackaprompt-to-corpus.py, which checks the re-drawn sample against the committed manifest ' +
+    'and the parquet file against its pinned sha256 before writing the corpus, && npx tsx src/eval/run-hackaprompt-benchmark.ts)';
+/** The ablation cited beside the headline (breakdown.in_sample_ablation[reference]). */
+const ABLATION_REFERENCE = 'without_rules_naming_source';
+const ABLATION_NOTE = 'Ablation reference, not an out-of-sample result. Rules whose YAML names HackAPrompt were written from ' +
+    'this corpus (several from this exact 4,780-sample set), so recall with them loaded is in-sample. ' +
+    `${ABLATION_REFERENCE} re-measures the same samples with every such rule removed; it is the figure to cite ` +
+    'beside the headline. It approximates an out-of-sample figure without guaranteeing one: a rule whose YAML ' +
+    'does not name HackAPrompt may still have been tuned on these samples or on near-duplicates of them, and ' +
+    'the removed rules may also catch attacks they were not mined from. without_batch_2026_05_12 removes only ' +
+    'ATR-2026-00452..00456, the batch mined from this exact sample on 2026-05-12.';
+function pct(n) {
     return `${(n * 100).toFixed(1)}%`;
 }
-function formatMs(n) {
-    return `${n.toFixed(2)}ms`;
+function figure(run) {
+    const { tp, fn } = run.report.overall.confusion;
+    return { lane: run.lane, detected: tp, attacks: tp + fn, recall: tp / (tp + fn), rules_in_lane: run.rulesInLane };
+}
+function log(label, f) {
+    console.log(`  ${label.padEnd(34)} ${f.lane.padEnd(8)} ${f.detected}/${f.attacks} = ${pct(f.recall)} (${f.rules_in_lane} rules in lane)`);
+}
+function rulesTreeSha(base) {
+    try {
+        return execFileSync('git', ['rev-parse', 'HEAD:rules'], { cwd: base, encoding: 'utf-8' }).trim();
+    }
+    catch {
+        return 'unknown';
+    }
+}
+async function ablation(base, rulesDir, ids, description) {
+    const { dir, removed } = rulesDirWithout(rulesDir, ids);
+    const hunt = figure(await runEval({ ...base, rulesDir: dir, lane: 'hunt' }));
+    const enforce = figure(await runEval({ ...base, rulesDir: dir, lane: 'enforce' }));
+    log(`without ${removed.length} rules: ${description}`, hunt);
+    log(`without ${removed.length} rules: ${description}`, enforce);
+    return { description, removed_rule_ids: removed, lanes: { hunt, enforce } };
 }
 async function main() {
     const base = resolve(join(import.meta.dirname ?? '.', '..', '..'));
     const rulesDir = join(base, 'rules');
+    const embeddingsPath = join(base, 'data', 'attack-embeddings.json');
     const dataPath = join(base, 'data', 'hackaprompt', 'hackaprompt-corpus.json');
+    const manifestPath = join(base, 'data', 'hackaprompt', 'hackaprompt-corpus.manifest.json');
     const outputPath = join(base, 'data', 'hackaprompt', 'hackaprompt-eval-report.json');
-    console.log('\n=== HackAPrompt Benchmark -- ATR Evaluation ===\n');
-    console.log(`Corpus: ${dataPath}`);
-    console.log(`Rules:  ${rulesDir}\n`);
+    const manifest = loadManifest(manifestPath);
+    const corpusSha = verifyCorpusAgainstManifest(dataPath, manifest);
     const corpus = loadHackaPromptCorpus(dataPath);
     const stats = getHackaPromptCorpusStats(corpus);
-    console.log(`Loaded ${stats.total} samples (${stats.attacks} attacks, ${stats.benign} benign)`);
-    console.log(`Categories: ${Object.entries(stats.byCategory).map(([k, v]) => `${k}=${v}`).join(', ')}`);
-    console.log(`Difficulty: ${Object.entries(stats.byDifficulty).map(([k, v]) => `${k}=${v}`).join(', ')}`);
-    // HackAPrompt is an all-adversarial corpus from a public global competition.
-    // It contains heavy paraphrasing, role-play, multilingual, and creative
-    // attacks. Recall is the headline number. FP rate is meaningless here
-    // because there are no benign samples in the corpus.
-    const hackaPromptThresholds = {
-        minRecall: 0.10,
-        maxFpRate: 1.0,
-        minF1: 0.0,
-        maxP95LatencyMs: 200,
-    };
-    const { report, tiersUsed, ruleQuality, eventShape } = await runEval({
+    if (stats.total !== manifest.samples) {
+        throw new Error(`corpus has ${stats.total} samples, manifest says ${manifest.samples}`);
+    }
+    console.log('\n=== HackAPrompt Benchmark -- ATR Evaluation ===\n');
+    console.log(`Dataset: ${manifest.dataset_id}@${manifest.dataset_revision}`);
+    console.log(`Corpus:  ${stats.total} samples, sha256 ${corpusSha}`);
+    const shared = { corpus, embeddingsPath };
+    const hunt = await runEval({
+        ...shared,
         rulesDir,
-        corpus,
-        thresholds: hackaPromptThresholds,
+        lane: 'hunt',
         outputPath,
+        thresholds: { minRecall: 0.1, maxFpRate: 1.0, minF1: 0.0, maxP95LatencyMs: 200 },
     });
-    console.log(`\nTiers: ${tiersUsed.join(' + ')}`);
-    console.log(`Event shape: ${eventShape}`);
-    console.log(`\n--- Overall ---`);
-    console.log(`  Recall:     ${formatPercent(report.overall.recall)}`);
-    console.log(`  Precision:  ${formatPercent(report.overall.precision)} (N/A meaning - no benign samples)`);
-    console.log(`  Confusion:  TP=${report.overall.confusion.tp} FN=${report.overall.confusion.fn}`);
-    console.log(`\n--- Latency ---`);
-    console.log(`  P50:  ${formatMs(report.latency.p50)}`);
-    console.log(`  P95:  ${formatMs(report.latency.p95)}`);
-    console.log(`  P99:  ${formatMs(report.latency.p99)}`);
-    console.log(`  Mean: ${formatMs(report.latency.mean)}`);
-    console.log(`\n--- By Category ---`);
-    for (const cat of report.byCategory) {
-        const m = cat.metrics;
-        const tp = m.confusion.tp;
-        const fn = m.confusion.fn;
-        const recall = tp + fn === 0 ? 0 : tp / (tp + fn);
-        console.log(`  ${cat.category}: recall=${formatPercent(recall)} (TP=${tp} FN=${fn})`);
-    }
-    console.log(`\n--- Top Firing Rules (top 15) ---`);
-    const fired = ruleQuality?.topRules ?? [];
-    for (const r of fired.slice(0, 15)) {
-        console.log(`  ${r.ruleId}: matches=${r.matchCount} TP=${r.tpCount} FP=${r.fpCount}`);
+    const alert = await runEval({ ...shared, rulesDir, lane: 'alert' });
+    const enforceRun = await runEval({ ...shared, rulesDir, lane: 'enforce' });
+    const enforceLane = enforceLaneFromEval(enforceRun, hunt);
+    for (const r of [alert, enforceRun]) {
+        if (r.eventShape !== hunt.eventShape || r.tiersUsed.join('+') !== hunt.tiersUsed.join('+')) {
+            throw new Error(`the ${r.lane} run differs from the hunt run in event shape or tiers`);
+        }
     }
-    console.log(`\nReport saved to: ${outputPath}`);
-    // Standardized Measurement file (version-pinned, immutable).
-    const { measurementPath } = writeMeasurementFromEvalReport(report, {
+    console.log(`\nTiers: ${hunt.tiersUsed.join(' + ')}   Event shape: ${hunt.eventShape}`);
+    const lanes = { hunt: figure(hunt), alert: figure(alert), enforce: figure(enforceRun) };
+    for (const f of Object.values(lanes))
+        log('all rules', f);
+    const namingHap = rulesNamingHackaPrompt(rulesDir);
+    const inSample = {
+        note: ABLATION_NOTE,
+        reference: ABLATION_REFERENCE,
+        [ABLATION_REFERENCE]: await ablation(shared, rulesDir, namingHap, 'every rule whose YAML text names HackAPrompt'),
+        without_batch_2026_05_12: await ablation(shared, rulesDir, HACKAPROMPT_BATCH_0512_RULE_IDS, 'ATR-2026-00452..00456 (commit e9efefdd)'),
+    };
+    // The embedding tier's contribution: the hunt lane once more, regex tier only.
+    const noEmbedding = figure(await runEval({ corpus, rulesDir, lane: 'hunt', enableEmbedding: false }));
+    log('all rules, embedding tier off', noEmbedding);
+    const embedding = hunt.tiersUsed.includes('tier2.5-embedding')
+        ? await embeddingProvenance(embeddingsPath, 'data/attack-embeddings.json')
+        : null;
+    const input = measurementInputFromEvalReport(hunt.report, {
         source: 'hackaprompt',
         source_version: 'v1',
-        source_url: 'https://huggingface.co/datasets/hackaprompt/hackaprompt-dataset',
-        notes: 'HackAPrompt competition dataset. 100% adversarial — fp_rate is undefined on this corpus. ' +
-            `Event shape: ${eventShape} (src/eval/eval-harness.ts EvalEventShapeMode).`,
+        source_url: `${SOURCE_URL}/tree/${manifest.dataset_revision}`,
+        enforceLane,
+        notes: 'HackAPrompt competition dataset (Schulhoff et al., EMNLP 2023). All samples adversarial, so ' +
+            'precision/fp_rate are convention values, not results. metrics.recall is the hunt lane (every ' +
+            'maturity); breakdown.lanes gives hunt/alert/enforce. breakdown.in_sample_ablation.' + ABLATION_REFERENCE +
+            ' is the ablation reference to cite beside it: the same samples without the rules whose YAML names ' +
+            'HackAPrompt (an approximation of out-of-sample recall, not a guarantee of it). breakdown.by_category ' +
+            "groups samples by HackAPrompt's `correct` field (whether the attack succeeded against its target model " +
+            "in the competition), not by attack type; breakdown.by_difficulty buckets the competition level " +
+            '(0-2 easy, 3-6 medium, 7-10 hard). Sample set: breakdown.provenance (pinned dataset revision, ' +
+            `seed, row indices in data/hackaprompt/hackaprompt-corpus.manifest.json). Event shape: ${hunt.eventShape}; ` +
+            `tiers: ${hunt.tiersUsed.join('+')}. Reproduce: ${REPRODUCE}. Method: data/hackaprompt/METHOD.md.`,
     });
-    console.log(`Measurement: ${measurementPath}`);
-    console.log('Done.');
+    const { measurementPath } = writeMeasurement({
+        ...input,
+        source_commit: manifest.dataset_revision,
+        breakdown: {
+            ...input.breakdown,
+            lanes,
+            by_category_field: {
+                field: 'correct',
+                meaning: "HackAPrompt's `correct`: the submission made its target model produce the level's target output, " +
+                    'i.e. the attack succeeded in the competition. Not an attack type.',
+                categories: HACKAPROMPT_OUTCOME_CATEGORY,
+            },
+            in_sample_ablation: inSample,
+            precision_not_citable: {
+                reason: 'HackAPrompt is 100% adversarial: there is no benign population, so precision and FP rate are undefined. ' +
+                    'metrics.precision and metrics.fp_rate are the convention values the schema requires, not results; ' +
+                    'data/stats.json projects precision, F1 and FP rate as null.',
+                benign_samples: 0,
+            },
+            provenance: {
+                dataset_id: manifest.dataset_id,
+                dataset_revision: manifest.dataset_revision,
+                parquet_sha256: manifest.parquet_sha256,
+                rows_total: manifest.rows_total,
+                selection: manifest.selection,
+                filter: manifest.filter,
+                dropped_short_or_empty: manifest.dropped_short_or_empty,
+                corpus_sha256: corpusSha,
+                sample_ids_sha256: manifest.sample_ids_sha256,
+                manifest: 'data/hackaprompt/hackaprompt-corpus.manifest.json',
+                rules_tree_sha: rulesTreeSha(base),
+                event_shape: hunt.eventShape,
+                tiers: hunt.tiersUsed,
+                embedding,
+                hunt_without_embedding_tier: noEmbedding,
+                reproduce: REPRODUCE,
+            },
+        },
+    }, { force: true });
+    console.log(`\nMeasurement: ${measurementPath}`);
 }
 main().catch((err) => {
     console.error('Error:', err);