agent-threat-rules 4.1.5 → 4.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +688 -1
- package/README.md +355 -125
- package/dist/adapters/openshell-filter.d.ts +10 -3
- package/dist/adapters/openshell-filter.d.ts.map +1 -1
- package/dist/adapters/openshell-filter.js +10 -3
- package/dist/adapters/openshell-filter.js.map +1 -1
- package/dist/adapters/stdio-adapter.js +3 -3
- package/dist/adapters/stdio-adapter.js.map +1 -1
- package/dist/cli/mcp-tools-document.d.ts +40 -0
- package/dist/cli/mcp-tools-document.d.ts.map +1 -0
- package/dist/cli/mcp-tools-document.js +95 -0
- package/dist/cli/mcp-tools-document.js.map +1 -0
- package/dist/cli/scan-handler.d.ts +8 -0
- package/dist/cli/scan-handler.d.ts.map +1 -1
- package/dist/cli/scan-handler.js +126 -34
- package/dist/cli/scan-handler.js.map +1 -1
- package/dist/cli.d.ts.map +1 -1
- package/dist/cli.js +80 -43
- package/dist/cli.js.map +1 -1
- package/dist/converters/sarif.d.ts +14 -2
- package/dist/converters/sarif.d.ts.map +1 -1
- package/dist/converters/sarif.js +7 -9
- package/dist/converters/sarif.js.map +1 -1
- package/dist/corpus-event.d.ts +47 -5
- package/dist/corpus-event.d.ts.map +1 -1
- package/dist/corpus-event.js +68 -29
- package/dist/corpus-event.js.map +1 -1
- package/dist/embedding/model-loader.d.ts +2 -0
- package/dist/embedding/model-loader.d.ts.map +1 -1
- package/dist/embedding/model-loader.js +3 -1
- package/dist/embedding/model-loader.js.map +1 -1
- package/dist/enforcement.d.ts +8 -0
- package/dist/enforcement.d.ts.map +1 -1
- package/dist/enforcement.js +10 -0
- package/dist/enforcement.js.map +1 -1
- package/dist/engine.d.ts +150 -6
- package/dist/engine.d.ts.map +1 -1
- package/dist/engine.js +337 -76
- package/dist/engine.js.map +1 -1
- package/dist/eval/embedding-provenance.d.ts +40 -0
- package/dist/eval/embedding-provenance.d.ts.map +1 -0
- package/dist/eval/embedding-provenance.js +71 -0
- package/dist/eval/embedding-provenance.js.map +1 -0
- package/dist/eval/eval-harness.d.ts +34 -0
- package/dist/eval/eval-harness.d.ts.map +1 -1
- package/dist/eval/eval-harness.js +49 -2
- package/dist/eval/eval-harness.js.map +1 -1
- package/dist/eval/hackaprompt-corpus.d.ts +12 -0
- package/dist/eval/hackaprompt-corpus.d.ts.map +1 -1
- package/dist/eval/hackaprompt-corpus.js +15 -1
- package/dist/eval/hackaprompt-corpus.js.map +1 -1
- package/dist/eval/hackaprompt-provenance.d.ts +52 -0
- package/dist/eval/hackaprompt-provenance.d.ts.map +1 -0
- package/dist/eval/hackaprompt-provenance.js +108 -0
- package/dist/eval/hackaprompt-provenance.js.map +1 -0
- package/dist/eval/run-hackaprompt-benchmark.d.ts +18 -8
- package/dist/eval/run-hackaprompt-benchmark.d.ts.map +1 -1
- package/dist/eval/run-hackaprompt-benchmark.js +154 -64
- package/dist/eval/run-hackaprompt-benchmark.js.map +1 -1
- package/dist/eval/run-pint-benchmark.js +11 -1
- package/dist/eval/run-pint-benchmark.js.map +1 -1
- package/dist/eval/skill-benchmark.d.ts +26 -6
- package/dist/eval/skill-benchmark.d.ts.map +1 -1
- package/dist/eval/skill-benchmark.js +49 -16
- package/dist/eval/skill-benchmark.js.map +1 -1
- package/dist/flywheel.d.ts +4 -0
- package/dist/flywheel.d.ts.map +1 -1
- package/dist/flywheel.js +12 -4
- package/dist/flywheel.js.map +1 -1
- package/dist/hook-handler.d.ts +11 -1
- package/dist/hook-handler.d.ts.map +1 -1
- package/dist/hook-handler.js +84 -16
- package/dist/hook-handler.js.map +1 -1
- package/dist/hook-tool-output.d.ts +47 -0
- package/dist/hook-tool-output.d.ts.map +1 -0
- package/dist/hook-tool-output.js +110 -0
- package/dist/hook-tool-output.js.map +1 -0
- package/dist/index.d.ts +5 -3
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +3 -2
- package/dist/index.js.map +1 -1
- package/dist/layer-integration.d.ts +4 -2
- package/dist/layer-integration.d.ts.map +1 -1
- package/dist/layer-integration.js +6 -2
- package/dist/layer-integration.js.map +1 -1
- package/dist/mcp-server.d.ts +8 -0
- package/dist/mcp-server.d.ts.map +1 -1
- package/dist/mcp-server.js +1 -0
- package/dist/mcp-server.js.map +1 -1
- package/dist/mcp-tools/scan.d.ts.map +1 -1
- package/dist/mcp-tools/scan.js +9 -1
- package/dist/mcp-tools/scan.js.map +1 -1
- package/dist/measurement/enforce-lane.d.ts +66 -0
- package/dist/measurement/enforce-lane.d.ts.map +1 -0
- package/dist/measurement/enforce-lane.js +89 -0
- package/dist/measurement/enforce-lane.js.map +1 -0
- package/dist/measurement/from-eval-harness.d.ts +16 -3
- package/dist/measurement/from-eval-harness.d.ts.map +1 -1
- package/dist/measurement/from-eval-harness.js +15 -5
- package/dist/measurement/from-eval-harness.js.map +1 -1
- package/dist/measurement/in-sample-ablation.d.ts +29 -0
- package/dist/measurement/in-sample-ablation.d.ts.map +1 -0
- package/dist/measurement/in-sample-ablation.js +45 -0
- package/dist/measurement/in-sample-ablation.js.map +1 -0
- package/dist/measurement/schema.d.ts +38 -1
- package/dist/measurement/schema.d.ts.map +1 -1
- package/dist/measurement/schema.js +43 -1
- package/dist/measurement/schema.js.map +1 -1
- package/dist/quality/action-eligibility.d.ts +4 -2
- package/dist/quality/action-eligibility.d.ts.map +1 -1
- package/dist/quality/action-eligibility.js.map +1 -1
- package/dist/rule-confidence.d.ts +31 -0
- package/dist/rule-confidence.d.ts.map +1 -0
- package/dist/rule-confidence.js +49 -0
- package/dist/rule-confidence.js.map +1 -0
- package/dist/rule-scaffolder.d.ts.map +1 -1
- package/dist/rule-scaffolder.js +43 -4
- package/dist/rule-scaffolder.js.map +1 -1
- package/dist/semantic-evaluator.d.ts.map +1 -1
- package/dist/semantic-evaluator.js +15 -13
- package/dist/semantic-evaluator.js.map +1 -1
- package/dist/shadow-evaluator.d.ts +7 -2
- package/dist/shadow-evaluator.d.ts.map +1 -1
- package/dist/shadow-evaluator.js +29 -25
- package/dist/shadow-evaluator.js.map +1 -1
- package/dist/testcase-event.d.ts +82 -1
- package/dist/testcase-event.d.ts.map +1 -1
- package/dist/testcase-event.js +135 -16
- package/dist/testcase-event.js.map +1 -1
- package/dist/tier0-invariant.d.ts.map +1 -1
- package/dist/tier0-invariant.js +1 -0
- package/dist/tier0-invariant.js.map +1 -1
- package/dist/tier1-blacklist.d.ts.map +1 -1
- package/dist/tier1-blacklist.js +1 -0
- package/dist/tier1-blacklist.js.map +1 -1
- package/dist/tool-definition-scope.d.ts +45 -0
- package/dist/tool-definition-scope.d.ts.map +1 -0
- package/dist/tool-definition-scope.js +68 -0
- package/dist/tool-definition-scope.js.map +1 -0
- package/dist/types.d.ts +42 -2
- package/dist/types.d.ts.map +1 -1
- package/dist/verdict.d.ts +46 -2
- package/dist/verdict.d.ts.map +1 -1
- package/dist/verdict.js +99 -14
- package/dist/verdict.js.map +1 -1
- package/package.json +2 -1
- package/rules/agent-manipulation/ATR-2026-00030-cross-agent-attack.yaml +2 -2
- package/rules/agent-manipulation/ATR-2026-00139-casual-authority-redirect.yaml +1 -1
- package/rules/agent-manipulation/ATR-2026-00328-named-mode-persona-override.yaml +1 -1
- package/rules/agent-manipulation/ATR-2026-00330-yoja-allcaps-settings-block-persona.yaml +1 -1
- package/rules/agent-manipulation/ATR-2026-00418-weknora-mcp-config-rce.yaml +5 -5
- package/rules/agent-manipulation/ATR-2026-00432-superagi-output-handler-eval-rce.yaml +1 -1
- package/rules/agent-manipulation/ATR-2026-00440-semantic-kernel-vector-store-eval-rce.yaml +3 -3
- package/rules/context-exfiltration/ATR-2026-00426-nl-output-injection-credential-leak.yaml +1 -1
- package/rules/context-exfiltration/ATR-2026-00702-ipi-credential-exfil-via-agent-action.yaml +1 -1
- package/rules/context-exfiltration/ATR-2026-00707-ipi-file-exfil-with-destroy-or-tamper.yaml +1 -1
- package/rules/context-exfiltration/ATR-2026-00851-indirect-pi-financial-payment-exfil.yaml +1 -1
- package/rules/context-exfiltration/ATR-2026-01457-sysprompt-completion-clone-attack.yaml +1 -1
- package/rules/context-exfiltration/ATR-2026-01462-sysprompt-override-pseudo-admin.yaml +1 -1
- package/rules/context-exfiltration/ATR-2026-01463-imend-imsystem-prompt-injection-format.yaml +1 -1
- package/rules/context-exfiltration/ATR-2026-01606-ssrf-internal-network-scan.yaml +1 -1
- package/rules/context-exfiltration/ATR-2026-02847-policy-justified-exfil-to-email.yaml +1 -1
- package/rules/context-exfiltration/ATR-2026-02855-cross-email-exfil-relay.yaml +79 -0
- package/rules/context-exfiltration/ATR-2026-02861-exfil-send-email-with-body-directive.yaml +83 -0
- package/rules/context-exfiltration/ATR-2026-02866-idna-label-separator-in-fetch-url-host.yaml +104 -0
- package/rules/excessive-autonomy/ATR-2026-00710-ipi-physical-access-grant.yaml +1 -1
- package/rules/model-abuse/ATR-2026-01897-host-sabotage.yaml +2 -2
- package/rules/model-security/ATR-2026-00433-modelcache-torch-load-deserialization-rce.yaml +1 -1
- package/rules/privilege-escalation/ATR-2026-00436-enclave-vm-sandbox-escape-rce.yaml +1 -1
- package/rules/privilege-escalation/ATR-2026-01601-sql-injection-destructive-ddl.yaml +2 -2
- package/rules/privilege-escalation/ATR-2026-01612-debug-mode-privilege-escalation.yaml +1 -1
- package/rules/privilege-escalation/ATR-2026-02867-identifier-typed-tool-argument-dot-dot-path-segment.yaml +106 -0
- package/rules/prompt-injection/ATR-2026-00004-system-prompt-override.yaml +2 -2
- package/rules/prompt-injection/ATR-2026-00080-encoding-evasion.yaml +1 -1
- package/rules/prompt-injection/ATR-2026-00086-visual-spoofing.yaml +1 -1
- package/rules/prompt-injection/ATR-2026-00203-context-pollution-skill-description.yaml +1 -1
- package/rules/prompt-injection/ATR-2026-00228-structured-jailbreak.yaml +2 -2
- package/rules/prompt-injection/ATR-2026-00233-structured-jailbreak.yaml +2 -2
- package/rules/prompt-injection/ATR-2026-00256-base-n-encoding-jailbreak.yaml +1 -1
- package/rules/prompt-injection/ATR-2026-00264-latent-injection-translation.yaml +3 -3
- package/rules/prompt-injection/ATR-2026-00265-latent-injection-rag-document.yaml +1 -1
- package/rules/prompt-injection/ATR-2026-00282-perez-prompt-injection-hijack.yaml +4 -4
- package/rules/prompt-injection/ATR-2026-00321-hyphenated-system-instruction-injection.yaml +1 -1
- package/rules/prompt-injection/ATR-2026-00394-backspace-deletion-character-injection.yaml +2 -2
- package/rules/prompt-injection/ATR-2026-00446-variable-assignment-payload-injection.yaml +3 -3
- package/rules/prompt-injection/ATR-2026-00476-garak-web-injection-javascript-alert-1.yaml +3 -3
- package/rules/prompt-injection/ATR-2026-00481-garak-dan-mixedunassigned.yaml +1 -1
- package/rules/prompt-injection/ATR-2026-00489-garak-encoding-javascript-alert-1.yaml +3 -3
- package/rules/prompt-injection/ATR-2026-00508-escape-delimiter-wrapped-goal-hijacking-in-user-input-prompt.yaml +3 -3
- package/rules/prompt-injection/ATR-2026-00512-rules-file-backdoor-injection.yaml +1 -1
- package/rules/prompt-injection/ATR-2026-01005-many-shot-repetition-override.yaml +2 -2
- package/rules/prompt-injection/ATR-2026-01006-payload-split-string-concat-injection.yaml +2 -2
- package/rules/prompt-injection/ATR-2026-01015-evasion-refusal-suppression.yaml +1 -1
- package/rules/prompt-injection/ATR-2026-01016-evasion-response-priming-forced-opener.yaml +1 -1
- package/rules/prompt-injection/ATR-2026-01026-evasion-unicode-glitch-token-anomaly.yaml +2 -2
- package/rules/prompt-injection/ATR-2026-01304-indirect-injection-carrier-important-override.yaml +1 -1
- package/rules/prompt-injection/ATR-2026-01759-dos-new-important-instructions-injection-in-document.yaml +1 -1
- package/rules/prompt-injection/ATR-2026-01865-llmail-fake-email-boundary-marker.yaml +1 -1
- package/rules/prompt-injection/ATR-2026-02010-emoji-only-payload.yaml +2 -2
- package/rules/prompt-injection/ATR-2026-02815-injected-halt-order-canned-reply-dos.yaml +80 -0
- package/rules/prompt-injection/ATR-2026-02820-tool-result-orders-agent-to-hide-its-own-action.yaml +106 -0
- package/rules/prompt-injection/ATR-2026-02858-context-suppression-directive.yaml +73 -0
- package/rules/prompt-injection/ATR-2026-02859-tool-output-agent-role-reassignment.yaml +71 -0
- package/rules/prompt-injection/ATR-2026-02860-forged-instruction-boundary-marker.yaml +70 -0
- package/rules/prompt-injection/ATR-2026-02862-fake-chat-template-turn-injection.yaml +79 -0
- package/rules/skill-compromise/ATR-2026-00060-skill-impersonation.yaml +1 -1
- package/rules/skill-compromise/ATR-2026-00063-skill-chain-attack.yaml +2 -2
- package/rules/skill-compromise/ATR-2026-00066-parameter-injection.yaml +2 -2
- package/rules/skill-compromise/ATR-2026-00149-skill-exfil-compound.yaml +6 -6
- package/rules/skill-compromise/ATR-2026-00223-reverse-shell-dropper.yaml +2 -2
- package/rules/skill-compromise/ATR-2026-01755-backdoor-pot-linguistic-trigger-phrase.yaml +1 -1
- package/rules/skill-compromise/ATR-2026-01756-backdoor-pot-symbol-emoticon-trigger.yaml +1 -1
- package/rules/skill-compromise/ATR-2026-02261-code-credential-exfil-chain.yaml +1 -1
- package/rules/tool-poisoning/ATR-2026-00012-unauthorized-tool-call.yaml +23 -23
- package/rules/tool-poisoning/ATR-2026-00013-tool-ssrf.yaml +16 -5
- package/rules/tool-poisoning/ATR-2026-00161-important-tag-cross-tool-shadowing.yaml +1 -1
- package/rules/tool-poisoning/ATR-2026-00277-echo-template-command-injection.yaml +2 -2
- package/rules/tool-poisoning/ATR-2026-00494-garak-exploitation-mixedunassigned.yaml +4 -4
- package/rules/tool-poisoning/ATR-2026-00581-mcp-tool-rug-pull-post-approval-redefinition.yaml +1 -1
- package/rules/tool-poisoning/ATR-2026-01303-tool-schema-enumeration-social-engineering.yaml +1 -1
- package/rules/tool-poisoning/ATR-2026-01307-mcp-dns-rebinding-attack.yaml +2 -2
- package/rules/tool-poisoning/ATR-2026-01775-semantic-mcp-tool-manifest-poisoning.yaml +1 -1
- package/rules/tool-poisoning/ATR-2026-01932-shadow-undeclared-mcp-server-registration.yaml +1 -1
- package/rules/tool-poisoning/ATR-2026-02025-mcp-full-schema-poisoning-nondescription-field.yaml +1 -1
- package/rules/tool-poisoning/ATR-2026-02817-untrusted-content-dictates-tool-arguments.yaml +83 -0
- package/rules/tool-poisoning/ATR-2026-02864-non-canonical-ipv4-host-in-url-tool-argument.yaml +102 -0
- package/rules/tool-poisoning/ATR-2026-02865-read-only-utility-write-flag-in-command-argument.yaml +157 -0
- package/rules/tool-poisoning/ATR-2026-02868-ssrf-ipv6-transition-address-embedded-private-ipv4.yaml +107 -0
- package/spec/README.md +6 -3
- package/spec/atr-event-v1.0.md +13 -8
- package/spec/atr-method-v1.1.md +2 -2
- package/spec/category-registry/gemara/README.md +102 -0
- package/spec/category-registry/gemara/ai-agent-capability-catalog.yaml +157 -0
- package/spec/category-registry/gemara/atr-attack-vector-catalog.yaml +130 -0
- package/spec/category-registry/gemara/atr-detection-control-catalog.yaml +327 -0
- package/spec/category-registry/gemara/atr-threat-catalog.yaml +246 -0
- package/spec/category-registry/gemara/src/editorial.yaml +743 -0
- package/spec/mappings/README.md +1 -1
- package/spec/mappings/atr-to-nist-csf-2.0.md +5 -4
- package/spec/schema/event.schema.json +1 -1
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"run-hackaprompt-benchmark.js","sourceRoot":"","sources":["../../src/eval/run-hackaprompt-benchmark.ts"],"names":[],"mappings":";AACA
|
|
1
|
+
{"version":3,"file":"run-hackaprompt-benchmark.js","sourceRoot":"","sources":["../../src/eval/run-hackaprompt-benchmark.ts"],"names":[],"mappings":";AACA;;;;;;;;;;;;;;;;;;;;;;;;;GAyBG;AAEH,OAAO,EAAE,YAAY,EAAE,MAAM,oBAAoB,CAAC;AAClD,OAAO,EAAE,OAAO,EAAE,IAAI,EAAE,MAAM,WAAW,CAAC;AAC1C,OAAO,EAAE,4BAA4B,EAAE,qBAAqB,EAAE,yBAAyB,EAAE,MAAM,yBAAyB,CAAC;AACzH,OAAO,EAAE,mBAAmB,EAAE,MAAM,2BAA2B,CAAC;AAChE,OAAO,EAAE,mBAAmB,EAAE,OAAO,EAAmB,MAAM,mBAAmB,CAAC;AAClF,OAAO,EACL,+BAA+B,EAC/B,YAAY,EACZ,eAAe,EACf,sBAAsB,EACtB,2BAA2B,GAC5B,MAAM,6BAA6B,CAAC;AACrC,OAAO,EAAE,8BAA8B,EAAE,MAAM,qCAAqC,CAAC;AACrF,OAAO,EAAE,gBAAgB,EAAE,MAAM,yBAAyB,CAAC;AAG3D,MAAM,UAAU,GAAG,iEAAiE,CAAC;AACrF,MAAM,SAAS,GACb,yGAAyG;IACzG,2CAA2C;IAC3C,+GAA+G;IAC/G,6HAA6H,CAAC;AAEhI,wFAAwF;AACxF,MAAM,kBAAkB,GAAG,6BAA6B,CAAC;AAEzD,MAAM,aAAa,GACjB,wGAAwG;IACxG,mGAAmG;IACnG,GAAG,kBAAkB,uFAAuF;IAC5G,2GAA2G;IAC3G,0GAA0G;IAC1G,2GAA2G;IAC3G,8EAA8E,CAAC;AAYjF,SAAS,GAAG,CAAC,CAAS;IACpB,OAAO,GAAG,CAAC,CAAC,GAAG,GAAG,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,GAAG,CAAC;AACpC,CAAC;AAED,SAAS,MAAM,CAAC,GAAQ;IACtB,MAAM,EAAE,EAAE,EAAE,EAAE,EAAE,GAAG,GAAG,CAAC,MAAM,CAAC,OAAO,CAAC,SAAS,CAAC;IAChD,OAAO,EAAE,IAAI,EAAE,GAAG,CAAC,IAAI,EAAE,QAAQ,EAAE,EAAE,EAAE,OAAO,EAAE,EAAE,GAAG,EAAE,EAAE,MAAM,EAAE,EAAE,GAAG,CAAC,EAAE,GAAG,EAAE,CAAC,EAAE,aAAa,EAAE,GAAG,CAAC,WAAW,EAAE,CAAC;AACpH,CAAC;AAED,SAAS,GAAG,CAAC,KAAa,EAAE,CAAa;IACvC,OAAO,CAAC,GAAG,CAAC,KAAK,KAAK,CAAC,MAAM,CAAC,EAAE,CAAC,IAAI,CAAC,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,QAAQ,IAAI,CAAC,CAAC,OAAO,MAAM,GAAG,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,aAAa,iBAAiB,CAAC,CAAC;AAC5I,CAAC;AAED,SAAS,YAAY,CAAC,IAAY;IAChC,IAAI,CAAC;QACH,OAAO,YAAY,CAAC,KAAK,EAAE,CAAC,WAAW,EAAE,YAAY,CAAC,EAAE,EAAE,GAAG,EAAE,IAAI,EAAE,QAAQ,EAAE,OAAO,EAAE,CAAC,CAAC,IAAI,EAAE,CAAC;IACnG,CAAC;IAAC,MAAM,CAAC;QACP,OAAO,SAAS,CAAC;IACnB,CAAC;AACH,CAAC;AAED,KAAK,UAAU,QAAQ,CACrB,IAA2C,EAC3C,QAAgB,EAChB,GAAsB,EACtB,WAAmB;IAEnB,MAAM,EAAE,GAAG,EAAE,OAAO,EAAE,GAAG,eAAe,CAAC,QAAQ,EAAE,GAAG,CAAC,CAAC;IACxD,MAAM,IAAI,GAAG,MAAM,CAAC,MAAM,OAAO,CAAC,EAAE,GAAG,IAAI,EAAE,QAAQ,EAAE,GAAG,EAAE,IAAI,EAAE,MAAM,EAAE,CAAC,CAAC,CAAC;IAC7E,MAAM,OAAO,GAAG,MAAM,CAAC,MAAM,OAAO,CAAC,EAAE,GAAG,IAAI,EAAE,QAAQ,EAAE,GAAG,EAAE,IAAI,EAAE,SAAS,EAAE,CAAC,CAAC,CAAC;IACnF,GAAG,CAAC,WAAW,OAAO,CAAC,MAAM,WAAW,WAAW,EAAE,EAAE,IAAI,CAAC,CAAC;IAC7D,GAAG,CAAC,WAAW,OAAO,CAAC,MAAM,WAAW,WAAW,EAAE,EAAE,OAAO,CAAC,CAAC;IAChE,OAAO,EAAE,WAAW,EAAE,gBAAgB,EAAE,OAAO,EAAE,KAAK,EAAE,EAAE,IAAI,EAAE,OAAO,EAAE,EAAE,CAAC;AAC9E,CAAC;AAED,KAAK,UAAU,IAAI;IACjB,MAAM,IAAI,GAAG,OAAO,CAAC,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,OAAO,IAAI,GAAG,EAAE,IAAI,EAAE,IAAI,CAAC,CAAC,CAAC;IACnE,MAAM,QAAQ,GAAG,IAAI,CAAC,IAAI,EAAE,OAAO,CAAC,CAAC;IACrC,MAAM,cAAc,GAAG,IAAI,CAAC,IAAI,EAAE,MAAM,EAAE,wBAAwB,CAAC,CAAC;IACpE,MAAM,QAAQ,GAAG,IAAI,CAAC,IAAI,EAAE,MAAM,EAAE,aAAa,EAAE,yBAAyB,CAAC,CAAC;IAC9E,MAAM,YAAY,GAAG,IAAI,CAAC,IAAI,EAAE,MAAM,EAAE,aAAa,EAAE,kCAAkC,CAAC,CAAC;IAC3F,MAAM,UAAU,GAAG,IAAI,CAAC,IAAI,EAAE,MAAM,EAAE,aAAa,EAAE,8BAA8B,CAAC,CAAC;IAErF,MAAM,QAAQ,GAAG,YAAY,CAAC,YAAY,CAAC,CAAC;IAC5C,MAAM,SAAS,GAAG,2BAA2B,CAAC,QAAQ,EAAE,QAAQ,CAAC,CAAC;IAClE,MAAM,MAAM,GAAG,qBAAqB,CAAC,QAAQ,CAAC,CAAC;IAC/C,MAAM,KAAK,GAAG,yBAAyB,CAAC,MAAM,CAAC,CAAC;IAChD,IAAI,KAAK,CAAC,KAAK,KAAK,QAAQ,CAAC,OAAO,EAAE,CAAC;QACrC,MAAM,IAAI,KAAK,CAAC,cAAc,KAAK,CAAC,KAAK,2BAA2B,QAAQ,CAAC,OAAO,EAAE,CAAC,CAAC;IAC1F,CAAC;IACD,OAAO,CAAC,GAAG,CAAC,qDAAqD,CAAC,CAAC;IACnE,OAAO,CAAC,GAAG,CAAC,YAAY,QAAQ,CAAC,UAAU,IAAI,QAAQ,CAAC,gBAAgB,EAAE,CAAC,CAAC;IAC5E,OAAO,CAAC,GAAG,CAAC,YAAY,KAAK,CAAC,KAAK,oBAAoB,SAAS,EAAE,CAAC,CAAC;IAEpE,MAAM,MAAM,GAAG,EAAE,MAAM,EAAE,cAAc,EAAW,CAAC;IACnD,MAAM,IAAI,GAAG,MAAM,OAAO,CAAC;QACzB,GAAG,MAAM;QACT,QAAQ;QACR,IAAI,EAAE,MAAM;QACZ,UAAU;QACV,UAAU,EAAE,EAAE,SAAS,EAAE,GAAG,EAAE,SAAS,EAAE,GAAG,EAAE,KAAK,EAAE,GAAG,EAAE,eAAe,EAAE,GAAG,EAAE;KACjF,CAAC,CAAC;IACH,MAAM,KAAK,GAAG,MAAM,OAAO,CAAC,EAAE,GAAG,MAAM,EAAE,QAAQ,EAAE,IAAI,EAAE,OAAO,EAAE,CAAC,CAAC;IACpE,MAAM,UAAU,GAAG,MAAM,OAAO,CAAC,EAAE,GAAG,MAAM,EAAE,QAAQ,EAAE,IAAI,EAAE,SAAS,EAAE,CAAC,CAAC;IAC3E,MAAM,WAAW,GAAG,mBAAmB,CAAC,UAAU,EAAE,IAAI,CAAC,CAAC;IAC1D,KAAK,MAAM,CAAC,IAAI,CAAC,KAAK,EAAE,UAAU,CAAC,EAAE,CAAC;QACpC,IAAI,CAAC,CAAC,UAAU,KAAK,IAAI,CAAC,UAAU,IAAI,CAAC,CAAC,SAAS,CAAC,IAAI,CAAC,GAAG,CAAC,KAAK,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC;YAC3F,MAAM,IAAI,KAAK,CAAC,OAAO,CAAC,CAAC,IAAI,wDAAwD,CAAC,CAAC;QACzF,CAAC;IACH,CAAC;IACD,OAAO,CAAC,GAAG,CAAC,YAAY,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,KAAK,CAAC,mBAAmB,IAAI,CAAC,UAAU,EAAE,CAAC,CAAC;IACxF,MAAM,KAAK,GAAG,EAAE,IAAI,EAAE,MAAM,CAAC,IAAI,CAAC,EAAE,KAAK,EAAE,MAAM,CAAC,KAAK,CAAC,EAAE,OAAO,EAAE,MAAM,CAAC,UAAU,CAAC,EAAE,CAAC;IACxF,KAAK,MAAM,CAAC,IAAI,MAAM,CAAC,MAAM,CAAC,KAAK,CAAC;QAAE,GAAG,CAAC,WAAW,EAAE,CAAC,CAAC,CAAC;IAE1D,MAAM,SAAS,GAAG,sBAAsB,CAAC,QAAQ,CAAC,CAAC;IACnD,MAAM,QAAQ,GAAG;QACf,IAAI,EAAE,aAAa;QACnB,SAAS,EAAE,kBAAkB;QAC7B,CAAC,kBAAkB,CAAC,EAAE,MAAM,QAAQ,CAAC,MAAM,EAAE,QAAQ,EAAE,SAAS,EAC9D,8CAA8C,CAAC;QACjD,wBAAwB,EAAE,MAAM,QAAQ,CAAC,MAAM,EAAE,QAAQ,EAAE,+BAA+B,EACxF,yCAAyC,CAAC;KAC7C,CAAC;IAEF,+EAA+E;IAC/E,MAAM,WAAW,GAAG,MAAM,CAAC,MAAM,OAAO,CAAC,EAAE,MAAM,EAAE,QAAQ,EAAE,IAAI,EAAE,MAAM,EAAE,eAAe,EAAE,KAAK,EAAE,CAAC,CAAC,CAAC;IACtG,GAAG,CAAC,+BAA+B,EAAE,WAAW,CAAC,CAAC;IAClD,MAAM,SAAS,GAAG,IAAI,CAAC,SAAS,CAAC,QAAQ,CAAC,mBAAmB,CAAC;QAC5D,CAAC,CAAC,MAAM,mBAAmB,CAAC,cAAc,EAAE,6BAA6B,CAAC;QAC1E,CAAC,CAAC,IAAI,CAAC;IAET,MAAM,KAAK,GAAG,8BAA8B,CAAC,IAAI,CAAC,MAAM,EAAE;QACxD,MAAM,EAAE,aAAa;QACrB,cAAc,EAAE,IAAI;QACpB,UAAU,EAAE,GAAG,UAAU,SAAS,QAAQ,CAAC,gBAAgB,EAAE;QAC7D,WAAW;QACX,KAAK,EACH,8FAA8F;YAC9F,+FAA+F;YAC/F,oFAAoF,GAAG,kBAAkB;YACzG,oGAAoG;YACpG,uGAAuG;YACvG,yGAAyG;YACzG,iGAAiG;YACjG,gGAAgG;YAChG,yFAAyF,IAAI,CAAC,UAAU,IAAI;YAC5G,UAAU,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,GAAG,CAAC,gBAAgB,SAAS,uCAAuC;KACrG,CAAC,CAAC;IACH,MAAM,EAAE,eAAe,EAAE,GAAG,gBAAgB,CAC1C;QACE,GAAG,KAAK;QACR,aAAa,EAAE,QAAQ,CAAC,gBAAgB;QACxC,SAAS,EAAE;YACT,GAAG,KAAK,CAAC,SAAS;YAClB,KAAK;YACL,iBAAiB,EAAE;gBACjB,KAAK,EAAE,SAAS;gBAChB,OAAO,EACL,mGAAmG;oBACnG,mEAAmE;gBACrE,UAAU,EAAE,4BAA4B;aACzC;YACD,kBAAkB,EAAE,QAAQ;YAC5B,qBAAqB,EAAE;gBACrB,MAAM,EACJ,0GAA0G;oBAC1G,oGAAoG;oBACpG,6DAA6D;gBAC/D,cAAc,EAAE,CAAC;aAClB;YACD,UAAU,EAAE;gBACV,UAAU,EAAE,QAAQ,CAAC,UAAU;gBAC/B,gBAAgB,EAAE,QAAQ,CAAC,gBAAgB;gBAC3C,cAAc,EAAE,QAAQ,CAAC,cAAc;gBACvC,UAAU,EAAE,QAAQ,CAAC,UAAU;gBAC/B,SAAS,EAAE,QAAQ,CAAC,SAAS;gBAC7B,MAAM,EAAE,QAAQ,CAAC,MAAM;gBACvB,sBAAsB,EAAE,QAAQ,CAAC,sBAAsB;gBACvD,aAAa,EAAE,SAAS;gBACxB,iBAAiB,EAAE,QAAQ,CAAC,iBAAiB;gBAC7C,QAAQ,EAAE,mDAAmD;gBAC7D,cAAc,EAAE,YAAY,CAAC,IAAI,CAAC;gBAClC,WAAW,EAAE,IAAI,CAAC,UAAU;gBAC5B,KAAK,EAAE,IAAI,CAAC,SAAS;gBACrB,SAAS;gBACT,2BAA2B,EAAE,WAAW;gBACxC,SAAS,EAAE,SAAS;aACrB;SACF;KACF,EACD,EAAE,KAAK,EAAE,IAAI,EAAE,CAChB,CAAC;IACF,OAAO,CAAC,GAAG,CAAC,kBAAkB,eAAe,EAAE,CAAC,CAAC;AACnD,CAAC;AAED,IAAI,EAAE,CAAC,KAAK,CAAC,CAAC,GAAG,EAAE,EAAE;IACnB,OAAO,CAAC,KAAK,CAAC,QAAQ,EAAE,GAAG,CAAC,CAAC;IAC7B,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;AAClB,CAAC,CAAC,CAAC"}
|
|
@@ -16,7 +16,7 @@
|
|
|
16
16
|
*/
|
|
17
17
|
import { resolve, join } from 'node:path';
|
|
18
18
|
import { loadPintCorpus, getPintCorpusStats } from './pint-corpus.js';
|
|
19
|
-
import { runEval } from './eval-harness.js';
|
|
19
|
+
import { enforceLaneFromEval, runEval } from './eval-harness.js';
|
|
20
20
|
import { writeMeasurementFromEvalReport } from '../measurement/from-eval-harness.js';
|
|
21
21
|
// ---------------------------------------------------------------------------
|
|
22
22
|
// Formatting helpers
|
|
@@ -149,9 +149,19 @@ async function main() {
|
|
|
149
149
|
}
|
|
150
150
|
}
|
|
151
151
|
console.log(`\nReport saved to: ${outputPath}`);
|
|
152
|
+
// The same corpus, event shape and tiers on an enforce-lane engine, recorded
|
|
153
|
+
// beside the headline as breakdown.enforce_lane (never instead of it).
|
|
154
|
+
const enforceLane = enforceLaneFromEval(await runEval({ rulesDir, corpus, lane: 'enforce' }), {
|
|
155
|
+
report,
|
|
156
|
+
eventShape,
|
|
157
|
+
tiersUsed,
|
|
158
|
+
});
|
|
159
|
+
console.log(`Enforce lane: ${enforceLane.detected}/${enforceLane.attacks} attacks = ${formatPercent(enforceLane.recall)} ` +
|
|
160
|
+
`(${enforceLane.rules_in_lane} rules in lane)`);
|
|
152
161
|
// Also write the standardized Measurement file (version-pinned, immutable).
|
|
153
162
|
const { measurementPath } = writeMeasurementFromEvalReport(report, {
|
|
154
163
|
source: 'pint',
|
|
164
|
+
enforceLane,
|
|
155
165
|
source_version: 'v1',
|
|
156
166
|
source_url: 'https://github.com/lakeraai/pint-benchmark',
|
|
157
167
|
notes: 'PINT-FORMAT corpus, self-built: 850 samples (deepset/prompt-injections + ' +
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"run-pint-benchmark.js","sourceRoot":"","sources":["../../src/eval/run-pint-benchmark.ts"],"names":[],"mappings":";AACA;;;;;;;;;;;;;;GAcG;AAEH,OAAO,EAAE,OAAO,EAAE,IAAI,EAAE,MAAM,WAAW,CAAC;AAC1C,OAAO,EAAE,cAAc,EAAE,kBAAkB,EAAE,MAAM,kBAAkB,CAAC;AACtE,OAAO,EAAE,OAAO,EAAE,MAAM,mBAAmB,CAAC;
|
|
1
|
+
{"version":3,"file":"run-pint-benchmark.js","sourceRoot":"","sources":["../../src/eval/run-pint-benchmark.ts"],"names":[],"mappings":";AACA;;;;;;;;;;;;;;GAcG;AAEH,OAAO,EAAE,OAAO,EAAE,IAAI,EAAE,MAAM,WAAW,CAAC;AAC1C,OAAO,EAAE,cAAc,EAAE,kBAAkB,EAAE,MAAM,kBAAkB,CAAC;AACtE,OAAO,EAAE,mBAAmB,EAAE,OAAO,EAAE,MAAM,mBAAmB,CAAC;AACjE,OAAO,EAAE,8BAA8B,EAAE,MAAM,qCAAqC,CAAC;AAErF,8EAA8E;AAC9E,qBAAqB;AACrB,8EAA8E;AAE9E,SAAS,aAAa,CAAC,CAAS;IAC9B,OAAO,GAAG,CAAC,CAAC,GAAG,GAAG,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,GAAG,CAAC;AACpC,CAAC;AAED,SAAS,QAAQ,CAAC,CAAS;IACzB,OAAO,GAAG,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,IAAI,CAAC;AAC7B,CAAC;AAED,8EAA8E;AAC9E,OAAO;AACP,8EAA8E;AAE9E,KAAK,UAAU,IAAI;IACjB,MAAM,IAAI,GAAG,OAAO,CAAC,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,OAAO,IAAI,GAAG,EAAE,IAAI,EAAE,IAAI,CAAC,CAAC,CAAC;IACnE,MAAM,QAAQ,GAAG,IAAI,CAAC,IAAI,EAAE,OAAO,CAAC,CAAC;IACrC,MAAM,QAAQ,GAAG,IAAI,CAAC,IAAI,EAAE,MAAM,EAAE,gBAAgB,EAAE,kBAAkB,CAAC,CAAC;IAC1E,MAAM,UAAU,GAAG,IAAI,CAAC,IAAI,EAAE,MAAM,EAAE,gBAAgB,EAAE,uBAAuB,CAAC,CAAC;IAEjF,OAAO,CAAC,GAAG,CAAC,8CAA8C,CAAC,CAAC;IAC5D,OAAO,CAAC,GAAG,CAAC,WAAW,QAAQ,EAAE,CAAC,CAAC;IACnC,OAAO,CAAC,GAAG,CAAC,WAAW,QAAQ,IAAI,CAAC,CAAC;IAErC,uBAAuB;IACvB,MAAM,MAAM,GAAG,cAAc,CAAC,QAAQ,CAAC,CAAC;IACxC,MAAM,KAAK,GAAG,kBAAkB,CAAC,MAAM,CAAC,CAAC;IAEzC,OAAO,CAAC,GAAG,CAAC,UAAU,KAAK,CAAC,KAAK,aAAa,KAAK,CAAC,OAAO,aAAa,KAAK,CAAC,MAAM,UAAU,CAAC,CAAC;IAChG,OAAO,CAAC,GAAG,CAAC,eAAe,MAAM,CAAC,OAAO,CAAC,KAAK,CAAC,UAAU,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,EAAE,EAAE,CAAC,GAAG,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;IACvG,OAAO,CAAC,GAAG,CAAC,eAAe,MAAM,CAAC,OAAO,CAAC,KAAK,CAAC,YAAY,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,EAAE,EAAE,CAAC,GAAG,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;IACzG,OAAO,CAAC,GAAG,CAAC,eAAe,MAAM,CAAC,OAAO,CAAC,KAAK,CAAC,UAAU,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,EAAE,EAAE,CAAC,GAAG,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;IAEvG,kEAAkE;IAClE,uEAAuE;IACvE,uEAAuE;IACvE,kEAAkE;IAClE,mFAAmF;IACnF,+EAA+E;IAC/E,MAAM,cAAc,GAAG;QACrB,SAAS,EAAE,IAAI;QACf,SAAS,EAAE,IAAI;QACf,KAAK,EAAE,IAAI;QACX,eAAe,EAAE,GAAG;KACrB,CAAC;IAEF,gCAAgC;IAChC,MAAM,EAAE,MAAM,EAAE,UAAU,EAAE,SAAS,EAAE,WAAW,EAAE,UAAU,EAAE,GAAG,MAAM,OAAO,CAAC;QAC/E,QAAQ;QACR,MAAM;QACN,UAAU,EAAE,cAAc;QAC1B,UAAU;KACX,CAAC,CAAC;IAEH,kBAAkB;IAClB,OAAO,CAAC,GAAG,CAAC,YAAY,SAAS,CAAC,IAAI,CAAC,KAAK,CAAC,EAAE,CAAC,CAAC;IACjD,OAAO,CAAC,GAAG,CAAC,gBAAgB,UAAU,EAAE,CAAC,CAAC;IAC1C,OAAO,CAAC,GAAG,CAAC,mBAAmB,CAAC,CAAC;IACjC,OAAO,CAAC,GAAG,CAAC,iBAAiB,aAAa,CAAC,MAAM,CAAC,OAAO,CAAC,SAAS,CAAC,EAAE,CAAC,CAAC;IACxE,OAAO,CAAC,GAAG,CAAC,iBAAiB,aAAa,CAAC,MAAM,CAAC,OAAO,CAAC,MAAM,CAAC,EAAE,CAAC,CAAC;IACrE,OAAO,CAAC,GAAG,CAAC,iBAAiB,aAAa,CAAC,MAAM,CAAC,OAAO,CAAC,EAAE,CAAC,EAAE,CAAC,CAAC;IACjE,OAAO,CAAC,GAAG,CAAC,iBAAiB,aAAa,CAAC,MAAM,CAAC,OAAO,CAAC,QAAQ,CAAC,EAAE,CAAC,CAAC;IACvE,OAAO,CAAC,GAAG,CAAC,iBAAiB,aAAa,CAAC,MAAM,CAAC,OAAO,CAAC,MAAM,CAAC,EAAE,CAAC,CAAC;IACrE,OAAO,CAAC,GAAG,CAAC,oBAAoB,MAAM,CAAC,OAAO,CAAC,SAAS,CAAC,EAAE,OAAO,MAAM,CAAC,OAAO,CAAC,SAAS,CAAC,EAAE,OAAO,MAAM,CAAC,OAAO,CAAC,SAAS,CAAC,EAAE,OAAO,MAAM,CAAC,OAAO,CAAC,SAAS,CAAC,EAAE,EAAE,CAAC,CAAC;IAErK,UAAU;IACV,OAAO,CAAC,GAAG,CAAC,mBAAmB,CAAC,CAAC;IACjC,OAAO,CAAC,GAAG,CAAC,WAAW,QAAQ,CAAC,MAAM,CAAC,OAAO,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC;IACvD,OAAO,CAAC,GAAG,CAAC,WAAW,QAAQ,CAAC,MAAM,CAAC,OAAO,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC;IACvD,OAAO,CAAC,GAAG,CAAC,WAAW,QAAQ,CAAC,MAAM,CAAC,OAAO,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC;IACvD,OAAO,CAAC,GAAG,CAAC,WAAW,QAAQ,CAAC,MAAM,CAAC,OAAO,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;IACxD,OAAO,CAAC,GAAG,CAAC,WAAW,QAAQ,CAAC,MAAM,CAAC,OAAO,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC;IAEvD,eAAe;IACf,OAAO,CAAC,GAAG,CAAC,uBAAuB,CAAC,CAAC;IACrC,KAAK,MAAM,GAAG,IAAI,MAAM,CAAC,UAAU,EAAE,CAAC;QACpC,MAAM,MAAM,GAAG,GAAG,CAAC,aAAa,CAAC,MAAM,CAAC;QACxC,MAAM,GAAG,GAAG,GAAG,CAAC,cAAc,CAAC,MAAM,CAAC;QACtC,OAAO,CAAC,GAAG,CACT,KAAK,GAAG,CAAC,QAAQ,YAAY,aAAa,CAAC,GAAG,CAAC,OAAO,CAAC,MAAM,CAAC,GAAG;YACjE,aAAa,aAAa,CAAC,GAAG,CAAC,OAAO,CAAC,SAAS,CAAC,GAAG;YACpD,MAAM,aAAa,CAAC,GAAG,CAAC,OAAO,CAAC,EAAE,CAAC,GAAG;YACtC,WAAW,MAAM,QAAQ,GAAG,GAAG,CAChC,CAAC;IACJ,CAAC;IAED,iBAAiB;IACjB,OAAO,CAAC,GAAG,CAAC,yBAAyB,CAAC,CAAC;IACvC,KAAK,MAAM,IAAI,IAAI,MAAM,CAAC,YAAY,EAAE,CAAC;QACvC,OAAO,CAAC,GAAG,CACT,KAAK,IAAI,CAAC,UAAU,YAAY,aAAa,CAAC,IAAI,CAAC,OAAO,CAAC,MAAM,CAAC,GAAG;YACrE,aAAa,aAAa,CAAC,IAAI,CAAC,OAAO,CAAC,SAAS,CAAC,GAAG;YACrD,MAAM,aAAa,CAAC,IAAI,CAAC,OAAO,CAAC,EAAE,CAAC,EAAE,CACvC,CAAC;IACJ,CAAC;IAED,iCAAiC;IACjC,IAAI,MAAM,CAAC,aAAa,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QACpC,MAAM,SAAS,GAAG,IAAI,CAAC,GAAG,CAAC,MAAM,CAAC,aAAa,CAAC,MAAM,EAAE,EAAE,CAAC,CAAC;QAC5D,OAAO,CAAC,GAAG,CAAC,yBAAyB,MAAM,CAAC,aAAa,CAAC,MAAM,mBAAmB,SAAS,OAAO,CAAC,CAAC;QACrG,KAAK,MAAM,CAAC,IAAI,MAAM,CAAC,aAAa,CAAC,KAAK,CAAC,CAAC,EAAE,SAAS,CAAC,EAAE,CAAC;YACzD,MAAM,IAAI,GAAG,MAAM,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,EAAE,KAAK,CAAC,CAAC,EAAE,CAAC,EAAE,MAAM,EAAE,CAAC,UAAU,CAAC,IAAI,GAAG,CAAC;YAC5E,OAAO,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC,EAAE,UAAU,IAAI,IAAI,CAAC,CAAC,QAAQ,IAAI,CAAC,CAAC,UAAU,EAAE,CAAC,CAAC;QACxE,CAAC;QACD,IAAI,MAAM,CAAC,aAAa,CAAC,MAAM,GAAG,SAAS,EAAE,CAAC;YAC5C,OAAO,CAAC,GAAG,CAAC,aAAa,MAAM,CAAC,aAAa,CAAC,MAAM,GAAG,SAAS,OAAO,CAAC,CAAC;QAC3E,CAAC;IACH,CAAC;IAED,kCAAkC;IAClC,IAAI,MAAM,CAAC,cAAc,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QACrC,MAAM,SAAS,GAAG,IAAI,CAAC,GAAG,CAAC,MAAM,CAAC,cAAc,CAAC,MAAM,EAAE,EAAE,CAAC,CAAC;QAC7D,OAAO,CAAC,GAAG,CAAC,0BAA0B,MAAM,CAAC,cAAc,CAAC,MAAM,mBAAmB,SAAS,OAAO,CAAC,CAAC;QACvG,KAAK,MAAM,EAAE,IAAI,MAAM,CAAC,cAAc,CAAC,KAAK,CAAC,CAAC,EAAE,SAAS,CAAC,EAAE,CAAC;YAC3D,OAAO,CAAC,GAAG,CAAC,MAAM,EAAE,CAAC,EAAE,YAAY,EAAE,CAAC,YAAY,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;QACnE,CAAC;QACD,IAAI,MAAM,CAAC,cAAc,CAAC,MAAM,GAAG,SAAS,EAAE,CAAC;YAC7C,OAAO,CAAC,GAAG,CAAC,aAAa,MAAM,CAAC,cAAc,CAAC,MAAM,GAAG,SAAS,OAAO,CAAC,CAAC;QAC5E,CAAC;IACH,CAAC;IAED,uBAAuB;IACvB,OAAO,CAAC,GAAG,CAAC,wBAAwB,CAAC,CAAC;IACtC,OAAO,CAAC,GAAG,CAAC,yBAAyB,WAAW,CAAC,mBAAmB,EAAE,CAAC,CAAC;IACxE,OAAO,CAAC,GAAG,CAAC,kBAAkB,WAAW,CAAC,UAAU,EAAE,CAAC,CAAC;IACxD,OAAO,CAAC,GAAG,CAAC,wBAAwB,WAAW,CAAC,eAAe,EAAE,CAAC,CAAC;IAEnE,IAAI,WAAW,CAAC,QAAQ,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QACpC,OAAO,CAAC,GAAG,CAAC,kCAAkC,CAAC,CAAC;QAChD,KAAK,MAAM,IAAI,IAAI,WAAW,CAAC,QAAQ,CAAC,KAAK,CAAC,CAAC,EAAE,EAAE,CAAC,EAAE,CAAC;YACrD,MAAM,SAAS,GAAG,IAAI,CAAC,UAAU,GAAG,CAAC;gBACnC,CAAC,CAAC,aAAa,CAAC,IAAI,CAAC,OAAO,GAAG,IAAI,CAAC,UAAU,CAAC;gBAC/C,CAAC,CAAC,KAAK,CAAC;YACV,OAAO,CAAC,GAAG,CACT,OAAO,IAAI,CAAC,MAAM,aAAa,IAAI,CAAC,UAAU,GAAG;gBACjD,MAAM,IAAI,CAAC,OAAO,OAAO,IAAI,CAAC,OAAO,GAAG;gBACxC,aAAa,SAAS,EAAE,CACzB,CAAC;QACJ,CAAC;IACH,CAAC;IAED,aAAa;IACb,OAAO,CAAC,GAAG,CAAC,8CAA8C,CAAC,CAAC;IAC5D,IAAI,UAAU,CAAC,MAAM,EAAE,CAAC;QACtB,OAAO,CAAC,GAAG,CAAC,UAAU,CAAC,CAAC;IAC1B,CAAC;SAAM,CAAC;QACN,OAAO,CAAC,GAAG,CAAC,WAAW,CAAC,CAAC;QACzB,KAAK,MAAM,CAAC,IAAI,UAAU,CAAC,UAAU,EAAE,CAAC;YACtC,OAAO,CAAC,GAAG,CAAC,SAAS,CAAC,EAAE,CAAC,CAAC;QAC5B,CAAC;IACH,CAAC;IAED,OAAO,CAAC,GAAG,CAAC,sBAAsB,UAAU,EAAE,CAAC,CAAC;IAEhD,6EAA6E;IAC7E,uEAAuE;IACvE,MAAM,WAAW,GAAG,mBAAmB,CAAC,MAAM,OAAO,CAAC,EAAE,QAAQ,EAAE,MAAM,EAAE,IAAI,EAAE,SAAS,EAAE,CAAC,EAAE;QAC5F,MAAM;QACN,UAAU;QACV,SAAS;KACV,CAAC,CAAC;IACH,OAAO,CAAC,GAAG,CACT,iBAAiB,WAAW,CAAC,QAAQ,IAAI,WAAW,CAAC,OAAO,cAAc,aAAa,CAAC,WAAW,CAAC,MAAM,CAAC,GAAG;QAC5G,IAAI,WAAW,CAAC,aAAa,iBAAiB,CACjD,CAAC;IAEF,4EAA4E;IAC5E,MAAM,EAAE,eAAe,EAAE,GAAG,8BAA8B,CAAC,MAAM,EAAE;QACjE,MAAM,EAAE,MAAM;QACd,WAAW;QACX,cAAc,EAAE,IAAI;QACpB,UAAU,EAAE,4CAA4C;QACxD,KAAK,EACH,2EAA2E;YAC3E,4EAA4E;YAC5E,+EAA+E;YAC/E,2EAA2E;YAC3E,2DAA2D,UAAU,KAAK;YAC1E,uEAAuE;YACvE,6EAA6E;YAC7E,0EAA0E;KAC7E,CAAC,CAAC;IACH,OAAO,CAAC,GAAG,CAAC,gBAAgB,eAAe,EAAE,CAAC,CAAC;IAC/C,OAAO,CAAC,GAAG,CAAC,SAAS,CAAC,CAAC;IAEvB,IAAI,CAAC,UAAU,CAAC,MAAM,EAAE,CAAC;QACvB,OAAO,CAAC,QAAQ,GAAG,CAAC,CAAC;IACvB,CAAC;AACH,CAAC;AAED,IAAI,EAAE,CAAC,KAAK,CAAC,CAAC,GAAG,EAAE,EAAE;IACnB,OAAO,CAAC,KAAK,CAAC,wBAAwB,EAAE,GAAG,CAAC,CAAC;IAC7C,OAAO,CAAC,QAAQ,GAAG,CAAC,CAAC;AACvB,CAAC,CAAC,CAAC"}
|
|
@@ -15,6 +15,8 @@
|
|
|
15
15
|
*
|
|
16
16
|
* @module agent-threat-rules/eval/skill-benchmark
|
|
17
17
|
*/
|
|
18
|
+
import { type MeasurementInput } from '../measurement/write.js';
|
|
19
|
+
import type { Lane } from '../quality/rule-contract.js';
|
|
18
20
|
interface SampleResult {
|
|
19
21
|
readonly file: string;
|
|
20
22
|
readonly label: 'malicious' | 'benign';
|
|
@@ -37,6 +39,10 @@ interface SkillBenchmarkReport {
|
|
|
37
39
|
readonly corpus_size: number;
|
|
38
40
|
/** Rules the engine loaded for this run; the denominator for latency budgets. */
|
|
39
41
|
readonly rule_count: number;
|
|
42
|
+
/** The lane the engine ran in: `hunt` unless the caller asked for another. */
|
|
43
|
+
readonly lane: Lane;
|
|
44
|
+
/** Rules that lane loads: status not draft/deprecated, maturity admitted by the lane. */
|
|
45
|
+
readonly rules_in_lane: number;
|
|
40
46
|
readonly malicious_count: number;
|
|
41
47
|
readonly benign_count: number;
|
|
42
48
|
readonly overall_recall: number;
|
|
@@ -61,19 +67,33 @@ interface SkillBenchmarkReport {
|
|
|
61
67
|
export declare function runSkillBenchmark(options?: {
|
|
62
68
|
readonly rulesDir?: string;
|
|
63
69
|
readonly corpusDir?: string;
|
|
64
|
-
|
|
70
|
+
/** Where the report JSON goes; `null` writes none (a second-lane run must not replace the headline's). */
|
|
71
|
+
readonly outputPath?: string | null;
|
|
72
|
+
/**
|
|
73
|
+
* Detection lane (default: the engine's default, `hunt`, every maturity). The
|
|
74
|
+
* measurement's headline is a hunt run; the CLI makes a second, `enforce` run
|
|
75
|
+
* over the same corpus and records it as breakdown.enforce_lane.
|
|
76
|
+
*/
|
|
77
|
+
readonly lane?: Lane;
|
|
65
78
|
}): Promise<SkillBenchmarkReport>;
|
|
66
79
|
/**
|
|
67
|
-
*
|
|
68
|
-
* report
|
|
69
|
-
*
|
|
70
|
-
*
|
|
80
|
+
* The standardized version-pinned Measurement for a SkillBenchmark run: the
|
|
81
|
+
* headline from `report`, and `enforce` — the same corpus run with
|
|
82
|
+
* `lane: 'enforce'` — as breakdown.enforce_lane. Pure, so tests can assert on
|
|
83
|
+
* exactly what the CLI records without touching data/measurements/.
|
|
84
|
+
*/
|
|
85
|
+
export declare function skillBenchmarkMeasurementInput(report: SkillBenchmarkReport, enforce: SkillBenchmarkReport): MeasurementInput;
|
|
86
|
+
/**
|
|
87
|
+
* Write the Measurement file for a SkillBenchmark run. Separated from
|
|
88
|
+
* runSkillBenchmark so unit tests can exercise the benchmark logic without
|
|
89
|
+
* mutating data/measurements/ on disk (which would make the CI
|
|
90
|
+
* 'sync-stats --check' drift gate flake).
|
|
71
91
|
*
|
|
72
92
|
* Called from the CLI block at the bottom of this file and from any
|
|
73
93
|
* external script that wants to persist the measurement. Safe to call
|
|
74
94
|
* repeatedly the same day — uses force=true.
|
|
75
95
|
*/
|
|
76
|
-
export declare function writeSkillBenchmarkMeasurement(report: SkillBenchmarkReport): void;
|
|
96
|
+
export declare function writeSkillBenchmarkMeasurement(report: SkillBenchmarkReport, enforce: SkillBenchmarkReport): void;
|
|
77
97
|
export declare function printReport(report: SkillBenchmarkReport): void;
|
|
78
98
|
export {};
|
|
79
99
|
//# sourceMappingURL=skill-benchmark.d.ts.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"skill-benchmark.d.ts","sourceRoot":"","sources":["../../src/eval/skill-benchmark.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;GAgBG;
|
|
1
|
+
{"version":3,"file":"skill-benchmark.d.ts","sourceRoot":"","sources":["../../src/eval/skill-benchmark.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;GAgBG;AAOH,OAAO,EAAoB,KAAK,gBAAgB,EAAE,MAAM,yBAAyB,CAAC;AAClF,OAAO,KAAK,EAAE,IAAI,EAAE,MAAM,6BAA6B,CAAC;AA0BxD,UAAU,YAAY;IACpB,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAC;IACtB,QAAQ,CAAC,KAAK,EAAE,WAAW,GAAG,QAAQ,CAAC;IACvC,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAC;IACvB,QAAQ,CAAC,WAAW,EAAE,MAAM,CAAC;IAC7B,QAAQ,CAAC,QAAQ,EAAE,OAAO,CAAC;IAC3B,QAAQ,CAAC,WAAW,EAAE,SAAS,MAAM,EAAE,CAAC;IACxC,QAAQ,CAAC,OAAO,EAAE,OAAO,CAAC;IAC1B,QAAQ,CAAC,UAAU,EAAE,MAAM,CAAC;IAC5B,QAAQ,CAAC,sBAAsB,EAAE,OAAO,CAAC;IACzC,QAAQ,CAAC,gBAAgB,EAAE,OAAO,CAAC;CACpC;AAED,UAAU,YAAY;IACpB,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAC;IACvB,QAAQ,CAAC,QAAQ,EAAE,MAAM,CAAC;IAC1B,QAAQ,CAAC,MAAM,EAAE,MAAM,CAAC;CACzB;AAED,UAAU,oBAAoB;IAC5B,QAAQ,CAAC,SAAS,EAAE,MAAM,CAAC;IAC3B,QAAQ,CAAC,WAAW,EAAE,MAAM,CAAC;IAC7B,iFAAiF;IACjF,QAAQ,CAAC,UAAU,EAAE,MAAM,CAAC;IAC5B,8EAA8E;IAC9E,QAAQ,CAAC,IAAI,EAAE,IAAI,CAAC;IACpB,yFAAyF;IACzF,QAAQ,CAAC,aAAa,EAAE,MAAM,CAAC;IAC/B,QAAQ,CAAC,eAAe,EAAE,MAAM,CAAC;IACjC,QAAQ,CAAC,YAAY,EAAE,MAAM,CAAC;IAC9B,QAAQ,CAAC,cAAc,EAAE,MAAM,CAAC;IAChC,QAAQ,CAAC,iBAAiB,EAAE,MAAM,CAAC;IACnC,QAAQ,CAAC,UAAU,EAAE,MAAM,CAAC;IAC5B,QAAQ,CAAC,OAAO,EAAE,MAAM,CAAC;IACzB,QAAQ,CAAC,OAAO,EAAE,YAAY,CAAC;IAC/B,QAAQ,CAAC,OAAO,EAAE,YAAY,CAAC;IAC/B,QAAQ,CAAC,OAAO,EAAE,YAAY,CAAC;IAC/B,QAAQ,CAAC,cAAc,EAAE,MAAM,CAAC;IAChC,QAAQ,CAAC,eAAe,EAAE,MAAM,CAAC;IACjC,QAAQ,CAAC,cAAc,EAAE,MAAM,CAAC;IAChC,QAAQ,CAAC,eAAe,EAAE,MAAM,CAAC;IACjC,QAAQ,CAAC,uBAAuB,EAAE,MAAM,CAAC;IACzC,QAAQ,CAAC,iBAAiB,EAAE,MAAM,CAAC;IACnC,QAAQ,CAAC,cAAc,EAAE,MAAM,CAAC;IAChC,QAAQ,CAAC,cAAc,EAAE,MAAM,CAAC;IAChC,QAAQ,CAAC,OAAO,EAAE,SAAS,YAAY,EAAE,CAAC;IAC1C,QAAQ,CAAC,cAAc,EAAE,SAAS,YAAY,EAAE,CAAC;IACjD,QAAQ,CAAC,YAAY,EAAE,SAAS,YAAY,EAAE,CAAC;CAChD;AAMD,wBAAsB,iBAAiB,CAAC,OAAO,CAAC,EAAE;IAChD,QAAQ,CAAC,QAAQ,CAAC,EAAE,MAAM,CAAC;IAC3B,QAAQ,CAAC,SAAS,CAAC,EAAE,MAAM,CAAC;IAC5B,0GAA0G;IAC1G,QAAQ,CAAC,UAAU,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;IACpC;;;;OAIG;IACH,QAAQ,CAAC,IAAI,CAAC,EAAE,IAAI,CAAC;CACtB,GAAG,OAAO,CAAC,oBAAoB,CAAC,CAgJhC;AAMD;;;;;GAKG;AACH,wBAAgB,8BAA8B,CAC5C,MAAM,EAAE,oBAAoB,EAC5B,OAAO,EAAE,oBAAoB,GAC5B,gBAAgB,CAkDlB;AAED;;;;;;;;;GASG;AACH,wBAAgB,8BAA8B,CAAC,MAAM,EAAE,oBAAoB,EAAE,OAAO,EAAE,oBAAoB,GAAG,IAAI,CAEhH;AAUD,wBAAgB,WAAW,CAAC,MAAM,EAAE,oBAAoB,GAAG,IAAI,CA6C9D"}
|
|
@@ -18,6 +18,8 @@
|
|
|
18
18
|
import { resolve, join } from 'node:path';
|
|
19
19
|
import { readFileSync, writeFileSync, existsSync } from 'node:fs';
|
|
20
20
|
import { ATREngine } from '../engine.js';
|
|
21
|
+
import { enforceLaneRecall, rulesInLane } from '../measurement/enforce-lane.js';
|
|
22
|
+
import { ENFORCE_LANE_BREAKDOWN_KEY } from '../measurement/schema.js';
|
|
21
23
|
import { writeMeasurement } from '../measurement/write.js';
|
|
22
24
|
// ---------------------------------------------------------------------------
|
|
23
25
|
// Benchmark runner
|
|
@@ -26,7 +28,9 @@ export async function runSkillBenchmark(options) {
|
|
|
26
28
|
const repoRoot = resolve(import.meta.dirname, '..', '..');
|
|
27
29
|
const rulesDir = options?.rulesDir ?? join(repoRoot, 'rules');
|
|
28
30
|
const corpusDir = options?.corpusDir ?? join(repoRoot, 'data', 'skill-benchmark');
|
|
29
|
-
const outputPath = options?.outputPath
|
|
31
|
+
const outputPath = options?.outputPath === null
|
|
32
|
+
? null
|
|
33
|
+
: options?.outputPath ?? join(repoRoot, 'data', 'skill-benchmark', 'benchmark-report.json');
|
|
30
34
|
// Load manifest
|
|
31
35
|
const manifestPath = join(corpusDir, 'manifest.json');
|
|
32
36
|
if (!existsSync(manifestPath)) {
|
|
@@ -41,8 +45,9 @@ export async function runSkillBenchmark(options) {
|
|
|
41
45
|
// Load engine. The rule count is reported because per-sample scan cost is
|
|
42
46
|
// roughly linear in it, so it is the denominator any latency budget for this
|
|
43
47
|
// benchmark has to be expressed in -- see tests/skill-benchmark.test.ts.
|
|
44
|
-
const engine = new ATREngine({ rulesDir });
|
|
48
|
+
const engine = new ATREngine({ rulesDir, lane: options?.lane });
|
|
45
49
|
const ruleCount = await engine.loadRules();
|
|
50
|
+
const lane = engine.getLane();
|
|
46
51
|
// Run each sample
|
|
47
52
|
const results = [];
|
|
48
53
|
for (const entry of manifest) {
|
|
@@ -114,6 +119,8 @@ export async function runSkillBenchmark(options) {
|
|
|
114
119
|
timestamp: new Date().toISOString(),
|
|
115
120
|
corpus_size: results.length,
|
|
116
121
|
rule_count: ruleCount,
|
|
122
|
+
lane,
|
|
123
|
+
rules_in_lane: rulesInLane(engine.getRules(), lane),
|
|
117
124
|
malicious_count: malicious.length,
|
|
118
125
|
benign_count: benign.length,
|
|
119
126
|
overall_recall: Math.round(recall * 1000) / 1000,
|
|
@@ -136,24 +143,27 @@ export async function runSkillBenchmark(options) {
|
|
|
136
143
|
false_alarms: benign.filter((r) => r.detected),
|
|
137
144
|
};
|
|
138
145
|
// Save report
|
|
139
|
-
|
|
146
|
+
if (outputPath !== null)
|
|
147
|
+
writeFileSync(outputPath, JSON.stringify(report, null, 2));
|
|
140
148
|
return report;
|
|
141
149
|
}
|
|
142
150
|
// ---------------------------------------------------------------------------
|
|
143
151
|
// Measurement adapter (CLI-only — tests call runSkillBenchmark and skip this)
|
|
144
152
|
// ---------------------------------------------------------------------------
|
|
145
153
|
/**
|
|
146
|
-
*
|
|
147
|
-
* report
|
|
148
|
-
*
|
|
149
|
-
*
|
|
150
|
-
*
|
|
151
|
-
* Called from the CLI block at the bottom of this file and from any
|
|
152
|
-
* external script that wants to persist the measurement. Safe to call
|
|
153
|
-
* repeatedly the same day — uses force=true.
|
|
154
|
+
* The standardized version-pinned Measurement for a SkillBenchmark run: the
|
|
155
|
+
* headline from `report`, and `enforce` — the same corpus run with
|
|
156
|
+
* `lane: 'enforce'` — as breakdown.enforce_lane. Pure, so tests can assert on
|
|
157
|
+
* exactly what the CLI records without touching data/measurements/.
|
|
154
158
|
*/
|
|
155
|
-
export function
|
|
156
|
-
|
|
159
|
+
export function skillBenchmarkMeasurementInput(report, enforce) {
|
|
160
|
+
if (enforce.lane !== 'enforce') {
|
|
161
|
+
throw new Error(`skill-benchmark: the second report ran in the ${enforce.lane} lane, not enforce`);
|
|
162
|
+
}
|
|
163
|
+
if (enforce.corpus_size !== report.corpus_size || enforce.malicious_count !== report.malicious_count) {
|
|
164
|
+
throw new Error('skill-benchmark: the enforce run did not evaluate the same corpus as the headline');
|
|
165
|
+
}
|
|
166
|
+
return {
|
|
157
167
|
source: 'skill-benchmark',
|
|
158
168
|
source_version: 'internal-498',
|
|
159
169
|
measured_at: report.timestamp,
|
|
@@ -187,9 +197,27 @@ export function writeSkillBenchmarkMeasurement(report) {
|
|
|
187
197
|
benign_count: report.benign_count,
|
|
188
198
|
expected_rules_accuracy: report.expected_rules_accuracy,
|
|
189
199
|
category_accuracy: report.category_accuracy,
|
|
200
|
+
[ENFORCE_LANE_BREAKDOWN_KEY]: enforceLaneRecall({
|
|
201
|
+
detected: enforce.true_positives,
|
|
202
|
+
attacks: enforce.malicious_count,
|
|
203
|
+
rulesInLane: enforce.rules_in_lane,
|
|
204
|
+
}),
|
|
190
205
|
},
|
|
191
206
|
notes: 'Internal 498-sample SKILL.md benchmark. Layer A = obvious payload, Layer B = obfuscated, Layer C = semantic.',
|
|
192
|
-
}
|
|
207
|
+
};
|
|
208
|
+
}
|
|
209
|
+
/**
|
|
210
|
+
* Write the Measurement file for a SkillBenchmark run. Separated from
|
|
211
|
+
* runSkillBenchmark so unit tests can exercise the benchmark logic without
|
|
212
|
+
* mutating data/measurements/ on disk (which would make the CI
|
|
213
|
+
* 'sync-stats --check' drift gate flake).
|
|
214
|
+
*
|
|
215
|
+
* Called from the CLI block at the bottom of this file and from any
|
|
216
|
+
* external script that wants to persist the measurement. Safe to call
|
|
217
|
+
* repeatedly the same day — uses force=true.
|
|
218
|
+
*/
|
|
219
|
+
export function writeSkillBenchmarkMeasurement(report, enforce) {
|
|
220
|
+
writeMeasurement(skillBenchmarkMeasurementInput(report, enforce), { force: true });
|
|
193
221
|
}
|
|
194
222
|
// ---------------------------------------------------------------------------
|
|
195
223
|
// CLI runner
|
|
@@ -239,12 +267,17 @@ export function printReport(report) {
|
|
|
239
267
|
// Direct execution
|
|
240
268
|
// ---------------------------------------------------------------------------
|
|
241
269
|
if (import.meta.url === `file://${process.argv[1]}` || process.argv[1]?.endsWith('skill-benchmark.ts')) {
|
|
242
|
-
runSkillBenchmark().then((report) => {
|
|
270
|
+
runSkillBenchmark().then(async (report) => {
|
|
243
271
|
printReport(report);
|
|
244
272
|
console.log(`Report saved to: data/skill-benchmark/benchmark-report.json`);
|
|
273
|
+
// The same corpus on an enforce-lane engine. No report file: the one above
|
|
274
|
+
// is the headline's.
|
|
275
|
+
const enforce = await runSkillBenchmark({ lane: 'enforce', outputPath: null });
|
|
276
|
+
console.log(`Enforce lane: ${enforce.true_positives}/${enforce.malicious_count} malicious = ` +
|
|
277
|
+
`${formatPercent(enforce.overall_recall)} (${enforce.rules_in_lane} rules in lane)`);
|
|
245
278
|
// Write the standardized version-pinned Measurement file. CLI-only —
|
|
246
279
|
// unit tests call runSkillBenchmark() directly and skip this.
|
|
247
|
-
writeSkillBenchmarkMeasurement(report);
|
|
280
|
+
writeSkillBenchmarkMeasurement(report, enforce);
|
|
248
281
|
console.log(`Measurement: data/measurements/skill-benchmark/`);
|
|
249
282
|
}).catch((err) => {
|
|
250
283
|
console.error('Benchmark failed:', err);
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"skill-benchmark.js","sourceRoot":"","sources":["../../src/eval/skill-benchmark.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;GAgBG;AAEH,OAAO,EAAE,OAAO,EAAE,IAAI,EAAE,MAAM,WAAW,CAAC;AAC1C,OAAO,EAAE,YAAY,EAAE,aAAa,EAAE,UAAU,EAAE,MAAM,SAAS,CAAC;AAClE,OAAO,EAAE,SAAS,EAAE,MAAM,cAAc,CAAC;AACzC,OAAO,EAAE,gBAAgB,EAAE,MAAM,yBAAyB,CAAC;AAwE3D,8EAA8E;AAC9E,mBAAmB;AACnB,8EAA8E;AAE9E,MAAM,CAAC,KAAK,UAAU,iBAAiB,CAAC,OAIvC;IACC,MAAM,QAAQ,GAAG,OAAO,CAAC,MAAM,CAAC,IAAI,CAAC,OAAO,EAAE,IAAI,EAAE,IAAI,CAAC,CAAC;IAC1D,MAAM,QAAQ,GAAG,OAAO,EAAE,QAAQ,IAAI,IAAI,CAAC,QAAQ,EAAE,OAAO,CAAC,CAAC;IAC9D,MAAM,SAAS,GAAG,OAAO,EAAE,SAAS,IAAI,IAAI,CAAC,QAAQ,EAAE,MAAM,EAAE,iBAAiB,CAAC,CAAC;IAClF,MAAM,UAAU,GAAG,OAAO,EAAE,UAAU,IAAI,IAAI,CAAC,QAAQ,EAAE,MAAM,EAAE,iBAAiB,EAAE,uBAAuB,CAAC,CAAC;IAE7G,gBAAgB;IAChB,MAAM,YAAY,GAAG,IAAI,CAAC,SAAS,EAAE,eAAe,CAAC,CAAC;IACtD,IAAI,CAAC,UAAU,CAAC,YAAY,CAAC,EAAE,CAAC;QAC9B,MAAM,IAAI,KAAK,CAAC,uBAAuB,YAAY,EAAE,CAAC,CAAC;IACzD,CAAC;IACD,MAAM,QAAQ,GAA6B,IAAI,CAAC,KAAK,CAAC,YAAY,CAAC,YAAY,EAAE,OAAO,CAAC,CAAC,CAAC;IAE3F,oDAAoD;IACpD,MAAM,YAAY,GAAG,IAAI,CAAC,SAAS,EAAE,wBAAwB,CAAC,CAAC;IAC/D,MAAM,WAAW,GAAkC,UAAU,CAAC,YAAY,CAAC;QACzE,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,YAAY,CAAC,YAAY,EAAE,OAAO,CAAC,CAAC;QACjD,CAAC,CAAC,EAAE,CAAC;IAEP,0EAA0E;IAC1E,6EAA6E;IAC7E,yEAAyE;IACzE,MAAM,MAAM,GAAG,IAAI,SAAS,CAAC,EAAE,QAAQ,EAAE,CAAC,CAAC;IAC3C,MAAM,SAAS,GAAG,MAAM,MAAM,CAAC,SAAS,EAAE,CAAC;IAE3C,kBAAkB;IAClB,MAAM,OAAO,GAAmB,EAAE,CAAC;IAEnC,KAAK,MAAM,KAAK,IAAI,QAAQ,EAAE,CAAC;QAC7B,MAAM,QAAQ,GAAG,IAAI,CAAC,SAAS,EAAE,KAAK,CAAC,IAAI,CAAC,CAAC;QAC7C,IAAI,CAAC,UAAU,CAAC,QAAQ,CAAC,EAAE,CAAC;YAC1B,OAAO,CAAC,KAAK,CAAC,yBAAyB,KAAK,CAAC,IAAI,EAAE,CAAC,CAAC;YACrD,SAAS;QACX,CAAC;QAED,MAAM,OAAO,GAAG,YAAY,CAAC,QAAQ,EAAE,OAAO,CAAC,CAAC;QAChD,MAAM,KAAK,GAAG,WAAW,CAAC,GAAG,EAAE,CAAC;QAChC,MAAM,OAAO,GAAG,MAAM,CAAC,SAAS,CAAC,OAAO,CAAC,CAAC;QAC1C,MAAM,OAAO,GAAG,WAAW,CAAC,GAAG,EAAE,GAAG,KAAK,CAAC;QAE1C,MAAM,QAAQ,GAAG,OAAO,CAAC,MAAM,GAAG,CAAC,CAAC;QACpC,MAAM,UAAU,GAAG,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;QAEjD,MAAM,OAAO,GACX,CAAC,KAAK,CAAC,KAAK,KAAK,WAAW,IAAI,QAAQ,CAAC;YACzC,CAAC,KAAK,CAAC,KAAK,KAAK,QAAQ,IAAI,CAAC,QAAQ,CAAC,CAAC;QAE1C,oEAAoE;QACpE,MAAM,QAAQ,GAAG,WAAW,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;QACzC,MAAM,oBAAoB,GAAG,QAAQ,EAAE,iBAAiB;YACtD,CAAC,CAAC,QAAQ,CAAC,iBAAiB,CAAC,KAAK,CAAC,CAAC,EAAE,EAAE,EAAE,CAAC,UAAU,CAAC,QAAQ,CAAC,EAAE,CAAC,OAAO,CAAC,CAAC;YAC3E,CAAC,CAAC,IAAI,CAAC,CAAC,gCAAgC;QAC1C,MAAM,kBAAkB,GAAG,QAAQ,EAAE,iBAAiB,EAAE,GAAG,CAAC,CAAC,EAAE,EAAE,EAAE,CAAC,EAAE,CAAC,QAAQ,CAAC,IAAI,EAAE,CAAC;QACvF,MAAM,gBAAgB,GAAG,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;QAClE,MAAM,eAAe,GAAG,kBAAkB,CAAC,KAAK,CAAC,CAAC,EAAE,EAAE,EAAE,CACtD,gBAAgB,CAAC,QAAQ,CAAC,EAAqC,CAAC,CACjE,CAAC;QAEF,OAAO,CAAC,IAAI,CAAC;YACX,IAAI,EAAE,KAAK,CAAC,IAAI;YAChB,KAAK,EAAE,KAAK,CAAC,KAAK;YAClB,KAAK,EAAE,KAAK,CAAC,KAAK,IAAI,EAAE;YACxB,WAAW,EAAE,KAAK,CAAC,WAAW;YAC9B,QAAQ;YACR,WAAW,EAAE,UAAU;YACvB,OAAO;YACP,UAAU,EAAE,IAAI,CAAC,KAAK,CAAC,OAAO,GAAG,GAAG,CAAC,GAAG,GAAG;YAC3C,sBAAsB,EAAE,oBAAoB;YAC5C,gBAAgB,EAAE,eAAe;SAClC,CAAC,CAAC;IACL,CAAC;IAED,kBAAkB;IAClB,MAAM,SAAS,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,KAAK,KAAK,WAAW,CAAC,CAAC;IACjE,MAAM,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,KAAK,KAAK,QAAQ,CAAC,CAAC;IAE3D,MAAM,EAAE,GAAG,SAAS,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC,MAAM,CAAC;IACtD,MAAM,EAAE,GAAG,SAAS,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC,MAAM,CAAC;IACvD,MAAM,EAAE,GAAG,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC,MAAM,CAAC;IACnD,MAAM,EAAE,GAAG,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC,MAAM,CAAC;IAEpD,MAAM,MAAM,GAAG,SAAS,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,EAAE,GAAG,SAAS,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,CAAC;IAChE,MAAM,SAAS,GAAG,CAAC,EAAE,GAAG,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,EAAE,GAAG,CAAC,EAAE,GAAG,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;IACrD,MAAM,EAAE,GAAG,CAAC,SAAS,GAAG,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,SAAS,GAAG,MAAM,CAAC,GAAG,CAAC,SAAS,GAAG,MAAM,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;IAC1F,MAAM,MAAM,GAAG,MAAM,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,EAAE,GAAG,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,CAAC;IAE1D,oBAAoB;IACpB,MAAM,YAAY,GAAG,CAAC,KAAa,EAAgB,EAAE;QACnD,MAAM,OAAO,GAAG,SAAS,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,KAAK,KAAK,KAAK,CAAC,CAAC;QAC3D,MAAM,QAAQ,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC,MAAM,CAAC;QAC1D,OAAO;YACL,KAAK,EAAE,OAAO,CAAC,MAAM;YACrB,QAAQ;YACR,MAAM,EAAE,OAAO,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,QAAQ,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC;SAC3D,CAAC;IACJ,CAAC,CAAC;IAEF,MAAM,SAAS,GAAG,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,UAAU,CAAC,CAAC;IAEnD,6BAA6B;IAC7B,MAAM,iBAAiB,GAAG,SAAS,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC;IAC9D,MAAM,qBAAqB,GAAG,iBAAiB,CAAC,MAAM,GAAG,CAAC;QACxD,CAAC,CAAC,iBAAiB,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,sBAAsB,CAAC,CAAC,MAAM,GAAG,iBAAiB,CAAC,MAAM;QAC7F,CAAC,CAAC,CAAC,CAAC;IACN,MAAM,gBAAgB,GAAG,iBAAiB,CAAC,MAAM,GAAG,CAAC;QACnD,CAAC,CAAC,iBAAiB,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,gBAAgB,CAAC,CAAC,MAAM,GAAG,iBAAiB,CAAC,MAAM;QACvF,CAAC,CAAC,CAAC,CAAC;IAEN,MAAM,MAAM,GAAyB;QACnC,SAAS,EAAE,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE;QACnC,WAAW,EAAE,OAAO,CAAC,MAAM;QAC3B,UAAU,EAAE,SAAS;QACrB,eAAe,EAAE,SAAS,CAAC,MAAM;QACjC,YAAY,EAAE,MAAM,CAAC,MAAM;QAC3B,cAAc,EAAE,IAAI,CAAC,KAAK,CAAC,MAAM,GAAG,IAAI,CAAC,GAAG,IAAI;QAChD,iBAAiB,EAAE,IAAI,CAAC,KAAK,CAAC,SAAS,GAAG,IAAI,CAAC,GAAG,IAAI;QACtD,UAAU,EAAE,IAAI,CAAC,KAAK,CAAC,EAAE,GAAG,IAAI,CAAC,GAAG,IAAI;QACxC,OAAO,EAAE,IAAI,CAAC,KAAK,CAAC,MAAM,GAAG,IAAI,CAAC,GAAG,IAAI;QACzC,OAAO,EAAE,YAAY,CAAC,GAAG,CAAC;QAC1B,OAAO,EAAE,YAAY,CAAC,GAAG,CAAC;QAC1B,OAAO,EAAE,YAAY,CAAC,GAAG,CAAC;QAC1B,cAAc,EAAE,EAAE;QAClB,eAAe,EAAE,EAAE;QACnB,cAAc,EAAE,EAAE;QAClB,eAAe,EAAE,EAAE;QACnB,uBAAuB,EAAE,IAAI,CAAC,KAAK,CAAC,qBAAqB,GAAG,IAAI,CAAC,GAAG,IAAI;QACxE,iBAAiB,EAAE,IAAI,CAAC,KAAK,CAAC,gBAAgB,GAAG,IAAI,CAAC,GAAG,IAAI;QAC7D,cAAc,EAAE,IAAI,CAAC,KAAK,CAAC,CAAC,SAAS,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC,GAAG,SAAS,CAAC,MAAM,CAAC,GAAG,GAAG,CAAC,GAAG,GAAG;QACjG,cAAc,EAAE,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,GAAG,CAAC,GAAG,SAAS,CAAC,GAAG,GAAG,CAAC,GAAG,GAAG;QAC9D,OAAO;QACP,cAAc,EAAE,SAAS,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,QAAQ,CAAC;QACpD,YAAY,EAAE,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,QAAQ,CAAC;KAC/C,CAAC;IAEF,cAAc;IACd,aAAa,CAAC,UAAU,EAAE,IAAI,CAAC,SAAS,CAAC,MAAM,EAAE,IAAI,EAAE,CAAC,CAAC,CAAC,CAAC;IAE3D,OAAO,MAAM,CAAC;AAChB,CAAC;AAED,8EAA8E;AAC9E,8EAA8E;AAC9E,8EAA8E;AAE9E;;;;;;;;;GASG;AACH,MAAM,UAAU,8BAA8B,CAAC,MAA4B;IACzE,gBAAgB,CACd;QACE,MAAM,EAAE,iBAAiB;QACzB,cAAc,EAAE,cAAc;QAC9B,WAAW,EAAE,MAAM,CAAC,SAAS;QAC7B,OAAO,EAAE,MAAM,CAAC,WAAW;QAC3B,OAAO,EAAE;YACP,MAAM,EAAE,MAAM,CAAC,cAAc;YAC7B,SAAS,EAAE,MAAM,CAAC,iBAAiB;YACnC,EAAE,EAAE,MAAM,CAAC,UAAU;YACrB,OAAO,EAAE,MAAM,CAAC,OAAO;SACxB;QACD,SAAS,EAAE;YACT,EAAE,EAAE,MAAM,CAAC,cAAc;YACzB,EAAE,EAAE,MAAM,CAAC,eAAe;YAC1B,EAAE,EAAE,MAAM,CAAC,cAAc;YACzB,EAAE,EAAE,MAAM,CAAC,eAAe;SAC3B;QACD,UAAU,EAAE;YACV,GAAG,EAAE,MAAM,CAAC,cAAc;YAC1B,GAAG,EAAE,MAAM,CAAC,cAAc;YAC1B,GAAG,EAAE,MAAM,CAAC,cAAc;YAC1B,IAAI,EAAE,MAAM,CAAC,cAAc;YAC3B,GAAG,EAAE,MAAM,CAAC,cAAc;SAC3B;QACD,SAAS,EAAE;YACT,MAAM,EAAE;gBACN,CAAC,EAAE,MAAM,CAAC,OAAO;gBACjB,CAAC,EAAE,MAAM,CAAC,OAAO;gBACjB,CAAC,EAAE,MAAM,CAAC,OAAO;aAClB;YACD,eAAe,EAAE,MAAM,CAAC,eAAe;YACvC,YAAY,EAAE,MAAM,CAAC,YAAY;YACjC,uBAAuB,EAAE,MAAM,CAAC,uBAAuB;YACvD,iBAAiB,EAAE,MAAM,CAAC,iBAAiB;SAC5C;QACD,KAAK,EACH,8GAA8G;KACjH,EACD,EAAE,KAAK,EAAE,IAAI,EAAE,CAChB,CAAC;AACJ,CAAC;AAED,8EAA8E;AAC9E,aAAa;AACb,8EAA8E;AAE9E,SAAS,aAAa,CAAC,CAAS;IAC9B,OAAO,GAAG,CAAC,CAAC,GAAG,GAAG,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,GAAG,CAAC;AACpC,CAAC;AAED,MAAM,UAAU,WAAW,CAAC,MAA4B;IACtD,OAAO,CAAC,GAAG,CAAC,wDAAwD,CAAC,CAAC;IACtE,OAAO,CAAC,GAAG,CAAC,qDAAqD,CAAC,CAAC;IACnE,OAAO,CAAC,GAAG,CAAC,wDAAwD,CAAC,CAAC;IAEtE,OAAO,CAAC,GAAG,CAAC,WAAW,MAAM,CAAC,WAAW,aAAa,MAAM,CAAC,eAAe,eAAe,MAAM,CAAC,YAAY,UAAU,CAAC,CAAC;IAC1H,OAAO,CAAC,GAAG,CAAC,cAAc,MAAM,CAAC,SAAS,IAAI,CAAC,CAAC;IAEhD,OAAO,CAAC,GAAG,CAAC,gCAAgC,CAAC,CAAC;IAC9C,OAAO,CAAC,GAAG,CAAC,uBAAuB,aAAa,CAAC,MAAM,CAAC,cAAc,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC;IACzF,OAAO,CAAC,GAAG,CAAC,uBAAuB,aAAa,CAAC,MAAM,CAAC,iBAAiB,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC;IAC5F,OAAO,CAAC,GAAG,CAAC,uBAAuB,aAAa,CAAC,MAAM,CAAC,UAAU,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC;IACrF,OAAO,CAAC,GAAG,CAAC,uBAAuB,aAAa,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC;IAClF,OAAO,CAAC,GAAG,CAAC,kCAAkC,CAAC,CAAC;IAEhD,OAAO,CAAC,GAAG,CAAC,mBAAmB,CAAC,CAAC;IACjC,OAAO,CAAC,GAAG,CAAC,2BAA2B,aAAa,CAAC,MAAM,CAAC,OAAO,CAAC,MAAM,CAAC,KAAK,MAAM,CAAC,OAAO,CAAC,QAAQ,IAAI,MAAM,CAAC,OAAO,CAAC,KAAK,GAAG,CAAC,CAAC;IACpI,OAAO,CAAC,GAAG,CAAC,2BAA2B,aAAa,CAAC,MAAM,CAAC,OAAO,CAAC,MAAM,CAAC,KAAK,MAAM,CAAC,OAAO,CAAC,QAAQ,IAAI,MAAM,CAAC,OAAO,CAAC,KAAK,GAAG,CAAC,CAAC;IACpI,OAAO,CAAC,GAAG,CAAC,2BAA2B,aAAa,CAAC,MAAM,CAAC,OAAO,CAAC,MAAM,CAAC,KAAK,MAAM,CAAC,OAAO,CAAC,QAAQ,IAAI,MAAM,CAAC,OAAO,CAAC,KAAK,KAAK,CAAC,CAAC;IAEtI,OAAO,CAAC,GAAG,CAAC,iCAAiC,CAAC,CAAC;IAC/C,OAAO,CAAC,GAAG,CAAC,6BAA6B,aAAa,CAAC,MAAM,CAAC,uBAAuB,CAAC,EAAE,CAAC,CAAC;IAC1F,OAAO,CAAC,GAAG,CAAC,6BAA6B,aAAa,CAAC,MAAM,CAAC,iBAAiB,CAAC,IAAI,CAAC,CAAC;IAEtF,OAAO,CAAC,GAAG,CAAC,mBAAmB,CAAC,CAAC;IACjC,OAAO,CAAC,GAAG,CAAC,SAAS,MAAM,CAAC,cAAc,SAAS,MAAM,CAAC,eAAe,EAAE,CAAC,CAAC;IAC7E,OAAO,CAAC,GAAG,CAAC,SAAS,MAAM,CAAC,eAAe,SAAS,MAAM,CAAC,cAAc,IAAI,CAAC,CAAC;IAE/E,OAAO,CAAC,GAAG,CAAC,gBAAgB,MAAM,CAAC,cAAc,WAAW,MAAM,CAAC,cAAc,MAAM,CAAC,CAAC;IAEzF,IAAI,MAAM,CAAC,cAAc,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QACrC,OAAO,CAAC,GAAG,CAAC,iBAAiB,CAAC,CAAC;QAC/B,KAAK,MAAM,CAAC,IAAI,MAAM,CAAC,cAAc,EAAE,CAAC;YACtC,OAAO,CAAC,GAAG,CAAC,QAAQ,CAAC,CAAC,KAAK,KAAK,CAAC,CAAC,IAAI,KAAK,CAAC,CAAC,WAAW,GAAG,CAAC,CAAC;QAC/D,CAAC;QACD,OAAO,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC;IAClB,CAAC;IAED,IAAI,MAAM,CAAC,YAAY,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QACnC,OAAO,CAAC,GAAG,CAAC,kBAAkB,CAAC,CAAC;QAChC,KAAK,MAAM,CAAC,IAAI,MAAM,CAAC,YAAY,EAAE,CAAC;YACpC,OAAO,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,IAAI,MAAM,CAAC,CAAC,WAAW,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;QAC7D,CAAC;QACD,OAAO,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC;IAClB,CAAC;AACH,CAAC;AAED,8EAA8E;AAC9E,mBAAmB;AACnB,8EAA8E;AAE9E,IAAI,MAAM,CAAC,IAAI,CAAC,GAAG,KAAK,UAAU,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,IAAI,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,QAAQ,CAAC,oBAAoB,CAAC,EAAE,CAAC;IACvG,iBAAiB,EAAE,CAAC,IAAI,CAAC,CAAC,MAAM,EAAE,EAAE;QAClC,WAAW,CAAC,MAAM,CAAC,CAAC;QACpB,OAAO,CAAC,GAAG,CAAC,6DAA6D,CAAC,CAAC;QAC3E,qEAAqE;QACrE,8DAA8D;QAC9D,8BAA8B,CAAC,MAAM,CAAC,CAAC;QACvC,OAAO,CAAC,GAAG,CAAC,iDAAiD,CAAC,CAAC;IACjE,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,GAAG,EAAE,EAAE;QACf,OAAO,CAAC,KAAK,CAAC,mBAAmB,EAAE,GAAG,CAAC,CAAC;QACxC,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;IAClB,CAAC,CAAC,CAAC;AACL,CAAC"}
|
|
1
|
+
{"version":3,"file":"skill-benchmark.js","sourceRoot":"","sources":["../../src/eval/skill-benchmark.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;GAgBG;AAEH,OAAO,EAAE,OAAO,EAAE,IAAI,EAAE,MAAM,WAAW,CAAC;AAC1C,OAAO,EAAE,YAAY,EAAE,aAAa,EAAE,UAAU,EAAE,MAAM,SAAS,CAAC;AAClE,OAAO,EAAE,SAAS,EAAE,MAAM,cAAc,CAAC;AACzC,OAAO,EAAE,iBAAiB,EAAE,WAAW,EAAE,MAAM,gCAAgC,CAAC;AAChF,OAAO,EAAE,0BAA0B,EAAE,MAAM,0BAA0B,CAAC;AACtE,OAAO,EAAE,gBAAgB,EAAyB,MAAM,yBAAyB,CAAC;AA6ElF,8EAA8E;AAC9E,mBAAmB;AACnB,8EAA8E;AAE9E,MAAM,CAAC,KAAK,UAAU,iBAAiB,CAAC,OAWvC;IACC,MAAM,QAAQ,GAAG,OAAO,CAAC,MAAM,CAAC,IAAI,CAAC,OAAO,EAAE,IAAI,EAAE,IAAI,CAAC,CAAC;IAC1D,MAAM,QAAQ,GAAG,OAAO,EAAE,QAAQ,IAAI,IAAI,CAAC,QAAQ,EAAE,OAAO,CAAC,CAAC;IAC9D,MAAM,SAAS,GAAG,OAAO,EAAE,SAAS,IAAI,IAAI,CAAC,QAAQ,EAAE,MAAM,EAAE,iBAAiB,CAAC,CAAC;IAClF,MAAM,UAAU,GACd,OAAO,EAAE,UAAU,KAAK,IAAI;QAC1B,CAAC,CAAC,IAAI;QACN,CAAC,CAAC,OAAO,EAAE,UAAU,IAAI,IAAI,CAAC,QAAQ,EAAE,MAAM,EAAE,iBAAiB,EAAE,uBAAuB,CAAC,CAAC;IAEhG,gBAAgB;IAChB,MAAM,YAAY,GAAG,IAAI,CAAC,SAAS,EAAE,eAAe,CAAC,CAAC;IACtD,IAAI,CAAC,UAAU,CAAC,YAAY,CAAC,EAAE,CAAC;QAC9B,MAAM,IAAI,KAAK,CAAC,uBAAuB,YAAY,EAAE,CAAC,CAAC;IACzD,CAAC;IACD,MAAM,QAAQ,GAA6B,IAAI,CAAC,KAAK,CAAC,YAAY,CAAC,YAAY,EAAE,OAAO,CAAC,CAAC,CAAC;IAE3F,oDAAoD;IACpD,MAAM,YAAY,GAAG,IAAI,CAAC,SAAS,EAAE,wBAAwB,CAAC,CAAC;IAC/D,MAAM,WAAW,GAAkC,UAAU,CAAC,YAAY,CAAC;QACzE,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,YAAY,CAAC,YAAY,EAAE,OAAO,CAAC,CAAC;QACjD,CAAC,CAAC,EAAE,CAAC;IAEP,0EAA0E;IAC1E,6EAA6E;IAC7E,yEAAyE;IACzE,MAAM,MAAM,GAAG,IAAI,SAAS,CAAC,EAAE,QAAQ,EAAE,IAAI,EAAE,OAAO,EAAE,IAAI,EAAE,CAAC,CAAC;IAChE,MAAM,SAAS,GAAG,MAAM,MAAM,CAAC,SAAS,EAAE,CAAC;IAC3C,MAAM,IAAI,GAAG,MAAM,CAAC,OAAO,EAAE,CAAC;IAE9B,kBAAkB;IAClB,MAAM,OAAO,GAAmB,EAAE,CAAC;IAEnC,KAAK,MAAM,KAAK,IAAI,QAAQ,EAAE,CAAC;QAC7B,MAAM,QAAQ,GAAG,IAAI,CAAC,SAAS,EAAE,KAAK,CAAC,IAAI,CAAC,CAAC;QAC7C,IAAI,CAAC,UAAU,CAAC,QAAQ,CAAC,EAAE,CAAC;YAC1B,OAAO,CAAC,KAAK,CAAC,yBAAyB,KAAK,CAAC,IAAI,EAAE,CAAC,CAAC;YACrD,SAAS;QACX,CAAC;QAED,MAAM,OAAO,GAAG,YAAY,CAAC,QAAQ,EAAE,OAAO,CAAC,CAAC;QAChD,MAAM,KAAK,GAAG,WAAW,CAAC,GAAG,EAAE,CAAC;QAChC,MAAM,OAAO,GAAG,MAAM,CAAC,SAAS,CAAC,OAAO,CAAC,CAAC;QAC1C,MAAM,OAAO,GAAG,WAAW,CAAC,GAAG,EAAE,GAAG,KAAK,CAAC;QAE1C,MAAM,QAAQ,GAAG,OAAO,CAAC,MAAM,GAAG,CAAC,CAAC;QACpC,MAAM,UAAU,GAAG,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;QAEjD,MAAM,OAAO,GACX,CAAC,KAAK,CAAC,KAAK,KAAK,WAAW,IAAI,QAAQ,CAAC;YACzC,CAAC,KAAK,CAAC,KAAK,KAAK,QAAQ,IAAI,CAAC,QAAQ,CAAC,CAAC;QAE1C,oEAAoE;QACpE,MAAM,QAAQ,GAAG,WAAW,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;QACzC,MAAM,oBAAoB,GAAG,QAAQ,EAAE,iBAAiB;YACtD,CAAC,CAAC,QAAQ,CAAC,iBAAiB,CAAC,KAAK,CAAC,CAAC,EAAE,EAAE,EAAE,CAAC,UAAU,CAAC,QAAQ,CAAC,EAAE,CAAC,OAAO,CAAC,CAAC;YAC3E,CAAC,CAAC,IAAI,CAAC,CAAC,gCAAgC;QAC1C,MAAM,kBAAkB,GAAG,QAAQ,EAAE,iBAAiB,EAAE,GAAG,CAAC,CAAC,EAAE,EAAE,EAAE,CAAC,EAAE,CAAC,QAAQ,CAAC,IAAI,EAAE,CAAC;QACvF,MAAM,gBAAgB,GAAG,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;QAClE,MAAM,eAAe,GAAG,kBAAkB,CAAC,KAAK,CAAC,CAAC,EAAE,EAAE,EAAE,CACtD,gBAAgB,CAAC,QAAQ,CAAC,EAAqC,CAAC,CACjE,CAAC;QAEF,OAAO,CAAC,IAAI,CAAC;YACX,IAAI,EAAE,KAAK,CAAC,IAAI;YAChB,KAAK,EAAE,KAAK,CAAC,KAAK;YAClB,KAAK,EAAE,KAAK,CAAC,KAAK,IAAI,EAAE;YACxB,WAAW,EAAE,KAAK,CAAC,WAAW;YAC9B,QAAQ;YACR,WAAW,EAAE,UAAU;YACvB,OAAO;YACP,UAAU,EAAE,IAAI,CAAC,KAAK,CAAC,OAAO,GAAG,GAAG,CAAC,GAAG,GAAG;YAC3C,sBAAsB,EAAE,oBAAoB;YAC5C,gBAAgB,EAAE,eAAe;SAClC,CAAC,CAAC;IACL,CAAC;IAED,kBAAkB;IAClB,MAAM,SAAS,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,KAAK,KAAK,WAAW,CAAC,CAAC;IACjE,MAAM,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,KAAK,KAAK,QAAQ,CAAC,CAAC;IAE3D,MAAM,EAAE,GAAG,SAAS,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC,MAAM,CAAC;IACtD,MAAM,EAAE,GAAG,SAAS,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC,MAAM,CAAC;IACvD,MAAM,EAAE,GAAG,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC,MAAM,CAAC;IACnD,MAAM,EAAE,GAAG,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC,MAAM,CAAC;IAEpD,MAAM,MAAM,GAAG,SAAS,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,EAAE,GAAG,SAAS,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,CAAC;IAChE,MAAM,SAAS,GAAG,CAAC,EAAE,GAAG,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,EAAE,GAAG,CAAC,EAAE,GAAG,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;IACrD,MAAM,EAAE,GAAG,CAAC,SAAS,GAAG,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,SAAS,GAAG,MAAM,CAAC,GAAG,CAAC,SAAS,GAAG,MAAM,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;IAC1F,MAAM,MAAM,GAAG,MAAM,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,EAAE,GAAG,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,CAAC;IAE1D,oBAAoB;IACpB,MAAM,YAAY,GAAG,CAAC,KAAa,EAAgB,EAAE;QACnD,MAAM,OAAO,GAAG,SAAS,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,KAAK,KAAK,KAAK,CAAC,CAAC;QAC3D,MAAM,QAAQ,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC,MAAM,CAAC;QAC1D,OAAO;YACL,KAAK,EAAE,OAAO,CAAC,MAAM;YACrB,QAAQ;YACR,MAAM,EAAE,OAAO,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,QAAQ,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC;SAC3D,CAAC;IACJ,CAAC,CAAC;IAEF,MAAM,SAAS,GAAG,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,UAAU,CAAC,CAAC;IAEnD,6BAA6B;IAC7B,MAAM,iBAAiB,GAAG,SAAS,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC;IAC9D,MAAM,qBAAqB,GAAG,iBAAiB,CAAC,MAAM,GAAG,CAAC;QACxD,CAAC,CAAC,iBAAiB,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,sBAAsB,CAAC,CAAC,MAAM,GAAG,iBAAiB,CAAC,MAAM;QAC7F,CAAC,CAAC,CAAC,CAAC;IACN,MAAM,gBAAgB,GAAG,iBAAiB,CAAC,MAAM,GAAG,CAAC;QACnD,CAAC,CAAC,iBAAiB,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,gBAAgB,CAAC,CAAC,MAAM,GAAG,iBAAiB,CAAC,MAAM;QACvF,CAAC,CAAC,CAAC,CAAC;IAEN,MAAM,MAAM,GAAyB;QACnC,SAAS,EAAE,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE;QACnC,WAAW,EAAE,OAAO,CAAC,MAAM;QAC3B,UAAU,EAAE,SAAS;QACrB,IAAI;QACJ,aAAa,EAAE,WAAW,CAAC,MAAM,CAAC,QAAQ,EAAE,EAAE,IAAI,CAAC;QACnD,eAAe,EAAE,SAAS,CAAC,MAAM;QACjC,YAAY,EAAE,MAAM,CAAC,MAAM;QAC3B,cAAc,EAAE,IAAI,CAAC,KAAK,CAAC,MAAM,GAAG,IAAI,CAAC,GAAG,IAAI;QAChD,iBAAiB,EAAE,IAAI,CAAC,KAAK,CAAC,SAAS,GAAG,IAAI,CAAC,GAAG,IAAI;QACtD,UAAU,EAAE,IAAI,CAAC,KAAK,CAAC,EAAE,GAAG,IAAI,CAAC,GAAG,IAAI;QACxC,OAAO,EAAE,IAAI,CAAC,KAAK,CAAC,MAAM,GAAG,IAAI,CAAC,GAAG,IAAI;QACzC,OAAO,EAAE,YAAY,CAAC,GAAG,CAAC;QAC1B,OAAO,EAAE,YAAY,CAAC,GAAG,CAAC;QAC1B,OAAO,EAAE,YAAY,CAAC,GAAG,CAAC;QAC1B,cAAc,EAAE,EAAE;QAClB,eAAe,EAAE,EAAE;QACnB,cAAc,EAAE,EAAE;QAClB,eAAe,EAAE,EAAE;QACnB,uBAAuB,EAAE,IAAI,CAAC,KAAK,CAAC,qBAAqB,GAAG,IAAI,CAAC,GAAG,IAAI;QACxE,iBAAiB,EAAE,IAAI,CAAC,KAAK,CAAC,gBAAgB,GAAG,IAAI,CAAC,GAAG,IAAI;QAC7D,cAAc,EAAE,IAAI,CAAC,KAAK,CAAC,CAAC,SAAS,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC,GAAG,SAAS,CAAC,MAAM,CAAC,GAAG,GAAG,CAAC,GAAG,GAAG;QACjG,cAAc,EAAE,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,GAAG,CAAC,GAAG,SAAS,CAAC,GAAG,GAAG,CAAC,GAAG,GAAG;QAC9D,OAAO;QACP,cAAc,EAAE,SAAS,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,QAAQ,CAAC;QACpD,YAAY,EAAE,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,QAAQ,CAAC;KAC/C,CAAC;IAEF,cAAc;IACd,IAAI,UAAU,KAAK,IAAI;QAAE,aAAa,CAAC,UAAU,EAAE,IAAI,CAAC,SAAS,CAAC,MAAM,EAAE,IAAI,EAAE,CAAC,CAAC,CAAC,CAAC;IAEpF,OAAO,MAAM,CAAC;AAChB,CAAC;AAED,8EAA8E;AAC9E,8EAA8E;AAC9E,8EAA8E;AAE9E;;;;;GAKG;AACH,MAAM,UAAU,8BAA8B,CAC5C,MAA4B,EAC5B,OAA6B;IAE7B,IAAI,OAAO,CAAC,IAAI,KAAK,SAAS,EAAE,CAAC;QAC/B,MAAM,IAAI,KAAK,CAAC,iDAAiD,OAAO,CAAC,IAAI,oBAAoB,CAAC,CAAC;IACrG,CAAC;IACD,IAAI,OAAO,CAAC,WAAW,KAAK,MAAM,CAAC,WAAW,IAAI,OAAO,CAAC,eAAe,KAAK,MAAM,CAAC,eAAe,EAAE,CAAC;QACrG,MAAM,IAAI,KAAK,CAAC,mFAAmF,CAAC,CAAC;IACvG,CAAC;IACD,OAAO;QACL,MAAM,EAAE,iBAAiB;QACzB,cAAc,EAAE,cAAc;QAC9B,WAAW,EAAE,MAAM,CAAC,SAAS;QAC7B,OAAO,EAAE,MAAM,CAAC,WAAW;QAC3B,OAAO,EAAE;YACP,MAAM,EAAE,MAAM,CAAC,cAAc;YAC7B,SAAS,EAAE,MAAM,CAAC,iBAAiB;YACnC,EAAE,EAAE,MAAM,CAAC,UAAU;YACrB,OAAO,EAAE,MAAM,CAAC,OAAO;SACxB;QACD,SAAS,EAAE;YACT,EAAE,EAAE,MAAM,CAAC,cAAc;YACzB,EAAE,EAAE,MAAM,CAAC,eAAe;YAC1B,EAAE,EAAE,MAAM,CAAC,cAAc;YACzB,EAAE,EAAE,MAAM,CAAC,eAAe;SAC3B;QACD,UAAU,EAAE;YACV,GAAG,EAAE,MAAM,CAAC,cAAc;YAC1B,GAAG,EAAE,MAAM,CAAC,cAAc;YAC1B,GAAG,EAAE,MAAM,CAAC,cAAc;YAC1B,IAAI,EAAE,MAAM,CAAC,cAAc;YAC3B,GAAG,EAAE,MAAM,CAAC,cAAc;SAC3B;QACD,SAAS,EAAE;YACT,MAAM,EAAE;gBACN,CAAC,EAAE,MAAM,CAAC,OAAO;gBACjB,CAAC,EAAE,MAAM,CAAC,OAAO;gBACjB,CAAC,EAAE,MAAM,CAAC,OAAO;aAClB;YACD,eAAe,EAAE,MAAM,CAAC,eAAe;YACvC,YAAY,EAAE,MAAM,CAAC,YAAY;YACjC,uBAAuB,EAAE,MAAM,CAAC,uBAAuB;YACvD,iBAAiB,EAAE,MAAM,CAAC,iBAAiB;YAC3C,CAAC,0BAA0B,CAAC,EAAE,iBAAiB,CAAC;gBAC9C,QAAQ,EAAE,OAAO,CAAC,cAAc;gBAChC,OAAO,EAAE,OAAO,CAAC,eAAe;gBAChC,WAAW,EAAE,OAAO,CAAC,aAAa;aACnC,CAAC;SACH;QACD,KAAK,EACH,8GAA8G;KACjH,CAAC;AACJ,CAAC;AAED;;;;;;;;;GASG;AACH,MAAM,UAAU,8BAA8B,CAAC,MAA4B,EAAE,OAA6B;IACxG,gBAAgB,CAAC,8BAA8B,CAAC,MAAM,EAAE,OAAO,CAAC,EAAE,EAAE,KAAK,EAAE,IAAI,EAAE,CAAC,CAAC;AACrF,CAAC;AAED,8EAA8E;AAC9E,aAAa;AACb,8EAA8E;AAE9E,SAAS,aAAa,CAAC,CAAS;IAC9B,OAAO,GAAG,CAAC,CAAC,GAAG,GAAG,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,GAAG,CAAC;AACpC,CAAC;AAED,MAAM,UAAU,WAAW,CAAC,MAA4B;IACtD,OAAO,CAAC,GAAG,CAAC,wDAAwD,CAAC,CAAC;IACtE,OAAO,CAAC,GAAG,CAAC,qDAAqD,CAAC,CAAC;IACnE,OAAO,CAAC,GAAG,CAAC,wDAAwD,CAAC,CAAC;IAEtE,OAAO,CAAC,GAAG,CAAC,WAAW,MAAM,CAAC,WAAW,aAAa,MAAM,CAAC,eAAe,eAAe,MAAM,CAAC,YAAY,UAAU,CAAC,CAAC;IAC1H,OAAO,CAAC,GAAG,CAAC,cAAc,MAAM,CAAC,SAAS,IAAI,CAAC,CAAC;IAEhD,OAAO,CAAC,GAAG,CAAC,gCAAgC,CAAC,CAAC;IAC9C,OAAO,CAAC,GAAG,CAAC,uBAAuB,aAAa,CAAC,MAAM,CAAC,cAAc,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC;IACzF,OAAO,CAAC,GAAG,CAAC,uBAAuB,aAAa,CAAC,MAAM,CAAC,iBAAiB,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC;IAC5F,OAAO,CAAC,GAAG,CAAC,uBAAuB,aAAa,CAAC,MAAM,CAAC,UAAU,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC;IACrF,OAAO,CAAC,GAAG,CAAC,uBAAuB,aAAa,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC;IAClF,OAAO,CAAC,GAAG,CAAC,kCAAkC,CAAC,CAAC;IAEhD,OAAO,CAAC,GAAG,CAAC,mBAAmB,CAAC,CAAC;IACjC,OAAO,CAAC,GAAG,CAAC,2BAA2B,aAAa,CAAC,MAAM,CAAC,OAAO,CAAC,MAAM,CAAC,KAAK,MAAM,CAAC,OAAO,CAAC,QAAQ,IAAI,MAAM,CAAC,OAAO,CAAC,KAAK,GAAG,CAAC,CAAC;IACpI,OAAO,CAAC,GAAG,CAAC,2BAA2B,aAAa,CAAC,MAAM,CAAC,OAAO,CAAC,MAAM,CAAC,KAAK,MAAM,CAAC,OAAO,CAAC,QAAQ,IAAI,MAAM,CAAC,OAAO,CAAC,KAAK,GAAG,CAAC,CAAC;IACpI,OAAO,CAAC,GAAG,CAAC,2BAA2B,aAAa,CAAC,MAAM,CAAC,OAAO,CAAC,MAAM,CAAC,KAAK,MAAM,CAAC,OAAO,CAAC,QAAQ,IAAI,MAAM,CAAC,OAAO,CAAC,KAAK,KAAK,CAAC,CAAC;IAEtI,OAAO,CAAC,GAAG,CAAC,iCAAiC,CAAC,CAAC;IAC/C,OAAO,CAAC,GAAG,CAAC,6BAA6B,aAAa,CAAC,MAAM,CAAC,uBAAuB,CAAC,EAAE,CAAC,CAAC;IAC1F,OAAO,CAAC,GAAG,CAAC,6BAA6B,aAAa,CAAC,MAAM,CAAC,iBAAiB,CAAC,IAAI,CAAC,CAAC;IAEtF,OAAO,CAAC,GAAG,CAAC,mBAAmB,CAAC,CAAC;IACjC,OAAO,CAAC,GAAG,CAAC,SAAS,MAAM,CAAC,cAAc,SAAS,MAAM,CAAC,eAAe,EAAE,CAAC,CAAC;IAC7E,OAAO,CAAC,GAAG,CAAC,SAAS,MAAM,CAAC,eAAe,SAAS,MAAM,CAAC,cAAc,IAAI,CAAC,CAAC;IAE/E,OAAO,CAAC,GAAG,CAAC,gBAAgB,MAAM,CAAC,cAAc,WAAW,MAAM,CAAC,cAAc,MAAM,CAAC,CAAC;IAEzF,IAAI,MAAM,CAAC,cAAc,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QACrC,OAAO,CAAC,GAAG,CAAC,iBAAiB,CAAC,CAAC;QAC/B,KAAK,MAAM,CAAC,IAAI,MAAM,CAAC,cAAc,EAAE,CAAC;YACtC,OAAO,CAAC,GAAG,CAAC,QAAQ,CAAC,CAAC,KAAK,KAAK,CAAC,CAAC,IAAI,KAAK,CAAC,CAAC,WAAW,GAAG,CAAC,CAAC;QAC/D,CAAC;QACD,OAAO,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC;IAClB,CAAC;IAED,IAAI,MAAM,CAAC,YAAY,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QACnC,OAAO,CAAC,GAAG,CAAC,kBAAkB,CAAC,CAAC;QAChC,KAAK,MAAM,CAAC,IAAI,MAAM,CAAC,YAAY,EAAE,CAAC;YACpC,OAAO,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,IAAI,MAAM,CAAC,CAAC,WAAW,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;QAC7D,CAAC;QACD,OAAO,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC;IAClB,CAAC;AACH,CAAC;AAED,8EAA8E;AAC9E,mBAAmB;AACnB,8EAA8E;AAE9E,IAAI,MAAM,CAAC,IAAI,CAAC,GAAG,KAAK,UAAU,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,IAAI,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,QAAQ,CAAC,oBAAoB,CAAC,EAAE,CAAC;IACvG,iBAAiB,EAAE,CAAC,IAAI,CAAC,KAAK,EAAE,MAAM,EAAE,EAAE;QACxC,WAAW,CAAC,MAAM,CAAC,CAAC;QACpB,OAAO,CAAC,GAAG,CAAC,6DAA6D,CAAC,CAAC;QAC3E,2EAA2E;QAC3E,qBAAqB;QACrB,MAAM,OAAO,GAAG,MAAM,iBAAiB,CAAC,EAAE,IAAI,EAAE,SAAS,EAAE,UAAU,EAAE,IAAI,EAAE,CAAC,CAAC;QAC/E,OAAO,CAAC,GAAG,CACT,iBAAiB,OAAO,CAAC,cAAc,IAAI,OAAO,CAAC,eAAe,eAAe;YAC/E,GAAG,aAAa,CAAC,OAAO,CAAC,cAAc,CAAC,KAAK,OAAO,CAAC,aAAa,iBAAiB,CACtF,CAAC;QACF,qEAAqE;QACrE,8DAA8D;QAC9D,8BAA8B,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;QAChD,OAAO,CAAC,GAAG,CAAC,iDAAiD,CAAC,CAAC;IACjE,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,GAAG,EAAE,EAAE;QACf,OAAO,CAAC,KAAK,CAAC,mBAAmB,EAAE,GAAG,CAAC,CAAC;QACxC,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;IAClB,CAAC,CAAC,CAAC;AACL,CAAC"}
|
package/dist/flywheel.d.ts
CHANGED
|
@@ -18,6 +18,8 @@ export interface FlywheelConfig {
|
|
|
18
18
|
readonly maxFPRate?: number;
|
|
19
19
|
/** Minimum shadow evaluations before promotion (default: 1000) */
|
|
20
20
|
readonly minEvaluations?: number;
|
|
21
|
+
/** Minimum matches with recorded feedback before promotion (default: 1; unreviewed matches never count) */
|
|
22
|
+
readonly minReviewedMatches?: number;
|
|
21
23
|
/** Callback when a rule is auto-promoted */
|
|
22
24
|
readonly onPromote?: (rule: ATRRule, stats: PromotionCandidate['stats']) => void | Promise<void>;
|
|
23
25
|
/** Callback when a new shadow rule is generated */
|
|
@@ -28,6 +30,8 @@ export declare class FlywheelManager {
|
|
|
28
30
|
private readonly shadow;
|
|
29
31
|
private readonly config;
|
|
30
32
|
private readonly existingIds;
|
|
33
|
+
/** Rules already handed to onPromote, so a periodic promoteReady() does not promote them again. */
|
|
34
|
+
private readonly promotedIds;
|
|
31
35
|
constructor(config?: FlywheelConfig);
|
|
32
36
|
/**
|
|
33
37
|
* Called when Tier 4 (LLM semantic) detects a novel threat.
|
package/dist/flywheel.d.ts.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"flywheel.d.ts","sourceRoot":"","sources":["../src/flywheel.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;GAYG;AAEH,OAAO,KAAK,EAAE,OAAO,EAAE,QAAQ,EAAE,UAAU,EAAE,MAAM,YAAY,CAAC;AAEhE,OAAO,EAAmB,KAAK,kBAAkB,EAAE,MAAM,uBAAuB,CAAC;AAEjF,MAAM,WAAW,cAAc;IAC7B,6DAA6D;IAC7D,QAAQ,CAAC,SAAS,CAAC,EAAE,MAAM,CAAC;IAC5B,kEAAkE;IAClE,QAAQ,CAAC,cAAc,CAAC,EAAE,MAAM,CAAC;IACjC,4CAA4C;IAC5C,QAAQ,CAAC,SAAS,CAAC,EAAE,CAAC,IAAI,EAAE,OAAO,EAAE,KAAK,EAAE,kBAAkB,CAAC,OAAO,CAAC,KAAK,IAAI,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC;IACjG,mDAAmD;IACnD,QAAQ,CAAC,YAAY,CAAC,EAAE,CAAC,IAAI,EAAE,OAAO,KAAK,IAAI,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC;CACjE;AAED,qBAAa,eAAe;IAC1B,OAAO,CAAC,QAAQ,CAAC,UAAU,CAAiB;IAC5C,OAAO,CAAC,QAAQ,CAAC,MAAM,CAAkB;IACzC,OAAO,CAAC,QAAQ,CAAC,MAAM,CAA2B;IAClD,OAAO,CAAC,QAAQ,CAAC,WAAW,CAAqB;gBAErC,MAAM,GAAE,cAAmB;
|
|
1
|
+
{"version":3,"file":"flywheel.d.ts","sourceRoot":"","sources":["../src/flywheel.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;GAYG;AAEH,OAAO,KAAK,EAAE,OAAO,EAAE,QAAQ,EAAE,UAAU,EAAE,MAAM,YAAY,CAAC;AAEhE,OAAO,EAAmB,KAAK,kBAAkB,EAAE,MAAM,uBAAuB,CAAC;AAEjF,MAAM,WAAW,cAAc;IAC7B,6DAA6D;IAC7D,QAAQ,CAAC,SAAS,CAAC,EAAE,MAAM,CAAC;IAC5B,kEAAkE;IAClE,QAAQ,CAAC,cAAc,CAAC,EAAE,MAAM,CAAC;IACjC,2GAA2G;IAC3G,QAAQ,CAAC,kBAAkB,CAAC,EAAE,MAAM,CAAC;IACrC,4CAA4C;IAC5C,QAAQ,CAAC,SAAS,CAAC,EAAE,CAAC,IAAI,EAAE,OAAO,EAAE,KAAK,EAAE,kBAAkB,CAAC,OAAO,CAAC,KAAK,IAAI,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC;IACjG,mDAAmD;IACnD,QAAQ,CAAC,YAAY,CAAC,EAAE,CAAC,IAAI,EAAE,OAAO,KAAK,IAAI,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC;CACjE;AAED,qBAAa,eAAe;IAC1B,OAAO,CAAC,QAAQ,CAAC,UAAU,CAAiB;IAC5C,OAAO,CAAC,QAAQ,CAAC,MAAM,CAAkB;IACzC,OAAO,CAAC,QAAQ,CAAC,MAAM,CAA2B;IAClD,OAAO,CAAC,QAAQ,CAAC,WAAW,CAAqB;IACjD,mGAAmG;IACnG,OAAO,CAAC,QAAQ,CAAC,WAAW,CAAqB;gBAErC,MAAM,GAAE,cAAmB;IAYvC;;;OAGG;IACG,gBAAgB,CAAC,KAAK,EAAE,QAAQ,EAAE,KAAK,EAAE,UAAU,GAAG,OAAO,CAAC,OAAO,GAAG,IAAI,CAAC;IAgEnF;;;OAGG;IACH,cAAc,CAAC,KAAK,EAAE,UAAU,GAAG,SAAS,QAAQ,EAAE;IAItD,6CAA6C;IAC7C,cAAc,CAAC,MAAM,EAAE,MAAM,EAAE,cAAc,EAAE,OAAO,GAAG,IAAI;IAI7D;;;OAGG;IACG,YAAY,IAAI,OAAO,CAAC,SAAS,kBAAkB,EAAE,CAAC;IAiB5D,iCAAiC;IACjC,cAAc,IAAI,WAAW,CAAC,MAAM,EAAE,OAAO,CAAC;IAI9C,qCAAqC;IACrC,eAAe,IAAI,MAAM;CAG1B"}
|
package/dist/flywheel.js
CHANGED
|
@@ -18,12 +18,15 @@ export class FlywheelManager {
|
|
|
18
18
|
shadow;
|
|
19
19
|
config;
|
|
20
20
|
existingIds = new Set();
|
|
21
|
+
/** Rules already handed to onPromote, so a periodic promoteReady() does not promote them again. */
|
|
22
|
+
promotedIds = new Set();
|
|
21
23
|
constructor(config = {}) {
|
|
22
24
|
this.scaffolder = new RuleScaffolder({ author: 'ATR Flywheel (auto-generated)' });
|
|
23
25
|
this.shadow = new ShadowEvaluator();
|
|
24
26
|
this.config = {
|
|
25
27
|
maxFPRate: config.maxFPRate ?? 0.001,
|
|
26
28
|
minEvaluations: config.minEvaluations ?? 1000,
|
|
29
|
+
minReviewedMatches: config.minReviewedMatches ?? 1,
|
|
27
30
|
onPromote: config.onPromote ?? (() => { }),
|
|
28
31
|
onShadowRule: config.onShadowRule ?? (() => { }),
|
|
29
32
|
};
|
|
@@ -44,12 +47,12 @@ export class FlywheelManager {
|
|
|
44
47
|
const payloads = [];
|
|
45
48
|
// 1. Matched patterns from the Tier 4 detection — these ARE the attack signals
|
|
46
49
|
if (match.matchedPatterns.length > 0) {
|
|
47
|
-
payloads.push(...match.matchedPatterns.filter((p) => p.length > 5));
|
|
50
|
+
payloads.push(...match.matchedPatterns.filter((p) => p.trim().length > 5));
|
|
48
51
|
}
|
|
49
52
|
// 2. Event fields (tool_args, tool_response, etc.) — more specific than content
|
|
50
53
|
if (event.fields) {
|
|
51
54
|
for (const value of Object.values(event.fields)) {
|
|
52
|
-
if (value && value.length > 10) {
|
|
55
|
+
if (value && value.trim().length > 10) {
|
|
53
56
|
payloads.push(value.slice(0, 500));
|
|
54
57
|
}
|
|
55
58
|
}
|
|
@@ -101,11 +104,16 @@ export class FlywheelManager {
|
|
|
101
104
|
* Call periodically (e.g., every 15 minutes).
|
|
102
105
|
*/
|
|
103
106
|
async promoteReady() {
|
|
104
|
-
const candidates = this.shadow
|
|
107
|
+
const candidates = this.shadow
|
|
108
|
+
.getPromotionCandidates(this.config.maxFPRate, this.config.minEvaluations, this.config.minReviewedMatches)
|
|
109
|
+
.filter((candidate) => !this.promotedIds.has(candidate.rule.id));
|
|
105
110
|
for (const candidate of candidates) {
|
|
106
|
-
// Promote: change status from experimental to stable
|
|
111
|
+
// Promote: change status from experimental to stable. maturity stays
|
|
112
|
+
// 'draft', so ATREngine's lane gate still keeps the rule out of the
|
|
113
|
+
// enforce and alert lanes until it passes canPromote().
|
|
107
114
|
const promoted = { ...candidate.rule, status: 'stable' };
|
|
108
115
|
await this.config.onPromote(promoted, candidate.stats);
|
|
116
|
+
this.promotedIds.add(candidate.rule.id);
|
|
109
117
|
}
|
|
110
118
|
return candidates;
|
|
111
119
|
}
|
package/dist/flywheel.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"flywheel.js","sourceRoot":"","sources":["../src/flywheel.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;GAYG;AAGH,OAAO,EAAE,cAAc,EAAsB,MAAM,sBAAsB,CAAC;AAC1E,OAAO,EAAE,eAAe,EAA2B,MAAM,uBAAuB,CAAC;
|
|
1
|
+
{"version":3,"file":"flywheel.js","sourceRoot":"","sources":["../src/flywheel.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;GAYG;AAGH,OAAO,EAAE,cAAc,EAAsB,MAAM,sBAAsB,CAAC;AAC1E,OAAO,EAAE,eAAe,EAA2B,MAAM,uBAAuB,CAAC;AAejF,MAAM,OAAO,eAAe;IACT,UAAU,CAAiB;IAC3B,MAAM,CAAkB;IACxB,MAAM,CAA2B;IACjC,WAAW,GAAG,IAAI,GAAG,EAAU,CAAC;IACjD,mGAAmG;IAClF,WAAW,GAAG,IAAI,GAAG,EAAU,CAAC;IAEjD,YAAY,SAAyB,EAAE;QACrC,IAAI,CAAC,UAAU,GAAG,IAAI,cAAc,CAAC,EAAE,MAAM,EAAE,+BAA+B,EAAE,CAAC,CAAC;QAClF,IAAI,CAAC,MAAM,GAAG,IAAI,eAAe,EAAE,CAAC;QACpC,IAAI,CAAC,MAAM,GAAG;YACZ,SAAS,EAAE,MAAM,CAAC,SAAS,IAAI,KAAK;YACpC,cAAc,EAAE,MAAM,CAAC,cAAc,IAAI,IAAI;YAC7C,kBAAkB,EAAE,MAAM,CAAC,kBAAkB,IAAI,CAAC;YAClD,SAAS,EAAE,MAAM,CAAC,SAAS,IAAI,CAAC,GAAG,EAAE,GAAE,CAAC,CAAC;YACzC,YAAY,EAAE,MAAM,CAAC,YAAY,IAAI,CAAC,GAAG,EAAE,GAAE,CAAC,CAAC;SAChD,CAAC;IACJ,CAAC;IAED;;;OAGG;IACH,KAAK,CAAC,gBAAgB,CAAC,KAAe,EAAE,KAAiB;QACvD,oDAAoD;QACpD,IAAI,KAAK,CAAC,UAAU,GAAG,GAAG;YAAE,OAAO,IAAI,CAAC;QAExC,+CAA+C;QAC/C,MAAM,QAAQ,GAAG,KAAK,CAAC,IAAI,CAAC,IAAI,EAAE,QAAQ,IAAI,kBAAkB,CAAC;QACjE,MAAM,QAAQ,GAAG,KAAK,CAAC,IAAI,CAAC,QAAQ,IAAI,QAAQ,CAAC;QAEjD,qEAAqE;QACrE,4DAA4D;QAC5D,MAAM,QAAQ,GAAa,EAAE,CAAC;QAE9B,+EAA+E;QAC/E,IAAI,KAAK,CAAC,eAAe,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YACrC,QAAQ,CAAC,IAAI,CAAC,GAAG,KAAK,CAAC,eAAe,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC;QAC7E,CAAC;QAED,gFAAgF;QAChF,IAAI,KAAK,CAAC,MAAM,EAAE,CAAC;YACjB,KAAK,MAAM,KAAK,IAAI,MAAM,CAAC,MAAM,CAAC,KAAK,CAAC,MAAM,CAAC,EAAE,CAAC;gBAChD,IAAI,KAAK,IAAI,KAAK,CAAC,IAAI,EAAE,CAAC,MAAM,GAAG,EAAE,EAAE,CAAC;oBACtC,QAAQ,CAAC,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC,EAAE,GAAG,CAAC,CAAC,CAAC;gBACrC,CAAC;YACH,CAAC;QACH,CAAC;QAED,0EAA0E;QAC1E,IAAI,QAAQ,CAAC,MAAM,KAAK,CAAC,IAAI,KAAK,CAAC,OAAO,EAAE,CAAC;YAC3C,QAAQ,CAAC,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC,EAAE,GAAG,CAAC,CAAC,CAAC;QAC7C,CAAC;QAED,8BAA8B;QAC9B,IAAI,QAAQ,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YAC1B,QAAQ,CAAC,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,WAAW,IAAI,KAAK,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;QAC5D,CAAC;QAED,MAAM,KAAK,GAAkB;YAC3B,KAAK,EAAE,SAAS,KAAK,CAAC,IAAI,CAAC,WAAW,EAAE,KAAK,CAAC,CAAC,EAAE,EAAE,CAAC,IAAI,KAAK,CAAC,IAAI,CAAC,KAAK,EAAE;YAC1E,QAAQ,EAAE,QAAqC;YAC/C,QAAQ,EAAE,QAAqC;YAC/C,iBAAiB,EAAE,KAAK,CAAC,IAAI,CAAC,WAAW,IAAI,KAAK,CAAC,eAAe,CAAC,IAAI,CAAC,IAAI,CAAC;YAC7E,eAAe,EAAE,QAAQ;SAC1B,CAAC;QAEF,IAAI,CAAC;YACH,MAAM,MAAM,GAAG,IAAI,CAAC,UAAU,CAAC,QAAQ,CAAC,KAAK,EAAE,IAAI,CAAC,WAAW,CAAC,CAAC;YACjE,MAAM,QAAQ,GAAG,MAAM,CAAC,IAAI,CAAC;YAE7B,+BAA+B;YAC/B,MAAM,EAAE,OAAO,EAAE,IAAI,EAAE,GAAG,MAAM,MAAM,CAAC,SAAS,CAAC,CAAC;YAClD,MAAM,IAAI,GAAG,IAAI,CAAC,IAAI,CAAC,QAAQ,CAAY,CAAC;YAC5C,IAAI,CAAC,MAAM,GAAG,cAAc,CAAC;YAE7B,IAAI,CAAC,WAAW,CAAC,GAAG,CAAC,MAAM,CAAC,EAAE,CAAC,CAAC;YAChC,IAAI,CAAC,MAAM,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC;YAE1B,MAAM,IAAI,CAAC,MAAM,CAAC,YAAY,CAAC,IAAI,CAAC,CAAC;YAErC,OAAO,IAAI,CAAC;QACd,CAAC;QAAC,MAAM,CAAC;YACP,OAAO,IAAI,CAAC;QACd,CAAC;IACH,CAAC;IAED;;;OAGG;IACH,cAAc,CAAC,KAAiB;QAC9B,OAAO,IAAI,CAAC,MAAM,CAAC,QAAQ,CAAC,KAAK,CAAC,CAAC;IACrC,CAAC;IAED,6CAA6C;IAC7C,cAAc,CAAC,MAAc,EAAE,cAAuB;QACpD,IAAI,CAAC,MAAM,CAAC,cAAc,CAAC,MAAM,EAAE,cAAc,CAAC,CAAC;IACrD,CAAC;IAED;;;OAGG;IACH,KAAK,CAAC,YAAY;QAChB,MAAM,UAAU,GAAG,IAAI,CAAC,MAAM;aAC3B,sBAAsB,CAAC,IAAI,CAAC,MAAM,CAAC,SAAS,EAAE,IAAI,CAAC,MAAM,CAAC,cAAc,EAAE,IAAI,CAAC,MAAM,CAAC,kBAAkB,CAAC;aACzG,MAAM,CAAC,CAAC,SAAS,EAAE,EAAE,CAAC,CAAC,IAAI,CAAC,WAAW,CAAC,GAAG,CAAC,SAAS,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC,CAAC;QAEnE,KAAK,MAAM,SAAS,IAAI,UAAU,EAAE,CAAC;YACnC,qEAAqE;YACrE,oEAAoE;YACpE,wDAAwD;YACxD,MAAM,QAAQ,GAAG,EAAE,GAAG,SAAS,CAAC,IAAI,EAAE,MAAM,EAAE,QAAiB,EAAE,CAAC;YAClE,MAAM,IAAI,CAAC,MAAM,CAAC,SAAS,CAAC,QAAQ,EAAE,SAAS,CAAC,KAAK,CAAC,CAAC;YACvD,IAAI,CAAC,WAAW,CAAC,GAAG,CAAC,SAAS,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;QAC1C,CAAC;QAED,OAAO,UAAU,CAAC;IACpB,CAAC;IAED,iCAAiC;IACjC,cAAc;QACZ,OAAO,IAAI,CAAC,MAAM,CAAC,WAAW,EAAE,CAAC;IACnC,CAAC;IAED,qCAAqC;IACrC,eAAe;QACb,OAAO,IAAI,CAAC,MAAM,CAAC,IAAI,EAAE,CAAC;IAC5B,CAAC;CACF"}
|
package/dist/hook-handler.d.ts
CHANGED
|
@@ -121,8 +121,13 @@ export declare function toClaudeCodePreToolUse(output: HookOutput, options?: Hoo
|
|
|
121
121
|
*
|
|
122
122
|
* ADVISORY MODE (the default): the sentinel is never set, and the findings
|
|
123
123
|
* still travel in atr_decision / atr_reason / matched_rules.
|
|
124
|
+
*
|
|
125
|
+
* PostToolUseFailure takes the same contract under its own name: the 2.1.280
|
|
126
|
+
* binary turns `decision: 'block'` into a blocking error it shows the model for
|
|
127
|
+
* that event as for PostToolUse, and its hookSpecificOutput carries only
|
|
128
|
+
* hookEventName and an optional additionalContext.
|
|
124
129
|
*/
|
|
125
|
-
export declare function toClaudeCodePostToolUse(output: HookOutput, options?: HookContractOptions): Record<string, unknown>;
|
|
130
|
+
export declare function toClaudeCodePostToolUse(output: HookOutput, options?: HookContractOptions, hookEventName?: 'PostToolUse' | 'PostToolUseFailure'): Record<string, unknown>;
|
|
126
131
|
/**
|
|
127
132
|
* The event name, under either spelling.
|
|
128
133
|
*
|
|
@@ -155,6 +160,11 @@ export declare class HookHandler {
|
|
|
155
160
|
* Scans the tool output for threats.
|
|
156
161
|
*/
|
|
157
162
|
handlePostToolUse(input: HookInput): Promise<HookOutput>;
|
|
163
|
+
/**
|
|
164
|
+
* Handle a PostToolUseFailure hook event: a tool call that failed. Scans the
|
|
165
|
+
* call's error, which carries a failed command's output.
|
|
166
|
+
*/
|
|
167
|
+
handlePostToolUseFailure(input: HookInput): Promise<HookOutput>;
|
|
158
168
|
/**
|
|
159
169
|
* Start a stdio JSON-lines loop.
|
|
160
170
|
*
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"hook-handler.d.ts","sourceRoot":"","sources":["../src/hook-handler.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;GA2BG;AAGH,OAAO,KAAK,
|
|
1
|
+
{"version":3,"file":"hook-handler.d.ts","sourceRoot":"","sources":["../src/hook-handler.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;GA2BG;AAGH,OAAO,KAAK,EAGV,SAAS,EACT,UAAU,EAEX,MAAM,YAAY,CAAC;AACpB,OAAO,KAAK,EAAE,SAAS,EAAE,MAAM,aAAa,CAAC;AAC7C,OAAO,KAAK,EAAE,cAAc,EAAE,MAAM,sBAAsB,CAAC;AAO3D,oDAAoD;AACpD,MAAM,WAAW,mBAAmB;IAClC;;;;;OAKG;IACH,QAAQ,CAAC,QAAQ,CAAC,EAAE,OAAO,CAAC;CAC7B;AAED,MAAM,WAAW,iBAAiB;IAChC,QAAQ,CAAC,MAAM,EAAE,SAAS,CAAC;IAC3B,QAAQ,CAAC,QAAQ,EAAE,cAAc,CAAC;IAClC,QAAQ,CAAC,SAAS,CAAC,EAAE,MAAM,CAAC;IAC5B,QAAQ,CAAC,QAAQ,CAAC,EAAE,OAAO,CAAC;IAC5B;;;;;;;;;OASG;IACH,QAAQ,CAAC,QAAQ,CAAC,EAAE,OAAO,CAAC;CAC7B;AAyGD;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA8CG;AACH,wBAAgB,sBAAsB,CACpC,MAAM,EAAE,UAAU,EAClB,OAAO,GAAE,mBAAwB,GAChC,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CA2BzB;AAED;;;;;;;;;;;;;;;;;;;;;;GAsBG;AACH,wBAAgB,uBAAuB,CACrC,MAAM,EAAE,UAAU,EAClB,OAAO,GAAE,mBAAwB,EACjC,aAAa,GAAE,aAAa,GAAG,oBAAoC,GAClE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAWzB;AA8BD;;;;;;;;;;;GAWG;AACH,wBAAgB,WAAW,CAAC,KAAK,EAAE,SAAS,GAAG,MAAM,GAAG,SAAS,CAEhE;AAED,qBAAa,WAAW;IACtB,OAAO,CAAC,QAAQ,CAAC,MAAM,CAAY;IACnC,OAAO,CAAC,QAAQ,CAAC,QAAQ,CAAiB;IAC1C,OAAO,CAAC,QAAQ,CAAC,SAAS,CAAS;IACnC,OAAO,CAAC,QAAQ,CAAC,QAAQ,CAAU;IACnC,OAAO,CAAC,QAAQ,CAAC,QAAQ,CAAU;gBAEvB,MAAM,EAAE,iBAAiB;IAQrC,wEAAwE;IACxE,UAAU,IAAI,OAAO;IAIrB;;;OAGG;IACG,gBAAgB,CAAC,KAAK,EAAE,SAAS,GAAG,OAAO,CAAC,UAAU,CAAC;IAS7D;;;OAGG;IACG,iBAAiB,CAAC,KAAK,EAAE,SAAS,GAAG,OAAO,CAAC,UAAU,CAAC;IAS9D;;;OAGG;IACG,wBAAwB,CAAC,KAAK,EAAE,SAAS,GAAG,OAAO,CAAC,UAAU,CAAC;IAIrE;;;;;;;OAOG;IACG,cAAc,CAAC,MAAM,GAAE,aAAa,GAAG,SAAyB,GAAG,OAAO,CAAC,IAAI,CAAC;IAyCtF;;OAEG;YACW,QAAQ;IAatB;;OAEG;YACW,kBAAkB;IAoBhC;;OAEG;IACH,OAAO,CAAC,WAAW;CAapB"}
|