agent-threat-rules 2.2.1 → 3.0.5
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +365 -327
- package/dist/engine.d.ts +46 -1
- package/dist/engine.d.ts.map +1 -1
- package/dist/engine.js +242 -1
- package/dist/engine.js.map +1 -1
- package/dist/eval/eval-harness.d.ts.map +1 -1
- package/dist/eval/eval-harness.js +9 -0
- package/dist/eval/eval-harness.js.map +1 -1
- package/dist/eval/run-hackaprompt-benchmark.js +9 -0
- package/dist/eval/run-hackaprompt-benchmark.js.map +1 -1
- package/dist/eval/run-pint-benchmark.js +9 -0
- package/dist/eval/run-pint-benchmark.js.map +1 -1
- package/dist/eval/skill-benchmark.d.ts +11 -0
- package/dist/eval/skill-benchmark.d.ts.map +1 -1
- package/dist/eval/skill-benchmark.js +57 -0
- package/dist/eval/skill-benchmark.js.map +1 -1
- package/dist/measurement/from-eval-harness.d.ts +70 -0
- package/dist/measurement/from-eval-harness.d.ts.map +1 -0
- package/dist/measurement/from-eval-harness.js +49 -0
- package/dist/measurement/from-eval-harness.js.map +1 -0
- package/dist/measurement/schema.d.ts +152 -0
- package/dist/measurement/schema.d.ts.map +1 -0
- package/dist/measurement/schema.js +178 -0
- package/dist/measurement/schema.js.map +1 -0
- package/dist/measurement/write.d.ts +64 -0
- package/dist/measurement/write.d.ts.map +1 -0
- package/dist/measurement/write.js +163 -0
- package/dist/measurement/write.js.map +1 -0
- package/dist/semantic-evaluator.d.ts +48 -0
- package/dist/semantic-evaluator.d.ts.map +1 -0
- package/dist/semantic-evaluator.js +107 -0
- package/dist/semantic-evaluator.js.map +1 -0
- package/dist/trace-evaluator.d.ts +22 -0
- package/dist/trace-evaluator.d.ts.map +1 -0
- package/dist/trace-evaluator.js +249 -0
- package/dist/trace-evaluator.js.map +1 -0
- package/dist/types.d.ts +143 -0
- package/dist/types.d.ts.map +1 -1
- package/package.json +5 -3
- package/rules/agent-manipulation/ATR-2026-00552-goal-drift-after-pressure-injection.yaml +216 -0
- package/rules/context-exfiltration/ATR-2026-00524-claude-code-anthropic-base-url-credential-exfil.yaml +257 -0
- package/rules/context-exfiltration/ATR-2026-00548-cross-agent-session-context-leak.yaml +177 -0
- package/rules/excessive-autonomy/ATR-2026-00553-runaway-tool-loop-behavioral.yaml +174 -0
- package/rules/privilege-escalation/ATR-2026-00528-praisonai-auth-disabled-default.yaml +192 -0
- package/rules/privilege-escalation/ATR-2026-00539-crewai-codeinterpreter-sandbox-escape-rce.yaml +292 -0
- package/rules/privilege-escalation/ATR-2026-00546-crewai-json-loader-local-file-read.yaml +162 -0
- package/rules/privilege-escalation/ATR-2026-00547-crewai-rag-url-ssrf-bypass.yaml +167 -0
- package/rules/privilege-escalation/ATR-2026-00549-destructive-tool-without-human-approval.yaml +193 -0
- package/rules/privilege-escalation/ATR-2026-00551-cross-conversation-memory-write.yaml +198 -0
- package/rules/prompt-injection/ATR-2026-00535-windsurf-ide-zero-click-prompt-injection.yaml +199 -0
- package/rules/prompt-injection/ATR-2026-00550-untrusted-retrieval-to-privileged-tool.yaml +199 -0
- package/rules/skill-compromise/ATR-2026-00123-skill-overreach-permissions.yaml +5 -2
- package/rules/skill-compromise/ATR-2026-00523-claude-code-hooks-session-start-pre-trust-rce.yaml +221 -0
- package/rules/skill-compromise/ATR-2026-00525-mini-shai-hulud-gh-token-monitor-persistence.yaml +220 -0
- package/rules/skill-compromise/ATR-2026-00527-skill-silent-git-remote-mirror-exfiltration.yaml +201 -0
- package/rules/tool-poisoning/ATR-2026-00526-claude-code-shell-metachar-in-double-quoted-path.yaml +167 -0
- package/rules/tool-poisoning/ATR-2026-00529-litellm-proxy-sqli-cisa-kev.yaml +158 -0
- package/rules/tool-poisoning/ATR-2026-00530-ms-agent-shell-tool-unsanitized-argv-rce.yaml +184 -0
- package/rules/tool-poisoning/ATR-2026-00531-praisonai-unauthenticated-agent-api.yaml +174 -0
- package/rules/tool-poisoning/ATR-2026-00532-apache-doris-mcp-sql-injection.yaml +155 -0
- package/rules/tool-poisoning/ATR-2026-00533-apache-pinot-mcp-unauthenticated-takeover.yaml +151 -0
- package/rules/tool-poisoning/ATR-2026-00534-alibaba-rds-mcp-unauthenticated-metadata-exfil.yaml +155 -0
- package/rules/tool-poisoning/ATR-2026-00536-nginx-ui-mcp-unauthenticated-command-execution.yaml +199 -0
- package/rules/tool-poisoning/ATR-2026-00537-fastmcp-server-name-cmd-injection-windows.yaml +226 -0
- package/rules/tool-poisoning/ATR-2026-00538-langchain-chatchat-mcp-stdio-unauthenticated-rce.yaml +244 -0
- package/rules/tool-poisoning/ATR-2026-00540-praisonai-parse-mcp-command-cli-injection.yaml +186 -0
- package/rules/tool-poisoning/ATR-2026-00541-agent-zero-mcp-config-command-injection.yaml +183 -0
- package/rules/tool-poisoning/ATR-2026-00542-upsonic-mcp-command-allowlist-bypass.yaml +166 -0
- package/rules/tool-poisoning/ATR-2026-00543-litellm-mcp-server-argv-injection.yaml +168 -0
- package/rules/tool-poisoning/ATR-2026-00544-praisonai-pth-file-path-traversal-rce.yaml +172 -0
- package/rules/tool-poisoning/ATR-2026-00545-praisonai-tool-override-unauth-rce.yaml +170 -0
- package/spec/README.md +279 -0
- package/spec/atr-correlation-v1.0.md +281 -0
- package/spec/atr-event-v1.0.md +294 -0
- package/spec/atr-language-detection-v1.0.md +218 -0
- package/spec/atr-method-v1.1.md +557 -0
- package/spec/atr-profile-v1.0.md +307 -0
- package/spec/atr-schema.yaml +279 -8
- package/spec/category-registry/v1.0.yaml +200 -0
- package/spec/conformance/README.md +244 -0
- package/spec/conformance/SIGNING.md +191 -0
- package/spec/conformance/baseline/fixtures/ATR-2026-00001-tp-001/expected.json +36 -0
- package/spec/conformance/baseline/fixtures/ATR-2026-00001-tp-001/input.json +16 -0
- package/spec/conformance/baseline/fixtures/README.md +120 -0
- package/spec/conformance/baseline/manifest.json +56 -0
- package/spec/conformance/expected-results.schema.json +121 -0
- package/spec/external-registries/cccs-yara.md +142 -0
- package/spec/internet-drafts/draft-lin-atr-core-00.html +1925 -0
- package/spec/internet-drafts/draft-lin-atr-core-00.md +288 -0
- package/spec/internet-drafts/draft-lin-atr-core-00.txt +560 -0
- package/spec/internet-drafts/draft-lin-atr-core-00.xml +424 -0
- package/spec/mappings/README.md +43 -0
- package/spec/mappings/atr-to-nist-csf-2.0.md +234 -0
- package/spec/schema/correlation.schema.json +144 -0
- package/spec/schema/event.schema.json +233 -0
- package/spec/schema/profile.schema.json +196 -0
- package/spec/schema/rule.schema.json +224 -0
- package/spec/stix-extension/README.md +76 -13
- package/spec/stix-extension/examples/atr-rule-trace-method-example.json +85 -0
- package/spec/stix-extension/extension-definition.json +23 -3
- package/spec/stix-extension/x-atr-rule-schema.json +107 -11
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"eval-harness.js","sourceRoot":"","sources":["../../src/eval/eval-harness.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;GAYG;AAEH,OAAO,EAAE,OAAO,EAAE,IAAI,EAAE,MAAM,WAAW,CAAC;AAC1C,OAAO,EAAE,UAAU,EAAE,YAAY,EAAE,aAAa,EAAE,MAAM,SAAS,CAAC;AAClE,OAAO,EAAE,SAAS,EAAE,MAAM,cAAc,CAAC;AACzC,OAAO,EAAE,eAAe,EAAE,MAAM,yBAAyB,CAAC;AAG1D,OAAO,EAAE,WAAW,EAAE,cAAc,EAAE,MAAM,aAAa,CAAC;AAE1D,OAAO,EAAE,iBAAiB,EAAE,eAAe,EAAE,MAAM,cAAc,CAAC;AAElE,OAAO,EAAE,kBAAkB,EAAE,MAAM,mBAAmB,CAAC;AAiBvD;;GAEG;AACH,SAAS,aAAa,CAAC,MAAoB;IACzC,OAAO;QACL,IAAI,EAAE,MAAM,CAAC,SAAS;QACtB,OAAO,EAAE,MAAM,CAAC,IAAI;QACpB,SAAS,EAAE,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE;QACnC,MAAM,EAAE,MAAM,CAAC,MAAM;KACtB,CAAC;AACJ,CAAC;AAED;;;GAGG;AACH,SAAS,mBAAmB,CAC1B,MAAiB,EACjB,MAAoB;IAEpB,MAAM,KAAK,GAAG,aAAa,CAAC,MAAM,CAAC,CAAC;IAEpC,IAAI,CAAC;QACH,MAAM,KAAK,GAAG,WAAW,CAAC,GAAG,EAAE,CAAC;QAChC,MAAM,OAAO,GAAG,MAAM,CAAC,QAAQ,CAAC,KAAK,CAAC,CAAC;QACvC,MAAM,SAAS,GAAG,WAAW,CAAC,GAAG,EAAE,GAAG,KAAK,CAAC;QAE5C,MAAM,QAAQ,GAAG,OAAO,CAAC,MAAM,GAAG,CAAC,CAAC;QACpC,MAAM,QAAQ,GAAG,OAAO,CAAC,CAAC,CAAC,CAAC;QAE5B,OAAO;YACL,EAAE,EAAE,MAAM,CAAC,EAAE;YACb,QAAQ,EAAE,MAAM,CAAC,QAAQ;YACzB,iBAAiB,EAAE,MAAM,CAAC,iBAAiB;YAC3C,eAAe,EAAE,QAAQ;YACzB,YAAY,EAAE,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,EAAE,CAAC;YAC3C,UAAU,EAAE,QAAQ,EAAE,UAAU,IAAI,CAAC;YACrC,SAAS;YACT,UAAU,EAAE,MAAM,CAAC,UAAU;YAC7B,IAAI,EAAE,MAAM,CAAC,IAAI;SAClB,CAAC;IACJ,CAAC;IAAC,MAAM,CAAC;QACP,OAAO;YACL,EAAE,EAAE,MAAM,CAAC,EAAE;YACb,QAAQ,EAAE,MAAM,CAAC,QAAQ;YACzB,iBAAiB,EAAE,MAAM,CAAC,iBAAiB;YAC3C,eAAe,EAAE,KAAK;YACtB,YAAY,EAAE,EAAE;YAChB,UAAU,EAAE,CAAC;YACb,SAAS,EAAE,CAAC;YACZ,UAAU,EAAE,MAAM,CAAC,UAAU;YAC7B,IAAI,EAAE,MAAM,CAAC,IAAI;SAClB,CAAC;IACJ,CAAC;AACH,CAAC;AAED;;GAEG;AACH,KAAK,UAAU,kBAAkB,CAC/B,MAAiB,EACjB,MAAoB;IAEpB,MAAM,KAAK,GAAG,aAAa,CAAC,MAAM,CAAC,CAAC;IAEpC,IAAI,CAAC;QACH,MAAM,KAAK,GAAG,WAAW,CAAC,GAAG,EAAE,CAAC;QAChC,MAAM,EAAE,OAAO,EAAE,GAAG,MAAM,MAAM,CAAC,mBAAmB,CAAC,KAAK,CAAC,CAAC;QAC5D,MAAM,SAAS,GAAG,WAAW,CAAC,GAAG,EAAE,GAAG,KAAK,CAAC;QAE5C,MAAM,QAAQ,GAAG,OAAO,CAAC,UAAU,GAAG,CAAC,CAAC;QAExC,OAAO;YACL,EAAE,EAAE,MAAM,CAAC,EAAE;YACb,QAAQ,EAAE,MAAM,CAAC,QAAQ;YACzB,iBAAiB,EAAE,MAAM,CAAC,iBAAiB;YAC3C,eAAe,EAAE,QAAQ;YACzB,YAAY,EAAE,OAAO,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,EAAE,CAAC;YACnD,UAAU,EAAE,OAAO,CAAC,iBAAiB;YACrC,SAAS;YACT,UAAU,EAAE,MAAM,CAAC,UAAU;YAC7B,IAAI,EAAE,MAAM,CAAC,IAAI;SAClB,CAAC;IACJ,CAAC;IAAC,MAAM,CAAC;QACP,OAAO;YACL,EAAE,EAAE,MAAM,CAAC,EAAE;YACb,QAAQ,EAAE,MAAM,CAAC,QAAQ;YACzB,iBAAiB,EAAE,MAAM,CAAC,iBAAiB;YAC3C,eAAe,EAAE,KAAK;YACtB,YAAY,EAAE,EAAE;YAChB,UAAU,EAAE,CAAC;YACb,SAAS,EAAE,CAAC;YACZ,UAAU,EAAE,MAAM,CAAC,UAAU;YAC7B,IAAI,EAAE,MAAM,CAAC,IAAI;SAClB,CAAC;IACJ,CAAC;AACH,CAAC;AAED;;GAEG;AACH,KAAK,UAAU,gBAAgB,CAAC,cAAsB;IACpD,IAAI,CAAC,UAAU,CAAC,cAAc,CAAC;QAAE,OAAO,IAAI,CAAC;IAE7C,IAAI,CAAC;QACH,MAAM,IAAI,GAAG,IAAI,CAAC,KAAK,CAAC,YAAY,CAAC,cAAc,EAAE,OAAO,CAAC,CAAC,CAAC;QAC/D,MAAM,MAAM,GAAG,IAAI,eAAe,CAAC;YACjC,iBAAiB,EAAE,IAAI;YACvB,mBAAmB,EAAE,IAAI;SAC1B,CAAC,CAAC;QACH,MAAM,MAAM,CAAC,UAAU,EAAE,CAAC;QAC1B,OAAO,MAAM,CAAC,WAAW,EAAE,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,IAAI,CAAC;IAC9C,CAAC;IAAC,MAAM,CAAC;QACP,OAAO,IAAI,CAAC;IACd,CAAC;AACH,CAAC;AAED;;;GAGG;AACH,MAAM,CAAC,KAAK,UAAU,OAAO,CAAC,MAAkB;IAO9C,MAAM,MAAM,GAAG,MAAM,CAAC,MAAM,IAAI,WAAW,CAAC;IAC5C,MAAM,IAAI,GAAG,OAAO,CAAC,MAAM,CAAC,QAAQ,EAAE,IAAI,CAAC,CAAC;IAC5C,MAAM,cAAc,GAAG,MAAM,CAAC,cAAc,IAAI,IAAI,CAAC,IAAI,EAAE,MAAM,EAAE,wBAAwB,CAAC,CAAC;IAE7F,+BAA+B;IAC/B,MAAM,WAAW,GAAG,MAAM,CAAC,eAAe,KAAK,KAAK,CAAC;IACrD,IAAI,eAAe,GAA2B,IAAI,CAAC;IACnD,MAAM,SAAS,GAAa,CAAC,aAAa,CAAC,CAAC;IAE5C,IAAI,WAAW,EAAE,CAAC;QAChB,eAAe,GAAG,MAAM,gBAAgB,CAAC,cAAc,CAAC,CAAC;QACzD,IAAI,eAAe,EAAE,CAAC;YACpB,SAAS,CAAC,IAAI,CAAC,mBAAmB,CAAC,CAAC;QACtC,CAAC;IACH,CAAC;IAED,oBAAoB;IACpB,MAAM,MAAM,GAAG,IAAI,SAAS,CAAC;QAC3B,QAAQ,EAAE,MAAM,CAAC,QAAQ;QACzB,eAAe,EAAE,eAAe,IAAI,SAAS;KAC9C,CAAC,CAAC;IACH,MAAM,SAAS,GAAG,MAAM,MAAM,CAAC,SAAS,EAAE,CAAC;IAE3C,IAAI,SAAS,KAAK,CAAC,EAAE,CAAC;QACpB,MAAM,IAAI,KAAK,CAAC,wBAAwB,MAAM,CAAC,QAAQ,EAAE,CAAC,CAAC;IAC7D,CAAC;IAED,kBAAkB;IAClB,MAAM,OAAO,GAAmB,EAAE,CAAC;IACnC,MAAM,eAAe,GAAG,eAAe,KAAK,IAAI,CAAC;IAEjD,KAAK,MAAM,MAAM,IAAI,MAAM,EAAE,CAAC;QAC5B,MAAM,MAAM,GAAG,eAAe;YAC5B,CAAC,CAAC,MAAM,kBAAkB,CAAC,MAAM,EAAE,MAAM,CAAC;YAC1C,CAAC,CAAC,mBAAmB,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;QACxC,OAAO,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;IACvB,CAAC;IAED,2BAA2B;IAC3B,IAAI,eAAe,EAAE,CAAC;QACpB,MAAM,eAAe,CAAC,OAAO,EAAE,CAAC;IAClC,CAAC;IAED,iBAAiB;IACjB,MAAM,MAAM,GAAG,iBAAiB,CAAC,OAAO,CAAC,CAAC;IAC1C,MAAM,UAAU,GAAG,eAAe,CAAC,MAAM,EAAE,MAAM,CAAC,UAAU,CAAC,CAAC;IAC9D,MAAM,WAAW,GAAG,cAAc,EAAE,CAAC;IAErC,2BAA2B;IAC3B,MAAM,aAAa,GAAG,MAAM,CAAC,QAAQ,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC;IACzD,MAAM,WAAW,GAAG,kBAAkB,CAAC,OAAO,EAAE,aAAa,CAAC,CAAC;IAE/D,uCAAuC;IACvC,IAAI,MAAM,CAAC,UAAU,EAAE,CAAC;QACtB,MAAM,MAAM,GAAG;YACb,MAAM;YACN,UAAU;YACV,WAAW;YACX,WAAW;YACX,SAAS;YACT,MAAM,EAAE,WAAW;YACnB,KAAK,EAAE,SAAS;SACjB,CAAC;QACF,aAAa,CAAC,MAAM,CAAC,UAAU,EAAE,IAAI,CAAC,SAAS,CAAC,MAAM,EAAE,IAAI,EAAE,CAAC,CAAC,CAAC,CAAC;IACpE,CAAC;IAED,OAAO,EAAE,MAAM,EAAE,UAAU,EAAE,WAAW,EAAE,SAAS,EAAE,WAAW,EAAE,CAAC;AACrE,CAAC;AAED,8EAA8E;AAC9E,kBAAkB;AAClB,8EAA8E;AAE9E,SAAS,aAAa,CAAC,CAAS;IAC9B,OAAO,GAAG,CAAC,CAAC,GAAG,GAAG,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,GAAG,CAAC;AACpC,CAAC;AAED,SAAS,QAAQ,CAAC,CAAS;IACzB,OAAO,GAAG,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,IAAI,CAAC;AAC7B,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,UAAU;IAC9B,MAAM,IAAI,GAAG,OAAO,CAAC,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,OAAO,IAAI,GAAG,EAAE,IAAI,EAAE,IAAI,CAAC,CAAC,CAAC;IACnE,MAAM,QAAQ,GAAG,IAAI,CAAC,IAAI,EAAE,OAAO,CAAC,CAAC;IACrC,MAAM,UAAU,GAAG,IAAI,CAAC,IAAI,EAAE,MAAM,EAAE,kBAAkB,CAAC,CAAC;IAE1D,OAAO,CAAC,GAAG,CAAC,oCAAoC,CAAC,CAAC;IAElD,MAAM,EAAE,MAAM,EAAE,UAAU,EAAE,WAAW,EAAE,SAAS,EAAE,WAAW,EAAE,GAAG,MAAM,OAAO,CAAC;QAChF,QAAQ;QACR,UAAU;KACX,CAAC,CAAC;IAEH,eAAe;IACf,OAAO,CAAC,GAAG,CAAC,WAAW,WAAW,CAAC,KAAK,aAAa,WAAW,CAAC,OAAO,aAAa,WAAW,CAAC,MAAM,UAAU,CAAC,CAAC;IACnH,OAAO,CAAC,GAAG,CAAC,eAAe,MAAM,CAAC,IAAI,CAAC,WAAW,CAAC,UAAU,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;IAC7E,OAAO,CAAC,GAAG,CAAC,UAAU,SAAS,CAAC,IAAI,CAAC,KAAK,CAAC,EAAE,CAAC,CAAC;IAE/C,kBAAkB;IAClB,OAAO,CAAC,GAAG,CAAC,mBAAmB,CAAC,CAAC;IACjC,OAAO,CAAC,GAAG,CAAC,iBAAiB,aAAa,CAAC,MAAM,CAAC,OAAO,CAAC,SAAS,CAAC,EAAE,CAAC,CAAC;IACxE,OAAO,CAAC,GAAG,CAAC,iBAAiB,aAAa,CAAC,MAAM,CAAC,OAAO,CAAC,MAAM,CAAC,EAAE,CAAC,CAAC;IACrE,OAAO,CAAC,GAAG,CAAC,iBAAiB,aAAa,CAAC,MAAM,CAAC,OAAO,CAAC,EAAE,CAAC,EAAE,CAAC,CAAC;IACjE,OAAO,CAAC,GAAG,CAAC,iBAAiB,aAAa,CAAC,MAAM,CAAC,OAAO,CAAC,QAAQ,CAAC,EAAE,CAAC,CAAC;IACvE,OAAO,CAAC,GAAG,CAAC,iBAAiB,aAAa,CAAC,MAAM,CAAC,OAAO,CAAC,MAAM,CAAC,EAAE,CAAC,CAAC;IACrE,OAAO,CAAC,GAAG,CAAC,oBAAoB,MAAM,CAAC,OAAO,CAAC,SAAS,CAAC,EAAE,OAAO,MAAM,CAAC,OAAO,CAAC,SAAS,CAAC,EAAE,OAAO,MAAM,CAAC,OAAO,CAAC,SAAS,CAAC,EAAE,OAAO,MAAM,CAAC,OAAO,CAAC,SAAS,CAAC,EAAE,EAAE,CAAC,CAAC;IAErK,UAAU;IACV,OAAO,CAAC,GAAG,CAAC,mBAAmB,CAAC,CAAC;IACjC,OAAO,CAAC,GAAG,CAAC,WAAW,QAAQ,CAAC,MAAM,CAAC,OAAO,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC;IACvD,OAAO,CAAC,GAAG,CAAC,WAAW,QAAQ,CAAC,MAAM,CAAC,OAAO,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC;IACvD,OAAO,CAAC,GAAG,CAAC,WAAW,QAAQ,CAAC,MAAM,CAAC,OAAO,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC;IACvD,OAAO,CAAC,GAAG,CAAC,WAAW,QAAQ,CAAC,MAAM,CAAC,OAAO,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;IACxD,OAAO,CAAC,GAAG,CAAC,WAAW,QAAQ,CAAC,MAAM,CAAC,OAAO,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC;IAEvD,eAAe;IACf,OAAO,CAAC,GAAG,CAAC,uBAAuB,CAAC,CAAC;IACrC,KAAK,MAAM,GAAG,IAAI,MAAM,CAAC,UAAU,EAAE,CAAC;QACpC,MAAM,MAAM,GAAG,GAAG,CAAC,aAAa,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,aAAa,GAAG,CAAC,aAAa,CAAC,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC;QAChG,MAAM,GAAG,GAAG,GAAG,CAAC,cAAc,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,SAAS,GAAG,CAAC,cAAc,CAAC,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC;QAC3F,OAAO,CAAC,GAAG,CAAC,KAAK,GAAG,CAAC,QAAQ,YAAY,aAAa,CAAC,GAAG,CAAC,OAAO,CAAC,MAAM,CAAC,cAAc,aAAa,CAAC,GAAG,CAAC,OAAO,CAAC,SAAS,CAAC,OAAO,aAAa,CAAC,GAAG,CAAC,OAAO,CAAC,EAAE,CAAC,GAAG,MAAM,GAAG,GAAG,EAAE,CAAC,CAAC;IACrL,CAAC;IAED,iBAAiB;IACjB,OAAO,CAAC,GAAG,CAAC,yBAAyB,CAAC,CAAC;IACvC,KAAK,MAAM,IAAI,IAAI,MAAM,CAAC,YAAY,EAAE,CAAC;QACvC,OAAO,CAAC,GAAG,CAAC,KAAK,IAAI,CAAC,UAAU,YAAY,aAAa,CAAC,IAAI,CAAC,OAAO,CAAC,MAAM,CAAC,cAAc,aAAa,CAAC,IAAI,CAAC,OAAO,CAAC,SAAS,CAAC,OAAO,aAAa,CAAC,IAAI,CAAC,OAAO,CAAC,EAAE,CAAC,EAAE,CAAC,CAAC;IAC5K,CAAC;IAED,iBAAiB;IACjB,IAAI,MAAM,CAAC,aAAa,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QACpC,OAAO,CAAC,GAAG,CAAC,yBAAyB,MAAM,CAAC,aAAa,CAAC,MAAM,OAAO,CAAC,CAAC;QACzE,KAAK,MAAM,CAAC,IAAI,MAAM,CAAC,aAAa,EAAE,CAAC;YACrC,OAAO,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC,EAAE,KAAK,CAAC,CAAC,QAAQ,IAAI,CAAC,CAAC,UAAU,IAAI,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC;QACrE,CAAC;IACH,CAAC;IAED,kBAAkB;IAClB,IAAI,MAAM,CAAC,cAAc,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QACrC,OAAO,CAAC,GAAG,CAAC,0BAA0B,MAAM,CAAC,cAAc,CAAC,MAAM,OAAO,CAAC,CAAC;QAC3E,KAAK,MAAM,EAAE,IAAI,MAAM,CAAC,cAAc,EAAE,CAAC;YACvC,OAAO,CAAC,GAAG,CAAC,MAAM,EAAE,CAAC,EAAE,YAAY,EAAE,CAAC,YAAY,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;QACnE,CAAC;IACH,CAAC;IAED,eAAe;IACf,OAAO,CAAC,GAAG,CAAC,wBAAwB,CAAC,CAAC;IACtC,OAAO,CAAC,GAAG,CAAC,yBAAyB,WAAW,CAAC,mBAAmB,EAAE,CAAC,CAAC;IACxE,OAAO,CAAC,GAAG,CAAC,kBAAkB,WAAW,CAAC,UAAU,EAAE,CAAC,CAAC;IACxD,OAAO,CAAC,GAAG,CAAC,wBAAwB,WAAW,CAAC,eAAe,EAAE,CAAC,CAAC;IAEnE,IAAI,WAAW,CAAC,QAAQ,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QACpC,OAAO,CAAC,GAAG,CAAC,kCAAkC,CAAC,CAAC;QAChD,KAAK,MAAM,IAAI,IAAI,WAAW,CAAC,QAAQ,CAAC,KAAK,CAAC,CAAC,EAAE,EAAE,CAAC,EAAE,CAAC;YACrD,MAAM,SAAS,GAAG,IAAI,CAAC,UAAU,GAAG,CAAC;gBACnC,CAAC,CAAC,aAAa,CAAC,IAAI,CAAC,OAAO,GAAG,IAAI,CAAC,UAAU,CAAC;gBAC/C,CAAC,CAAC,KAAK,CAAC;YACV,OAAO,CAAC,GAAG,CAAC,OAAO,IAAI,CAAC,MAAM,aAAa,IAAI,CAAC,UAAU,OAAO,IAAI,CAAC,OAAO,OAAO,IAAI,CAAC,OAAO,cAAc,SAAS,YAAY,IAAI,CAAC,aAAa,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC;QACtK,CAAC;IACH,CAAC;IAED,IAAI,WAAW,CAAC,SAAS,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QACrC,OAAO,CAAC,GAAG,CAAC,6CAA6C,CAAC,CAAC;QAC3D,KAAK,MAAM,IAAI,IAAI,WAAW,CAAC,SAAS,CAAC,KAAK,CAAC,CAAC,EAAE,EAAE,CAAC,EAAE,CAAC;YACtD,OAAO,CAAC,GAAG,CAAC,OAAO,IAAI,CAAC,MAAM,aAAa,IAAI,CAAC,UAAU,OAAO,IAAI,CAAC,OAAO,OAAO,IAAI,CAAC,OAAO,gBAAgB,IAAI,CAAC,UAAU,CAAC,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;QACjJ,CAAC;IACH,CAAC;IAED,IAAI,WAAW,CAAC,iBAAiB,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QAC7C,OAAO,CAAC,GAAG,CAAC,0BAA0B,WAAW,CAAC,iBAAiB,CAAC,MAAM,IAAI,CAAC,CAAC;QAChF,KAAK,MAAM,EAAE,IAAI,WAAW,CAAC,iBAAiB,CAAC,KAAK,CAAC,CAAC,EAAE,EAAE,CAAC,EAAE,CAAC;YAC5D,OAAO,CAAC,GAAG,CAAC,OAAO,EAAE,EAAE,CAAC,CAAC;QAC3B,CAAC;QACD,IAAI,WAAW,CAAC,iBAAiB,CAAC,MAAM,GAAG,EAAE,EAAE,CAAC;YAC9C,OAAO,CAAC,GAAG,CAAC,eAAe,WAAW,CAAC,iBAAiB,CAAC,MAAM,GAAG,EAAE,OAAO,CAAC,CAAC;QAC/E,CAAC;IACH,CAAC;IAED,mBAAmB;IACnB,OAAO,CAAC,GAAG,CAAC,4BAA4B,CAAC,CAAC;IAC1C,IAAI,UAAU,CAAC,MAAM,EAAE,CAAC;QACtB,OAAO,CAAC,GAAG,CAAC,UAAU,CAAC,CAAC;IAC1B,CAAC;SAAM,CAAC;QACN,OAAO,CAAC,GAAG,CAAC,WAAW,CAAC,CAAC;QACzB,KAAK,MAAM,CAAC,IAAI,UAAU,CAAC,UAAU,EAAE,CAAC;YACtC,OAAO,CAAC,GAAG,CAAC,SAAS,CAAC,EAAE,CAAC,CAAC;QAC5B,CAAC;IACH,CAAC;IAED,OAAO,CAAC,GAAG,CAAC,sBAAsB,UAAU,EAAE,CAAC,CAAC;IAChD,OAAO,CAAC,GAAG,CAAC,SAAS,CAAC,CAAC;IAEvB,IAAI,CAAC,UAAU,CAAC,MAAM,EAAE,CAAC;QACvB,OAAO,CAAC,QAAQ,GAAG,CAAC,CAAC;IACvB,CAAC;AACH,CAAC"}
|
|
1
|
+
{"version":3,"file":"eval-harness.js","sourceRoot":"","sources":["../../src/eval/eval-harness.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;GAYG;AAEH,OAAO,EAAE,OAAO,EAAE,IAAI,EAAE,MAAM,WAAW,CAAC;AAC1C,OAAO,EAAE,UAAU,EAAE,YAAY,EAAE,aAAa,EAAE,MAAM,SAAS,CAAC;AAClE,OAAO,EAAE,8BAA8B,EAAE,MAAM,qCAAqC,CAAC;AACrF,OAAO,EAAE,SAAS,EAAE,MAAM,cAAc,CAAC;AACzC,OAAO,EAAE,eAAe,EAAE,MAAM,yBAAyB,CAAC;AAG1D,OAAO,EAAE,WAAW,EAAE,cAAc,EAAE,MAAM,aAAa,CAAC;AAE1D,OAAO,EAAE,iBAAiB,EAAE,eAAe,EAAE,MAAM,cAAc,CAAC;AAElE,OAAO,EAAE,kBAAkB,EAAE,MAAM,mBAAmB,CAAC;AAiBvD;;GAEG;AACH,SAAS,aAAa,CAAC,MAAoB;IACzC,OAAO;QACL,IAAI,EAAE,MAAM,CAAC,SAAS;QACtB,OAAO,EAAE,MAAM,CAAC,IAAI;QACpB,SAAS,EAAE,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE;QACnC,MAAM,EAAE,MAAM,CAAC,MAAM;KACtB,CAAC;AACJ,CAAC;AAED;;;GAGG;AACH,SAAS,mBAAmB,CAC1B,MAAiB,EACjB,MAAoB;IAEpB,MAAM,KAAK,GAAG,aAAa,CAAC,MAAM,CAAC,CAAC;IAEpC,IAAI,CAAC;QACH,MAAM,KAAK,GAAG,WAAW,CAAC,GAAG,EAAE,CAAC;QAChC,MAAM,OAAO,GAAG,MAAM,CAAC,QAAQ,CAAC,KAAK,CAAC,CAAC;QACvC,MAAM,SAAS,GAAG,WAAW,CAAC,GAAG,EAAE,GAAG,KAAK,CAAC;QAE5C,MAAM,QAAQ,GAAG,OAAO,CAAC,MAAM,GAAG,CAAC,CAAC;QACpC,MAAM,QAAQ,GAAG,OAAO,CAAC,CAAC,CAAC,CAAC;QAE5B,OAAO;YACL,EAAE,EAAE,MAAM,CAAC,EAAE;YACb,QAAQ,EAAE,MAAM,CAAC,QAAQ;YACzB,iBAAiB,EAAE,MAAM,CAAC,iBAAiB;YAC3C,eAAe,EAAE,QAAQ;YACzB,YAAY,EAAE,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,EAAE,CAAC;YAC3C,UAAU,EAAE,QAAQ,EAAE,UAAU,IAAI,CAAC;YACrC,SAAS;YACT,UAAU,EAAE,MAAM,CAAC,UAAU;YAC7B,IAAI,EAAE,MAAM,CAAC,IAAI;SAClB,CAAC;IACJ,CAAC;IAAC,MAAM,CAAC;QACP,OAAO;YACL,EAAE,EAAE,MAAM,CAAC,EAAE;YACb,QAAQ,EAAE,MAAM,CAAC,QAAQ;YACzB,iBAAiB,EAAE,MAAM,CAAC,iBAAiB;YAC3C,eAAe,EAAE,KAAK;YACtB,YAAY,EAAE,EAAE;YAChB,UAAU,EAAE,CAAC;YACb,SAAS,EAAE,CAAC;YACZ,UAAU,EAAE,MAAM,CAAC,UAAU;YAC7B,IAAI,EAAE,MAAM,CAAC,IAAI;SAClB,CAAC;IACJ,CAAC;AACH,CAAC;AAED;;GAEG;AACH,KAAK,UAAU,kBAAkB,CAC/B,MAAiB,EACjB,MAAoB;IAEpB,MAAM,KAAK,GAAG,aAAa,CAAC,MAAM,CAAC,CAAC;IAEpC,IAAI,CAAC;QACH,MAAM,KAAK,GAAG,WAAW,CAAC,GAAG,EAAE,CAAC;QAChC,MAAM,EAAE,OAAO,EAAE,GAAG,MAAM,MAAM,CAAC,mBAAmB,CAAC,KAAK,CAAC,CAAC;QAC5D,MAAM,SAAS,GAAG,WAAW,CAAC,GAAG,EAAE,GAAG,KAAK,CAAC;QAE5C,MAAM,QAAQ,GAAG,OAAO,CAAC,UAAU,GAAG,CAAC,CAAC;QAExC,OAAO;YACL,EAAE,EAAE,MAAM,CAAC,EAAE;YACb,QAAQ,EAAE,MAAM,CAAC,QAAQ;YACzB,iBAAiB,EAAE,MAAM,CAAC,iBAAiB;YAC3C,eAAe,EAAE,QAAQ;YACzB,YAAY,EAAE,OAAO,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,EAAE,CAAC;YACnD,UAAU,EAAE,OAAO,CAAC,iBAAiB;YACrC,SAAS;YACT,UAAU,EAAE,MAAM,CAAC,UAAU;YAC7B,IAAI,EAAE,MAAM,CAAC,IAAI;SAClB,CAAC;IACJ,CAAC;IAAC,MAAM,CAAC;QACP,OAAO;YACL,EAAE,EAAE,MAAM,CAAC,EAAE;YACb,QAAQ,EAAE,MAAM,CAAC,QAAQ;YACzB,iBAAiB,EAAE,MAAM,CAAC,iBAAiB;YAC3C,eAAe,EAAE,KAAK;YACtB,YAAY,EAAE,EAAE;YAChB,UAAU,EAAE,CAAC;YACb,SAAS,EAAE,CAAC;YACZ,UAAU,EAAE,MAAM,CAAC,UAAU;YAC7B,IAAI,EAAE,MAAM,CAAC,IAAI;SAClB,CAAC;IACJ,CAAC;AACH,CAAC;AAED;;GAEG;AACH,KAAK,UAAU,gBAAgB,CAAC,cAAsB;IACpD,IAAI,CAAC,UAAU,CAAC,cAAc,CAAC;QAAE,OAAO,IAAI,CAAC;IAE7C,IAAI,CAAC;QACH,MAAM,IAAI,GAAG,IAAI,CAAC,KAAK,CAAC,YAAY,CAAC,cAAc,EAAE,OAAO,CAAC,CAAC,CAAC;QAC/D,MAAM,MAAM,GAAG,IAAI,eAAe,CAAC;YACjC,iBAAiB,EAAE,IAAI;YACvB,mBAAmB,EAAE,IAAI;SAC1B,CAAC,CAAC;QACH,MAAM,MAAM,CAAC,UAAU,EAAE,CAAC;QAC1B,OAAO,MAAM,CAAC,WAAW,EAAE,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,IAAI,CAAC;IAC9C,CAAC;IAAC,MAAM,CAAC;QACP,OAAO,IAAI,CAAC;IACd,CAAC;AACH,CAAC;AAED;;;GAGG;AACH,MAAM,CAAC,KAAK,UAAU,OAAO,CAAC,MAAkB;IAO9C,MAAM,MAAM,GAAG,MAAM,CAAC,MAAM,IAAI,WAAW,CAAC;IAC5C,MAAM,IAAI,GAAG,OAAO,CAAC,MAAM,CAAC,QAAQ,EAAE,IAAI,CAAC,CAAC;IAC5C,MAAM,cAAc,GAAG,MAAM,CAAC,cAAc,IAAI,IAAI,CAAC,IAAI,EAAE,MAAM,EAAE,wBAAwB,CAAC,CAAC;IAE7F,+BAA+B;IAC/B,MAAM,WAAW,GAAG,MAAM,CAAC,eAAe,KAAK,KAAK,CAAC;IACrD,IAAI,eAAe,GAA2B,IAAI,CAAC;IACnD,MAAM,SAAS,GAAa,CAAC,aAAa,CAAC,CAAC;IAE5C,IAAI,WAAW,EAAE,CAAC;QAChB,eAAe,GAAG,MAAM,gBAAgB,CAAC,cAAc,CAAC,CAAC;QACzD,IAAI,eAAe,EAAE,CAAC;YACpB,SAAS,CAAC,IAAI,CAAC,mBAAmB,CAAC,CAAC;QACtC,CAAC;IACH,CAAC;IAED,oBAAoB;IACpB,MAAM,MAAM,GAAG,IAAI,SAAS,CAAC;QAC3B,QAAQ,EAAE,MAAM,CAAC,QAAQ;QACzB,eAAe,EAAE,eAAe,IAAI,SAAS;KAC9C,CAAC,CAAC;IACH,MAAM,SAAS,GAAG,MAAM,MAAM,CAAC,SAAS,EAAE,CAAC;IAE3C,IAAI,SAAS,KAAK,CAAC,EAAE,CAAC;QACpB,MAAM,IAAI,KAAK,CAAC,wBAAwB,MAAM,CAAC,QAAQ,EAAE,CAAC,CAAC;IAC7D,CAAC;IAED,kBAAkB;IAClB,MAAM,OAAO,GAAmB,EAAE,CAAC;IACnC,MAAM,eAAe,GAAG,eAAe,KAAK,IAAI,CAAC;IAEjD,KAAK,MAAM,MAAM,IAAI,MAAM,EAAE,CAAC;QAC5B,MAAM,MAAM,GAAG,eAAe;YAC5B,CAAC,CAAC,MAAM,kBAAkB,CAAC,MAAM,EAAE,MAAM,CAAC;YAC1C,CAAC,CAAC,mBAAmB,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;QACxC,OAAO,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;IACvB,CAAC;IAED,2BAA2B;IAC3B,IAAI,eAAe,EAAE,CAAC;QACpB,MAAM,eAAe,CAAC,OAAO,EAAE,CAAC;IAClC,CAAC;IAED,iBAAiB;IACjB,MAAM,MAAM,GAAG,iBAAiB,CAAC,OAAO,CAAC,CAAC;IAC1C,MAAM,UAAU,GAAG,eAAe,CAAC,MAAM,EAAE,MAAM,CAAC,UAAU,CAAC,CAAC;IAC9D,MAAM,WAAW,GAAG,cAAc,EAAE,CAAC;IAErC,2BAA2B;IAC3B,MAAM,aAAa,GAAG,MAAM,CAAC,QAAQ,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC;IACzD,MAAM,WAAW,GAAG,kBAAkB,CAAC,OAAO,EAAE,aAAa,CAAC,CAAC;IAE/D,uCAAuC;IACvC,IAAI,MAAM,CAAC,UAAU,EAAE,CAAC;QACtB,MAAM,MAAM,GAAG;YACb,MAAM;YACN,UAAU;YACV,WAAW;YACX,WAAW;YACX,SAAS;YACT,MAAM,EAAE,WAAW;YACnB,KAAK,EAAE,SAAS;SACjB,CAAC;QACF,aAAa,CAAC,MAAM,CAAC,UAAU,EAAE,IAAI,CAAC,SAAS,CAAC,MAAM,EAAE,IAAI,EAAE,CAAC,CAAC,CAAC,CAAC;IACpE,CAAC;IAED,OAAO,EAAE,MAAM,EAAE,UAAU,EAAE,WAAW,EAAE,SAAS,EAAE,WAAW,EAAE,CAAC;AACrE,CAAC;AAED,8EAA8E;AAC9E,kBAAkB;AAClB,8EAA8E;AAE9E,SAAS,aAAa,CAAC,CAAS;IAC9B,OAAO,GAAG,CAAC,CAAC,GAAG,GAAG,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,GAAG,CAAC;AACpC,CAAC;AAED,SAAS,QAAQ,CAAC,CAAS;IACzB,OAAO,GAAG,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,IAAI,CAAC;AAC7B,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,UAAU;IAC9B,MAAM,IAAI,GAAG,OAAO,CAAC,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,OAAO,IAAI,GAAG,EAAE,IAAI,EAAE,IAAI,CAAC,CAAC,CAAC;IACnE,MAAM,QAAQ,GAAG,IAAI,CAAC,IAAI,EAAE,OAAO,CAAC,CAAC;IACrC,MAAM,UAAU,GAAG,IAAI,CAAC,IAAI,EAAE,MAAM,EAAE,kBAAkB,CAAC,CAAC;IAE1D,OAAO,CAAC,GAAG,CAAC,oCAAoC,CAAC,CAAC;IAElD,MAAM,EAAE,MAAM,EAAE,UAAU,EAAE,WAAW,EAAE,SAAS,EAAE,WAAW,EAAE,GAAG,MAAM,OAAO,CAAC;QAChF,QAAQ;QACR,UAAU;KACX,CAAC,CAAC;IAEH,eAAe;IACf,OAAO,CAAC,GAAG,CAAC,WAAW,WAAW,CAAC,KAAK,aAAa,WAAW,CAAC,OAAO,aAAa,WAAW,CAAC,MAAM,UAAU,CAAC,CAAC;IACnH,OAAO,CAAC,GAAG,CAAC,eAAe,MAAM,CAAC,IAAI,CAAC,WAAW,CAAC,UAAU,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;IAC7E,OAAO,CAAC,GAAG,CAAC,UAAU,SAAS,CAAC,IAAI,CAAC,KAAK,CAAC,EAAE,CAAC,CAAC;IAE/C,kBAAkB;IAClB,OAAO,CAAC,GAAG,CAAC,mBAAmB,CAAC,CAAC;IACjC,OAAO,CAAC,GAAG,CAAC,iBAAiB,aAAa,CAAC,MAAM,CAAC,OAAO,CAAC,SAAS,CAAC,EAAE,CAAC,CAAC;IACxE,OAAO,CAAC,GAAG,CAAC,iBAAiB,aAAa,CAAC,MAAM,CAAC,OAAO,CAAC,MAAM,CAAC,EAAE,CAAC,CAAC;IACrE,OAAO,CAAC,GAAG,CAAC,iBAAiB,aAAa,CAAC,MAAM,CAAC,OAAO,CAAC,EAAE,CAAC,EAAE,CAAC,CAAC;IACjE,OAAO,CAAC,GAAG,CAAC,iBAAiB,aAAa,CAAC,MAAM,CAAC,OAAO,CAAC,QAAQ,CAAC,EAAE,CAAC,CAAC;IACvE,OAAO,CAAC,GAAG,CAAC,iBAAiB,aAAa,CAAC,MAAM,CAAC,OAAO,CAAC,MAAM,CAAC,EAAE,CAAC,CAAC;IACrE,OAAO,CAAC,GAAG,CAAC,oBAAoB,MAAM,CAAC,OAAO,CAAC,SAAS,CAAC,EAAE,OAAO,MAAM,CAAC,OAAO,CAAC,SAAS,CAAC,EAAE,OAAO,MAAM,CAAC,OAAO,CAAC,SAAS,CAAC,EAAE,OAAO,MAAM,CAAC,OAAO,CAAC,SAAS,CAAC,EAAE,EAAE,CAAC,CAAC;IAErK,UAAU;IACV,OAAO,CAAC,GAAG,CAAC,mBAAmB,CAAC,CAAC;IACjC,OAAO,CAAC,GAAG,CAAC,WAAW,QAAQ,CAAC,MAAM,CAAC,OAAO,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC;IACvD,OAAO,CAAC,GAAG,CAAC,WAAW,QAAQ,CAAC,MAAM,CAAC,OAAO,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC;IACvD,OAAO,CAAC,GAAG,CAAC,WAAW,QAAQ,CAAC,MAAM,CAAC,OAAO,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC;IACvD,OAAO,CAAC,GAAG,CAAC,WAAW,QAAQ,CAAC,MAAM,CAAC,OAAO,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;IACxD,OAAO,CAAC,GAAG,CAAC,WAAW,QAAQ,CAAC,MAAM,CAAC,OAAO,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC;IAEvD,eAAe;IACf,OAAO,CAAC,GAAG,CAAC,uBAAuB,CAAC,CAAC;IACrC,KAAK,MAAM,GAAG,IAAI,MAAM,CAAC,UAAU,EAAE,CAAC;QACpC,MAAM,MAAM,GAAG,GAAG,CAAC,aAAa,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,aAAa,GAAG,CAAC,aAAa,CAAC,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC;QAChG,MAAM,GAAG,GAAG,GAAG,CAAC,cAAc,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,SAAS,GAAG,CAAC,cAAc,CAAC,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC;QAC3F,OAAO,CAAC,GAAG,CAAC,KAAK,GAAG,CAAC,QAAQ,YAAY,aAAa,CAAC,GAAG,CAAC,OAAO,CAAC,MAAM,CAAC,cAAc,aAAa,CAAC,GAAG,CAAC,OAAO,CAAC,SAAS,CAAC,OAAO,aAAa,CAAC,GAAG,CAAC,OAAO,CAAC,EAAE,CAAC,GAAG,MAAM,GAAG,GAAG,EAAE,CAAC,CAAC;IACrL,CAAC;IAED,iBAAiB;IACjB,OAAO,CAAC,GAAG,CAAC,yBAAyB,CAAC,CAAC;IACvC,KAAK,MAAM,IAAI,IAAI,MAAM,CAAC,YAAY,EAAE,CAAC;QACvC,OAAO,CAAC,GAAG,CAAC,KAAK,IAAI,CAAC,UAAU,YAAY,aAAa,CAAC,IAAI,CAAC,OAAO,CAAC,MAAM,CAAC,cAAc,aAAa,CAAC,IAAI,CAAC,OAAO,CAAC,SAAS,CAAC,OAAO,aAAa,CAAC,IAAI,CAAC,OAAO,CAAC,EAAE,CAAC,EAAE,CAAC,CAAC;IAC5K,CAAC;IAED,iBAAiB;IACjB,IAAI,MAAM,CAAC,aAAa,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QACpC,OAAO,CAAC,GAAG,CAAC,yBAAyB,MAAM,CAAC,aAAa,CAAC,MAAM,OAAO,CAAC,CAAC;QACzE,KAAK,MAAM,CAAC,IAAI,MAAM,CAAC,aAAa,EAAE,CAAC;YACrC,OAAO,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC,EAAE,KAAK,CAAC,CAAC,QAAQ,IAAI,CAAC,CAAC,UAAU,IAAI,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC;QACrE,CAAC;IACH,CAAC;IAED,kBAAkB;IAClB,IAAI,MAAM,CAAC,cAAc,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QACrC,OAAO,CAAC,GAAG,CAAC,0BAA0B,MAAM,CAAC,cAAc,CAAC,MAAM,OAAO,CAAC,CAAC;QAC3E,KAAK,MAAM,EAAE,IAAI,MAAM,CAAC,cAAc,EAAE,CAAC;YACvC,OAAO,CAAC,GAAG,CAAC,MAAM,EAAE,CAAC,EAAE,YAAY,EAAE,CAAC,YAAY,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;QACnE,CAAC;IACH,CAAC;IAED,eAAe;IACf,OAAO,CAAC,GAAG,CAAC,wBAAwB,CAAC,CAAC;IACtC,OAAO,CAAC,GAAG,CAAC,yBAAyB,WAAW,CAAC,mBAAmB,EAAE,CAAC,CAAC;IACxE,OAAO,CAAC,GAAG,CAAC,kBAAkB,WAAW,CAAC,UAAU,EAAE,CAAC,CAAC;IACxD,OAAO,CAAC,GAAG,CAAC,wBAAwB,WAAW,CAAC,eAAe,EAAE,CAAC,CAAC;IAEnE,IAAI,WAAW,CAAC,QAAQ,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QACpC,OAAO,CAAC,GAAG,CAAC,kCAAkC,CAAC,CAAC;QAChD,KAAK,MAAM,IAAI,IAAI,WAAW,CAAC,QAAQ,CAAC,KAAK,CAAC,CAAC,EAAE,EAAE,CAAC,EAAE,CAAC;YACrD,MAAM,SAAS,GAAG,IAAI,CAAC,UAAU,GAAG,CAAC;gBACnC,CAAC,CAAC,aAAa,CAAC,IAAI,CAAC,OAAO,GAAG,IAAI,CAAC,UAAU,CAAC;gBAC/C,CAAC,CAAC,KAAK,CAAC;YACV,OAAO,CAAC,GAAG,CAAC,OAAO,IAAI,CAAC,MAAM,aAAa,IAAI,CAAC,UAAU,OAAO,IAAI,CAAC,OAAO,OAAO,IAAI,CAAC,OAAO,cAAc,SAAS,YAAY,IAAI,CAAC,aAAa,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC;QACtK,CAAC;IACH,CAAC;IAED,IAAI,WAAW,CAAC,SAAS,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QACrC,OAAO,CAAC,GAAG,CAAC,6CAA6C,CAAC,CAAC;QAC3D,KAAK,MAAM,IAAI,IAAI,WAAW,CAAC,SAAS,CAAC,KAAK,CAAC,CAAC,EAAE,EAAE,CAAC,EAAE,CAAC;YACtD,OAAO,CAAC,GAAG,CAAC,OAAO,IAAI,CAAC,MAAM,aAAa,IAAI,CAAC,UAAU,OAAO,IAAI,CAAC,OAAO,OAAO,IAAI,CAAC,OAAO,gBAAgB,IAAI,CAAC,UAAU,CAAC,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;QACjJ,CAAC;IACH,CAAC;IAED,IAAI,WAAW,CAAC,iBAAiB,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QAC7C,OAAO,CAAC,GAAG,CAAC,0BAA0B,WAAW,CAAC,iBAAiB,CAAC,MAAM,IAAI,CAAC,CAAC;QAChF,KAAK,MAAM,EAAE,IAAI,WAAW,CAAC,iBAAiB,CAAC,KAAK,CAAC,CAAC,EAAE,EAAE,CAAC,EAAE,CAAC;YAC5D,OAAO,CAAC,GAAG,CAAC,OAAO,EAAE,EAAE,CAAC,CAAC;QAC3B,CAAC;QACD,IAAI,WAAW,CAAC,iBAAiB,CAAC,MAAM,GAAG,EAAE,EAAE,CAAC;YAC9C,OAAO,CAAC,GAAG,CAAC,eAAe,WAAW,CAAC,iBAAiB,CAAC,MAAM,GAAG,EAAE,OAAO,CAAC,CAAC;QAC/E,CAAC;IACH,CAAC;IAED,mBAAmB;IACnB,OAAO,CAAC,GAAG,CAAC,4BAA4B,CAAC,CAAC;IAC1C,IAAI,UAAU,CAAC,MAAM,EAAE,CAAC;QACtB,OAAO,CAAC,GAAG,CAAC,UAAU,CAAC,CAAC;IAC1B,CAAC;SAAM,CAAC;QACN,OAAO,CAAC,GAAG,CAAC,WAAW,CAAC,CAAC;QACzB,KAAK,MAAM,CAAC,IAAI,UAAU,CAAC,UAAU,EAAE,CAAC;YACtC,OAAO,CAAC,GAAG,CAAC,SAAS,CAAC,EAAE,CAAC,CAAC;QAC5B,CAAC;IACH,CAAC;IAED,OAAO,CAAC,GAAG,CAAC,sBAAsB,UAAU,EAAE,CAAC,CAAC;IAEhD,yEAAyE;IACzE,2EAA2E;IAC3E,MAAM,EAAE,eAAe,EAAE,GAAG,8BAA8B,CAAC,MAAM,EAAE;QACjE,MAAM,EAAE,eAAe;QACvB,cAAc,EAAE,UAAU;QAC1B,KAAK,EAAE,0EAA0E;KAClF,CAAC,CAAC;IACH,OAAO,CAAC,GAAG,CAAC,gBAAgB,eAAe,EAAE,CAAC,CAAC;IAC/C,OAAO,CAAC,GAAG,CAAC,SAAS,CAAC,CAAC;IAEvB,IAAI,CAAC,UAAU,CAAC,MAAM,EAAE,CAAC;QACvB,OAAO,CAAC,QAAQ,GAAG,CAAC,CAAC;IACvB,CAAC;AACH,CAAC"}
|
|
@@ -18,6 +18,7 @@
|
|
|
18
18
|
import { resolve, join } from 'node:path';
|
|
19
19
|
import { loadHackaPromptCorpus, getHackaPromptCorpusStats } from './hackaprompt-corpus.js';
|
|
20
20
|
import { runEval } from './eval-harness.js';
|
|
21
|
+
import { writeMeasurementFromEvalReport } from '../measurement/from-eval-harness.js';
|
|
21
22
|
function formatPercent(n) {
|
|
22
23
|
return `${(n * 100).toFixed(1)}%`;
|
|
23
24
|
}
|
|
@@ -77,6 +78,14 @@ async function main() {
|
|
|
77
78
|
console.log(` ${r.ruleId}: matches=${r.matchCount} TP=${r.tpCount} FP=${r.fpCount}`);
|
|
78
79
|
}
|
|
79
80
|
console.log(`\nReport saved to: ${outputPath}`);
|
|
81
|
+
// Standardized Measurement file (version-pinned, immutable).
|
|
82
|
+
const { measurementPath } = writeMeasurementFromEvalReport(report, {
|
|
83
|
+
source: 'hackaprompt',
|
|
84
|
+
source_version: 'v1',
|
|
85
|
+
source_url: 'https://huggingface.co/datasets/hackaprompt/hackaprompt-dataset',
|
|
86
|
+
notes: 'HackAPrompt competition dataset. 100% adversarial — fp_rate is undefined on this corpus.',
|
|
87
|
+
});
|
|
88
|
+
console.log(`Measurement: ${measurementPath}`);
|
|
80
89
|
console.log('Done.');
|
|
81
90
|
}
|
|
82
91
|
main().catch((err) => {
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"run-hackaprompt-benchmark.js","sourceRoot":"","sources":["../../src/eval/run-hackaprompt-benchmark.ts"],"names":[],"mappings":";AACA;;;;;;;;;;;;;;;GAeG;AAEH,OAAO,EAAE,OAAO,EAAE,IAAI,EAAE,MAAM,WAAW,CAAC;AAC1C,OAAO,EAAE,qBAAqB,EAAE,yBAAyB,EAAE,MAAM,yBAAyB,CAAC;AAC3F,OAAO,EAAE,OAAO,EAAE,MAAM,mBAAmB,CAAC;
|
|
1
|
+
{"version":3,"file":"run-hackaprompt-benchmark.js","sourceRoot":"","sources":["../../src/eval/run-hackaprompt-benchmark.ts"],"names":[],"mappings":";AACA;;;;;;;;;;;;;;;GAeG;AAEH,OAAO,EAAE,OAAO,EAAE,IAAI,EAAE,MAAM,WAAW,CAAC;AAC1C,OAAO,EAAE,qBAAqB,EAAE,yBAAyB,EAAE,MAAM,yBAAyB,CAAC;AAC3F,OAAO,EAAE,OAAO,EAAE,MAAM,mBAAmB,CAAC;AAC5C,OAAO,EAAE,8BAA8B,EAAE,MAAM,qCAAqC,CAAC;AAErF,SAAS,aAAa,CAAC,CAAS;IAC9B,OAAO,GAAG,CAAC,CAAC,GAAG,GAAG,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,GAAG,CAAC;AACpC,CAAC;AAED,SAAS,QAAQ,CAAC,CAAS;IACzB,OAAO,GAAG,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,IAAI,CAAC;AAC7B,CAAC;AAED,KAAK,UAAU,IAAI;IACjB,MAAM,IAAI,GAAG,OAAO,CAAC,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,OAAO,IAAI,GAAG,EAAE,IAAI,EAAE,IAAI,CAAC,CAAC,CAAC;IACnE,MAAM,QAAQ,GAAG,IAAI,CAAC,IAAI,EAAE,OAAO,CAAC,CAAC;IACrC,MAAM,QAAQ,GAAG,IAAI,CAAC,IAAI,EAAE,MAAM,EAAE,aAAa,EAAE,yBAAyB,CAAC,CAAC;IAC9E,MAAM,UAAU,GAAG,IAAI,CAAC,IAAI,EAAE,MAAM,EAAE,aAAa,EAAE,8BAA8B,CAAC,CAAC;IAErF,OAAO,CAAC,GAAG,CAAC,qDAAqD,CAAC,CAAC;IACnE,OAAO,CAAC,GAAG,CAAC,WAAW,QAAQ,EAAE,CAAC,CAAC;IACnC,OAAO,CAAC,GAAG,CAAC,WAAW,QAAQ,IAAI,CAAC,CAAC;IAErC,MAAM,MAAM,GAAG,qBAAqB,CAAC,QAAQ,CAAC,CAAC;IAC/C,MAAM,KAAK,GAAG,yBAAyB,CAAC,MAAM,CAAC,CAAC;IAEhD,OAAO,CAAC,GAAG,CAAC,UAAU,KAAK,CAAC,KAAK,aAAa,KAAK,CAAC,OAAO,aAAa,KAAK,CAAC,MAAM,UAAU,CAAC,CAAC;IAChG,OAAO,CAAC,GAAG,CAAC,eAAe,MAAM,CAAC,OAAO,CAAC,KAAK,CAAC,UAAU,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,EAAE,EAAE,CAAC,GAAG,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;IACvG,OAAO,CAAC,GAAG,CAAC,eAAe,MAAM,CAAC,OAAO,CAAC,KAAK,CAAC,YAAY,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,EAAE,EAAE,CAAC,GAAG,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;IAEzG,6EAA6E;IAC7E,wEAAwE;IACxE,sEAAsE;IACtE,qDAAqD;IACrD,MAAM,qBAAqB,GAAG;QAC5B,SAAS,EAAE,IAAI;QACf,SAAS,EAAE,GAAG;QACd,KAAK,EAAE,GAAG;QACV,eAAe,EAAE,GAAG;KACrB,CAAC;IAEF,MAAM,EAAE,MAAM,EAAE,SAAS,EAAE,WAAW,EAAE,GAAG,MAAM,OAAO,CAAC;QACvD,QAAQ;QACR,MAAM;QACN,UAAU,EAAE,qBAAqB;QACjC,UAAU;KACX,CAAC,CAAC;IAEH,OAAO,CAAC,GAAG,CAAC,YAAY,SAAS,CAAC,IAAI,CAAC,KAAK,CAAC,EAAE,CAAC,CAAC;IACjD,OAAO,CAAC,GAAG,CAAC,mBAAmB,CAAC,CAAC;IACjC,OAAO,CAAC,GAAG,CAAC,iBAAiB,aAAa,CAAC,MAAM,CAAC,OAAO,CAAC,MAAM,CAAC,EAAE,CAAC,CAAC;IACrE,OAAO,CAAC,GAAG,CAAC,iBAAiB,aAAa,CAAC,MAAM,CAAC,OAAO,CAAC,SAAS,CAAC,oCAAoC,CAAC,CAAC;IAC1G,OAAO,CAAC,GAAG,CAAC,oBAAoB,MAAM,CAAC,OAAO,CAAC,SAAS,CAAC,EAAE,OAAO,MAAM,CAAC,OAAO,CAAC,SAAS,CAAC,EAAE,EAAE,CAAC,CAAC;IAEjG,OAAO,CAAC,GAAG,CAAC,mBAAmB,CAAC,CAAC;IACjC,OAAO,CAAC,GAAG,CAAC,WAAW,QAAQ,CAAC,MAAM,CAAC,OAAO,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC;IACvD,OAAO,CAAC,GAAG,CAAC,WAAW,QAAQ,CAAC,MAAM,CAAC,OAAO,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC;IACvD,OAAO,CAAC,GAAG,CAAC,WAAW,QAAQ,CAAC,MAAM,CAAC,OAAO,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC;IACvD,OAAO,CAAC,GAAG,CAAC,WAAW,QAAQ,CAAC,MAAM,CAAC,OAAO,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;IAExD,OAAO,CAAC,GAAG,CAAC,uBAAuB,CAAC,CAAC;IACrC,KAAK,MAAM,GAAG,IAAI,MAAM,CAAC,UAAU,EAAE,CAAC;QACpC,MAAM,CAAC,GAAG,GAAG,CAAC,OAAO,CAAC;QACtB,MAAM,EAAE,GAAG,CAAC,CAAC,SAAS,CAAC,EAAE,CAAC;QAC1B,MAAM,EAAE,GAAG,CAAC,CAAC,SAAS,CAAC,EAAE,CAAC;QAC1B,MAAM,MAAM,GAAG,EAAE,GAAG,EAAE,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,EAAE,GAAG,CAAC,EAAE,GAAG,EAAE,CAAC,CAAC;QAClD,OAAO,CAAC,GAAG,CAAC,KAAK,GAAG,CAAC,QAAQ,YAAY,aAAa,CAAC,MAAM,CAAC,QAAQ,EAAE,OAAO,EAAE,GAAG,CAAC,CAAC;IACxF,CAAC;IAED,OAAO,CAAC,GAAG,CAAC,qCAAqC,CAAC,CAAC;IACnD,MAAM,KAAK,GAAG,WAAW,EAAE,QAAQ,IAAI,EAAE,CAAC;IAC1C,KAAK,MAAM,CAAC,IAAI,KAAK,CAAC,KAAK,CAAC,CAAC,EAAE,EAAE,CAAC,EAAE,CAAC;QACnC,OAAO,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC,MAAM,aAAa,CAAC,CAAC,UAAU,OAAO,CAAC,CAAC,OAAO,OAAO,CAAC,CAAC,OAAO,EAAE,CAAC,CAAC;IACxF,CAAC;IAED,OAAO,CAAC,GAAG,CAAC,sBAAsB,UAAU,EAAE,CAAC,CAAC;IAEhD,6DAA6D;IAC7D,MAAM,EAAE,eAAe,EAAE,GAAG,8BAA8B,CAAC,MAAM,EAAE;QACjE,MAAM,EAAE,aAAa;QACrB,cAAc,EAAE,IAAI;QACpB,UAAU,EAAE,iEAAiE;QAC7E,KAAK,EAAE,0FAA0F;KAClG,CAAC,CAAC;IACH,OAAO,CAAC,GAAG,CAAC,gBAAgB,eAAe,EAAE,CAAC,CAAC;IAC/C,OAAO,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC;AACvB,CAAC;AAED,IAAI,EAAE,CAAC,KAAK,CAAC,CAAC,GAAG,EAAE,EAAE;IACnB,OAAO,CAAC,KAAK,CAAC,QAAQ,EAAE,GAAG,CAAC,CAAC;IAC7B,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;AAClB,CAAC,CAAC,CAAC"}
|
|
@@ -17,6 +17,7 @@
|
|
|
17
17
|
import { resolve, join } from 'node:path';
|
|
18
18
|
import { loadPintCorpus, getPintCorpusStats } from './pint-corpus.js';
|
|
19
19
|
import { runEval } from './eval-harness.js';
|
|
20
|
+
import { writeMeasurementFromEvalReport } from '../measurement/from-eval-harness.js';
|
|
20
21
|
// ---------------------------------------------------------------------------
|
|
21
22
|
// Formatting helpers
|
|
22
23
|
// ---------------------------------------------------------------------------
|
|
@@ -147,6 +148,14 @@ async function main() {
|
|
|
147
148
|
}
|
|
148
149
|
}
|
|
149
150
|
console.log(`\nReport saved to: ${outputPath}`);
|
|
151
|
+
// Also write the standardized Measurement file (version-pinned, immutable).
|
|
152
|
+
const { measurementPath } = writeMeasurementFromEvalReport(report, {
|
|
153
|
+
source: 'pint',
|
|
154
|
+
source_version: 'v1',
|
|
155
|
+
source_url: 'https://github.com/lakeraai/pint-benchmark',
|
|
156
|
+
notes: 'Invariant Labs PINT benchmark — 850-sample adversarial prompt-injection corpus.',
|
|
157
|
+
});
|
|
158
|
+
console.log(`Measurement: ${measurementPath}`);
|
|
150
159
|
console.log('Done.\n');
|
|
151
160
|
if (!regression.passed) {
|
|
152
161
|
process.exitCode = 1;
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"run-pint-benchmark.js","sourceRoot":"","sources":["../../src/eval/run-pint-benchmark.ts"],"names":[],"mappings":";AACA;;;;;;;;;;;;;;GAcG;AAEH,OAAO,EAAE,OAAO,EAAE,IAAI,EAAE,MAAM,WAAW,CAAC;AAC1C,OAAO,EAAE,cAAc,EAAE,kBAAkB,EAAE,MAAM,kBAAkB,CAAC;AACtE,OAAO,EAAE,OAAO,EAAE,MAAM,mBAAmB,CAAC;
|
|
1
|
+
{"version":3,"file":"run-pint-benchmark.js","sourceRoot":"","sources":["../../src/eval/run-pint-benchmark.ts"],"names":[],"mappings":";AACA;;;;;;;;;;;;;;GAcG;AAEH,OAAO,EAAE,OAAO,EAAE,IAAI,EAAE,MAAM,WAAW,CAAC;AAC1C,OAAO,EAAE,cAAc,EAAE,kBAAkB,EAAE,MAAM,kBAAkB,CAAC;AACtE,OAAO,EAAE,OAAO,EAAE,MAAM,mBAAmB,CAAC;AAC5C,OAAO,EAAE,8BAA8B,EAAE,MAAM,qCAAqC,CAAC;AAErF,8EAA8E;AAC9E,qBAAqB;AACrB,8EAA8E;AAE9E,SAAS,aAAa,CAAC,CAAS;IAC9B,OAAO,GAAG,CAAC,CAAC,GAAG,GAAG,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,GAAG,CAAC;AACpC,CAAC;AAED,SAAS,QAAQ,CAAC,CAAS;IACzB,OAAO,GAAG,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,IAAI,CAAC;AAC7B,CAAC;AAED,8EAA8E;AAC9E,OAAO;AACP,8EAA8E;AAE9E,KAAK,UAAU,IAAI;IACjB,MAAM,IAAI,GAAG,OAAO,CAAC,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,OAAO,IAAI,GAAG,EAAE,IAAI,EAAE,IAAI,CAAC,CAAC,CAAC;IACnE,MAAM,QAAQ,GAAG,IAAI,CAAC,IAAI,EAAE,OAAO,CAAC,CAAC;IACrC,MAAM,QAAQ,GAAG,IAAI,CAAC,IAAI,EAAE,MAAM,EAAE,gBAAgB,EAAE,kBAAkB,CAAC,CAAC;IAC1E,MAAM,UAAU,GAAG,IAAI,CAAC,IAAI,EAAE,MAAM,EAAE,gBAAgB,EAAE,uBAAuB,CAAC,CAAC;IAEjF,OAAO,CAAC,GAAG,CAAC,8CAA8C,CAAC,CAAC;IAC5D,OAAO,CAAC,GAAG,CAAC,WAAW,QAAQ,EAAE,CAAC,CAAC;IACnC,OAAO,CAAC,GAAG,CAAC,WAAW,QAAQ,IAAI,CAAC,CAAC;IAErC,uBAAuB;IACvB,MAAM,MAAM,GAAG,cAAc,CAAC,QAAQ,CAAC,CAAC;IACxC,MAAM,KAAK,GAAG,kBAAkB,CAAC,MAAM,CAAC,CAAC;IAEzC,OAAO,CAAC,GAAG,CAAC,UAAU,KAAK,CAAC,KAAK,aAAa,KAAK,CAAC,OAAO,aAAa,KAAK,CAAC,MAAM,UAAU,CAAC,CAAC;IAChG,OAAO,CAAC,GAAG,CAAC,eAAe,MAAM,CAAC,OAAO,CAAC,KAAK,CAAC,UAAU,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,EAAE,EAAE,CAAC,GAAG,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;IACvG,OAAO,CAAC,GAAG,CAAC,eAAe,MAAM,CAAC,OAAO,CAAC,KAAK,CAAC,YAAY,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,EAAE,EAAE,CAAC,GAAG,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;IACzG,OAAO,CAAC,GAAG,CAAC,eAAe,MAAM,CAAC,OAAO,CAAC,KAAK,CAAC,UAAU,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,EAAE,EAAE,CAAC,GAAG,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;IAEvG,kEAAkE;IAClE,uEAAuE;IACvE,uEAAuE;IACvE,kEAAkE;IAClE,mFAAmF;IACnF,+EAA+E;IAC/E,MAAM,cAAc,GAAG;QACrB,SAAS,EAAE,IAAI;QACf,SAAS,EAAE,IAAI;QACf,KAAK,EAAE,IAAI;QACX,eAAe,EAAE,GAAG;KACrB,CAAC;IAEF,gCAAgC;IAChC,MAAM,EAAE,MAAM,EAAE,UAAU,EAAE,SAAS,EAAE,WAAW,EAAE,GAAG,MAAM,OAAO,CAAC;QACnE,QAAQ;QACR,MAAM;QACN,UAAU,EAAE,cAAc;QAC1B,UAAU;KACX,CAAC,CAAC;IAEH,kBAAkB;IAClB,OAAO,CAAC,GAAG,CAAC,YAAY,SAAS,CAAC,IAAI,CAAC,KAAK,CAAC,EAAE,CAAC,CAAC;IACjD,OAAO,CAAC,GAAG,CAAC,mBAAmB,CAAC,CAAC;IACjC,OAAO,CAAC,GAAG,CAAC,iBAAiB,aAAa,CAAC,MAAM,CAAC,OAAO,CAAC,SAAS,CAAC,EAAE,CAAC,CAAC;IACxE,OAAO,CAAC,GAAG,CAAC,iBAAiB,aAAa,CAAC,MAAM,CAAC,OAAO,CAAC,MAAM,CAAC,EAAE,CAAC,CAAC;IACrE,OAAO,CAAC,GAAG,CAAC,iBAAiB,aAAa,CAAC,MAAM,CAAC,OAAO,CAAC,EAAE,CAAC,EAAE,CAAC,CAAC;IACjE,OAAO,CAAC,GAAG,CAAC,iBAAiB,aAAa,CAAC,MAAM,CAAC,OAAO,CAAC,QAAQ,CAAC,EAAE,CAAC,CAAC;IACvE,OAAO,CAAC,GAAG,CAAC,iBAAiB,aAAa,CAAC,MAAM,CAAC,OAAO,CAAC,MAAM,CAAC,EAAE,CAAC,CAAC;IACrE,OAAO,CAAC,GAAG,CAAC,oBAAoB,MAAM,CAAC,OAAO,CAAC,SAAS,CAAC,EAAE,OAAO,MAAM,CAAC,OAAO,CAAC,SAAS,CAAC,EAAE,OAAO,MAAM,CAAC,OAAO,CAAC,SAAS,CAAC,EAAE,OAAO,MAAM,CAAC,OAAO,CAAC,SAAS,CAAC,EAAE,EAAE,CAAC,CAAC;IAErK,UAAU;IACV,OAAO,CAAC,GAAG,CAAC,mBAAmB,CAAC,CAAC;IACjC,OAAO,CAAC,GAAG,CAAC,WAAW,QAAQ,CAAC,MAAM,CAAC,OAAO,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC;IACvD,OAAO,CAAC,GAAG,CAAC,WAAW,QAAQ,CAAC,MAAM,CAAC,OAAO,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC;IACvD,OAAO,CAAC,GAAG,CAAC,WAAW,QAAQ,CAAC,MAAM,CAAC,OAAO,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC;IACvD,OAAO,CAAC,GAAG,CAAC,WAAW,QAAQ,CAAC,MAAM,CAAC,OAAO,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;IACxD,OAAO,CAAC,GAAG,CAAC,WAAW,QAAQ,CAAC,MAAM,CAAC,OAAO,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC;IAEvD,eAAe;IACf,OAAO,CAAC,GAAG,CAAC,uBAAuB,CAAC,CAAC;IACrC,KAAK,MAAM,GAAG,IAAI,MAAM,CAAC,UAAU,EAAE,CAAC;QACpC,MAAM,MAAM,GAAG,GAAG,CAAC,aAAa,CAAC,MAAM,CAAC;QACxC,MAAM,GAAG,GAAG,GAAG,CAAC,cAAc,CAAC,MAAM,CAAC;QACtC,OAAO,CAAC,GAAG,CACT,KAAK,GAAG,CAAC,QAAQ,YAAY,aAAa,CAAC,GAAG,CAAC,OAAO,CAAC,MAAM,CAAC,GAAG;YACjE,aAAa,aAAa,CAAC,GAAG,CAAC,OAAO,CAAC,SAAS,CAAC,GAAG;YACpD,MAAM,aAAa,CAAC,GAAG,CAAC,OAAO,CAAC,EAAE,CAAC,GAAG;YACtC,WAAW,MAAM,QAAQ,GAAG,GAAG,CAChC,CAAC;IACJ,CAAC;IAED,iBAAiB;IACjB,OAAO,CAAC,GAAG,CAAC,yBAAyB,CAAC,CAAC;IACvC,KAAK,MAAM,IAAI,IAAI,MAAM,CAAC,YAAY,EAAE,CAAC;QACvC,OAAO,CAAC,GAAG,CACT,KAAK,IAAI,CAAC,UAAU,YAAY,aAAa,CAAC,IAAI,CAAC,OAAO,CAAC,MAAM,CAAC,GAAG;YACrE,aAAa,aAAa,CAAC,IAAI,CAAC,OAAO,CAAC,SAAS,CAAC,GAAG;YACrD,MAAM,aAAa,CAAC,IAAI,CAAC,OAAO,CAAC,EAAE,CAAC,EAAE,CACvC,CAAC;IACJ,CAAC;IAED,iCAAiC;IACjC,IAAI,MAAM,CAAC,aAAa,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QACpC,MAAM,SAAS,GAAG,IAAI,CAAC,GAAG,CAAC,MAAM,CAAC,aAAa,CAAC,MAAM,EAAE,EAAE,CAAC,CAAC;QAC5D,OAAO,CAAC,GAAG,CAAC,yBAAyB,MAAM,CAAC,aAAa,CAAC,MAAM,mBAAmB,SAAS,OAAO,CAAC,CAAC;QACrG,KAAK,MAAM,CAAC,IAAI,MAAM,CAAC,aAAa,CAAC,KAAK,CAAC,CAAC,EAAE,SAAS,CAAC,EAAE,CAAC;YACzD,MAAM,IAAI,GAAG,MAAM,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,EAAE,KAAK,CAAC,CAAC,EAAE,CAAC,EAAE,MAAM,EAAE,CAAC,UAAU,CAAC,IAAI,GAAG,CAAC;YAC5E,OAAO,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC,EAAE,UAAU,IAAI,IAAI,CAAC,CAAC,QAAQ,IAAI,CAAC,CAAC,UAAU,EAAE,CAAC,CAAC;QACxE,CAAC;QACD,IAAI,MAAM,CAAC,aAAa,CAAC,MAAM,GAAG,SAAS,EAAE,CAAC;YAC5C,OAAO,CAAC,GAAG,CAAC,aAAa,MAAM,CAAC,aAAa,CAAC,MAAM,GAAG,SAAS,OAAO,CAAC,CAAC;QAC3E,CAAC;IACH,CAAC;IAED,kCAAkC;IAClC,IAAI,MAAM,CAAC,cAAc,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QACrC,MAAM,SAAS,GAAG,IAAI,CAAC,GAAG,CAAC,MAAM,CAAC,cAAc,CAAC,MAAM,EAAE,EAAE,CAAC,CAAC;QAC7D,OAAO,CAAC,GAAG,CAAC,0BAA0B,MAAM,CAAC,cAAc,CAAC,MAAM,mBAAmB,SAAS,OAAO,CAAC,CAAC;QACvG,KAAK,MAAM,EAAE,IAAI,MAAM,CAAC,cAAc,CAAC,KAAK,CAAC,CAAC,EAAE,SAAS,CAAC,EAAE,CAAC;YAC3D,OAAO,CAAC,GAAG,CAAC,MAAM,EAAE,CAAC,EAAE,YAAY,EAAE,CAAC,YAAY,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;QACnE,CAAC;QACD,IAAI,MAAM,CAAC,cAAc,CAAC,MAAM,GAAG,SAAS,EAAE,CAAC;YAC7C,OAAO,CAAC,GAAG,CAAC,aAAa,MAAM,CAAC,cAAc,CAAC,MAAM,GAAG,SAAS,OAAO,CAAC,CAAC;QAC5E,CAAC;IACH,CAAC;IAED,uBAAuB;IACvB,OAAO,CAAC,GAAG,CAAC,wBAAwB,CAAC,CAAC;IACtC,OAAO,CAAC,GAAG,CAAC,yBAAyB,WAAW,CAAC,mBAAmB,EAAE,CAAC,CAAC;IACxE,OAAO,CAAC,GAAG,CAAC,kBAAkB,WAAW,CAAC,UAAU,EAAE,CAAC,CAAC;IACxD,OAAO,CAAC,GAAG,CAAC,wBAAwB,WAAW,CAAC,eAAe,EAAE,CAAC,CAAC;IAEnE,IAAI,WAAW,CAAC,QAAQ,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QACpC,OAAO,CAAC,GAAG,CAAC,kCAAkC,CAAC,CAAC;QAChD,KAAK,MAAM,IAAI,IAAI,WAAW,CAAC,QAAQ,CAAC,KAAK,CAAC,CAAC,EAAE,EAAE,CAAC,EAAE,CAAC;YACrD,MAAM,SAAS,GAAG,IAAI,CAAC,UAAU,GAAG,CAAC;gBACnC,CAAC,CAAC,aAAa,CAAC,IAAI,CAAC,OAAO,GAAG,IAAI,CAAC,UAAU,CAAC;gBAC/C,CAAC,CAAC,KAAK,CAAC;YACV,OAAO,CAAC,GAAG,CACT,OAAO,IAAI,CAAC,MAAM,aAAa,IAAI,CAAC,UAAU,GAAG;gBACjD,MAAM,IAAI,CAAC,OAAO,OAAO,IAAI,CAAC,OAAO,GAAG;gBACxC,aAAa,SAAS,EAAE,CACzB,CAAC;QACJ,CAAC;IACH,CAAC;IAED,aAAa;IACb,OAAO,CAAC,GAAG,CAAC,8CAA8C,CAAC,CAAC;IAC5D,IAAI,UAAU,CAAC,MAAM,EAAE,CAAC;QACtB,OAAO,CAAC,GAAG,CAAC,UAAU,CAAC,CAAC;IAC1B,CAAC;SAAM,CAAC;QACN,OAAO,CAAC,GAAG,CAAC,WAAW,CAAC,CAAC;QACzB,KAAK,MAAM,CAAC,IAAI,UAAU,CAAC,UAAU,EAAE,CAAC;YACtC,OAAO,CAAC,GAAG,CAAC,SAAS,CAAC,EAAE,CAAC,CAAC;QAC5B,CAAC;IACH,CAAC;IAED,OAAO,CAAC,GAAG,CAAC,sBAAsB,UAAU,EAAE,CAAC,CAAC;IAEhD,4EAA4E;IAC5E,MAAM,EAAE,eAAe,EAAE,GAAG,8BAA8B,CAAC,MAAM,EAAE;QACjE,MAAM,EAAE,MAAM;QACd,cAAc,EAAE,IAAI;QACpB,UAAU,EAAE,4CAA4C;QACxD,KAAK,EAAE,iFAAiF;KACzF,CAAC,CAAC;IACH,OAAO,CAAC,GAAG,CAAC,gBAAgB,eAAe,EAAE,CAAC,CAAC;IAC/C,OAAO,CAAC,GAAG,CAAC,SAAS,CAAC,CAAC;IAEvB,IAAI,CAAC,UAAU,CAAC,MAAM,EAAE,CAAC;QACvB,OAAO,CAAC,QAAQ,GAAG,CAAC,CAAC;IACvB,CAAC;AACH,CAAC;AAED,IAAI,EAAE,CAAC,KAAK,CAAC,CAAC,GAAG,EAAE,EAAE;IACnB,OAAO,CAAC,KAAK,CAAC,wBAAwB,EAAE,GAAG,CAAC,CAAC;IAC7C,OAAO,CAAC,QAAQ,GAAG,CAAC,CAAC;AACvB,CAAC,CAAC,CAAC"}
|
|
@@ -61,6 +61,17 @@ export declare function runSkillBenchmark(options?: {
|
|
|
61
61
|
readonly corpusDir?: string;
|
|
62
62
|
readonly outputPath?: string;
|
|
63
63
|
}): Promise<SkillBenchmarkReport>;
|
|
64
|
+
/**
|
|
65
|
+
* Write the standardized version-pinned Measurement file for a SkillBenchmark
|
|
66
|
+
* report. Separated from runSkillBenchmark so unit tests can exercise the
|
|
67
|
+
* benchmark logic without mutating data/measurements/ on disk (which would
|
|
68
|
+
* make the CI 'sync-stats --check' drift gate flake).
|
|
69
|
+
*
|
|
70
|
+
* Called from the CLI block at the bottom of this file and from any
|
|
71
|
+
* external script that wants to persist the measurement. Safe to call
|
|
72
|
+
* repeatedly the same day — uses force=true.
|
|
73
|
+
*/
|
|
74
|
+
export declare function writeSkillBenchmarkMeasurement(report: SkillBenchmarkReport): void;
|
|
64
75
|
export declare function printReport(report: SkillBenchmarkReport): void;
|
|
65
76
|
export {};
|
|
66
77
|
//# sourceMappingURL=skill-benchmark.d.ts.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"skill-benchmark.d.ts","sourceRoot":"","sources":["../../src/eval/skill-benchmark.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;GAgBG;
|
|
1
|
+
{"version":3,"file":"skill-benchmark.d.ts","sourceRoot":"","sources":["../../src/eval/skill-benchmark.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;GAgBG;AA+BH,UAAU,YAAY;IACpB,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAC;IACtB,QAAQ,CAAC,KAAK,EAAE,WAAW,GAAG,QAAQ,CAAC;IACvC,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAC;IACvB,QAAQ,CAAC,WAAW,EAAE,MAAM,CAAC;IAC7B,QAAQ,CAAC,QAAQ,EAAE,OAAO,CAAC;IAC3B,QAAQ,CAAC,WAAW,EAAE,SAAS,MAAM,EAAE,CAAC;IACxC,QAAQ,CAAC,OAAO,EAAE,OAAO,CAAC;IAC1B,QAAQ,CAAC,UAAU,EAAE,MAAM,CAAC;IAC5B,QAAQ,CAAC,sBAAsB,EAAE,OAAO,CAAC;IACzC,QAAQ,CAAC,gBAAgB,EAAE,OAAO,CAAC;CACpC;AAED,UAAU,YAAY;IACpB,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAC;IACvB,QAAQ,CAAC,QAAQ,EAAE,MAAM,CAAC;IAC1B,QAAQ,CAAC,MAAM,EAAE,MAAM,CAAC;CACzB;AAED,UAAU,oBAAoB;IAC5B,QAAQ,CAAC,SAAS,EAAE,MAAM,CAAC;IAC3B,QAAQ,CAAC,WAAW,EAAE,MAAM,CAAC;IAC7B,QAAQ,CAAC,eAAe,EAAE,MAAM,CAAC;IACjC,QAAQ,CAAC,YAAY,EAAE,MAAM,CAAC;IAC9B,QAAQ,CAAC,cAAc,EAAE,MAAM,CAAC;IAChC,QAAQ,CAAC,iBAAiB,EAAE,MAAM,CAAC;IACnC,QAAQ,CAAC,UAAU,EAAE,MAAM,CAAC;IAC5B,QAAQ,CAAC,OAAO,EAAE,MAAM,CAAC;IACzB,QAAQ,CAAC,OAAO,EAAE,YAAY,CAAC;IAC/B,QAAQ,CAAC,OAAO,EAAE,YAAY,CAAC;IAC/B,QAAQ,CAAC,OAAO,EAAE,YAAY,CAAC;IAC/B,QAAQ,CAAC,cAAc,EAAE,MAAM,CAAC;IAChC,QAAQ,CAAC,eAAe,EAAE,MAAM,CAAC;IACjC,QAAQ,CAAC,cAAc,EAAE,MAAM,CAAC;IAChC,QAAQ,CAAC,eAAe,EAAE,MAAM,CAAC;IACjC,QAAQ,CAAC,uBAAuB,EAAE,MAAM,CAAC;IACzC,QAAQ,CAAC,iBAAiB,EAAE,MAAM,CAAC;IACnC,QAAQ,CAAC,cAAc,EAAE,MAAM,CAAC;IAChC,QAAQ,CAAC,cAAc,EAAE,MAAM,CAAC;IAChC,QAAQ,CAAC,OAAO,EAAE,SAAS,YAAY,EAAE,CAAC;IAC1C,QAAQ,CAAC,cAAc,EAAE,SAAS,YAAY,EAAE,CAAC;IACjD,QAAQ,CAAC,YAAY,EAAE,SAAS,YAAY,EAAE,CAAC;CAChD;AAMD,wBAAsB,iBAAiB,CAAC,OAAO,CAAC,EAAE;IAChD,QAAQ,CAAC,QAAQ,CAAC,EAAE,MAAM,CAAC;IAC3B,QAAQ,CAAC,SAAS,CAAC,EAAE,MAAM,CAAC;IAC5B,QAAQ,CAAC,UAAU,CAAC,EAAE,MAAM,CAAC;CAC9B,GAAG,OAAO,CAAC,oBAAoB,CAAC,CAuIhC;AAMD;;;;;;;;;GASG;AACH,wBAAgB,8BAA8B,CAAC,MAAM,EAAE,oBAAoB,GAAG,IAAI,CA0CjF;AAUD,wBAAgB,WAAW,CAAC,MAAM,EAAE,oBAAoB,GAAG,IAAI,CA6C9D"}
|
|
@@ -18,6 +18,7 @@
|
|
|
18
18
|
import { resolve, join } from 'node:path';
|
|
19
19
|
import { readFileSync, writeFileSync, existsSync } from 'node:fs';
|
|
20
20
|
import { ATREngine } from '../engine.js';
|
|
21
|
+
import { writeMeasurement } from '../measurement/write.js';
|
|
21
22
|
// ---------------------------------------------------------------------------
|
|
22
23
|
// Benchmark runner
|
|
23
24
|
// ---------------------------------------------------------------------------
|
|
@@ -136,6 +137,58 @@ export async function runSkillBenchmark(options) {
|
|
|
136
137
|
return report;
|
|
137
138
|
}
|
|
138
139
|
// ---------------------------------------------------------------------------
|
|
140
|
+
// Measurement adapter (CLI-only — tests call runSkillBenchmark and skip this)
|
|
141
|
+
// ---------------------------------------------------------------------------
|
|
142
|
+
/**
|
|
143
|
+
* Write the standardized version-pinned Measurement file for a SkillBenchmark
|
|
144
|
+
* report. Separated from runSkillBenchmark so unit tests can exercise the
|
|
145
|
+
* benchmark logic without mutating data/measurements/ on disk (which would
|
|
146
|
+
* make the CI 'sync-stats --check' drift gate flake).
|
|
147
|
+
*
|
|
148
|
+
* Called from the CLI block at the bottom of this file and from any
|
|
149
|
+
* external script that wants to persist the measurement. Safe to call
|
|
150
|
+
* repeatedly the same day — uses force=true.
|
|
151
|
+
*/
|
|
152
|
+
export function writeSkillBenchmarkMeasurement(report) {
|
|
153
|
+
writeMeasurement({
|
|
154
|
+
source: 'skill-benchmark',
|
|
155
|
+
source_version: 'internal-498',
|
|
156
|
+
measured_at: report.timestamp,
|
|
157
|
+
samples: report.corpus_size,
|
|
158
|
+
metrics: {
|
|
159
|
+
recall: report.overall_recall,
|
|
160
|
+
precision: report.overall_precision,
|
|
161
|
+
f1: report.overall_f1,
|
|
162
|
+
fp_rate: report.fp_rate,
|
|
163
|
+
},
|
|
164
|
+
confusion: {
|
|
165
|
+
tp: report.true_positives,
|
|
166
|
+
fp: report.false_positives,
|
|
167
|
+
tn: report.true_negatives,
|
|
168
|
+
fn: report.false_negatives,
|
|
169
|
+
},
|
|
170
|
+
latency_ms: {
|
|
171
|
+
p50: report.avg_latency_ms,
|
|
172
|
+
p95: report.max_latency_ms,
|
|
173
|
+
p99: report.max_latency_ms,
|
|
174
|
+
mean: report.avg_latency_ms,
|
|
175
|
+
max: report.max_latency_ms,
|
|
176
|
+
},
|
|
177
|
+
breakdown: {
|
|
178
|
+
layers: {
|
|
179
|
+
a: report.layer_a,
|
|
180
|
+
b: report.layer_b,
|
|
181
|
+
c: report.layer_c,
|
|
182
|
+
},
|
|
183
|
+
malicious_count: report.malicious_count,
|
|
184
|
+
benign_count: report.benign_count,
|
|
185
|
+
expected_rules_accuracy: report.expected_rules_accuracy,
|
|
186
|
+
category_accuracy: report.category_accuracy,
|
|
187
|
+
},
|
|
188
|
+
notes: 'Internal 498-sample SKILL.md benchmark. Layer A = obvious payload, Layer B = obfuscated, Layer C = semantic.',
|
|
189
|
+
}, { force: true });
|
|
190
|
+
}
|
|
191
|
+
// ---------------------------------------------------------------------------
|
|
139
192
|
// CLI runner
|
|
140
193
|
// ---------------------------------------------------------------------------
|
|
141
194
|
function formatPercent(n) {
|
|
@@ -186,6 +239,10 @@ if (import.meta.url === `file://${process.argv[1]}` || process.argv[1]?.endsWith
|
|
|
186
239
|
runSkillBenchmark().then((report) => {
|
|
187
240
|
printReport(report);
|
|
188
241
|
console.log(`Report saved to: data/skill-benchmark/benchmark-report.json`);
|
|
242
|
+
// Write the standardized version-pinned Measurement file. CLI-only —
|
|
243
|
+
// unit tests call runSkillBenchmark() directly and skip this.
|
|
244
|
+
writeSkillBenchmarkMeasurement(report);
|
|
245
|
+
console.log(`Measurement: data/measurements/skill-benchmark/`);
|
|
189
246
|
}).catch((err) => {
|
|
190
247
|
console.error('Benchmark failed:', err);
|
|
191
248
|
process.exit(1);
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"skill-benchmark.js","sourceRoot":"","sources":["../../src/eval/skill-benchmark.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;GAgBG;AAEH,OAAO,EAAE,OAAO,EAAE,IAAI,EAAE,MAAM,WAAW,CAAC;AAC1C,OAAO,EAAE,YAAY,EAAE,aAAa,EAAE,UAAU,EAAE,MAAM,SAAS,CAAC;AAClE,OAAO,EAAE,SAAS,EAAE,MAAM,cAAc,CAAC;AAsEzC,8EAA8E;AAC9E,mBAAmB;AACnB,8EAA8E;AAE9E,MAAM,CAAC,KAAK,UAAU,iBAAiB,CAAC,OAIvC;IACC,MAAM,QAAQ,GAAG,OAAO,CAAC,MAAM,CAAC,IAAI,CAAC,OAAO,EAAE,IAAI,EAAE,IAAI,CAAC,CAAC;IAC1D,MAAM,QAAQ,GAAG,OAAO,EAAE,QAAQ,IAAI,IAAI,CAAC,QAAQ,EAAE,OAAO,CAAC,CAAC;IAC9D,MAAM,SAAS,GAAG,OAAO,EAAE,SAAS,IAAI,IAAI,CAAC,QAAQ,EAAE,MAAM,EAAE,iBAAiB,CAAC,CAAC;IAClF,MAAM,UAAU,GAAG,OAAO,EAAE,UAAU,IAAI,IAAI,CAAC,QAAQ,EAAE,MAAM,EAAE,iBAAiB,EAAE,uBAAuB,CAAC,CAAC;IAE7G,gBAAgB;IAChB,MAAM,YAAY,GAAG,IAAI,CAAC,SAAS,EAAE,eAAe,CAAC,CAAC;IACtD,IAAI,CAAC,UAAU,CAAC,YAAY,CAAC,EAAE,CAAC;QAC9B,MAAM,IAAI,KAAK,CAAC,uBAAuB,YAAY,EAAE,CAAC,CAAC;IACzD,CAAC;IACD,MAAM,QAAQ,GAA6B,IAAI,CAAC,KAAK,CAAC,YAAY,CAAC,YAAY,EAAE,OAAO,CAAC,CAAC,CAAC;IAE3F,oDAAoD;IACpD,MAAM,YAAY,GAAG,IAAI,CAAC,SAAS,EAAE,wBAAwB,CAAC,CAAC;IAC/D,MAAM,WAAW,GAAkC,UAAU,CAAC,YAAY,CAAC;QACzE,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,YAAY,CAAC,YAAY,EAAE,OAAO,CAAC,CAAC;QACjD,CAAC,CAAC,EAAE,CAAC;IAEP,cAAc;IACd,MAAM,MAAM,GAAG,IAAI,SAAS,CAAC,EAAE,QAAQ,EAAE,CAAC,CAAC;IAC3C,MAAM,MAAM,CAAC,SAAS,EAAE,CAAC;IAEzB,kBAAkB;IAClB,MAAM,OAAO,GAAmB,EAAE,CAAC;IAEnC,KAAK,MAAM,KAAK,IAAI,QAAQ,EAAE,CAAC;QAC7B,MAAM,QAAQ,GAAG,IAAI,CAAC,SAAS,EAAE,KAAK,CAAC,IAAI,CAAC,CAAC;QAC7C,IAAI,CAAC,UAAU,CAAC,QAAQ,CAAC,EAAE,CAAC;YAC1B,OAAO,CAAC,KAAK,CAAC,yBAAyB,KAAK,CAAC,IAAI,EAAE,CAAC,CAAC;YACrD,SAAS;QACX,CAAC;QAED,MAAM,OAAO,GAAG,YAAY,CAAC,QAAQ,EAAE,OAAO,CAAC,CAAC;QAChD,MAAM,KAAK,GAAG,WAAW,CAAC,GAAG,EAAE,CAAC;QAChC,MAAM,OAAO,GAAG,MAAM,CAAC,SAAS,CAAC,OAAO,CAAC,CAAC;QAC1C,MAAM,OAAO,GAAG,WAAW,CAAC,GAAG,EAAE,GAAG,KAAK,CAAC;QAE1C,MAAM,QAAQ,GAAG,OAAO,CAAC,MAAM,GAAG,CAAC,CAAC;QACpC,MAAM,UAAU,GAAG,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;QAEjD,MAAM,OAAO,GACX,CAAC,KAAK,CAAC,KAAK,KAAK,WAAW,IAAI,QAAQ,CAAC;YACzC,CAAC,KAAK,CAAC,KAAK,KAAK,QAAQ,IAAI,CAAC,QAAQ,CAAC,CAAC;QAE1C,oEAAoE;QACpE,MAAM,QAAQ,GAAG,WAAW,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;QACzC,MAAM,oBAAoB,GAAG,QAAQ,EAAE,iBAAiB;YACtD,CAAC,CAAC,QAAQ,CAAC,iBAAiB,CAAC,KAAK,CAAC,CAAC,EAAE,EAAE,EAAE,CAAC,UAAU,CAAC,QAAQ,CAAC,EAAE,CAAC,OAAO,CAAC,CAAC;YAC3E,CAAC,CAAC,IAAI,CAAC,CAAC,gCAAgC;QAC1C,MAAM,kBAAkB,GAAG,QAAQ,EAAE,iBAAiB,EAAE,GAAG,CAAC,CAAC,EAAE,EAAE,EAAE,CAAC,EAAE,CAAC,QAAQ,CAAC,IAAI,EAAE,CAAC;QACvF,MAAM,gBAAgB,GAAG,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;QAClE,MAAM,eAAe,GAAG,kBAAkB,CAAC,KAAK,CAAC,CAAC,EAAE,EAAE,EAAE,CACtD,gBAAgB,CAAC,QAAQ,CAAC,EAAqC,CAAC,CACjE,CAAC;QAEF,OAAO,CAAC,IAAI,CAAC;YACX,IAAI,EAAE,KAAK,CAAC,IAAI;YAChB,KAAK,EAAE,KAAK,CAAC,KAAK;YAClB,KAAK,EAAE,KAAK,CAAC,KAAK,IAAI,EAAE;YACxB,WAAW,EAAE,KAAK,CAAC,WAAW;YAC9B,QAAQ;YACR,WAAW,EAAE,UAAU;YACvB,OAAO;YACP,UAAU,EAAE,IAAI,CAAC,KAAK,CAAC,OAAO,GAAG,GAAG,CAAC,GAAG,GAAG;YAC3C,sBAAsB,EAAE,oBAAoB;YAC5C,gBAAgB,EAAE,eAAe;SAClC,CAAC,CAAC;IACL,CAAC;IAED,kBAAkB;IAClB,MAAM,SAAS,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,KAAK,KAAK,WAAW,CAAC,CAAC;IACjE,MAAM,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,KAAK,KAAK,QAAQ,CAAC,CAAC;IAE3D,MAAM,EAAE,GAAG,SAAS,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC,MAAM,CAAC;IACtD,MAAM,EAAE,GAAG,SAAS,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC,MAAM,CAAC;IACvD,MAAM,EAAE,GAAG,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC,MAAM,CAAC;IACnD,MAAM,EAAE,GAAG,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC,MAAM,CAAC;IAEpD,MAAM,MAAM,GAAG,SAAS,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,EAAE,GAAG,SAAS,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,CAAC;IAChE,MAAM,SAAS,GAAG,CAAC,EAAE,GAAG,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,EAAE,GAAG,CAAC,EAAE,GAAG,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;IACrD,MAAM,EAAE,GAAG,CAAC,SAAS,GAAG,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,SAAS,GAAG,MAAM,CAAC,GAAG,CAAC,SAAS,GAAG,MAAM,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;IAC1F,MAAM,MAAM,GAAG,MAAM,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,EAAE,GAAG,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,CAAC;IAE1D,oBAAoB;IACpB,MAAM,YAAY,GAAG,CAAC,KAAa,EAAgB,EAAE;QACnD,MAAM,OAAO,GAAG,SAAS,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,KAAK,KAAK,KAAK,CAAC,CAAC;QAC3D,MAAM,QAAQ,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC,MAAM,CAAC;QAC1D,OAAO;YACL,KAAK,EAAE,OAAO,CAAC,MAAM;YACrB,QAAQ;YACR,MAAM,EAAE,OAAO,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,QAAQ,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC;SAC3D,CAAC;IACJ,CAAC,CAAC;IAEF,MAAM,SAAS,GAAG,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,UAAU,CAAC,CAAC;IAEnD,6BAA6B;IAC7B,MAAM,iBAAiB,GAAG,SAAS,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC;IAC9D,MAAM,qBAAqB,GAAG,iBAAiB,CAAC,MAAM,GAAG,CAAC;QACxD,CAAC,CAAC,iBAAiB,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,sBAAsB,CAAC,CAAC,MAAM,GAAG,iBAAiB,CAAC,MAAM;QAC7F,CAAC,CAAC,CAAC,CAAC;IACN,MAAM,gBAAgB,GAAG,iBAAiB,CAAC,MAAM,GAAG,CAAC;QACnD,CAAC,CAAC,iBAAiB,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,gBAAgB,CAAC,CAAC,MAAM,GAAG,iBAAiB,CAAC,MAAM;QACvF,CAAC,CAAC,CAAC,CAAC;IAEN,MAAM,MAAM,GAAyB;QACnC,SAAS,EAAE,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE;QACnC,WAAW,EAAE,OAAO,CAAC,MAAM;QAC3B,eAAe,EAAE,SAAS,CAAC,MAAM;QACjC,YAAY,EAAE,MAAM,CAAC,MAAM;QAC3B,cAAc,EAAE,IAAI,CAAC,KAAK,CAAC,MAAM,GAAG,IAAI,CAAC,GAAG,IAAI;QAChD,iBAAiB,EAAE,IAAI,CAAC,KAAK,CAAC,SAAS,GAAG,IAAI,CAAC,GAAG,IAAI;QACtD,UAAU,EAAE,IAAI,CAAC,KAAK,CAAC,EAAE,GAAG,IAAI,CAAC,GAAG,IAAI;QACxC,OAAO,EAAE,IAAI,CAAC,KAAK,CAAC,MAAM,GAAG,IAAI,CAAC,GAAG,IAAI;QACzC,OAAO,EAAE,YAAY,CAAC,GAAG,CAAC;QAC1B,OAAO,EAAE,YAAY,CAAC,GAAG,CAAC;QAC1B,OAAO,EAAE,YAAY,CAAC,GAAG,CAAC;QAC1B,cAAc,EAAE,EAAE;QAClB,eAAe,EAAE,EAAE;QACnB,cAAc,EAAE,EAAE;QAClB,eAAe,EAAE,EAAE;QACnB,uBAAuB,EAAE,IAAI,CAAC,KAAK,CAAC,qBAAqB,GAAG,IAAI,CAAC,GAAG,IAAI;QACxE,iBAAiB,EAAE,IAAI,CAAC,KAAK,CAAC,gBAAgB,GAAG,IAAI,CAAC,GAAG,IAAI;QAC7D,cAAc,EAAE,IAAI,CAAC,KAAK,CAAC,CAAC,SAAS,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC,GAAG,SAAS,CAAC,MAAM,CAAC,GAAG,GAAG,CAAC,GAAG,GAAG;QACjG,cAAc,EAAE,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,GAAG,CAAC,GAAG,SAAS,CAAC,GAAG,GAAG,CAAC,GAAG,GAAG;QAC9D,OAAO;QACP,cAAc,EAAE,SAAS,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,QAAQ,CAAC;QACpD,YAAY,EAAE,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,QAAQ,CAAC;KAC/C,CAAC;IAEF,cAAc;IACd,aAAa,CAAC,UAAU,EAAE,IAAI,CAAC,SAAS,CAAC,MAAM,EAAE,IAAI,EAAE,CAAC,CAAC,CAAC,CAAC;IAE3D,OAAO,MAAM,CAAC;AAChB,CAAC;AAED,8EAA8E;AAC9E,aAAa;AACb,8EAA8E;AAE9E,SAAS,aAAa,CAAC,CAAS;IAC9B,OAAO,GAAG,CAAC,CAAC,GAAG,GAAG,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,GAAG,CAAC;AACpC,CAAC;AAED,MAAM,UAAU,WAAW,CAAC,MAA4B;IACtD,OAAO,CAAC,GAAG,CAAC,wDAAwD,CAAC,CAAC;IACtE,OAAO,CAAC,GAAG,CAAC,qDAAqD,CAAC,CAAC;IACnE,OAAO,CAAC,GAAG,CAAC,wDAAwD,CAAC,CAAC;IAEtE,OAAO,CAAC,GAAG,CAAC,WAAW,MAAM,CAAC,WAAW,aAAa,MAAM,CAAC,eAAe,eAAe,MAAM,CAAC,YAAY,UAAU,CAAC,CAAC;IAC1H,OAAO,CAAC,GAAG,CAAC,cAAc,MAAM,CAAC,SAAS,IAAI,CAAC,CAAC;IAEhD,OAAO,CAAC,GAAG,CAAC,gCAAgC,CAAC,CAAC;IAC9C,OAAO,CAAC,GAAG,CAAC,uBAAuB,aAAa,CAAC,MAAM,CAAC,cAAc,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC;IACzF,OAAO,CAAC,GAAG,CAAC,uBAAuB,aAAa,CAAC,MAAM,CAAC,iBAAiB,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC;IAC5F,OAAO,CAAC,GAAG,CAAC,uBAAuB,aAAa,CAAC,MAAM,CAAC,UAAU,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC;IACrF,OAAO,CAAC,GAAG,CAAC,uBAAuB,aAAa,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC;IAClF,OAAO,CAAC,GAAG,CAAC,kCAAkC,CAAC,CAAC;IAEhD,OAAO,CAAC,GAAG,CAAC,mBAAmB,CAAC,CAAC;IACjC,OAAO,CAAC,GAAG,CAAC,2BAA2B,aAAa,CAAC,MAAM,CAAC,OAAO,CAAC,MAAM,CAAC,KAAK,MAAM,CAAC,OAAO,CAAC,QAAQ,IAAI,MAAM,CAAC,OAAO,CAAC,KAAK,GAAG,CAAC,CAAC;IACpI,OAAO,CAAC,GAAG,CAAC,2BAA2B,aAAa,CAAC,MAAM,CAAC,OAAO,CAAC,MAAM,CAAC,KAAK,MAAM,CAAC,OAAO,CAAC,QAAQ,IAAI,MAAM,CAAC,OAAO,CAAC,KAAK,GAAG,CAAC,CAAC;IACpI,OAAO,CAAC,GAAG,CAAC,2BAA2B,aAAa,CAAC,MAAM,CAAC,OAAO,CAAC,MAAM,CAAC,KAAK,MAAM,CAAC,OAAO,CAAC,QAAQ,IAAI,MAAM,CAAC,OAAO,CAAC,KAAK,KAAK,CAAC,CAAC;IAEtI,OAAO,CAAC,GAAG,CAAC,iCAAiC,CAAC,CAAC;IAC/C,OAAO,CAAC,GAAG,CAAC,6BAA6B,aAAa,CAAC,MAAM,CAAC,uBAAuB,CAAC,EAAE,CAAC,CAAC;IAC1F,OAAO,CAAC,GAAG,CAAC,6BAA6B,aAAa,CAAC,MAAM,CAAC,iBAAiB,CAAC,IAAI,CAAC,CAAC;IAEtF,OAAO,CAAC,GAAG,CAAC,mBAAmB,CAAC,CAAC;IACjC,OAAO,CAAC,GAAG,CAAC,SAAS,MAAM,CAAC,cAAc,SAAS,MAAM,CAAC,eAAe,EAAE,CAAC,CAAC;IAC7E,OAAO,CAAC,GAAG,CAAC,SAAS,MAAM,CAAC,eAAe,SAAS,MAAM,CAAC,cAAc,IAAI,CAAC,CAAC;IAE/E,OAAO,CAAC,GAAG,CAAC,gBAAgB,MAAM,CAAC,cAAc,WAAW,MAAM,CAAC,cAAc,MAAM,CAAC,CAAC;IAEzF,IAAI,MAAM,CAAC,cAAc,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QACrC,OAAO,CAAC,GAAG,CAAC,iBAAiB,CAAC,CAAC;QAC/B,KAAK,MAAM,CAAC,IAAI,MAAM,CAAC,cAAc,EAAE,CAAC;YACtC,OAAO,CAAC,GAAG,CAAC,QAAQ,CAAC,CAAC,KAAK,KAAK,CAAC,CAAC,IAAI,KAAK,CAAC,CAAC,WAAW,GAAG,CAAC,CAAC;QAC/D,CAAC;QACD,OAAO,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC;IAClB,CAAC;IAED,IAAI,MAAM,CAAC,YAAY,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QACnC,OAAO,CAAC,GAAG,CAAC,kBAAkB,CAAC,CAAC;QAChC,KAAK,MAAM,CAAC,IAAI,MAAM,CAAC,YAAY,EAAE,CAAC;YACpC,OAAO,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,IAAI,MAAM,CAAC,CAAC,WAAW,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;QAC7D,CAAC;QACD,OAAO,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC;IAClB,CAAC;AACH,CAAC;AAED,8EAA8E;AAC9E,mBAAmB;AACnB,8EAA8E;AAE9E,IAAI,MAAM,CAAC,IAAI,CAAC,GAAG,KAAK,UAAU,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,IAAI,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,QAAQ,CAAC,oBAAoB,CAAC,EAAE,CAAC;IACvG,iBAAiB,EAAE,CAAC,IAAI,CAAC,CAAC,MAAM,EAAE,EAAE;QAClC,WAAW,CAAC,MAAM,CAAC,CAAC;QACpB,OAAO,CAAC,GAAG,CAAC,6DAA6D,CAAC,CAAC;IAC7E,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,GAAG,EAAE,EAAE;QACf,OAAO,CAAC,KAAK,CAAC,mBAAmB,EAAE,GAAG,CAAC,CAAC;QACxC,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;IAClB,CAAC,CAAC,CAAC;AACL,CAAC"}
|
|
1
|
+
{"version":3,"file":"skill-benchmark.js","sourceRoot":"","sources":["../../src/eval/skill-benchmark.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;GAgBG;AAEH,OAAO,EAAE,OAAO,EAAE,IAAI,EAAE,MAAM,WAAW,CAAC;AAC1C,OAAO,EAAE,YAAY,EAAE,aAAa,EAAE,UAAU,EAAE,MAAM,SAAS,CAAC;AAClE,OAAO,EAAE,SAAS,EAAE,MAAM,cAAc,CAAC;AACzC,OAAO,EAAE,gBAAgB,EAAE,MAAM,yBAAyB,CAAC;AAsE3D,8EAA8E;AAC9E,mBAAmB;AACnB,8EAA8E;AAE9E,MAAM,CAAC,KAAK,UAAU,iBAAiB,CAAC,OAIvC;IACC,MAAM,QAAQ,GAAG,OAAO,CAAC,MAAM,CAAC,IAAI,CAAC,OAAO,EAAE,IAAI,EAAE,IAAI,CAAC,CAAC;IAC1D,MAAM,QAAQ,GAAG,OAAO,EAAE,QAAQ,IAAI,IAAI,CAAC,QAAQ,EAAE,OAAO,CAAC,CAAC;IAC9D,MAAM,SAAS,GAAG,OAAO,EAAE,SAAS,IAAI,IAAI,CAAC,QAAQ,EAAE,MAAM,EAAE,iBAAiB,CAAC,CAAC;IAClF,MAAM,UAAU,GAAG,OAAO,EAAE,UAAU,IAAI,IAAI,CAAC,QAAQ,EAAE,MAAM,EAAE,iBAAiB,EAAE,uBAAuB,CAAC,CAAC;IAE7G,gBAAgB;IAChB,MAAM,YAAY,GAAG,IAAI,CAAC,SAAS,EAAE,eAAe,CAAC,CAAC;IACtD,IAAI,CAAC,UAAU,CAAC,YAAY,CAAC,EAAE,CAAC;QAC9B,MAAM,IAAI,KAAK,CAAC,uBAAuB,YAAY,EAAE,CAAC,CAAC;IACzD,CAAC;IACD,MAAM,QAAQ,GAA6B,IAAI,CAAC,KAAK,CAAC,YAAY,CAAC,YAAY,EAAE,OAAO,CAAC,CAAC,CAAC;IAE3F,oDAAoD;IACpD,MAAM,YAAY,GAAG,IAAI,CAAC,SAAS,EAAE,wBAAwB,CAAC,CAAC;IAC/D,MAAM,WAAW,GAAkC,UAAU,CAAC,YAAY,CAAC;QACzE,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,YAAY,CAAC,YAAY,EAAE,OAAO,CAAC,CAAC;QACjD,CAAC,CAAC,EAAE,CAAC;IAEP,cAAc;IACd,MAAM,MAAM,GAAG,IAAI,SAAS,CAAC,EAAE,QAAQ,EAAE,CAAC,CAAC;IAC3C,MAAM,MAAM,CAAC,SAAS,EAAE,CAAC;IAEzB,kBAAkB;IAClB,MAAM,OAAO,GAAmB,EAAE,CAAC;IAEnC,KAAK,MAAM,KAAK,IAAI,QAAQ,EAAE,CAAC;QAC7B,MAAM,QAAQ,GAAG,IAAI,CAAC,SAAS,EAAE,KAAK,CAAC,IAAI,CAAC,CAAC;QAC7C,IAAI,CAAC,UAAU,CAAC,QAAQ,CAAC,EAAE,CAAC;YAC1B,OAAO,CAAC,KAAK,CAAC,yBAAyB,KAAK,CAAC,IAAI,EAAE,CAAC,CAAC;YACrD,SAAS;QACX,CAAC;QAED,MAAM,OAAO,GAAG,YAAY,CAAC,QAAQ,EAAE,OAAO,CAAC,CAAC;QAChD,MAAM,KAAK,GAAG,WAAW,CAAC,GAAG,EAAE,CAAC;QAChC,MAAM,OAAO,GAAG,MAAM,CAAC,SAAS,CAAC,OAAO,CAAC,CAAC;QAC1C,MAAM,OAAO,GAAG,WAAW,CAAC,GAAG,EAAE,GAAG,KAAK,CAAC;QAE1C,MAAM,QAAQ,GAAG,OAAO,CAAC,MAAM,GAAG,CAAC,CAAC;QACpC,MAAM,UAAU,GAAG,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;QAEjD,MAAM,OAAO,GACX,CAAC,KAAK,CAAC,KAAK,KAAK,WAAW,IAAI,QAAQ,CAAC;YACzC,CAAC,KAAK,CAAC,KAAK,KAAK,QAAQ,IAAI,CAAC,QAAQ,CAAC,CAAC;QAE1C,oEAAoE;QACpE,MAAM,QAAQ,GAAG,WAAW,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;QACzC,MAAM,oBAAoB,GAAG,QAAQ,EAAE,iBAAiB;YACtD,CAAC,CAAC,QAAQ,CAAC,iBAAiB,CAAC,KAAK,CAAC,CAAC,EAAE,EAAE,EAAE,CAAC,UAAU,CAAC,QAAQ,CAAC,EAAE,CAAC,OAAO,CAAC,CAAC;YAC3E,CAAC,CAAC,IAAI,CAAC,CAAC,gCAAgC;QAC1C,MAAM,kBAAkB,GAAG,QAAQ,EAAE,iBAAiB,EAAE,GAAG,CAAC,CAAC,EAAE,EAAE,EAAE,CAAC,EAAE,CAAC,QAAQ,CAAC,IAAI,EAAE,CAAC;QACvF,MAAM,gBAAgB,GAAG,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;QAClE,MAAM,eAAe,GAAG,kBAAkB,CAAC,KAAK,CAAC,CAAC,EAAE,EAAE,EAAE,CACtD,gBAAgB,CAAC,QAAQ,CAAC,EAAqC,CAAC,CACjE,CAAC;QAEF,OAAO,CAAC,IAAI,CAAC;YACX,IAAI,EAAE,KAAK,CAAC,IAAI;YAChB,KAAK,EAAE,KAAK,CAAC,KAAK;YAClB,KAAK,EAAE,KAAK,CAAC,KAAK,IAAI,EAAE;YACxB,WAAW,EAAE,KAAK,CAAC,WAAW;YAC9B,QAAQ;YACR,WAAW,EAAE,UAAU;YACvB,OAAO;YACP,UAAU,EAAE,IAAI,CAAC,KAAK,CAAC,OAAO,GAAG,GAAG,CAAC,GAAG,GAAG;YAC3C,sBAAsB,EAAE,oBAAoB;YAC5C,gBAAgB,EAAE,eAAe;SAClC,CAAC,CAAC;IACL,CAAC;IAED,kBAAkB;IAClB,MAAM,SAAS,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,KAAK,KAAK,WAAW,CAAC,CAAC;IACjE,MAAM,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,KAAK,KAAK,QAAQ,CAAC,CAAC;IAE3D,MAAM,EAAE,GAAG,SAAS,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC,MAAM,CAAC;IACtD,MAAM,EAAE,GAAG,SAAS,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC,MAAM,CAAC;IACvD,MAAM,EAAE,GAAG,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC,MAAM,CAAC;IACnD,MAAM,EAAE,GAAG,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC,MAAM,CAAC;IAEpD,MAAM,MAAM,GAAG,SAAS,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,EAAE,GAAG,SAAS,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,CAAC;IAChE,MAAM,SAAS,GAAG,CAAC,EAAE,GAAG,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,EAAE,GAAG,CAAC,EAAE,GAAG,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;IACrD,MAAM,EAAE,GAAG,CAAC,SAAS,GAAG,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,SAAS,GAAG,MAAM,CAAC,GAAG,CAAC,SAAS,GAAG,MAAM,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;IAC1F,MAAM,MAAM,GAAG,MAAM,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,EAAE,GAAG,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,CAAC;IAE1D,oBAAoB;IACpB,MAAM,YAAY,GAAG,CAAC,KAAa,EAAgB,EAAE;QACnD,MAAM,OAAO,GAAG,SAAS,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,KAAK,KAAK,KAAK,CAAC,CAAC;QAC3D,MAAM,QAAQ,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC,MAAM,CAAC;QAC1D,OAAO;YACL,KAAK,EAAE,OAAO,CAAC,MAAM;YACrB,QAAQ;YACR,MAAM,EAAE,OAAO,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,QAAQ,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC;SAC3D,CAAC;IACJ,CAAC,CAAC;IAEF,MAAM,SAAS,GAAG,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,UAAU,CAAC,CAAC;IAEnD,6BAA6B;IAC7B,MAAM,iBAAiB,GAAG,SAAS,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC;IAC9D,MAAM,qBAAqB,GAAG,iBAAiB,CAAC,MAAM,GAAG,CAAC;QACxD,CAAC,CAAC,iBAAiB,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,sBAAsB,CAAC,CAAC,MAAM,GAAG,iBAAiB,CAAC,MAAM;QAC7F,CAAC,CAAC,CAAC,CAAC;IACN,MAAM,gBAAgB,GAAG,iBAAiB,CAAC,MAAM,GAAG,CAAC;QACnD,CAAC,CAAC,iBAAiB,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,gBAAgB,CAAC,CAAC,MAAM,GAAG,iBAAiB,CAAC,MAAM;QACvF,CAAC,CAAC,CAAC,CAAC;IAEN,MAAM,MAAM,GAAyB;QACnC,SAAS,EAAE,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE;QACnC,WAAW,EAAE,OAAO,CAAC,MAAM;QAC3B,eAAe,EAAE,SAAS,CAAC,MAAM;QACjC,YAAY,EAAE,MAAM,CAAC,MAAM;QAC3B,cAAc,EAAE,IAAI,CAAC,KAAK,CAAC,MAAM,GAAG,IAAI,CAAC,GAAG,IAAI;QAChD,iBAAiB,EAAE,IAAI,CAAC,KAAK,CAAC,SAAS,GAAG,IAAI,CAAC,GAAG,IAAI;QACtD,UAAU,EAAE,IAAI,CAAC,KAAK,CAAC,EAAE,GAAG,IAAI,CAAC,GAAG,IAAI;QACxC,OAAO,EAAE,IAAI,CAAC,KAAK,CAAC,MAAM,GAAG,IAAI,CAAC,GAAG,IAAI;QACzC,OAAO,EAAE,YAAY,CAAC,GAAG,CAAC;QAC1B,OAAO,EAAE,YAAY,CAAC,GAAG,CAAC;QAC1B,OAAO,EAAE,YAAY,CAAC,GAAG,CAAC;QAC1B,cAAc,EAAE,EAAE;QAClB,eAAe,EAAE,EAAE;QACnB,cAAc,EAAE,EAAE;QAClB,eAAe,EAAE,EAAE;QACnB,uBAAuB,EAAE,IAAI,CAAC,KAAK,CAAC,qBAAqB,GAAG,IAAI,CAAC,GAAG,IAAI;QACxE,iBAAiB,EAAE,IAAI,CAAC,KAAK,CAAC,gBAAgB,GAAG,IAAI,CAAC,GAAG,IAAI;QAC7D,cAAc,EAAE,IAAI,CAAC,KAAK,CAAC,CAAC,SAAS,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC,GAAG,SAAS,CAAC,MAAM,CAAC,GAAG,GAAG,CAAC,GAAG,GAAG;QACjG,cAAc,EAAE,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,GAAG,CAAC,GAAG,SAAS,CAAC,GAAG,GAAG,CAAC,GAAG,GAAG;QAC9D,OAAO;QACP,cAAc,EAAE,SAAS,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,QAAQ,CAAC;QACpD,YAAY,EAAE,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,QAAQ,CAAC;KAC/C,CAAC;IAEF,cAAc;IACd,aAAa,CAAC,UAAU,EAAE,IAAI,CAAC,SAAS,CAAC,MAAM,EAAE,IAAI,EAAE,CAAC,CAAC,CAAC,CAAC;IAE3D,OAAO,MAAM,CAAC;AAChB,CAAC;AAED,8EAA8E;AAC9E,8EAA8E;AAC9E,8EAA8E;AAE9E;;;;;;;;;GASG;AACH,MAAM,UAAU,8BAA8B,CAAC,MAA4B;IACzE,gBAAgB,CACd;QACE,MAAM,EAAE,iBAAiB;QACzB,cAAc,EAAE,cAAc;QAC9B,WAAW,EAAE,MAAM,CAAC,SAAS;QAC7B,OAAO,EAAE,MAAM,CAAC,WAAW;QAC3B,OAAO,EAAE;YACP,MAAM,EAAE,MAAM,CAAC,cAAc;YAC7B,SAAS,EAAE,MAAM,CAAC,iBAAiB;YACnC,EAAE,EAAE,MAAM,CAAC,UAAU;YACrB,OAAO,EAAE,MAAM,CAAC,OAAO;SACxB;QACD,SAAS,EAAE;YACT,EAAE,EAAE,MAAM,CAAC,cAAc;YACzB,EAAE,EAAE,MAAM,CAAC,eAAe;YAC1B,EAAE,EAAE,MAAM,CAAC,cAAc;YACzB,EAAE,EAAE,MAAM,CAAC,eAAe;SAC3B;QACD,UAAU,EAAE;YACV,GAAG,EAAE,MAAM,CAAC,cAAc;YAC1B,GAAG,EAAE,MAAM,CAAC,cAAc;YAC1B,GAAG,EAAE,MAAM,CAAC,cAAc;YAC1B,IAAI,EAAE,MAAM,CAAC,cAAc;YAC3B,GAAG,EAAE,MAAM,CAAC,cAAc;SAC3B;QACD,SAAS,EAAE;YACT,MAAM,EAAE;gBACN,CAAC,EAAE,MAAM,CAAC,OAAO;gBACjB,CAAC,EAAE,MAAM,CAAC,OAAO;gBACjB,CAAC,EAAE,MAAM,CAAC,OAAO;aAClB;YACD,eAAe,EAAE,MAAM,CAAC,eAAe;YACvC,YAAY,EAAE,MAAM,CAAC,YAAY;YACjC,uBAAuB,EAAE,MAAM,CAAC,uBAAuB;YACvD,iBAAiB,EAAE,MAAM,CAAC,iBAAiB;SAC5C;QACD,KAAK,EACH,8GAA8G;KACjH,EACD,EAAE,KAAK,EAAE,IAAI,EAAE,CAChB,CAAC;AACJ,CAAC;AAED,8EAA8E;AAC9E,aAAa;AACb,8EAA8E;AAE9E,SAAS,aAAa,CAAC,CAAS;IAC9B,OAAO,GAAG,CAAC,CAAC,GAAG,GAAG,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,GAAG,CAAC;AACpC,CAAC;AAED,MAAM,UAAU,WAAW,CAAC,MAA4B;IACtD,OAAO,CAAC,GAAG,CAAC,wDAAwD,CAAC,CAAC;IACtE,OAAO,CAAC,GAAG,CAAC,qDAAqD,CAAC,CAAC;IACnE,OAAO,CAAC,GAAG,CAAC,wDAAwD,CAAC,CAAC;IAEtE,OAAO,CAAC,GAAG,CAAC,WAAW,MAAM,CAAC,WAAW,aAAa,MAAM,CAAC,eAAe,eAAe,MAAM,CAAC,YAAY,UAAU,CAAC,CAAC;IAC1H,OAAO,CAAC,GAAG,CAAC,cAAc,MAAM,CAAC,SAAS,IAAI,CAAC,CAAC;IAEhD,OAAO,CAAC,GAAG,CAAC,gCAAgC,CAAC,CAAC;IAC9C,OAAO,CAAC,GAAG,CAAC,uBAAuB,aAAa,CAAC,MAAM,CAAC,cAAc,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC;IACzF,OAAO,CAAC,GAAG,CAAC,uBAAuB,aAAa,CAAC,MAAM,CAAC,iBAAiB,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC;IAC5F,OAAO,CAAC,GAAG,CAAC,uBAAuB,aAAa,CAAC,MAAM,CAAC,UAAU,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC;IACrF,OAAO,CAAC,GAAG,CAAC,uBAAuB,aAAa,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC;IAClF,OAAO,CAAC,GAAG,CAAC,kCAAkC,CAAC,CAAC;IAEhD,OAAO,CAAC,GAAG,CAAC,mBAAmB,CAAC,CAAC;IACjC,OAAO,CAAC,GAAG,CAAC,2BAA2B,aAAa,CAAC,MAAM,CAAC,OAAO,CAAC,MAAM,CAAC,KAAK,MAAM,CAAC,OAAO,CAAC,QAAQ,IAAI,MAAM,CAAC,OAAO,CAAC,KAAK,GAAG,CAAC,CAAC;IACpI,OAAO,CAAC,GAAG,CAAC,2BAA2B,aAAa,CAAC,MAAM,CAAC,OAAO,CAAC,MAAM,CAAC,KAAK,MAAM,CAAC,OAAO,CAAC,QAAQ,IAAI,MAAM,CAAC,OAAO,CAAC,KAAK,GAAG,CAAC,CAAC;IACpI,OAAO,CAAC,GAAG,CAAC,2BAA2B,aAAa,CAAC,MAAM,CAAC,OAAO,CAAC,MAAM,CAAC,KAAK,MAAM,CAAC,OAAO,CAAC,QAAQ,IAAI,MAAM,CAAC,OAAO,CAAC,KAAK,KAAK,CAAC,CAAC;IAEtI,OAAO,CAAC,GAAG,CAAC,iCAAiC,CAAC,CAAC;IAC/C,OAAO,CAAC,GAAG,CAAC,6BAA6B,aAAa,CAAC,MAAM,CAAC,uBAAuB,CAAC,EAAE,CAAC,CAAC;IAC1F,OAAO,CAAC,GAAG,CAAC,6BAA6B,aAAa,CAAC,MAAM,CAAC,iBAAiB,CAAC,IAAI,CAAC,CAAC;IAEtF,OAAO,CAAC,GAAG,CAAC,mBAAmB,CAAC,CAAC;IACjC,OAAO,CAAC,GAAG,CAAC,SAAS,MAAM,CAAC,cAAc,SAAS,MAAM,CAAC,eAAe,EAAE,CAAC,CAAC;IAC7E,OAAO,CAAC,GAAG,CAAC,SAAS,MAAM,CAAC,eAAe,SAAS,MAAM,CAAC,cAAc,IAAI,CAAC,CAAC;IAE/E,OAAO,CAAC,GAAG,CAAC,gBAAgB,MAAM,CAAC,cAAc,WAAW,MAAM,CAAC,cAAc,MAAM,CAAC,CAAC;IAEzF,IAAI,MAAM,CAAC,cAAc,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QACrC,OAAO,CAAC,GAAG,CAAC,iBAAiB,CAAC,CAAC;QAC/B,KAAK,MAAM,CAAC,IAAI,MAAM,CAAC,cAAc,EAAE,CAAC;YACtC,OAAO,CAAC,GAAG,CAAC,QAAQ,CAAC,CAAC,KAAK,KAAK,CAAC,CAAC,IAAI,KAAK,CAAC,CAAC,WAAW,GAAG,CAAC,CAAC;QAC/D,CAAC;QACD,OAAO,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC;IAClB,CAAC;IAED,IAAI,MAAM,CAAC,YAAY,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QACnC,OAAO,CAAC,GAAG,CAAC,kBAAkB,CAAC,CAAC;QAChC,KAAK,MAAM,CAAC,IAAI,MAAM,CAAC,YAAY,EAAE,CAAC;YACpC,OAAO,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,IAAI,MAAM,CAAC,CAAC,WAAW,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;QAC7D,CAAC;QACD,OAAO,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC;IAClB,CAAC;AACH,CAAC;AAED,8EAA8E;AAC9E,mBAAmB;AACnB,8EAA8E;AAE9E,IAAI,MAAM,CAAC,IAAI,CAAC,GAAG,KAAK,UAAU,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,IAAI,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,QAAQ,CAAC,oBAAoB,CAAC,EAAE,CAAC;IACvG,iBAAiB,EAAE,CAAC,IAAI,CAAC,CAAC,MAAM,EAAE,EAAE;QAClC,WAAW,CAAC,MAAM,CAAC,CAAC;QACpB,OAAO,CAAC,GAAG,CAAC,6DAA6D,CAAC,CAAC;QAC3E,qEAAqE;QACrE,8DAA8D;QAC9D,8BAA8B,CAAC,MAAM,CAAC,CAAC;QACvC,OAAO,CAAC,GAAG,CAAC,iDAAiD,CAAC,CAAC;IACjE,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,GAAG,EAAE,EAAE;QACf,OAAO,CAAC,KAAK,CAAC,mBAAmB,EAAE,GAAG,CAAC,CAAC;QACxC,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;IAClB,CAAC,CAAC,CAAC;AACL,CAAC"}
|
|
@@ -0,0 +1,70 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* src/measurement/from-eval-harness.ts
|
|
3
|
+
*
|
|
4
|
+
* Adapter: convert an `src/eval/eval-harness.ts` `EvalReport` into a
|
|
5
|
+
* `MeasurementInput`. Used by the four first-class eval runners (PINT,
|
|
6
|
+
* HackAPrompt, atr-self-test, the SKILL.md runner) to write a Measurement
|
|
7
|
+
* file at the end of an eval run without each runner duplicating the
|
|
8
|
+
* conversion logic.
|
|
9
|
+
*
|
|
10
|
+
* The shape of `EvalReport` is defined in `src/eval/eval-harness.ts`; we
|
|
11
|
+
* accept a structurally-typed input here to avoid creating a cross-module
|
|
12
|
+
* type dependency in either direction.
|
|
13
|
+
*/
|
|
14
|
+
/**
|
|
15
|
+
* Structural type matching the fields we read from `EvalReport`. Kept here so
|
|
16
|
+
* `src/eval/*` does not need to export its internal types just to use the
|
|
17
|
+
* adapter.
|
|
18
|
+
*/
|
|
19
|
+
export interface EvalHarnessReportShape {
|
|
20
|
+
timestamp?: string;
|
|
21
|
+
corpusSize: number;
|
|
22
|
+
overall: {
|
|
23
|
+
recall: number;
|
|
24
|
+
precision: number;
|
|
25
|
+
f1: number;
|
|
26
|
+
fpRate: number;
|
|
27
|
+
confusion: {
|
|
28
|
+
tp: number;
|
|
29
|
+
fp: number;
|
|
30
|
+
tn: number;
|
|
31
|
+
fn: number;
|
|
32
|
+
};
|
|
33
|
+
sampleCount?: number;
|
|
34
|
+
};
|
|
35
|
+
latency?: {
|
|
36
|
+
p50: number;
|
|
37
|
+
p95: number;
|
|
38
|
+
p99: number;
|
|
39
|
+
mean: number;
|
|
40
|
+
max: number;
|
|
41
|
+
};
|
|
42
|
+
byCategory?: unknown;
|
|
43
|
+
byDifficulty?: unknown;
|
|
44
|
+
}
|
|
45
|
+
export interface FromEvalHarnessOpts {
|
|
46
|
+
/** Source identifier (lowercase, hyphenated). Example: "pint". */
|
|
47
|
+
source: string;
|
|
48
|
+
/** Source version. Example: "v1", "corpus-2026-05-20". */
|
|
49
|
+
source_version: string;
|
|
50
|
+
/** Optional canonical upstream URL. */
|
|
51
|
+
source_url?: string;
|
|
52
|
+
/** Optional human-readable context. */
|
|
53
|
+
notes?: string;
|
|
54
|
+
/**
|
|
55
|
+
* Force overwrite if the same filename already exists (same source +
|
|
56
|
+
* source_version + atr_version + date). Defaults to false (append-only).
|
|
57
|
+
*/
|
|
58
|
+
force?: boolean;
|
|
59
|
+
}
|
|
60
|
+
/**
|
|
61
|
+
* Build a `MeasurementInput` from an `EvalReport` plus source metadata, then
|
|
62
|
+
* write it via `writeMeasurement()`. Returns the same paths/objects that
|
|
63
|
+
* `writeMeasurement()` returns.
|
|
64
|
+
*/
|
|
65
|
+
export declare function writeMeasurementFromEvalReport(report: EvalHarnessReportShape, opts: FromEvalHarnessOpts): {
|
|
66
|
+
measurementPath: string;
|
|
67
|
+
latestPath: string;
|
|
68
|
+
measurement: import("./schema.js").Measurement;
|
|
69
|
+
};
|
|
70
|
+
//# sourceMappingURL=from-eval-harness.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"from-eval-harness.d.ts","sourceRoot":"","sources":["../../src/measurement/from-eval-harness.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;GAYG;AAIH;;;;GAIG;AACH,MAAM,WAAW,sBAAsB;IACrC,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,UAAU,EAAE,MAAM,CAAC;IACnB,OAAO,EAAE;QACP,MAAM,EAAE,MAAM,CAAC;QACf,SAAS,EAAE,MAAM,CAAC;QAClB,EAAE,EAAE,MAAM,CAAC;QACX,MAAM,EAAE,MAAM,CAAC;QACf,SAAS,EAAE;YAAE,EAAE,EAAE,MAAM,CAAC;YAAC,EAAE,EAAE,MAAM,CAAC;YAAC,EAAE,EAAE,MAAM,CAAC;YAAC,EAAE,EAAE,MAAM,CAAA;SAAE,CAAC;QAC9D,WAAW,CAAC,EAAE,MAAM,CAAC;KACtB,CAAC;IACF,OAAO,CAAC,EAAE;QAAE,GAAG,EAAE,MAAM,CAAC;QAAC,GAAG,EAAE,MAAM,CAAC;QAAC,GAAG,EAAE,MAAM,CAAC;QAAC,IAAI,EAAE,MAAM,CAAC;QAAC,GAAG,EAAE,MAAM,CAAA;KAAE,CAAC;IAC/E,UAAU,CAAC,EAAE,OAAO,CAAC;IACrB,YAAY,CAAC,EAAE,OAAO,CAAC;CACxB;AAED,MAAM,WAAW,mBAAmB;IAClC,kEAAkE;IAClE,MAAM,EAAE,MAAM,CAAC;IACf,0DAA0D;IAC1D,cAAc,EAAE,MAAM,CAAC;IACvB,uCAAuC;IACvC,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,uCAAuC;IACvC,KAAK,CAAC,EAAE,MAAM,CAAC;IACf;;;OAGG;IACH,KAAK,CAAC,EAAE,OAAO,CAAC;CACjB;AAED;;;;GAIG;AACH,wBAAgB,8BAA8B,CAC5C,MAAM,EAAE,sBAAsB,EAC9B,IAAI,EAAE,mBAAmB;;;;EAuB1B"}
|
|
@@ -0,0 +1,49 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* src/measurement/from-eval-harness.ts
|
|
3
|
+
*
|
|
4
|
+
* Adapter: convert an `src/eval/eval-harness.ts` `EvalReport` into a
|
|
5
|
+
* `MeasurementInput`. Used by the four first-class eval runners (PINT,
|
|
6
|
+
* HackAPrompt, atr-self-test, the SKILL.md runner) to write a Measurement
|
|
7
|
+
* file at the end of an eval run without each runner duplicating the
|
|
8
|
+
* conversion logic.
|
|
9
|
+
*
|
|
10
|
+
* The shape of `EvalReport` is defined in `src/eval/eval-harness.ts`; we
|
|
11
|
+
* accept a structurally-typed input here to avoid creating a cross-module
|
|
12
|
+
* type dependency in either direction.
|
|
13
|
+
*/
|
|
14
|
+
import { writeMeasurement } from "./write.js";
|
|
15
|
+
/**
|
|
16
|
+
* Build a `MeasurementInput` from an `EvalReport` plus source metadata, then
|
|
17
|
+
* write it via `writeMeasurement()`. Returns the same paths/objects that
|
|
18
|
+
* `writeMeasurement()` returns.
|
|
19
|
+
*/
|
|
20
|
+
export function writeMeasurementFromEvalReport(report, opts) {
|
|
21
|
+
const input = {
|
|
22
|
+
source: opts.source,
|
|
23
|
+
source_version: opts.source_version,
|
|
24
|
+
measured_at: report.timestamp,
|
|
25
|
+
samples: report.corpusSize,
|
|
26
|
+
metrics: {
|
|
27
|
+
recall: report.overall.recall,
|
|
28
|
+
precision: report.overall.precision,
|
|
29
|
+
f1: report.overall.f1,
|
|
30
|
+
fp_rate: report.overall.fpRate,
|
|
31
|
+
},
|
|
32
|
+
confusion: report.overall.confusion,
|
|
33
|
+
};
|
|
34
|
+
if (opts.source_url)
|
|
35
|
+
input.source_url = opts.source_url;
|
|
36
|
+
if (opts.notes)
|
|
37
|
+
input.notes = opts.notes;
|
|
38
|
+
if (report.latency)
|
|
39
|
+
input.latency_ms = report.latency;
|
|
40
|
+
const breakdown = {};
|
|
41
|
+
if (report.byCategory)
|
|
42
|
+
breakdown.by_category = report.byCategory;
|
|
43
|
+
if (report.byDifficulty)
|
|
44
|
+
breakdown.by_difficulty = report.byDifficulty;
|
|
45
|
+
if (Object.keys(breakdown).length > 0)
|
|
46
|
+
input.breakdown = breakdown;
|
|
47
|
+
return writeMeasurement(input, { force: opts.force ?? true });
|
|
48
|
+
}
|
|
49
|
+
//# sourceMappingURL=from-eval-harness.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"from-eval-harness.js","sourceRoot":"","sources":["../../src/measurement/from-eval-harness.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;GAYG;AAEH,OAAO,EAAE,gBAAgB,EAAyB,MAAM,YAAY,CAAC;AAuCrE;;;;GAIG;AACH,MAAM,UAAU,8BAA8B,CAC5C,MAA8B,EAC9B,IAAyB;IAEzB,MAAM,KAAK,GAAqB;QAC9B,MAAM,EAAE,IAAI,CAAC,MAAM;QACnB,cAAc,EAAE,IAAI,CAAC,cAAc;QACnC,WAAW,EAAE,MAAM,CAAC,SAAS;QAC7B,OAAO,EAAE,MAAM,CAAC,UAAU;QAC1B,OAAO,EAAE;YACP,MAAM,EAAE,MAAM,CAAC,OAAO,CAAC,MAAM;YAC7B,SAAS,EAAE,MAAM,CAAC,OAAO,CAAC,SAAS;YACnC,EAAE,EAAE,MAAM,CAAC,OAAO,CAAC,EAAE;YACrB,OAAO,EAAE,MAAM,CAAC,OAAO,CAAC,MAAM;SAC/B;QACD,SAAS,EAAE,MAAM,CAAC,OAAO,CAAC,SAAS;KACpC,CAAC;IACF,IAAI,IAAI,CAAC,UAAU;QAAE,KAAK,CAAC,UAAU,GAAG,IAAI,CAAC,UAAU,CAAC;IACxD,IAAI,IAAI,CAAC,KAAK;QAAE,KAAK,CAAC,KAAK,GAAG,IAAI,CAAC,KAAK,CAAC;IACzC,IAAI,MAAM,CAAC,OAAO;QAAE,KAAK,CAAC,UAAU,GAAG,MAAM,CAAC,OAAO,CAAC;IACtD,MAAM,SAAS,GAA4B,EAAE,CAAC;IAC9C,IAAI,MAAM,CAAC,UAAU;QAAE,SAAS,CAAC,WAAW,GAAG,MAAM,CAAC,UAAU,CAAC;IACjE,IAAI,MAAM,CAAC,YAAY;QAAE,SAAS,CAAC,aAAa,GAAG,MAAM,CAAC,YAAY,CAAC;IACvE,IAAI,MAAM,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC,MAAM,GAAG,CAAC;QAAE,KAAK,CAAC,SAAS,GAAG,SAAS,CAAC;IACnE,OAAO,gBAAgB,CAAC,KAAK,EAAE,EAAE,KAAK,EAAE,IAAI,CAAC,KAAK,IAAI,IAAI,EAAE,CAAC,CAAC;AAChE,CAAC"}
|
|
@@ -0,0 +1,152 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* src/measurement/schema.ts
|
|
3
|
+
*
|
|
4
|
+
* Schema for ATR benchmark measurement files.
|
|
5
|
+
*
|
|
6
|
+
* Every public ATR recall / precision / FP-rate claim must reference a
|
|
7
|
+
* measurement file conforming to this schema. The contract is documented in
|
|
8
|
+
* `data/measurements/README.md`.
|
|
9
|
+
*
|
|
10
|
+
* Design constraints:
|
|
11
|
+
* - No external dependencies (no zod, no ajv). Schemas evolve slowly; the
|
|
12
|
+
* dependency surface should not.
|
|
13
|
+
* - Strict at the boundary. `parseMeasurement()` throws on any deviation;
|
|
14
|
+
* it does not silently coerce, drop fields, or accept missing required
|
|
15
|
+
* fields.
|
|
16
|
+
* - Forward-compatible. `schema_version` is mandatory. Future readers can
|
|
17
|
+
* decide how to handle older versions.
|
|
18
|
+
*/
|
|
19
|
+
/** Bump this when the schema breaks backward compatibility. */
|
|
20
|
+
export declare const CURRENT_SCHEMA_VERSION: "1";
|
|
21
|
+
/**
|
|
22
|
+
* Core metrics that every measurement reports.
|
|
23
|
+
*
|
|
24
|
+
* `recall` is the most-cited number externally. `precision` and `fp_rate`
|
|
25
|
+
* anchor the recall claim against overclaim risk.
|
|
26
|
+
*/
|
|
27
|
+
export interface Metrics {
|
|
28
|
+
/** True-positive rate. matched_attacks / total_attacks. Range [0, 1]. */
|
|
29
|
+
recall: number;
|
|
30
|
+
/** matched / (matched + false_positives). Range [0, 1]. */
|
|
31
|
+
precision: number;
|
|
32
|
+
/** 2 * precision * recall / (precision + recall). Range [0, 1]. */
|
|
33
|
+
f1: number;
|
|
34
|
+
/** false_positives / total_benign. Range [0, 1]. May be 0 if the corpus has no benign samples. */
|
|
35
|
+
fp_rate: number;
|
|
36
|
+
}
|
|
37
|
+
/** Confusion matrix. Strongly recommended; CI does not require it. */
|
|
38
|
+
export interface Confusion {
|
|
39
|
+
/** True positives — adversarial sample matched by a rule. */
|
|
40
|
+
tp: number;
|
|
41
|
+
/** False positives — benign sample matched by a rule. */
|
|
42
|
+
fp: number;
|
|
43
|
+
/** True negatives — benign sample NOT matched by any rule. */
|
|
44
|
+
tn: number;
|
|
45
|
+
/** False negatives — adversarial sample NOT matched (the recall gap). */
|
|
46
|
+
fn: number;
|
|
47
|
+
}
|
|
48
|
+
/** Engine latency profile in milliseconds. Optional. */
|
|
49
|
+
export interface LatencyMs {
|
|
50
|
+
p50: number;
|
|
51
|
+
p95: number;
|
|
52
|
+
p99: number;
|
|
53
|
+
mean: number;
|
|
54
|
+
max: number;
|
|
55
|
+
}
|
|
56
|
+
/**
|
|
57
|
+
* Source-defined bespoke breakdown. Each source uses this shape differently:
|
|
58
|
+
* - garak: by attack family (dan, latentinjection, sysprompt_extraction, …)
|
|
59
|
+
* - skill-benchmark: by layer (layer_a, layer_b, layer_c)
|
|
60
|
+
* - mega-scan: by severity (critical, high, medium, low)
|
|
61
|
+
* - eval-harness: by category and by difficulty
|
|
62
|
+
*
|
|
63
|
+
* Kept as `Record<string, unknown>` because the structure is source-defined.
|
|
64
|
+
* Consumers of this field must validate per-source.
|
|
65
|
+
*/
|
|
66
|
+
export type Breakdown = Record<string, unknown>;
|
|
67
|
+
/**
|
|
68
|
+
* A single measurement run.
|
|
69
|
+
*
|
|
70
|
+
* Required fields are the minimum for a public-citable claim.
|
|
71
|
+
*/
|
|
72
|
+
export interface Measurement {
|
|
73
|
+
/** Schema version. Currently `"1"`. */
|
|
74
|
+
schema_version: typeof CURRENT_SCHEMA_VERSION;
|
|
75
|
+
/** Stable source identifier. Lowercase, hyphen-separated. Example: `"garak"`, `"pint"`, `"hh-rlhf"`. */
|
|
76
|
+
source: string;
|
|
77
|
+
/** Upstream version. Example: `"v0.10.3"`, `"corpus-2026-04-15"`. */
|
|
78
|
+
source_version: string;
|
|
79
|
+
/** Optional canonical URL for the exact upstream release/commit. */
|
|
80
|
+
source_url?: string;
|
|
81
|
+
/** Optional upstream git SHA (for git-pinnable sources). */
|
|
82
|
+
source_commit?: string;
|
|
83
|
+
/** ATR version at measurement time. Read from `package.json`. */
|
|
84
|
+
atr_version: string;
|
|
85
|
+
/** ATR git commit at measurement time (short SHA). */
|
|
86
|
+
atr_commit: string;
|
|
87
|
+
/** Total rule count in the engine at measurement time. */
|
|
88
|
+
rules_loaded: number;
|
|
89
|
+
/** ISO 8601 UTC timestamp. Example: `"2026-05-23T03:57:58.869Z"`. */
|
|
90
|
+
measured_at: string;
|
|
91
|
+
/** Total sample count in the evaluated corpus. */
|
|
92
|
+
samples: number;
|
|
93
|
+
/** Core metrics. All four required. */
|
|
94
|
+
metrics: Metrics;
|
|
95
|
+
/** Confusion matrix. Optional; strongly recommended. */
|
|
96
|
+
confusion?: Confusion;
|
|
97
|
+
/** Engine latency profile. Optional. */
|
|
98
|
+
latency_ms?: LatencyMs;
|
|
99
|
+
/** Source-defined per-category / per-family / per-severity breakdown. Optional. */
|
|
100
|
+
breakdown?: Breakdown;
|
|
101
|
+
/** Free-text context. Optional. Keep brief. */
|
|
102
|
+
notes?: string;
|
|
103
|
+
}
|
|
104
|
+
/**
|
|
105
|
+
* `latest.json` per source. Points to the most recent measurement file.
|
|
106
|
+
*
|
|
107
|
+
* The pointer is a relative path from the `<source>/` directory.
|
|
108
|
+
* Consumers should NOT cache; the file is small and re-reading is cheap.
|
|
109
|
+
*/
|
|
110
|
+
export interface LatestPointer {
|
|
111
|
+
source: string;
|
|
112
|
+
/** Filename of the latest measurement, relative to the `<source>/` directory. */
|
|
113
|
+
file: string;
|
|
114
|
+
/** Mirror of the measurement's `measured_at`, for fast inspection. */
|
|
115
|
+
measured_at: string;
|
|
116
|
+
/** Mirror of `metrics` for fast aggregation without reading the underlying file. */
|
|
117
|
+
metrics: Metrics;
|
|
118
|
+
/** Mirror of `source_version` and `atr_version` for fast inspection. */
|
|
119
|
+
source_version: string;
|
|
120
|
+
atr_version: string;
|
|
121
|
+
/** Mirror of `samples`. */
|
|
122
|
+
samples: number;
|
|
123
|
+
}
|
|
124
|
+
/**
|
|
125
|
+
* Error thrown when a measurement file fails schema validation.
|
|
126
|
+
* Includes the field path and reason for fast debugging.
|
|
127
|
+
*/
|
|
128
|
+
export declare class MeasurementSchemaError extends Error {
|
|
129
|
+
readonly path: string;
|
|
130
|
+
readonly reason: string;
|
|
131
|
+
constructor(path: string, reason: string);
|
|
132
|
+
}
|
|
133
|
+
/**
|
|
134
|
+
* Parse a `Measurement` from an arbitrary value. Throws `MeasurementSchemaError`
|
|
135
|
+
* on any deviation from the schema. Returns a strongly-typed `Measurement`.
|
|
136
|
+
*
|
|
137
|
+
* Unknown extra top-level keys are allowed but ignored (forward-compat); they
|
|
138
|
+
* are NOT preserved in the returned object.
|
|
139
|
+
*/
|
|
140
|
+
export declare function parseMeasurement(raw: unknown): Measurement;
|
|
141
|
+
/** Parse a `LatestPointer`. Throws on schema violation. */
|
|
142
|
+
export declare function parseLatestPointer(raw: unknown): LatestPointer;
|
|
143
|
+
/**
|
|
144
|
+
* Compute the canonical filename for a measurement.
|
|
145
|
+
*
|
|
146
|
+
* Format: `<YYYY-MM-DD>_<source>-<source_version>_atr-<atr_version>.json`
|
|
147
|
+
*
|
|
148
|
+
* `source_version` and `atr_version` are slugified (lowercase, non-alphanumeric
|
|
149
|
+
* → `-`, leading/trailing `-` removed, collapsed runs of `-`).
|
|
150
|
+
*/
|
|
151
|
+
export declare function measurementFilename(m: Pick<Measurement, "measured_at" | "source" | "source_version" | "atr_version">): string;
|
|
152
|
+
//# sourceMappingURL=schema.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"schema.d.ts","sourceRoot":"","sources":["../../src/measurement/schema.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;GAiBG;AAIH,+DAA+D;AAC/D,eAAO,MAAM,sBAAsB,EAAG,GAAY,CAAC;AAInD;;;;;GAKG;AACH,MAAM,WAAW,OAAO;IACtB,yEAAyE;IACzE,MAAM,EAAE,MAAM,CAAC;IACf,2DAA2D;IAC3D,SAAS,EAAE,MAAM,CAAC;IAClB,mEAAmE;IACnE,EAAE,EAAE,MAAM,CAAC;IACX,kGAAkG;IAClG,OAAO,EAAE,MAAM,CAAC;CACjB;AAED,sEAAsE;AACtE,MAAM,WAAW,SAAS;IACxB,6DAA6D;IAC7D,EAAE,EAAE,MAAM,CAAC;IACX,yDAAyD;IACzD,EAAE,EAAE,MAAM,CAAC;IACX,8DAA8D;IAC9D,EAAE,EAAE,MAAM,CAAC;IACX,yEAAyE;IACzE,EAAE,EAAE,MAAM,CAAC;CACZ;AAED,wDAAwD;AACxD,MAAM,WAAW,SAAS;IACxB,GAAG,EAAE,MAAM,CAAC;IACZ,GAAG,EAAE,MAAM,CAAC;IACZ,GAAG,EAAE,MAAM,CAAC;IACZ,IAAI,EAAE,MAAM,CAAC;IACb,GAAG,EAAE,MAAM,CAAC;CACb;AAED;;;;;;;;;GASG;AACH,MAAM,MAAM,SAAS,GAAG,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;AAEhD;;;;GAIG;AACH,MAAM,WAAW,WAAW;IAC1B,uCAAuC;IACvC,cAAc,EAAE,OAAO,sBAAsB,CAAC;IAE9C,wGAAwG;IACxG,MAAM,EAAE,MAAM,CAAC;IAEf,qEAAqE;IACrE,cAAc,EAAE,MAAM,CAAC;IAEvB,oEAAoE;IACpE,UAAU,CAAC,EAAE,MAAM,CAAC;IAEpB,4DAA4D;IAC5D,aAAa,CAAC,EAAE,MAAM,CAAC;IAEvB,iEAAiE;IACjE,WAAW,EAAE,MAAM,CAAC;IAEpB,sDAAsD;IACtD,UAAU,EAAE,MAAM,CAAC;IAEnB,0DAA0D;IAC1D,YAAY,EAAE,MAAM,CAAC;IAErB,qEAAqE;IACrE,WAAW,EAAE,MAAM,CAAC;IAEpB,kDAAkD;IAClD,OAAO,EAAE,MAAM,CAAC;IAEhB,uCAAuC;IACvC,OAAO,EAAE,OAAO,CAAC;IAEjB,wDAAwD;IACxD,SAAS,CAAC,EAAE,SAAS,CAAC;IAEtB,wCAAwC;IACxC,UAAU,CAAC,EAAE,SAAS,CAAC;IAEvB,mFAAmF;IACnF,SAAS,CAAC,EAAE,SAAS,CAAC;IAEtB,+CAA+C;IAC/C,KAAK,CAAC,EAAE,MAAM,CAAC;CAChB;AAED;;;;;GAKG;AACH,MAAM,WAAW,aAAa;IAC5B,MAAM,EAAE,MAAM,CAAC;IACf,iFAAiF;IACjF,IAAI,EAAE,MAAM,CAAC;IACb,sEAAsE;IACtE,WAAW,EAAE,MAAM,CAAC;IACpB,oFAAoF;IACpF,OAAO,EAAE,OAAO,CAAC;IACjB,wEAAwE;IACxE,cAAc,EAAE,MAAM,CAAC;IACvB,WAAW,EAAE,MAAM,CAAC;IACpB,2BAA2B;IAC3B,OAAO,EAAE,MAAM,CAAC;CACjB;AAID;;;GAGG;AACH,qBAAa,sBAAuB,SAAQ,KAAK;aAE7B,IAAI,EAAE,MAAM;aACZ,MAAM,EAAE,MAAM;gBADd,IAAI,EAAE,MAAM,EACZ,MAAM,EAAE,MAAM;CAKjC;AAqED;;;;;;GAMG;AACH,wBAAgB,gBAAgB,CAAC,GAAG,EAAE,OAAO,GAAG,WAAW,CAyC1D;AAED,2DAA2D;AAC3D,wBAAgB,kBAAkB,CAAC,GAAG,EAAE,OAAO,GAAG,aAAa,CAW9D;AAED;;;;;;;GAOG;AACH,wBAAgB,mBAAmB,CAAC,CAAC,EAAE,IAAI,CAAC,WAAW,EAAE,aAAa,GAAG,QAAQ,GAAG,gBAAgB,GAAG,aAAa,CAAC,GAAG,MAAM,CAM7H"}
|