agent-threat-rules 4.1.5 → 4.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +688 -1
- package/README.md +355 -125
- package/dist/adapters/openshell-filter.d.ts +10 -3
- package/dist/adapters/openshell-filter.d.ts.map +1 -1
- package/dist/adapters/openshell-filter.js +10 -3
- package/dist/adapters/openshell-filter.js.map +1 -1
- package/dist/adapters/stdio-adapter.js +3 -3
- package/dist/adapters/stdio-adapter.js.map +1 -1
- package/dist/cli/mcp-tools-document.d.ts +40 -0
- package/dist/cli/mcp-tools-document.d.ts.map +1 -0
- package/dist/cli/mcp-tools-document.js +95 -0
- package/dist/cli/mcp-tools-document.js.map +1 -0
- package/dist/cli/scan-handler.d.ts +8 -0
- package/dist/cli/scan-handler.d.ts.map +1 -1
- package/dist/cli/scan-handler.js +126 -34
- package/dist/cli/scan-handler.js.map +1 -1
- package/dist/cli.d.ts.map +1 -1
- package/dist/cli.js +80 -43
- package/dist/cli.js.map +1 -1
- package/dist/converters/sarif.d.ts +14 -2
- package/dist/converters/sarif.d.ts.map +1 -1
- package/dist/converters/sarif.js +7 -9
- package/dist/converters/sarif.js.map +1 -1
- package/dist/corpus-event.d.ts +47 -5
- package/dist/corpus-event.d.ts.map +1 -1
- package/dist/corpus-event.js +68 -29
- package/dist/corpus-event.js.map +1 -1
- package/dist/embedding/model-loader.d.ts +2 -0
- package/dist/embedding/model-loader.d.ts.map +1 -1
- package/dist/embedding/model-loader.js +3 -1
- package/dist/embedding/model-loader.js.map +1 -1
- package/dist/enforcement.d.ts +8 -0
- package/dist/enforcement.d.ts.map +1 -1
- package/dist/enforcement.js +10 -0
- package/dist/enforcement.js.map +1 -1
- package/dist/engine.d.ts +150 -6
- package/dist/engine.d.ts.map +1 -1
- package/dist/engine.js +337 -76
- package/dist/engine.js.map +1 -1
- package/dist/eval/embedding-provenance.d.ts +40 -0
- package/dist/eval/embedding-provenance.d.ts.map +1 -0
- package/dist/eval/embedding-provenance.js +71 -0
- package/dist/eval/embedding-provenance.js.map +1 -0
- package/dist/eval/eval-harness.d.ts +34 -0
- package/dist/eval/eval-harness.d.ts.map +1 -1
- package/dist/eval/eval-harness.js +49 -2
- package/dist/eval/eval-harness.js.map +1 -1
- package/dist/eval/hackaprompt-corpus.d.ts +12 -0
- package/dist/eval/hackaprompt-corpus.d.ts.map +1 -1
- package/dist/eval/hackaprompt-corpus.js +15 -1
- package/dist/eval/hackaprompt-corpus.js.map +1 -1
- package/dist/eval/hackaprompt-provenance.d.ts +52 -0
- package/dist/eval/hackaprompt-provenance.d.ts.map +1 -0
- package/dist/eval/hackaprompt-provenance.js +108 -0
- package/dist/eval/hackaprompt-provenance.js.map +1 -0
- package/dist/eval/run-hackaprompt-benchmark.d.ts +18 -8
- package/dist/eval/run-hackaprompt-benchmark.d.ts.map +1 -1
- package/dist/eval/run-hackaprompt-benchmark.js +154 -64
- package/dist/eval/run-hackaprompt-benchmark.js.map +1 -1
- package/dist/eval/run-pint-benchmark.js +11 -1
- package/dist/eval/run-pint-benchmark.js.map +1 -1
- package/dist/eval/skill-benchmark.d.ts +26 -6
- package/dist/eval/skill-benchmark.d.ts.map +1 -1
- package/dist/eval/skill-benchmark.js +49 -16
- package/dist/eval/skill-benchmark.js.map +1 -1
- package/dist/flywheel.d.ts +4 -0
- package/dist/flywheel.d.ts.map +1 -1
- package/dist/flywheel.js +12 -4
- package/dist/flywheel.js.map +1 -1
- package/dist/hook-handler.d.ts +11 -1
- package/dist/hook-handler.d.ts.map +1 -1
- package/dist/hook-handler.js +84 -16
- package/dist/hook-handler.js.map +1 -1
- package/dist/hook-tool-output.d.ts +47 -0
- package/dist/hook-tool-output.d.ts.map +1 -0
- package/dist/hook-tool-output.js +110 -0
- package/dist/hook-tool-output.js.map +1 -0
- package/dist/index.d.ts +5 -3
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +3 -2
- package/dist/index.js.map +1 -1
- package/dist/layer-integration.d.ts +4 -2
- package/dist/layer-integration.d.ts.map +1 -1
- package/dist/layer-integration.js +6 -2
- package/dist/layer-integration.js.map +1 -1
- package/dist/mcp-server.d.ts +8 -0
- package/dist/mcp-server.d.ts.map +1 -1
- package/dist/mcp-server.js +1 -0
- package/dist/mcp-server.js.map +1 -1
- package/dist/mcp-tools/scan.d.ts.map +1 -1
- package/dist/mcp-tools/scan.js +9 -1
- package/dist/mcp-tools/scan.js.map +1 -1
- package/dist/measurement/enforce-lane.d.ts +66 -0
- package/dist/measurement/enforce-lane.d.ts.map +1 -0
- package/dist/measurement/enforce-lane.js +89 -0
- package/dist/measurement/enforce-lane.js.map +1 -0
- package/dist/measurement/from-eval-harness.d.ts +16 -3
- package/dist/measurement/from-eval-harness.d.ts.map +1 -1
- package/dist/measurement/from-eval-harness.js +15 -5
- package/dist/measurement/from-eval-harness.js.map +1 -1
- package/dist/measurement/in-sample-ablation.d.ts +29 -0
- package/dist/measurement/in-sample-ablation.d.ts.map +1 -0
- package/dist/measurement/in-sample-ablation.js +45 -0
- package/dist/measurement/in-sample-ablation.js.map +1 -0
- package/dist/measurement/schema.d.ts +38 -1
- package/dist/measurement/schema.d.ts.map +1 -1
- package/dist/measurement/schema.js +43 -1
- package/dist/measurement/schema.js.map +1 -1
- package/dist/quality/action-eligibility.d.ts +4 -2
- package/dist/quality/action-eligibility.d.ts.map +1 -1
- package/dist/quality/action-eligibility.js.map +1 -1
- package/dist/rule-confidence.d.ts +31 -0
- package/dist/rule-confidence.d.ts.map +1 -0
- package/dist/rule-confidence.js +49 -0
- package/dist/rule-confidence.js.map +1 -0
- package/dist/rule-scaffolder.d.ts.map +1 -1
- package/dist/rule-scaffolder.js +43 -4
- package/dist/rule-scaffolder.js.map +1 -1
- package/dist/semantic-evaluator.d.ts.map +1 -1
- package/dist/semantic-evaluator.js +15 -13
- package/dist/semantic-evaluator.js.map +1 -1
- package/dist/shadow-evaluator.d.ts +7 -2
- package/dist/shadow-evaluator.d.ts.map +1 -1
- package/dist/shadow-evaluator.js +29 -25
- package/dist/shadow-evaluator.js.map +1 -1
- package/dist/testcase-event.d.ts +82 -1
- package/dist/testcase-event.d.ts.map +1 -1
- package/dist/testcase-event.js +135 -16
- package/dist/testcase-event.js.map +1 -1
- package/dist/tier0-invariant.d.ts.map +1 -1
- package/dist/tier0-invariant.js +1 -0
- package/dist/tier0-invariant.js.map +1 -1
- package/dist/tier1-blacklist.d.ts.map +1 -1
- package/dist/tier1-blacklist.js +1 -0
- package/dist/tier1-blacklist.js.map +1 -1
- package/dist/tool-definition-scope.d.ts +45 -0
- package/dist/tool-definition-scope.d.ts.map +1 -0
- package/dist/tool-definition-scope.js +68 -0
- package/dist/tool-definition-scope.js.map +1 -0
- package/dist/types.d.ts +42 -2
- package/dist/types.d.ts.map +1 -1
- package/dist/verdict.d.ts +46 -2
- package/dist/verdict.d.ts.map +1 -1
- package/dist/verdict.js +99 -14
- package/dist/verdict.js.map +1 -1
- package/package.json +2 -1
- package/rules/agent-manipulation/ATR-2026-00030-cross-agent-attack.yaml +2 -2
- package/rules/agent-manipulation/ATR-2026-00139-casual-authority-redirect.yaml +1 -1
- package/rules/agent-manipulation/ATR-2026-00328-named-mode-persona-override.yaml +1 -1
- package/rules/agent-manipulation/ATR-2026-00330-yoja-allcaps-settings-block-persona.yaml +1 -1
- package/rules/agent-manipulation/ATR-2026-00418-weknora-mcp-config-rce.yaml +5 -5
- package/rules/agent-manipulation/ATR-2026-00432-superagi-output-handler-eval-rce.yaml +1 -1
- package/rules/agent-manipulation/ATR-2026-00440-semantic-kernel-vector-store-eval-rce.yaml +3 -3
- package/rules/context-exfiltration/ATR-2026-00426-nl-output-injection-credential-leak.yaml +1 -1
- package/rules/context-exfiltration/ATR-2026-00702-ipi-credential-exfil-via-agent-action.yaml +1 -1
- package/rules/context-exfiltration/ATR-2026-00707-ipi-file-exfil-with-destroy-or-tamper.yaml +1 -1
- package/rules/context-exfiltration/ATR-2026-00851-indirect-pi-financial-payment-exfil.yaml +1 -1
- package/rules/context-exfiltration/ATR-2026-01457-sysprompt-completion-clone-attack.yaml +1 -1
- package/rules/context-exfiltration/ATR-2026-01462-sysprompt-override-pseudo-admin.yaml +1 -1
- package/rules/context-exfiltration/ATR-2026-01463-imend-imsystem-prompt-injection-format.yaml +1 -1
- package/rules/context-exfiltration/ATR-2026-01606-ssrf-internal-network-scan.yaml +1 -1
- package/rules/context-exfiltration/ATR-2026-02847-policy-justified-exfil-to-email.yaml +1 -1
- package/rules/context-exfiltration/ATR-2026-02855-cross-email-exfil-relay.yaml +79 -0
- package/rules/context-exfiltration/ATR-2026-02861-exfil-send-email-with-body-directive.yaml +83 -0
- package/rules/context-exfiltration/ATR-2026-02866-idna-label-separator-in-fetch-url-host.yaml +104 -0
- package/rules/excessive-autonomy/ATR-2026-00710-ipi-physical-access-grant.yaml +1 -1
- package/rules/model-abuse/ATR-2026-01897-host-sabotage.yaml +2 -2
- package/rules/model-security/ATR-2026-00433-modelcache-torch-load-deserialization-rce.yaml +1 -1
- package/rules/privilege-escalation/ATR-2026-00436-enclave-vm-sandbox-escape-rce.yaml +1 -1
- package/rules/privilege-escalation/ATR-2026-01601-sql-injection-destructive-ddl.yaml +2 -2
- package/rules/privilege-escalation/ATR-2026-01612-debug-mode-privilege-escalation.yaml +1 -1
- package/rules/privilege-escalation/ATR-2026-02867-identifier-typed-tool-argument-dot-dot-path-segment.yaml +106 -0
- package/rules/prompt-injection/ATR-2026-00004-system-prompt-override.yaml +2 -2
- package/rules/prompt-injection/ATR-2026-00080-encoding-evasion.yaml +1 -1
- package/rules/prompt-injection/ATR-2026-00086-visual-spoofing.yaml +1 -1
- package/rules/prompt-injection/ATR-2026-00203-context-pollution-skill-description.yaml +1 -1
- package/rules/prompt-injection/ATR-2026-00228-structured-jailbreak.yaml +2 -2
- package/rules/prompt-injection/ATR-2026-00233-structured-jailbreak.yaml +2 -2
- package/rules/prompt-injection/ATR-2026-00256-base-n-encoding-jailbreak.yaml +1 -1
- package/rules/prompt-injection/ATR-2026-00264-latent-injection-translation.yaml +3 -3
- package/rules/prompt-injection/ATR-2026-00265-latent-injection-rag-document.yaml +1 -1
- package/rules/prompt-injection/ATR-2026-00282-perez-prompt-injection-hijack.yaml +4 -4
- package/rules/prompt-injection/ATR-2026-00321-hyphenated-system-instruction-injection.yaml +1 -1
- package/rules/prompt-injection/ATR-2026-00394-backspace-deletion-character-injection.yaml +2 -2
- package/rules/prompt-injection/ATR-2026-00446-variable-assignment-payload-injection.yaml +3 -3
- package/rules/prompt-injection/ATR-2026-00476-garak-web-injection-javascript-alert-1.yaml +3 -3
- package/rules/prompt-injection/ATR-2026-00481-garak-dan-mixedunassigned.yaml +1 -1
- package/rules/prompt-injection/ATR-2026-00489-garak-encoding-javascript-alert-1.yaml +3 -3
- package/rules/prompt-injection/ATR-2026-00508-escape-delimiter-wrapped-goal-hijacking-in-user-input-prompt.yaml +3 -3
- package/rules/prompt-injection/ATR-2026-00512-rules-file-backdoor-injection.yaml +1 -1
- package/rules/prompt-injection/ATR-2026-01005-many-shot-repetition-override.yaml +2 -2
- package/rules/prompt-injection/ATR-2026-01006-payload-split-string-concat-injection.yaml +2 -2
- package/rules/prompt-injection/ATR-2026-01015-evasion-refusal-suppression.yaml +1 -1
- package/rules/prompt-injection/ATR-2026-01016-evasion-response-priming-forced-opener.yaml +1 -1
- package/rules/prompt-injection/ATR-2026-01026-evasion-unicode-glitch-token-anomaly.yaml +2 -2
- package/rules/prompt-injection/ATR-2026-01304-indirect-injection-carrier-important-override.yaml +1 -1
- package/rules/prompt-injection/ATR-2026-01759-dos-new-important-instructions-injection-in-document.yaml +1 -1
- package/rules/prompt-injection/ATR-2026-01865-llmail-fake-email-boundary-marker.yaml +1 -1
- package/rules/prompt-injection/ATR-2026-02010-emoji-only-payload.yaml +2 -2
- package/rules/prompt-injection/ATR-2026-02815-injected-halt-order-canned-reply-dos.yaml +80 -0
- package/rules/prompt-injection/ATR-2026-02820-tool-result-orders-agent-to-hide-its-own-action.yaml +106 -0
- package/rules/prompt-injection/ATR-2026-02858-context-suppression-directive.yaml +73 -0
- package/rules/prompt-injection/ATR-2026-02859-tool-output-agent-role-reassignment.yaml +71 -0
- package/rules/prompt-injection/ATR-2026-02860-forged-instruction-boundary-marker.yaml +70 -0
- package/rules/prompt-injection/ATR-2026-02862-fake-chat-template-turn-injection.yaml +79 -0
- package/rules/skill-compromise/ATR-2026-00060-skill-impersonation.yaml +1 -1
- package/rules/skill-compromise/ATR-2026-00063-skill-chain-attack.yaml +2 -2
- package/rules/skill-compromise/ATR-2026-00066-parameter-injection.yaml +2 -2
- package/rules/skill-compromise/ATR-2026-00149-skill-exfil-compound.yaml +6 -6
- package/rules/skill-compromise/ATR-2026-00223-reverse-shell-dropper.yaml +2 -2
- package/rules/skill-compromise/ATR-2026-01755-backdoor-pot-linguistic-trigger-phrase.yaml +1 -1
- package/rules/skill-compromise/ATR-2026-01756-backdoor-pot-symbol-emoticon-trigger.yaml +1 -1
- package/rules/skill-compromise/ATR-2026-02261-code-credential-exfil-chain.yaml +1 -1
- package/rules/tool-poisoning/ATR-2026-00012-unauthorized-tool-call.yaml +23 -23
- package/rules/tool-poisoning/ATR-2026-00013-tool-ssrf.yaml +16 -5
- package/rules/tool-poisoning/ATR-2026-00161-important-tag-cross-tool-shadowing.yaml +1 -1
- package/rules/tool-poisoning/ATR-2026-00277-echo-template-command-injection.yaml +2 -2
- package/rules/tool-poisoning/ATR-2026-00494-garak-exploitation-mixedunassigned.yaml +4 -4
- package/rules/tool-poisoning/ATR-2026-00581-mcp-tool-rug-pull-post-approval-redefinition.yaml +1 -1
- package/rules/tool-poisoning/ATR-2026-01303-tool-schema-enumeration-social-engineering.yaml +1 -1
- package/rules/tool-poisoning/ATR-2026-01307-mcp-dns-rebinding-attack.yaml +2 -2
- package/rules/tool-poisoning/ATR-2026-01775-semantic-mcp-tool-manifest-poisoning.yaml +1 -1
- package/rules/tool-poisoning/ATR-2026-01932-shadow-undeclared-mcp-server-registration.yaml +1 -1
- package/rules/tool-poisoning/ATR-2026-02025-mcp-full-schema-poisoning-nondescription-field.yaml +1 -1
- package/rules/tool-poisoning/ATR-2026-02817-untrusted-content-dictates-tool-arguments.yaml +83 -0
- package/rules/tool-poisoning/ATR-2026-02864-non-canonical-ipv4-host-in-url-tool-argument.yaml +102 -0
- package/rules/tool-poisoning/ATR-2026-02865-read-only-utility-write-flag-in-command-argument.yaml +157 -0
- package/rules/tool-poisoning/ATR-2026-02868-ssrf-ipv6-transition-address-embedded-private-ipv4.yaml +107 -0
- package/spec/README.md +6 -3
- package/spec/atr-event-v1.0.md +13 -8
- package/spec/atr-method-v1.1.md +2 -2
- package/spec/category-registry/gemara/README.md +102 -0
- package/spec/category-registry/gemara/ai-agent-capability-catalog.yaml +157 -0
- package/spec/category-registry/gemara/atr-attack-vector-catalog.yaml +130 -0
- package/spec/category-registry/gemara/atr-detection-control-catalog.yaml +327 -0
- package/spec/category-registry/gemara/atr-threat-catalog.yaml +246 -0
- package/spec/category-registry/gemara/src/editorial.yaml +743 -0
- package/spec/mappings/README.md +1 -1
- package/spec/mappings/atr-to-nist-csf-2.0.md +5 -4
- package/spec/schema/event.schema.json +1 -1
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"scan.d.ts","sourceRoot":"","sources":["../../src/mcp-tools/scan.ts"],"names":[],"mappings":"AAAA;;;;;;;;;GASG;AAEH,OAAO,KAAK,EAAE,SAAS,EAAE,MAAM,cAAc,CAAC;
|
|
1
|
+
{"version":3,"file":"scan.d.ts","sourceRoot":"","sources":["../../src/mcp-tools/scan.ts"],"names":[],"mappings":"AAAA;;;;;;;;;GASG;AAEH,OAAO,KAAK,EAAE,SAAS,EAAE,MAAM,cAAc,CAAC;AAI9C,MAAM,WAAW,SAAS;IACxB,OAAO,EAAE,MAAM,CAAC;IAChB,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,YAAY,CAAC,EAAE,MAAM,CAAC;CACvB;AAmBD,wBAAsB,UAAU,CAAC,MAAM,EAAE,SAAS,EAAE,IAAI,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,GAAG,OAAO,CAAC;IAC1F,OAAO,EAAE,KAAK,CAAC;QAAE,IAAI,EAAE,MAAM,CAAC;QAAC,IAAI,EAAE,MAAM,CAAA;KAAE,CAAC,CAAC;IAC/C,OAAO,CAAC,EAAE,OAAO,CAAC;CACnB,CAAC,CAqFD"}
|
package/dist/mcp-tools/scan.js
CHANGED
|
@@ -8,6 +8,7 @@
|
|
|
8
8
|
*
|
|
9
9
|
* @module agent-threat-rules/mcp-tools/scan
|
|
10
10
|
*/
|
|
11
|
+
import { isModelDecided, observeActions } from '../verdict.js';
|
|
11
12
|
const SEVERITY_ORDER = {
|
|
12
13
|
informational: 0,
|
|
13
14
|
low: 1,
|
|
@@ -67,6 +68,9 @@ export async function handleScan(engine, args) {
|
|
|
67
68
|
// Use evaluateWithVerdict() to run all configured layers
|
|
68
69
|
const { verdict, layersUsed } = await engine.evaluateWithVerdict(event);
|
|
69
70
|
const filtered = verdict.matches.filter((m) => (SEVERITY_ORDER[m.rule.severity] ?? 0) >= minIdx);
|
|
71
|
+
// A model's decision the engine reports without enforcing: its recommended
|
|
72
|
+
// actions stop at the observe tier, as the verdict's actions do.
|
|
73
|
+
const heldBack = (m) => engine.getModelDecisions() === 'advisory' && isModelDecided(m);
|
|
70
74
|
const result = {
|
|
71
75
|
threats_found: filtered.length,
|
|
72
76
|
scan_timestamp: event.timestamp,
|
|
@@ -84,7 +88,11 @@ export async function handleScan(engine, args) {
|
|
|
84
88
|
confidence: Math.round(m.confidence * 100),
|
|
85
89
|
description: m.rule.description,
|
|
86
90
|
matched_patterns: m.matchedPatterns,
|
|
87
|
-
recommended_actions: m.rule.response.actions,
|
|
91
|
+
recommended_actions: heldBack(m) ? observeActions([m]) : m.rule.response.actions,
|
|
92
|
+
// A model's decision (semantic judge, embedding, semantic layer) is reported
|
|
93
|
+
// without counting toward verdict.outcome unless the engine enforces them.
|
|
94
|
+
decided_by: m.decidedBy ?? null,
|
|
95
|
+
enforced: !heldBack(m),
|
|
88
96
|
})),
|
|
89
97
|
};
|
|
90
98
|
return {
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"scan.js","sourceRoot":"","sources":["../../src/mcp-tools/scan.ts"],"names":[],"mappings":"AAAA;;;;;;;;;GASG;
|
|
1
|
+
{"version":3,"file":"scan.js","sourceRoot":"","sources":["../../src/mcp-tools/scan.ts"],"names":[],"mappings":"AAAA;;;;;;;;;GASG;AAGH,OAAO,EAAE,cAAc,EAAE,cAAc,EAAE,MAAM,eAAe,CAAC;AAS/D,MAAM,cAAc,GAA2B;IAC7C,aAAa,EAAE,CAAC;IAChB,GAAG,EAAE,CAAC;IACN,MAAM,EAAE,CAAC;IACT,IAAI,EAAE,CAAC;IACP,QAAQ,EAAE,CAAC;CACZ,CAAC;AAEF,MAAM,iBAAiB,GAAwB,IAAI,GAAG,CAAC;IACrD,WAAW;IACX,YAAY;IACZ,WAAW;IACX,eAAe;IACf,gBAAgB;IAChB,qBAAqB;CACtB,CAAC,CAAC;AAEH,MAAM,CAAC,KAAK,UAAU,UAAU,CAAC,MAAiB,EAAE,IAA6B;IAI/E,MAAM,kBAAkB,GAAG,OAAO,CAAC;IACnC,MAAM,OAAO,GAAG,IAAI,CAAC,SAAS,CAAC,CAAC;IAChC,IAAI,OAAO,OAAO,KAAK,QAAQ,IAAI,OAAO,CAAC,IAAI,EAAE,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QAC/D,OAAO;YACL,OAAO,EAAE,CAAC,EAAE,IAAI,EAAE,MAAM,EAAE,IAAI,EAAE,8DAA8D,EAAE,CAAC;YACjG,OAAO,EAAE,IAAI;SACd,CAAC;IACJ,CAAC;IACD,IAAI,OAAO,CAAC,MAAM,GAAG,kBAAkB,EAAE,CAAC;QACxC,OAAO;YACL,OAAO,EAAE,CAAC,EAAE,IAAI,EAAE,MAAM,EAAE,IAAI,EAAE,0CAA0C,kBAAkB,cAAc,EAAE,CAAC;YAC7G,OAAO,EAAE,IAAI;SACd,CAAC;IACJ,CAAC;IAED,MAAM,YAAY,GAAI,IAAI,CAAC,YAAY,CAAY,IAAI,WAAW,CAAC;IACnE,IAAI,CAAC,iBAAiB,CAAC,GAAG,CAAC,YAAY,CAAC,EAAE,CAAC;QACzC,OAAO;YACL,OAAO,EAAE,CAAC,EAAE,IAAI,EAAE,MAAM,EAAE,IAAI,EAAE,8BAA8B,YAAY,mBAAmB,CAAC,GAAG,iBAAiB,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,EAAE,CAAC;YACnI,OAAO,EAAE,IAAI;SACd,CAAC;IACJ,CAAC;IAED,MAAM,WAAW,GAAG,CAAE,IAAI,CAAC,cAAc,CAAY,IAAI,eAAe,CAAC,CAAC,WAAW,EAAE,CAAC;IACxF,IAAI,CAAC,CAAC,WAAW,IAAI,cAAc,CAAC,EAAE,CAAC;QACrC,OAAO;YACL,OAAO,EAAE,CAAC,EAAE,IAAI,EAAE,MAAM,EAAE,IAAI,EAAE,gCAAgC,WAAW,sDAAsD,EAAE,CAAC;YACpI,OAAO,EAAE,IAAI;SACd,CAAC;IACJ,CAAC;IAED,MAAM,MAAM,GAAG,cAAc,CAAC,WAAW,CAAC,IAAI,CAAC,CAAC;IAEhD,MAAM,KAAK,GAAe;QACxB,IAAI,EAAE,YAA8B;QACpC,SAAS,EAAE,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE;QACnC,OAAO,EAAE,OAAO;QAChB,MAAM,EAAE;YACN,UAAU,EAAE,OAAO;YACnB,YAAY,EAAE,OAAO;YACrB,aAAa,EAAE,OAAO;YACtB,OAAO,EAAE,OAAO;SACjB;KACF,CAAC;IAEF,yDAAyD;IACzD,MAAM,EAAE,OAAO,EAAE,UAAU,EAAE,GAAG,MAAM,MAAM,CAAC,mBAAmB,CAAC,KAAK,CAAC,CAAC;IAExE,MAAM,QAAQ,GAAG,OAAO,CAAC,OAAO,CAAC,MAAM,CACrC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,cAAc,CAAC,CAAC,CAAC,IAAI,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAC,IAAI,MAAM,CACxD,CAAC;IACF,2EAA2E;IAC3E,iEAAiE;IACjE,MAAM,QAAQ,GAAG,CAAC,CAA4B,EAAW,EAAE,CACzD,MAAM,CAAC,iBAAiB,EAAE,KAAK,UAAU,IAAI,cAAc,CAAC,CAAC,CAAC,CAAC;IAEjE,MAAM,MAAM,GAAG;QACb,aAAa,EAAE,QAAQ,CAAC,MAAM;QAC9B,cAAc,EAAE,KAAK,CAAC,SAAS;QAC/B,UAAU,EAAE,YAAY;QACxB,WAAW,EAAE,UAAU;QACvB,OAAO,EAAE;YACP,OAAO,EAAE,OAAO,CAAC,OAAO;YACxB,MAAM,EAAE,OAAO,CAAC,MAAM;SACvB;QACD,OAAO,EAAE,QAAQ,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC;YAC5B,OAAO,EAAE,CAAC,CAAC,IAAI,CAAC,EAAE;YAClB,KAAK,EAAE,CAAC,CAAC,IAAI,CAAC,KAAK;YACnB,QAAQ,EAAE,CAAC,CAAC,IAAI,CAAC,QAAQ;YACzB,QAAQ,EAAE,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,QAAQ;YAC9B,UAAU,EAAE,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,UAAU,GAAG,GAAG,CAAC;YAC1C,WAAW,EAAE,CAAC,CAAC,IAAI,CAAC,WAAW;YAC/B,gBAAgB,EAAE,CAAC,CAAC,eAAe;YACnC,mBAAmB,EAAE,QAAQ,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,cAAc,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,QAAQ,CAAC,OAAO;YAChF,6EAA6E;YAC7E,2EAA2E;YAC3E,UAAU,EAAE,CAAC,CAAC,SAAS,IAAI,IAAI;YAC/B,QAAQ,EAAE,CAAC,QAAQ,CAAC,CAAC,CAAC;SACvB,CAAC,CAAC;KACJ,CAAC;IAEF,OAAO;QACL,OAAO,EAAE,CAAC,EAAE,IAAI,EAAE,MAAM,EAAE,IAAI,EAAE,IAAI,CAAC,SAAS,CAAC,MAAM,EAAE,IAAI,EAAE,CAAC,CAAC,EAAE,CAAC;KACnE,CAAC;AACJ,CAAC"}
|
|
@@ -0,0 +1,66 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* src/measurement/enforce-lane.ts
|
|
3
|
+
*
|
|
4
|
+
* The enforce-lane figure every benchmark runner records beside its headline,
|
|
5
|
+
* as `breakdown.enforce_lane` (shape and validation: ./schema.ts).
|
|
6
|
+
*
|
|
7
|
+
* WHY THIS EXISTS
|
|
8
|
+
*
|
|
9
|
+
* A runner's headline recall comes from `new ATREngine({ rulesDir })`, and an
|
|
10
|
+
* engine given no lane is pinned to `hunt`, which loads every maturity. The
|
|
11
|
+
* enforce lane — the one to run with blocking on, where a detection stops the
|
|
12
|
+
* agent with no human in the loop — loads only `maturity: stable`. On ATR 4.1.5
|
|
13
|
+
* the headline garak corpus scored 600 of 650 in hunt and 6 of 650 in enforce,
|
|
14
|
+
* and the SKILL.md corpus 32 of 32 against 0 of 32. Those enforce figures were
|
|
15
|
+
* one-off replays: no measurement recorded them, so README could cite them only
|
|
16
|
+
* as figures nothing refreshed, and stats.json dropped the one it had.
|
|
17
|
+
*
|
|
18
|
+
* THE RULE: SAME SAMPLES, SAME LOOP, OTHER LANE
|
|
19
|
+
*
|
|
20
|
+
* A runner passes the predicate that decides its own headline — the same
|
|
21
|
+
* function, not a re-implementation — to `enforceLaneRecallOf`, which runs it
|
|
22
|
+
* on an engine constructed with `lane: 'enforce'` over the same attack samples.
|
|
23
|
+
* Anything else (a wider event shape, a different sample filter) would make the
|
|
24
|
+
* enforce figure a measurement of something the headline is not.
|
|
25
|
+
*/
|
|
26
|
+
import { ATREngine, type ATREngineConfig } from "../engine.js";
|
|
27
|
+
import { type Lane } from "../quality/rule-contract.js";
|
|
28
|
+
import { type EnforceLaneRecall } from "./schema.js";
|
|
29
|
+
export type { EnforceLaneRecall } from "./schema.js";
|
|
30
|
+
/** Stored in every record, so the figure explains itself wherever it is read. */
|
|
31
|
+
export declare const ENFORCE_LANE_NOTE: string;
|
|
32
|
+
/**
|
|
33
|
+
* Rules an engine in `lane` loads: the engine's own two gates, in its order —
|
|
34
|
+
* status `draft` / `deprecated` is skipped outright (src/engine.ts), then the
|
|
35
|
+
* maturity -> lane mapping every pipeline imports (laneAllows). A loaded rule
|
|
36
|
+
* marked `confirm: embedding` still fires only once embedding similarity
|
|
37
|
+
* confirms it, outside the hunt lane; the synchronous evaluate() most runners
|
|
38
|
+
* use cannot confirm, so on that path such a rule is counted here but never
|
|
39
|
+
* fires.
|
|
40
|
+
*/
|
|
41
|
+
export declare function rulesInLane(rules: readonly {
|
|
42
|
+
status?: unknown;
|
|
43
|
+
maturity?: unknown;
|
|
44
|
+
}[], lane: Lane): number;
|
|
45
|
+
/** An engine in the enforce lane, and how many rules that lane loads. */
|
|
46
|
+
export interface EnforceLaneEngine {
|
|
47
|
+
readonly engine: ATREngine;
|
|
48
|
+
readonly rulesInLane: number;
|
|
49
|
+
}
|
|
50
|
+
/**
|
|
51
|
+
* Construct and load an enforce-lane engine. Throws when no rule loads: an
|
|
52
|
+
* engine with no rules reports 0 recall, which reads as a measured result.
|
|
53
|
+
*/
|
|
54
|
+
export declare function loadEnforceLaneEngine(config: Omit<ATREngineConfig, "lane">): Promise<EnforceLaneEngine>;
|
|
55
|
+
/** The record, validated by the same parser verify.ts runs over every file. */
|
|
56
|
+
export declare function enforceLaneRecall(input: {
|
|
57
|
+
detected: number;
|
|
58
|
+
attacks: number;
|
|
59
|
+
rulesInLane: number;
|
|
60
|
+
}): EnforceLaneRecall;
|
|
61
|
+
/**
|
|
62
|
+
* Run `detect` — the runner's own headline predicate — once per attack sample
|
|
63
|
+
* on the enforce-lane engine, and record how many it detected.
|
|
64
|
+
*/
|
|
65
|
+
export declare function enforceLaneRecallOf<T>(samples: readonly T[], detect: (engine: ATREngine, sample: T) => boolean, lane: EnforceLaneEngine): EnforceLaneRecall;
|
|
66
|
+
//# sourceMappingURL=enforce-lane.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"enforce-lane.d.ts","sourceRoot":"","sources":["../../src/measurement/enforce-lane.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;GAwBG;AAEH,OAAO,EAAE,SAAS,EAAE,KAAK,eAAe,EAAE,MAAM,cAAc,CAAC;AAC/D,OAAO,EAAc,KAAK,IAAI,EAAE,MAAM,6BAA6B,CAAC;AACpE,OAAO,EAA0B,KAAK,iBAAiB,EAAE,MAAM,aAAa,CAAC;AAE7E,YAAY,EAAE,iBAAiB,EAAE,MAAM,aAAa,CAAC;AAErD,iFAAiF;AACjF,eAAO,MAAM,iBAAiB,QAK2D,CAAC;AAE1F;;;;;;;;GAQG;AACH,wBAAgB,WAAW,CAAC,KAAK,EAAE,SAAS;IAAE,MAAM,CAAC,EAAE,OAAO,CAAC;IAAC,QAAQ,CAAC,EAAE,OAAO,CAAA;CAAE,EAAE,EAAE,IAAI,EAAE,IAAI,GAAG,MAAM,CAG1G;AAED,yEAAyE;AACzE,MAAM,WAAW,iBAAiB;IAChC,QAAQ,CAAC,MAAM,EAAE,SAAS,CAAC;IAC3B,QAAQ,CAAC,WAAW,EAAE,MAAM,CAAC;CAC9B;AAED;;;GAGG;AACH,wBAAsB,qBAAqB,CAAC,MAAM,EAAE,IAAI,CAAC,eAAe,EAAE,MAAM,CAAC,GAAG,OAAO,CAAC,iBAAiB,CAAC,CAO7G;AAED,+EAA+E;AAC/E,wBAAgB,iBAAiB,CAAC,KAAK,EAAE;IAAE,QAAQ,EAAE,MAAM,CAAC;IAAC,OAAO,EAAE,MAAM,CAAC;IAAC,WAAW,EAAE,MAAM,CAAA;CAAE,GAAG,iBAAiB,CAgBtH;AAED;;;GAGG;AACH,wBAAgB,mBAAmB,CAAC,CAAC,EACnC,OAAO,EAAE,SAAS,CAAC,EAAE,EACrB,MAAM,EAAE,CAAC,MAAM,EAAE,SAAS,EAAE,MAAM,EAAE,CAAC,KAAK,OAAO,EACjD,IAAI,EAAE,iBAAiB,GACtB,iBAAiB,CAMnB"}
|
|
@@ -0,0 +1,89 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* src/measurement/enforce-lane.ts
|
|
3
|
+
*
|
|
4
|
+
* The enforce-lane figure every benchmark runner records beside its headline,
|
|
5
|
+
* as `breakdown.enforce_lane` (shape and validation: ./schema.ts).
|
|
6
|
+
*
|
|
7
|
+
* WHY THIS EXISTS
|
|
8
|
+
*
|
|
9
|
+
* A runner's headline recall comes from `new ATREngine({ rulesDir })`, and an
|
|
10
|
+
* engine given no lane is pinned to `hunt`, which loads every maturity. The
|
|
11
|
+
* enforce lane — the one to run with blocking on, where a detection stops the
|
|
12
|
+
* agent with no human in the loop — loads only `maturity: stable`. On ATR 4.1.5
|
|
13
|
+
* the headline garak corpus scored 600 of 650 in hunt and 6 of 650 in enforce,
|
|
14
|
+
* and the SKILL.md corpus 32 of 32 against 0 of 32. Those enforce figures were
|
|
15
|
+
* one-off replays: no measurement recorded them, so README could cite them only
|
|
16
|
+
* as figures nothing refreshed, and stats.json dropped the one it had.
|
|
17
|
+
*
|
|
18
|
+
* THE RULE: SAME SAMPLES, SAME LOOP, OTHER LANE
|
|
19
|
+
*
|
|
20
|
+
* A runner passes the predicate that decides its own headline — the same
|
|
21
|
+
* function, not a re-implementation — to `enforceLaneRecallOf`, which runs it
|
|
22
|
+
* on an engine constructed with `lane: 'enforce'` over the same attack samples.
|
|
23
|
+
* Anything else (a wider event shape, a different sample filter) would make the
|
|
24
|
+
* enforce figure a measurement of something the headline is not.
|
|
25
|
+
*/
|
|
26
|
+
import { ATREngine } from "../engine.js";
|
|
27
|
+
import { laneAllows } from "../quality/rule-contract.js";
|
|
28
|
+
import { parseEnforceLaneRecall } from "./schema.js";
|
|
29
|
+
/** Stored in every record, so the figure explains itself wherever it is read. */
|
|
30
|
+
export const ENFORCE_LANE_NOTE = "Recall of the same attack samples, through the same detection loop as metrics.recall, " +
|
|
31
|
+
"on an engine constructed with lane: 'enforce', which loads only maturity: stable rules: " +
|
|
32
|
+
"the lane README recommends when blocking is on. A match is not necessarily a block (src/verdict.ts " +
|
|
33
|
+
"decides that from severity and confidence), so this is an upper bound on what blocking stops. " +
|
|
34
|
+
"metrics.recall is the hunt lane (the engine default, every maturity). Not a headline.";
|
|
35
|
+
/**
|
|
36
|
+
* Rules an engine in `lane` loads: the engine's own two gates, in its order —
|
|
37
|
+
* status `draft` / `deprecated` is skipped outright (src/engine.ts), then the
|
|
38
|
+
* maturity -> lane mapping every pipeline imports (laneAllows). A loaded rule
|
|
39
|
+
* marked `confirm: embedding` still fires only once embedding similarity
|
|
40
|
+
* confirms it, outside the hunt lane; the synchronous evaluate() most runners
|
|
41
|
+
* use cannot confirm, so on that path such a rule is counted here but never
|
|
42
|
+
* fires.
|
|
43
|
+
*/
|
|
44
|
+
export function rulesInLane(rules, lane) {
|
|
45
|
+
return rules.filter((r) => r.status !== "draft" && r.status !== "deprecated" && laneAllows(r.maturity, lane))
|
|
46
|
+
.length;
|
|
47
|
+
}
|
|
48
|
+
/**
|
|
49
|
+
* Construct and load an enforce-lane engine. Throws when no rule loads: an
|
|
50
|
+
* engine with no rules reports 0 recall, which reads as a measured result.
|
|
51
|
+
*/
|
|
52
|
+
export async function loadEnforceLaneEngine(config) {
|
|
53
|
+
const engine = new ATREngine({ ...config, lane: "enforce" });
|
|
54
|
+
const loaded = await engine.loadRules();
|
|
55
|
+
if (loaded === 0) {
|
|
56
|
+
throw new Error(`enforce lane: no rules loaded from ${config.rulesDir ?? "the bundled rules directory"}`);
|
|
57
|
+
}
|
|
58
|
+
return { engine, rulesInLane: rulesInLane(engine.getRules(), "enforce") };
|
|
59
|
+
}
|
|
60
|
+
/** The record, validated by the same parser verify.ts runs over every file. */
|
|
61
|
+
export function enforceLaneRecall(input) {
|
|
62
|
+
const { detected, attacks, rulesInLane: inLane } = input;
|
|
63
|
+
if (!Number.isInteger(attacks) || attacks < 1) {
|
|
64
|
+
throw new Error(`enforce lane: attacks must be a positive integer, got ${attacks}`);
|
|
65
|
+
}
|
|
66
|
+
if (!Number.isInteger(detected) || detected < 0 || detected > attacks) {
|
|
67
|
+
throw new Error(`enforce lane: detected must be an integer in [0, ${attacks}], got ${detected}`);
|
|
68
|
+
}
|
|
69
|
+
return parseEnforceLaneRecall({
|
|
70
|
+
lane: "enforce",
|
|
71
|
+
detected,
|
|
72
|
+
attacks,
|
|
73
|
+
recall: detected / attacks,
|
|
74
|
+
rules_in_lane: inLane,
|
|
75
|
+
note: ENFORCE_LANE_NOTE,
|
|
76
|
+
});
|
|
77
|
+
}
|
|
78
|
+
/**
|
|
79
|
+
* Run `detect` — the runner's own headline predicate — once per attack sample
|
|
80
|
+
* on the enforce-lane engine, and record how many it detected.
|
|
81
|
+
*/
|
|
82
|
+
export function enforceLaneRecallOf(samples, detect, lane) {
|
|
83
|
+
if (lane.engine.getLane() !== "enforce") {
|
|
84
|
+
throw new Error(`enforce lane: the engine is in the ${lane.engine.getLane()} lane, not enforce`);
|
|
85
|
+
}
|
|
86
|
+
const detected = samples.filter((s) => detect(lane.engine, s)).length;
|
|
87
|
+
return enforceLaneRecall({ detected, attacks: samples.length, rulesInLane: lane.rulesInLane });
|
|
88
|
+
}
|
|
89
|
+
//# sourceMappingURL=enforce-lane.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"enforce-lane.js","sourceRoot":"","sources":["../../src/measurement/enforce-lane.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;GAwBG;AAEH,OAAO,EAAE,SAAS,EAAwB,MAAM,cAAc,CAAC;AAC/D,OAAO,EAAE,UAAU,EAAa,MAAM,6BAA6B,CAAC;AACpE,OAAO,EAAE,sBAAsB,EAA0B,MAAM,aAAa,CAAC;AAI7E,iFAAiF;AACjF,MAAM,CAAC,MAAM,iBAAiB,GAC5B,wFAAwF;IACxF,0FAA0F;IAC1F,qGAAqG;IACrG,gGAAgG;IAChG,uFAAuF,CAAC;AAE1F;;;;;;;;GAQG;AACH,MAAM,UAAU,WAAW,CAAC,KAA0D,EAAE,IAAU;IAChG,OAAO,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,MAAM,KAAK,OAAO,IAAI,CAAC,CAAC,MAAM,KAAK,YAAY,IAAI,UAAU,CAAC,CAAC,CAAC,QAAQ,EAAE,IAAI,CAAC,CAAC;SAC1G,MAAM,CAAC;AACZ,CAAC;AAQD;;;GAGG;AACH,MAAM,CAAC,KAAK,UAAU,qBAAqB,CAAC,MAAqC;IAC/E,MAAM,MAAM,GAAG,IAAI,SAAS,CAAC,EAAE,GAAG,MAAM,EAAE,IAAI,EAAE,SAAS,EAAE,CAAC,CAAC;IAC7D,MAAM,MAAM,GAAG,MAAM,MAAM,CAAC,SAAS,EAAE,CAAC;IACxC,IAAI,MAAM,KAAK,CAAC,EAAE,CAAC;QACjB,MAAM,IAAI,KAAK,CAAC,sCAAsC,MAAM,CAAC,QAAQ,IAAI,6BAA6B,EAAE,CAAC,CAAC;IAC5G,CAAC;IACD,OAAO,EAAE,MAAM,EAAE,WAAW,EAAE,WAAW,CAAC,MAAM,CAAC,QAAQ,EAAE,EAAE,SAAS,CAAC,EAAE,CAAC;AAC5E,CAAC;AAED,+EAA+E;AAC/E,MAAM,UAAU,iBAAiB,CAAC,KAAiE;IACjG,MAAM,EAAE,QAAQ,EAAE,OAAO,EAAE,WAAW,EAAE,MAAM,EAAE,GAAG,KAAK,CAAC;IACzD,IAAI,CAAC,MAAM,CAAC,SAAS,CAAC,OAAO,CAAC,IAAI,OAAO,GAAG,CAAC,EAAE,CAAC;QAC9C,MAAM,IAAI,KAAK,CAAC,yDAAyD,OAAO,EAAE,CAAC,CAAC;IACtF,CAAC;IACD,IAAI,CAAC,MAAM,CAAC,SAAS,CAAC,QAAQ,CAAC,IAAI,QAAQ,GAAG,CAAC,IAAI,QAAQ,GAAG,OAAO,EAAE,CAAC;QACtE,MAAM,IAAI,KAAK,CAAC,oDAAoD,OAAO,UAAU,QAAQ,EAAE,CAAC,CAAC;IACnG,CAAC;IACD,OAAO,sBAAsB,CAAC;QAC5B,IAAI,EAAE,SAAS;QACf,QAAQ;QACR,OAAO;QACP,MAAM,EAAE,QAAQ,GAAG,OAAO;QAC1B,aAAa,EAAE,MAAM;QACrB,IAAI,EAAE,iBAAiB;KACxB,CAAC,CAAC;AACL,CAAC;AAED;;;GAGG;AACH,MAAM,UAAU,mBAAmB,CACjC,OAAqB,EACrB,MAAiD,EACjD,IAAuB;IAEvB,IAAI,IAAI,CAAC,MAAM,CAAC,OAAO,EAAE,KAAK,SAAS,EAAE,CAAC;QACxC,MAAM,IAAI,KAAK,CAAC,sCAAsC,IAAI,CAAC,MAAM,CAAC,OAAO,EAAE,oBAAoB,CAAC,CAAC;IACnG,CAAC;IACD,MAAM,QAAQ,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,MAAM,CAAC,IAAI,CAAC,MAAM,EAAE,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC;IACtE,OAAO,iBAAiB,CAAC,EAAE,QAAQ,EAAE,OAAO,EAAE,OAAO,CAAC,MAAM,EAAE,WAAW,EAAE,IAAI,CAAC,WAAW,EAAE,CAAC,CAAC;AACjG,CAAC"}
|
|
@@ -11,6 +11,8 @@
|
|
|
11
11
|
* accept a structurally-typed input here to avoid creating a cross-module
|
|
12
12
|
* type dependency in either direction.
|
|
13
13
|
*/
|
|
14
|
+
import { type EnforceLaneRecall } from "./schema.js";
|
|
15
|
+
import { type MeasurementInput } from "./write.js";
|
|
14
16
|
/**
|
|
15
17
|
* Structural type matching the fields we read from `EvalReport`. Kept here so
|
|
16
18
|
* `src/eval/*` does not need to export its internal types just to use the
|
|
@@ -51,6 +53,12 @@ export interface FromEvalHarnessOpts {
|
|
|
51
53
|
source_url?: string;
|
|
52
54
|
/** Optional human-readable context. */
|
|
53
55
|
notes?: string;
|
|
56
|
+
/**
|
|
57
|
+
* The same corpus run on an enforce-lane engine (src/eval/eval-harness.ts
|
|
58
|
+
* enforceLaneFromEval). Recorded as breakdown.enforce_lane; the headline
|
|
59
|
+
* metrics stay the report's own (hunt) figures.
|
|
60
|
+
*/
|
|
61
|
+
enforceLane?: EnforceLaneRecall;
|
|
54
62
|
/**
|
|
55
63
|
* Force overwrite if the same filename already exists (same source +
|
|
56
64
|
* source_version + atr_version + date). Defaults to false (append-only).
|
|
@@ -58,9 +66,14 @@ export interface FromEvalHarnessOpts {
|
|
|
58
66
|
force?: boolean;
|
|
59
67
|
}
|
|
60
68
|
/**
|
|
61
|
-
* Build a `MeasurementInput` from an `EvalReport` plus source metadata
|
|
62
|
-
*
|
|
63
|
-
|
|
69
|
+
* Build a `MeasurementInput` from an `EvalReport` plus source metadata. Pure:
|
|
70
|
+
* nothing is written, so tests can assert on exactly what a runner would record.
|
|
71
|
+
*/
|
|
72
|
+
export declare function measurementInputFromEvalReport(report: EvalHarnessReportShape, opts: FromEvalHarnessOpts): MeasurementInput;
|
|
73
|
+
/**
|
|
74
|
+
* Build the `MeasurementInput` (measurementInputFromEvalReport) and write it via
|
|
75
|
+
* `writeMeasurement()`. Returns the same paths/objects that `writeMeasurement()`
|
|
76
|
+
* returns.
|
|
64
77
|
*/
|
|
65
78
|
export declare function writeMeasurementFromEvalReport(report: EvalHarnessReportShape, opts: FromEvalHarnessOpts): {
|
|
66
79
|
measurementPath: string;
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"from-eval-harness.d.ts","sourceRoot":"","sources":["../../src/measurement/from-eval-harness.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;GAYG;
|
|
1
|
+
{"version":3,"file":"from-eval-harness.d.ts","sourceRoot":"","sources":["../../src/measurement/from-eval-harness.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;GAYG;AAEH,OAAO,EAA8B,KAAK,iBAAiB,EAAE,MAAM,aAAa,CAAC;AACjF,OAAO,EAAoB,KAAK,gBAAgB,EAAE,MAAM,YAAY,CAAC;AAErE;;;;GAIG;AACH,MAAM,WAAW,sBAAsB;IACrC,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,UAAU,EAAE,MAAM,CAAC;IACnB,OAAO,EAAE;QACP,MAAM,EAAE,MAAM,CAAC;QACf,SAAS,EAAE,MAAM,CAAC;QAClB,EAAE,EAAE,MAAM,CAAC;QACX,MAAM,EAAE,MAAM,CAAC;QACf,SAAS,EAAE;YAAE,EAAE,EAAE,MAAM,CAAC;YAAC,EAAE,EAAE,MAAM,CAAC;YAAC,EAAE,EAAE,MAAM,CAAC;YAAC,EAAE,EAAE,MAAM,CAAA;SAAE,CAAC;QAC9D,WAAW,CAAC,EAAE,MAAM,CAAC;KACtB,CAAC;IACF,OAAO,CAAC,EAAE;QAAE,GAAG,EAAE,MAAM,CAAC;QAAC,GAAG,EAAE,MAAM,CAAC;QAAC,GAAG,EAAE,MAAM,CAAC;QAAC,IAAI,EAAE,MAAM,CAAC;QAAC,GAAG,EAAE,MAAM,CAAA;KAAE,CAAC;IAC/E,UAAU,CAAC,EAAE,OAAO,CAAC;IACrB,YAAY,CAAC,EAAE,OAAO,CAAC;CACxB;AAED,MAAM,WAAW,mBAAmB;IAClC,kEAAkE;IAClE,MAAM,EAAE,MAAM,CAAC;IACf,0DAA0D;IAC1D,cAAc,EAAE,MAAM,CAAC;IACvB,uCAAuC;IACvC,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,uCAAuC;IACvC,KAAK,CAAC,EAAE,MAAM,CAAC;IACf;;;;OAIG;IACH,WAAW,CAAC,EAAE,iBAAiB,CAAC;IAChC;;;OAGG;IACH,KAAK,CAAC,EAAE,OAAO,CAAC;CACjB;AAED;;;GAGG;AACH,wBAAgB,8BAA8B,CAC5C,MAAM,EAAE,sBAAsB,EAC9B,IAAI,EAAE,mBAAmB,GACxB,gBAAgB,CAuBlB;AAED;;;;GAIG;AACH,wBAAgB,8BAA8B,CAC5C,MAAM,EAAE,sBAAsB,EAC9B,IAAI,EAAE,mBAAmB;;;;EAG1B"}
|
|
@@ -11,13 +11,13 @@
|
|
|
11
11
|
* accept a structurally-typed input here to avoid creating a cross-module
|
|
12
12
|
* type dependency in either direction.
|
|
13
13
|
*/
|
|
14
|
+
import { ENFORCE_LANE_BREAKDOWN_KEY } from "./schema.js";
|
|
14
15
|
import { writeMeasurement } from "./write.js";
|
|
15
16
|
/**
|
|
16
|
-
* Build a `MeasurementInput` from an `EvalReport` plus source metadata
|
|
17
|
-
*
|
|
18
|
-
* `writeMeasurement()` returns.
|
|
17
|
+
* Build a `MeasurementInput` from an `EvalReport` plus source metadata. Pure:
|
|
18
|
+
* nothing is written, so tests can assert on exactly what a runner would record.
|
|
19
19
|
*/
|
|
20
|
-
export function
|
|
20
|
+
export function measurementInputFromEvalReport(report, opts) {
|
|
21
21
|
const input = {
|
|
22
22
|
source: opts.source,
|
|
23
23
|
source_version: opts.source_version,
|
|
@@ -42,8 +42,18 @@ export function writeMeasurementFromEvalReport(report, opts) {
|
|
|
42
42
|
breakdown.by_category = report.byCategory;
|
|
43
43
|
if (report.byDifficulty)
|
|
44
44
|
breakdown.by_difficulty = report.byDifficulty;
|
|
45
|
+
if (opts.enforceLane)
|
|
46
|
+
breakdown[ENFORCE_LANE_BREAKDOWN_KEY] = opts.enforceLane;
|
|
45
47
|
if (Object.keys(breakdown).length > 0)
|
|
46
48
|
input.breakdown = breakdown;
|
|
47
|
-
return
|
|
49
|
+
return input;
|
|
50
|
+
}
|
|
51
|
+
/**
|
|
52
|
+
* Build the `MeasurementInput` (measurementInputFromEvalReport) and write it via
|
|
53
|
+
* `writeMeasurement()`. Returns the same paths/objects that `writeMeasurement()`
|
|
54
|
+
* returns.
|
|
55
|
+
*/
|
|
56
|
+
export function writeMeasurementFromEvalReport(report, opts) {
|
|
57
|
+
return writeMeasurement(measurementInputFromEvalReport(report, opts), { force: opts.force ?? true });
|
|
48
58
|
}
|
|
49
59
|
//# sourceMappingURL=from-eval-harness.js.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"from-eval-harness.js","sourceRoot":"","sources":["../../src/measurement/from-eval-harness.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;GAYG;AAEH,OAAO,EAAE,gBAAgB,EAAyB,MAAM,YAAY,CAAC;
|
|
1
|
+
{"version":3,"file":"from-eval-harness.js","sourceRoot":"","sources":["../../src/measurement/from-eval-harness.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;GAYG;AAEH,OAAO,EAAE,0BAA0B,EAA0B,MAAM,aAAa,CAAC;AACjF,OAAO,EAAE,gBAAgB,EAAyB,MAAM,YAAY,CAAC;AA6CrE;;;GAGG;AACH,MAAM,UAAU,8BAA8B,CAC5C,MAA8B,EAC9B,IAAyB;IAEzB,MAAM,KAAK,GAAqB;QAC9B,MAAM,EAAE,IAAI,CAAC,MAAM;QACnB,cAAc,EAAE,IAAI,CAAC,cAAc;QACnC,WAAW,EAAE,MAAM,CAAC,SAAS;QAC7B,OAAO,EAAE,MAAM,CAAC,UAAU;QAC1B,OAAO,EAAE;YACP,MAAM,EAAE,MAAM,CAAC,OAAO,CAAC,MAAM;YAC7B,SAAS,EAAE,MAAM,CAAC,OAAO,CAAC,SAAS;YACnC,EAAE,EAAE,MAAM,CAAC,OAAO,CAAC,EAAE;YACrB,OAAO,EAAE,MAAM,CAAC,OAAO,CAAC,MAAM;SAC/B;QACD,SAAS,EAAE,MAAM,CAAC,OAAO,CAAC,SAAS;KACpC,CAAC;IACF,IAAI,IAAI,CAAC,UAAU;QAAE,KAAK,CAAC,UAAU,GAAG,IAAI,CAAC,UAAU,CAAC;IACxD,IAAI,IAAI,CAAC,KAAK;QAAE,KAAK,CAAC,KAAK,GAAG,IAAI,CAAC,KAAK,CAAC;IACzC,IAAI,MAAM,CAAC,OAAO;QAAE,KAAK,CAAC,UAAU,GAAG,MAAM,CAAC,OAAO,CAAC;IACtD,MAAM,SAAS,GAA4B,EAAE,CAAC;IAC9C,IAAI,MAAM,CAAC,UAAU;QAAE,SAAS,CAAC,WAAW,GAAG,MAAM,CAAC,UAAU,CAAC;IACjE,IAAI,MAAM,CAAC,YAAY;QAAE,SAAS,CAAC,aAAa,GAAG,MAAM,CAAC,YAAY,CAAC;IACvE,IAAI,IAAI,CAAC,WAAW;QAAE,SAAS,CAAC,0BAA0B,CAAC,GAAG,IAAI,CAAC,WAAW,CAAC;IAC/E,IAAI,MAAM,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC,MAAM,GAAG,CAAC;QAAE,KAAK,CAAC,SAAS,GAAG,SAAS,CAAC;IACnE,OAAO,KAAK,CAAC;AACf,CAAC;AAED;;;;GAIG;AACH,MAAM,UAAU,8BAA8B,CAC5C,MAA8B,EAC9B,IAAyB;IAEzB,OAAO,gBAAgB,CAAC,8BAA8B,CAAC,MAAM,EAAE,IAAI,CAAC,EAAE,EAAE,KAAK,EAAE,IAAI,CAAC,KAAK,IAAI,IAAI,EAAE,CAAC,CAAC;AACvG,CAAC"}
|
|
@@ -0,0 +1,29 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* `breakdown.in_sample_ablation` of a measurement: the same samples re-measured
|
|
3
|
+
* with the rules written from that corpus removed. `reference` names the
|
|
4
|
+
* ablation to cite beside the headline. It approximates an out-of-sample
|
|
5
|
+
* figure; it is not one, so every surface that quotes it calls it the ablation
|
|
6
|
+
* reference.
|
|
7
|
+
*
|
|
8
|
+
* @module agent-threat-rules/measurement/in-sample-ablation
|
|
9
|
+
*/
|
|
10
|
+
export declare const IN_SAMPLE_ABLATION_BREAKDOWN_KEY: "in_sample_ablation";
|
|
11
|
+
export interface AblationLaneFigure {
|
|
12
|
+
readonly detected: number;
|
|
13
|
+
readonly attacks: number;
|
|
14
|
+
readonly recall: number;
|
|
15
|
+
}
|
|
16
|
+
export interface AblationReference {
|
|
17
|
+
/** Key of the cited ablation inside breakdown.in_sample_ablation. */
|
|
18
|
+
readonly key: string;
|
|
19
|
+
readonly removed_rule_ids: readonly string[];
|
|
20
|
+
readonly hunt: AblationLaneFigure;
|
|
21
|
+
readonly enforce: AblationLaneFigure;
|
|
22
|
+
}
|
|
23
|
+
/**
|
|
24
|
+
* The cited ablation, or null when the measurement records none. A record
|
|
25
|
+
* that is present but malformed throws: quoting a headline without the
|
|
26
|
+
* reference it is meant to travel with is the failure this record prevents.
|
|
27
|
+
*/
|
|
28
|
+
export declare function parseAblationReference(breakdown: unknown): AblationReference | null;
|
|
29
|
+
//# sourceMappingURL=in-sample-ablation.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"in-sample-ablation.d.ts","sourceRoot":"","sources":["../../src/measurement/in-sample-ablation.ts"],"names":[],"mappings":"AAAA;;;;;;;;GAQG;AAEH,eAAO,MAAM,gCAAgC,EAAG,oBAA6B,CAAC;AAE9E,MAAM,WAAW,kBAAkB;IACjC,QAAQ,CAAC,QAAQ,EAAE,MAAM,CAAC;IAC1B,QAAQ,CAAC,OAAO,EAAE,MAAM,CAAC;IACzB,QAAQ,CAAC,MAAM,EAAE,MAAM,CAAC;CACzB;AAED,MAAM,WAAW,iBAAiB;IAChC,qEAAqE;IACrE,QAAQ,CAAC,GAAG,EAAE,MAAM,CAAC;IACrB,QAAQ,CAAC,gBAAgB,EAAE,SAAS,MAAM,EAAE,CAAC;IAC7C,QAAQ,CAAC,IAAI,EAAE,kBAAkB,CAAC;IAClC,QAAQ,CAAC,OAAO,EAAE,kBAAkB,CAAC;CACtC;AAcD;;;;GAIG;AACH,wBAAgB,sBAAsB,CAAC,SAAS,EAAE,OAAO,GAAG,iBAAiB,GAAG,IAAI,CAgBnF"}
|
|
@@ -0,0 +1,45 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* `breakdown.in_sample_ablation` of a measurement: the same samples re-measured
|
|
3
|
+
* with the rules written from that corpus removed. `reference` names the
|
|
4
|
+
* ablation to cite beside the headline. It approximates an out-of-sample
|
|
5
|
+
* figure; it is not one, so every surface that quotes it calls it the ablation
|
|
6
|
+
* reference.
|
|
7
|
+
*
|
|
8
|
+
* @module agent-threat-rules/measurement/in-sample-ablation
|
|
9
|
+
*/
|
|
10
|
+
export const IN_SAMPLE_ABLATION_BREAKDOWN_KEY = 'in_sample_ablation';
|
|
11
|
+
const isRecord = (v) => typeof v === 'object' && v !== null && !Array.isArray(v);
|
|
12
|
+
const isCount = (v) => typeof v === 'number' && Number.isInteger(v) && v >= 0;
|
|
13
|
+
function laneFigure(raw, where) {
|
|
14
|
+
if (!isRecord(raw) || !isCount(raw.detected) || !isCount(raw.attacks) || raw.attacks === 0
|
|
15
|
+
|| typeof raw.recall !== 'number' || Math.abs(raw.recall - raw.detected / raw.attacks) > 1e-9) {
|
|
16
|
+
throw new Error(`${where} is malformed (detected / attacks / recall)`);
|
|
17
|
+
}
|
|
18
|
+
return { detected: raw.detected, attacks: raw.attacks, recall: raw.recall };
|
|
19
|
+
}
|
|
20
|
+
/**
|
|
21
|
+
* The cited ablation, or null when the measurement records none. A record
|
|
22
|
+
* that is present but malformed throws: quoting a headline without the
|
|
23
|
+
* reference it is meant to travel with is the failure this record prevents.
|
|
24
|
+
*/
|
|
25
|
+
export function parseAblationReference(breakdown) {
|
|
26
|
+
if (!isRecord(breakdown))
|
|
27
|
+
return null;
|
|
28
|
+
const raw = breakdown[IN_SAMPLE_ABLATION_BREAKDOWN_KEY];
|
|
29
|
+
if (raw === undefined)
|
|
30
|
+
return null;
|
|
31
|
+
const where = `breakdown.${IN_SAMPLE_ABLATION_BREAKDOWN_KEY}`;
|
|
32
|
+
if (!isRecord(raw) || typeof raw.reference !== 'string')
|
|
33
|
+
throw new Error(`${where}.reference is missing`);
|
|
34
|
+
const rec = raw[raw.reference];
|
|
35
|
+
if (!isRecord(rec) || !Array.isArray(rec.removed_rule_ids) || !isRecord(rec.lanes)) {
|
|
36
|
+
throw new Error(`${where}.${raw.reference} is malformed`);
|
|
37
|
+
}
|
|
38
|
+
return {
|
|
39
|
+
key: raw.reference,
|
|
40
|
+
removed_rule_ids: rec.removed_rule_ids.filter((x) => typeof x === 'string'),
|
|
41
|
+
hunt: laneFigure(rec.lanes.hunt, `${where}.${raw.reference}.lanes.hunt`),
|
|
42
|
+
enforce: laneFigure(rec.lanes.enforce, `${where}.${raw.reference}.lanes.enforce`),
|
|
43
|
+
};
|
|
44
|
+
}
|
|
45
|
+
//# sourceMappingURL=in-sample-ablation.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"in-sample-ablation.js","sourceRoot":"","sources":["../../src/measurement/in-sample-ablation.ts"],"names":[],"mappings":"AAAA;;;;;;;;GAQG;AAEH,MAAM,CAAC,MAAM,gCAAgC,GAAG,oBAA6B,CAAC;AAgB9E,MAAM,QAAQ,GAAG,CAAC,CAAU,EAAgC,EAAE,CAC5D,OAAO,CAAC,KAAK,QAAQ,IAAI,CAAC,KAAK,IAAI,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC;AAC3D,MAAM,OAAO,GAAG,CAAC,CAAU,EAAe,EAAE,CAAC,OAAO,CAAC,KAAK,QAAQ,IAAI,MAAM,CAAC,SAAS,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;AAEpG,SAAS,UAAU,CAAC,GAAY,EAAE,KAAa;IAC7C,IAAI,CAAC,QAAQ,CAAC,GAAG,CAAC,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,QAAQ,CAAC,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,OAAO,CAAC,IAAI,GAAG,CAAC,OAAO,KAAK,CAAC;WACrF,OAAO,GAAG,CAAC,MAAM,KAAK,QAAQ,IAAI,IAAI,CAAC,GAAG,CAAC,GAAG,CAAC,MAAM,GAAG,GAAG,CAAC,QAAQ,GAAG,GAAG,CAAC,OAAO,CAAC,GAAG,IAAI,EAAE,CAAC;QAChG,MAAM,IAAI,KAAK,CAAC,GAAG,KAAK,6CAA6C,CAAC,CAAC;IACzE,CAAC;IACD,OAAO,EAAE,QAAQ,EAAE,GAAG,CAAC,QAAQ,EAAE,OAAO,EAAE,GAAG,CAAC,OAAO,EAAE,MAAM,EAAE,GAAG,CAAC,MAAM,EAAE,CAAC;AAC9E,CAAC;AAED;;;;GAIG;AACH,MAAM,UAAU,sBAAsB,CAAC,SAAkB;IACvD,IAAI,CAAC,QAAQ,CAAC,SAAS,CAAC;QAAE,OAAO,IAAI,CAAC;IACtC,MAAM,GAAG,GAAG,SAAS,CAAC,gCAAgC,CAAC,CAAC;IACxD,IAAI,GAAG,KAAK,SAAS;QAAE,OAAO,IAAI,CAAC;IACnC,MAAM,KAAK,GAAG,aAAa,gCAAgC,EAAE,CAAC;IAC9D,IAAI,CAAC,QAAQ,CAAC,GAAG,CAAC,IAAI,OAAO,GAAG,CAAC,SAAS,KAAK,QAAQ;QAAE,MAAM,IAAI,KAAK,CAAC,GAAG,KAAK,uBAAuB,CAAC,CAAC;IAC1G,MAAM,GAAG,GAAG,GAAG,CAAC,GAAG,CAAC,SAAS,CAAC,CAAC;IAC/B,IAAI,CAAC,QAAQ,CAAC,GAAG,CAAC,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,GAAG,CAAC,gBAAgB,CAAC,IAAI,CAAC,QAAQ,CAAC,GAAG,CAAC,KAAK,CAAC,EAAE,CAAC;QACnF,MAAM,IAAI,KAAK,CAAC,GAAG,KAAK,IAAI,GAAG,CAAC,SAAS,eAAe,CAAC,CAAC;IAC5D,CAAC;IACD,OAAO;QACL,GAAG,EAAE,GAAG,CAAC,SAAS;QAClB,gBAAgB,EAAE,GAAG,CAAC,gBAAgB,CAAC,MAAM,CAAC,CAAC,CAAC,EAAe,EAAE,CAAC,OAAO,CAAC,KAAK,QAAQ,CAAC;QACxF,IAAI,EAAE,UAAU,CAAC,GAAG,CAAC,KAAK,CAAC,IAAI,EAAE,GAAG,KAAK,IAAI,GAAG,CAAC,SAAS,aAAa,CAAC;QACxE,OAAO,EAAE,UAAU,CAAC,GAAG,CAAC,KAAK,CAAC,OAAO,EAAE,GAAG,KAAK,IAAI,GAAG,CAAC,SAAS,gBAAgB,CAAC;KAClF,CAAC;AACJ,CAAC"}
|
|
@@ -61,9 +61,41 @@ export interface LatencyMs {
|
|
|
61
61
|
* - eval-harness: by category and by difficulty
|
|
62
62
|
*
|
|
63
63
|
* Kept as `Record<string, unknown>` because the structure is source-defined.
|
|
64
|
-
* Consumers of this field must validate per-source.
|
|
64
|
+
* Consumers of this field must validate per-source. The one exception is
|
|
65
|
+
* `enforce_lane` (below), which every runner records the same way and this
|
|
66
|
+
* module validates.
|
|
65
67
|
*/
|
|
66
68
|
export type Breakdown = Record<string, unknown>;
|
|
69
|
+
/**
|
|
70
|
+
* `breakdown.enforce_lane`: the same samples, through the same detection loop
|
|
71
|
+
* as `metrics.recall`, on an engine constructed with `lane: 'enforce'`.
|
|
72
|
+
*
|
|
73
|
+
* `metrics.recall` is a hunt-lane figure (an engine given no lane loads every
|
|
74
|
+
* maturity); the enforce lane, the one to run with blocking on, loads only
|
|
75
|
+
* `maturity: stable`. The two can differ by two orders of magnitude, so a
|
|
76
|
+
* runner records both — and this one in the breakdown, deliberately not as a
|
|
77
|
+
* second headline: `metrics`, latest.json and data/stats.json stay hunt.
|
|
78
|
+
* Built by src/measurement/enforce-lane.ts.
|
|
79
|
+
*/
|
|
80
|
+
export interface EnforceLaneRecall {
|
|
81
|
+
lane: "enforce";
|
|
82
|
+
/** Attack samples the enforce-lane engine detected. */
|
|
83
|
+
detected: number;
|
|
84
|
+
/** Attack samples evaluated — the headline's `confusion.tp + confusion.fn`. */
|
|
85
|
+
attacks: number;
|
|
86
|
+
/** detected / attacks. */
|
|
87
|
+
recall: number;
|
|
88
|
+
/**
|
|
89
|
+
* Rules the enforce lane loads: status not draft/deprecated, maturity stable.
|
|
90
|
+
* Includes any `confirm: embedding` rule, which the synchronous evaluate()
|
|
91
|
+
* path most runners use never fires outside the hunt lane.
|
|
92
|
+
*/
|
|
93
|
+
rules_in_lane: number;
|
|
94
|
+
/** What the figure is, in the record itself. */
|
|
95
|
+
note: string;
|
|
96
|
+
}
|
|
97
|
+
/** Breakdown key every runner records the enforce-lane figure under. */
|
|
98
|
+
export declare const ENFORCE_LANE_BREAKDOWN_KEY: "enforce_lane";
|
|
67
99
|
/**
|
|
68
100
|
* A single measurement run.
|
|
69
101
|
*
|
|
@@ -130,6 +162,11 @@ export declare class MeasurementSchemaError extends Error {
|
|
|
130
162
|
readonly reason: string;
|
|
131
163
|
constructor(path: string, reason: string);
|
|
132
164
|
}
|
|
165
|
+
/**
|
|
166
|
+
* Parse `breakdown.enforce_lane`. Strict: a figure whose recall disagrees with
|
|
167
|
+
* its own counts, or that names another lane, is a number nothing backs.
|
|
168
|
+
*/
|
|
169
|
+
export declare function parseEnforceLaneRecall(raw: unknown, path?: string): EnforceLaneRecall;
|
|
133
170
|
/**
|
|
134
171
|
* Parse a `Measurement` from an arbitrary value. Throws `MeasurementSchemaError`
|
|
135
172
|
* on any deviation from the schema. Returns a strongly-typed `Measurement`.
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"schema.d.ts","sourceRoot":"","sources":["../../src/measurement/schema.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;GAiBG;AAIH,+DAA+D;AAC/D,eAAO,MAAM,sBAAsB,EAAG,GAAY,CAAC;AAInD;;;;;GAKG;AACH,MAAM,WAAW,OAAO;IACtB,yEAAyE;IACzE,MAAM,EAAE,MAAM,CAAC;IACf,2DAA2D;IAC3D,SAAS,EAAE,MAAM,CAAC;IAClB,mEAAmE;IACnE,EAAE,EAAE,MAAM,CAAC;IACX,kGAAkG;IAClG,OAAO,EAAE,MAAM,CAAC;CACjB;AAED,sEAAsE;AACtE,MAAM,WAAW,SAAS;IACxB,6DAA6D;IAC7D,EAAE,EAAE,MAAM,CAAC;IACX,yDAAyD;IACzD,EAAE,EAAE,MAAM,CAAC;IACX,8DAA8D;IAC9D,EAAE,EAAE,MAAM,CAAC;IACX,yEAAyE;IACzE,EAAE,EAAE,MAAM,CAAC;CACZ;AAED,wDAAwD;AACxD,MAAM,WAAW,SAAS;IACxB,GAAG,EAAE,MAAM,CAAC;IACZ,GAAG,EAAE,MAAM,CAAC;IACZ,GAAG,EAAE,MAAM,CAAC;IACZ,IAAI,EAAE,MAAM,CAAC;IACb,GAAG,EAAE,MAAM,CAAC;CACb;AAED
|
|
1
|
+
{"version":3,"file":"schema.d.ts","sourceRoot":"","sources":["../../src/measurement/schema.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;GAiBG;AAIH,+DAA+D;AAC/D,eAAO,MAAM,sBAAsB,EAAG,GAAY,CAAC;AAInD;;;;;GAKG;AACH,MAAM,WAAW,OAAO;IACtB,yEAAyE;IACzE,MAAM,EAAE,MAAM,CAAC;IACf,2DAA2D;IAC3D,SAAS,EAAE,MAAM,CAAC;IAClB,mEAAmE;IACnE,EAAE,EAAE,MAAM,CAAC;IACX,kGAAkG;IAClG,OAAO,EAAE,MAAM,CAAC;CACjB;AAED,sEAAsE;AACtE,MAAM,WAAW,SAAS;IACxB,6DAA6D;IAC7D,EAAE,EAAE,MAAM,CAAC;IACX,yDAAyD;IACzD,EAAE,EAAE,MAAM,CAAC;IACX,8DAA8D;IAC9D,EAAE,EAAE,MAAM,CAAC;IACX,yEAAyE;IACzE,EAAE,EAAE,MAAM,CAAC;CACZ;AAED,wDAAwD;AACxD,MAAM,WAAW,SAAS;IACxB,GAAG,EAAE,MAAM,CAAC;IACZ,GAAG,EAAE,MAAM,CAAC;IACZ,GAAG,EAAE,MAAM,CAAC;IACZ,IAAI,EAAE,MAAM,CAAC;IACb,GAAG,EAAE,MAAM,CAAC;CACb;AAED;;;;;;;;;;;GAWG;AACH,MAAM,MAAM,SAAS,GAAG,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;AAEhD;;;;;;;;;;GAUG;AACH,MAAM,WAAW,iBAAiB;IAChC,IAAI,EAAE,SAAS,CAAC;IAChB,uDAAuD;IACvD,QAAQ,EAAE,MAAM,CAAC;IACjB,+EAA+E;IAC/E,OAAO,EAAE,MAAM,CAAC;IAChB,0BAA0B;IAC1B,MAAM,EAAE,MAAM,CAAC;IACf;;;;OAIG;IACH,aAAa,EAAE,MAAM,CAAC;IACtB,gDAAgD;IAChD,IAAI,EAAE,MAAM,CAAC;CACd;AAED,wEAAwE;AACxE,eAAO,MAAM,0BAA0B,EAAG,cAAuB,CAAC;AAElE;;;;GAIG;AACH,MAAM,WAAW,WAAW;IAC1B,uCAAuC;IACvC,cAAc,EAAE,OAAO,sBAAsB,CAAC;IAE9C,wGAAwG;IACxG,MAAM,EAAE,MAAM,CAAC;IAEf,qEAAqE;IACrE,cAAc,EAAE,MAAM,CAAC;IAEvB,oEAAoE;IACpE,UAAU,CAAC,EAAE,MAAM,CAAC;IAEpB,4DAA4D;IAC5D,aAAa,CAAC,EAAE,MAAM,CAAC;IAEvB,iEAAiE;IACjE,WAAW,EAAE,MAAM,CAAC;IAEpB,sDAAsD;IACtD,UAAU,EAAE,MAAM,CAAC;IAEnB,0DAA0D;IAC1D,YAAY,EAAE,MAAM,CAAC;IAErB,qEAAqE;IACrE,WAAW,EAAE,MAAM,CAAC;IAEpB,kDAAkD;IAClD,OAAO,EAAE,MAAM,CAAC;IAEhB,uCAAuC;IACvC,OAAO,EAAE,OAAO,CAAC;IAEjB,wDAAwD;IACxD,SAAS,CAAC,EAAE,SAAS,CAAC;IAEtB,wCAAwC;IACxC,UAAU,CAAC,EAAE,SAAS,CAAC;IAEvB,mFAAmF;IACnF,SAAS,CAAC,EAAE,SAAS,CAAC;IAEtB,+CAA+C;IAC/C,KAAK,CAAC,EAAE,MAAM,CAAC;CAChB;AAED;;;;;GAKG;AACH,MAAM,WAAW,aAAa;IAC5B,MAAM,EAAE,MAAM,CAAC;IACf,iFAAiF;IACjF,IAAI,EAAE,MAAM,CAAC;IACb,sEAAsE;IACtE,WAAW,EAAE,MAAM,CAAC;IACpB,oFAAoF;IACpF,OAAO,EAAE,OAAO,CAAC;IACjB,wEAAwE;IACxE,cAAc,EAAE,MAAM,CAAC;IACvB,WAAW,EAAE,MAAM,CAAC;IACpB,2BAA2B;IAC3B,OAAO,EAAE,MAAM,CAAC;CACjB;AAID;;;GAGG;AACH,qBAAa,sBAAuB,SAAQ,KAAK;aAE7B,IAAI,EAAE,MAAM;aACZ,MAAM,EAAE,MAAM;gBADd,IAAI,EAAE,MAAM,EACZ,MAAM,EAAE,MAAM;CAKjC;AAkED;;;GAGG;AACH,wBAAgB,sBAAsB,CAAC,GAAG,EAAE,OAAO,EAAE,IAAI,SAA6B,GAAG,iBAAiB,CAmBzG;AAsBD;;;;;;GAMG;AACH,wBAAgB,gBAAgB,CAAC,GAAG,EAAE,OAAO,GAAG,WAAW,CA4C1D;AAED,2DAA2D;AAC3D,wBAAgB,kBAAkB,CAAC,GAAG,EAAE,OAAO,GAAG,aAAa,CAW9D;AAED;;;;;;;GAOG;AACH,wBAAgB,mBAAmB,CAAC,CAAC,EAAE,IAAI,CAAC,WAAW,EAAE,aAAa,GAAG,QAAQ,GAAG,gBAAgB,GAAG,aAAa,CAAC,GAAG,MAAM,CAM7H"}
|
|
@@ -19,6 +19,8 @@
|
|
|
19
19
|
// ─── Schema version ─────────────────────────────────────────────────────────
|
|
20
20
|
/** Bump this when the schema breaks backward compatibility. */
|
|
21
21
|
export const CURRENT_SCHEMA_VERSION = "1";
|
|
22
|
+
/** Breakdown key every runner records the enforce-lane figure under. */
|
|
23
|
+
export const ENFORCE_LANE_BREAKDOWN_KEY = "enforce_lane";
|
|
22
24
|
// ─── Validation ─────────────────────────────────────────────────────────────
|
|
23
25
|
/**
|
|
24
26
|
* Error thrown when a measurement file fails schema validation.
|
|
@@ -92,6 +94,44 @@ function parseLatency(raw, path) {
|
|
|
92
94
|
max: assertNumber(obj.max, `${path}.max`, { min: 0 }),
|
|
93
95
|
};
|
|
94
96
|
}
|
|
97
|
+
/**
|
|
98
|
+
* Parse `breakdown.enforce_lane`. Strict: a figure whose recall disagrees with
|
|
99
|
+
* its own counts, or that names another lane, is a number nothing backs.
|
|
100
|
+
*/
|
|
101
|
+
export function parseEnforceLaneRecall(raw, path = "$.breakdown.enforce_lane") {
|
|
102
|
+
const obj = assertObject(raw, path);
|
|
103
|
+
if (obj.lane !== "enforce") {
|
|
104
|
+
throw new MeasurementSchemaError(`${path}.lane`, `expected "enforce", got ${JSON.stringify(obj.lane)}`);
|
|
105
|
+
}
|
|
106
|
+
const attacks = assertNumber(obj.attacks, `${path}.attacks`, { min: 1, integer: true });
|
|
107
|
+
const detected = assertNumber(obj.detected, `${path}.detected`, { min: 0, max: attacks, integer: true });
|
|
108
|
+
const recall = assertNumber(obj.recall, `${path}.recall`, { min: 0, max: 1 });
|
|
109
|
+
if (Math.abs(recall - detected / attacks) > 1e-9) {
|
|
110
|
+
throw new MeasurementSchemaError(`${path}.recall`, `${recall} != detected / attacks = ${detected}/${attacks}`);
|
|
111
|
+
}
|
|
112
|
+
return {
|
|
113
|
+
lane: "enforce",
|
|
114
|
+
detected,
|
|
115
|
+
attacks,
|
|
116
|
+
recall,
|
|
117
|
+
rules_in_lane: assertNumber(obj.rules_in_lane, `${path}.rules_in_lane`, { min: 0, integer: true }),
|
|
118
|
+
note: assertString(obj.note, `${path}.note`, { nonEmpty: true }),
|
|
119
|
+
};
|
|
120
|
+
}
|
|
121
|
+
/**
|
|
122
|
+
* Validate `breakdown.enforce_lane` when present, and hold it to the headline's
|
|
123
|
+
* sample set: the enforce run covers the attacks the headline counted
|
|
124
|
+
* (`tp + fn`), or the two figures are not about the same corpus.
|
|
125
|
+
*/
|
|
126
|
+
function checkEnforceLane(breakdown, confusion) {
|
|
127
|
+
if (breakdown[ENFORCE_LANE_BREAKDOWN_KEY] === undefined)
|
|
128
|
+
return;
|
|
129
|
+
const lane = parseEnforceLaneRecall(breakdown[ENFORCE_LANE_BREAKDOWN_KEY]);
|
|
130
|
+
if (confusion !== undefined && lane.attacks !== confusion.tp + confusion.fn) {
|
|
131
|
+
throw new MeasurementSchemaError("$.breakdown.enforce_lane.attacks", `${lane.attacks} != confusion.tp + confusion.fn = ${confusion.tp + confusion.fn}: ` +
|
|
132
|
+
"the enforce run must cover the samples the headline counted");
|
|
133
|
+
}
|
|
134
|
+
}
|
|
95
135
|
const ISO8601_RE = /^\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}(?:\.\d+)?Z$/;
|
|
96
136
|
const SOURCE_RE = /^[a-z0-9][a-z0-9-]*$/;
|
|
97
137
|
/**
|
|
@@ -134,8 +174,10 @@ export function parseMeasurement(raw) {
|
|
|
134
174
|
m.confusion = parseConfusion(obj.confusion, "$.confusion");
|
|
135
175
|
if (obj.latency_ms !== undefined)
|
|
136
176
|
m.latency_ms = parseLatency(obj.latency_ms, "$.latency_ms");
|
|
137
|
-
if (obj.breakdown !== undefined)
|
|
177
|
+
if (obj.breakdown !== undefined) {
|
|
138
178
|
m.breakdown = assertObject(obj.breakdown, "$.breakdown");
|
|
179
|
+
checkEnforceLane(m.breakdown, m.confusion);
|
|
180
|
+
}
|
|
139
181
|
if (obj.notes !== undefined)
|
|
140
182
|
m.notes = assertString(obj.notes, "$.notes");
|
|
141
183
|
return m;
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"schema.js","sourceRoot":"","sources":["../../src/measurement/schema.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;GAiBG;AAEH,+EAA+E;AAE/E,+DAA+D;AAC/D,MAAM,CAAC,MAAM,sBAAsB,GAAG,GAAY,CAAC;
|
|
1
|
+
{"version":3,"file":"schema.js","sourceRoot":"","sources":["../../src/measurement/schema.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;GAiBG;AAEH,+EAA+E;AAE/E,+DAA+D;AAC/D,MAAM,CAAC,MAAM,sBAAsB,GAAG,GAAY,CAAC;AAqFnD,wEAAwE;AACxE,MAAM,CAAC,MAAM,0BAA0B,GAAG,cAAuB,CAAC;AA2ElE,+EAA+E;AAE/E;;;GAGG;AACH,MAAM,OAAO,sBAAuB,SAAQ,KAAK;IAE7B;IACA;IAFlB,YACkB,IAAY,EACZ,MAAc;QAE9B,KAAK,CAAC,uBAAuB,IAAI,KAAK,MAAM,EAAE,CAAC,CAAC;QAHhC,SAAI,GAAJ,IAAI,CAAQ;QACZ,WAAM,GAAN,MAAM,CAAQ;QAG9B,IAAI,CAAC,IAAI,GAAG,wBAAwB,CAAC;IACvC,CAAC;CACF;AAED,SAAS,YAAY,CAAC,CAAU,EAAE,IAAY,EAAE,IAA6B;IAC3E,IAAI,OAAO,CAAC,KAAK,QAAQ,EAAE,CAAC;QAC1B,MAAM,IAAI,sBAAsB,CAAC,IAAI,EAAE,wBAAwB,OAAO,CAAC,EAAE,CAAC,CAAC;IAC7E,CAAC;IACD,IAAI,IAAI,EAAE,QAAQ,IAAI,CAAC,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QACrC,MAAM,IAAI,sBAAsB,CAAC,IAAI,EAAE,2BAA2B,CAAC,CAAC;IACtE,CAAC;IACD,OAAO,CAAC,CAAC;AACX,CAAC;AAED,SAAS,YAAY,CAAC,CAAU,EAAE,IAAY,EAAE,IAAwD;IACtG,IAAI,OAAO,CAAC,KAAK,QAAQ,IAAI,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC,IAAI,CAAC,MAAM,CAAC,QAAQ,CAAC,CAAC,CAAC,EAAE,CAAC;QACpE,MAAM,IAAI,sBAAsB,CAAC,IAAI,EAAE,+BAA+B,IAAI,CAAC,SAAS,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC;IAC7F,CAAC;IACD,IAAI,IAAI,EAAE,OAAO,IAAI,CAAC,MAAM,CAAC,SAAS,CAAC,CAAC,CAAC,EAAE,CAAC;QAC1C,MAAM,IAAI,sBAAsB,CAAC,IAAI,EAAE,yBAAyB,CAAC,EAAE,CAAC,CAAC;IACvE,CAAC;IACD,IAAI,IAAI,EAAE,GAAG,KAAK,SAAS,IAAI,CAAC,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;QAC5C,MAAM,IAAI,sBAAsB,CAAC,IAAI,EAAE,eAAe,IAAI,CAAC,GAAG,SAAS,CAAC,EAAE,CAAC,CAAC;IAC9E,CAAC;IACD,IAAI,IAAI,EAAE,GAAG,KAAK,SAAS,IAAI,CAAC,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;QAC5C,MAAM,IAAI,sBAAsB,CAAC,IAAI,EAAE,eAAe,IAAI,CAAC,GAAG,SAAS,CAAC,EAAE,CAAC,CAAC;IAC9E,CAAC;IACD,OAAO,CAAC,CAAC;AACX,CAAC;AAED,SAAS,YAAY,CAAC,CAAU,EAAE,IAAY;IAC5C,IAAI,CAAC,KAAK,IAAI,IAAI,OAAO,CAAC,KAAK,QAAQ,IAAI,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,CAAC;QAC5D,MAAM,IAAI,sBAAsB,CAAC,IAAI,EAAE,wBAAwB,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,OAAO,CAAC,EAAE,CAAC,CAAC;IAC1G,CAAC;IACD,OAAO,CAA4B,CAAC;AACtC,CAAC;AAED,SAAS,YAAY,CAAC,GAAY,EAAE,IAAY;IAC9C,MAAM,GAAG,GAAG,YAAY,CAAC,GAAG,EAAE,IAAI,CAAC,CAAC;IACpC,OAAO;QACL,MAAM,EAAE,YAAY,CAAC,GAAG,CAAC,MAAM,EAAE,GAAG,IAAI,SAAS,EAAE,EAAE,GAAG,EAAE,CAAC,EAAE,GAAG,EAAE,CAAC,EAAE,CAAC;QACtE,SAAS,EAAE,YAAY,CAAC,GAAG,CAAC,SAAS,EAAE,GAAG,IAAI,YAAY,EAAE,EAAE,GAAG,EAAE,CAAC,EAAE,GAAG,EAAE,CAAC,EAAE,CAAC;QAC/E,EAAE,EAAE,YAAY,CAAC,GAAG,CAAC,EAAE,EAAE,GAAG,IAAI,KAAK,EAAE,EAAE,GAAG,EAAE,CAAC,EAAE,GAAG,EAAE,CAAC,EAAE,CAAC;QAC1D,OAAO,EAAE,YAAY,CAAC,GAAG,CAAC,OAAO,EAAE,GAAG,IAAI,UAAU,EAAE,EAAE,GAAG,EAAE,CAAC,EAAE,GAAG,EAAE,CAAC,EAAE,CAAC;KAC1E,CAAC;AACJ,CAAC;AAED,SAAS,cAAc,CAAC,GAAY,EAAE,IAAY;IAChD,MAAM,GAAG,GAAG,YAAY,CAAC,GAAG,EAAE,IAAI,CAAC,CAAC;IACpC,OAAO;QACL,EAAE,EAAE,YAAY,CAAC,GAAG,CAAC,EAAE,EAAE,GAAG,IAAI,KAAK,EAAE,EAAE,GAAG,EAAE,CAAC,EAAE,OAAO,EAAE,IAAI,EAAE,CAAC;QACjE,EAAE,EAAE,YAAY,CAAC,GAAG,CAAC,EAAE,EAAE,GAAG,IAAI,KAAK,EAAE,EAAE,GAAG,EAAE,CAAC,EAAE,OAAO,EAAE,IAAI,EAAE,CAAC;QACjE,EAAE,EAAE,YAAY,CAAC,GAAG,CAAC,EAAE,EAAE,GAAG,IAAI,KAAK,EAAE,EAAE,GAAG,EAAE,CAAC,EAAE,OAAO,EAAE,IAAI,EAAE,CAAC;QACjE,EAAE,EAAE,YAAY,CAAC,GAAG,CAAC,EAAE,EAAE,GAAG,IAAI,KAAK,EAAE,EAAE,GAAG,EAAE,CAAC,EAAE,OAAO,EAAE,IAAI,EAAE,CAAC;KAClE,CAAC;AACJ,CAAC;AAED,SAAS,YAAY,CAAC,GAAY,EAAE,IAAY;IAC9C,MAAM,GAAG,GAAG,YAAY,CAAC,GAAG,EAAE,IAAI,CAAC,CAAC;IACpC,OAAO;QACL,GAAG,EAAE,YAAY,CAAC,GAAG,CAAC,GAAG,EAAE,GAAG,IAAI,MAAM,EAAE,EAAE,GAAG,EAAE,CAAC,EAAE,CAAC;QACrD,GAAG,EAAE,YAAY,CAAC,GAAG,CAAC,GAAG,EAAE,GAAG,IAAI,MAAM,EAAE,EAAE,GAAG,EAAE,CAAC,EAAE,CAAC;QACrD,GAAG,EAAE,YAAY,CAAC,GAAG,CAAC,GAAG,EAAE,GAAG,IAAI,MAAM,EAAE,EAAE,GAAG,EAAE,CAAC,EAAE,CAAC;QACrD,IAAI,EAAE,YAAY,CAAC,GAAG,CAAC,IAAI,EAAE,GAAG,IAAI,OAAO,EAAE,EAAE,GAAG,EAAE,CAAC,EAAE,CAAC;QACxD,GAAG,EAAE,YAAY,CAAC,GAAG,CAAC,GAAG,EAAE,GAAG,IAAI,MAAM,EAAE,EAAE,GAAG,EAAE,CAAC,EAAE,CAAC;KACtD,CAAC;AACJ,CAAC;AAED;;;GAGG;AACH,MAAM,UAAU,sBAAsB,CAAC,GAAY,EAAE,IAAI,GAAG,0BAA0B;IACpF,MAAM,GAAG,GAAG,YAAY,CAAC,GAAG,EAAE,IAAI,CAAC,CAAC;IACpC,IAAI,GAAG,CAAC,IAAI,KAAK,SAAS,EAAE,CAAC;QAC3B,MAAM,IAAI,sBAAsB,CAAC,GAAG,IAAI,OAAO,EAAE,2BAA2B,IAAI,CAAC,SAAS,CAAC,GAAG,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;IAC1G,CAAC;IACD,MAAM,OAAO,GAAG,YAAY,CAAC,GAAG,CAAC,OAAO,EAAE,GAAG,IAAI,UAAU,EAAE,EAAE,GAAG,EAAE,CAAC,EAAE,OAAO,EAAE,IAAI,EAAE,CAAC,CAAC;IACxF,MAAM,QAAQ,GAAG,YAAY,CAAC,GAAG,CAAC,QAAQ,EAAE,GAAG,IAAI,WAAW,EAAE,EAAE,GAAG,EAAE,CAAC,EAAE,GAAG,EAAE,OAAO,EAAE,OAAO,EAAE,IAAI,EAAE,CAAC,CAAC;IACzG,MAAM,MAAM,GAAG,YAAY,CAAC,GAAG,CAAC,MAAM,EAAE,GAAG,IAAI,SAAS,EAAE,EAAE,GAAG,EAAE,CAAC,EAAE,GAAG,EAAE,CAAC,EAAE,CAAC,CAAC;IAC9E,IAAI,IAAI,CAAC,GAAG,CAAC,MAAM,GAAG,QAAQ,GAAG,OAAO,CAAC,GAAG,IAAI,EAAE,CAAC;QACjD,MAAM,IAAI,sBAAsB,CAAC,GAAG,IAAI,SAAS,EAAE,GAAG,MAAM,4BAA4B,QAAQ,IAAI,OAAO,EAAE,CAAC,CAAC;IACjH,CAAC;IACD,OAAO;QACL,IAAI,EAAE,SAAS;QACf,QAAQ;QACR,OAAO;QACP,MAAM;QACN,aAAa,EAAE,YAAY,CAAC,GAAG,CAAC,aAAa,EAAE,GAAG,IAAI,gBAAgB,EAAE,EAAE,GAAG,EAAE,CAAC,EAAE,OAAO,EAAE,IAAI,EAAE,CAAC;QAClG,IAAI,EAAE,YAAY,CAAC,GAAG,CAAC,IAAI,EAAE,GAAG,IAAI,OAAO,EAAE,EAAE,QAAQ,EAAE,IAAI,EAAE,CAAC;KACjE,CAAC;AACJ,CAAC;AAED;;;;GAIG;AACH,SAAS,gBAAgB,CAAC,SAAoB,EAAE,SAAgC;IAC9E,IAAI,SAAS,CAAC,0BAA0B,CAAC,KAAK,SAAS;QAAE,OAAO;IAChE,MAAM,IAAI,GAAG,sBAAsB,CAAC,SAAS,CAAC,0BAA0B,CAAC,CAAC,CAAC;IAC3E,IAAI,SAAS,KAAK,SAAS,IAAI,IAAI,CAAC,OAAO,KAAK,SAAS,CAAC,EAAE,GAAG,SAAS,CAAC,EAAE,EAAE,CAAC;QAC5E,MAAM,IAAI,sBAAsB,CAC9B,kCAAkC,EAClC,GAAG,IAAI,CAAC,OAAO,qCAAqC,SAAS,CAAC,EAAE,GAAG,SAAS,CAAC,EAAE,IAAI;YACjF,6DAA6D,CAChE,CAAC;IACJ,CAAC;AACH,CAAC;AAED,MAAM,UAAU,GAAG,kDAAkD,CAAC;AACtE,MAAM,SAAS,GAAG,sBAAsB,CAAC;AAEzC;;;;;;GAMG;AACH,MAAM,UAAU,gBAAgB,CAAC,GAAY;IAC3C,MAAM,GAAG,GAAG,YAAY,CAAC,GAAG,EAAE,GAAG,CAAC,CAAC;IACnC,MAAM,cAAc,GAAG,YAAY,CAAC,GAAG,CAAC,cAAc,EAAE,kBAAkB,EAAE,EAAE,QAAQ,EAAE,IAAI,EAAE,CAAC,CAAC;IAChG,IAAI,cAAc,KAAK,sBAAsB,EAAE,CAAC;QAC9C,MAAM,IAAI,sBAAsB,CAC9B,kBAAkB,EAClB,aAAa,sBAAsB,WAAW,cAAc,kCAAkC,CAC/F,CAAC;IACJ,CAAC;IACD,MAAM,MAAM,GAAG,YAAY,CAAC,GAAG,CAAC,MAAM,EAAE,UAAU,EAAE,EAAE,QAAQ,EAAE,IAAI,EAAE,CAAC,CAAC;IACxE,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,MAAM,CAAC,EAAE,CAAC;QAC5B,MAAM,IAAI,sBAAsB,CAC9B,UAAU,EACV,2CAA2C,MAAM,GAAG,CACrD,CAAC;IACJ,CAAC;IACD,MAAM,WAAW,GAAG,YAAY,CAAC,GAAG,CAAC,WAAW,EAAE,eAAe,EAAE,EAAE,QAAQ,EAAE,IAAI,EAAE,CAAC,CAAC;IACvF,IAAI,CAAC,UAAU,CAAC,IAAI,CAAC,WAAW,CAAC,EAAE,CAAC;QAClC,MAAM,IAAI,sBAAsB,CAC9B,eAAe,EACf,4DAA4D,WAAW,GAAG,CAC3E,CAAC;IACJ,CAAC;IACD,MAAM,CAAC,GAAgB;QACrB,cAAc,EAAE,sBAAsB;QACtC,MAAM;QACN,cAAc,EAAE,YAAY,CAAC,GAAG,CAAC,cAAc,EAAE,kBAAkB,EAAE,EAAE,QAAQ,EAAE,IAAI,EAAE,CAAC;QACxF,WAAW,EAAE,YAAY,CAAC,GAAG,CAAC,WAAW,EAAE,eAAe,EAAE,EAAE,QAAQ,EAAE,IAAI,EAAE,CAAC;QAC/E,UAAU,EAAE,YAAY,CAAC,GAAG,CAAC,UAAU,EAAE,cAAc,EAAE,EAAE,QAAQ,EAAE,IAAI,EAAE,CAAC;QAC5E,YAAY,EAAE,YAAY,CAAC,GAAG,CAAC,YAAY,EAAE,gBAAgB,EAAE,EAAE,GAAG,EAAE,CAAC,EAAE,OAAO,EAAE,IAAI,EAAE,CAAC;QACzF,WAAW;QACX,OAAO,EAAE,YAAY,CAAC,GAAG,CAAC,OAAO,EAAE,WAAW,EAAE,EAAE,GAAG,EAAE,CAAC,EAAE,OAAO,EAAE,IAAI,EAAE,CAAC;QAC1E,OAAO,EAAE,YAAY,CAAC,GAAG,CAAC,OAAO,EAAE,WAAW,CAAC;KAChD,CAAC;IACF,IAAI,GAAG,CAAC,UAAU,KAAK,SAAS;QAAE,CAAC,CAAC,UAAU,GAAG,YAAY,CAAC,GAAG,CAAC,UAAU,EAAE,cAAc,EAAE,EAAE,QAAQ,EAAE,IAAI,EAAE,CAAC,CAAC;IAClH,IAAI,GAAG,CAAC,aAAa,KAAK,SAAS;QAAE,CAAC,CAAC,aAAa,GAAG,YAAY,CAAC,GAAG,CAAC,aAAa,EAAE,iBAAiB,EAAE,EAAE,QAAQ,EAAE,IAAI,EAAE,CAAC,CAAC;IAC9H,IAAI,GAAG,CAAC,SAAS,KAAK,SAAS;QAAE,CAAC,CAAC,SAAS,GAAG,cAAc,CAAC,GAAG,CAAC,SAAS,EAAE,aAAa,CAAC,CAAC;IAC5F,IAAI,GAAG,CAAC,UAAU,KAAK,SAAS;QAAE,CAAC,CAAC,UAAU,GAAG,YAAY,CAAC,GAAG,CAAC,UAAU,EAAE,cAAc,CAAC,CAAC;IAC9F,IAAI,GAAG,CAAC,SAAS,KAAK,SAAS,EAAE,CAAC;QAChC,CAAC,CAAC,SAAS,GAAG,YAAY,CAAC,GAAG,CAAC,SAAS,EAAE,aAAa,CAAC,CAAC;QACzD,gBAAgB,CAAC,CAAC,CAAC,SAAS,EAAE,CAAC,CAAC,SAAS,CAAC,CAAC;IAC7C,CAAC;IACD,IAAI,GAAG,CAAC,KAAK,KAAK,SAAS;QAAE,CAAC,CAAC,KAAK,GAAG,YAAY,CAAC,GAAG,CAAC,KAAK,EAAE,SAAS,CAAC,CAAC;IAC1E,OAAO,CAAC,CAAC;AACX,CAAC;AAED,2DAA2D;AAC3D,MAAM,UAAU,kBAAkB,CAAC,GAAY;IAC7C,MAAM,GAAG,GAAG,YAAY,CAAC,GAAG,EAAE,GAAG,CAAC,CAAC;IACnC,OAAO;QACL,MAAM,EAAE,YAAY,CAAC,GAAG,CAAC,MAAM,EAAE,UAAU,EAAE,EAAE,QAAQ,EAAE,IAAI,EAAE,CAAC;QAChE,IAAI,EAAE,YAAY,CAAC,GAAG,CAAC,IAAI,EAAE,QAAQ,EAAE,EAAE,QAAQ,EAAE,IAAI,EAAE,CAAC;QAC1D,WAAW,EAAE,YAAY,CAAC,GAAG,CAAC,WAAW,EAAE,eAAe,EAAE,EAAE,QAAQ,EAAE,IAAI,EAAE,CAAC;QAC/E,OAAO,EAAE,YAAY,CAAC,GAAG,CAAC,OAAO,EAAE,WAAW,CAAC;QAC/C,cAAc,EAAE,YAAY,CAAC,GAAG,CAAC,cAAc,EAAE,kBAAkB,EAAE,EAAE,QAAQ,EAAE,IAAI,EAAE,CAAC;QACxF,WAAW,EAAE,YAAY,CAAC,GAAG,CAAC,WAAW,EAAE,eAAe,EAAE,EAAE,QAAQ,EAAE,IAAI,EAAE,CAAC;QAC/E,OAAO,EAAE,YAAY,CAAC,GAAG,CAAC,OAAO,EAAE,WAAW,EAAE,EAAE,GAAG,EAAE,CAAC,EAAE,OAAO,EAAE,IAAI,EAAE,CAAC;KAC3E,CAAC;AACJ,CAAC;AAED;;;;;;;GAOG;AACH,MAAM,UAAU,mBAAmB,CAAC,CAAiF;IACnH,MAAM,IAAI,GAAG,CAAC,CAAC,WAAW,CAAC,KAAK,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC;IACxC,MAAM,UAAU,GAAG,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC;IACrC,MAAM,aAAa,GAAG,OAAO,CAAC,CAAC,CAAC,cAAc,CAAC,CAAC;IAChD,MAAM,UAAU,GAAG,OAAO,CAAC,CAAC,CAAC,WAAW,CAAC,CAAC;IAC1C,OAAO,GAAG,IAAI,IAAI,UAAU,IAAI,aAAa,QAAQ,UAAU,OAAO,CAAC;AACzE,CAAC;AAED,SAAS,OAAO,CAAC,CAAS;IACxB,OAAO,CAAC;SACL,WAAW,EAAE;SACb,OAAO,CAAC,aAAa,EAAE,GAAG,CAAC;SAC3B,OAAO,CAAC,UAAU,EAAE,EAAE,CAAC;SACvB,OAAO,CAAC,QAAQ,EAAE,GAAG,CAAC,CAAC;AAC5B,CAAC"}
|
|
@@ -126,8 +126,10 @@ export interface FpEvidence {
|
|
|
126
126
|
readonly sampleCount?: number;
|
|
127
127
|
/**
|
|
128
128
|
* True when the harness reported that some of the rule's declared conditions
|
|
129
|
-
* could not be exercised at all (fieldCoverage
|
|
130
|
-
*
|
|
129
|
+
* could not be exercised at all (fieldCoverage: a field no sample fills, or a
|
|
130
|
+
* method the synchronous engine cannot run — trace, behavioral, signature, or
|
|
131
|
+
* semantic without a pattern fallback). A partial measurement that found
|
|
132
|
+
* nothing bounds nothing, so it is treated as no measurement.
|
|
131
133
|
*/
|
|
132
134
|
readonly partial?: boolean;
|
|
133
135
|
/**
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"action-eligibility.d.ts","sourceRoot":"","sources":["../../src/quality/action-eligibility.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAiDG;AAIH,6FAA6F;AAC7F,eAAO,MAAM,YAAY;IACvB,iFAAiF;;IAEjF,iFAAiF;;IAEjF,gGAAgG;;IAEhG,oFAAoF;;EAE3E,CAAC;AAEZ,MAAM,MAAM,UAAU,GAAG,CAAC,OAAO,YAAY,CAAC,CAAC,MAAM,OAAO,YAAY,CAAC,CAAC;AAE1E,kDAAkD;AAClD,eAAO,MAAM,UAAU,EAAE,SAAS,MAAM,EAKtC,CAAC;AA2BH,8EAA8E;AAC9E,eAAO,MAAM,cAAc,EAAE,SAAS,MAAM,EAE1C,CAAC;AAEH;;;;;;;;GAQG;AACH,wBAAgB,UAAU,CAAC,MAAM,EAAE,MAAM,GAAG,UAAU,CAErD;AAED,8DAA8D;AAC9D,wBAAgB,WAAW,CAAC,OAAO,EAAE,SAAS,MAAM,EAAE,GAAG,UAAU,CAKlE;AAED;;;;;GAKG;AACH,eAAO,MAAM,qBAAqB,MAAM,CAAC;AAEzC;;;;;GAKG;AACH,eAAO,MAAM,eAAe,IAAM,CAAC;AAEnC;;;;;;;;;;;;;;GAcG;AACH,wBAAgB,oBAAoB,CAAC,IAAI,EAAE;IACzC,SAAS,CAAC,EAAE,OAAO,CAAC;IACpB,IAAI,CAAC,EAAE;QAAE,WAAW,CAAC,EAAE,OAAO,CAAA;KAAE,CAAC;IACjC,YAAY,CAAC,EAAE;QAAE,IAAI,CAAC,EAAE,OAAO,CAAA;KAAE,CAAC;CACnC,GAAG,MAAM,CAoBT;AAED,oEAAoE;AACpE,MAAM,WAAW,UAAU;IACzB;;;;OAIG;IACH,QAAQ,CAAC,OAAO,CAAC,EAAE,MAAM,CAAC;IAC1B,iFAAiF;IACjF,QAAQ,CAAC,WAAW,CAAC,EAAE,MAAM,CAAC;IAC9B
|
|
1
|
+
{"version":3,"file":"action-eligibility.d.ts","sourceRoot":"","sources":["../../src/quality/action-eligibility.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAiDG;AAIH,6FAA6F;AAC7F,eAAO,MAAM,YAAY;IACvB,iFAAiF;;IAEjF,iFAAiF;;IAEjF,gGAAgG;;IAEhG,oFAAoF;;EAE3E,CAAC;AAEZ,MAAM,MAAM,UAAU,GAAG,CAAC,OAAO,YAAY,CAAC,CAAC,MAAM,OAAO,YAAY,CAAC,CAAC;AAE1E,kDAAkD;AAClD,eAAO,MAAM,UAAU,EAAE,SAAS,MAAM,EAKtC,CAAC;AA2BH,8EAA8E;AAC9E,eAAO,MAAM,cAAc,EAAE,SAAS,MAAM,EAE1C,CAAC;AAEH;;;;;;;;GAQG;AACH,wBAAgB,UAAU,CAAC,MAAM,EAAE,MAAM,GAAG,UAAU,CAErD;AAED,8DAA8D;AAC9D,wBAAgB,WAAW,CAAC,OAAO,EAAE,SAAS,MAAM,EAAE,GAAG,UAAU,CAKlE;AAED;;;;;GAKG;AACH,eAAO,MAAM,qBAAqB,MAAM,CAAC;AAEzC;;;;;GAKG;AACH,eAAO,MAAM,eAAe,IAAM,CAAC;AAEnC;;;;;;;;;;;;;;GAcG;AACH,wBAAgB,oBAAoB,CAAC,IAAI,EAAE;IACzC,SAAS,CAAC,EAAE,OAAO,CAAC;IACpB,IAAI,CAAC,EAAE;QAAE,WAAW,CAAC,EAAE,OAAO,CAAA;KAAE,CAAC;IACjC,YAAY,CAAC,EAAE;QAAE,IAAI,CAAC,EAAE,OAAO,CAAA;KAAE,CAAC;CACnC,GAAG,MAAM,CAoBT;AAED,oEAAoE;AACpE,MAAM,WAAW,UAAU;IACzB;;;;OAIG;IACH,QAAQ,CAAC,OAAO,CAAC,EAAE,MAAM,CAAC;IAC1B,iFAAiF;IACjF,QAAQ,CAAC,WAAW,CAAC,EAAE,MAAM,CAAC;IAC9B;;;;;;OAMG;IACH,QAAQ,CAAC,OAAO,CAAC,EAAE,OAAO,CAAC;IAC3B;;;;OAIG;IACH,QAAQ,CAAC,QAAQ,CAAC,EAAE,MAAM,CAAC;CAC5B;AAED,qFAAqF;AACrF,MAAM,WAAW,mBAAmB;IAClC,QAAQ,CAAC,OAAO,EAAE,UAAU,CAAC;IAC7B,QAAQ,CAAC,SAAS,EAAE,MAAM,GAAG,IAAI,CAAC;IAClC,QAAQ,CAAC,MAAM,EAAE,MAAM,CAAC;CACzB;AAED;;;;;;;GAOG;AACH,wBAAgB,QAAQ,CAAC,EAAE,EAAE,UAAU,GAAG,OAAO,CAShD;AAED;;;;;;GAMG;AACH,wBAAgB,UAAU,CAAC,EAAE,EAAE,UAAU,GAAG,OAAO,CAGlD;AAED;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAiDG;AACH,wBAAgB,UAAU,CAAC,EAAE,EAAE,UAAU,GAAG,mBAAmB,CAqD9D;AAED,+EAA+E;AAC/E,wBAAgB,iBAAiB,CAC/B,OAAO,EAAE,SAAS,MAAM,EAAE,EAC1B,OAAO,EAAE,UAAU,GAClB,SAAS,MAAM,EAAE,CAEnB;AAED;;;;;;GAMG;AACH,wBAAgB,eAAe,CAC7B,OAAO,EAAE,SAAS,MAAM,EAAE,EAC1B,OAAO,EAAE,UAAU,GAClB,SAAS,MAAM,EAAE,CAGnB;AAED;;;;;;;;;;;;;;;;;;;;;;;GAuBG;AACH,wBAAgB,YAAY,CAC1B,QAAQ,EAAE,MAAM,EAChB,OAAO,EAAE,SAAS,MAAM,EAAE,EAC1B,EAAE,EAAE;IACF,QAAQ,CAAC,UAAU,EAAE,CAAC,CAAC,EAAE,MAAM,KAAK,OAAO,CAAC;IAC5C,QAAQ,CAAC,WAAW,EAAE,CAAC,CAAC,EAAE,MAAM,KAAK,MAAM,EAAE,CAAC;IAC9C,QAAQ,CAAC,QAAQ,EAAE,CAAC,CAAC,EAAE,MAAM,KAAK;QAAE,WAAW,IAAI,OAAO,CAAA;KAAE,CAAC;IAC7D,QAAQ,CAAC,YAAY,EAAE,CAAC,CAAC,EAAE,MAAM,KAAK,MAAM,CAAC;CAC9C,EACD,IAAI,EAAE;IAAE,QAAQ,CAAC,IAAI,EAAE,CAAC,GAAG,CAAC,EAAE,MAAM,EAAE,KAAK,MAAM,CAAC;IAAC,QAAQ,CAAC,QAAQ,EAAE,CAAC,CAAC,EAAE,MAAM,EAAE,CAAC,EAAE,MAAM,KAAK,MAAM,CAAA;CAAE,EACxG,UAAU,EAAE,CAAC,GAAG,EAAE,MAAM,KAAK;IAC3B,MAAM,CAAC,CAAC,EAAE,MAAM,GAAG,MAAM,GAAG,OAAO,CAAC;IACpC,MAAM,CAAC,GAAG,EAAE,MAAM,GAAG,MAAM,CAAC;CAC7B,GACA,MAAM,CA0BR;AAED,eAAO,MAAM,mBAAmB,EAAE,SAAS,MAAM,EAI/C,CAAC"}
|