@waratahlabs/canopy 0.5.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +21 -0
- package/README.md +323 -0
- package/SKILL.md +97 -0
- package/demo/data/openclaw-demo.tc.json +397 -0
- package/demo/data/openclaw-nemoclaw-demo.tc.json +475 -0
- package/demo/index.html +556 -0
- package/dist/adapters/a2a.d.ts +82 -0
- package/dist/adapters/a2a.d.ts.map +1 -0
- package/dist/adapters/a2a.js +215 -0
- package/dist/adapters/a2a.js.map +1 -0
- package/dist/adapters/a2a.test.d.ts +2 -0
- package/dist/adapters/a2a.test.d.ts.map +1 -0
- package/dist/adapters/a2a.test.js +196 -0
- package/dist/adapters/a2a.test.js.map +1 -0
- package/dist/adapters/langfuse.d.ts +28 -0
- package/dist/adapters/langfuse.d.ts.map +1 -0
- package/dist/adapters/langfuse.js +192 -0
- package/dist/adapters/langfuse.js.map +1 -0
- package/dist/adapters/n8n.d.ts +4 -0
- package/dist/adapters/n8n.d.ts.map +1 -0
- package/dist/adapters/n8n.js +89 -0
- package/dist/adapters/n8n.js.map +1 -0
- package/dist/adapters/openclaw.d.ts +58 -0
- package/dist/adapters/openclaw.d.ts.map +1 -0
- package/dist/adapters/openclaw.js +275 -0
- package/dist/adapters/openclaw.js.map +1 -0
- package/dist/adapters/types.d.ts +33 -0
- package/dist/adapters/types.d.ts.map +1 -0
- package/dist/adapters/types.js +2 -0
- package/dist/adapters/types.js.map +1 -0
- package/dist/aivss/atlas/coverage-matrix.d.ts +12 -0
- package/dist/aivss/atlas/coverage-matrix.d.ts.map +1 -0
- package/dist/aivss/atlas/coverage-matrix.js +272 -0
- package/dist/aivss/atlas/coverage-matrix.js.map +1 -0
- package/dist/aivss/atlas/loader.d.ts +28 -0
- package/dist/aivss/atlas/loader.d.ts.map +1 -0
- package/dist/aivss/atlas/loader.js +126 -0
- package/dist/aivss/atlas/loader.js.map +1 -0
- package/dist/aivss/atlas/loader.test.d.ts +11 -0
- package/dist/aivss/atlas/loader.test.d.ts.map +1 -0
- package/dist/aivss/atlas/loader.test.js +111 -0
- package/dist/aivss/atlas/loader.test.js.map +1 -0
- package/dist/aivss/atlas/resolve-path.d.ts +34 -0
- package/dist/aivss/atlas/resolve-path.d.ts.map +1 -0
- package/dist/aivss/atlas/resolve-path.js +58 -0
- package/dist/aivss/atlas/resolve-path.js.map +1 -0
- package/dist/aivss/batch/batch-builder.d.ts +11 -0
- package/dist/aivss/batch/batch-builder.d.ts.map +1 -0
- package/dist/aivss/batch/batch-builder.js +75 -0
- package/dist/aivss/batch/batch-builder.js.map +1 -0
- package/dist/aivss/index.d.ts +21 -0
- package/dist/aivss/index.d.ts.map +1 -0
- package/dist/aivss/index.js +19 -0
- package/dist/aivss/index.js.map +1 -0
- package/dist/aivss/knowledge-base/kb-client.d.ts +9 -0
- package/dist/aivss/knowledge-base/kb-client.d.ts.map +1 -0
- package/dist/aivss/knowledge-base/kb-client.js +139 -0
- package/dist/aivss/knowledge-base/kb-client.js.map +1 -0
- package/dist/aivss/knowledge-base/kb-types.d.ts +21 -0
- package/dist/aivss/knowledge-base/kb-types.d.ts.map +1 -0
- package/dist/aivss/knowledge-base/kb-types.js +2 -0
- package/dist/aivss/knowledge-base/kb-types.js.map +1 -0
- package/dist/aivss/knowledge-base/search-builder.d.ts +7 -0
- package/dist/aivss/knowledge-base/search-builder.d.ts.map +1 -0
- package/dist/aivss/knowledge-base/search-builder.js +119 -0
- package/dist/aivss/knowledge-base/search-builder.js.map +1 -0
- package/dist/aivss/metrics/aa.d.ts +3 -0
- package/dist/aivss/metrics/aa.d.ts.map +1 -0
- package/dist/aivss/metrics/aa.js +26 -0
- package/dist/aivss/metrics/aa.js.map +1 -0
- package/dist/aivss/metrics/ad.d.ts +3 -0
- package/dist/aivss/metrics/ad.d.ts.map +1 -0
- package/dist/aivss/metrics/ad.js +30 -0
- package/dist/aivss/metrics/ad.js.map +1 -0
- package/dist/aivss/metrics/cs.d.ts +3 -0
- package/dist/aivss/metrics/cs.d.ts.map +1 -0
- package/dist/aivss/metrics/cs.js +50 -0
- package/dist/aivss/metrics/cs.js.map +1 -0
- package/dist/aivss/metrics/dc.d.ts +3 -0
- package/dist/aivss/metrics/dc.d.ts.map +1 -0
- package/dist/aivss/metrics/dc.js +30 -0
- package/dist/aivss/metrics/dc.js.map +1 -0
- package/dist/aivss/metrics/ds.d.ts +3 -0
- package/dist/aivss/metrics/ds.d.ts.map +1 -0
- package/dist/aivss/metrics/ds.js +26 -0
- package/dist/aivss/metrics/ds.js.map +1 -0
- package/dist/aivss/metrics/ei.d.ts +3 -0
- package/dist/aivss/metrics/ei.d.ts.map +1 -0
- package/dist/aivss/metrics/ei.js +30 -0
- package/dist/aivss/metrics/ei.js.map +1 -0
- package/dist/aivss/metrics/gv.d.ts +3 -0
- package/dist/aivss/metrics/gv.d.ts.map +1 -0
- package/dist/aivss/metrics/gv.js +34 -0
- package/dist/aivss/metrics/gv.js.map +1 -0
- package/dist/aivss/metrics/ll.d.ts +3 -0
- package/dist/aivss/metrics/ll.d.ts.map +1 -0
- package/dist/aivss/metrics/ll.js +30 -0
- package/dist/aivss/metrics/ll.js.map +1 -0
- package/dist/aivss/metrics/mr.d.ts +3 -0
- package/dist/aivss/metrics/mr.d.ts.map +1 -0
- package/dist/aivss/metrics/mr.js +26 -0
- package/dist/aivss/metrics/mr.js.map +1 -0
- package/dist/aivss/metrics/shared.d.ts +22 -0
- package/dist/aivss/metrics/shared.d.ts.map +1 -0
- package/dist/aivss/metrics/shared.js +70 -0
- package/dist/aivss/metrics/shared.js.map +1 -0
- package/dist/aivss/prompts/base-system-prompt.d.ts +2 -0
- package/dist/aivss/prompts/base-system-prompt.d.ts.map +1 -0
- package/dist/aivss/prompts/base-system-prompt.js +31 -0
- package/dist/aivss/prompts/base-system-prompt.js.map +1 -0
- package/dist/aivss/prompts/prompt-composer.d.ts +13 -0
- package/dist/aivss/prompts/prompt-composer.d.ts.map +1 -0
- package/dist/aivss/prompts/prompt-composer.js +21 -0
- package/dist/aivss/prompts/prompt-composer.js.map +1 -0
- package/dist/aivss/scoring/a2a-corpus.test.d.ts +28 -0
- package/dist/aivss/scoring/a2a-corpus.test.d.ts.map +1 -0
- package/dist/aivss/scoring/a2a-corpus.test.js +109 -0
- package/dist/aivss/scoring/a2a-corpus.test.js.map +1 -0
- package/dist/aivss/scoring/determinism.test.d.ts +16 -0
- package/dist/aivss/scoring/determinism.test.d.ts.map +1 -0
- package/dist/aivss/scoring/determinism.test.js +169 -0
- package/dist/aivss/scoring/determinism.test.js.map +1 -0
- package/dist/aivss/scoring/heuristic-scorer.d.ts +21 -0
- package/dist/aivss/scoring/heuristic-scorer.d.ts.map +1 -0
- package/dist/aivss/scoring/heuristic-scorer.js +299 -0
- package/dist/aivss/scoring/heuristic-scorer.js.map +1 -0
- package/dist/aivss/scoring/heuristic-scorer.test.d.ts +8 -0
- package/dist/aivss/scoring/heuristic-scorer.test.d.ts.map +1 -0
- package/dist/aivss/scoring/heuristic-scorer.test.js +150 -0
- package/dist/aivss/scoring/heuristic-scorer.test.js.map +1 -0
- package/dist/aivss/scoring/scoring-mode.d.ts +48 -0
- package/dist/aivss/scoring/scoring-mode.d.ts.map +1 -0
- package/dist/aivss/scoring/scoring-mode.js +64 -0
- package/dist/aivss/scoring/scoring-mode.js.map +1 -0
- package/dist/aivss/scoring/v4-engine.d.ts +15 -0
- package/dist/aivss/scoring/v4-engine.d.ts.map +1 -0
- package/dist/aivss/scoring/v4-engine.js +84 -0
- package/dist/aivss/scoring/v4-engine.js.map +1 -0
- package/dist/aivss/scoring/v4-engine.test.d.ts +16 -0
- package/dist/aivss/scoring/v4-engine.test.d.ts.map +1 -0
- package/dist/aivss/scoring/v4-engine.test.js +156 -0
- package/dist/aivss/scoring/v4-engine.test.js.map +1 -0
- package/dist/aivss/types.d.ts +92 -0
- package/dist/aivss/types.d.ts.map +1 -0
- package/dist/aivss/types.js +25 -0
- package/dist/aivss/types.js.map +1 -0
- package/dist/baseline/baseline.test.d.ts +2 -0
- package/dist/baseline/baseline.test.d.ts.map +1 -0
- package/dist/baseline/baseline.test.js +219 -0
- package/dist/baseline/baseline.test.js.map +1 -0
- package/dist/baseline/diff.d.ts +42 -0
- package/dist/baseline/diff.d.ts.map +1 -0
- package/dist/baseline/diff.js +124 -0
- package/dist/baseline/diff.js.map +1 -0
- package/dist/baseline/store.d.ts +51 -0
- package/dist/baseline/store.d.ts.map +1 -0
- package/dist/baseline/store.js +92 -0
- package/dist/baseline/store.js.map +1 -0
- package/dist/batch/anthropic-batch-runner.d.ts +50 -0
- package/dist/batch/anthropic-batch-runner.d.ts.map +1 -0
- package/dist/batch/anthropic-batch-runner.js +91 -0
- package/dist/batch/anthropic-batch-runner.js.map +1 -0
- package/dist/batch/bedrock-batch-runner.d.ts +90 -0
- package/dist/batch/bedrock-batch-runner.d.ts.map +1 -0
- package/dist/batch/bedrock-batch-runner.js +199 -0
- package/dist/batch/bedrock-batch-runner.js.map +1 -0
- package/dist/batch/providers/anthropic.d.ts +55 -0
- package/dist/batch/providers/anthropic.d.ts.map +1 -0
- package/dist/batch/providers/anthropic.js +103 -0
- package/dist/batch/providers/anthropic.js.map +1 -0
- package/dist/batch/providers/bedrock.d.ts +31 -0
- package/dist/batch/providers/bedrock.d.ts.map +1 -0
- package/dist/batch/providers/bedrock.js +42 -0
- package/dist/batch/providers/bedrock.js.map +1 -0
- package/dist/batch/providers/harness.d.ts +36 -0
- package/dist/batch/providers/harness.d.ts.map +1 -0
- package/dist/batch/providers/harness.js +65 -0
- package/dist/batch/providers/harness.js.map +1 -0
- package/dist/batch/providers/harness.test.d.ts +7 -0
- package/dist/batch/providers/harness.test.d.ts.map +1 -0
- package/dist/batch/providers/harness.test.js +84 -0
- package/dist/batch/providers/harness.test.js.map +1 -0
- package/dist/batch/providers/index.d.ts +82 -0
- package/dist/batch/providers/index.d.ts.map +1 -0
- package/dist/batch/providers/index.js +82 -0
- package/dist/batch/providers/index.js.map +1 -0
- package/dist/batch/providers/ollama.d.ts +68 -0
- package/dist/batch/providers/ollama.d.ts.map +1 -0
- package/dist/batch/providers/ollama.js +108 -0
- package/dist/batch/providers/ollama.js.map +1 -0
- package/dist/batch/response-parser.d.ts +45 -0
- package/dist/batch/response-parser.d.ts.map +1 -0
- package/dist/batch/response-parser.js +161 -0
- package/dist/batch/response-parser.js.map +1 -0
- package/dist/batch/types.d.ts +87 -0
- package/dist/batch/types.d.ts.map +1 -0
- package/dist/batch/types.js +8 -0
- package/dist/batch/types.js.map +1 -0
- package/dist/cli.d.ts +15 -0
- package/dist/cli.d.ts.map +1 -0
- package/dist/cli.js +1037 -0
- package/dist/cli.js.map +1 -0
- package/dist/examples/a2a-assessment.d.ts +20 -0
- package/dist/examples/a2a-assessment.d.ts.map +1 -0
- package/dist/examples/a2a-assessment.js +115 -0
- package/dist/examples/a2a-assessment.js.map +1 -0
- package/dist/examples/batch-assessment.d.ts +18 -0
- package/dist/examples/batch-assessment.d.ts.map +1 -0
- package/dist/examples/batch-assessment.js +112 -0
- package/dist/examples/batch-assessment.js.map +1 -0
- package/dist/examples/batch-submission-flow.d.ts +2 -0
- package/dist/examples/batch-submission-flow.d.ts.map +1 -0
- package/dist/examples/batch-submission-flow.js +279 -0
- package/dist/examples/batch-submission-flow.js.map +1 -0
- package/dist/examples/demo-agent-definition.json +58 -0
- package/dist/examples/langfuse-assessment.d.ts +2 -0
- package/dist/examples/langfuse-assessment.d.ts.map +1 -0
- package/dist/examples/langfuse-assessment.js +46 -0
- package/dist/examples/langfuse-assessment.js.map +1 -0
- package/dist/examples/model-eval.d.ts +25 -0
- package/dist/examples/model-eval.d.ts.map +1 -0
- package/dist/examples/model-eval.js +94 -0
- package/dist/examples/model-eval.js.map +1 -0
- package/dist/examples/openclaw-assessment.d.ts +17 -0
- package/dist/examples/openclaw-assessment.d.ts.map +1 -0
- package/dist/examples/openclaw-assessment.js +79 -0
- package/dist/examples/openclaw-assessment.js.map +1 -0
- package/dist/examples/simple-assessment.d.ts +2 -0
- package/dist/examples/simple-assessment.d.ts.map +1 -0
- package/dist/examples/simple-assessment.js +146 -0
- package/dist/examples/simple-assessment.js.map +1 -0
- package/dist/examples/threat-model-demo.d.ts +2 -0
- package/dist/examples/threat-model-demo.d.ts.map +1 -0
- package/dist/examples/threat-model-demo.js +103 -0
- package/dist/examples/threat-model-demo.js.map +1 -0
- package/dist/index.d.ts +60 -0
- package/dist/index.d.ts.map +1 -0
- package/dist/index.js +155 -0
- package/dist/index.js.map +1 -0
- package/dist/output/likec4-generator.d.ts +43 -0
- package/dist/output/likec4-generator.d.ts.map +1 -0
- package/dist/output/likec4-generator.js +287 -0
- package/dist/output/likec4-generator.js.map +1 -0
- package/dist/output/slot-filler.d.ts +6 -0
- package/dist/output/slot-filler.d.ts.map +1 -0
- package/dist/output/slot-filler.js +71 -0
- package/dist/output/slot-filler.js.map +1 -0
- package/dist/output/tc-assembler.d.ts +30 -0
- package/dist/output/tc-assembler.d.ts.map +1 -0
- package/dist/output/tc-assembler.js +227 -0
- package/dist/output/tc-assembler.js.map +1 -0
- package/dist/output/tc-types.d.ts +41 -0
- package/dist/output/tc-types.d.ts.map +1 -0
- package/dist/output/tc-types.js +5 -0
- package/dist/output/tc-types.js.map +1 -0
- package/dist/output/tc-validator.test.d.ts +9 -0
- package/dist/output/tc-validator.test.d.ts.map +1 -0
- package/dist/output/tc-validator.test.js +194 -0
- package/dist/output/tc-validator.test.js.map +1 -0
- package/dist/providers/anthropic.d.ts +9 -0
- package/dist/providers/anthropic.d.ts.map +1 -0
- package/dist/providers/anthropic.js +30 -0
- package/dist/providers/anthropic.js.map +1 -0
- package/dist/providers/bedrock.d.ts +9 -0
- package/dist/providers/bedrock.d.ts.map +1 -0
- package/dist/providers/bedrock.js +38 -0
- package/dist/providers/bedrock.js.map +1 -0
- package/dist/providers/index.d.ts +6 -0
- package/dist/providers/index.d.ts.map +1 -0
- package/dist/providers/index.js +14 -0
- package/dist/providers/index.js.map +1 -0
- package/dist/providers/types.d.ts +12 -0
- package/dist/providers/types.d.ts.map +1 -0
- package/dist/providers/types.js +5 -0
- package/dist/providers/types.js.map +1 -0
- package/dist/references/acsc-risks.d.ts +18 -0
- package/dist/references/acsc-risks.d.ts.map +1 -0
- package/dist/references/acsc-risks.js +232 -0
- package/dist/references/acsc-risks.js.map +1 -0
- package/dist/references/aicm.d.ts +41 -0
- package/dist/references/aicm.d.ts.map +1 -0
- package/dist/references/aicm.js +447 -0
- package/dist/references/aicm.js.map +1 -0
- package/dist/references/dstg-meaid.d.ts +59 -0
- package/dist/references/dstg-meaid.d.ts.map +1 -0
- package/dist/references/dstg-meaid.js +352 -0
- package/dist/references/dstg-meaid.js.map +1 -0
- package/dist/references/index.d.ts +20 -0
- package/dist/references/index.d.ts.map +1 -0
- package/dist/references/index.js +31 -0
- package/dist/references/index.js.map +1 -0
- package/dist/references/nemoclaw.d.ts +48 -0
- package/dist/references/nemoclaw.d.ts.map +1 -0
- package/dist/references/nemoclaw.js +140 -0
- package/dist/references/nemoclaw.js.map +1 -0
- package/dist/references/nist-ai-rmf.d.ts +31 -0
- package/dist/references/nist-ai-rmf.d.ts.map +1 -0
- package/dist/references/nist-ai-rmf.js +627 -0
- package/dist/references/nist-ai-rmf.js.map +1 -0
- package/dist/references/owasp-agentic-top10.d.ts +50 -0
- package/dist/references/owasp-agentic-top10.d.ts.map +1 -0
- package/dist/references/owasp-agentic-top10.js +202 -0
- package/dist/references/owasp-agentic-top10.js.map +1 -0
- package/dist/references/registry-parity.test.d.ts +17 -0
- package/dist/references/registry-parity.test.d.ts.map +1 -0
- package/dist/references/registry-parity.test.js +87 -0
- package/dist/references/registry-parity.test.js.map +1 -0
- package/dist/references/registry.d.ts +90 -0
- package/dist/references/registry.d.ts.map +1 -0
- package/dist/references/registry.js +29 -0
- package/dist/references/registry.js.map +1 -0
- package/dist/tools/model-eval.d.ts +97 -0
- package/dist/tools/model-eval.d.ts.map +1 -0
- package/dist/tools/model-eval.js +307 -0
- package/dist/tools/model-eval.js.map +1 -0
- package/package.json +105 -0
- package/schemas/threat-composer-v1.schema.json +966 -0
- package/vendor/atlas-release/ATLAS-2026.07.yaml +17877 -0
- package/vendor/atlas-release/README.md +41 -0
|
@@ -0,0 +1,109 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* A2A corpus regression — deterministic scoring against real agent definitions.
|
|
3
|
+
*
|
|
4
|
+
* The unit tests prove the scorer is pure over a hand-written fixture. This
|
|
5
|
+
* proves it stays stable over a corpus of ~25 real agent topologies harvested
|
|
6
|
+
* from a2aproject/a2a-samples, which is where scoring changes actually show up:
|
|
7
|
+
* a threshold tweak that looks harmless against one synthetic agent will move
|
|
8
|
+
* severities across a real corpus.
|
|
9
|
+
*
|
|
10
|
+
* Regenerate the corpus with a checkout of a2a-samples:
|
|
11
|
+
*
|
|
12
|
+
* bun scripts/harvest-a2a-samples.ts path/to/a2a-samples
|
|
13
|
+
*
|
|
14
|
+
* That writes tests/fixtures/a2a/input/<agent>.json (the synthesized
|
|
15
|
+
* AgentDefinitions) and tests/fixtures/a2a/expected/<agent>.tc.json (the
|
|
16
|
+
* assessed artifact). This test re-derives the expected output from the input
|
|
17
|
+
* and diffs.
|
|
18
|
+
*
|
|
19
|
+
* A diff here is not automatically a bug. It means emitted scores moved, which
|
|
20
|
+
* per CONTRIBUTING.md is a breaking change requiring a major version bump — so
|
|
21
|
+
* the diff should be reviewed, then the corpus regenerated in the same commit
|
|
22
|
+
* that changes the scoring.
|
|
23
|
+
*
|
|
24
|
+
* The suite skips cleanly when the corpus is absent, so a fresh clone without
|
|
25
|
+
* fixtures is green rather than red.
|
|
26
|
+
*/
|
|
27
|
+
import { describe, expect, test } from "bun:test";
|
|
28
|
+
import * as fs from "node:fs";
|
|
29
|
+
import * as path from "node:path";
|
|
30
|
+
import { fileURLToPath } from "node:url";
|
|
31
|
+
import { heuristicSeverity } from "./heuristic-scorer.js";
|
|
32
|
+
import { assembleTCFormat, severityKey, slotKey } from "../../output/tc-assembler.js";
|
|
33
|
+
import { ATLAS_COVERAGE_MATRIX } from "../atlas/coverage-matrix.js";
|
|
34
|
+
import { loadAtlasData } from "../atlas/loader.js";
|
|
35
|
+
import { resolveAtlasYaml } from "../atlas/resolve-path.js";
|
|
36
|
+
const repoRoot = path.resolve(path.dirname(fileURLToPath(import.meta.url)), "../../..");
|
|
37
|
+
const INPUT_DIR = path.join(repoRoot, "tests/fixtures/a2a/input");
|
|
38
|
+
const EXPECTED_DIR = path.join(repoRoot, "tests/fixtures/a2a/expected");
|
|
39
|
+
function corpusCases() {
|
|
40
|
+
if (!fs.existsSync(INPUT_DIR))
|
|
41
|
+
return [];
|
|
42
|
+
return fs
|
|
43
|
+
.readdirSync(INPUT_DIR)
|
|
44
|
+
.filter((f) => f.endsWith(".json"))
|
|
45
|
+
.map((f) => f.replace(/\.json$/, ""))
|
|
46
|
+
.sort();
|
|
47
|
+
}
|
|
48
|
+
const cases = corpusCases();
|
|
49
|
+
// Normalise away the only field that legitimately varies between runs.
|
|
50
|
+
const normalise = (doc) => JSON.stringify(doc).replace(/"Generated: [^"\\]+? \\n/, '"Generated: <ts> \\n');
|
|
51
|
+
function assess(input) {
|
|
52
|
+
const atlasData = loadAtlasData(resolveAtlasYaml());
|
|
53
|
+
const severityBySubCategory = new Map();
|
|
54
|
+
const filledSlots = new Map();
|
|
55
|
+
for (const agent of input.agents) {
|
|
56
|
+
for (const entry of ATLAS_COVERAGE_MATRIX) {
|
|
57
|
+
severityBySubCategory.set(severityKey(agent.id, entry.metric, entry.subCategory), heuristicSeverity(agent, entry.metric, entry.subCategory));
|
|
58
|
+
const techniqueId = entry.atlasTechniqueIds[0];
|
|
59
|
+
if (!techniqueId)
|
|
60
|
+
continue;
|
|
61
|
+
const technique = atlasData.techniqueById.get(techniqueId);
|
|
62
|
+
if (!technique)
|
|
63
|
+
continue;
|
|
64
|
+
filledSlots.set(slotKey(agent.id, techniqueId, entry.metric, entry.subCategory), {
|
|
65
|
+
threatSource: "A threat actor",
|
|
66
|
+
prerequisites: `with access to ${agent.name}`,
|
|
67
|
+
threatAction: `exploit ${entry.subCategory.toLowerCase()} via ${technique.id}`,
|
|
68
|
+
threatImpact: `compromise ${agent.name}`,
|
|
69
|
+
impactedGoal: ["confidentiality", "integrity"],
|
|
70
|
+
impactedAssets: [agent.name.toLowerCase()],
|
|
71
|
+
});
|
|
72
|
+
}
|
|
73
|
+
}
|
|
74
|
+
return assembleTCFormat({
|
|
75
|
+
agents: input.agents,
|
|
76
|
+
handoffs: input.handoffs ?? [],
|
|
77
|
+
metricResponses: [],
|
|
78
|
+
atlasData,
|
|
79
|
+
coverageMatrix: ATLAS_COVERAGE_MATRIX,
|
|
80
|
+
filledSlots,
|
|
81
|
+
severityBySubCategory,
|
|
82
|
+
applicationName: input.applicationName,
|
|
83
|
+
scoringMode: "deterministic",
|
|
84
|
+
});
|
|
85
|
+
}
|
|
86
|
+
describe.skipIf(cases.length === 0)("a2a corpus regression", () => {
|
|
87
|
+
test("corpus has both halves of every golden pair", () => {
|
|
88
|
+
for (const name of cases) {
|
|
89
|
+
expect(fs.existsSync(path.join(EXPECTED_DIR, `${name}.tc.json`))).toBe(true);
|
|
90
|
+
}
|
|
91
|
+
});
|
|
92
|
+
test.each(cases)("%s scores identically to its committed artifact", (name) => {
|
|
93
|
+
const input = JSON.parse(fs.readFileSync(path.join(INPUT_DIR, `${name}.json`), "utf8"));
|
|
94
|
+
const expected = JSON.parse(fs.readFileSync(path.join(EXPECTED_DIR, `${name}.tc.json`), "utf8"));
|
|
95
|
+
expect(normalise(assess(input))).toBe(normalise(expected));
|
|
96
|
+
});
|
|
97
|
+
test.each(cases)("%s re-assesses identically within a single run", (name) => {
|
|
98
|
+
const input = JSON.parse(fs.readFileSync(path.join(INPUT_DIR, `${name}.json`), "utf8"));
|
|
99
|
+
expect(normalise(assess(input))).toBe(normalise(assess(input)));
|
|
100
|
+
});
|
|
101
|
+
});
|
|
102
|
+
// Visible when the corpus is missing, so an absent harness reads as a stated
|
|
103
|
+
// gap rather than silent green.
|
|
104
|
+
describe.skipIf(cases.length > 0)("a2a corpus regression (not present)", () => {
|
|
105
|
+
test("corpus is absent — regenerate with scripts/harvest-a2a-samples.ts", () => {
|
|
106
|
+
expect(cases.length).toBe(0);
|
|
107
|
+
});
|
|
108
|
+
});
|
|
109
|
+
//# sourceMappingURL=a2a-corpus.test.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"a2a-corpus.test.js","sourceRoot":"","sources":["../../../src/aivss/scoring/a2a-corpus.test.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;GAyBG;AAEH,OAAO,EAAE,QAAQ,EAAE,MAAM,EAAE,IAAI,EAAE,MAAM,UAAU,CAAC;AAClD,OAAO,KAAK,EAAE,MAAM,SAAS,CAAC;AAC9B,OAAO,KAAK,IAAI,MAAM,WAAW,CAAC;AAClC,OAAO,EAAE,aAAa,EAAE,MAAM,UAAU,CAAC;AACzC,OAAO,EAAE,iBAAiB,EAAE,MAAM,uBAAuB,CAAC;AAC1D,OAAO,EAAE,gBAAgB,EAAE,WAAW,EAAE,OAAO,EAAE,MAAM,8BAA8B,CAAC;AACtF,OAAO,EAAE,qBAAqB,EAAE,MAAM,6BAA6B,CAAC;AACpE,OAAO,EAAE,aAAa,EAAE,MAAM,oBAAoB,CAAC;AACnD,OAAO,EAAE,gBAAgB,EAAE,MAAM,0BAA0B,CAAC;AAI5D,MAAM,QAAQ,GAAG,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC,OAAO,CAAC,aAAa,CAAC,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,UAAU,CAAC,CAAC;AACxF,MAAM,SAAS,GAAG,IAAI,CAAC,IAAI,CAAC,QAAQ,EAAE,0BAA0B,CAAC,CAAC;AAClE,MAAM,YAAY,GAAG,IAAI,CAAC,IAAI,CAAC,QAAQ,EAAE,6BAA6B,CAAC,CAAC;AAQxE,SAAS,WAAW;IAClB,IAAI,CAAC,EAAE,CAAC,UAAU,CAAC,SAAS,CAAC;QAAE,OAAO,EAAE,CAAC;IACzC,OAAO,EAAE;SACN,WAAW,CAAC,SAAS,CAAC;SACtB,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,QAAQ,CAAC,OAAO,CAAC,CAAC;SAClC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,OAAO,CAAC,SAAS,EAAE,EAAE,CAAC,CAAC;SACpC,IAAI,EAAE,CAAC;AACZ,CAAC;AAED,MAAM,KAAK,GAAG,WAAW,EAAE,CAAC;AAE5B,uEAAuE;AACvE,MAAM,SAAS,GAAG,CAAC,GAAY,EAAU,EAAE,CACzC,IAAI,CAAC,SAAS,CAAC,GAAG,CAAC,CAAC,OAAO,CAAC,2BAA2B,EAAE,uBAAuB,CAAC,CAAC;AAEpF,SAAS,MAAM,CAAC,KAAkB;IAChC,MAAM,SAAS,GAAG,aAAa,CAAC,gBAAgB,EAAE,CAAC,CAAC;IACpD,MAAM,qBAAqB,GAAG,IAAI,GAAG,EAAyB,CAAC;IAC/D,MAAM,WAAW,GAAG,IAAI,GAAG,EAAE,CAAC;IAE9B,KAAK,MAAM,KAAK,IAAI,KAAK,CAAC,MAAM,EAAE,CAAC;QACjC,KAAK,MAAM,KAAK,IAAI,qBAAqB,EAAE,CAAC;YAC1C,qBAAqB,CAAC,GAAG,CACvB,WAAW,CAAC,KAAK,CAAC,EAAE,EAAE,KAAK,CAAC,MAAM,EAAE,KAAK,CAAC,WAAW,CAAC,EACtD,iBAAiB,CAAC,KAAK,EAAE,KAAK,CAAC,MAAM,EAAE,KAAK,CAAC,WAAW,CAAC,CAC1D,CAAC;YACF,MAAM,WAAW,GAAG,KAAK,CAAC,iBAAiB,CAAC,CAAC,CAAC,CAAC;YAC/C,IAAI,CAAC,WAAW;gBAAE,SAAS;YAC3B,MAAM,SAAS,GAAG,SAAS,CAAC,aAAa,CAAC,GAAG,CAAC,WAAW,CAAC,CAAC;YAC3D,IAAI,CAAC,SAAS;gBAAE,SAAS;YACzB,WAAW,CAAC,GAAG,CAAC,OAAO,CAAC,KAAK,CAAC,EAAE,EAAE,WAAW,EAAE,KAAK,CAAC,MAAM,EAAE,KAAK,CAAC,WAAW,CAAC,EAAE;gBAC/E,YAAY,EAAE,gBAAgB;gBAC9B,aAAa,EAAE,kBAAkB,KAAK,CAAC,IAAI,EAAE;gBAC7C,YAAY,EAAE,WAAW,KAAK,CAAC,WAAW,CAAC,WAAW,EAAE,QAAQ,SAAS,CAAC,EAAE,EAAE;gBAC9E,YAAY,EAAE,cAAc,KAAK,CAAC,IAAI,EAAE;gBACxC,YAAY,EAAE,CAAC,iBAAiB,EAAE,WAAW,CAAC;gBAC9C,cAAc,EAAE,CAAC,KAAK,CAAC,IAAI,CAAC,WAAW,EAAE,CAAC;aAC3C,CAAC,CAAC;QACL,CAAC;IACH,CAAC;IAED,OAAO,gBAAgB,CAAC;QACtB,MAAM,EAAE,KAAK,CAAC,MAAM;QACpB,QAAQ,EAAE,KAAK,CAAC,QAAQ,IAAI,EAAE;QAC9B,eAAe,EAAE,EAAE;QACnB,SAAS;QACT,cAAc,EAAE,qBAAqB;QACrC,WAAW;QACX,qBAAqB;QACrB,eAAe,EAAE,KAAK,CAAC,eAAe;QACtC,WAAW,EAAE,eAAe;KAC7B,CAAC,CAAC;AACL,CAAC;AAED,QAAQ,CAAC,MAAM,CAAC,KAAK,CAAC,MAAM,KAAK,CAAC,CAAC,CAAC,uBAAuB,EAAE,GAAG,EAAE;IAChE,IAAI,CAAC,6CAA6C,EAAE,GAAG,EAAE;QACvD,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE,CAAC;YACzB,MAAM,CAAC,EAAE,CAAC,UAAU,CAAC,IAAI,CAAC,IAAI,CAAC,YAAY,EAAE,GAAG,IAAI,UAAU,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QAC/E,CAAC;IACH,CAAC,CAAC,CAAC;IAEH,IAAI,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,iDAAiD,EAAE,CAAC,IAAI,EAAE,EAAE;QAC3E,MAAM,KAAK,GAAG,IAAI,CAAC,KAAK,CACtB,EAAE,CAAC,YAAY,CAAC,IAAI,CAAC,IAAI,CAAC,SAAS,EAAE,GAAG,IAAI,OAAO,CAAC,EAAE,MAAM,CAAC,CAC/C,CAAC;QACjB,MAAM,QAAQ,GAAG,IAAI,CAAC,KAAK,CAAC,EAAE,CAAC,YAAY,CAAC,IAAI,CAAC,IAAI,CAAC,YAAY,EAAE,GAAG,IAAI,UAAU,CAAC,EAAE,MAAM,CAAC,CAAC,CAAC;QAEjG,MAAM,CAAC,SAAS,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,SAAS,CAAC,QAAQ,CAAC,CAAC,CAAC;IAC7D,CAAC,CAAC,CAAC;IAEH,IAAI,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,gDAAgD,EAAE,CAAC,IAAI,EAAE,EAAE;QAC1E,MAAM,KAAK,GAAG,IAAI,CAAC,KAAK,CACtB,EAAE,CAAC,YAAY,CAAC,IAAI,CAAC,IAAI,CAAC,SAAS,EAAE,GAAG,IAAI,OAAO,CAAC,EAAE,MAAM,CAAC,CAC/C,CAAC;QACjB,MAAM,CAAC,SAAS,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,SAAS,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC;IAClE,CAAC,CAAC,CAAC;AACL,CAAC,CAAC,CAAC;AAEH,6EAA6E;AAC7E,gCAAgC;AAChC,QAAQ,CAAC,MAAM,CAAC,KAAK,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,qCAAqC,EAAE,GAAG,EAAE;IAC5E,IAAI,CAAC,mEAAmE,EAAE,GAAG,EAAE;QAC7E,MAAM,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;IAC/B,CAAC,CAAC,CAAC;AACL,CAAC,CAAC,CAAC"}
|
|
@@ -0,0 +1,16 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Determinism — the property that makes a Canopy score defensible.
|
|
3
|
+
*
|
|
4
|
+
* A security engineer has to be able to say where a number came from. CVSS is
|
|
5
|
+
* trusted because the vector string lets anyone recompute it; OpenSSF Scorecard
|
|
6
|
+
* because every check's rule is readable. Canopy's deterministic mode makes the
|
|
7
|
+
* same promise, and these tests are what keep it honest.
|
|
8
|
+
*
|
|
9
|
+
* Two distinct claims are covered:
|
|
10
|
+
* 1. The heuristic scorer is a pure function of the agent definition.
|
|
11
|
+
* 2. The assembled artifact is byte-stable apart from its generation
|
|
12
|
+
* timestamp — which required replacing random threat ids with
|
|
13
|
+
* content-derived ones.
|
|
14
|
+
*/
|
|
15
|
+
export {};
|
|
16
|
+
//# sourceMappingURL=determinism.test.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"determinism.test.d.ts","sourceRoot":"","sources":["../../../src/aivss/scoring/determinism.test.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;GAaG"}
|
|
@@ -0,0 +1,169 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Determinism — the property that makes a Canopy score defensible.
|
|
3
|
+
*
|
|
4
|
+
* A security engineer has to be able to say where a number came from. CVSS is
|
|
5
|
+
* trusted because the vector string lets anyone recompute it; OpenSSF Scorecard
|
|
6
|
+
* because every check's rule is readable. Canopy's deterministic mode makes the
|
|
7
|
+
* same promise, and these tests are what keep it honest.
|
|
8
|
+
*
|
|
9
|
+
* Two distinct claims are covered:
|
|
10
|
+
* 1. The heuristic scorer is a pure function of the agent definition.
|
|
11
|
+
* 2. The assembled artifact is byte-stable apart from its generation
|
|
12
|
+
* timestamp — which required replacing random threat ids with
|
|
13
|
+
* content-derived ones.
|
|
14
|
+
*/
|
|
15
|
+
import { describe, expect, test } from "bun:test";
|
|
16
|
+
import { heuristicSeverity } from "./heuristic-scorer.js";
|
|
17
|
+
import { SCORING_MODE_INFO, SCORING_MODES, reproducibilityWarning } from "./scoring-mode.js";
|
|
18
|
+
import { assembleTCFormat, severityKey, slotKey } from "../../output/tc-assembler.js";
|
|
19
|
+
import { ATLAS_COVERAGE_MATRIX } from "../atlas/coverage-matrix.js";
|
|
20
|
+
import { loadAtlasData } from "../atlas/loader.js";
|
|
21
|
+
import { resolveAtlasYaml } from "../atlas/resolve-path.js";
|
|
22
|
+
const AGENT = {
|
|
23
|
+
id: "agent-det-1",
|
|
24
|
+
name: "Determinism Fixture",
|
|
25
|
+
model: "claude-sonnet-4-6",
|
|
26
|
+
systemPrompt: "You process mortgage applications end to end.",
|
|
27
|
+
tools: [
|
|
28
|
+
{ name: "shell", description: "Run shell commands", dangerous: true },
|
|
29
|
+
{ name: "credit_check", description: "Query the credit bureau", dangerous: true },
|
|
30
|
+
],
|
|
31
|
+
permissions: ["read:applications", "write:decisions", "execute:scripts"],
|
|
32
|
+
deploymentContext: "FinancialServices",
|
|
33
|
+
modelComplexity: "Complex",
|
|
34
|
+
mitigationMultiplier: 1.0,
|
|
35
|
+
finetuned: false,
|
|
36
|
+
dependencies: ["document-store", "credit-api"],
|
|
37
|
+
secretManagement: "env",
|
|
38
|
+
deployment: "AWS ECS Fargate",
|
|
39
|
+
};
|
|
40
|
+
describe("heuristic scorer is a pure function", () => {
|
|
41
|
+
test("repeated scoring of the same input yields identical severities", () => {
|
|
42
|
+
const runOnce = () => ATLAS_COVERAGE_MATRIX.map((e) => heuristicSeverity(AGENT, e.metric, e.subCategory));
|
|
43
|
+
const first = runOnce();
|
|
44
|
+
for (let i = 0; i < 25; i++) {
|
|
45
|
+
expect(runOnce()).toEqual(first);
|
|
46
|
+
}
|
|
47
|
+
});
|
|
48
|
+
test("scoring does not mutate the agent definition", () => {
|
|
49
|
+
const before = JSON.stringify(AGENT);
|
|
50
|
+
for (const e of ATLAS_COVERAGE_MATRIX)
|
|
51
|
+
heuristicSeverity(AGENT, e.metric, e.subCategory);
|
|
52
|
+
expect(JSON.stringify(AGENT)).toBe(before);
|
|
53
|
+
});
|
|
54
|
+
test("a materially riskier agent does not score lower", () => {
|
|
55
|
+
// Guards the direction of the scoring rules: removing every mitigation and
|
|
56
|
+
// granting dangerous capability must not reduce severity. Catches an
|
|
57
|
+
// inverted comparison, which a pure-determinism test would happily pass.
|
|
58
|
+
const hardened = { ...AGENT, mitigationMultiplier: 0.5, tools: [], permissions: ["read:applications"] };
|
|
59
|
+
const rank = { None: 0, Low: 1, Medium: 2, High: 3, Critical: 4 };
|
|
60
|
+
let exposedTotal = 0;
|
|
61
|
+
let hardenedTotal = 0;
|
|
62
|
+
for (const e of ATLAS_COVERAGE_MATRIX) {
|
|
63
|
+
exposedTotal += rank[heuristicSeverity(AGENT, e.metric, e.subCategory)];
|
|
64
|
+
hardenedTotal += rank[heuristicSeverity(hardened, e.metric, e.subCategory)];
|
|
65
|
+
}
|
|
66
|
+
expect(exposedTotal).toBeGreaterThan(hardenedTotal);
|
|
67
|
+
});
|
|
68
|
+
});
|
|
69
|
+
describe("assembled artifact is byte-stable", () => {
|
|
70
|
+
const atlasData = loadAtlasData(resolveAtlasYaml());
|
|
71
|
+
function build() {
|
|
72
|
+
const severityBySubCategory = new Map();
|
|
73
|
+
const filledSlots = new Map();
|
|
74
|
+
for (const e of ATLAS_COVERAGE_MATRIX) {
|
|
75
|
+
const severity = heuristicSeverity(AGENT, e.metric, e.subCategory);
|
|
76
|
+
severityBySubCategory.set(severityKey(AGENT.id, e.metric, e.subCategory), severity);
|
|
77
|
+
const techniqueId = e.atlasTechniqueIds[0];
|
|
78
|
+
if (!techniqueId || !atlasData.techniqueById.get(techniqueId))
|
|
79
|
+
continue;
|
|
80
|
+
filledSlots.set(slotKey(AGENT.id, techniqueId, e.metric, e.subCategory), {
|
|
81
|
+
threatSource: "A threat actor",
|
|
82
|
+
prerequisites: `with access to ${AGENT.name}`,
|
|
83
|
+
threatAction: `exploit ${e.subCategory.toLowerCase()}`,
|
|
84
|
+
threatImpact: `compromise ${AGENT.name}`,
|
|
85
|
+
impactedGoal: ["confidentiality"],
|
|
86
|
+
impactedAssets: [AGENT.name.toLowerCase()],
|
|
87
|
+
});
|
|
88
|
+
}
|
|
89
|
+
return assembleTCFormat({
|
|
90
|
+
agents: [AGENT],
|
|
91
|
+
handoffs: [],
|
|
92
|
+
metricResponses: [],
|
|
93
|
+
atlasData,
|
|
94
|
+
coverageMatrix: ATLAS_COVERAGE_MATRIX,
|
|
95
|
+
filledSlots,
|
|
96
|
+
severityBySubCategory,
|
|
97
|
+
applicationName: "Determinism Fixture App",
|
|
98
|
+
scoringMode: "deterministic",
|
|
99
|
+
});
|
|
100
|
+
}
|
|
101
|
+
// The audit trail carries a wall-clock generation time, which is legitimately
|
|
102
|
+
// different every run. Everything else must match byte for byte.
|
|
103
|
+
const stripTimestamp = (d) => JSON.stringify({
|
|
104
|
+
...d,
|
|
105
|
+
applicationInfo: {
|
|
106
|
+
...d.applicationInfo,
|
|
107
|
+
description: (d.applicationInfo?.description ?? "").replace(/Generated: [^\s]+/, "Generated: <ts>"),
|
|
108
|
+
},
|
|
109
|
+
});
|
|
110
|
+
test("two runs produce identical output apart from the timestamp", () => {
|
|
111
|
+
expect(stripTimestamp(build())).toBe(stripTimestamp(build()));
|
|
112
|
+
});
|
|
113
|
+
test("threat ids are content-derived, not random", () => {
|
|
114
|
+
// Regression guard: ids were crypto.randomUUID(), so structurally identical
|
|
115
|
+
// artifacts differed byte-wise every run and diffs were unusable.
|
|
116
|
+
const a = build();
|
|
117
|
+
const b = build();
|
|
118
|
+
const aThreats = a.threats ?? [];
|
|
119
|
+
const bThreats = b.threats ?? [];
|
|
120
|
+
expect(aThreats.map((t) => t.id)).toEqual(bThreats.map((t) => t.id));
|
|
121
|
+
expect(new Set(aThreats.map((t) => t.id)).size).toBe(aThreats.length);
|
|
122
|
+
for (const t of aThreats) {
|
|
123
|
+
expect(t.id).toMatch(/^[0-9a-f]{8}-[0-9a-f]{4}-5[0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$/);
|
|
124
|
+
}
|
|
125
|
+
});
|
|
126
|
+
test("every threat records the scoring mode that produced it", () => {
|
|
127
|
+
const doc = build();
|
|
128
|
+
const docThreats = doc.threats ?? [];
|
|
129
|
+
expect(docThreats.length).toBeGreaterThan(0);
|
|
130
|
+
for (const threat of docThreats) {
|
|
131
|
+
const mode = threat.metadata?.find((m) => m.key === "custom:scoring-mode")?.value;
|
|
132
|
+
expect(mode).toBe("deterministic");
|
|
133
|
+
}
|
|
134
|
+
});
|
|
135
|
+
test("the audit trail states the mode and its reproducibility", () => {
|
|
136
|
+
const description = build().applicationInfo?.description ?? "";
|
|
137
|
+
expect(description).toContain("Scoring mode: deterministic");
|
|
138
|
+
expect(description).toContain("Reproducible: yes");
|
|
139
|
+
});
|
|
140
|
+
test("a non-reproducible mode is labelled as such", () => {
|
|
141
|
+
const severityBySubCategory = new Map();
|
|
142
|
+
const doc = assembleTCFormat({
|
|
143
|
+
agents: [AGENT],
|
|
144
|
+
handoffs: [],
|
|
145
|
+
metricResponses: [],
|
|
146
|
+
atlasData,
|
|
147
|
+
coverageMatrix: ATLAS_COVERAGE_MATRIX,
|
|
148
|
+
filledSlots: new Map(),
|
|
149
|
+
severityBySubCategory,
|
|
150
|
+
applicationName: "Harness Run",
|
|
151
|
+
scoringMode: "harness",
|
|
152
|
+
});
|
|
153
|
+
expect(doc.applicationInfo?.description ?? "").toContain("Scoring mode: harness");
|
|
154
|
+
expect(doc.applicationInfo?.description ?? "").toContain("Reproducible: no");
|
|
155
|
+
});
|
|
156
|
+
});
|
|
157
|
+
describe("scoring mode metadata", () => {
|
|
158
|
+
test("exactly one mode is reproducible, and it needs no credentials", () => {
|
|
159
|
+
const reproducible = SCORING_MODES.filter((m) => SCORING_MODE_INFO[m].reproducible);
|
|
160
|
+
expect(reproducible).toEqual(["deterministic"]);
|
|
161
|
+
expect(SCORING_MODE_INFO.deterministic.requiresCredentials).toBe(false);
|
|
162
|
+
});
|
|
163
|
+
test("non-reproducible modes warn, the reproducible one does not", () => {
|
|
164
|
+
expect(reproducibilityWarning("deterministic")).toBeNull();
|
|
165
|
+
expect(reproducibilityWarning("model")).toContain("not reproducible");
|
|
166
|
+
expect(reproducibilityWarning("harness")).toContain("not reproducible");
|
|
167
|
+
});
|
|
168
|
+
});
|
|
169
|
+
//# sourceMappingURL=determinism.test.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"determinism.test.js","sourceRoot":"","sources":["../../../src/aivss/scoring/determinism.test.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;GAaG;AAEH,OAAO,EAAE,QAAQ,EAAE,MAAM,EAAE,IAAI,EAAE,MAAM,UAAU,CAAC;AAClD,OAAO,EAAE,iBAAiB,EAAE,MAAM,uBAAuB,CAAC;AAC1D,OAAO,EAAE,iBAAiB,EAAE,aAAa,EAAE,sBAAsB,EAAE,MAAM,mBAAmB,CAAC;AAC7F,OAAO,EAAE,gBAAgB,EAAE,WAAW,EAAE,OAAO,EAAE,MAAM,8BAA8B,CAAC;AACtF,OAAO,EAAE,qBAAqB,EAAE,MAAM,6BAA6B,CAAC;AACpE,OAAO,EAAE,aAAa,EAAE,MAAM,oBAAoB,CAAC;AACnD,OAAO,EAAE,gBAAgB,EAAE,MAAM,0BAA0B,CAAC;AAG5D,MAAM,KAAK,GAAoB;IAC7B,EAAE,EAAE,aAAa;IACjB,IAAI,EAAE,qBAAqB;IAC3B,KAAK,EAAE,mBAAmB;IAC1B,YAAY,EAAE,+CAA+C;IAC7D,KAAK,EAAE;QACL,EAAE,IAAI,EAAE,OAAO,EAAE,WAAW,EAAE,oBAAoB,EAAE,SAAS,EAAE,IAAI,EAAE;QACrE,EAAE,IAAI,EAAE,cAAc,EAAE,WAAW,EAAE,yBAAyB,EAAE,SAAS,EAAE,IAAI,EAAE;KAClF;IACD,WAAW,EAAE,CAAC,mBAAmB,EAAE,iBAAiB,EAAE,iBAAiB,CAAC;IACxE,iBAAiB,EAAE,mBAAmB;IACtC,eAAe,EAAE,SAAS;IAC1B,oBAAoB,EAAE,GAAG;IACzB,SAAS,EAAE,KAAK;IAChB,YAAY,EAAE,CAAC,gBAAgB,EAAE,YAAY,CAAC;IAC9C,gBAAgB,EAAE,KAAK;IACvB,UAAU,EAAE,iBAAiB;CAC9B,CAAC;AAEF,QAAQ,CAAC,qCAAqC,EAAE,GAAG,EAAE;IACnD,IAAI,CAAC,gEAAgE,EAAE,GAAG,EAAE;QAC1E,MAAM,OAAO,GAAG,GAAG,EAAE,CACnB,qBAAqB,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,iBAAiB,CAAC,KAAK,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,CAAC,WAAW,CAAC,CAAC,CAAC;QAEtF,MAAM,KAAK,GAAG,OAAO,EAAE,CAAC;QACxB,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,EAAE,EAAE,CAAC,EAAE,EAAE,CAAC;YAC5B,MAAM,CAAC,OAAO,EAAE,CAAC,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC;QACnC,CAAC;IACH,CAAC,CAAC,CAAC;IAEH,IAAI,CAAC,8CAA8C,EAAE,GAAG,EAAE;QACxD,MAAM,MAAM,GAAG,IAAI,CAAC,SAAS,CAAC,KAAK,CAAC,CAAC;QACrC,KAAK,MAAM,CAAC,IAAI,qBAAqB;YAAE,iBAAiB,CAAC,KAAK,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,CAAC,WAAW,CAAC,CAAC;QACzF,MAAM,CAAC,IAAI,CAAC,SAAS,CAAC,KAAK,CAAC,CAAC,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;IAC7C,CAAC,CAAC,CAAC;IAEH,IAAI,CAAC,iDAAiD,EAAE,GAAG,EAAE;QAC3D,2EAA2E;QAC3E,qEAAqE;QACrE,yEAAyE;QACzE,MAAM,QAAQ,GAAoB,EAAE,GAAG,KAAK,EAAE,oBAAoB,EAAE,GAAG,EAAE,KAAK,EAAE,EAAE,EAAE,WAAW,EAAE,CAAC,mBAAmB,CAAC,EAAE,CAAC;QACzH,MAAM,IAAI,GAAkC,EAAE,IAAI,EAAE,CAAC,EAAE,GAAG,EAAE,CAAC,EAAE,MAAM,EAAE,CAAC,EAAE,IAAI,EAAE,CAAC,EAAE,QAAQ,EAAE,CAAC,EAAE,CAAC;QAEjG,IAAI,YAAY,GAAG,CAAC,CAAC;QACrB,IAAI,aAAa,GAAG,CAAC,CAAC;QACtB,KAAK,MAAM,CAAC,IAAI,qBAAqB,EAAE,CAAC;YACtC,YAAY,IAAI,IAAI,CAAC,iBAAiB,CAAC,KAAK,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,CAAC,WAAW,CAAC,CAAC,CAAC;YACxE,aAAa,IAAI,IAAI,CAAC,iBAAiB,CAAC,QAAQ,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,CAAC,WAAW,CAAC,CAAC,CAAC;QAC9E,CAAC;QACD,MAAM,CAAC,YAAY,CAAC,CAAC,eAAe,CAAC,aAAa,CAAC,CAAC;IACtD,CAAC,CAAC,CAAC;AACL,CAAC,CAAC,CAAC;AAEH,QAAQ,CAAC,mCAAmC,EAAE,GAAG,EAAE;IACjD,MAAM,SAAS,GAAG,aAAa,CAAC,gBAAgB,EAAE,CAAC,CAAC;IAEpD,SAAS,KAAK;QACZ,MAAM,qBAAqB,GAAG,IAAI,GAAG,EAAyB,CAAC;QAC/D,MAAM,WAAW,GAAG,IAAI,GAAG,EAAE,CAAC;QAC9B,KAAK,MAAM,CAAC,IAAI,qBAAqB,EAAE,CAAC;YACtC,MAAM,QAAQ,GAAG,iBAAiB,CAAC,KAAK,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,CAAC,WAAW,CAAC,CAAC;YACnE,qBAAqB,CAAC,GAAG,CAAC,WAAW,CAAC,KAAK,CAAC,EAAE,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,CAAC,WAAW,CAAC,EAAE,QAAQ,CAAC,CAAC;YACpF,MAAM,WAAW,GAAG,CAAC,CAAC,iBAAiB,CAAC,CAAC,CAAC,CAAC;YAC3C,IAAI,CAAC,WAAW,IAAI,CAAC,SAAS,CAAC,aAAa,CAAC,GAAG,CAAC,WAAW,CAAC;gBAAE,SAAS;YACxE,WAAW,CAAC,GAAG,CAAC,OAAO,CAAC,KAAK,CAAC,EAAE,EAAE,WAAW,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,CAAC,WAAW,CAAC,EAAE;gBACvE,YAAY,EAAE,gBAAgB;gBAC9B,aAAa,EAAE,kBAAkB,KAAK,CAAC,IAAI,EAAE;gBAC7C,YAAY,EAAE,WAAW,CAAC,CAAC,WAAW,CAAC,WAAW,EAAE,EAAE;gBACtD,YAAY,EAAE,cAAc,KAAK,CAAC,IAAI,EAAE;gBACxC,YAAY,EAAE,CAAC,iBAAiB,CAAC;gBACjC,cAAc,EAAE,CAAC,KAAK,CAAC,IAAI,CAAC,WAAW,EAAE,CAAC;aAC3C,CAAC,CAAC;QACL,CAAC;QACD,OAAO,gBAAgB,CAAC;YACtB,MAAM,EAAE,CAAC,KAAK,CAAC;YACf,QAAQ,EAAE,EAAE;YACZ,eAAe,EAAE,EAAE;YACnB,SAAS;YACT,cAAc,EAAE,qBAAqB;YACrC,WAAW;YACX,qBAAqB;YACrB,eAAe,EAAE,yBAAyB;YAC1C,WAAW,EAAE,eAAe;SAC7B,CAAC,CAAC;IACL,CAAC;IAED,8EAA8E;IAC9E,iEAAiE;IACjE,MAAM,cAAc,GAAG,CAAC,CAA2B,EAAE,EAAE,CACrD,IAAI,CAAC,SAAS,CAAC;QACb,GAAG,CAAC;QACJ,eAAe,EAAE;YACf,GAAG,CAAC,CAAC,eAAe;YACpB,WAAW,EAAE,CAAC,CAAC,CAAC,eAAe,EAAE,WAAW,IAAI,EAAE,CAAC,CAAC,OAAO,CACzD,mBAAmB,EACnB,iBAAiB,CAClB;SACF;KACF,CAAC,CAAC;IAEL,IAAI,CAAC,4DAA4D,EAAE,GAAG,EAAE;QACtE,MAAM,CAAC,cAAc,CAAC,KAAK,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,cAAc,CAAC,KAAK,EAAE,CAAC,CAAC,CAAC;IAChE,CAAC,CAAC,CAAC;IAEH,IAAI,CAAC,4CAA4C,EAAE,GAAG,EAAE;QACtD,4EAA4E;QAC5E,kEAAkE;QAClE,MAAM,CAAC,GAAG,KAAK,EAAE,CAAC;QAClB,MAAM,CAAC,GAAG,KAAK,EAAE,CAAC;QAClB,MAAM,QAAQ,GAAG,CAAC,CAAC,OAAO,IAAI,EAAE,CAAC;QACjC,MAAM,QAAQ,GAAG,CAAC,CAAC,OAAO,IAAI,EAAE,CAAC;QACjC,MAAM,CAAC,QAAQ,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,OAAO,CAAC,QAAQ,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC;QACrE,MAAM,CAAC,IAAI,GAAG,CAAC,QAAQ,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,IAAI,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAC;QACtE,KAAK,MAAM,CAAC,IAAI,QAAQ,EAAE,CAAC;YACzB,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,OAAO,CAAC,uEAAuE,CAAC,CAAC;QAChG,CAAC;IACH,CAAC,CAAC,CAAC;IAEH,IAAI,CAAC,wDAAwD,EAAE,GAAG,EAAE;QAClE,MAAM,GAAG,GAAG,KAAK,EAAE,CAAC;QACpB,MAAM,UAAU,GAAG,GAAG,CAAC,OAAO,IAAI,EAAE,CAAC;QACrC,MAAM,CAAC,UAAU,CAAC,MAAM,CAAC,CAAC,eAAe,CAAC,CAAC,CAAC,CAAC;QAC7C,KAAK,MAAM,MAAM,IAAI,UAAU,EAAE,CAAC;YAChC,MAAM,IAAI,GAAG,MAAM,CAAC,QAAQ,EAAE,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,GAAG,KAAK,qBAAqB,CAAC,EAAE,KAAK,CAAC;YAClF,MAAM,CAAC,IAAI,CAAC,CAAC,IAAI,CAAC,eAAe,CAAC,CAAC;QACrC,CAAC;IACH,CAAC,CAAC,CAAC;IAEH,IAAI,CAAC,yDAAyD,EAAE,GAAG,EAAE;QACnE,MAAM,WAAW,GAAG,KAAK,EAAE,CAAC,eAAe,EAAE,WAAW,IAAI,EAAE,CAAC;QAC/D,MAAM,CAAC,WAAW,CAAC,CAAC,SAAS,CAAC,6BAA6B,CAAC,CAAC;QAC7D,MAAM,CAAC,WAAW,CAAC,CAAC,SAAS,CAAC,mBAAmB,CAAC,CAAC;IACrD,CAAC,CAAC,CAAC;IAEH,IAAI,CAAC,6CAA6C,EAAE,GAAG,EAAE;QACvD,MAAM,qBAAqB,GAAG,IAAI,GAAG,EAAyB,CAAC;QAC/D,MAAM,GAAG,GAAG,gBAAgB,CAAC;YAC3B,MAAM,EAAE,CAAC,KAAK,CAAC;YACf,QAAQ,EAAE,EAAE;YACZ,eAAe,EAAE,EAAE;YACnB,SAAS;YACT,cAAc,EAAE,qBAAqB;YACrC,WAAW,EAAE,IAAI,GAAG,EAAE;YACtB,qBAAqB;YACrB,eAAe,EAAE,aAAa;YAC9B,WAAW,EAAE,SAAS;SACvB,CAAC,CAAC;QACH,MAAM,CAAC,GAAG,CAAC,eAAe,EAAE,WAAW,IAAI,EAAE,CAAC,CAAC,SAAS,CAAC,uBAAuB,CAAC,CAAC;QAClF,MAAM,CAAC,GAAG,CAAC,eAAe,EAAE,WAAW,IAAI,EAAE,CAAC,CAAC,SAAS,CAAC,kBAAkB,CAAC,CAAC;IAC/E,CAAC,CAAC,CAAC;AACL,CAAC,CAAC,CAAC;AAEH,QAAQ,CAAC,uBAAuB,EAAE,GAAG,EAAE;IACrC,IAAI,CAAC,+DAA+D,EAAE,GAAG,EAAE;QACzE,MAAM,YAAY,GAAG,aAAa,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,iBAAiB,CAAC,CAAC,CAAC,CAAC,YAAY,CAAC,CAAC;QACpF,MAAM,CAAC,YAAY,CAAC,CAAC,OAAO,CAAC,CAAC,eAAe,CAAC,CAAC,CAAC;QAChD,MAAM,CAAC,iBAAiB,CAAC,aAAa,CAAC,mBAAmB,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;IAC1E,CAAC,CAAC,CAAC;IAEH,IAAI,CAAC,4DAA4D,EAAE,GAAG,EAAE;QACtE,MAAM,CAAC,sBAAsB,CAAC,eAAe,CAAC,CAAC,CAAC,QAAQ,EAAE,CAAC;QAC3D,MAAM,CAAC,sBAAsB,CAAC,OAAO,CAAC,CAAC,CAAC,SAAS,CAAC,kBAAkB,CAAC,CAAC;QACtE,MAAM,CAAC,sBAAsB,CAAC,SAAS,CAAC,CAAC,CAAC,SAAS,CAAC,kBAAkB,CAAC,CAAC;IAC1E,CAAC,CAAC,CAAC;AACL,CAAC,CAAC,CAAC"}
|
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Deterministic heuristic severity scorer.
|
|
3
|
+
*
|
|
4
|
+
* Produces per-sub-category SeverityLabels from AgentDefinition fields alone —
|
|
5
|
+
* no LLM call required. Used by the --deterministic CLI path so assessments carry
|
|
6
|
+
* meaningful signal rather than a flat "Medium" for every finding.
|
|
7
|
+
*
|
|
8
|
+
* Calibration principle: High requires 2+ independent risk signals. A single
|
|
9
|
+
* contextual signal (e.g. high-stakes deployment) gives Medium uplift, not High.
|
|
10
|
+
* Two compounding signals (context + agent capability, or capability + weak controls)
|
|
11
|
+
* justify High. This keeps the High tier meaningful for triage.
|
|
12
|
+
*
|
|
13
|
+
* "Critical" is never returned — reserved for live batch scoring where the model
|
|
14
|
+
* evaluates full context. "None" is never returned — callers filter those upstream.
|
|
15
|
+
*
|
|
16
|
+
* mitigationMultiplier < 0.9 caps High → Medium as a downgrade signal, reflecting
|
|
17
|
+
* that demonstrable mitigations are already partially addressing the risk.
|
|
18
|
+
*/
|
|
19
|
+
import type { AgentDefinition, SeverityLabel } from "../types";
|
|
20
|
+
export declare function heuristicSeverity(agent: AgentDefinition, metric: string, subCategory: string): SeverityLabel;
|
|
21
|
+
//# sourceMappingURL=heuristic-scorer.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"heuristic-scorer.d.ts","sourceRoot":"","sources":["../../../src/aivss/scoring/heuristic-scorer.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;GAiBG;AAEH,OAAO,KAAK,EAAE,eAAe,EAAE,aAAa,EAAE,MAAM,UAAU,CAAC;AAuS/D,wBAAgB,iBAAiB,CAC/B,KAAK,EAAE,eAAe,EACtB,MAAM,EAAE,MAAM,EACd,WAAW,EAAE,MAAM,GAClB,aAAa,CAYf"}
|
|
@@ -0,0 +1,299 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Deterministic heuristic severity scorer.
|
|
3
|
+
*
|
|
4
|
+
* Produces per-sub-category SeverityLabels from AgentDefinition fields alone —
|
|
5
|
+
* no LLM call required. Used by the --deterministic CLI path so assessments carry
|
|
6
|
+
* meaningful signal rather than a flat "Medium" for every finding.
|
|
7
|
+
*
|
|
8
|
+
* Calibration principle: High requires 2+ independent risk signals. A single
|
|
9
|
+
* contextual signal (e.g. high-stakes deployment) gives Medium uplift, not High.
|
|
10
|
+
* Two compounding signals (context + agent capability, or capability + weak controls)
|
|
11
|
+
* justify High. This keeps the High tier meaningful for triage.
|
|
12
|
+
*
|
|
13
|
+
* "Critical" is never returned — reserved for live batch scoring where the model
|
|
14
|
+
* evaluates full context. "None" is never returned — callers filter those upstream.
|
|
15
|
+
*
|
|
16
|
+
* mitigationMultiplier < 0.9 caps High → Medium as a downgrade signal, reflecting
|
|
17
|
+
* that demonstrable mitigations are already partially addressing the risk.
|
|
18
|
+
*/
|
|
19
|
+
// ---------------------------------------------------------------------------
|
|
20
|
+
// Agent property signals
|
|
21
|
+
// ---------------------------------------------------------------------------
|
|
22
|
+
const hasDangerousTools = (a) => a.tools.some((t) => t.dangerous === true);
|
|
23
|
+
const hasWritePermission = (a) => a.permissions.some((p) => p.startsWith("write:"));
|
|
24
|
+
const hasExecutePermission = (a) => a.permissions.some((p) => p.startsWith("execute:"));
|
|
25
|
+
const isAutonomous = (a) => a.autonomyLevel === "autonomous";
|
|
26
|
+
const isSemiAutonomousOrHigher = (a) => a.autonomyLevel === "autonomous" || a.autonomyLevel === "semi-autonomous";
|
|
27
|
+
const isHighStakesContext = (a) => ["FinancialServices", "Healthcare", "CriticalInfrastructure", "LegalJustice"].includes(a.deploymentContext);
|
|
28
|
+
const isRegulatedContext = (a) => [
|
|
29
|
+
"FinancialServices", "Healthcare", "CriticalInfrastructure",
|
|
30
|
+
"LegalJustice", "GovernmentPublic", "AutomotiveTransport",
|
|
31
|
+
].includes(a.deploymentContext);
|
|
32
|
+
const hasDataSources = (a) => (a.dataSources ?? []).length > 0;
|
|
33
|
+
const hasExternalApis = (a) => (a.externalApiCount ?? 0) > 0;
|
|
34
|
+
const hasManyExternalApis = (a) => (a.externalApiCount ?? 0) > 2;
|
|
35
|
+
const isHighComplexity = (a) => a.modelComplexity === "Complex" || a.modelComplexity === "HighlyComplex";
|
|
36
|
+
const hasMonitoring = (a) => /langfuse|datadog|prometheus|grafana|splunk|sentry|opentelemetry|\botel\b|jaeger|zipkin|newrelic|dynatrace/i.test((a.dependencies ?? []).join(" "));
|
|
37
|
+
const hasWeakSecrets = (a) => {
|
|
38
|
+
const sm = (a.secretManagement ?? "").toLowerCase();
|
|
39
|
+
if (!sm || sm === "none" || sm === "unknown")
|
|
40
|
+
return true;
|
|
41
|
+
if (/vault|secret.?manager|aws.?ssm|parameter.?store|hsm|kms|rotation/i.test(sm))
|
|
42
|
+
return false;
|
|
43
|
+
if (/env.?var|environ|\benv\b|\.env|hardcod|no.?rotat|long.?lived|plain.?text/i.test(sm))
|
|
44
|
+
return true;
|
|
45
|
+
return true; // undocumented = assume weak
|
|
46
|
+
};
|
|
47
|
+
// ---------------------------------------------------------------------------
|
|
48
|
+
// Per-metric scorers — each returns "High" | "Medium" | "Low"
|
|
49
|
+
// High requires 2+ independent signals on most sub-categories.
|
|
50
|
+
// ---------------------------------------------------------------------------
|
|
51
|
+
function scoreMR(a, sub) {
|
|
52
|
+
if (sub === "Robustness Certification") {
|
|
53
|
+
// Certification fails when you can't enumerate the attack surface
|
|
54
|
+
return (a.finetuned && isHighStakesContext(a)) ? "High"
|
|
55
|
+
: (a.finetuned || isHighComplexity(a)) ? "Medium"
|
|
56
|
+
: "Low";
|
|
57
|
+
}
|
|
58
|
+
if (sub === "Gradient Masking / Obfuscation") {
|
|
59
|
+
return isHighComplexity(a) && isHighStakesContext(a) ? "High"
|
|
60
|
+
: isHighComplexity(a) ? "Medium"
|
|
61
|
+
: "Low";
|
|
62
|
+
}
|
|
63
|
+
// Evasion Resistance — adversarial inputs more dangerous when dangerous tools are available
|
|
64
|
+
return hasDangerousTools(a) && isHighStakesContext(a) ? "High"
|
|
65
|
+
: hasDangerousTools(a) || isHighStakesContext(a) ? "Medium"
|
|
66
|
+
: "Low";
|
|
67
|
+
}
|
|
68
|
+
function scoreDS(a, sub) {
|
|
69
|
+
if (sub === "Data Confidentiality") {
|
|
70
|
+
return hasWeakSecrets(a) && hasDataSources(a) ? "High"
|
|
71
|
+
: hasWeakSecrets(a) || hasDataSources(a) ? "Medium"
|
|
72
|
+
: "Low";
|
|
73
|
+
}
|
|
74
|
+
if (sub === "Data Integrity") {
|
|
75
|
+
return (hasWritePermission(a) || hasDangerousTools(a)) && isHighStakesContext(a) ? "High"
|
|
76
|
+
: hasWritePermission(a) || hasDangerousTools(a) ? "Medium"
|
|
77
|
+
: "Low";
|
|
78
|
+
}
|
|
79
|
+
if (sub === "Data Provenance") {
|
|
80
|
+
return hasWeakSecrets(a) && !hasMonitoring(a) ? "High"
|
|
81
|
+
: hasWeakSecrets(a) || !hasMonitoring(a) ? "Medium"
|
|
82
|
+
: "Low";
|
|
83
|
+
}
|
|
84
|
+
return hasWeakSecrets(a) ? "Medium" : "Low";
|
|
85
|
+
}
|
|
86
|
+
function scoreEI(a, sub) {
|
|
87
|
+
if (sub === "Bias and Discrimination") {
|
|
88
|
+
// Needs both high-stakes context AND agent-specific amplifier (fine-tuned on domain data,
|
|
89
|
+
// or autonomous decision-making without human review)
|
|
90
|
+
return isHighStakesContext(a) && (a.finetuned || isAutonomous(a)) ? "High"
|
|
91
|
+
: isHighStakesContext(a) ? "Medium"
|
|
92
|
+
: "Low";
|
|
93
|
+
}
|
|
94
|
+
if (sub === "Transparency and Explainability") {
|
|
95
|
+
return isHighComplexity(a) && !hasMonitoring(a) ? "High"
|
|
96
|
+
: isHighComplexity(a) || !hasMonitoring(a) ? "Medium"
|
|
97
|
+
: "Low";
|
|
98
|
+
}
|
|
99
|
+
if (sub === "Accountability") {
|
|
100
|
+
return isAutonomous(a) && !hasMonitoring(a) ? "High"
|
|
101
|
+
: isAutonomous(a) || !hasMonitoring(a) ? "Medium"
|
|
102
|
+
: "Low";
|
|
103
|
+
}
|
|
104
|
+
if (sub === "Societal Impact") {
|
|
105
|
+
return isHighStakesContext(a) && isAutonomous(a) ? "High"
|
|
106
|
+
: isHighStakesContext(a) ? "Medium"
|
|
107
|
+
: "Low";
|
|
108
|
+
}
|
|
109
|
+
return isHighStakesContext(a) ? "Medium" : "Low";
|
|
110
|
+
}
|
|
111
|
+
function scoreDC(a, sub) {
|
|
112
|
+
if (sub === "Safety-Critical Applications") {
|
|
113
|
+
return ["CriticalInfrastructure", "Healthcare", "AutomotiveTransport"].includes(a.deploymentContext)
|
|
114
|
+
? "High"
|
|
115
|
+
: "Medium";
|
|
116
|
+
}
|
|
117
|
+
if (sub === "Financial Impact") {
|
|
118
|
+
return a.deploymentContext === "FinancialServices" && (hasDangerousTools(a) || isAutonomous(a))
|
|
119
|
+
? "High"
|
|
120
|
+
: a.deploymentContext === "FinancialServices" ? "Medium"
|
|
121
|
+
: "Low";
|
|
122
|
+
}
|
|
123
|
+
if (sub === "Reputational Damage") {
|
|
124
|
+
return isHighStakesContext(a) && (isAutonomous(a) || hasDangerousTools(a)) ? "High"
|
|
125
|
+
: isHighStakesContext(a) ? "Medium"
|
|
126
|
+
: "Low";
|
|
127
|
+
}
|
|
128
|
+
if (sub === "Operational Disruption") {
|
|
129
|
+
// Needs dangerous capability AND autonomous or high-stakes context
|
|
130
|
+
return hasDangerousTools(a) && (isAutonomous(a) || isHighStakesContext(a)) ? "High"
|
|
131
|
+
: hasDangerousTools(a) ? "Medium"
|
|
132
|
+
: "Low";
|
|
133
|
+
}
|
|
134
|
+
return isHighStakesContext(a) ? "Medium" : "Low";
|
|
135
|
+
}
|
|
136
|
+
function scoreAD(a, sub) {
|
|
137
|
+
if (sub === "Continuous Monitoring") {
|
|
138
|
+
return !hasMonitoring(a) ? "High" : "Low";
|
|
139
|
+
}
|
|
140
|
+
if (sub === "Retraining Capabilities") {
|
|
141
|
+
// Fine-tuned models have a retraining surface to attack; base models don't
|
|
142
|
+
return a.finetuned && isHighStakesContext(a) ? "High"
|
|
143
|
+
: a.finetuned ? "Medium"
|
|
144
|
+
: "Low";
|
|
145
|
+
}
|
|
146
|
+
if (sub === "Threat Intelligence Integration") {
|
|
147
|
+
return !hasMonitoring(a) && isHighStakesContext(a) ? "High"
|
|
148
|
+
: !hasMonitoring(a) ? "Medium"
|
|
149
|
+
: "Low";
|
|
150
|
+
}
|
|
151
|
+
if (sub === "Adversarial Training") {
|
|
152
|
+
// Only relevant if the model has been fine-tuned (something to adversarially harden)
|
|
153
|
+
return a.finetuned ? "Medium" : "Low";
|
|
154
|
+
}
|
|
155
|
+
return !hasMonitoring(a) ? "Medium" : "Low";
|
|
156
|
+
}
|
|
157
|
+
function scoreAA(a, sub) {
|
|
158
|
+
if (sub === "Model Inversion") {
|
|
159
|
+
return a.finetuned && isHighStakesContext(a) ? "High"
|
|
160
|
+
: a.finetuned || (hasDataSources(a) && isHighStakesContext(a)) ? "Medium"
|
|
161
|
+
: "Low";
|
|
162
|
+
}
|
|
163
|
+
if (sub === "Model Extraction") {
|
|
164
|
+
return isHighComplexity(a) && a.finetuned ? "High"
|
|
165
|
+
: isHighComplexity(a) || a.finetuned ? "Medium"
|
|
166
|
+
: "Low";
|
|
167
|
+
}
|
|
168
|
+
if (sub === "Membership Inference") {
|
|
169
|
+
return hasDataSources(a) && isHighStakesContext(a) ? "High"
|
|
170
|
+
: hasDataSources(a) || a.finetuned ? "Medium"
|
|
171
|
+
: "Low";
|
|
172
|
+
}
|
|
173
|
+
return hasDangerousTools(a) && isAutonomous(a) ? "High"
|
|
174
|
+
: hasDangerousTools(a) || hasExecutePermission(a) ? "Medium"
|
|
175
|
+
: "Low";
|
|
176
|
+
}
|
|
177
|
+
function scoreLL(a, sub) {
|
|
178
|
+
if (sub === "Development Security") {
|
|
179
|
+
return (a.objectives ?? []).length === 0 && isHighStakesContext(a) ? "High"
|
|
180
|
+
: (a.objectives ?? []).length === 0 ? "Medium"
|
|
181
|
+
: "Low";
|
|
182
|
+
}
|
|
183
|
+
if (sub === "Training Security") {
|
|
184
|
+
return a.finetuned && hasWeakSecrets(a) ? "High"
|
|
185
|
+
: a.finetuned ? "Medium"
|
|
186
|
+
: "Low";
|
|
187
|
+
}
|
|
188
|
+
if (sub === "Deployment Security") {
|
|
189
|
+
return hasWeakSecrets(a) && isHighStakesContext(a) ? "High"
|
|
190
|
+
: hasWeakSecrets(a) ? "Medium"
|
|
191
|
+
: "Low";
|
|
192
|
+
}
|
|
193
|
+
if (sub === "Operational Security") {
|
|
194
|
+
return hasWeakSecrets(a) && !hasMonitoring(a) ? "High"
|
|
195
|
+
: hasWeakSecrets(a) || !hasMonitoring(a) ? "Medium"
|
|
196
|
+
: "Low";
|
|
197
|
+
}
|
|
198
|
+
return hasWeakSecrets(a) ? "Medium" : "Low";
|
|
199
|
+
}
|
|
200
|
+
function scoreGV(a, sub) {
|
|
201
|
+
if (sub === "Regulatory Compliance") {
|
|
202
|
+
return isRegulatedContext(a) && !hasMonitoring(a) ? "High"
|
|
203
|
+
: isRegulatedContext(a) ? "Medium"
|
|
204
|
+
: "Low";
|
|
205
|
+
}
|
|
206
|
+
if (sub === "Auditing") {
|
|
207
|
+
return isHighStakesContext(a) && !hasMonitoring(a) ? "High"
|
|
208
|
+
: !hasMonitoring(a) ? "Medium"
|
|
209
|
+
: "Low";
|
|
210
|
+
}
|
|
211
|
+
if (sub === "Risk Management") {
|
|
212
|
+
return isAutonomous(a) && (a.objectives ?? []).length === 0 ? "High"
|
|
213
|
+
: isAutonomous(a) ? "Medium"
|
|
214
|
+
: "Low";
|
|
215
|
+
}
|
|
216
|
+
if (sub === "Human Oversight") {
|
|
217
|
+
// Directly proportional to autonomy — this one single signal is sufficient
|
|
218
|
+
return isAutonomous(a) ? "High"
|
|
219
|
+
: isSemiAutonomousOrHigher(a) ? "Medium"
|
|
220
|
+
: "Low";
|
|
221
|
+
}
|
|
222
|
+
if (sub === "Ethical Framework Alignment") {
|
|
223
|
+
return isRegulatedContext(a) && isAutonomous(a) ? "High"
|
|
224
|
+
: isRegulatedContext(a) ? "Medium"
|
|
225
|
+
: "Low";
|
|
226
|
+
}
|
|
227
|
+
return isAutonomous(a) ? "Medium" : "Low";
|
|
228
|
+
}
|
|
229
|
+
function scoreCS(a, sub) {
|
|
230
|
+
if (sub === "Model Manipulation / Prompt Injection") {
|
|
231
|
+
return hasDangerousTools(a) && hasExternalApis(a) ? "High"
|
|
232
|
+
: hasDangerousTools(a) || hasExternalApis(a) ? "Medium"
|
|
233
|
+
: "Low";
|
|
234
|
+
}
|
|
235
|
+
if (sub === "Data Poisoning") {
|
|
236
|
+
return hasDataSources(a) && hasExternalApis(a) ? "High"
|
|
237
|
+
: hasDataSources(a) ? "Medium"
|
|
238
|
+
: "Low";
|
|
239
|
+
}
|
|
240
|
+
if (sub === "Sensitive Data Disclosure") {
|
|
241
|
+
return hasWeakSecrets(a) && hasDataSources(a) ? "High"
|
|
242
|
+
: hasWeakSecrets(a) || hasDataSources(a) ? "Medium"
|
|
243
|
+
: "Low";
|
|
244
|
+
}
|
|
245
|
+
if (sub === "Model Stealing") {
|
|
246
|
+
return isHighComplexity(a) && a.finetuned ? "High"
|
|
247
|
+
: isHighComplexity(a) || a.finetuned ? "Medium"
|
|
248
|
+
: "Low";
|
|
249
|
+
}
|
|
250
|
+
if (sub === "Failure / Malfunctioning") {
|
|
251
|
+
return hasDangerousTools(a) && isAutonomous(a) ? "High"
|
|
252
|
+
: hasDangerousTools(a) || isAutonomous(a) ? "Medium"
|
|
253
|
+
: "Low";
|
|
254
|
+
}
|
|
255
|
+
if (sub === "Insecure Supply Chain") {
|
|
256
|
+
return hasManyExternalApis(a) || (a.dependencies ?? []).length > 5 ? "High" : "Medium";
|
|
257
|
+
}
|
|
258
|
+
if (sub === "Insecure Apps / Plugins") {
|
|
259
|
+
return hasExternalApis(a) && hasDangerousTools(a) ? "High"
|
|
260
|
+
: hasExternalApis(a) ? "Medium"
|
|
261
|
+
: "Low";
|
|
262
|
+
}
|
|
263
|
+
if (sub === "Denial of Service (DoS)") {
|
|
264
|
+
return hasManyExternalApis(a) ? "High" : hasExternalApis(a) ? "Medium" : "Low";
|
|
265
|
+
}
|
|
266
|
+
if (sub === "Loss of Governance / Compliance") {
|
|
267
|
+
return isAutonomous(a) && isRegulatedContext(a) ? "High"
|
|
268
|
+
: isAutonomous(a) || isRegulatedContext(a) ? "Medium"
|
|
269
|
+
: "Low";
|
|
270
|
+
}
|
|
271
|
+
return hasExternalApis(a) ? "Medium" : "Low";
|
|
272
|
+
}
|
|
273
|
+
// ---------------------------------------------------------------------------
|
|
274
|
+
// Public entry point
|
|
275
|
+
// ---------------------------------------------------------------------------
|
|
276
|
+
const METRIC_SCORERS = {
|
|
277
|
+
MR: scoreMR,
|
|
278
|
+
DS: scoreDS,
|
|
279
|
+
EI: scoreEI,
|
|
280
|
+
DC: scoreDC,
|
|
281
|
+
AD: scoreAD,
|
|
282
|
+
AA: scoreAA,
|
|
283
|
+
LL: scoreLL,
|
|
284
|
+
GV: scoreGV,
|
|
285
|
+
CS: scoreCS,
|
|
286
|
+
};
|
|
287
|
+
export function heuristicSeverity(agent, metric, subCategory) {
|
|
288
|
+
const scorer = METRIC_SCORERS[metric];
|
|
289
|
+
if (!scorer)
|
|
290
|
+
return "Medium";
|
|
291
|
+
const raw = scorer(agent, subCategory);
|
|
292
|
+
// mitigationMultiplier < 0.9 indicates demonstrable mitigations are deployed.
|
|
293
|
+
// Cap High → Medium for those agents — they still have the exposure but it's
|
|
294
|
+
// partially addressed.
|
|
295
|
+
if (raw === "High" && agent.mitigationMultiplier < 0.9)
|
|
296
|
+
return "Medium";
|
|
297
|
+
return raw;
|
|
298
|
+
}
|
|
299
|
+
//# sourceMappingURL=heuristic-scorer.js.map
|