@waratahlabs/canopy 0.5.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +21 -0
- package/README.md +323 -0
- package/SKILL.md +97 -0
- package/demo/data/openclaw-demo.tc.json +397 -0
- package/demo/data/openclaw-nemoclaw-demo.tc.json +475 -0
- package/demo/index.html +556 -0
- package/dist/adapters/a2a.d.ts +82 -0
- package/dist/adapters/a2a.d.ts.map +1 -0
- package/dist/adapters/a2a.js +215 -0
- package/dist/adapters/a2a.js.map +1 -0
- package/dist/adapters/a2a.test.d.ts +2 -0
- package/dist/adapters/a2a.test.d.ts.map +1 -0
- package/dist/adapters/a2a.test.js +196 -0
- package/dist/adapters/a2a.test.js.map +1 -0
- package/dist/adapters/langfuse.d.ts +28 -0
- package/dist/adapters/langfuse.d.ts.map +1 -0
- package/dist/adapters/langfuse.js +192 -0
- package/dist/adapters/langfuse.js.map +1 -0
- package/dist/adapters/n8n.d.ts +4 -0
- package/dist/adapters/n8n.d.ts.map +1 -0
- package/dist/adapters/n8n.js +89 -0
- package/dist/adapters/n8n.js.map +1 -0
- package/dist/adapters/openclaw.d.ts +58 -0
- package/dist/adapters/openclaw.d.ts.map +1 -0
- package/dist/adapters/openclaw.js +275 -0
- package/dist/adapters/openclaw.js.map +1 -0
- package/dist/adapters/types.d.ts +33 -0
- package/dist/adapters/types.d.ts.map +1 -0
- package/dist/adapters/types.js +2 -0
- package/dist/adapters/types.js.map +1 -0
- package/dist/aivss/atlas/coverage-matrix.d.ts +12 -0
- package/dist/aivss/atlas/coverage-matrix.d.ts.map +1 -0
- package/dist/aivss/atlas/coverage-matrix.js +272 -0
- package/dist/aivss/atlas/coverage-matrix.js.map +1 -0
- package/dist/aivss/atlas/loader.d.ts +28 -0
- package/dist/aivss/atlas/loader.d.ts.map +1 -0
- package/dist/aivss/atlas/loader.js +126 -0
- package/dist/aivss/atlas/loader.js.map +1 -0
- package/dist/aivss/atlas/loader.test.d.ts +11 -0
- package/dist/aivss/atlas/loader.test.d.ts.map +1 -0
- package/dist/aivss/atlas/loader.test.js +111 -0
- package/dist/aivss/atlas/loader.test.js.map +1 -0
- package/dist/aivss/atlas/resolve-path.d.ts +34 -0
- package/dist/aivss/atlas/resolve-path.d.ts.map +1 -0
- package/dist/aivss/atlas/resolve-path.js +58 -0
- package/dist/aivss/atlas/resolve-path.js.map +1 -0
- package/dist/aivss/batch/batch-builder.d.ts +11 -0
- package/dist/aivss/batch/batch-builder.d.ts.map +1 -0
- package/dist/aivss/batch/batch-builder.js +75 -0
- package/dist/aivss/batch/batch-builder.js.map +1 -0
- package/dist/aivss/index.d.ts +21 -0
- package/dist/aivss/index.d.ts.map +1 -0
- package/dist/aivss/index.js +19 -0
- package/dist/aivss/index.js.map +1 -0
- package/dist/aivss/knowledge-base/kb-client.d.ts +9 -0
- package/dist/aivss/knowledge-base/kb-client.d.ts.map +1 -0
- package/dist/aivss/knowledge-base/kb-client.js +139 -0
- package/dist/aivss/knowledge-base/kb-client.js.map +1 -0
- package/dist/aivss/knowledge-base/kb-types.d.ts +21 -0
- package/dist/aivss/knowledge-base/kb-types.d.ts.map +1 -0
- package/dist/aivss/knowledge-base/kb-types.js +2 -0
- package/dist/aivss/knowledge-base/kb-types.js.map +1 -0
- package/dist/aivss/knowledge-base/search-builder.d.ts +7 -0
- package/dist/aivss/knowledge-base/search-builder.d.ts.map +1 -0
- package/dist/aivss/knowledge-base/search-builder.js +119 -0
- package/dist/aivss/knowledge-base/search-builder.js.map +1 -0
- package/dist/aivss/metrics/aa.d.ts +3 -0
- package/dist/aivss/metrics/aa.d.ts.map +1 -0
- package/dist/aivss/metrics/aa.js +26 -0
- package/dist/aivss/metrics/aa.js.map +1 -0
- package/dist/aivss/metrics/ad.d.ts +3 -0
- package/dist/aivss/metrics/ad.d.ts.map +1 -0
- package/dist/aivss/metrics/ad.js +30 -0
- package/dist/aivss/metrics/ad.js.map +1 -0
- package/dist/aivss/metrics/cs.d.ts +3 -0
- package/dist/aivss/metrics/cs.d.ts.map +1 -0
- package/dist/aivss/metrics/cs.js +50 -0
- package/dist/aivss/metrics/cs.js.map +1 -0
- package/dist/aivss/metrics/dc.d.ts +3 -0
- package/dist/aivss/metrics/dc.d.ts.map +1 -0
- package/dist/aivss/metrics/dc.js +30 -0
- package/dist/aivss/metrics/dc.js.map +1 -0
- package/dist/aivss/metrics/ds.d.ts +3 -0
- package/dist/aivss/metrics/ds.d.ts.map +1 -0
- package/dist/aivss/metrics/ds.js +26 -0
- package/dist/aivss/metrics/ds.js.map +1 -0
- package/dist/aivss/metrics/ei.d.ts +3 -0
- package/dist/aivss/metrics/ei.d.ts.map +1 -0
- package/dist/aivss/metrics/ei.js +30 -0
- package/dist/aivss/metrics/ei.js.map +1 -0
- package/dist/aivss/metrics/gv.d.ts +3 -0
- package/dist/aivss/metrics/gv.d.ts.map +1 -0
- package/dist/aivss/metrics/gv.js +34 -0
- package/dist/aivss/metrics/gv.js.map +1 -0
- package/dist/aivss/metrics/ll.d.ts +3 -0
- package/dist/aivss/metrics/ll.d.ts.map +1 -0
- package/dist/aivss/metrics/ll.js +30 -0
- package/dist/aivss/metrics/ll.js.map +1 -0
- package/dist/aivss/metrics/mr.d.ts +3 -0
- package/dist/aivss/metrics/mr.d.ts.map +1 -0
- package/dist/aivss/metrics/mr.js +26 -0
- package/dist/aivss/metrics/mr.js.map +1 -0
- package/dist/aivss/metrics/shared.d.ts +22 -0
- package/dist/aivss/metrics/shared.d.ts.map +1 -0
- package/dist/aivss/metrics/shared.js +70 -0
- package/dist/aivss/metrics/shared.js.map +1 -0
- package/dist/aivss/prompts/base-system-prompt.d.ts +2 -0
- package/dist/aivss/prompts/base-system-prompt.d.ts.map +1 -0
- package/dist/aivss/prompts/base-system-prompt.js +31 -0
- package/dist/aivss/prompts/base-system-prompt.js.map +1 -0
- package/dist/aivss/prompts/prompt-composer.d.ts +13 -0
- package/dist/aivss/prompts/prompt-composer.d.ts.map +1 -0
- package/dist/aivss/prompts/prompt-composer.js +21 -0
- package/dist/aivss/prompts/prompt-composer.js.map +1 -0
- package/dist/aivss/scoring/a2a-corpus.test.d.ts +28 -0
- package/dist/aivss/scoring/a2a-corpus.test.d.ts.map +1 -0
- package/dist/aivss/scoring/a2a-corpus.test.js +109 -0
- package/dist/aivss/scoring/a2a-corpus.test.js.map +1 -0
- package/dist/aivss/scoring/determinism.test.d.ts +16 -0
- package/dist/aivss/scoring/determinism.test.d.ts.map +1 -0
- package/dist/aivss/scoring/determinism.test.js +169 -0
- package/dist/aivss/scoring/determinism.test.js.map +1 -0
- package/dist/aivss/scoring/heuristic-scorer.d.ts +21 -0
- package/dist/aivss/scoring/heuristic-scorer.d.ts.map +1 -0
- package/dist/aivss/scoring/heuristic-scorer.js +299 -0
- package/dist/aivss/scoring/heuristic-scorer.js.map +1 -0
- package/dist/aivss/scoring/heuristic-scorer.test.d.ts +8 -0
- package/dist/aivss/scoring/heuristic-scorer.test.d.ts.map +1 -0
- package/dist/aivss/scoring/heuristic-scorer.test.js +150 -0
- package/dist/aivss/scoring/heuristic-scorer.test.js.map +1 -0
- package/dist/aivss/scoring/scoring-mode.d.ts +48 -0
- package/dist/aivss/scoring/scoring-mode.d.ts.map +1 -0
- package/dist/aivss/scoring/scoring-mode.js +64 -0
- package/dist/aivss/scoring/scoring-mode.js.map +1 -0
- package/dist/aivss/scoring/v4-engine.d.ts +15 -0
- package/dist/aivss/scoring/v4-engine.d.ts.map +1 -0
- package/dist/aivss/scoring/v4-engine.js +84 -0
- package/dist/aivss/scoring/v4-engine.js.map +1 -0
- package/dist/aivss/scoring/v4-engine.test.d.ts +16 -0
- package/dist/aivss/scoring/v4-engine.test.d.ts.map +1 -0
- package/dist/aivss/scoring/v4-engine.test.js +156 -0
- package/dist/aivss/scoring/v4-engine.test.js.map +1 -0
- package/dist/aivss/types.d.ts +92 -0
- package/dist/aivss/types.d.ts.map +1 -0
- package/dist/aivss/types.js +25 -0
- package/dist/aivss/types.js.map +1 -0
- package/dist/baseline/baseline.test.d.ts +2 -0
- package/dist/baseline/baseline.test.d.ts.map +1 -0
- package/dist/baseline/baseline.test.js +219 -0
- package/dist/baseline/baseline.test.js.map +1 -0
- package/dist/baseline/diff.d.ts +42 -0
- package/dist/baseline/diff.d.ts.map +1 -0
- package/dist/baseline/diff.js +124 -0
- package/dist/baseline/diff.js.map +1 -0
- package/dist/baseline/store.d.ts +51 -0
- package/dist/baseline/store.d.ts.map +1 -0
- package/dist/baseline/store.js +92 -0
- package/dist/baseline/store.js.map +1 -0
- package/dist/batch/anthropic-batch-runner.d.ts +50 -0
- package/dist/batch/anthropic-batch-runner.d.ts.map +1 -0
- package/dist/batch/anthropic-batch-runner.js +91 -0
- package/dist/batch/anthropic-batch-runner.js.map +1 -0
- package/dist/batch/bedrock-batch-runner.d.ts +90 -0
- package/dist/batch/bedrock-batch-runner.d.ts.map +1 -0
- package/dist/batch/bedrock-batch-runner.js +199 -0
- package/dist/batch/bedrock-batch-runner.js.map +1 -0
- package/dist/batch/providers/anthropic.d.ts +55 -0
- package/dist/batch/providers/anthropic.d.ts.map +1 -0
- package/dist/batch/providers/anthropic.js +103 -0
- package/dist/batch/providers/anthropic.js.map +1 -0
- package/dist/batch/providers/bedrock.d.ts +31 -0
- package/dist/batch/providers/bedrock.d.ts.map +1 -0
- package/dist/batch/providers/bedrock.js +42 -0
- package/dist/batch/providers/bedrock.js.map +1 -0
- package/dist/batch/providers/harness.d.ts +36 -0
- package/dist/batch/providers/harness.d.ts.map +1 -0
- package/dist/batch/providers/harness.js +65 -0
- package/dist/batch/providers/harness.js.map +1 -0
- package/dist/batch/providers/harness.test.d.ts +7 -0
- package/dist/batch/providers/harness.test.d.ts.map +1 -0
- package/dist/batch/providers/harness.test.js +84 -0
- package/dist/batch/providers/harness.test.js.map +1 -0
- package/dist/batch/providers/index.d.ts +82 -0
- package/dist/batch/providers/index.d.ts.map +1 -0
- package/dist/batch/providers/index.js +82 -0
- package/dist/batch/providers/index.js.map +1 -0
- package/dist/batch/providers/ollama.d.ts +68 -0
- package/dist/batch/providers/ollama.d.ts.map +1 -0
- package/dist/batch/providers/ollama.js +108 -0
- package/dist/batch/providers/ollama.js.map +1 -0
- package/dist/batch/response-parser.d.ts +45 -0
- package/dist/batch/response-parser.d.ts.map +1 -0
- package/dist/batch/response-parser.js +161 -0
- package/dist/batch/response-parser.js.map +1 -0
- package/dist/batch/types.d.ts +87 -0
- package/dist/batch/types.d.ts.map +1 -0
- package/dist/batch/types.js +8 -0
- package/dist/batch/types.js.map +1 -0
- package/dist/cli.d.ts +15 -0
- package/dist/cli.d.ts.map +1 -0
- package/dist/cli.js +1037 -0
- package/dist/cli.js.map +1 -0
- package/dist/examples/a2a-assessment.d.ts +20 -0
- package/dist/examples/a2a-assessment.d.ts.map +1 -0
- package/dist/examples/a2a-assessment.js +115 -0
- package/dist/examples/a2a-assessment.js.map +1 -0
- package/dist/examples/batch-assessment.d.ts +18 -0
- package/dist/examples/batch-assessment.d.ts.map +1 -0
- package/dist/examples/batch-assessment.js +112 -0
- package/dist/examples/batch-assessment.js.map +1 -0
- package/dist/examples/batch-submission-flow.d.ts +2 -0
- package/dist/examples/batch-submission-flow.d.ts.map +1 -0
- package/dist/examples/batch-submission-flow.js +279 -0
- package/dist/examples/batch-submission-flow.js.map +1 -0
- package/dist/examples/demo-agent-definition.json +58 -0
- package/dist/examples/langfuse-assessment.d.ts +2 -0
- package/dist/examples/langfuse-assessment.d.ts.map +1 -0
- package/dist/examples/langfuse-assessment.js +46 -0
- package/dist/examples/langfuse-assessment.js.map +1 -0
- package/dist/examples/model-eval.d.ts +25 -0
- package/dist/examples/model-eval.d.ts.map +1 -0
- package/dist/examples/model-eval.js +94 -0
- package/dist/examples/model-eval.js.map +1 -0
- package/dist/examples/openclaw-assessment.d.ts +17 -0
- package/dist/examples/openclaw-assessment.d.ts.map +1 -0
- package/dist/examples/openclaw-assessment.js +79 -0
- package/dist/examples/openclaw-assessment.js.map +1 -0
- package/dist/examples/simple-assessment.d.ts +2 -0
- package/dist/examples/simple-assessment.d.ts.map +1 -0
- package/dist/examples/simple-assessment.js +146 -0
- package/dist/examples/simple-assessment.js.map +1 -0
- package/dist/examples/threat-model-demo.d.ts +2 -0
- package/dist/examples/threat-model-demo.d.ts.map +1 -0
- package/dist/examples/threat-model-demo.js +103 -0
- package/dist/examples/threat-model-demo.js.map +1 -0
- package/dist/index.d.ts +60 -0
- package/dist/index.d.ts.map +1 -0
- package/dist/index.js +155 -0
- package/dist/index.js.map +1 -0
- package/dist/output/likec4-generator.d.ts +43 -0
- package/dist/output/likec4-generator.d.ts.map +1 -0
- package/dist/output/likec4-generator.js +287 -0
- package/dist/output/likec4-generator.js.map +1 -0
- package/dist/output/slot-filler.d.ts +6 -0
- package/dist/output/slot-filler.d.ts.map +1 -0
- package/dist/output/slot-filler.js +71 -0
- package/dist/output/slot-filler.js.map +1 -0
- package/dist/output/tc-assembler.d.ts +30 -0
- package/dist/output/tc-assembler.d.ts.map +1 -0
- package/dist/output/tc-assembler.js +227 -0
- package/dist/output/tc-assembler.js.map +1 -0
- package/dist/output/tc-types.d.ts +41 -0
- package/dist/output/tc-types.d.ts.map +1 -0
- package/dist/output/tc-types.js +5 -0
- package/dist/output/tc-types.js.map +1 -0
- package/dist/output/tc-validator.test.d.ts +9 -0
- package/dist/output/tc-validator.test.d.ts.map +1 -0
- package/dist/output/tc-validator.test.js +194 -0
- package/dist/output/tc-validator.test.js.map +1 -0
- package/dist/providers/anthropic.d.ts +9 -0
- package/dist/providers/anthropic.d.ts.map +1 -0
- package/dist/providers/anthropic.js +30 -0
- package/dist/providers/anthropic.js.map +1 -0
- package/dist/providers/bedrock.d.ts +9 -0
- package/dist/providers/bedrock.d.ts.map +1 -0
- package/dist/providers/bedrock.js +38 -0
- package/dist/providers/bedrock.js.map +1 -0
- package/dist/providers/index.d.ts +6 -0
- package/dist/providers/index.d.ts.map +1 -0
- package/dist/providers/index.js +14 -0
- package/dist/providers/index.js.map +1 -0
- package/dist/providers/types.d.ts +12 -0
- package/dist/providers/types.d.ts.map +1 -0
- package/dist/providers/types.js +5 -0
- package/dist/providers/types.js.map +1 -0
- package/dist/references/acsc-risks.d.ts +18 -0
- package/dist/references/acsc-risks.d.ts.map +1 -0
- package/dist/references/acsc-risks.js +232 -0
- package/dist/references/acsc-risks.js.map +1 -0
- package/dist/references/aicm.d.ts +41 -0
- package/dist/references/aicm.d.ts.map +1 -0
- package/dist/references/aicm.js +447 -0
- package/dist/references/aicm.js.map +1 -0
- package/dist/references/dstg-meaid.d.ts +59 -0
- package/dist/references/dstg-meaid.d.ts.map +1 -0
- package/dist/references/dstg-meaid.js +352 -0
- package/dist/references/dstg-meaid.js.map +1 -0
- package/dist/references/index.d.ts +20 -0
- package/dist/references/index.d.ts.map +1 -0
- package/dist/references/index.js +31 -0
- package/dist/references/index.js.map +1 -0
- package/dist/references/nemoclaw.d.ts +48 -0
- package/dist/references/nemoclaw.d.ts.map +1 -0
- package/dist/references/nemoclaw.js +140 -0
- package/dist/references/nemoclaw.js.map +1 -0
- package/dist/references/nist-ai-rmf.d.ts +31 -0
- package/dist/references/nist-ai-rmf.d.ts.map +1 -0
- package/dist/references/nist-ai-rmf.js +627 -0
- package/dist/references/nist-ai-rmf.js.map +1 -0
- package/dist/references/owasp-agentic-top10.d.ts +50 -0
- package/dist/references/owasp-agentic-top10.d.ts.map +1 -0
- package/dist/references/owasp-agentic-top10.js +202 -0
- package/dist/references/owasp-agentic-top10.js.map +1 -0
- package/dist/references/registry-parity.test.d.ts +17 -0
- package/dist/references/registry-parity.test.d.ts.map +1 -0
- package/dist/references/registry-parity.test.js +87 -0
- package/dist/references/registry-parity.test.js.map +1 -0
- package/dist/references/registry.d.ts +90 -0
- package/dist/references/registry.d.ts.map +1 -0
- package/dist/references/registry.js +29 -0
- package/dist/references/registry.js.map +1 -0
- package/dist/tools/model-eval.d.ts +97 -0
- package/dist/tools/model-eval.d.ts.map +1 -0
- package/dist/tools/model-eval.js +307 -0
- package/dist/tools/model-eval.js.map +1 -0
- package/package.json +105 -0
- package/schemas/threat-composer-v1.schema.json +966 -0
- package/vendor/atlas-release/ATLAS-2026.07.yaml +17877 -0
- package/vendor/atlas-release/README.md +41 -0
|
@@ -0,0 +1,97 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Model Evaluation Tool — Canopy internal research utility.
|
|
3
|
+
*
|
|
4
|
+
* Runs the same AIVSS assessment through N Ollama models in parallel and
|
|
5
|
+
* produces a structured comparison report. Used to vet which models generate
|
|
6
|
+
* meaningful threat assessments before recommending them to users.
|
|
7
|
+
*
|
|
8
|
+
* This is a developer/researcher tool, not a user-facing feature. Output
|
|
9
|
+
* informs guidance like "qwen2.5:7b is recommended; qwen3.5:2b produces
|
|
10
|
+
* shallow findings with low metric coverage."
|
|
11
|
+
*
|
|
12
|
+
* @example
|
|
13
|
+
* const report = await runModelEval({
|
|
14
|
+
* agents,
|
|
15
|
+
* models: ["llama3.2", "qwen2.5:7b", "mistral", "gemma2:9b"],
|
|
16
|
+
* baseUrl: "http://localhost:11434",
|
|
17
|
+
* });
|
|
18
|
+
* console.log(report.recommendations);
|
|
19
|
+
*/
|
|
20
|
+
import type { AgentDefinition, MetricAssessmentResponse, MetricCode } from "../aivss/types.js";
|
|
21
|
+
export type ModelVerdict = "recommended" | "conditional" | "not-recommended";
|
|
22
|
+
export interface ModelRecommendation {
|
|
23
|
+
model: string;
|
|
24
|
+
verdict: ModelVerdict;
|
|
25
|
+
/** Concise reasons for the verdict. */
|
|
26
|
+
reasons: string[];
|
|
27
|
+
/** Metrics where this model performed well (produced detailed, high-specificity scores). */
|
|
28
|
+
strengths: MetricCode[];
|
|
29
|
+
/** Metrics where this model underperformed (shallow findings, parse failures, default scores). */
|
|
30
|
+
weaknesses: MetricCode[];
|
|
31
|
+
/** Mean severity score across all sub-categories (0–4 scale). */
|
|
32
|
+
meanSeverity: number;
|
|
33
|
+
/** % of sub-categories that parsed successfully (vs. falling back to parse-error default). */
|
|
34
|
+
parseSuccessRate: number;
|
|
35
|
+
/** % of sub-categories with non-empty findings arrays. */
|
|
36
|
+
findingCoverageRate: number;
|
|
37
|
+
}
|
|
38
|
+
export interface MetricAgreement {
|
|
39
|
+
metric: MetricCode;
|
|
40
|
+
/** Mean pairwise agreement rate across all model pairs (0–1). */
|
|
41
|
+
agreementRate: number;
|
|
42
|
+
/** Mean severity by model on this metric (0–4). */
|
|
43
|
+
meanSeverityByModel: Record<string, number>;
|
|
44
|
+
/** Model that most consistently diverges from the group median, if any. */
|
|
45
|
+
outlier?: string;
|
|
46
|
+
}
|
|
47
|
+
export interface ModelEvalReport {
|
|
48
|
+
/** ISO timestamp of eval run. */
|
|
49
|
+
runDate: string;
|
|
50
|
+
models: string[];
|
|
51
|
+
agentCount: number;
|
|
52
|
+
/** Total wall time in milliseconds. */
|
|
53
|
+
durationMs: number;
|
|
54
|
+
/** Per-model raw results. */
|
|
55
|
+
perModel: Record<string, {
|
|
56
|
+
responses: MetricAssessmentResponse[];
|
|
57
|
+
durationMs: number;
|
|
58
|
+
parseSuccessRate: number;
|
|
59
|
+
findingCoverageRate: number;
|
|
60
|
+
meanSeverity: number;
|
|
61
|
+
}>;
|
|
62
|
+
/** Per-metric agreement analysis across all models. */
|
|
63
|
+
metricAgreement: MetricAgreement[];
|
|
64
|
+
/** Pairwise agreement matrix: modelA → modelB → % severity agreement (0–1). */
|
|
65
|
+
agreementMatrix: Record<string, Record<string, number>>;
|
|
66
|
+
/** Actionable recommendations per model. */
|
|
67
|
+
recommendations: ModelRecommendation[];
|
|
68
|
+
/** Plain-English summary suitable for README or documentation. */
|
|
69
|
+
summary: string;
|
|
70
|
+
}
|
|
71
|
+
export interface ModelEvalOptions {
|
|
72
|
+
agents: AgentDefinition[];
|
|
73
|
+
/** Ollama model tags to evaluate (e.g. ["llama3.2", "qwen2.5:7b", "mistral"]). */
|
|
74
|
+
models: string[];
|
|
75
|
+
/**
|
|
76
|
+
* Ollama API base URL. Applies to all models.
|
|
77
|
+
* Default: "http://localhost:11434"
|
|
78
|
+
*/
|
|
79
|
+
baseUrl?: string;
|
|
80
|
+
/** API key for cloud endpoint. */
|
|
81
|
+
apiKey?: string;
|
|
82
|
+
/** Concurrency per model. Default: 1. */
|
|
83
|
+
concurrency?: number;
|
|
84
|
+
maxTokens?: number;
|
|
85
|
+
temperature?: number;
|
|
86
|
+
/** Called when any model completes its run. */
|
|
87
|
+
onModelComplete?: (model: string, durationMs: number) => void;
|
|
88
|
+
}
|
|
89
|
+
/**
|
|
90
|
+
* Run the same AIVSS assessment through multiple Ollama models and produce a
|
|
91
|
+
* structured comparison report with per-model recommendations.
|
|
92
|
+
*
|
|
93
|
+
* Models run in parallel. Results are compared by metric, sub-category agreement
|
|
94
|
+
* rate, parse success, and finding coverage.
|
|
95
|
+
*/
|
|
96
|
+
export declare function runModelEval(options: ModelEvalOptions): Promise<ModelEvalReport>;
|
|
97
|
+
//# sourceMappingURL=model-eval.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"model-eval.d.ts","sourceRoot":"","sources":["../../src/tools/model-eval.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;GAkBG;AAIH,OAAO,KAAK,EAAE,eAAe,EAAE,wBAAwB,EAAE,UAAU,EAAiB,MAAM,mBAAmB,CAAC;AAc9G,MAAM,MAAM,YAAY,GAAG,aAAa,GAAG,aAAa,GAAG,iBAAiB,CAAC;AAE7E,MAAM,WAAW,mBAAmB;IAClC,KAAK,EAAE,MAAM,CAAC;IACd,OAAO,EAAE,YAAY,CAAC;IACtB,uCAAuC;IACvC,OAAO,EAAE,MAAM,EAAE,CAAC;IAClB,4FAA4F;IAC5F,SAAS,EAAE,UAAU,EAAE,CAAC;IACxB,kGAAkG;IAClG,UAAU,EAAE,UAAU,EAAE,CAAC;IACzB,iEAAiE;IACjE,YAAY,EAAE,MAAM,CAAC;IACrB,8FAA8F;IAC9F,gBAAgB,EAAE,MAAM,CAAC;IACzB,0DAA0D;IAC1D,mBAAmB,EAAE,MAAM,CAAC;CAC7B;AAED,MAAM,WAAW,eAAe;IAC9B,MAAM,EAAE,UAAU,CAAC;IACnB,iEAAiE;IACjE,aAAa,EAAE,MAAM,CAAC;IACtB,mDAAmD;IACnD,mBAAmB,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;IAC5C,2EAA2E;IAC3E,OAAO,CAAC,EAAE,MAAM,CAAC;CAClB;AAED,MAAM,WAAW,eAAe;IAC9B,iCAAiC;IACjC,OAAO,EAAE,MAAM,CAAC;IAChB,MAAM,EAAE,MAAM,EAAE,CAAC;IACjB,UAAU,EAAE,MAAM,CAAC;IACnB,uCAAuC;IACvC,UAAU,EAAE,MAAM,CAAC;IAEnB,6BAA6B;IAC7B,QAAQ,EAAE,MAAM,CACd,MAAM,EACN;QACE,SAAS,EAAE,wBAAwB,EAAE,CAAC;QACtC,UAAU,EAAE,MAAM,CAAC;QACnB,gBAAgB,EAAE,MAAM,CAAC;QACzB,mBAAmB,EAAE,MAAM,CAAC;QAC5B,YAAY,EAAE,MAAM,CAAC;KACtB,CACF,CAAC;IAEF,uDAAuD;IACvD,eAAe,EAAE,eAAe,EAAE,CAAC;IAEnC,+EAA+E;IAC/E,eAAe,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC,CAAC;IAExD,4CAA4C;IAC5C,eAAe,EAAE,mBAAmB,EAAE,CAAC;IAEvC,kEAAkE;IAClE,OAAO,EAAE,MAAM,CAAC;CACjB;AAID,MAAM,WAAW,gBAAgB;IAC/B,MAAM,EAAE,eAAe,EAAE,CAAC;IAC1B,kFAAkF;IAClF,MAAM,EAAE,MAAM,EAAE,CAAC;IACjB;;;OAGG;IACH,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,kCAAkC;IAClC,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,yCAAyC;IACzC,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,+CAA+C;IAC/C,eAAe,CAAC,EAAE,CAAC,KAAK,EAAE,MAAM,EAAE,UAAU,EAAE,MAAM,KAAK,IAAI,CAAC;CAC/D;AA0ND;;;;;;GAMG;AACH,wBAAsB,YAAY,CAAC,OAAO,EAAE,gBAAgB,GAAG,OAAO,CAAC,eAAe,CAAC,CAwGtF"}
|
|
@@ -0,0 +1,307 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Model Evaluation Tool — Canopy internal research utility.
|
|
3
|
+
*
|
|
4
|
+
* Runs the same AIVSS assessment through N Ollama models in parallel and
|
|
5
|
+
* produces a structured comparison report. Used to vet which models generate
|
|
6
|
+
* meaningful threat assessments before recommending them to users.
|
|
7
|
+
*
|
|
8
|
+
* This is a developer/researcher tool, not a user-facing feature. Output
|
|
9
|
+
* informs guidance like "qwen2.5:7b is recommended; qwen3.5:2b produces
|
|
10
|
+
* shallow findings with low metric coverage."
|
|
11
|
+
*
|
|
12
|
+
* @example
|
|
13
|
+
* const report = await runModelEval({
|
|
14
|
+
* agents,
|
|
15
|
+
* models: ["llama3.2", "qwen2.5:7b", "mistral", "gemma2:9b"],
|
|
16
|
+
* baseUrl: "http://localhost:11434",
|
|
17
|
+
* });
|
|
18
|
+
* console.log(report.recommendations);
|
|
19
|
+
*/
|
|
20
|
+
import { OllamaBatchProvider } from "../batch/providers/ollama.js";
|
|
21
|
+
import { METRIC_CODES } from "../aivss/types.js";
|
|
22
|
+
// ── Report types ─────────────────────────────────────────────────────────────
|
|
23
|
+
/** Normalized numeric weight for each SeverityLabel (higher = more severe). */
|
|
24
|
+
const SEVERITY_WEIGHT = {
|
|
25
|
+
None: 0,
|
|
26
|
+
Low: 1,
|
|
27
|
+
Medium: 2,
|
|
28
|
+
High: 3,
|
|
29
|
+
Critical: 4,
|
|
30
|
+
};
|
|
31
|
+
// ── Analysis helpers ─────────────────────────────────────────────────────────
|
|
32
|
+
function meanSeverity(responses) {
|
|
33
|
+
const scores = [];
|
|
34
|
+
for (const r of responses) {
|
|
35
|
+
for (const v of Object.values(r.subCategoryScores)) {
|
|
36
|
+
scores.push(SEVERITY_WEIGHT[v] ?? 0);
|
|
37
|
+
}
|
|
38
|
+
}
|
|
39
|
+
if (scores.length === 0)
|
|
40
|
+
return 0;
|
|
41
|
+
return scores.reduce((a, b) => a + b, 0) / scores.length;
|
|
42
|
+
}
|
|
43
|
+
function parseSuccessRate(responses) {
|
|
44
|
+
let total = 0;
|
|
45
|
+
let successes = 0;
|
|
46
|
+
for (const r of responses) {
|
|
47
|
+
const isParseError = r.findings.some((f) => f.startsWith("[parse-error]"));
|
|
48
|
+
const subCount = Object.keys(r.subCategoryScores).length;
|
|
49
|
+
total++;
|
|
50
|
+
if (!isParseError && subCount > 0)
|
|
51
|
+
successes++;
|
|
52
|
+
}
|
|
53
|
+
return total === 0 ? 0 : successes / total;
|
|
54
|
+
}
|
|
55
|
+
function findingCoverageRate(responses) {
|
|
56
|
+
let total = 0;
|
|
57
|
+
let withFindings = 0;
|
|
58
|
+
for (const r of responses) {
|
|
59
|
+
total++;
|
|
60
|
+
if (r.findings.length > 0 && !r.findings[0]?.startsWith("[parse-error]"))
|
|
61
|
+
withFindings++;
|
|
62
|
+
}
|
|
63
|
+
return total === 0 ? 0 : withFindings / total;
|
|
64
|
+
}
|
|
65
|
+
/**
|
|
66
|
+
* Pairwise severity agreement: for each sub-category, do both models give the
|
|
67
|
+
* same SeverityLabel? Returns the fraction that match.
|
|
68
|
+
*/
|
|
69
|
+
function pairwiseAgreement(a, b) {
|
|
70
|
+
const bIndex = new Map();
|
|
71
|
+
for (const r of b) {
|
|
72
|
+
bIndex.set(r.metricCode, r.subCategoryScores);
|
|
73
|
+
}
|
|
74
|
+
let matches = 0;
|
|
75
|
+
let total = 0;
|
|
76
|
+
for (const ra of a) {
|
|
77
|
+
const rb = bIndex.get(ra.metricCode);
|
|
78
|
+
if (!rb)
|
|
79
|
+
continue;
|
|
80
|
+
for (const [sub, scoreA] of Object.entries(ra.subCategoryScores)) {
|
|
81
|
+
const scoreB = rb[sub];
|
|
82
|
+
if (scoreB !== undefined) {
|
|
83
|
+
total++;
|
|
84
|
+
if (scoreA === scoreB)
|
|
85
|
+
matches++;
|
|
86
|
+
}
|
|
87
|
+
}
|
|
88
|
+
}
|
|
89
|
+
return total === 0 ? 0 : matches / total;
|
|
90
|
+
}
|
|
91
|
+
function metricMeanSeverity(responses, metric) {
|
|
92
|
+
const scores = [];
|
|
93
|
+
for (const r of responses) {
|
|
94
|
+
if (r.metricCode !== metric)
|
|
95
|
+
continue;
|
|
96
|
+
for (const v of Object.values(r.subCategoryScores)) {
|
|
97
|
+
scores.push(SEVERITY_WEIGHT[v] ?? 0);
|
|
98
|
+
}
|
|
99
|
+
}
|
|
100
|
+
return scores.length === 0 ? 0 : scores.reduce((a, b) => a + b, 0) / scores.length;
|
|
101
|
+
}
|
|
102
|
+
function buildRecommendation(model, responses, allModels, agreementMatrix) {
|
|
103
|
+
const psr = parseSuccessRate(responses);
|
|
104
|
+
const fcr = findingCoverageRate(responses);
|
|
105
|
+
const ms = meanSeverity(responses);
|
|
106
|
+
// Per-metric quality
|
|
107
|
+
const strengths = [];
|
|
108
|
+
const weaknesses = [];
|
|
109
|
+
for (const metric of METRIC_CODES) {
|
|
110
|
+
const metricResponses = responses.filter((r) => r.metricCode === metric);
|
|
111
|
+
const hasParseError = metricResponses.some((r) => r.findings.some((f) => f.startsWith("[parse-error]")));
|
|
112
|
+
const subCount = metricResponses.reduce((n, r) => n + Object.keys(r.subCategoryScores).length, 0);
|
|
113
|
+
if (!hasParseError && subCount > 0 && metricResponses.some((r) => r.findings.length > 0)) {
|
|
114
|
+
strengths.push(metric);
|
|
115
|
+
}
|
|
116
|
+
else if (hasParseError || subCount === 0) {
|
|
117
|
+
weaknesses.push(metric);
|
|
118
|
+
}
|
|
119
|
+
}
|
|
120
|
+
// Mean pairwise agreement with other models
|
|
121
|
+
const agreements = allModels
|
|
122
|
+
.filter((m) => m !== model && agreementMatrix[model]?.[m] !== undefined)
|
|
123
|
+
.map((m) => agreementMatrix[model][m]);
|
|
124
|
+
const meanAgreement = agreements.length
|
|
125
|
+
? agreements.reduce((a, b) => a + b, 0) / agreements.length
|
|
126
|
+
: 1;
|
|
127
|
+
const reasons = [];
|
|
128
|
+
let verdict;
|
|
129
|
+
if (psr < 0.5) {
|
|
130
|
+
reasons.push(`High parse failure rate (${(psr * 100).toFixed(0)}% success) — model output is not structured JSON`);
|
|
131
|
+
}
|
|
132
|
+
if (fcr < 0.5) {
|
|
133
|
+
reasons.push(`Low finding coverage (${(fcr * 100).toFixed(0)}%) — model produces few actionable findings`);
|
|
134
|
+
}
|
|
135
|
+
if (ms < 0.5) {
|
|
136
|
+
reasons.push(`Very low mean severity (${ms.toFixed(2)}/4.0) — model may be systematically under-scoring threats`);
|
|
137
|
+
}
|
|
138
|
+
if (ms > 3.5) {
|
|
139
|
+
reasons.push(`Very high mean severity (${ms.toFixed(2)}/4.0) — model may be over-flagging; review for false positives`);
|
|
140
|
+
}
|
|
141
|
+
if (meanAgreement < 0.4) {
|
|
142
|
+
reasons.push(`Low agreement with peer models (${(meanAgreement * 100).toFixed(0)}%) — divergent scoring warrants manual validation`);
|
|
143
|
+
}
|
|
144
|
+
if (weaknesses.length >= 4) {
|
|
145
|
+
reasons.push(`Parse failures on ${weaknesses.length}/9 metrics: ${weaknesses.join(", ")}`);
|
|
146
|
+
}
|
|
147
|
+
if (psr >= 0.85 && fcr >= 0.7 && ms >= 1.0 && ms <= 3.5 && weaknesses.length <= 2) {
|
|
148
|
+
verdict = "recommended";
|
|
149
|
+
if (reasons.length === 0) {
|
|
150
|
+
reasons.push(`Strong metric coverage (${strengths.length}/9), consistent scoring, good finding quality`);
|
|
151
|
+
}
|
|
152
|
+
}
|
|
153
|
+
else if (psr >= 0.6 && fcr >= 0.4 && weaknesses.length <= 4) {
|
|
154
|
+
verdict = "conditional";
|
|
155
|
+
if (reasons.length === 0) {
|
|
156
|
+
reasons.push("Acceptable performance with caveats — validate on your specific agent topology");
|
|
157
|
+
}
|
|
158
|
+
}
|
|
159
|
+
else {
|
|
160
|
+
verdict = "not-recommended";
|
|
161
|
+
if (reasons.length === 0) {
|
|
162
|
+
reasons.push("Insufficient metric coverage or output quality for reliable threat assessment");
|
|
163
|
+
}
|
|
164
|
+
}
|
|
165
|
+
return { model, verdict, reasons, strengths, weaknesses, meanSeverity: ms, parseSuccessRate: psr, findingCoverageRate: fcr };
|
|
166
|
+
}
|
|
167
|
+
function buildSummary(report) {
|
|
168
|
+
const lines = [
|
|
169
|
+
`## Canopy Model Evaluation — ${report.runDate.slice(0, 10)}`,
|
|
170
|
+
"",
|
|
171
|
+
`Evaluated ${report.models.length} models across ${report.agentCount} agent(s) in ${(report.durationMs / 1000).toFixed(1)}s.`,
|
|
172
|
+
"",
|
|
173
|
+
"### Verdicts",
|
|
174
|
+
"",
|
|
175
|
+
];
|
|
176
|
+
for (const rec of report.recommendations) {
|
|
177
|
+
const icon = rec.verdict === "recommended" ? "✅" : rec.verdict === "conditional" ? "⚠️" : "❌";
|
|
178
|
+
lines.push(`**${icon} ${rec.model}** (${rec.verdict})`);
|
|
179
|
+
for (const r of rec.reasons)
|
|
180
|
+
lines.push(` - ${r}`);
|
|
181
|
+
if (rec.strengths.length > 0)
|
|
182
|
+
lines.push(` - Strong on: ${rec.strengths.join(", ")}`);
|
|
183
|
+
if (rec.weaknesses.length > 0)
|
|
184
|
+
lines.push(` - Weak on: ${rec.weaknesses.join(", ")}`);
|
|
185
|
+
lines.push("");
|
|
186
|
+
}
|
|
187
|
+
// Highest agreement pair
|
|
188
|
+
const pairs = [];
|
|
189
|
+
const models = report.models;
|
|
190
|
+
for (let i = 0; i < models.length; i++) {
|
|
191
|
+
for (let j = i + 1; j < models.length; j++) {
|
|
192
|
+
const a = models[i];
|
|
193
|
+
const b = models[j];
|
|
194
|
+
const agreement = report.agreementMatrix[a]?.[b] ?? 0;
|
|
195
|
+
pairs.push({ a, b, agreement });
|
|
196
|
+
}
|
|
197
|
+
}
|
|
198
|
+
pairs.sort((x, y) => y.agreement - x.agreement);
|
|
199
|
+
if (pairs.length > 0) {
|
|
200
|
+
lines.push("### Cross-Model Agreement");
|
|
201
|
+
lines.push("");
|
|
202
|
+
for (const { a, b, agreement } of pairs.slice(0, 3)) {
|
|
203
|
+
lines.push(`- **${a}** ↔ **${b}**: ${(agreement * 100).toFixed(0)}% severity agreement`);
|
|
204
|
+
}
|
|
205
|
+
lines.push("");
|
|
206
|
+
}
|
|
207
|
+
// Most contested metric
|
|
208
|
+
const contested = [...report.metricAgreement].sort((a, b) => a.agreementRate - b.agreementRate)[0];
|
|
209
|
+
if (contested && contested.agreementRate < 0.7) {
|
|
210
|
+
lines.push(`> ⚠️ **${contested.metric}** has the lowest cross-model agreement (${(contested.agreementRate * 100).toFixed(0)}%) — findings on this metric warrant human review regardless of model.`);
|
|
211
|
+
lines.push("");
|
|
212
|
+
}
|
|
213
|
+
return lines.join("\n");
|
|
214
|
+
}
|
|
215
|
+
// ── Public API ────────────────────────────────────────────────────────────────
|
|
216
|
+
/**
|
|
217
|
+
* Run the same AIVSS assessment through multiple Ollama models and produce a
|
|
218
|
+
* structured comparison report with per-model recommendations.
|
|
219
|
+
*
|
|
220
|
+
* Models run in parallel. Results are compared by metric, sub-category agreement
|
|
221
|
+
* rate, parse success, and finding coverage.
|
|
222
|
+
*/
|
|
223
|
+
export async function runModelEval(options) {
|
|
224
|
+
const { agents, models, baseUrl, apiKey, concurrency, maxTokens, temperature, onModelComplete } = options;
|
|
225
|
+
if (models.length === 0)
|
|
226
|
+
throw new Error("runModelEval: models array is empty");
|
|
227
|
+
if (agents.length === 0)
|
|
228
|
+
throw new Error("runModelEval: agents array is empty");
|
|
229
|
+
const wallStart = Date.now();
|
|
230
|
+
// Run all models in parallel
|
|
231
|
+
const modelResults = await Promise.all(models.map(async (model) => {
|
|
232
|
+
const provider = new OllamaBatchProvider({
|
|
233
|
+
model,
|
|
234
|
+
baseUrl,
|
|
235
|
+
apiKey,
|
|
236
|
+
concurrency: concurrency ?? 1,
|
|
237
|
+
maxTokens,
|
|
238
|
+
});
|
|
239
|
+
const start = Date.now();
|
|
240
|
+
const responses = await provider.runBatch(agents, { temperature });
|
|
241
|
+
const dur = Date.now() - start;
|
|
242
|
+
onModelComplete?.(model, dur);
|
|
243
|
+
return { model, responses, durationMs: dur };
|
|
244
|
+
}));
|
|
245
|
+
const perModel = {};
|
|
246
|
+
for (const { model, responses, durationMs } of modelResults) {
|
|
247
|
+
perModel[model] = {
|
|
248
|
+
responses,
|
|
249
|
+
durationMs,
|
|
250
|
+
parseSuccessRate: parseSuccessRate(responses),
|
|
251
|
+
findingCoverageRate: findingCoverageRate(responses),
|
|
252
|
+
meanSeverity: meanSeverity(responses),
|
|
253
|
+
};
|
|
254
|
+
}
|
|
255
|
+
// Agreement matrix
|
|
256
|
+
const agreementMatrix = {};
|
|
257
|
+
for (const { model: a, responses: ra } of modelResults) {
|
|
258
|
+
agreementMatrix[a] = {};
|
|
259
|
+
for (const { model: b, responses: rb } of modelResults) {
|
|
260
|
+
if (a !== b) {
|
|
261
|
+
agreementMatrix[a][b] = pairwiseAgreement(ra, rb);
|
|
262
|
+
}
|
|
263
|
+
}
|
|
264
|
+
}
|
|
265
|
+
// Per-metric agreement
|
|
266
|
+
const metricAgreement = METRIC_CODES.map((metric) => {
|
|
267
|
+
const meanByModel = {};
|
|
268
|
+
for (const { model, responses } of modelResults) {
|
|
269
|
+
meanByModel[model] = metricMeanSeverity(responses, metric);
|
|
270
|
+
}
|
|
271
|
+
// Agreement: pairs that give the same score on this metric's sub-categories
|
|
272
|
+
const pairAgreements = [];
|
|
273
|
+
for (let i = 0; i < modelResults.length; i++) {
|
|
274
|
+
for (let j = i + 1; j < modelResults.length; j++) {
|
|
275
|
+
const ra = modelResults[i].responses.filter((r) => r.metricCode === metric);
|
|
276
|
+
const rb = modelResults[j].responses.filter((r) => r.metricCode === metric);
|
|
277
|
+
pairAgreements.push(pairwiseAgreement(ra, rb));
|
|
278
|
+
}
|
|
279
|
+
}
|
|
280
|
+
const agreementRate = pairAgreements.length === 0
|
|
281
|
+
? 1
|
|
282
|
+
: pairAgreements.reduce((a, b) => a + b, 0) / pairAgreements.length;
|
|
283
|
+
// Outlier: model whose mean differs most from the group median
|
|
284
|
+
const values = Object.values(meanByModel);
|
|
285
|
+
const median = [...values].sort((a, b) => a - b)[Math.floor(values.length / 2)] ?? 0;
|
|
286
|
+
const outlierEntry = Object.entries(meanByModel).reduce((best, [m, v]) => (Math.abs(v - median) > Math.abs(best[1] - median) ? [m, v] : best), ["", 0]);
|
|
287
|
+
const outlier = Math.abs(outlierEntry[1] - median) > 0.75 ? outlierEntry[0] : undefined;
|
|
288
|
+
return { metric, agreementRate, meanSeverityByModel: meanByModel, outlier };
|
|
289
|
+
});
|
|
290
|
+
// Recommendations
|
|
291
|
+
const recommendations = modelResults.map(({ model, responses }) => buildRecommendation(model, responses, models, agreementMatrix));
|
|
292
|
+
const partialReport = {
|
|
293
|
+
runDate: new Date().toISOString(),
|
|
294
|
+
models,
|
|
295
|
+
agentCount: agents.length,
|
|
296
|
+
durationMs: Date.now() - wallStart,
|
|
297
|
+
perModel,
|
|
298
|
+
metricAgreement,
|
|
299
|
+
agreementMatrix,
|
|
300
|
+
recommendations,
|
|
301
|
+
};
|
|
302
|
+
return {
|
|
303
|
+
...partialReport,
|
|
304
|
+
summary: buildSummary(partialReport),
|
|
305
|
+
};
|
|
306
|
+
}
|
|
307
|
+
//# sourceMappingURL=model-eval.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"model-eval.js","sourceRoot":"","sources":["../../src/tools/model-eval.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;GAkBG;AAEH,OAAO,EAAE,mBAAmB,EAAE,MAAM,8BAA8B,CAAC;AAGnE,OAAO,EAAE,YAAY,EAAE,MAAM,mBAAmB,CAAC;AAEjD,gFAAgF;AAEhF,+EAA+E;AAC/E,MAAM,eAAe,GAAkC;IACrD,IAAI,EAAE,CAAC;IACP,GAAG,EAAE,CAAC;IACN,MAAM,EAAE,CAAC;IACT,IAAI,EAAE,CAAC;IACP,QAAQ,EAAE,CAAC;CACZ,CAAC;AAqFF,gFAAgF;AAEhF,SAAS,YAAY,CAAC,SAAqC;IACzD,MAAM,MAAM,GAAa,EAAE,CAAC;IAC5B,KAAK,MAAM,CAAC,IAAI,SAAS,EAAE,CAAC;QAC1B,KAAK,MAAM,CAAC,IAAI,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC,iBAAiB,CAAC,EAAE,CAAC;YACnD,MAAM,CAAC,IAAI,CAAC,eAAe,CAAC,CAAkB,CAAC,IAAI,CAAC,CAAC,CAAC;QACxD,CAAC;IACH,CAAC;IACD,IAAI,MAAM,CAAC,MAAM,KAAK,CAAC;QAAE,OAAO,CAAC,CAAC;IAClC,OAAO,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC,GAAG,MAAM,CAAC,MAAM,CAAC;AAC3D,CAAC;AAED,SAAS,gBAAgB,CAAC,SAAqC;IAC7D,IAAI,KAAK,GAAG,CAAC,CAAC;IACd,IAAI,SAAS,GAAG,CAAC,CAAC;IAClB,KAAK,MAAM,CAAC,IAAI,SAAS,EAAE,CAAC;QAC1B,MAAM,YAAY,GAAG,CAAC,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,UAAU,CAAC,eAAe,CAAC,CAAC,CAAC;QAC3E,MAAM,QAAQ,GAAG,MAAM,CAAC,IAAI,CAAC,CAAC,CAAC,iBAAiB,CAAC,CAAC,MAAM,CAAC;QACzD,KAAK,EAAE,CAAC;QACR,IAAI,CAAC,YAAY,IAAI,QAAQ,GAAG,CAAC;YAAE,SAAS,EAAE,CAAC;IACjD,CAAC;IACD,OAAO,KAAK,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,SAAS,GAAG,KAAK,CAAC;AAC7C,CAAC;AAED,SAAS,mBAAmB,CAAC,SAAqC;IAChE,IAAI,KAAK,GAAG,CAAC,CAAC;IACd,IAAI,YAAY,GAAG,CAAC,CAAC;IACrB,KAAK,MAAM,CAAC,IAAI,SAAS,EAAE,CAAC;QAC1B,KAAK,EAAE,CAAC;QACR,IAAI,CAAC,CAAC,QAAQ,CAAC,MAAM,GAAG,CAAC,IAAI,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC,EAAE,UAAU,CAAC,eAAe,CAAC;YAAE,YAAY,EAAE,CAAC;IAC3F,CAAC;IACD,OAAO,KAAK,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,YAAY,GAAG,KAAK,CAAC;AAChD,CAAC;AAED;;;GAGG;AACH,SAAS,iBAAiB,CACxB,CAA6B,EAC7B,CAA6B;IAE7B,MAAM,MAAM,GAAG,IAAI,GAAG,EAAyC,CAAC;IAChE,KAAK,MAAM,CAAC,IAAI,CAAC,EAAE,CAAC;QAClB,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,UAAU,EAAE,CAAC,CAAC,iBAAiB,CAAC,CAAC;IAChD,CAAC;IAED,IAAI,OAAO,GAAG,CAAC,CAAC;IAChB,IAAI,KAAK,GAAG,CAAC,CAAC;IAEd,KAAK,MAAM,EAAE,IAAI,CAAC,EAAE,CAAC;QACnB,MAAM,EAAE,GAAG,MAAM,CAAC,GAAG,CAAC,EAAE,CAAC,UAAU,CAAC,CAAC;QACrC,IAAI,CAAC,EAAE;YAAE,SAAS;QAClB,KAAK,MAAM,CAAC,GAAG,EAAE,MAAM,CAAC,IAAI,MAAM,CAAC,OAAO,CAAC,EAAE,CAAC,iBAAiB,CAAC,EAAE,CAAC;YACjE,MAAM,MAAM,GAAG,EAAE,CAAC,GAAG,CAAC,CAAC;YACvB,IAAI,MAAM,KAAK,SAAS,EAAE,CAAC;gBACzB,KAAK,EAAE,CAAC;gBACR,IAAI,MAAM,KAAK,MAAM;oBAAE,OAAO,EAAE,CAAC;YACnC,CAAC;QACH,CAAC;IACH,CAAC;IAED,OAAO,KAAK,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,OAAO,GAAG,KAAK,CAAC;AAC3C,CAAC;AAED,SAAS,kBAAkB,CACzB,SAAqC,EACrC,MAAkB;IAElB,MAAM,MAAM,GAAa,EAAE,CAAC;IAC5B,KAAK,MAAM,CAAC,IAAI,SAAS,EAAE,CAAC;QAC1B,IAAI,CAAC,CAAC,UAAU,KAAK,MAAM;YAAE,SAAS;QACtC,KAAK,MAAM,CAAC,IAAI,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC,iBAAiB,CAAC,EAAE,CAAC;YACnD,MAAM,CAAC,IAAI,CAAC,eAAe,CAAC,CAAkB,CAAC,IAAI,CAAC,CAAC,CAAC;QACxD,CAAC;IACH,CAAC;IACD,OAAO,MAAM,CAAC,MAAM,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC,GAAG,MAAM,CAAC,MAAM,CAAC;AACrF,CAAC;AAED,SAAS,mBAAmB,CAC1B,KAAa,EACb,SAAqC,EACrC,SAAmB,EACnB,eAAuD;IAEvD,MAAM,GAAG,GAAG,gBAAgB,CAAC,SAAS,CAAC,CAAC;IACxC,MAAM,GAAG,GAAG,mBAAmB,CAAC,SAAS,CAAC,CAAC;IAC3C,MAAM,EAAE,GAAG,YAAY,CAAC,SAAS,CAAC,CAAC;IAEnC,qBAAqB;IACrB,MAAM,SAAS,GAAiB,EAAE,CAAC;IACnC,MAAM,UAAU,GAAiB,EAAE,CAAC;IACpC,KAAK,MAAM,MAAM,IAAI,YAAY,EAAE,CAAC;QAClC,MAAM,eAAe,GAAG,SAAS,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,UAAU,KAAK,MAAM,CAAC,CAAC;QACzE,MAAM,aAAa,GAAG,eAAe,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAC/C,CAAC,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,UAAU,CAAC,eAAe,CAAC,CAAC,CACtD,CAAC;QACF,MAAM,QAAQ,GAAG,eAAe,CAAC,MAAM,CACrC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,GAAG,MAAM,CAAC,IAAI,CAAC,CAAC,CAAC,iBAAiB,CAAC,CAAC,MAAM,EACrD,CAAC,CACF,CAAC;QACF,IAAI,CAAC,aAAa,IAAI,QAAQ,GAAG,CAAC,IAAI,eAAe,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,QAAQ,CAAC,MAAM,GAAG,CAAC,CAAC,EAAE,CAAC;YACzF,SAAS,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;QACzB,CAAC;aAAM,IAAI,aAAa,IAAI,QAAQ,KAAK,CAAC,EAAE,CAAC;YAC3C,UAAU,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;QAC1B,CAAC;IACH,CAAC;IAED,4CAA4C;IAC5C,MAAM,UAAU,GAAG,SAAS;SACzB,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,KAAK,KAAK,IAAI,eAAe,CAAC,KAAK,CAAC,EAAE,CAAC,CAAC,CAAC,KAAK,SAAS,CAAC;SACvE,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,eAAe,CAAC,KAAK,CAAE,CAAC,CAAC,CAAE,CAAC,CAAC;IAC3C,MAAM,aAAa,GAAG,UAAU,CAAC,MAAM;QACrC,CAAC,CAAC,UAAU,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC,GAAG,UAAU,CAAC,MAAM;QAC3D,CAAC,CAAC,CAAC,CAAC;IAEN,MAAM,OAAO,GAAa,EAAE,CAAC;IAC7B,IAAI,OAAqB,CAAC;IAE1B,IAAI,GAAG,GAAG,GAAG,EAAE,CAAC;QACd,OAAO,CAAC,IAAI,CAAC,4BAA4B,CAAC,GAAG,GAAG,GAAG,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,kDAAkD,CAAC,CAAC;IACrH,CAAC;IACD,IAAI,GAAG,GAAG,GAAG,EAAE,CAAC;QACd,OAAO,CAAC,IAAI,CAAC,yBAAyB,CAAC,GAAG,GAAG,GAAG,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,6CAA6C,CAAC,CAAC;IAC7G,CAAC;IACD,IAAI,EAAE,GAAG,GAAG,EAAE,CAAC;QACb,OAAO,CAAC,IAAI,CAAC,2BAA2B,EAAE,CAAC,OAAO,CAAC,CAAC,CAAC,2DAA2D,CAAC,CAAC;IACpH,CAAC;IACD,IAAI,EAAE,GAAG,GAAG,EAAE,CAAC;QACb,OAAO,CAAC,IAAI,CAAC,4BAA4B,EAAE,CAAC,OAAO,CAAC,CAAC,CAAC,gEAAgE,CAAC,CAAC;IAC1H,CAAC;IACD,IAAI,aAAa,GAAG,GAAG,EAAE,CAAC;QACxB,OAAO,CAAC,IAAI,CAAC,mCAAmC,CAAC,aAAa,GAAG,GAAG,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,mDAAmD,CAAC,CAAC;IACvI,CAAC;IACD,IAAI,UAAU,CAAC,MAAM,IAAI,CAAC,EAAE,CAAC;QAC3B,OAAO,CAAC,IAAI,CAAC,qBAAqB,UAAU,CAAC,MAAM,eAAe,UAAU,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;IAC7F,CAAC;IAED,IAAI,GAAG,IAAI,IAAI,IAAI,GAAG,IAAI,GAAG,IAAI,EAAE,IAAI,GAAG,IAAI,EAAE,IAAI,GAAG,IAAI,UAAU,CAAC,MAAM,IAAI,CAAC,EAAE,CAAC;QAClF,OAAO,GAAG,aAAa,CAAC;QACxB,IAAI,OAAO,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YACzB,OAAO,CAAC,IAAI,CAAC,2BAA2B,SAAS,CAAC,MAAM,+CAA+C,CAAC,CAAC;QAC3G,CAAC;IACH,CAAC;SAAM,IAAI,GAAG,IAAI,GAAG,IAAI,GAAG,IAAI,GAAG,IAAI,UAAU,CAAC,MAAM,IAAI,CAAC,EAAE,CAAC;QAC9D,OAAO,GAAG,aAAa,CAAC;QACxB,IAAI,OAAO,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YACzB,OAAO,CAAC,IAAI,CAAC,gFAAgF,CAAC,CAAC;QACjG,CAAC;IACH,CAAC;SAAM,CAAC;QACN,OAAO,GAAG,iBAAiB,CAAC;QAC5B,IAAI,OAAO,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YACzB,OAAO,CAAC,IAAI,CAAC,+EAA+E,CAAC,CAAC;QAChG,CAAC;IACH,CAAC;IAED,OAAO,EAAE,KAAK,EAAE,OAAO,EAAE,OAAO,EAAE,SAAS,EAAE,UAAU,EAAE,YAAY,EAAE,EAAE,EAAE,gBAAgB,EAAE,GAAG,EAAE,mBAAmB,EAAE,GAAG,EAAE,CAAC;AAC/H,CAAC;AAED,SAAS,YAAY,CAAC,MAAwC;IAC5D,MAAM,KAAK,GAAa;QACtB,gCAAgC,MAAM,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC,EAAE,EAAE,CAAC,EAAE;QAC7D,EAAE;QACF,aAAa,MAAM,CAAC,MAAM,CAAC,MAAM,kBAAkB,MAAM,CAAC,UAAU,gBAAgB,CAAC,MAAM,CAAC,UAAU,GAAG,IAAI,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,IAAI;QAC7H,EAAE;QACF,cAAc;QACd,EAAE;KACH,CAAC;IAEF,KAAK,MAAM,GAAG,IAAI,MAAM,CAAC,eAAe,EAAE,CAAC;QACzC,MAAM,IAAI,GAAG,GAAG,CAAC,OAAO,KAAK,aAAa,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,GAAG,CAAC,OAAO,KAAK,aAAa,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,GAAG,CAAC;QAC9F,KAAK,CAAC,IAAI,CAAC,KAAK,IAAI,IAAI,GAAG,CAAC,KAAK,OAAO,GAAG,CAAC,OAAO,GAAG,CAAC,CAAC;QACxD,KAAK,MAAM,CAAC,IAAI,GAAG,CAAC,OAAO;YAAE,KAAK,CAAC,IAAI,CAAC,OAAO,CAAC,EAAE,CAAC,CAAC;QACpD,IAAI,GAAG,CAAC,SAAS,CAAC,MAAM,GAAG,CAAC;YAC1B,KAAK,CAAC,IAAI,CAAC,kBAAkB,GAAG,CAAC,SAAS,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;QAC3D,IAAI,GAAG,CAAC,UAAU,CAAC,MAAM,GAAG,CAAC;YAC3B,KAAK,CAAC,IAAI,CAAC,gBAAgB,GAAG,CAAC,UAAU,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;QAC1D,KAAK,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;IACjB,CAAC;IAED,yBAAyB;IACzB,MAAM,KAAK,GAAuD,EAAE,CAAC;IACrE,MAAM,MAAM,GAAG,MAAM,CAAC,MAAM,CAAC;IAC7B,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,MAAM,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;QACvC,KAAK,IAAI,CAAC,GAAG,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,MAAM,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;YAC3C,MAAM,CAAC,GAAG,MAAM,CAAC,CAAC,CAAE,CAAC;YACrB,MAAM,CAAC,GAAG,MAAM,CAAC,CAAC,CAAE,CAAC;YACrB,MAAM,SAAS,GAAG,MAAM,CAAC,eAAe,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC;YACtD,KAAK,CAAC,IAAI,CAAC,EAAE,CAAC,EAAE,CAAC,EAAE,SAAS,EAAE,CAAC,CAAC;QAClC,CAAC;IACH,CAAC;IACD,KAAK,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,SAAS,GAAG,CAAC,CAAC,SAAS,CAAC,CAAC;IAEhD,IAAI,KAAK,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QACrB,KAAK,CAAC,IAAI,CAAC,2BAA2B,CAAC,CAAC;QACxC,KAAK,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;QACf,KAAK,MAAM,EAAE,CAAC,EAAE,CAAC,EAAE,SAAS,EAAE,IAAI,KAAK,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC,EAAE,CAAC;YACpD,KAAK,CAAC,IAAI,CAAC,OAAO,CAAC,UAAU,CAAC,OAAO,CAAC,SAAS,GAAG,GAAG,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,sBAAsB,CAAC,CAAC;QAC3F,CAAC;QACD,KAAK,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;IACjB,CAAC;IAED,wBAAwB;IACxB,MAAM,SAAS,GAAG,CAAC,GAAG,MAAM,CAAC,eAAe,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,aAAa,GAAG,CAAC,CAAC,aAAa,CAAC,CAAC,CAAC,CAAC,CAAC;IACnG,IAAI,SAAS,IAAI,SAAS,CAAC,aAAa,GAAG,GAAG,EAAE,CAAC;QAC/C,KAAK,CAAC,IAAI,CACR,UAAU,SAAS,CAAC,MAAM,4CAA4C,CAAC,SAAS,CAAC,aAAa,GAAG,GAAG,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,wEAAwE,CACzL,CAAC;QACF,KAAK,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;IACjB,CAAC;IAED,OAAO,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;AAC1B,CAAC;AAED,iFAAiF;AAEjF;;;;;;GAMG;AACH,MAAM,CAAC,KAAK,UAAU,YAAY,CAAC,OAAyB;IAC1D,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,OAAO,EAAE,MAAM,EAAE,WAAW,EAAE,SAAS,EAAE,WAAW,EAAE,eAAe,EAAE,GAAG,OAAO,CAAC;IAE1G,IAAI,MAAM,CAAC,MAAM,KAAK,CAAC;QAAE,MAAM,IAAI,KAAK,CAAC,qCAAqC,CAAC,CAAC;IAChF,IAAI,MAAM,CAAC,MAAM,KAAK,CAAC;QAAE,MAAM,IAAI,KAAK,CAAC,qCAAqC,CAAC,CAAC;IAEhF,MAAM,SAAS,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;IAE7B,6BAA6B;IAC7B,MAAM,YAAY,GAAG,MAAM,OAAO,CAAC,GAAG,CACpC,MAAM,CAAC,GAAG,CAAC,KAAK,EAAE,KAAK,EAAE,EAAE;QACzB,MAAM,QAAQ,GAAG,IAAI,mBAAmB,CAAC;YACvC,KAAK;YACL,OAAO;YACP,MAAM;YACN,WAAW,EAAE,WAAW,IAAI,CAAC;YAC7B,SAAS;SACkB,CAAC,CAAC;QAE/B,MAAM,KAAK,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;QACzB,MAAM,SAAS,GAAG,MAAM,QAAQ,CAAC,QAAQ,CAAC,MAAM,EAAE,EAAE,WAAW,EAAE,CAAC,CAAC;QACnE,MAAM,GAAG,GAAG,IAAI,CAAC,GAAG,EAAE,GAAG,KAAK,CAAC;QAE/B,eAAe,EAAE,CAAC,KAAK,EAAE,GAAG,CAAC,CAAC;QAC9B,OAAO,EAAE,KAAK,EAAE,SAAS,EAAE,UAAU,EAAE,GAAG,EAAE,CAAC;IAC/C,CAAC,CAAC,CACH,CAAC;IAEF,MAAM,QAAQ,GAAgC,EAAE,CAAC;IACjD,KAAK,MAAM,EAAE,KAAK,EAAE,SAAS,EAAE,UAAU,EAAE,IAAI,YAAY,EAAE,CAAC;QAC5D,QAAQ,CAAC,KAAK,CAAC,GAAG;YAChB,SAAS;YACT,UAAU;YACV,gBAAgB,EAAE,gBAAgB,CAAC,SAAS,CAAC;YAC7C,mBAAmB,EAAE,mBAAmB,CAAC,SAAS,CAAC;YACnD,YAAY,EAAE,YAAY,CAAC,SAAS,CAAC;SACtC,CAAC;IACJ,CAAC;IAED,mBAAmB;IACnB,MAAM,eAAe,GAA2C,EAAE,CAAC;IACnE,KAAK,MAAM,EAAE,KAAK,EAAE,CAAC,EAAE,SAAS,EAAE,EAAE,EAAE,IAAI,YAAY,EAAE,CAAC;QACvD,eAAe,CAAC,CAAC,CAAC,GAAG,EAAE,CAAC;QACxB,KAAK,MAAM,EAAE,KAAK,EAAE,CAAC,EAAE,SAAS,EAAE,EAAE,EAAE,IAAI,YAAY,EAAE,CAAC;YACvD,IAAI,CAAC,KAAK,CAAC,EAAE,CAAC;gBACZ,eAAe,CAAC,CAAC,CAAE,CAAC,CAAC,CAAC,GAAG,iBAAiB,CAAC,EAAE,EAAE,EAAE,CAAC,CAAC;YACrD,CAAC;QACH,CAAC;IACH,CAAC;IAED,uBAAuB;IACvB,MAAM,eAAe,GAAsB,YAAY,CAAC,GAAG,CAAC,CAAC,MAAM,EAAE,EAAE;QACrE,MAAM,WAAW,GAA2B,EAAE,CAAC;QAC/C,KAAK,MAAM,EAAE,KAAK,EAAE,SAAS,EAAE,IAAI,YAAY,EAAE,CAAC;YAChD,WAAW,CAAC,KAAK,CAAC,GAAG,kBAAkB,CAAC,SAAS,EAAE,MAAM,CAAC,CAAC;QAC7D,CAAC;QAED,4EAA4E;QAC5E,MAAM,cAAc,GAAa,EAAE,CAAC;QACpC,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,YAAY,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;YAC7C,KAAK,IAAI,CAAC,GAAG,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,YAAY,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;gBACjD,MAAM,EAAE,GAAG,YAAY,CAAC,CAAC,CAAE,CAAC,SAAS,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,UAAU,KAAK,MAAM,CAAC,CAAC;gBAC7E,MAAM,EAAE,GAAG,YAAY,CAAC,CAAC,CAAE,CAAC,SAAS,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,UAAU,KAAK,MAAM,CAAC,CAAC;gBAC7E,cAAc,CAAC,IAAI,CAAC,iBAAiB,CAAC,EAAE,EAAE,EAAE,CAAC,CAAC,CAAC;YACjD,CAAC;QACH,CAAC;QACD,MAAM,aAAa,GACjB,cAAc,CAAC,MAAM,KAAK,CAAC;YACzB,CAAC,CAAC,CAAC;YACH,CAAC,CAAC,cAAc,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC,GAAG,cAAc,CAAC,MAAM,CAAC;QAExE,+DAA+D;QAC/D,MAAM,MAAM,GAAG,MAAM,CAAC,MAAM,CAAC,WAAW,CAAC,CAAC;QAC1C,MAAM,MAAM,GAAG,CAAC,GAAG,MAAM,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,MAAM,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC;QACrF,MAAM,YAAY,GAAG,MAAM,CAAC,OAAO,CAAC,WAAW,CAAC,CAAC,MAAM,CACrD,CAAC,IAAI,EAAE,CAAC,CAAC,EAAE,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,GAAG,MAAM,CAAC,GAAG,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC,CAAC,GAAG,MAAM,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,EACrF,CAAC,EAAE,EAAE,CAAC,CAAqB,CAC5B,CAAC;QACF,MAAM,OAAO,GACX,IAAI,CAAC,GAAG,CAAC,YAAY,CAAC,CAAC,CAAC,GAAG,MAAM,CAAC,GAAG,IAAI,CAAC,CAAC,CAAC,YAAY,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,SAAS,CAAC;QAE1E,OAAO,EAAE,MAAM,EAAE,aAAa,EAAE,mBAAmB,EAAE,WAAW,EAAE,OAAO,EAAE,CAAC;IAC9E,CAAC,CAAC,CAAC;IAEH,kBAAkB;IAClB,MAAM,eAAe,GAAG,YAAY,CAAC,GAAG,CAAC,CAAC,EAAE,KAAK,EAAE,SAAS,EAAE,EAAE,EAAE,CAChE,mBAAmB,CAAC,KAAK,EAAE,SAAS,EAAE,MAAM,EAAE,eAAe,CAAC,CAC/D,CAAC;IAEF,MAAM,aAAa,GAAG;QACpB,OAAO,EAAE,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE;QACjC,MAAM;QACN,UAAU,EAAE,MAAM,CAAC,MAAM;QACzB,UAAU,EAAE,IAAI,CAAC,GAAG,EAAE,GAAG,SAAS;QAClC,QAAQ;QACR,eAAe;QACf,eAAe;QACf,eAAe;KAChB,CAAC;IAEF,OAAO;QACL,GAAG,aAAa;QAChB,OAAO,EAAE,YAAY,CAAC,aAAa,CAAC;KACrC,CAAC;AACJ,CAAC"}
|
package/package.json
ADDED
|
@@ -0,0 +1,105 @@
|
|
|
1
|
+
{
|
|
2
|
+
"name": "@waratahlabs/canopy",
|
|
3
|
+
"version": "0.5.0",
|
|
4
|
+
"description": "Multi-framework threat modelling for agentic AI systems \u2014 OWASP AIVSS V4 scoring and MITRE ATLAS technique mapping, with findings annotated against six risk frameworks",
|
|
5
|
+
"canopy": {
|
|
6
|
+
"comment": "Framework adapters registered by this build. Asserted against FRAMEWORK_REGISTRY by src/references/registry-parity.test.ts \u2014 a framework listed here but not registered (or vice versa) fails the suite. Update both together.",
|
|
7
|
+
"frameworks": [
|
|
8
|
+
"acsc",
|
|
9
|
+
"aicm",
|
|
10
|
+
"dstg-meaid",
|
|
11
|
+
"nemoclaw",
|
|
12
|
+
"nist-ai-rmf",
|
|
13
|
+
"owasp-agentic-top10"
|
|
14
|
+
]
|
|
15
|
+
},
|
|
16
|
+
"type": "module",
|
|
17
|
+
"main": "dist/index.js",
|
|
18
|
+
"types": "dist/index.d.ts",
|
|
19
|
+
"bin": {
|
|
20
|
+
"canopy": "dist/cli.js"
|
|
21
|
+
},
|
|
22
|
+
"exports": {
|
|
23
|
+
".": {
|
|
24
|
+
"types": "./dist/index.d.ts",
|
|
25
|
+
"default": "./dist/index.js"
|
|
26
|
+
},
|
|
27
|
+
"./aivss": {
|
|
28
|
+
"types": "./dist/aivss/index.d.ts",
|
|
29
|
+
"default": "./dist/aivss/index.js"
|
|
30
|
+
},
|
|
31
|
+
"./types": {
|
|
32
|
+
"types": "./dist/aivss/types.d.ts",
|
|
33
|
+
"default": "./dist/aivss/types.js"
|
|
34
|
+
}
|
|
35
|
+
},
|
|
36
|
+
"files": [
|
|
37
|
+
"dist",
|
|
38
|
+
"vendor/atlas-release",
|
|
39
|
+
"schemas",
|
|
40
|
+
"demo",
|
|
41
|
+
"SKILL.md",
|
|
42
|
+
"README.md",
|
|
43
|
+
"LICENSE"
|
|
44
|
+
],
|
|
45
|
+
"repository": {
|
|
46
|
+
"type": "git",
|
|
47
|
+
"url": "git+https://github.com/waratahlabs/canopy.git"
|
|
48
|
+
},
|
|
49
|
+
"bugs": {
|
|
50
|
+
"url": "https://github.com/waratahlabs/canopy/issues"
|
|
51
|
+
},
|
|
52
|
+
"homepage": "https://github.com/waratahlabs/canopy#readme",
|
|
53
|
+
"engines": {
|
|
54
|
+
"node": ">=20"
|
|
55
|
+
},
|
|
56
|
+
"scripts": {
|
|
57
|
+
"build": "rm -rf dist && bunx tsc",
|
|
58
|
+
"dev": "bun src/examples/simple-assessment.ts",
|
|
59
|
+
"batch-flow": "bun src/examples/batch-submission-flow.ts",
|
|
60
|
+
"batch": "bun src/examples/batch-assessment.ts",
|
|
61
|
+
"eval": "bun src/examples/model-eval.ts",
|
|
62
|
+
"openclaw": "bun src/examples/openclaw-assessment.ts",
|
|
63
|
+
"demo": "cd demo && bunx serve . --port 3000",
|
|
64
|
+
"threat-model": "bun src/examples/threat-model-demo.ts",
|
|
65
|
+
"test": "bun test src/",
|
|
66
|
+
"prepublishOnly": "rm -rf dist && bunx tsc && bun test src/",
|
|
67
|
+
"changeset": "changeset"
|
|
68
|
+
},
|
|
69
|
+
"keywords": [
|
|
70
|
+
"threat-modelling",
|
|
71
|
+
"agentic-ai",
|
|
72
|
+
"AIVSS",
|
|
73
|
+
"MITRE-ATLAS",
|
|
74
|
+
"OWASP-agentic-top-10",
|
|
75
|
+
"NIST-AI-RMF",
|
|
76
|
+
"AICM",
|
|
77
|
+
"DSTG-MEAID",
|
|
78
|
+
"ACSC",
|
|
79
|
+
"threat-composer",
|
|
80
|
+
"AI-security",
|
|
81
|
+
"n8n"
|
|
82
|
+
],
|
|
83
|
+
"author": "Waratah Labs",
|
|
84
|
+
"license": "MIT",
|
|
85
|
+
"publishConfig": {
|
|
86
|
+
"registry": "https://registry.npmjs.org",
|
|
87
|
+
"access": "public"
|
|
88
|
+
},
|
|
89
|
+
"dependencies": {
|
|
90
|
+
"@anthropic-ai/sdk": "^0.100.1",
|
|
91
|
+
"@aws-sdk/client-bedrock": "^3.1057.0",
|
|
92
|
+
"@aws-sdk/client-bedrock-runtime": "^3.1053.0",
|
|
93
|
+
"@aws-sdk/client-s3": "^3.1057.0",
|
|
94
|
+
"js-yaml": "^4.1.1"
|
|
95
|
+
},
|
|
96
|
+
"devDependencies": {
|
|
97
|
+
"@changesets/changelog-github": "^1.0.0",
|
|
98
|
+
"@changesets/cli": "^3.0.1",
|
|
99
|
+
"@types/bun": "^1.3.14",
|
|
100
|
+
"@types/js-yaml": "^4.0.9",
|
|
101
|
+
"@types/node": "^20.11.0",
|
|
102
|
+
"tsx": "^4.7.0",
|
|
103
|
+
"typescript": "^5.3.3"
|
|
104
|
+
}
|
|
105
|
+
}
|