akm-cli 0.9.0-beta.56 → 0.9.0-beta.58

package/dist/commands/improve/distill-promotion-policy.js

@@ -42,9 +42,6 @@ function deriveDescription(body, description) {
         ?.replace(/^#+\s*/, "")
         ?.slice(0, 160);
 }
-function memoryContent(frontmatter, body) {
-    return ["---", ...frontmatter, "---", "", body, ""].join("\n");
-}
 export function deriveKnowledgeRef(inputRef) {
     const parsed = parseAssetRef(inputRef);
     return `knowledge:${parsed.name}`;
@@ -419,7 +416,7 @@ const METADATA_BASELINE = {
         };
     },
 };
-function selectPromotionPolicy(corpus) {
+export function selectPromotionPolicy(corpus) {
     const trainingCases = corpus.filter((testCase) => (testCase.split ?? "train") === "train");
     const heldOutCases = corpus.filter((testCase) => (testCase.split ?? "train") === "heldout");
     let bestPolicy;
@@ -481,171 +478,833 @@ function selectPromotionPolicy(corpus) {
         strictlyBeatsBaselines,
     };
 }
-function benchmarkCase(name, expectPromote, split, frontmatter, body, feedbackSignals, outcome) {
-    return {
-        name,
-        expectPromote,
-        split,
-        input: {
-            inputRef: `memory:${name}`,
-            assetContent: memoryContent(frontmatter, body),
-            feedbackEvents: feedbackSignals.map((signal) => ({ metadata: { signal } })),
-        },
-        ...outcome,
-    };
-}
-export const DEFAULT_PROMOTION_POLICY_CORPUS = [
-    benchmarkCase("deploy-vpn-required", true, "train", [
-        "description: VPN required before deploy",
-        "source: skill:deploy",
-        "observed_at: 2026-04-20",
-        "confidence: 0.95",
-        "tags: [deploy, ops]",
-    ], "Always connect the VPN before starting production deploys.", ["positive", "positive"]),
-    benchmarkCase("release-smoke-test", true, "train", [
-        "description: Smoke test gates release",
-        "quality: curated",
-        "source: skill:release",
-        "observed_at: 2026-04-18",
-        "confidence: 0.85",
-    ], "Run the smoke test before tagging a release candidate.", ["positive", "positive", "positive"]),
-    benchmarkCase("kubernetes-rollout-check", true, "train", [
-        "description: Verify rollout status after apply",
-        "source: skill:k8s",
-        "observed_at: 2026-04-15",
-        "confidence: 0.95",
-        "tags: [k8s]",
-    ], "Check rollout status after kubectl apply before declaring success.", ["positive", "positive"]),
-    benchmarkCase("incident-channel-rule", true, "train", [
-        "description: Incident bridge stays single-threaded",
-        "quality: curated",
-        "source: skill:incident",
-        "observed_at: 2026-04-12",
-        "confidence: 0.95",
-    ], "Keep one operator narrating decisions in the incident bridge to avoid conflicting instructions.", ["positive", "positive", "positive"]),
-    benchmarkCase("weak-single-signal", false, "train", [
-        "description: VPN required before deploy",
-        "source: skill:deploy",
-        "observed_at: 2026-04-20",
-        "confidence: 0.95",
-        "tags: [deploy]",
-    ], "Always connect the VPN before starting production deploys.", ["positive"]),
-    benchmarkCase("contested-fact", false, "train", [
-        "description: VPN required before deploy",
-        "quality: curated",
-        "source: skill:deploy",
-        "observed_at: 2026-04-20",
-        "confidence: 0.95",
-    ], "Always connect the VPN before starting production deploys.", ["positive", "negative", "positive"], { falsePromoteCost: 5 }),
-    benchmarkCase("tentative-fact", false, "train", ["description: Deploy may require VPN", "source: skill:deploy", "observed_at: 2026-04-20", "confidence: 0.95"], "Maybe connect the VPN before starting production deploys.", ["positive", "positive"]),
-    benchmarkCase("subjective-preference", false, "train", [
-        "description: VPN required before deploy",
-        "subjective: true",
-        "source: skill:deploy",
-        "observed_at: 2026-04-20",
-        "confidence: 0.95",
-    ], "I prefer connecting the VPN before starting production deploys.", ["positive", "positive"]),
-    benchmarkCase("feedback-conflict", false, "train", ["description: VPN required before deploy", "source: skill:deploy", "observed_at: 2026-04-20", "confidence: 0.95"], "Always connect the VPN before starting production deploys.", ["positive", "positive"]),
-    {
-        name: "feedback-conflict",
-        expectPromote: false,
-        split: "train",
-        input: {
-            inputRef: "memory:feedback-conflict",
-            assetContent: memoryContent([
-                "description: VPN required before deploy",
-                "source: skill:deploy",
-                "observed_at: 2026-04-20",
-                "confidence: 0.95",
-            ], "Always connect the VPN before starting production deploys."),
-            feedbackEvents: [{ metadata: { signal: "positive" } }, { metadata: { signal: "positive", conflict: true } }],
-        },
+// Frozen module-load selection. The promotion policy's selected model was
+// historically computed by running `selectPromotionPolicy` over a large
+// hardcoded benchmark corpus at import time (a full grid search on every module
+// load). That corpus and the grid search now live in the bench test
+// (tests/commands/distill/distill-promotion-policy.bench.test.ts), which
+// re-runs `selectPromotionPolicy(DEFAULT_PROMOTION_POLICY_CORPUS)` and asserts
+// deep equality with this constant so the freeze stays honest.
+export const DEFAULT_PROMOTION_POLICY_SELECTION = {
+    corpusSize: 21,
+    trainingSize: 14,
+    heldOutSize: 7,
+    selectedModel: {
+        name: "balanced-evidence",
+        threshold: 3.8,
     },
-    benchmarkCase("derived-memory", false, "train", ["description: VPN required before deploy", "source: skill:deploy", "confidence: 0.95"], "Always connect the VPN before starting production deploys.", ["positive", "positive"]),
-    {
-        name: "derived-memory",
-        expectPromote: false,
-        split: "train",
-        input: {
-            inputRef: "memory:derived-memory.derived",
-            assetContent: memoryContent(["description: VPN required before deploy", "source: skill:deploy", "confidence: 0.95"], "Always connect the VPN before starting production deploys."),
-            feedbackEvents: [{ metadata: { signal: "positive" } }, { metadata: { signal: "positive" } }],
-        },
+    training: {
+        total: 14,
+        correct: 13,
+        falsePositives: 1,
+        falseNegatives: 0,
+        accuracy: 0.9285714285714286,
+        precision: 0.8333333333333334,
+        recall: 1,
+        f1: 0.9090909090909091,
+        truePositives: 5,
+        trueNegatives: 8,
+        netOutcomeScore: 11,
+        capturedPromoteValue: 15,
+        preventedFalsePromotionCost: 33,
+        results: [
+            {
+                name: "deploy-vpn-required",
+                expectPromote: true,
+                assessment: {
+                    applicable: true,
+                    promote: true,
+                    score: 4.250000000000001,
+                    threshold: 3.8,
+                    knowledgeRef: "knowledge:deploy-vpn-required",
+                    content: "---\ndescription: VPN required before deploy\ntags:\n  - deploy\n  - ops\nobserved_at: 2026-04-20\nsources:\n  - memory:deploy-vpn-required\n  - skill:deploy\n---\n\nAlways connect the VPN before starting production deploys.\n",
+                    blockedBy: [],
+                    positiveSignals: [
+                        "2 positive feedback events",
+                        "repeated reinforcement",
+                        "strong confidence",
+                        "linked source",
+                        "observed_at present",
+                        "description present",
+                        "tagged memory",
+                        "substantive body",
+                    ],
+                    negativeSignals: [],
+                    modelName: "balanced-evidence",
+                },
+                passed: true,
+            },
+            {
+                name: "release-smoke-test",
+                expectPromote: true,
+                assessment: {
+                    applicable: true,
+                    promote: true,
+                    score: 5.240000000000002,
+                    threshold: 3.8,
+                    knowledgeRef: "knowledge:release-smoke-test",
+                    content: "---\ndescription: Smoke test gates release\nobserved_at: 2026-04-18\nsources:\n  - memory:release-smoke-test\n  - skill:release\n---\n\nRun the smoke test before tagging a release candidate.\n",
+                    blockedBy: [],
+                    positiveSignals: [
+                        "3 positive feedback events",
+                        "repeated reinforcement",
+                        "curated quality",
+                        "strong confidence",
+                        "linked source",
+                        "observed_at present",
+                        "description present",
+                        "substantive body",
+                    ],
+                    negativeSignals: [],
+                    modelName: "balanced-evidence",
+                },
+                passed: true,
+            },
+            {
+                name: "kubernetes-rollout-check",
+                expectPromote: true,
+                assessment: {
+                    applicable: true,
+                    promote: true,
+                    score: 4.250000000000001,
+                    threshold: 3.8,
+                    knowledgeRef: "knowledge:kubernetes-rollout-check",
+                    content: "---\ndescription: Verify rollout status after apply\ntags:\n  - k8s\nobserved_at: 2026-04-15\nsources:\n  - memory:kubernetes-rollout-check\n  - skill:k8s\n---\n\nCheck rollout status after kubectl apply before declaring success.\n",
+                    blockedBy: [],
+                    positiveSignals: [
+                        "2 positive feedback events",
+                        "repeated reinforcement",
+                        "strong confidence",
+                        "linked source",
+                        "observed_at present",
+                        "description present",
+                        "tagged memory",
+                        "substantive body",
+                    ],
+                    negativeSignals: [],
+                    modelName: "balanced-evidence",
+                },
+                passed: true,
+            },
+            {
+                name: "incident-channel-rule",
+                expectPromote: true,
+                assessment: {
+                    applicable: true,
+                    promote: true,
+                    score: 5.450000000000002,
+                    threshold: 3.8,
+                    knowledgeRef: "knowledge:incident-channel-rule",
+                    content: "---\ndescription: Incident bridge stays single-threaded\nobserved_at: 2026-04-12\nsources:\n  - memory:incident-channel-rule\n  - skill:incident\n---\n\nKeep one operator narrating decisions in the incident bridge to avoid conflicting instructions.\n",
+                    blockedBy: [],
+                    positiveSignals: [
+                        "3 positive feedback events",
+                        "repeated reinforcement",
+                        "curated quality",
+                        "strong confidence",
+                        "linked source",
+                        "observed_at present",
+                        "description present",
+                        "substantive body",
+                    ],
+                    negativeSignals: [],
+                    modelName: "balanced-evidence",
+                },
+                passed: true,
+            },
+            {
+                name: "weak-single-signal",
+                expectPromote: false,
+                assessment: {
+                    applicable: true,
+                    promote: false,
+                    score: 2.1,
+                    threshold: 3.8,
+                    knowledgeRef: "knowledge:weak-single-signal",
+                    blockedBy: [],
+                    positiveSignals: [
+                        "1 positive feedback event",
+                        "strong confidence",
+                        "linked source",
+                        "observed_at present",
+                        "description present",
+                        "tagged memory",
+                        "substantive body",
+                    ],
+                    negativeSignals: ["only one reinforcing feedback event"],
+                    modelName: "balanced-evidence",
+                },
+                passed: true,
+            },
+            {
+                name: "contested-fact",
+                expectPromote: false,
+                assessment: {
+                    applicable: true,
+                    promote: false,
+                    score: 2.65,
+                    threshold: 3.8,
+                    knowledgeRef: "knowledge:contested-fact",
+                    blockedBy: [],
+                    positiveSignals: [
+                        "2 positive feedback events",
+                        "repeated reinforcement",
+                        "curated quality",
+                        "strong confidence",
+                        "linked source",
+                        "observed_at present",
+                        "description present",
+                        "substantive body",
+                    ],
+                    negativeSignals: ["1 negative feedback event"],
+                    modelName: "balanced-evidence",
+                },
+                passed: true,
+            },
+            {
+                name: "tentative-fact",
+                expectPromote: false,
+                assessment: {
+                    applicable: true,
+                    promote: false,
+                    score: 3.0000000000000004,
+                    threshold: 3.8,
+                    knowledgeRef: "knowledge:tentative-fact",
+                    blockedBy: [],
+                    positiveSignals: [
+                        "2 positive feedback events",
+                        "repeated reinforcement",
+                        "strong confidence",
+                        "linked source",
+                        "observed_at present",
+                        "description present",
+                        "substantive body",
+                    ],
+                    negativeSignals: ["tentative language"],
+                    modelName: "balanced-evidence",
+                },
+                passed: true,
+            },
+            {
+                name: "subjective-preference",
+                expectPromote: false,
+                assessment: {
+                    applicable: true,
+                    promote: false,
+                    score: 0,
+                    threshold: 3.8,
+                    knowledgeRef: "knowledge:subjective-preference",
+                    blockedBy: ["subjective-memory"],
+                    positiveSignals: [],
+                    negativeSignals: [],
+                    modelName: "balanced-evidence",
+                },
+                passed: true,
+            },
+            {
+                name: "feedback-conflict",
+                expectPromote: false,
+                assessment: {
+                    applicable: true,
+                    promote: true,
+                    score: 4.1000000000000005,
+                    threshold: 3.8,
+                    knowledgeRef: "knowledge:feedback-conflict",
+                    content: "---\ndescription: VPN required before deploy\nobserved_at: 2026-04-20\nsources:\n  - memory:feedback-conflict\n  - skill:deploy\n---\n\nAlways connect the VPN before starting production deploys.\n",
+                    blockedBy: [],
+                    positiveSignals: [
+                        "2 positive feedback events",
+                        "repeated reinforcement",
+                        "strong confidence",
+                        "linked source",
+                        "observed_at present",
+                        "description present",
+                        "substantive body",
+                    ],
+                    negativeSignals: [],
+                    modelName: "balanced-evidence",
+                },
+                passed: false,
+            },
+            {
+                name: "derived-memory",
+                expectPromote: false,
+                assessment: {
+                    applicable: true,
+                    promote: false,
+                    score: 3.7,
+                    threshold: 3.8,
+                    knowledgeRef: "knowledge:derived-memory",
+                    blockedBy: [],
+                    positiveSignals: [
+                        "2 positive feedback events",
+                        "repeated reinforcement",
+                        "strong confidence",
+                        "linked source",
+                        "description present",
+                        "substantive body",
+                    ],
+                    negativeSignals: [],
+                    modelName: "balanced-evidence",
+                },
+                passed: true,
+            },
+            {
+                name: "staging-cutover-order",
+                expectPromote: true,
+                assessment: {
+                    applicable: true,
+                    promote: true,
+                    score: 4.840000000000001,
+                    threshold: 3.8,
+                    knowledgeRef: "knowledge:staging-cutover-order",
+                    content: "---\ndescription: Cut over staging after migrations\ntags:\n  - db\n  - deploy\nobserved_at: 2026-04-10\nsources:\n  - memory:staging-cutover-order\n  - skill:database\n---\n\nRun database migrations before shifting staging traffic onto the new release.\n",
+                    blockedBy: [],
+                    positiveSignals: [
+                        "3 positive feedback events",
+                        "repeated reinforcement",
+                        "strong confidence",
+                        "linked source",
+                        "observed_at present",
+                        "description present",
+                        "tagged memory",
+                        "substantive body",
+                    ],
+                    negativeSignals: [],
+                    modelName: "balanced-evidence",
+                },
+                passed: true,
+            },
+            {
+                name: "temporary-token-workaround",
+                expectPromote: false,
+                assessment: {
+                    applicable: true,
+                    promote: false,
+                    score: 0,
+                    threshold: 3.8,
+                    knowledgeRef: "knowledge:temporary-token-workaround",
+                    blockedBy: ["expiring-memory"],
+                    positiveSignals: [],
+                    negativeSignals: [],
+                    modelName: "balanced-evidence",
+                },
+                passed: true,
+            },
+            {
+                name: "thin-metadata-memory",
+                expectPromote: false,
+                assessment: {
+                    applicable: true,
+                    promote: false,
+                    score: 3,
+                    threshold: 3.8,
+                    knowledgeRef: "knowledge:thin-metadata-memory",
+                    blockedBy: [],
+                    positiveSignals: [
+                        "2 positive feedback events",
+                        "repeated reinforcement",
+                        "linked source",
+                        "description present",
+                        "substantive body",
+                    ],
+                    negativeSignals: [],
+                    modelName: "balanced-evidence",
+                },
+                passed: true,
+            },
+            {
+                name: "promoted-quality-memory",
+                expectPromote: false,
+                assessment: {
+                    applicable: true,
+                    promote: false,
+                    score: 0,
+                    threshold: 3.8,
+                    knowledgeRef: "knowledge:promoted-quality-memory",
+                    blockedBy: ["proposed-quality"],
+                    positiveSignals: [],
+                    negativeSignals: [],
+                    modelName: "balanced-evidence",
+                },
+                passed: true,
+            },
+        ],
     },
-    benchmarkCase("staging-cutover-order", true, "train", [
-        "description: Cut over staging after migrations",
-        "source: skill:database",
-        "observed_at: 2026-04-10",
-        "confidence: 0.85",
-        "tags: [db, deploy]",
-    ], "Run database migrations before shifting staging traffic onto the new release.", ["positive", "positive", "positive"]),
-    benchmarkCase("temporary-token-workaround", false, "train", [
-        "description: Temporary deploy token workaround",
-        "source: skill:deploy",
-        "observed_at: 2026-04-20",
-        "confidence: 0.95",
-        "expires: 2026-06-01",
-    ], "Use the temporary deploy token workaround until the incident is closed.", ["positive", "positive"]),
-    benchmarkCase("thin-metadata-memory", false, "train", ["description: VPN required before deploy", "source: skill:deploy"], "Always connect the VPN before starting production deploys.", ["positive", "positive"]),
-    benchmarkCase("promoted-quality-memory", false, "train", [
-        "description: VPN required before deploy",
-        "quality: proposed",
-        "source: skill:deploy",
-        "observed_at: 2026-04-20",
-        "confidence: 0.95",
-    ], "Always connect the VPN before starting production deploys.", ["positive", "positive"]),
-    benchmarkCase("kafka-rebalance-note", true, "heldout", [
-        "description: Pause consumers during rebalance",
-        "quality: curated",
-        "source: skill:kafka",
-        "observed_at: 2026-04-08",
-        "confidence: 0.95",
-        "tags: [kafka, ops]",
-    ], "Pause consumers during partition rebalances to avoid duplicate processing while assignments settle.", ["positive", "positive", "positive"], { promoteValue: 4 }),
-    benchmarkCase("gha-token-scope", true, "heldout", [
-        "description: Minimize GitHub token scopes",
-        "source: skill:github-actions",
-        "observed_at: 2026-04-07",
-        "confidence: 0.85",
-        "tags: [gha, security]",
-    ], "Use the narrowest GitHub token scope that still allows the workflow step to succeed.", ["positive", "positive"], { promoteValue: 4 }),
-    benchmarkCase("helm-debug-guess", false, "heldout", [
-        "description: Helm upgrade might need --debug",
-        "source: skill:helm",
-        "observed_at: 2026-04-05",
-        "confidence: 0.85",
-    ], "It might help to add --debug to helm upgrade output during failures.", ["positive", "positive"]),
-    benchmarkCase("terraform-state-location", true, "heldout", [
-        "description: Use remote state locks",
-        "quality: curated",
-        "source: skill:terraform",
-        "observed_at: 2026-04-04",
-        "confidence: 0.95",
-        "tags: [terraform]",
-    ], "Use remote state with locking enabled before applying shared Terraform stacks.", ["positive", "positive", "positive"]),
-    benchmarkCase("mixed-signal-rollback", false, "heldout", [
-        "description: Rollback the cluster immediately",
-        "quality: curated",
-        "source: skill:incident",
-        "observed_at: 2026-04-03",
-        "confidence: 0.95",
-    ], "Rollback the cluster immediately after any 5xx spike.", ["positive", "negative", "positive"], { falsePromoteCost: 6 }),
-    benchmarkCase("cache-ttl-fact", true, "heldout", [
-        "description: Cache TTL defaults to five minutes",
-        "source: skill:platform",
-        "observed_at: 2026-04-02",
-        "confidence: 0.95",
-        "tags: [cache, platform]",
-    ], "The shared platform cache TTL defaults to five minutes unless the service opts out.", ["positive", "positive"]),
-    benchmarkCase("personal-shell-alias", false, "heldout", [
-        "description: Preferred shell alias",
-        "subjective: true",
-        "source: skill:shell",
-        "observed_at: 2026-04-01",
-        "confidence: 0.95",
-    ], "I prefer aliasing kubectl to k.", ["positive", "positive"]),
-];
-export const DEFAULT_PROMOTION_POLICY_SELECTION = selectPromotionPolicy(DEFAULT_PROMOTION_POLICY_CORPUS);
+    heldOut: {
+        total: 7,
+        correct: 7,
+        falsePositives: 0,
+        falseNegatives: 0,
+        accuracy: 1,
+        precision: 1,
+        recall: 1,
+        f1: 1,
+        truePositives: 4,
+        trueNegatives: 3,
+        netOutcomeScore: 14,
+        capturedPromoteValue: 14,
+        preventedFalsePromotionCost: 14,
+        results: [
+            {
+                name: "kafka-rebalance-note",
+                expectPromote: true,
+                assessment: {
+                    applicable: true,
+                    promote: true,
+                    score: 5.600000000000002,
+                    threshold: 3.8,
+                    knowledgeRef: "knowledge:kafka-rebalance-note",
+                    content: "---\ndescription: Pause consumers during rebalance\ntags:\n  - kafka\n  - ops\nobserved_at: 2026-04-08\nsources:\n  - memory:kafka-rebalance-note\n  - skill:kafka\n---\n\nPause consumers during partition rebalances to avoid duplicate processing while assignments settle.\n",
+                    blockedBy: [],
+                    positiveSignals: [
+                        "3 positive feedback events",
+                        "repeated reinforcement",
+                        "curated quality",
+                        "strong confidence",
+                        "linked source",
+                        "observed_at present",
+                        "description present",
+                        "tagged memory",
+                        "substantive body",
+                    ],
+                    negativeSignals: [],
+                    modelName: "balanced-evidence",
+                },
+                passed: true,
+            },
+            {
+                name: "gha-token-scope",
+                expectPromote: true,
+                assessment: {
+                    applicable: true,
+                    promote: true,
+                    score: 4.04,
+                    threshold: 3.8,
+                    knowledgeRef: "knowledge:gha-token-scope",
+                    content: "---\ndescription: Minimize GitHub token scopes\ntags:\n  - gha\n  - security\nobserved_at: 2026-04-07\nsources:\n  - memory:gha-token-scope\n  - skill:github-actions\n---\n\nUse the narrowest GitHub token scope that still allows the workflow step to succeed.\n",
+                    blockedBy: [],
+                    positiveSignals: [
+                        "2 positive feedback events",
+                        "repeated reinforcement",
+                        "strong confidence",
+                        "linked source",
+                        "observed_at present",
+                        "description present",
+                        "tagged memory",
+                        "substantive body",
+                    ],
+                    negativeSignals: [],
+                    modelName: "balanced-evidence",
+                },
+                passed: true,
+            },
+            {
+                name: "helm-debug-guess",
+                expectPromote: false,
+                assessment: {
+                    applicable: true,
+                    promote: false,
+                    score: 2.7899999999999996,
+                    threshold: 3.8,
+                    knowledgeRef: "knowledge:helm-debug-guess",
+                    blockedBy: [],
+                    positiveSignals: [
+                        "2 positive feedback events",
+                        "repeated reinforcement",
+                        "strong confidence",
+                        "linked source",
+                        "observed_at present",
+                        "description present",
+                        "substantive body",
+                    ],
+                    negativeSignals: ["tentative language"],
+                    modelName: "balanced-evidence",
+                },
+                passed: true,
+            },
+            {
+                name: "terraform-state-location",
+                expectPromote: true,
+                assessment: {
+                    applicable: true,
+                    promote: true,
+                    score: 5.600000000000002,
+                    threshold: 3.8,
+                    knowledgeRef: "knowledge:terraform-state-location",
+                    content: "---\ndescription: Use remote state locks\ntags:\n  - terraform\nobserved_at: 2026-04-04\nsources:\n  - memory:terraform-state-location\n  - skill:terraform\n---\n\nUse remote state with locking enabled before applying shared Terraform stacks.\n",
+                    blockedBy: [],
+                    positiveSignals: [
+                        "3 positive feedback events",
+                        "repeated reinforcement",
+                        "curated quality",
+                        "strong confidence",
+                        "linked source",
+                        "observed_at present",
+                        "description present",
+                        "tagged memory",
+                        "substantive body",
+                    ],
+                    negativeSignals: [],
+                    modelName: "balanced-evidence",
+                },
+                passed: true,
+            },
+            {
+                name: "mixed-signal-rollback",
+                expectPromote: false,
+                assessment: {
+                    applicable: true,
+                    promote: false,
+                    score: 2.65,
+                    threshold: 3.8,
+                    knowledgeRef: "knowledge:mixed-signal-rollback",
+                    blockedBy: [],
+                    positiveSignals: [
+                        "2 positive feedback events",
+                        "repeated reinforcement",
+                        "curated quality",
+                        "strong confidence",
+                        "linked source",
+                        "observed_at present",
+                        "description present",
+                        "substantive body",
+                    ],
+                    negativeSignals: ["1 negative feedback event"],
+                    modelName: "balanced-evidence",
+                },
+                passed: true,
+            },
+            {
+                name: "cache-ttl-fact",
+                expectPromote: true,
+                assessment: {
+                    applicable: true,
+                    promote: true,
+                    score: 4.250000000000001,
+                    threshold: 3.8,
+                    knowledgeRef: "knowledge:cache-ttl-fact",
+                    content: "---\ndescription: Cache TTL defaults to five minutes\ntags:\n  - cache\n  - platform\nobserved_at: 2026-04-02\nsources:\n  - memory:cache-ttl-fact\n  - skill:platform\n---\n\nThe shared platform cache TTL defaults to five minutes unless the service opts out.\n",
+                    blockedBy: [],
+                    positiveSignals: [
+                        "2 positive feedback events",
+                        "repeated reinforcement",
+                        "strong confidence",
+                        "linked source",
+                        "observed_at present",
+                        "description present",
+                        "tagged memory",
+                        "substantive body",
+                    ],
+                    negativeSignals: [],
+                    modelName: "balanced-evidence",
+                },
+                passed: true,
+            },
+            {
+                name: "personal-shell-alias",
+                expectPromote: false,
+                assessment: {
+                    applicable: true,
+                    promote: false,
+                    score: 0,
+                    threshold: 3.8,
+                    knowledgeRef: "knowledge:personal-shell-alias",
+                    blockedBy: ["subjective-memory"],
+                    positiveSignals: [],
+                    negativeSignals: [],
+                    modelName: "balanced-evidence",
+                },
+                passed: true,
+            },
+        ],
+    },
+    baselines: [
+        {
+            name: "baseline-positive-feedback",
+            heldOut: {
+                total: 7,
+                correct: 5,
+                falsePositives: 2,
+                falseNegatives: 0,
+                accuracy: 0.7142857142857143,
+                precision: 0.6666666666666666,
+                recall: 1,
+                f1: 0.8,
+                truePositives: 4,
+                trueNegatives: 1,
+                netOutcomeScore: 4,
+                capturedPromoteValue: 14,
+                preventedFalsePromotionCost: 4,
+                results: [
+                    {
+                        name: "kafka-rebalance-note",
+                        expectPromote: true,
+                        assessment: {
+                            applicable: true,
+                            promote: true,
+                            score: 3,
+                            threshold: 2,
+                            knowledgeRef: "knowledge:kafka-rebalance-note",
+                            content: "---\ndescription: Pause consumers during rebalance\ntags:\n  - kafka\n  - ops\nobserved_at: 2026-04-08\nsources:\n  - memory:kafka-rebalance-note\n  - skill:kafka\n---\n\nPause consumers during partition rebalances to avoid duplicate processing while assignments settle.\n",
+                            blockedBy: [],
+                            positiveSignals: ["baseline positive feedback rule"],
+                            negativeSignals: [],
+                            modelName: "baseline-positive-feedback",
+                        },
+                        passed: true,
+                    },
+                    {
+                        name: "gha-token-scope",
+                        expectPromote: true,
+                        assessment: {
+                            applicable: true,
+                            promote: true,
+                            score: 2,
+                            threshold: 2,
+                            knowledgeRef: "knowledge:gha-token-scope",
+                            content: "---\ndescription: Minimize GitHub token scopes\ntags:\n  - gha\n  - security\nobserved_at: 2026-04-07\nsources:\n  - memory:gha-token-scope\n  - skill:github-actions\n---\n\nUse the narrowest GitHub token scope that still allows the workflow step to succeed.\n",
+                            blockedBy: [],
+                            positiveSignals: ["baseline positive feedback rule"],
+                            negativeSignals: [],
+                            modelName: "baseline-positive-feedback",
+                        },
+                        passed: true,
+                    },
+                    {
+                        name: "helm-debug-guess",
+                        expectPromote: false,
+                        assessment: {
+                            applicable: true,
+                            promote: true,
+                            score: 2,
+                            threshold: 2,
+                            knowledgeRef: "knowledge:helm-debug-guess",
+                            content: "---\ndescription: Helm upgrade might need --debug\nobserved_at: 2026-04-05\nsources:\n  - memory:helm-debug-guess\n  - skill:helm\n---\n\nIt might help to add --debug to helm upgrade output during failures.\n",
+                            blockedBy: [],
+                            positiveSignals: ["baseline positive feedback rule"],
+                            negativeSignals: [],
+                            modelName: "baseline-positive-feedback",
+                        },
+                        passed: false,
+                    },
+                    {
+                        name: "terraform-state-location",
+                        expectPromote: true,
+                        assessment: {
+                            applicable: true,
+                            promote: true,
+                            score: 3,
+                            threshold: 2,
+                            knowledgeRef: "knowledge:terraform-state-location",
+                            content: "---\ndescription: Use remote state locks\ntags:\n  - terraform\nobserved_at: 2026-04-04\nsources:\n  - memory:terraform-state-location\n  - skill:terraform\n---\n\nUse remote state with locking enabled before applying shared Terraform stacks.\n",
+                            blockedBy: [],
+                            positiveSignals: ["baseline positive feedback rule"],
+                            negativeSignals: [],
+                            modelName: "baseline-positive-feedback",
+                        },
+                        passed: true,
+                    },
+                    {
+                        name: "mixed-signal-rollback",
+                        expectPromote: false,
+                        assessment: {
+                            applicable: true,
+                            promote: true,
+                            score: 2,
+                            threshold: 2,
+                            knowledgeRef: "knowledge:mixed-signal-rollback",
+                            content: "---\ndescription: Rollback the cluster immediately\nobserved_at: 2026-04-03\nsources:\n  - memory:mixed-signal-rollback\n  - skill:incident\n---\n\nRollback the cluster immediately after any 5xx spike.\n",
+                            blockedBy: [],
+                            positiveSignals: ["baseline positive feedback rule"],
+                            negativeSignals: [],
+                            modelName: "baseline-positive-feedback",
+                        },
+                        passed: false,
+                    },
+                    {
+                        name: "cache-ttl-fact",
+                        expectPromote: true,
+                        assessment: {
+                            applicable: true,
+                            promote: true,
+                            score: 2,
+                            threshold: 2,
+                            knowledgeRef: "knowledge:cache-ttl-fact",
+                            content: "---\ndescription: Cache TTL defaults to five minutes\ntags:\n  - cache\n  - platform\nobserved_at: 2026-04-02\nsources:\n  - memory:cache-ttl-fact\n  - skill:platform\n---\n\nThe shared platform cache TTL defaults to five minutes unless the service opts out.\n",
+                            blockedBy: [],
+                            positiveSignals: ["baseline positive feedback rule"],
+                            negativeSignals: [],
+                            modelName: "baseline-positive-feedback",
+                        },
+                        passed: true,
+                    },
+                    {
+                        name: "personal-shell-alias",
+                        expectPromote: false,
+                        assessment: {
+                            applicable: true,
+                            promote: false,
+                            score: 0,
+                            threshold: 2,
+                            knowledgeRef: "knowledge:personal-shell-alias",
+                            blockedBy: ["subjective-memory"],
+                            positiveSignals: [],
+                            negativeSignals: [],
+                            modelName: "baseline-positive-feedback",
+                        },
+                        passed: true,
+                    },
+                ],
+            },
+            noWorseThanSelected: true,
+            strictWin: true,
+            strictWinMetrics: ["f1", "netOutcomeScore", "accuracy"],
+        },
+        {
+            name: "baseline-metadata",
+            heldOut: {
+                total: 7,
+                correct: 5,
+                falsePositives: 2,
+                falseNegatives: 0,
+                accuracy: 0.7142857142857143,
+                precision: 0.6666666666666666,
+                recall: 1,
+                f1: 0.8,
+                truePositives: 4,
+                trueNegatives: 1,
+                netOutcomeScore: 4,
+                capturedPromoteValue: 14,
+                preventedFalsePromotionCost: 4,
+                results: [
+                    {
+                        name: "kafka-rebalance-note",
+                        expectPromote: true,
+                        assessment: {
+                            applicable: true,
+                            promote: true,
+                            score: 2,
+                            threshold: 3,
+                            knowledgeRef: "knowledge:kafka-rebalance-note",
+                            content: "---\ndescription: Pause consumers during rebalance\ntags:\n  - kafka\n  - ops\nobserved_at: 2026-04-08\nsources:\n  - memory:kafka-rebalance-note\n  - skill:kafka\n---\n\nPause consumers during partition rebalances to avoid duplicate processing while assignments settle.\n",
+                            blockedBy: [],
+                            positiveSignals: ["baseline metadata rule"],
+                            negativeSignals: [],
+                            modelName: "baseline-metadata",
+                        },
+                        passed: true,
+                    },
+                    {
+                        name: "gha-token-scope",
+                        expectPromote: true,
+                        assessment: {
+                            applicable: true,
+                            promote: true,
+                            score: 2,
+                            threshold: 3,
+                            knowledgeRef: "knowledge:gha-token-scope",
+                            content: "---\ndescription: Minimize GitHub token scopes\ntags:\n  - gha\n  - security\nobserved_at: 2026-04-07\nsources:\n  - memory:gha-token-scope\n  - skill:github-actions\n---\n\nUse the narrowest GitHub token scope that still allows the workflow step to succeed.\n",
+                            blockedBy: [],
+                            positiveSignals: ["baseline metadata rule"],
+                            negativeSignals: [],
+                            modelName: "baseline-metadata",
+                        },
+                        passed: true,
+                    },
+                    {
+                        name: "helm-debug-guess",
+                        expectPromote: false,
+                        assessment: {
+                            applicable: true,
+                            promote: true,
+                            score: 2,
+                            threshold: 3,
+                            knowledgeRef: "knowledge:helm-debug-guess",
+                            content: "---\ndescription: Helm upgrade might need --debug\nobserved_at: 2026-04-05\nsources:\n  - memory:helm-debug-guess\n  - skill:helm\n---\n\nIt might help to add --debug to helm upgrade output during failures.\n",
+                            blockedBy: [],
+                            positiveSignals: ["baseline metadata rule"],
+                            negativeSignals: [],
+                            modelName: "baseline-metadata",
+                        },
+                        passed: false,
+                    },
+                    {
+                        name: "terraform-state-location",
+                        expectPromote: true,
+                        assessment: {
+                            applicable: true,
+                            promote: true,
+                            score: 2,
+                            threshold: 3,
+                            knowledgeRef: "knowledge:terraform-state-location",
+                            content: "---\ndescription: Use remote state locks\ntags:\n  - terraform\nobserved_at: 2026-04-04\nsources:\n  - memory:terraform-state-location\n  - skill:terraform\n---\n\nUse remote state with locking enabled before applying shared Terraform stacks.\n",
+                            blockedBy: [],
+                            positiveSignals: ["baseline metadata rule"],
+                            negativeSignals: [],
+                            modelName: "baseline-metadata",
+                        },
+                        passed: true,
+                    },
+                    {
+                        name: "mixed-signal-rollback",
+                        expectPromote: false,
+                        assessment: {
+                            applicable: true,
+                            promote: true,
+                            score: 2,
+                            threshold: 3,
+                            knowledgeRef: "knowledge:mixed-signal-rollback",
+                            content: "---\ndescription: Rollback the cluster immediately\nobserved_at: 2026-04-03\nsources:\n  - memory:mixed-signal-rollback\n  - skill:incident\n---\n\nRollback the cluster immediately after any 5xx spike.\n",
+                            blockedBy: [],
+                            positiveSignals: ["baseline metadata rule"],
+                            negativeSignals: [],
+                            modelName: "baseline-metadata",
+                        },
+                        passed: false,
+                    },
+                    {
+                        name: "cache-ttl-fact",
+                        expectPromote: true,
+                        assessment: {
+                            applicable: true,
+                            promote: true,
+                            score: 2,
+                            threshold: 3,
+                            knowledgeRef: "knowledge:cache-ttl-fact",
+                            content: "---\ndescription: Cache TTL defaults to five minutes\ntags:\n  - cache\n  - platform\nobserved_at: 2026-04-02\nsources:\n  - memory:cache-ttl-fact\n  - skill:platform\n---\n\nThe shared platform cache TTL defaults to five minutes unless the service opts out.\n",
+                            blockedBy: [],
+                            positiveSignals: ["baseline metadata rule"],
+                            negativeSignals: [],
+                            modelName: "baseline-metadata",
+                        },
+                        passed: true,
+                    },
+                    {
+                        name: "personal-shell-alias",
+                        expectPromote: false,
+                        assessment: {
+                            applicable: true,
+                            promote: false,
+                            score: 0,
+                            threshold: 2,
+                            knowledgeRef: "knowledge:personal-shell-alias",
+                            blockedBy: ["subjective-memory"],
+                            positiveSignals: [],
+                            negativeSignals: [],
+                            modelName: "baseline-metadata",
+                        },
+                        passed: true,
+                    },
+                ],
+            },
+            noWorseThanSelected: true,
+            strictWin: true,
+            strictWinMetrics: ["f1", "netOutcomeScore", "accuracy"],
+        },
+    ],
+    strictlyBeatsBaselines: true,
+};
 const SELECTED_MODEL = CANDIDATE_MODELS.find((model) => model.name === DEFAULT_PROMOTION_POLICY_SELECTION.selectedModel.name);
 export const DEFAULT_PROMOTION_POLICY = {
     name: DEFAULT_PROMOTION_POLICY_SELECTION.selectedModel.name,