akm-cli 0.9.0-beta.9 → 0.9.0-rc.0

package/dist/commands/improve/recombine.js

@@ -0,0 +1,818 @@
+// This Source Code Form is subject to the terms of the Mozilla Public
+// License, v. 2.0. If a copy of the MPL was not distributed with this
+// file, You can obtain one at https://mozilla.org/MPL/2.0/.
+/**
+ * #609 — recombine / synthesize pass.
+ *
+ * A whole-corpus synthesis stage that runs AFTER consolidation and is OPT-IN
+ * (default disabled via `IMPROVE_PROCESS_DEFAULTS.recombine`). It clusters
+ * memories by RELATEDNESS (shared tags / graph entities — NEVER embedding
+ * similarity), issues ONE bounded LLM call per cluster to induce a single
+ * cross-episodic generalization, and emits the result as a NORMAL pending
+ * proposal with frontmatter `type: hypothesis` through the existing proposal
+ * queue + quality gate.
+ *
+ * Two-pass contract: the first pass ONLY ever emits `type: hypothesis`
+ * proposals — never a `type: lesson`. Promotion to a lesson happens on a later
+ * confirmation run once the same generalization has been re-induced
+ * `confirmThreshold` times (#625). The confirmation count is persisted in the
+ * `recombine_hypotheses` state.db table (migration 014), keyed by the
+ * deterministic `deriveRecombineLessonRef` value so re-induction of the SAME
+ * member-set maps back to the SAME row. When the count reaches the threshold,
+ * the run emits ONE `type: lesson` promotion proposal through the SAME proposal
+ * queue + quality gate (createProposal + validateProposalFrontmatter), NEVER a
+ * direct stash write, then marks the row promoted (resetting its count) so it is
+ * not re-promoted on every subsequent run. Hypotheses NOT re-induced in a run
+ * have their consecutive streak reset (decay-to-zero).
+ *
+ * NAMESPACE note: the ref stays `lesson:recombined/<slug>-<hash>` for BOTH
+ * passes. The ref is the promotion TARGET asset (a lesson in both the hypothesis
+ * and promoted states), so re-induction must map to the same ref and the ref
+ * cannot encode the proposal type. The hypothesis-vs-lesson distinction is
+ * carried ONLY by the proposal frontmatter `type` field. On promotion the prior
+ * pending `type: hypothesis` proposal for that ref is superseded (rejected) so
+ * the queue never shows two proposals for one ref.
+ *
+ * A justified null (the LLM determines no defensible generalization exists) is
+ * an acceptable outcome: it produces no proposal and records a
+ * `recombine_invoked` event with `outcome: 'null_returned'`.
+ */
+import { createHash } from "node:crypto";
+import fs from "node:fs";
+import recombineSystemPrompt from "../../assets/prompts/recombine-system.md" with { type: "text" };
+import { assembleAssetFromString } from "../../core/asset/asset-serialize.js";
+import { parseFrontmatter } from "../../core/asset/frontmatter.js";
+import { resolveStashDir } from "../../core/common.js";
+import { loadConfig } from "../../core/config/config.js";
+import { appendEvent } from "../../core/events.js";
+import { parseEmbeddedJsonResponse } from "../../core/parse.js";
+import { resolveStashStandards } from "../../core/standards/resolve-stash-standards.js";
+import { withStateDbAsync } from "../../core/state-db.js";
+import { closeDatabase, getAllEntries, getEntitiesByEntryIds, openExistingDatabase, } from "../../indexer/db/db.js";
+import { decayUnseenRecombineHypotheses, findMatchingRecombineHypothesis, getRecombineHypothesis, markRecombineHypothesisPromoted, recordRecombineInduction, } from "../../storage/repositories/recombine-repository.js";
+import { archiveProposal, createProposal, isProposalSkipped, listProposals } from "../proposal/repository.js";
+import { isValidDescription, isValidWhenToUse, validateProposalFrontmatter, } from "../proposal/validators/proposal-quality-validators.js";
+import { isConsolidationEligibleMemoryName } from "./consolidate.js";
+import { resolveImproveLlmFn } from "./shared.js";
+const RECOMBINE_SYSTEM_PROMPT = recombineSystemPrompt;
+const DEFAULT_MIN_CLUSTER_SIZE = 3;
+const DEFAULT_MAX_CLUSTERS_PER_RUN = 5;
+// #632 — slots in each run's processed budget reserved for the top TAG clusters
+// when entity clusters are present, so tag-only topics (a topic with a good tag
+// but no extracted graph entity) are never fully starved by entity preference.
+// Entities still take the rest of the budget; whichever kind is short, the other
+// backfills. UNSET of entities (tags-only stash) ignores this and takes top-N tags.
+const RESERVED_TAG_SLOTS = 3;
+// #632 — a tag cluster larger than this is treated as an over-broad project
+// mega-bucket (low-coherence, the bland signal #632 de-emphasizes), so the
+// reserved tag slots prefer TIGHTER clusters at or below this size first.
+const TAG_RESERVE_SOFT_CAP = 20;
+// #632 — default to the UNION of tag + graph-entity relatedness, with entity
+// clusters PREFERRED at selection time (see the rank in buildRelatednessClusters).
+// Entity clustering surfaces coherent, subject-scoped clusters (a tool/subsystem)
+// that the coarse stash-wide tag buckets miss, while tags still cover memories the
+// graph has no entity for. The `entity:` vs `tag:` signature namespaces are
+// independent, so a pure tag cluster's confirmation streak is only re-baselined
+// when its OWN membership changes — which is exactly what the session-capture pool
+// exclusion intends for the telemetry-polluted buckets (re-baselining a noisy
+// cluster's streak is correct, not a regression). A stash with no extracted graph
+// entities falls through to tag-only.
+const DEFAULT_RELATEDNESS_SOURCE = "both";
+/** #625 — re-induction count required before a hypothesis promotes to a lesson. */
+const DEFAULT_CONFIRM_THRESHOLD = 2;
+/**
+ * #633 — Jaccard membership-overlap threshold for matching a freshly-induced
+ * hypothesis to an existing pending row under the SAME signature. A growing
+ * stash drifts the exact member set every run; an overlap >= this lets the
+ * confirmation streak keep accumulating under one row instead of resetting to 1.
+ */
+const DEFAULT_RECOMBINE_OVERLAP = 0.7;
+// ── Clustering by relatedness (NOT similarity) ────────────────────────────────
+/**
+ * #632 — English stopwords that occasionally leak into frontmatter tags
+ * (`is`, `the`, `for`, …). They carry no topical signal, so a cluster keyed on
+ * one is meaningless. Lowercased; matched case-insensitively.
+ */
+const JUNK_STOPWORD_TAGS = new Set([
+    "a",
+    "an",
+    "and",
+    "the",
+    "to",
+    "of",
+    "in",
+    "on",
+    "for",
+    "is",
+    "are",
+    "be",
+    "no",
+    "not",
+    "or",
+    "if",
+    "it",
+    "as",
+    "at",
+    "by",
+    "we",
+    "us",
+    "do",
+    "so",
+    "when",
+    "then",
+    "than",
+    "with",
+    "from",
+    "this",
+    "that",
+    "uses",
+    "use",
+    "via",
+]);
+/**
+ * #632 — a tag carries no clustering signal (and must be skipped) when it is
+ * purely a number / date / hash / version string, a single char, or a common
+ * stopword. Unlike `excludeTags` (a fixed project list), this catches the
+ * OPEN-ENDED junk — every new date or commit hash — without config upkeep.
+ */
+export function isJunkTag(tag) {
+    const t = tag.trim().toLowerCase();
+    if (t.length <= 1)
+        return true;
+    if (JUNK_STOPWORD_TAGS.has(t))
+        return true;
+    if (/^\d+$/.test(t))
+        return true; // pure numbers + dates: 2026, 05, 23, 20260529
+    if (/^v?\d+(?:\.\d+)+$/.test(t))
+        return true; // versions: 0.8.0, v1.2
+    if (/^v\d+$/.test(t))
+        return true; // v0, v2
+    if (/^[0-9a-f]{4,}$/.test(t) && /\d/.test(t))
+        return true; // short hex hashes: 002c624c, 192d
+    return false;
+}
+/**
+ * #632 — generic extraction-artefact entities the graph routinely emits: session
+ * bookkeeping (`session_id`, `session_checkpoint`), structured-log field names
+ * (`reason`, `harness`, `structured event log`), and the like. They are
+ * stash-wide and carry no topical signal, so an `entity:<norm>` cluster keyed on
+ * one is exactly the bland mega-bucket #632 aims to remove. Lowercased; matched
+ * against the already-normalised `entity_norm`.
+ */
+const JUNK_ENTITY_NORMS = new Set([
+    "session",
+    "session_id",
+    "session_checkpoint",
+    "checkpoint",
+    "reason",
+    "harness",
+    "event",
+    "event log",
+    "structured event",
+    "structured event log",
+    "timestamp",
+    "metadata",
+    "status",
+]);
+/**
+ * #632 — an entity carries no clustering signal (and must be skipped) when it is
+ * a generic extraction artefact (session / structured-log bookkeeping), a raw
+ * filesystem path (absolute paths the extractor lifts verbatim), or the same
+ * number / date / hash / version / stopword junk `isJunkTag` rejects. Mirrors
+ * `isJunkTag` so the graph relatedness source does not reintroduce the very
+ * bland buckets entity clustering is meant to replace. Unlike `excludeEntities`
+ * (a fixed user list), this catches the OPEN-ENDED junk without config upkeep.
+ */
+export function isJunkEntity(entity) {
+    const e = entity.trim().toLowerCase();
+    if (e.length <= 1)
+        return true;
+    if (JUNK_ENTITY_NORMS.has(e))
+        return true;
+    if (JUNK_STOPWORD_TAGS.has(e))
+        return true;
+    if (e.includes("/") || e.includes("\\"))
+        return true; // raw file paths
+    if (/^\d+$/.test(e))
+        return true; // pure numbers + dates
+    if (/^v?\d+(?:\.\d+)+$/.test(e))
+        return true; // versions
+    if (/^v\d+$/.test(e))
+        return true; // v0, v2
+    if (/^[0-9a-f]{4,}$/.test(e) && /\d/.test(e))
+        return true; // short hex hashes
+    return false;
+}
+/**
+ * Build relatedness clusters from the memory pool. Clustering is driven purely
+ * by shared tags / graph entities — it MUST NOT use embedding similarity, so
+ * textually near-identical memories that share no relatedness signal never
+ * cluster together.
+ *
+ * For `relatednessSource`:
+ *   - `"tags"`  — group by each frontmatter tag.
+ *   - `"graph"` — group by shared `graph_file_entities.entity_norm`; falls back
+ *                 to tags when the graph table is empty (fail-open).
+ *   - `"both"`  — union of the tag and entity grouping keys.
+ *
+ * A cluster is a signal whose member set is >= `minClusterSize`. Overlapping
+ * clusters are de-duplicated by member-set identity, and the result is RANKED
+ * by member-count descending (deterministic alphabetical tiebreak). The
+ * `maxClustersPerRun` cap is NOT applied here — call {@link selectClustersForRun}
+ * (entity-aware blend) or {@link capClusters} (tags-only) on the result for the
+ * processed slice; the full ranked list is retained so the cap-aware decay sweep
+ * can tell cap-displacement from corpus absence (#658).
+ */
+export function buildRelatednessClusters(entries, opts) {
+    // Only consolidation-eligible memories participate (exclude `.derived`).
+    const memories = entries.filter((e) => e.entry.type === "memory" && isConsolidationEligibleMemoryName(e.entry.name));
+    // signal -> member entries
+    const groups = new Map();
+    const add = (signal, entry) => {
+        const key = signal.trim();
+        if (!key)
+            return;
+        const list = groups.get(key);
+        if (list) {
+            if (!list.includes(entry))
+                list.push(entry);
+        }
+        else {
+            groups.set(key, [entry]);
+        }
+    };
+    const useTags = opts.relatednessSource === "tags" || opts.relatednessSource === "both";
+    // Graph relatedness falls open to tags when no entities are available.
+    const hasEntities = !!opts.entityByEntryId && opts.entityByEntryId.size > 0;
+    const useGraph = (opts.relatednessSource === "graph" || opts.relatednessSource === "both") && hasEntities;
+    const tagsFallback = !useTags && opts.relatednessSource === "graph" && !hasEntities;
+    // #632 — tags/entities excluded from clustering (applies regardless of
+    // source). UNSET/[] leaves tag clustering byte-identical to the pre-#632 path.
+    const excludeTags = new Set(opts.excludeTags ?? []);
+    // `entity_norm` is always lowercased (graph-dedup.ts), so normalise the
+    // user-supplied exclusion list to match — `excludeEntities: ["OpenCode"]`
+    // should suppress the stored `opencode` entity (Reviewer A, #632).
+    const excludeEntities = new Set((opts.excludeEntities ?? []).map((e) => e.toLowerCase()));
+    for (const entry of memories) {
+        if (useTags || tagsFallback) {
+            for (const tag of entry.entry.tags ?? []) {
+                if (excludeTags.has(tag))
+                    continue;
+                if (isJunkTag(tag))
+                    continue; // #632 — skip numeric/date/hash/version/stopword junk
+                add(`tag:${tag}`, entry);
+            }
+        }
+        if (useGraph && opts.entityByEntryId) {
+            for (const ent of opts.entityByEntryId.get(entry.id) ?? []) {
+                if (excludeEntities.has(ent))
+                    continue;
+                if (isJunkEntity(ent))
+                    continue; // #632 — skip generic extraction-artefact / path entities
+                add(`entity:${ent}`, entry);
+            }
+        }
+    }
+    // Keep only groups at or above the minimum cluster size. #632 — when
+    // maxClusterSize is set, also SKIP groups strictly larger than the cap so an
+    // over-broad bucket never reaches (and starves) the largest-first slice.
+    // UNSET = no upper bound = identical to the pre-#632 behaviour.
+    let clusters = [];
+    for (const [signature, members] of groups) {
+        if (members.length < opts.minClusterSize)
+            continue;
+        if (opts.maxClusterSize != null && members.length > opts.maxClusterSize)
+            continue;
+        clusters.push({ signature, members });
+    }
+    // De-duplicate clusters that share the exact same member set (e.g. a tag and
+    // an entity that co-occur on the same trio). Keep the first by signature.
+    const seenMemberKeys = new Set();
+    clusters = clusters.filter((c) => {
+        const memberKey = c.members
+            .map((m) => m.id)
+            .sort((a, b) => a - b)
+            .join(",");
+        if (seenMemberKeys.has(memberKey))
+            return false;
+        seenMemberKeys.add(memberKey);
+        return true;
+    });
+    // #632 — rank ENTITY clusters ahead of tag clusters, then largest-first within
+    // each kind (deterministic alphabetical tiebreak). A graph entity is an
+    // EXTRACTED SUBJECT (a tool / subsystem / component), so it is a far
+    // higher-signal cluster key than an auto-tokenized frontmatter tag, whose
+    // broadest buckets (`tag:<project>` — e.g. every memory tagged `akm`) are the
+    // coarse, bland clusters #632 set out to kill. Largest-first ALONE let those
+    // tag mega-buckets fill the `maxClustersPerRun` slice every run and starve the
+    // coherent entity clusters this pass produces. Preferring entities keeps tag
+    // clustering as the fallback (a stash with no graph entities, or a topic with a
+    // tag but no extracted entity, still clusters) while ensuring the better signal
+    // wins the cap. The processed slice is chosen by the caller via
+    // {@link selectClustersForRun} (NOT here), so the FULL formed set stays
+    // available for the cap-aware decay sweep —
+    // a cluster displaced by the cap must not be confused with a cluster that
+    // vanished from the corpus (#658).
+    const entityRank = (sig) => (sig.startsWith("entity:") ? 0 : 1);
+    clusters.sort((a, b) => entityRank(a.signature) - entityRank(b.signature) ||
+        b.members.length - a.members.length ||
+        a.signature.localeCompare(b.signature));
+    return clusters;
+}
+/**
+ * #658 — apply the `maxClustersPerRun` cap to a largest-first ranked cluster
+ * list. Split out from {@link buildRelatednessClusters} so callers retain the
+ * full pre-cap set: the clusters BELOW the cap still re-formed this run and must
+ * spare their hypotheses from decay (cap-displacement is a SCHEDULING miss, not
+ * a substance miss). Callers that only need the processed slice call this; the
+ * full ranked list feeds {@link decayUnseenRecombineHypotheses}.
+ */
+export function capClusters(ranked, maxClustersPerRun) {
+    return ranked.slice(0, Math.max(0, maxClustersPerRun));
+}
+/**
+ * #632 — pick the per-run PROCESSED slice from the full ranked list, BLENDING
+ * entity and tag clusters so neither starves the other:
+ *
+ *   - No entity clusters → top `maxClustersPerRun` TAG clusters (a tags-only
+ *     stash is byte-identical to {@link capClusters}).
+ *   - Entity clusters present → ENTITIES LEAD the budget (the higher-signal key —
+ *     an extracted subject vs an auto-tokenized filename tag), but RESERVE up to
+ *     `RESERVED_TAG_SLOTS` for tag clusters so tag-only topics still surface every
+ *     run. Entities are never starved below one slot when present and the budget
+ *     allows. Whichever kind is short, the other backfills, so the full budget is
+ *     always used when enough clusters exist.
+ *
+ * The reserved tag slots prefer TIGHTER tag clusters (size <= `TAG_RESERVE_SOFT_CAP`,
+ * largest-first within that band, then the over-cap buckets): a broad
+ * auto-tokenized `tag:<project>` mega-bucket is exactly the bland, low-coherence
+ * signal #632 de-emphasizes, so the reserve should not spend its protected slots
+ * on the largest tags. Entities themselves stay largest-first — an entity is a
+ * coherent subject at any size.
+ *
+ * Entities lead the returned order. The FULL pre-cap `ranked` list (not this
+ * slice) still feeds the cap-aware decay sweep (#658), so a cluster left out of
+ * the processed slice this run is NOT decayed as if it vanished.
+ */
+export function selectClustersForRun(ranked, maxClustersPerRun) {
+    const max = Math.max(0, maxClustersPerRun);
+    if (max === 0)
+        return [];
+    const entities = ranked.filter((c) => c.signature.startsWith("entity:"));
+    if (entities.length === 0)
+        return ranked.slice(0, max); // tags-only → top-N (capClusters parity)
+    const tags = ranked.filter((c) => !c.signature.startsWith("entity:"));
+    // Reserve up to RESERVED_TAG_SLOTS for tags, but never below one slot for the
+    // leading entities when the budget allows (so a small `maxClustersPerRun` does
+    // not silently invert the entity preference).
+    const reservedForTags = Math.min(tags.length, RESERVED_TAG_SLOTS, Math.max(0, max - 1));
+    const entityTake = Math.min(entities.length, max - reservedForTags);
+    const tagTake = Math.min(tags.length, max - entityTake); // tags take their reserve + any slot entities left
+    // Prefer tight tags (<= soft cap) over broad mega-buckets for the reserve.
+    const reserveTags = [...tags]
+        .sort((a, b) => {
+        const aOver = a.members.length > TAG_RESERVE_SOFT_CAP ? 1 : 0;
+        const bOver = b.members.length > TAG_RESERVE_SOFT_CAP ? 1 : 0;
+        return aOver - bOver || b.members.length - a.members.length || a.signature.localeCompare(b.signature);
+    })
+        .slice(0, tagTake);
+    return [...entities.slice(0, entityTake), ...reserveTags];
+}
+// ── Prompt + ref derivation ───────────────────────────────────────────────────
+/** Read a memory body (frontmatter stripped) for the cluster prompt. */
+function readBody(entry) {
+    try {
+        const raw = fs.readFileSync(entry.filePath, "utf8");
+        return parseFrontmatter(raw).content.trim();
+    }
+    catch {
+        return "";
+    }
+}
+/** Assemble the per-cluster user prompt fed to the recombine LLM. */
+export function buildClusterPrompt(cluster, standardsContext = "") {
+    const lines = [
+        `Shared signal: ${cluster.signature}`,
+        `Cluster of ${cluster.members.length} related memories:`,
+        "",
+    ];
+    if (standardsContext.trim()) {
+        lines.push("Standards to follow (the rulebook for this target):");
+        lines.push(standardsContext.trim());
+        lines.push("");
+    }
+    for (const m of cluster.members) {
+        lines.push(`[memory:${m.entry.name}]`);
+        if (m.entry.description)
+            lines.push(`Description: ${m.entry.description}`);
+        const body = readBody(m);
+        if (body)
+            lines.push(body);
+        lines.push("");
+    }
+    lines.push("Induce ONE cross-episodic generalization these memories support, or return an explicit null if none is defensible.");
+    return lines.join("\n");
+}
+/**
+ * Stable lesson ref for a cluster. The hash of the sorted member refs keeps the
+ * ref deterministic across runs (so re-induction maps to the same ref + the
+ * content-hash dedup in createProposal suppresses queue churn).
+ */
+export function deriveRecombineLessonRef(cluster) {
+    const slug = cluster.signature
+        .replace(/^(tag|entity):/, "")
+        .toLowerCase()
+        .replace(/[^a-z0-9-]+/g, "-")
+        .replace(/-+/g, "-")
+        .replace(/^-|-$/g, "");
+    const memberKey = recombineMemberKey(cluster);
+    const hash = createHash("sha256").update(memberKey, "utf8").digest("hex").slice(0, 8);
+    return `lesson:recombined/${slug || "cluster"}-${hash}`;
+}
+function validatePromotedLessonFrontmatter(ref, frontmatter) {
+    const descCheck = isValidDescription(frontmatter.description, ref);
+    if (!descCheck.ok)
+        return { ok: false, reason: descCheck.reason };
+    const whenToUseCheck = isValidWhenToUse(frontmatter.when_to_use, ref);
+    if (!whenToUseCheck.ok)
+        return { ok: false, reason: whenToUseCheck.reason };
+    if (typeof frontmatter.description === "string" &&
+        typeof frontmatter.when_to_use === "string" &&
+        frontmatter.description.trim().toLowerCase() === frontmatter.when_to_use.trim().toLowerCase()) {
+        return { ok: false, reason: "description and when_to_use are identical" };
+    }
+    return { ok: true };
+}
+/**
+ * The membership fingerprint of a cluster: its member entryKeys sorted and
+ * joined. Single source of truth shared by {@link deriveRecombineLessonRef}'s
+ * hash and the `recombine_hypotheses.member_key` column, so the table key and
+ * the ref hash always derive from the SAME member set. Adding/removing one
+ * memory yields a different fingerprint → a different ref → a fresh row (the
+ * old streak is correctly NOT inherited).
+ */
+export function recombineMemberKey(cluster) {
+    return cluster.members
+        .map((m) => m.entryKey)
+        .sort()
+        .join("|");
+}
+/** Parse the raw LLM output into a generalization, or `null` for the justified-null path. */
+function parseGeneralization(raw) {
+    if (raw === null)
+        return null;
+    const trimmed = raw.trim();
+    if (!trimmed || trimmed.toLowerCase() === "null")
+        return null;
+    const parsed = parseEmbeddedJsonResponse(trimmed);
+    if (parsed === undefined || parsed === null)
+        return null;
+    if (typeof parsed !== "object")
+        return null;
+    const obj = parsed;
+    const description = typeof obj.description === "string" ? obj.description : "";
+    const body = typeof obj.body === "string" ? obj.body : "";
+    const when_to_use = typeof obj.when_to_use === "string" ? obj.when_to_use : undefined;
+    // An empty object / all-empty fields is treated as a justified null.
+    if (!description && !body)
+        return null;
+    return { description, body, ...(when_to_use ? { when_to_use } : {}) };
+}
+// ── Main entry point ───────────────────────────────────────────────────────────
+export async function akmRecombine(opts) {
+    const startMs = Date.now();
+    const config = opts.config ?? loadConfig();
+    const stashDir = opts.stashDir ?? resolveStashDir();
+    const sourceRun = opts.sourceRun ?? `recombine-${startMs}`;
+    const eligibilitySource = opts.eligibilitySource ?? "recombine";
+    const minClusterSize = opts.minClusterSize ?? DEFAULT_MIN_CLUSTER_SIZE;
+    const maxClustersPerRun = opts.maxClustersPerRun ?? DEFAULT_MAX_CLUSTERS_PER_RUN;
+    const relatednessSource = opts.relatednessSource ?? DEFAULT_RELATEDNESS_SOURCE;
+    const confirmThreshold = opts.confirmThreshold ?? DEFAULT_CONFIRM_THRESHOLD;
+    const warnings = [];
+    const finish = (over) => ({
+        schemaVersion: 1,
+        ok: true,
+        clustersFormed: 0,
+        proposalsEmitted: 0,
+        lessonsPromoted: 0,
+        nullsReturned: 0,
+        durationMs: Date.now() - startMs,
+        warnings,
+        ...over,
+    });
+    // Budget guard: an already-aborted signal short-circuits before any LLM call.
+    if (opts.signal?.aborted) {
+        return finish({ ok: false, warnings: [...warnings, "aborted-before-start"] });
+    }
+    // Load the memory pool + (optionally) graph entities from the index.
+    let entries = [];
+    let entityByEntryId;
+    let db;
+    try {
+        db = openExistingDatabase();
+        entries = getAllEntries(db, "memory");
+        if (relatednessSource === "graph" || relatednessSource === "both") {
+            try {
+                entityByEntryId = getEntitiesByEntryIds(db, entries.map((e) => e.id));
+            }
+            catch {
+                // Fail open to tag relatedness.
+                entityByEntryId = undefined;
+            }
+        }
+    }
+    catch (e) {
+        warnings.push(`recombine: failed to open index — ${String(e)}`);
+        return finish({ ok: false });
+    }
+    finally {
+        if (db)
+            closeDatabase(db);
+    }
+    // #658 — `rankedClusters` is the FULL set that re-formed this run (ranked,
+    // pre-cap); `clusters` is the processed top-`maxClustersPerRun` slice. The
+    // decay sweep below uses the full set so a cap-displaced (but present)
+    // cluster spares its hypothesis from reset.
+    const rankedClusters = buildRelatednessClusters(entries, {
+        minClusterSize,
+        relatednessSource,
+        ...(entityByEntryId ? { entityByEntryId } : {}),
+        ...(opts.maxClusterSize != null ? { maxClusterSize: opts.maxClusterSize } : {}),
+        ...(opts.excludeTags ? { excludeTags: opts.excludeTags } : {}),
+        ...(opts.excludeEntities ? { excludeEntities: opts.excludeEntities } : {}),
+    });
+    const clusters = selectClustersForRun(rankedClusters, maxClustersPerRun);
+    let clustersFormed = 0;
+    let proposalsEmitted = 0;
+    let lessonsPromoted = 0;
+    let nullsReturned = 0;
+    const llmFn = opts.recombineLlmFn ??
+        resolveImproveLlmFn(config, {
+            processKey: "recombine",
+            systemPrompt: RECOMBINE_SYSTEM_PROMPT,
+            tag: "[recombine]",
+            signal: opts.signal,
+            activeProfile: opts.improveProfile,
+        });
+    if (!llmFn) {
+        warnings.push("recombine: no LLM configured — skipping");
+        return finish({ clustersFormed: 0 });
+    }
+    // Refs re-induced (defensible generalization passed the quality gate) THIS
+    // run — everything else is decayed after the loop.
+    const seenThisRun = new Set();
+    // Recombine output is knowledge/lesson (non-wiki) → stash authoring
+    // standards. Resolved ONCE per run and passed to each cluster prompt.
+    const standardsContext = resolveStashStandards(stashDir);
+    // #625 — open the confirmation-count store once per run via the ctx seam,
+    // reusing a long-lived ctx.db handle when the caller provided one (mirrors
+    // proposals.ts). Only handles WE opened are closed by the seam.
+    await withStateDbAsync(async (stateDb) => {
+        for (const cluster of clusters) {
+            if (opts.signal?.aborted) {
+                warnings.push("aborted-mid-run");
+                break;
+            }
+            clustersFormed += 1;
+            // #9 — promotion is terminal. If this cluster Jaccard-matches a
+            // hypothesis that was ALREADY promoted, generating again can at best
+            // re-queue a redundant `type: hypothesis` for a settled ref (promotion
+            // gates on !alreadyPromoted below), so the LLM output is wasted. Skip the
+            // per-cluster call entirely. The decay sweep still spares the row — it
+            // matches a present cluster via `presentClusters` (independent of
+            // `seenThisRun`), so the promotion record is preserved.
+            if (stateDb) {
+                const promotedMatch = findMatchingRecombineHypothesis(stateDb, {
+                    signature: cluster.signature,
+                    memberKey: recombineMemberKey(cluster),
+                    minOverlap: DEFAULT_RECOMBINE_OVERLAP,
+                });
+                if (promotedMatch?.hypothesis_ref &&
+                    getRecombineHypothesis(stateDb, promotedMatch.hypothesis_ref)?.promoted_at != null) {
+                    appendEvent({
+                        eventType: "recombine_invoked",
+                        ref: promotedMatch.hypothesis_ref,
+                        metadata: {
+                            signal: cluster.signature,
+                            memberCount: cluster.members.length,
+                            outcome: "skipped_promoted",
+                            sourceRun,
+                        },
+                    }, opts.ctx);
+                    continue;
+                }
+            }
+            const prompt = buildClusterPrompt(cluster, standardsContext);
+            const raw = await llmFn(prompt);
+            const generalization = parseGeneralization(raw);
+            if (!generalization) {
+                nullsReturned += 1;
+                appendEvent({
+                    eventType: "recombine_invoked",
+                    ref: deriveRecombineLessonRef(cluster),
+                    metadata: {
+                        signal: cluster.signature,
+                        memberCount: cluster.members.length,
+                        outcome: "null_returned",
+                        sourceRun,
+                    },
+                }, opts.ctx);
+                continue;
+            }
+            // #633 — the confirmation identity is decoupled from the EXACT member
+            // set. We first look for an existing pending hypothesis row under the
+            // SAME signature whose membership overlaps this cluster (Jaccard >=
+            // threshold) and, if found, REUSE that row's stable ref so a
+            // drifting-but-overlapping cluster keeps accumulating its streak under one
+            // row instead of spawning a fresh row (count=1) every run. With no match
+            // (first induction, or membership drifted past the overlap floor) we fall
+            // back to the deterministic member-set ref exactly as before.
+            const memberKey = recombineMemberKey(cluster);
+            const derivedRef = deriveRecombineLessonRef(cluster);
+            const matchedRow = stateDb
+                ? findMatchingRecombineHypothesis(stateDb, {
+                    signature: cluster.signature,
+                    memberKey,
+                    minOverlap: DEFAULT_RECOMBINE_OVERLAP,
+                })
+                : undefined;
+            const lessonRef = matchedRow?.hypothesis_ref ?? derivedRef;
+            const sourceRefs = cluster.members.map((m) => `memory:${m.entry.name}`);
+            const priorRow = stateDb ? getRecombineHypothesis(stateDb, lessonRef) : undefined;
+            const alreadyPromoted = priorRow?.promoted_at != null;
+            const nextCount = stateDb == null
+                ? 0
+                : priorRow == null
+                    ? 1
+                    : priorRow.last_run === sourceRun
+                        ? priorRow.consecutive_count
+                        : priorRow.consecutive_count + 1;
+            // Quality gate (always-run): the frontmatter description must be present
+            // and non-truncated. Promotion adds the full lesson frontmatter check so
+            // `when_to_use` never bypasses validation on the promote=true path.
+            const fmCheck = validateProposalFrontmatter({ description: generalization.description });
+            if (!fmCheck.ok) {
+                appendEvent({
+                    eventType: "recombine_invoked",
+                    ref: lessonRef,
+                    metadata: {
+                        signal: cluster.signature,
+                        memberCount: cluster.members.length,
+                        outcome: "quality_rejected",
+                        reason: fmCheck.reason,
+                        sourceRun,
+                    },
+                }, opts.ctx);
+                continue;
+            }
+            const promote = stateDb != null && !alreadyPromoted && nextCount >= confirmThreshold;
+            if (promote) {
+                const lessonFmCheck = validatePromotedLessonFrontmatter(lessonRef, {
+                    description: generalization.description,
+                    when_to_use: generalization.when_to_use,
+                });
+                if (!lessonFmCheck.ok) {
+                    appendEvent({
+                        eventType: "recombine_invoked",
+                        ref: lessonRef,
+                        metadata: {
+                            signal: cluster.signature,
+                            memberCount: cluster.members.length,
+                            outcome: "quality_rejected",
+                            reason: lessonFmCheck.reason,
+                            sourceRun,
+                        },
+                    }, opts.ctx);
+                    continue;
+                }
+            }
+            // A defensible generalization was produced this run — record it so it is
+            // NOT decayed by the unseen sweep below.
+            seenThisRun.add(lessonRef);
+            // #625/#633 — record the re-induction only AFTER the quality gate passed.
+            // Quality-rejected outputs must not advance the confirmation streak.
+            const count = stateDb
+                ? recordRecombineInduction(stateDb, {
+                    hypothesisRef: lessonRef,
+                    signature: cluster.signature,
+                    memberKey,
+                    seenAt: new Date().toISOString(),
+                    run: sourceRun,
+                })
+                : 0;
+            // Promote to a `type: lesson` proposal when the confirmation streak
+            // reaches the threshold AND the hypothesis has not already been promoted.
+            const proposalType = promote ? "lesson" : "hypothesis";
+            const frontmatter = {
+                type: proposalType,
+                description: generalization.description,
+                ...(generalization.when_to_use ? { when_to_use: generalization.when_to_use } : {}),
+                source_refs: sourceRefs,
+            };
+            const content = assembleContent(frontmatter, generalization.body);
+            if (promote && stateDb) {
+                // Supersede the prior pending `type: hypothesis` proposal for this ref so
+                // the queue never shows two proposals for one ref. The promoted lesson
+                // proposal has different content (type changed), so content-hash dedup
+                // would otherwise let both co-exist.
+                for (const stale of listProposals(stashDir, { status: "pending", ref: lessonRef }, opts.ctx)) {
+                    if (stale.source === "recombine") {
+                        archiveProposal(stashDir, stale.id, "rejected", "superseded by recombine lesson promotion", opts.ctx);
+                    }
+                }
+            }
+            const proposalResult = createProposal(stashDir, {
+                ref: lessonRef,
+                source: "recombine",
+                sourceRun,
+                payload: { content, frontmatter },
+                eligibilitySource,
+                // The promotion is a distinct asset (lesson) for the same ref; force
+                // past the duplicate-pending guard (the stale hypothesis was just
+                // superseded, but force keeps the path robust to ordering).
+                ...(promote ? { force: true } : {}),
+            }, opts.ctx);
+            if (isProposalSkipped(proposalResult)) {
+                appendEvent({
+                    eventType: "recombine_invoked",
+                    ref: lessonRef,
+                    metadata: {
+                        signal: cluster.signature,
+                        memberCount: cluster.members.length,
+                        outcome: "skipped",
+                        skipReason: proposalResult.reason,
+                        sourceRun,
+                    },
+                }, opts.ctx);
+                continue;
+            }
+            if (promote && stateDb) {
+                markRecombineHypothesisPromoted(stateDb, lessonRef, new Date().toISOString());
+                lessonsPromoted += 1;
+                appendEvent({
+                    eventType: "recombine_invoked",
+                    ref: lessonRef,
+                    metadata: {
+                        signal: cluster.signature,
+                        memberCount: cluster.members.length,
+                        outcome: "promoted",
+                        proposalId: proposalResult.id,
+                        confirmationCount: count,
+                        sourceRun,
+                    },
+                }, opts.ctx);
+            }
+            else {
+                proposalsEmitted += 1;
+                appendEvent({
+                    eventType: "recombine_invoked",
+                    ref: lessonRef,
+                    metadata: {
+                        signal: cluster.signature,
+                        memberCount: cluster.members.length,
+                        outcome: "queued",
+                        proposalId: proposalResult.id,
+                        confirmationCount: count,
+                        sourceRun,
+                    },
+                }, opts.ctx);
+            }
+        }
+        // #625 — decay hypotheses NOT re-induced this run (reset their consecutive
+        // streak) so confirmation is per-consecutive-run and conservative (AC4).
+        // #658 — but a hypothesis whose cluster genuinely re-formed this run and was
+        // merely cap-displaced (outside the top-`maxClustersPerRun` slice) must NOT
+        // be decayed — that is a scheduling miss, not a substance miss. We pass
+        // EVERY cluster that formed this run (the full pre-cap `rankedClusters`) as
+        // `presentClusters`; decay spares any row that Jaccard-matches a present
+        // cluster under the SAME overlap rule used for re-induction. Only rows with
+        // no matching current cluster (the corpus stopped supporting them) decay.
+        if (stateDb) {
+            const presentClusters = rankedClusters.map((c) => ({
+                signature: c.signature,
+                memberKey: recombineMemberKey(c),
+            }));
+            const decayedCount = decayUnseenRecombineHypotheses(stateDb, sourceRun, [...seenThisRun], {
+                presentClusters,
+                minOverlap: DEFAULT_RECOMBINE_OVERLAP,
+            });
+            if (decayedCount > 0) {
+                appendEvent({
+                    eventType: "recombine_invoked",
+                    metadata: { outcome: "decayed", decayedCount, sourceRun },
+                }, opts.ctx);
+            }
+        }
+    }, { path: opts.ctx?.dbPath, borrowed: opts.ctx?.db });
+    return finish({ clustersFormed, proposalsEmitted, lessonsPromoted, nullsReturned });
+}
+/** Serialize frontmatter + body into a markdown asset string. */
+function assembleContent(frontmatter, body) {
+    const fmLines = Object.entries(frontmatter)
+        .map(([key, value]) => Array.isArray(value)
+        ? `${key}: [${value.map((v) => JSON.stringify(v)).join(", ")}]`
+        : `${key}: ${typeof value === "string" ? value : JSON.stringify(value)}`)
+        .join("\n");
+    return assembleAssetFromString(fmLines, body);
+}