@gamaze/hicortex 0.20.3 → 0.20.5

package/dist/dedup.d.ts

@@ -1,52 +1,101 @@
 /**
- * `hicortex dedup` — cluster + merge near-duplicate memories (issue #100).
+ * `hicortex dedup` + the nightly deterministic merge zone (issues #100, #392).
  *
  * Corpus-quality companion to `hicortex relink`/`classify-domains`: instead of
- * discovering NEW structure, this command collapses memories that are
- * near-identical (top-10 KNN cosine >= dedupMergeThreshold, default 0.92,
- * union-find clustered — same math as the #191 D1 duplicate-rate audit; see
- * cluster.ts). Default is a DRY RUN: report only, zero writes. `--apply`
- * executes the merge.
+ * discovering NEW structure, this collapses memories that are near-identical
+ * (top-10 KNN cosine >= threshold, default 0.92, union-find clustered — same
+ * math as the #191 D1 duplicate-rate audit; see cluster.ts). Two surfaces,
+ * ONE core:
  *
- * Per cluster:
+ *  - `runDedup` — the manual CLI. Default is a DRY RUN: report only, zero
+ *    writes. `--apply` executes the merge. Threshold resolution:
+ *    `--threshold` > config `dedupAutoMergeThreshold` > legacy config
+ *    `dedupMergeThreshold` > 0.92.
+ *  - `runDeterministicMergeZone` (#392) — the nightly's LLM-free merge zone:
+ *    pairs at/above the ceiling merge deterministically, ZERO LLM calls, under
+ *    its own pacing cap (`dedupNightlyMaxMerges`). Called from the
+ *    reconsolidation stage (and from the quiet-night skip path in
+ *    consolidate.ts) so one stage report covers all resolution work.
+ *
+ * Per cluster (shared `planDedup`/`mergeCluster` core — no forks):
  *   - Canonical = highest access_count (tie: oldest created_at, then
  *     lexicographically smallest id — fully deterministic for audit).
  *   - Losers' links are re-pointed onto the canonical (a link that would
  *     become a self-link, or one whose (canonical, target) ordered pair
  *     ALREADY holds an edge, is skipped rather than overwritten — see
- *     planLinkRepoints for why `relationship` cannot be part of that guard).
+ *     planLinkRepoints for why `relationship` cannot be part of that guard);
+ *     the losers' own link rows are then deleted (previously cascade-deleted
+ *     with the row).
  *   - canonical.access_count/shown_count = summed across the cluster;
  *     last_accessed = max; base_strength = max.
  *   - Tags are UNIONED onto the canonical (weights NULL — the next nightly's
  *     reconsolidation pass recomputes weights and the derived primary from
- *     the merged tag set).
- *   - A `dedup_log` row is written per loser BEFORE it is deleted — audit
- *     trail AND the safety net /distill consults (mcp-server.ts) so a
- *     deleted loser's `source_session` marker still blocks a re-ingest.
- *   - Losers are deleted via storage.deleteMemory (cascades links/tags/
- *     vectors/FTS).
+ *     the merged tag set); the losers' tag rows are cleared and their domain
+ *     set NULL — a loser must not count in moduleIndex/tag recomputes.
+ *   - A `dedup_log` row is written per loser — audit trail AND the safety net
+ *     /distill consults (mcp-server.ts) so an absorbed loser's
+ *     `source_session` marker still blocks a re-ingest.
+ *   - Losers are ABSORBED (storage.absorbMemory), not deleted (#392): status
+ *     'absorbed', vector + FTS rows dropped, plain row retained — invisible
+ *     to recall, fetchable by id as evidence. Same vocabulary as the
+ *     reconsolidation rewrite path. Merges are NOT history-rollback-able —
+ *     dedup_log (loser_id → canonical_id) + the retained loser row is the
+ *     record.
  *
- * A cluster whose members disagree on project, privacy, or source_agent is
- * SKIPPED entirely and listed for manual review — no --force in this release.
+ * A cluster whose members disagree on project or source_agent is SKIPPED
+ * entirely and listed for manual review — no --force in this release.
  *
- * Safety rails on --apply:
+ * Safety rails when applying (CLI and zone alike):
  *   - A full DB backup (SQLite backup API) is taken FIRST, to
- *     ~/.hicortex/backups/pre-dedup-<ISO>.db. Abort (no merges attempted) if
- *     the backup fails.
+ *     <state>/backups/pre-dedup-<ISO>.db, pruned to `backupRetention` newest
+ *     (pattern-scoped: full `hicortex-*.tar.gz` artifacts keep their own
+ *     count). The CLI aborts (no merges attempted) if the backup fails; the
+ *     nightly zone is fail-soft (backup_failed flag, zero merges).
  *   - The existing single-flight capture lock (capture.ts) is held for the
  *     duration of the merge so a concurrent nightly/capture run can't race
- *     the dedup_log bookkeeping the merge relies on.
+ *     the dedup_log bookkeeping the merge relies on. The CLI fails fast on a
+ *     busy lock; the zone reports lock_busy and merges nothing.
  *
  * Server-mode only (needs the local DB), like relink/classify-domains.
  */
 import type Database from "better-sqlite3";
 import { type ClusterMetadataMismatch } from "./cluster.js";
 import { acquireCaptureLock } from "./capture.js";
+import type { DeterministicMergeZoneReport } from "./types.js";
 /**
  * Default merge threshold. Measured on the #191 mechanical audit corpus:
  * 89 clusters / 110 excess rows at 0.92 (data/audit-20260729/eval-report.md).
+ * #392: also the default `dedupAutoMergeThreshold` — the deterministic/LLM
+ * boundary of the unified resolution pass.
  */
 export declare const DEFAULT_DEDUP_MERGE_THRESHOLD = 0.92;
+/**
+ * Default pacing cap on merge OPERATIONS per nightly run (#392): the zone's
+ * clusters and the stage's judged pair merges count against ONE cap. Bounds a
+ * misbehaving-distiller burst; a large backlog drains over a few nights.
+ * `0` disables the merge machinery entirely.
+ */
+export declare const DEFAULT_DEDUP_NIGHTLY_MAX_MERGES = 250;
+/**
+ * Row shape read from `memories` for merge decisions — a superset of the
+ * fields the Memory type declares (shown_count isn't on that interface yet).
+ * `status` rides along so judged-pair merges can defensively drop rows that
+ * were absorbed between verdict and apply.
+ */
+interface DedupMemberRow {
+    id: string;
+    content: string;
+    access_count: number;
+    shown_count: number | null;
+    last_accessed: string | null;
+    base_strength: number;
+    created_at: string;
+    project: string | null;
+    privacy: string | null;
+    source_agent: string;
+    source_session: string | null;
+    status: string | null;
+}
 export interface DedupClusterPlan {
     size: number;
     canonicalId: string;
@@ -94,8 +143,8 @@ export interface DedupReport {
     linksSkippedExisting: number;
     /** --apply only: clusters actually merged. */
     merged?: number;
-    /** --apply only: loser rows deleted. */
-    losersDeleted?: number;
+    /** --apply only: loser rows absorbed (hidden from recall, kept as evidence). */
+    losersAbsorbed?: number;
     /** --apply only: clusters that errored mid-merge (rolled back; left for a re-run). */
     failedClusters?: number;
     /** --apply only: path to the pre-merge backup. */
@@ -104,7 +153,7 @@ export interface DedupReport {
 export interface DedupOptions {
     /** Execute the merge. Default false = dry run (report only, zero writes). */
     apply?: boolean;
-    /** Override config.dedupMergeThreshold for one run. */
+    /** Override the configured threshold for one run. */
     threshold?: number;
     /** DB path override (tests / manual snapshot verification). Defaults to resolveDbPath(). */
     dbPath?: string;
@@ -116,7 +165,7 @@ export interface DedupOptions {
     acquireLock?: typeof acquireCaptureLock;
     /**
      * Test-only failure injection: called once per cluster merge, after the
-     * link/tag/counter writes but before the audit-log + delete step. Throwing
+     * link/tag/counter writes but before the audit-log + absorb step. Throwing
      * here proves a mid-merge error rolls the WHOLE cluster's writes back
      * (better-sqlite3 transaction semantics) rather than leaving a half-merged
      * cluster. Never set in production.
@@ -135,10 +184,93 @@ export interface LinkRepointPlan {
     skippedSelfLink: number;
     skippedExisting: number;
 }
+/** One cluster's execution plan from `planDedup` — canonical, losers, members. */
+export interface DedupMergePlan {
+    canonical: DedupMemberRow;
+    losers: DedupMemberRow[];
+    /** All member rows, oldest first (CLI preview lines derive from this). */
+    membersOldestFirst: DedupMemberRow[];
+}
+export interface PlanDedupResult {
+    /** Every cluster found at the threshold (mergeable + mismatch-skipped). */
+    clusterCount: number;
+    /** Clusters that passed the metadata rails, in discovery order. */
+    mergePlans: DedupMergePlan[];
+    mismatchSkipped: DedupMismatchCluster[];
+}
+/**
+ * Discovery + merge planning at a cosine threshold (read-only — no writes).
+ * The ONE clustering core shared by the manual CLI (`runDedup`) and the
+ * nightly deterministic merge zone (`runDeterministicMergeZone`): KNN edges
+ * (k=10) → union-find clusters → member load → metadata-rail classification →
+ * canonical pick. Never forked.
+ */
+export declare function planDedup(db: Database.Database, threshold: number): PlanDedupResult;
+export type MergeMemoryIdsResult = {
+    ok: true;
+    canonicalId: string;
+    loserIds: string[];
+    linksRepointed: number;
+} | {
+    ok: false;
+    reason: "metadata_mismatch" | "no_members";
+};
+/**
+ * Merge an explicit set of memories (the judged-pair phase of #392: the
+ * reconsolidation stage queues verdict-confirmed pairs and applies them
+ * through THIS function so the merge math stays single-definition). Loads the
+ * LIVE rows at apply time — members that vanished or were absorbed between
+ * verdict and apply are dropped defensively; a metadata disagreement refuses
+ * the merge (both memories stay live). One transaction for the whole set.
+ */
+export declare function mergeMemoryIds(db: Database.Database, ids: string[]): MergeMemoryIdsResult;
+/**
+ * Take a pre-merge DB backup to <stateDir>/backups/pre-dedup-<ISO>.db and
+ * prune older pre-dedup backups to `backupRetention` (config, default 7;
+ * pattern-scoped so full `hicortex-*.tar.gz` artifacts keep their own,
+ * independent retention count). THROWS on failure — the callers own the
+ * policy: the CLI aborts, the nightly zone is fail-soft. Returns the path.
+ */
+export declare function takePreDedupBackup(db: Database.Database, stateDir: string, config?: Record<string, unknown> | null): Promise<string>;
+export interface DeterministicMergeZoneOptions {
+    /** State dir (lock + backup + band-stats persistence). Defaults to ~/.hicortex. */
+    stateDir?: string;
+    /** Cosine ceiling; validated (0,1] → DEFAULT_DEDUP_MERGE_THRESHOLD. */
+    threshold?: number;
+    /**
+     * Pacing cap on merge operations this run; validated >= 0 →
+     * DEFAULT_DEDUP_NIGHTLY_MAX_MERGES. `0` disables the machinery entirely
+     * (discovery skipped, zeroed report).
+     */
+    maxMerges?: number;
+    /** Discovery + bounded preview only — zero writes, no lock, no backup. */
+    dryRun?: boolean;
+    /** Config override (backupRetention) — defaults to reading stateDir/config.json. */
+    config?: Record<string, unknown> | null;
+    /** Capture-lock acquirer override (tests). Defaults to the real capture.ts lock. */
+    acquireLock?: typeof acquireCaptureLock;
+}
+/**
+ * The >= dedupAutoMergeThreshold band of the unified resolution pass (#392):
+ * planDedup discovery + per-cluster mergeCluster — LLM-free, budget-free, so
+ * an LLM-less night still drains duplicates. Its own short capture-lock
+ * window and pre-merge backup; fail-soft on a busy lock (lock_busy) and on a
+ * backup failure (backup_failed) — zero merges either way, never a throw.
+ *
+ * Also persists the deterministic band's cumulative statistics to state.json
+ * `resolutionBandStats` (label `>=threshold`; losers count as merge verdicts
+ * at confidence 1.0, mismatch clusters as metadata_skipped) — skipped
+ * entirely on dry-run. Called from the reconsolidation stage (main path) and
+ * from runConsolidation's quiet-night skip path — exactly one of the two per
+ * run.
+ */
+export declare function runDeterministicMergeZone(db: Database.Database, opts?: DeterministicMergeZoneOptions): Promise<DeterministicMergeZoneReport>;
 /**
  * Run `hicortex dedup`. Dry run by default (options.apply falsy) — discovery
  * + merge planning only, zero writes. `options.apply` executes: backup, then
- * one transaction per cluster.
+ * one transaction per cluster. Fails fast (throws) on a busy capture lock or
+ * a failed backup — a deliberate manual command should be retried by the
+ * operator, not silently deferred (the nightly zone is the fail-soft twin).
  */
 export declare function runDedup(options?: DedupOptions): Promise<DedupReport>;
 /** Escape SQL LIKE wildcards — session ids (e.g. Hermes) can contain "_"/"%". */