@gamaze/hicortex 0.14.4 → 0.15.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -0,0 +1,275 @@
1
+ #!/usr/bin/env node
2
+ "use strict";
3
+ /**
4
+ * #191 mechanical audit baseline — orchestrates every section (D1 duplicate
5
+ * rate, D4 decay/prune/no-fit + #192 adoption, D6 link-graph health, the
6
+ * reflection census) into one markdown report.
7
+ *
8
+ * Read-only end to end: opens the snapshot via eval-db.ts (`openSnapshot`,
9
+ * never `initDb()`), never writes to the DB. Not wired into `cli.ts` — this
10
+ * is an internal measurement tool, run via the `eval` npm script:
11
+ *
12
+ * npm run eval -- <snapshot.db> [report.md]
13
+ *
14
+ * `state.json` is expected alongside the snapshot DB (same directory) for
15
+ * the `domainCursor`/`relinkCursor` watermarks. Report path defaults to
16
+ * `eval-report.md` next to the DB.
17
+ */
18
+ Object.defineProperty(exports, "__esModule", { value: true });
19
+ const node_path_1 = require("node:path");
20
+ const node_fs_1 = require("node:fs");
21
+ const eval_db_js_1 = require("./eval-db.js");
22
+ const dups_js_1 = require("./dups.js");
23
+ const decay_eval_js_1 = require("./decay-eval.js");
24
+ const graph_eval_js_1 = require("./graph-eval.js");
25
+ const reflection_census_js_1 = require("./reflection-census.js");
26
+ const retrieval_js_1 = require("../retrieval.js");
27
+ // ---------------------------------------------------------------------------
28
+ // Formatting helpers
29
+ // ---------------------------------------------------------------------------
30
+ function pct(n) {
31
+ return `${(n * 100).toFixed(1)}%`;
32
+ }
33
+ function round(n, places = 4) {
34
+ const f = 10 ** places;
35
+ return Math.round(n * f) / f;
36
+ }
37
+ /** Extract the leading number from a bucket label (e.g. "[0.2, 0.3)" -> 0.2) so buckets render in ascending order. */
38
+ function bucketSortKey(label) {
39
+ const match = label.match(/-?\d+(\.\d+)?/);
40
+ return match ? Number(match[0]) : Number.POSITIVE_INFINITY;
41
+ }
42
+ function histTable(h) {
43
+ const entries = Object.entries(h).sort((a, b) => bucketSortKey(a[0]) - bucketSortKey(b[0]));
44
+ if (entries.length === 0)
45
+ return "_(no data)_\n";
46
+ return entries.map(([bucket, count]) => `| ${bucket} | ${count} |`).join("\n") + "\n";
47
+ }
48
+ // ---------------------------------------------------------------------------
49
+ // Report sections
50
+ // ---------------------------------------------------------------------------
51
+ function renderDups(d) {
52
+ const lines = [];
53
+ lines.push("## D1 — Duplicate rate\n");
54
+ lines.push(`Top-${d.knnK} vector KNN per memory (${d.totalMemories} memories), pairs kept at cosine >= 0.90 and clustered per threshold (union-find). "Excess" = rows that would disappear if every cluster merged to one.\n`);
55
+ lines.push("| Threshold | Clusters | Excess (rows) |");
56
+ lines.push("|---|---|---|");
57
+ for (const t of d.thresholds) {
58
+ lines.push(`| ${t.threshold} | ${t.clusterCount} | ${t.excess} |`);
59
+ }
60
+ lines.push("");
61
+ lines.push(`**Pair attribution** (over all ${d.pairAttribution.totalPairs} pairs at/above the lowest threshold, 0.90): ` +
62
+ `${d.pairAttribution.recoveryReingest} recovery/re-ingest-suspect (${pct(d.pairAttribution.totalPairs > 0 ? d.pairAttribution.recoveryReingest / d.pairAttribution.totalPairs : 0)}), ${d.pairAttribution.organic} organic (${pct(d.pairAttribution.totalPairs > 0 ? d.pairAttribution.organic / d.pairAttribution.totalPairs : 0)}).\n`);
63
+ lines.push(`### Top ${d.topClusters.length} clusters (0.90 threshold, largest first)\n`);
64
+ d.topClusters.forEach((cluster, i) => {
65
+ const mismatch = cluster.metadataMismatch;
66
+ const mismatchFlags = [
67
+ mismatch.projectMismatch ? "project" : null,
68
+ mismatch.privacyMismatch ? "privacy" : null,
69
+ mismatch.sourceAgentMismatch ? "source_agent" : null,
70
+ ].filter(Boolean);
71
+ lines.push(`**Cluster ${i + 1}** — size ${cluster.size}, attribution: ${cluster.attribution.recoveryReingest} recovery-pair(s) / ${cluster.attribution.organic} organic-pair(s)` +
72
+ (mismatchFlags.length > 0 ? `, metadata mismatch: ${mismatchFlags.join(", ")}` : ", metadata consistent"));
73
+ for (const m of cluster.members) {
74
+ lines.push(` - \`${m.id.slice(0, 8)}\` ${m.created_at} — ${m.preview.replace(/\n/g, " ")}`);
75
+ }
76
+ lines.push("");
77
+ });
78
+ return lines.join("\n");
79
+ }
80
+ function renderDecay(backlog, prune, structural, adoption) {
81
+ const lines = [];
82
+ lines.push("## D4 — Decay / Prune / No-fit lifecycle + #192 adoption\n");
83
+ lines.push("### Domain classification backlog\n");
84
+ lines.push(`${backlog.nullDomainTotal} memories have NULL domain (of ${backlog.maxRowid} max rowid). Partitioned by ` +
85
+ `\`domainCursor\` (${backlog.domainCursor ?? "unset — classify-domains never run"}), documented in ` +
86
+ `classify-domains.ts as the last fully-committed rowid:\n`);
87
+ lines.push(`- **Never-classified backlog** (rowid > cursor): ${backlog.neverClassifiedBacklog}`);
88
+ lines.push(`- **Classified but no fitting domain** (rowid <= cursor, no-fit path): ${backlog.classifiedButEmpty}\n`);
89
+ lines.push("### Prune dry-run (real production predicate: `stageDecayPrune`, `dryRun: true`)\n");
90
+ lines.push(`Decay clock configured to \`decayHalfLifeDays: ${prune.decayHalfLifeDaysUsed}\` (bedrock has no override — this is the shipped default) before running.\n`);
91
+ lines.push(`- Candidates (would prune today): **${prune.candidates}**`);
92
+ lines.push(`- Pruned: ${prune.pruned} (0 expected — dry run)`);
93
+ lines.push(`- Failed: ${prune.failed}\n`);
94
+ lines.push("### Structural strength — who can EVER prune\n");
95
+ lines.push(`A memory can only ever cross the prune floor (effective strength < 0.01) if its asymptote is below that floor: ` +
96
+ `\`base_strength < sqrt(0.1) ≈ ${round(structural.everPrunableCeiling, 4)}\` (floor = base_strength × importance × 0.1, ` +
97
+ `importance defaults to base_strength — independent of access/link hardening, which only slows the approach).\n`);
98
+ lines.push(`**Ever-prunable: ${structural.everPrunableCount} / ${structural.totalMemories} (${pct(structural.totalMemories > 0 ? structural.everPrunableCount / structural.totalMemories : 0)})** — everyone else has a strength floor that never drops low enough to prune, no matter how long it goes unaccessed.\n`);
99
+ for (const [label, hist] of [
100
+ ["Effective strength now", structural.effectiveStrengthNowHistogram],
101
+ ["Simulated +180 days (no further access)", structural.effectiveStrengthAt180dHistogram],
102
+ ["Simulated +365 days (no further access)", structural.effectiveStrengthAt365dHistogram],
103
+ ["Asymptotic floor (never, base_strength² × 0.1)", structural.effectiveStrengthNeverHistogram],
104
+ ]) {
105
+ lines.push(`**${label}**\n`);
106
+ lines.push("| Bucket | Count |");
107
+ lines.push("|---|---|");
108
+ lines.push(histTable(hist));
109
+ }
110
+ lines.push("### Adoption (#192 shown_count / access_count)\n");
111
+ lines.push("`shown_count` (migration v8) was deployed ~2 days before this snapshot — near-zero shown counts reflect the " +
112
+ "feature's youth as much as its effectiveness; read this section as a baseline to compare against, not a verdict.\n");
113
+ lines.push(`- Total shown (sum): ${adoption.totalShown}, total access (sum): ${adoption.totalAccess}`);
114
+ lines.push(`- Uses per showing (overall): ${adoption.usesPerShowingOverall !== null ? round(adoption.usesPerShowingOverall) : "n/a (nothing shown yet)"}`);
115
+ lines.push(`- Cold (shown=0 AND access=0): ${adoption.coldShare.coldCount} / ${adoption.coldShare.total} (${pct(adoption.coldShare.share)})\n`);
116
+ lines.push("**shown_count distribution**\n\n| Bucket | Count |\n|---|---|\n" + histTable(adoption.shownCountHistogram));
117
+ lines.push("**access_count distribution**\n\n| Bucket | Count |\n|---|---|\n" + histTable(adoption.accessCountHistogram));
118
+ lines.push("**Uses per showing by source_agent** (agents with at least one showing)\n");
119
+ lines.push("| source_agent | shown | access | ratio |");
120
+ lines.push("|---|---|---|---|");
121
+ const byAgent = Object.entries(adoption.usesPerShowingBySourceAgent)
122
+ .filter(([, v]) => v.shown > 0 || v.access > 0)
123
+ .sort((a, b) => b[1].shown - a[1].shown);
124
+ for (const [agent, v] of byAgent) {
125
+ lines.push(`| ${agent} | ${v.shown} | ${v.access} | ${v.ratio !== null ? round(v.ratio) : "n/a"} |`);
126
+ }
127
+ lines.push("");
128
+ lines.push("**Exposure age profile** — are new memories getting shown?\n");
129
+ lines.push("| Age bucket | Total | Ever shown | Share shown |");
130
+ lines.push("|---|---|---|---|");
131
+ for (const b of adoption.exposureAgeProfile) {
132
+ lines.push(`| ${b.bucket} | ${b.total} | ${b.everShown} | ${pct(b.shareShown)} |`);
133
+ }
134
+ lines.push("");
135
+ return lines.join("\n");
136
+ }
137
+ function renderGraph(g) {
138
+ const lines = [];
139
+ lines.push("## D6 — Link-graph health\n");
140
+ lines.push(`Total links: **${g.totalLinks}**\n`);
141
+ lines.push("| Relationship | Count |");
142
+ lines.push("|---|---|");
143
+ for (const [rel, count] of Object.entries(g.byRelationship)) {
144
+ lines.push(`| ${rel} | ${count} |`);
145
+ }
146
+ lines.push("");
147
+ lines.push(`### Degree distribution (${g.degree.memoriesWithLinks} / ${g.degree.totalMemories} memories have at least one link)\n`);
148
+ lines.push("| Degree bucket | Count |");
149
+ lines.push("|---|---|");
150
+ lines.push(histTable(g.degree.degreeHistogram));
151
+ lines.push(`### Top ${g.degree.topHubs.length} hubs\n`);
152
+ lines.push("| id | degree | project | domain | preview |");
153
+ lines.push("|---|---|---|---|---|");
154
+ for (const h of g.degree.topHubs) {
155
+ lines.push(`| \`${h.id.slice(0, 8)}\` | ${h.degree} | ${h.project ?? "-"} | ${h.domain ?? "-"} | ${h.preview.replace(/\|/g, "/")} |`);
156
+ }
157
+ lines.push("");
158
+ lines.push(`### Cosine drift (stored \`memory_links.strength\` vs recomputed, ${g.drift.sampleSize}-link random sample` +
159
+ (g.drift.skippedMissingEmbedding > 0 ? `, ${g.drift.skippedMissingEmbedding} skipped for missing embeddings` : "") +
160
+ ")\n");
161
+ lines.push(`Mean |drift|: ${round(g.drift.meanAbsDrift, 8)}, max |drift|: ${round(g.drift.maxAbsDrift, 8)} ` +
162
+ "(float32 rounding-noise scale (~1e-7) counts as zero drift — strength was set from the same embeddings at link-creation time, so no staleness is expected).\n");
163
+ lines.push("| |drift| bucket | Count |");
164
+ lines.push("|---|---|");
165
+ lines.push(histTable(g.drift.driftHistogram));
166
+ lines.push(`### Dup-noise links (endpoints >= ${g.dupNoise.threshold} cosine, measured over ${g.dupNoise.measured} links with both embeddings)\n`);
167
+ lines.push(`${g.dupNoise.count} / ${g.dupNoise.measured} (${pct(g.dupNoise.share)}) of links connect near-duplicate memories.\n`);
168
+ lines.push(`### Relink migration partition (\`relinkCursor\`: ${g.relinkCursor ?? "unset — relink never run"})\n`);
169
+ lines.push("| Partition | Links | Dup-noise share |");
170
+ lines.push("|---|---|---|");
171
+ lines.push(`| Relinked (source rowid <= cursor) | ${g.relinkedPartition.linkCount} | ${pct(g.relinkedPartition.dupNoiseShare)} |`);
172
+ lines.push(`| Not yet relinked (source rowid > cursor) | ${g.notYetRelinkedPartition.linkCount} | ${pct(g.notYetRelinkedPartition.dupNoiseShare)} |`);
173
+ lines.push("");
174
+ return lines.join("\n");
175
+ }
176
+ function renderReflection(r) {
177
+ const lines = [];
178
+ lines.push("## Reflection census\n");
179
+ lines.push(`Lessons: ${r.totalLessons}, episodes: ${r.totalEpisodes}, yield ratio (lessons/episodes): ${r.yieldRatio !== null ? round(r.yieldRatio) : "n/a"}\n`);
180
+ lines.push("_Not a quality read — actionable-vs-noise sampling is deferred Phase-A grading work._\n");
181
+ if (r.lessonsByDate.length > 0) {
182
+ lines.push("| Date | Lessons |");
183
+ lines.push("|---|---|");
184
+ for (const row of r.lessonsByDate) {
185
+ lines.push(`| ${row.date} | ${row.count} |`);
186
+ }
187
+ lines.push("");
188
+ }
189
+ return lines.join("\n");
190
+ }
191
+ function renderPhaseB(dups, prune, structural, backlog, graph, adoption) {
192
+ const lines = [];
193
+ lines.push("## Phase-B decision inputs\n");
194
+ lines.push("**1. Merge-threshold curve (dup excess at each cosine)**\n");
195
+ for (const t of dups.thresholds) {
196
+ lines.push(`- ${t.threshold}: ${t.clusterCount} clusters, ${t.excess} excess rows`);
197
+ }
198
+ lines.push("");
199
+ lines.push("**2. Prune posture — inert by design or miscalibration?**\n");
200
+ lines.push(`- Dry-run candidates today: ${prune.candidates} (half-life ${prune.decayHalfLifeDaysUsed}d)`);
201
+ lines.push(`- Ever-prunable ceiling: ${structural.everPrunableCount} / ${structural.totalMemories} (${pct(structural.totalMemories > 0 ? structural.everPrunableCount / structural.totalMemories : 0)}) memories can EVER cross the floor regardless of age`);
202
+ lines.push(`- If the current trend holds, compare the now/+180d/+365d histograms above for how many additional memories drift into low-strength bands`);
203
+ lines.push("");
204
+ lines.push("**3. Classification backlog size**\n");
205
+ lines.push(`- ${backlog.neverClassifiedBacklog} never-classified (beyond domainCursor), ${backlog.classifiedButEmpty} classified-but-empty (no-fit), of ${backlog.nullDomainTotal} total NULL-domain`);
206
+ lines.push("");
207
+ lines.push("**4. Link-threshold pile-up evidence**\n");
208
+ lines.push(`- ${graph.dupNoise.count} / ${graph.dupNoise.measured} links (${pct(graph.dupNoise.share)}) connect near-duplicate memories (>= ${graph.dupNoise.threshold} cosine)`);
209
+ lines.push(`- Relinked partition dup-noise share: ${pct(graph.relinkedPartition.dupNoiseShare)} vs not-yet-relinked: ${pct(graph.notYetRelinkedPartition.dupNoiseShare)}`);
210
+ lines.push("");
211
+ lines.push("**5. Adoption snapshot**\n");
212
+ lines.push(`- Cold (never shown, never accessed): ${adoption.coldShare.coldCount} / ${adoption.coldShare.total} (${pct(adoption.coldShare.share)})`);
213
+ lines.push(`- Uses per showing overall: ${adoption.usesPerShowingOverall !== null ? round(adoption.usesPerShowingOverall) : "n/a"} — CAVEAT: shown_count is ~2 days old at snapshot time, read directionally only`);
214
+ lines.push("");
215
+ return lines.join("\n");
216
+ }
217
+ function renderReport(input) {
218
+ const parts = [];
219
+ parts.push("# Hicortex Mechanical Audit Baseline (#191)\n");
220
+ parts.push(`Snapshot: \`${input.dbPath}\` \nGenerated: ${input.generatedAt} \nRead-only, zero LLM, zero human grading — mechanical measurement only.\n`);
221
+ parts.push(renderDups(input.dups));
222
+ parts.push(renderDecay(input.domainBacklog, input.pruneDryRun, input.structural, input.adoption));
223
+ parts.push(renderGraph(input.graph));
224
+ parts.push(renderReflection(input.reflection));
225
+ parts.push(renderPhaseB(input.dups, input.pruneDryRun, input.structural, input.domainBacklog, input.graph, input.adoption));
226
+ return parts.join("\n");
227
+ }
228
+ // ---------------------------------------------------------------------------
229
+ // CLI entry
230
+ // ---------------------------------------------------------------------------
231
+ function main() {
232
+ const [, , dbPathArg, reportPathArg] = process.argv;
233
+ if (!dbPathArg) {
234
+ console.error("Usage: run-eval.js <snapshot.db> [report.md]");
235
+ process.exitCode = 1;
236
+ return;
237
+ }
238
+ const dbPath = dbPathArg;
239
+ const stateDir = (0, node_path_1.dirname)(dbPath);
240
+ const reportPath = reportPathArg ?? (0, node_path_1.join)(stateDir, "eval-report.md");
241
+ const db = (0, eval_db_js_1.openSnapshot)(dbPath);
242
+ try {
243
+ console.log("[eval] D1 duplicate audit...");
244
+ const dups = (0, dups_js_1.runDupAudit)(db);
245
+ console.log("[eval] D4 domain backlog...");
246
+ const domainBacklog = (0, decay_eval_js_1.runDomainBacklogAudit)(db, stateDir);
247
+ console.log("[eval] D4 prune dry-run...");
248
+ const pruneDryRun = (0, decay_eval_js_1.runPruneDryRun)(db, retrieval_js_1.DEFAULT_DECAY_HALF_LIFE_DAYS);
249
+ console.log("[eval] D4 structural strength stats...");
250
+ const structural = (0, decay_eval_js_1.runStructuralStrengthStats)(db);
251
+ console.log("[eval] D4 adoption stats...");
252
+ const adoption = (0, decay_eval_js_1.runAdoptionStats)(db);
253
+ console.log("[eval] D6 link-graph audit...");
254
+ const graph = (0, graph_eval_js_1.runGraphAudit)(db, stateDir);
255
+ console.log("[eval] reflection census...");
256
+ const reflection = (0, reflection_census_js_1.runReflectionCensus)(db);
257
+ const report = renderReport({
258
+ dbPath,
259
+ generatedAt: new Date().toISOString(),
260
+ dups,
261
+ domainBacklog,
262
+ pruneDryRun,
263
+ structural,
264
+ adoption,
265
+ graph,
266
+ reflection,
267
+ });
268
+ (0, node_fs_1.writeFileSync)(reportPath, report, "utf-8");
269
+ console.log(`[eval] report written to ${reportPath}`);
270
+ }
271
+ finally {
272
+ db.close();
273
+ }
274
+ }
275
+ main();
@@ -69,6 +69,7 @@ const memory_instructions_js_1 = require("./memory-instructions.js");
69
69
  const recall_index_js_1 = require("./recall-index.js");
70
70
  const seed_lesson_js_1 = require("./seed-lesson.js");
71
71
  const distiller_js_1 = require("./distiller.js");
72
+ const dedup_js_1 = require("./dedup.js");
72
73
  // ---------------------------------------------------------------------------
73
74
  // Server state
74
75
  // ---------------------------------------------------------------------------
@@ -790,30 +791,31 @@ async function startServer(options = {}) {
790
791
  res.status(400).json({ error: "Provide either 'text' (string) or 'messages' (array)" });
791
792
  return;
792
793
  }
793
- // Escape LIKE wildcards — Hermes ids contain "_" (e.g. 20260701_045744_...).
794
- const escapeLike = (s) => s.replace(/[\\%_]/g, (m) => "\\" + m);
795
794
  // Segment-exact dedup (#189): an incremental capture POST carries
796
795
  // segment_id "<start>-<end>[.pN]". Skip iff THIS exact segment's chunks are
797
796
  // already stored (keys "<sid>#<segment_id>#<i>"). This is what lets a failed
798
797
  // segment be safely retried with the same id, and a legacy session-level row
799
798
  // (key "<sid>#<i>", no segment) does NOT match — so the #189 recovery
800
799
  // re-ingest is never blocked by night-1's whole-session rows.
800
+ // ALSO consults dedup_log (#100): a merged-away loser's source_session
801
+ // marker survives there after the `memories` row is deleted, so a
802
+ // `hicortex dedup --apply` merge can't be undone by a retried/recaptured
803
+ // segment silently re-ingesting the same content.
801
804
  if (session_id && segment_id) {
802
- const likePrefix = `${escapeLike(session_id)}#${escapeLike(segment_id)}#%`;
803
- const existing = db.prepare("SELECT COUNT(*) as c FROM memories WHERE source_session LIKE ? ESCAPE '\\'").get(likePrefix);
804
- if (existing.c > 0) {
805
- res.status(200).json({ skipped: true, existing_count: existing.c });
805
+ const existingCount = (0, dedup_js_1.countExistingSegment)(db, session_id, segment_id);
806
+ if (existingCount > 0) {
807
+ res.status(200).json({ skipped: true, existing_count: existingCount });
806
808
  return;
807
809
  }
808
810
  }
809
811
  // Session-level dedup: when session_id is present and this is a whole-session
810
812
  // POST (no segment_id — legacy ≤0.13.1 clients), skip if any chunk of this
811
- // session is already stored. Unchanged: legacy clients keep exact behaviour.
813
+ // session is already stored (memories OR dedup_log see above).
814
+ // Unchanged: legacy clients keep exact behaviour.
812
815
  if (session_id && !segment_id) {
813
- const likePrefix = `${escapeLike(session_id)}#%`;
814
- const existing = db.prepare("SELECT COUNT(*) as c FROM memories WHERE source_session = ? OR source_session LIKE ? ESCAPE '\\'").get(session_id, likePrefix);
815
- if (existing.c > 0) {
816
- res.status(200).json({ skipped: true, existing_count: existing.c });
816
+ const existingCount = (0, dedup_js_1.countExistingSession)(db, session_id);
817
+ if (existingCount > 0) {
818
+ res.status(200).json({ skipped: true, existing_count: existingCount });
817
819
  return;
818
820
  }
819
821
  }
package/dist/nightly.js CHANGED
@@ -390,6 +390,10 @@ async function runNightly(options = {}) {
390
390
  domains: cfgDomains,
391
391
  contentDomainsReady,
392
392
  weakPrimaryFloor: (0, nofit_js_1.resolveWeakPrimaryFloor)(savedConfig),
393
+ }, {
394
+ minSimilarity: savedConfig?.supersessionMinSimilarity,
395
+ maxCalls: savedConfig?.supersessionMaxCalls,
396
+ penalty: savedConfig?.supersessionPenalty,
393
397
  });
394
398
  console.log(`[hicortex] Consolidation ${report.status} in ${report.elapsed_seconds}s` +
395
399
  (report.stages.reflection ? ` (${report.stages.reflection.lessons_generated} lessons)` : ""));
package/dist/relink.d.ts CHANGED
@@ -37,7 +37,6 @@
37
37
  * Server-mode only: relink needs the local DB. Client installs must run it
38
38
  * on the server machine.
39
39
  */
40
- import type Database from "better-sqlite3";
41
40
  import type { EmbedFn } from "./retrieval.js";
42
41
  export interface RelinkOptions {
43
42
  /** Full discovery + would-be counts, zero writes, cursor untouched. */
@@ -87,8 +86,10 @@ export interface RelinkReport {
87
86
  /**
88
87
  * Read the stored embedding for a memory from memory_vectors.
89
88
  * Returns null when the row is missing (caller falls back to re-embedding).
89
+ * @deprecated moved to storage.ts (shared with consolidate.ts's supersession
90
+ * stage); re-exported here so existing importers of relink.ts keep working.
90
91
  */
91
- export declare function getStoredEmbedding(db: Database.Database, memoryId: string): Float32Array | null;
92
+ export { getStoredEmbedding } from "./storage.js";
92
93
  /**
93
94
  * Run the relink pass. Returns a structured report.
94
95
  * Throws on unrecoverable errors (client mode, DB write failure) — the cursor
package/dist/relink.js CHANGED
@@ -72,7 +72,7 @@ var __importStar = (this && this.__importStar) || (function () {
72
72
  };
73
73
  })();
74
74
  Object.defineProperty(exports, "__esModule", { value: true });
75
- exports.getStoredEmbedding = getStoredEmbedding;
75
+ exports.getStoredEmbedding = void 0;
76
76
  exports.runRelink = runRelink;
77
77
  const paths_js_1 = require("./paths.js");
78
78
  const node_fs_1 = require("node:fs");
@@ -107,16 +107,11 @@ function loadExistingPairs(db) {
107
107
  /**
108
108
  * Read the stored embedding for a memory from memory_vectors.
109
109
  * Returns null when the row is missing (caller falls back to re-embedding).
110
+ * @deprecated moved to storage.ts (shared with consolidate.ts's supersession
111
+ * stage); re-exported here so existing importers of relink.ts keep working.
110
112
  */
111
- function getStoredEmbedding(db, memoryId) {
112
- const row = db
113
- .prepare("SELECT embedding FROM memory_vectors WHERE id = ?")
114
- .get(memoryId);
115
- if (!row?.embedding)
116
- return null;
117
- const buf = row.embedding;
118
- return new Float32Array(buf.buffer, buf.byteOffset, buf.byteLength / 4);
119
- }
113
+ var storage_js_1 = require("./storage.js");
114
+ Object.defineProperty(exports, "getStoredEmbedding", { enumerable: true, get: function () { return storage_js_1.getStoredEmbedding; } });
120
115
  /**
121
116
  * Run the relink pass. Returns a structured report.
122
117
  * Throws on unrecoverable errors (client mode, DB write failure) — the cursor
@@ -193,7 +188,7 @@ async function runRelink(options = {}) {
193
188
  const { __rowid: _ignored, ...memRow } = row;
194
189
  const mem = memRow;
195
190
  try {
196
- let embedding = getStoredEmbedding(db, mem.id);
191
+ let embedding = storage.getStoredEmbedding(db, mem.id);
197
192
  if (!embedding) {
198
193
  embedding = await (await getEmbedFn())(mem.content);
199
194
  }
@@ -3,5 +3,5 @@
3
3
  * Shared between OC plugin and CC MCP server.
4
4
  */
5
5
  import type Database from "better-sqlite3";
6
- export declare const SEED_LESSON = "## Lesson: 1% Daily Self-Improvement\n\nSelf-improvement protocol: Review your sessions daily. Scan for mistakes, gaps, user corrections, and friction. Identify 1-2 issues maximum. Apply one targeted fix to your instruction files (AGENT.md, IDENTITY.md, TOOLS.md, SOUL.md, or CLAUDE.md). Never overcorrect \u2014 feedback is guidance, not emergency. One fix per issue. Never rewrite entire systems. Before changing something, ask: is this a real pattern or am I reacting to noise?\n\nThe flywheel: Sessions \u2192 Hicortex captures \u2192 Nightly consolidation \u2192 Lessons generated \u2192 Lessons injected into your context \u2192 You apply one fix \u2192 Better sessions tomorrow.\n\nUse hicortex_lessons to check for new insights. If a lesson has high confidence, apply it. If medium, observe for one more day. If nothing needs fixing today, skip \u2014 that's fine.\n\n**Severity:** critical\n**Confidence:** high\n**Generated:** seed-lesson";
6
+ export declare const SEED_LESSON = "## Lesson: 1% Daily Self-Improvement\n\nSelf-improvement protocol: Review your sessions daily. Scan for mistakes, gaps, user corrections, and friction. Identify 1-2 issues maximum. Apply one targeted fix to your agent instruction files (e.g. SOUL.md / MEMORY.md for Hermes profiles, CLAUDE.md for Claude Code). Never overcorrect \u2014 feedback is guidance, not emergency. One fix per issue. Never rewrite entire systems. Before changing something, ask: is this a real pattern or am I reacting to noise?\n\nThe flywheel: Sessions \u2192 Hicortex captures \u2192 Nightly consolidation \u2192 Lessons generated \u2192 Lessons injected into your context \u2192 You apply one fix \u2192 Better sessions tomorrow.\n\nUse hicortex_lessons to check for new insights. If a lesson has high confidence, apply it. If medium, observe for one more day. If nothing needs fixing today, skip \u2014 that's fine.\n\n**Severity:** critical\n**Confidence:** high\n**Generated:** seed-lesson";
7
7
  export declare function injectSeedLesson(database: Database.Database, log?: (msg: string) => void): Promise<void>;
@@ -43,7 +43,7 @@ const embedder_js_1 = require("./embedder.js");
43
43
  const storage = __importStar(require("./storage.js"));
44
44
  exports.SEED_LESSON = `## Lesson: 1% Daily Self-Improvement
45
45
 
46
- Self-improvement protocol: Review your sessions daily. Scan for mistakes, gaps, user corrections, and friction. Identify 1-2 issues maximum. Apply one targeted fix to your instruction files (AGENT.md, IDENTITY.md, TOOLS.md, SOUL.md, or CLAUDE.md). Never overcorrect — feedback is guidance, not emergency. One fix per issue. Never rewrite entire systems. Before changing something, ask: is this a real pattern or am I reacting to noise?
46
+ Self-improvement protocol: Review your sessions daily. Scan for mistakes, gaps, user corrections, and friction. Identify 1-2 issues maximum. Apply one targeted fix to your agent instruction files (e.g. SOUL.md / MEMORY.md for Hermes profiles, CLAUDE.md for Claude Code). Never overcorrect — feedback is guidance, not emergency. One fix per issue. Never rewrite entire systems. Before changing something, ask: is this a real pattern or am I reacting to noise?
47
47
 
48
48
  The flywheel: Sessions → Hicortex captures → Nightly consolidation → Lessons generated → Lessons injected into your context → You apply one fix → Better sessions tomorrow.
49
49
 
package/dist/state.d.ts CHANGED
@@ -50,6 +50,16 @@ export interface HicortexState {
50
50
  * Same discipline as relinkCursor.
51
51
  */
52
52
  domainCursor?: number;
53
+ /**
54
+ * Resume cursor for the nightly's supersession-detection stage (#191 Phase
55
+ * B) — highest memories.rowid whose decision/correction candidates have
56
+ * been evaluated (or infra-skipped) this run. Absent/0 = never run. Unlike
57
+ * relinkCursor/domainCursor (separate resumable CLI commands), this cursor
58
+ * advances a SMALL amount per night (config `supersessionMaxCalls`, default
59
+ * 30) as part of the regular nightly — the corpus is back-processed
60
+ * gradually over many nights.
61
+ */
62
+ supersessionCursor?: number;
53
63
  }
54
64
  /**
55
65
  * Load the state file. Returns an empty state if the file is missing
package/dist/storage.d.ts CHANGED
@@ -89,6 +89,16 @@ export declare function getMemoryTagsWeighted(db: Database.Database, memoryId: s
89
89
  tag: string;
90
90
  weight: number | null;
91
91
  }>;
92
+ /**
93
+ * Read the stored embedding for a memory from memory_vectors.
94
+ * Returns null when the row is missing (caller falls back to re-embedding).
95
+ *
96
+ * Shared by `hicortex relink` and the nightly's supersession stage
97
+ * (consolidate.ts) — lives here (not in relink.ts) so consolidate.ts can use
98
+ * it without importing from relink.ts, which itself imports from
99
+ * consolidate.ts (BudgetTracker, discoverLinkCandidates).
100
+ */
101
+ export declare function getStoredEmbedding(db: Database.Database, memoryId: string): Float32Array | null;
92
102
  /**
93
103
  * Find similar memories by vector distance. Returns memories with distance field.
94
104
  */
package/dist/storage.js CHANGED
@@ -15,6 +15,7 @@ exports.deleteMemory = deleteMemory;
15
15
  exports.setMemoryTags = setMemoryTags;
16
16
  exports.getMemoryTags = getMemoryTags;
17
17
  exports.getMemoryTagsWeighted = getMemoryTagsWeighted;
18
+ exports.getStoredEmbedding = getStoredEmbedding;
18
19
  exports.vectorSearch = vectorSearch;
19
20
  exports.searchFts = searchFts;
20
21
  exports.addLink = addLink;
@@ -115,6 +116,10 @@ const ALLOWED_UPDATE_FIELDS = new Set([
115
116
  "base_strength",
116
117
  "last_accessed",
117
118
  "access_count",
119
+ // shown_count is normally maintained by touchMemoriesShown (bulk +1 per
120
+ // recall-index appearance); it's also allowed here so `hicortex dedup`
121
+ // (#100) can sum a merged cluster's counters onto the canonical row.
122
+ "shown_count",
118
123
  "source_agent",
119
124
  "source_session",
120
125
  "project",
@@ -239,6 +244,24 @@ function getMemoryTagsWeighted(db, memoryId) {
239
244
  .all(memoryId);
240
245
  return rows;
241
246
  }
247
+ /**
248
+ * Read the stored embedding for a memory from memory_vectors.
249
+ * Returns null when the row is missing (caller falls back to re-embedding).
250
+ *
251
+ * Shared by `hicortex relink` and the nightly's supersession stage
252
+ * (consolidate.ts) — lives here (not in relink.ts) so consolidate.ts can use
253
+ * it without importing from relink.ts, which itself imports from
254
+ * consolidate.ts (BudgetTracker, discoverLinkCandidates).
255
+ */
256
+ function getStoredEmbedding(db, memoryId) {
257
+ const row = db
258
+ .prepare("SELECT embedding FROM memory_vectors WHERE id = ?")
259
+ .get(memoryId);
260
+ if (!row?.embedding)
261
+ return null;
262
+ const buf = row.embedding;
263
+ return new Float32Array(buf.buffer, buf.byteOffset, buf.byteLength / 4);
264
+ }
242
265
  // ---------------------------------------------------------------------------
243
266
  // Vector search
244
267
  // ---------------------------------------------------------------------------
package/dist/types.d.ts CHANGED
@@ -108,6 +108,21 @@ export interface ConsolidationReport {
108
108
  heuristic_fallback?: number;
109
109
  failed: number;
110
110
  };
111
+ /** Supersession detection (#191 Phase B) — runs after linking, before decay/prune. */
112
+ supersession?: {
113
+ /** Decision/correction-shaped candidates examined this run. */
114
+ scanned: number;
115
+ /** Older-neighbor pairs actually sent to the classify-tier LLM. */
116
+ evaluated: number;
117
+ /** Pairs the LLM judged superseded — a `superseded_by` link was created. */
118
+ superseded: number;
119
+ /** Pairs skipped on a parse/infra error (retried naturally next night). */
120
+ skipped_infra: number;
121
+ /** Pairs skipped because a superseded_by link already existed (either direction). */
122
+ skipped_idempotent: number;
123
+ /** supersessionCursor after this run (unchanged in dry-run). */
124
+ cursor: number;
125
+ };
111
126
  decay_prune?: {
112
127
  candidates: number;
113
128
  pruned: number;
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "@gamaze/hicortex",
3
- "version": "0.14.4",
3
+ "version": "0.15.0",
4
4
  "description": "Self-learning memory for AI agents — experience captured automatically, distilled into lessons overnight, shared across your whole fleet. Works with Hermes, OpenClaw, Claude Code, and Pi.",
5
5
  "main": "dist/index.js",
6
6
  "bin": {
@@ -37,6 +37,7 @@
37
37
  "build": "rm -rf dist && tsc",
38
38
  "test": "vitest run",
39
39
  "test:watch": "vitest",
40
+ "eval": "node dist/eval/run-eval.js",
40
41
  "prepack": "npm run build && rm -rf ./hermes-plugin && mkdir -p ./hermes-plugin && cp -r ../../hermes-plugin/hicortex ./hermes-plugin/ && find ./hermes-plugin -name __pycache__ -type d -exec rm -rf {} + 2>/dev/null || true",
41
42
  "prepublishOnly": "npm run build && rm -rf ./hermes-plugin && mkdir -p ./hermes-plugin && cp -r ../../hermes-plugin/hicortex ./hermes-plugin/ && find ./hermes-plugin -name __pycache__ -type d -exec rm -rf {} + 2>/dev/null || true"
42
43
  },