@gamaze/hicortex 0.14.3 → 0.15.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -0,0 +1,275 @@
1
+ #!/usr/bin/env node
2
+ "use strict";
3
+ /**
4
+ * #191 mechanical audit baseline — orchestrates every section (D1 duplicate
5
+ * rate, D4 decay/prune/no-fit + #192 adoption, D6 link-graph health, the
6
+ * reflection census) into one markdown report.
7
+ *
8
+ * Read-only end to end: opens the snapshot via eval-db.ts (`openSnapshot`,
9
+ * never `initDb()`), never writes to the DB. Not wired into `cli.ts` — this
10
+ * is an internal measurement tool, run via the `eval` npm script:
11
+ *
12
+ * npm run eval -- <snapshot.db> [report.md]
13
+ *
14
+ * `state.json` is expected alongside the snapshot DB (same directory) for
15
+ * the `domainCursor`/`relinkCursor` watermarks. Report path defaults to
16
+ * `eval-report.md` next to the DB.
17
+ */
18
+ Object.defineProperty(exports, "__esModule", { value: true });
19
+ const node_path_1 = require("node:path");
20
+ const node_fs_1 = require("node:fs");
21
+ const eval_db_js_1 = require("./eval-db.js");
22
+ const dups_js_1 = require("./dups.js");
23
+ const decay_eval_js_1 = require("./decay-eval.js");
24
+ const graph_eval_js_1 = require("./graph-eval.js");
25
+ const reflection_census_js_1 = require("./reflection-census.js");
26
+ const retrieval_js_1 = require("../retrieval.js");
27
+ // ---------------------------------------------------------------------------
28
+ // Formatting helpers
29
+ // ---------------------------------------------------------------------------
30
+ function pct(n) {
31
+ return `${(n * 100).toFixed(1)}%`;
32
+ }
33
+ function round(n, places = 4) {
34
+ const f = 10 ** places;
35
+ return Math.round(n * f) / f;
36
+ }
37
+ /** Extract the leading number from a bucket label (e.g. "[0.2, 0.3)" -> 0.2) so buckets render in ascending order. */
38
+ function bucketSortKey(label) {
39
+ const match = label.match(/-?\d+(\.\d+)?/);
40
+ return match ? Number(match[0]) : Number.POSITIVE_INFINITY;
41
+ }
42
+ function histTable(h) {
43
+ const entries = Object.entries(h).sort((a, b) => bucketSortKey(a[0]) - bucketSortKey(b[0]));
44
+ if (entries.length === 0)
45
+ return "_(no data)_\n";
46
+ return entries.map(([bucket, count]) => `| ${bucket} | ${count} |`).join("\n") + "\n";
47
+ }
48
+ // ---------------------------------------------------------------------------
49
+ // Report sections
50
+ // ---------------------------------------------------------------------------
51
+ function renderDups(d) {
52
+ const lines = [];
53
+ lines.push("## D1 — Duplicate rate\n");
54
+ lines.push(`Top-${d.knnK} vector KNN per memory (${d.totalMemories} memories), pairs kept at cosine >= 0.90 and clustered per threshold (union-find). "Excess" = rows that would disappear if every cluster merged to one.\n`);
55
+ lines.push("| Threshold | Clusters | Excess (rows) |");
56
+ lines.push("|---|---|---|");
57
+ for (const t of d.thresholds) {
58
+ lines.push(`| ${t.threshold} | ${t.clusterCount} | ${t.excess} |`);
59
+ }
60
+ lines.push("");
61
+ lines.push(`**Pair attribution** (over all ${d.pairAttribution.totalPairs} pairs at/above the lowest threshold, 0.90): ` +
62
+ `${d.pairAttribution.recoveryReingest} recovery/re-ingest-suspect (${pct(d.pairAttribution.totalPairs > 0 ? d.pairAttribution.recoveryReingest / d.pairAttribution.totalPairs : 0)}), ${d.pairAttribution.organic} organic (${pct(d.pairAttribution.totalPairs > 0 ? d.pairAttribution.organic / d.pairAttribution.totalPairs : 0)}).\n`);
63
+ lines.push(`### Top ${d.topClusters.length} clusters (0.90 threshold, largest first)\n`);
64
+ d.topClusters.forEach((cluster, i) => {
65
+ const mismatch = cluster.metadataMismatch;
66
+ const mismatchFlags = [
67
+ mismatch.projectMismatch ? "project" : null,
68
+ mismatch.privacyMismatch ? "privacy" : null,
69
+ mismatch.sourceAgentMismatch ? "source_agent" : null,
70
+ ].filter(Boolean);
71
+ lines.push(`**Cluster ${i + 1}** — size ${cluster.size}, attribution: ${cluster.attribution.recoveryReingest} recovery-pair(s) / ${cluster.attribution.organic} organic-pair(s)` +
72
+ (mismatchFlags.length > 0 ? `, metadata mismatch: ${mismatchFlags.join(", ")}` : ", metadata consistent"));
73
+ for (const m of cluster.members) {
74
+ lines.push(` - \`${m.id.slice(0, 8)}\` ${m.created_at} — ${m.preview.replace(/\n/g, " ")}`);
75
+ }
76
+ lines.push("");
77
+ });
78
+ return lines.join("\n");
79
+ }
80
+ function renderDecay(backlog, prune, structural, adoption) {
81
+ const lines = [];
82
+ lines.push("## D4 — Decay / Prune / No-fit lifecycle + #192 adoption\n");
83
+ lines.push("### Domain classification backlog\n");
84
+ lines.push(`${backlog.nullDomainTotal} memories have NULL domain (of ${backlog.maxRowid} max rowid). Partitioned by ` +
85
+ `\`domainCursor\` (${backlog.domainCursor ?? "unset — classify-domains never run"}), documented in ` +
86
+ `classify-domains.ts as the last fully-committed rowid:\n`);
87
+ lines.push(`- **Never-classified backlog** (rowid > cursor): ${backlog.neverClassifiedBacklog}`);
88
+ lines.push(`- **Classified but no fitting domain** (rowid <= cursor, no-fit path): ${backlog.classifiedButEmpty}\n`);
89
+ lines.push("### Prune dry-run (real production predicate: `stageDecayPrune`, `dryRun: true`)\n");
90
+ lines.push(`Decay clock configured to \`decayHalfLifeDays: ${prune.decayHalfLifeDaysUsed}\` (bedrock has no override — this is the shipped default) before running.\n`);
91
+ lines.push(`- Candidates (would prune today): **${prune.candidates}**`);
92
+ lines.push(`- Pruned: ${prune.pruned} (0 expected — dry run)`);
93
+ lines.push(`- Failed: ${prune.failed}\n`);
94
+ lines.push("### Structural strength — who can EVER prune\n");
95
+ lines.push(`A memory can only ever cross the prune floor (effective strength < 0.01) if its asymptote is below that floor: ` +
96
+ `\`base_strength < sqrt(0.1) ≈ ${round(structural.everPrunableCeiling, 4)}\` (floor = base_strength × importance × 0.1, ` +
97
+ `importance defaults to base_strength — independent of access/link hardening, which only slows the approach).\n`);
98
+ lines.push(`**Ever-prunable: ${structural.everPrunableCount} / ${structural.totalMemories} (${pct(structural.totalMemories > 0 ? structural.everPrunableCount / structural.totalMemories : 0)})** — everyone else has a strength floor that never drops low enough to prune, no matter how long it goes unaccessed.\n`);
99
+ for (const [label, hist] of [
100
+ ["Effective strength now", structural.effectiveStrengthNowHistogram],
101
+ ["Simulated +180 days (no further access)", structural.effectiveStrengthAt180dHistogram],
102
+ ["Simulated +365 days (no further access)", structural.effectiveStrengthAt365dHistogram],
103
+ ["Asymptotic floor (never, base_strength² × 0.1)", structural.effectiveStrengthNeverHistogram],
104
+ ]) {
105
+ lines.push(`**${label}**\n`);
106
+ lines.push("| Bucket | Count |");
107
+ lines.push("|---|---|");
108
+ lines.push(histTable(hist));
109
+ }
110
+ lines.push("### Adoption (#192 shown_count / access_count)\n");
111
+ lines.push("`shown_count` (migration v8) was deployed ~2 days before this snapshot — near-zero shown counts reflect the " +
112
+ "feature's youth as much as its effectiveness; read this section as a baseline to compare against, not a verdict.\n");
113
+ lines.push(`- Total shown (sum): ${adoption.totalShown}, total access (sum): ${adoption.totalAccess}`);
114
+ lines.push(`- Uses per showing (overall): ${adoption.usesPerShowingOverall !== null ? round(adoption.usesPerShowingOverall) : "n/a (nothing shown yet)"}`);
115
+ lines.push(`- Cold (shown=0 AND access=0): ${adoption.coldShare.coldCount} / ${adoption.coldShare.total} (${pct(adoption.coldShare.share)})\n`);
116
+ lines.push("**shown_count distribution**\n\n| Bucket | Count |\n|---|---|\n" + histTable(adoption.shownCountHistogram));
117
+ lines.push("**access_count distribution**\n\n| Bucket | Count |\n|---|---|\n" + histTable(adoption.accessCountHistogram));
118
+ lines.push("**Uses per showing by source_agent** (agents with at least one showing)\n");
119
+ lines.push("| source_agent | shown | access | ratio |");
120
+ lines.push("|---|---|---|---|");
121
+ const byAgent = Object.entries(adoption.usesPerShowingBySourceAgent)
122
+ .filter(([, v]) => v.shown > 0 || v.access > 0)
123
+ .sort((a, b) => b[1].shown - a[1].shown);
124
+ for (const [agent, v] of byAgent) {
125
+ lines.push(`| ${agent} | ${v.shown} | ${v.access} | ${v.ratio !== null ? round(v.ratio) : "n/a"} |`);
126
+ }
127
+ lines.push("");
128
+ lines.push("**Exposure age profile** — are new memories getting shown?\n");
129
+ lines.push("| Age bucket | Total | Ever shown | Share shown |");
130
+ lines.push("|---|---|---|---|");
131
+ for (const b of adoption.exposureAgeProfile) {
132
+ lines.push(`| ${b.bucket} | ${b.total} | ${b.everShown} | ${pct(b.shareShown)} |`);
133
+ }
134
+ lines.push("");
135
+ return lines.join("\n");
136
+ }
137
+ function renderGraph(g) {
138
+ const lines = [];
139
+ lines.push("## D6 — Link-graph health\n");
140
+ lines.push(`Total links: **${g.totalLinks}**\n`);
141
+ lines.push("| Relationship | Count |");
142
+ lines.push("|---|---|");
143
+ for (const [rel, count] of Object.entries(g.byRelationship)) {
144
+ lines.push(`| ${rel} | ${count} |`);
145
+ }
146
+ lines.push("");
147
+ lines.push(`### Degree distribution (${g.degree.memoriesWithLinks} / ${g.degree.totalMemories} memories have at least one link)\n`);
148
+ lines.push("| Degree bucket | Count |");
149
+ lines.push("|---|---|");
150
+ lines.push(histTable(g.degree.degreeHistogram));
151
+ lines.push(`### Top ${g.degree.topHubs.length} hubs\n`);
152
+ lines.push("| id | degree | project | domain | preview |");
153
+ lines.push("|---|---|---|---|---|");
154
+ for (const h of g.degree.topHubs) {
155
+ lines.push(`| \`${h.id.slice(0, 8)}\` | ${h.degree} | ${h.project ?? "-"} | ${h.domain ?? "-"} | ${h.preview.replace(/\|/g, "/")} |`);
156
+ }
157
+ lines.push("");
158
+ lines.push(`### Cosine drift (stored \`memory_links.strength\` vs recomputed, ${g.drift.sampleSize}-link random sample` +
159
+ (g.drift.skippedMissingEmbedding > 0 ? `, ${g.drift.skippedMissingEmbedding} skipped for missing embeddings` : "") +
160
+ ")\n");
161
+ lines.push(`Mean |drift|: ${round(g.drift.meanAbsDrift, 8)}, max |drift|: ${round(g.drift.maxAbsDrift, 8)} ` +
162
+ "(float32 rounding-noise scale (~1e-7) counts as zero drift — strength was set from the same embeddings at link-creation time, so no staleness is expected).\n");
163
+ lines.push("| |drift| bucket | Count |");
164
+ lines.push("|---|---|");
165
+ lines.push(histTable(g.drift.driftHistogram));
166
+ lines.push(`### Dup-noise links (endpoints >= ${g.dupNoise.threshold} cosine, measured over ${g.dupNoise.measured} links with both embeddings)\n`);
167
+ lines.push(`${g.dupNoise.count} / ${g.dupNoise.measured} (${pct(g.dupNoise.share)}) of links connect near-duplicate memories.\n`);
168
+ lines.push(`### Relink migration partition (\`relinkCursor\`: ${g.relinkCursor ?? "unset — relink never run"})\n`);
169
+ lines.push("| Partition | Links | Dup-noise share |");
170
+ lines.push("|---|---|---|");
171
+ lines.push(`| Relinked (source rowid <= cursor) | ${g.relinkedPartition.linkCount} | ${pct(g.relinkedPartition.dupNoiseShare)} |`);
172
+ lines.push(`| Not yet relinked (source rowid > cursor) | ${g.notYetRelinkedPartition.linkCount} | ${pct(g.notYetRelinkedPartition.dupNoiseShare)} |`);
173
+ lines.push("");
174
+ return lines.join("\n");
175
+ }
176
+ function renderReflection(r) {
177
+ const lines = [];
178
+ lines.push("## Reflection census\n");
179
+ lines.push(`Lessons: ${r.totalLessons}, episodes: ${r.totalEpisodes}, yield ratio (lessons/episodes): ${r.yieldRatio !== null ? round(r.yieldRatio) : "n/a"}\n`);
180
+ lines.push("_Not a quality read — actionable-vs-noise sampling is deferred Phase-A grading work._\n");
181
+ if (r.lessonsByDate.length > 0) {
182
+ lines.push("| Date | Lessons |");
183
+ lines.push("|---|---|");
184
+ for (const row of r.lessonsByDate) {
185
+ lines.push(`| ${row.date} | ${row.count} |`);
186
+ }
187
+ lines.push("");
188
+ }
189
+ return lines.join("\n");
190
+ }
191
+ function renderPhaseB(dups, prune, structural, backlog, graph, adoption) {
192
+ const lines = [];
193
+ lines.push("## Phase-B decision inputs\n");
194
+ lines.push("**1. Merge-threshold curve (dup excess at each cosine)**\n");
195
+ for (const t of dups.thresholds) {
196
+ lines.push(`- ${t.threshold}: ${t.clusterCount} clusters, ${t.excess} excess rows`);
197
+ }
198
+ lines.push("");
199
+ lines.push("**2. Prune posture — inert by design or miscalibration?**\n");
200
+ lines.push(`- Dry-run candidates today: ${prune.candidates} (half-life ${prune.decayHalfLifeDaysUsed}d)`);
201
+ lines.push(`- Ever-prunable ceiling: ${structural.everPrunableCount} / ${structural.totalMemories} (${pct(structural.totalMemories > 0 ? structural.everPrunableCount / structural.totalMemories : 0)}) memories can EVER cross the floor regardless of age`);
202
+ lines.push(`- If the current trend holds, compare the now/+180d/+365d histograms above for how many additional memories drift into low-strength bands`);
203
+ lines.push("");
204
+ lines.push("**3. Classification backlog size**\n");
205
+ lines.push(`- ${backlog.neverClassifiedBacklog} never-classified (beyond domainCursor), ${backlog.classifiedButEmpty} classified-but-empty (no-fit), of ${backlog.nullDomainTotal} total NULL-domain`);
206
+ lines.push("");
207
+ lines.push("**4. Link-threshold pile-up evidence**\n");
208
+ lines.push(`- ${graph.dupNoise.count} / ${graph.dupNoise.measured} links (${pct(graph.dupNoise.share)}) connect near-duplicate memories (>= ${graph.dupNoise.threshold} cosine)`);
209
+ lines.push(`- Relinked partition dup-noise share: ${pct(graph.relinkedPartition.dupNoiseShare)} vs not-yet-relinked: ${pct(graph.notYetRelinkedPartition.dupNoiseShare)}`);
210
+ lines.push("");
211
+ lines.push("**5. Adoption snapshot**\n");
212
+ lines.push(`- Cold (never shown, never accessed): ${adoption.coldShare.coldCount} / ${adoption.coldShare.total} (${pct(adoption.coldShare.share)})`);
213
+ lines.push(`- Uses per showing overall: ${adoption.usesPerShowingOverall !== null ? round(adoption.usesPerShowingOverall) : "n/a"} — CAVEAT: shown_count is ~2 days old at snapshot time, read directionally only`);
214
+ lines.push("");
215
+ return lines.join("\n");
216
+ }
217
+ function renderReport(input) {
218
+ const parts = [];
219
+ parts.push("# Hicortex Mechanical Audit Baseline (#191)\n");
220
+ parts.push(`Snapshot: \`${input.dbPath}\` \nGenerated: ${input.generatedAt} \nRead-only, zero LLM, zero human grading — mechanical measurement only.\n`);
221
+ parts.push(renderDups(input.dups));
222
+ parts.push(renderDecay(input.domainBacklog, input.pruneDryRun, input.structural, input.adoption));
223
+ parts.push(renderGraph(input.graph));
224
+ parts.push(renderReflection(input.reflection));
225
+ parts.push(renderPhaseB(input.dups, input.pruneDryRun, input.structural, input.domainBacklog, input.graph, input.adoption));
226
+ return parts.join("\n");
227
+ }
228
+ // ---------------------------------------------------------------------------
229
+ // CLI entry
230
+ // ---------------------------------------------------------------------------
231
+ function main() {
232
+ const [, , dbPathArg, reportPathArg] = process.argv;
233
+ if (!dbPathArg) {
234
+ console.error("Usage: run-eval.js <snapshot.db> [report.md]");
235
+ process.exitCode = 1;
236
+ return;
237
+ }
238
+ const dbPath = dbPathArg;
239
+ const stateDir = (0, node_path_1.dirname)(dbPath);
240
+ const reportPath = reportPathArg ?? (0, node_path_1.join)(stateDir, "eval-report.md");
241
+ const db = (0, eval_db_js_1.openSnapshot)(dbPath);
242
+ try {
243
+ console.log("[eval] D1 duplicate audit...");
244
+ const dups = (0, dups_js_1.runDupAudit)(db);
245
+ console.log("[eval] D4 domain backlog...");
246
+ const domainBacklog = (0, decay_eval_js_1.runDomainBacklogAudit)(db, stateDir);
247
+ console.log("[eval] D4 prune dry-run...");
248
+ const pruneDryRun = (0, decay_eval_js_1.runPruneDryRun)(db, retrieval_js_1.DEFAULT_DECAY_HALF_LIFE_DAYS);
249
+ console.log("[eval] D4 structural strength stats...");
250
+ const structural = (0, decay_eval_js_1.runStructuralStrengthStats)(db);
251
+ console.log("[eval] D4 adoption stats...");
252
+ const adoption = (0, decay_eval_js_1.runAdoptionStats)(db);
253
+ console.log("[eval] D6 link-graph audit...");
254
+ const graph = (0, graph_eval_js_1.runGraphAudit)(db, stateDir);
255
+ console.log("[eval] reflection census...");
256
+ const reflection = (0, reflection_census_js_1.runReflectionCensus)(db);
257
+ const report = renderReport({
258
+ dbPath,
259
+ generatedAt: new Date().toISOString(),
260
+ dups,
261
+ domainBacklog,
262
+ pruneDryRun,
263
+ structural,
264
+ adoption,
265
+ graph,
266
+ reflection,
267
+ });
268
+ (0, node_fs_1.writeFileSync)(reportPath, report, "utf-8");
269
+ console.log(`[eval] report written to ${reportPath}`);
270
+ }
271
+ finally {
272
+ db.close();
273
+ }
274
+ }
275
+ main();
@@ -65,9 +65,11 @@ const viz_js_1 = require("./viz.js");
65
65
  const context_store_js_1 = require("./context-store.js");
66
66
  const retrieval = __importStar(require("./retrieval.js"));
67
67
  const recall_registry_js_1 = require("./recall-registry.js");
68
+ const memory_instructions_js_1 = require("./memory-instructions.js");
68
69
  const recall_index_js_1 = require("./recall-index.js");
69
70
  const seed_lesson_js_1 = require("./seed-lesson.js");
70
71
  const distiller_js_1 = require("./distiller.js");
72
+ const dedup_js_1 = require("./dedup.js");
71
73
  // ---------------------------------------------------------------------------
72
74
  // Server state
73
75
  // ---------------------------------------------------------------------------
@@ -90,6 +92,8 @@ let contextAgents = {};
90
92
  // Pushed-recall dedup registry (#192) + options; configured at boot.
91
93
  let recallRegistry = new recall_registry_js_1.SessionRecallRegistry();
92
94
  let recallIndexOptions = {};
95
+ // Product-owned memory instructions (#192): on unless config says false.
96
+ let memoryInstructionsEnabled = true;
93
97
  // Cache detectChunkSize results keyed by "<provider>/<model>@<baseUrl>" so we
94
98
  // probe each endpoint once per server boot rather than once per /distill request.
95
99
  const chunkSizeCache = new Map();
@@ -493,6 +497,7 @@ async function startServer(options = {}) {
493
497
  maxItems: savedConfig?.recallMaxItems,
494
498
  minPromptLength: savedConfig?.recallMinPromptChars,
495
499
  };
500
+ memoryInstructionsEnabled = savedConfig?.memoryInstructions !== false;
496
501
  if (resolvedAgents.dropped.length > 0) {
497
502
  console.warn(`[hicortex] Ignoring invalid contextAgents entries: ${resolvedAgents.dropped.join(", ")} ` +
498
503
  `(keys must match ^[a-z0-9][a-z0-9_-]*$; modes must be override|global|off)`);
@@ -727,6 +732,13 @@ async function startServer(options = {}) {
727
732
  app.get("/context", (req, res) => {
728
733
  try {
729
734
  const r = (0, context_store_js_1.handleContextGet)((0, node_path_1.join)(stateDir, "context"), contextClients, req.query, contextAgents);
735
+ // #192: product-owned memory instructions ride as a synthetic read-only
736
+ // `memory` section (config memoryInstructions !== false; agent mode
737
+ // "off" respected inside the helper). Every harness renders it via the
738
+ // shared section renderer — zero client changes.
739
+ if (r.status === 200) {
740
+ (0, memory_instructions_js_1.injectMemorySection)(r.body, memoryInstructionsEnabled);
741
+ }
730
742
  res.status(r.status).json(r.body);
731
743
  }
732
744
  catch (err) {
@@ -735,6 +747,12 @@ async function startServer(options = {}) {
735
747
  });
736
748
  app.put("/context", (req, res) => {
737
749
  try {
750
+ // Reserved product section: never writable, loud error (no silent skip).
751
+ const putSections = req.body?.sections;
752
+ if (putSections && Object.keys(putSections).some((n) => (0, memory_instructions_js_1.isReservedSectionName)(n))) {
753
+ res.status(400).json({ error: `Section name '${memory_instructions_js_1.MEMORY_SECTION_NAME}' is reserved for the product-owned memory instructions (config memoryInstructions to disable them)` });
754
+ return;
755
+ }
738
756
  const r = (0, context_store_js_1.handleContextPut)((0, node_path_1.join)(stateDir, "context"), req.body, req.query, contextAgents);
739
757
  if (r.warn)
740
758
  console.warn(`[hicortex] ${r.warn}`);
@@ -773,30 +791,31 @@ async function startServer(options = {}) {
773
791
  res.status(400).json({ error: "Provide either 'text' (string) or 'messages' (array)" });
774
792
  return;
775
793
  }
776
- // Escape LIKE wildcards — Hermes ids contain "_" (e.g. 20260701_045744_...).
777
- const escapeLike = (s) => s.replace(/[\\%_]/g, (m) => "\\" + m);
778
794
  // Segment-exact dedup (#189): an incremental capture POST carries
779
795
  // segment_id "<start>-<end>[.pN]". Skip iff THIS exact segment's chunks are
780
796
  // already stored (keys "<sid>#<segment_id>#<i>"). This is what lets a failed
781
797
  // segment be safely retried with the same id, and a legacy session-level row
782
798
  // (key "<sid>#<i>", no segment) does NOT match — so the #189 recovery
783
799
  // re-ingest is never blocked by night-1's whole-session rows.
800
+ // ALSO consults dedup_log (#100): a merged-away loser's source_session
801
+ // marker survives there after the `memories` row is deleted, so a
802
+ // `hicortex dedup --apply` merge can't be undone by a retried/recaptured
803
+ // segment silently re-ingesting the same content.
784
804
  if (session_id && segment_id) {
785
- const likePrefix = `${escapeLike(session_id)}#${escapeLike(segment_id)}#%`;
786
- const existing = db.prepare("SELECT COUNT(*) as c FROM memories WHERE source_session LIKE ? ESCAPE '\\'").get(likePrefix);
787
- if (existing.c > 0) {
788
- res.status(200).json({ skipped: true, existing_count: existing.c });
805
+ const existingCount = (0, dedup_js_1.countExistingSegment)(db, session_id, segment_id);
806
+ if (existingCount > 0) {
807
+ res.status(200).json({ skipped: true, existing_count: existingCount });
789
808
  return;
790
809
  }
791
810
  }
792
811
  // Session-level dedup: when session_id is present and this is a whole-session
793
812
  // POST (no segment_id — legacy ≤0.13.1 clients), skip if any chunk of this
794
- // session is already stored. Unchanged: legacy clients keep exact behaviour.
813
+ // session is already stored (memories OR dedup_log see above).
814
+ // Unchanged: legacy clients keep exact behaviour.
795
815
  if (session_id && !segment_id) {
796
- const likePrefix = `${escapeLike(session_id)}#%`;
797
- const existing = db.prepare("SELECT COUNT(*) as c FROM memories WHERE source_session = ? OR source_session LIKE ? ESCAPE '\\'").get(session_id, likePrefix);
798
- if (existing.c > 0) {
799
- res.status(200).json({ skipped: true, existing_count: existing.c });
816
+ const existingCount = (0, dedup_js_1.countExistingSession)(db, session_id);
817
+ if (existingCount > 0) {
818
+ res.status(200).json({ skipped: true, existing_count: existingCount });
800
819
  return;
801
820
  }
802
821
  }
@@ -0,0 +1,38 @@
1
+ /**
2
+ * Product-owned memory instructions (#192, owner decision 28.07.2026).
3
+ *
4
+ * The instructions for HOW agents use Hicortex are shipped BY the product,
5
+ * versioned with the server, and injected as a synthetic read-only `memory`
6
+ * section in the GET /context response. Rationale ("enforced, built-in"):
7
+ * - Harness personas (SOUL.md etc.) carry ZERO hicortex content — mechanics
8
+ * described there rot silently when the product changes (field evidence:
9
+ * stale "captured via hooks" sentences; an agent shell-spelunking its own
10
+ * plugin infrastructure when told "the plugin was updated").
11
+ * - User context files (user.md / rules.md) stay purely personal — norms the
12
+ * product depends on must not live in user-editable files (same principle
13
+ * as the built-in citation norm, 0.14.1).
14
+ * - Because every harness already renders `## Context` sections through the
15
+ * shared gate/render path, a synthetic section ships fleet-wide with zero
16
+ * client changes — including plugins that predate this feature.
17
+ *
18
+ * The section name is RESERVED: PUT /context rejects it, and the synthetic
19
+ * text overrides any user file of the same name (enforced means enforced).
20
+ * Off-switch: config `memoryInstructions: false`.
21
+ */
22
+ export declare const MEMORY_SECTION_NAME = "memory";
23
+ /** The product-authored instruction text. Keep compact (~120 tokens): it is
24
+ * injected once per session into every agent on the fleet. */
25
+ export declare function renderMemoryInstructions(): string;
26
+ /** True for the reserved product section name (case-insensitive guard —
27
+ * section names are lowercase by allowlist, but be safe). */
28
+ export declare function isReservedSectionName(name: unknown): boolean;
29
+ /**
30
+ * Inject the synthetic section into a successful GET /context body.
31
+ * Pure: returns the same body object with sections.memory set. Skips agent
32
+ * mode "off" (operator explicitly silenced context for that agent) and
33
+ * non-object bodies (error shapes). Overrides a user file named memory.md.
34
+ */
35
+ export declare function injectMemorySection<T extends {
36
+ sections?: Record<string, string>;
37
+ mode?: string;
38
+ }>(body: T, enabled: boolean): T;
@@ -0,0 +1,63 @@
1
+ "use strict";
2
+ /**
3
+ * Product-owned memory instructions (#192, owner decision 28.07.2026).
4
+ *
5
+ * The instructions for HOW agents use Hicortex are shipped BY the product,
6
+ * versioned with the server, and injected as a synthetic read-only `memory`
7
+ * section in the GET /context response. Rationale ("enforced, built-in"):
8
+ * - Harness personas (SOUL.md etc.) carry ZERO hicortex content — mechanics
9
+ * described there rot silently when the product changes (field evidence:
10
+ * stale "captured via hooks" sentences; an agent shell-spelunking its own
11
+ * plugin infrastructure when told "the plugin was updated").
12
+ * - User context files (user.md / rules.md) stay purely personal — norms the
13
+ * product depends on must not live in user-editable files (same principle
14
+ * as the built-in citation norm, 0.14.1).
15
+ * - Because every harness already renders `## Context` sections through the
16
+ * shared gate/render path, a synthetic section ships fleet-wide with zero
17
+ * client changes — including plugins that predate this feature.
18
+ *
19
+ * The section name is RESERVED: PUT /context rejects it, and the synthetic
20
+ * text overrides any user file of the same name (enforced means enforced).
21
+ * Off-switch: config `memoryInstructions: false`.
22
+ */
23
+ Object.defineProperty(exports, "__esModule", { value: true });
24
+ exports.MEMORY_SECTION_NAME = void 0;
25
+ exports.renderMemoryInstructions = renderMemoryInstructions;
26
+ exports.isReservedSectionName = isReservedSectionName;
27
+ exports.injectMemorySection = injectMemorySection;
28
+ exports.MEMORY_SECTION_NAME = "memory";
29
+ /** The product-authored instruction text. Keep compact (~120 tokens): it is
30
+ * injected once per session into every agent on the fleet. */
31
+ function renderMemoryInstructions() {
32
+ return [
33
+ "Your long-term memory is Hicortex — shared across all agents and sessions.",
34
+ "- A `## Memory recall (auto)` index may arrive with prompts: it is a MENU, not content. Fetch a full memory with `hicortex_get(id)` only when the entry is relevant to your current task.",
35
+ "- Recall before assuming: `hicortex_search` for prior decisions/facts/preferences, `hicortex_recent` to catch up on a project.",
36
+ "- Cite any memory you rely on (id, date); on conflicts, newer memories supersede older.",
37
+ "- Capture is automatic (nightly). Do not manually ingest routine content — `hicortex_ingest` is for explicitly requested learnings only.",
38
+ "- Never inspect, test, or modify memory/plugin/gateway infrastructure (configs, services, tokens). If a memory tool seems missing or broken, say so and stop.",
39
+ ].join("\n");
40
+ }
41
+ /** True for the reserved product section name (case-insensitive guard —
42
+ * section names are lowercase by allowlist, but be safe). */
43
+ function isReservedSectionName(name) {
44
+ return typeof name === "string" && name.trim().toLowerCase() === exports.MEMORY_SECTION_NAME;
45
+ }
46
+ /**
47
+ * Inject the synthetic section into a successful GET /context body.
48
+ * Pure: returns the same body object with sections.memory set. Skips agent
49
+ * mode "off" (operator explicitly silenced context for that agent) and
50
+ * non-object bodies (error shapes). Overrides a user file named memory.md.
51
+ */
52
+ function injectMemorySection(body, enabled) {
53
+ if (!enabled)
54
+ return body;
55
+ if (!body || typeof body !== "object")
56
+ return body;
57
+ if (body.mode === "off")
58
+ return body;
59
+ if (!body.sections || typeof body.sections !== "object")
60
+ return body;
61
+ body.sections[exports.MEMORY_SECTION_NAME] = renderMemoryInstructions();
62
+ return body;
63
+ }
package/dist/nightly.js CHANGED
@@ -390,6 +390,10 @@ async function runNightly(options = {}) {
390
390
  domains: cfgDomains,
391
391
  contentDomainsReady,
392
392
  weakPrimaryFloor: (0, nofit_js_1.resolveWeakPrimaryFloor)(savedConfig),
393
+ }, {
394
+ minSimilarity: savedConfig?.supersessionMinSimilarity,
395
+ maxCalls: savedConfig?.supersessionMaxCalls,
396
+ penalty: savedConfig?.supersessionPenalty,
393
397
  });
394
398
  console.log(`[hicortex] Consolidation ${report.status} in ${report.elapsed_seconds}s` +
395
399
  (report.stages.reflection ? ` (${report.stages.reflection.lessons_generated} lessons)` : ""));
package/dist/relink.d.ts CHANGED
@@ -37,7 +37,6 @@
37
37
  * Server-mode only: relink needs the local DB. Client installs must run it
38
38
  * on the server machine.
39
39
  */
40
- import type Database from "better-sqlite3";
41
40
  import type { EmbedFn } from "./retrieval.js";
42
41
  export interface RelinkOptions {
43
42
  /** Full discovery + would-be counts, zero writes, cursor untouched. */
@@ -87,8 +86,10 @@ export interface RelinkReport {
87
86
  /**
88
87
  * Read the stored embedding for a memory from memory_vectors.
89
88
  * Returns null when the row is missing (caller falls back to re-embedding).
89
+ * @deprecated moved to storage.ts (shared with consolidate.ts's supersession
90
+ * stage); re-exported here so existing importers of relink.ts keep working.
90
91
  */
91
- export declare function getStoredEmbedding(db: Database.Database, memoryId: string): Float32Array | null;
92
+ export { getStoredEmbedding } from "./storage.js";
92
93
  /**
93
94
  * Run the relink pass. Returns a structured report.
94
95
  * Throws on unrecoverable errors (client mode, DB write failure) — the cursor
package/dist/relink.js CHANGED
@@ -72,7 +72,7 @@ var __importStar = (this && this.__importStar) || (function () {
72
72
  };
73
73
  })();
74
74
  Object.defineProperty(exports, "__esModule", { value: true });
75
- exports.getStoredEmbedding = getStoredEmbedding;
75
+ exports.getStoredEmbedding = void 0;
76
76
  exports.runRelink = runRelink;
77
77
  const paths_js_1 = require("./paths.js");
78
78
  const node_fs_1 = require("node:fs");
@@ -107,16 +107,11 @@ function loadExistingPairs(db) {
107
107
  /**
108
108
  * Read the stored embedding for a memory from memory_vectors.
109
109
  * Returns null when the row is missing (caller falls back to re-embedding).
110
+ * @deprecated moved to storage.ts (shared with consolidate.ts's supersession
111
+ * stage); re-exported here so existing importers of relink.ts keep working.
110
112
  */
111
- function getStoredEmbedding(db, memoryId) {
112
- const row = db
113
- .prepare("SELECT embedding FROM memory_vectors WHERE id = ?")
114
- .get(memoryId);
115
- if (!row?.embedding)
116
- return null;
117
- const buf = row.embedding;
118
- return new Float32Array(buf.buffer, buf.byteOffset, buf.byteLength / 4);
119
- }
113
+ var storage_js_1 = require("./storage.js");
114
+ Object.defineProperty(exports, "getStoredEmbedding", { enumerable: true, get: function () { return storage_js_1.getStoredEmbedding; } });
120
115
  /**
121
116
  * Run the relink pass. Returns a structured report.
122
117
  * Throws on unrecoverable errors (client mode, DB write failure) — the cursor
@@ -193,7 +188,7 @@ async function runRelink(options = {}) {
193
188
  const { __rowid: _ignored, ...memRow } = row;
194
189
  const mem = memRow;
195
190
  try {
196
- let embedding = getStoredEmbedding(db, mem.id);
191
+ let embedding = storage.getStoredEmbedding(db, mem.id);
197
192
  if (!embedding) {
198
193
  embedding = await (await getEmbedFn())(mem.content);
199
194
  }
@@ -3,5 +3,5 @@
3
3
  * Shared between OC plugin and CC MCP server.
4
4
  */
5
5
  import type Database from "better-sqlite3";
6
- export declare const SEED_LESSON = "## Lesson: 1% Daily Self-Improvement\n\nSelf-improvement protocol: Review your sessions daily. Scan for mistakes, gaps, user corrections, and friction. Identify 1-2 issues maximum. Apply one targeted fix to your instruction files (AGENT.md, IDENTITY.md, TOOLS.md, SOUL.md, or CLAUDE.md). Never overcorrect \u2014 feedback is guidance, not emergency. One fix per issue. Never rewrite entire systems. Before changing something, ask: is this a real pattern or am I reacting to noise?\n\nThe flywheel: Sessions \u2192 Hicortex captures \u2192 Nightly consolidation \u2192 Lessons generated \u2192 Lessons injected into your context \u2192 You apply one fix \u2192 Better sessions tomorrow.\n\nUse hicortex_lessons to check for new insights. If a lesson has high confidence, apply it. If medium, observe for one more day. If nothing needs fixing today, skip \u2014 that's fine.\n\n**Severity:** critical\n**Confidence:** high\n**Generated:** seed-lesson";
6
+ export declare const SEED_LESSON = "## Lesson: 1% Daily Self-Improvement\n\nSelf-improvement protocol: Review your sessions daily. Scan for mistakes, gaps, user corrections, and friction. Identify 1-2 issues maximum. Apply one targeted fix to your agent instruction files (e.g. SOUL.md / MEMORY.md for Hermes profiles, CLAUDE.md for Claude Code). Never overcorrect \u2014 feedback is guidance, not emergency. One fix per issue. Never rewrite entire systems. Before changing something, ask: is this a real pattern or am I reacting to noise?\n\nThe flywheel: Sessions \u2192 Hicortex captures \u2192 Nightly consolidation \u2192 Lessons generated \u2192 Lessons injected into your context \u2192 You apply one fix \u2192 Better sessions tomorrow.\n\nUse hicortex_lessons to check for new insights. If a lesson has high confidence, apply it. If medium, observe for one more day. If nothing needs fixing today, skip \u2014 that's fine.\n\n**Severity:** critical\n**Confidence:** high\n**Generated:** seed-lesson";
7
7
  export declare function injectSeedLesson(database: Database.Database, log?: (msg: string) => void): Promise<void>;
@@ -43,7 +43,7 @@ const embedder_js_1 = require("./embedder.js");
43
43
  const storage = __importStar(require("./storage.js"));
44
44
  exports.SEED_LESSON = `## Lesson: 1% Daily Self-Improvement
45
45
 
46
- Self-improvement protocol: Review your sessions daily. Scan for mistakes, gaps, user corrections, and friction. Identify 1-2 issues maximum. Apply one targeted fix to your instruction files (AGENT.md, IDENTITY.md, TOOLS.md, SOUL.md, or CLAUDE.md). Never overcorrect — feedback is guidance, not emergency. One fix per issue. Never rewrite entire systems. Before changing something, ask: is this a real pattern or am I reacting to noise?
46
+ Self-improvement protocol: Review your sessions daily. Scan for mistakes, gaps, user corrections, and friction. Identify 1-2 issues maximum. Apply one targeted fix to your agent instruction files (e.g. SOUL.md / MEMORY.md for Hermes profiles, CLAUDE.md for Claude Code). Never overcorrect — feedback is guidance, not emergency. One fix per issue. Never rewrite entire systems. Before changing something, ask: is this a real pattern or am I reacting to noise?
47
47
 
48
48
  The flywheel: Sessions → Hicortex captures → Nightly consolidation → Lessons generated → Lessons injected into your context → You apply one fix → Better sessions tomorrow.
49
49
 
package/dist/state.d.ts CHANGED
@@ -50,6 +50,16 @@ export interface HicortexState {
50
50
  * Same discipline as relinkCursor.
51
51
  */
52
52
  domainCursor?: number;
53
+ /**
54
+ * Resume cursor for the nightly's supersession-detection stage (#191 Phase
55
+ * B) — highest memories.rowid whose decision/correction candidates have
56
+ * been evaluated (or infra-skipped) this run. Absent/0 = never run. Unlike
57
+ * relinkCursor/domainCursor (separate resumable CLI commands), this cursor
58
+ * advances a SMALL amount per night (config `supersessionMaxCalls`, default
59
+ * 30) as part of the regular nightly — the corpus is back-processed
60
+ * gradually over many nights.
61
+ */
62
+ supersessionCursor?: number;
53
63
  }
54
64
  /**
55
65
  * Load the state file. Returns an empty state if the file is missing
package/dist/storage.d.ts CHANGED
@@ -89,6 +89,16 @@ export declare function getMemoryTagsWeighted(db: Database.Database, memoryId: s
89
89
  tag: string;
90
90
  weight: number | null;
91
91
  }>;
92
+ /**
93
+ * Read the stored embedding for a memory from memory_vectors.
94
+ * Returns null when the row is missing (caller falls back to re-embedding).
95
+ *
96
+ * Shared by `hicortex relink` and the nightly's supersession stage
97
+ * (consolidate.ts) — lives here (not in relink.ts) so consolidate.ts can use
98
+ * it without importing from relink.ts, which itself imports from
99
+ * consolidate.ts (BudgetTracker, discoverLinkCandidates).
100
+ */
101
+ export declare function getStoredEmbedding(db: Database.Database, memoryId: string): Float32Array | null;
92
102
  /**
93
103
  * Find similar memories by vector distance. Returns memories with distance field.
94
104
  */