@gamaze/hicortex 0.10.1 → 0.11.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (49) hide show
  1. package/README.md +42 -0
  2. package/THIRD_PARTY_NOTICES.md +108 -0
  3. package/assets/vendor/3d-force-graph.min.js +5 -0
  4. package/assets/vendor/force-graph.min.js +5 -0
  5. package/assets/vendor/three.core.min.js +6 -0
  6. package/assets/vendor/three.module.min.js +6 -0
  7. package/assets/viz.html +1128 -0
  8. package/dist/classify-domains.d.ts +98 -0
  9. package/dist/classify-domains.js +340 -0
  10. package/dist/cli.d.ts +1 -0
  11. package/dist/cli.js +63 -0
  12. package/dist/consolidate.d.ts +139 -2
  13. package/dist/consolidate.js +302 -87
  14. package/dist/db.js +70 -0
  15. package/dist/domain-classify.d.ts +164 -0
  16. package/dist/domain-classify.js +300 -0
  17. package/dist/extensions.d.ts +12 -0
  18. package/dist/graph.d.ts +56 -0
  19. package/dist/graph.js +145 -0
  20. package/dist/index.js +1 -1
  21. package/dist/init.d.ts +25 -0
  22. package/dist/init.js +54 -0
  23. package/dist/lesson-selection.js +12 -5
  24. package/dist/lessons-context.js +2 -1
  25. package/dist/llm.d.ts +67 -0
  26. package/dist/llm.js +122 -0
  27. package/dist/mcp-server.js +86 -28
  28. package/dist/nightly-status.js +9 -28
  29. package/dist/nightly.js +42 -32
  30. package/dist/nofit.d.ts +111 -0
  31. package/dist/nofit.js +176 -0
  32. package/dist/prompts.d.ts +0 -5
  33. package/dist/prompts.js +5 -29
  34. package/dist/relink.d.ts +100 -0
  35. package/dist/relink.js +277 -0
  36. package/dist/retrieval.d.ts +16 -1
  37. package/dist/retrieval.js +34 -2
  38. package/dist/schema-prototypes.d.ts +149 -0
  39. package/dist/schema-prototypes.js +329 -0
  40. package/dist/state.d.ts +32 -0
  41. package/dist/state.js +29 -0
  42. package/dist/status.js +12 -19
  43. package/dist/storage.d.ts +44 -1
  44. package/dist/storage.js +70 -1
  45. package/dist/types.d.ts +90 -0
  46. package/dist/viz.d.ts +69 -0
  47. package/dist/viz.js +180 -0
  48. package/domains.example.json +36 -0
  49. package/package.json +6 -3
package/dist/prompts.js CHANGED
@@ -8,7 +8,6 @@ exports.importanceScoring = importanceScoring;
8
8
  exports.reflection = reflection;
9
9
  exports.distillation = distillation;
10
10
  exports.domainCuration = domainCuration;
11
- exports.edgeClassification = edgeClassification;
12
11
  /**
13
12
  * Importance scoring prompt. Takes a {memories_block} with indexed memories.
14
13
  */
@@ -170,31 +169,8 @@ Rules:
170
169
 
171
170
  Respond with ONLY a JSON array. No explanations.`;
172
171
  }
173
- /**
174
- * Edge classification prompt. Presents memory pairs and asks the LLM to
175
- * choose the most specific relationship type for each.
176
- */
177
- function edgeClassification(pairsBlock) {
178
- return `You are a memory graph analyst. Classify the relationship between each memory pair.
179
-
180
- VALID RELATIONSHIP TYPES:
181
- - derives: A lesson or fact was derived from episodes (lesson ← episode)
182
- - updates: A newer memory updates/replaces an older one on the same topic
183
- - extends: Memory adds detail to another within the same project
184
- - relates_to: Generic association (use ONLY when no specific type fits)
185
- - CONTRADICTS: Memories give opposite advice or conflicting information
186
- - SUPERSEDES: One memory fully replaces another (stronger than "updates")
187
- - DEPENDS_ON: One memory's validity requires the other (prerequisite)
188
- - CAUSED_BY: One event/decision directly caused the other
189
- - VALIDATES: One memory confirms or provides evidence for the other
190
-
191
- Choose the MOST SPECIFIC type. Prefer specific types over "relates_to".
192
-
193
- MEMORY PAIRS:
194
- ${pairsBlock}
195
-
196
- Respond with ONLY a JSON array of relationship type strings, one per pair, in order.
197
- Example for 3 pairs: ["CAUSED_BY", "extends", "VALIDATES"]
198
-
199
- No explanations. Just the JSON array.`;
200
- }
172
+ // edgeClassification prompt REMOVED (2026-07). LLM edge classification is
173
+ // retired: the 672-link audit found the LLM-classified UPPERCASE relationship
174
+ // types near-useless (CONTRADICTS 4% acceptable). Linking is now heuristic-only
175
+ // (extends/relates_to) in consolidate.ts. A classification prompt may return
176
+ // only when a future classifier passes the audit harness at >= 70% acceptable.
@@ -0,0 +1,100 @@
1
+ /**
2
+ * `hicortex relink` — deliberate, resumable link-discovery pass over the
3
+ * ENTIRE memories corpus (issue #143).
4
+ *
5
+ * The nightly's stageLinks only processes memories that are new since the
6
+ * last consolidation, so everything ingested before the TS linking stage
7
+ * (e.g. a migrated corpus) has never been through link discovery. This
8
+ * command back-fills the graph by reusing the exact same discovery +
9
+ * classification machinery (discoverLinkCandidates / classifyLinkCandidates
10
+ * from consolidate.ts) — no new linking logic.
11
+ *
12
+ * Classification is HEURISTIC-ONLY (2026-07). LLM edge classification was
13
+ * retired after the 672-link audit (see consolidate.ts Stage 3 header) found
14
+ * the LLM-classified UPPERCASE types near-useless. relink therefore never needs
15
+ * an LLM; the old `--no-llm`, `--max-llm-calls`, and `--llm-base-url/--llm-model`
16
+ * flags are gone. The classifyLinkCandidates call is retained (shared with the
17
+ * nightly) but takes no live LLM.
18
+ *
19
+ * Design:
20
+ * - Scope: all memories ordered by rowid, processed in batches (default 200).
21
+ * - Resumable: `relinkCursor` (last fully-committed rowid) is persisted in
22
+ * state.json after each batch. Interruption never loses more than the
23
+ * current batch. `--reset` restarts from rowid 0.
24
+ * - Candidates: reuses the STORED embedding from memory_vectors (no
25
+ * re-embedding); falls back to embedding the content only when a vector
26
+ * row is missing. Same rules as the nightly: top-10 neighbors, cosine
27
+ * above CONSOLIDATE_LINK_THRESHOLD (CROSS_PROJECT_LINK_THRESHOLD for
28
+ * cross-project pairs), capped at CONSOLIDATE_LINK_TOP_K.
29
+ * - Pair dedup: a candidate is skipped when a link already exists in EITHER
30
+ * direction (memory_links PK is directional, so reverse duplicates must
31
+ * be filtered here). Existing pairs are loaded once at start and the set
32
+ * is maintained as links are added during the run.
33
+ * - Classification: heuristic `extends`/`relates_to` only (classifyRelationship).
34
+ * - `--dry-run`: full discovery, zero writes, cursor untouched. Would-be
35
+ * links are reported with a breakdown by heuristic type.
36
+ *
37
+ * Server-mode only: relink needs the local DB. Client installs must run it
38
+ * on the server machine.
39
+ */
40
+ import type Database from "better-sqlite3";
41
+ import type { EmbedFn } from "./retrieval.js";
42
+ export interface RelinkOptions {
43
+ /** Full discovery + would-be counts, zero writes, cursor untouched. */
44
+ dryRun?: boolean;
45
+ /** Memories per batch (default 200). Cursor advances per committed batch. */
46
+ batchSize?: number;
47
+ /** Ignore the saved cursor and restart from rowid 0. */
48
+ reset?: boolean;
49
+ /** DB path override (tests). Defaults to resolveDbPath(). */
50
+ dbPath?: string;
51
+ /** State dir override (tests). Defaults to ~/.hicortex. */
52
+ stateDir?: string;
53
+ /**
54
+ * Fallback embedder for memories missing a memory_vectors row (tests).
55
+ * Defaults to the local ONNX embedder, loaded lazily on first miss.
56
+ */
57
+ embedFn?: EmbedFn;
58
+ }
59
+ export interface RelinkReport {
60
+ dryRun: boolean;
61
+ /** Memories examined in this invocation. */
62
+ scanned: number;
63
+ /** Candidate pairs above the similarity threshold (before dedup). */
64
+ candidatesFound: number;
65
+ /** Candidates skipped: link existed in either direction BEFORE this run. */
66
+ skippedExisting: number;
67
+ /** Candidates skipped: pair already handled earlier in this run (a pair is
68
+ * discovered from both sides — the reverse discovery is not a new link). */
69
+ skippedDuplicate: number;
70
+ /** Links created (or, in dry-run, links that would be created). */
71
+ linksCreated: number;
72
+ /** Candidates classified by an LLM. Always 0 — LLM classification retired. */
73
+ llmClassified: number;
74
+ /** Candidates classified by the heuristic (always all of them now). */
75
+ heuristicFallback: number;
76
+ /** Final relationship-type breakdown of created (or would-be) links. */
77
+ byType: Record<string, number>;
78
+ /** Memories whose discovery failed (missing vector AND embed failure). */
79
+ failed: number;
80
+ /** Batches processed. */
81
+ batches: number;
82
+ /** Cursor after this run (unchanged in dry-run). */
83
+ cursor: number;
84
+ /** Why the run ended. Only "complete" now (the LLM budget path is retired). */
85
+ stoppedReason: "complete";
86
+ }
87
+ /**
88
+ * Read the stored embedding for a memory from memory_vectors.
89
+ * Returns null when the row is missing (caller falls back to re-embedding).
90
+ */
91
+ export declare function getStoredEmbedding(db: Database.Database, memoryId: string): Float32Array | null;
92
+ /**
93
+ * Run the relink pass. Returns a structured report.
94
+ * Throws on unrecoverable errors (client mode, DB write failure) — the cursor
95
+ * always reflects the last committed batch.
96
+ *
97
+ * Classification is heuristic-only (LLM edge classification retired 2026-07),
98
+ * so relink never contacts an LLM.
99
+ */
100
+ export declare function runRelink(options?: RelinkOptions): Promise<RelinkReport>;
package/dist/relink.js ADDED
@@ -0,0 +1,277 @@
1
+ "use strict";
2
+ /**
3
+ * `hicortex relink` — deliberate, resumable link-discovery pass over the
4
+ * ENTIRE memories corpus (issue #143).
5
+ *
6
+ * The nightly's stageLinks only processes memories that are new since the
7
+ * last consolidation, so everything ingested before the TS linking stage
8
+ * (e.g. a migrated corpus) has never been through link discovery. This
9
+ * command back-fills the graph by reusing the exact same discovery +
10
+ * classification machinery (discoverLinkCandidates / classifyLinkCandidates
11
+ * from consolidate.ts) — no new linking logic.
12
+ *
13
+ * Classification is HEURISTIC-ONLY (2026-07). LLM edge classification was
14
+ * retired after the 672-link audit (see consolidate.ts Stage 3 header) found
15
+ * the LLM-classified UPPERCASE types near-useless. relink therefore never needs
16
+ * an LLM; the old `--no-llm`, `--max-llm-calls`, and `--llm-base-url/--llm-model`
17
+ * flags are gone. The classifyLinkCandidates call is retained (shared with the
18
+ * nightly) but takes no live LLM.
19
+ *
20
+ * Design:
21
+ * - Scope: all memories ordered by rowid, processed in batches (default 200).
22
+ * - Resumable: `relinkCursor` (last fully-committed rowid) is persisted in
23
+ * state.json after each batch. Interruption never loses more than the
24
+ * current batch. `--reset` restarts from rowid 0.
25
+ * - Candidates: reuses the STORED embedding from memory_vectors (no
26
+ * re-embedding); falls back to embedding the content only when a vector
27
+ * row is missing. Same rules as the nightly: top-10 neighbors, cosine
28
+ * above CONSOLIDATE_LINK_THRESHOLD (CROSS_PROJECT_LINK_THRESHOLD for
29
+ * cross-project pairs), capped at CONSOLIDATE_LINK_TOP_K.
30
+ * - Pair dedup: a candidate is skipped when a link already exists in EITHER
31
+ * direction (memory_links PK is directional, so reverse duplicates must
32
+ * be filtered here). Existing pairs are loaded once at start and the set
33
+ * is maintained as links are added during the run.
34
+ * - Classification: heuristic `extends`/`relates_to` only (classifyRelationship).
35
+ * - `--dry-run`: full discovery, zero writes, cursor untouched. Would-be
36
+ * links are reported with a breakdown by heuristic type.
37
+ *
38
+ * Server-mode only: relink needs the local DB. Client installs must run it
39
+ * on the server machine.
40
+ */
41
+ var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
42
+ if (k2 === undefined) k2 = k;
43
+ var desc = Object.getOwnPropertyDescriptor(m, k);
44
+ if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
45
+ desc = { enumerable: true, get: function() { return m[k]; } };
46
+ }
47
+ Object.defineProperty(o, k2, desc);
48
+ }) : (function(o, m, k, k2) {
49
+ if (k2 === undefined) k2 = k;
50
+ o[k2] = m[k];
51
+ }));
52
+ var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
53
+ Object.defineProperty(o, "default", { enumerable: true, value: v });
54
+ }) : function(o, v) {
55
+ o["default"] = v;
56
+ });
57
+ var __importStar = (this && this.__importStar) || (function () {
58
+ var ownKeys = function(o) {
59
+ ownKeys = Object.getOwnPropertyNames || function (o) {
60
+ var ar = [];
61
+ for (var k in o) if (Object.prototype.hasOwnProperty.call(o, k)) ar[ar.length] = k;
62
+ return ar;
63
+ };
64
+ return ownKeys(o);
65
+ };
66
+ return function (mod) {
67
+ if (mod && mod.__esModule) return mod;
68
+ var result = {};
69
+ if (mod != null) for (var k = ownKeys(mod), i = 0; i < k.length; i++) if (k[i] !== "default") __createBinding(result, mod, k[i]);
70
+ __setModuleDefault(result, mod);
71
+ return result;
72
+ };
73
+ })();
74
+ Object.defineProperty(exports, "__esModule", { value: true });
75
+ exports.getStoredEmbedding = getStoredEmbedding;
76
+ exports.runRelink = runRelink;
77
+ const node_fs_1 = require("node:fs");
78
+ const node_path_1 = require("node:path");
79
+ const node_os_1 = require("node:os");
80
+ const db_js_1 = require("./db.js");
81
+ const storage = __importStar(require("./storage.js"));
82
+ const consolidate_js_1 = require("./consolidate.js");
83
+ const state_js_1 = require("./state.js");
84
+ const HICORTEX_HOME = (0, node_path_1.join)((0, node_os_1.homedir)(), ".hicortex");
85
+ function readConfig(stateDir) {
86
+ try {
87
+ return JSON.parse((0, node_fs_1.readFileSync)((0, node_path_1.join)(stateDir, "config.json"), "utf-8"));
88
+ }
89
+ catch {
90
+ return null;
91
+ }
92
+ }
93
+ /** Canonical unordered key for a memory pair — direction-insensitive dedup. */
94
+ function pairKey(a, b) {
95
+ return a < b ? `${a}|${b}` : `${b}|${a}`;
96
+ }
97
+ /** Load every existing link as an unordered pair set (both directions collapse). */
98
+ function loadExistingPairs(db) {
99
+ const rows = db
100
+ .prepare("SELECT source_id, target_id FROM memory_links")
101
+ .all();
102
+ const pairs = new Set();
103
+ for (const row of rows)
104
+ pairs.add(pairKey(row.source_id, row.target_id));
105
+ return pairs;
106
+ }
107
+ /**
108
+ * Read the stored embedding for a memory from memory_vectors.
109
+ * Returns null when the row is missing (caller falls back to re-embedding).
110
+ */
111
+ function getStoredEmbedding(db, memoryId) {
112
+ const row = db
113
+ .prepare("SELECT embedding FROM memory_vectors WHERE id = ?")
114
+ .get(memoryId);
115
+ if (!row?.embedding)
116
+ return null;
117
+ const buf = row.embedding;
118
+ return new Float32Array(buf.buffer, buf.byteOffset, buf.byteLength / 4);
119
+ }
120
+ /**
121
+ * Run the relink pass. Returns a structured report.
122
+ * Throws on unrecoverable errors (client mode, DB write failure) — the cursor
123
+ * always reflects the last committed batch.
124
+ *
125
+ * Classification is heuristic-only (LLM edge classification retired 2026-07),
126
+ * so relink never contacts an LLM.
127
+ */
128
+ async function runRelink(options = {}) {
129
+ const dryRun = options.dryRun ?? false;
130
+ const batchSize = options.batchSize ?? 200;
131
+ const stateDir = options.stateDir ?? HICORTEX_HOME;
132
+ if (!Number.isInteger(batchSize) || batchSize < 1) {
133
+ throw new Error(`[hicortex] relink: invalid --batch value: ${options.batchSize}`);
134
+ }
135
+ // Server-mode only — client installs have no local DB.
136
+ const config = readConfig(stateDir);
137
+ if (config?.mode === "client") {
138
+ throw new Error("[hicortex] relink is server-mode only (it needs the local DB). " +
139
+ `This machine is a client of ${config.serverUrl ?? "a remote server"} — run relink on the server.`);
140
+ }
141
+ // Classification is heuristic-only — no LLM client, no budget cap.
142
+ const budget = new consolidate_js_1.BudgetTracker(Number.MAX_SAFE_INTEGER);
143
+ const dbPath = (0, db_js_1.resolveDbPath)(options.dbPath);
144
+ const db = (0, db_js_1.initDb)(dbPath);
145
+ const report = {
146
+ dryRun,
147
+ scanned: 0,
148
+ candidatesFound: 0,
149
+ skippedExisting: 0,
150
+ skippedDuplicate: 0,
151
+ linksCreated: 0,
152
+ llmClassified: 0,
153
+ heuristicFallback: 0,
154
+ byType: {},
155
+ failed: 0,
156
+ batches: 0,
157
+ cursor: 0,
158
+ stoppedReason: "complete",
159
+ };
160
+ try {
161
+ const totalMemories = storage.countMemories(db);
162
+ let cursor = options.reset ? 0 : ((0, state_js_1.loadState)(stateDir).relinkCursor ?? 0);
163
+ report.cursor = cursor;
164
+ console.log(`[hicortex] relink starting (${dryRun ? "dry-run" : "heuristic"}): ${totalMemories} memories, ` +
165
+ `batch ${batchSize}, cursor ${cursor}${options.reset ? " (reset)" : ""}`);
166
+ // Load all existing links once as an unordered pair set (471 rows on the
167
+ // live corpus — trivial). `seenPairs` additionally accumulates pairs
168
+ // handled during this run so re-runs AND within-run reverse candidates
169
+ // never create duplicate edges.
170
+ const preExistingPairs = loadExistingPairs(db);
171
+ const seenPairs = new Set(preExistingPairs);
172
+ const batchStmt = db.prepare("SELECT rowid AS __rowid, * FROM memories WHERE rowid > ? ORDER BY rowid ASC LIMIT ?");
173
+ // Lazy fallback embedder — only loaded if a memory_vectors row is missing.
174
+ let embedFn = options.embedFn ?? null;
175
+ const getEmbedFn = async () => {
176
+ if (!embedFn) {
177
+ const { embed } = await import("./embedder.js");
178
+ embedFn = embed;
179
+ }
180
+ return embedFn;
181
+ };
182
+ for (;;) {
183
+ const rows = batchStmt.all(cursor, batchSize);
184
+ if (rows.length === 0)
185
+ break;
186
+ const lastRowid = rows[rows.length - 1].__rowid;
187
+ // Phase A: discovery — reuse stored embeddings, same candidate rules
188
+ // as the nightly (discoverLinkCandidates).
189
+ const candidates = [];
190
+ let batchSkippedExisting = 0;
191
+ let batchSkippedDuplicate = 0;
192
+ for (const row of rows) {
193
+ const { __rowid: _ignored, ...memRow } = row;
194
+ const mem = memRow;
195
+ try {
196
+ let embedding = getStoredEmbedding(db, mem.id);
197
+ if (!embedding) {
198
+ embedding = await (await getEmbedFn())(mem.content);
199
+ }
200
+ for (const cand of (0, consolidate_js_1.discoverLinkCandidates)(db, mem, embedding)) {
201
+ report.candidatesFound++;
202
+ const key = pairKey(cand.source.id, cand.target.id);
203
+ if (seenPairs.has(key)) {
204
+ if (preExistingPairs.has(key))
205
+ batchSkippedExisting++;
206
+ else
207
+ batchSkippedDuplicate++;
208
+ continue;
209
+ }
210
+ // Reserve the pair immediately so the reverse direction later in
211
+ // this run (or this batch) is deduped too.
212
+ seenPairs.add(key);
213
+ candidates.push(cand);
214
+ }
215
+ }
216
+ catch {
217
+ report.failed++;
218
+ }
219
+ }
220
+ report.skippedExisting += batchSkippedExisting;
221
+ report.skippedDuplicate += batchSkippedDuplicate;
222
+ // Phase B: classification — shared heuristic-only path (LLM retired).
223
+ // classifyLinkCandidates ignores the null LLM/budget and returns each
224
+ // candidate's heuristic type (extends/relates_to).
225
+ const classified = await (0, consolidate_js_1.classifyLinkCandidates)(candidates, null, budget);
226
+ const types = classified.types;
227
+ report.llmClassified += classified.llmClassified; // always 0
228
+ report.heuristicFallback += classified.heuristicFallback;
229
+ // Phase C: store — one transaction per batch, then persist the cursor.
230
+ // A crash between commit and cursor save is safe: the re-run re-scans
231
+ // the batch but the pair dedup (loaded from the DB) skips every link.
232
+ if (!dryRun) {
233
+ const tx = db.transaction(() => {
234
+ for (let i = 0; i < candidates.length; i++) {
235
+ const c = candidates[i];
236
+ storage.addLink(db, c.source.id, c.target.id, types[i], c.similarity);
237
+ }
238
+ });
239
+ tx();
240
+ (0, state_js_1.updateState)((s) => {
241
+ s.relinkCursor = lastRowid;
242
+ }, stateDir);
243
+ report.cursor = lastRowid;
244
+ }
245
+ for (const t of types)
246
+ report.byType[t] = (report.byType[t] ?? 0) + 1;
247
+ report.linksCreated += candidates.length;
248
+ report.scanned += rows.length;
249
+ report.batches++;
250
+ cursor = lastRowid;
251
+ console.log(`[hicortex] batch ${report.batches}: scanned ${rows.length}, ` +
252
+ `candidates ${candidates.length + batchSkippedExisting + batchSkippedDuplicate}, ` +
253
+ `skipped ${batchSkippedExisting} already-linked + ${batchSkippedDuplicate} duplicate, ` +
254
+ `${dryRun ? "would create" : "created"} ${candidates.length} ` +
255
+ `(cursor ${lastRowid}, ${report.scanned} scanned this run)`);
256
+ }
257
+ // Final summary
258
+ const typeBreakdown = Object.entries(report.byType)
259
+ .sort((a, b) => b[1] - a[1])
260
+ .map(([t, n]) => `${t}=${n}`)
261
+ .join(", ") || "none";
262
+ console.log(`[hicortex] relink complete` +
263
+ `${dryRun ? " (dry-run, nothing written)" : ""}: ` +
264
+ `${report.scanned} memories scanned, ${report.candidatesFound} candidates, ` +
265
+ `${report.skippedExisting} already linked, ${report.skippedDuplicate} reverse-duplicates, ` +
266
+ `${report.linksCreated} links ${dryRun ? "would be created" : "created"} ` +
267
+ `(heuristic ${report.heuristicFallback})`);
268
+ console.log(`[hicortex] by type: ${typeBreakdown}`);
269
+ if (report.failed > 0) {
270
+ console.warn(`[hicortex] ${report.failed} memories failed discovery (see errors above)`);
271
+ }
272
+ return report;
273
+ }
274
+ finally {
275
+ db.close();
276
+ }
277
+ }
@@ -14,7 +14,17 @@
14
14
  * effective = floor + (base - floor) * decay_rate^hours
15
15
  */
16
16
  import type Database from "better-sqlite3";
17
- import type { MemorySearchResult } from "./types.js";
17
+ import type { Memory, MemorySearchResult } from "./types.js";
18
+ /**
19
+ * Convert an L2 distance (as returned by sqlite-vec's vec0 `distance`) to
20
+ * cosine similarity. Valid because our embeddings are L2-normalized
21
+ * (embedder.ts, `normalize: true`): for unit vectors, d² = 2 − 2·cos,
22
+ * hence cos = 1 − d²/2. Exact anchors: d=0 → 1, d=√2 → 0, d=2 → −1.
23
+ *
24
+ * Lives here (the dependency-root of the scoring code) and is re-exported
25
+ * by consolidate.ts so pre-#145 importers keep working.
26
+ */
27
+ export declare function l2ToCosine(distance: number): number;
18
28
  /**
19
29
  * Compute decayed strength with adaptive decay (B+E+D model).
20
30
  * Exported for use by consolidation decay/prune stage.
@@ -24,6 +34,11 @@ export declare function effectiveStrength(baseStrength: number, lastAccessed: st
24
34
  accessCount?: number;
25
35
  linkCount?: number;
26
36
  }): number;
37
+ /**
38
+ * Return a composite relevance score in [0, 1] for a candidate memory.
39
+ * Exported for exact-value tests of the similarity component (#145).
40
+ */
41
+ export declare function computeScore(memory: Memory, distance: number, connectionCount: number, maxConnections: number, now: Date): number;
27
42
  export interface EmbedFn {
28
43
  (text: string): Promise<Float32Array>;
29
44
  }
package/dist/retrieval.js CHANGED
@@ -48,13 +48,35 @@ var __importStar = (this && this.__importStar) || (function () {
48
48
  };
49
49
  })();
50
50
  Object.defineProperty(exports, "__esModule", { value: true });
51
+ exports.l2ToCosine = l2ToCosine;
51
52
  exports.effectiveStrength = effectiveStrength;
53
+ exports.computeScore = computeScore;
52
54
  exports.retrieve = retrieve;
53
55
  exports.searchContext = searchContext;
54
56
  const storage = __importStar(require("./storage.js"));
55
57
  const BASE_DECAY = 0.0005;
56
- const DEFAULT_GRAPH_DISTANCE = 0.5;
58
+ /**
59
+ * Placeholder L2 distance for candidates that have no measured vector
60
+ * distance (FTS-only hits and graph-discovered neighbors). Chosen so that
61
+ * l2ToCosine(1.0) = 0.5 — a neutral mid-scale similarity. Before the #145
62
+ * fix the value was 0.5 on the accidental 1−L2 scale, which also yielded
63
+ * similarity 0.5; keeping 0.5 under the corrected formula would have jumped
64
+ * these candidates to cosine 0.875, outranking most true vector matches.
65
+ */
66
+ const DEFAULT_GRAPH_DISTANCE = 1.0;
57
67
  const RRF_K = 60;
68
+ /**
69
+ * Convert an L2 distance (as returned by sqlite-vec's vec0 `distance`) to
70
+ * cosine similarity. Valid because our embeddings are L2-normalized
71
+ * (embedder.ts, `normalize: true`): for unit vectors, d² = 2 − 2·cos,
72
+ * hence cos = 1 − d²/2. Exact anchors: d=0 → 1, d=√2 → 0, d=2 → −1.
73
+ *
74
+ * Lives here (the dependency-root of the scoring code) and is re-exported
75
+ * by consolidate.ts so pre-#145 importers keep working.
76
+ */
77
+ function l2ToCosine(distance) {
78
+ return 1 - (distance * distance) / 2;
79
+ }
58
80
  // ---------------------------------------------------------------------------
59
81
  // Timestamp parsing
60
82
  // ---------------------------------------------------------------------------
@@ -96,9 +118,19 @@ function effectiveStrength(baseStrength, lastAccessed, now, options) {
96
118
  }
97
119
  /**
98
120
  * Return a composite relevance score in [0, 1] for a candidate memory.
121
+ * Exported for exact-value tests of the similarity component (#145).
99
122
  */
100
123
  function computeScore(memory, distance, connectionCount, maxConnections, now) {
101
- const similarity = Math.max(0, 1.0 - distance);
124
+ // TRUE cosine similarity (#145). The old `1 distance` compressed real
125
+ // cosines (cos 0.8 scored 0.37) and the 0-clamp at that scale flattened
126
+ // everything below cos 0.5 to exactly 0, killing mid-relevance
127
+ // discrimination. The clamp stays at 0 — a negative cosine means truly
128
+ // unrelated — but now at the correct scale. NOTE: the similarity values
129
+ // roughly DOUBLE for related content on the new scale; the blend weights
130
+ // below are deliberately unchanged in this pass so the before/after
131
+ // retrieval comparison is measured, not guessed. Rebalancing the weights
132
+ // is a data-driven follow-up if the eval shows it is needed.
133
+ const similarity = Math.max(0, l2ToCosine(distance));
102
134
  const effStrength = effectiveStrength(memory.base_strength ?? 0.5, memory.last_accessed, now, {
103
135
  accessCount: memory.access_count ?? 0,
104
136
  linkCount: connectionCount,
@@ -0,0 +1,149 @@
1
+ /**
2
+ * Graded schema membership — domain prototypes + per-tag association weights
3
+ * (spec: specs/2026-07-07-graded-schema-memory-tags.md).
4
+ *
5
+ * MODEL (cognitive grounding → mechanism)
6
+ * ---------------------------------------
7
+ * A configured domain is a SCHEMA with graded membership (Rosch prototypes):
8
+ * - prototype(domain) = L2-normalized mean of the embeddings of memories
9
+ * whose tag set includes the domain. Cold start / thin domains
10
+ * (member_count < PROTOTYPE_MIN_MEMBERS) seed the prototype from the
11
+ * embedding of the domain's config description instead.
12
+ * - weight(memory, tag) = cosine(memory embedding, prototype(tag)). Both
13
+ * vectors are L2-normalized, so cosine reduces to a dot product.
14
+ * - PRIMARY (memories.domain) = argmax-weight tag, overridden by any tagged
15
+ * domain flagged `compartment: true` (deliberate compartmentalization —
16
+ * the owner's Work firewall). Fully mechanical, no LLM.
17
+ *
18
+ * The LLM decides ONLY the discrete part (which schemas apply — see
19
+ * domain-classify.ts); ALL gradation is derived from embeddings here.
20
+ * Prototypes + weights are recomputed each nightly, so categories drift with
21
+ * the data ("reconsolidation") without any re-classification runs.
22
+ *
23
+ * Persistence: `domain_prototypes(domain, embedding, member_count, updated_at)`
24
+ * and `memory_tags.weight` (both migration v7).
25
+ */
26
+ import type Database from "better-sqlite3";
27
+ import type { DomainDef } from "./types.js";
28
+ import type { EmbedFn } from "./retrieval.js";
29
+ /**
30
+ * Below this member count a domain's prototype is seeded from its config
31
+ * description instead of the member centroid (cold start / thin domains).
32
+ */
33
+ export declare const PROTOTYPE_MIN_MEMBERS = 5;
34
+ /** Deserialize a BLOB column into a Float32Array (same layout as sqlite-vec). */
35
+ export declare function blobToVec(buf: Buffer): Float32Array;
36
+ /**
37
+ * L2-normalize a vector IN A COPY. A zero vector (norm 0 — e.g. test fixtures)
38
+ * is returned as an all-zero copy rather than dividing by zero.
39
+ */
40
+ export declare function l2Normalize(vec: Float32Array): Float32Array;
41
+ /**
42
+ * Association weight of a memory for a tag = cosine(memory embedding, domain
43
+ * prototype). Both inputs are L2-normalized (embedder.ts normalizes memory
44
+ * embeddings; computeDomainPrototypes normalizes prototypes), so cosine is
45
+ * exactly the dot product.
46
+ */
47
+ export declare function tagWeight(memoryEmbedding: Float32Array, prototype: Float32Array): number;
48
+ /** One tag with its association weight (null = never computed / no prototype). */
49
+ export interface WeightedTag {
50
+ tag: string;
51
+ weight: number | null;
52
+ }
53
+ /** The configured compartment domain names (DomainDef.compartment === true). */
54
+ export declare function compartmentSet(domains: DomainDef[]): Set<string>;
55
+ /**
56
+ * Derive the PRIMARY tag (memories.domain) from a weighted tag set.
57
+ *
58
+ * Rules (deterministic, no LLM):
59
+ * 1. Any tagged compartment domain wins — first one in array order if the
60
+ * (unusual) case of several arises.
61
+ * 2. Else the argmax-weight tag. `tags` MUST be in LLM most-relevant-first
62
+ * order: ties (and all-null weights) resolve to the EARLIEST array
63
+ * position — strict `>` comparison keeps the first maximum.
64
+ * 3. A null weight loses to any numeric weight (treated as -Infinity).
65
+ *
66
+ * Throws on an empty tag set — callers guarantee >= 1 tag (an empty tag set
67
+ * from the classifier is a NO-FIT and must be routed through nofit.ts, never
68
+ * here); an empty set reaching this function is a programming error.
69
+ */
70
+ export declare function derivePrimary(tags: WeightedTag[], compartments: Set<string>): string;
71
+ export interface PrototypeStat {
72
+ domain: string;
73
+ memberCount: number;
74
+ /** true = description seed (member_count < PROTOTYPE_MIN_MEMBERS). */
75
+ seeded: boolean;
76
+ }
77
+ /** Load all stored prototypes into a name → vector map. */
78
+ export declare function loadDomainPrototypes(db: Database.Database): Map<string, Float32Array>;
79
+ /**
80
+ * Compute + persist the prototype of every configured domain.
81
+ *
82
+ * Per domain: mean of the embeddings of memories whose memory_tags include the
83
+ * domain, L2-normalized. When member_count < PROTOTYPE_MIN_MEMBERS the
84
+ * prototype is instead the embedding of `"<Name>: <description>"` (the same
85
+ * "name: description" line the classifier prompt shows) — this seeds cold
86
+ * starts and keeps thin domains from collapsing onto 1-2 outliers.
87
+ *
88
+ * `getEmbedFn` is LAZY (a function returning a promise of the embedder) so the
89
+ * ~130 MB ONNX embedder is loaded ONLY when at least one domain actually needs
90
+ * a description seed — same pattern as relink.ts's lazy fallback embedder.
91
+ *
92
+ * Returns the fresh prototype map (for immediate classification-time weights)
93
+ * plus per-domain stats. Rows in domain_prototypes are REPLACED per domain;
94
+ * domains removed from the config keep a stale row until the next config-set
95
+ * change (harmless — nothing reads prototypes outside the configured set).
96
+ */
97
+ export declare function computeDomainPrototypes(db: Database.Database, domains: DomainDef[], getEmbedFn: () => Promise<EmbedFn>): Promise<{
98
+ prototypes: Map<string, Float32Array>;
99
+ stats: PrototypeStat[];
100
+ }>;
101
+ /**
102
+ * Compute the per-tag weights for ONE memory from the current prototypes
103
+ * (classification-time path: newly tagged memories get weights immediately).
104
+ * A missing memory vector or missing prototype yields null (stored as NULL;
105
+ * repaired by the next nightly recompute).
106
+ */
107
+ export declare function computeTagWeights(db: Database.Database, memoryId: string, tags: string[], prototypes: Map<string, Float32Array>): Record<string, number | null>;
108
+ /**
109
+ * Best prototype match for one memory across ALL configured domains:
110
+ * argmax of cosine(memory embedding, prototype(domain)).
111
+ *
112
+ * Used by the no-fit path (nofit.ts, owner amendment 07.07): when the LLM
113
+ * says no domain fits, the memory can still earn a WEAK primary from pure
114
+ * embedding association — provided the best cosine clears the configured
115
+ * weakPrimaryFloor (the caller checks the floor; this function just reports
116
+ * the argmax).
117
+ *
118
+ * Returns null when the memory has no stored vector or no configured domain
119
+ * has a prototype (nothing to associate against). Ties resolve to the FIRST
120
+ * domain in config order (strict `>` comparison), mirroring derivePrimary.
121
+ */
122
+ export declare function bestPrototypeMatch(db: Database.Database, memoryId: string, domains: DomainDef[], prototypes: Map<string, Float32Array>): {
123
+ domain: string;
124
+ weight: number;
125
+ } | null;
126
+ /**
127
+ * One pass over ALL memory_tags rows: weight = cosine(memory embedding,
128
+ * prototype(tag)). Rows whose memory has no stored vector, or whose tag has no
129
+ * prototype (out-of-vocabulary leftovers), are set to NULL. Cheap: one point
130
+ * lookup per distinct memory + one dot product per tag row (3–5k on the
131
+ * production corpus).
132
+ */
133
+ export declare function recomputeAllTagWeights(db: Database.Database, prototypes: Map<string, Float32Array>): {
134
+ updated: number;
135
+ nulled: number;
136
+ };
137
+ /**
138
+ * Re-derive the PRIMARY (memories.domain) of every tagged memory from its
139
+ * current tag weights: compartment override first, else argmax weight, LLM
140
+ * order (memory_tags insertion order = rowid, written most-relevant-first by
141
+ * storage.setMemoryTags) breaking exact-weight ties.
142
+ *
143
+ * Memories with NO memory_tags rows are untouched (e.g. infra-skipped rows
144
+ * awaiting classification — issue #150 discipline).
145
+ */
146
+ export declare function refreshPrimaries(db: Database.Database, domains: DomainDef[]): {
147
+ examined: number;
148
+ updated: number;
149
+ };