@gamaze/hicortex 0.10.0 → 0.11.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +58 -1
- package/THIRD_PARTY_NOTICES.md +108 -0
- package/assets/vendor/3d-force-graph.min.js +5 -0
- package/assets/vendor/force-graph.min.js +5 -0
- package/assets/vendor/three.core.min.js +6 -0
- package/assets/vendor/three.module.min.js +6 -0
- package/assets/viz.html +1126 -0
- package/dist/classify-domains.d.ts +98 -0
- package/dist/classify-domains.js +340 -0
- package/dist/cli.d.ts +1 -0
- package/dist/cli.js +63 -0
- package/dist/consolidate.d.ts +139 -2
- package/dist/consolidate.js +302 -87
- package/dist/db.js +70 -0
- package/dist/domain-classify.d.ts +164 -0
- package/dist/domain-classify.js +300 -0
- package/dist/extensions.d.ts +12 -0
- package/dist/graph.d.ts +56 -0
- package/dist/graph.js +145 -0
- package/dist/index.js +1 -1
- package/dist/init.d.ts +25 -0
- package/dist/init.js +54 -0
- package/dist/lesson-selection.js +12 -5
- package/dist/lessons-context.js +2 -1
- package/dist/llm.d.ts +67 -0
- package/dist/llm.js +122 -0
- package/dist/mcp-server.js +82 -27
- package/dist/nightly-status.js +9 -28
- package/dist/nightly.js +42 -32
- package/dist/nofit.d.ts +111 -0
- package/dist/nofit.js +176 -0
- package/dist/prompts.d.ts +0 -5
- package/dist/prompts.js +5 -29
- package/dist/relink.d.ts +100 -0
- package/dist/relink.js +277 -0
- package/dist/retrieval.d.ts +16 -1
- package/dist/retrieval.js +34 -2
- package/dist/schema-prototypes.d.ts +149 -0
- package/dist/schema-prototypes.js +329 -0
- package/dist/state.d.ts +32 -0
- package/dist/state.js +29 -0
- package/dist/status.js +12 -19
- package/dist/storage.d.ts +44 -1
- package/dist/storage.js +70 -1
- package/dist/types.d.ts +90 -0
- package/dist/viz.d.ts +69 -0
- package/dist/viz.js +180 -0
- package/domains.example.json +36 -0
- package/package.json +6 -3
package/dist/prompts.js
CHANGED
|
@@ -8,7 +8,6 @@ exports.importanceScoring = importanceScoring;
|
|
|
8
8
|
exports.reflection = reflection;
|
|
9
9
|
exports.distillation = distillation;
|
|
10
10
|
exports.domainCuration = domainCuration;
|
|
11
|
-
exports.edgeClassification = edgeClassification;
|
|
12
11
|
/**
|
|
13
12
|
* Importance scoring prompt. Takes a {memories_block} with indexed memories.
|
|
14
13
|
*/
|
|
@@ -170,31 +169,8 @@ Rules:
|
|
|
170
169
|
|
|
171
170
|
Respond with ONLY a JSON array. No explanations.`;
|
|
172
171
|
}
|
|
173
|
-
|
|
174
|
-
|
|
175
|
-
|
|
176
|
-
|
|
177
|
-
|
|
178
|
-
return `You are a memory graph analyst. Classify the relationship between each memory pair.
|
|
179
|
-
|
|
180
|
-
VALID RELATIONSHIP TYPES:
|
|
181
|
-
- derives: A lesson or fact was derived from episodes (lesson ← episode)
|
|
182
|
-
- updates: A newer memory updates/replaces an older one on the same topic
|
|
183
|
-
- extends: Memory adds detail to another within the same project
|
|
184
|
-
- relates_to: Generic association (use ONLY when no specific type fits)
|
|
185
|
-
- CONTRADICTS: Memories give opposite advice or conflicting information
|
|
186
|
-
- SUPERSEDES: One memory fully replaces another (stronger than "updates")
|
|
187
|
-
- DEPENDS_ON: One memory's validity requires the other (prerequisite)
|
|
188
|
-
- CAUSED_BY: One event/decision directly caused the other
|
|
189
|
-
- VALIDATES: One memory confirms or provides evidence for the other
|
|
190
|
-
|
|
191
|
-
Choose the MOST SPECIFIC type. Prefer specific types over "relates_to".
|
|
192
|
-
|
|
193
|
-
MEMORY PAIRS:
|
|
194
|
-
${pairsBlock}
|
|
195
|
-
|
|
196
|
-
Respond with ONLY a JSON array of relationship type strings, one per pair, in order.
|
|
197
|
-
Example for 3 pairs: ["CAUSED_BY", "extends", "VALIDATES"]
|
|
198
|
-
|
|
199
|
-
No explanations. Just the JSON array.`;
|
|
200
|
-
}
|
|
172
|
+
// edgeClassification prompt REMOVED (2026-07). LLM edge classification is
|
|
173
|
+
// retired: the 672-link audit found the LLM-classified UPPERCASE relationship
|
|
174
|
+
// types near-useless (CONTRADICTS 4% acceptable). Linking is now heuristic-only
|
|
175
|
+
// (extends/relates_to) in consolidate.ts. A classification prompt may return
|
|
176
|
+
// only when a future classifier passes the audit harness at >= 70% acceptable.
|
package/dist/relink.d.ts
ADDED
|
@@ -0,0 +1,100 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* `hicortex relink` — deliberate, resumable link-discovery pass over the
|
|
3
|
+
* ENTIRE memories corpus (issue #143).
|
|
4
|
+
*
|
|
5
|
+
* The nightly's stageLinks only processes memories that are new since the
|
|
6
|
+
* last consolidation, so everything ingested before the TS linking stage
|
|
7
|
+
* (e.g. a migrated corpus) has never been through link discovery. This
|
|
8
|
+
* command back-fills the graph by reusing the exact same discovery +
|
|
9
|
+
* classification machinery (discoverLinkCandidates / classifyLinkCandidates
|
|
10
|
+
* from consolidate.ts) — no new linking logic.
|
|
11
|
+
*
|
|
12
|
+
* Classification is HEURISTIC-ONLY (2026-07). LLM edge classification was
|
|
13
|
+
* retired after the 672-link audit (see consolidate.ts Stage 3 header) found
|
|
14
|
+
* the LLM-classified UPPERCASE types near-useless. relink therefore never needs
|
|
15
|
+
* an LLM; the old `--no-llm`, `--max-llm-calls`, and `--llm-base-url/--llm-model`
|
|
16
|
+
* flags are gone. The classifyLinkCandidates call is retained (shared with the
|
|
17
|
+
* nightly) but takes no live LLM.
|
|
18
|
+
*
|
|
19
|
+
* Design:
|
|
20
|
+
* - Scope: all memories ordered by rowid, processed in batches (default 200).
|
|
21
|
+
* - Resumable: `relinkCursor` (last fully-committed rowid) is persisted in
|
|
22
|
+
* state.json after each batch. Interruption never loses more than the
|
|
23
|
+
* current batch. `--reset` restarts from rowid 0.
|
|
24
|
+
* - Candidates: reuses the STORED embedding from memory_vectors (no
|
|
25
|
+
* re-embedding); falls back to embedding the content only when a vector
|
|
26
|
+
* row is missing. Same rules as the nightly: top-10 neighbors, cosine
|
|
27
|
+
* above CONSOLIDATE_LINK_THRESHOLD (CROSS_PROJECT_LINK_THRESHOLD for
|
|
28
|
+
* cross-project pairs), capped at CONSOLIDATE_LINK_TOP_K.
|
|
29
|
+
* - Pair dedup: a candidate is skipped when a link already exists in EITHER
|
|
30
|
+
* direction (memory_links PK is directional, so reverse duplicates must
|
|
31
|
+
* be filtered here). Existing pairs are loaded once at start and the set
|
|
32
|
+
* is maintained as links are added during the run.
|
|
33
|
+
* - Classification: heuristic `extends`/`relates_to` only (classifyRelationship).
|
|
34
|
+
* - `--dry-run`: full discovery, zero writes, cursor untouched. Would-be
|
|
35
|
+
* links are reported with a breakdown by heuristic type.
|
|
36
|
+
*
|
|
37
|
+
* Server-mode only: relink needs the local DB. Client installs must run it
|
|
38
|
+
* on the server machine.
|
|
39
|
+
*/
|
|
40
|
+
import type Database from "better-sqlite3";
|
|
41
|
+
import type { EmbedFn } from "./retrieval.js";
|
|
42
|
+
export interface RelinkOptions {
|
|
43
|
+
/** Full discovery + would-be counts, zero writes, cursor untouched. */
|
|
44
|
+
dryRun?: boolean;
|
|
45
|
+
/** Memories per batch (default 200). Cursor advances per committed batch. */
|
|
46
|
+
batchSize?: number;
|
|
47
|
+
/** Ignore the saved cursor and restart from rowid 0. */
|
|
48
|
+
reset?: boolean;
|
|
49
|
+
/** DB path override (tests). Defaults to resolveDbPath(). */
|
|
50
|
+
dbPath?: string;
|
|
51
|
+
/** State dir override (tests). Defaults to ~/.hicortex. */
|
|
52
|
+
stateDir?: string;
|
|
53
|
+
/**
|
|
54
|
+
* Fallback embedder for memories missing a memory_vectors row (tests).
|
|
55
|
+
* Defaults to the local ONNX embedder, loaded lazily on first miss.
|
|
56
|
+
*/
|
|
57
|
+
embedFn?: EmbedFn;
|
|
58
|
+
}
|
|
59
|
+
export interface RelinkReport {
|
|
60
|
+
dryRun: boolean;
|
|
61
|
+
/** Memories examined in this invocation. */
|
|
62
|
+
scanned: number;
|
|
63
|
+
/** Candidate pairs above the similarity threshold (before dedup). */
|
|
64
|
+
candidatesFound: number;
|
|
65
|
+
/** Candidates skipped: link existed in either direction BEFORE this run. */
|
|
66
|
+
skippedExisting: number;
|
|
67
|
+
/** Candidates skipped: pair already handled earlier in this run (a pair is
|
|
68
|
+
* discovered from both sides — the reverse discovery is not a new link). */
|
|
69
|
+
skippedDuplicate: number;
|
|
70
|
+
/** Links created (or, in dry-run, links that would be created). */
|
|
71
|
+
linksCreated: number;
|
|
72
|
+
/** Candidates classified by an LLM. Always 0 — LLM classification retired. */
|
|
73
|
+
llmClassified: number;
|
|
74
|
+
/** Candidates classified by the heuristic (always all of them now). */
|
|
75
|
+
heuristicFallback: number;
|
|
76
|
+
/** Final relationship-type breakdown of created (or would-be) links. */
|
|
77
|
+
byType: Record<string, number>;
|
|
78
|
+
/** Memories whose discovery failed (missing vector AND embed failure). */
|
|
79
|
+
failed: number;
|
|
80
|
+
/** Batches processed. */
|
|
81
|
+
batches: number;
|
|
82
|
+
/** Cursor after this run (unchanged in dry-run). */
|
|
83
|
+
cursor: number;
|
|
84
|
+
/** Why the run ended. Only "complete" now (the LLM budget path is retired). */
|
|
85
|
+
stoppedReason: "complete";
|
|
86
|
+
}
|
|
87
|
+
/**
|
|
88
|
+
* Read the stored embedding for a memory from memory_vectors.
|
|
89
|
+
* Returns null when the row is missing (caller falls back to re-embedding).
|
|
90
|
+
*/
|
|
91
|
+
export declare function getStoredEmbedding(db: Database.Database, memoryId: string): Float32Array | null;
|
|
92
|
+
/**
|
|
93
|
+
* Run the relink pass. Returns a structured report.
|
|
94
|
+
* Throws on unrecoverable errors (client mode, DB write failure) — the cursor
|
|
95
|
+
* always reflects the last committed batch.
|
|
96
|
+
*
|
|
97
|
+
* Classification is heuristic-only (LLM edge classification retired 2026-07),
|
|
98
|
+
* so relink never contacts an LLM.
|
|
99
|
+
*/
|
|
100
|
+
export declare function runRelink(options?: RelinkOptions): Promise<RelinkReport>;
|
package/dist/relink.js
ADDED
|
@@ -0,0 +1,277 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
/**
|
|
3
|
+
* `hicortex relink` — deliberate, resumable link-discovery pass over the
|
|
4
|
+
* ENTIRE memories corpus (issue #143).
|
|
5
|
+
*
|
|
6
|
+
* The nightly's stageLinks only processes memories that are new since the
|
|
7
|
+
* last consolidation, so everything ingested before the TS linking stage
|
|
8
|
+
* (e.g. a migrated corpus) has never been through link discovery. This
|
|
9
|
+
* command back-fills the graph by reusing the exact same discovery +
|
|
10
|
+
* classification machinery (discoverLinkCandidates / classifyLinkCandidates
|
|
11
|
+
* from consolidate.ts) — no new linking logic.
|
|
12
|
+
*
|
|
13
|
+
* Classification is HEURISTIC-ONLY (2026-07). LLM edge classification was
|
|
14
|
+
* retired after the 672-link audit (see consolidate.ts Stage 3 header) found
|
|
15
|
+
* the LLM-classified UPPERCASE types near-useless. relink therefore never needs
|
|
16
|
+
* an LLM; the old `--no-llm`, `--max-llm-calls`, and `--llm-base-url/--llm-model`
|
|
17
|
+
* flags are gone. The classifyLinkCandidates call is retained (shared with the
|
|
18
|
+
* nightly) but takes no live LLM.
|
|
19
|
+
*
|
|
20
|
+
* Design:
|
|
21
|
+
* - Scope: all memories ordered by rowid, processed in batches (default 200).
|
|
22
|
+
* - Resumable: `relinkCursor` (last fully-committed rowid) is persisted in
|
|
23
|
+
* state.json after each batch. Interruption never loses more than the
|
|
24
|
+
* current batch. `--reset` restarts from rowid 0.
|
|
25
|
+
* - Candidates: reuses the STORED embedding from memory_vectors (no
|
|
26
|
+
* re-embedding); falls back to embedding the content only when a vector
|
|
27
|
+
* row is missing. Same rules as the nightly: top-10 neighbors, cosine
|
|
28
|
+
* above CONSOLIDATE_LINK_THRESHOLD (CROSS_PROJECT_LINK_THRESHOLD for
|
|
29
|
+
* cross-project pairs), capped at CONSOLIDATE_LINK_TOP_K.
|
|
30
|
+
* - Pair dedup: a candidate is skipped when a link already exists in EITHER
|
|
31
|
+
* direction (memory_links PK is directional, so reverse duplicates must
|
|
32
|
+
* be filtered here). Existing pairs are loaded once at start and the set
|
|
33
|
+
* is maintained as links are added during the run.
|
|
34
|
+
* - Classification: heuristic `extends`/`relates_to` only (classifyRelationship).
|
|
35
|
+
* - `--dry-run`: full discovery, zero writes, cursor untouched. Would-be
|
|
36
|
+
* links are reported with a breakdown by heuristic type.
|
|
37
|
+
*
|
|
38
|
+
* Server-mode only: relink needs the local DB. Client installs must run it
|
|
39
|
+
* on the server machine.
|
|
40
|
+
*/
|
|
41
|
+
var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
|
|
42
|
+
if (k2 === undefined) k2 = k;
|
|
43
|
+
var desc = Object.getOwnPropertyDescriptor(m, k);
|
|
44
|
+
if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
|
|
45
|
+
desc = { enumerable: true, get: function() { return m[k]; } };
|
|
46
|
+
}
|
|
47
|
+
Object.defineProperty(o, k2, desc);
|
|
48
|
+
}) : (function(o, m, k, k2) {
|
|
49
|
+
if (k2 === undefined) k2 = k;
|
|
50
|
+
o[k2] = m[k];
|
|
51
|
+
}));
|
|
52
|
+
var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
|
|
53
|
+
Object.defineProperty(o, "default", { enumerable: true, value: v });
|
|
54
|
+
}) : function(o, v) {
|
|
55
|
+
o["default"] = v;
|
|
56
|
+
});
|
|
57
|
+
var __importStar = (this && this.__importStar) || (function () {
|
|
58
|
+
var ownKeys = function(o) {
|
|
59
|
+
ownKeys = Object.getOwnPropertyNames || function (o) {
|
|
60
|
+
var ar = [];
|
|
61
|
+
for (var k in o) if (Object.prototype.hasOwnProperty.call(o, k)) ar[ar.length] = k;
|
|
62
|
+
return ar;
|
|
63
|
+
};
|
|
64
|
+
return ownKeys(o);
|
|
65
|
+
};
|
|
66
|
+
return function (mod) {
|
|
67
|
+
if (mod && mod.__esModule) return mod;
|
|
68
|
+
var result = {};
|
|
69
|
+
if (mod != null) for (var k = ownKeys(mod), i = 0; i < k.length; i++) if (k[i] !== "default") __createBinding(result, mod, k[i]);
|
|
70
|
+
__setModuleDefault(result, mod);
|
|
71
|
+
return result;
|
|
72
|
+
};
|
|
73
|
+
})();
|
|
74
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
75
|
+
exports.getStoredEmbedding = getStoredEmbedding;
|
|
76
|
+
exports.runRelink = runRelink;
|
|
77
|
+
const node_fs_1 = require("node:fs");
|
|
78
|
+
const node_path_1 = require("node:path");
|
|
79
|
+
const node_os_1 = require("node:os");
|
|
80
|
+
const db_js_1 = require("./db.js");
|
|
81
|
+
const storage = __importStar(require("./storage.js"));
|
|
82
|
+
const consolidate_js_1 = require("./consolidate.js");
|
|
83
|
+
const state_js_1 = require("./state.js");
|
|
84
|
+
const HICORTEX_HOME = (0, node_path_1.join)((0, node_os_1.homedir)(), ".hicortex");
|
|
85
|
+
function readConfig(stateDir) {
|
|
86
|
+
try {
|
|
87
|
+
return JSON.parse((0, node_fs_1.readFileSync)((0, node_path_1.join)(stateDir, "config.json"), "utf-8"));
|
|
88
|
+
}
|
|
89
|
+
catch {
|
|
90
|
+
return null;
|
|
91
|
+
}
|
|
92
|
+
}
|
|
93
|
+
/** Canonical unordered key for a memory pair — direction-insensitive dedup. */
|
|
94
|
+
function pairKey(a, b) {
|
|
95
|
+
return a < b ? `${a}|${b}` : `${b}|${a}`;
|
|
96
|
+
}
|
|
97
|
+
/** Load every existing link as an unordered pair set (both directions collapse). */
|
|
98
|
+
function loadExistingPairs(db) {
|
|
99
|
+
const rows = db
|
|
100
|
+
.prepare("SELECT source_id, target_id FROM memory_links")
|
|
101
|
+
.all();
|
|
102
|
+
const pairs = new Set();
|
|
103
|
+
for (const row of rows)
|
|
104
|
+
pairs.add(pairKey(row.source_id, row.target_id));
|
|
105
|
+
return pairs;
|
|
106
|
+
}
|
|
107
|
+
/**
|
|
108
|
+
* Read the stored embedding for a memory from memory_vectors.
|
|
109
|
+
* Returns null when the row is missing (caller falls back to re-embedding).
|
|
110
|
+
*/
|
|
111
|
+
function getStoredEmbedding(db, memoryId) {
|
|
112
|
+
const row = db
|
|
113
|
+
.prepare("SELECT embedding FROM memory_vectors WHERE id = ?")
|
|
114
|
+
.get(memoryId);
|
|
115
|
+
if (!row?.embedding)
|
|
116
|
+
return null;
|
|
117
|
+
const buf = row.embedding;
|
|
118
|
+
return new Float32Array(buf.buffer, buf.byteOffset, buf.byteLength / 4);
|
|
119
|
+
}
|
|
120
|
+
/**
|
|
121
|
+
* Run the relink pass. Returns a structured report.
|
|
122
|
+
* Throws on unrecoverable errors (client mode, DB write failure) — the cursor
|
|
123
|
+
* always reflects the last committed batch.
|
|
124
|
+
*
|
|
125
|
+
* Classification is heuristic-only (LLM edge classification retired 2026-07),
|
|
126
|
+
* so relink never contacts an LLM.
|
|
127
|
+
*/
|
|
128
|
+
async function runRelink(options = {}) {
|
|
129
|
+
const dryRun = options.dryRun ?? false;
|
|
130
|
+
const batchSize = options.batchSize ?? 200;
|
|
131
|
+
const stateDir = options.stateDir ?? HICORTEX_HOME;
|
|
132
|
+
if (!Number.isInteger(batchSize) || batchSize < 1) {
|
|
133
|
+
throw new Error(`[hicortex] relink: invalid --batch value: ${options.batchSize}`);
|
|
134
|
+
}
|
|
135
|
+
// Server-mode only — client installs have no local DB.
|
|
136
|
+
const config = readConfig(stateDir);
|
|
137
|
+
if (config?.mode === "client") {
|
|
138
|
+
throw new Error("[hicortex] relink is server-mode only (it needs the local DB). " +
|
|
139
|
+
`This machine is a client of ${config.serverUrl ?? "a remote server"} — run relink on the server.`);
|
|
140
|
+
}
|
|
141
|
+
// Classification is heuristic-only — no LLM client, no budget cap.
|
|
142
|
+
const budget = new consolidate_js_1.BudgetTracker(Number.MAX_SAFE_INTEGER);
|
|
143
|
+
const dbPath = (0, db_js_1.resolveDbPath)(options.dbPath);
|
|
144
|
+
const db = (0, db_js_1.initDb)(dbPath);
|
|
145
|
+
const report = {
|
|
146
|
+
dryRun,
|
|
147
|
+
scanned: 0,
|
|
148
|
+
candidatesFound: 0,
|
|
149
|
+
skippedExisting: 0,
|
|
150
|
+
skippedDuplicate: 0,
|
|
151
|
+
linksCreated: 0,
|
|
152
|
+
llmClassified: 0,
|
|
153
|
+
heuristicFallback: 0,
|
|
154
|
+
byType: {},
|
|
155
|
+
failed: 0,
|
|
156
|
+
batches: 0,
|
|
157
|
+
cursor: 0,
|
|
158
|
+
stoppedReason: "complete",
|
|
159
|
+
};
|
|
160
|
+
try {
|
|
161
|
+
const totalMemories = storage.countMemories(db);
|
|
162
|
+
let cursor = options.reset ? 0 : ((0, state_js_1.loadState)(stateDir).relinkCursor ?? 0);
|
|
163
|
+
report.cursor = cursor;
|
|
164
|
+
console.log(`[hicortex] relink starting (${dryRun ? "dry-run" : "heuristic"}): ${totalMemories} memories, ` +
|
|
165
|
+
`batch ${batchSize}, cursor ${cursor}${options.reset ? " (reset)" : ""}`);
|
|
166
|
+
// Load all existing links once as an unordered pair set (471 rows on the
|
|
167
|
+
// live corpus — trivial). `seenPairs` additionally accumulates pairs
|
|
168
|
+
// handled during this run so re-runs AND within-run reverse candidates
|
|
169
|
+
// never create duplicate edges.
|
|
170
|
+
const preExistingPairs = loadExistingPairs(db);
|
|
171
|
+
const seenPairs = new Set(preExistingPairs);
|
|
172
|
+
const batchStmt = db.prepare("SELECT rowid AS __rowid, * FROM memories WHERE rowid > ? ORDER BY rowid ASC LIMIT ?");
|
|
173
|
+
// Lazy fallback embedder — only loaded if a memory_vectors row is missing.
|
|
174
|
+
let embedFn = options.embedFn ?? null;
|
|
175
|
+
const getEmbedFn = async () => {
|
|
176
|
+
if (!embedFn) {
|
|
177
|
+
const { embed } = await import("./embedder.js");
|
|
178
|
+
embedFn = embed;
|
|
179
|
+
}
|
|
180
|
+
return embedFn;
|
|
181
|
+
};
|
|
182
|
+
for (;;) {
|
|
183
|
+
const rows = batchStmt.all(cursor, batchSize);
|
|
184
|
+
if (rows.length === 0)
|
|
185
|
+
break;
|
|
186
|
+
const lastRowid = rows[rows.length - 1].__rowid;
|
|
187
|
+
// Phase A: discovery — reuse stored embeddings, same candidate rules
|
|
188
|
+
// as the nightly (discoverLinkCandidates).
|
|
189
|
+
const candidates = [];
|
|
190
|
+
let batchSkippedExisting = 0;
|
|
191
|
+
let batchSkippedDuplicate = 0;
|
|
192
|
+
for (const row of rows) {
|
|
193
|
+
const { __rowid: _ignored, ...memRow } = row;
|
|
194
|
+
const mem = memRow;
|
|
195
|
+
try {
|
|
196
|
+
let embedding = getStoredEmbedding(db, mem.id);
|
|
197
|
+
if (!embedding) {
|
|
198
|
+
embedding = await (await getEmbedFn())(mem.content);
|
|
199
|
+
}
|
|
200
|
+
for (const cand of (0, consolidate_js_1.discoverLinkCandidates)(db, mem, embedding)) {
|
|
201
|
+
report.candidatesFound++;
|
|
202
|
+
const key = pairKey(cand.source.id, cand.target.id);
|
|
203
|
+
if (seenPairs.has(key)) {
|
|
204
|
+
if (preExistingPairs.has(key))
|
|
205
|
+
batchSkippedExisting++;
|
|
206
|
+
else
|
|
207
|
+
batchSkippedDuplicate++;
|
|
208
|
+
continue;
|
|
209
|
+
}
|
|
210
|
+
// Reserve the pair immediately so the reverse direction later in
|
|
211
|
+
// this run (or this batch) is deduped too.
|
|
212
|
+
seenPairs.add(key);
|
|
213
|
+
candidates.push(cand);
|
|
214
|
+
}
|
|
215
|
+
}
|
|
216
|
+
catch {
|
|
217
|
+
report.failed++;
|
|
218
|
+
}
|
|
219
|
+
}
|
|
220
|
+
report.skippedExisting += batchSkippedExisting;
|
|
221
|
+
report.skippedDuplicate += batchSkippedDuplicate;
|
|
222
|
+
// Phase B: classification — shared heuristic-only path (LLM retired).
|
|
223
|
+
// classifyLinkCandidates ignores the null LLM/budget and returns each
|
|
224
|
+
// candidate's heuristic type (extends/relates_to).
|
|
225
|
+
const classified = await (0, consolidate_js_1.classifyLinkCandidates)(candidates, null, budget);
|
|
226
|
+
const types = classified.types;
|
|
227
|
+
report.llmClassified += classified.llmClassified; // always 0
|
|
228
|
+
report.heuristicFallback += classified.heuristicFallback;
|
|
229
|
+
// Phase C: store — one transaction per batch, then persist the cursor.
|
|
230
|
+
// A crash between commit and cursor save is safe: the re-run re-scans
|
|
231
|
+
// the batch but the pair dedup (loaded from the DB) skips every link.
|
|
232
|
+
if (!dryRun) {
|
|
233
|
+
const tx = db.transaction(() => {
|
|
234
|
+
for (let i = 0; i < candidates.length; i++) {
|
|
235
|
+
const c = candidates[i];
|
|
236
|
+
storage.addLink(db, c.source.id, c.target.id, types[i], c.similarity);
|
|
237
|
+
}
|
|
238
|
+
});
|
|
239
|
+
tx();
|
|
240
|
+
(0, state_js_1.updateState)((s) => {
|
|
241
|
+
s.relinkCursor = lastRowid;
|
|
242
|
+
}, stateDir);
|
|
243
|
+
report.cursor = lastRowid;
|
|
244
|
+
}
|
|
245
|
+
for (const t of types)
|
|
246
|
+
report.byType[t] = (report.byType[t] ?? 0) + 1;
|
|
247
|
+
report.linksCreated += candidates.length;
|
|
248
|
+
report.scanned += rows.length;
|
|
249
|
+
report.batches++;
|
|
250
|
+
cursor = lastRowid;
|
|
251
|
+
console.log(`[hicortex] batch ${report.batches}: scanned ${rows.length}, ` +
|
|
252
|
+
`candidates ${candidates.length + batchSkippedExisting + batchSkippedDuplicate}, ` +
|
|
253
|
+
`skipped ${batchSkippedExisting} already-linked + ${batchSkippedDuplicate} duplicate, ` +
|
|
254
|
+
`${dryRun ? "would create" : "created"} ${candidates.length} ` +
|
|
255
|
+
`(cursor ${lastRowid}, ${report.scanned} scanned this run)`);
|
|
256
|
+
}
|
|
257
|
+
// Final summary
|
|
258
|
+
const typeBreakdown = Object.entries(report.byType)
|
|
259
|
+
.sort((a, b) => b[1] - a[1])
|
|
260
|
+
.map(([t, n]) => `${t}=${n}`)
|
|
261
|
+
.join(", ") || "none";
|
|
262
|
+
console.log(`[hicortex] relink complete` +
|
|
263
|
+
`${dryRun ? " (dry-run, nothing written)" : ""}: ` +
|
|
264
|
+
`${report.scanned} memories scanned, ${report.candidatesFound} candidates, ` +
|
|
265
|
+
`${report.skippedExisting} already linked, ${report.skippedDuplicate} reverse-duplicates, ` +
|
|
266
|
+
`${report.linksCreated} links ${dryRun ? "would be created" : "created"} ` +
|
|
267
|
+
`(heuristic ${report.heuristicFallback})`);
|
|
268
|
+
console.log(`[hicortex] by type: ${typeBreakdown}`);
|
|
269
|
+
if (report.failed > 0) {
|
|
270
|
+
console.warn(`[hicortex] ${report.failed} memories failed discovery (see errors above)`);
|
|
271
|
+
}
|
|
272
|
+
return report;
|
|
273
|
+
}
|
|
274
|
+
finally {
|
|
275
|
+
db.close();
|
|
276
|
+
}
|
|
277
|
+
}
|
package/dist/retrieval.d.ts
CHANGED
|
@@ -14,7 +14,17 @@
|
|
|
14
14
|
* effective = floor + (base - floor) * decay_rate^hours
|
|
15
15
|
*/
|
|
16
16
|
import type Database from "better-sqlite3";
|
|
17
|
-
import type { MemorySearchResult } from "./types.js";
|
|
17
|
+
import type { Memory, MemorySearchResult } from "./types.js";
|
|
18
|
+
/**
|
|
19
|
+
* Convert an L2 distance (as returned by sqlite-vec's vec0 `distance`) to
|
|
20
|
+
* cosine similarity. Valid because our embeddings are L2-normalized
|
|
21
|
+
* (embedder.ts, `normalize: true`): for unit vectors, d² = 2 − 2·cos,
|
|
22
|
+
* hence cos = 1 − d²/2. Exact anchors: d=0 → 1, d=√2 → 0, d=2 → −1.
|
|
23
|
+
*
|
|
24
|
+
* Lives here (the dependency-root of the scoring code) and is re-exported
|
|
25
|
+
* by consolidate.ts so pre-#145 importers keep working.
|
|
26
|
+
*/
|
|
27
|
+
export declare function l2ToCosine(distance: number): number;
|
|
18
28
|
/**
|
|
19
29
|
* Compute decayed strength with adaptive decay (B+E+D model).
|
|
20
30
|
* Exported for use by consolidation decay/prune stage.
|
|
@@ -24,6 +34,11 @@ export declare function effectiveStrength(baseStrength: number, lastAccessed: st
|
|
|
24
34
|
accessCount?: number;
|
|
25
35
|
linkCount?: number;
|
|
26
36
|
}): number;
|
|
37
|
+
/**
|
|
38
|
+
* Return a composite relevance score in [0, 1] for a candidate memory.
|
|
39
|
+
* Exported for exact-value tests of the similarity component (#145).
|
|
40
|
+
*/
|
|
41
|
+
export declare function computeScore(memory: Memory, distance: number, connectionCount: number, maxConnections: number, now: Date): number;
|
|
27
42
|
export interface EmbedFn {
|
|
28
43
|
(text: string): Promise<Float32Array>;
|
|
29
44
|
}
|
package/dist/retrieval.js
CHANGED
|
@@ -48,13 +48,35 @@ var __importStar = (this && this.__importStar) || (function () {
|
|
|
48
48
|
};
|
|
49
49
|
})();
|
|
50
50
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
51
|
+
exports.l2ToCosine = l2ToCosine;
|
|
51
52
|
exports.effectiveStrength = effectiveStrength;
|
|
53
|
+
exports.computeScore = computeScore;
|
|
52
54
|
exports.retrieve = retrieve;
|
|
53
55
|
exports.searchContext = searchContext;
|
|
54
56
|
const storage = __importStar(require("./storage.js"));
|
|
55
57
|
const BASE_DECAY = 0.0005;
|
|
56
|
-
|
|
58
|
+
/**
|
|
59
|
+
* Placeholder L2 distance for candidates that have no measured vector
|
|
60
|
+
* distance (FTS-only hits and graph-discovered neighbors). Chosen so that
|
|
61
|
+
* l2ToCosine(1.0) = 0.5 — a neutral mid-scale similarity. Before the #145
|
|
62
|
+
* fix the value was 0.5 on the accidental 1−L2 scale, which also yielded
|
|
63
|
+
* similarity 0.5; keeping 0.5 under the corrected formula would have jumped
|
|
64
|
+
* these candidates to cosine 0.875, outranking most true vector matches.
|
|
65
|
+
*/
|
|
66
|
+
const DEFAULT_GRAPH_DISTANCE = 1.0;
|
|
57
67
|
const RRF_K = 60;
|
|
68
|
+
/**
|
|
69
|
+
* Convert an L2 distance (as returned by sqlite-vec's vec0 `distance`) to
|
|
70
|
+
* cosine similarity. Valid because our embeddings are L2-normalized
|
|
71
|
+
* (embedder.ts, `normalize: true`): for unit vectors, d² = 2 − 2·cos,
|
|
72
|
+
* hence cos = 1 − d²/2. Exact anchors: d=0 → 1, d=√2 → 0, d=2 → −1.
|
|
73
|
+
*
|
|
74
|
+
* Lives here (the dependency-root of the scoring code) and is re-exported
|
|
75
|
+
* by consolidate.ts so pre-#145 importers keep working.
|
|
76
|
+
*/
|
|
77
|
+
function l2ToCosine(distance) {
|
|
78
|
+
return 1 - (distance * distance) / 2;
|
|
79
|
+
}
|
|
58
80
|
// ---------------------------------------------------------------------------
|
|
59
81
|
// Timestamp parsing
|
|
60
82
|
// ---------------------------------------------------------------------------
|
|
@@ -96,9 +118,19 @@ function effectiveStrength(baseStrength, lastAccessed, now, options) {
|
|
|
96
118
|
}
|
|
97
119
|
/**
|
|
98
120
|
* Return a composite relevance score in [0, 1] for a candidate memory.
|
|
121
|
+
* Exported for exact-value tests of the similarity component (#145).
|
|
99
122
|
*/
|
|
100
123
|
function computeScore(memory, distance, connectionCount, maxConnections, now) {
|
|
101
|
-
|
|
124
|
+
// TRUE cosine similarity (#145). The old `1 − distance` compressed real
|
|
125
|
+
// cosines (cos 0.8 scored 0.37) and the 0-clamp at that scale flattened
|
|
126
|
+
// everything below cos 0.5 to exactly 0, killing mid-relevance
|
|
127
|
+
// discrimination. The clamp stays at 0 — a negative cosine means truly
|
|
128
|
+
// unrelated — but now at the correct scale. NOTE: the similarity values
|
|
129
|
+
// roughly DOUBLE for related content on the new scale; the blend weights
|
|
130
|
+
// below are deliberately unchanged in this pass so the before/after
|
|
131
|
+
// retrieval comparison is measured, not guessed. Rebalancing the weights
|
|
132
|
+
// is a data-driven follow-up if the eval shows it is needed.
|
|
133
|
+
const similarity = Math.max(0, l2ToCosine(distance));
|
|
102
134
|
const effStrength = effectiveStrength(memory.base_strength ?? 0.5, memory.last_accessed, now, {
|
|
103
135
|
accessCount: memory.access_count ?? 0,
|
|
104
136
|
linkCount: connectionCount,
|
|
@@ -0,0 +1,149 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Graded schema membership — domain prototypes + per-tag association weights
|
|
3
|
+
* (spec: specs/2026-07-07-graded-schema-memory-tags.md).
|
|
4
|
+
*
|
|
5
|
+
* MODEL (cognitive grounding → mechanism)
|
|
6
|
+
* ---------------------------------------
|
|
7
|
+
* A configured domain is a SCHEMA with graded membership (Rosch prototypes):
|
|
8
|
+
* - prototype(domain) = L2-normalized mean of the embeddings of memories
|
|
9
|
+
* whose tag set includes the domain. Cold start / thin domains
|
|
10
|
+
* (member_count < PROTOTYPE_MIN_MEMBERS) seed the prototype from the
|
|
11
|
+
* embedding of the domain's config description instead.
|
|
12
|
+
* - weight(memory, tag) = cosine(memory embedding, prototype(tag)). Both
|
|
13
|
+
* vectors are L2-normalized, so cosine reduces to a dot product.
|
|
14
|
+
* - PRIMARY (memories.domain) = argmax-weight tag, overridden by any tagged
|
|
15
|
+
* domain flagged `compartment: true` (deliberate compartmentalization —
|
|
16
|
+
* the owner's Work firewall). Fully mechanical, no LLM.
|
|
17
|
+
*
|
|
18
|
+
* The LLM decides ONLY the discrete part (which schemas apply — see
|
|
19
|
+
* domain-classify.ts); ALL gradation is derived from embeddings here.
|
|
20
|
+
* Prototypes + weights are recomputed each nightly, so categories drift with
|
|
21
|
+
* the data ("reconsolidation") without any re-classification runs.
|
|
22
|
+
*
|
|
23
|
+
* Persistence: `domain_prototypes(domain, embedding, member_count, updated_at)`
|
|
24
|
+
* and `memory_tags.weight` (both migration v7).
|
|
25
|
+
*/
|
|
26
|
+
import type Database from "better-sqlite3";
|
|
27
|
+
import type { DomainDef } from "./types.js";
|
|
28
|
+
import type { EmbedFn } from "./retrieval.js";
|
|
29
|
+
/**
|
|
30
|
+
* Below this member count a domain's prototype is seeded from its config
|
|
31
|
+
* description instead of the member centroid (cold start / thin domains).
|
|
32
|
+
*/
|
|
33
|
+
export declare const PROTOTYPE_MIN_MEMBERS = 5;
|
|
34
|
+
/** Deserialize a BLOB column into a Float32Array (same layout as sqlite-vec). */
|
|
35
|
+
export declare function blobToVec(buf: Buffer): Float32Array;
|
|
36
|
+
/**
|
|
37
|
+
* L2-normalize a vector IN A COPY. A zero vector (norm 0 — e.g. test fixtures)
|
|
38
|
+
* is returned as an all-zero copy rather than dividing by zero.
|
|
39
|
+
*/
|
|
40
|
+
export declare function l2Normalize(vec: Float32Array): Float32Array;
|
|
41
|
+
/**
|
|
42
|
+
* Association weight of a memory for a tag = cosine(memory embedding, domain
|
|
43
|
+
* prototype). Both inputs are L2-normalized (embedder.ts normalizes memory
|
|
44
|
+
* embeddings; computeDomainPrototypes normalizes prototypes), so cosine is
|
|
45
|
+
* exactly the dot product.
|
|
46
|
+
*/
|
|
47
|
+
export declare function tagWeight(memoryEmbedding: Float32Array, prototype: Float32Array): number;
|
|
48
|
+
/** One tag with its association weight (null = never computed / no prototype). */
|
|
49
|
+
export interface WeightedTag {
|
|
50
|
+
tag: string;
|
|
51
|
+
weight: number | null;
|
|
52
|
+
}
|
|
53
|
+
/** The configured compartment domain names (DomainDef.compartment === true). */
|
|
54
|
+
export declare function compartmentSet(domains: DomainDef[]): Set<string>;
|
|
55
|
+
/**
|
|
56
|
+
* Derive the PRIMARY tag (memories.domain) from a weighted tag set.
|
|
57
|
+
*
|
|
58
|
+
* Rules (deterministic, no LLM):
|
|
59
|
+
* 1. Any tagged compartment domain wins — first one in array order if the
|
|
60
|
+
* (unusual) case of several arises.
|
|
61
|
+
* 2. Else the argmax-weight tag. `tags` MUST be in LLM most-relevant-first
|
|
62
|
+
* order: ties (and all-null weights) resolve to the EARLIEST array
|
|
63
|
+
* position — strict `>` comparison keeps the first maximum.
|
|
64
|
+
* 3. A null weight loses to any numeric weight (treated as -Infinity).
|
|
65
|
+
*
|
|
66
|
+
* Throws on an empty tag set — callers guarantee >= 1 tag (an empty tag set
|
|
67
|
+
* from the classifier is a NO-FIT and must be routed through nofit.ts, never
|
|
68
|
+
* here); an empty set reaching this function is a programming error.
|
|
69
|
+
*/
|
|
70
|
+
export declare function derivePrimary(tags: WeightedTag[], compartments: Set<string>): string;
|
|
71
|
+
export interface PrototypeStat {
|
|
72
|
+
domain: string;
|
|
73
|
+
memberCount: number;
|
|
74
|
+
/** true = description seed (member_count < PROTOTYPE_MIN_MEMBERS). */
|
|
75
|
+
seeded: boolean;
|
|
76
|
+
}
|
|
77
|
+
/** Load all stored prototypes into a name → vector map. */
|
|
78
|
+
export declare function loadDomainPrototypes(db: Database.Database): Map<string, Float32Array>;
|
|
79
|
+
/**
|
|
80
|
+
* Compute + persist the prototype of every configured domain.
|
|
81
|
+
*
|
|
82
|
+
* Per domain: mean of the embeddings of memories whose memory_tags include the
|
|
83
|
+
* domain, L2-normalized. When member_count < PROTOTYPE_MIN_MEMBERS the
|
|
84
|
+
* prototype is instead the embedding of `"<Name>: <description>"` (the same
|
|
85
|
+
* "name: description" line the classifier prompt shows) — this seeds cold
|
|
86
|
+
* starts and keeps thin domains from collapsing onto 1-2 outliers.
|
|
87
|
+
*
|
|
88
|
+
* `getEmbedFn` is LAZY (a function returning a promise of the embedder) so the
|
|
89
|
+
* ~130 MB ONNX embedder is loaded ONLY when at least one domain actually needs
|
|
90
|
+
* a description seed — same pattern as relink.ts's lazy fallback embedder.
|
|
91
|
+
*
|
|
92
|
+
* Returns the fresh prototype map (for immediate classification-time weights)
|
|
93
|
+
* plus per-domain stats. Rows in domain_prototypes are REPLACED per domain;
|
|
94
|
+
* domains removed from the config keep a stale row until the next config-set
|
|
95
|
+
* change (harmless — nothing reads prototypes outside the configured set).
|
|
96
|
+
*/
|
|
97
|
+
export declare function computeDomainPrototypes(db: Database.Database, domains: DomainDef[], getEmbedFn: () => Promise<EmbedFn>): Promise<{
|
|
98
|
+
prototypes: Map<string, Float32Array>;
|
|
99
|
+
stats: PrototypeStat[];
|
|
100
|
+
}>;
|
|
101
|
+
/**
|
|
102
|
+
* Compute the per-tag weights for ONE memory from the current prototypes
|
|
103
|
+
* (classification-time path: newly tagged memories get weights immediately).
|
|
104
|
+
* A missing memory vector or missing prototype yields null (stored as NULL;
|
|
105
|
+
* repaired by the next nightly recompute).
|
|
106
|
+
*/
|
|
107
|
+
export declare function computeTagWeights(db: Database.Database, memoryId: string, tags: string[], prototypes: Map<string, Float32Array>): Record<string, number | null>;
|
|
108
|
+
/**
|
|
109
|
+
* Best prototype match for one memory across ALL configured domains:
|
|
110
|
+
* argmax of cosine(memory embedding, prototype(domain)).
|
|
111
|
+
*
|
|
112
|
+
* Used by the no-fit path (nofit.ts, owner amendment 07.07): when the LLM
|
|
113
|
+
* says no domain fits, the memory can still earn a WEAK primary from pure
|
|
114
|
+
* embedding association — provided the best cosine clears the configured
|
|
115
|
+
* weakPrimaryFloor (the caller checks the floor; this function just reports
|
|
116
|
+
* the argmax).
|
|
117
|
+
*
|
|
118
|
+
* Returns null when the memory has no stored vector or no configured domain
|
|
119
|
+
* has a prototype (nothing to associate against). Ties resolve to the FIRST
|
|
120
|
+
* domain in config order (strict `>` comparison), mirroring derivePrimary.
|
|
121
|
+
*/
|
|
122
|
+
export declare function bestPrototypeMatch(db: Database.Database, memoryId: string, domains: DomainDef[], prototypes: Map<string, Float32Array>): {
|
|
123
|
+
domain: string;
|
|
124
|
+
weight: number;
|
|
125
|
+
} | null;
|
|
126
|
+
/**
|
|
127
|
+
* One pass over ALL memory_tags rows: weight = cosine(memory embedding,
|
|
128
|
+
* prototype(tag)). Rows whose memory has no stored vector, or whose tag has no
|
|
129
|
+
* prototype (out-of-vocabulary leftovers), are set to NULL. Cheap: one point
|
|
130
|
+
* lookup per distinct memory + one dot product per tag row (3–5k on the
|
|
131
|
+
* production corpus).
|
|
132
|
+
*/
|
|
133
|
+
export declare function recomputeAllTagWeights(db: Database.Database, prototypes: Map<string, Float32Array>): {
|
|
134
|
+
updated: number;
|
|
135
|
+
nulled: number;
|
|
136
|
+
};
|
|
137
|
+
/**
|
|
138
|
+
* Re-derive the PRIMARY (memories.domain) of every tagged memory from its
|
|
139
|
+
* current tag weights: compartment override first, else argmax weight, LLM
|
|
140
|
+
* order (memory_tags insertion order = rowid, written most-relevant-first by
|
|
141
|
+
* storage.setMemoryTags) breaking exact-weight ties.
|
|
142
|
+
*
|
|
143
|
+
* Memories with NO memory_tags rows are untouched (e.g. infra-skipped rows
|
|
144
|
+
* awaiting classification — issue #150 discipline).
|
|
145
|
+
*/
|
|
146
|
+
export declare function refreshPrimaries(db: Database.Database, domains: DomainDef[]): {
|
|
147
|
+
examined: number;
|
|
148
|
+
updated: number;
|
|
149
|
+
};
|