@gamaze/hicortex 0.10.1 → 0.11.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +42 -0
- package/THIRD_PARTY_NOTICES.md +108 -0
- package/assets/vendor/3d-force-graph.min.js +5 -0
- package/assets/vendor/force-graph.min.js +5 -0
- package/assets/vendor/three.core.min.js +6 -0
- package/assets/vendor/three.module.min.js +6 -0
- package/assets/viz.html +1128 -0
- package/dist/classify-domains.d.ts +98 -0
- package/dist/classify-domains.js +340 -0
- package/dist/cli.d.ts +1 -0
- package/dist/cli.js +63 -0
- package/dist/consolidate.d.ts +139 -2
- package/dist/consolidate.js +302 -87
- package/dist/db.js +70 -0
- package/dist/domain-classify.d.ts +164 -0
- package/dist/domain-classify.js +300 -0
- package/dist/extensions.d.ts +12 -0
- package/dist/graph.d.ts +56 -0
- package/dist/graph.js +145 -0
- package/dist/index.js +1 -1
- package/dist/init.d.ts +25 -0
- package/dist/init.js +54 -0
- package/dist/lesson-selection.js +12 -5
- package/dist/lessons-context.js +2 -1
- package/dist/llm.d.ts +67 -0
- package/dist/llm.js +122 -0
- package/dist/mcp-server.js +86 -28
- package/dist/nightly-status.js +9 -28
- package/dist/nightly.js +42 -32
- package/dist/nofit.d.ts +111 -0
- package/dist/nofit.js +176 -0
- package/dist/prompts.d.ts +0 -5
- package/dist/prompts.js +5 -29
- package/dist/relink.d.ts +100 -0
- package/dist/relink.js +277 -0
- package/dist/retrieval.d.ts +16 -1
- package/dist/retrieval.js +34 -2
- package/dist/schema-prototypes.d.ts +149 -0
- package/dist/schema-prototypes.js +329 -0
- package/dist/state.d.ts +32 -0
- package/dist/state.js +29 -0
- package/dist/status.js +12 -19
- package/dist/storage.d.ts +44 -1
- package/dist/storage.js +70 -1
- package/dist/types.d.ts +90 -0
- package/dist/viz.d.ts +69 -0
- package/dist/viz.js +180 -0
- package/domains.example.json +36 -0
- package/package.json +6 -3
|
@@ -0,0 +1,98 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* `hicortex classify-domains` — deliberate, resumable content-based domain
|
|
3
|
+
* classification pass over the memories corpus (feat/content-domains).
|
|
4
|
+
*
|
|
5
|
+
* The nightly's content-domain stage only files NULL/stale rows incrementally.
|
|
6
|
+
* This command back-fills the whole corpus on demand, with the same discipline
|
|
7
|
+
* as `hicortex relink`:
|
|
8
|
+
* - Scope: memories ordered by rowid, processed in batches (default 200).
|
|
9
|
+
* Default scope = rows whose domain is NULL or not in the configured set.
|
|
10
|
+
* `--all` reclassifies EVERY memory.
|
|
11
|
+
* - Resumable: `domainCursor` (last fully-committed rowid) persisted in
|
|
12
|
+
* state.json after each batch. Interruption never loses more than the
|
|
13
|
+
* current batch. `--reset` restarts from rowid 0.
|
|
14
|
+
* - Classification: one constrained MULTI-TAG LLM call per memory via the
|
|
15
|
+
* CLASSIFY tier (classifyBaseUrl/classifyModel when configured, else the
|
|
16
|
+
* reflect tier), same as the nightly. The LLM emits ONLY the ordered tag
|
|
17
|
+
* set; per-tag weights come from the domain prototypes (computed once at
|
|
18
|
+
* run start) and the PRIMARY (memories.domain) is derived (argmax weight,
|
|
19
|
+
* compartment override, LLM order breaking ties) inside
|
|
20
|
+
* storage.setMemoryTags. After a completed (non-aborted) run the
|
|
21
|
+
* prototypes, all weights, and all primaries are recomputed from the
|
|
22
|
+
* final tag sets — same reconsolidation pass as the nightly.
|
|
23
|
+
* - No-fit (owner amendment 07.07): an LLM reply of {"tags": []} means no
|
|
24
|
+
* configured domain fits — there is NO fallback category. The memory gets
|
|
25
|
+
* a WEAK primary (argmax prototype cosine, when >= weakPrimaryFloor) or,
|
|
26
|
+
* below the floor, accelerated decay (base_strength halved, domain left
|
|
27
|
+
* NULL so later runs re-attempt it). See nofit.ts.
|
|
28
|
+
* - LLM pre-flight: the endpoint classification will actually use (when a
|
|
29
|
+
* separate Ollama) is probed before any work. If unreachable, abort
|
|
30
|
+
* cleanly (nothing written, cursor untouched) — strict, like distill.
|
|
31
|
+
* - Infra-error abort (issue #150): if the classifier returns null mid-run
|
|
32
|
+
* (endpoint died AFTER preflight), the run aborts after committing
|
|
33
|
+
* the last full batch. The failing memory is left completely untouched; the
|
|
34
|
+
* cursor sits at the last committed batch so a re-run resumes cleanly.
|
|
35
|
+
* - Server-mode only: needs the local DB.
|
|
36
|
+
*
|
|
37
|
+
* Requires a `domains` list in ~/.hicortex/config.json — without it there is
|
|
38
|
+
* nothing to classify into and the command exits with a clear message.
|
|
39
|
+
*/
|
|
40
|
+
import { LlmClient } from "./llm.js";
|
|
41
|
+
import type { EmbedFn } from "./retrieval.js";
|
|
42
|
+
export interface ClassifyDomainsOptions {
|
|
43
|
+
/** Reclassify EVERY memory, not just NULL/stale-domain rows. */
|
|
44
|
+
all?: boolean;
|
|
45
|
+
/** Memories per batch (default 200). Cursor advances per committed batch. */
|
|
46
|
+
batchSize?: number;
|
|
47
|
+
/** Ignore the saved cursor and restart from rowid 0. */
|
|
48
|
+
reset?: boolean;
|
|
49
|
+
/** DB path override (tests). Defaults to resolveDbPath(). */
|
|
50
|
+
dbPath?: string;
|
|
51
|
+
/** State dir override (tests). Defaults to ~/.hicortex. */
|
|
52
|
+
stateDir?: string;
|
|
53
|
+
/** LLM override (tests). Bypasses config resolution + preflight. */
|
|
54
|
+
llm?: LlmClient;
|
|
55
|
+
/** Config override (tests). Defaults to reading stateDir/config.json. */
|
|
56
|
+
config?: Record<string, unknown> | null;
|
|
57
|
+
/**
|
|
58
|
+
* Embedder override (tests). Used only for domain-description prototype
|
|
59
|
+
* seeds; defaults to the local ONNX embedder, loaded lazily on first need
|
|
60
|
+
* (same pattern as relink).
|
|
61
|
+
*/
|
|
62
|
+
embedFn?: EmbedFn;
|
|
63
|
+
}
|
|
64
|
+
export interface ClassifyDomainsReport {
|
|
65
|
+
/** Memories examined in this invocation. */
|
|
66
|
+
scanned: number;
|
|
67
|
+
/** Memories whose tags were written (primary changed or first-set). */
|
|
68
|
+
classified: number;
|
|
69
|
+
/** Memories whose primary was already the classified value (no rewrite). */
|
|
70
|
+
unchanged: number;
|
|
71
|
+
/** Memories skipped due to an infra error (classifier returned null). */
|
|
72
|
+
failed: number;
|
|
73
|
+
/** No-fit memories that earned a WEAK primary (prototype argmax >= floor). */
|
|
74
|
+
weakPrimary: number;
|
|
75
|
+
/** No-fit memories below the floor — untagged, base_strength halved. */
|
|
76
|
+
noAssociationDecayed: number;
|
|
77
|
+
/** Batches processed. */
|
|
78
|
+
batches: number;
|
|
79
|
+
/** Cursor after this run. */
|
|
80
|
+
cursor: number;
|
|
81
|
+
/** Whether the run aborted early on an infra error (classifier null). */
|
|
82
|
+
aborted: boolean;
|
|
83
|
+
/** Final per-PRIMARY memory counts (whole corpus, post-run). */
|
|
84
|
+
byDomain: Record<string, number>;
|
|
85
|
+
/** Total tag assignments across memory_tags (whole corpus, post-run). */
|
|
86
|
+
totalTags: number;
|
|
87
|
+
/** Post-run reconsolidation: memory_tags rows whose weight was recomputed. */
|
|
88
|
+
weightsRecomputed: number;
|
|
89
|
+
/** Post-run reconsolidation: memories whose derived primary changed. */
|
|
90
|
+
primariesUpdated: number;
|
|
91
|
+
}
|
|
92
|
+
/**
|
|
93
|
+
* Run the classify-domains pass. Returns a structured report.
|
|
94
|
+
* Throws on unrecoverable setup errors (client mode, no domains, no LLM,
|
|
95
|
+
* classification endpoint down) — the cursor always reflects the last
|
|
96
|
+
* committed batch.
|
|
97
|
+
*/
|
|
98
|
+
export declare function runClassifyDomains(options?: ClassifyDomainsOptions): Promise<ClassifyDomainsReport>;
|
|
@@ -0,0 +1,340 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
/**
|
|
3
|
+
* `hicortex classify-domains` — deliberate, resumable content-based domain
|
|
4
|
+
* classification pass over the memories corpus (feat/content-domains).
|
|
5
|
+
*
|
|
6
|
+
* The nightly's content-domain stage only files NULL/stale rows incrementally.
|
|
7
|
+
* This command back-fills the whole corpus on demand, with the same discipline
|
|
8
|
+
* as `hicortex relink`:
|
|
9
|
+
* - Scope: memories ordered by rowid, processed in batches (default 200).
|
|
10
|
+
* Default scope = rows whose domain is NULL or not in the configured set.
|
|
11
|
+
* `--all` reclassifies EVERY memory.
|
|
12
|
+
* - Resumable: `domainCursor` (last fully-committed rowid) persisted in
|
|
13
|
+
* state.json after each batch. Interruption never loses more than the
|
|
14
|
+
* current batch. `--reset` restarts from rowid 0.
|
|
15
|
+
* - Classification: one constrained MULTI-TAG LLM call per memory via the
|
|
16
|
+
* CLASSIFY tier (classifyBaseUrl/classifyModel when configured, else the
|
|
17
|
+
* reflect tier), same as the nightly. The LLM emits ONLY the ordered tag
|
|
18
|
+
* set; per-tag weights come from the domain prototypes (computed once at
|
|
19
|
+
* run start) and the PRIMARY (memories.domain) is derived (argmax weight,
|
|
20
|
+
* compartment override, LLM order breaking ties) inside
|
|
21
|
+
* storage.setMemoryTags. After a completed (non-aborted) run the
|
|
22
|
+
* prototypes, all weights, and all primaries are recomputed from the
|
|
23
|
+
* final tag sets — same reconsolidation pass as the nightly.
|
|
24
|
+
* - No-fit (owner amendment 07.07): an LLM reply of {"tags": []} means no
|
|
25
|
+
* configured domain fits — there is NO fallback category. The memory gets
|
|
26
|
+
* a WEAK primary (argmax prototype cosine, when >= weakPrimaryFloor) or,
|
|
27
|
+
* below the floor, accelerated decay (base_strength halved, domain left
|
|
28
|
+
* NULL so later runs re-attempt it). See nofit.ts.
|
|
29
|
+
* - LLM pre-flight: the endpoint classification will actually use (when a
|
|
30
|
+
* separate Ollama) is probed before any work. If unreachable, abort
|
|
31
|
+
* cleanly (nothing written, cursor untouched) — strict, like distill.
|
|
32
|
+
* - Infra-error abort (issue #150): if the classifier returns null mid-run
|
|
33
|
+
* (endpoint died AFTER preflight), the run aborts after committing
|
|
34
|
+
* the last full batch. The failing memory is left completely untouched; the
|
|
35
|
+
* cursor sits at the last committed batch so a re-run resumes cleanly.
|
|
36
|
+
* - Server-mode only: needs the local DB.
|
|
37
|
+
*
|
|
38
|
+
* Requires a `domains` list in ~/.hicortex/config.json — without it there is
|
|
39
|
+
* nothing to classify into and the command exits with a clear message.
|
|
40
|
+
*/
|
|
41
|
+
var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
|
|
42
|
+
if (k2 === undefined) k2 = k;
|
|
43
|
+
var desc = Object.getOwnPropertyDescriptor(m, k);
|
|
44
|
+
if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
|
|
45
|
+
desc = { enumerable: true, get: function() { return m[k]; } };
|
|
46
|
+
}
|
|
47
|
+
Object.defineProperty(o, k2, desc);
|
|
48
|
+
}) : (function(o, m, k, k2) {
|
|
49
|
+
if (k2 === undefined) k2 = k;
|
|
50
|
+
o[k2] = m[k];
|
|
51
|
+
}));
|
|
52
|
+
var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
|
|
53
|
+
Object.defineProperty(o, "default", { enumerable: true, value: v });
|
|
54
|
+
}) : function(o, v) {
|
|
55
|
+
o["default"] = v;
|
|
56
|
+
});
|
|
57
|
+
var __importStar = (this && this.__importStar) || (function () {
|
|
58
|
+
var ownKeys = function(o) {
|
|
59
|
+
ownKeys = Object.getOwnPropertyNames || function (o) {
|
|
60
|
+
var ar = [];
|
|
61
|
+
for (var k in o) if (Object.prototype.hasOwnProperty.call(o, k)) ar[ar.length] = k;
|
|
62
|
+
return ar;
|
|
63
|
+
};
|
|
64
|
+
return ownKeys(o);
|
|
65
|
+
};
|
|
66
|
+
return function (mod) {
|
|
67
|
+
if (mod && mod.__esModule) return mod;
|
|
68
|
+
var result = {};
|
|
69
|
+
if (mod != null) for (var k = ownKeys(mod), i = 0; i < k.length; i++) if (k[i] !== "default") __createBinding(result, mod, k[i]);
|
|
70
|
+
__setModuleDefault(result, mod);
|
|
71
|
+
return result;
|
|
72
|
+
};
|
|
73
|
+
})();
|
|
74
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
75
|
+
exports.runClassifyDomains = runClassifyDomains;
|
|
76
|
+
const node_fs_1 = require("node:fs");
|
|
77
|
+
const node_path_1 = require("node:path");
|
|
78
|
+
const node_os_1 = require("node:os");
|
|
79
|
+
const db_js_1 = require("./db.js");
|
|
80
|
+
const state_js_1 = require("./state.js");
|
|
81
|
+
const storage = __importStar(require("./storage.js"));
|
|
82
|
+
const llm_js_1 = require("./llm.js");
|
|
83
|
+
const domain_classify_js_1 = require("./domain-classify.js");
|
|
84
|
+
const consolidate_js_1 = require("./consolidate.js");
|
|
85
|
+
const schema_prototypes_js_1 = require("./schema-prototypes.js");
|
|
86
|
+
const nofit_js_1 = require("./nofit.js");
|
|
87
|
+
const HICORTEX_HOME = (0, node_path_1.join)((0, node_os_1.homedir)(), ".hicortex");
|
|
88
|
+
function readConfig(stateDir) {
|
|
89
|
+
try {
|
|
90
|
+
return JSON.parse((0, node_fs_1.readFileSync)((0, node_path_1.join)(stateDir, "config.json"), "utf-8"));
|
|
91
|
+
}
|
|
92
|
+
catch {
|
|
93
|
+
return null;
|
|
94
|
+
}
|
|
95
|
+
}
|
|
96
|
+
/**
|
|
97
|
+
* Pre-flight the endpoint classification will ACTUALLY use — the classify
|
|
98
|
+
* tier (classifyModel/classifyBaseUrl) when configured, else the reflect tier.
|
|
99
|
+
* Target resolution is the pure resolveClassifyProbeTarget (llm.ts), the same
|
|
100
|
+
* source of truth as the nightly's contentDomainsReady gate.
|
|
101
|
+
*
|
|
102
|
+
* Returns null when ready, or a reason string when it is unreachable (caller
|
|
103
|
+
* aborts clean). Only a separate Ollama endpoint can go unreachable mid-run;
|
|
104
|
+
* API providers are cloud-reachable.
|
|
105
|
+
*/
|
|
106
|
+
async function preflightClassify(config) {
|
|
107
|
+
const target = (0, llm_js_1.resolveClassifyProbeTarget)(config);
|
|
108
|
+
if (!target)
|
|
109
|
+
return null;
|
|
110
|
+
const health = await (0, llm_js_1.probeOllamaModel)(target.baseUrl, target.model);
|
|
111
|
+
if (!health.ok) {
|
|
112
|
+
return health.reason === "unreachable"
|
|
113
|
+
? `${target.tier} endpoint unreachable (${target.baseUrl})`
|
|
114
|
+
: `${target.tier} model not loaded (${target.model} missing on ${target.baseUrl})`;
|
|
115
|
+
}
|
|
116
|
+
return null;
|
|
117
|
+
}
|
|
118
|
+
/**
|
|
119
|
+
* Run the classify-domains pass. Returns a structured report.
|
|
120
|
+
* Throws on unrecoverable setup errors (client mode, no domains, no LLM,
|
|
121
|
+
* classification endpoint down) — the cursor always reflects the last
|
|
122
|
+
* committed batch.
|
|
123
|
+
*/
|
|
124
|
+
async function runClassifyDomains(options = {}) {
|
|
125
|
+
const batchSize = options.batchSize ?? 200;
|
|
126
|
+
const stateDir = options.stateDir ?? HICORTEX_HOME;
|
|
127
|
+
const all = options.all ?? false;
|
|
128
|
+
if (!Number.isInteger(batchSize) || batchSize < 1) {
|
|
129
|
+
throw new Error(`[hicortex] classify-domains: invalid --batch value: ${options.batchSize}`);
|
|
130
|
+
}
|
|
131
|
+
const config = options.config !== undefined ? options.config : readConfig(stateDir);
|
|
132
|
+
// Server-mode only — client installs have no local DB.
|
|
133
|
+
if (config?.mode === "client") {
|
|
134
|
+
throw new Error("[hicortex] classify-domains is server-mode only (it needs the local DB). " +
|
|
135
|
+
`This machine is a client of ${config.serverUrl ?? "a remote server"} — run it on the server.`);
|
|
136
|
+
}
|
|
137
|
+
const domains = (0, domain_classify_js_1.parseConfigDomains)(config);
|
|
138
|
+
if (!domains) {
|
|
139
|
+
throw new Error("[hicortex] classify-domains needs a `domains` list in ~/.hicortex/config.json. " +
|
|
140
|
+
'Add e.g. { "domains": [{ "name": "Work", "description": "..." }, ' +
|
|
141
|
+
'{ "name": "Boating", "description": "..." }] } and re-run. ' +
|
|
142
|
+
"No fallback bucket is needed — no-fit memories are handled automatically.");
|
|
143
|
+
}
|
|
144
|
+
const weakPrimaryFloor = (0, nofit_js_1.resolveWeakPrimaryFloor)(config);
|
|
145
|
+
// Resolve the LLM (classify tier does the classifying; falls back to reflect).
|
|
146
|
+
let llm;
|
|
147
|
+
if (options.llm) {
|
|
148
|
+
llm = options.llm;
|
|
149
|
+
}
|
|
150
|
+
else {
|
|
151
|
+
const resolved = (0, llm_js_1.resolveSavedLlmConfig)(config);
|
|
152
|
+
if (!resolved.config) {
|
|
153
|
+
throw new Error("[hicortex] classify-domains: no LLM configured — run `npx @gamaze/hicortex init`.");
|
|
154
|
+
}
|
|
155
|
+
const classifyDown = await preflightClassify(resolved.config);
|
|
156
|
+
if (classifyDown) {
|
|
157
|
+
throw new Error(`[hicortex] classify-domains aborted: ${classifyDown}. ` +
|
|
158
|
+
"Nothing written, cursor untouched — retry when the endpoint is up.");
|
|
159
|
+
}
|
|
160
|
+
llm = new llm_js_1.LlmClient(resolved.config);
|
|
161
|
+
}
|
|
162
|
+
const dbPath = (0, db_js_1.resolveDbPath)(options.dbPath);
|
|
163
|
+
const db = (0, db_js_1.initDb)(dbPath);
|
|
164
|
+
const report = {
|
|
165
|
+
scanned: 0,
|
|
166
|
+
classified: 0,
|
|
167
|
+
unchanged: 0,
|
|
168
|
+
failed: 0,
|
|
169
|
+
weakPrimary: 0,
|
|
170
|
+
noAssociationDecayed: 0,
|
|
171
|
+
batches: 0,
|
|
172
|
+
cursor: 0,
|
|
173
|
+
aborted: false,
|
|
174
|
+
byDomain: {},
|
|
175
|
+
totalTags: 0,
|
|
176
|
+
weightsRecomputed: 0,
|
|
177
|
+
primariesUpdated: 0,
|
|
178
|
+
};
|
|
179
|
+
try {
|
|
180
|
+
let cursor = options.reset ? 0 : ((0, state_js_1.loadState)(stateDir).domainCursor ?? 0);
|
|
181
|
+
report.cursor = cursor;
|
|
182
|
+
console.log(`[hicortex] classify-domains starting: ${domains.length} domains, ` +
|
|
183
|
+
`scope ${all ? "ALL" : "null/stale"}, batch ${batchSize}, cursor ${cursor}` +
|
|
184
|
+
`${options.reset ? " (reset)" : ""}`);
|
|
185
|
+
// Lazy embedder — only loaded if a domain needs a description seed
|
|
186
|
+
// (member_count < 5). Same pattern as relink's fallback embedder.
|
|
187
|
+
let embedFn = options.embedFn ?? null;
|
|
188
|
+
const getEmbedFn = async () => {
|
|
189
|
+
if (!embedFn) {
|
|
190
|
+
const { embed } = await import("./embedder.js");
|
|
191
|
+
embedFn = embed;
|
|
192
|
+
}
|
|
193
|
+
return embedFn;
|
|
194
|
+
};
|
|
195
|
+
// Prototypes once at run start — newly classified memories get their
|
|
196
|
+
// weights from these; the post-run reconsolidation pass refreshes
|
|
197
|
+
// everything from the final tag sets.
|
|
198
|
+
const compartments = (0, schema_prototypes_js_1.compartmentSet)(domains);
|
|
199
|
+
const { prototypes } = await (0, schema_prototypes_js_1.computeDomainPrototypes)(db, domains, getEmbedFn);
|
|
200
|
+
// Scope filter: default = NULL / not-in-set / no tags yet; --all = everything.
|
|
201
|
+
const placeholders = domains.map(() => "?").join(", ");
|
|
202
|
+
const scopeSql = all
|
|
203
|
+
? "rowid > ?"
|
|
204
|
+
: `rowid > ? AND (domain IS NULL OR domain NOT IN (${placeholders}) ` +
|
|
205
|
+
`OR id NOT IN (SELECT DISTINCT memory_id FROM memory_tags))`;
|
|
206
|
+
const batchStmt = db.prepare(`SELECT rowid AS __rowid, id, content, project, domain FROM memories
|
|
207
|
+
WHERE ${scopeSql} ORDER BY rowid ASC LIMIT ?`);
|
|
208
|
+
// Set true when the classifier returns null (infra error): finish the
|
|
209
|
+
// current batch's already-classified writes, commit, advance cursor to the
|
|
210
|
+
// last SUCCESSFULLY-classified row, then stop (the failing row is untouched).
|
|
211
|
+
let infraAbort = false;
|
|
212
|
+
for (;;) {
|
|
213
|
+
const params = all ? [cursor, batchSize] : [cursor, ...domains.map((d) => d.name), batchSize];
|
|
214
|
+
const rows = batchStmt.all(...params);
|
|
215
|
+
if (rows.length === 0)
|
|
216
|
+
break;
|
|
217
|
+
let batchClassified = 0;
|
|
218
|
+
let batchUnchanged = 0;
|
|
219
|
+
let batchWeakPrimary = 0;
|
|
220
|
+
let batchNoAssociation = 0;
|
|
221
|
+
let scannedInBatch = 0;
|
|
222
|
+
// Highest rowid we can safely advance the cursor to (last row we fully
|
|
223
|
+
// resolved — classified or unchanged — before any infra abort).
|
|
224
|
+
let committedRowid = cursor;
|
|
225
|
+
// Classify (network) OUTSIDE the write transaction; collect results.
|
|
226
|
+
// No-fit resolution (resolveNoFit) is read-only, so it also happens in
|
|
227
|
+
// the scan phase; only the writes are deferred to the transaction.
|
|
228
|
+
const writes = [];
|
|
229
|
+
for (const row of rows) {
|
|
230
|
+
const result = await (0, domain_classify_js_1.classifyMemoryTags)(row.content, row.project, domains, llm);
|
|
231
|
+
if (result === null) {
|
|
232
|
+
// Infra error — stop scanning; leave this row untouched for retry.
|
|
233
|
+
infraAbort = true;
|
|
234
|
+
break;
|
|
235
|
+
}
|
|
236
|
+
scannedInBatch++;
|
|
237
|
+
if (result.tags.length === 0) {
|
|
238
|
+
// Genuine no-fit (owner amendment 07.07): weak primary from the
|
|
239
|
+
// prototype argmax when it clears the floor, else accelerated
|
|
240
|
+
// decay. Each rowid is visited at most once per run (cursor is
|
|
241
|
+
// strictly increasing), so a run never double-halves.
|
|
242
|
+
const resolution = (0, nofit_js_1.resolveNoFit)(db, row.id, domains, prototypes, weakPrimaryFloor);
|
|
243
|
+
if (resolution.kind === "weak_primary") {
|
|
244
|
+
batchWeakPrimary++;
|
|
245
|
+
if (resolution.domain === row.domain)
|
|
246
|
+
batchUnchanged++;
|
|
247
|
+
else
|
|
248
|
+
batchClassified++;
|
|
249
|
+
}
|
|
250
|
+
else {
|
|
251
|
+
batchNoAssociation++;
|
|
252
|
+
}
|
|
253
|
+
writes.push({ kind: "nofit", id: row.id, resolution });
|
|
254
|
+
committedRowid = row.__rowid;
|
|
255
|
+
continue;
|
|
256
|
+
}
|
|
257
|
+
// Derived primary (argmax weight from the run-start prototypes,
|
|
258
|
+
// compartment override, LLM order breaking ties) — the same value
|
|
259
|
+
// setMemoryTags will write below.
|
|
260
|
+
const weights = (0, schema_prototypes_js_1.computeTagWeights)(db, row.id, result.tags, prototypes);
|
|
261
|
+
const derived = (0, schema_prototypes_js_1.derivePrimary)(result.tags.map((tag) => ({ tag, weight: weights[tag] ?? null })), compartments);
|
|
262
|
+
if (derived === row.domain) {
|
|
263
|
+
batchUnchanged++;
|
|
264
|
+
}
|
|
265
|
+
else {
|
|
266
|
+
batchClassified++;
|
|
267
|
+
}
|
|
268
|
+
writes.push({ kind: "tags", id: row.id, tags: result.tags, weights });
|
|
269
|
+
committedRowid = row.__rowid;
|
|
270
|
+
}
|
|
271
|
+
// Commit the resolved writes, then persist the cursor at the last fully
|
|
272
|
+
// resolved rowid (crash-safe + infra-abort-safe: a re-run resumes there).
|
|
273
|
+
const tx = db.transaction(() => {
|
|
274
|
+
for (const w of writes) {
|
|
275
|
+
if (w.kind === "tags") {
|
|
276
|
+
storage.setMemoryTags(db, w.id, w.tags, { weights: w.weights, compartments });
|
|
277
|
+
}
|
|
278
|
+
else if (w.resolution.kind === "weak_primary") {
|
|
279
|
+
(0, nofit_js_1.applyWeakPrimary)(db, w.id, w.resolution.domain, w.resolution.weight, compartments);
|
|
280
|
+
}
|
|
281
|
+
else {
|
|
282
|
+
(0, nofit_js_1.applyNoAssociationDecay)(db, w.id);
|
|
283
|
+
}
|
|
284
|
+
}
|
|
285
|
+
});
|
|
286
|
+
tx();
|
|
287
|
+
(0, state_js_1.updateState)((s) => { s.domainCursor = committedRowid; }, stateDir);
|
|
288
|
+
report.scanned += scannedInBatch;
|
|
289
|
+
report.classified += batchClassified;
|
|
290
|
+
report.unchanged += batchUnchanged;
|
|
291
|
+
report.weakPrimary += batchWeakPrimary;
|
|
292
|
+
report.noAssociationDecayed += batchNoAssociation;
|
|
293
|
+
report.batches++;
|
|
294
|
+
report.cursor = committedRowid;
|
|
295
|
+
cursor = committedRowid;
|
|
296
|
+
console.log(`[hicortex] batch ${report.batches}: resolved ${scannedInBatch}, ` +
|
|
297
|
+
`classified ${batchClassified}, unchanged ${batchUnchanged}, ` +
|
|
298
|
+
`weak-primary ${batchWeakPrimary}, no-association ${batchNoAssociation} ` +
|
|
299
|
+
`(cursor ${committedRowid})${infraAbort ? " [infra abort]" : ""}`);
|
|
300
|
+
if (infraAbort) {
|
|
301
|
+
report.aborted = true;
|
|
302
|
+
report.failed++;
|
|
303
|
+
console.warn("[hicortex] classify-domains ABORTED on a classify-endpoint error. " +
|
|
304
|
+
"The failing memory is untouched; cursor at last committed batch — re-run when the endpoint is back up.");
|
|
305
|
+
break;
|
|
306
|
+
}
|
|
307
|
+
}
|
|
308
|
+
// Post-run reconsolidation (same pass as the nightly, skipped on infra
|
|
309
|
+
// abort — the corpus is partially classified; the next full run or nightly
|
|
310
|
+
// repairs it): recompute prototypes from the FINAL tag sets, refresh every
|
|
311
|
+
// weight and derived primary, then rebuild the moduleIndex counts from the
|
|
312
|
+
// refreshed primaries.
|
|
313
|
+
if (!report.aborted) {
|
|
314
|
+
const { prototypes: finalPrototypes } = await (0, schema_prototypes_js_1.computeDomainPrototypes)(db, domains, getEmbedFn);
|
|
315
|
+
report.weightsRecomputed = (0, schema_prototypes_js_1.recomputeAllTagWeights)(db, finalPrototypes).updated;
|
|
316
|
+
report.primariesUpdated = (0, schema_prototypes_js_1.refreshPrimaries)(db, domains).updated;
|
|
317
|
+
(0, consolidate_js_1.rebuildContentModuleIndex)(db, domains, stateDir);
|
|
318
|
+
}
|
|
319
|
+
// Final per-PRIMARY counts across the whole corpus.
|
|
320
|
+
const counts = db
|
|
321
|
+
.prepare(`SELECT domain, COUNT(*) AS cnt FROM memories WHERE domain IS NOT NULL GROUP BY domain ORDER BY cnt DESC`)
|
|
322
|
+
.all();
|
|
323
|
+
for (const c of counts)
|
|
324
|
+
report.byDomain[c.domain] = c.cnt;
|
|
325
|
+
// Total tag assignments across memory_tags (multi-label breadth).
|
|
326
|
+
report.totalTags = db.prepare("SELECT COUNT(*) AS cnt FROM memory_tags").get().cnt;
|
|
327
|
+
const breakdown = counts.map((c) => `${c.domain}=${c.cnt}`).join(", ") || "none";
|
|
328
|
+
console.log(`[hicortex] classify-domains ${report.aborted ? "ABORTED" : "complete"}: ` +
|
|
329
|
+
`${report.scanned} resolved, ${report.classified} (re)filed, ` +
|
|
330
|
+
`${report.unchanged} unchanged, ${report.weakPrimary} weak-primary, ` +
|
|
331
|
+
`${report.noAssociationDecayed} no-association decayed, ${report.failed} infra-skipped, ` +
|
|
332
|
+
`${report.totalTags} total tags, ${report.weightsRecomputed} weights recomputed, ` +
|
|
333
|
+
`${report.primariesUpdated} primaries updated (hash ${(0, domain_classify_js_1.domainSetHash)(domains).slice(0, 8)})`);
|
|
334
|
+
console.log(`[hicortex] by primary: ${breakdown}`);
|
|
335
|
+
return report;
|
|
336
|
+
}
|
|
337
|
+
finally {
|
|
338
|
+
db.close();
|
|
339
|
+
}
|
|
340
|
+
}
|
package/dist/cli.d.ts
CHANGED
|
@@ -8,6 +8,7 @@
|
|
|
8
8
|
* nightly Run capture + consolidate (manual trigger)
|
|
9
9
|
* nightly --capture-only Capture only, skip consolidation
|
|
10
10
|
* nightly --status Show nightly pipeline health check
|
|
11
|
+
* relink Resumable link-discovery pass over the entire corpus (issue #143)
|
|
11
12
|
* status Show config, DB stats, adapter status
|
|
12
13
|
* uninstall Clean removal of CC integration
|
|
13
14
|
*/
|
package/dist/cli.js
CHANGED
|
@@ -9,6 +9,7 @@
|
|
|
9
9
|
* nightly Run capture + consolidate (manual trigger)
|
|
10
10
|
* nightly --capture-only Capture only, skip consolidation
|
|
11
11
|
* nightly --status Show nightly pipeline health check
|
|
12
|
+
* relink Resumable link-discovery pass over the entire corpus (issue #143)
|
|
12
13
|
* status Show config, DB stats, adapter status
|
|
13
14
|
* uninstall Clean removal of CC integration
|
|
14
15
|
*/
|
|
@@ -61,6 +62,58 @@ switch (command) {
|
|
|
61
62
|
}
|
|
62
63
|
break;
|
|
63
64
|
}
|
|
65
|
+
case "relink": {
|
|
66
|
+
const args = process.argv.slice(3);
|
|
67
|
+
const intFlag = (name) => {
|
|
68
|
+
const idx = args.indexOf(name);
|
|
69
|
+
if (idx === -1)
|
|
70
|
+
return undefined;
|
|
71
|
+
const val = parseInt(args[idx + 1], 10);
|
|
72
|
+
if (isNaN(val)) {
|
|
73
|
+
console.error(`[hicortex] relink: ${name} requires an integer value`);
|
|
74
|
+
process.exit(1);
|
|
75
|
+
}
|
|
76
|
+
return val;
|
|
77
|
+
};
|
|
78
|
+
const relinkOptions = {
|
|
79
|
+
dryRun: args.includes("--dry-run"),
|
|
80
|
+
reset: args.includes("--reset"),
|
|
81
|
+
batchSize: intFlag("--batch"),
|
|
82
|
+
};
|
|
83
|
+
import("./relink.js").then(({ runRelink }) => {
|
|
84
|
+
runRelink(relinkOptions).catch((err) => {
|
|
85
|
+
console.error(err instanceof Error ? err.message : `[hicortex] Relink failed: ${err}`);
|
|
86
|
+
process.exit(1);
|
|
87
|
+
});
|
|
88
|
+
});
|
|
89
|
+
break;
|
|
90
|
+
}
|
|
91
|
+
case "classify-domains": {
|
|
92
|
+
const args = process.argv.slice(3);
|
|
93
|
+
const intFlag = (name) => {
|
|
94
|
+
const idx = args.indexOf(name);
|
|
95
|
+
if (idx === -1)
|
|
96
|
+
return undefined;
|
|
97
|
+
const val = parseInt(args[idx + 1], 10);
|
|
98
|
+
if (isNaN(val)) {
|
|
99
|
+
console.error(`[hicortex] classify-domains: ${name} requires an integer value`);
|
|
100
|
+
process.exit(1);
|
|
101
|
+
}
|
|
102
|
+
return val;
|
|
103
|
+
};
|
|
104
|
+
const classifyOptions = {
|
|
105
|
+
all: args.includes("--all"),
|
|
106
|
+
reset: args.includes("--reset"),
|
|
107
|
+
batchSize: intFlag("--batch"),
|
|
108
|
+
};
|
|
109
|
+
import("./classify-domains.js").then(({ runClassifyDomains }) => {
|
|
110
|
+
runClassifyDomains(classifyOptions).catch((err) => {
|
|
111
|
+
console.error(err instanceof Error ? err.message : `[hicortex] classify-domains failed: ${err}`);
|
|
112
|
+
process.exit(1);
|
|
113
|
+
});
|
|
114
|
+
});
|
|
115
|
+
break;
|
|
116
|
+
}
|
|
64
117
|
case "status":
|
|
65
118
|
import("./status.js").then(({ runStatus }) => {
|
|
66
119
|
runStatus().catch((err) => {
|
|
@@ -99,8 +152,12 @@ Usage: hicortex <command> [options]
|
|
|
99
152
|
Commands:
|
|
100
153
|
server Start the MCP HTTP/SSE server (server mode)
|
|
101
154
|
init Set up Hicortex (server mode, local DB + daemon)
|
|
155
|
+
Scaffolds 5 editable default memory domains (Work, Personal,
|
|
156
|
+
People, Health, Finance) in ~/.hicortex/config.json
|
|
102
157
|
init --server <url> Set up as client (remote server)
|
|
103
158
|
nightly Run nightly denoise + capture + consolidate
|
|
159
|
+
relink Resumable link-discovery pass over the ENTIRE corpus (server mode)
|
|
160
|
+
classify-domains Backfill content-based domain tags over the corpus (server mode, needs config.domains)
|
|
104
161
|
lessons-context Fetch lessons and print Markdown to stdout (CC SessionStart hook)
|
|
105
162
|
status Show current configuration and stats
|
|
106
163
|
uninstall Remove CC integration (preserves DB)
|
|
@@ -111,6 +168,12 @@ Options:
|
|
|
111
168
|
nightly --dry-run Preview without changes
|
|
112
169
|
nightly --capture-only Capture only, skip consolidation (safe to run multiple times/day)
|
|
113
170
|
nightly --status Show nightly pipeline health
|
|
171
|
+
relink --dry-run Discovery + counts only, zero writes, cursor untouched
|
|
172
|
+
relink --batch <n> Memories per batch (default: 200)
|
|
173
|
+
relink --reset Restart from the beginning (ignore saved cursor)
|
|
174
|
+
classify-domains --all Reclassify every memory (default: only NULL/stale-domain rows)
|
|
175
|
+
classify-domains --batch <n> Memories per batch (default: 200)
|
|
176
|
+
classify-domains --reset Restart from the beginning (ignore saved cursor)
|
|
114
177
|
|
|
115
178
|
Examples:
|
|
116
179
|
npx @gamaze/hicortex server
|