@dzhechkov/harness-core 0.3.50 → 0.3.54
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/brain.d.ts +146 -4
- package/dist/brain.d.ts.map +1 -1
- package/dist/brain.js +437 -78
- package/dist/brain.js.map +1 -1
- package/dist/index.d.ts +1 -1
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +1 -1
- package/dist/index.js.map +1 -1
- package/package.json +3 -3
- package/src/brain.ts +500 -86
- package/src/index.ts +8 -0
package/dist/brain.js
CHANGED
|
@@ -14,7 +14,7 @@
|
|
|
14
14
|
*
|
|
15
15
|
* @packageDocumentation
|
|
16
16
|
*/
|
|
17
|
-
import { join } from 'node:path';
|
|
17
|
+
import { join, dirname } from 'node:path';
|
|
18
18
|
import { homedir } from 'node:os';
|
|
19
19
|
import { existsSync, mkdirSync, readFileSync, writeFileSync } from 'node:fs';
|
|
20
20
|
import { pathToFileURL } from 'node:url';
|
|
@@ -82,27 +82,57 @@ function resolveSqlite(depsRoot) {
|
|
|
82
82
|
return { error: 'better-sqlite3 not installed (run: dz setup --memory agentdb)' };
|
|
83
83
|
}
|
|
84
84
|
}
|
|
85
|
-
/**
|
|
86
|
-
|
|
87
|
-
const
|
|
88
|
-
|
|
89
|
-
|
|
85
|
+
/** Reconstruct a {@link BookKU} from a raw `book_knowledge` row (shared by every reader below). */
|
|
86
|
+
function rowToKu(r) {
|
|
87
|
+
const pages = jsonPages(r.pages);
|
|
88
|
+
const meta = jsonObj(r.metadata);
|
|
89
|
+
return {
|
|
90
|
+
book: r.book,
|
|
91
|
+
kuId: r.ku_id,
|
|
92
|
+
corpusVersion: r.corpus_version,
|
|
93
|
+
type: r.type,
|
|
94
|
+
name: r.name,
|
|
95
|
+
problem: r.problem,
|
|
96
|
+
content: r.content,
|
|
97
|
+
...(r.chapter !== null ? { chapter: r.chapter } : {}),
|
|
98
|
+
...(pages !== undefined ? { pages } : {}),
|
|
99
|
+
...(Object.keys(meta).length > 0 ? { metadata: meta } : {}),
|
|
100
|
+
};
|
|
101
|
+
}
|
|
102
|
+
/**
|
|
103
|
+
* Read reconstructed KUs from ANY `books.sqlite`-format lexical store (a project store, the brain
|
|
104
|
+
* store, or a per-source slice) — the single shared reader behind {@link promoteProjectToBrain},
|
|
105
|
+
* {@link buildPrimer}, {@link exportBrainSlice}, and {@link importBrainSlice}. `source` narrows to
|
|
106
|
+
* one book; default reads all. Synchronous (native `better-sqlite3` via `createRequire`) and
|
|
107
|
+
* best-effort: an absent store or unresolved dependency returns an honest `error`, never throws.
|
|
108
|
+
*/
|
|
109
|
+
export function readBookKus(opts) {
|
|
110
|
+
const depsRoot = opts.depsRoot ?? process.cwd();
|
|
111
|
+
if (!existsSync(opts.storePath))
|
|
112
|
+
return { kus: [], error: `no book store at ${opts.storePath}` };
|
|
113
|
+
let Database;
|
|
90
114
|
try {
|
|
91
|
-
const
|
|
92
|
-
|
|
115
|
+
const req = createRequire(join(depsRoot, 'package.json'));
|
|
116
|
+
Database = req('better-sqlite3');
|
|
117
|
+
}
|
|
118
|
+
catch {
|
|
119
|
+
return { kus: [], error: 'better-sqlite3 not installed (run: dz setup --memory agentdb)' };
|
|
120
|
+
}
|
|
121
|
+
try {
|
|
122
|
+
const db = new Database(opts.storePath, { readonly: true });
|
|
93
123
|
try {
|
|
94
124
|
const cols = 'book, ku_id, corpus_version, type, name, problem, content, chapter, pages, metadata';
|
|
95
|
-
const rows = (source !== undefined
|
|
96
|
-
? db.prepare(`SELECT ${cols} FROM book_knowledge WHERE book = ?`).all(source)
|
|
125
|
+
const rows = (opts.source !== undefined
|
|
126
|
+
? db.prepare(`SELECT ${cols} FROM book_knowledge WHERE book = ?`).all(opts.source)
|
|
97
127
|
: db.prepare(`SELECT ${cols} FROM book_knowledge`).all());
|
|
98
|
-
return { rows };
|
|
128
|
+
return { kus: rows.map(rowToKu) };
|
|
99
129
|
}
|
|
100
130
|
finally {
|
|
101
131
|
db.close();
|
|
102
132
|
}
|
|
103
133
|
}
|
|
104
134
|
catch (err) {
|
|
105
|
-
return { error: `read
|
|
135
|
+
return { kus: [], error: `read book KB failed: ${err instanceof Error ? err.message : String(err)}` };
|
|
106
136
|
}
|
|
107
137
|
}
|
|
108
138
|
/**
|
|
@@ -168,119 +198,446 @@ function worthScore(meta) {
|
|
|
168
198
|
return 0.5;
|
|
169
199
|
}
|
|
170
200
|
}
|
|
201
|
+
// ──────────────────────────────────────── Primers ───────────────────────────────────────────
|
|
202
|
+
/** Clamp + whitespace-collapse a one-line problem for the primer (deterministic). */
|
|
203
|
+
function primerLine(s, max = 120) {
|
|
204
|
+
const flat = s.replace(/\s+/g, ' ').trim();
|
|
205
|
+
return flat.length <= max ? flat : `${flat.slice(0, max - 1).trimEnd()}…`;
|
|
206
|
+
}
|
|
207
|
+
/**
|
|
208
|
+
* Render a source's capability card (ADR-001 §5.4) from its registry entry + KUs — PURE and
|
|
209
|
+
* DETERMINISTIC (no clock, no query): a header (slug/kind/kuCount/corpusVersion/isbn/lang/license),
|
|
210
|
+
* a KU-TYPE histogram (count per type, count-desc then type-asc), and the top ~8 decision/framework
|
|
211
|
+
* KUs (name + one-line problem, page-anchored), ranked by declared `worth` then `kuId`.
|
|
212
|
+
*/
|
|
213
|
+
function buildPrimerMarkdown(source, kus) {
|
|
214
|
+
const lines = [`# Primer: ${source.slug}`, ''];
|
|
215
|
+
lines.push(`- kind: ${source.kind}`);
|
|
216
|
+
lines.push(`- KUs: ${source.kuCount}`);
|
|
217
|
+
if (source.corpusVersion !== undefined && source.corpusVersion !== '')
|
|
218
|
+
lines.push(`- corpusVersion: ${source.corpusVersion}`);
|
|
219
|
+
if (source.isbn !== undefined)
|
|
220
|
+
lines.push(`- isbn: ${source.isbn}`);
|
|
221
|
+
if (source.lang !== undefined)
|
|
222
|
+
lines.push(`- lang: ${source.lang}`);
|
|
223
|
+
if (source.license !== undefined)
|
|
224
|
+
lines.push(`- license: ${source.license}`);
|
|
225
|
+
lines.push('');
|
|
226
|
+
// KU-type histogram — count per type, count-desc then type-asc (stable, deterministic).
|
|
227
|
+
const counts = new Map();
|
|
228
|
+
for (const k of kus)
|
|
229
|
+
counts.set(k.type, (counts.get(k.type) ?? 0) + 1);
|
|
230
|
+
const hist = [...counts.entries()].sort((a, b) => b[1] - a[1] || a[0].localeCompare(b[0]));
|
|
231
|
+
lines.push('## KU types');
|
|
232
|
+
for (const [type, n] of hist)
|
|
233
|
+
lines.push(`- ${type}: ${n}`);
|
|
234
|
+
lines.push('');
|
|
235
|
+
// Top decision-grade KUs — worth-desc then kuId-asc, first 8; page-anchored provenance. Match the
|
|
236
|
+
// ACTUAL KU type vocabulary (decision-framework / tradeoff-table / methodology / heuristic), not the
|
|
237
|
+
// legacy 'decision'/'framework' shorthands that never occur in real KUs (primer showed _none_).
|
|
238
|
+
const DECISION_TYPES = new Set(['decision-framework', 'tradeoff-table', 'methodology', 'heuristic', 'decision', 'framework']);
|
|
239
|
+
const top = kus
|
|
240
|
+
.filter((k) => DECISION_TYPES.has(k.type))
|
|
241
|
+
.sort((a, b) => worthScore(b.metadata ?? {}) - worthScore(a.metadata ?? {}) || a.kuId.localeCompare(b.kuId))
|
|
242
|
+
.slice(0, 8);
|
|
243
|
+
lines.push('## Top decision moments');
|
|
244
|
+
if (top.length === 0) {
|
|
245
|
+
lines.push('_none_');
|
|
246
|
+
}
|
|
247
|
+
else {
|
|
248
|
+
top.forEach((k, i) => {
|
|
249
|
+
const ch = k.chapter !== undefined && k.chapter !== '' ? ` гл.${k.chapter}` : '';
|
|
250
|
+
const pg = k.pages !== undefined && k.pages.length > 0 ? ` с.${k.pages.join('–')}` : '';
|
|
251
|
+
lines.push(`${i + 1}. ${k.name} — ${primerLine(k.problem)}${ch}${pg}`);
|
|
252
|
+
});
|
|
253
|
+
}
|
|
254
|
+
lines.push('');
|
|
255
|
+
return lines.join('\n');
|
|
256
|
+
}
|
|
257
|
+
/** Synthesize a {@link BrainSource} for a primer when the registry has no entry yet (metadata-derived). */
|
|
258
|
+
function sourceFromKus(slug, kus) {
|
|
259
|
+
const meta0 = kus[0]?.metadata ?? {};
|
|
260
|
+
const corpusVersion = kus[0]?.corpusVersion ?? '';
|
|
261
|
+
return {
|
|
262
|
+
slug,
|
|
263
|
+
kind: 'book',
|
|
264
|
+
kuCount: kus.length,
|
|
265
|
+
addedTs: '',
|
|
266
|
+
...(corpusVersion !== '' ? { corpusVersion } : {}),
|
|
267
|
+
...(typeof meta0['lang'] === 'string' ? { lang: meta0['lang'] } : {}),
|
|
268
|
+
...(typeof meta0['isbn'] === 'string' ? { isbn: meta0['isbn'] } : {}),
|
|
269
|
+
...(typeof meta0['license'] === 'string' ? { license: meta0['license'] } : {}),
|
|
270
|
+
};
|
|
271
|
+
}
|
|
171
272
|
/**
|
|
172
|
-
*
|
|
173
|
-
*
|
|
174
|
-
*
|
|
175
|
-
|
|
273
|
+
* Build a source's capability card (ADR-001 §5.4) by reading its KUs from the brain lexical store
|
|
274
|
+
* ({@link readBookKus}) + its registry entry, then rendering {@link buildPrimerMarkdown}. Best-effort:
|
|
275
|
+
* a missing store/source returns an honest `error` with empty `markdown`, never throws.
|
|
276
|
+
*/
|
|
277
|
+
export async function buildPrimer(opts) {
|
|
278
|
+
const home = opts.brainHome ?? brainHome();
|
|
279
|
+
const depsRoot = opts.depsRoot ?? process.cwd();
|
|
280
|
+
const read = readBookKus({ storePath: brainBooksPath(home), depsRoot, source: opts.slug });
|
|
281
|
+
if (read.error !== undefined)
|
|
282
|
+
return { markdown: '', error: read.error };
|
|
283
|
+
if (read.kus.length === 0)
|
|
284
|
+
return { markdown: '', error: `no KUs for source '${opts.slug}' in brain` };
|
|
285
|
+
const source = readRegistry(home).sources[opts.slug] ?? sourceFromKus(opts.slug, read.kus);
|
|
286
|
+
return { markdown: buildPrimerMarkdown(source, read.kus) };
|
|
287
|
+
}
|
|
288
|
+
/** Write a source's primer to `<brainHome>/primers/<slug>.md` (mkdir). Best-effort; honest error. */
|
|
289
|
+
export async function writePrimer(opts) {
|
|
290
|
+
const home = opts.brainHome ?? brainHome();
|
|
291
|
+
const path = join(home, 'primers', `${opts.slug}.md`);
|
|
292
|
+
const built = await buildPrimer(opts);
|
|
293
|
+
if (built.error !== undefined)
|
|
294
|
+
return { path, error: built.error };
|
|
295
|
+
try {
|
|
296
|
+
mkdirSync(dirname(path), { recursive: true });
|
|
297
|
+
writeFileSync(path, built.markdown);
|
|
298
|
+
return { path };
|
|
299
|
+
}
|
|
300
|
+
catch (err) {
|
|
301
|
+
return { path, error: `write primer failed: ${err instanceof Error ? err.message : String(err)}` };
|
|
302
|
+
}
|
|
303
|
+
}
|
|
304
|
+
// ──────────────────────────────────────── Mirror ────────────────────────────────────────────
|
|
305
|
+
/**
|
|
306
|
+
* The ONE shared "mirror KUs into the brain + register + primer" path behind
|
|
307
|
+
* {@link promoteProjectToBrain}, {@link importBrainSlice}, and {@link registerKusToBrain}. Groups
|
|
308
|
+
* the KUs by `book` and, per book: (1) lexical-mirrors via the idempotent {@link putBookKnowledge},
|
|
309
|
+
* (2) re-embeds via {@link indexPatternsToAgentdb} after pre-deleting the book's stale vectors, and
|
|
310
|
+
* (3) writes/refreshes the registry entry — then generates `primers/<slug>.md` best-effort (a primer
|
|
311
|
+
* failure never fails the mirror) and stores its relative path on the entry.
|
|
176
312
|
*
|
|
177
|
-
* **Non-clobbering:**
|
|
313
|
+
* **Non-clobbering:** mirroring the same book twice does not duplicate — the lexical upsert keys on
|
|
178
314
|
* `(book, ku_id, corpus_version)` and the vector rows are pre-deleted before re-index.
|
|
179
315
|
*/
|
|
180
|
-
|
|
181
|
-
const home = opts
|
|
182
|
-
|
|
183
|
-
const projectDbPath = bookKbPath(opts.projectRoot);
|
|
184
|
-
if (!existsSync(projectDbPath))
|
|
185
|
-
return { sources: [], kus: 0, error: 'no project book KB' };
|
|
186
|
-
const read = await readProjectRows(depsRoot, projectDbPath, opts.source);
|
|
187
|
-
if ('error' in read)
|
|
188
|
-
return { sources: [], kus: 0, error: read.error };
|
|
189
|
-
if (read.rows.length === 0)
|
|
316
|
+
async function mirrorKusToBrain(opts) {
|
|
317
|
+
const { home, depsRoot } = opts;
|
|
318
|
+
if (opts.kus.length === 0)
|
|
190
319
|
return { sources: [], kus: 0 };
|
|
191
|
-
// Group
|
|
320
|
+
// Group KUs by book (each book is one mirror unit + one registry entry).
|
|
192
321
|
const byBook = new Map();
|
|
193
|
-
for (const
|
|
194
|
-
const bucket = byBook.get(
|
|
322
|
+
for (const k of opts.kus) {
|
|
323
|
+
const bucket = byBook.get(k.book);
|
|
195
324
|
if (bucket)
|
|
196
|
-
bucket.push(
|
|
325
|
+
bucket.push(k);
|
|
197
326
|
else
|
|
198
|
-
byBook.set(
|
|
327
|
+
byBook.set(k.book, [k]);
|
|
199
328
|
}
|
|
200
329
|
const reg = readRegistry(home);
|
|
201
330
|
const sources = { ...reg.sources };
|
|
202
|
-
const
|
|
331
|
+
const mirrored = [];
|
|
203
332
|
let totalKus = 0;
|
|
204
333
|
let firstError;
|
|
205
|
-
for (const [book,
|
|
334
|
+
for (const [book, kus] of byBook) {
|
|
206
335
|
// 1. Lexical mirror — reuse the idempotent upsert, pointed at the brain store.
|
|
207
|
-
const kus = rows.map((r) => {
|
|
208
|
-
const pages = jsonPages(r.pages);
|
|
209
|
-
const meta = jsonObj(r.metadata);
|
|
210
|
-
return {
|
|
211
|
-
book: r.book,
|
|
212
|
-
kuId: r.ku_id,
|
|
213
|
-
corpusVersion: r.corpus_version,
|
|
214
|
-
type: r.type,
|
|
215
|
-
name: r.name,
|
|
216
|
-
problem: r.problem,
|
|
217
|
-
content: r.content,
|
|
218
|
-
...(r.chapter !== null ? { chapter: r.chapter } : {}),
|
|
219
|
-
...(pages !== undefined ? { pages } : {}),
|
|
220
|
-
...(Object.keys(meta).length > 0 ? { metadata: meta } : {}),
|
|
221
|
-
};
|
|
222
|
-
});
|
|
223
336
|
const lex = await putBookKnowledge(depsRoot, kus, { dbPath: brainBooksPath(home) });
|
|
224
337
|
if (lex.error !== undefined && firstError === undefined)
|
|
225
338
|
firstError = lex.error;
|
|
226
339
|
// 2. Vector re-embed — pre-delete this book's stale vectors (insert-only primitive), then index.
|
|
227
340
|
await deleteBrainVectors(depsRoot, brainAgentdbPath(home), book);
|
|
228
|
-
const vecRows =
|
|
229
|
-
|
|
230
|
-
|
|
231
|
-
|
|
232
|
-
|
|
233
|
-
|
|
234
|
-
|
|
235
|
-
|
|
236
|
-
|
|
237
|
-
|
|
238
|
-
|
|
239
|
-
|
|
240
|
-
},
|
|
241
|
-
};
|
|
242
|
-
});
|
|
341
|
+
const vecRows = kus.map((k) => ({
|
|
342
|
+
taskType: 'book-knowledge',
|
|
343
|
+
text: `${k.name}: ${k.problem}`,
|
|
344
|
+
score: worthScore(k.metadata ?? {}),
|
|
345
|
+
tags: ['book', k.book, k.type],
|
|
346
|
+
metadata: {
|
|
347
|
+
chapter: k.chapter ?? null,
|
|
348
|
+
pages: k.pages ?? null,
|
|
349
|
+
ku_id: k.kuId,
|
|
350
|
+
corpus_version: k.corpusVersion,
|
|
351
|
+
},
|
|
352
|
+
}));
|
|
243
353
|
const vec = await indexPatternsToAgentdb(depsRoot, vecRows, { dbPath: brainAgentdbPath(home) });
|
|
244
354
|
if (vec.error !== undefined && firstError === undefined)
|
|
245
355
|
firstError = vec.error;
|
|
246
|
-
// 3. Registry entry
|
|
247
|
-
const meta0 =
|
|
248
|
-
const corpusVersion =
|
|
249
|
-
const lang = typeof meta0['lang'] === 'string' ? meta0['lang'] : undefined;
|
|
356
|
+
// 3. Registry entry — corpus_version/lang/isbn/license from opts (explicit) then first KU's metadata.
|
|
357
|
+
const meta0 = kus[0].metadata ?? {};
|
|
358
|
+
const corpusVersion = kus[0].corpusVersion;
|
|
359
|
+
const lang = opts.lang ?? (typeof meta0['lang'] === 'string' ? meta0['lang'] : undefined);
|
|
250
360
|
const isbn = typeof meta0['isbn'] === 'string' ? meta0['isbn'] : undefined;
|
|
251
|
-
|
|
361
|
+
const license = opts.license ?? (typeof meta0['license'] === 'string' ? meta0['license'] : undefined);
|
|
362
|
+
const entry = {
|
|
252
363
|
slug: book,
|
|
253
|
-
kind:
|
|
254
|
-
kuCount:
|
|
364
|
+
kind: opts.kind,
|
|
365
|
+
kuCount: kus.length,
|
|
255
366
|
addedTs: opts.addedTs,
|
|
256
367
|
...(corpusVersion !== undefined && corpusVersion !== '' ? { corpusVersion } : {}),
|
|
257
368
|
...(lang !== undefined ? { lang } : {}),
|
|
258
369
|
...(isbn !== undefined ? { isbn } : {}),
|
|
370
|
+
...(license !== undefined ? { license } : {}),
|
|
259
371
|
};
|
|
260
|
-
|
|
261
|
-
|
|
372
|
+
// 4. Primer — best-effort: a primer failure must not fail the mirror. Store the relative path.
|
|
373
|
+
let primerRel;
|
|
374
|
+
try {
|
|
375
|
+
const rel = join('primers', `${book}.md`);
|
|
376
|
+
const primerPath = join(home, rel);
|
|
377
|
+
mkdirSync(dirname(primerPath), { recursive: true });
|
|
378
|
+
writeFileSync(primerPath, buildPrimerMarkdown(entry, kus));
|
|
379
|
+
primerRel = rel;
|
|
380
|
+
}
|
|
381
|
+
catch {
|
|
382
|
+
/* primer is advisory — never fail the mirror on it */
|
|
383
|
+
}
|
|
384
|
+
sources[book] = primerRel !== undefined ? { ...entry, primer: primerRel } : entry;
|
|
385
|
+
mirrored.push(book);
|
|
386
|
+
totalKus += kus.length;
|
|
262
387
|
}
|
|
263
388
|
writeRegistry(home, { version: 1, sources });
|
|
264
389
|
return firstError !== undefined
|
|
265
|
-
? { sources:
|
|
266
|
-
: { sources:
|
|
390
|
+
? { sources: mirrored, kus: totalKus, error: firstError }
|
|
391
|
+
: { sources: mirrored, kus: totalKus };
|
|
392
|
+
}
|
|
393
|
+
// ──────────────────────────────────────── Promote ───────────────────────────────────────────
|
|
394
|
+
/**
|
|
395
|
+
* Promote a PROJECT's digitized book KB into the durable cross-project brain (ADR-001 P0
|
|
396
|
+
* `book-brain-register`). Reads the project's lexical `books.sqlite` ({@link readBookKus}) and mirrors
|
|
397
|
+
* every KU into the brain via the shared {@link mirrorKusToBrain} path (lexical + vector + registry +
|
|
398
|
+
* primer). `kind: 'book'`.
|
|
399
|
+
*
|
|
400
|
+
* **Non-clobbering:** promoting the same book twice does not duplicate — the lexical upsert keys on
|
|
401
|
+
* `(book, ku_id, corpus_version)` and the vector rows are pre-deleted before re-index.
|
|
402
|
+
*/
|
|
403
|
+
export async function promoteProjectToBrain(opts) {
|
|
404
|
+
const home = opts.brainHome ?? brainHome();
|
|
405
|
+
const depsRoot = opts.depsRoot ?? opts.projectRoot;
|
|
406
|
+
const projectDbPath = bookKbPath(opts.projectRoot);
|
|
407
|
+
if (!existsSync(projectDbPath))
|
|
408
|
+
return { sources: [], kus: 0, error: 'no project book KB' };
|
|
409
|
+
const read = readBookKus({ storePath: projectDbPath, depsRoot, ...(opts.source !== undefined ? { source: opts.source } : {}) });
|
|
410
|
+
if (read.error !== undefined)
|
|
411
|
+
return { sources: [], kus: 0, error: read.error };
|
|
412
|
+
if (read.kus.length === 0)
|
|
413
|
+
return { sources: [], kus: 0 };
|
|
414
|
+
return mirrorKusToBrain({ home, depsRoot, kus: read.kus, kind: 'book', addedTs: opts.addedTs });
|
|
415
|
+
}
|
|
416
|
+
// ────────────────────────────────────────── Update ──────────────────────────────────────────
|
|
417
|
+
/**
|
|
418
|
+
* `dz brain update <slug>` backend (ADR-001 §11 P3 — non-destructive refresh). The project has
|
|
419
|
+
* re-ingested a source's book at a NEW `corpus_version`; this re-reads that source's CURRENT KUs
|
|
420
|
+
* from the PROJECT lexical store ({@link readBookKus} at the project `bookKbPath`, `source=slug`)
|
|
421
|
+
* and re-mirrors them into the brain via the shared {@link mirrorKusToBrain} path.
|
|
422
|
+
*
|
|
423
|
+
* **Non-clobbering + non-destructive:** the per-source stale-corpus eviction inside the reused
|
|
424
|
+
* `putBookKnowledge` upsert evicts THIS source's old-corpus rows and upserts the new ones, while
|
|
425
|
+
* OTHER sources are untouched. The primer is refreshed as part of the mirror. Reports the
|
|
426
|
+
* before/after KU counts (read directly from the brain, so `after` reflects the post-eviction set)
|
|
427
|
+
* and the new `corpusVersion`.
|
|
428
|
+
*
|
|
429
|
+
* Honest failure, no partial state: an unregistered source, or a project with no KUs for it, returns
|
|
430
|
+
* an `error` before any write happens.
|
|
431
|
+
*/
|
|
432
|
+
export async function updateBrainSource(opts) {
|
|
433
|
+
const home = opts.brainHome ?? brainHome();
|
|
434
|
+
const depsRoot = opts.depsRoot ?? opts.projectRoot;
|
|
435
|
+
// Gate 1: the source must already be in the registry (update refreshes, it does not add).
|
|
436
|
+
const existing = readRegistry(home).sources[opts.slug];
|
|
437
|
+
if (existing === undefined) {
|
|
438
|
+
return { before: 0, after: 0, error: `source '${opts.slug}' is not registered in the brain (use promote/register first)` };
|
|
439
|
+
}
|
|
440
|
+
// `before` = the source's CURRENT row count in the brain (read directly, not the cached registry
|
|
441
|
+
// count) so the delta reflects real state.
|
|
442
|
+
const beforeRead = readBookKus({ storePath: brainBooksPath(home), depsRoot, source: opts.slug });
|
|
443
|
+
const before = beforeRead.kus.length;
|
|
444
|
+
// Gate 2: the project must hold the re-ingested KUs for THIS source (no partial state on failure).
|
|
445
|
+
const projectDbPath = bookKbPath(opts.projectRoot);
|
|
446
|
+
if (!existsSync(projectDbPath)) {
|
|
447
|
+
return { before, after: before, error: 'no project book KB' };
|
|
448
|
+
}
|
|
449
|
+
const read = readBookKus({ storePath: projectDbPath, depsRoot, source: opts.slug });
|
|
450
|
+
if (read.error !== undefined)
|
|
451
|
+
return { before, after: before, error: read.error };
|
|
452
|
+
if (read.kus.length === 0) {
|
|
453
|
+
return { before, after: before, error: `project has no KUs for source '${opts.slug}'` };
|
|
454
|
+
}
|
|
455
|
+
// Re-mirror through the shared path — the stale-corpus eviction makes this refresh non-destructive
|
|
456
|
+
// for this source and non-clobbering for others; the primer is rewritten inside the mirror.
|
|
457
|
+
const mirror = await mirrorKusToBrain({
|
|
458
|
+
home,
|
|
459
|
+
depsRoot,
|
|
460
|
+
kus: read.kus,
|
|
461
|
+
kind: existing.kind,
|
|
462
|
+
addedTs: opts.addedTs,
|
|
463
|
+
...(existing.lang !== undefined ? { lang: existing.lang } : {}),
|
|
464
|
+
...(existing.license !== undefined ? { license: existing.license } : {}),
|
|
465
|
+
});
|
|
466
|
+
// `after` from the brain itself — proves the eviction landed (no orphan old-corpus rows).
|
|
467
|
+
const afterRead = readBookKus({ storePath: brainBooksPath(home), depsRoot, source: opts.slug });
|
|
468
|
+
const after = afterRead.kus.length;
|
|
469
|
+
const newCorpus = read.kus[0]?.corpusVersion;
|
|
470
|
+
const out = { before, after };
|
|
471
|
+
if (newCorpus !== undefined && newCorpus !== '')
|
|
472
|
+
out.corpusVersion = newCorpus;
|
|
473
|
+
if (mirror.error !== undefined)
|
|
474
|
+
out.error = mirror.error;
|
|
475
|
+
return out;
|
|
476
|
+
}
|
|
477
|
+
// ────────────────────────────────────── Slices / ingest ─────────────────────────────────────
|
|
478
|
+
/**
|
|
479
|
+
* Export ONE source's KUs from the brain as a STANDALONE lexical-only `books.sqlite` slice at
|
|
480
|
+
* `outPath` (ADR-001 §8.1). Reads via {@link readBookKus} and writes via {@link putBookKnowledge} —
|
|
481
|
+
* lexical only; vectors re-embed on import. This is the portable, per-book shareable unit.
|
|
482
|
+
*/
|
|
483
|
+
export async function exportBrainSlice(opts) {
|
|
484
|
+
const home = opts.brainHome ?? brainHome();
|
|
485
|
+
const depsRoot = opts.depsRoot ?? process.cwd();
|
|
486
|
+
const read = readBookKus({ storePath: brainBooksPath(home), depsRoot, source: opts.slug });
|
|
487
|
+
if (read.error !== undefined)
|
|
488
|
+
return { kuCount: 0, error: read.error };
|
|
489
|
+
if (read.kus.length === 0)
|
|
490
|
+
return { kuCount: 0, error: `no KUs for source '${opts.slug}' in brain` };
|
|
491
|
+
const put = await putBookKnowledge(depsRoot, read.kus, { dbPath: opts.outPath });
|
|
492
|
+
if (put.error !== undefined)
|
|
493
|
+
return { kuCount: 0, error: put.error };
|
|
494
|
+
return { kuCount: read.kus.length };
|
|
495
|
+
}
|
|
496
|
+
/**
|
|
497
|
+
* Import a per-book slice ({@link exportBrainSlice} output, or a pack's `brain/<slug>.sqlite`) into
|
|
498
|
+
* the brain (ADR-001 §8.1). Reads the slice's KUs ({@link readBookKus}) and mirrors them via the same
|
|
499
|
+
* non-clobbering {@link mirrorKusToBrain} path as promote (upsert + re-embed + registry + primer).
|
|
500
|
+
*/
|
|
501
|
+
export async function importBrainSlice(opts) {
|
|
502
|
+
const home = opts.brainHome ?? brainHome();
|
|
503
|
+
const depsRoot = opts.depsRoot ?? process.cwd();
|
|
504
|
+
const read = readBookKus({ storePath: opts.slicePath, depsRoot });
|
|
505
|
+
if (read.error !== undefined)
|
|
506
|
+
return { sources: [], kus: 0, error: read.error };
|
|
507
|
+
if (read.kus.length === 0)
|
|
508
|
+
return { sources: [], kus: 0 };
|
|
509
|
+
return mirrorKusToBrain({ home, depsRoot, kus: read.kus, kind: 'book', addedTs: opts.addedTs });
|
|
510
|
+
}
|
|
511
|
+
/**
|
|
512
|
+
* Register an array of already-shaped KUs (from a repo deep-walk, §6, or raw JSON) into the brain via
|
|
513
|
+
* the shared {@link mirrorKusToBrain} path — the CLI `--from-kus` backend. `slug` is authoritative:
|
|
514
|
+
* every KU is registered under it (so a repo's `book` field is normalized to the source slug). The
|
|
515
|
+
* registry entry carries `kind` and, for `repo`, `license`.
|
|
516
|
+
*/
|
|
517
|
+
/**
|
|
518
|
+
* SPDX ids the license gate treats as clearly-permissive (auto-pass for repo ingest). Anything else
|
|
519
|
+
* — including an absent license — is refused unless `override:true`. Kept deliberately small and
|
|
520
|
+
* conservative (ADR §8: repos carry their own licenses; the brain must not silently ingest
|
|
521
|
+
* unknown/incompatible source into a redistributable slice).
|
|
522
|
+
*/
|
|
523
|
+
export const PERMISSIVE_LICENSES = [
|
|
524
|
+
'MIT', 'Apache-2.0', 'BSD-2-Clause', 'BSD-3-Clause', 'ISC', 'Unlicense', 'CC0-1.0', '0BSD', 'MIT-0',
|
|
525
|
+
];
|
|
526
|
+
export async function registerKusToBrain(opts) {
|
|
527
|
+
const home = opts.brainHome ?? brainHome();
|
|
528
|
+
const depsRoot = opts.depsRoot ?? process.cwd();
|
|
529
|
+
if (opts.kus.length === 0)
|
|
530
|
+
return { kus: 0 };
|
|
531
|
+
// License gate (ADR §8) — repo/paper sources must declare a permissive license, else refuse unless
|
|
532
|
+
// explicitly overridden. Books promoted from a digitized pack are exempt (own CP5 flow). No writes
|
|
533
|
+
// happen before this check, so a refusal never leaves a partial brain.
|
|
534
|
+
if (opts.kind !== 'book' && opts.override !== true) {
|
|
535
|
+
const lic = opts.license;
|
|
536
|
+
if (lic === undefined || lic === '') {
|
|
537
|
+
return { kus: 0, error: `refusing to ingest '${opts.slug}' (${opts.kind}) with no --license; pass a permissive SPDX id (${PERMISSIVE_LICENSES.join(', ')}) or --override` };
|
|
538
|
+
}
|
|
539
|
+
if (!PERMISSIVE_LICENSES.includes(lic)) {
|
|
540
|
+
return { kus: 0, error: `refusing to ingest '${opts.slug}': license '${lic}' is not in the permissive allow-list (${PERMISSIVE_LICENSES.join(', ')}); pass --override to ingest anyway` };
|
|
541
|
+
}
|
|
542
|
+
}
|
|
543
|
+
// Normalize every KU under the source slug so the batch shares one `book` (putBookKnowledge's key).
|
|
544
|
+
const kus = opts.kus.map((k) => (k.book === opts.slug ? k : { ...k, book: opts.slug }));
|
|
545
|
+
const mirror = await mirrorKusToBrain({
|
|
546
|
+
home,
|
|
547
|
+
depsRoot,
|
|
548
|
+
kus,
|
|
549
|
+
kind: opts.kind,
|
|
550
|
+
addedTs: opts.addedTs,
|
|
551
|
+
...(opts.lang !== undefined ? { lang: opts.lang } : {}),
|
|
552
|
+
...(opts.license !== undefined ? { license: opts.license } : {}),
|
|
553
|
+
});
|
|
554
|
+
return mirror.error !== undefined ? { kus: mirror.kus, error: mirror.error } : { kus: mirror.kus };
|
|
555
|
+
}
|
|
556
|
+
// ──────────────────────────────────────── Rerank ────────────────────────────────────────────
|
|
557
|
+
/** FTS5's default `limit` when none is passed (mirrors {@link queryBookKnowledge}). */
|
|
558
|
+
const DEFAULT_QUERY_LIMIT = 10;
|
|
559
|
+
/**
|
|
560
|
+
* Field weights for the deterministic reranker (ADR-001 §11 P3 / G3): a query term matched in a
|
|
561
|
+
* KU's `name` outranks one only in its `problem`, which outranks one only in `content`. Because a
|
|
562
|
+
* term contributes its BEST field's weight once, summing over distinct terms folds together (a)
|
|
563
|
+
* term COVERAGE — more distinct query terms present ⇒ a higher sum — and (b) FIELD WEIGHT.
|
|
564
|
+
*/
|
|
565
|
+
const RERANK_FIELD_WEIGHT = { name: 3, problem: 2, content: 1 };
|
|
566
|
+
/**
|
|
567
|
+
* KU types that carry a small, deterministic type PRIOR — a decision-grade unit
|
|
568
|
+
* (decision-framework / tradeoff-table / methodology) sits slightly above a bare `definition` when
|
|
569
|
+
* coverage + field weight tie. Kept below the smallest field-weight step (1) so it only breaks
|
|
570
|
+
* near-ties and never overrides a genuinely better term/field match.
|
|
571
|
+
*/
|
|
572
|
+
const RERANK_TYPE_PRIOR = new Set(['decision-framework', 'tradeoff-table', 'methodology']);
|
|
573
|
+
const RERANK_PRIOR_BONUS = 0.5;
|
|
574
|
+
/** Over-fetch cap for the rerank pass — bounds work while leaving room for a `limit*3` window. */
|
|
575
|
+
const RERANK_OVERFETCH_CAP = 200;
|
|
576
|
+
/**
|
|
577
|
+
* A DETERMINISTIC lexical reranker (ADR-001 §11 P3 / G3) — lifts precision on the top-K without a
|
|
578
|
+
* model dependency, so the sync grounding path stays fast + offline. Scores each hit against the
|
|
579
|
+
* query's CONTENT TERMS (reusing {@link contentTerms}) by (a) term COVERAGE, (b) a FIELD WEIGHT
|
|
580
|
+
* (name > problem > content), and (c) a small type PRIOR, then returns the top-`limit` reordered.
|
|
581
|
+
* Fully deterministic: no clock, no random, stable tie-break by `kuId`.
|
|
582
|
+
*
|
|
583
|
+
* This is deliberately lexical: an ML cross-encoder reranker is OUT OF SCOPE for the sync path (it
|
|
584
|
+
* would make grounding slow + online + non-deterministic). A future model reranker is a drop-in
|
|
585
|
+
* swap BEHIND this same `(query, hits, opts) → hits` signature — callers never change.
|
|
586
|
+
*/
|
|
587
|
+
export function rerankHits(query, hits, opts) {
|
|
588
|
+
const terms = contentTerms(query);
|
|
589
|
+
const scored = hits.map((hit, idx) => {
|
|
590
|
+
const name = hit.name.toLowerCase();
|
|
591
|
+
const problem = hit.problem.toLowerCase();
|
|
592
|
+
const content = hit.content.toLowerCase();
|
|
593
|
+
let score = 0;
|
|
594
|
+
for (const t of terms) {
|
|
595
|
+
// Each distinct term contributes its BEST field's weight once (coverage × field weight).
|
|
596
|
+
if (name.includes(t))
|
|
597
|
+
score += RERANK_FIELD_WEIGHT.name;
|
|
598
|
+
else if (problem.includes(t))
|
|
599
|
+
score += RERANK_FIELD_WEIGHT.problem;
|
|
600
|
+
else if (content.includes(t))
|
|
601
|
+
score += RERANK_FIELD_WEIGHT.content;
|
|
602
|
+
}
|
|
603
|
+
if (RERANK_TYPE_PRIOR.has(hit.type))
|
|
604
|
+
score += RERANK_PRIOR_BONUS;
|
|
605
|
+
return { hit, score, idx };
|
|
606
|
+
});
|
|
607
|
+
// Deterministic order: score desc, then stable tie-break by kuId (never by input position/clock).
|
|
608
|
+
scored.sort((a, b) => b.score - a.score || a.hit.kuId.localeCompare(b.hit.kuId));
|
|
609
|
+
const limit = opts?.limit ?? hits.length;
|
|
610
|
+
return scored.slice(0, limit).map((s) => s.hit);
|
|
267
611
|
}
|
|
268
612
|
// ───────────────────────────────────────── Query ────────────────────────────────────────────
|
|
269
613
|
/**
|
|
270
614
|
* Cross-source lexical recall over the whole brain — a thin, brain-home-scoped wrapper over
|
|
271
615
|
* {@link queryBookKnowledge}. `source` narrows to one source; default is cross-source. Never throws.
|
|
616
|
+
*
|
|
617
|
+
* `rerank` (default **false** — pure FTS order stays the default so nothing regresses): when true,
|
|
618
|
+
* over-fetch (`limit*3`, capped) then {@link rerankHits} down to `limit` for on-point top-K.
|
|
272
619
|
*/
|
|
273
620
|
export async function queryBrain(opts) {
|
|
274
621
|
const home = opts.brainHome ?? brainHome();
|
|
275
622
|
const depsRoot = opts.depsRoot ?? process.cwd();
|
|
623
|
+
const rerank = opts.rerank === true;
|
|
624
|
+
const effLimit = opts.limit ?? DEFAULT_QUERY_LIMIT;
|
|
276
625
|
const qopts = { dbPath: brainBooksPath(home) };
|
|
277
626
|
if (opts.source !== undefined)
|
|
278
627
|
qopts.book = opts.source;
|
|
279
|
-
if (
|
|
628
|
+
if (rerank) {
|
|
629
|
+
// Over-fetch a wider window so the reranker has real candidates to reorder, then trim to limit.
|
|
630
|
+
qopts.limit = Math.min(effLimit * 3, RERANK_OVERFETCH_CAP);
|
|
631
|
+
}
|
|
632
|
+
else if (opts.limit !== undefined) {
|
|
280
633
|
qopts.limit = opts.limit;
|
|
634
|
+
}
|
|
281
635
|
if (opts.match !== undefined)
|
|
282
636
|
qopts.match = opts.match;
|
|
283
|
-
|
|
637
|
+
const res = await queryBookKnowledge(depsRoot, opts.query, qopts);
|
|
638
|
+
if (!rerank || res.error !== undefined)
|
|
639
|
+
return res;
|
|
640
|
+
return { hits: rerankHits(opts.query, res.hits, { limit: effLimit }) };
|
|
284
641
|
}
|
|
285
642
|
// ──────────────────────────────────────── Grounding ─────────────────────────────────────────
|
|
286
643
|
/**
|
|
@@ -360,6 +717,8 @@ export async function groundPrompt(opts) {
|
|
|
360
717
|
query: terms.join(' '),
|
|
361
718
|
limit: opts.k ?? 5,
|
|
362
719
|
match: 'any',
|
|
720
|
+
// Grounding wants the most ON-POINT citation first — pass through the deterministic reranker.
|
|
721
|
+
rerank: true,
|
|
363
722
|
};
|
|
364
723
|
if (opts.brainHome !== undefined)
|
|
365
724
|
query.brainHome = opts.brainHome;
|