gnosys 5.12.2 → 5.13.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/index.js +36 -5
- package/dist/lib/archive.js +10 -4
- package/dist/lib/db.d.ts +15 -0
- package/dist/lib/db.js +94 -34
- package/dist/lib/dbWrite.js +12 -0
- package/dist/lib/dream.d.ts +9 -0
- package/dist/lib/dream.js +98 -4
- package/dist/lib/dreamRunLog.d.ts +4 -4
- package/dist/lib/dreamRunLog.js +9 -6
- package/dist/lib/embedDb.d.ts +53 -0
- package/dist/lib/embedDb.js +84 -0
- package/dist/lib/embedQueue.d.ts +28 -0
- package/dist/lib/embedQueue.js +80 -0
- package/dist/lib/ftsQuery.d.ts +25 -0
- package/dist/lib/ftsQuery.js +42 -0
- package/dist/lib/hybridSearch.d.ts +21 -0
- package/dist/lib/hybridSearch.js +37 -1
- package/dist/lib/hybridSearchCommand.js +5 -0
- package/dist/lib/reindexCommand.js +26 -0
- package/dist/lib/resolver.d.ts +17 -0
- package/dist/lib/resolver.js +43 -17
- package/dist/lib/search.js +33 -20
- package/package.json +1 -1
package/dist/index.js
CHANGED
|
@@ -1752,10 +1752,17 @@ regTool("gnosys_hybrid_search", "Search memories using hybrid keyword + semantic
|
|
|
1752
1752
|
};
|
|
1753
1753
|
}
|
|
1754
1754
|
try {
|
|
1755
|
-
const
|
|
1755
|
+
const requestedMode = mode || "hybrid";
|
|
1756
|
+
const results = await hybridSearch.hybridSearch(query, limit || 15, requestedMode);
|
|
1757
|
+
// v5.12.3: hybrid used to degrade to keyword-only silently when the
|
|
1758
|
+
// semantic leg can't run (embeddings are only built by gnosys_reindex,
|
|
1759
|
+
// and DB mode also needs the store-local query embedder). Say so loudly.
|
|
1760
|
+
const degradeWarning = requestedMode !== "keyword" && !hybridSearch.canRunSemantic()
|
|
1761
|
+
? `⚠️ Semantic embeddings unavailable — ${requestedMode} search ran keyword-only. Run gnosys_reindex to build embeddings and enable semantic recall.\n\n`
|
|
1762
|
+
: "";
|
|
1756
1763
|
if (results.length === 0) {
|
|
1757
1764
|
return {
|
|
1758
|
-
content: [{ type: "text", text:
|
|
1765
|
+
content: [{ type: "text", text: `${degradeWarning}No results for "${query}". Try different keywords.` }],
|
|
1759
1766
|
};
|
|
1760
1767
|
}
|
|
1761
1768
|
const formatted = results
|
|
@@ -1774,7 +1781,7 @@ regTool("gnosys_hybrid_search", "Search memories using hybrid keyword + semantic
|
|
|
1774
1781
|
content: [
|
|
1775
1782
|
{
|
|
1776
1783
|
type: "text",
|
|
1777
|
-
text:
|
|
1784
|
+
text: `${degradeWarning}Found ${results.length} results for "${query}" (${embCount} embeddings indexed):\n\n${formatted}`,
|
|
1778
1785
|
},
|
|
1779
1786
|
],
|
|
1780
1787
|
};
|
|
@@ -1802,10 +1809,19 @@ regTool("gnosys_semantic_search", "Search memories using semantic similarity onl
|
|
|
1802
1809
|
};
|
|
1803
1810
|
}
|
|
1804
1811
|
try {
|
|
1812
|
+
// v5.12.3: semantic search without a working semantic leg either
|
|
1813
|
+
// returns nothing or (in DB mode) keyword hits mislabeled as semantic —
|
|
1814
|
+
// refuse with the exact reason instead.
|
|
1815
|
+
if (!hybridSearch.canRunSemantic()) {
|
|
1816
|
+
return {
|
|
1817
|
+
content: [{ type: "text", text: `⚠️ Semantic embeddings unavailable — semantic search cannot run. Run gnosys_reindex to build embeddings, then retry.` }],
|
|
1818
|
+
isError: true,
|
|
1819
|
+
};
|
|
1820
|
+
}
|
|
1805
1821
|
const results = await hybridSearch.hybridSearch(query, limit || 15, "semantic");
|
|
1806
1822
|
if (results.length === 0) {
|
|
1807
1823
|
return {
|
|
1808
|
-
content: [{ type: "text", text: `No semantic results for "${query}".
|
|
1824
|
+
content: [{ type: "text", text: `No semantic results for "${query}". Try a broader query.` }],
|
|
1809
1825
|
};
|
|
1810
1826
|
}
|
|
1811
1827
|
const formatted = results
|
|
@@ -1837,11 +1853,18 @@ regTool("gnosys_reindex", "Rebuild all semantic embeddings from every memory fil
|
|
|
1837
1853
|
// Also rebuild FTS5 index
|
|
1838
1854
|
await reindexAllStores();
|
|
1839
1855
|
const count = await hybridSearch.reindex();
|
|
1856
|
+
// v5.13.0: also (re)build the central-DB embedding column — the
|
|
1857
|
+
// vectors DB-mode hybrid/semantic search actually reads.
|
|
1858
|
+
const dbResult = await hybridSearch.reindexCentralDb();
|
|
1859
|
+
const parts = [`${count} file-store memories embedded`];
|
|
1860
|
+
if (dbResult.total > 0) {
|
|
1861
|
+
parts.push(`${dbResult.embedded}/${dbResult.total} central-DB memories embedded`);
|
|
1862
|
+
}
|
|
1840
1863
|
return {
|
|
1841
1864
|
content: [
|
|
1842
1865
|
{
|
|
1843
1866
|
type: "text",
|
|
1844
|
-
text: `Reindex complete: ${
|
|
1867
|
+
text: `Reindex complete: ${parts.join(", ")}. Hybrid search is now available.`,
|
|
1845
1868
|
},
|
|
1846
1869
|
],
|
|
1847
1870
|
};
|
|
@@ -3169,6 +3192,14 @@ async function initHeavyDeps() {
|
|
|
3169
3192
|
const embeddings = new GnosysEmbeddings(writeTarget.store.getStorePath());
|
|
3170
3193
|
hybridSearch = new GnosysHybridSearch(search, embeddings, resolver, writeTarget.store.getStorePath(), gnosysDb || undefined);
|
|
3171
3194
|
askEngine = new GnosysAsk(hybridSearch, config, resolver, writeTarget.store.getStorePath());
|
|
3195
|
+
// v5.13.0: write-time embeddings (serve mode only) — new and updated
|
|
3196
|
+
// memories get vectors without waiting for a manual reindex. Best-effort:
|
|
3197
|
+
// queued writes drain on an unref'd timer and never block a tool call.
|
|
3198
|
+
if (centralDb?.isAvailable() && centralDb.isMigrated()) {
|
|
3199
|
+
const { enableWriteTimeEmbedding } = await import("./lib/embedQueue.js");
|
|
3200
|
+
enableWriteTimeEmbedding(() => centralDb, embeddings);
|
|
3201
|
+
console.error("Write-time embeddings: enabled (central DB)");
|
|
3202
|
+
}
|
|
3172
3203
|
const embCount = embeddings.hasEmbeddings() ? embeddings.count() : 0;
|
|
3173
3204
|
console.error(`Hybrid search: ${embCount > 0 ? `ready (${embCount} embeddings)` : "available (run gnosys_reindex to build embeddings)"}`);
|
|
3174
3205
|
console.error(`Ask engine: ${askEngine.isLLMAvailable ? `ready (${askEngine.providerName}/${askEngine.modelName})` : "disabled (configure LLM provider)"}`);
|
package/dist/lib/archive.js
CHANGED
|
@@ -23,6 +23,7 @@ import { statSync } from "fs";
|
|
|
23
23
|
import { syncMemoryToDb, syncDearchiveToDb } from "./dbWrite.js";
|
|
24
24
|
import { enableWAL } from "./lock.js";
|
|
25
25
|
import { auditLog } from "./audit.js";
|
|
26
|
+
import { ftsTerms, ftsAndQuery, ftsOrQuery } from "./ftsQuery.js";
|
|
26
27
|
// ─── Archive Manager ────────────────────────────────────────────────────
|
|
27
28
|
export class GnosysArchive {
|
|
28
29
|
db = null;
|
|
@@ -217,8 +218,8 @@ export class GnosysArchive {
|
|
|
217
218
|
searchArchive(query, limit = 20) {
|
|
218
219
|
if (!this.db)
|
|
219
220
|
return [];
|
|
220
|
-
const
|
|
221
|
-
if (
|
|
221
|
+
const terms = ftsTerms(query);
|
|
222
|
+
if (terms.length === 0)
|
|
222
223
|
return [];
|
|
223
224
|
const stmt = this.db.prepare(`
|
|
224
225
|
SELECT
|
|
@@ -235,7 +236,12 @@ export class GnosysArchive {
|
|
|
235
236
|
LIMIT ?
|
|
236
237
|
`);
|
|
237
238
|
try {
|
|
238
|
-
|
|
239
|
+
// v5.12.3: AND first (precision), OR retry when AND finds nothing —
|
|
240
|
+
// multi-word queries previously required every term to match.
|
|
241
|
+
const results = stmt.all(ftsAndQuery(terms), limit);
|
|
242
|
+
if (results.length > 0 || terms.length === 1)
|
|
243
|
+
return results;
|
|
244
|
+
return stmt.all(ftsOrQuery(terms), limit);
|
|
239
245
|
}
|
|
240
246
|
catch {
|
|
241
247
|
// FTS5 query syntax failed — try LIKE fallback
|
|
@@ -251,7 +257,7 @@ export class GnosysArchive {
|
|
|
251
257
|
WHERE content LIKE ? OR title LIKE ? OR tags LIKE ?
|
|
252
258
|
LIMIT ?
|
|
253
259
|
`);
|
|
254
|
-
const pattern = `%${
|
|
260
|
+
const pattern = `%${terms.join(" ")}%`;
|
|
255
261
|
return likeStmt.all(pattern, pattern, pattern, limit);
|
|
256
262
|
}
|
|
257
263
|
}
|
package/dist/lib/db.d.ts
CHANGED
|
@@ -317,6 +317,21 @@ export declare class GnosysDB {
|
|
|
317
317
|
operation?: string;
|
|
318
318
|
limit?: number;
|
|
319
319
|
}): DbAuditEntry[];
|
|
320
|
+
/**
|
|
321
|
+
* v5.13.0: rows needed to (re)build the embedding column. All tiers and
|
|
322
|
+
* statuses are included — matching FTS, which indexes every row and
|
|
323
|
+
* filters at query time. Newest-first so capped backfills prioritize
|
|
324
|
+
* recent memories.
|
|
325
|
+
*/
|
|
326
|
+
getMemoriesForEmbedding(mode: "missing" | "all", limit?: number): Array<{
|
|
327
|
+
id: string;
|
|
328
|
+
title: string;
|
|
329
|
+
relevance: string | null;
|
|
330
|
+
tags: string;
|
|
331
|
+
content: string;
|
|
332
|
+
}>;
|
|
333
|
+
/** v5.13.0: memories whose embedding column is NULL (embedding-health check). */
|
|
334
|
+
countMemoriesMissingEmbedding(): number;
|
|
320
335
|
updateEmbedding(id: string, embedding: Buffer): void;
|
|
321
336
|
getEmbedding(id: string): Buffer | null;
|
|
322
337
|
getAllEmbeddings(): Array<{
|
package/dist/lib/db.js
CHANGED
|
@@ -21,6 +21,7 @@ import { enableWAL } from "./lock.js";
|
|
|
21
21
|
import { getGnosysHome as getGnosysHomeImpl, getCentralDbPath as getCentralDbPathImpl } from "./paths.js";
|
|
22
22
|
import { readMachineConfig } from "./machineConfig.js";
|
|
23
23
|
import { logError } from "./log.js";
|
|
24
|
+
import { ftsTerms, ftsAndQuery, ftsOrQuery } from "./ftsQuery.js";
|
|
24
25
|
import { ulid } from "ulidx";
|
|
25
26
|
// ─── Schema ─────────────────────────────────────────────────────────────
|
|
26
27
|
const SCHEMA_VERSION = 5;
|
|
@@ -1088,27 +1089,33 @@ export class GnosysDB {
|
|
|
1088
1089
|
}
|
|
1089
1090
|
// ─── FTS5 Search ────────────────────────────────────────────────────
|
|
1090
1091
|
searchFts(query, limit = 20) {
|
|
1091
|
-
const
|
|
1092
|
-
if (
|
|
1092
|
+
const terms = ftsTerms(query);
|
|
1093
|
+
if (terms.length === 0)
|
|
1093
1094
|
return [];
|
|
1094
1095
|
// v5.8.0 (#7): join memories so callers can render project-prefixed IDs.
|
|
1095
1096
|
return this.withRecovery(() => {
|
|
1096
1097
|
try {
|
|
1097
|
-
|
|
1098
|
-
|
|
1099
|
-
|
|
1100
|
-
|
|
1101
|
-
|
|
1102
|
-
|
|
1103
|
-
|
|
1104
|
-
|
|
1105
|
-
|
|
1106
|
-
|
|
1107
|
-
|
|
1098
|
+
const run = (match) => this.prep(`
|
|
1099
|
+
SELECT m.id AS id, m.title AS title,
|
|
1100
|
+
snippet(memories_fts, 5, '>>>', '<<<', '...', 40) as snippet,
|
|
1101
|
+
fts.rank AS rank,
|
|
1102
|
+
m.project_id AS project_id
|
|
1103
|
+
FROM memories_fts fts
|
|
1104
|
+
JOIN memories m ON m.id = fts.id
|
|
1105
|
+
WHERE memories_fts MATCH ?
|
|
1106
|
+
ORDER BY fts.rank
|
|
1107
|
+
LIMIT ?
|
|
1108
|
+
`).all(match, limit);
|
|
1109
|
+
// v5.12.3: AND first (precision), OR retry when AND finds nothing —
|
|
1110
|
+
// multi-word queries previously required every term to match.
|
|
1111
|
+
const results = run(ftsAndQuery(terms));
|
|
1112
|
+
if (results.length > 0 || terms.length === 1)
|
|
1113
|
+
return results;
|
|
1114
|
+
return run(ftsOrQuery(terms));
|
|
1108
1115
|
}
|
|
1109
1116
|
catch {
|
|
1110
1117
|
// FTS5 syntax error — fallback to LIKE
|
|
1111
|
-
const pattern = `%${
|
|
1118
|
+
const pattern = `%${terms.join(" ")}%`;
|
|
1112
1119
|
return this.prep(`
|
|
1113
1120
|
SELECT id, title, substr(content, 1, 200) as snippet, 0 as rank, project_id
|
|
1114
1121
|
FROM memories WHERE content LIKE ? OR title LIKE ? OR tags LIKE ?
|
|
@@ -1118,8 +1125,8 @@ export class GnosysDB {
|
|
|
1118
1125
|
});
|
|
1119
1126
|
}
|
|
1120
1127
|
discoverFts(query, limit = 20) {
|
|
1121
|
-
const
|
|
1122
|
-
if (
|
|
1128
|
+
const terms = ftsTerms(query);
|
|
1129
|
+
if (terms.length === 0)
|
|
1123
1130
|
return [];
|
|
1124
1131
|
// v5.7.1 (#14): join `memories` so callers can render project-prefixed IDs.
|
|
1125
1132
|
const select = `
|
|
@@ -1130,26 +1137,37 @@ export class GnosysDB {
|
|
|
1130
1137
|
ORDER BY fts.rank
|
|
1131
1138
|
LIMIT ?
|
|
1132
1139
|
`;
|
|
1133
|
-
|
|
1140
|
+
// Let corruption escape to withRecovery (reopen + retry); plain FTS
|
|
1141
|
+
// syntax failures still degrade gracefully to "no results".
|
|
1142
|
+
const tryRun = (match) => {
|
|
1134
1143
|
try {
|
|
1135
|
-
|
|
1136
|
-
const results = this.prep(select).all(colQuery, limit);
|
|
1137
|
-
if (results.length > 0)
|
|
1138
|
-
return results;
|
|
1139
|
-
return this.prep(select).all(safeQuery, limit);
|
|
1144
|
+
return this.prep(select).all(match, limit);
|
|
1140
1145
|
}
|
|
1141
|
-
catch {
|
|
1142
|
-
|
|
1143
|
-
|
|
1144
|
-
|
|
1145
|
-
catch (err) {
|
|
1146
|
-
// Let corruption escape to withRecovery (reopen + retry); plain FTS
|
|
1147
|
-
// syntax failures still degrade gracefully to "no results".
|
|
1148
|
-
if (GnosysDB.isCorruptionError(err))
|
|
1149
|
-
throw err;
|
|
1150
|
-
return [];
|
|
1151
|
-
}
|
|
1146
|
+
catch (err) {
|
|
1147
|
+
if (GnosysDB.isCorruptionError(err))
|
|
1148
|
+
throw err;
|
|
1149
|
+
return [];
|
|
1152
1150
|
}
|
|
1151
|
+
};
|
|
1152
|
+
// v5.12.3: precision-to-recall ladder. AND on the metadata columns,
|
|
1153
|
+
// AND anywhere, then OR retries — multi-word queries previously
|
|
1154
|
+
// required every term to match, so long queries returned nothing.
|
|
1155
|
+
// Parens scope the column filter to the whole expression (a bare
|
|
1156
|
+
// `{cols} : a b` only filtered the first term).
|
|
1157
|
+
const colScoped = (expr) => `{relevance title tags} : (${expr})`;
|
|
1158
|
+
const andExpr = ftsAndQuery(terms);
|
|
1159
|
+
return this.withRecovery(() => {
|
|
1160
|
+
let results = tryRun(colScoped(andExpr));
|
|
1161
|
+
if (results.length > 0)
|
|
1162
|
+
return results;
|
|
1163
|
+
results = tryRun(andExpr);
|
|
1164
|
+
if (results.length > 0 || terms.length === 1)
|
|
1165
|
+
return results;
|
|
1166
|
+
const orExpr = ftsOrQuery(terms);
|
|
1167
|
+
results = tryRun(colScoped(orExpr));
|
|
1168
|
+
if (results.length > 0)
|
|
1169
|
+
return results;
|
|
1170
|
+
return tryRun(orExpr);
|
|
1153
1171
|
});
|
|
1154
1172
|
}
|
|
1155
1173
|
// ─── Relationships ──────────────────────────────────────────────────
|
|
@@ -1235,9 +1253,51 @@ export class GnosysDB {
|
|
|
1235
1253
|
});
|
|
1236
1254
|
}
|
|
1237
1255
|
// ─── Embeddings ─────────────────────────────────────────────────────
|
|
1256
|
+
/**
|
|
1257
|
+
* v5.13.0: rows needed to (re)build the embedding column. All tiers and
|
|
1258
|
+
* statuses are included — matching FTS, which indexes every row and
|
|
1259
|
+
* filters at query time. Newest-first so capped backfills prioritize
|
|
1260
|
+
* recent memories.
|
|
1261
|
+
*/
|
|
1262
|
+
getMemoriesForEmbedding(mode, limit) {
|
|
1263
|
+
const where = mode === "missing" ? "WHERE embedding IS NULL" : "";
|
|
1264
|
+
const lim = limit && limit > 0 ? ` LIMIT ${Math.floor(limit)}` : "";
|
|
1265
|
+
return this.withRecovery(() => this.prep(`SELECT id, title, relevance, tags, content FROM memories ${where} ORDER BY modified DESC${lim}`).all());
|
|
1266
|
+
}
|
|
1267
|
+
/** v5.13.0: memories whose embedding column is NULL (embedding-health check). */
|
|
1268
|
+
countMemoriesMissingEmbedding() {
|
|
1269
|
+
return this.withRecovery(() => {
|
|
1270
|
+
const row = this.prep("SELECT COUNT(*) as cnt FROM memories WHERE embedding IS NULL").get();
|
|
1271
|
+
return row.cnt;
|
|
1272
|
+
});
|
|
1273
|
+
}
|
|
1238
1274
|
updateEmbedding(id, embedding) {
|
|
1239
1275
|
this.withRecovery(() => {
|
|
1240
|
-
|
|
1276
|
+
const sql = "UPDATE memories SET embedding = ? WHERE id = ?";
|
|
1277
|
+
try {
|
|
1278
|
+
this.db.prepare(sql).run(embedding, id);
|
|
1279
|
+
}
|
|
1280
|
+
catch {
|
|
1281
|
+
// v5.13.0: same failure mode updateMemory works around — the FTS5
|
|
1282
|
+
// AFTER UPDATE trigger's 'delete' command errors on standalone FTS5
|
|
1283
|
+
// tables. embedding isn't an FTS column, so drop the trigger,
|
|
1284
|
+
// update, recreate; the FTS entry itself needs no rebuild.
|
|
1285
|
+
this.db.exec("DROP TRIGGER IF EXISTS memories_fts_au");
|
|
1286
|
+
this.db.prepare(sql).run(embedding, id);
|
|
1287
|
+
try {
|
|
1288
|
+
this.db.exec(`
|
|
1289
|
+
CREATE TRIGGER IF NOT EXISTS memories_fts_au AFTER UPDATE ON memories BEGIN
|
|
1290
|
+
INSERT INTO memories_fts(memories_fts, id, title, category, tags, relevance, content, summary)
|
|
1291
|
+
VALUES ('delete', old.id, old.title, old.category, old.tags, old.relevance, old.content, old.summary);
|
|
1292
|
+
INSERT INTO memories_fts(id, title, category, tags, relevance, content, summary)
|
|
1293
|
+
VALUES (new.id, new.title, new.category, new.tags, new.relevance, new.content, new.summary);
|
|
1294
|
+
END;
|
|
1295
|
+
`);
|
|
1296
|
+
}
|
|
1297
|
+
catch {
|
|
1298
|
+
// Trigger recreation failed — not critical
|
|
1299
|
+
}
|
|
1300
|
+
}
|
|
1241
1301
|
});
|
|
1242
1302
|
}
|
|
1243
1303
|
getEmbedding(id) {
|
package/dist/lib/dbWrite.js
CHANGED
|
@@ -14,6 +14,7 @@
|
|
|
14
14
|
* become optional — controlled by config.
|
|
15
15
|
*/
|
|
16
16
|
import { fnv1a } from "./db.js";
|
|
17
|
+
import { queueMemoryEmbedding } from "./embedQueue.js";
|
|
17
18
|
/** Coerce Date objects (from gray-matter parsing) to ISO date strings. */
|
|
18
19
|
function toDateStr(value) {
|
|
19
20
|
if (value instanceof Date)
|
|
@@ -61,6 +62,9 @@ export function syncMemoryToDb(db, frontmatter, content, sourcePath, projectId,
|
|
|
61
62
|
project_id: projectId || null,
|
|
62
63
|
scope: scope || "project",
|
|
63
64
|
});
|
|
65
|
+
// v5.13.0: best-effort write-time embedding (no-op unless the MCP server
|
|
66
|
+
// enabled the queue). Never blocks or fails the write.
|
|
67
|
+
queueMemoryEmbedding(frontmatter.id);
|
|
64
68
|
}
|
|
65
69
|
/**
|
|
66
70
|
* Sync a memory update to gnosys.db after it's been updated in .md.
|
|
@@ -113,6 +117,14 @@ export function syncUpdateToDb(db, id, updates, newContent) {
|
|
|
113
117
|
}
|
|
114
118
|
dbUpdates.modified = new Date().toISOString().split("T")[0];
|
|
115
119
|
db.updateMemory(id, dbUpdates);
|
|
120
|
+
// v5.13.0: re-embed when searchable text changed (title/content/tags/
|
|
121
|
+
// relevance feed the embedding recipe). No-op unless the queue is enabled.
|
|
122
|
+
if (newContent !== undefined ||
|
|
123
|
+
updates.title !== undefined ||
|
|
124
|
+
updates.tags !== undefined ||
|
|
125
|
+
updates.relevance !== undefined) {
|
|
126
|
+
queueMemoryEmbedding(id);
|
|
127
|
+
}
|
|
116
128
|
}
|
|
117
129
|
/**
|
|
118
130
|
* Sync an archive operation to gnosys.db.
|
package/dist/lib/dream.d.ts
CHANGED
|
@@ -79,6 +79,12 @@ export interface DreamReport {
|
|
|
79
79
|
llmCalls?: DreamLLMCallRecord[];
|
|
80
80
|
totals?: DreamRunRecord["totals"];
|
|
81
81
|
effectiveness?: DreamEffectivenessRecord;
|
|
82
|
+
/** v5.13.0: embedding coverage found/repaired by the embedding-health phase. */
|
|
83
|
+
embeddingHealth?: {
|
|
84
|
+
total: number;
|
|
85
|
+
missingBefore: number;
|
|
86
|
+
embedded: number;
|
|
87
|
+
};
|
|
82
88
|
}
|
|
83
89
|
export interface ReviewSuggestion {
|
|
84
90
|
memoryId: string;
|
|
@@ -99,9 +105,12 @@ export declare class GnosysDreamEngine {
|
|
|
99
105
|
private dreamState;
|
|
100
106
|
private pendingFingerprints;
|
|
101
107
|
private llmCallsMade;
|
|
108
|
+
/** v5.13.0: explicit dream-state dir (defense against test pollution of ~/.gnosys). */
|
|
109
|
+
private stateDir?;
|
|
102
110
|
constructor(db: GnosysDB, config: GnosysConfig, dreamConfig?: Partial<DreamConfig>, options?: {
|
|
103
111
|
trigger?: DreamTrigger;
|
|
104
112
|
machineId?: string;
|
|
113
|
+
stateDir?: string;
|
|
105
114
|
});
|
|
106
115
|
/** Captured at construction if getLLMProvider throws. Used in dream() to write a Layer 2 audit entry. */
|
|
107
116
|
private providerInitError;
|
package/dist/lib/dream.js
CHANGED
|
@@ -23,6 +23,7 @@ import { createProvider } from "./llm.js";
|
|
|
23
23
|
import { notifyDesktop } from "./desktopNotify.js";
|
|
24
24
|
import { syncConfidenceToDb, auditToDb } from "./dbWrite.js";
|
|
25
25
|
import { logError } from "./log.js";
|
|
26
|
+
import { getGnosysHome } from "./paths.js";
|
|
26
27
|
import { estimateCost, acquireDreamLock, estimateTokens, fingerprintMemories, memoryWatermark, readDreamState, writeDreamState, } from "./dreamRunLog.js";
|
|
27
28
|
/** Layer 4 alert threshold: fire desktop notification at this many consecutive provider failures. */
|
|
28
29
|
const DREAM_FAILURE_NOTIFY_THRESHOLD = 3;
|
|
@@ -44,6 +45,9 @@ export const DEFAULT_DREAM_CONFIG = {
|
|
|
44
45
|
};
|
|
45
46
|
// ─── Decay Constants ─────────────────────────────────────────────────────
|
|
46
47
|
const DECAY_LAMBDA = 0.005;
|
|
48
|
+
// v5.13.0: embedding-health backfill cap per dream run. Bounds idle-time
|
|
49
|
+
// work (~128-row batches, local model) — large gaps close over a few runs.
|
|
50
|
+
const EMBEDDING_BACKFILL_MAX_PER_RUN = 512;
|
|
47
51
|
const STALE_THRESHOLD = 0.3;
|
|
48
52
|
// ─── Dream Engine ────────────────────────────────────────────────────────
|
|
49
53
|
export class GnosysDreamEngine {
|
|
@@ -55,9 +59,11 @@ export class GnosysDreamEngine {
|
|
|
55
59
|
startTime = 0;
|
|
56
60
|
trigger;
|
|
57
61
|
machineId;
|
|
58
|
-
dreamState
|
|
62
|
+
dreamState;
|
|
59
63
|
pendingFingerprints = {};
|
|
60
64
|
llmCallsMade = 0;
|
|
65
|
+
/** v5.13.0: explicit dream-state dir (defense against test pollution of ~/.gnosys). */
|
|
66
|
+
stateDir;
|
|
61
67
|
constructor(db, config, dreamConfig, options) {
|
|
62
68
|
this.db = db;
|
|
63
69
|
this.config = config;
|
|
@@ -68,6 +74,8 @@ export class GnosysDreamEngine {
|
|
|
68
74
|
};
|
|
69
75
|
this.trigger = options?.trigger ?? "manual";
|
|
70
76
|
this.machineId = options?.machineId;
|
|
77
|
+
this.stateDir = options?.stateDir;
|
|
78
|
+
this.dreamState = readDreamState(this.stateDir);
|
|
71
79
|
// Initialize LLM provider for dream operations.
|
|
72
80
|
// v5.4.2: Failure here is no longer silent — when dream tries to actually
|
|
73
81
|
// run (in dream()), we record the unavailability to audit_log so the
|
|
@@ -128,7 +136,7 @@ export class GnosysDreamEngine {
|
|
|
128
136
|
...this.dreamState.analyzedFingerprints,
|
|
129
137
|
...this.pendingFingerprints,
|
|
130
138
|
},
|
|
131
|
-
});
|
|
139
|
+
}, this.stateDir);
|
|
132
140
|
}
|
|
133
141
|
catch {
|
|
134
142
|
// Best-effort: a failed checkpoint only costs re-analysis on resume.
|
|
@@ -261,7 +269,7 @@ export class GnosysDreamEngine {
|
|
|
261
269
|
this.llmCallsMade = 0;
|
|
262
270
|
this.llmCalls = [];
|
|
263
271
|
this.pendingFingerprints = {};
|
|
264
|
-
this.dreamState = readDreamState();
|
|
272
|
+
this.dreamState = readDreamState(this.stateDir);
|
|
265
273
|
const log = onProgress || (() => { });
|
|
266
274
|
const startedAt = new Date().toISOString();
|
|
267
275
|
const report = {
|
|
@@ -368,6 +376,92 @@ export class GnosysDreamEngine {
|
|
|
368
376
|
report.abortReason = check.reason;
|
|
369
377
|
return this.finalize(report);
|
|
370
378
|
}
|
|
379
|
+
// ─── Phase 1.5: Embedding Health (v5.13.0) ───────────────────────────
|
|
380
|
+
// Semantic search reads memories.embedding; writes only best-effort
|
|
381
|
+
// embed. Dream is the safety net: report coverage and backfill missing
|
|
382
|
+
// vectors during idle time. No LLM involved — local embedding model.
|
|
383
|
+
{
|
|
384
|
+
log("embedding-health", "Phase 1.5: Embedding health check...");
|
|
385
|
+
const embedPhase = this.createPhase("embedding-health");
|
|
386
|
+
const embedStart = Date.now();
|
|
387
|
+
report.phases.push(embedPhase);
|
|
388
|
+
try {
|
|
389
|
+
const total = this.db.getMemoryCount().total;
|
|
390
|
+
const missingBefore = this.db.countMemoriesMissingEmbedding();
|
|
391
|
+
const embeddedCount = this.db.getEmbeddingCount();
|
|
392
|
+
if (missingBefore === 0) {
|
|
393
|
+
embedPhase.status = "skipped";
|
|
394
|
+
embedPhase.reason = "all memories embedded";
|
|
395
|
+
report.embeddingHealth = { total, missingBefore: 0, embedded: 0 };
|
|
396
|
+
log("embedding-health", `All ${total} memories embedded`);
|
|
397
|
+
}
|
|
398
|
+
else if (embeddedCount === 0) {
|
|
399
|
+
// Embeddings never initialized on this install — that's an explicit
|
|
400
|
+
// user choice (reindex downloads the 80 MB model); Dream repairs
|
|
401
|
+
// drift, it doesn't opt users in. Report it loudly and move on.
|
|
402
|
+
embedPhase.status = "skipped";
|
|
403
|
+
embedPhase.reason = "embeddings not initialized — run gnosys_reindex";
|
|
404
|
+
report.embeddingHealth = { total, missingBefore, embedded: 0 };
|
|
405
|
+
auditToDb(this.db, "dream_embedding_health", undefined, {
|
|
406
|
+
missing: missingBefore,
|
|
407
|
+
total,
|
|
408
|
+
initialized: false,
|
|
409
|
+
});
|
|
410
|
+
log("embedding-health", `${missingBefore}/${total} memories lack embeddings — run gnosys_reindex to enable semantic search`);
|
|
411
|
+
}
|
|
412
|
+
else {
|
|
413
|
+
auditToDb(this.db, "dream_embedding_health", undefined, {
|
|
414
|
+
missing: missingBefore,
|
|
415
|
+
total,
|
|
416
|
+
initialized: true,
|
|
417
|
+
});
|
|
418
|
+
const { backfillCentralDbEmbeddings } = await import("./embedDb.js");
|
|
419
|
+
const { GnosysEmbeddings } = await import("./embeddings.js");
|
|
420
|
+
// Model-only use — we never touch the embedder's store-local db.
|
|
421
|
+
const embedder = new GnosysEmbeddings(getGnosysHome());
|
|
422
|
+
let embedded = 0;
|
|
423
|
+
try {
|
|
424
|
+
// Chunked so shouldStop() (abort / max-runtime) is honored
|
|
425
|
+
// between batches; capped per run to bound idle-time work.
|
|
426
|
+
while (embedded < EMBEDDING_BACKFILL_MAX_PER_RUN) {
|
|
427
|
+
if (this.shouldStop().stop)
|
|
428
|
+
break;
|
|
429
|
+
const chunk = Math.min(128, EMBEDDING_BACKFILL_MAX_PER_RUN - embedded);
|
|
430
|
+
const result = await backfillCentralDbEmbeddings(this.db, embedder, {
|
|
431
|
+
mode: "missing",
|
|
432
|
+
limit: chunk,
|
|
433
|
+
});
|
|
434
|
+
embedded += result.embedded;
|
|
435
|
+
if (result.embedded === 0)
|
|
436
|
+
break;
|
|
437
|
+
}
|
|
438
|
+
}
|
|
439
|
+
finally {
|
|
440
|
+
embedder.close();
|
|
441
|
+
}
|
|
442
|
+
report.embeddingHealth = { total, missingBefore, embedded };
|
|
443
|
+
if (embedded > 0) {
|
|
444
|
+
auditToDb(this.db, "dream_embedding_backfill", undefined, {
|
|
445
|
+
embedded,
|
|
446
|
+
remaining: missingBefore - embedded,
|
|
447
|
+
});
|
|
448
|
+
}
|
|
449
|
+
log("embedding-health", `Embedded ${embedded}/${missingBefore} missing (${total} total)`);
|
|
450
|
+
}
|
|
451
|
+
}
|
|
452
|
+
catch (err) {
|
|
453
|
+
report.errors.push(`Embedding health: ${err instanceof Error ? err.message : String(err)}`);
|
|
454
|
+
}
|
|
455
|
+
finally {
|
|
456
|
+
this.finishPhase(embedPhase, embedStart);
|
|
457
|
+
}
|
|
458
|
+
check = this.shouldStop();
|
|
459
|
+
if (check.stop) {
|
|
460
|
+
report.aborted = true;
|
|
461
|
+
report.abortReason = check.reason;
|
|
462
|
+
return this.finalize(report);
|
|
463
|
+
}
|
|
464
|
+
}
|
|
371
465
|
// ─── Phase 2: Self-Critique (Review Suggestions) ─────────────────────
|
|
372
466
|
if (this.dreamConfig.selfCritique) {
|
|
373
467
|
log("critique", "Phase 2: Self-critique...");
|
|
@@ -487,7 +581,7 @@ export class GnosysDreamEngine {
|
|
|
487
581
|
...this.dreamState.analyzedFingerprints,
|
|
488
582
|
...this.pendingFingerprints,
|
|
489
583
|
},
|
|
490
|
-
});
|
|
584
|
+
}, this.stateDir);
|
|
491
585
|
// v5.4.2: A run is considered "successful with LLM work" if any of the
|
|
492
586
|
// LLM-dependent counters moved. Resetting the consecutive-failure count
|
|
493
587
|
// here ensures Layer 4 doesn't keep firing once dream is healthy again.
|
|
@@ -9,7 +9,7 @@ export interface DreamRunGateResult {
|
|
|
9
9
|
details?: Record<string, unknown>;
|
|
10
10
|
}
|
|
11
11
|
export interface DreamRunPhaseRecord {
|
|
12
|
-
name: "decay" | "critique" | "summaries" | "relationships";
|
|
12
|
+
name: "decay" | "embedding-health" | "critique" | "summaries" | "relationships";
|
|
13
13
|
status: "ran" | "skipped";
|
|
14
14
|
reason?: string;
|
|
15
15
|
durationMs: number;
|
|
@@ -86,7 +86,7 @@ export interface DreamReadOptions {
|
|
|
86
86
|
status?: DreamRunStatus;
|
|
87
87
|
}
|
|
88
88
|
export declare function getDreamRunsPath(): string;
|
|
89
|
-
export declare function getDreamStatePath(): string;
|
|
89
|
+
export declare function getDreamStatePath(baseDir?: string): string;
|
|
90
90
|
export declare function getDreamLockPath(): string;
|
|
91
91
|
export declare function acquireDreamLock(depth?: number): {
|
|
92
92
|
acquired: true;
|
|
@@ -97,8 +97,8 @@ export declare function acquireDreamLock(depth?: number): {
|
|
|
97
97
|
};
|
|
98
98
|
export declare function estimateTokens(text: string): number;
|
|
99
99
|
export declare function estimateCost(model: string | undefined, inputTokens: number, outputTokens: number): number;
|
|
100
|
-
export declare function readDreamState(): DreamState;
|
|
101
|
-
export declare function writeDreamState(state: DreamState): void;
|
|
100
|
+
export declare function readDreamState(baseDir?: string): DreamState;
|
|
101
|
+
export declare function writeDreamState(state: DreamState, baseDir?: string): void;
|
|
102
102
|
export declare function appendDreamRun(record: DreamRunRecord): void;
|
|
103
103
|
export declare function readDreamRuns(opts?: DreamReadOptions): DreamRunRecord[];
|
|
104
104
|
export declare function fingerprintMemories(kind: "summary" | "critique" | "relationship", memories: DbMemory[]): string;
|
package/dist/lib/dreamRunLog.js
CHANGED
|
@@ -20,8 +20,11 @@ const MODEL_PRICES_USD_PER_MILLION = {
|
|
|
20
20
|
export function getDreamRunsPath() {
|
|
21
21
|
return path.join(getGnosysHome(), "dream-runs.jsonl");
|
|
22
22
|
}
|
|
23
|
-
|
|
24
|
-
|
|
23
|
+
// v5.13.0: baseDir injection (defense in depth against test pollution) —
|
|
24
|
+
// in-process engines can target an explicit state dir instead of the
|
|
25
|
+
// developer's real ~/.gnosys.
|
|
26
|
+
export function getDreamStatePath(baseDir) {
|
|
27
|
+
return path.join(baseDir || getGnosysHome(), "dream-state.json");
|
|
25
28
|
}
|
|
26
29
|
export function getDreamLockPath() {
|
|
27
30
|
return path.join(getGnosysHome(), "dream.lock");
|
|
@@ -103,9 +106,9 @@ export function estimateCost(model, inputTokens, outputTokens) {
|
|
|
103
106
|
const cost = (inputTokens / 1_000_000) * fuzzy.input + (outputTokens / 1_000_000) * fuzzy.output;
|
|
104
107
|
return Math.round(cost * 1_000_000) / 1_000_000;
|
|
105
108
|
}
|
|
106
|
-
export function readDreamState() {
|
|
109
|
+
export function readDreamState(baseDir) {
|
|
107
110
|
try {
|
|
108
|
-
const raw = fs.readFileSync(getDreamStatePath(), "utf8");
|
|
111
|
+
const raw = fs.readFileSync(getDreamStatePath(baseDir), "utf8");
|
|
109
112
|
const parsed = JSON.parse(raw);
|
|
110
113
|
return {
|
|
111
114
|
...DEFAULT_STATE,
|
|
@@ -117,8 +120,8 @@ export function readDreamState() {
|
|
|
117
120
|
return { ...DEFAULT_STATE };
|
|
118
121
|
}
|
|
119
122
|
}
|
|
120
|
-
export function writeDreamState(state) {
|
|
121
|
-
const file = getDreamStatePath();
|
|
123
|
+
export function writeDreamState(state, baseDir) {
|
|
124
|
+
const file = getDreamStatePath(baseDir);
|
|
122
125
|
fs.mkdirSync(path.dirname(file), { recursive: true });
|
|
123
126
|
fs.writeFileSync(file, `${JSON.stringify(state, null, 2)}\n`, "utf8");
|
|
124
127
|
}
|
|
@@ -0,0 +1,53 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Central-DB embedding utilities (v5.13.0).
|
|
3
|
+
*
|
|
4
|
+
* GnosysDbSearch reads vectors from the memories.embedding column of
|
|
5
|
+
* gnosys.db, but before v5.13.0 nothing ever wrote that column: the
|
|
6
|
+
* reindex flow embedded file-store memories into the store-local
|
|
7
|
+
* embeddings.db, so DB-mode hybrid search could never run its semantic
|
|
8
|
+
* leg. This module owns the column — full backfill for reindex,
|
|
9
|
+
* missing-only backfill for Dream/maintenance, and single-memory
|
|
10
|
+
* embedding for the write-time queue.
|
|
11
|
+
*
|
|
12
|
+
* Embeddings are derived, rebuildable data: regenerating them is always
|
|
13
|
+
* safe, and a sync snapshot refresh replacing them is not a loss.
|
|
14
|
+
*/
|
|
15
|
+
import type { GnosysDB } from "./db.js";
|
|
16
|
+
import type { GnosysEmbeddings } from "./embeddings.js";
|
|
17
|
+
/** Row fields needed to build embedding text. */
|
|
18
|
+
export interface EmbeddableMemoryRow {
|
|
19
|
+
id: string;
|
|
20
|
+
title: string;
|
|
21
|
+
relevance: string | null;
|
|
22
|
+
tags: string;
|
|
23
|
+
content: string;
|
|
24
|
+
}
|
|
25
|
+
/**
|
|
26
|
+
* The text recipe — identical to the file-store reindex in hybridSearch.ts
|
|
27
|
+
* (title, relevance keywords, tags, content) so query-vs-document
|
|
28
|
+
* similarity behaves the same in both modes.
|
|
29
|
+
*/
|
|
30
|
+
export declare function embeddingText(m: EmbeddableMemoryRow): string;
|
|
31
|
+
/** View a Float32Array as the Buffer shape memories.embedding stores. */
|
|
32
|
+
export declare function float32ToBuffer(vec: Float32Array): Buffer;
|
|
33
|
+
/**
|
|
34
|
+
* (Re)build the memories.embedding column.
|
|
35
|
+
*
|
|
36
|
+
* mode "all" regenerates every memory (reindex semantics); "missing" only
|
|
37
|
+
* fills NULL rows (write-gap backfill for Dream/maintenance, newest first).
|
|
38
|
+
* Batched through embedBatch for model efficiency.
|
|
39
|
+
*/
|
|
40
|
+
export declare function backfillCentralDbEmbeddings(db: GnosysDB, embeddings: GnosysEmbeddings, opts?: {
|
|
41
|
+
mode?: "missing" | "all";
|
|
42
|
+
batchSize?: number;
|
|
43
|
+
limit?: number;
|
|
44
|
+
onProgress?: (current: number, total: number, id: string) => void;
|
|
45
|
+
}): Promise<{
|
|
46
|
+
embedded: number;
|
|
47
|
+
total: number;
|
|
48
|
+
}>;
|
|
49
|
+
/**
|
|
50
|
+
* Embed one memory and store its vector. Returns false if the memory
|
|
51
|
+
* doesn't exist; throws on embedder failure — callers decide how loud.
|
|
52
|
+
*/
|
|
53
|
+
export declare function embedMemoryIntoDb(db: GnosysDB, embeddings: GnosysEmbeddings, id: string): Promise<boolean>;
|