gnosys 5.12.2 → 5.13.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/dist/index.js CHANGED
@@ -1752,10 +1752,17 @@ regTool("gnosys_hybrid_search", "Search memories using hybrid keyword + semantic
1752
1752
  };
1753
1753
  }
1754
1754
  try {
1755
- const results = await hybridSearch.hybridSearch(query, limit || 15, mode || "hybrid");
1755
+ const requestedMode = mode || "hybrid";
1756
+ const results = await hybridSearch.hybridSearch(query, limit || 15, requestedMode);
1757
+ // v5.12.3: hybrid used to degrade to keyword-only silently when the
1758
+ // semantic leg can't run (embeddings are only built by gnosys_reindex,
1759
+ // and DB mode also needs the store-local query embedder). Say so loudly.
1760
+ const degradeWarning = requestedMode !== "keyword" && !hybridSearch.canRunSemantic()
1761
+ ? `⚠️ Semantic embeddings unavailable — ${requestedMode} search ran keyword-only. Run gnosys_reindex to build embeddings and enable semantic recall.\n\n`
1762
+ : "";
1756
1763
  if (results.length === 0) {
1757
1764
  return {
1758
- content: [{ type: "text", text: `No results for "${query}". Try gnosys_reindex to build embeddings, or different keywords.` }],
1765
+ content: [{ type: "text", text: `${degradeWarning}No results for "${query}". Try different keywords.` }],
1759
1766
  };
1760
1767
  }
1761
1768
  const formatted = results
@@ -1774,7 +1781,7 @@ regTool("gnosys_hybrid_search", "Search memories using hybrid keyword + semantic
1774
1781
  content: [
1775
1782
  {
1776
1783
  type: "text",
1777
- text: `Found ${results.length} results for "${query}" (${embCount} embeddings indexed):\n\n${formatted}`,
1784
+ text: `${degradeWarning}Found ${results.length} results for "${query}" (${embCount} embeddings indexed):\n\n${formatted}`,
1778
1785
  },
1779
1786
  ],
1780
1787
  };
@@ -1802,10 +1809,19 @@ regTool("gnosys_semantic_search", "Search memories using semantic similarity onl
1802
1809
  };
1803
1810
  }
1804
1811
  try {
1812
+ // v5.12.3: semantic search without a working semantic leg either
1813
+ // returns nothing or (in DB mode) keyword hits mislabeled as semantic —
1814
+ // refuse with the exact reason instead.
1815
+ if (!hybridSearch.canRunSemantic()) {
1816
+ return {
1817
+ content: [{ type: "text", text: `⚠️ Semantic embeddings unavailable — semantic search cannot run. Run gnosys_reindex to build embeddings, then retry.` }],
1818
+ isError: true,
1819
+ };
1820
+ }
1805
1821
  const results = await hybridSearch.hybridSearch(query, limit || 15, "semantic");
1806
1822
  if (results.length === 0) {
1807
1823
  return {
1808
- content: [{ type: "text", text: `No semantic results for "${query}". Run gnosys_reindex first to build embeddings.` }],
1824
+ content: [{ type: "text", text: `No semantic results for "${query}". Try a broader query.` }],
1809
1825
  };
1810
1826
  }
1811
1827
  const formatted = results
@@ -1837,11 +1853,18 @@ regTool("gnosys_reindex", "Rebuild all semantic embeddings from every memory fil
1837
1853
  // Also rebuild FTS5 index
1838
1854
  await reindexAllStores();
1839
1855
  const count = await hybridSearch.reindex();
1856
+ // v5.13.0: also (re)build the central-DB embedding column — the
1857
+ // vectors DB-mode hybrid/semantic search actually reads.
1858
+ const dbResult = await hybridSearch.reindexCentralDb();
1859
+ const parts = [`${count} file-store memories embedded`];
1860
+ if (dbResult.total > 0) {
1861
+ parts.push(`${dbResult.embedded}/${dbResult.total} central-DB memories embedded`);
1862
+ }
1840
1863
  return {
1841
1864
  content: [
1842
1865
  {
1843
1866
  type: "text",
1844
- text: `Reindex complete: ${count} memories embedded. Hybrid search is now available.`,
1867
+ text: `Reindex complete: ${parts.join(", ")}. Hybrid search is now available.`,
1845
1868
  },
1846
1869
  ],
1847
1870
  };
@@ -3169,6 +3192,14 @@ async function initHeavyDeps() {
3169
3192
  const embeddings = new GnosysEmbeddings(writeTarget.store.getStorePath());
3170
3193
  hybridSearch = new GnosysHybridSearch(search, embeddings, resolver, writeTarget.store.getStorePath(), gnosysDb || undefined);
3171
3194
  askEngine = new GnosysAsk(hybridSearch, config, resolver, writeTarget.store.getStorePath());
3195
+ // v5.13.0: write-time embeddings (serve mode only) — new and updated
3196
+ // memories get vectors without waiting for a manual reindex. Best-effort:
3197
+ // queued writes drain on an unref'd timer and never block a tool call.
3198
+ if (centralDb?.isAvailable() && centralDb.isMigrated()) {
3199
+ const { enableWriteTimeEmbedding } = await import("./lib/embedQueue.js");
3200
+ enableWriteTimeEmbedding(() => centralDb, embeddings);
3201
+ console.error("Write-time embeddings: enabled (central DB)");
3202
+ }
3172
3203
  const embCount = embeddings.hasEmbeddings() ? embeddings.count() : 0;
3173
3204
  console.error(`Hybrid search: ${embCount > 0 ? `ready (${embCount} embeddings)` : "available (run gnosys_reindex to build embeddings)"}`);
3174
3205
  console.error(`Ask engine: ${askEngine.isLLMAvailable ? `ready (${askEngine.providerName}/${askEngine.modelName})` : "disabled (configure LLM provider)"}`);
@@ -23,6 +23,7 @@ import { statSync } from "fs";
23
23
  import { syncMemoryToDb, syncDearchiveToDb } from "./dbWrite.js";
24
24
  import { enableWAL } from "./lock.js";
25
25
  import { auditLog } from "./audit.js";
26
+ import { ftsTerms, ftsAndQuery, ftsOrQuery } from "./ftsQuery.js";
26
27
  // ─── Archive Manager ────────────────────────────────────────────────────
27
28
  export class GnosysArchive {
28
29
  db = null;
@@ -217,8 +218,8 @@ export class GnosysArchive {
217
218
  searchArchive(query, limit = 20) {
218
219
  if (!this.db)
219
220
  return [];
220
- const safeQuery = query.replace(/['"]/g, "").trim();
221
- if (!safeQuery)
221
+ const terms = ftsTerms(query);
222
+ if (terms.length === 0)
222
223
  return [];
223
224
  const stmt = this.db.prepare(`
224
225
  SELECT
@@ -235,7 +236,12 @@ export class GnosysArchive {
235
236
  LIMIT ?
236
237
  `);
237
238
  try {
238
- return stmt.all(safeQuery, limit);
239
+ // v5.12.3: AND first (precision), OR retry when AND finds nothing —
240
+ // multi-word queries previously required every term to match.
241
+ const results = stmt.all(ftsAndQuery(terms), limit);
242
+ if (results.length > 0 || terms.length === 1)
243
+ return results;
244
+ return stmt.all(ftsOrQuery(terms), limit);
239
245
  }
240
246
  catch {
241
247
  // FTS5 query syntax failed — try LIKE fallback
@@ -251,7 +257,7 @@ export class GnosysArchive {
251
257
  WHERE content LIKE ? OR title LIKE ? OR tags LIKE ?
252
258
  LIMIT ?
253
259
  `);
254
- const pattern = `%${safeQuery}%`;
260
+ const pattern = `%${terms.join(" ")}%`;
255
261
  return likeStmt.all(pattern, pattern, pattern, limit);
256
262
  }
257
263
  }
package/dist/lib/db.d.ts CHANGED
@@ -317,6 +317,21 @@ export declare class GnosysDB {
317
317
  operation?: string;
318
318
  limit?: number;
319
319
  }): DbAuditEntry[];
320
+ /**
321
+ * v5.13.0: rows needed to (re)build the embedding column. All tiers and
322
+ * statuses are included — matching FTS, which indexes every row and
323
+ * filters at query time. Newest-first so capped backfills prioritize
324
+ * recent memories.
325
+ */
326
+ getMemoriesForEmbedding(mode: "missing" | "all", limit?: number): Array<{
327
+ id: string;
328
+ title: string;
329
+ relevance: string | null;
330
+ tags: string;
331
+ content: string;
332
+ }>;
333
+ /** v5.13.0: memories whose embedding column is NULL (embedding-health check). */
334
+ countMemoriesMissingEmbedding(): number;
320
335
  updateEmbedding(id: string, embedding: Buffer): void;
321
336
  getEmbedding(id: string): Buffer | null;
322
337
  getAllEmbeddings(): Array<{
package/dist/lib/db.js CHANGED
@@ -21,6 +21,7 @@ import { enableWAL } from "./lock.js";
21
21
  import { getGnosysHome as getGnosysHomeImpl, getCentralDbPath as getCentralDbPathImpl } from "./paths.js";
22
22
  import { readMachineConfig } from "./machineConfig.js";
23
23
  import { logError } from "./log.js";
24
+ import { ftsTerms, ftsAndQuery, ftsOrQuery } from "./ftsQuery.js";
24
25
  import { ulid } from "ulidx";
25
26
  // ─── Schema ─────────────────────────────────────────────────────────────
26
27
  const SCHEMA_VERSION = 5;
@@ -1088,27 +1089,33 @@ export class GnosysDB {
1088
1089
  }
1089
1090
  // ─── FTS5 Search ────────────────────────────────────────────────────
1090
1091
  searchFts(query, limit = 20) {
1091
- const safeQuery = query.replace(/['"]/g, "").trim();
1092
- if (!safeQuery)
1092
+ const terms = ftsTerms(query);
1093
+ if (terms.length === 0)
1093
1094
  return [];
1094
1095
  // v5.8.0 (#7): join memories so callers can render project-prefixed IDs.
1095
1096
  return this.withRecovery(() => {
1096
1097
  try {
1097
- return this.prep(`
1098
- SELECT m.id AS id, m.title AS title,
1099
- snippet(memories_fts, 5, '>>>', '<<<', '...', 40) as snippet,
1100
- fts.rank AS rank,
1101
- m.project_id AS project_id
1102
- FROM memories_fts fts
1103
- JOIN memories m ON m.id = fts.id
1104
- WHERE memories_fts MATCH ?
1105
- ORDER BY fts.rank
1106
- LIMIT ?
1107
- `).all(safeQuery, limit);
1098
+ const run = (match) => this.prep(`
1099
+ SELECT m.id AS id, m.title AS title,
1100
+ snippet(memories_fts, 5, '>>>', '<<<', '...', 40) as snippet,
1101
+ fts.rank AS rank,
1102
+ m.project_id AS project_id
1103
+ FROM memories_fts fts
1104
+ JOIN memories m ON m.id = fts.id
1105
+ WHERE memories_fts MATCH ?
1106
+ ORDER BY fts.rank
1107
+ LIMIT ?
1108
+ `).all(match, limit);
1109
+ // v5.12.3: AND first (precision), OR retry when AND finds nothing —
1110
+ // multi-word queries previously required every term to match.
1111
+ const results = run(ftsAndQuery(terms));
1112
+ if (results.length > 0 || terms.length === 1)
1113
+ return results;
1114
+ return run(ftsOrQuery(terms));
1108
1115
  }
1109
1116
  catch {
1110
1117
  // FTS5 syntax error — fallback to LIKE
1111
- const pattern = `%${safeQuery}%`;
1118
+ const pattern = `%${terms.join(" ")}%`;
1112
1119
  return this.prep(`
1113
1120
  SELECT id, title, substr(content, 1, 200) as snippet, 0 as rank, project_id
1114
1121
  FROM memories WHERE content LIKE ? OR title LIKE ? OR tags LIKE ?
@@ -1118,8 +1125,8 @@ export class GnosysDB {
1118
1125
  });
1119
1126
  }
1120
1127
  discoverFts(query, limit = 20) {
1121
- const safeQuery = query.replace(/['"]/g, "").trim();
1122
- if (!safeQuery)
1128
+ const terms = ftsTerms(query);
1129
+ if (terms.length === 0)
1123
1130
  return [];
1124
1131
  // v5.7.1 (#14): join `memories` so callers can render project-prefixed IDs.
1125
1132
  const select = `
@@ -1130,26 +1137,37 @@ export class GnosysDB {
1130
1137
  ORDER BY fts.rank
1131
1138
  LIMIT ?
1132
1139
  `;
1133
- return this.withRecovery(() => {
1140
+ // Let corruption escape to withRecovery (reopen + retry); plain FTS
1141
+ // syntax failures still degrade gracefully to "no results".
1142
+ const tryRun = (match) => {
1134
1143
  try {
1135
- const colQuery = `{relevance title tags} : ${safeQuery}`;
1136
- const results = this.prep(select).all(colQuery, limit);
1137
- if (results.length > 0)
1138
- return results;
1139
- return this.prep(select).all(safeQuery, limit);
1144
+ return this.prep(select).all(match, limit);
1140
1145
  }
1141
- catch {
1142
- try {
1143
- return this.prep(select).all(safeQuery, limit);
1144
- }
1145
- catch (err) {
1146
- // Let corruption escape to withRecovery (reopen + retry); plain FTS
1147
- // syntax failures still degrade gracefully to "no results".
1148
- if (GnosysDB.isCorruptionError(err))
1149
- throw err;
1150
- return [];
1151
- }
1146
+ catch (err) {
1147
+ if (GnosysDB.isCorruptionError(err))
1148
+ throw err;
1149
+ return [];
1152
1150
  }
1151
+ };
1152
+ // v5.12.3: precision-to-recall ladder. AND on the metadata columns,
1153
+ // AND anywhere, then OR retries — multi-word queries previously
1154
+ // required every term to match, so long queries returned nothing.
1155
+ // Parens scope the column filter to the whole expression (a bare
1156
+ // `{cols} : a b` only filtered the first term).
1157
+ const colScoped = (expr) => `{relevance title tags} : (${expr})`;
1158
+ const andExpr = ftsAndQuery(terms);
1159
+ return this.withRecovery(() => {
1160
+ let results = tryRun(colScoped(andExpr));
1161
+ if (results.length > 0)
1162
+ return results;
1163
+ results = tryRun(andExpr);
1164
+ if (results.length > 0 || terms.length === 1)
1165
+ return results;
1166
+ const orExpr = ftsOrQuery(terms);
1167
+ results = tryRun(colScoped(orExpr));
1168
+ if (results.length > 0)
1169
+ return results;
1170
+ return tryRun(orExpr);
1153
1171
  });
1154
1172
  }
1155
1173
  // ─── Relationships ──────────────────────────────────────────────────
@@ -1235,9 +1253,51 @@ export class GnosysDB {
1235
1253
  });
1236
1254
  }
1237
1255
  // ─── Embeddings ─────────────────────────────────────────────────────
1256
+ /**
1257
+ * v5.13.0: rows needed to (re)build the embedding column. All tiers and
1258
+ * statuses are included — matching FTS, which indexes every row and
1259
+ * filters at query time. Newest-first so capped backfills prioritize
1260
+ * recent memories.
1261
+ */
1262
+ getMemoriesForEmbedding(mode, limit) {
1263
+ const where = mode === "missing" ? "WHERE embedding IS NULL" : "";
1264
+ const lim = limit && limit > 0 ? ` LIMIT ${Math.floor(limit)}` : "";
1265
+ return this.withRecovery(() => this.prep(`SELECT id, title, relevance, tags, content FROM memories ${where} ORDER BY modified DESC${lim}`).all());
1266
+ }
1267
+ /** v5.13.0: memories whose embedding column is NULL (embedding-health check). */
1268
+ countMemoriesMissingEmbedding() {
1269
+ return this.withRecovery(() => {
1270
+ const row = this.prep("SELECT COUNT(*) as cnt FROM memories WHERE embedding IS NULL").get();
1271
+ return row.cnt;
1272
+ });
1273
+ }
1238
1274
  updateEmbedding(id, embedding) {
1239
1275
  this.withRecovery(() => {
1240
- this.db.prepare("UPDATE memories SET embedding = ? WHERE id = ?").run(embedding, id);
1276
+ const sql = "UPDATE memories SET embedding = ? WHERE id = ?";
1277
+ try {
1278
+ this.db.prepare(sql).run(embedding, id);
1279
+ }
1280
+ catch {
1281
+ // v5.13.0: same failure mode updateMemory works around — the FTS5
1282
+ // AFTER UPDATE trigger's 'delete' command errors on standalone FTS5
1283
+ // tables. embedding isn't an FTS column, so drop the trigger,
1284
+ // update, recreate; the FTS entry itself needs no rebuild.
1285
+ this.db.exec("DROP TRIGGER IF EXISTS memories_fts_au");
1286
+ this.db.prepare(sql).run(embedding, id);
1287
+ try {
1288
+ this.db.exec(`
1289
+ CREATE TRIGGER IF NOT EXISTS memories_fts_au AFTER UPDATE ON memories BEGIN
1290
+ INSERT INTO memories_fts(memories_fts, id, title, category, tags, relevance, content, summary)
1291
+ VALUES ('delete', old.id, old.title, old.category, old.tags, old.relevance, old.content, old.summary);
1292
+ INSERT INTO memories_fts(id, title, category, tags, relevance, content, summary)
1293
+ VALUES (new.id, new.title, new.category, new.tags, new.relevance, new.content, new.summary);
1294
+ END;
1295
+ `);
1296
+ }
1297
+ catch {
1298
+ // Trigger recreation failed — not critical
1299
+ }
1300
+ }
1241
1301
  });
1242
1302
  }
1243
1303
  getEmbedding(id) {
@@ -14,6 +14,7 @@
14
14
  * become optional — controlled by config.
15
15
  */
16
16
  import { fnv1a } from "./db.js";
17
+ import { queueMemoryEmbedding } from "./embedQueue.js";
17
18
  /** Coerce Date objects (from gray-matter parsing) to ISO date strings. */
18
19
  function toDateStr(value) {
19
20
  if (value instanceof Date)
@@ -61,6 +62,9 @@ export function syncMemoryToDb(db, frontmatter, content, sourcePath, projectId,
61
62
  project_id: projectId || null,
62
63
  scope: scope || "project",
63
64
  });
65
+ // v5.13.0: best-effort write-time embedding (no-op unless the MCP server
66
+ // enabled the queue). Never blocks or fails the write.
67
+ queueMemoryEmbedding(frontmatter.id);
64
68
  }
65
69
  /**
66
70
  * Sync a memory update to gnosys.db after it's been updated in .md.
@@ -113,6 +117,14 @@ export function syncUpdateToDb(db, id, updates, newContent) {
113
117
  }
114
118
  dbUpdates.modified = new Date().toISOString().split("T")[0];
115
119
  db.updateMemory(id, dbUpdates);
120
+ // v5.13.0: re-embed when searchable text changed (title/content/tags/
121
+ // relevance feed the embedding recipe). No-op unless the queue is enabled.
122
+ if (newContent !== undefined ||
123
+ updates.title !== undefined ||
124
+ updates.tags !== undefined ||
125
+ updates.relevance !== undefined) {
126
+ queueMemoryEmbedding(id);
127
+ }
116
128
  }
117
129
  /**
118
130
  * Sync an archive operation to gnosys.db.
@@ -79,6 +79,12 @@ export interface DreamReport {
79
79
  llmCalls?: DreamLLMCallRecord[];
80
80
  totals?: DreamRunRecord["totals"];
81
81
  effectiveness?: DreamEffectivenessRecord;
82
+ /** v5.13.0: embedding coverage found/repaired by the embedding-health phase. */
83
+ embeddingHealth?: {
84
+ total: number;
85
+ missingBefore: number;
86
+ embedded: number;
87
+ };
82
88
  }
83
89
  export interface ReviewSuggestion {
84
90
  memoryId: string;
@@ -99,9 +105,12 @@ export declare class GnosysDreamEngine {
99
105
  private dreamState;
100
106
  private pendingFingerprints;
101
107
  private llmCallsMade;
108
+ /** v5.13.0: explicit dream-state dir (defense against test pollution of ~/.gnosys). */
109
+ private stateDir?;
102
110
  constructor(db: GnosysDB, config: GnosysConfig, dreamConfig?: Partial<DreamConfig>, options?: {
103
111
  trigger?: DreamTrigger;
104
112
  machineId?: string;
113
+ stateDir?: string;
105
114
  });
106
115
  /** Captured at construction if getLLMProvider throws. Used in dream() to write a Layer 2 audit entry. */
107
116
  private providerInitError;
package/dist/lib/dream.js CHANGED
@@ -23,6 +23,7 @@ import { createProvider } from "./llm.js";
23
23
  import { notifyDesktop } from "./desktopNotify.js";
24
24
  import { syncConfidenceToDb, auditToDb } from "./dbWrite.js";
25
25
  import { logError } from "./log.js";
26
+ import { getGnosysHome } from "./paths.js";
26
27
  import { estimateCost, acquireDreamLock, estimateTokens, fingerprintMemories, memoryWatermark, readDreamState, writeDreamState, } from "./dreamRunLog.js";
27
28
  /** Layer 4 alert threshold: fire desktop notification at this many consecutive provider failures. */
28
29
  const DREAM_FAILURE_NOTIFY_THRESHOLD = 3;
@@ -44,6 +45,9 @@ export const DEFAULT_DREAM_CONFIG = {
44
45
  };
45
46
  // ─── Decay Constants ─────────────────────────────────────────────────────
46
47
  const DECAY_LAMBDA = 0.005;
48
+ // v5.13.0: embedding-health backfill cap per dream run. Bounds idle-time
49
+ // work (~128-row batches, local model) — large gaps close over a few runs.
50
+ const EMBEDDING_BACKFILL_MAX_PER_RUN = 512;
47
51
  const STALE_THRESHOLD = 0.3;
48
52
  // ─── Dream Engine ────────────────────────────────────────────────────────
49
53
  export class GnosysDreamEngine {
@@ -55,9 +59,11 @@ export class GnosysDreamEngine {
55
59
  startTime = 0;
56
60
  trigger;
57
61
  machineId;
58
- dreamState = readDreamState();
62
+ dreamState;
59
63
  pendingFingerprints = {};
60
64
  llmCallsMade = 0;
65
+ /** v5.13.0: explicit dream-state dir (defense against test pollution of ~/.gnosys). */
66
+ stateDir;
61
67
  constructor(db, config, dreamConfig, options) {
62
68
  this.db = db;
63
69
  this.config = config;
@@ -68,6 +74,8 @@ export class GnosysDreamEngine {
68
74
  };
69
75
  this.trigger = options?.trigger ?? "manual";
70
76
  this.machineId = options?.machineId;
77
+ this.stateDir = options?.stateDir;
78
+ this.dreamState = readDreamState(this.stateDir);
71
79
  // Initialize LLM provider for dream operations.
72
80
  // v5.4.2: Failure here is no longer silent — when dream tries to actually
73
81
  // run (in dream()), we record the unavailability to audit_log so the
@@ -128,7 +136,7 @@ export class GnosysDreamEngine {
128
136
  ...this.dreamState.analyzedFingerprints,
129
137
  ...this.pendingFingerprints,
130
138
  },
131
- });
139
+ }, this.stateDir);
132
140
  }
133
141
  catch {
134
142
  // Best-effort: a failed checkpoint only costs re-analysis on resume.
@@ -261,7 +269,7 @@ export class GnosysDreamEngine {
261
269
  this.llmCallsMade = 0;
262
270
  this.llmCalls = [];
263
271
  this.pendingFingerprints = {};
264
- this.dreamState = readDreamState();
272
+ this.dreamState = readDreamState(this.stateDir);
265
273
  const log = onProgress || (() => { });
266
274
  const startedAt = new Date().toISOString();
267
275
  const report = {
@@ -368,6 +376,92 @@ export class GnosysDreamEngine {
368
376
  report.abortReason = check.reason;
369
377
  return this.finalize(report);
370
378
  }
379
+ // ─── Phase 1.5: Embedding Health (v5.13.0) ───────────────────────────
380
+ // Semantic search reads memories.embedding; writes only best-effort
381
+ // embed. Dream is the safety net: report coverage and backfill missing
382
+ // vectors during idle time. No LLM involved — local embedding model.
383
+ {
384
+ log("embedding-health", "Phase 1.5: Embedding health check...");
385
+ const embedPhase = this.createPhase("embedding-health");
386
+ const embedStart = Date.now();
387
+ report.phases.push(embedPhase);
388
+ try {
389
+ const total = this.db.getMemoryCount().total;
390
+ const missingBefore = this.db.countMemoriesMissingEmbedding();
391
+ const embeddedCount = this.db.getEmbeddingCount();
392
+ if (missingBefore === 0) {
393
+ embedPhase.status = "skipped";
394
+ embedPhase.reason = "all memories embedded";
395
+ report.embeddingHealth = { total, missingBefore: 0, embedded: 0 };
396
+ log("embedding-health", `All ${total} memories embedded`);
397
+ }
398
+ else if (embeddedCount === 0) {
399
+ // Embeddings never initialized on this install — that's an explicit
400
+ // user choice (reindex downloads the 80 MB model); Dream repairs
401
+ // drift, it doesn't opt users in. Report it loudly and move on.
402
+ embedPhase.status = "skipped";
403
+ embedPhase.reason = "embeddings not initialized — run gnosys_reindex";
404
+ report.embeddingHealth = { total, missingBefore, embedded: 0 };
405
+ auditToDb(this.db, "dream_embedding_health", undefined, {
406
+ missing: missingBefore,
407
+ total,
408
+ initialized: false,
409
+ });
410
+ log("embedding-health", `${missingBefore}/${total} memories lack embeddings — run gnosys_reindex to enable semantic search`);
411
+ }
412
+ else {
413
+ auditToDb(this.db, "dream_embedding_health", undefined, {
414
+ missing: missingBefore,
415
+ total,
416
+ initialized: true,
417
+ });
418
+ const { backfillCentralDbEmbeddings } = await import("./embedDb.js");
419
+ const { GnosysEmbeddings } = await import("./embeddings.js");
420
+ // Model-only use — we never touch the embedder's store-local db.
421
+ const embedder = new GnosysEmbeddings(getGnosysHome());
422
+ let embedded = 0;
423
+ try {
424
+ // Chunked so shouldStop() (abort / max-runtime) is honored
425
+ // between batches; capped per run to bound idle-time work.
426
+ while (embedded < EMBEDDING_BACKFILL_MAX_PER_RUN) {
427
+ if (this.shouldStop().stop)
428
+ break;
429
+ const chunk = Math.min(128, EMBEDDING_BACKFILL_MAX_PER_RUN - embedded);
430
+ const result = await backfillCentralDbEmbeddings(this.db, embedder, {
431
+ mode: "missing",
432
+ limit: chunk,
433
+ });
434
+ embedded += result.embedded;
435
+ if (result.embedded === 0)
436
+ break;
437
+ }
438
+ }
439
+ finally {
440
+ embedder.close();
441
+ }
442
+ report.embeddingHealth = { total, missingBefore, embedded };
443
+ if (embedded > 0) {
444
+ auditToDb(this.db, "dream_embedding_backfill", undefined, {
445
+ embedded,
446
+ remaining: missingBefore - embedded,
447
+ });
448
+ }
449
+ log("embedding-health", `Embedded ${embedded}/${missingBefore} missing (${total} total)`);
450
+ }
451
+ }
452
+ catch (err) {
453
+ report.errors.push(`Embedding health: ${err instanceof Error ? err.message : String(err)}`);
454
+ }
455
+ finally {
456
+ this.finishPhase(embedPhase, embedStart);
457
+ }
458
+ check = this.shouldStop();
459
+ if (check.stop) {
460
+ report.aborted = true;
461
+ report.abortReason = check.reason;
462
+ return this.finalize(report);
463
+ }
464
+ }
371
465
  // ─── Phase 2: Self-Critique (Review Suggestions) ─────────────────────
372
466
  if (this.dreamConfig.selfCritique) {
373
467
  log("critique", "Phase 2: Self-critique...");
@@ -487,7 +581,7 @@ export class GnosysDreamEngine {
487
581
  ...this.dreamState.analyzedFingerprints,
488
582
  ...this.pendingFingerprints,
489
583
  },
490
- });
584
+ }, this.stateDir);
491
585
  // v5.4.2: A run is considered "successful with LLM work" if any of the
492
586
  // LLM-dependent counters moved. Resetting the consecutive-failure count
493
587
  // here ensures Layer 4 doesn't keep firing once dream is healthy again.
@@ -9,7 +9,7 @@ export interface DreamRunGateResult {
9
9
  details?: Record<string, unknown>;
10
10
  }
11
11
  export interface DreamRunPhaseRecord {
12
- name: "decay" | "critique" | "summaries" | "relationships";
12
+ name: "decay" | "embedding-health" | "critique" | "summaries" | "relationships";
13
13
  status: "ran" | "skipped";
14
14
  reason?: string;
15
15
  durationMs: number;
@@ -86,7 +86,7 @@ export interface DreamReadOptions {
86
86
  status?: DreamRunStatus;
87
87
  }
88
88
  export declare function getDreamRunsPath(): string;
89
- export declare function getDreamStatePath(): string;
89
+ export declare function getDreamStatePath(baseDir?: string): string;
90
90
  export declare function getDreamLockPath(): string;
91
91
  export declare function acquireDreamLock(depth?: number): {
92
92
  acquired: true;
@@ -97,8 +97,8 @@ export declare function acquireDreamLock(depth?: number): {
97
97
  };
98
98
  export declare function estimateTokens(text: string): number;
99
99
  export declare function estimateCost(model: string | undefined, inputTokens: number, outputTokens: number): number;
100
- export declare function readDreamState(): DreamState;
101
- export declare function writeDreamState(state: DreamState): void;
100
+ export declare function readDreamState(baseDir?: string): DreamState;
101
+ export declare function writeDreamState(state: DreamState, baseDir?: string): void;
102
102
  export declare function appendDreamRun(record: DreamRunRecord): void;
103
103
  export declare function readDreamRuns(opts?: DreamReadOptions): DreamRunRecord[];
104
104
  export declare function fingerprintMemories(kind: "summary" | "critique" | "relationship", memories: DbMemory[]): string;
@@ -20,8 +20,11 @@ const MODEL_PRICES_USD_PER_MILLION = {
20
20
  export function getDreamRunsPath() {
21
21
  return path.join(getGnosysHome(), "dream-runs.jsonl");
22
22
  }
23
- export function getDreamStatePath() {
24
- return path.join(getGnosysHome(), "dream-state.json");
23
+ // v5.13.0: baseDir injection (defense in depth against test pollution)
24
+ // in-process engines can target an explicit state dir instead of the
25
+ // developer's real ~/.gnosys.
26
+ export function getDreamStatePath(baseDir) {
27
+ return path.join(baseDir || getGnosysHome(), "dream-state.json");
25
28
  }
26
29
  export function getDreamLockPath() {
27
30
  return path.join(getGnosysHome(), "dream.lock");
@@ -103,9 +106,9 @@ export function estimateCost(model, inputTokens, outputTokens) {
103
106
  const cost = (inputTokens / 1_000_000) * fuzzy.input + (outputTokens / 1_000_000) * fuzzy.output;
104
107
  return Math.round(cost * 1_000_000) / 1_000_000;
105
108
  }
106
- export function readDreamState() {
109
+ export function readDreamState(baseDir) {
107
110
  try {
108
- const raw = fs.readFileSync(getDreamStatePath(), "utf8");
111
+ const raw = fs.readFileSync(getDreamStatePath(baseDir), "utf8");
109
112
  const parsed = JSON.parse(raw);
110
113
  return {
111
114
  ...DEFAULT_STATE,
@@ -117,8 +120,8 @@ export function readDreamState() {
117
120
  return { ...DEFAULT_STATE };
118
121
  }
119
122
  }
120
- export function writeDreamState(state) {
121
- const file = getDreamStatePath();
123
+ export function writeDreamState(state, baseDir) {
124
+ const file = getDreamStatePath(baseDir);
122
125
  fs.mkdirSync(path.dirname(file), { recursive: true });
123
126
  fs.writeFileSync(file, `${JSON.stringify(state, null, 2)}\n`, "utf8");
124
127
  }
@@ -0,0 +1,53 @@
1
+ /**
2
+ * Central-DB embedding utilities (v5.13.0).
3
+ *
4
+ * GnosysDbSearch reads vectors from the memories.embedding column of
5
+ * gnosys.db, but before v5.13.0 nothing ever wrote that column: the
6
+ * reindex flow embedded file-store memories into the store-local
7
+ * embeddings.db, so DB-mode hybrid search could never run its semantic
8
+ * leg. This module owns the column — full backfill for reindex,
9
+ * missing-only backfill for Dream/maintenance, and single-memory
10
+ * embedding for the write-time queue.
11
+ *
12
+ * Embeddings are derived, rebuildable data: regenerating them is always
13
+ * safe, and a sync snapshot refresh replacing them is not a loss.
14
+ */
15
+ import type { GnosysDB } from "./db.js";
16
+ import type { GnosysEmbeddings } from "./embeddings.js";
17
+ /** Row fields needed to build embedding text. */
18
+ export interface EmbeddableMemoryRow {
19
+ id: string;
20
+ title: string;
21
+ relevance: string | null;
22
+ tags: string;
23
+ content: string;
24
+ }
25
+ /**
26
+ * The text recipe — identical to the file-store reindex in hybridSearch.ts
27
+ * (title, relevance keywords, tags, content) so query-vs-document
28
+ * similarity behaves the same in both modes.
29
+ */
30
+ export declare function embeddingText(m: EmbeddableMemoryRow): string;
31
+ /** View a Float32Array as the Buffer shape memories.embedding stores. */
32
+ export declare function float32ToBuffer(vec: Float32Array): Buffer;
33
+ /**
34
+ * (Re)build the memories.embedding column.
35
+ *
36
+ * mode "all" regenerates every memory (reindex semantics); "missing" only
37
+ * fills NULL rows (write-gap backfill for Dream/maintenance, newest first).
38
+ * Batched through embedBatch for model efficiency.
39
+ */
40
+ export declare function backfillCentralDbEmbeddings(db: GnosysDB, embeddings: GnosysEmbeddings, opts?: {
41
+ mode?: "missing" | "all";
42
+ batchSize?: number;
43
+ limit?: number;
44
+ onProgress?: (current: number, total: number, id: string) => void;
45
+ }): Promise<{
46
+ embedded: number;
47
+ total: number;
48
+ }>;
49
+ /**
50
+ * Embed one memory and store its vector. Returns false if the memory
51
+ * doesn't exist; throws on embedder failure — callers decide how loud.
52
+ */
53
+ export declare function embedMemoryIntoDb(db: GnosysDB, embeddings: GnosysEmbeddings, id: string): Promise<boolean>;