kiro-memory 1.7.1 → 1.8.0

package/plugin/dist/services/search/index.js

@@ -58,7 +58,7 @@ function searchObservationsFTS(db, query, filters = {}) {
       sql += " AND o.created_at_epoch <= ?";
       params.push(filters.dateEnd);
     }
-    sql += " ORDER BY rank LIMIT ?";
+    sql += ` ORDER BY bm25(observations_fts, ${BM25_WEIGHTS}) LIMIT ?`;
     params.push(limit);
     const stmt = db.query(sql);
     return stmt.all(...params);
@@ -72,7 +72,7 @@ function searchObservationsFTSWithRank(db, query, filters = {}) {
     const safeQuery = sanitizeFTS5Query(query);
     if (!safeQuery) return [];
     let sql = `
-      SELECT o.*, rank as fts5_rank FROM observations o
+      SELECT o.*, bm25(observations_fts, ${BM25_WEIGHTS}) as fts5_rank FROM observations o
       JOIN observations_fts fts ON o.id = fts.rowid
       WHERE observations_fts MATCH ?
     `;
@@ -93,7 +93,7 @@ function searchObservationsFTSWithRank(db, query, filters = {}) {
       sql += " AND o.created_at_epoch <= ?";
       params.push(filters.dateEnd);
     }
-    sql += " ORDER BY rank LIMIT ?";
+    sql += ` ORDER BY bm25(observations_fts, ${BM25_WEIGHTS}) LIMIT ?`;
     params.push(limit);
     const stmt = db.query(sql);
     return stmt.all(...params);
@@ -197,11 +197,23 @@ function getProjectStats(db, project) {
   const sumStmt = db.query("SELECT COUNT(*) as count FROM summaries WHERE project = ?");
   const sesStmt = db.query("SELECT COUNT(*) as count FROM sessions WHERE project = ?");
   const prmStmt = db.query("SELECT COUNT(*) as count FROM prompts WHERE project = ?");
+  const discoveryStmt = db.query(
+    "SELECT COALESCE(SUM(discovery_tokens), 0) as total FROM observations WHERE project = ?"
+  );
+  const discoveryTokens = discoveryStmt.get(project)?.total || 0;
+  const readStmt = db.query(
+    `SELECT COALESCE(SUM(
+      CAST((LENGTH(COALESCE(title, '')) + LENGTH(COALESCE(narrative, ''))) / 4 AS INTEGER)
+    ), 0) as total FROM observations WHERE project = ?`
+  );
+  const readTokens = readStmt.get(project)?.total || 0;
+  const savings = Math.max(0, discoveryTokens - readTokens);
   return {
     observations: obsStmt.get(project)?.count || 0,
     summaries: sumStmt.get(project)?.count || 0,
     sessions: sesStmt.get(project)?.count || 0,
-    prompts: prmStmt.get(project)?.count || 0
+    prompts: prmStmt.get(project)?.count || 0,
+    tokenEconomics: { discoveryTokens, readTokens, savings }
   };
 }
 function getStaleObservations(db, project) {
@@ -243,9 +255,11 @@ function markObservationsStale(db, ids, stale) {
     [stale ? 1 : 0, ...validIds]
   );
 }
+var BM25_WEIGHTS;
 var init_Search = __esm({
   "src/services/sqlite/Search.ts"() {
     "use strict";
+    BM25_WEIGHTS = "10.0, 1.0, 5.0, 3.0";
   }
 });
 
@@ -257,19 +271,28 @@ __export(Observations_exports, {
   deleteObservation: () => deleteObservation,
   getObservationsByProject: () => getObservationsByProject,
   getObservationsBySession: () => getObservationsBySession,
+  isDuplicateObservation: () => isDuplicateObservation,
   searchObservations: () => searchObservations,
   updateLastAccessed: () => updateLastAccessed
 });
 function escapeLikePattern2(input) {
   return input.replace(/[%_\\]/g, "\\$&");
 }
-function createObservation(db, memorySessionId, project, type, title, subtitle, text, narrative, facts, concepts, filesRead, filesModified, promptNumber) {
+function isDuplicateObservation(db, contentHash, windowMs = 3e4) {
+  if (!contentHash) return false;
+  const threshold = Date.now() - windowMs;
+  const result = db.query(
+    "SELECT id FROM observations WHERE content_hash = ? AND created_at_epoch > ? LIMIT 1"
+  ).get(contentHash, threshold);
+  return !!result;
+}
+function createObservation(db, memorySessionId, project, type, title, subtitle, text, narrative, facts, concepts, filesRead, filesModified, promptNumber, contentHash = null, discoveryTokens = 0) {
   const now = /* @__PURE__ */ new Date();
   const result = db.run(
-    `INSERT INTO observations 
-     (memory_session_id, project, type, title, subtitle, text, narrative, facts, concepts, files_read, files_modified, prompt_number, created_at, created_at_epoch)
-     VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`,
-    [memorySessionId, project, type, title, subtitle, text, narrative, facts, concepts, filesRead, filesModified, promptNumber, now.toISOString(), now.getTime()]
+    `INSERT INTO observations
+     (memory_session_id, project, type, title, subtitle, text, narrative, facts, concepts, files_read, files_modified, prompt_number, created_at, created_at_epoch, content_hash, discovery_tokens)
+     VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`,
+    [memorySessionId, project, type, title, subtitle, text, narrative, facts, concepts, filesRead, filesModified, promptNumber, now.toISOString(), now.getTime(), contentHash, discoveryTokens]
   );
   return Number(result.lastInsertRowid);
 }
@@ -325,39 +348,42 @@ function consolidateObservations(db, project, options = {}) {
   if (groups.length === 0) return { merged: 0, removed: 0 };
   let totalMerged = 0;
   let totalRemoved = 0;
-  for (const group of groups) {
-    const obsIds = group.ids.split(",").map(Number);
-    const placeholders = obsIds.map(() => "?").join(",");
-    const observations = db.query(
-      `SELECT * FROM observations WHERE id IN (${placeholders}) ORDER BY created_at_epoch DESC`
-    ).all(...obsIds);
-    if (observations.length < minGroupSize) continue;
-    if (options.dryRun) {
-      totalMerged += 1;
-      totalRemoved += observations.length - 1;
-      continue;
-    }
-    const keeper = observations[0];
-    const others = observations.slice(1);
-    const uniqueTexts = /* @__PURE__ */ new Set();
-    if (keeper.text) uniqueTexts.add(keeper.text);
-    for (const obs of others) {
-      if (obs.text && !uniqueTexts.has(obs.text)) {
-        uniqueTexts.add(obs.text);
+  const runConsolidation = db.transaction(() => {
+    for (const group of groups) {
+      const obsIds = group.ids.split(",").map(Number);
+      const placeholders = obsIds.map(() => "?").join(",");
+      const observations = db.query(
+        `SELECT * FROM observations WHERE id IN (${placeholders}) ORDER BY created_at_epoch DESC`
+      ).all(...obsIds);
+      if (observations.length < minGroupSize) continue;
+      if (options.dryRun) {
+        totalMerged += 1;
+        totalRemoved += observations.length - 1;
+        continue;
       }
+      const keeper = observations[0];
+      const others = observations.slice(1);
+      const uniqueTexts = /* @__PURE__ */ new Set();
+      if (keeper.text) uniqueTexts.add(keeper.text);
+      for (const obs of others) {
+        if (obs.text && !uniqueTexts.has(obs.text)) {
+          uniqueTexts.add(obs.text);
+        }
+      }
+      const consolidatedText = Array.from(uniqueTexts).join("\n---\n").substring(0, 1e5);
+      db.run(
+        "UPDATE observations SET text = ?, title = ? WHERE id = ?",
+        [consolidatedText, `[consolidato x${observations.length}] ${keeper.title}`, keeper.id]
+      );
+      const removeIds = others.map((o) => o.id);
+      const removePlaceholders = removeIds.map(() => "?").join(",");
+      db.run(`DELETE FROM observations WHERE id IN (${removePlaceholders})`, removeIds);
+      db.run(`DELETE FROM observation_embeddings WHERE observation_id IN (${removePlaceholders})`, removeIds);
+      totalMerged += 1;
+      totalRemoved += removeIds.length;
     }
-    const consolidatedText = Array.from(uniqueTexts).join("\n---\n").substring(0, 1e5);
-    db.run(
-      "UPDATE observations SET text = ?, title = ? WHERE id = ?",
-      [consolidatedText, `[consolidato x${observations.length}] ${keeper.title}`, keeper.id]
-    );
-    const removeIds = others.map((o) => o.id);
-    const removePlaceholders = removeIds.map(() => "?").join(",");
-    db.run(`DELETE FROM observations WHERE id IN (${removePlaceholders})`, removeIds);
-    db.run(`DELETE FROM observation_embeddings WHERE observation_id IN (${removePlaceholders})`, removeIds);
-    totalMerged += 1;
-    totalRemoved += removeIds.length;
-  }
+  });
+  runConsolidation();
   return { merged: totalMerged, removed: totalRemoved };
 }
 var init_Observations = __esm({

package/plugin/dist/services/sqlite/Database.js

@@ -685,6 +685,29 @@ var MigrationRunner = class {
           db.run("CREATE INDEX IF NOT EXISTS idx_checkpoints_project ON checkpoints(project)");
           db.run("CREATE INDEX IF NOT EXISTS idx_checkpoints_epoch ON checkpoints(created_at_epoch)");
         }
+      },
+      {
+        version: 7,
+        up: (db) => {
+          db.run("ALTER TABLE observations ADD COLUMN content_hash TEXT");
+          db.run("CREATE INDEX IF NOT EXISTS idx_observations_hash ON observations(content_hash)");
+        }
+      },
+      {
+        version: 8,
+        up: (db) => {
+          db.run("ALTER TABLE observations ADD COLUMN discovery_tokens INTEGER DEFAULT 0");
+          db.run("ALTER TABLE summaries ADD COLUMN discovery_tokens INTEGER DEFAULT 0");
+        }
+      },
+      {
+        version: 9,
+        up: (db) => {
+          db.run("CREATE INDEX IF NOT EXISTS idx_observations_project_epoch ON observations(project, created_at_epoch DESC)");
+          db.run("CREATE INDEX IF NOT EXISTS idx_observations_project_type ON observations(project, type)");
+          db.run("CREATE INDEX IF NOT EXISTS idx_summaries_project_epoch ON summaries(project, created_at_epoch DESC)");
+          db.run("CREATE INDEX IF NOT EXISTS idx_prompts_project_epoch ON prompts(project, created_at_epoch DESC)");
+        }
       }
     ];
   }

package/plugin/dist/services/sqlite/Observations.js

@@ -4,13 +4,21 @@ import { createRequire } from 'module';const require = createRequire(import.meta
 function escapeLikePattern(input) {
   return input.replace(/[%_\\]/g, "\\$&");
 }
-function createObservation(db, memorySessionId, project, type, title, subtitle, text, narrative, facts, concepts, filesRead, filesModified, promptNumber) {
+function isDuplicateObservation(db, contentHash, windowMs = 3e4) {
+  if (!contentHash) return false;
+  const threshold = Date.now() - windowMs;
+  const result = db.query(
+    "SELECT id FROM observations WHERE content_hash = ? AND created_at_epoch > ? LIMIT 1"
+  ).get(contentHash, threshold);
+  return !!result;
+}
+function createObservation(db, memorySessionId, project, type, title, subtitle, text, narrative, facts, concepts, filesRead, filesModified, promptNumber, contentHash = null, discoveryTokens = 0) {
   const now = /* @__PURE__ */ new Date();
   const result = db.run(
-    `INSERT INTO observations 
-     (memory_session_id, project, type, title, subtitle, text, narrative, facts, concepts, files_read, files_modified, prompt_number, created_at, created_at_epoch)
-     VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`,
-    [memorySessionId, project, type, title, subtitle, text, narrative, facts, concepts, filesRead, filesModified, promptNumber, now.toISOString(), now.getTime()]
+    `INSERT INTO observations
+     (memory_session_id, project, type, title, subtitle, text, narrative, facts, concepts, files_read, files_modified, prompt_number, created_at, created_at_epoch, content_hash, discovery_tokens)
+     VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`,
+    [memorySessionId, project, type, title, subtitle, text, narrative, facts, concepts, filesRead, filesModified, promptNumber, now.toISOString(), now.getTime(), contentHash, discoveryTokens]
   );
   return Number(result.lastInsertRowid);
 }
@@ -66,39 +74,42 @@ function consolidateObservations(db, project, options = {}) {
   if (groups.length === 0) return { merged: 0, removed: 0 };
   let totalMerged = 0;
   let totalRemoved = 0;
-  for (const group of groups) {
-    const obsIds = group.ids.split(",").map(Number);
-    const placeholders = obsIds.map(() => "?").join(",");
-    const observations = db.query(
-      `SELECT * FROM observations WHERE id IN (${placeholders}) ORDER BY created_at_epoch DESC`
-    ).all(...obsIds);
-    if (observations.length < minGroupSize) continue;
-    if (options.dryRun) {
-      totalMerged += 1;
-      totalRemoved += observations.length - 1;
-      continue;
-    }
-    const keeper = observations[0];
-    const others = observations.slice(1);
-    const uniqueTexts = /* @__PURE__ */ new Set();
-    if (keeper.text) uniqueTexts.add(keeper.text);
-    for (const obs of others) {
-      if (obs.text && !uniqueTexts.has(obs.text)) {
-        uniqueTexts.add(obs.text);
+  const runConsolidation = db.transaction(() => {
+    for (const group of groups) {
+      const obsIds = group.ids.split(",").map(Number);
+      const placeholders = obsIds.map(() => "?").join(",");
+      const observations = db.query(
+        `SELECT * FROM observations WHERE id IN (${placeholders}) ORDER BY created_at_epoch DESC`
+      ).all(...obsIds);
+      if (observations.length < minGroupSize) continue;
+      if (options.dryRun) {
+        totalMerged += 1;
+        totalRemoved += observations.length - 1;
+        continue;
       }
+      const keeper = observations[0];
+      const others = observations.slice(1);
+      const uniqueTexts = /* @__PURE__ */ new Set();
+      if (keeper.text) uniqueTexts.add(keeper.text);
+      for (const obs of others) {
+        if (obs.text && !uniqueTexts.has(obs.text)) {
+          uniqueTexts.add(obs.text);
+        }
+      }
+      const consolidatedText = Array.from(uniqueTexts).join("\n---\n").substring(0, 1e5);
+      db.run(
+        "UPDATE observations SET text = ?, title = ? WHERE id = ?",
+        [consolidatedText, `[consolidato x${observations.length}] ${keeper.title}`, keeper.id]
+      );
+      const removeIds = others.map((o) => o.id);
+      const removePlaceholders = removeIds.map(() => "?").join(",");
+      db.run(`DELETE FROM observations WHERE id IN (${removePlaceholders})`, removeIds);
+      db.run(`DELETE FROM observation_embeddings WHERE observation_id IN (${removePlaceholders})`, removeIds);
+      totalMerged += 1;
+      totalRemoved += removeIds.length;
     }
-    const consolidatedText = Array.from(uniqueTexts).join("\n---\n").substring(0, 1e5);
-    db.run(
-      "UPDATE observations SET text = ?, title = ? WHERE id = ?",
-      [consolidatedText, `[consolidato x${observations.length}] ${keeper.title}`, keeper.id]
-    );
-    const removeIds = others.map((o) => o.id);
-    const removePlaceholders = removeIds.map(() => "?").join(",");
-    db.run(`DELETE FROM observations WHERE id IN (${removePlaceholders})`, removeIds);
-    db.run(`DELETE FROM observation_embeddings WHERE observation_id IN (${removePlaceholders})`, removeIds);
-    totalMerged += 1;
-    totalRemoved += removeIds.length;
-  }
+  });
+  runConsolidation();
   return { merged: totalMerged, removed: totalRemoved };
 }
 export {
@@ -107,6 +118,7 @@ export {
   deleteObservation,
   getObservationsByProject,
   getObservationsBySession,
+  isDuplicateObservation,
   searchObservations,
   updateLastAccessed
 };

package/plugin/dist/services/sqlite/Search.js

@@ -2,6 +2,7 @@ import { createRequire } from 'module';const require = createRequire(import.meta
 
 // src/services/sqlite/Search.ts
 import { existsSync, statSync } from "fs";
+var BM25_WEIGHTS = "10.0, 1.0, 5.0, 3.0";
 function escapeLikePattern(input) {
   return input.replace(/[%_\\]/g, "\\$&");
 }
@@ -37,7 +38,7 @@ function searchObservationsFTS(db, query, filters = {}) {
       sql += " AND o.created_at_epoch <= ?";
       params.push(filters.dateEnd);
     }
-    sql += " ORDER BY rank LIMIT ?";
+    sql += ` ORDER BY bm25(observations_fts, ${BM25_WEIGHTS}) LIMIT ?`;
     params.push(limit);
     const stmt = db.query(sql);
     return stmt.all(...params);
@@ -51,7 +52,7 @@ function searchObservationsFTSWithRank(db, query, filters = {}) {
     const safeQuery = sanitizeFTS5Query(query);
     if (!safeQuery) return [];
     let sql = `
-      SELECT o.*, rank as fts5_rank FROM observations o
+      SELECT o.*, bm25(observations_fts, ${BM25_WEIGHTS}) as fts5_rank FROM observations o
       JOIN observations_fts fts ON o.id = fts.rowid
       WHERE observations_fts MATCH ?
     `;
@@ -72,7 +73,7 @@ function searchObservationsFTSWithRank(db, query, filters = {}) {
       sql += " AND o.created_at_epoch <= ?";
       params.push(filters.dateEnd);
     }
-    sql += " ORDER BY rank LIMIT ?";
+    sql += ` ORDER BY bm25(observations_fts, ${BM25_WEIGHTS}) LIMIT ?`;
     params.push(limit);
     const stmt = db.query(sql);
     return stmt.all(...params);
@@ -176,11 +177,23 @@ function getProjectStats(db, project) {
   const sumStmt = db.query("SELECT COUNT(*) as count FROM summaries WHERE project = ?");
   const sesStmt = db.query("SELECT COUNT(*) as count FROM sessions WHERE project = ?");
   const prmStmt = db.query("SELECT COUNT(*) as count FROM prompts WHERE project = ?");
+  const discoveryStmt = db.query(
+    "SELECT COALESCE(SUM(discovery_tokens), 0) as total FROM observations WHERE project = ?"
+  );
+  const discoveryTokens = discoveryStmt.get(project)?.total || 0;
+  const readStmt = db.query(
+    `SELECT COALESCE(SUM(
+      CAST((LENGTH(COALESCE(title, '')) + LENGTH(COALESCE(narrative, ''))) / 4 AS INTEGER)
+    ), 0) as total FROM observations WHERE project = ?`
+  );
+  const readTokens = readStmt.get(project)?.total || 0;
+  const savings = Math.max(0, discoveryTokens - readTokens);
   return {
     observations: obsStmt.get(project)?.count || 0,
     summaries: sumStmt.get(project)?.count || 0,
     sessions: sesStmt.get(project)?.count || 0,
-    prompts: prmStmt.get(project)?.count || 0
+    prompts: prmStmt.get(project)?.count || 0,
+    tokenEconomics: { discoveryTokens, readTokens, savings }
   };
 }
 function getStaleObservations(db, project) {

package/plugin/dist/services/sqlite/index.js

@@ -685,6 +685,29 @@ var MigrationRunner = class {
           db.run("CREATE INDEX IF NOT EXISTS idx_checkpoints_project ON checkpoints(project)");
           db.run("CREATE INDEX IF NOT EXISTS idx_checkpoints_epoch ON checkpoints(created_at_epoch)");
         }
+      },
+      {
+        version: 7,
+        up: (db) => {
+          db.run("ALTER TABLE observations ADD COLUMN content_hash TEXT");
+          db.run("CREATE INDEX IF NOT EXISTS idx_observations_hash ON observations(content_hash)");
+        }
+      },
+      {
+        version: 8,
+        up: (db) => {
+          db.run("ALTER TABLE observations ADD COLUMN discovery_tokens INTEGER DEFAULT 0");
+          db.run("ALTER TABLE summaries ADD COLUMN discovery_tokens INTEGER DEFAULT 0");
+        }
+      },
+      {
+        version: 9,
+        up: (db) => {
+          db.run("CREATE INDEX IF NOT EXISTS idx_observations_project_epoch ON observations(project, created_at_epoch DESC)");
+          db.run("CREATE INDEX IF NOT EXISTS idx_observations_project_type ON observations(project, type)");
+          db.run("CREATE INDEX IF NOT EXISTS idx_summaries_project_epoch ON summaries(project, created_at_epoch DESC)");
+          db.run("CREATE INDEX IF NOT EXISTS idx_prompts_project_epoch ON prompts(project, created_at_epoch DESC)");
+        }
       }
     ];
   }
@@ -755,13 +778,21 @@ function getSessionsByProject(db, project, limit = 100) {
 function escapeLikePattern(input) {
   return input.replace(/[%_\\]/g, "\\$&");
 }
-function createObservation(db, memorySessionId, project, type, title, subtitle, text, narrative, facts, concepts, filesRead, filesModified, promptNumber) {
+function isDuplicateObservation(db, contentHash, windowMs = 3e4) {
+  if (!contentHash) return false;
+  const threshold = Date.now() - windowMs;
+  const result = db.query(
+    "SELECT id FROM observations WHERE content_hash = ? AND created_at_epoch > ? LIMIT 1"
+  ).get(contentHash, threshold);
+  return !!result;
+}
+function createObservation(db, memorySessionId, project, type, title, subtitle, text, narrative, facts, concepts, filesRead, filesModified, promptNumber, contentHash = null, discoveryTokens = 0) {
   const now = /* @__PURE__ */ new Date();
   const result = db.run(
-    `INSERT INTO observations 
-     (memory_session_id, project, type, title, subtitle, text, narrative, facts, concepts, files_read, files_modified, prompt_number, created_at, created_at_epoch)
-     VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`,
-    [memorySessionId, project, type, title, subtitle, text, narrative, facts, concepts, filesRead, filesModified, promptNumber, now.toISOString(), now.getTime()]
+    `INSERT INTO observations
+     (memory_session_id, project, type, title, subtitle, text, narrative, facts, concepts, files_read, files_modified, prompt_number, created_at, created_at_epoch, content_hash, discovery_tokens)
+     VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`,
+    [memorySessionId, project, type, title, subtitle, text, narrative, facts, concepts, filesRead, filesModified, promptNumber, now.toISOString(), now.getTime(), contentHash, discoveryTokens]
   );
   return Number(result.lastInsertRowid);
 }
@@ -817,39 +848,42 @@ function consolidateObservations(db, project, options = {}) {
   if (groups.length === 0) return { merged: 0, removed: 0 };
   let totalMerged = 0;
   let totalRemoved = 0;
-  for (const group of groups) {
-    const obsIds = group.ids.split(",").map(Number);
-    const placeholders = obsIds.map(() => "?").join(",");
-    const observations = db.query(
-      `SELECT * FROM observations WHERE id IN (${placeholders}) ORDER BY created_at_epoch DESC`
-    ).all(...obsIds);
-    if (observations.length < minGroupSize) continue;
-    if (options.dryRun) {
-      totalMerged += 1;
-      totalRemoved += observations.length - 1;
-      continue;
-    }
-    const keeper = observations[0];
-    const others = observations.slice(1);
-    const uniqueTexts = /* @__PURE__ */ new Set();
-    if (keeper.text) uniqueTexts.add(keeper.text);
-    for (const obs of others) {
-      if (obs.text && !uniqueTexts.has(obs.text)) {
-        uniqueTexts.add(obs.text);
+  const runConsolidation = db.transaction(() => {
+    for (const group of groups) {
+      const obsIds = group.ids.split(",").map(Number);
+      const placeholders = obsIds.map(() => "?").join(",");
+      const observations = db.query(
+        `SELECT * FROM observations WHERE id IN (${placeholders}) ORDER BY created_at_epoch DESC`
+      ).all(...obsIds);
+      if (observations.length < minGroupSize) continue;
+      if (options.dryRun) {
+        totalMerged += 1;
+        totalRemoved += observations.length - 1;
+        continue;
+      }
+      const keeper = observations[0];
+      const others = observations.slice(1);
+      const uniqueTexts = /* @__PURE__ */ new Set();
+      if (keeper.text) uniqueTexts.add(keeper.text);
+      for (const obs of others) {
+        if (obs.text && !uniqueTexts.has(obs.text)) {
+          uniqueTexts.add(obs.text);
+        }
       }
+      const consolidatedText = Array.from(uniqueTexts).join("\n---\n").substring(0, 1e5);
+      db.run(
+        "UPDATE observations SET text = ?, title = ? WHERE id = ?",
+        [consolidatedText, `[consolidato x${observations.length}] ${keeper.title}`, keeper.id]
+      );
+      const removeIds = others.map((o) => o.id);
+      const removePlaceholders = removeIds.map(() => "?").join(",");
+      db.run(`DELETE FROM observations WHERE id IN (${removePlaceholders})`, removeIds);
+      db.run(`DELETE FROM observation_embeddings WHERE observation_id IN (${removePlaceholders})`, removeIds);
+      totalMerged += 1;
+      totalRemoved += removeIds.length;
     }
-    const consolidatedText = Array.from(uniqueTexts).join("\n---\n").substring(0, 1e5);
-    db.run(
-      "UPDATE observations SET text = ?, title = ? WHERE id = ?",
-      [consolidatedText, `[consolidato x${observations.length}] ${keeper.title}`, keeper.id]
-    );
-    const removeIds = others.map((o) => o.id);
-    const removePlaceholders = removeIds.map(() => "?").join(",");
-    db.run(`DELETE FROM observations WHERE id IN (${removePlaceholders})`, removeIds);
-    db.run(`DELETE FROM observation_embeddings WHERE observation_id IN (${removePlaceholders})`, removeIds);
-    totalMerged += 1;
-    totalRemoved += removeIds.length;
-  }
+  });
+  runConsolidation();
   return { merged: totalMerged, removed: totalRemoved };
 }
 
@@ -1090,6 +1124,7 @@ function getReportData(db, project, startEpoch, endEpoch) {
 
 // src/services/sqlite/Search.ts
 import { existsSync as existsSync3, statSync } from "fs";
+var BM25_WEIGHTS = "10.0, 1.0, 5.0, 3.0";
 function escapeLikePattern3(input) {
   return input.replace(/[%_\\]/g, "\\$&");
 }
@@ -1125,7 +1160,7 @@ function searchObservationsFTS(db, query, filters = {}) {
       sql += " AND o.created_at_epoch <= ?";
       params.push(filters.dateEnd);
     }
-    sql += " ORDER BY rank LIMIT ?";
+    sql += ` ORDER BY bm25(observations_fts, ${BM25_WEIGHTS}) LIMIT ?`;
     params.push(limit);
     const stmt = db.query(sql);
     return stmt.all(...params);
@@ -1139,7 +1174,7 @@ function searchObservationsFTSWithRank(db, query, filters = {}) {
     const safeQuery = sanitizeFTS5Query(query);
     if (!safeQuery) return [];
     let sql = `
-      SELECT o.*, rank as fts5_rank FROM observations o
+      SELECT o.*, bm25(observations_fts, ${BM25_WEIGHTS}) as fts5_rank FROM observations o
       JOIN observations_fts fts ON o.id = fts.rowid
       WHERE observations_fts MATCH ?
     `;
@@ -1160,7 +1195,7 @@ function searchObservationsFTSWithRank(db, query, filters = {}) {
       sql += " AND o.created_at_epoch <= ?";
       params.push(filters.dateEnd);
     }
-    sql += " ORDER BY rank LIMIT ?";
+    sql += ` ORDER BY bm25(observations_fts, ${BM25_WEIGHTS}) LIMIT ?`;
     params.push(limit);
     const stmt = db.query(sql);
     return stmt.all(...params);
@@ -1264,11 +1299,23 @@ function getProjectStats(db, project) {
   const sumStmt = db.query("SELECT COUNT(*) as count FROM summaries WHERE project = ?");
   const sesStmt = db.query("SELECT COUNT(*) as count FROM sessions WHERE project = ?");
   const prmStmt = db.query("SELECT COUNT(*) as count FROM prompts WHERE project = ?");
+  const discoveryStmt = db.query(
+    "SELECT COALESCE(SUM(discovery_tokens), 0) as total FROM observations WHERE project = ?"
+  );
+  const discoveryTokens = discoveryStmt.get(project)?.total || 0;
+  const readStmt = db.query(
+    `SELECT COALESCE(SUM(
+      CAST((LENGTH(COALESCE(title, '')) + LENGTH(COALESCE(narrative, ''))) / 4 AS INTEGER)
+    ), 0) as total FROM observations WHERE project = ?`
+  );
+  const readTokens = readStmt.get(project)?.total || 0;
+  const savings = Math.max(0, discoveryTokens - readTokens);
   return {
     observations: obsStmt.get(project)?.count || 0,
     summaries: sumStmt.get(project)?.count || 0,
     sessions: sesStmt.get(project)?.count || 0,
-    prompts: prmStmt.get(project)?.count || 0
+    prompts: prmStmt.get(project)?.count || 0,
+    tokenEconomics: { discoveryTokens, readTokens, savings }
   };
 }
 function getStaleObservations(db, project) {
@@ -1346,6 +1393,7 @@ export {
   getSummaryBySession,
   getTimeline,
   initializeDatabase,
+  isDuplicateObservation,
   markObservationsStale,
   searchObservations,
   searchObservationsFTS,

package/plugin/dist/viewer.html

@@ -75,6 +75,10 @@
               '0%, 100%': { opacity: '1' },
               '50%': { opacity: '0.5' },
             },
+            'slide-in-left': {
+              '0%': { opacity: '0', transform: 'translateX(-100%)' },
+              '100%': { opacity: '1', transform: 'translateX(0)' },
+            },
           },
           animation: {
             'slide-up': 'slide-up 0.35s ease-out forwards',
@@ -82,6 +86,7 @@
             'scale-in': 'scale-in 0.2s ease-out forwards',
             'spin': 'spin 0.8s linear infinite',
             'pulse-dot': 'pulse-dot 2s ease-in-out infinite',
+            'slide-in-left': 'slide-in-left 0.25s ease-out forwards',
           },
         },
       },