kiro-memory 1.7.1 → 1.8.0

package/plugin/dist/index.js

@@ -23,19 +23,28 @@ __export(Observations_exports, {
   deleteObservation: () => deleteObservation,
   getObservationsByProject: () => getObservationsByProject,
   getObservationsBySession: () => getObservationsBySession,
+  isDuplicateObservation: () => isDuplicateObservation,
   searchObservations: () => searchObservations,
   updateLastAccessed: () => updateLastAccessed
 });
 function escapeLikePattern(input) {
   return input.replace(/[%_\\]/g, "\\$&");
 }
-function createObservation(db, memorySessionId, project, type, title, subtitle, text, narrative, facts, concepts, filesRead, filesModified, promptNumber) {
+function isDuplicateObservation(db, contentHash, windowMs = 3e4) {
+  if (!contentHash) return false;
+  const threshold = Date.now() - windowMs;
+  const result = db.query(
+    "SELECT id FROM observations WHERE content_hash = ? AND created_at_epoch > ? LIMIT 1"
+  ).get(contentHash, threshold);
+  return !!result;
+}
+function createObservation(db, memorySessionId, project, type, title, subtitle, text, narrative, facts, concepts, filesRead, filesModified, promptNumber, contentHash = null, discoveryTokens = 0) {
   const now = /* @__PURE__ */ new Date();
   const result = db.run(
-    `INSERT INTO observations 
-     (memory_session_id, project, type, title, subtitle, text, narrative, facts, concepts, files_read, files_modified, prompt_number, created_at, created_at_epoch)
-     VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`,
-    [memorySessionId, project, type, title, subtitle, text, narrative, facts, concepts, filesRead, filesModified, promptNumber, now.toISOString(), now.getTime()]
+    `INSERT INTO observations
+     (memory_session_id, project, type, title, subtitle, text, narrative, facts, concepts, files_read, files_modified, prompt_number, created_at, created_at_epoch, content_hash, discovery_tokens)
+     VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`,
+    [memorySessionId, project, type, title, subtitle, text, narrative, facts, concepts, filesRead, filesModified, promptNumber, now.toISOString(), now.getTime(), contentHash, discoveryTokens]
   );
   return Number(result.lastInsertRowid);
 }
@@ -91,39 +100,42 @@ function consolidateObservations(db, project, options = {}) {
   if (groups.length === 0) return { merged: 0, removed: 0 };
   let totalMerged = 0;
   let totalRemoved = 0;
-  for (const group of groups) {
-    const obsIds = group.ids.split(",").map(Number);
-    const placeholders = obsIds.map(() => "?").join(",");
-    const observations = db.query(
-      `SELECT * FROM observations WHERE id IN (${placeholders}) ORDER BY created_at_epoch DESC`
-    ).all(...obsIds);
-    if (observations.length < minGroupSize) continue;
-    if (options.dryRun) {
-      totalMerged += 1;
-      totalRemoved += observations.length - 1;
-      continue;
-    }
-    const keeper = observations[0];
-    const others = observations.slice(1);
-    const uniqueTexts = /* @__PURE__ */ new Set();
-    if (keeper.text) uniqueTexts.add(keeper.text);
-    for (const obs of others) {
-      if (obs.text && !uniqueTexts.has(obs.text)) {
-        uniqueTexts.add(obs.text);
+  const runConsolidation = db.transaction(() => {
+    for (const group of groups) {
+      const obsIds = group.ids.split(",").map(Number);
+      const placeholders = obsIds.map(() => "?").join(",");
+      const observations = db.query(
+        `SELECT * FROM observations WHERE id IN (${placeholders}) ORDER BY created_at_epoch DESC`
+      ).all(...obsIds);
+      if (observations.length < minGroupSize) continue;
+      if (options.dryRun) {
+        totalMerged += 1;
+        totalRemoved += observations.length - 1;
+        continue;
+      }
+      const keeper = observations[0];
+      const others = observations.slice(1);
+      const uniqueTexts = /* @__PURE__ */ new Set();
+      if (keeper.text) uniqueTexts.add(keeper.text);
+      for (const obs of others) {
+        if (obs.text && !uniqueTexts.has(obs.text)) {
+          uniqueTexts.add(obs.text);
+        }
       }
+      const consolidatedText = Array.from(uniqueTexts).join("\n---\n").substring(0, 1e5);
+      db.run(
+        "UPDATE observations SET text = ?, title = ? WHERE id = ?",
+        [consolidatedText, `[consolidato x${observations.length}] ${keeper.title}`, keeper.id]
+      );
+      const removeIds = others.map((o) => o.id);
+      const removePlaceholders = removeIds.map(() => "?").join(",");
+      db.run(`DELETE FROM observations WHERE id IN (${removePlaceholders})`, removeIds);
+      db.run(`DELETE FROM observation_embeddings WHERE observation_id IN (${removePlaceholders})`, removeIds);
+      totalMerged += 1;
+      totalRemoved += removeIds.length;
     }
-    const consolidatedText = Array.from(uniqueTexts).join("\n---\n").substring(0, 1e5);
-    db.run(
-      "UPDATE observations SET text = ?, title = ? WHERE id = ?",
-      [consolidatedText, `[consolidato x${observations.length}] ${keeper.title}`, keeper.id]
-    );
-    const removeIds = others.map((o) => o.id);
-    const removePlaceholders = removeIds.map(() => "?").join(",");
-    db.run(`DELETE FROM observations WHERE id IN (${removePlaceholders})`, removeIds);
-    db.run(`DELETE FROM observation_embeddings WHERE observation_id IN (${removePlaceholders})`, removeIds);
-    totalMerged += 1;
-    totalRemoved += removeIds.length;
-  }
+  });
+  runConsolidation();
   return { merged: totalMerged, removed: totalRemoved };
 }
 var init_Observations = __esm({
@@ -181,7 +193,7 @@ function searchObservationsFTS(db, query, filters = {}) {
       sql += " AND o.created_at_epoch <= ?";
       params.push(filters.dateEnd);
     }
-    sql += " ORDER BY rank LIMIT ?";
+    sql += ` ORDER BY bm25(observations_fts, ${BM25_WEIGHTS}) LIMIT ?`;
     params.push(limit);
     const stmt = db.query(sql);
     return stmt.all(...params);
@@ -195,7 +207,7 @@ function searchObservationsFTSWithRank(db, query, filters = {}) {
     const safeQuery = sanitizeFTS5Query(query);
     if (!safeQuery) return [];
     let sql = `
-      SELECT o.*, rank as fts5_rank FROM observations o
+      SELECT o.*, bm25(observations_fts, ${BM25_WEIGHTS}) as fts5_rank FROM observations o
       JOIN observations_fts fts ON o.id = fts.rowid
       WHERE observations_fts MATCH ?
     `;
@@ -216,7 +228,7 @@ function searchObservationsFTSWithRank(db, query, filters = {}) {
       sql += " AND o.created_at_epoch <= ?";
       params.push(filters.dateEnd);
     }
-    sql += " ORDER BY rank LIMIT ?";
+    sql += ` ORDER BY bm25(observations_fts, ${BM25_WEIGHTS}) LIMIT ?`;
     params.push(limit);
     const stmt = db.query(sql);
     return stmt.all(...params);
@@ -320,11 +332,23 @@ function getProjectStats(db, project) {
   const sumStmt = db.query("SELECT COUNT(*) as count FROM summaries WHERE project = ?");
   const sesStmt = db.query("SELECT COUNT(*) as count FROM sessions WHERE project = ?");
   const prmStmt = db.query("SELECT COUNT(*) as count FROM prompts WHERE project = ?");
+  const discoveryStmt = db.query(
+    "SELECT COALESCE(SUM(discovery_tokens), 0) as total FROM observations WHERE project = ?"
+  );
+  const discoveryTokens = discoveryStmt.get(project)?.total || 0;
+  const readStmt = db.query(
+    `SELECT COALESCE(SUM(
+      CAST((LENGTH(COALESCE(title, '')) + LENGTH(COALESCE(narrative, ''))) / 4 AS INTEGER)
+    ), 0) as total FROM observations WHERE project = ?`
+  );
+  const readTokens = readStmt.get(project)?.total || 0;
+  const savings = Math.max(0, discoveryTokens - readTokens);
   return {
     observations: obsStmt.get(project)?.count || 0,
     summaries: sumStmt.get(project)?.count || 0,
     sessions: sesStmt.get(project)?.count || 0,
-    prompts: prmStmt.get(project)?.count || 0
+    prompts: prmStmt.get(project)?.count || 0,
+    tokenEconomics: { discoveryTokens, readTokens, savings }
   };
 }
 function getStaleObservations(db, project) {
@@ -366,9 +390,11 @@ function markObservationsStale(db, ids, stale) {
     [stale ? 1 : 0, ...validIds]
   );
 }
+var BM25_WEIGHTS;
 var init_Search = __esm({
   "src/services/sqlite/Search.ts"() {
     "use strict";
+    BM25_WEIGHTS = "10.0, 1.0, 5.0, 3.0";
   }
 });
 
@@ -1057,6 +1083,29 @@ var MigrationRunner = class {
           db.run("CREATE INDEX IF NOT EXISTS idx_checkpoints_project ON checkpoints(project)");
           db.run("CREATE INDEX IF NOT EXISTS idx_checkpoints_epoch ON checkpoints(created_at_epoch)");
         }
+      },
+      {
+        version: 7,
+        up: (db) => {
+          db.run("ALTER TABLE observations ADD COLUMN content_hash TEXT");
+          db.run("CREATE INDEX IF NOT EXISTS idx_observations_hash ON observations(content_hash)");
+        }
+      },
+      {
+        version: 8,
+        up: (db) => {
+          db.run("ALTER TABLE observations ADD COLUMN discovery_tokens INTEGER DEFAULT 0");
+          db.run("ALTER TABLE summaries ADD COLUMN discovery_tokens INTEGER DEFAULT 0");
+        }
+      },
+      {
+        version: 9,
+        up: (db) => {
+          db.run("CREATE INDEX IF NOT EXISTS idx_observations_project_epoch ON observations(project, created_at_epoch DESC)");
+          db.run("CREATE INDEX IF NOT EXISTS idx_observations_project_type ON observations(project, type)");
+          db.run("CREATE INDEX IF NOT EXISTS idx_summaries_project_epoch ON summaries(project, created_at_epoch DESC)");
+          db.run("CREATE INDEX IF NOT EXISTS idx_prompts_project_epoch ON prompts(project, created_at_epoch DESC)");
+        }
       }
     ];
   }
@@ -1311,6 +1360,7 @@ init_Search();
 
 // src/sdk/index.ts
 init_Observations();
+import { createHash } from "crypto";
 init_Search();
 
 // src/services/search/EmbeddingService.ts
@@ -1869,31 +1919,71 @@ var KiroMemorySDK = class {
       logger.debug("SDK", `Embedding generation fallita per obs ${observationId}: ${error}`);
     }
   }
+  /**
+   * Genera content hash SHA256 per deduplicazione basata su contenuto.
+   * Usa (project + type + title + narrative) come tupla di identità semantica.
+   * NON include sessionId perché è unico ad ogni invocazione.
+   */
+  generateContentHash(type, title, narrative) {
+    const payload = `${this.project}|${type}|${title}|${narrative || ""}`;
+    return createHash("sha256").update(payload).digest("hex");
+  }
+  /**
+   * Finestre di deduplicazione per tipo (ms).
+   * Tipi con molte ripetizioni hanno finestre più ampie.
+   */
+  getDeduplicationWindow(type) {
+    switch (type) {
+      case "file-read":
+        return 6e4;
+      // 60s — letture frequenti sugli stessi file
+      case "file-write":
+        return 1e4;
+      // 10s — scritture rapide consecutive
+      case "command":
+        return 3e4;
+      // 30s — standard
+      case "research":
+        return 12e4;
+      // 120s — web search e fetch ripetuti
+      case "delegation":
+        return 6e4;
+      // 60s — delegazioni rapide
+      default:
+        return 3e4;
+    }
+  }
   /**
    * Store a new observation
    */
   async storeObservation(data) {
     this.validateObservationInput(data);
+    const sessionId = "sdk-" + Date.now();
+    const contentHash = this.generateContentHash(data.type, data.title, data.narrative);
+    const dedupWindow = this.getDeduplicationWindow(data.type);
+    if (isDuplicateObservation(this.db.db, contentHash, dedupWindow)) {
+      logger.debug("SDK", `Osservazione duplicata scartata (${data.type}, ${dedupWindow}ms): ${data.title}`);
+      return -1;
+    }
+    const filesRead = data.filesRead || (data.type === "file-read" ? data.files : void 0);
+    const filesModified = data.filesModified || (data.type === "file-write" ? data.files : void 0);
+    const discoveryTokens = Math.ceil(data.content.length / 4);
     const observationId = createObservation(
       this.db.db,
-      "sdk-" + Date.now(),
+      sessionId,
       this.project,
       data.type,
       data.title,
-      null,
-      // subtitle
+      data.subtitle || null,
       data.content,
-      null,
-      // narrative
-      null,
-      // facts
+      data.narrative || null,
+      data.facts || null,
       data.concepts?.join(", ") || null,
-      data.files?.join(", ") || null,
-      // files_read
-      data.files?.join(", ") || null,
-      // files_modified
-      0
-      // prompt_number
+      filesRead?.join(", ") || null,
+      filesModified?.join(", ") || null,
+      0,
+      contentHash,
+      discoveryTokens
     );
     this.generateEmbeddingAsync(observationId, data.title, data.content, data.concepts).catch(() => {
     });
@@ -1936,9 +2026,16 @@ var KiroMemorySDK = class {
           };
       }
     })();
+    const sessionId = "sdk-" + Date.now();
+    const contentHash = this.generateContentHash(data.type, data.title);
+    if (isDuplicateObservation(this.db.db, contentHash)) {
+      logger.debug("SDK", `Knowledge duplicata scartata: ${data.title}`);
+      return -1;
+    }
+    const discoveryTokens = Math.ceil(data.content.length / 4);
     const observationId = createObservation(
       this.db.db,
-      "sdk-" + Date.now(),
+      sessionId,
       data.project || this.project,
       data.knowledgeType,
       // type = knowledgeType
@@ -1952,9 +2049,12 @@ var KiroMemorySDK = class {
       // facts = metadati JSON
       data.concepts?.join(", ") || null,
       data.files?.join(", ") || null,
-      data.files?.join(", ") || null,
-      0
+      null,
+      // filesModified: knowledge non modifica file
+      0,
       // prompt_number
+      contentHash,
+      discoveryTokens
     );
     this.generateEmbeddingAsync(observationId, data.title, data.content, data.concepts).catch(() => {
     });
@@ -1970,11 +2070,11 @@ var KiroMemorySDK = class {
       "sdk-" + Date.now(),
       this.project,
       data.request || null,
-      null,
+      data.investigated || null,
       data.learned || null,
       data.completed || null,
       data.nextSteps || null,
-      null
+      data.notes || null
     );
   }
   /**
@@ -2158,7 +2258,14 @@ var KiroMemorySDK = class {
       }));
     } else {
       const observations = getObservationsByProject(this.db.db, this.project, 30);
-      items = observations.map((obs) => {
+      const knowledgeTypes = new Set(KNOWLEDGE_TYPES);
+      const knowledgeObs = [];
+      const normalObs = [];
+      for (const obs of observations) {
+        if (knowledgeTypes.has(obs.type)) knowledgeObs.push(obs);
+        else normalObs.push(obs);
+      }
+      const scoreObs = (obs) => {
         const signals = {
           semantic: 0,
           fts5: 0,
@@ -2166,7 +2273,6 @@ var KiroMemorySDK = class {
           projectMatch: projectMatchScore(obs.project, this.project)
         };
         const baseScore = computeCompositeScore(signals, CONTEXT_WEIGHTS);
-        const boostedScore = Math.min(1, baseScore * knowledgeTypeBoost(obs.type));
         return {
           id: obs.id,
           title: obs.title,
@@ -2175,17 +2281,23 @@ var KiroMemorySDK = class {
           project: obs.project,
           created_at: obs.created_at,
           created_at_epoch: obs.created_at_epoch,
-          score: boostedScore,
+          score: Math.min(1, baseScore * knowledgeTypeBoost(obs.type)),
           signals
         };
-      });
-      items.sort((a, b) => b.score - a.score);
+      };
+      const scoredKnowledge = knowledgeObs.map(scoreObs).sort((a, b) => b.score - a.score);
+      const scoredNormal = normalObs.map(scoreObs).sort((a, b) => b.score - a.score);
+      items = [...scoredKnowledge, ...scoredNormal];
     }
     let tokensUsed = 0;
+    const budgetItems = [];
     for (const item of items) {
-      tokensUsed += Math.ceil((item.title.length + item.content.length) / 4);
-      if (tokensUsed > tokenBudget) break;
+      const itemTokens = Math.ceil((item.title.length + item.content.length) / 4);
+      if (tokensUsed + itemTokens > tokenBudget) break;
+      tokensUsed += itemTokens;
+      budgetItems.push(item);
     }
+    items = budgetItems;
     return {
       project: this.project,
       items,