kiro-memory 1.7.1 → 1.8.0

package/plugin/dist/worker-service.js

@@ -1570,19 +1570,28 @@ __export(Observations_exports, {
   deleteObservation: () => deleteObservation,
   getObservationsByProject: () => getObservationsByProject,
   getObservationsBySession: () => getObservationsBySession,
+  isDuplicateObservation: () => isDuplicateObservation,
   searchObservations: () => searchObservations,
   updateLastAccessed: () => updateLastAccessed
 });
 function escapeLikePattern(input) {
   return input.replace(/[%_\\]/g, "\\$&");
 }
-function createObservation(db2, memorySessionId, project, type, title, subtitle, text, narrative, facts, concepts, filesRead, filesModified, promptNumber) {
+function isDuplicateObservation(db2, contentHash, windowMs = 3e4) {
+  if (!contentHash) return false;
+  const threshold = Date.now() - windowMs;
+  const result = db2.query(
+    "SELECT id FROM observations WHERE content_hash = ? AND created_at_epoch > ? LIMIT 1"
+  ).get(contentHash, threshold);
+  return !!result;
+}
+function createObservation(db2, memorySessionId, project, type, title, subtitle, text, narrative, facts, concepts, filesRead, filesModified, promptNumber, contentHash = null, discoveryTokens = 0) {
   const now = /* @__PURE__ */ new Date();
   const result = db2.run(
-    `INSERT INTO observations 
-     (memory_session_id, project, type, title, subtitle, text, narrative, facts, concepts, files_read, files_modified, prompt_number, created_at, created_at_epoch)
-     VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`,
-    [memorySessionId, project, type, title, subtitle, text, narrative, facts, concepts, filesRead, filesModified, promptNumber, now.toISOString(), now.getTime()]
+    `INSERT INTO observations
+     (memory_session_id, project, type, title, subtitle, text, narrative, facts, concepts, files_read, files_modified, prompt_number, created_at, created_at_epoch, content_hash, discovery_tokens)
+     VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`,
+    [memorySessionId, project, type, title, subtitle, text, narrative, facts, concepts, filesRead, filesModified, promptNumber, now.toISOString(), now.getTime(), contentHash, discoveryTokens]
   );
   return Number(result.lastInsertRowid);
 }
@@ -1638,39 +1647,42 @@ function consolidateObservations(db2, project, options = {}) {
   if (groups.length === 0) return { merged: 0, removed: 0 };
   let totalMerged = 0;
   let totalRemoved = 0;
-  for (const group of groups) {
-    const obsIds = group.ids.split(",").map(Number);
-    const placeholders = obsIds.map(() => "?").join(",");
-    const observations = db2.query(
-      `SELECT * FROM observations WHERE id IN (${placeholders}) ORDER BY created_at_epoch DESC`
-    ).all(...obsIds);
-    if (observations.length < minGroupSize) continue;
-    if (options.dryRun) {
-      totalMerged += 1;
-      totalRemoved += observations.length - 1;
-      continue;
-    }
-    const keeper = observations[0];
-    const others = observations.slice(1);
-    const uniqueTexts = /* @__PURE__ */ new Set();
-    if (keeper.text) uniqueTexts.add(keeper.text);
-    for (const obs of others) {
-      if (obs.text && !uniqueTexts.has(obs.text)) {
-        uniqueTexts.add(obs.text);
+  const runConsolidation = db2.transaction(() => {
+    for (const group of groups) {
+      const obsIds = group.ids.split(",").map(Number);
+      const placeholders = obsIds.map(() => "?").join(",");
+      const observations = db2.query(
+        `SELECT * FROM observations WHERE id IN (${placeholders}) ORDER BY created_at_epoch DESC`
+      ).all(...obsIds);
+      if (observations.length < minGroupSize) continue;
+      if (options.dryRun) {
+        totalMerged += 1;
+        totalRemoved += observations.length - 1;
+        continue;
       }
+      const keeper = observations[0];
+      const others = observations.slice(1);
+      const uniqueTexts = /* @__PURE__ */ new Set();
+      if (keeper.text) uniqueTexts.add(keeper.text);
+      for (const obs of others) {
+        if (obs.text && !uniqueTexts.has(obs.text)) {
+          uniqueTexts.add(obs.text);
+        }
+      }
+      const consolidatedText = Array.from(uniqueTexts).join("\n---\n").substring(0, 1e5);
+      db2.run(
+        "UPDATE observations SET text = ?, title = ? WHERE id = ?",
+        [consolidatedText, `[consolidato x${observations.length}] ${keeper.title}`, keeper.id]
+      );
+      const removeIds = others.map((o) => o.id);
+      const removePlaceholders = removeIds.map(() => "?").join(",");
+      db2.run(`DELETE FROM observations WHERE id IN (${removePlaceholders})`, removeIds);
+      db2.run(`DELETE FROM observation_embeddings WHERE observation_id IN (${removePlaceholders})`, removeIds);
+      totalMerged += 1;
+      totalRemoved += removeIds.length;
     }
-    const consolidatedText = Array.from(uniqueTexts).join("\n---\n").substring(0, 1e5);
-    db2.run(
-      "UPDATE observations SET text = ?, title = ? WHERE id = ?",
-      [consolidatedText, `[consolidato x${observations.length}] ${keeper.title}`, keeper.id]
-    );
-    const removeIds = others.map((o) => o.id);
-    const removePlaceholders = removeIds.map(() => "?").join(",");
-    db2.run(`DELETE FROM observations WHERE id IN (${removePlaceholders})`, removeIds);
-    db2.run(`DELETE FROM observation_embeddings WHERE observation_id IN (${removePlaceholders})`, removeIds);
-    totalMerged += 1;
-    totalRemoved += removeIds.length;
-  }
+  });
+  runConsolidation();
   return { merged: totalMerged, removed: totalRemoved };
 }
 var init_Observations = __esm({
@@ -1728,7 +1740,7 @@ function searchObservationsFTS(db2, query, filters = {}) {
       sql += " AND o.created_at_epoch <= ?";
       params.push(filters.dateEnd);
     }
-    sql += " ORDER BY rank LIMIT ?";
+    sql += ` ORDER BY bm25(observations_fts, ${BM25_WEIGHTS}) LIMIT ?`;
     params.push(limit);
     const stmt = db2.query(sql);
     return stmt.all(...params);
@@ -1742,7 +1754,7 @@ function searchObservationsFTSWithRank(db2, query, filters = {}) {
     const safeQuery = sanitizeFTS5Query(query);
     if (!safeQuery) return [];
     let sql = `
-      SELECT o.*, rank as fts5_rank FROM observations o
+      SELECT o.*, bm25(observations_fts, ${BM25_WEIGHTS}) as fts5_rank FROM observations o
       JOIN observations_fts fts ON o.id = fts.rowid
       WHERE observations_fts MATCH ?
     `;
@@ -1763,7 +1775,7 @@ function searchObservationsFTSWithRank(db2, query, filters = {}) {
       sql += " AND o.created_at_epoch <= ?";
       params.push(filters.dateEnd);
     }
-    sql += " ORDER BY rank LIMIT ?";
+    sql += ` ORDER BY bm25(observations_fts, ${BM25_WEIGHTS}) LIMIT ?`;
     params.push(limit);
     const stmt = db2.query(sql);
     return stmt.all(...params);
@@ -1867,11 +1879,23 @@ function getProjectStats(db2, project) {
   const sumStmt = db2.query("SELECT COUNT(*) as count FROM summaries WHERE project = ?");
   const sesStmt = db2.query("SELECT COUNT(*) as count FROM sessions WHERE project = ?");
   const prmStmt = db2.query("SELECT COUNT(*) as count FROM prompts WHERE project = ?");
+  const discoveryStmt = db2.query(
+    "SELECT COALESCE(SUM(discovery_tokens), 0) as total FROM observations WHERE project = ?"
+  );
+  const discoveryTokens = discoveryStmt.get(project)?.total || 0;
+  const readStmt = db2.query(
+    `SELECT COALESCE(SUM(
+      CAST((LENGTH(COALESCE(title, '')) + LENGTH(COALESCE(narrative, ''))) / 4 AS INTEGER)
+    ), 0) as total FROM observations WHERE project = ?`
+  );
+  const readTokens = readStmt.get(project)?.total || 0;
+  const savings = Math.max(0, discoveryTokens - readTokens);
   return {
     observations: obsStmt.get(project)?.count || 0,
     summaries: sumStmt.get(project)?.count || 0,
     sessions: sesStmt.get(project)?.count || 0,
-    prompts: prmStmt.get(project)?.count || 0
+    prompts: prmStmt.get(project)?.count || 0,
+    tokenEconomics: { discoveryTokens, readTokens, savings }
   };
 }
 function getStaleObservations(db2, project) {
@@ -1913,9 +1937,11 @@ function markObservationsStale(db2, ids, stale) {
     [stale ? 1 : 0, ...validIds]
   );
 }
+var BM25_WEIGHTS;
 var init_Search = __esm({
   "src/services/sqlite/Search.ts"() {
     "use strict";
+    BM25_WEIGHTS = "10.0, 1.0, 5.0, 3.0";
   }
 });
 
@@ -3945,6 +3971,29 @@ var MigrationRunner = class {
           db2.run("CREATE INDEX IF NOT EXISTS idx_checkpoints_project ON checkpoints(project)");
           db2.run("CREATE INDEX IF NOT EXISTS idx_checkpoints_epoch ON checkpoints(created_at_epoch)");
         }
+      },
+      {
+        version: 7,
+        up: (db2) => {
+          db2.run("ALTER TABLE observations ADD COLUMN content_hash TEXT");
+          db2.run("CREATE INDEX IF NOT EXISTS idx_observations_hash ON observations(content_hash)");
+        }
+      },
+      {
+        version: 8,
+        up: (db2) => {
+          db2.run("ALTER TABLE observations ADD COLUMN discovery_tokens INTEGER DEFAULT 0");
+          db2.run("ALTER TABLE summaries ADD COLUMN discovery_tokens INTEGER DEFAULT 0");
+        }
+      },
+      {
+        version: 9,
+        up: (db2) => {
+          db2.run("CREATE INDEX IF NOT EXISTS idx_observations_project_epoch ON observations(project, created_at_epoch DESC)");
+          db2.run("CREATE INDEX IF NOT EXISTS idx_observations_project_type ON observations(project, type)");
+          db2.run("CREATE INDEX IF NOT EXISTS idx_summaries_project_epoch ON summaries(project, created_at_epoch DESC)");
+          db2.run("CREATE INDEX IF NOT EXISTS idx_prompts_project_epoch ON prompts(project, created_at_epoch DESC)");
+        }
       }
     ];
   }
@@ -4510,6 +4559,14 @@ function getAnalyticsOverview(db2, project) {
   const knowledgeSql = project ? `SELECT COUNT(*) as count FROM observations WHERE project = ? AND type IN ('constraint', 'decision', 'heuristic', 'rejected')` : `SELECT COUNT(*) as count FROM observations WHERE type IN ('constraint', 'decision', 'heuristic', 'rejected')`;
   const knowledgeStmt = db2.query(knowledgeSql);
   const knowledgeCount = project ? knowledgeStmt.get(project)?.count || 0 : knowledgeStmt.get()?.count || 0;
+  const discoverySql = project ? "SELECT COALESCE(SUM(discovery_tokens), 0) as total FROM observations WHERE project = ?" : "SELECT COALESCE(SUM(discovery_tokens), 0) as total FROM observations";
+  const discoveryStmt = db2.query(discoverySql);
+  const discoveryTokens = project ? discoveryStmt.get(project)?.total || 0 : discoveryStmt.get()?.total || 0;
+  const readSql = project ? `SELECT COALESCE(SUM(CAST((LENGTH(COALESCE(title, '')) + LENGTH(COALESCE(narrative, ''))) / 4 AS INTEGER)), 0) as total FROM observations WHERE project = ?` : `SELECT COALESCE(SUM(CAST((LENGTH(COALESCE(title, '')) + LENGTH(COALESCE(narrative, ''))) / 4 AS INTEGER)), 0) as total FROM observations`;
+  const readStmt = db2.query(readSql);
+  const readTokens = project ? readStmt.get(project)?.total || 0 : readStmt.get()?.total || 0;
+  const savings = Math.max(0, discoveryTokens - readTokens);
+  const reductionPct = discoveryTokens > 0 ? Math.round((1 - readTokens / discoveryTokens) * 100) : 0;
   return {
     observations,
     summaries,
@@ -4518,7 +4575,8 @@ function getAnalyticsOverview(db2, project) {
     observationsToday,
     observationsThisWeek,
     staleCount,
-    knowledgeCount
+    knowledgeCount,
+    tokenEconomics: { discoveryTokens, readTokens, savings, reductionPct }
   };
 }
 
@@ -4951,6 +5009,7 @@ app.post("/api/observations", (req, res) => {
       0
     );
     broadcast("observation-created", { id, project, title });
+    projectsCache.ts = 0;
     generateEmbeddingForObservation(id, title, content, concepts).catch(() => {
     });
     res.json({ id, success: true });
@@ -5301,8 +5360,12 @@ app.get("/api/project-aliases", (_req, res) => {
 app.put("/api/project-aliases/:project", (req, res) => {
   const { project } = req.params;
   const { displayName } = req.body;
-  if (!displayName || typeof displayName !== "string") {
-    res.status(400).json({ error: 'Field "displayName" (string) is required' });
+  if (!isValidProject(project)) {
+    res.status(400).json({ error: "Invalid project name" });
+    return;
+  }
+  if (!displayName || typeof displayName !== "string" || displayName.trim().length === 0 || displayName.length > 100) {
+    res.status(400).json({ error: 'Field "displayName" is required (string, max 100 chars)' });
     return;
   }
   try {
@@ -5319,8 +5382,15 @@ app.put("/api/project-aliases/:project", (req, res) => {
     res.status(500).json({ error: "Failed to update project alias" });
   }
 });
+var projectsCache = { data: [], ts: 0 };
+var PROJECTS_CACHE_TTL = 6e4;
 app.get("/api/projects", (_req, res) => {
   try {
+    const now = Date.now();
+    if (now - projectsCache.ts < PROJECTS_CACHE_TTL && projectsCache.data.length > 0) {
+      res.json(projectsCache.data);
+      return;
+    }
     const stmt = db.db.query(
       `SELECT DISTINCT project FROM (
         SELECT project FROM observations
@@ -5331,7 +5401,8 @@ app.get("/api/projects", (_req, res) => {
       ) ORDER BY project ASC`
     );
     const rows = stmt.all();
-    res.json(rows.map((r) => r.project));
+    projectsCache = { data: rows.map((r) => r.project), ts: now };
+    res.json(projectsCache.data);
   } catch (error) {
     logger.error("WORKER", "Lista progetti fallita", {}, error);
     res.status(500).json({ error: "Failed to list projects" });
@@ -5477,6 +5548,154 @@ app.get("/api/report", (req, res) => {
     res.status(500).json({ error: "Report generation failed" });
   }
 });
+app.post("/api/memory/save", (req, res) => {
+  const { project, title, content, type, concepts } = req.body;
+  if (!isValidProject(project)) {
+    res.status(400).json({ error: 'Invalid or missing "project"' });
+    return;
+  }
+  if (!isValidString(title, 500)) {
+    res.status(400).json({ error: 'Invalid or missing "title" (max 500 chars)' });
+    return;
+  }
+  if (!isValidString(content, 1e5)) {
+    res.status(400).json({ error: 'Invalid or missing "content" (max 100KB)' });
+    return;
+  }
+  const obsType = type || "research";
+  const conceptStr = Array.isArray(concepts) ? concepts.join(", ") : concepts || null;
+  try {
+    const id = createObservation(
+      db.db,
+      "memory-save-" + Date.now(),
+      project,
+      obsType,
+      title,
+      null,
+      // subtitle
+      content,
+      // text
+      content,
+      // narrative
+      null,
+      // facts
+      conceptStr,
+      null,
+      // filesRead
+      null,
+      // filesModified
+      0
+      // promptNumber
+    );
+    broadcast("observation-created", { id, project, title });
+    projectsCache.ts = 0;
+    generateEmbeddingForObservation(id, title, content, Array.isArray(concepts) ? concepts : void 0).catch(() => {
+    });
+    res.json({ id, success: true });
+  } catch (error) {
+    logger.error("WORKER", "Memory save fallito", {}, error);
+    res.status(500).json({ error: "Failed to save memory" });
+  }
+});
+app.post("/api/retention/cleanup", (req, res) => {
+  const { maxAgeDays, dryRun } = req.body || {};
+  const days = parseIntSafe(String(maxAgeDays), 90, 7, 730);
+  const threshold = Date.now() - days * 864e5;
+  try {
+    if (dryRun) {
+      const obsCount = db.db.query("SELECT COUNT(*) as c FROM observations WHERE created_at_epoch < ?").get(threshold).c;
+      const sumCount = db.db.query("SELECT COUNT(*) as c FROM summaries WHERE created_at_epoch < ?").get(threshold).c;
+      const promptCount = db.db.query("SELECT COUNT(*) as c FROM prompts WHERE created_at_epoch < ?").get(threshold).c;
+      res.json({ dryRun: true, maxAgeDays: days, wouldDelete: { observations: obsCount, summaries: sumCount, prompts: promptCount } });
+      return;
+    }
+    const cleanup = db.db.transaction(() => {
+      db.db.run("DELETE FROM observation_embeddings WHERE observation_id IN (SELECT id FROM observations WHERE created_at_epoch < ?)", [threshold]);
+      const obsResult = db.db.run("DELETE FROM observations WHERE created_at_epoch < ?", [threshold]);
+      const sumResult = db.db.run("DELETE FROM summaries WHERE created_at_epoch < ?", [threshold]);
+      const promptResult = db.db.run("DELETE FROM prompts WHERE created_at_epoch < ?", [threshold]);
+      return {
+        observations: obsResult.changes,
+        summaries: sumResult.changes,
+        prompts: promptResult.changes
+      };
+    });
+    const deleted = cleanup();
+    projectsCache.ts = 0;
+    logger.info("WORKER", `Retention cleanup: eliminati ${deleted.observations} obs, ${deleted.summaries} sum, ${deleted.prompts} prompts (> ${days}gg)`);
+    res.json({ success: true, maxAgeDays: days, deleted });
+  } catch (error) {
+    logger.error("WORKER", "Retention cleanup fallito", { maxAgeDays: days }, error);
+    res.status(500).json({ error: "Retention cleanup failed" });
+  }
+});
+app.get("/api/export", (req, res) => {
+  const { project, format: fmt, type, days } = req.query;
+  if (project && !isValidProject(project)) {
+    res.status(400).json({ error: "Invalid project name" });
+    return;
+  }
+  const daysBack = parseIntSafe(days, 30, 1, 365);
+  const threshold = Date.now() - daysBack * 864e5;
+  try {
+    let sql = "SELECT * FROM observations WHERE created_at_epoch > ?";
+    const params = [threshold];
+    if (project) {
+      sql += " AND project = ?";
+      params.push(project);
+    }
+    if (type) {
+      sql += " AND type = ?";
+      params.push(type);
+    }
+    sql += " ORDER BY created_at_epoch DESC LIMIT 1000";
+    const observations = db.db.query(sql).all(...params);
+    let sumSql = "SELECT * FROM summaries WHERE created_at_epoch > ?";
+    const sumParams = [threshold];
+    if (project) {
+      sumSql += " AND project = ?";
+      sumParams.push(project);
+    }
+    sumSql += " ORDER BY created_at_epoch DESC LIMIT 100";
+    const summaries = db.db.query(sumSql).all(...sumParams);
+    if (fmt === "markdown" || fmt === "md") {
+      const lines = [
+        `# Kiro Memory Export`,
+        `> Project: ${project || "All"} | Period: ${daysBack} days | Generated: ${(/* @__PURE__ */ new Date()).toISOString()}`,
+        "",
+        `## Observations (${observations.length})`,
+        ""
+      ];
+      for (const obs of observations) {
+        const date = new Date(obs.created_at_epoch).toISOString().split("T")[0];
+        lines.push(`### [${obs.type}] ${obs.title}`);
+        lines.push(`- **Date**: ${date} | **Project**: ${obs.project} | **ID**: #${obs.id}`);
+        if (obs.narrative) lines.push(`- ${obs.narrative}`);
+        if (obs.concepts) lines.push(`- **Concepts**: ${obs.concepts}`);
+        lines.push("");
+      }
+      lines.push(`## Summaries (${summaries.length})`, "");
+      for (const sum of summaries) {
+        const date = new Date(sum.created_at_epoch).toISOString().split("T")[0];
+        lines.push(`### Session ${sum.session_id} (${date})`);
+        if (sum.request) lines.push(`- **Request**: ${sum.request}`);
+        if (sum.completed) lines.push(`- **Completed**: ${sum.completed}`);
+        if (sum.next_steps) lines.push(`- **Next steps**: ${sum.next_steps}`);
+        lines.push("");
+      }
+      res.type("text/markdown").send(lines.join("\n"));
+    } else {
+      res.json({
+        meta: { project: project || "all", daysBack, exportedAt: (/* @__PURE__ */ new Date()).toISOString() },
+        observations,
+        summaries
+      });
+    }
+  } catch (error) {
+    logger.error("WORKER", "Export fallito", { project, fmt }, error);
+    res.status(500).json({ error: "Export failed" });
+  }
+});
 app.use(express.static(__worker_dirname, {
   index: false,
   maxAge: "1h"