metame-cli 1.4.34 → 1.5.1

package/scripts/memory-nightly-reflect.js

@@ -26,15 +26,17 @@ const REFLECT_LOG_FILE = path.join(METAME_DIR, 'memory_reflect_log.jsonl');
 const MEMORY_DIR = path.join(HOME, '.metame', 'memory');
 const DECISIONS_DIR = path.join(MEMORY_DIR, 'decisions');
 const LESSONS_DIR = path.join(MEMORY_DIR, 'lessons');
+const CAPSULES_DIR = path.join(MEMORY_DIR, 'capsules');
 
 // Hot zone thresholds
 const MIN_SEARCH_COUNT = 3;
 const WINDOW_DAYS = 7;
 const MAX_FACTS = 20;
 const LOCK_TIMEOUT_MS = 10 * 60 * 1000; // 10 minutes
+const EXCLUDED_RELATIONS = ['project_milestone', 'synthesized_insight', 'knowledge_capsule', 'bug_lesson'];
 
 // Ensure output directories exist at startup
-[MEMORY_DIR, DECISIONS_DIR, LESSONS_DIR].forEach(d => fs.mkdirSync(d, { recursive: true }));
+[MEMORY_DIR, DECISIONS_DIR, LESSONS_DIR, CAPSULES_DIR].forEach(d => fs.mkdirSync(d, { recursive: true }));
 
 /**
  * Load callHaiku + buildDistillEnv from deployed path, fallback to scripts dir.
@@ -107,18 +109,19 @@ function writeReflectLog(record) {
  * Returns array of plain objects.
  */
 function queryHotFacts(db) {
+  const relationPlaceholders = EXCLUDED_RELATIONS.map(() => '?').join(', ');
   const stmt = db.prepare(`
-    SELECT entity, relation, value, confidence, search_count, created_at
+    SELECT id, entity, relation, value, confidence, search_count, created_at
     FROM facts
     WHERE search_count >= ${MIN_SEARCH_COUNT}
       AND created_at >= datetime('now', '-${WINDOW_DAYS} days')
       AND superseded_by IS NULL
       AND (conflict_status IS NULL OR conflict_status = 'OK')
-      AND relation != 'project_milestone'
+      AND relation NOT IN (${relationPlaceholders})
     ORDER BY search_count DESC, created_at DESC
     LIMIT ${MAX_FACTS}
   `);
-  return stmt.all();
+  return stmt.all(...EXCLUDED_RELATIONS);
 }
 
 /**
@@ -142,6 +145,101 @@ ${sections}
   fs.writeFileSync(filePath, content, 'utf8');
 }
 
+function sanitizeSlug(input, fallback = 'capsule') {
+  const v = String(input || '')
+    .trim()
+    .toLowerCase()
+    .replace(/[^a-z0-9\u4e00-\u9fff_-]+/g, '-')
+    .replace(/-+/g, '-')
+    .replace(/^-|-$/g, '');
+  if (!v) return fallback;
+  return v.slice(0, 50);
+}
+
+function stripMd(text) {
+  return String(text || '').replace(/[#*_`>\[\]\(\)]/g, ' ').replace(/\s+/g, ' ').trim();
+}
+
+function buildSynthesizedFacts(today, decisions, lessons) {
+  const all = []
+    .concat(Array.isArray(decisions) ? decisions : [])
+    .concat(Array.isArray(lessons) ? lessons : []);
+  const out = [];
+  for (const item of all) {
+    const title = String(item && item.title ? item.title : '').trim();
+    const content = String(item && item.content ? item.content : '').trim();
+    if (!title || !content) continue;
+    const value = stripMd(`${title}: ${content}`).slice(0, 280);
+    if (value.length < 20) continue;
+    out.push({
+      entity: `nightly.reflect.${today}`,
+      relation: 'synthesized_insight',
+      value,
+      confidence: 'high',
+      tags: ['nightly', 'reflection'],
+    });
+  }
+  return out;
+}
+
+function entityPrefix(entity) {
+  const src = String(entity || '').trim();
+  if (!src) return '';
+  const parts = src.split('.').map(s => s.trim()).filter(Boolean);
+  if (parts.length === 0) return '';
+  if (parts.length === 1) return parts[0];
+  return `${parts[0]}.${parts[1]}`;
+}
+
+function collectCapsuleGroups(facts, minGroupSize = 3) {
+  const groups = new Map();
+  for (const fact of Array.isArray(facts) ? facts : []) {
+    const prefix = entityPrefix(fact && fact.entity);
+    if (!prefix) continue;
+    if (!groups.has(prefix)) groups.set(prefix, []);
+    groups.get(prefix).push(fact);
+  }
+  return [...groups.entries()]
+    .map(([prefix, items]) => ({ prefix, items }))
+    .filter(g => g.items.length >= minGroupSize)
+    .sort((a, b) => b.items.length - a.items.length);
+}
+
+function parseJsonFromLlm(raw) {
+  const text = String(raw || '');
+  const cleaned = text.replace(/```json\n?/g, '').replace(/```\n?/g, '').trim();
+  if (!cleaned) return null;
+  try { return JSON.parse(cleaned); } catch { return null; }
+}
+
+function writeCapsuleFile(filePath, capsule, facts, today, prefix) {
+  const related = Array.isArray(capsule.related_concepts) ? capsule.related_concepts.slice(0, 8) : [];
+  const supporting = Array.isArray(capsule.supporting_facts) ? capsule.supporting_facts.slice(0, 8) : [];
+  const content = `---
+date: ${today}
+source: nightly-reflect
+type: knowledge-capsule
+entity_prefix: ${prefix}
+facts_analyzed: ${Array.isArray(facts) ? facts.length : 0}
+---
+
+# ${capsule.title}
+
+## 核心结论
+${capsule.core_conclusion}
+
+## 适用场景
+${capsule.applicable_scenarios}
+
+## 关联概念
+${related.length > 0 ? related.map(x => `- ${x}`).join('\n') : '- (none)'}
+
+## 支撑事实
+${supporting.length > 0 ? supporting.map(x => `- ${x}`).join('\n') : '- (none)'}
+`;
+  fs.writeFileSync(filePath, content, 'utf8');
+}
+
 /**
  * Main nightly reflect run.
  */
@@ -238,12 +336,9 @@ Rules:
     }
 
     // Parse Haiku response
-    let parsed;
-    try {
-      const cleaned = raw.replace(/```json\n?/g, '').replace(/```\n?/g, '').trim();
-      parsed = JSON.parse(cleaned);
-    } catch (e) {
-      console.log(`[NIGHTLY-REFLECT] Failed to parse Haiku output: ${e.message}`);
+    const parsed = parseJsonFromLlm(raw);
+    if (!parsed || typeof parsed !== 'object') {
+      console.log('[NIGHTLY-REFLECT] Failed to parse Haiku output.');
       writeReflectLog({ status: 'error', reason: 'parse_failed', facts_found: hotFacts.length });
       return;
     }
@@ -265,12 +360,92 @@ Rules:
       console.log(`[NIGHTLY-REFLECT] Lessons written: ${lessonFile}`);
     }
 
+    let synthesizedSaved = 0;
+    let capsulesWritten = 0;
+    let capsuleFactsSaved = 0;
+    let memory = null;
+    try {
+      try { memory = require('./memory'); } catch { /* optional */ }
+
+      // 3B: write distilled insights back into memory.db for closed-loop retrieval.
+      if (memory && typeof memory.saveFacts === 'function') {
+        const synthesizedFacts = buildSynthesizedFacts(today, decisions, lessons);
+        if (synthesizedFacts.length > 0) {
+          const writeRes = memory.saveFacts(`nightly-reflect-${today}`, '*', synthesizedFacts, { scope: '*' });
+          synthesizedSaved = Number(writeRes && writeRes.saved) || 0;
+        }
+      }
+
+      // 3C: knowledge capsule aggregation by entity prefix.
+      const capsuleGroups = collectCapsuleGroups(hotFacts, 3).slice(0, 3);
+      for (const group of capsuleGroups) {
+        const groupFacts = group.items.map(f => ({
+          entity: f.entity,
+          relation: f.relation,
+          value: f.value,
+          search_count: f.search_count,
+        }));
+        const capsulePrompt = `你是知识胶囊生成器。请将同一主题下的事实聚合成结构化胶囊。
+
+entity_prefix: ${group.prefix}
+facts(json): ${JSON.stringify(groupFacts, null, 2).slice(0, 5000)}
+
+输出 JSON：
+{
+  "title":"标题",
+  "core_conclusion":"一句核心结论",
+  "applicable_scenarios":"适用场景（1-2句）",
+  "related_concepts":["概念1","概念2"],
+  "supporting_facts":["支撑点1","支撑点2"]
+}
+
+规则：
+- 只基于输入事实，不虚构
+- 每个字段简洁具体
+- 仅输出 JSON`;
+
+        let capsule = null;
+        try {
+          const rawCapsule = await Promise.race([
+            callHaiku(capsulePrompt, distillEnv, 60000),
+            new Promise((_, reject) => setTimeout(() => reject(new Error('timeout')), 65000)),
+          ]);
+          capsule = parseJsonFromLlm(rawCapsule);
+        } catch { /* non-fatal */ }
+        if (!capsule || !capsule.title || !capsule.core_conclusion || !capsule.applicable_scenarios) continue;
+
+        const capsuleSlug = sanitizeSlug(group.prefix.replace(/\./g, '-'), 'capsule');
+        const capsuleFile = path.join(CAPSULES_DIR, `${capsuleSlug}-${today}.md`);
+        writeCapsuleFile(capsuleFile, capsule, group.items, today, group.prefix);
+        capsulesWritten++;
+
+        if (memory && typeof memory.saveFacts === 'function') {
+          const capsuleValue = stripMd(`${capsule.title}: ${capsule.core_conclusion}`).slice(0, 280);
+          if (capsuleValue.length >= 20) {
+            const saveCapsule = memory.saveFacts(`capsule-${today}-${capsuleSlug}`, '*', [{
+              entity: `capsule.${group.prefix.replace(/\./g, '_')}`,
+              relation: 'knowledge_capsule',
+              value: capsuleValue,
+              confidence: 'high',
+              tags: ['capsule'],
+            }], { scope: '*' });
+            capsuleFactsSaved += Number(saveCapsule && saveCapsule.saved) || 0;
+          }
+        }
+      }
+    } finally {
+      try { if (memory && typeof memory.close === 'function') memory.close(); } catch { /* non-fatal */ }
+    }
+
     // Write audit log
     writeReflectLog({
       status: 'success',
       facts_analyzed: hotFacts.length,
       decisions_written: decisions.length,
       lessons_written: lessons.length,
+      synthesized_insights_saved: synthesizedSaved,
+      capsules_written: capsulesWritten,
+      capsule_facts_saved: capsuleFactsSaved,
       decision_file: decisions.length > 0 ? decisionFile : null,
       lesson_file: lessons.length > 0 ? lessonFile : null,
     });
@@ -290,10 +465,23 @@ Rules:
 if (require.main === module) {
   run().then(() => {
     console.log('✅ nightly-reflect complete');
+    // Report estimated token usage for daemon budget tracking
+    // ~5k tokens per reflection + capsule generation
+    console.log('__TOKENS__:5000');
   }).catch(e => {
     console.error(`[NIGHTLY-REFLECT] Fatal: ${e.message}`);
     process.exit(1);
   });
 }
 
-module.exports = { run };
+module.exports = {
+  run,
+  _private: {
+    queryHotFacts,
+    buildSynthesizedFacts,
+    collectCapsuleGroups,
+    entityPrefix,
+    parseJsonFromLlm,
+    EXCLUDED_RELATIONS,
+  },
+};

package/scripts/memory.js

@@ -122,6 +122,17 @@ function getDb() {
     )
   `);
 
+  // Optional concept label side-table (non-invasive, no ALTER on facts schema)
+  _db.exec(`
+    CREATE TABLE IF NOT EXISTS fact_labels (
+      fact_id     TEXT NOT NULL REFERENCES facts(id) ON DELETE CASCADE,
+      label       TEXT NOT NULL,
+      domain      TEXT,
+      created_at  TEXT NOT NULL DEFAULT CURRENT_TIMESTAMP,
+      PRIMARY KEY (fact_id, label)
+    )
+  `);
+
   // FTS5 index for facts (separate from sessions_fts, zero compatibility risk)
   try {
     _db.exec(`
@@ -159,6 +170,8 @@ function getDb() {
   try { _db.exec('CREATE INDEX IF NOT EXISTS idx_facts_entity  ON facts(entity)'); } catch {}
   try { _db.exec('CREATE INDEX IF NOT EXISTS idx_facts_entity_relation ON facts(entity, relation)'); } catch {}
   try { _db.exec('CREATE INDEX IF NOT EXISTS idx_facts_project ON facts(project)'); } catch {}
+  try { _db.exec('CREATE INDEX IF NOT EXISTS idx_fact_labels_label ON fact_labels(label)'); } catch {}
+  try { _db.exec('CREATE INDEX IF NOT EXISTS idx_fact_labels_domain ON fact_labels(domain)'); } catch {}
 
   // Backward-compatible migration for old DBs without `scope`
   try { _db.exec('ALTER TABLE facts ADD COLUMN scope TEXT DEFAULT NULL'); } catch {}
@@ -369,7 +382,43 @@ function saveFacts(sessionId, project, facts, { scope = null } = {}) {
 
   if (conflicts > 0) log('WARN', `[MEMORY] ${conflicts} conflict(s) detected`);
 
-  return { saved, skipped, superseded, conflicts };
+  return { saved, skipped, superseded, conflicts, savedFacts };
+}
+
+/**
+ * Save concept labels for facts (side-table).
+ *
+ * @param {Array<{fact_id:string,label:string,domain?:string}>} rows
+ * @returns {{ saved: number, skipped: number }}
+ */
+function saveFactLabels(rows) {
+  if (!Array.isArray(rows) || rows.length === 0) return { saved: 0, skipped: 0 };
+  const db = getDb();
+  const upsert = db.prepare(`
+    INSERT INTO fact_labels (fact_id, label, domain)
+    VALUES (?, ?, ?)
+    ON CONFLICT(fact_id, label) DO UPDATE SET
+      domain = COALESCE(excluded.domain, fact_labels.domain)
+  `);
+
+  let saved = 0;
+  let skipped = 0;
+  for (const row of rows) {
+    const factId = String(row && row.fact_id ? row.fact_id : '').trim();
+    const label = String(row && row.label ? row.label : '').trim();
+    const domainRaw = row && row.domain != null ? String(row.domain).trim() : '';
+    const domain = domainRaw || null;
+    if (!factId || !label) { skipped++; continue; }
+    if (label.length > 60) { skipped++; continue; }
+    if (domain && domain.length > 60) { skipped++; continue; }
+    try {
+      upsert.run(factId, label, domain);
+      saved++;
+    } catch {
+      skipped++;
+    }
+  }
+  return { saved, skipped };
 }
 
 /**
@@ -613,12 +662,46 @@ function searchFacts(query, { limit = 5, project = null, scope = null } = {}) {
     }
     const ftsResults = db.prepare(sql).all(...params);
     if (ftsResults.length > 0) {
+      // Supplement with fact_labels matches (concepts written by memory-extract).
+      const ftsIds = new Set(ftsResults.map(r => r.id));
+      const remaining = limit - ftsResults.length;
+      if (remaining > 0) {
+        try {
+          const labelLike = '%' + query.trim() + '%';
+          let labelSql = `
+            SELECT DISTINCT f.id, f.entity, f.relation, f.value, f.confidence, f.project, f.scope, f.tags, f.created_at
+            FROM fact_labels fl JOIN facts f ON f.id = fl.fact_id
+            WHERE fl.label LIKE ?
+              AND f.superseded_by IS NULL
+              AND (f.conflict_status IS NULL OR f.conflict_status NOT IN ('ARCHIVED', 'CONFLICT'))`;
+          const labelParams = [labelLike];
+          if (scope && project) {
+            labelSql += ` AND ((f.scope = ? OR f.scope = '*') OR (f.scope IS NULL AND (f.project = ? OR f.project = '*')))`;
+            labelParams.push(scope, project);
+          } else if (scope) {
+            labelSql += ` AND (f.scope = ? OR f.scope = '*')`;
+            labelParams.push(scope);
+          } else if (project) {
+            labelSql += ` AND (f.project = ? OR f.project = '*')`;
+            labelParams.push(project);
+          }
+          labelSql += ` LIMIT ?`;
+          labelParams.push(remaining + ftsResults.length);
+          const labelRows = db.prepare(labelSql).all(...labelParams);
+          for (const row of labelRows) {
+            if (!ftsIds.has(row.id) && ftsResults.length < limit) {
+              ftsIds.add(row.id);
+              ftsResults.push(row);
+            }
+          }
+        } catch { /* fact_labels table may not exist yet */ }
+      }
       _trackSearch(ftsResults.map(r => r.id));
       return ftsResults;
     }
   } catch { /* FTS error, fall through */ }
 
-  // LIKE fallback
+  // LIKE fallback (also check fact_labels)
   const like = '%' + query.trim() + '%';
   const likeSql = scope && project
     ? `SELECT id, entity, relation, value, confidence, project, scope, tags, created_at
@@ -651,6 +734,39 @@ function searchFacts(query, { limit = 5, project = null, scope = null } = {}) {
       : project
         ? db.prepare(likeSql).all(like, like, like, project, limit)
         : db.prepare(likeSql).all(like, like, like, limit);
+  // Supplement LIKE results with fact_labels matches.
+  if (likeResults.length < limit) {
+    try {
+      const labelLike = '%' + query.trim() + '%';
+      const likeIds = new Set(likeResults.map(r => r.id));
+      let labelSql2 = `
+        SELECT DISTINCT f.id, f.entity, f.relation, f.value, f.confidence, f.project, f.scope, f.tags, f.created_at
+        FROM fact_labels fl JOIN facts f ON f.id = fl.fact_id
+        WHERE fl.label LIKE ?
+          AND f.superseded_by IS NULL
+          AND (f.conflict_status IS NULL OR f.conflict_status NOT IN ('ARCHIVED', 'CONFLICT'))`;
+      const labelParams2 = [labelLike];
+      if (scope && project) {
+        labelSql2 += ` AND ((f.scope = ? OR f.scope = '*') OR (f.scope IS NULL AND (f.project = ? OR f.project = '*')))`;
+        labelParams2.push(scope, project);
+      } else if (scope) {
+        labelSql2 += ` AND (f.scope = ? OR f.scope = '*')`;
+        labelParams2.push(scope);
+      } else if (project) {
+        labelSql2 += ` AND (f.project = ? OR f.project = '*')`;
+        labelParams2.push(project);
+      }
+      labelSql2 += ` LIMIT ?`;
+      labelParams2.push(limit);
+      const labelRows = db.prepare(labelSql2).all(...labelParams2);
+      for (const row of labelRows) {
+        if (!likeIds.has(row.id) && likeResults.length < limit) {
+          likeIds.add(row.id);
+          likeResults.push(row);
+        }
+      }
+    } catch { /* fact_labels table may not exist yet */ }
+  }
   if (likeResults.length > 0) _trackSearch(likeResults.map(r => r.id));
   return likeResults;
 }
@@ -839,4 +955,19 @@ function forceClose() {
   if (_db) { _db.close(); _db = null; }
 }
 
-module.exports = { saveSession, saveFacts, searchFacts, searchFactsAsync, searchSessions, recentSessions, getSession, stats, acquire, release, close, forceClose, DB_PATH };
+module.exports = {
+  saveSession,
+  saveFacts,
+  saveFactLabels,
+  searchFacts,
+  searchFactsAsync,
+  searchSessions,
+  recentSessions,
+  getSession,
+  stats,
+  acquire,
+  release,
+  close,
+  forceClose,
+  DB_PATH,
+};