@joshuaswarren/openclaw-engram 8.3.82 → 8.3.84

package/dist/index.js

@@ -3655,6 +3655,9 @@ function scoreImportance(content, category, tags = []) {
     keywords
   };
 }
+function rescoreMemoryImportance(memory) {
+  return scoreImportance(memory.content, memory.frontmatter.category, memory.frontmatter.tags ?? []);
+}
 
 // src/qmd.ts
 import { spawn } from "child_process";
@@ -6039,6 +6042,46 @@ ${memory.content}
     await appendFile(this.behaviorSignalsPath, payload, "utf-8");
     return deduped.length;
   }
+  async appendReextractJobs(events) {
+    if (events.length === 0) return 0;
+    await this.ensureDirectories();
+    const filePath = path4.join(this.stateDir, "reextract-jobs.jsonl");
+    const lines = events.map((event) => JSON.stringify(event)).join("\n") + "\n";
+    try {
+      await appendFile(filePath, lines, "utf-8");
+      return events.length;
+    } catch {
+      return 0;
+    }
+  }
+  async readReextractJobs(limit = 200) {
+    const safeLimit = Number.isFinite(limit) ? Math.max(1, Math.min(1e3, Math.floor(limit))) : 200;
+    const filePath = path4.join(this.stateDir, "reextract-jobs.jsonl");
+    try {
+      const raw = await readFile2(filePath, "utf-8");
+      const lines = raw.split("\n").filter((line) => line.trim().length > 0);
+      const parsed = [];
+      for (const line of lines) {
+        try {
+          const record = JSON.parse(line);
+          if (typeof record.memoryId !== "string" || record.memoryId.length === 0 || typeof record.model !== "string" || record.model.length === 0 || typeof record.requestedAt !== "string" || record.requestedAt.length === 0 || record.source !== "cli-migrate") {
+            continue;
+          }
+          parsed.push({
+            memoryId: record.memoryId,
+            model: record.model,
+            requestedAt: record.requestedAt,
+            source: "cli-migrate"
+          });
+        } catch {
+          continue;
+        }
+      }
+      return parsed.slice(-safeLimit);
+    } catch {
+      return [];
+    }
+  }
   async readBehaviorSignals(limit = 200) {
     const cappedLimit = Math.max(0, Math.floor(limit));
     if (cappedLimit === 0) return [];
@@ -12108,6 +12151,58 @@ function safeSlug(s) {
 function ymd(d) {
   return d.toISOString().slice(0, 10);
 }
+var CROSS_SIGNAL_STOPWORDS = /* @__PURE__ */ new Set([
+  "a",
+  "an",
+  "and",
+  "are",
+  "as",
+  "at",
+  "be",
+  "by",
+  "for",
+  "from",
+  "has",
+  "have",
+  "in",
+  "is",
+  "it",
+  "of",
+  "on",
+  "or",
+  "that",
+  "the",
+  "this",
+  "to",
+  "was",
+  "were",
+  "with",
+  "agent",
+  "output",
+  "today",
+  "daily",
+  "notes",
+  "note",
+  "summary"
+]);
+function extractTopicTokens(text, maxTokens = 12) {
+  const seen = /* @__PURE__ */ new Set();
+  const out = [];
+  const tokens = text.toLowerCase().replace(/[^a-z0-9\s]/g, " ").split(/\s+/).map((token) => token.trim()).filter((token) => token.length >= 4).filter((token) => !CROSS_SIGNAL_STOPWORDS.has(token));
+  for (const token of tokens) {
+    if (seen.has(token)) continue;
+    seen.add(token);
+    out.push(token);
+    if (out.length >= maxTokens) break;
+  }
+  return out;
+}
+function stripYamlFrontmatter(text) {
+  if (!text.startsWith("---\n")) return text;
+  const closing = text.indexOf("\n---\n", 4);
+  if (closing === -1) return text;
+  return text.slice(closing + 5);
+}
 var SharedContextManager = class {
   constructor(config) {
     this.config = config;
@@ -12238,7 +12333,7 @@ title: ${opts.title.replace(/\n/g, " ").slice(0, 200)}
             const p = path22.join(dayDir, f);
             const raw = await readFile15(p, "utf-8");
             const title = (raw.match(/^title:\s*(.+)$/m)?.[1] ?? f).trim();
-            outputs.push({ path: p, title });
+            outputs.push({ agent: a.name, path: p, title, raw });
           }
         } catch {
         }
@@ -12260,25 +12355,86 @@ title: ${opts.title.replace(/\n/g, " ").slice(0, 200)}
       }
     } catch {
     }
+    const sources = outputs.map((output) => {
+      const body = stripYamlFrontmatter(output.raw);
+      return {
+        agent: output.agent,
+        path: output.path,
+        title: output.title,
+        topics: extractTopicTokens(`${output.title}
+${body}`)
+      };
+    });
+    const overlapMap = /* @__PURE__ */ new Map();
+    for (const source of sources) {
+      for (const token of source.topics) {
+        const existing = overlapMap.get(token);
+        if (existing) {
+          existing.agents.add(source.agent);
+          existing.sourcePaths.add(source.path);
+        } else {
+          overlapMap.set(token, {
+            agents: /* @__PURE__ */ new Set([source.agent]),
+            sourcePaths: /* @__PURE__ */ new Set([source.path])
+          });
+        }
+      }
+    }
+    const overlaps = [...overlapMap.entries()].map(([token, v]) => ({
+      token,
+      agents: [...v.agents].sort(),
+      sourcePaths: [...v.sourcePaths].sort(),
+      agentCount: v.agents.size
+    })).filter((entry) => entry.agentCount >= 2).sort((a, b) => b.agentCount - a.agentCount || a.token.localeCompare(b.token));
+    const feedbackByDecision = {
+      approved: 0,
+      approved_with_feedback: 0,
+      rejected: 0
+    };
+    for (const entry of feedback) {
+      feedbackByDecision[entry.decision] += 1;
+    }
+    const crossSignalReport = {
+      date,
+      generatedAt: (/* @__PURE__ */ new Date()).toISOString(),
+      sourceCount: sources.length,
+      feedbackCount: feedback.length,
+      feedbackByDecision,
+      sources,
+      overlaps
+    };
+    const crossSignalsPath = path22.join(this.crossSignalsDir, `${date}.json`);
+    await writeFile14(crossSignalsPath, `${JSON.stringify(crossSignalReport, null, 2)}
+`, "utf-8");
+    const overlapBullets = overlaps.length === 0 ? ["- No multi-agent topic overlap detected."] : overlaps.slice(0, 8).map((entry) => `- \`${entry.token}\` (${entry.agentCount} agents: ${entry.agents.join(", ")})`);
     const md = [
       `# Roundtable \u2014 ${date}`,
       "",
       "## Notable Agent Outputs",
-      ...outputs.length === 0 ? ["- (none)"] : outputs.map((o) => `- ${o.title} (${o.path})`),
+      ...sources.length === 0 ? ["- (none)"] : sources.map((o) => `- ${o.title} (${o.path})`),
       "",
       "## Feedback (Approve/Reject)",
       ...feedback.length === 0 ? ["- (none)"] : feedback.map((f) => `- [${f.agent}] ${f.decision}: ${f.reason}`),
       "",
       "## Cross-Signals",
-      "- (baseline) not computed in deterministic mode",
+      `- Source outputs analyzed: ${sources.length}`,
+      `- Feedback entries analyzed: ${feedback.length}`,
+      `- Decision totals: approved=${feedbackByDecision.approved}, approved_with_feedback=${feedbackByDecision.approved_with_feedback}, rejected=${feedbackByDecision.rejected}`,
+      `- Cross-signals file: ${crossSignalsPath}`,
+      ...overlapBullets,
       ""
     ];
     const out = md.join("\n");
     const trimmed = out.length > maxChars ? out.slice(0, maxChars) + "\n\n...(trimmed)\n" : out;
-    const fp = path22.join(this.roundtableDir, `${date}.md`);
-    await writeFile14(fp, trimmed, "utf-8");
-    log.info(`shared-context curated daily roundtable: ${fp}`);
-    return fp;
+    const roundtablePath = path22.join(this.roundtableDir, `${date}.md`);
+    await writeFile14(roundtablePath, trimmed, "utf-8");
+    log.info(`shared-context curated daily roundtable: ${roundtablePath}`);
+    return {
+      date,
+      roundtablePath,
+      crossSignalsPath,
+      overlapCount: overlaps.length
+    };
   }
 };
 
@@ -19790,8 +19946,14 @@ Best for:
           );
         }
         const { date } = params;
-        const fp = await orchestrator.sharedContext.curateDaily({ date });
-        return toolResult(`Wrote: ${fp}`);
+        const result = await orchestrator.sharedContext.curateDaily({ date });
+        return toolResult(
+          [
+            `Roundtable: ${result.roundtablePath}`,
+            `Cross-signals: ${result.crossSignalsPath}`,
+            `Overlap count: ${result.overlapCount}`
+          ].join("\n")
+        );
       }
     },
     { name: "shared_context_curate_daily" }
@@ -22122,6 +22284,187 @@ async function runTierMigrateCliCommand(orchestrator, options = {}) {
     limit: options.limit
   });
 }
+var MIGRATE_LIMIT_CAP = 2e3;
+var REEXTRACT_LIMIT_CAP = 500;
+function clampMigrateLimit(limit, cap, fallback) {
+  if (typeof limit !== "number" || !Number.isFinite(limit)) return fallback;
+  return Math.max(1, Math.min(cap, Math.floor(limit)));
+}
+async function readMigrateCandidateMemories(storage, options) {
+  const merged = /* @__PURE__ */ new Map();
+  const addMany = (items) => {
+    for (const item of items) {
+      if (!item.frontmatter?.id) continue;
+      merged.set(item.path, item);
+    }
+  };
+  addMany(await storage.readAllMemories());
+  if (options.includeArchived) {
+    addMany(await storage.readArchivedMemories());
+  }
+  return [...merged.values()].sort((a, b) => a.path.localeCompare(b.path));
+}
+function sameImportance(a, b) {
+  if (!a && !b) return true;
+  if (!a || !b) return false;
+  if (Math.abs(a.score - b.score) > 1e-6) return false;
+  if (a.level !== b.level) return false;
+  if (a.reasons.join("|") !== b.reasons.join("|")) return false;
+  if (a.keywords.join("|") !== b.keywords.join("|")) return false;
+  return true;
+}
+function sameChunkContent(existing, desired) {
+  if (existing.length !== desired.length) return false;
+  for (let i = 0; i < desired.length; i += 1) {
+    const current = existing[i]?.content?.trim() ?? "";
+    if (current !== desired[i]?.trim()) {
+      return false;
+    }
+  }
+  return true;
+}
+async function runMigrateNormalizeFrontmatterCliCommand(orchestrator, options = {}) {
+  const limit = clampMigrateLimit(options.limit, MIGRATE_LIMIT_CAP, 200);
+  const storage = await orchestrator.getStorage(orchestrator.config.defaultNamespace);
+  const candidates = (await readMigrateCandidateMemories(storage, { includeArchived: true })).slice(0, limit);
+  if (options.write === true) {
+    for (const memory of candidates) {
+      await storage.writeMemoryFrontmatter(memory, {});
+    }
+  }
+  return {
+    action: "normalize-frontmatter",
+    dryRun: options.write !== true,
+    scanned: candidates.length,
+    changed: candidates.length,
+    queued: 0,
+    limit
+  };
+}
+async function runMigrateRescoreImportanceCliCommand(orchestrator, options = {}) {
+  const limit = clampMigrateLimit(options.limit, MIGRATE_LIMIT_CAP, 200);
+  const storage = await orchestrator.getStorage(orchestrator.config.defaultNamespace);
+  const candidates = (await readMigrateCandidateMemories(storage, { includeArchived: true })).slice(0, limit);
+  let changed = 0;
+  for (const memory of candidates) {
+    const nextImportance = rescoreMemoryImportance(memory);
+    if (sameImportance(memory.frontmatter.importance, nextImportance)) continue;
+    changed += 1;
+    if (options.write === true) {
+      await storage.writeMemoryFrontmatter(memory, {
+        importance: nextImportance,
+        updated: (/* @__PURE__ */ new Date()).toISOString()
+      });
+    }
+  }
+  return {
+    action: "rescore-importance",
+    dryRun: options.write !== true,
+    scanned: candidates.length,
+    changed,
+    queued: 0,
+    limit
+  };
+}
+async function runMigrateRechunkCliCommand(orchestrator, options = {}) {
+  const limit = clampMigrateLimit(options.limit, MIGRATE_LIMIT_CAP, 200);
+  const storage = await orchestrator.getStorage(orchestrator.config.defaultNamespace);
+  const candidates = (await readMigrateCandidateMemories(storage, { includeArchived: false })).filter((memory) => memory.frontmatter.parentId === void 0).slice(0, limit);
+  let changed = 0;
+  for (const memory of candidates) {
+    const existing = await storage.getChunksForParent(memory.frontmatter.id);
+    const chunked = chunkContent(memory.content);
+    if (!chunked.chunked) {
+      if (existing.length === 0) continue;
+      changed += 1;
+      if (options.write === true) {
+        for (const stale of existing) {
+          await storage.invalidateMemory(stale.frontmatter.id);
+        }
+      }
+      continue;
+    }
+    const desired = chunked.chunks.map((chunk) => chunk.content);
+    if (sameChunkContent(existing, desired)) continue;
+    changed += 1;
+    if (options.write !== true) continue;
+    const total = chunked.chunks.length;
+    for (const chunk of chunked.chunks) {
+      const existingChunk = existing[chunk.index];
+      if (existingChunk) {
+        await storage.updateMemory(existingChunk.frontmatter.id, chunk.content);
+        await storage.updateMemoryFrontmatter(existingChunk.frontmatter.id, {
+          chunkIndex: chunk.index,
+          chunkTotal: total,
+          updated: (/* @__PURE__ */ new Date()).toISOString()
+        });
+        continue;
+      }
+      await storage.writeChunk(
+        memory.frontmatter.id,
+        chunk.index,
+        total,
+        memory.frontmatter.category,
+        chunk.content,
+        {
+          confidence: memory.frontmatter.confidence,
+          tags: memory.frontmatter.tags,
+          entityRef: memory.frontmatter.entityRef,
+          source: "migration-rechunk",
+          importance: memory.frontmatter.importance,
+          intentGoal: memory.frontmatter.intentGoal,
+          intentActionType: memory.frontmatter.intentActionType,
+          intentEntityTypes: memory.frontmatter.intentEntityTypes,
+          memoryKind: memory.frontmatter.memoryKind
+        }
+      );
+    }
+    for (let idx = total; idx < existing.length; idx += 1) {
+      const stale = existing[idx];
+      if (stale?.frontmatter?.id) {
+        await storage.invalidateMemory(stale.frontmatter.id);
+      }
+    }
+  }
+  return {
+    action: "rechunk",
+    dryRun: options.write !== true,
+    scanned: candidates.length,
+    changed,
+    queued: 0,
+    limit
+  };
+}
+async function runMigrateReextractCliCommand(orchestrator, options) {
+  const model = options.model.trim();
+  if (model.length === 0) {
+    throw new Error("missing --model for migrate reextract");
+  }
+  const limit = clampMigrateLimit(options.limit, REEXTRACT_LIMIT_CAP, 100);
+  const storage = await orchestrator.getStorage(orchestrator.config.defaultNamespace);
+  const candidates = (await readMigrateCandidateMemories(storage, { includeArchived: false })).filter((memory) => memory.frontmatter.parentId === void 0);
+  const selected = candidates.slice(0, limit);
+  let queued = 0;
+  if (options.write === true && selected.length > 0) {
+    queued = await storage.appendReextractJobs(
+      selected.map((memory) => ({
+        memoryId: memory.frontmatter.id,
+        model,
+        requestedAt: (/* @__PURE__ */ new Date()).toISOString(),
+        source: "cli-migrate"
+      }))
+    );
+  }
+  return {
+    action: "reextract",
+    dryRun: options.write !== true,
+    scanned: selected.length,
+    changed: selected.length,
+    queued,
+    limit,
+    model
+  };
+}
 function effectivePolicyValuesForVersion(values, config) {
   const candidate = {
     recencyWeight: values.recencyWeight ?? config.recencyWeight,
@@ -23023,6 +23366,49 @@ function registerCli(api, orchestrator) {
         console.log(JSON.stringify(report, null, 2));
         console.log("OK");
       });
+      const migrateCmd = cmd.command("migrate").description("Run memory migration helpers (dry-run by default)");
+      migrateCmd.command("normalize-frontmatter").description("Normalize memory frontmatter serialization").option("--write", "Apply frontmatter rewrites (default: dry-run)").option("--limit <n>", "Maximum memories to scan", "200").action(async (...args) => {
+        const options = args[0] ?? {};
+        const limitRaw = parseInt(String(options.limit ?? "200"), 10);
+        const report = await runMigrateNormalizeFrontmatterCliCommand(orchestrator, {
+          write: options.write === true,
+          limit: Number.isFinite(limitRaw) ? limitRaw : 200
+        });
+        console.log(JSON.stringify(report, null, 2));
+        console.log("OK");
+      });
+      migrateCmd.command("rescore-importance").description("Recompute memory importance scores using current local heuristics").option("--write", "Apply frontmatter updates (default: dry-run)").option("--limit <n>", "Maximum memories to scan", "200").action(async (...args) => {
+        const options = args[0] ?? {};
+        const limitRaw = parseInt(String(options.limit ?? "200"), 10);
+        const report = await runMigrateRescoreImportanceCliCommand(orchestrator, {
+          write: options.write === true,
+          limit: Number.isFinite(limitRaw) ? limitRaw : 200
+        });
+        console.log(JSON.stringify(report, null, 2));
+        console.log("OK");
+      });
+      migrateCmd.command("rechunk").description("Rebuild chunk files from current chunking heuristics").option("--write", "Apply chunk rewrites (default: dry-run)").option("--limit <n>", "Maximum parent memories to scan", "200").action(async (...args) => {
+        const options = args[0] ?? {};
+        const limitRaw = parseInt(String(options.limit ?? "200"), 10);
+        const report = await runMigrateRechunkCliCommand(orchestrator, {
+          write: options.write === true,
+          limit: Number.isFinite(limitRaw) ? limitRaw : 200
+        });
+        console.log(JSON.stringify(report, null, 2));
+        console.log("OK");
+      });
+      migrateCmd.command("reextract").description("Queue bounded memory re-extraction jobs for an explicit model").option("--model <id>", "Model id used for re-extraction request").option("--write", "Queue re-extraction jobs (default: dry-run)").option("--limit <n>", "Maximum memories to queue", "100").action(async (...args) => {
+        const options = args[0] ?? {};
+        const model = typeof options.model === "string" ? options.model : "";
+        const limitRaw = parseInt(String(options.limit ?? "100"), 10);
+        const report = await runMigrateReextractCliCommand(orchestrator, {
+          model,
+          write: options.write === true,
+          limit: Number.isFinite(limitRaw) ? limitRaw : 100
+        });
+        console.log(JSON.stringify(report, null, 2));
+        console.log("OK");
+      });
       cmd.command("action-audit").description("Show namespace-aware memory action policy outcomes").option("--namespace <name>", "Filter to a single namespace").option("--limit <n>", "Max events to read per namespace", "200").action(async (...args) => {
         const options = args[0] ?? {};
         const limitRaw = parseInt(String(options.limit ?? "200"), 10);