@agentmemory/agentmemory 0.7.0

package/src/functions/consolidate.ts

@@ -0,0 +1,212 @@
+import type { ISdk } from "iii-sdk";
+import { getContext } from "iii-sdk";
+import type {
+  CompressedObservation,
+  Memory,
+  Session,
+  MemoryProvider,
+} from "../types.js";
+import { KV, generateId } from "../state/schema.js";
+import { StateKV } from "../state/kv.js";
+
+const CONSOLIDATION_SYSTEM = `You are a memory consolidation engine. Given a set of related observations from coding sessions, synthesize them into a single long-term memory.
+
+Output XML:
+<memory>
+  <type>pattern|preference|architecture|bug|workflow|fact</type>
+  <title>Concise memory title (max 80 chars)</title>
+  <content>2-4 sentence description of the learned insight</content>
+  <concepts>
+    <concept>key term</concept>
+  </concepts>
+  <files>
+    <file>relevant/file/path</file>
+  </files>
+  <strength>1-10 how confident/important this memory is</strength>
+</memory>`;
+
+import { getXmlTag, getXmlChildren } from "../prompts/xml.js";
+
+function parseMemoryXml(
+  xml: string,
+  sessionIds: string[],
+): Omit<Memory, "id" | "createdAt" | "updatedAt"> | null {
+  const type = getXmlTag(xml, "type");
+  const title = getXmlTag(xml, "title");
+  const content = getXmlTag(xml, "content");
+  if (!type || !title || !content) return null;
+
+  const validTypes = new Set([
+    "pattern",
+    "preference",
+    "architecture",
+    "bug",
+    "workflow",
+    "fact",
+  ]);
+
+  return {
+    type: (validTypes.has(type) ? type : "fact") as Memory["type"],
+    title,
+    content,
+    concepts: getXmlChildren(xml, "concepts", "concept"),
+    files: getXmlChildren(xml, "files", "file"),
+    sessionIds,
+    strength: Math.max(
+      1,
+      Math.min(10, parseInt(getXmlTag(xml, "strength") || "5", 10) || 5),
+    ),
+    version: 1,
+    isLatest: true,
+  };
+}
+
+export function registerConsolidateFunction(
+  sdk: ISdk,
+  kv: StateKV,
+  provider: MemoryProvider,
+): void {
+  sdk.registerFunction(
+    { id: "mem::consolidate" },
+    async (data: { project?: string; minObservations?: number }) => {
+      const ctx = getContext();
+      const minObs = data.minObservations ?? 10;
+
+      const sessions = await kv.list<Session>(KV.sessions);
+      const filtered = data.project
+        ? sessions.filter((s) => s.project === data.project)
+        : sessions;
+
+      const allObs: Array<CompressedObservation & { sid: string }> = [];
+      const obsPerSession: CompressedObservation[][] = [];
+      for (let batch = 0; batch < filtered.length; batch += 10) {
+        const chunk = filtered.slice(batch, batch + 10);
+        const results = await Promise.all(
+          chunk.map((s) =>
+            kv
+              .list<CompressedObservation>(KV.observations(s.id))
+              .catch(() => [] as CompressedObservation[]),
+          ),
+        );
+        obsPerSession.push(...results);
+      }
+      for (let i = 0; i < filtered.length; i++) {
+        for (const obs of obsPerSession[i]) {
+          if (obs.title && obs.importance >= 5) {
+            allObs.push({ ...obs, sid: filtered[i].id });
+          }
+        }
+      }
+
+      if (allObs.length < minObs) {
+        return { consolidated: 0, reason: "insufficient_observations" };
+      }
+
+      const conceptGroups = new Map<string, typeof allObs>();
+      for (const obs of allObs) {
+        for (const concept of obs.concepts) {
+          const key = concept.toLowerCase();
+          if (!conceptGroups.has(key)) conceptGroups.set(key, []);
+          conceptGroups.get(key)!.push(obs);
+        }
+      }
+
+      let consolidated = 0;
+      const existingMemories = await kv.list<Memory>(KV.memories);
+      const existingTitles = new Set(
+        existingMemories.map((m) => m.title.toLowerCase()),
+      );
+
+      const MAX_LLM_CALLS = 10;
+      let llmCallCount = 0;
+
+      const sortedGroups = [...conceptGroups.entries()]
+        .filter(([, g]) => g.length >= 3)
+        .sort((a, b) => b[1].length - a[1].length);
+
+      for (const [concept, obsGroup] of sortedGroups) {
+        if (llmCallCount >= MAX_LLM_CALLS) break;
+
+        const top = obsGroup
+          .sort((a, b) => b.importance - a.importance)
+          .slice(0, 8);
+        const sessionIds = [...new Set(top.map((o) => o.sid))];
+
+        const prompt = top
+          .map(
+            (o) =>
+              `[${o.type}] ${o.title}\n${o.narrative}\nFiles: ${o.files.join(", ")}\nImportance: ${o.importance}`,
+          )
+          .join("\n\n");
+
+        try {
+          const response = await Promise.race([
+            provider.compress(
+              CONSOLIDATION_SYSTEM,
+              `Concept: "${concept}"\n\nObservations:\n${prompt}`,
+            ),
+            new Promise<never>((_, reject) =>
+              setTimeout(() => reject(new Error("compress timeout")), 30_000),
+            ),
+          ]);
+          llmCallCount++;
+          const parsed = parseMemoryXml(response, sessionIds);
+          if (!parsed) continue;
+
+          const existingMatch = existingMemories.find(
+            (m) => m.title.toLowerCase() === parsed.title.toLowerCase(),
+          );
+
+          const now = new Date().toISOString();
+          const obsIds = [...new Set(top.map((o) => o.id))];
+          if (existingMatch) {
+            existingMatch.isLatest = false;
+            await kv.set(KV.memories, existingMatch.id, existingMatch);
+
+            const evolved: Memory = {
+              id: generateId("mem"),
+              createdAt: now,
+              updatedAt: now,
+              ...parsed,
+              version: (existingMatch.version || 1) + 1,
+              parentId: existingMatch.id,
+              supersedes: [
+                existingMatch.id,
+                ...(existingMatch.supersedes || []),
+              ],
+              sourceObservationIds: obsIds,
+              isLatest: true,
+            };
+            await kv.set(KV.memories, evolved.id, evolved);
+            existingTitles.add(evolved.title.toLowerCase());
+            consolidated++;
+          } else {
+            const memory: Memory = {
+              id: generateId("mem"),
+              createdAt: now,
+              updatedAt: now,
+              ...parsed,
+              sourceObservationIds: obsIds,
+              version: 1,
+              isLatest: true,
+            };
+            await kv.set(KV.memories, memory.id, memory);
+            existingTitles.add(memory.title.toLowerCase());
+            consolidated++;
+          }
+        } catch (err) {
+          ctx.logger.warn("Consolidation failed for concept", {
+            concept,
+            error: err instanceof Error ? err.message : String(err),
+          });
+        }
+      }
+
+      ctx.logger.info("Consolidation complete", {
+        consolidated,
+        totalObs: allObs.length,
+      });
+      return { consolidated, totalObservations: allObs.length };
+    },
+  );
+}

package/src/functions/consolidation-pipeline.ts

@@ -0,0 +1,258 @@
+import type { ISdk } from "iii-sdk";
+import { getContext } from "iii-sdk";
+import type {
+  SemanticMemory,
+  ProceduralMemory,
+  SessionSummary,
+  Memory,
+  MemoryProvider,
+} from "../types.js";
+import { KV, generateId } from "../state/schema.js";
+import type { StateKV } from "../state/kv.js";
+import {
+  SEMANTIC_MERGE_SYSTEM,
+  buildSemanticMergePrompt,
+  PROCEDURAL_EXTRACTION_SYSTEM,
+  buildProceduralExtractionPrompt,
+} from "../prompts/consolidation.js";
+import { recordAudit } from "./audit.js";
+import { getConsolidationDecayDays, isConsolidationEnabled } from "../config.js";
+
+function applyDecay(
+  items: Array<{
+    strength: number;
+    lastAccessedAt?: string;
+    updatedAt: string;
+  }>,
+  decayDays: number,
+): void {
+  if (decayDays <= 0 || !Number.isFinite(decayDays)) return;
+  const now = Date.now();
+  for (const item of items) {
+    const lastAccess = item.lastAccessedAt || item.updatedAt;
+    const daysSince =
+      (now - new Date(lastAccess).getTime()) / (1000 * 60 * 60 * 24);
+    if (daysSince > decayDays) {
+      const decayPeriods = Math.floor(daysSince / decayDays);
+      item.strength = Math.max(
+        0.1,
+        item.strength * Math.pow(0.9, decayPeriods),
+      );
+    }
+  }
+}
+
+export function registerConsolidationPipelineFunction(
+  sdk: ISdk,
+  kv: StateKV,
+  provider: MemoryProvider,
+): void {
+  sdk.registerFunction(
+    { id: "mem::consolidate-pipeline" },
+    async (data?: { tier?: string; force?: boolean }) => {
+      if (!data?.force && !isConsolidationEnabled()) {
+        return { success: false, skipped: true, reason: "CONSOLIDATION_ENABLED is not set to true" };
+      }
+      const ctx = getContext();
+      const tier = data?.tier || "all";
+      const decayDays = getConsolidationDecayDays();
+      const results: Record<string, unknown> = {};
+
+      if (tier === "all" || tier === "semantic") {
+        const summaries = await kv.list<SessionSummary>(KV.summaries);
+        const existingSemantic = await kv.list<SemanticMemory>(KV.semantic);
+
+        if (summaries.length >= 5) {
+          const recentSummaries = summaries
+            .sort(
+              (a, b) =>
+                new Date(b.createdAt).getTime() -
+                new Date(a.createdAt).getTime(),
+            )
+            .slice(0, 20);
+
+          const prompt = buildSemanticMergePrompt(
+            recentSummaries.map((s) => ({
+              title: s.title,
+              narrative: s.narrative,
+              concepts: s.concepts,
+            })),
+          );
+
+          try {
+            const response = await provider.summarize(
+              SEMANTIC_MERGE_SYSTEM,
+              prompt,
+            );
+
+            const factRegex = /<fact\s+confidence="([^"]+)">([^<]+)<\/fact>/g;
+            let match;
+            let newFacts = 0;
+            const now = new Date().toISOString();
+
+            while ((match = factRegex.exec(response)) !== null) {
+              const parsedConf = parseFloat(match[1]);
+              const confidence = Number.isNaN(parsedConf) ? 0.5 : parsedConf;
+              const fact = match[2].trim();
+
+              const existing = existingSemantic.find(
+                (s) => s.fact.toLowerCase() === fact.toLowerCase(),
+              );
+              if (existing) {
+                existing.accessCount++;
+                existing.lastAccessedAt = now;
+                existing.updatedAt = now;
+                existing.confidence = Math.max(existing.confidence, confidence);
+                await kv.set(KV.semantic, existing.id, existing);
+              } else {
+                const sem: SemanticMemory = {
+                  id: generateId("sem"),
+                  fact,
+                  confidence,
+                  sourceSessionIds: recentSummaries.map((s) => s.sessionId),
+                  sourceMemoryIds: [],
+                  accessCount: 1,
+                  lastAccessedAt: now,
+                  strength: confidence,
+                  createdAt: now,
+                  updatedAt: now,
+                };
+                await kv.set(KV.semantic, sem.id, sem);
+                newFacts++;
+              }
+            }
+            results.semantic = { newFacts, totalSummaries: summaries.length };
+          } catch (err) {
+            const msg = err instanceof Error ? err.message : String(err);
+            ctx.logger.error("Semantic consolidation failed", { error: msg });
+            results.semantic = { error: msg };
+          }
+        } else {
+          results.semantic = {
+            skipped: true,
+            reason: "fewer than 5 summaries",
+          };
+        }
+      }
+
+      if (tier === "all" || tier === "procedural") {
+        const memories = await kv.list<Memory>(KV.memories);
+        const patterns = memories
+          .filter((m) => m.isLatest && m.type === "pattern")
+          .map((m) => ({
+            content: m.content,
+            frequency: m.sessionIds.length || 1,
+          }))
+          .filter((p) => p.frequency >= 2);
+
+        if (patterns.length >= 2) {
+          const prompt = buildProceduralExtractionPrompt(patterns);
+
+          try {
+            const response = await provider.summarize(
+              PROCEDURAL_EXTRACTION_SYSTEM,
+              prompt,
+            );
+
+            const procRegex =
+              /<procedure\s+name="([^"]+)"\s+trigger="([^"]+)">([\s\S]*?)<\/procedure>/g;
+            let match;
+            let newProcs = 0;
+            const now = new Date().toISOString();
+            const existingProcs = await kv.list<ProceduralMemory>(
+              KV.procedural,
+            );
+
+            while ((match = procRegex.exec(response)) !== null) {
+              const name = match[1];
+              const trigger = match[2];
+              const stepsBlock = match[3];
+              const steps: string[] = [];
+
+              const stepRegex = /<step>([^<]+)<\/step>/g;
+              let stepMatch;
+              while ((stepMatch = stepRegex.exec(stepsBlock)) !== null) {
+                steps.push(stepMatch[1].trim());
+              }
+
+              const existing = existingProcs.find(
+                (p) => p.name.toLowerCase() === name.toLowerCase(),
+              );
+              if (existing) {
+                existing.frequency++;
+                existing.updatedAt = now;
+                existing.strength = Math.min(1, existing.strength + 0.1);
+                await kv.set(KV.procedural, existing.id, existing);
+              } else {
+                const proc: ProceduralMemory = {
+                  id: generateId("proc"),
+                  name,
+                  steps,
+                  triggerCondition: trigger,
+                  frequency: 1,
+                  sourceSessionIds: [],
+                  strength: 0.5,
+                  createdAt: now,
+                  updatedAt: now,
+                };
+                await kv.set(KV.procedural, proc.id, proc);
+                newProcs++;
+              }
+            }
+            results.procedural = {
+              newProcedures: newProcs,
+              patternsAnalyzed: patterns.length,
+            };
+          } catch (err) {
+            const msg = err instanceof Error ? err.message : String(err);
+            ctx.logger.error("Procedural extraction failed", { error: msg });
+            results.procedural = { error: msg };
+          }
+        } else {
+          results.procedural = {
+            skipped: true,
+            reason: "fewer than 2 recurring patterns",
+          };
+        }
+      }
+
+      if (tier === "all" || tier === "decay") {
+        const semantic = await kv.list<SemanticMemory>(KV.semantic);
+        applyDecay(semantic, decayDays);
+        for (const s of semantic) {
+          await kv.set(KV.semantic, s.id, s);
+        }
+
+        const procedural = await kv.list<ProceduralMemory>(KV.procedural);
+        applyDecay(procedural, decayDays);
+        for (const p of procedural) {
+          await kv.set(KV.procedural, p.id, p);
+        }
+
+        results.decay = {
+          semantic: semantic.length,
+          procedural: procedural.length,
+        };
+      }
+
+      if (process.env["OBSIDIAN_AUTO_EXPORT"] === "true") {
+        try {
+          await sdk.trigger("mem::obsidian-export", {});
+          results.obsidianExport = { success: true };
+        } catch (err) {
+          const msg = err instanceof Error ? err.message : String(err);
+          ctx.logger.warn("Obsidian auto-export failed", { error: msg });
+          results.obsidianExport = { success: false, error: msg };
+        }
+      }
+
+      await recordAudit(kv, "consolidate", "mem::consolidate-pipeline", [], {
+        tier,
+        results,
+      });
+
+      ctx.logger.info("Consolidation pipeline complete", { tier, results });
+      return { success: true, results };
+    },
+  );
+}

package/src/functions/context.ts

@@ -0,0 +1,169 @@
+import type { ISdk } from "iii-sdk";
+import { getContext } from "iii-sdk";
+import type {
+  Session,
+  CompressedObservation,
+  SessionSummary,
+  ContextBlock,
+  ProjectProfile,
+} from "../types.js";
+import { KV } from "../state/schema.js";
+import { StateKV } from "../state/kv.js";
+
+function estimateTokens(text: string): number {
+  return Math.ceil(text.length / 3);
+}
+
+function escapeXmlAttr(s: string): string {
+  return s
+    .replace(/&/g, "&")
+    .replace(/"/g, """)
+    .replace(/</g, "&lt;")
+    .replace(/>/g, "&gt;");
+}
+
+export function registerContextFunction(
+  sdk: ISdk,
+  kv: StateKV,
+  tokenBudget: number,
+): void {
+  sdk.registerFunction(
+    {
+      id: "mem::context",
+      description: "Generate context for session injection",
+    },
+    async (data: { sessionId: string; project: string; budget?: number }) => {
+      const ctx = getContext();
+      const budget = data.budget || tokenBudget;
+      const blocks: ContextBlock[] = [];
+
+      const profile = await kv
+        .get<ProjectProfile>(KV.profiles, data.project)
+        .catch(() => null);
+      if (profile) {
+        const profileParts = [];
+        if (profile.topConcepts.length > 0) {
+          profileParts.push(
+            `Concepts: ${profile.topConcepts
+              .slice(0, 8)
+              .map((c) => c.concept)
+              .join(", ")}`,
+          );
+        }
+        if (profile.topFiles.length > 0) {
+          profileParts.push(
+            `Key files: ${profile.topFiles
+              .slice(0, 5)
+              .map((f) => f.file)
+              .join(", ")}`,
+          );
+        }
+        if (profile.conventions.length > 0) {
+          profileParts.push(`Conventions: ${profile.conventions.join("; ")}`);
+        }
+        if (profile.commonErrors.length > 0) {
+          profileParts.push(
+            `Common errors: ${profile.commonErrors.slice(0, 3).join("; ")}`,
+          );
+        }
+        if (profileParts.length > 0) {
+          const profileContent = `## Project Profile\n${profileParts.join("\n")}`;
+          blocks.push({
+            type: "memory",
+            content: profileContent,
+            tokens: estimateTokens(profileContent),
+            recency: new Date(profile.updatedAt).getTime(),
+          });
+        }
+      }
+
+      const allSessions = await kv.list<Session>(KV.sessions);
+      const sessions = allSessions
+        .filter((s) => s.project === data.project && s.id !== data.sessionId)
+        .sort(
+          (a, b) =>
+            new Date(b.startedAt).getTime() - new Date(a.startedAt).getTime(),
+        )
+        .slice(0, 10);
+
+      const summariesPerSession = await Promise.all(
+        sessions.map((s) =>
+          kv.get<SessionSummary>(KV.summaries, s.id).catch(() => null),
+        ),
+      );
+
+      const sessionsNeedingObs: number[] = [];
+      for (let i = 0; i < sessions.length; i++) {
+        const summary = summariesPerSession[i];
+        if (summary) {
+          const content = `## ${summary.title}\n${summary.narrative}\nDecisions: ${summary.keyDecisions.join("; ")}\nFiles: ${summary.filesModified.join(", ")}`;
+          blocks.push({
+            type: "summary",
+            content,
+            tokens: estimateTokens(content),
+            recency: new Date(summary.createdAt).getTime(),
+          });
+        } else {
+          sessionsNeedingObs.push(i);
+        }
+      }
+
+      const obsResults = await Promise.all(
+        sessionsNeedingObs.map((i) =>
+          kv
+            .list<CompressedObservation>(KV.observations(sessions[i].id))
+            .catch(() => []),
+        ),
+      );
+
+      for (let j = 0; j < sessionsNeedingObs.length; j++) {
+        const i = sessionsNeedingObs[j];
+        const observations = obsResults[j];
+        const important = observations.filter(
+          (o) => o.title && o.importance >= 5,
+        );
+
+        if (important.length > 0) {
+          const items = important
+            .sort((a, b) => b.importance - a.importance)
+            .slice(0, 5)
+            .map((o) => `- [${o.type}] ${o.title}: ${o.narrative}`)
+            .join("\n");
+          const content = `## Session ${sessions[i].id.slice(0, 8)} (${sessions[i].startedAt})\n${items}`;
+          blocks.push({
+            type: "observation",
+            content,
+            tokens: estimateTokens(content),
+            recency: new Date(sessions[i].startedAt).getTime(),
+          });
+        }
+      }
+
+      blocks.sort((a, b) => b.recency - a.recency);
+
+      let usedTokens = 0;
+      const selected: string[] = [];
+      const header = `<agentmemory-context project="${escapeXmlAttr(data.project)}">`;
+      const footer = `</agentmemory-context>`;
+      usedTokens += estimateTokens(header) + estimateTokens(footer);
+
+      for (const block of blocks) {
+        if (usedTokens + block.tokens > budget) break;
+        selected.push(block.content);
+        usedTokens += block.tokens;
+      }
+
+      if (selected.length === 0) {
+        ctx.logger.info("No context available", { project: data.project });
+        return { context: "", blocks: 0, tokens: 0 };
+      }
+
+      const result = `${header}\n${selected.join("\n\n")}\n${footer}`;
+      ctx.logger.info("Context generated", {
+        blocks: selected.length,
+        tokens: usedTokens,
+      });
+      return { context: result, blocks: selected.length, tokens: usedTokens };
+    },
+  );
+}