@agentmemory/agentmemory 0.7.2 → 0.7.3

package/src/functions/compress.ts

@@ -1,194 +0,0 @@
-import type { ISdk } from "iii-sdk";
-import { getContext } from "iii-sdk";
-import type {
-  RawObservation,
-  CompressedObservation,
-  ObservationType,
-  MemoryProvider,
-} from "../types.js";
-import { KV, STREAM } from "../state/schema.js";
-import { StateKV } from "../state/kv.js";
-import {
-  COMPRESSION_SYSTEM,
-  buildCompressionPrompt,
-} from "../prompts/compression.js";
-import { getXmlTag, getXmlChildren } from "../prompts/xml.js";
-import { getSearchIndex } from "./search.js";
-import { CompressOutputSchema } from "../eval/schemas.js";
-import { validateOutput } from "../eval/validator.js";
-import { scoreCompression } from "../eval/quality.js";
-import { compressWithRetry } from "../eval/self-correct.js";
-import type { MetricsStore } from "../eval/metrics-store.js";
-
-const VALID_TYPES = new Set<string>([
-  "file_read",
-  "file_write",
-  "file_edit",
-  "command_run",
-  "search",
-  "web_fetch",
-  "conversation",
-  "error",
-  "decision",
-  "discovery",
-  "subagent",
-  "notification",
-  "task",
-  "other",
-]);
-
-function parseCompressionXml(
-  xml: string,
-): Omit<CompressedObservation, "id" | "sessionId" | "timestamp"> | null {
-  const rawType = getXmlTag(xml, "type");
-  const title = getXmlTag(xml, "title");
-  if (!rawType || !title) return null;
-  const type = VALID_TYPES.has(rawType) ? rawType : "other";
-
-  return {
-    type: type as ObservationType,
-    title,
-    subtitle: getXmlTag(xml, "subtitle") || undefined,
-    facts: getXmlChildren(xml, "facts", "fact"),
-    narrative: getXmlTag(xml, "narrative"),
-    concepts: getXmlChildren(xml, "concepts", "concept"),
-    files: getXmlChildren(xml, "files", "file"),
-    importance: Math.max(
-      1,
-      Math.min(10, parseInt(getXmlTag(xml, "importance") || "5", 10) || 5),
-    ),
-  };
-}
-
-export function registerCompressFunction(
-  sdk: ISdk,
-  kv: StateKV,
-  provider: MemoryProvider,
-  metricsStore?: MetricsStore,
-): void {
-  sdk.registerFunction(
-    {
-      id: "mem::compress",
-      description: "Compress a raw observation using LLM",
-    },
-    async (data: {
-      observationId: string;
-      sessionId: string;
-      raw: RawObservation;
-    }) => {
-      const ctx = getContext();
-      const startMs = Date.now();
-      const prompt = buildCompressionPrompt({
-        hookType: data.raw.hookType,
-        toolName: data.raw.toolName,
-        toolInput: data.raw.toolInput,
-        toolOutput: data.raw.toolOutput,
-        userPrompt: data.raw.userPrompt,
-        timestamp: data.raw.timestamp,
-      });
-
-      try {
-        const validator = (response: string) => {
-          const parsed = parseCompressionXml(response);
-          if (!parsed) return { valid: false, errors: ["xml_parse_failed"] };
-          const result = validateOutput(
-            CompressOutputSchema,
-            parsed,
-            "mem::compress",
-          );
-          return result.valid
-            ? { valid: true }
-            : { valid: false, errors: result.result.errors };
-        };
-
-        const { response, retried } = await compressWithRetry(
-          provider,
-          COMPRESSION_SYSTEM,
-          prompt,
-          validator,
-          1,
-        );
-
-        const parsed = parseCompressionXml(response);
-        if (!parsed) {
-          const latencyMs = Date.now() - startMs;
-          if (metricsStore) {
-            await metricsStore.record("mem::compress", latencyMs, false);
-          }
-          ctx.logger.warn("Failed to parse compression XML", {
-            obsId: data.observationId,
-            retried,
-          });
-          return { success: false, error: "parse_failed" };
-        }
-
-        const qualityScore = scoreCompression(parsed);
-
-        const compressed: CompressedObservation = {
-          id: data.observationId,
-          sessionId: data.sessionId,
-          timestamp: data.raw.timestamp,
-          ...parsed,
-          confidence: qualityScore / 100,
-        };
-
-        await kv.set(
-          KV.observations(data.sessionId),
-          data.observationId,
-          compressed,
-        );
-
-        getSearchIndex().add(compressed);
-
-        sdk.triggerVoid("stream::set", {
-          stream_name: STREAM.name,
-          group_id: STREAM.group(data.sessionId),
-          item_id: data.observationId,
-          data: { type: "compressed", observation: compressed },
-        });
-
-        sdk.triggerVoid("stream::set", {
-          stream_name: STREAM.name,
-          group_id: STREAM.viewerGroup,
-          item_id: data.observationId,
-          data: {
-            type: "compressed",
-            observation: compressed,
-            sessionId: data.sessionId,
-          },
-        });
-
-        const latencyMs = Date.now() - startMs;
-        if (metricsStore) {
-          await metricsStore.record(
-            "mem::compress",
-            latencyMs,
-            true,
-            qualityScore,
-          );
-        }
-
-        ctx.logger.info("Observation compressed", {
-          obsId: data.observationId,
-          type: compressed.type,
-          importance: compressed.importance,
-          qualityScore,
-          retried,
-        });
-
-        return { success: true, compressed, qualityScore };
-      } catch (err) {
-        const msg = err instanceof Error ? err.message : String(err);
-        const latencyMs = Date.now() - startMs;
-        if (metricsStore) {
-          await metricsStore.record("mem::compress", latencyMs, false);
-        }
-        ctx.logger.error("Compression failed", {
-          obsId: data.observationId,
-          error: msg,
-        });
-        return { success: false, error: "compression_failed" };
-      }
-    },
-  );
-}

package/src/functions/consolidate.ts

@@ -1,212 +0,0 @@
-import type { ISdk } from "iii-sdk";
-import { getContext } from "iii-sdk";
-import type {
-  CompressedObservation,
-  Memory,
-  Session,
-  MemoryProvider,
-} from "../types.js";
-import { KV, generateId } from "../state/schema.js";
-import { StateKV } from "../state/kv.js";
-
-const CONSOLIDATION_SYSTEM = `You are a memory consolidation engine. Given a set of related observations from coding sessions, synthesize them into a single long-term memory.
-
-Output XML:
-<memory>
-  <type>pattern|preference|architecture|bug|workflow|fact</type>
-  <title>Concise memory title (max 80 chars)</title>
-  <content>2-4 sentence description of the learned insight</content>
-  <concepts>
-    <concept>key term</concept>
-  </concepts>
-  <files>
-    <file>relevant/file/path</file>
-  </files>
-  <strength>1-10 how confident/important this memory is</strength>
-</memory>`;
-
-import { getXmlTag, getXmlChildren } from "../prompts/xml.js";
-
-function parseMemoryXml(
-  xml: string,
-  sessionIds: string[],
-): Omit<Memory, "id" | "createdAt" | "updatedAt"> | null {
-  const type = getXmlTag(xml, "type");
-  const title = getXmlTag(xml, "title");
-  const content = getXmlTag(xml, "content");
-  if (!type || !title || !content) return null;
-
-  const validTypes = new Set([
-    "pattern",
-    "preference",
-    "architecture",
-    "bug",
-    "workflow",
-    "fact",
-  ]);
-
-  return {
-    type: (validTypes.has(type) ? type : "fact") as Memory["type"],
-    title,
-    content,
-    concepts: getXmlChildren(xml, "concepts", "concept"),
-    files: getXmlChildren(xml, "files", "file"),
-    sessionIds,
-    strength: Math.max(
-      1,
-      Math.min(10, parseInt(getXmlTag(xml, "strength") || "5", 10) || 5),
-    ),
-    version: 1,
-    isLatest: true,
-  };
-}
-
-export function registerConsolidateFunction(
-  sdk: ISdk,
-  kv: StateKV,
-  provider: MemoryProvider,
-): void {
-  sdk.registerFunction(
-    { id: "mem::consolidate" },
-    async (data: { project?: string; minObservations?: number }) => {
-      const ctx = getContext();
-      const minObs = data.minObservations ?? 10;
-
-      const sessions = await kv.list<Session>(KV.sessions);
-      const filtered = data.project
-        ? sessions.filter((s) => s.project === data.project)
-        : sessions;
-
-      const allObs: Array<CompressedObservation & { sid: string }> = [];
-      const obsPerSession: CompressedObservation[][] = [];
-      for (let batch = 0; batch < filtered.length; batch += 10) {
-        const chunk = filtered.slice(batch, batch + 10);
-        const results = await Promise.all(
-          chunk.map((s) =>
-            kv
-              .list<CompressedObservation>(KV.observations(s.id))
-              .catch(() => [] as CompressedObservation[]),
-          ),
-        );
-        obsPerSession.push(...results);
-      }
-      for (let i = 0; i < filtered.length; i++) {
-        for (const obs of obsPerSession[i]) {
-          if (obs.title && obs.importance >= 5) {
-            allObs.push({ ...obs, sid: filtered[i].id });
-          }
-        }
-      }
-
-      if (allObs.length < minObs) {
-        return { consolidated: 0, reason: "insufficient_observations" };
-      }
-
-      const conceptGroups = new Map<string, typeof allObs>();
-      for (const obs of allObs) {
-        for (const concept of obs.concepts) {
-          const key = concept.toLowerCase();
-          if (!conceptGroups.has(key)) conceptGroups.set(key, []);
-          conceptGroups.get(key)!.push(obs);
-        }
-      }
-
-      let consolidated = 0;
-      const existingMemories = await kv.list<Memory>(KV.memories);
-      const existingTitles = new Set(
-        existingMemories.map((m) => m.title.toLowerCase()),
-      );
-
-      const MAX_LLM_CALLS = 10;
-      let llmCallCount = 0;
-
-      const sortedGroups = [...conceptGroups.entries()]
-        .filter(([, g]) => g.length >= 3)
-        .sort((a, b) => b[1].length - a[1].length);
-
-      for (const [concept, obsGroup] of sortedGroups) {
-        if (llmCallCount >= MAX_LLM_CALLS) break;
-
-        const top = obsGroup
-          .sort((a, b) => b.importance - a.importance)
-          .slice(0, 8);
-        const sessionIds = [...new Set(top.map((o) => o.sid))];
-
-        const prompt = top
-          .map(
-            (o) =>
-              `[${o.type}] ${o.title}\n${o.narrative}\nFiles: ${o.files.join(", ")}\nImportance: ${o.importance}`,
-          )
-          .join("\n\n");
-
-        try {
-          const response = await Promise.race([
-            provider.compress(
-              CONSOLIDATION_SYSTEM,
-              `Concept: "${concept}"\n\nObservations:\n${prompt}`,
-            ),
-            new Promise<never>((_, reject) =>
-              setTimeout(() => reject(new Error("compress timeout")), 30_000),
-            ),
-          ]);
-          llmCallCount++;
-          const parsed = parseMemoryXml(response, sessionIds);
-          if (!parsed) continue;
-
-          const existingMatch = existingMemories.find(
-            (m) => m.title.toLowerCase() === parsed.title.toLowerCase(),
-          );
-
-          const now = new Date().toISOString();
-          const obsIds = [...new Set(top.map((o) => o.id))];
-          if (existingMatch) {
-            existingMatch.isLatest = false;
-            await kv.set(KV.memories, existingMatch.id, existingMatch);
-
-            const evolved: Memory = {
-              id: generateId("mem"),
-              createdAt: now,
-              updatedAt: now,
-              ...parsed,
-              version: (existingMatch.version || 1) + 1,
-              parentId: existingMatch.id,
-              supersedes: [
-                existingMatch.id,
-                ...(existingMatch.supersedes || []),
-              ],
-              sourceObservationIds: obsIds,
-              isLatest: true,
-            };
-            await kv.set(KV.memories, evolved.id, evolved);
-            existingTitles.add(evolved.title.toLowerCase());
-            consolidated++;
-          } else {
-            const memory: Memory = {
-              id: generateId("mem"),
-              createdAt: now,
-              updatedAt: now,
-              ...parsed,
-              sourceObservationIds: obsIds,
-              version: 1,
-              isLatest: true,
-            };
-            await kv.set(KV.memories, memory.id, memory);
-            existingTitles.add(memory.title.toLowerCase());
-            consolidated++;
-          }
-        } catch (err) {
-          ctx.logger.warn("Consolidation failed for concept", {
-            concept,
-            error: err instanceof Error ? err.message : String(err),
-          });
-        }
-      }
-
-      ctx.logger.info("Consolidation complete", {
-        consolidated,
-        totalObs: allObs.length,
-      });
-      return { consolidated, totalObservations: allObs.length };
-    },
-  );
-}

package/src/functions/consolidation-pipeline.ts

@@ -1,258 +0,0 @@
-import type { ISdk } from "iii-sdk";
-import { getContext } from "iii-sdk";
-import type {
-  SemanticMemory,
-  ProceduralMemory,
-  SessionSummary,
-  Memory,
-  MemoryProvider,
-} from "../types.js";
-import { KV, generateId } from "../state/schema.js";
-import type { StateKV } from "../state/kv.js";
-import {
-  SEMANTIC_MERGE_SYSTEM,
-  buildSemanticMergePrompt,
-  PROCEDURAL_EXTRACTION_SYSTEM,
-  buildProceduralExtractionPrompt,
-} from "../prompts/consolidation.js";
-import { recordAudit } from "./audit.js";
-import { getConsolidationDecayDays, isConsolidationEnabled } from "../config.js";
-
-function applyDecay(
-  items: Array<{
-    strength: number;
-    lastAccessedAt?: string;
-    updatedAt: string;
-  }>,
-  decayDays: number,
-): void {
-  if (decayDays <= 0 || !Number.isFinite(decayDays)) return;
-  const now = Date.now();
-  for (const item of items) {
-    const lastAccess = item.lastAccessedAt || item.updatedAt;
-    const daysSince =
-      (now - new Date(lastAccess).getTime()) / (1000 * 60 * 60 * 24);
-    if (daysSince > decayDays) {
-      const decayPeriods = Math.floor(daysSince / decayDays);
-      item.strength = Math.max(
-        0.1,
-        item.strength * Math.pow(0.9, decayPeriods),
-      );
-    }
-  }
-}
-
-export function registerConsolidationPipelineFunction(
-  sdk: ISdk,
-  kv: StateKV,
-  provider: MemoryProvider,
-): void {
-  sdk.registerFunction(
-    { id: "mem::consolidate-pipeline" },
-    async (data?: { tier?: string; force?: boolean }) => {
-      if (!data?.force && !isConsolidationEnabled()) {
-        return { success: false, skipped: true, reason: "CONSOLIDATION_ENABLED is not set to true" };
-      }
-      const ctx = getContext();
-      const tier = data?.tier || "all";
-      const decayDays = getConsolidationDecayDays();
-      const results: Record<string, unknown> = {};
-
-      if (tier === "all" || tier === "semantic") {
-        const summaries = await kv.list<SessionSummary>(KV.summaries);
-        const existingSemantic = await kv.list<SemanticMemory>(KV.semantic);
-
-        if (summaries.length >= 5) {
-          const recentSummaries = summaries
-            .sort(
-              (a, b) =>
-                new Date(b.createdAt).getTime() -
-                new Date(a.createdAt).getTime(),
-            )
-            .slice(0, 20);
-
-          const prompt = buildSemanticMergePrompt(
-            recentSummaries.map((s) => ({
-              title: s.title,
-              narrative: s.narrative,
-              concepts: s.concepts,
-            })),
-          );
-
-          try {
-            const response = await provider.summarize(
-              SEMANTIC_MERGE_SYSTEM,
-              prompt,
-            );
-
-            const factRegex = /<fact\s+confidence="([^"]+)">([^<]+)<\/fact>/g;
-            let match;
-            let newFacts = 0;
-            const now = new Date().toISOString();
-
-            while ((match = factRegex.exec(response)) !== null) {
-              const parsedConf = parseFloat(match[1]);
-              const confidence = Number.isNaN(parsedConf) ? 0.5 : parsedConf;
-              const fact = match[2].trim();
-
-              const existing = existingSemantic.find(
-                (s) => s.fact.toLowerCase() === fact.toLowerCase(),
-              );
-              if (existing) {
-                existing.accessCount++;
-                existing.lastAccessedAt = now;
-                existing.updatedAt = now;
-                existing.confidence = Math.max(existing.confidence, confidence);
-                await kv.set(KV.semantic, existing.id, existing);
-              } else {
-                const sem: SemanticMemory = {
-                  id: generateId("sem"),
-                  fact,
-                  confidence,
-                  sourceSessionIds: recentSummaries.map((s) => s.sessionId),
-                  sourceMemoryIds: [],
-                  accessCount: 1,
-                  lastAccessedAt: now,
-                  strength: confidence,
-                  createdAt: now,
-                  updatedAt: now,
-                };
-                await kv.set(KV.semantic, sem.id, sem);
-                newFacts++;
-              }
-            }
-            results.semantic = { newFacts, totalSummaries: summaries.length };
-          } catch (err) {
-            const msg = err instanceof Error ? err.message : String(err);
-            ctx.logger.error("Semantic consolidation failed", { error: msg });
-            results.semantic = { error: msg };
-          }
-        } else {
-          results.semantic = {
-            skipped: true,
-            reason: "fewer than 5 summaries",
-          };
-        }
-      }
-
-      if (tier === "all" || tier === "procedural") {
-        const memories = await kv.list<Memory>(KV.memories);
-        const patterns = memories
-          .filter((m) => m.isLatest && m.type === "pattern")
-          .map((m) => ({
-            content: m.content,
-            frequency: m.sessionIds.length || 1,
-          }))
-          .filter((p) => p.frequency >= 2);
-
-        if (patterns.length >= 2) {
-          const prompt = buildProceduralExtractionPrompt(patterns);
-
-          try {
-            const response = await provider.summarize(
-              PROCEDURAL_EXTRACTION_SYSTEM,
-              prompt,
-            );
-
-            const procRegex =
-              /<procedure\s+name="([^"]+)"\s+trigger="([^"]+)">([\s\S]*?)<\/procedure>/g;
-            let match;
-            let newProcs = 0;
-            const now = new Date().toISOString();
-            const existingProcs = await kv.list<ProceduralMemory>(
-              KV.procedural,
-            );
-
-            while ((match = procRegex.exec(response)) !== null) {
-              const name = match[1];
-              const trigger = match[2];
-              const stepsBlock = match[3];
-              const steps: string[] = [];
-
-              const stepRegex = /<step>([^<]+)<\/step>/g;
-              let stepMatch;
-              while ((stepMatch = stepRegex.exec(stepsBlock)) !== null) {
-                steps.push(stepMatch[1].trim());
-              }
-
-              const existing = existingProcs.find(
-                (p) => p.name.toLowerCase() === name.toLowerCase(),
-              );
-              if (existing) {
-                existing.frequency++;
-                existing.updatedAt = now;
-                existing.strength = Math.min(1, existing.strength + 0.1);
-                await kv.set(KV.procedural, existing.id, existing);
-              } else {
-                const proc: ProceduralMemory = {
-                  id: generateId("proc"),
-                  name,
-                  steps,
-                  triggerCondition: trigger,
-                  frequency: 1,
-                  sourceSessionIds: [],
-                  strength: 0.5,
-                  createdAt: now,
-                  updatedAt: now,
-                };
-                await kv.set(KV.procedural, proc.id, proc);
-                newProcs++;
-              }
-            }
-            results.procedural = {
-              newProcedures: newProcs,
-              patternsAnalyzed: patterns.length,
-            };
-          } catch (err) {
-            const msg = err instanceof Error ? err.message : String(err);
-            ctx.logger.error("Procedural extraction failed", { error: msg });
-            results.procedural = { error: msg };
-          }
-        } else {
-          results.procedural = {
-            skipped: true,
-            reason: "fewer than 2 recurring patterns",
-          };
-        }
-      }
-
-      if (tier === "all" || tier === "decay") {
-        const semantic = await kv.list<SemanticMemory>(KV.semantic);
-        applyDecay(semantic, decayDays);
-        for (const s of semantic) {
-          await kv.set(KV.semantic, s.id, s);
-        }
-
-        const procedural = await kv.list<ProceduralMemory>(KV.procedural);
-        applyDecay(procedural, decayDays);
-        for (const p of procedural) {
-          await kv.set(KV.procedural, p.id, p);
-        }
-
-        results.decay = {
-          semantic: semantic.length,
-          procedural: procedural.length,
-        };
-      }
-
-      if (process.env["OBSIDIAN_AUTO_EXPORT"] === "true") {
-        try {
-          await sdk.trigger("mem::obsidian-export", {});
-          results.obsidianExport = { success: true };
-        } catch (err) {
-          const msg = err instanceof Error ? err.message : String(err);
-          ctx.logger.warn("Obsidian auto-export failed", { error: msg });
-          results.obsidianExport = { success: false, error: msg };
-        }
-      }
-
-      await recordAudit(kv, "consolidate", "mem::consolidate-pipeline", [], {
-        tier,
-        results,
-      });
-
-      ctx.logger.info("Consolidation pipeline complete", { tier, results });
-      return { success: true, results };
-    },
-  );
-}