@agentmemory/agentmemory 0.7.0

package/src/functions/migrate.ts

@@ -0,0 +1,165 @@
+import type { ISdk } from "iii-sdk";
+import { getContext } from "iii-sdk";
+import { resolve } from "node:path";
+import { homedir } from "node:os";
+import { KV, generateId } from "../state/schema.js";
+import { StateKV } from "../state/kv.js";
+import type {
+  Session,
+  CompressedObservation,
+  SessionSummary,
+} from "../types.js";
+
+const ALLOWED_DIRS = [resolve(homedir(), ".agentmemory")];
+
+function isAllowedPath(dbPath: string): boolean {
+  const resolved = resolve(dbPath);
+  return ALLOWED_DIRS.some((dir) => resolved.startsWith(dir + "/"));
+}
+
+export function registerMigrateFunction(sdk: ISdk, kv: StateKV): void {
+  sdk.registerFunction(
+    {
+      id: "mem::migrate",
+      description: "Import data from SQLite database",
+    },
+    async (data: { dbPath: string }) => {
+      const ctx = getContext();
+      ctx.logger.info("Migration started", { dbPath: data.dbPath });
+
+      if (!isAllowedPath(data.dbPath)) {
+        return {
+          success: false,
+          error: `Path not allowed. Must be under: ${ALLOWED_DIRS.join(", ")}`,
+        };
+      }
+
+      let Database: any;
+      try {
+        // @ts-expect-error optional dependency
+        Database = (await import("better-sqlite3")).default;
+      } catch {
+        return {
+          success: false,
+          error:
+            "better-sqlite3 not installed. Run: npm install better-sqlite3",
+        };
+      }
+
+      const fs = await import("node:fs");
+      if (!fs.existsSync(data.dbPath)) {
+        return { success: false, error: `Database not found: ${data.dbPath}` };
+      }
+
+      try {
+        const db = Database(data.dbPath, { readonly: true });
+        let sessionCount = 0;
+        let obsCount = 0;
+        let summaryCount = 0;
+
+        const sessions = db
+          .prepare("SELECT * FROM sessions ORDER BY created_at DESC")
+          .all() as any[];
+        for (const row of sessions) {
+          const session: Session = {
+            id: row.session_id || row.id,
+            project: row.project_path || row.project || "unknown",
+            cwd: row.cwd || row.project_path || "",
+            startedAt:
+              row.created_at || row.started_at || new Date().toISOString(),
+            endedAt: row.ended_at || row.updated_at,
+            status: "completed",
+            observationCount: 0,
+          };
+          await kv.set(KV.sessions, session.id, session);
+          sessionCount++;
+        }
+
+        let observations: any[] = [];
+        try {
+          observations = db
+            .prepare("SELECT * FROM observations ORDER BY created_at ASC")
+            .all() as any[];
+        } catch {
+          try {
+            observations = db
+              .prepare(
+                "SELECT * FROM compressed_observations ORDER BY created_at ASC",
+              )
+              .all() as any[];
+          } catch {
+            ctx.logger.warn("No observation tables found");
+          }
+        }
+
+        for (const row of observations) {
+          const sessionId = row.session_id || "migrated";
+          const obs: CompressedObservation = {
+            id: row.id || generateId("mig"),
+            sessionId,
+            timestamp: row.created_at || new Date().toISOString(),
+            type: row.type || "other",
+            title: row.title || row.summary || "Migrated observation",
+            subtitle: row.subtitle,
+            facts: safeJsonParse(row.facts, []),
+            narrative: row.narrative || row.content || "",
+            concepts: safeJsonParse(row.concepts, []),
+            files: safeJsonParse(row.files, []),
+            importance: row.importance || 5,
+          };
+          await kv.set(KV.observations(sessionId), obs.id, obs);
+          obsCount++;
+        }
+
+        let summaries: any[] = [];
+        try {
+          summaries = db
+            .prepare("SELECT * FROM session_summaries")
+            .all() as any[];
+        } catch {
+          ctx.logger.warn("No summaries table found");
+        }
+
+        for (const row of summaries) {
+          const summary: SessionSummary = {
+            sessionId: row.session_id,
+            project: row.project || "unknown",
+            createdAt: row.created_at || new Date().toISOString(),
+            title: row.title || "Migrated session",
+            narrative: row.narrative || row.summary || "",
+            keyDecisions: safeJsonParse(row.key_decisions, []),
+            filesModified: safeJsonParse(row.files_modified, []),
+            concepts: safeJsonParse(row.concepts, []),
+            observationCount: row.observation_count || 0,
+          };
+          await kv.set(KV.summaries, row.session_id, summary);
+          summaryCount++;
+        }
+
+        db.close();
+        ctx.logger.info("Migration complete", {
+          sessionCount,
+          obsCount,
+          summaryCount,
+        });
+        return { success: true, sessionCount, obsCount, summaryCount };
+      } catch (err) {
+        const msg = err instanceof Error ? err.message : String(err);
+        ctx.logger.error("Migration failed", { error: msg });
+        return { success: false, error: "Migration failed" };
+      }
+    },
+  );
+}
+
+function safeJsonParse<T>(value: unknown, fallback: T): T {
+  if (Array.isArray(value)) return value as T;
+  if (typeof value === "string") {
+    try {
+      return JSON.parse(value) as T;
+    } catch {
+      return fallback;
+    }
+  }
+  return fallback;
+}

package/src/functions/observe.ts

@@ -0,0 +1,144 @@
+import type { ISdk } from "iii-sdk";
+import { getContext } from "iii-sdk";
+import type { RawObservation, HookPayload, Session } from "../types.js";
+import { KV, STREAM, generateId } from "../state/schema.js";
+import { StateKV } from "../state/kv.js";
+import { stripPrivateData } from "./privacy.js";
+import { DedupMap } from "./dedup.js";
+import { withKeyedLock } from "../state/keyed-mutex.js";
+
+export function registerObserveFunction(
+  sdk: ISdk,
+  kv: StateKV,
+  dedupMap?: DedupMap,
+  maxObservationsPerSession?: number,
+): void {
+  sdk.registerFunction(
+    {
+      id: "mem::observe",
+      description: "Capture and store a tool-use observation",
+    },
+    async (payload: HookPayload) => {
+      const ctx = getContext();
+
+      if (
+        !payload?.sessionId ||
+        typeof payload.sessionId !== "string" ||
+        !payload.hookType ||
+        typeof payload.hookType !== "string" ||
+        !payload.timestamp ||
+        typeof payload.timestamp !== "string"
+      ) {
+        return {
+          success: false,
+          error:
+            "Invalid payload: sessionId, hookType, and timestamp are required",
+        };
+      }
+
+      const obsId = generateId("obs");
+
+      let dedupHash: string | undefined;
+      if (dedupMap) {
+        const d =
+          typeof payload.data === "object" && payload.data !== null
+            ? (payload.data as Record<string, unknown>)
+            : {};
+        const toolName = (d["tool_name"] as string) || payload.hookType;
+        dedupHash = dedupMap.computeHash(
+          payload.sessionId,
+          toolName,
+          d["tool_input"],
+        );
+        if (dedupMap.isDuplicate(dedupHash)) {
+          return { deduplicated: true, sessionId: payload.sessionId };
+        }
+      }
+
+      let sanitizedRaw: unknown = payload.data;
+      try {
+        const jsonStr = JSON.stringify(payload.data);
+        const sanitized = stripPrivateData(jsonStr);
+        sanitizedRaw = JSON.parse(sanitized);
+      } catch {
+        sanitizedRaw = stripPrivateData(String(payload.data));
+      }
+
+      const raw: RawObservation = {
+        id: obsId,
+        sessionId: payload.sessionId,
+        timestamp: payload.timestamp,
+        hookType: payload.hookType,
+        raw: sanitizedRaw,
+      };
+
+      if (typeof sanitizedRaw === "object" && sanitizedRaw !== null) {
+        const d = sanitizedRaw as Record<string, unknown>;
+        if (
+          payload.hookType === "post_tool_use" ||
+          payload.hookType === "post_tool_failure"
+        ) {
+          raw.toolName = d["tool_name"] as string | undefined;
+          raw.toolInput = d["tool_input"];
+          raw.toolOutput = d["tool_output"] || d["error"];
+        }
+        if (payload.hookType === "prompt_submit") {
+          raw.userPrompt = d["prompt"] as string | undefined;
+        }
+      }
+
+      return withKeyedLock(`obs:${payload.sessionId}`, async () => {
+        if (maxObservationsPerSession && maxObservationsPerSession > 0) {
+          const existing = await kv.list(KV.observations(payload.sessionId));
+          if (existing.length >= maxObservationsPerSession) {
+            return {
+              success: false,
+              error: `Session observation limit reached (${maxObservationsPerSession})`,
+            };
+          }
+        }
+
+        await kv.set(KV.observations(payload.sessionId), obsId, raw);
+
+        if (dedupMap && dedupHash) {
+          dedupMap.record(dedupHash);
+        }
+
+        sdk.triggerVoid("stream::set", {
+          stream_name: STREAM.name,
+          group_id: STREAM.group(payload.sessionId),
+          item_id: obsId,
+          data: { type: "raw", observation: raw },
+        });
+
+        sdk.triggerVoid("stream::set", {
+          stream_name: STREAM.name,
+          group_id: STREAM.viewerGroup,
+          item_id: obsId,
+          data: { type: "raw", observation: raw, sessionId: payload.sessionId },
+        });
+
+        const session = await kv.get<Session>(KV.sessions, payload.sessionId);
+        if (session) {
+          await kv.set(KV.sessions, payload.sessionId, {
+            ...session,
+            observationCount: (session.observationCount || 0) + 1,
+          });
+        }
+
+        sdk.triggerVoid("mem::compress", {
+          observationId: obsId,
+          sessionId: payload.sessionId,
+          raw,
+        });
+
+        ctx.logger.info("Observation captured", {
+          obsId,
+          sessionId: payload.sessionId,
+          hook: payload.hookType,
+        });
+        return { observationId: obsId };
+      });
+    },
+  );
+}

package/src/functions/obsidian-export.ts

@@ -0,0 +1,310 @@
+import { mkdir, writeFile } from "node:fs/promises";
+import { join } from "node:path";
+import { homedir } from "node:os";
+import type { ISdk } from "iii-sdk";
+import type { StateKV } from "../state/kv.js";
+import { KV } from "../state/schema.js";
+import type {
+  Memory,
+  Lesson,
+  Crystal,
+  Session,
+} from "../types.js";
+import { recordAudit } from "./audit.js";
+
+const DEFAULT_VAULT = join(homedir(), ".agentmemory", "vault");
+
+function sanitize(name: string): string {
+  return name.replace(/[<>:"/\\|?*\x00-\x1f]/g, "_").slice(0, 100);
+}
+
+function toFrontmatter(obj: Record<string, unknown>): string {
+  const lines = ["---"];
+  for (const [key, value] of Object.entries(obj)) {
+    if (value === undefined || value === null) continue;
+    if (Array.isArray(value)) {
+      lines.push(`${key}: [${value.map((v) => JSON.stringify(String(v))).join(", ")}]`);
+    } else {
+      lines.push(`${key}: ${JSON.stringify(value)}`);
+    }
+  }
+  lines.push("---");
+  return lines.join("\n");
+}
+
+function memoryToMd(m: Memory): string {
+  const fm = toFrontmatter({
+    id: m.id,
+    type: m.type,
+    created: m.createdAt,
+    updated: m.updatedAt,
+    strength: m.strength,
+    version: m.version,
+    concepts: m.concepts,
+    files: m.files,
+  });
+
+  const related = (m.relatedIds || [])
+    .map((id) => `- [[${id}]]`)
+    .join("\n");
+  const supersedes = (m.supersedes || [])
+    .map((id) => `- [[${id}]] (superseded)`)
+    .join("\n");
+
+  const sections = [
+    fm,
+    "",
+    `# ${m.title}`,
+    "",
+    m.content,
+  ];
+
+  if (m.concepts.length > 0) {
+    sections.push("", "## Concepts", m.concepts.map((c) => `#${c.replace(/\s+/g, "-")}`).join(" "));
+  }
+  if (related) {
+    sections.push("", "## Related", related);
+  }
+  if (supersedes) {
+    sections.push("", "## Supersedes", supersedes);
+  }
+
+  return sections.join("\n");
+}
+
+function lessonToMd(l: Lesson): string {
+  const fm = toFrontmatter({
+    id: l.id,
+    type: "lesson",
+    source: l.source,
+    confidence: l.confidence,
+    reinforcements: l.reinforcements,
+    created: l.createdAt,
+    updated: l.updatedAt,
+    project: l.project,
+    tags: l.tags,
+    decayRate: l.decayRate,
+  });
+
+  const sourceLinks = l.sourceIds
+    .map((id) => `- [[${id}]]`)
+    .join("\n");
+
+  const sections = [
+    fm,
+    "",
+    `# Lesson: ${l.content.slice(0, 80)}`,
+    "",
+    l.content,
+  ];
+
+  if (l.context) {
+    sections.push("", "## Context", l.context);
+  }
+  if (l.tags.length > 0) {
+    sections.push("", "## Tags", l.tags.map((t) => `#${t.replace(/\s+/g, "-")}`).join(" "));
+  }
+  if (sourceLinks) {
+    sections.push("", "## Sources", sourceLinks);
+  }
+
+  return sections.join("\n");
+}
+
+function crystalToMd(c: Crystal): string {
+  const fm = toFrontmatter({
+    id: c.id,
+    type: "crystal",
+    created: c.createdAt,
+    project: c.project,
+    sessionId: c.sessionId,
+    filesAffected: c.filesAffected,
+  });
+
+  const actionLinks = c.sourceActionIds
+    .map((id) => `- [[${id}]]`)
+    .join("\n");
+
+  const sections = [
+    fm,
+    "",
+    `# Crystal: ${c.narrative.slice(0, 80)}`,
+    "",
+    c.narrative,
+    "",
+    "## Key Outcomes",
+    ...c.keyOutcomes.map((o) => `- ${o}`),
+  ];
+
+  if (c.lessons.length > 0) {
+    sections.push("", "## Lessons", ...c.lessons.map((l) => `- ${l}`));
+  }
+  if (c.filesAffected.length > 0) {
+    sections.push("", "## Files", ...c.filesAffected.map((f) => `- \`${f}\``));
+  }
+  if (actionLinks) {
+    sections.push("", "## Source Actions", actionLinks);
+  }
+
+  return sections.join("\n");
+}
+
+function sessionToMd(s: Session): string {
+  const fm = toFrontmatter({
+    id: s.id,
+    type: "session",
+    project: s.project,
+    status: s.status,
+    started: s.startedAt,
+    ended: s.endedAt,
+    observations: s.observationCount,
+  });
+
+  return [
+    fm,
+    "",
+    `# Session: ${s.project}`,
+    "",
+    `**Status:** ${s.status}`,
+    `**Started:** ${s.startedAt}`,
+    s.endedAt ? `**Ended:** ${s.endedAt}` : "",
+    `**Observations:** ${s.observationCount}`,
+    `**CWD:** \`${s.cwd}\``,
+  ]
+    .filter(Boolean)
+    .join("\n");
+}
+
+interface ExportError {
+  id: string;
+  path: string;
+  error: string;
+}
+
+export function registerObsidianExportFunction(
+  sdk: ISdk,
+  kv: StateKV,
+): void {
+  sdk.registerFunction(
+    { id: "mem::obsidian-export" },
+    async (data: { vaultDir?: string; types?: string[] }) => {
+      const vaultDir = data.vaultDir || DEFAULT_VAULT;
+      const exportTypes = new Set(
+        data.types || ["memories", "lessons", "crystals", "sessions"],
+      );
+
+      const dirs = {
+        memories: join(vaultDir, "memories"),
+        lessons: join(vaultDir, "lessons"),
+        crystals: join(vaultDir, "crystals"),
+        sessions: join(vaultDir, "sessions"),
+      };
+
+      await Promise.all(
+        Object.values(dirs).map((dir) => mkdir(dir, { recursive: true })),
+      );
+
+      const stats = { memories: 0, lessons: 0, crystals: 0, sessions: 0 };
+      const errors: ExportError[] = [];
+      const memoryMoc: string[] = [];
+      const lessonMoc: string[] = [];
+      const crystalMoc: string[] = [];
+      const sessionMoc: string[] = [];
+
+      const [memories, lessons, crystals, sessions] = await Promise.all([
+        exportTypes.has("memories") ? kv.list<Memory>(KV.memories) : Promise.resolve([] as Memory[]),
+        exportTypes.has("lessons") ? kv.list<Lesson>(KV.lessons) : Promise.resolve([] as Lesson[]),
+        exportTypes.has("crystals") ? kv.list<Crystal>(KV.crystals) : Promise.resolve([] as Crystal[]),
+        exportTypes.has("sessions") ? kv.list<Session>(KV.sessions) : Promise.resolve([] as Session[]),
+      ]);
+
+      for (const m of memories.filter((m) => m.isLatest)) {
+        const filename = `${sanitize(m.id)}.md`;
+        const filepath = join(dirs.memories, filename);
+        try {
+          await writeFile(filepath, memoryToMd(m));
+          stats.memories++;
+          memoryMoc.push(`- [[memories/${sanitize(m.id)}|${m.title}]] (${m.type}, strength: ${m.strength})`);
+        } catch (err) {
+          errors.push({ id: m.id, path: filepath, error: err instanceof Error ? err.message : String(err) });
+        }
+      }
+
+      for (const l of lessons.filter((l) => !l.deleted)) {
+        const filename = `${sanitize(l.id)}.md`;
+        const filepath = join(dirs.lessons, filename);
+        try {
+          await writeFile(filepath, lessonToMd(l));
+          stats.lessons++;
+          lessonMoc.push(`- [[lessons/${sanitize(l.id)}|${l.content.slice(0, 60)}]] (confidence: ${l.confidence})`);
+        } catch (err) {
+          errors.push({ id: l.id, path: filepath, error: err instanceof Error ? err.message : String(err) });
+        }
+      }
+
+      for (const c of crystals) {
+        const filename = `${sanitize(c.id)}.md`;
+        const filepath = join(dirs.crystals, filename);
+        try {
+          await writeFile(filepath, crystalToMd(c));
+          stats.crystals++;
+          crystalMoc.push(`- [[crystals/${sanitize(c.id)}|${c.narrative.slice(0, 60)}]]`);
+        } catch (err) {
+          errors.push({ id: c.id, path: filepath, error: err instanceof Error ? err.message : String(err) });
+        }
+      }
+
+      const recent = sessions
+        .sort(
+          (a, b) =>
+            new Date(b.startedAt).getTime() -
+            new Date(a.startedAt).getTime(),
+        )
+        .slice(0, 50);
+      for (const s of recent) {
+        const filename = `${sanitize(s.id)}.md`;
+        const filepath = join(dirs.sessions, filename);
+        try {
+          await writeFile(filepath, sessionToMd(s));
+          stats.sessions++;
+          sessionMoc.push(`- [[sessions/${sanitize(s.id)}|${s.project} (${s.status})]]`);
+        } catch (err) {
+          errors.push({ id: s.id, path: filepath, error: err instanceof Error ? err.message : String(err) });
+        }
+      }
+
+      const exportedAt = new Date().toISOString();
+      const moc = [
+        "---",
+        "type: moc",
+        `exported: ${exportedAt}`,
+        "---",
+        "",
+        "# agentmemory vault",
+        "",
+        `Exported: ${exportedAt}`,
+        "",
+        `## Memories (${stats.memories})`,
+        ...memoryMoc,
+        "",
+        `## Lessons (${stats.lessons})`,
+        ...lessonMoc,
+        "",
+        `## Crystals (${stats.crystals})`,
+        ...crystalMoc,
+        "",
+        `## Sessions (${stats.sessions})`,
+        ...sessionMoc,
+      ].join("\n");
+
+      await writeFile(join(vaultDir, "MOC.md"), moc);
+
+      await recordAudit(kv, "obsidian_export", "mem::obsidian-export", [], {
+        vaultDir,
+        stats,
+      });
+
+      return { success: true, exported: stats, errors: errors.length > 0 ? errors : undefined, vaultDir };
+    },
+  );
+}