@aisystemresources/emdee 0.1.0

package/src/lib/owner/identity.ts

@@ -0,0 +1,65 @@
+// SPRINT-058 (SIG-006): owner-node naming.
+//
+// The owner node is the user's personal subtree under EMDEE root. Per the
+// canonical 4-node vault structure (VAULT / SHARED / GRAVEYARD / MINE),
+// MINE is named per-user — by default derived from the user's primary
+// email local-part, then renameable any time via the standard rename_doc
+// MCP tool.
+
+/**
+ * Derive the default owner-node title from an email address.
+ *
+ *   "elz.work@gmail.com"  → "ELZ-WORK"
+ *   "junior_lin@..."      → "JUNIOR-LIN"
+ *   "song.wenjuan@..."    → "SONG-WENJUAN"
+ *   ""                    → "OWNER" (fallback)
+ *
+ * Rules:
+ *   - take local-part (everything before @)
+ *   - uppercase ASCII
+ *   - dots / underscores → hyphens
+ *   - drop everything that isn't [A-Z0-9-]
+ *   - collapse repeated hyphens
+ *   - strip leading/trailing hyphens
+ *   - fall back to "OWNER" if the result is empty (e.g. all-non-ASCII)
+ *
+ * Matches SPRINT-055's filename uppercase convention so the file written
+ * to the vault is `<TITLE>.md`, identical shape to other EMDEE filenames.
+ */
+/**
+ * Normalise any free-text name into a valid owner-node title.
+ * Same rules as email-derived titles so renames stay consistent.
+ */
+export function normalizeOwnerTitle(input: string): string {
+  const normalized = input
+    .trim()
+    .toUpperCase()
+    .replace(/[._\s]/g, "-")
+    .replace(/[^A-Z0-9-]/g, "")
+    .replace(/-+/g, "-")
+    .replace(/^-|-$/g, "");
+  return normalized || "OWNER";
+}
+
+export function ownerTitleFromEmail(email: string): string {
+  const localPart = (email.split("@")[0] ?? "").trim();
+  return normalizeOwnerTitle(localPart);
+}
+
+/** The default markdown scaffold for a fresh owner node. */
+export function ownerNodeScaffold(title: string): string {
+  return `# ${title}
+
+> Your personal subtree. Top-level content (projects, people, notes, etc.) lives here. Renameable any time via \`rename_doc\` — inbound wiki-link references update atomically across the vault.
+
+## Child of
+
+* [[EMDEE]]
+
+## Parent of
+
+## Associated with
+
+## Notes
+`;
+}

package/src/lib/storage/FilesystemStorage.ts

@@ -0,0 +1,88 @@
+import { mkdir, readdir, readFile, rm, stat, writeFile } from "node:fs/promises";
+import path from "node:path";
+import type { VaultFile, VaultStorage } from "./VaultStorage";
+
+/**
+ * Filesystem-backed vault used in local development. Paths are relative to
+ * `rootDir`. Used by /api/index, /api/doc, etc. when `EMDEE_DOCS` is set so
+ * the dev experience reads/writes ./docs directly instead of round-tripping
+ * through Supabase.
+ */
+export class FilesystemStorage implements VaultStorage {
+  constructor(private rootDir: string) {}
+
+  async list(prefix?: string): Promise<VaultFile[]> {
+    const start = prefix
+      ? path.join(this.rootDir, prefix.replace(/\/$/, ""))
+      : this.rootDir;
+    try {
+      await stat(start);
+    } catch {
+      return [];
+    }
+    const collected: VaultFile[] = [];
+    await this.walk(start, collected);
+    return collected;
+  }
+
+  private async walk(dir: string, out: VaultFile[]): Promise<void> {
+    const entries = await readdir(dir, { withFileTypes: true });
+    for (const e of entries) {
+      if (e.name.startsWith(".")) continue;
+      const full = path.join(dir, e.name);
+      if (e.isDirectory()) {
+        await this.walk(full, out);
+      } else if (e.isFile() && e.name.endsWith(".md")) {
+        const s = await stat(full);
+        out.push({
+          path: path.relative(this.rootDir, full),
+          content: "",
+          updatedAt: s.mtime.toISOString(),
+        });
+      }
+    }
+  }
+
+  /** FS already lists metadata without bodies — listMeta is an alias. */
+  async listMeta(prefix?: string): Promise<VaultFile[]> {
+    return this.list(prefix);
+  }
+
+  async listWithContent(prefix?: string): Promise<VaultFile[]> {
+    const listed = await this.list(prefix);
+    return Promise.all(
+      listed.map(async (f) => ({
+        path: f.path,
+        content: (await this.read(f.path)) ?? "",
+        updatedAt: f.updatedAt,
+      }))
+    );
+  }
+
+  async read(filePath: string): Promise<string | null> {
+    try {
+      return await readFile(path.join(this.rootDir, filePath), "utf8");
+    } catch {
+      return null;
+    }
+  }
+
+  async write(filePath: string, content: string): Promise<void> {
+    const full = path.join(this.rootDir, filePath);
+    await mkdir(path.dirname(full), { recursive: true });
+    await writeFile(full, content, "utf8");
+  }
+
+  async delete(filePath: string): Promise<void> {
+    await rm(path.join(this.rootDir, filePath), { force: true });
+  }
+
+  async exists(filePath: string): Promise<boolean> {
+    try {
+      await stat(path.join(this.rootDir, filePath));
+      return true;
+    } catch {
+      return false;
+    }
+  }
+}

package/src/lib/storage/SupabaseStorage.ts

@@ -0,0 +1,358 @@
+import { adminClient } from "../supabase/admin";
+import { syncDocEdges, deleteDocEdges } from "../../core/syncDocEdges";
+import { bustVaultCache } from "../cache/bust";
+import {
+  publishNamespaceInvalidate,
+  subscribeNamespaceInvalidate,
+} from "../cache/invalidation";
+import { hashBody } from "../mcp/tools/sections";
+import { deriveSummary } from "../../core/indexer";
+import type { VaultFile, VaultStorage } from "./VaultStorage";
+
+const BUCKET = "vaults";
+const CACHE_TABLE = "vault_files";
+
+// SPRINT-035: module-scope TTL cache for list / listMeta / listWithContent.
+// Eliminates the storage.search amplification documented in the
+// 2026-05-31 IDEAS investigation. Promise-cached so concurrent misses on
+// the same key dedupe to one underlying call. Writes invalidate via the
+// shared pub/sub hub.
+//
+// Key format: `${method}:${prefix ?? ""}`. Methods: "list", "listMeta",
+// "listWithContent". TTL bounded so silent external mutations are
+// visible within one minute.
+interface ListCacheEntry {
+  promise: Promise<VaultFile[]>;
+  expiresAt: number;
+}
+const listCache = new Map<string, ListCacheEntry>();
+const LIST_CACHE_TTL_MS = 60_000;
+
+const cacheStats = {
+  listHits: 0,
+  listMisses: 0,
+  invalidations: 0,
+};
+
+function namespaceFromKey(key: string): string {
+  // key = `${method}:${prefix}`; prefix = `${ns}/...` or `${ns}` or "".
+  const colon = key.indexOf(":");
+  const prefix = key.slice(colon + 1).replace(/\/$/, "");
+  const slash = prefix.indexOf("/");
+  return slash === -1 ? prefix : prefix.slice(0, slash);
+}
+
+function evictListCacheForNamespace(namespace: string): void {
+  let n = 0;
+  for (const key of listCache.keys()) {
+    if (namespaceFromKey(key) === namespace) {
+      listCache.delete(key);
+      n++;
+    }
+  }
+  cacheStats.invalidations += n;
+}
+
+// Subscribe at module init so any write through any SupabaseStorage
+// instance — or any other namespace-publishing path — clears the cache.
+subscribeNamespaceInvalidate(evictListCacheForNamespace);
+
+async function withListCache(
+  method: string,
+  prefix: string | undefined,
+  slowPath: () => Promise<VaultFile[]>,
+): Promise<VaultFile[]> {
+  const key = `${method}:${prefix ?? ""}`;
+  const hit = listCache.get(key);
+  if (hit && hit.expiresAt > Date.now()) {
+    cacheStats.listHits++;
+    return hit.promise;
+  }
+  const promise = slowPath().catch((e) => {
+    // Don't cache failures — a transient blip shouldn't poison the cache.
+    listCache.delete(key);
+    throw e;
+  });
+  listCache.set(key, { promise, expiresAt: Date.now() + LIST_CACHE_TTL_MS });
+  cacheStats.listMisses++;
+  return promise;
+}
+
+export function getStorageCacheStats(): {
+  hits: number;
+  misses: number;
+  invalidations: number;
+  size: number;
+  hitRate: number;
+} {
+  const total = cacheStats.listHits + cacheStats.listMisses;
+  return {
+    hits: cacheStats.listHits,
+    misses: cacheStats.listMisses,
+    invalidations: cacheStats.invalidations,
+    size: listCache.size,
+    hitRate: total === 0 ? 0 : cacheStats.listHits / total,
+  };
+}
+
+/**
+ * Split a full storage path ("user_X/foo/bar.md") into the cache table's
+ * composite key (namespace, file_path). Returns null for unprefixed paths
+ * which shouldn't happen in practice but we tolerate.
+ */
+function splitNs(p: string): { namespace: string; file_path: string } | null {
+  const i = p.indexOf("/");
+  if (i === -1) return null;
+  return { namespace: p.slice(0, i), file_path: p.slice(i + 1) };
+}
+
+export class SupabaseStorage implements VaultStorage {
+  private bucket() {
+    return adminClient().storage.from(BUCKET);
+  }
+
+  async list(prefix?: string): Promise<VaultFile[]> {
+    return withListCache("list", prefix, async () => {
+      const folder = prefix ? prefix.replace(/\/$/, "") : "";
+      return this.walkFolder(folder);
+    });
+  }
+
+  /**
+   * Metadata-only list. SPRINT-024 Phase 2 — selects only the columns the
+   * caller needs (namespace, file_path, updated_at) so the wire payload is
+   * a fraction of `listWithContent`'s. Used by change-polling and other
+   * surfaces that just want "what files exist + when did they last move".
+   *
+   * Like `listWithContent`, the fast path requires a single-segment
+   * namespace prefix; deeper prefixes fall back to walking Storage and
+   * stamping empty bodies.
+   */
+  async listMeta(prefix?: string): Promise<VaultFile[]> {
+    return withListCache("listMeta", prefix, async () => {
+      const folder = prefix ? prefix.replace(/\/$/, "") : "";
+      if (!folder || folder.includes("/")) {
+        return this.walkFolder(folder);
+      }
+      const { data, error } = await adminClient()
+        .from(CACHE_TABLE)
+        .select("file_path, updated_at")
+        .eq("namespace", folder);
+      if (!error && data) {
+        return data.map((r) => ({
+          path: `${folder}/${r.file_path}`,
+          content: "",
+          updatedAt: r.updated_at as string,
+        }));
+      }
+      return this.walkFolder(folder);
+    });
+  }
+
+  private async walkFolder(folder: string): Promise<VaultFile[]> {
+    const { data, error } = await this.bucket().list(folder || undefined, { limit: 1000 });
+    if (error || !data) return [];
+
+    const results: VaultFile[] = [];
+    await Promise.all(
+      data.map(async (item) => {
+        const itemPath = folder ? `${folder}/${item.name}` : item.name;
+        if (item.id === null) {
+          results.push(...(await this.walkFolder(itemPath)));
+        } else if (item.name.endsWith(".md")) {
+          results.push({
+            path: itemPath,
+            content: "",
+            updatedAt: item.updated_at ?? new Date(0).toISOString(),
+          });
+        }
+      })
+    );
+    return results;
+  }
+
+  /**
+   * Bulk-fetch every file under `prefix` via the vault_files cache. The
+   * cache is a derived index of Storage — if it's empty for a namespace
+   * we fall back to listing Storage and downloading each file, then
+   * repopulate the cache so subsequent reads are fast.
+   */
+  async listWithContent(prefix?: string): Promise<VaultFile[]> {
+    return withListCache("listWithContent", prefix, async () => {
+      const folder = prefix ? prefix.replace(/\/$/, "") : "";
+      // Postgres-cache is keyed by namespace; only fast-path when we have one.
+      if (!folder || folder.includes("/")) {
+        return this.bulkReadFromStorage(folder);
+      }
+
+      const admin = adminClient();
+      const { data, error } = await admin
+        .from(CACHE_TABLE)
+        .select("file_path, content, updated_at")
+        .eq("namespace", folder);
+
+      if (!error && data && data.length > 0) {
+        return data.map((r) => ({
+          path: `${folder}/${r.file_path}`,
+          content: r.content,
+          updatedAt: r.updated_at,
+        }));
+      }
+
+      // Cache miss (or error) — pay the slow path once and repopulate.
+      const files = await this.bulkReadFromStorage(folder);
+      if (files.length > 0) {
+        const rows = files
+          .map((f) => {
+            const split = splitNs(f.path);
+            if (!split) return null;
+            return { namespace: split.namespace, file_path: split.file_path, content: f.content };
+          })
+          .filter((r): r is { namespace: string; file_path: string; content: string } => r !== null);
+        if (rows.length > 0) {
+          await admin.from(CACHE_TABLE).upsert(rows, { onConflict: "namespace,file_path" });
+        }
+      }
+      return files;
+    });
+  }
+
+  private async bulkReadFromStorage(folder: string): Promise<VaultFile[]> {
+    const listed = await this.walkFolder(folder);
+    return Promise.all(
+      listed.map(async (f) => ({
+        path: f.path,
+        content: (await this.read(f.path)) ?? "",
+        updatedAt: f.updatedAt,
+      }))
+    );
+  }
+
+  async read(filePath: string): Promise<string | null> {
+    const split = splitNs(filePath);
+    if (split) {
+      const { data: cacheRow } = await adminClient()
+        .from(CACHE_TABLE)
+        .select("content")
+        .match({ namespace: split.namespace, file_path: split.file_path })
+        .maybeSingle();
+      if (cacheRow) return (cacheRow.content as string) ?? "";
+    }
+    // Fall back to Storage for unsplit paths or cache misses; repopulate
+    // the cache on success so subsequent reads stay fast.
+    const { data, error } = await this.bucket().download(filePath);
+    if (error || !data) return null;
+    const content = await data.text();
+    if (split) {
+      try {
+        await adminClient()
+          .from(CACHE_TABLE)
+          .upsert(
+            { namespace: split.namespace, file_path: split.file_path, content, updated_at: new Date().toISOString() },
+            { onConflict: "namespace,file_path" }
+          );
+      } catch {}
+    }
+    return content;
+  }
+
+  async write(filePath: string, content: string): Promise<void> {
+    // Storage is canonical — fail here propagates.
+    const blob = new Blob([content], { type: "text/markdown; charset=utf-8" });
+    const { error } = await this.bucket().upload(filePath, blob, {
+      upsert: true,
+      contentType: "text/markdown; charset=utf-8",
+    });
+    if (error) throw new Error(`storage write failed: ${error.message}`);
+
+    // Cache mirror — best-effort. If this fails the cache is stale; on
+    // next read we'd return stale content, but a rebuild script
+    // (scripts/backfill-vault-files.mjs) restores parity.
+    const split = splitNs(filePath);
+    if (!split) return;
+    try {
+      const admin = adminClient();
+
+      // SPRINT-081: record summary drift hashes. Snapshot the current
+      // content hash into `content_hash_at_summary_write` only when the
+      // summary itself changed — that's how drift accretes (body edits
+      // shift the live content hash without shifting the snapshot).
+      const summary = deriveSummary(content);
+      const summaryHash = summary ? hashBody(summary) : "";
+      const contentHash = hashBody(content);
+
+      const { data: existing } = await admin
+        .from(CACHE_TABLE)
+        .select("summary_hash")
+        .match({ namespace: split.namespace, file_path: split.file_path })
+        .maybeSingle();
+      const oldSummaryHash = (existing?.summary_hash as string | null) ?? null;
+      const summaryChanged = oldSummaryHash !== summaryHash;
+
+      const row: Record<string, unknown> = {
+        namespace: split.namespace,
+        file_path: split.file_path,
+        content,
+        updated_at: new Date().toISOString(),
+        summary_hash: summaryHash,
+      };
+      if (summaryChanged) row.content_hash_at_summary_write = contentHash;
+
+      const { error: cacheErr } = await admin
+        .from(CACHE_TABLE)
+        .upsert(row, { onConflict: "namespace,file_path" });
+      if (cacheErr) console.error(`vault_files cache write failed for ${filePath}:`, cacheErr.message);
+    } catch (e) {
+      console.error(`vault_files cache write threw for ${filePath}:`, e);
+    }
+
+    // SPRINT-018 Phase 2: keep doc_edges in sync after the cache mirror.
+    // Sync errors propagate — the bucket + cache have already committed
+    // (documented asymmetry), but a failed sync means the edges are
+    // wrong and the caller should know about it (HTTP 500).
+    await syncDocEdges(adminClient(), split.namespace, split.file_path, content);
+
+    // SPRINT-024 Phase 3: bust the edge cache for this namespace + path.
+    // Personal namespaces aren't cached (no-store) so this is effectively
+    // a no-op for them; for `public` it invalidates the s-maxage tier.
+    bustVaultCache(split.namespace, split.file_path);
+
+    // SPRINT-035: clear the in-memory list cache + any subscriber's
+    // namespace memo (e.g. vault.ts loadVaultIndex memo) so subsequent
+    // reads see the write without waiting for TTL.
+    publishNamespaceInvalidate(split.namespace);
+  }
+
+  async delete(filePath: string): Promise<void> {
+    await this.bucket().remove([filePath]);
+    const split = splitNs(filePath);
+    if (!split) return;
+    try {
+      const { error: cacheErr } = await adminClient()
+        .from(CACHE_TABLE)
+        .delete()
+        .match({ namespace: split.namespace, file_path: split.file_path });
+      if (cacheErr) console.error(`vault_files cache delete failed for ${filePath}:`, cacheErr.message);
+    } catch (e) {
+      console.error(`vault_files cache delete threw for ${filePath}:`, e);
+    }
+
+    // SPRINT-018 Phase 2: drop every edge touching this doc. Sync errors
+    // propagate — see comment on write() above for the asymmetry note.
+    await deleteDocEdges(adminClient(), split.namespace, split.file_path);
+
+    // SPRINT-024 Phase 3: see write() for cache busting rationale.
+    bustVaultCache(split.namespace, split.file_path);
+
+    // SPRINT-035: see write() — invalidate in-memory caches for this ns.
+    publishNamespaceInvalidate(split.namespace);
+  }
+
+  async exists(filePath: string): Promise<boolean> {
+    const parts = filePath.split("/");
+    const name = parts.pop()!;
+    const folder = parts.join("/");
+    const { data } = await this.bucket().list(folder || undefined, { search: name, limit: 1 });
+    return (data ?? []).some((f) => f.name === name && f.id !== null);
+  }
+}

package/src/lib/storage/VaultStorage.ts

@@ -0,0 +1,35 @@
+export interface VaultFile {
+  path: string;       // relative path, e.g. "projects/SILENTMANE/BUILD.md"
+  content: string;
+  updatedAt: string;  // ISO 8601
+}
+
+export interface VaultStorage {
+  /** List all .md files, optionally filtered to a path prefix. */
+  list(prefix?: string): Promise<VaultFile[]>;
+  /**
+   * Metadata-only list (path + updatedAt, no body). Cheaper than `list()`
+   * for cloud — pulls from the vault_files cache instead of recursing
+   * Storage. Use this whenever the caller doesn't need bodies (change
+   * polling, summary endpoints). Returns the same VaultFile shape with
+   * `content: ""` for compat.
+   *
+   * SPRINT-024 Phase 2.
+   */
+  listMeta(prefix?: string): Promise<VaultFile[]>;
+  /**
+   * Bulk-read every .md file under `prefix` in one shot. Implementations
+   * are encouraged to use a single round-trip — SupabaseStorage hits the
+   * vault_files Postgres cache so /api/index doesn't pay per-file HTTPS.
+   * FilesystemStorage just reads from disk.
+   */
+  listWithContent(prefix?: string): Promise<VaultFile[]>;
+  /** Read a single file. Returns null if not found. */
+  read(path: string): Promise<string | null>;
+  /** Write (create or overwrite) a file. */
+  write(path: string, content: string): Promise<void>;
+  /** Delete a file. No-op if not found. */
+  delete(path: string): Promise<void>;
+  /** Returns true if a file exists at path. */
+  exists(path: string): Promise<boolean>;
+}

package/src/lib/storage/index.ts

@@ -0,0 +1,41 @@
+import path from "node:path";
+import { SupabaseStorage } from "./SupabaseStorage";
+import { FilesystemStorage } from "./FilesystemStorage";
+import type { VaultStorage } from "./VaultStorage";
+
+export type { VaultStorage, VaultFile } from "./VaultStorage";
+export { SupabaseStorage } from "./SupabaseStorage";
+export { FilesystemStorage } from "./FilesystemStorage";
+
+/**
+ * Resolve the active vault storage based on the environment.
+ *
+ * - **Local dev** (`EMDEE_DOCS` set): every namespace maps to the same
+ *   on-disk folder, so the dev server reads/writes ./docs directly. The
+ *   {userId} in URLs is ignored — there's one local vault.
+ * - **Cloud** (no `EMDEE_DOCS`): SupabaseStorage scoped to `{ns}/`, so
+ *   each Clerk user has an isolated namespace inside the `vaults` bucket.
+ *
+ * Returns the storage, the path-prefix to prepend to relative paths, and
+ * a flag callers can use to skip auth gates / namespace checks in local
+ * mode (the filesystem is single-tenant by definition).
+ */
+export function getVaultStorage(ns: string): {
+  storage: VaultStorage;
+  prefix: string;
+  isLocal: boolean;
+} {
+  const docsDir = process.env.EMDEE_DOCS;
+  if (docsDir) {
+    return {
+      storage: new FilesystemStorage(path.resolve(docsDir)),
+      prefix: "",
+      isLocal: true,
+    };
+  }
+  return {
+    storage: new SupabaseStorage(),
+    prefix: `${ns}/`,
+    isLocal: false,
+  };
+}

package/src/lib/supabase/admin.ts

@@ -0,0 +1,16 @@
+import { createClient } from "@supabase/supabase-js";
+
+// Service-role client — bypasses RLS, only used in server-side API routes.
+export function adminClient() {
+  const key = process.env.SUPABASE_SECRET_KEY ?? process.env.SUPABASE_SERVICE_ROLE_KEY;
+  return createClient(
+    process.env.NEXT_PUBLIC_SUPABASE_URL!,
+    key!,
+    { auth: { autoRefreshToken: false, persistSession: false } }
+  );
+}
+
+export async function hashToken(token: string): Promise<string> {
+  const buf = await crypto.subtle.digest("SHA-256", new TextEncoder().encode(token));
+  return Array.from(new Uint8Array(buf), (b) => b.toString(16).padStart(2, "0")).join("");
+}

package/src/lib/supabase/client.ts

@@ -0,0 +1,8 @@
+import { createBrowserClient } from "@supabase/ssr";
+
+export function createClient() {
+  return createBrowserClient(
+    process.env.NEXT_PUBLIC_SUPABASE_URL!,
+    process.env.NEXT_PUBLIC_SUPABASE_ANON_KEY!
+  );
+}