@agenticmail/core 0.7.5 → 0.9.0

package/README.md

@@ -10,7 +10,16 @@ This is the foundation layer that everything else builds on. If the API server,
 
 Every other AgenticMail package depends on this one.
 
-## ✨ What's new in 0.7.4
+## ✨ What's new in 0.9.0
+
+- **🧠 New `threading/` module.** Three exported primitives for the dispatcher's wake-context layer:
+  - **`threadIdFor({subject})`** — stable, subject-only thread id (replies collapse into the same thread regardless of sender). SHA-256 base64url truncated to 16 chars. Strips chained `Re:`/`Fwd:` prefixes + `[FINAL]/[DONE]/[CLOSED]/[WRAP]` markers + collapses whitespace + lower-cases.
+  - **`ThreadCache`** — per-thread ring buffer of the last K message envelopes + previews on disk at `~/.agenticmail/thread-cache/<t>.json`. Configurable K, LRU-bounded directory eviction (default cap 5000 threads ≈ 25 MB). Atomic writes via rename.
+  - **`AgentMemoryStore`** — per-`(agent, thread)` markdown files at `~/.agenticmail/agent-memory/<agentId>/<t>.md`. Agent writes structured fields (summary, commitments, openQuestions, lastAction, lastUid) at end-of-wake; on the next wake the dispatcher reads the raw markdown back into the prompt. Per-agent isolation — one agent's memory is invisible to another sharing the same thread.
+
+23 new unit tests covering normalization, hash stability, push/dedup/cap/delete, write/read round-trip, and per-agent isolation.
+
+## ✨ Earlier — 0.7.4
 
 - **⭐ `MailReceiver.setStarred(uid, starred, mailbox?)`** — toggles IMAP's `\Flagged` flag via `messageFlagsAdd` / `messageFlagsRemove`. Same on-disk bit Gmail's star uses. Underpins the new `POST /mail/messages/:uid/star` route in `@agenticmail/api`.
 - **📐 Task `output_schema` column** — migration `014_task_output_schema.sql` adds an optional `output_schema TEXT` column to `agent_tasks`. Stores the assigner-supplied JSON Schema for typed task contracts. `NULL` means "no schema, accept anything", fully back-compat with the v0.8.x task model.

package/dist/index.cjs

@@ -711,6 +711,7 @@ __export(index_exports, {
   AGENT_ROLES: () => AGENT_ROLES,
   AccountManager: () => AccountManager,
   AgentDeletionService: () => AgentDeletionService,
+  AgentMemoryStore: () => AgentMemoryStore,
   AgenticMailClient: () => AgenticMailClient,
   CloudflareClient: () => CloudflareClient,
   DEFAULT_AGENT_NAME: () => DEFAULT_AGENT_NAME,
@@ -734,6 +735,7 @@ __export(index_exports, {
   SmsManager: () => SmsManager,
   SmsPoller: () => SmsPoller,
   StalwartAdmin: () => StalwartAdmin,
+  ThreadCache: () => ThreadCache,
   TunnelManager: () => TunnelManager,
   WARNING_THRESHOLD: () => WARNING_THRESHOLD,
   buildInboundSecurityAdvisory: () => buildInboundSecurityAdvisory,
@@ -748,7 +750,9 @@ __export(index_exports, {
   getDatabase: () => getDatabase,
   isInternalEmail: () => isInternalEmail,
   isValidPhoneNumber: () => isValidPhoneNumber,
+  normalizeAddress: () => normalizeAddress,
   normalizePhoneNumber: () => normalizePhoneNumber,
+  normalizeSubject: () => normalizeSubject,
   parseEmail: () => parseEmail,
   parseGoogleVoiceSms: () => parseGoogleVoiceSms,
   recordToolCall: () => recordToolCall,
@@ -758,7 +762,8 @@ __export(index_exports, {
   scanOutboundEmail: () => scanOutboundEmail,
   scoreEmail: () => scoreEmail,
   setTelemetryVersion: () => setTelemetryVersion,
-  startRelayBridge: () => startRelayBridge
+  startRelayBridge: () => startRelayBridge,
+  threadIdFor: () => threadIdFor
 });
 module.exports = __toCommonJS(index_exports);
 
@@ -1841,14 +1846,14 @@ var StalwartAdmin = class {
     if (!isValidDomain(domain)) {
       throw new Error(`Invalid domain format: "${domain}"`);
     }
-    const { readFileSync: readFileSync5, writeFileSync: writeFileSync6 } = await import("fs");
-    const { homedir: homedir9 } = await import("os");
-    const { join: join10 } = await import("path");
-    const configPath = join10(homedir9(), ".agenticmail", "stalwart.toml");
+    const { readFileSync: readFileSync7, writeFileSync: writeFileSync8 } = await import("fs");
+    const { homedir: homedir11 } = await import("os");
+    const { join: join12 } = await import("path");
+    const configPath = join12(homedir11(), ".agenticmail", "stalwart.toml");
     try {
-      let config = readFileSync5(configPath, "utf-8");
+      let config = readFileSync7(configPath, "utf-8");
       config = config.replace(/^hostname\s*=\s*"[^"]*"/m, `hostname = "${escapeTomlString(domain)}"`);
-      writeFileSync6(configPath, config);
+      writeFileSync8(configPath, config);
       console.log(`[Stalwart] Updated hostname to "${domain}" in stalwart.toml`);
     } catch (err) {
       throw new Error(`Failed to set config server.hostname=${domain}`);
@@ -1857,15 +1862,15 @@ var StalwartAdmin = class {
   // --- DKIM ---
   /** Path to the host-side stalwart.toml (mounted read-only into container) */
   get configPath() {
-    const { homedir: homedir9 } = require("os");
-    const { join: join10 } = require("path");
-    return join10(homedir9(), ".agenticmail", "stalwart.toml");
+    const { homedir: homedir11 } = require("os");
+    const { join: join12 } = require("path");
+    return join12(homedir11(), ".agenticmail", "stalwart.toml");
   }
   /** Path to host-side DKIM key directory */
   get dkimDir() {
-    const { homedir: homedir9 } = require("os");
-    const { join: join10 } = require("path");
-    return join10(homedir9(), ".agenticmail");
+    const { homedir: homedir11 } = require("os");
+    const { join: join12 } = require("path");
+    return join12(homedir11(), ".agenticmail");
   }
   /**
    * Create/reuse a DKIM signing key for a domain.
@@ -1966,12 +1971,12 @@ var StalwartAdmin = class {
    * This bypasses the need for a PTR record on the sending IP.
    */
   async configureOutboundRelay(config) {
-    const { readFileSync: readFileSync5, writeFileSync: writeFileSync6 } = await import("fs");
-    const { homedir: homedir9 } = await import("os");
-    const { join: join10 } = await import("path");
+    const { readFileSync: readFileSync7, writeFileSync: writeFileSync8 } = await import("fs");
+    const { homedir: homedir11 } = await import("os");
+    const { join: join12 } = await import("path");
     const routeName = config.routeName ?? "gmail";
-    const tomlPath = join10(homedir9(), ".agenticmail", "stalwart.toml");
-    let toml = readFileSync5(tomlPath, "utf-8");
+    const tomlPath = join12(homedir11(), ".agenticmail", "stalwart.toml");
+    let toml = readFileSync7(tomlPath, "utf-8");
     toml = toml.replace(/\n\[queue\.route\.gmail\][\s\S]*?(?=\n\[|$)/, "");
     toml = toml.replace(/\n\[queue\.strategy\][\s\S]*?(?=\n\[|$)/, "");
     const safeRouteName = routeName.replace(/[^a-zA-Z0-9_-]/g, "");
@@ -1991,7 +1996,7 @@ auth.secret = "${escapeTomlString(config.password)}"
 route = [ { if = "is_local_domain('', rcpt_domain)", then = "'local'" },
            { else = "'${safeRouteName}'" } ]
 `;
-    writeFileSync6(tomlPath, toml, "utf-8");
+    writeFileSync8(tomlPath, toml, "utf-8");
     await this.restartContainer();
   }
 };
@@ -3604,6 +3609,18 @@ CREATE INDEX IF NOT EXISTS idx_pending_notification ON pending_outbound(notifica
 -- Column is optional; NULL means "no schema, accept anything" (the
 -- v0.8.x behaviour, fully back-compat).
 ALTER TABLE agent_tasks ADD COLUMN output_schema TEXT;
+`,
+  "015_draft_attachments.sql": `
+-- Persist attachments alongside their draft.
+--
+-- Stored as a JSON array on the row: each entry is
+-- { filename, contentType, content (base64), size }. The web UI
+-- cap is 20 MB total per draft, which SQLite handles fine without
+-- bloating other queries \u2014 the column is only fetched on the
+-- per-draft GET (not on the list endpoint) so the Drafts sidebar
+-- stays snappy. NULL means "no attachments", fully back-compat
+-- with rows from before this migration.
+ALTER TABLE drafts ADD COLUMN attachments TEXT;
 `
 };
 function runMigrations(database) {
@@ -5852,12 +5869,12 @@ var GatewayManager = class {
       zone = await this.cfClient.createZone(domain);
     }
     const existingRecords = await this.cfClient.listDnsRecords(zone.id);
-    const { homedir: homedir9 } = await import("os");
-    const backupDir = (0, import_node_path4.join)(homedir9(), ".agenticmail");
+    const { homedir: homedir11 } = await import("os");
+    const backupDir = (0, import_node_path4.join)(homedir11(), ".agenticmail");
     const backupPath = (0, import_node_path4.join)(backupDir, `dns-backup-${domain}-${Date.now()}.json`);
-    const { writeFileSync: writeFileSync6, mkdirSync: mkdirSync7 } = await import("fs");
-    mkdirSync7(backupDir, { recursive: true });
-    writeFileSync6(backupPath, JSON.stringify({
+    const { writeFileSync: writeFileSync8, mkdirSync: mkdirSync9 } = await import("fs");
+    mkdirSync9(backupDir, { recursive: true });
+    writeFileSync8(backupPath, JSON.stringify({
       domain,
       zoneId: zone.id,
       backedUpAt: (/* @__PURE__ */ new Date()).toISOString(),
@@ -8040,11 +8057,286 @@ secret = "${password}"
     return (0, import_node_fs7.existsSync)(configPath);
   }
 };
+
+// src/threading/thread-id.ts
+var import_node_crypto4 = require("crypto");
+function stripReplyPrefixes(subject) {
+  let s = subject;
+  for (; ; ) {
+    const next = s.replace(/^\s*(?:re|fwd?|fw)\s*(?:\[\d+\])?\s*:\s*/i, "");
+    if (next === s) break;
+    s = next;
+  }
+  return s;
+}
+function stripCoordinationMarkers(subject) {
+  return subject.replace(/\[\s*(?:final|done|closed|wrap)\s*\]/gi, " ");
+}
+function normalizeSubject(subject) {
+  if (!subject) return "(no subject)";
+  let s = stripReplyPrefixes(subject);
+  s = stripCoordinationMarkers(s);
+  s = s.replace(/\s+/g, " ").trim().toLowerCase();
+  return s || "(no subject)";
+}
+function normalizeAddress(addr) {
+  if (!addr) return "(unknown)";
+  const m = addr.match(/<([^>]+)>/);
+  const raw = m ? m[1] : addr;
+  return raw.trim().toLowerCase();
+}
+function threadIdFor(input) {
+  const subject = normalizeSubject(input.subject);
+  return (0, import_node_crypto4.createHash)("sha256").update(subject).digest("base64url").slice(0, 16);
+}
+
+// src/threading/thread-cache.ts
+var import_node_fs8 = require("fs");
+var import_node_os8 = require("os");
+var import_node_path9 = require("path");
+var CACHE_DIR_DEFAULT = (0, import_node_path9.join)((0, import_node_os8.homedir)(), ".agenticmail", "thread-cache");
+var DEFAULT_K_MESSAGES = 10;
+var DEFAULT_LRU_CAP = 5e3;
+var PREVIEW_MAX_CHARS = 240;
+var ThreadCache = class {
+  dir;
+  k;
+  lruCap;
+  constructor(opts = {}) {
+    this.dir = opts.cacheDir ?? CACHE_DIR_DEFAULT;
+    this.k = opts.k ?? DEFAULT_K_MESSAGES;
+    this.lruCap = opts.lruCap ?? DEFAULT_LRU_CAP;
+    try {
+      (0, import_node_fs8.mkdirSync)(this.dir, { recursive: true });
+    } catch {
+    }
+  }
+  pathFor(threadId) {
+    return (0, import_node_path9.join)(this.dir, `${threadId}.json`);
+  }
+  read(threadId) {
+    const p = this.pathFor(threadId);
+    if (!(0, import_node_fs8.existsSync)(p)) return null;
+    try {
+      const raw = (0, import_node_fs8.readFileSync)(p, "utf-8");
+      return JSON.parse(raw);
+    } catch {
+      try {
+        (0, import_node_fs8.rmSync)(p, { force: true });
+      } catch {
+      }
+      return null;
+    }
+  }
+  /**
+   * Append a message to the thread's cache, pruning to the K
+   * newest entries. Creates the cache entry on first write.
+   *
+   * `rootFromAddr` is the sender of the ROOT message on the
+   * thread; on a brand-new thread this is just `env.fromAddr`,
+   * on a reply it's read off the existing cache entry (callers
+   * should pass the existing entry's rootFromAddr when known).
+   */
+  pushMessage(threadId, env, meta) {
+    const existing = this.read(threadId);
+    const entry = existing ? {
+      ...existing,
+      // We re-affirm subject + rootFromAddr to existing values —
+      // the first-seen values win. Reply messages carry the
+      // replier's `from`, not the original sender's, so we'd
+      // corrupt the thread root if we overwrote here.
+      subject: existing.subject,
+      rootFromAddr: existing.rootFromAddr,
+      lastUpdated: Date.now(),
+      messages: dedupAndCap([env, ...existing.messages], this.k)
+    } : {
+      threadId,
+      subject: meta.subject,
+      rootFromAddr: meta.rootFromAddr,
+      lastUpdated: Date.now(),
+      messages: [env]
+    };
+    this.writeAtomic(threadId, entry);
+    this.maybeEvict();
+    return entry;
+  }
+  /** Permanently remove a thread's cache (called on [FINAL] / [DONE] / [CLOSED] / [WRAP]). */
+  delete(threadId) {
+    try {
+      (0, import_node_fs8.rmSync)(this.pathFor(threadId), { force: true });
+    } catch {
+    }
+  }
+  /**
+   * Render the cache as a compact text block for the wake prompt.
+   * One line per message, newest first. Empty string when the
+   * cache is empty — caller decides whether to suppress the
+   * header in that case.
+   */
+  renderForPrompt(entry) {
+    if (!entry || entry.messages.length === 0) return "";
+    return entry.messages.map((m) => {
+      const preview = m.preview.replace(/\s+/g, " ").slice(0, PREVIEW_MAX_CHARS);
+      return `- UID ${m.uid} \xB7 ${m.from} \xB7 ${m.date} \xB7 "${m.subject}" \xB7 ${preview}`;
+    }).join("\n");
+  }
+  writeAtomic(threadId, entry) {
+    const p = this.pathFor(threadId);
+    const tmp = `${p}.tmp`;
+    (0, import_node_fs8.writeFileSync)(tmp, JSON.stringify(entry), "utf-8");
+    (0, import_node_fs8.renameSync)(tmp, p);
+  }
+  /**
+   * Best-effort LRU eviction. Runs at most every 256 writes (we
+   * don't track a precise counter — `Math.random()` sampling keeps
+   * the write path cheap). When the directory has more files than
+   * `lruCap`, sort by mtime ascending and delete the oldest 10%.
+   */
+  maybeEvict() {
+    if (Math.random() > 1 / 256) return;
+    let files;
+    try {
+      files = (0, import_node_fs8.readdirSync)(this.dir).filter((f) => f.endsWith(".json"));
+    } catch {
+      return;
+    }
+    if (files.length <= this.lruCap) return;
+    const stats = files.map((f) => {
+      const p = (0, import_node_path9.join)(this.dir, f);
+      try {
+        return { p, mtime: (0, import_node_fs8.statSync)(p).mtimeMs };
+      } catch {
+        return { p, mtime: 0 };
+      }
+    });
+    stats.sort((a, b) => a.mtime - b.mtime);
+    const dropCount = Math.max(1, Math.floor(this.lruCap * 0.1));
+    for (let i = 0; i < dropCount; i++) {
+      try {
+        (0, import_node_fs8.rmSync)(stats[i].p, { force: true });
+      } catch {
+      }
+    }
+  }
+};
+function dedupAndCap(messages, k) {
+  const seen = /* @__PURE__ */ new Set();
+  const out = [];
+  for (const m of messages) {
+    if (seen.has(m.uid)) continue;
+    seen.add(m.uid);
+    out.push(m);
+    if (out.length >= k) break;
+  }
+  return out;
+}
+
+// src/threading/agent-memory.ts
+var import_node_fs9 = require("fs");
+var import_node_os9 = require("os");
+var import_node_path10 = require("path");
+var MEMORY_DIR_DEFAULT = (0, import_node_path10.join)((0, import_node_os9.homedir)(), ".agenticmail", "agent-memory");
+var AgentMemoryStore = class {
+  dir;
+  constructor(opts = {}) {
+    this.dir = opts.memoryDir ?? MEMORY_DIR_DEFAULT;
+    try {
+      (0, import_node_fs9.mkdirSync)(this.dir, { recursive: true });
+    } catch {
+    }
+  }
+  dirFor(agentId) {
+    return (0, import_node_path10.join)(this.dir, sanitizeId(agentId));
+  }
+  pathFor(agentId, threadId) {
+    return (0, import_node_path10.join)(this.dirFor(agentId), `${sanitizeId(threadId)}.md`);
+  }
+  read(agentId, threadId) {
+    const p = this.pathFor(agentId, threadId);
+    if (!(0, import_node_fs9.existsSync)(p)) return null;
+    try {
+      const raw = (0, import_node_fs9.readFileSync)(p, "utf-8");
+      const parsed = parse(raw);
+      return { ...parsed, raw };
+    } catch {
+      return null;
+    }
+  }
+  write(agentId, threadId, fields) {
+    const agentDir = this.dirFor(agentId);
+    try {
+      (0, import_node_fs9.mkdirSync)(agentDir, { recursive: true });
+    } catch {
+    }
+    const body = render({ ...fields, updatedAt: (/* @__PURE__ */ new Date()).toISOString() });
+    const p = this.pathFor(agentId, threadId);
+    const tmp = `${p}.tmp`;
+    (0, import_node_fs9.writeFileSync)(tmp, body, "utf-8");
+    (0, import_node_fs9.renameSync)(tmp, p);
+  }
+  delete(agentId, threadId) {
+    try {
+      (0, import_node_fs9.rmSync)(this.pathFor(agentId, threadId), { force: true });
+    } catch {
+    }
+  }
+  /** Render an agent's memory for injection into a wake prompt.
+   *  Returns the raw markdown if present; empty string when there's
+   *  no prior memory (the caller decides whether to suppress the
+   *  whole "Your own memory" block). */
+  renderForPrompt(memory) {
+    if (!memory) return "";
+    return memory.raw;
+  }
+};
+function sanitizeId(id) {
+  return id.replace(/[^a-zA-Z0-9._-]/g, "_");
+}
+function render(fields) {
+  const fm = ["---"];
+  fm.push(`updated_at: ${fields.updatedAt}`);
+  if (typeof fields.lastUid === "number") fm.push(`last_uid: ${fields.lastUid}`);
+  fm.push("---", "");
+  const sections = [];
+  if (fields.summary && fields.summary.trim()) {
+    sections.push(fields.summary.trim());
+  }
+  if (fields.commitments && fields.commitments.length > 0) {
+    sections.push(`### Commitments
+${fields.commitments.map((c) => `- ${c}`).join("\n")}`);
+  }
+  if (fields.openQuestions && fields.openQuestions.length > 0) {
+    sections.push(`### Open
+${fields.openQuestions.map((q) => `- ${q}`).join("\n")}`);
+  }
+  if (fields.lastAction && fields.lastAction.trim()) {
+    sections.push(`### Last action
+${fields.lastAction.trim()}`);
+  }
+  return fm.join("\n") + sections.join("\n\n") + "\n";
+}
+function parse(raw) {
+  const out = {};
+  const m = raw.match(/^---\n([\s\S]*?)\n---\n/);
+  if (m) {
+    for (const line of m[1].split("\n")) {
+      const kv = line.match(/^(\w+):\s*(.*)$/);
+      if (!kv) continue;
+      if (kv[1] === "updated_at") out.updatedAt = kv[2].trim();
+      else if (kv[1] === "last_uid") {
+        const n = parseInt(kv[2], 10);
+        if (!isNaN(n)) out.lastUid = n;
+      }
+    }
+  }
+  return out;
+}
 // Annotate the CommonJS export names for ESM import in node:
 0 && (module.exports = {
   AGENT_ROLES,
   AccountManager,
   AgentDeletionService,
+  AgentMemoryStore,
   AgenticMailClient,
   CloudflareClient,
   DEFAULT_AGENT_NAME,
@@ -8068,6 +8360,7 @@ secret = "${password}"
   SmsManager,
   SmsPoller,
   StalwartAdmin,
+  ThreadCache,
   TunnelManager,
   WARNING_THRESHOLD,
   buildInboundSecurityAdvisory,
@@ -8082,7 +8375,9 @@ secret = "${password}"
   getDatabase,
   isInternalEmail,
   isValidPhoneNumber,
+  normalizeAddress,
   normalizePhoneNumber,
+  normalizeSubject,
   parseEmail,
   parseGoogleVoiceSms,
   recordToolCall,
@@ -8092,5 +8387,6 @@ secret = "${password}"
   scanOutboundEmail,
   scoreEmail,
   setTelemetryVersion,
-  startRelayBridge
+  startRelayBridge,
+  threadIdFor
 });