@zaclinley/dsh-mailharbor 0.17.0 → 0.19.0

package/lib/index.js

@@ -1114,6 +1114,16 @@ function ensureMailSchema(file) {
     if (!columns.some((column) => column.name === "disposition")) {
       db.exec("ALTER TABLE attachments ADD COLUMN disposition TEXT");
     }
+    const emailColumns = db.prepare("PRAGMA table_info(emails)").all();
+    if (!emailColumns.some((column) => column.name === "in_reply_to")) {
+      db.exec("ALTER TABLE emails ADD COLUMN in_reply_to TEXT");
+    }
+    if (!emailColumns.some((column) => column.name === "references")) {
+      db.exec('ALTER TABLE emails ADD COLUMN "references" TEXT');
+    }
+    if (!emailColumns.some((column) => column.name === "refs_indexed")) {
+      db.exec("ALTER TABLE emails ADD COLUMN refs_indexed INTEGER DEFAULT 0");
+    }
   } finally {
     db.close();
   }
@@ -1124,6 +1134,7 @@ var MailDatabase = class {
   db;
   ftsChecked = false;
   ftsAvailable = false;
+  threadRefs = /* @__PURE__ */ new Map();
   constructor(file, options = {}) {
     this.file = file;
     this.options = options;
@@ -1150,6 +1161,14 @@ var MailDatabase = class {
       return false;
     }
   }
+  hasRefColumns() {
+    try {
+      const columns = this.open().prepare("PRAGMA table_info(emails)").all();
+      return columns.some((column) => column.name === "in_reply_to");
+    } catch {
+      return false;
+    }
+  }
   attachmentCountSql() {
     if (!this.hasAttachmentColumns()) {
       return "(SELECT COUNT(*) FROM attachments a WHERE a.email_id = e.id) AS attachment_count";
@@ -1169,6 +1188,7 @@ var MailDatabase = class {
     } catch {
     }
     this.db = void 0;
+    this.threadRefs.clear();
   }
   status() {
     try {
@@ -1360,13 +1380,174 @@ var MailDatabase = class {
       (entry) => text2 === "" || entry.address.toLowerCase().includes(text2) || entry.name.toLowerCase().includes(text2)
     ).sort((left, right) => right.count - left.count).slice(0, limit);
   }
-  /** All archived messages in the same conversation (normalized subject). */
-  thread(subject, limit = 200) {
+  /**
+   * Conversation for one message.
+   *
+   * Subject grouping is only the candidate pool; inside it we read
+   * In-Reply-To/References from the stored raw message and keep the connected
+   * component of the seed. That stops unrelated mail with the same subject
+   * from being merged while still working for old backups.
+   */
+  thread(seed, limit = 200) {
     const normalize2 = (value) => value.replace(/^(\s*(re|fw|fwd|回复|转发)\s*[:：]\s*)+/gi, "").replace(/\s+/g, " ").trim().toLowerCase();
-    const base = normalize2(subject);
+    const base = normalize2(seed.subject);
     if (base === "") return [];
-    const page = this.list({ mailbox: "all", query: base, limit: Math.min(500, limit * 2) });
-    return page.items.filter((item) => normalize2(item.subject) === base).slice(0, limit);
+    const seedIdForQuery = (seed.messageId ?? "").replace(/^<|>$/g, "").toLowerCase();
+    const candidates = this.threadCandidates(base, limit, seedIdForQuery);
+    if (candidates.length <= 1) return candidates;
+    const refs = this.referencesFor(candidates.slice(0, 120).map((item) => Number(item.id)));
+    const byMessageId = /* @__PURE__ */ new Map();
+    for (const item of candidates) {
+      const id = item.messageId === void 0 ? "" : item.messageId.replace(/^<|>$/g, "").toLowerCase();
+      if (id !== "") byMessageId.set(id, Number(item.id));
+    }
+    const seedId = (seed.messageId ?? "").replace(/^<|>$/g, "").toLowerCase();
+    const startId = byMessageId.has(seedId) ? seedId : (candidates[0]?.messageId ?? "").replace(/^<|>$/g, "").toLowerCase();
+    if (startId === "" || !byMessageId.has(startId)) return candidates;
+    const neighbours = /* @__PURE__ */ new Map();
+    const link = (left, right) => {
+      if (left === right) return;
+      if (!neighbours.has(left)) neighbours.set(left, /* @__PURE__ */ new Set());
+      if (!neighbours.has(right)) neighbours.set(right, /* @__PURE__ */ new Set());
+      neighbours.get(left)?.add(right);
+      neighbours.get(right)?.add(left);
+    };
+    let edges = 0;
+    for (const item of candidates) {
+      const local = refs.get(Number(item.id));
+      if (local === void 0 || local.refs.length === 0) continue;
+      for (const ref of local.refs) {
+        const target = byMessageId.get(ref);
+        if (target !== void 0) {
+          link(Number(item.id), target);
+          edges += 1;
+        }
+      }
+    }
+    if (process.env.MAILHARBOR_DEBUG_THREAD === "1") {
+      console.error("THREAD", {
+        base,
+        candidates: candidates.length,
+        edges,
+        refs: [...refs.entries()].slice(0, 3)
+      });
+    }
+    if (edges === 0) return candidates;
+    const seen = /* @__PURE__ */ new Set();
+    const queue = [byMessageId.get(startId)];
+    while (queue.length > 0) {
+      const id = queue.shift();
+      if (seen.has(id)) continue;
+      seen.add(id);
+      for (const next of neighbours.get(id) ?? []) {
+        if (!seen.has(next)) queue.push(next);
+      }
+    }
+    if (seen.size <= 1) return candidates;
+    return candidates.filter((item) => seen.has(Number(item.id)));
+  }
+  /** Subject candidates via a distinctive token, not FTS (punctuation-safe). */
+  threadCandidates(base, limit, seedId = "") {
+    const normalize2 = (value) => value.replace(/^(\s*(re|fw|fwd|回复|转发)\s*[:：]\s*)+/gi, "").replace(/\s+/g, " ").trim().toLowerCase();
+    const words = base.split(/[^\p{L}\p{N}]+/u).filter((word) => word.length >= 3);
+    const token = words.sort((left, right) => right.length - left.length)[0] ?? base.slice(0, 8);
+    const pattern = `%${token.replace(/[%_]/g, "\\$&")}%`;
+    try {
+      const rows = this.open().prepare(
+        `SELECT e.id, e.mailbox, e.uid, e.subject, e.from_name, e.from_addr, e.to_addr, e.cc,
+                  e.message_id, e.date, length(e.body_text) AS text_length,
+                  (SELECT COUNT(*) FROM attachments a WHERE a.email_id = e.id) AS attachment_count,
+                  substr(COALESCE(e.body_text, ''), 1, 240) AS snippet
+           FROM emails e WHERE lower(e.subject) LIKE ? ESCAPE '\\'
+           ORDER BY e.date DESC LIMIT ?`
+      ).all(pattern, 500);
+      const items = rows.map((row) => rowToSummary(row)).filter((item) => normalize2(item.subject) === base);
+      if (seedId !== "" && this.hasRefColumns()) {
+        const linked = this.open().prepare(
+          `SELECT e.id, e.mailbox, e.uid, e.subject, e.from_name, e.from_addr, e.to_addr, e.cc,
+                    e.message_id, e.date, length(e.body_text) AS text_length,
+                    (SELECT COUNT(*) FROM attachments a WHERE a.email_id = e.id) AS attachment_count,
+                    substr(COALESCE(e.body_text, ''), 1, 240) AS snippet
+             FROM emails e
+             WHERE lower(e.in_reply_to) = ? OR lower(e."references") LIKE ? ESCAPE '\\'
+             ORDER BY e.date DESC LIMIT 100`
+        ).all(seedId, `%${seedId.replace(/[%_]/g, "\\$&")}%`);
+        const known = new Set(items.map((item) => item.id));
+        try {
+          const seedRow = this.open().prepare(
+            'SELECT in_reply_to, "references", raw_eml FROM emails WHERE lower(message_id) = ? LIMIT 1'
+          ).get(seedId);
+          const parents = `${seedRow?.references ?? ""} ${seedRow?.in_reply_to ?? ""}`.split(/\s+/).map((item) => item.replace(/^<|>$/g, "").toLowerCase()).filter((item) => item.includes("@"));
+          if (parents.length === 0 && seedRow?.raw_eml !== void 0 && seedRow.raw_eml !== null) {
+            for (const header of ["references", "in-reply-to"]) {
+              for (const found of headerIds(seedRow.raw_eml, header)) parents.push(found);
+            }
+          }
+          if (parents.length > 0) {
+            const marks = parents.map(() => "?").join(",");
+            const ancestors = this.open().prepare(
+              `SELECT e.id, e.mailbox, e.uid, e.subject, e.from_name, e.from_addr, e.to_addr, e.cc,
+                        e.message_id, e.date, length(e.body_text) AS text_length,
+                        (SELECT COUNT(*) FROM attachments a WHERE a.email_id = e.id) AS attachment_count,
+                        substr(COALESCE(e.body_text, ''), 1, 240) AS snippet
+                 FROM emails e WHERE lower(e.message_id) IN (${marks}) LIMIT 100`
+            ).all(...parents);
+            for (const row of ancestors) {
+              const item = rowToSummary(row);
+              if (!known.has(item.id)) {
+                items.push(item);
+                known.add(item.id);
+              }
+            }
+          }
+        } catch {
+        }
+        for (const row of linked) {
+          const item = rowToSummary(row);
+          if (!known.has(item.id)) {
+            items.push(item);
+            known.add(item.id);
+          }
+        }
+      }
+      return items.slice(0, limit);
+    } catch {
+      return [];
+    }
+  }
+  /** Read (and cache) In-Reply-To / References for a set of local ids. */
+  referencesFor(ids) {
+    const missing = ids.filter((id) => !this.threadRefs.has(id));
+    if (missing.length > 0) {
+      const placeholders = missing.map(() => "?").join(",");
+      try {
+        const withRefs = this.hasRefColumns();
+        const columns = withRefs ? 'id, message_id, raw_eml, in_reply_to, "references"' : "id, message_id, raw_eml";
+        const rows = this.open().prepare(`SELECT ${columns} FROM emails WHERE id IN (${placeholders})`).all(...missing);
+        for (const row of rows) {
+          const id = Number(row.id);
+          const raw = row.raw_eml;
+          const refs = [];
+          for (const stored of [row.references ?? "", row.in_reply_to ?? ""]) {
+            for (const found of String(stored).split(/\s+/)) {
+              const value = found.replace(/^<|>$/g, "").toLowerCase();
+              if (value.includes("@")) refs.push(value);
+            }
+          }
+          if (refs.length === 0 && raw !== void 0 && raw !== null) {
+            for (const header of ["references", "in-reply-to"]) {
+              for (const found of headerIds(raw, header)) refs.push(found);
+            }
+          }
+          this.threadRefs.set(id, {
+            messageId: String(row.message_id ?? "").replace(/^<|>$/g, "").toLowerCase(),
+            refs: [...new Set(refs)]
+          });
+        }
+      } catch {
+      }
+    }
+    return this.threadRefs;
   }
   /** Number of unread keys that still exist in this backup. */
   countUnread(keys) {
@@ -1550,6 +1731,15 @@ function truncate(value, max) {
 function quoteMatch(text2) {
   return `"${text2.replaceAll('"', '""')}"`;
 }
+function headerIds(raw, header) {
+  const text2 = Buffer.from(raw.subarray(0, Math.min(raw.byteLength, 2e5))).toString("utf8");
+  const separator = text2.search(/\r?\n\r?\n/);
+  const head = separator >= 0 ? text2.slice(0, separator) : text2;
+  const unfolded = head.replace(/\r?\n[ \t]+/g, " ");
+  const match = new RegExp(`^${header}:\\s*(.*)$`, "im").exec(unfolded);
+  if (match === null) return [];
+  return String(match[1] ?? "").split(/\s+/).map((item) => item.replace(/^<|>$/g, "").toLowerCase()).filter((item) => item.includes("@"));
+}
 function rowToSummary(row) {
   const fromName = String(row.from_name ?? "");
   const fromAddr = String(row.from_addr ?? "");
@@ -1679,8 +1869,8 @@ var BackupEngine = class {
     const inserted = db.prepare(
       `INSERT OR IGNORE INTO emails
            (mailbox, uid, subject, from_name, from_addr, to_addr, cc, date, message_id,
-            body_text, body_html, raw_eml, hash, backup_log_id)
-         VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`
+            in_reply_to, "references", body_text, body_html, raw_eml, hash, backup_log_id)
+         VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`
     ).run(
       mailbox,
       Number(message.uid),
@@ -1691,6 +1881,8 @@ var BackupEngine = class {
       ccText,
       stamp(parsed.date ?? message.internalDate ?? message.envelope?.date),
       messageId === "" ? null : messageId,
+      normalizeId(parsed.inReplyTo ?? "") || null,
+      String(parsed.references ?? "").split(/\s+/).map(normalizeId).filter((item) => item !== "").join(" ") || null,
       text2,
       html,
       source,
@@ -2780,7 +2972,7 @@ ${conversation.trim()}
 }
 
 // src/host/version.ts
-var VERSION = "0.17.0";
+var VERSION = "0.19.0";
 
 // src/host/api.ts
 init_smtp();
@@ -3002,6 +3194,15 @@ function buildRouter(runtime) {
     }
     return { ok: true, count };
   });
+  router.get("/maintenance/references", ({ url }) => {
+    const { account } = accountFrom(runtime, url);
+    return runtime.referencesStatus(account);
+  });
+  router.post("/maintenance/references", ({ url }) => {
+    const { account } = accountFrom(runtime, url);
+    runtime.startReferencesBackfill(account);
+    return runtime.referencesStatus(account);
+  });
   router.get("/transfer/export", () => {
     const config = runtime.current;
     return {
@@ -3074,10 +3275,11 @@ function buildRouter(runtime) {
     runtime.refresh();
     return { ok: true, addedAccounts: added, state: appState(runtime) };
   });
-  router.get("/messages/thread", ({ url }) => {
+  router.get("/messages/thread", async ({ url }) => {
     const { account } = accountFrom(runtime, url);
     const subject = url.searchParams.get("subject") ?? "";
-    const items = runtime.database(account).thread(subject);
+    const messageId = url.searchParams.get("messageId") ?? "";
+    const items = await runtime.database(account).thread({ subject, messageId });
     return { items: items.map((item) => localSummary(runtime, account, item)) };
   });
   router.get("/contacts", ({ url }) => {
@@ -4114,7 +4316,7 @@ ${signature}`;
 }
 
 // src/host/runtime.ts
-import { statSync } from "node:fs";
+import { existsSync as existsSync8, statSync } from "node:fs";
 import { join as join7 } from "node:path";
 
 // src/host/secrets.ts
@@ -4496,11 +4698,63 @@ var MailFlagsStore = class {
   }
 };
 
+// src/host/references.ts
+import { DatabaseSync as DatabaseSync4 } from "node:sqlite";
+function extractReferences(raw, messageId) {
+  const empty = { inReplyTo: null, references: null };
+  if (raw === null || raw === void 0) return empty;
+  const text2 = Buffer.from(raw.subarray(0, Math.min(raw.byteLength, 2e5))).toString("utf8");
+  const separator = text2.search(/\r?\n\r?\n/);
+  const head = separator >= 0 ? text2.slice(0, separator) : text2;
+  const unfolded = head.replace(/\r?\n[ \t]+/g, " ");
+  const value = (name2) => {
+    const match = new RegExp(`^${name2}:\\s*(.*)$`, "im").exec(unfolded);
+    return match === null ? "" : String(match[1] ?? "");
+  };
+  const ids = (value2) => value2.split(/\s+/).map((item) => item.replace(/^<|>$/g, "").toLowerCase()).filter((item) => item.includes("@"));
+  const referenceIds = ids(value("references"));
+  const replyTo = ids(value("in-reply-to"));
+  const filteredRefs = referenceIds.filter((id) => id !== messageId.toLowerCase());
+  return {
+    inReplyTo: replyTo[0] ?? null,
+    references: filteredRefs.length > 0 ? filteredRefs.join(" ") : null
+  };
+}
+function backfillReferences(file, batchSize = 300) {
+  ensureMailSchema(file);
+  const db = new DatabaseSync4(file);
+  try {
+    db.exec("PRAGMA busy_timeout = 10000");
+    const rows = db.prepare("SELECT id, message_id, raw_eml FROM emails WHERE refs_indexed = 0 ORDER BY id LIMIT ?").all(batchSize);
+    if (rows.length === 0) return { updated: 0, remaining: 0 };
+    const update = db.prepare(
+      'UPDATE emails SET in_reply_to = ?, "references" = ?, refs_indexed = 1 WHERE id = ?'
+    );
+    db.exec("BEGIN");
+    try {
+      for (const row of rows) {
+        const headers = extractReferences(row.raw_eml, String(row.message_id ?? ""));
+        update.run(headers.inReplyTo, headers.references, Number(row.id));
+      }
+      db.exec("COMMIT");
+    } catch (error) {
+      db.exec("ROLLBACK");
+      throw error;
+    }
+    const remaining = Number(
+      db.prepare("SELECT COUNT(*) AS count FROM emails WHERE refs_indexed = 0").get()?.count ?? 0
+    );
+    return { updated: rows.length, remaining };
+  } finally {
+    db.close();
+  }
+}
+
 // src/host/tasks.ts
 import { randomUUID as randomUUID4 } from "node:crypto";
 import { mkdirSync as mkdirSync7 } from "node:fs";
 import { dirname as dirname4 } from "node:path";
-import { DatabaseSync as DatabaseSync4 } from "node:sqlite";
+import { DatabaseSync as DatabaseSync5 } from "node:sqlite";
 var STATUSES = /* @__PURE__ */ new Set(["NEEDS-ACTION", "IN-PROCESS", "COMPLETED", "CANCELLED"]);
 function now() {
   return (/* @__PURE__ */ new Date()).toISOString();
@@ -4549,7 +4803,7 @@ var TaskStore = class {
   open() {
     if (this.db !== null) return this.db;
     mkdirSync7(dirname4(this.file), { recursive: true });
-    const db = new DatabaseSync4(this.file);
+    const db = new DatabaseSync5(this.file);
     db.exec("PRAGMA journal_mode=WAL");
     db.exec("PRAGMA busy_timeout=10000");
     db.exec(`
@@ -4975,6 +5229,7 @@ var MailHarborRuntime = class {
   projectStore = null;
   taskStore = null;
   flagStores = /* @__PURE__ */ new Map();
+  referenceJobs = /* @__PURE__ */ new Map();
   disposed = false;
   constructor(options = {}) {
     this.config = loadConfig();
@@ -5074,6 +5329,43 @@ var MailHarborRuntime = class {
   paths(account) {
     return accountRuntime(account);
   }
+  referencesStatus(account) {
+    return this.referenceJobs.get(account.id) ?? {
+      running: false,
+      indexed: 0,
+      remaining: null,
+      error: null
+    };
+  }
+  /** Backfill In-Reply-To/References for archived mail, one batch at a time. */
+  startReferencesBackfill(account) {
+    const existing = this.referenceJobs.get(account.id);
+    if (existing?.running === true) return;
+    const dbFile = this.paths(account).mailDb;
+    if (!existsSync8(dbFile)) return;
+    const status = {
+      running: true,
+      indexed: 0,
+      remaining: null,
+      error: null
+    };
+    this.referenceJobs.set(account.id, status);
+    void (async () => {
+      try {
+        for (; ; ) {
+          const result = backfillReferences(dbFile, 300);
+          status.indexed += result.updated;
+          status.remaining = result.remaining;
+          if (result.remaining <= 0 || result.updated === 0) break;
+          await new Promise((resolve) => setTimeout(resolve, 40));
+        }
+      } catch (error) {
+        status.error = error instanceof Error ? error.message : String(error);
+      } finally {
+        status.running = false;
+      }
+    })();
+  }
   /** Credentials for an account, with a clear message when the user must log in. */
   credentialsFor(account) {
     const resolved = this.credentials.resolve(account);
@@ -5643,6 +5935,9 @@ function apply(ctx) {
   }
   const handler = createApiHandler(runtime);
   const disposers = [];
+  if (activeAccount2 !== void 0) {
+    setTimeout(() => runtime.startReferencesBackfill(activeAccount2), 3e3);
+  }
   const start = () => {
     disposers.push(ctx.webServer.register({ kind: "prefix", path: ROUTE, handler }));
     if (ctx.tools !== void 0) disposers.push(...registerMailTools(ctx.tools, runtime));