@zaclinley/dsh-mailharbor 0.16.1 → 0.18.0

package/lib/index.js

@@ -1114,6 +1114,13 @@ function ensureMailSchema(file) {
     if (!columns.some((column) => column.name === "disposition")) {
       db.exec("ALTER TABLE attachments ADD COLUMN disposition TEXT");
     }
+    const emailColumns = db.prepare("PRAGMA table_info(emails)").all();
+    if (!emailColumns.some((column) => column.name === "in_reply_to")) {
+      db.exec("ALTER TABLE emails ADD COLUMN in_reply_to TEXT");
+    }
+    if (!emailColumns.some((column) => column.name === "references")) {
+      db.exec('ALTER TABLE emails ADD COLUMN "references" TEXT');
+    }
   } finally {
     db.close();
   }
@@ -1124,6 +1131,7 @@ var MailDatabase = class {
   db;
   ftsChecked = false;
   ftsAvailable = false;
+  threadRefs = /* @__PURE__ */ new Map();
   constructor(file, options = {}) {
     this.file = file;
     this.options = options;
@@ -1150,6 +1158,14 @@ var MailDatabase = class {
       return false;
     }
   }
+  hasRefColumns() {
+    try {
+      const columns = this.open().prepare("PRAGMA table_info(emails)").all();
+      return columns.some((column) => column.name === "in_reply_to");
+    } catch {
+      return false;
+    }
+  }
   attachmentCountSql() {
     if (!this.hasAttachmentColumns()) {
       return "(SELECT COUNT(*) FROM attachments a WHERE a.email_id = e.id) AS attachment_count";
@@ -1169,6 +1185,7 @@ var MailDatabase = class {
     } catch {
     }
     this.db = void 0;
+    this.threadRefs.clear();
   }
   status() {
     try {
@@ -1209,6 +1226,18 @@ var MailDatabase = class {
       where.push("e.date >= ?");
       params.push(query.since);
     }
+    if (typeof query.before === "string" && query.before !== "") {
+      where.push("e.date < ?");
+      params.push(query.before.length === 10 ? `${query.before} 23:59:59` : query.before);
+    }
+    if (typeof query.from === "string" && query.from.trim() !== "") {
+      const needle = `%${query.from.trim().toLowerCase()}%`;
+      where.push("(lower(e.from_addr) LIKE ? OR lower(e.from_name) LIKE ?)");
+      params.push(needle, needle);
+    }
+    if (query.attachmentsOnly === true) {
+      where.push("EXISTS (SELECT 1 FROM attachments a WHERE a.email_id = e.id)");
+    }
     if (query.unread !== void 0) {
       const mids = query.unread.mids.slice(0, 500);
       const ids = query.unread.ids.slice(0, 500);
@@ -1348,13 +1377,174 @@ var MailDatabase = class {
       (entry) => text2 === "" || entry.address.toLowerCase().includes(text2) || entry.name.toLowerCase().includes(text2)
     ).sort((left, right) => right.count - left.count).slice(0, limit);
   }
-  /** All archived messages in the same conversation (normalized subject). */
-  thread(subject, limit = 200) {
+  /**
+   * Conversation for one message.
+   *
+   * Subject grouping is only the candidate pool; inside it we read
+   * In-Reply-To/References from the stored raw message and keep the connected
+   * component of the seed. That stops unrelated mail with the same subject
+   * from being merged while still working for old backups.
+   */
+  thread(seed, limit = 200) {
     const normalize2 = (value) => value.replace(/^(\s*(re|fw|fwd|回复|转发)\s*[:：]\s*)+/gi, "").replace(/\s+/g, " ").trim().toLowerCase();
-    const base = normalize2(subject);
+    const base = normalize2(seed.subject);
     if (base === "") return [];
-    const page = this.list({ mailbox: "all", query: base, limit: Math.min(500, limit * 2) });
-    return page.items.filter((item) => normalize2(item.subject) === base).slice(0, limit);
+    const seedIdForQuery = (seed.messageId ?? "").replace(/^<|>$/g, "").toLowerCase();
+    const candidates = this.threadCandidates(base, limit, seedIdForQuery);
+    if (candidates.length <= 1) return candidates;
+    const refs = this.referencesFor(candidates.slice(0, 120).map((item) => Number(item.id)));
+    const byMessageId = /* @__PURE__ */ new Map();
+    for (const item of candidates) {
+      const id = item.messageId === void 0 ? "" : item.messageId.replace(/^<|>$/g, "").toLowerCase();
+      if (id !== "") byMessageId.set(id, Number(item.id));
+    }
+    const seedId = (seed.messageId ?? "").replace(/^<|>$/g, "").toLowerCase();
+    const startId = byMessageId.has(seedId) ? seedId : (candidates[0]?.messageId ?? "").replace(/^<|>$/g, "").toLowerCase();
+    if (startId === "" || !byMessageId.has(startId)) return candidates;
+    const neighbours = /* @__PURE__ */ new Map();
+    const link = (left, right) => {
+      if (left === right) return;
+      if (!neighbours.has(left)) neighbours.set(left, /* @__PURE__ */ new Set());
+      if (!neighbours.has(right)) neighbours.set(right, /* @__PURE__ */ new Set());
+      neighbours.get(left)?.add(right);
+      neighbours.get(right)?.add(left);
+    };
+    let edges = 0;
+    for (const item of candidates) {
+      const local = refs.get(Number(item.id));
+      if (local === void 0 || local.refs.length === 0) continue;
+      for (const ref of local.refs) {
+        const target = byMessageId.get(ref);
+        if (target !== void 0) {
+          link(Number(item.id), target);
+          edges += 1;
+        }
+      }
+    }
+    if (process.env.MAILHARBOR_DEBUG_THREAD === "1") {
+      console.error("THREAD", {
+        base,
+        candidates: candidates.length,
+        edges,
+        refs: [...refs.entries()].slice(0, 3)
+      });
+    }
+    if (edges === 0) return candidates;
+    const seen = /* @__PURE__ */ new Set();
+    const queue = [byMessageId.get(startId)];
+    while (queue.length > 0) {
+      const id = queue.shift();
+      if (seen.has(id)) continue;
+      seen.add(id);
+      for (const next of neighbours.get(id) ?? []) {
+        if (!seen.has(next)) queue.push(next);
+      }
+    }
+    if (seen.size <= 1) return candidates;
+    return candidates.filter((item) => seen.has(Number(item.id)));
+  }
+  /** Subject candidates via a distinctive token, not FTS (punctuation-safe). */
+  threadCandidates(base, limit, seedId = "") {
+    const normalize2 = (value) => value.replace(/^(\s*(re|fw|fwd|回复|转发)\s*[:：]\s*)+/gi, "").replace(/\s+/g, " ").trim().toLowerCase();
+    const words = base.split(/[^\p{L}\p{N}]+/u).filter((word) => word.length >= 3);
+    const token = words.sort((left, right) => right.length - left.length)[0] ?? base.slice(0, 8);
+    const pattern = `%${token.replace(/[%_]/g, "\\$&")}%`;
+    try {
+      const rows = this.open().prepare(
+        `SELECT e.id, e.mailbox, e.uid, e.subject, e.from_name, e.from_addr, e.to_addr, e.cc,
+                  e.message_id, e.date, length(e.body_text) AS text_length,
+                  (SELECT COUNT(*) FROM attachments a WHERE a.email_id = e.id) AS attachment_count,
+                  substr(COALESCE(e.body_text, ''), 1, 240) AS snippet
+           FROM emails e WHERE lower(e.subject) LIKE ? ESCAPE '\\'
+           ORDER BY e.date DESC LIMIT ?`
+      ).all(pattern, 500);
+      const items = rows.map((row) => rowToSummary(row)).filter((item) => normalize2(item.subject) === base);
+      if (seedId !== "" && this.hasRefColumns()) {
+        const linked = this.open().prepare(
+          `SELECT e.id, e.mailbox, e.uid, e.subject, e.from_name, e.from_addr, e.to_addr, e.cc,
+                    e.message_id, e.date, length(e.body_text) AS text_length,
+                    (SELECT COUNT(*) FROM attachments a WHERE a.email_id = e.id) AS attachment_count,
+                    substr(COALESCE(e.body_text, ''), 1, 240) AS snippet
+             FROM emails e
+             WHERE lower(e.in_reply_to) = ? OR lower(e."references") LIKE ? ESCAPE '\\'
+             ORDER BY e.date DESC LIMIT 100`
+        ).all(seedId, `%${seedId.replace(/[%_]/g, "\\$&")}%`);
+        const known = new Set(items.map((item) => item.id));
+        try {
+          const seedRow = this.open().prepare(
+            'SELECT in_reply_to, "references", raw_eml FROM emails WHERE lower(message_id) = ? LIMIT 1'
+          ).get(seedId);
+          const parents = `${seedRow?.references ?? ""} ${seedRow?.in_reply_to ?? ""}`.split(/\s+/).map((item) => item.replace(/^<|>$/g, "").toLowerCase()).filter((item) => item.includes("@"));
+          if (parents.length === 0 && seedRow?.raw_eml !== void 0 && seedRow.raw_eml !== null) {
+            for (const header of ["references", "in-reply-to"]) {
+              for (const found of headerIds(seedRow.raw_eml, header)) parents.push(found);
+            }
+          }
+          if (parents.length > 0) {
+            const marks = parents.map(() => "?").join(",");
+            const ancestors = this.open().prepare(
+              `SELECT e.id, e.mailbox, e.uid, e.subject, e.from_name, e.from_addr, e.to_addr, e.cc,
+                        e.message_id, e.date, length(e.body_text) AS text_length,
+                        (SELECT COUNT(*) FROM attachments a WHERE a.email_id = e.id) AS attachment_count,
+                        substr(COALESCE(e.body_text, ''), 1, 240) AS snippet
+                 FROM emails e WHERE lower(e.message_id) IN (${marks}) LIMIT 100`
+            ).all(...parents);
+            for (const row of ancestors) {
+              const item = rowToSummary(row);
+              if (!known.has(item.id)) {
+                items.push(item);
+                known.add(item.id);
+              }
+            }
+          }
+        } catch {
+        }
+        for (const row of linked) {
+          const item = rowToSummary(row);
+          if (!known.has(item.id)) {
+            items.push(item);
+            known.add(item.id);
+          }
+        }
+      }
+      return items.slice(0, limit);
+    } catch {
+      return [];
+    }
+  }
+  /** Read (and cache) In-Reply-To / References for a set of local ids. */
+  referencesFor(ids) {
+    const missing = ids.filter((id) => !this.threadRefs.has(id));
+    if (missing.length > 0) {
+      const placeholders = missing.map(() => "?").join(",");
+      try {
+        const withRefs = this.hasRefColumns();
+        const columns = withRefs ? 'id, message_id, raw_eml, in_reply_to, "references"' : "id, message_id, raw_eml";
+        const rows = this.open().prepare(`SELECT ${columns} FROM emails WHERE id IN (${placeholders})`).all(...missing);
+        for (const row of rows) {
+          const id = Number(row.id);
+          const raw = row.raw_eml;
+          const refs = [];
+          for (const stored of [row.references ?? "", row.in_reply_to ?? ""]) {
+            for (const found of String(stored).split(/\s+/)) {
+              const value = found.replace(/^<|>$/g, "").toLowerCase();
+              if (value.includes("@")) refs.push(value);
+            }
+          }
+          if (refs.length === 0 && raw !== void 0 && raw !== null) {
+            for (const header of ["references", "in-reply-to"]) {
+              for (const found of headerIds(raw, header)) refs.push(found);
+            }
+          }
+          this.threadRefs.set(id, {
+            messageId: String(row.message_id ?? "").replace(/^<|>$/g, "").toLowerCase(),
+            refs: [...new Set(refs)]
+          });
+        }
+      } catch {
+      }
+    }
+    return this.threadRefs;
   }
   /** Number of unread keys that still exist in this backup. */
   countUnread(keys) {
@@ -1538,6 +1728,15 @@ function truncate(value, max) {
 function quoteMatch(text2) {
   return `"${text2.replaceAll('"', '""')}"`;
 }
+function headerIds(raw, header) {
+  const text2 = Buffer.from(raw.subarray(0, Math.min(raw.byteLength, 2e5))).toString("utf8");
+  const separator = text2.search(/\r?\n\r?\n/);
+  const head = separator >= 0 ? text2.slice(0, separator) : text2;
+  const unfolded = head.replace(/\r?\n[ \t]+/g, " ");
+  const match = new RegExp(`^${header}:\\s*(.*)$`, "im").exec(unfolded);
+  if (match === null) return [];
+  return String(match[1] ?? "").split(/\s+/).map((item) => item.replace(/^<|>$/g, "").toLowerCase()).filter((item) => item.includes("@"));
+}
 function rowToSummary(row) {
   const fromName = String(row.from_name ?? "");
   const fromAddr = String(row.from_addr ?? "");
@@ -1667,8 +1866,8 @@ var BackupEngine = class {
     const inserted = db.prepare(
       `INSERT OR IGNORE INTO emails
            (mailbox, uid, subject, from_name, from_addr, to_addr, cc, date, message_id,
-            body_text, body_html, raw_eml, hash, backup_log_id)
-         VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`
+            in_reply_to, "references", body_text, body_html, raw_eml, hash, backup_log_id)
+         VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`
     ).run(
       mailbox,
       Number(message.uid),
@@ -1679,6 +1878,8 @@ var BackupEngine = class {
       ccText,
       stamp(parsed.date ?? message.internalDate ?? message.envelope?.date),
       messageId === "" ? null : messageId,
+      normalizeId(parsed.inReplyTo ?? "") || null,
+      String(parsed.references ?? "").split(/\s+/).map(normalizeId).filter((item) => item !== "").join(" ") || null,
       text2,
       html,
       source,
@@ -2768,7 +2969,7 @@ ${conversation.trim()}
 }
 
 // src/host/version.ts
-var VERSION = "0.16.1";
+var VERSION = "0.18.0";
 
 // src/host/api.ts
 init_smtp();
@@ -2872,11 +3073,19 @@ function buildRouter(runtime) {
       return { ...items, offset, limit };
     }
     const unreadOnly = url.searchParams.get("unread") === "1";
+    const from = url.searchParams.get("from") ?? "";
+    const since = url.searchParams.get("since") ?? "";
+    const before = url.searchParams.get("before") ?? "";
+    const attachmentsOnly = url.searchParams.get("attachment") === "1";
     const page = runtime.database(account).list({
       mailbox: mailbox === "" ? "all" : mailbox,
       query,
       offset,
       limit,
+      ...from.trim() === "" ? {} : { from },
+      ...since === "" ? {} : { since },
+      ...before === "" ? {} : { before },
+      ...attachmentsOnly ? { attachmentsOnly: true } : {},
       ...unreadOnly ? { unread: unreadKeys(runtime, account) } : {}
     });
     return { ...page, items: page.items.map((item) => localSummary(runtime, account, item)) };
@@ -3054,10 +3263,11 @@ function buildRouter(runtime) {
     runtime.refresh();
     return { ok: true, addedAccounts: added, state: appState(runtime) };
   });
-  router.get("/messages/thread", ({ url }) => {
+  router.get("/messages/thread", async ({ url }) => {
     const { account } = accountFrom(runtime, url);
     const subject = url.searchParams.get("subject") ?? "";
-    const items = runtime.database(account).thread(subject);
+    const messageId = url.searchParams.get("messageId") ?? "";
+    const items = await runtime.database(account).thread({ subject, messageId });
     return { items: items.map((item) => localSummary(runtime, account, item)) };
   });
   router.get("/contacts", ({ url }) => {