@zaclinley/dsh-mailharbor 0.18.0 → 0.19.1

package/lib/index.js

@@ -1121,6 +1121,14 @@ function ensureMailSchema(file) {
     if (!emailColumns.some((column) => column.name === "references")) {
       db.exec('ALTER TABLE emails ADD COLUMN "references" TEXT');
     }
+    if (!emailColumns.some((column) => column.name === "refs_indexed")) {
+      db.exec("ALTER TABLE emails ADD COLUMN refs_indexed INTEGER DEFAULT 0");
+    }
+    try {
+      db.exec("CREATE INDEX IF NOT EXISTS idx_emails_in_reply ON emails(in_reply_to)");
+      db.exec("CREATE INDEX IF NOT EXISTS idx_emails_message_id_lower ON emails(lower(message_id))");
+    } catch {
+    }
   } finally {
     db.close();
   }
@@ -1132,6 +1140,7 @@ var MailDatabase = class {
   ftsChecked = false;
   ftsAvailable = false;
   threadRefs = /* @__PURE__ */ new Map();
+  threadCache = /* @__PURE__ */ new Map();
   constructor(file, options = {}) {
     this.file = file;
     this.options = options;
@@ -1186,6 +1195,7 @@ var MailDatabase = class {
     }
     this.db = void 0;
     this.threadRefs.clear();
+    this.threadCache.clear();
   }
   status() {
     try {
@@ -1389,9 +1399,15 @@ var MailDatabase = class {
     const normalize2 = (value) => value.replace(/^(\s*(re|fw|fwd|回复|转发)\s*[:：]\s*)+/gi, "").replace(/\s+/g, " ").trim().toLowerCase();
     const base = normalize2(seed.subject);
     if (base === "") return [];
+    const cacheKey = `${seed.messageId ?? ""}|${base}`;
+    const cached = this.threadCache.get(cacheKey);
+    if (cached !== void 0 && Date.now() - cached.at < 3e4) return cached.items;
     const seedIdForQuery = (seed.messageId ?? "").replace(/^<|>$/g, "").toLowerCase();
     const candidates = this.threadCandidates(base, limit, seedIdForQuery);
-    if (candidates.length <= 1) return candidates;
+    if (candidates.length <= 1) {
+      this.threadCache.set(cacheKey, { at: Date.now(), items: candidates });
+      return candidates;
+    }
     const refs = this.referencesFor(candidates.slice(0, 120).map((item) => Number(item.id)));
     const byMessageId = /* @__PURE__ */ new Map();
     for (const item of candidates) {
@@ -1429,7 +1445,10 @@ var MailDatabase = class {
         refs: [...refs.entries()].slice(0, 3)
       });
     }
-    if (edges === 0) return candidates;
+    if (edges === 0) {
+      this.threadCache.set(cacheKey, { at: Date.now(), items: candidates });
+      return candidates;
+    }
     const seen = /* @__PURE__ */ new Set();
     const queue = [byMessageId.get(startId)];
     while (queue.length > 0) {
@@ -1440,24 +1459,27 @@ var MailDatabase = class {
         if (!seen.has(next)) queue.push(next);
       }
     }
-    if (seen.size <= 1) return candidates;
-    return candidates.filter((item) => seen.has(Number(item.id)));
+    const result = seen.size <= 1 ? candidates : candidates.filter((item) => seen.has(Number(item.id)));
+    this.threadCache.set(cacheKey, { at: Date.now(), items: result });
+    return result;
   }
   /** Subject candidates via a distinctive token, not FTS (punctuation-safe). */
   threadCandidates(base, limit, seedId = "") {
     const normalize2 = (value) => value.replace(/^(\s*(re|fw|fwd|回复|转发)\s*[:：]\s*)+/gi, "").replace(/\s+/g, " ").trim().toLowerCase();
     const words = base.split(/[^\p{L}\p{N}]+/u).filter((word) => word.length >= 3);
     const token = words.sort((left, right) => right.length - left.length)[0] ?? base.slice(0, 8);
-    const pattern = `%${token.replace(/[%_]/g, "\\$&")}%`;
-    try {
-      const rows = this.open().prepare(
-        `SELECT e.id, e.mailbox, e.uid, e.subject, e.from_name, e.from_addr, e.to_addr, e.cc,
+    const select = `SELECT e.id, e.mailbox, e.uid, e.subject, e.from_name, e.from_addr, e.to_addr, e.cc,
                   e.message_id, e.date, length(e.body_text) AS text_length,
                   (SELECT COUNT(*) FROM attachments a WHERE a.email_id = e.id) AS attachment_count,
                   substr(COALESCE(e.body_text, ''), 1, 240) AS snippet
-           FROM emails e WHERE lower(e.subject) LIKE ? ESCAPE '\\'
-           ORDER BY e.date DESC LIMIT ?`
-      ).all(pattern, 500);
+           FROM emails e`;
+    try {
+      const ftsQuery = `subject:"${token.replace(/"/g, '""')}"`;
+      const rows = this.open().prepare(
+        `${select} JOIN emails_fts f ON f.rowid = e.id
+           WHERE emails_fts MATCH ?
+           ORDER BY e.date DESC LIMIT 500`
+      ).all(ftsQuery);
       const items = rows.map((row) => rowToSummary(row)).filter((item) => normalize2(item.subject) === base);
       if (seedId !== "" && this.hasRefColumns()) {
         const linked = this.open().prepare(
@@ -1466,9 +1488,9 @@ var MailDatabase = class {
                     (SELECT COUNT(*) FROM attachments a WHERE a.email_id = e.id) AS attachment_count,
                     substr(COALESCE(e.body_text, ''), 1, 240) AS snippet
              FROM emails e
-             WHERE lower(e.in_reply_to) = ? OR lower(e."references") LIKE ? ESCAPE '\\'
+             WHERE e.in_reply_to = ?
              ORDER BY e.date DESC LIMIT 100`
-        ).all(seedId, `%${seedId.replace(/[%_]/g, "\\$&")}%`);
+        ).all(seedId);
         const known = new Set(items.map((item) => item.id));
         try {
           const seedRow = this.open().prepare(
@@ -1509,7 +1531,8 @@ var MailDatabase = class {
       }
       return items.slice(0, limit);
     } catch {
-      return [];
+      const page = this.list({ mailbox: "all", query: base, limit: Math.min(500, limit * 2) });
+      return page.items.filter((item) => normalize2(item.subject) === base).slice(0, limit);
     }
   }
   /** Read (and cache) In-Reply-To / References for a set of local ids. */
@@ -1519,11 +1542,11 @@ var MailDatabase = class {
       const placeholders = missing.map(() => "?").join(",");
       try {
         const withRefs = this.hasRefColumns();
-        const columns = withRefs ? 'id, message_id, raw_eml, in_reply_to, "references"' : "id, message_id, raw_eml";
+        const columns = withRefs ? 'id, message_id, in_reply_to, "references", refs_indexed' : "id, message_id, raw_eml";
         const rows = this.open().prepare(`SELECT ${columns} FROM emails WHERE id IN (${placeholders})`).all(...missing);
+        const needRaw = [];
         for (const row of rows) {
           const id = Number(row.id);
-          const raw = row.raw_eml;
           const refs = [];
           for (const stored of [row.references ?? "", row.in_reply_to ?? ""]) {
             for (const found of String(stored).split(/\s+/)) {
@@ -1531,16 +1554,28 @@ var MailDatabase = class {
               if (value.includes("@")) refs.push(value);
             }
           }
-          if (refs.length === 0 && raw !== void 0 && raw !== null) {
-            for (const header of ["references", "in-reply-to"]) {
-              for (const found of headerIds(raw, header)) refs.push(found);
-            }
+          if (refs.length === 0 && (!withRefs || Number(row.refs_indexed ?? 0) !== 1)) {
+            needRaw.push(id);
           }
           this.threadRefs.set(id, {
             messageId: String(row.message_id ?? "").replace(/^<|>$/g, "").toLowerCase(),
             refs: [...new Set(refs)]
           });
         }
+        if (needRaw.length > 0) {
+          const marks = needRaw.map(() => "?").join(",");
+          const rawRows = this.open().prepare(`SELECT id, raw_eml FROM emails WHERE id IN (${marks})`).all(...needRaw);
+          for (const row of rawRows) {
+            const id = Number(row.id);
+            const entry = this.threadRefs.get(id);
+            if (entry === void 0 || row.raw_eml === null || row.raw_eml === void 0) continue;
+            const refs = [...entry.refs];
+            for (const header of ["references", "in-reply-to"]) {
+              for (const found of headerIds(row.raw_eml, header)) refs.push(found);
+            }
+            this.threadRefs.set(id, { ...entry, refs: [...new Set(refs)] });
+          }
+        }
       } catch {
       }
     }
@@ -2969,7 +3004,7 @@ ${conversation.trim()}
 }
 
 // src/host/version.ts
-var VERSION = "0.18.0";
+var VERSION = "0.19.1";
 
 // src/host/api.ts
 init_smtp();
@@ -3191,6 +3226,15 @@ function buildRouter(runtime) {
     }
     return { ok: true, count };
   });
+  router.get("/maintenance/references", ({ url }) => {
+    const { account } = accountFrom(runtime, url);
+    return runtime.referencesStatus(account);
+  });
+  router.post("/maintenance/references", ({ url }) => {
+    const { account } = accountFrom(runtime, url);
+    runtime.startReferencesBackfill(account);
+    return runtime.referencesStatus(account);
+  });
   router.get("/transfer/export", () => {
     const config = runtime.current;
     return {
@@ -4304,7 +4348,7 @@ ${signature}`;
 }
 
 // src/host/runtime.ts
-import { statSync } from "node:fs";
+import { existsSync as existsSync8, statSync } from "node:fs";
 import { join as join7 } from "node:path";
 
 // src/host/secrets.ts
@@ -4686,11 +4730,63 @@ var MailFlagsStore = class {
   }
 };
 
+// src/host/references.ts
+import { DatabaseSync as DatabaseSync4 } from "node:sqlite";
+function extractReferences(raw, messageId) {
+  const empty = { inReplyTo: null, references: null };
+  if (raw === null || raw === void 0) return empty;
+  const text2 = Buffer.from(raw.subarray(0, Math.min(raw.byteLength, 2e5))).toString("utf8");
+  const separator = text2.search(/\r?\n\r?\n/);
+  const head = separator >= 0 ? text2.slice(0, separator) : text2;
+  const unfolded = head.replace(/\r?\n[ \t]+/g, " ");
+  const value = (name2) => {
+    const match = new RegExp(`^${name2}:\\s*(.*)$`, "im").exec(unfolded);
+    return match === null ? "" : String(match[1] ?? "");
+  };
+  const ids = (value2) => value2.split(/\s+/).map((item) => item.replace(/^<|>$/g, "").toLowerCase()).filter((item) => item.includes("@"));
+  const referenceIds = ids(value("references"));
+  const replyTo = ids(value("in-reply-to"));
+  const filteredRefs = referenceIds.filter((id) => id !== messageId.toLowerCase());
+  return {
+    inReplyTo: replyTo[0] ?? null,
+    references: filteredRefs.length > 0 ? filteredRefs.join(" ") : null
+  };
+}
+function backfillReferences(file, batchSize = 300) {
+  ensureMailSchema(file);
+  const db = new DatabaseSync4(file);
+  try {
+    db.exec("PRAGMA busy_timeout = 10000");
+    const rows = db.prepare("SELECT id, message_id, raw_eml FROM emails WHERE refs_indexed = 0 ORDER BY id LIMIT ?").all(batchSize);
+    if (rows.length === 0) return { updated: 0, remaining: 0 };
+    const update = db.prepare(
+      'UPDATE emails SET in_reply_to = ?, "references" = ?, refs_indexed = 1 WHERE id = ?'
+    );
+    db.exec("BEGIN");
+    try {
+      for (const row of rows) {
+        const headers = extractReferences(row.raw_eml, String(row.message_id ?? ""));
+        update.run(headers.inReplyTo, headers.references, Number(row.id));
+      }
+      db.exec("COMMIT");
+    } catch (error) {
+      db.exec("ROLLBACK");
+      throw error;
+    }
+    const remaining = Number(
+      db.prepare("SELECT COUNT(*) AS count FROM emails WHERE refs_indexed = 0").get()?.count ?? 0
+    );
+    return { updated: rows.length, remaining };
+  } finally {
+    db.close();
+  }
+}
+
 // src/host/tasks.ts
 import { randomUUID as randomUUID4 } from "node:crypto";
 import { mkdirSync as mkdirSync7 } from "node:fs";
 import { dirname as dirname4 } from "node:path";
-import { DatabaseSync as DatabaseSync4 } from "node:sqlite";
+import { DatabaseSync as DatabaseSync5 } from "node:sqlite";
 var STATUSES = /* @__PURE__ */ new Set(["NEEDS-ACTION", "IN-PROCESS", "COMPLETED", "CANCELLED"]);
 function now() {
   return (/* @__PURE__ */ new Date()).toISOString();
@@ -4739,7 +4835,7 @@ var TaskStore = class {
   open() {
     if (this.db !== null) return this.db;
     mkdirSync7(dirname4(this.file), { recursive: true });
-    const db = new DatabaseSync4(this.file);
+    const db = new DatabaseSync5(this.file);
     db.exec("PRAGMA journal_mode=WAL");
     db.exec("PRAGMA busy_timeout=10000");
     db.exec(`
@@ -5165,6 +5261,7 @@ var MailHarborRuntime = class {
   projectStore = null;
   taskStore = null;
   flagStores = /* @__PURE__ */ new Map();
+  referenceJobs = /* @__PURE__ */ new Map();
   disposed = false;
   constructor(options = {}) {
     this.config = loadConfig();
@@ -5264,6 +5361,43 @@ var MailHarborRuntime = class {
   paths(account) {
     return accountRuntime(account);
   }
+  referencesStatus(account) {
+    return this.referenceJobs.get(account.id) ?? {
+      running: false,
+      indexed: 0,
+      remaining: null,
+      error: null
+    };
+  }
+  /** Backfill In-Reply-To/References for archived mail, one batch at a time. */
+  startReferencesBackfill(account) {
+    const existing = this.referenceJobs.get(account.id);
+    if (existing?.running === true) return;
+    const dbFile = this.paths(account).mailDb;
+    if (!existsSync8(dbFile)) return;
+    const status = {
+      running: true,
+      indexed: 0,
+      remaining: null,
+      error: null
+    };
+    this.referenceJobs.set(account.id, status);
+    void (async () => {
+      try {
+        for (; ; ) {
+          const result = backfillReferences(dbFile, 300);
+          status.indexed += result.updated;
+          status.remaining = result.remaining;
+          if (result.remaining <= 0 || result.updated === 0) break;
+          await new Promise((resolve) => setTimeout(resolve, 40));
+        }
+      } catch (error) {
+        status.error = error instanceof Error ? error.message : String(error);
+      } finally {
+        status.running = false;
+      }
+    })();
+  }
   /** Credentials for an account, with a clear message when the user must log in. */
   credentialsFor(account) {
     const resolved = this.credentials.resolve(account);
@@ -5833,6 +5967,9 @@ function apply(ctx) {
   }
   const handler = createApiHandler(runtime);
   const disposers = [];
+  if (activeAccount2 !== void 0) {
+    setTimeout(() => runtime.startReferencesBackfill(activeAccount2), 3e3);
+  }
   const start = () => {
     disposers.push(ctx.webServer.register({ kind: "prefix", path: ROUTE, handler }));
     if (ctx.tools !== void 0) disposers.push(...registerMailTools(ctx.tools, runtime));