@zaclinley/dsh-mailharbor 0.19.1 → 0.20.0

package/lib/index.js

@@ -1027,20 +1027,21 @@ async function draftMime(account, draft) {
   return Buffer.from(info.message);
 }
 
-// src/host/backup.ts
-init_address();
-import { createHash as createHash2 } from "node:crypto";
-import { existsSync as existsSync3, mkdirSync as mkdirSync2, writeFileSync as writeFileSync2 } from "node:fs";
-import { join as join3 } from "node:path";
-import { ImapFlow } from "imapflow";
-import { simpleParser } from "mailparser";
+// src/host/references.ts
 import { DatabaseSync as DatabaseSync3 } from "node:sqlite";
 
 // src/host/mail-db.ts
 init_address();
-init_inline_images();
 import { existsSync as existsSync2, readFileSync as readFileSync2 } from "node:fs";
 import { DatabaseSync as DatabaseSync2 } from "node:sqlite";
+
+// src/host/subject-key.ts
+function subjectKey(subject) {
+  return subject.replace(/^(\s*(re|fw|fwd|回复|答复|转发)\s*[:：]\s*)+/gi, "").replace(/\s+/g, " ").trim().toLowerCase();
+}
+
+// src/host/mail-db.ts
+init_inline_images();
 var SCHEMA = `
   CREATE TABLE IF NOT EXISTS emails (
     id INTEGER PRIMARY KEY AUTOINCREMENT,
@@ -1125,6 +1126,8 @@ function ensureMailSchema(file) {
       db.exec("ALTER TABLE emails ADD COLUMN refs_indexed INTEGER DEFAULT 0");
     }
     try {
+      db.exec("CREATE TABLE IF NOT EXISTS email_subjects (email_id INTEGER PRIMARY KEY, key TEXT NOT NULL)");
+      db.exec("CREATE INDEX IF NOT EXISTS idx_email_subjects_key ON email_subjects(key)");
       db.exec("CREATE INDEX IF NOT EXISTS idx_emails_in_reply ON emails(in_reply_to)");
       db.exec("CREATE INDEX IF NOT EXISTS idx_emails_message_id_lower ON emails(lower(message_id))");
     } catch {
@@ -1463,64 +1466,45 @@ var MailDatabase = class {
     this.threadCache.set(cacheKey, { at: Date.now(), items: result });
     return result;
   }
-  /** Subject candidates via a distinctive token, not FTS (punctuation-safe). */
+  /** Subject candidates from a normalized-key table, then FTS, then LIKE. */
   threadCandidates(base, limit, seedId = "") {
-    const normalize2 = (value) => value.replace(/^(\s*(re|fw|fwd|回复|转发)\s*[:：]\s*)+/gi, "").replace(/\s+/g, " ").trim().toLowerCase();
-    const words = base.split(/[^\p{L}\p{N}]+/u).filter((word) => word.length >= 3);
-    const token = words.sort((left, right) => right.length - left.length)[0] ?? base.slice(0, 8);
     const select = `SELECT e.id, e.mailbox, e.uid, e.subject, e.from_name, e.from_addr, e.to_addr, e.cc,
                   e.message_id, e.date, length(e.body_text) AS text_length,
                   (SELECT COUNT(*) FROM attachments a WHERE a.email_id = e.id) AS attachment_count,
                   substr(COALESCE(e.body_text, ''), 1, 240) AS snippet
            FROM emails e`;
+    let items = [];
     try {
-      const ftsQuery = `subject:"${token.replace(/"/g, '""')}"`;
-      const rows = this.open().prepare(
-        `${select} JOIN emails_fts f ON f.rowid = e.id
-           WHERE emails_fts MATCH ?
+      const byKey = this.open().prepare(
+        `${select} JOIN email_subjects s ON s.email_id = e.id
+           WHERE s.key = ?
            ORDER BY e.date DESC LIMIT 500`
-      ).all(ftsQuery);
-      const items = rows.map((row) => rowToSummary(row)).filter((item) => normalize2(item.subject) === base);
-      if (seedId !== "" && this.hasRefColumns()) {
+      ).all(base);
+      if (byKey.length > 0) {
+        items = byKey.map((row) => rowToSummary(row));
+      } else {
+        const words = base.split(/[^\p{L}\p{N}]+/u).filter((word) => word.length >= 3);
+        const token = words.sort((left, right) => right.length - left.length)[0] ?? base.slice(0, 8);
+        const ftsQuery = `subject:"${token.replace(/"/g, '""')}"`;
+        const rows = this.open().prepare(
+          `${select} JOIN emails_fts f ON f.rowid = e.id
+             WHERE emails_fts MATCH ?
+             ORDER BY e.date DESC LIMIT 500`
+        ).all(ftsQuery);
+        items = rows.map((row) => rowToSummary(row)).filter((item) => subjectKey(item.subject) === base);
+      }
+    } catch {
+      const page = this.list({ mailbox: "all", query: base, limit: Math.min(500, limit * 2) });
+      return page.items.filter((item) => subjectKey(item.subject) === base).slice(0, limit);
+    }
+    if (seedId !== "" && this.hasRefColumns()) {
+      try {
         const linked = this.open().prepare(
-          `SELECT e.id, e.mailbox, e.uid, e.subject, e.from_name, e.from_addr, e.to_addr, e.cc,
-                    e.message_id, e.date, length(e.body_text) AS text_length,
-                    (SELECT COUNT(*) FROM attachments a WHERE a.email_id = e.id) AS attachment_count,
-                    substr(COALESCE(e.body_text, ''), 1, 240) AS snippet
-             FROM emails e
+          `${select}
              WHERE e.in_reply_to = ?
              ORDER BY e.date DESC LIMIT 100`
         ).all(seedId);
         const known = new Set(items.map((item) => item.id));
-        try {
-          const seedRow = this.open().prepare(
-            'SELECT in_reply_to, "references", raw_eml FROM emails WHERE lower(message_id) = ? LIMIT 1'
-          ).get(seedId);
-          const parents = `${seedRow?.references ?? ""} ${seedRow?.in_reply_to ?? ""}`.split(/\s+/).map((item) => item.replace(/^<|>$/g, "").toLowerCase()).filter((item) => item.includes("@"));
-          if (parents.length === 0 && seedRow?.raw_eml !== void 0 && seedRow.raw_eml !== null) {
-            for (const header of ["references", "in-reply-to"]) {
-              for (const found of headerIds(seedRow.raw_eml, header)) parents.push(found);
-            }
-          }
-          if (parents.length > 0) {
-            const marks = parents.map(() => "?").join(",");
-            const ancestors = this.open().prepare(
-              `SELECT e.id, e.mailbox, e.uid, e.subject, e.from_name, e.from_addr, e.to_addr, e.cc,
-                        e.message_id, e.date, length(e.body_text) AS text_length,
-                        (SELECT COUNT(*) FROM attachments a WHERE a.email_id = e.id) AS attachment_count,
-                        substr(COALESCE(e.body_text, ''), 1, 240) AS snippet
-                 FROM emails e WHERE lower(e.message_id) IN (${marks}) LIMIT 100`
-            ).all(...parents);
-            for (const row of ancestors) {
-              const item = rowToSummary(row);
-              if (!known.has(item.id)) {
-                items.push(item);
-                known.add(item.id);
-              }
-            }
-          }
-        } catch {
-        }
         for (const row of linked) {
           const item = rowToSummary(row);
           if (!known.has(item.id)) {
@@ -1528,12 +1512,33 @@ var MailDatabase = class {
             known.add(item.id);
           }
         }
+        const seedRow = this.open().prepare(
+          'SELECT in_reply_to, "references", raw_eml FROM emails WHERE lower(message_id) = ? LIMIT 1'
+        ).get(seedId);
+        const parents = `${seedRow?.references ?? ""} ${seedRow?.in_reply_to ?? ""}`.split(/\s+/).map((item) => item.replace(/^<|>$/g, "").toLowerCase()).filter((item) => item.includes("@"));
+        if (parents.length === 0 && seedRow?.raw_eml !== void 0 && seedRow.raw_eml !== null) {
+          for (const header of ["references", "in-reply-to"]) {
+            for (const found of headerIds(seedRow.raw_eml, header)) parents.push(found);
+          }
+        }
+        if (parents.length > 0) {
+          const marks = parents.map(() => "?").join(",");
+          const ancestors = this.open().prepare(
+            `${select}
+               WHERE lower(e.message_id) IN (${marks}) LIMIT 100`
+          ).all(...parents);
+          for (const row of ancestors) {
+            const item = rowToSummary(row);
+            if (!known.has(item.id)) {
+              items.push(item);
+              known.add(item.id);
+            }
+          }
+        }
+      } catch {
       }
-      return items.slice(0, limit);
-    } catch {
-      const page = this.list({ mailbox: "all", query: base, limit: Math.min(500, limit * 2) });
-      return page.items.filter((item) => normalize2(item.subject) === base).slice(0, limit);
     }
+    return items.slice(0, limit);
   }
   /** Read (and cache) In-Reply-To / References for a set of local ids. */
   referencesFor(ids) {
@@ -1792,7 +1797,117 @@ function rowToSummary(row) {
   };
 }
 
+// src/host/references.ts
+function extractReferences(raw, messageId) {
+  const empty = { inReplyTo: null, references: null };
+  if (raw === null || raw === void 0) return empty;
+  const text2 = Buffer.from(raw.subarray(0, Math.min(raw.byteLength, 2e5))).toString("utf8");
+  const separator = text2.search(/\r?\n\r?\n/);
+  const head = separator >= 0 ? text2.slice(0, separator) : text2;
+  const unfolded = head.replace(/\r?\n[ \t]+/g, " ");
+  const value = (name2) => {
+    const match = new RegExp(`^${name2}:\\s*(.*)$`, "im").exec(unfolded);
+    return match === null ? "" : String(match[1] ?? "");
+  };
+  const ids = (value2) => value2.split(/\s+/).map((item) => item.replace(/^<|>$/g, "").toLowerCase()).filter((item) => item.includes("@"));
+  const referenceIds = ids(value("references"));
+  const replyTo = ids(value("in-reply-to"));
+  const filteredRefs = referenceIds.filter((id) => id !== messageId.toLowerCase());
+  return {
+    inReplyTo: replyTo[0] ?? null,
+    references: filteredRefs.length > 0 ? filteredRefs.join(" ") : null
+  };
+}
+function backfillReferences(file, batchSize = 300) {
+  ensureMailSchema(file);
+  const db = new DatabaseSync3(file);
+  try {
+    db.exec("PRAGMA busy_timeout = 10000");
+    const rows = db.prepare("SELECT id, message_id, raw_eml FROM emails WHERE refs_indexed = 0 ORDER BY id LIMIT ?").all(batchSize);
+    if (rows.length === 0) return { updated: 0, remaining: 0 };
+    const update = db.prepare(
+      'UPDATE emails SET in_reply_to = ?, "references" = ?, refs_indexed = 1 WHERE id = ?'
+    );
+    db.exec("BEGIN");
+    try {
+      for (const row of rows) {
+        const headers = extractReferences(row.raw_eml, String(row.message_id ?? ""));
+        update.run(headers.inReplyTo, headers.references, Number(row.id));
+      }
+      db.exec("COMMIT");
+    } catch (error) {
+      db.exec("ROLLBACK");
+      throw error;
+    }
+    const remaining = Number(
+      db.prepare("SELECT COUNT(*) AS count FROM emails WHERE refs_indexed = 0").get()?.count ?? 0
+    );
+    return { updated: rows.length, remaining };
+  } finally {
+    db.close();
+  }
+}
+function optimizeDatabase(file) {
+  ensureMailSchema(file);
+  const db = new DatabaseSync3(file);
+  try {
+    db.exec("PRAGMA busy_timeout = 10000");
+    db.exec("PRAGMA optimize");
+    try {
+      db.exec("INSERT INTO emails_fts(emails_fts) VALUES('optimize')");
+    } catch {
+    }
+    try {
+      db.exec("PRAGMA wal_checkpoint(TRUNCATE)");
+    } catch {
+    }
+    return { ok: true };
+  } catch (error) {
+    return { ok: false, error: error instanceof Error ? error.message : String(error) };
+  } finally {
+    db.close();
+  }
+}
+function backfillSubjects(file, batchSize = 300) {
+  ensureMailSchema(file);
+  const db = new DatabaseSync3(file);
+  try {
+    db.exec("PRAGMA busy_timeout = 10000");
+    const rows = db.prepare(
+      `SELECT e.id, e.subject FROM emails e
+         WHERE NOT EXISTS (SELECT 1 FROM email_subjects s WHERE s.email_id = e.id)
+         ORDER BY e.id LIMIT ?`
+    ).all(batchSize);
+    if (rows.length === 0) return { updated: 0, remaining: 0 };
+    const insert = db.prepare("INSERT OR REPLACE INTO email_subjects (email_id, key) VALUES (?, ?)");
+    db.exec("BEGIN");
+    try {
+      for (const row of rows) insert.run(Number(row.id), subjectKey(String(row.subject ?? "")));
+      db.exec("COMMIT");
+    } catch (error) {
+      db.exec("ROLLBACK");
+      throw error;
+    }
+    const remaining = Number(
+      db.prepare(
+        `SELECT COUNT(*) AS count FROM emails e
+           WHERE NOT EXISTS (SELECT 1 FROM email_subjects s WHERE s.email_id = e.id)`
+      ).get()?.count ?? 0
+    );
+    return { updated: rows.length, remaining };
+  } finally {
+    db.close();
+  }
+}
+
 // src/host/backup.ts
+init_address();
+import { createHash as createHash2 } from "node:crypto";
+import { existsSync as existsSync3, mkdirSync as mkdirSync2, writeFileSync as writeFileSync2 } from "node:fs";
+import { join as join3 } from "node:path";
+import { ImapFlow } from "imapflow";
+import { simpleParser } from "mailparser";
+import { DatabaseSync as DatabaseSync4 } from "node:sqlite";
 var BATCH = 25;
 function sha256(data) {
   return createHash2("sha256").update(data).digest("hex");
@@ -1874,7 +1989,7 @@ var BackupEngine = class {
   }
   openDb() {
     ensureMailSchema(this.runtime.mailDb);
-    const db = new DatabaseSync3(this.runtime.mailDb);
+    const db = new DatabaseSync4(this.runtime.mailDb);
     db.exec("PRAGMA journal_mode=WAL");
     db.exec("PRAGMA busy_timeout=10000");
     db.exec("PRAGMA synchronous=NORMAL");
@@ -1923,6 +2038,10 @@ var BackupEngine = class {
     );
     if (inserted.changes === 0) return 0;
     const emailId = Number(inserted.lastInsertRowid);
+    db.prepare("INSERT OR REPLACE INTO email_subjects (email_id, key) VALUES (?, ?)").run(
+      emailId,
+      subjectKey(subject)
+    );
     const attachments = parsed.attachments ?? [];
     const insertAttachment = db.prepare(
       "INSERT INTO attachments (email_id, filename, content_type, size, hash, content_id, disposition) VALUES (?, ?, ?, ?, ?, ?, ?)"
@@ -2128,7 +2247,7 @@ var BackupEngine = class {
     };
     if (!existsSync3(dbFile)) return empty;
     try {
-      const db = new DatabaseSync3(dbFile, { readOnly: true });
+      const db = new DatabaseSync4(dbFile, { readOnly: true });
       try {
         const latest = db.prepare("SELECT * FROM backup_logs ORDER BY id DESC LIMIT 1").get();
         const success = db.prepare("SELECT completed_at FROM backup_logs WHERE status = 'completed' ORDER BY id DESC LIMIT 1").get();
@@ -3004,7 +3123,7 @@ ${conversation.trim()}
 }
 
 // src/host/version.ts
-var VERSION = "0.19.1";
+var VERSION = "0.20.0";
 
 // src/host/api.ts
 init_smtp();
@@ -3226,6 +3345,10 @@ function buildRouter(runtime) {
     }
     return { ok: true, count };
   });
+  router.post("/maintenance/optimize", ({ url }) => {
+    const { account } = accountFrom(runtime, url);
+    return optimizeDatabase(runtime.paths(account).mailDb);
+  });
   router.get("/maintenance/references", ({ url }) => {
     const { account } = accountFrom(runtime, url);
     return runtime.referencesStatus(account);
@@ -4276,7 +4399,7 @@ function writeAttachment(res, filename, contentType, data, inline = false) {
     "content-disposition",
     `${inline ? "inline" : "attachment"}; filename*=UTF-8''${encodeURIComponent(filename)}`
   );
-  res.setHeader("cache-control", "no-store");
+  res.setHeader("cache-control", "private, max-age=86400");
   res.setHeader("x-content-type-options", "nosniff");
   res.setHeader(
     "content-security-policy",
@@ -4730,58 +4853,6 @@ var MailFlagsStore = class {
   }
 };
 
-// src/host/references.ts
-import { DatabaseSync as DatabaseSync4 } from "node:sqlite";
-function extractReferences(raw, messageId) {
-  const empty = { inReplyTo: null, references: null };
-  if (raw === null || raw === void 0) return empty;
-  const text2 = Buffer.from(raw.subarray(0, Math.min(raw.byteLength, 2e5))).toString("utf8");
-  const separator = text2.search(/\r?\n\r?\n/);
-  const head = separator >= 0 ? text2.slice(0, separator) : text2;
-  const unfolded = head.replace(/\r?\n[ \t]+/g, " ");
-  const value = (name2) => {
-    const match = new RegExp(`^${name2}:\\s*(.*)$`, "im").exec(unfolded);
-    return match === null ? "" : String(match[1] ?? "");
-  };
-  const ids = (value2) => value2.split(/\s+/).map((item) => item.replace(/^<|>$/g, "").toLowerCase()).filter((item) => item.includes("@"));
-  const referenceIds = ids(value("references"));
-  const replyTo = ids(value("in-reply-to"));
-  const filteredRefs = referenceIds.filter((id) => id !== messageId.toLowerCase());
-  return {
-    inReplyTo: replyTo[0] ?? null,
-    references: filteredRefs.length > 0 ? filteredRefs.join(" ") : null
-  };
-}
-function backfillReferences(file, batchSize = 300) {
-  ensureMailSchema(file);
-  const db = new DatabaseSync4(file);
-  try {
-    db.exec("PRAGMA busy_timeout = 10000");
-    const rows = db.prepare("SELECT id, message_id, raw_eml FROM emails WHERE refs_indexed = 0 ORDER BY id LIMIT ?").all(batchSize);
-    if (rows.length === 0) return { updated: 0, remaining: 0 };
-    const update = db.prepare(
-      'UPDATE emails SET in_reply_to = ?, "references" = ?, refs_indexed = 1 WHERE id = ?'
-    );
-    db.exec("BEGIN");
-    try {
-      for (const row of rows) {
-        const headers = extractReferences(row.raw_eml, String(row.message_id ?? ""));
-        update.run(headers.inReplyTo, headers.references, Number(row.id));
-      }
-      db.exec("COMMIT");
-    } catch (error) {
-      db.exec("ROLLBACK");
-      throw error;
-    }
-    const remaining = Number(
-      db.prepare("SELECT COUNT(*) AS count FROM emails WHERE refs_indexed = 0").get()?.count ?? 0
-    );
-    return { updated: rows.length, remaining };
-  } finally {
-    db.close();
-  }
-}
-
 // src/host/tasks.ts
 import { randomUUID as randomUUID4 } from "node:crypto";
 import { mkdirSync as mkdirSync7 } from "node:fs";
@@ -5391,6 +5462,13 @@ var MailHarborRuntime = class {
           if (result.remaining <= 0 || result.updated === 0) break;
           await new Promise((resolve) => setTimeout(resolve, 40));
         }
+        for (; ; ) {
+          const result = backfillSubjects(dbFile, 300);
+          status.indexed += result.updated;
+          status.remaining = result.remaining;
+          if (result.remaining <= 0 || result.updated === 0) break;
+          await new Promise((resolve) => setTimeout(resolve, 40));
+        }
       } catch (error) {
         status.error = error instanceof Error ? error.message : String(error);
       } finally {