@lengelhard/imap-email-mcp 1.9.0 → 1.9.1

package/CHANGELOG.md

@@ -1,5 +1,11 @@
 # Changelog
 
+## 1.9.1
+
+### Fixed
+- The threading-header backfill no longer runs inside each folder's sync. On a server that reads In-Reply-To and References from disk at a few messages per second (seen on Migadu: about 3 per second on a large Archive), the 3,000-row slice took longer than the run cap, so every scheduled run was cut off in Archive and the folders after it were never synced. The backfill now runs once per scheduled run, after every folder has had its normal sync and `last_full_sync` has been stamped, under its own time budget (`INDEX_BACKFILL_SECONDS`, default 90, `0` disables). Folders with the fewest pending rows go first, so the working folders reach full coverage long before a large Archive. `sync_index` and refresh-triggered syncs never backfill, so they stay quick.
+- `last_run` records `thread_headers_backfilled` for each scheduled run.
+
 ## 1.9.0
 
 ### Added

package/README.md

@@ -107,6 +107,7 @@ Add to your config file:
 | `INDEX_EXCLUDE_FOLDERS` | none | Comma-separated folders to skip, on top of Trash/Junk |
 | `INDEX_SYNC_INTERVAL_MINUTES` | `5` | Background sync interval, measured from the end of the previous run |
 | `INDEX_SYNC_MAX_MINUTES` | `15` | Wall-clock cap on one sync run; a run that outlives it is aborted and its connection dropped (`0` disables) |
+| `INDEX_BACKFILL_SECONDS` | `90` | Time budget per scheduled run for backfilling threading headers on rows indexed before 1.9.0 (`0` disables) |
 
 ### Local search index
 
@@ -136,7 +137,7 @@ A conversation can end up spread across folders without any message being duplic
 - **Why headers, not subject.** References lists the Message-IDs of every earlier message in the conversation, so a reply keeps its thread even after someone renames the subject, and two unrelated conversations with the same subject line stay apart. Subject matching gets both of those wrong.
 - **Scope.** `scope: "split"` (default) reports only threads present in more than one of the searched folders; `scope: "all"` lists every thread. The folders you pass decide what counts as split: a thread with three messages in Delegated and two in Hold is split when both folders are searched and not when only Delegated is.
 - **Copies are not splits.** A thread whose only spread is the same Message-ID present in two folders is marked `copies_only`. That is a duplicate, and `find_duplicates` is the tool for it.
-- **Honest coverage.** Rows indexed before 1.9.0 have no threading headers until the background sync backfills them (3,000 rows per folder per run). `coverage.without_thread_headers` and `index_status.thread_headers_pending` show how many are still waiting, and a thread touching such rows is marked `threading: "partial"`.
+- **Honest coverage.** Rows indexed before 1.9.0 have no threading headers until the background sync backfills them, which it does at the end of each scheduled run for up to `INDEX_BACKFILL_SECONDS`, smallest folders first, without ever delaying the indexing of new mail. `coverage.without_thread_headers` and `index_status.thread_headers_pending` show how many are still waiting, and a thread touching such rows is marked `threading: "partial"`.
 - **Nothing is moved.** The tool only reports. `move_email` or `move_thread` are the tools for acting on it.
 
 ### Provider Settings

package/index.js

@@ -2142,7 +2142,8 @@ async function main() {
         folders: list(process.env.INDEX_FOLDERS),
         excludeFolders: list(process.env.INDEX_EXCLUDE_FOLDERS),
         intervalMinutes: parseFloat(process.env.INDEX_SYNC_INTERVAL_MINUTES || '5'),
-        maxRunMinutes: parseFloat(process.env.INDEX_SYNC_MAX_MINUTES || '15')
+        maxRunMinutes: parseFloat(process.env.INDEX_SYNC_MAX_MINUTES || '15'),
+        backfillSeconds: parseFloat(process.env.INDEX_BACKFILL_SECONDS || '90')
       }).open();
       localIndex.start();
       console.error(`Local index enabled: ${localIndex.dbPath}`);

package/lib/local-index.js

@@ -57,7 +57,9 @@ function pidAlive(pid) {
   }
 }
 const HEADER_FIELDS = 'HEADER.FIELDS (FROM TO CC SUBJECT DATE MESSAGE-ID IN-REPLY-TO REFERENCES)';
-const THREAD_BACKFILL_PER_FOLDER = 3000;
+const DEFAULT_BACKFILL_SECONDS = 90;
+// Small batches for the backfill: on a slow server a 100-row header fetch has taken longer than the command timeout.
+const BACKFILL_BATCH = 25;
 const DEFAULT_EXCLUDE = ['trash', 'junk', 'spam', 'deleted items', 'deleted messages', 'bin'];
 const SPECIAL_EXCLUDE = ['\\trash', '\\junk'];
 
@@ -152,7 +154,7 @@ export function parseMessageIdList(value) {
 }
 
 export class LocalIndex {
-  constructor({ imapConfig, dbPath, folders = null, excludeFolders = [], intervalMinutes = 5, maxRunMinutes = DEFAULT_MAX_RUN_MINUTES, batchSize = 100, maxBodyChars = 100000, log = console.error }) {
+  constructor({ imapConfig, dbPath, folders = null, excludeFolders = [], intervalMinutes = 5, maxRunMinutes = DEFAULT_MAX_RUN_MINUTES, backfillSeconds = DEFAULT_BACKFILL_SECONDS, batchSize = 100, maxBodyChars = 100000, log = console.error }) {
     this.imapConfig = imapConfig;
     this.dbPath = dbPath || defaultDbPath(imapConfig.imap.user, imapConfig.imap.host);
     this.folders = folders && folders.length ? folders : null;
@@ -161,6 +163,9 @@ export class LocalIndex {
     // Wall-clock cap on one sync run. 0 disables it. Fractions are allowed for tests.
     const cap = Number(maxRunMinutes);
     this.maxRunMs = Number.isFinite(cap) && cap >= 0 ? cap * 60 * 1000 : DEFAULT_MAX_RUN_MINUTES * 60 * 1000;
+    // Time budget for the threading-header backfill at the end of each scheduled run. 0 disables it.
+    const bf = Number(backfillSeconds);
+    this.backfillMs = Number.isFinite(bf) && bf >= 0 ? bf * 1000 : DEFAULT_BACKFILL_SECONDS * 1000;
     this.batchSize = batchSize;
     this.maxBodyChars = maxBodyChars;
     this.log = log;
@@ -248,7 +253,7 @@ export class LocalIndex {
     const kick = async () => {
       try {
         const last = Number(this._meta('last_full_sync') || 0);
-        if (Date.now() - last >= this.intervalMs / 2) await this.sync();
+        if (Date.now() - last >= this.intervalMs / 2) await this.sync({ backfill: true });
       } catch (err) {
         this.log(`[index] sync failed: ${err.message}`);
       } finally {
@@ -394,7 +399,7 @@ export class LocalIndex {
    * Run a sync. Single-flight: concurrent callers share the running sync.
    * maxPerFolder caps new messages per folder per pass; passes caps the loop.
    */
-  sync({ folders = null, maxPerFolder = 1000, passes = Infinity } = {}) {
+  sync({ folders = null, maxPerFolder = 1000, passes = Infinity, backfill = false } = {}) {
     if (this.running) return this.running;
     this.runningSince = Date.now();
     this.running = (async () => {
@@ -417,7 +422,7 @@ export class LocalIndex {
       }, LOCK_HEARTBEAT_MS);
       if (heartbeat.unref) heartbeat.unref();
       try {
-        const summary = await this._runCapped(() => this._sync({ folders, maxPerFolder, passes }));
+        const summary = await this._runCapped(() => this._sync({ folders, maxPerFolder, passes, backfill }));
         if (this.db) {
           if (!folders && passes === Infinity && !summary.aborted) this._setMeta('last_full_sync', Date.now());
           this._setMeta('last_run', JSON.stringify({
@@ -427,6 +432,7 @@ export class LocalIndex {
             added: summary.added,
             removed: summary.removed,
             duration_ms: summary.duration_ms,
+            ...(summary.backfilled != null && { thread_headers_backfilled: summary.backfilled }),
             ...(summary.aborted && { aborted: summary.aborted })
           }));
         }
@@ -482,7 +488,7 @@ export class LocalIndex {
     return true;
   }
 
-  async _sync({ folders, maxPerFolder, passes }) {
+  async _sync({ folders, maxPerFolder, passes, backfill = false }) {
     const started = Date.now();
     const summary = { added: 0, removed: 0, folders: {} };
     let targets;
@@ -551,6 +557,21 @@ export class LocalIndex {
         }
       }
     }
+    // The index is current at this point; last_full_sync is stamped on it by
+    // the caller. The threading-header backfill runs after that, under its
+    // own time budget, so it can never hold up the indexing of new mail.
+    if (backfill && !summary.aborted && this.backfillMs > 0) {
+      try {
+        summary.backfilled = await this._backfillThreadHeaders(targets, Date.now() + this.backfillMs);
+      } catch (err) {
+        if (err.code === 'IMAP_TIMEOUT' || err.code === 'LOCK_LOST' || !this.db) {
+          summary.aborted = err.message;
+          this.log(`[index] backfill stopped: ${err.message}`);
+        } else {
+          this.log(`[index] backfill: ${err.message}`);
+        }
+      }
+    }
     summary.duration_ms = Date.now() - started;
     return summary;
   }
@@ -617,8 +638,6 @@ export class LocalIndex {
         added += await this._indexBatch(conn, folder, uidvalidity, todo.slice(i, i + this.batchSize));
       }
 
-      await this._backfillThreadHeaders(conn, folder);
-
       const pending = missing.length - todo.length;
       const indexed = this.db.prepare('SELECT count(*) AS n FROM messages WHERE folder = ?').get(folder).n;
       this.db.prepare(`INSERT INTO folders (name, uidvalidity, server_count, indexed_count, backfill_complete, last_sync, last_error)
@@ -635,40 +654,56 @@ export class LocalIndex {
 
   /**
    * Fetch In-Reply-To and References for rows indexed before 1.9.0, headers
-   * only, in batches, up to THREAD_BACKFILL_PER_FOLDER rows per run so a large
-   * Archive is spread over a few syncs rather than one long one.
+   * only, in batches, until the deadline. Some servers read these headers
+   * from disk at a few messages per second, so the work is spread over many
+   * scheduled runs. Folders with the fewest pending rows go first, so the
+   * small working folders reach full coverage long before a large Archive.
    */
-  async _backfillThreadHeaders(conn, folder) {
-    const rows = this.db.prepare('SELECT uid FROM messages WHERE folder = ? AND thread_headers = 0 ORDER BY uid DESC LIMIT ?')
-      .all(folder, THREAD_BACKFILL_PER_FOLDER);
-    if (!rows.length) return 0;
-    const parseHeader = conn.imap.constructor.parseHeader;
+  async _backfillThreadHeaders(folders, deadline) {
+    const counts = this.db.prepare('SELECT folder, count(*) AS n FROM messages WHERE thread_headers = 0 GROUP BY folder ORDER BY n ASC').all()
+      .filter((r) => folders.includes(r.folder));
+    if (!counts.length) return 0;
     const upd = this.db.prepare('UPDATE messages SET in_reply_to = ?, references_ids = ?, thread_headers = 1 WHERE folder = ? AND uid = ?');
-    let done = 0;
-    for (let i = 0; i < rows.length; i += this.batchSize) {
+    let total = 0;
+    for (const { folder } of counts) {
+      if (Date.now() >= deadline) break;
       this._assertLock();
-      this.progressAt = Date.now();
-      const uids = rows.slice(i, i + this.batchSize).map((r) => Number(r.uid));
-      const fetched = await fetchMessages(conn, uids, { bodies: ['HEADER.FIELDS (IN-REPLY-TO REFERENCES)'] });
-      this._assertLock();
-      this.db.exec('BEGIN');
+      const conn = await this.session.acquire();
+      let done = 0;
       try {
-        for (const uid of uids) {
-          const rec = fetched.get(uid);
-          // A uid the server no longer has is marked done; the next set
-          // difference removes the row.
-          const h = rec ? headerOf(rec, parseHeader) : {};
-          upd.run(parseMessageIdList(h['in-reply-to'])[0] || null, parseMessageIdList(h.references).join(' ') || null, folder, uid);
-          done++;
+        await openBox(conn, folder, true);
+        const parseHeader = conn.imap.constructor.parseHeader;
+        while (Date.now() < deadline) {
+          this._assertLock();
+          this.progressAt = Date.now();
+          const uids = this.db.prepare('SELECT uid FROM messages WHERE folder = ? AND thread_headers = 0 ORDER BY uid DESC LIMIT ?')
+            .all(folder, Math.min(this.batchSize, BACKFILL_BATCH)).map((r) => Number(r.uid));
+          if (!uids.length) break;
+          const fetched = await fetchMessages(conn, uids, { bodies: ['HEADER.FIELDS (IN-REPLY-TO REFERENCES)'] });
+          this._assertLock();
+          this.db.exec('BEGIN');
+          try {
+            for (const uid of uids) {
+              const rec = fetched.get(uid);
+              // A uid the server no longer has is marked done; the next set
+              // difference removes the row.
+              const h = rec ? headerOf(rec, parseHeader) : {};
+              upd.run(parseMessageIdList(h['in-reply-to'])[0] || null, parseMessageIdList(h.references).join(' ') || null, folder, uid);
+              done++;
+            }
+            this.db.exec('COMMIT');
+          } catch (err) {
+            this.db.exec('ROLLBACK');
+            throw err;
+          }
         }
-        this.db.exec('COMMIT');
-      } catch (err) {
-        this.db.exec('ROLLBACK');
-        throw err;
+      } finally {
+        conn.end();
+        if (done) this.log(`[index] ${folder}: threading headers backfilled for ${done} message(s)`);
+        total += done;
       }
     }
-    if (done) this.log(`[index] ${folder}: threading headers backfilled for ${done} message(s)`);
-    return done;
+    return total;
   }
 
   async _indexBatch(conn, folder, uidvalidity, uids) {

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@lengelhard/imap-email-mcp",
-  "version": "1.9.0",
+  "version": "1.9.1",
   "description": "MCP server for Claude Code that provides email capabilities through IMAP/SMTP. Read, search, compose, and manage emails from any IMAP provider.",
   "type": "module",
   "main": "index.js",