@bobfrankston/mailx-store 0.1.27 → 0.1.29

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/db.d.ts CHANGED
@@ -347,6 +347,14 @@ export declare class MailxDB {
347
347
  beginTransaction(): void;
348
348
  commitTransaction(): void;
349
349
  rollbackTransaction(): void;
350
+ /** Run `fn` inside a transaction — nesting-safe. `node:sqlite` throws on
351
+ * a nested BEGIN, and the async chunked walkers (seedContactsFromMessages,
352
+ * applyContactsConfig) can interleave with the sync backfill's own
353
+ * transactions across their `setImmediate` yields. If a transaction is
354
+ * already open this just runs `fn` (its writes join the open txn);
355
+ * otherwise it owns a fresh BEGIN/COMMIT. `fn` MUST be synchronous so it
356
+ * can't span an await and leave a transaction open across a yield. */
357
+ runInTxn<T>(fn: () => T): T;
350
358
  /** Record an address used in sent mail */
351
359
  recordSentAddress(name: string, email: string): void;
352
360
  /** True if `email` (lowercased) appears in the active denylist. Cached
@@ -399,7 +407,16 @@ export declare class MailxDB {
399
407
  * Discovered is a single tier; sub-distinctions like sent-vs-received
400
408
  * collapse here because the user-facing UI shows them as one "discovered"
401
409
  * source. Recency-weighted use_count differentiates within the tier. */
402
- seedContactsFromMessages(): number;
410
+ /** ASYNC + chunked. This walks every cached message's address fields —
411
+ * on a large account that is hundreds of thousands of rows. The old
412
+ * synchronous version did `.all()` on the whole messages table in one
413
+ * call, materialising every row and blocking the event loop for tens
414
+ * of seconds to minutes (profiled 2026-05-15: 99% of daemon ticks were
415
+ * in native SQLite while this ran — every getMessage IPC, every preview
416
+ * click, queued behind it; that IS the "loading body takes forever"
417
+ * bug). Now keyset-paginated by `m.id` with a `setImmediate` yield
418
+ * between pages, so user IPC lands in the gaps. */
419
+ seedContactsFromMessages(): Promise<number>;
403
420
  /** Apply the contents of contacts.jsonc — replaces all preferred-tier rows
404
421
  * with the entries in `preferred[]`, merges `discovered[]` into the local
405
422
  * cache, sets the in-memory denylist, and purges any discovered rows
@@ -428,12 +445,12 @@ export declare class MailxDB {
428
445
  useCount?: number;
429
446
  lastUsed?: number;
430
447
  }[];
431
- }): {
448
+ }): Promise<{
432
449
  preferred: number;
433
450
  discovered: number;
434
451
  purged: number;
435
452
  conflicts: string[];
436
- };
453
+ }>;
437
454
  /** Build the contacts.jsonc shape from current DB state — for round-trip
438
455
  * to GDrive. Preferred-tier rows come from anything not in the reserved
439
456
  * system sources; discovered comes from `source='discovered'` rows;
package/db.js CHANGED
@@ -1355,13 +1355,18 @@ export class MailxDB {
1355
1355
  // quoted-strings (RFC-strictly forbidden, common in the wild). Run
1356
1356
  // libmime over Subject and every address display name once; safe
1357
1357
  // to call on already-decoded text (no `=?...?=` markers → no-op).
1358
+ // The `address` is decoded too: a legit address never carries an
1359
+ // encoded-word, so it's a no-op there — but spam sometimes packs the
1360
+ // whole `name <addr>` into the mailbox local-part as one encoded
1361
+ // word (`=?utf-8?q?...=3C...=40...=3E?=@host`). Decoding it at least
1362
+ // renders readable text instead of raw `=3C`/`=C3=A9` gibberish.
1358
1363
  msg.subject = decodeHeaderWords(msg.subject);
1359
1364
  if (msg.from)
1360
- msg.from = { name: decodeHeaderWords(msg.from.name || ""), address: msg.from.address || "" };
1365
+ msg.from = { name: decodeHeaderWords(msg.from.name || ""), address: decodeHeaderWords(msg.from.address || "") };
1361
1366
  if (msg.to)
1362
- msg.to = msg.to.map(a => ({ name: decodeHeaderWords(a.name || ""), address: a.address || "" }));
1367
+ msg.to = msg.to.map(a => ({ name: decodeHeaderWords(a.name || ""), address: decodeHeaderWords(a.address || "") }));
1363
1368
  if (msg.cc)
1364
- msg.cc = msg.cc.map(a => ({ name: decodeHeaderWords(a.name || ""), address: a.address || "" }));
1369
+ msg.cc = msg.cc.map(a => ({ name: decodeHeaderWords(a.name || ""), address: decodeHeaderWords(a.address || "") }));
1365
1370
  const existing = this.db.prepare("SELECT id, provider_id FROM messages WHERE account_id = ? AND folder_id = ? AND uid = ?").get(msg.accountId, msg.folderId, msg.uid);
1366
1371
  if (existing) {
1367
1372
  // Backfill provider_id on existing rows that predate this column —
@@ -1901,6 +1906,30 @@ export class MailxDB {
1901
1906
  beginTransaction() { this.db.exec("BEGIN"); }
1902
1907
  commitTransaction() { this.db.exec("COMMIT"); }
1903
1908
  rollbackTransaction() { this.db.exec("ROLLBACK"); }
1909
+ /** Run `fn` inside a transaction — nesting-safe. `node:sqlite` throws on
1910
+ * a nested BEGIN, and the async chunked walkers (seedContactsFromMessages,
1911
+ * applyContactsConfig) can interleave with the sync backfill's own
1912
+ * transactions across their `setImmediate` yields. If a transaction is
1913
+ * already open this just runs `fn` (its writes join the open txn);
1914
+ * otherwise it owns a fresh BEGIN/COMMIT. `fn` MUST be synchronous so it
1915
+ * can't span an await and leave a transaction open across a yield. */
1916
+ runInTxn(fn) {
1917
+ if (this.db.isTransaction)
1918
+ return fn();
1919
+ this.db.exec("BEGIN");
1920
+ try {
1921
+ const r = fn();
1922
+ this.db.exec("COMMIT");
1923
+ return r;
1924
+ }
1925
+ catch (e) {
1926
+ try {
1927
+ this.db.exec("ROLLBACK");
1928
+ }
1929
+ catch { /* already rolled back */ }
1930
+ throw e;
1931
+ }
1932
+ }
1904
1933
  // ── Contacts ──
1905
1934
  /** Record an address used in sent mail */
1906
1935
  recordSentAddress(name, email) {
@@ -1992,7 +2021,16 @@ export class MailxDB {
1992
2021
  * Discovered is a single tier; sub-distinctions like sent-vs-received
1993
2022
  * collapse here because the user-facing UI shows them as one "discovered"
1994
2023
  * source. Recency-weighted use_count differentiates within the tier. */
1995
- seedContactsFromMessages() {
2024
+ /** ASYNC + chunked. This walks every cached message's address fields —
2025
+ * on a large account that is hundreds of thousands of rows. The old
2026
+ * synchronous version did `.all()` on the whole messages table in one
2027
+ * call, materialising every row and blocking the event loop for tens
2028
+ * of seconds to minutes (profiled 2026-05-15: 99% of daemon ticks were
2029
+ * in native SQLite while this ran — every getMessage IPC, every preview
2030
+ * click, queued behind it; that IS the "loading body takes forever"
2031
+ * bug). Now keyset-paginated by `m.id` with a `setImmediate` yield
2032
+ * between pages, so user IPC lands in the gaps. */
2033
+ async seedContactsFromMessages() {
1996
2034
  const VALID = /^[^\s<>@]+@[^\s<>@]+\.[^\s<>@]+$/;
1997
2035
  const now = Date.now();
1998
2036
  const agg = new Map();
@@ -2016,57 +2054,75 @@ export class MailxDB {
2016
2054
  agg.set(email, { name: name || "", cnt: 1, last: date || 0 });
2017
2055
  }
2018
2056
  };
2019
- // Sent folder: recipients only (skip the user's own From address).
2020
- const sentRows = this.db.prepare(`SELECT m.to_json, m.cc_json, m.bcc_json, m.date
2021
- FROM messages m
2022
- JOIN folders f ON m.folder_id = f.id
2023
- WHERE f.special_use = 'sent'`).all();
2024
- for (const r of sentRows) {
2025
- const date = r.date || 0;
2026
- for (const field of [r.to_json, r.cc_json, r.bcc_json]) {
2027
- if (!field)
2028
- continue;
2029
- let parsed;
2030
- try {
2031
- parsed = JSON.parse(field);
2032
- }
2033
- catch {
2034
- continue;
2035
- }
2036
- if (!Array.isArray(parsed))
2057
+ const eatRecipients = (field, date) => {
2058
+ if (!field)
2059
+ return;
2060
+ let parsed;
2061
+ try {
2062
+ parsed = JSON.parse(field);
2063
+ }
2064
+ catch {
2065
+ return;
2066
+ }
2067
+ if (!Array.isArray(parsed))
2068
+ return;
2069
+ for (const a of parsed) {
2070
+ if (!a)
2037
2071
  continue;
2038
- for (const a of parsed) {
2039
- if (!a)
2040
- continue;
2041
- bump(a.name || "", a.address || a.email || "", date);
2072
+ bump(a.name || "", a.address || a.email || "", date);
2073
+ }
2074
+ };
2075
+ const yieldLoop = () => new Promise(r => setImmediate(r));
2076
+ const PAGE = 2000;
2077
+ // Sent folder: recipients only (skip the user's own From address).
2078
+ // Keyset pagination by m.id — each page is a bounded `.all()`, and
2079
+ // the event loop is handed back between pages.
2080
+ {
2081
+ const stmt = this.db.prepare(`SELECT m.id AS id, m.to_json, m.cc_json, m.bcc_json, m.date
2082
+ FROM messages m
2083
+ JOIN folders f ON m.folder_id = f.id
2084
+ WHERE f.special_use = 'sent' AND m.id > ?
2085
+ ORDER BY m.id LIMIT ?`);
2086
+ let lastId = 0;
2087
+ for (;;) {
2088
+ const rows = stmt.all(lastId, PAGE);
2089
+ if (rows.length === 0)
2090
+ break;
2091
+ for (const r of rows) {
2092
+ const date = r.date || 0;
2093
+ eatRecipients(r.to_json, date);
2094
+ eatRecipients(r.cc_json, date);
2095
+ eatRecipients(r.bcc_json, date);
2042
2096
  }
2097
+ lastId = rows[rows.length - 1].id;
2098
+ if (rows.length < PAGE)
2099
+ break;
2100
+ await yieldLoop();
2043
2101
  }
2044
2102
  }
2045
2103
  // Other folders: From + recipients.
2046
- const recvRows = this.db.prepare(`SELECT m.from_name, m.from_address, m.to_json, m.cc_json, m.bcc_json, m.date
2047
- FROM messages m
2048
- LEFT JOIN folders f ON m.folder_id = f.id
2049
- WHERE f.special_use IS NULL OR f.special_use != 'sent'`).all();
2050
- for (const r of recvRows) {
2051
- const date = r.date || 0;
2052
- bump(r.from_name, r.from_address, date);
2053
- for (const field of [r.to_json, r.cc_json, r.bcc_json]) {
2054
- if (!field)
2055
- continue;
2056
- let parsed;
2057
- try {
2058
- parsed = JSON.parse(field);
2059
- }
2060
- catch {
2061
- continue;
2062
- }
2063
- if (!Array.isArray(parsed))
2064
- continue;
2065
- for (const a of parsed) {
2066
- if (!a)
2067
- continue;
2068
- bump(a.name || "", a.address || a.email || "", date);
2104
+ {
2105
+ const stmt = this.db.prepare(`SELECT m.id AS id, m.from_name, m.from_address, m.to_json, m.cc_json, m.bcc_json, m.date
2106
+ FROM messages m
2107
+ LEFT JOIN folders f ON m.folder_id = f.id
2108
+ WHERE (f.special_use IS NULL OR f.special_use != 'sent') AND m.id > ?
2109
+ ORDER BY m.id LIMIT ?`);
2110
+ let lastId = 0;
2111
+ for (;;) {
2112
+ const rows = stmt.all(lastId, PAGE);
2113
+ if (rows.length === 0)
2114
+ break;
2115
+ for (const r of rows) {
2116
+ const date = r.date || 0;
2117
+ bump(r.from_name, r.from_address, date);
2118
+ eatRecipients(r.to_json, date);
2119
+ eatRecipients(r.cc_json, date);
2120
+ eatRecipients(r.bcc_json, date);
2069
2121
  }
2122
+ lastId = rows[rows.length - 1].id;
2123
+ if (rows.length < PAGE)
2124
+ break;
2125
+ await yieldLoop();
2070
2126
  }
2071
2127
  }
2072
2128
  let added = 0;
@@ -2077,16 +2133,30 @@ export class MailxDB {
2077
2133
  name = CASE WHEN name = '' AND ? != '' THEN ? ELSE name END,
2078
2134
  updated_at = ?
2079
2135
  WHERE id = ?`);
2080
- for (const [email, info] of agg) {
2081
- const existing = this.db.prepare("SELECT id FROM contacts WHERE source = 'discovered' AND lower(email) = ?").get(email);
2082
- if (!existing) {
2083
- insStmt.run(info.name, email, info.last, info.cnt, now);
2084
- added++;
2085
- }
2086
- else {
2087
- updStmt.run(info.cnt, info.last, info.name, info.name, now, existing.id);
2088
- bumped++;
2089
- }
2136
+ const findStmt = this.db.prepare("SELECT id FROM contacts WHERE source = 'discovered' AND lower(email) = ?");
2137
+ // Write phase — chunked. Each chunk runs via `db.transaction()`
2138
+ // (one synchronous turn, one commit) rather than raw BEGIN/COMMIT
2139
+ // straddling an `await` — the wrapper is nesting-safe (savepoints)
2140
+ // so it can't collide with the sync backfill's own transactions
2141
+ // when the two interleave across the yield below.
2142
+ const WRITE_CHUNK = 500;
2143
+ const entries = [...agg.entries()];
2144
+ for (let i = 0; i < entries.length; i += WRITE_CHUNK) {
2145
+ const slice = entries.slice(i, i + WRITE_CHUNK);
2146
+ this.runInTxn(() => {
2147
+ for (const [email, info] of slice) {
2148
+ const existing = findStmt.get(email);
2149
+ if (!existing) {
2150
+ insStmt.run(info.name, email, info.last, info.cnt, now);
2151
+ added++;
2152
+ }
2153
+ else {
2154
+ updStmt.run(info.cnt, info.last, info.name, info.name, now, existing.id);
2155
+ bumped++;
2156
+ }
2157
+ }
2158
+ });
2159
+ await yieldLoop();
2090
2160
  }
2091
2161
  if (added > 0 || bumped > 0) {
2092
2162
  console.log(` [contacts] seed: ${added} new + ${bumped} refreshed (discovered)`);
@@ -2104,7 +2174,7 @@ export class MailxDB {
2104
2174
  * Discovered rows from the file are MERGED with whatever the local
2105
2175
  * message-corpus seeder has produced. Each device contributes its
2106
2176
  * observed addresses; over time GDrive accumulates the union. */
2107
- applyContactsConfig(cfg) {
2177
+ async applyContactsConfig(cfg) {
2108
2178
  const preferred = Array.isArray(cfg.preferred) ? cfg.preferred : [];
2109
2179
  const denylist = Array.isArray(cfg.denylist) ? cfg.denylist : [];
2110
2180
  const denylistPatterns = Array.isArray(cfg.denylistPatterns) ? cfg.denylistPatterns : [];
@@ -2119,41 +2189,65 @@ export class MailxDB {
2119
2189
  .filter(e => e && e.priority === true && e.email)
2120
2190
  .map(e => e.email);
2121
2191
  this.setPriorityIndex(prioritySenders, priorityDomains);
2192
+ const VALID = /^[^\s<>@]+@[^\s<>@]+\.[^\s<>@]+$/;
2193
+ const now = Date.now();
2194
+ const denySet = new Set(denylist.map(e => (e || "").trim().toLowerCase()).filter(Boolean));
2195
+ const conflicts = [];
2196
+ const yieldLoop = () => new Promise(r => setImmediate(r));
2197
+ const CHUNK = 500;
2122
2198
  // Wipe and rewrite preferred-tier rows owned by contacts.jsonc.
2123
2199
  // The address-book UI's legacy `upsertContact` still writes
2124
2200
  // source='manual' rows; those are owned by the address-book code
2125
2201
  // path, not contacts.jsonc, so we leave them alone here.
2126
- this.db.exec("DELETE FROM contacts WHERE source NOT IN ('google', 'discovered', 'manual')");
2127
- const VALID = /^[^\s<>@]+@[^\s<>@]+\.[^\s<>@]+$/;
2128
- const now = Date.now();
2202
+ // ALL writes below run inside transactions and yield the event
2203
+ // loop between chunks. The prior version did one autocommitting
2204
+ // INSERT/UPDATE per row with NO transaction — on a contacts.jsonc
2205
+ // that has accumulated thousands of `discovered` entries across
2206
+ // devices, that was thousands of fsyncs back-to-back, blocking the
2207
+ // daemon for 25-30 s (profiled 2026-05-15: applyContactsConfig was
2208
+ // ~97% of the wedge). It also re-`prepare()`d a `lower(email)`
2209
+ // full-scan SELECT every iteration. Now: one statement prep, one
2210
+ // up-front map of existing discovered rows (kills the N×N scan),
2211
+ // chunked transactions, yields between chunks.
2129
2212
  const ins = this.db.prepare(`INSERT OR IGNORE INTO contacts (source, name, email, organization, last_used, use_count, updated_at)
2130
2213
  VALUES (?, ?, ?, ?, 0, 0, ?)`);
2131
- const denySet = new Set(denylist.map(e => (e || "").trim().toLowerCase()).filter(Boolean));
2132
- const conflicts = [];
2133
2214
  let inserted = 0;
2134
- for (const entry of preferred) {
2135
- if (!entry)
2136
- continue;
2137
- const email = (entry.email || "").trim();
2138
- if (!email || !VALID.test(email))
2139
- continue;
2140
- if (denySet.has(email.toLowerCase())) {
2141
- conflicts.push(email);
2142
- continue;
2143
- }
2144
- const source = (entry.source || "preferred").trim() || "preferred";
2145
- const name = (entry.name || "").trim();
2146
- const org = (entry.organization || entry.org || "").trim();
2147
- try {
2148
- const r = ins.run(source, name, email, org, now);
2149
- if (r.changes)
2150
- inserted++;
2215
+ // runInTxn: nesting-safe and atomic within one synchronous turn, so
2216
+ // these writes can't collide with the sync backfill's transactions
2217
+ // when the two interleave across the yields below.
2218
+ this.runInTxn(() => {
2219
+ this.db.exec("DELETE FROM contacts WHERE source NOT IN ('google', 'discovered', 'manual')");
2220
+ for (const entry of preferred) {
2221
+ if (!entry)
2222
+ continue;
2223
+ const email = (entry.email || "").trim();
2224
+ if (!email || !VALID.test(email))
2225
+ continue;
2226
+ if (denySet.has(email.toLowerCase())) {
2227
+ conflicts.push(email);
2228
+ continue;
2229
+ }
2230
+ const source = (entry.source || "preferred").trim() || "preferred";
2231
+ const name = (entry.name || "").trim();
2232
+ const org = (entry.organization || entry.org || "").trim();
2233
+ try {
2234
+ const r = ins.run(source, name, email, org, now);
2235
+ if (r.changes)
2236
+ inserted++;
2237
+ }
2238
+ catch { /* dup row, skip */ }
2151
2239
  }
2152
- catch { /* dup row, skip */ }
2153
- }
2240
+ });
2241
+ await yieldLoop();
2154
2242
  // Merge discovered[] from cloud into local cache. For each entry:
2155
2243
  // existing row wins on use_count (max), name fills if empty, lastUsed
2156
2244
  // is max. Missing rows are inserted. Denylisted entries skipped.
2245
+ // Existing discovered rows are mapped ONCE up front so the merge is
2246
+ // O(N) hash lookups, not O(N) `lower(email)` table scans.
2247
+ const existingDiscovered = new Map();
2248
+ for (const row of this.db.prepare("SELECT id, lower(email) AS le FROM contacts WHERE source = 'discovered'").all()) {
2249
+ existingDiscovered.set(row.le, row.id);
2250
+ }
2157
2251
  const insDiscovered = this.db.prepare("INSERT INTO contacts (source, name, email, last_used, use_count, updated_at) VALUES ('discovered', ?, ?, ?, ?, ?)");
2158
2252
  const updDiscovered = this.db.prepare(`UPDATE contacts SET use_count = max(use_count, ?),
2159
2253
  last_used = max(last_used, ?),
@@ -2161,36 +2255,44 @@ export class MailxDB {
2161
2255
  updated_at = ?
2162
2256
  WHERE id = ?`);
2163
2257
  let discoveredAdded = 0;
2164
- for (const entry of discovered) {
2165
- if (!entry)
2166
- continue;
2167
- const email = (entry.email || "").trim();
2168
- if (!email || !VALID.test(email))
2169
- continue;
2170
- const lower = email.toLowerCase();
2171
- if (denySet.has(lower))
2172
- continue;
2173
- if (isJunkContact(lower, entry.name || ""))
2174
- continue;
2175
- const name = (entry.name || "").trim();
2176
- const useCount = Math.max(0, entry.useCount || 0);
2177
- const lastUsed = Math.max(0, entry.lastUsed || 0);
2178
- const existing = this.db.prepare("SELECT id FROM contacts WHERE source = 'discovered' AND lower(email) = ?").get(lower);
2179
- if (!existing) {
2180
- insDiscovered.run(name, email, lastUsed, useCount, now);
2181
- discoveredAdded++;
2182
- }
2183
- else {
2184
- updDiscovered.run(useCount, lastUsed, name, name, now, existing.id);
2185
- }
2258
+ for (let i = 0; i < discovered.length; i += CHUNK) {
2259
+ const slice = discovered.slice(i, i + CHUNK);
2260
+ this.runInTxn(() => {
2261
+ for (const entry of slice) {
2262
+ if (!entry)
2263
+ continue;
2264
+ const email = (entry.email || "").trim();
2265
+ if (!email || !VALID.test(email))
2266
+ continue;
2267
+ const lower = email.toLowerCase();
2268
+ if (denySet.has(lower))
2269
+ continue;
2270
+ if (isJunkContact(lower, entry.name || ""))
2271
+ continue;
2272
+ const name = (entry.name || "").trim();
2273
+ const useCount = Math.max(0, entry.useCount || 0);
2274
+ const lastUsed = Math.max(0, entry.lastUsed || 0);
2275
+ const existingId = existingDiscovered.get(lower);
2276
+ if (existingId === undefined) {
2277
+ insDiscovered.run(name, email, lastUsed, useCount, now);
2278
+ discoveredAdded++;
2279
+ }
2280
+ else {
2281
+ updDiscovered.run(useCount, lastUsed, name, name, now, existingId);
2282
+ }
2283
+ }
2284
+ });
2285
+ await yieldLoop();
2186
2286
  }
2187
2287
  // Purge discovered rows for any denylisted email.
2188
2288
  const purge = this.db.prepare("DELETE FROM contacts WHERE source = 'discovered' AND lower(email) = ?");
2189
2289
  let purged = 0;
2190
- for (const e of denySet) {
2191
- const r = purge.run(e);
2192
- purged += Number(r.changes || 0);
2193
- }
2290
+ this.runInTxn(() => {
2291
+ for (const e of denySet) {
2292
+ const r = purge.run(e);
2293
+ purged += Number(r.changes || 0);
2294
+ }
2295
+ });
2194
2296
  if (conflicts.length > 0) {
2195
2297
  console.warn(` [contacts] config: ${conflicts.length} preferred entries also appear in denylist — denylist wins, entries skipped: ${conflicts.join(", ")}`);
2196
2298
  }
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "@bobfrankston/mailx-store",
3
- "version": "0.1.27",
3
+ "version": "0.1.29",
4
4
  "type": "module",
5
5
  "main": "index.js",
6
6
  "types": "index.d.ts",
@@ -9,8 +9,8 @@
9
9
  },
10
10
  "license": "ISC",
11
11
  "dependencies": {
12
- "@bobfrankston/mailx-types": "^0.1.14",
13
- "@bobfrankston/mailx-settings": "^0.1.17",
12
+ "@bobfrankston/mailx-types": "^0.1.15",
13
+ "@bobfrankston/mailx-settings": "^0.1.18",
14
14
  "@bobfrankston/mailx-bus": "^0.1.2",
15
15
  "mailparser": "^3.7.2"
16
16
  },
@@ -29,8 +29,8 @@
29
29
  },
30
30
  ".transformedSnapshot": {
31
31
  "dependencies": {
32
- "@bobfrankston/mailx-types": "^0.1.14",
33
- "@bobfrankston/mailx-settings": "^0.1.17",
32
+ "@bobfrankston/mailx-types": "^0.1.15",
33
+ "@bobfrankston/mailx-settings": "^0.1.18",
34
34
  "@bobfrankston/mailx-bus": "^0.1.2",
35
35
  "mailparser": "^3.7.2"
36
36
  }
package/parse-serial.js CHANGED
@@ -73,7 +73,10 @@ function getWorker() {
73
73
  });
74
74
  w.on("error", (e) => {
75
75
  // Fatal worker crash — fail every in-flight parse, then null the
76
- // singleton so the next call respawns.
76
+ // singleton so the next call respawns. Logged loudly: a silent
77
+ // respawn loop forces every Nth parse to eat a fresh cold-start,
78
+ // which reads as a random multi-second "loading body" stall.
79
+ console.error(` [parse-worker] CRASHED (${e.message}) — ${_inflight.size} in-flight parse(s) failed, will respawn`);
77
80
  for (const entry of _inflight.values())
78
81
  entry.reject(e);
79
82
  _inflight.clear();
@@ -81,6 +84,9 @@ function getWorker() {
81
84
  signalCompletion();
82
85
  });
83
86
  w.on("exit", (code) => {
87
+ // code 0 = clean exit (shouldn't happen — parentPort keeps it alive);
88
+ // non-zero = crash. Either way the next parse respawns + re-warms.
89
+ console.warn(` [parse-worker] exited (code=${code}), ${_inflight.size} in-flight — next parse respawns`);
84
90
  if (_inflight.size > 0) {
85
91
  const err = new Error(`parse-worker exited (code=${code}) with ${_inflight.size} in-flight`);
86
92
  for (const entry of _inflight.values())
package/parse-worker.js CHANGED
@@ -83,7 +83,17 @@ parentPort.on("message", async (msg) => {
83
83
  const u = source;
84
84
  normalized = Buffer.from(u.buffer, u.byteOffset, u.byteLength);
85
85
  }
86
+ const _t0 = Date.now();
86
87
  const result = await simpleParser(normalized);
88
+ const _ms = Date.now() - _t0;
89
+ // Worker-side parse time. If THIS is slow, the bug is in simpleParser
90
+ // on a given input; if this is fast but the main thread still saw a
91
+ // multi-second `[parse]` figure, the cost is queue wait / respawn —
92
+ // splitting the two was the open question of the 2026-05-15 hunt.
93
+ if (_ms > 1000) {
94
+ const size = Buffer.isBuffer(normalized) ? normalized.byteLength : String(normalized).length;
95
+ console.warn(` [parse-worker] simpleParser took ${_ms}ms for ${(size / 1024).toFixed(0)} KB (worker-side, not queue wait)`);
96
+ }
87
97
  parentPort.postMessage({ id, ok: true, result });
88
98
  }
89
99
  catch (e) {
package/store.d.ts CHANGED
@@ -78,6 +78,7 @@ export declare class Store {
78
78
  private parsedLru;
79
79
  private parsedLruGet;
80
80
  private parsedLruPut;
81
+ private parseInflight;
81
82
  private _allowlistCache;
82
83
  private _settingsCache;
83
84
  private getCachedAllowlist;
package/store.js CHANGED
@@ -95,6 +95,15 @@ export class Store {
95
95
  this.parsedLru.delete(oldest);
96
96
  }
97
97
  }
98
+ // In-flight parse coalescing. Concurrent getMessage calls for the same
99
+ // body (same path|mtime cacheKey) must share ONE parse — otherwise a
100
+ // viewer that re-requests while the first parse is still queued enqueues
101
+ // a duplicate, and N retries become N parses stacked in the parse-worker
102
+ // queue. Observed 2026-05-15: the same .eml parsed 3-4× in a burst, and
103
+ // a genuine click then waited 25 s behind the pile-up. The map entry is
104
+ // removed when the parse settles (success OR failure — a failed parse
105
+ // must not poison the key).
106
+ parseInflight = new Map();
98
107
  // Allowlist + settings caches. Both files live on the GDrive-mounted
99
108
  // shared dir; their sync `readFileSync` calls in `loadAllowlist()` and
100
109
  // `loadSettings()` can stall for seconds per call. getMessage runs on
@@ -262,13 +271,32 @@ export class Store {
262
271
  // queue-contention parses. Synchronous parse makes the whole
263
272
  // class structurally impossible: getMessage either returns the
264
273
  // body or returns `cached:false` meaning exactly "fetch it."
265
- const raw = await this.bodyStore.readByPath(storedPath);
266
- const adjusted = sniffAndFixCharset(raw);
267
- const _parseT0 = Date.now();
268
- const parsed = await parseSerial(adjusted);
269
- const _parseMs = Date.now() - _parseT0;
270
- if (_parseMs > 50) {
271
- console.log(` [parse] simpleParser ${_parseMs}ms for ${(raw.length / 1024).toFixed(0)} KB (${storedPath})`);
274
+ // Coalesce: if another getMessage is already reading+parsing this
275
+ // exact body, await its parse instead of starting a duplicate.
276
+ let parsed;
277
+ const existingParse = this.parseInflight.get(cacheKey);
278
+ if (existingParse) {
279
+ parsed = await existingParse;
280
+ }
281
+ else {
282
+ const _parseT0 = Date.now();
283
+ const fresh = (async () => {
284
+ const raw = await this.bodyStore.readByPath(storedPath);
285
+ const adjusted = sniffAndFixCharset(raw);
286
+ const result = await parseSerial(adjusted);
287
+ const _parseMs = Date.now() - _parseT0;
288
+ if (_parseMs > 50) {
289
+ console.log(` [parse] simpleParser ${_parseMs}ms for ${(raw.length / 1024).toFixed(0)} KB (${storedPath})`);
290
+ }
291
+ return result;
292
+ })();
293
+ this.parseInflight.set(cacheKey, fresh);
294
+ try {
295
+ parsed = await fresh;
296
+ }
297
+ finally {
298
+ this.parseInflight.delete(cacheKey);
299
+ }
272
300
  }
273
301
  let bodyHtml = parsed.html || "";
274
302
  const bodyText = parsed.text || "";