@bobfrankston/mailx-store 0.1.27 → 0.1.29
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/db.d.ts +20 -3
- package/db.js +212 -110
- package/package.json +5 -5
- package/parse-serial.js +7 -1
- package/parse-worker.js +10 -0
- package/store.d.ts +1 -0
- package/store.js +35 -7
package/db.d.ts
CHANGED
|
@@ -347,6 +347,14 @@ export declare class MailxDB {
|
|
|
347
347
|
beginTransaction(): void;
|
|
348
348
|
commitTransaction(): void;
|
|
349
349
|
rollbackTransaction(): void;
|
|
350
|
+
/** Run `fn` inside a transaction — nesting-safe. `node:sqlite` throws on
|
|
351
|
+
* a nested BEGIN, and the async chunked walkers (seedContactsFromMessages,
|
|
352
|
+
* applyContactsConfig) can interleave with the sync backfill's own
|
|
353
|
+
* transactions across their `setImmediate` yields. If a transaction is
|
|
354
|
+
* already open this just runs `fn` (its writes join the open txn);
|
|
355
|
+
* otherwise it owns a fresh BEGIN/COMMIT. `fn` MUST be synchronous so it
|
|
356
|
+
* can't span an await and leave a transaction open across a yield. */
|
|
357
|
+
runInTxn<T>(fn: () => T): T;
|
|
350
358
|
/** Record an address used in sent mail */
|
|
351
359
|
recordSentAddress(name: string, email: string): void;
|
|
352
360
|
/** True if `email` (lowercased) appears in the active denylist. Cached
|
|
@@ -399,7 +407,16 @@ export declare class MailxDB {
|
|
|
399
407
|
* Discovered is a single tier; sub-distinctions like sent-vs-received
|
|
400
408
|
* collapse here because the user-facing UI shows them as one "discovered"
|
|
401
409
|
* source. Recency-weighted use_count differentiates within the tier. */
|
|
402
|
-
|
|
410
|
+
/** ASYNC + chunked. This walks every cached message's address fields —
|
|
411
|
+
* on a large account that is hundreds of thousands of rows. The old
|
|
412
|
+
* synchronous version did `.all()` on the whole messages table in one
|
|
413
|
+
* call, materialising every row and blocking the event loop for tens
|
|
414
|
+
* of seconds to minutes (profiled 2026-05-15: 99% of daemon ticks were
|
|
415
|
+
* in native SQLite while this ran — every getMessage IPC, every preview
|
|
416
|
+
* click, queued behind it; that IS the "loading body takes forever"
|
|
417
|
+
* bug). Now keyset-paginated by `m.id` with a `setImmediate` yield
|
|
418
|
+
* between pages, so user IPC lands in the gaps. */
|
|
419
|
+
seedContactsFromMessages(): Promise<number>;
|
|
403
420
|
/** Apply the contents of contacts.jsonc — replaces all preferred-tier rows
|
|
404
421
|
* with the entries in `preferred[]`, merges `discovered[]` into the local
|
|
405
422
|
* cache, sets the in-memory denylist, and purges any discovered rows
|
|
@@ -428,12 +445,12 @@ export declare class MailxDB {
|
|
|
428
445
|
useCount?: number;
|
|
429
446
|
lastUsed?: number;
|
|
430
447
|
}[];
|
|
431
|
-
}): {
|
|
448
|
+
}): Promise<{
|
|
432
449
|
preferred: number;
|
|
433
450
|
discovered: number;
|
|
434
451
|
purged: number;
|
|
435
452
|
conflicts: string[];
|
|
436
|
-
}
|
|
453
|
+
}>;
|
|
437
454
|
/** Build the contacts.jsonc shape from current DB state — for round-trip
|
|
438
455
|
* to GDrive. Preferred-tier rows come from anything not in the reserved
|
|
439
456
|
* system sources; discovered comes from `source='discovered'` rows;
|
package/db.js
CHANGED
|
@@ -1355,13 +1355,18 @@ export class MailxDB {
|
|
|
1355
1355
|
// quoted-strings (RFC-strictly forbidden, common in the wild). Run
|
|
1356
1356
|
// libmime over Subject and every address display name once; safe
|
|
1357
1357
|
// to call on already-decoded text (no `=?...?=` markers → no-op).
|
|
1358
|
+
// The `address` is decoded too: a legit address never carries an
|
|
1359
|
+
// encoded-word, so it's a no-op there — but spam sometimes packs the
|
|
1360
|
+
// whole `name <addr>` into the mailbox local-part as one encoded
|
|
1361
|
+
// word (`=?utf-8?q?...=3C...=40...=3E?=@host`). Decoding it at least
|
|
1362
|
+
// renders readable text instead of raw `=3C`/`=C3=A9` gibberish.
|
|
1358
1363
|
msg.subject = decodeHeaderWords(msg.subject);
|
|
1359
1364
|
if (msg.from)
|
|
1360
|
-
msg.from = { name: decodeHeaderWords(msg.from.name || ""), address: msg.from.address || "" };
|
|
1365
|
+
msg.from = { name: decodeHeaderWords(msg.from.name || ""), address: decodeHeaderWords(msg.from.address || "") };
|
|
1361
1366
|
if (msg.to)
|
|
1362
|
-
msg.to = msg.to.map(a => ({ name: decodeHeaderWords(a.name || ""), address: a.address || "" }));
|
|
1367
|
+
msg.to = msg.to.map(a => ({ name: decodeHeaderWords(a.name || ""), address: decodeHeaderWords(a.address || "") }));
|
|
1363
1368
|
if (msg.cc)
|
|
1364
|
-
msg.cc = msg.cc.map(a => ({ name: decodeHeaderWords(a.name || ""), address: a.address || "" }));
|
|
1369
|
+
msg.cc = msg.cc.map(a => ({ name: decodeHeaderWords(a.name || ""), address: decodeHeaderWords(a.address || "") }));
|
|
1365
1370
|
const existing = this.db.prepare("SELECT id, provider_id FROM messages WHERE account_id = ? AND folder_id = ? AND uid = ?").get(msg.accountId, msg.folderId, msg.uid);
|
|
1366
1371
|
if (existing) {
|
|
1367
1372
|
// Backfill provider_id on existing rows that predate this column —
|
|
@@ -1901,6 +1906,30 @@ export class MailxDB {
|
|
|
1901
1906
|
beginTransaction() { this.db.exec("BEGIN"); }
|
|
1902
1907
|
commitTransaction() { this.db.exec("COMMIT"); }
|
|
1903
1908
|
rollbackTransaction() { this.db.exec("ROLLBACK"); }
|
|
1909
|
+
/** Run `fn` inside a transaction — nesting-safe. `node:sqlite` throws on
|
|
1910
|
+
* a nested BEGIN, and the async chunked walkers (seedContactsFromMessages,
|
|
1911
|
+
* applyContactsConfig) can interleave with the sync backfill's own
|
|
1912
|
+
* transactions across their `setImmediate` yields. If a transaction is
|
|
1913
|
+
* already open this just runs `fn` (its writes join the open txn);
|
|
1914
|
+
* otherwise it owns a fresh BEGIN/COMMIT. `fn` MUST be synchronous so it
|
|
1915
|
+
* can't span an await and leave a transaction open across a yield. */
|
|
1916
|
+
runInTxn(fn) {
|
|
1917
|
+
if (this.db.isTransaction)
|
|
1918
|
+
return fn();
|
|
1919
|
+
this.db.exec("BEGIN");
|
|
1920
|
+
try {
|
|
1921
|
+
const r = fn();
|
|
1922
|
+
this.db.exec("COMMIT");
|
|
1923
|
+
return r;
|
|
1924
|
+
}
|
|
1925
|
+
catch (e) {
|
|
1926
|
+
try {
|
|
1927
|
+
this.db.exec("ROLLBACK");
|
|
1928
|
+
}
|
|
1929
|
+
catch { /* already rolled back */ }
|
|
1930
|
+
throw e;
|
|
1931
|
+
}
|
|
1932
|
+
}
|
|
1904
1933
|
// ── Contacts ──
|
|
1905
1934
|
/** Record an address used in sent mail */
|
|
1906
1935
|
recordSentAddress(name, email) {
|
|
@@ -1992,7 +2021,16 @@ export class MailxDB {
|
|
|
1992
2021
|
* Discovered is a single tier; sub-distinctions like sent-vs-received
|
|
1993
2022
|
* collapse here because the user-facing UI shows them as one "discovered"
|
|
1994
2023
|
* source. Recency-weighted use_count differentiates within the tier. */
|
|
1995
|
-
|
|
2024
|
+
/** ASYNC + chunked. This walks every cached message's address fields —
|
|
2025
|
+
* on a large account that is hundreds of thousands of rows. The old
|
|
2026
|
+
* synchronous version did `.all()` on the whole messages table in one
|
|
2027
|
+
* call, materialising every row and blocking the event loop for tens
|
|
2028
|
+
* of seconds to minutes (profiled 2026-05-15: 99% of daemon ticks were
|
|
2029
|
+
* in native SQLite while this ran — every getMessage IPC, every preview
|
|
2030
|
+
* click, queued behind it; that IS the "loading body takes forever"
|
|
2031
|
+
* bug). Now keyset-paginated by `m.id` with a `setImmediate` yield
|
|
2032
|
+
* between pages, so user IPC lands in the gaps. */
|
|
2033
|
+
async seedContactsFromMessages() {
|
|
1996
2034
|
const VALID = /^[^\s<>@]+@[^\s<>@]+\.[^\s<>@]+$/;
|
|
1997
2035
|
const now = Date.now();
|
|
1998
2036
|
const agg = new Map();
|
|
@@ -2016,57 +2054,75 @@ export class MailxDB {
|
|
|
2016
2054
|
agg.set(email, { name: name || "", cnt: 1, last: date || 0 });
|
|
2017
2055
|
}
|
|
2018
2056
|
};
|
|
2019
|
-
|
|
2020
|
-
|
|
2021
|
-
|
|
2022
|
-
|
|
2023
|
-
|
|
2024
|
-
|
|
2025
|
-
|
|
2026
|
-
|
|
2027
|
-
|
|
2028
|
-
|
|
2029
|
-
|
|
2030
|
-
|
|
2031
|
-
|
|
2032
|
-
|
|
2033
|
-
catch {
|
|
2034
|
-
continue;
|
|
2035
|
-
}
|
|
2036
|
-
if (!Array.isArray(parsed))
|
|
2057
|
+
const eatRecipients = (field, date) => {
|
|
2058
|
+
if (!field)
|
|
2059
|
+
return;
|
|
2060
|
+
let parsed;
|
|
2061
|
+
try {
|
|
2062
|
+
parsed = JSON.parse(field);
|
|
2063
|
+
}
|
|
2064
|
+
catch {
|
|
2065
|
+
return;
|
|
2066
|
+
}
|
|
2067
|
+
if (!Array.isArray(parsed))
|
|
2068
|
+
return;
|
|
2069
|
+
for (const a of parsed) {
|
|
2070
|
+
if (!a)
|
|
2037
2071
|
continue;
|
|
2038
|
-
|
|
2039
|
-
|
|
2040
|
-
|
|
2041
|
-
|
|
2072
|
+
bump(a.name || "", a.address || a.email || "", date);
|
|
2073
|
+
}
|
|
2074
|
+
};
|
|
2075
|
+
const yieldLoop = () => new Promise(r => setImmediate(r));
|
|
2076
|
+
const PAGE = 2000;
|
|
2077
|
+
// Sent folder: recipients only (skip the user's own From address).
|
|
2078
|
+
// Keyset pagination by m.id — each page is a bounded `.all()`, and
|
|
2079
|
+
// the event loop is handed back between pages.
|
|
2080
|
+
{
|
|
2081
|
+
const stmt = this.db.prepare(`SELECT m.id AS id, m.to_json, m.cc_json, m.bcc_json, m.date
|
|
2082
|
+
FROM messages m
|
|
2083
|
+
JOIN folders f ON m.folder_id = f.id
|
|
2084
|
+
WHERE f.special_use = 'sent' AND m.id > ?
|
|
2085
|
+
ORDER BY m.id LIMIT ?`);
|
|
2086
|
+
let lastId = 0;
|
|
2087
|
+
for (;;) {
|
|
2088
|
+
const rows = stmt.all(lastId, PAGE);
|
|
2089
|
+
if (rows.length === 0)
|
|
2090
|
+
break;
|
|
2091
|
+
for (const r of rows) {
|
|
2092
|
+
const date = r.date || 0;
|
|
2093
|
+
eatRecipients(r.to_json, date);
|
|
2094
|
+
eatRecipients(r.cc_json, date);
|
|
2095
|
+
eatRecipients(r.bcc_json, date);
|
|
2042
2096
|
}
|
|
2097
|
+
lastId = rows[rows.length - 1].id;
|
|
2098
|
+
if (rows.length < PAGE)
|
|
2099
|
+
break;
|
|
2100
|
+
await yieldLoop();
|
|
2043
2101
|
}
|
|
2044
2102
|
}
|
|
2045
2103
|
// Other folders: From + recipients.
|
|
2046
|
-
|
|
2047
|
-
|
|
2048
|
-
|
|
2049
|
-
|
|
2050
|
-
|
|
2051
|
-
|
|
2052
|
-
|
|
2053
|
-
for (
|
|
2054
|
-
|
|
2055
|
-
|
|
2056
|
-
|
|
2057
|
-
|
|
2058
|
-
|
|
2059
|
-
|
|
2060
|
-
|
|
2061
|
-
|
|
2062
|
-
|
|
2063
|
-
if (!Array.isArray(parsed))
|
|
2064
|
-
continue;
|
|
2065
|
-
for (const a of parsed) {
|
|
2066
|
-
if (!a)
|
|
2067
|
-
continue;
|
|
2068
|
-
bump(a.name || "", a.address || a.email || "", date);
|
|
2104
|
+
{
|
|
2105
|
+
const stmt = this.db.prepare(`SELECT m.id AS id, m.from_name, m.from_address, m.to_json, m.cc_json, m.bcc_json, m.date
|
|
2106
|
+
FROM messages m
|
|
2107
|
+
LEFT JOIN folders f ON m.folder_id = f.id
|
|
2108
|
+
WHERE (f.special_use IS NULL OR f.special_use != 'sent') AND m.id > ?
|
|
2109
|
+
ORDER BY m.id LIMIT ?`);
|
|
2110
|
+
let lastId = 0;
|
|
2111
|
+
for (;;) {
|
|
2112
|
+
const rows = stmt.all(lastId, PAGE);
|
|
2113
|
+
if (rows.length === 0)
|
|
2114
|
+
break;
|
|
2115
|
+
for (const r of rows) {
|
|
2116
|
+
const date = r.date || 0;
|
|
2117
|
+
bump(r.from_name, r.from_address, date);
|
|
2118
|
+
eatRecipients(r.to_json, date);
|
|
2119
|
+
eatRecipients(r.cc_json, date);
|
|
2120
|
+
eatRecipients(r.bcc_json, date);
|
|
2069
2121
|
}
|
|
2122
|
+
lastId = rows[rows.length - 1].id;
|
|
2123
|
+
if (rows.length < PAGE)
|
|
2124
|
+
break;
|
|
2125
|
+
await yieldLoop();
|
|
2070
2126
|
}
|
|
2071
2127
|
}
|
|
2072
2128
|
let added = 0;
|
|
@@ -2077,16 +2133,30 @@ export class MailxDB {
|
|
|
2077
2133
|
name = CASE WHEN name = '' AND ? != '' THEN ? ELSE name END,
|
|
2078
2134
|
updated_at = ?
|
|
2079
2135
|
WHERE id = ?`);
|
|
2080
|
-
|
|
2081
|
-
|
|
2082
|
-
|
|
2083
|
-
|
|
2084
|
-
|
|
2085
|
-
|
|
2086
|
-
|
|
2087
|
-
|
|
2088
|
-
|
|
2089
|
-
|
|
2136
|
+
const findStmt = this.db.prepare("SELECT id FROM contacts WHERE source = 'discovered' AND lower(email) = ?");
|
|
2137
|
+
// Write phase — chunked. Each chunk runs via `db.transaction()`
|
|
2138
|
+
// (one synchronous turn, one commit) rather than raw BEGIN/COMMIT
|
|
2139
|
+
// straddling an `await` — the wrapper is nesting-safe (savepoints)
|
|
2140
|
+
// so it can't collide with the sync backfill's own transactions
|
|
2141
|
+
// when the two interleave across the yield below.
|
|
2142
|
+
const WRITE_CHUNK = 500;
|
|
2143
|
+
const entries = [...agg.entries()];
|
|
2144
|
+
for (let i = 0; i < entries.length; i += WRITE_CHUNK) {
|
|
2145
|
+
const slice = entries.slice(i, i + WRITE_CHUNK);
|
|
2146
|
+
this.runInTxn(() => {
|
|
2147
|
+
for (const [email, info] of slice) {
|
|
2148
|
+
const existing = findStmt.get(email);
|
|
2149
|
+
if (!existing) {
|
|
2150
|
+
insStmt.run(info.name, email, info.last, info.cnt, now);
|
|
2151
|
+
added++;
|
|
2152
|
+
}
|
|
2153
|
+
else {
|
|
2154
|
+
updStmt.run(info.cnt, info.last, info.name, info.name, now, existing.id);
|
|
2155
|
+
bumped++;
|
|
2156
|
+
}
|
|
2157
|
+
}
|
|
2158
|
+
});
|
|
2159
|
+
await yieldLoop();
|
|
2090
2160
|
}
|
|
2091
2161
|
if (added > 0 || bumped > 0) {
|
|
2092
2162
|
console.log(` [contacts] seed: ${added} new + ${bumped} refreshed (discovered)`);
|
|
@@ -2104,7 +2174,7 @@ export class MailxDB {
|
|
|
2104
2174
|
* Discovered rows from the file are MERGED with whatever the local
|
|
2105
2175
|
* message-corpus seeder has produced. Each device contributes its
|
|
2106
2176
|
* observed addresses; over time GDrive accumulates the union. */
|
|
2107
|
-
applyContactsConfig(cfg) {
|
|
2177
|
+
async applyContactsConfig(cfg) {
|
|
2108
2178
|
const preferred = Array.isArray(cfg.preferred) ? cfg.preferred : [];
|
|
2109
2179
|
const denylist = Array.isArray(cfg.denylist) ? cfg.denylist : [];
|
|
2110
2180
|
const denylistPatterns = Array.isArray(cfg.denylistPatterns) ? cfg.denylistPatterns : [];
|
|
@@ -2119,41 +2189,65 @@ export class MailxDB {
|
|
|
2119
2189
|
.filter(e => e && e.priority === true && e.email)
|
|
2120
2190
|
.map(e => e.email);
|
|
2121
2191
|
this.setPriorityIndex(prioritySenders, priorityDomains);
|
|
2192
|
+
const VALID = /^[^\s<>@]+@[^\s<>@]+\.[^\s<>@]+$/;
|
|
2193
|
+
const now = Date.now();
|
|
2194
|
+
const denySet = new Set(denylist.map(e => (e || "").trim().toLowerCase()).filter(Boolean));
|
|
2195
|
+
const conflicts = [];
|
|
2196
|
+
const yieldLoop = () => new Promise(r => setImmediate(r));
|
|
2197
|
+
const CHUNK = 500;
|
|
2122
2198
|
// Wipe and rewrite preferred-tier rows owned by contacts.jsonc.
|
|
2123
2199
|
// The address-book UI's legacy `upsertContact` still writes
|
|
2124
2200
|
// source='manual' rows; those are owned by the address-book code
|
|
2125
2201
|
// path, not contacts.jsonc, so we leave them alone here.
|
|
2126
|
-
|
|
2127
|
-
|
|
2128
|
-
|
|
2202
|
+
// ALL writes below run inside transactions and yield the event
|
|
2203
|
+
// loop between chunks. The prior version did one autocommitting
|
|
2204
|
+
// INSERT/UPDATE per row with NO transaction — on a contacts.jsonc
|
|
2205
|
+
// that has accumulated thousands of `discovered` entries across
|
|
2206
|
+
// devices, that was thousands of fsyncs back-to-back, blocking the
|
|
2207
|
+
// daemon for 25-30 s (profiled 2026-05-15: applyContactsConfig was
|
|
2208
|
+
// ~97% of the wedge). It also re-`prepare()`d a `lower(email)`
|
|
2209
|
+
// full-scan SELECT every iteration. Now: one statement prep, one
|
|
2210
|
+
// up-front map of existing discovered rows (kills the N×N scan),
|
|
2211
|
+
// chunked transactions, yields between chunks.
|
|
2129
2212
|
const ins = this.db.prepare(`INSERT OR IGNORE INTO contacts (source, name, email, organization, last_used, use_count, updated_at)
|
|
2130
2213
|
VALUES (?, ?, ?, ?, 0, 0, ?)`);
|
|
2131
|
-
const denySet = new Set(denylist.map(e => (e || "").trim().toLowerCase()).filter(Boolean));
|
|
2132
|
-
const conflicts = [];
|
|
2133
2214
|
let inserted = 0;
|
|
2134
|
-
|
|
2135
|
-
|
|
2136
|
-
|
|
2137
|
-
|
|
2138
|
-
|
|
2139
|
-
|
|
2140
|
-
|
|
2141
|
-
|
|
2142
|
-
|
|
2143
|
-
|
|
2144
|
-
|
|
2145
|
-
|
|
2146
|
-
|
|
2147
|
-
|
|
2148
|
-
|
|
2149
|
-
|
|
2150
|
-
|
|
2215
|
+
// runInTxn: nesting-safe and atomic within one synchronous turn, so
|
|
2216
|
+
// these writes can't collide with the sync backfill's transactions
|
|
2217
|
+
// when the two interleave across the yields below.
|
|
2218
|
+
this.runInTxn(() => {
|
|
2219
|
+
this.db.exec("DELETE FROM contacts WHERE source NOT IN ('google', 'discovered', 'manual')");
|
|
2220
|
+
for (const entry of preferred) {
|
|
2221
|
+
if (!entry)
|
|
2222
|
+
continue;
|
|
2223
|
+
const email = (entry.email || "").trim();
|
|
2224
|
+
if (!email || !VALID.test(email))
|
|
2225
|
+
continue;
|
|
2226
|
+
if (denySet.has(email.toLowerCase())) {
|
|
2227
|
+
conflicts.push(email);
|
|
2228
|
+
continue;
|
|
2229
|
+
}
|
|
2230
|
+
const source = (entry.source || "preferred").trim() || "preferred";
|
|
2231
|
+
const name = (entry.name || "").trim();
|
|
2232
|
+
const org = (entry.organization || entry.org || "").trim();
|
|
2233
|
+
try {
|
|
2234
|
+
const r = ins.run(source, name, email, org, now);
|
|
2235
|
+
if (r.changes)
|
|
2236
|
+
inserted++;
|
|
2237
|
+
}
|
|
2238
|
+
catch { /* dup row, skip */ }
|
|
2151
2239
|
}
|
|
2152
|
-
|
|
2153
|
-
|
|
2240
|
+
});
|
|
2241
|
+
await yieldLoop();
|
|
2154
2242
|
// Merge discovered[] from cloud into local cache. For each entry:
|
|
2155
2243
|
// existing row wins on use_count (max), name fills if empty, lastUsed
|
|
2156
2244
|
// is max. Missing rows are inserted. Denylisted entries skipped.
|
|
2245
|
+
// Existing discovered rows are mapped ONCE up front so the merge is
|
|
2246
|
+
// O(N) hash lookups, not O(N) `lower(email)` table scans.
|
|
2247
|
+
const existingDiscovered = new Map();
|
|
2248
|
+
for (const row of this.db.prepare("SELECT id, lower(email) AS le FROM contacts WHERE source = 'discovered'").all()) {
|
|
2249
|
+
existingDiscovered.set(row.le, row.id);
|
|
2250
|
+
}
|
|
2157
2251
|
const insDiscovered = this.db.prepare("INSERT INTO contacts (source, name, email, last_used, use_count, updated_at) VALUES ('discovered', ?, ?, ?, ?, ?)");
|
|
2158
2252
|
const updDiscovered = this.db.prepare(`UPDATE contacts SET use_count = max(use_count, ?),
|
|
2159
2253
|
last_used = max(last_used, ?),
|
|
@@ -2161,36 +2255,44 @@ export class MailxDB {
|
|
|
2161
2255
|
updated_at = ?
|
|
2162
2256
|
WHERE id = ?`);
|
|
2163
2257
|
let discoveredAdded = 0;
|
|
2164
|
-
for (
|
|
2165
|
-
|
|
2166
|
-
|
|
2167
|
-
|
|
2168
|
-
|
|
2169
|
-
|
|
2170
|
-
|
|
2171
|
-
|
|
2172
|
-
|
|
2173
|
-
|
|
2174
|
-
|
|
2175
|
-
|
|
2176
|
-
|
|
2177
|
-
|
|
2178
|
-
|
|
2179
|
-
|
|
2180
|
-
|
|
2181
|
-
|
|
2182
|
-
|
|
2183
|
-
|
|
2184
|
-
|
|
2185
|
-
|
|
2258
|
+
for (let i = 0; i < discovered.length; i += CHUNK) {
|
|
2259
|
+
const slice = discovered.slice(i, i + CHUNK);
|
|
2260
|
+
this.runInTxn(() => {
|
|
2261
|
+
for (const entry of slice) {
|
|
2262
|
+
if (!entry)
|
|
2263
|
+
continue;
|
|
2264
|
+
const email = (entry.email || "").trim();
|
|
2265
|
+
if (!email || !VALID.test(email))
|
|
2266
|
+
continue;
|
|
2267
|
+
const lower = email.toLowerCase();
|
|
2268
|
+
if (denySet.has(lower))
|
|
2269
|
+
continue;
|
|
2270
|
+
if (isJunkContact(lower, entry.name || ""))
|
|
2271
|
+
continue;
|
|
2272
|
+
const name = (entry.name || "").trim();
|
|
2273
|
+
const useCount = Math.max(0, entry.useCount || 0);
|
|
2274
|
+
const lastUsed = Math.max(0, entry.lastUsed || 0);
|
|
2275
|
+
const existingId = existingDiscovered.get(lower);
|
|
2276
|
+
if (existingId === undefined) {
|
|
2277
|
+
insDiscovered.run(name, email, lastUsed, useCount, now);
|
|
2278
|
+
discoveredAdded++;
|
|
2279
|
+
}
|
|
2280
|
+
else {
|
|
2281
|
+
updDiscovered.run(useCount, lastUsed, name, name, now, existingId);
|
|
2282
|
+
}
|
|
2283
|
+
}
|
|
2284
|
+
});
|
|
2285
|
+
await yieldLoop();
|
|
2186
2286
|
}
|
|
2187
2287
|
// Purge discovered rows for any denylisted email.
|
|
2188
2288
|
const purge = this.db.prepare("DELETE FROM contacts WHERE source = 'discovered' AND lower(email) = ?");
|
|
2189
2289
|
let purged = 0;
|
|
2190
|
-
|
|
2191
|
-
const
|
|
2192
|
-
|
|
2193
|
-
|
|
2290
|
+
this.runInTxn(() => {
|
|
2291
|
+
for (const e of denySet) {
|
|
2292
|
+
const r = purge.run(e);
|
|
2293
|
+
purged += Number(r.changes || 0);
|
|
2294
|
+
}
|
|
2295
|
+
});
|
|
2194
2296
|
if (conflicts.length > 0) {
|
|
2195
2297
|
console.warn(` [contacts] config: ${conflicts.length} preferred entries also appear in denylist — denylist wins, entries skipped: ${conflicts.join(", ")}`);
|
|
2196
2298
|
}
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@bobfrankston/mailx-store",
|
|
3
|
-
"version": "0.1.
|
|
3
|
+
"version": "0.1.29",
|
|
4
4
|
"type": "module",
|
|
5
5
|
"main": "index.js",
|
|
6
6
|
"types": "index.d.ts",
|
|
@@ -9,8 +9,8 @@
|
|
|
9
9
|
},
|
|
10
10
|
"license": "ISC",
|
|
11
11
|
"dependencies": {
|
|
12
|
-
"@bobfrankston/mailx-types": "^0.1.
|
|
13
|
-
"@bobfrankston/mailx-settings": "^0.1.
|
|
12
|
+
"@bobfrankston/mailx-types": "^0.1.15",
|
|
13
|
+
"@bobfrankston/mailx-settings": "^0.1.18",
|
|
14
14
|
"@bobfrankston/mailx-bus": "^0.1.2",
|
|
15
15
|
"mailparser": "^3.7.2"
|
|
16
16
|
},
|
|
@@ -29,8 +29,8 @@
|
|
|
29
29
|
},
|
|
30
30
|
".transformedSnapshot": {
|
|
31
31
|
"dependencies": {
|
|
32
|
-
"@bobfrankston/mailx-types": "^0.1.
|
|
33
|
-
"@bobfrankston/mailx-settings": "^0.1.
|
|
32
|
+
"@bobfrankston/mailx-types": "^0.1.15",
|
|
33
|
+
"@bobfrankston/mailx-settings": "^0.1.18",
|
|
34
34
|
"@bobfrankston/mailx-bus": "^0.1.2",
|
|
35
35
|
"mailparser": "^3.7.2"
|
|
36
36
|
}
|
package/parse-serial.js
CHANGED
|
@@ -73,7 +73,10 @@ function getWorker() {
|
|
|
73
73
|
});
|
|
74
74
|
w.on("error", (e) => {
|
|
75
75
|
// Fatal worker crash — fail every in-flight parse, then null the
|
|
76
|
-
// singleton so the next call respawns.
|
|
76
|
+
// singleton so the next call respawns. Logged loudly: a silent
|
|
77
|
+
// respawn loop forces every Nth parse to eat a fresh cold-start,
|
|
78
|
+
// which reads as a random multi-second "loading body" stall.
|
|
79
|
+
console.error(` [parse-worker] CRASHED (${e.message}) — ${_inflight.size} in-flight parse(s) failed, will respawn`);
|
|
77
80
|
for (const entry of _inflight.values())
|
|
78
81
|
entry.reject(e);
|
|
79
82
|
_inflight.clear();
|
|
@@ -81,6 +84,9 @@ function getWorker() {
|
|
|
81
84
|
signalCompletion();
|
|
82
85
|
});
|
|
83
86
|
w.on("exit", (code) => {
|
|
87
|
+
// code 0 = clean exit (shouldn't happen — parentPort keeps it alive);
|
|
88
|
+
// non-zero = crash. Either way the next parse respawns + re-warms.
|
|
89
|
+
console.warn(` [parse-worker] exited (code=${code}), ${_inflight.size} in-flight — next parse respawns`);
|
|
84
90
|
if (_inflight.size > 0) {
|
|
85
91
|
const err = new Error(`parse-worker exited (code=${code}) with ${_inflight.size} in-flight`);
|
|
86
92
|
for (const entry of _inflight.values())
|
package/parse-worker.js
CHANGED
|
@@ -83,7 +83,17 @@ parentPort.on("message", async (msg) => {
|
|
|
83
83
|
const u = source;
|
|
84
84
|
normalized = Buffer.from(u.buffer, u.byteOffset, u.byteLength);
|
|
85
85
|
}
|
|
86
|
+
const _t0 = Date.now();
|
|
86
87
|
const result = await simpleParser(normalized);
|
|
88
|
+
const _ms = Date.now() - _t0;
|
|
89
|
+
// Worker-side parse time. If THIS is slow, the bug is in simpleParser
|
|
90
|
+
// on a given input; if this is fast but the main thread still saw a
|
|
91
|
+
// multi-second `[parse]` figure, the cost is queue wait / respawn —
|
|
92
|
+
// splitting the two was the open question of the 2026-05-15 hunt.
|
|
93
|
+
if (_ms > 1000) {
|
|
94
|
+
const size = Buffer.isBuffer(normalized) ? normalized.byteLength : String(normalized).length;
|
|
95
|
+
console.warn(` [parse-worker] simpleParser took ${_ms}ms for ${(size / 1024).toFixed(0)} KB (worker-side, not queue wait)`);
|
|
96
|
+
}
|
|
87
97
|
parentPort.postMessage({ id, ok: true, result });
|
|
88
98
|
}
|
|
89
99
|
catch (e) {
|
package/store.d.ts
CHANGED
package/store.js
CHANGED
|
@@ -95,6 +95,15 @@ export class Store {
|
|
|
95
95
|
this.parsedLru.delete(oldest);
|
|
96
96
|
}
|
|
97
97
|
}
|
|
98
|
+
// In-flight parse coalescing. Concurrent getMessage calls for the same
|
|
99
|
+
// body (same path|mtime cacheKey) must share ONE parse — otherwise a
|
|
100
|
+
// viewer that re-requests while the first parse is still queued enqueues
|
|
101
|
+
// a duplicate, and N retries become N parses stacked in the parse-worker
|
|
102
|
+
// queue. Observed 2026-05-15: the same .eml parsed 3-4× in a burst, and
|
|
103
|
+
// a genuine click then waited 25 s behind the pile-up. The map entry is
|
|
104
|
+
// removed when the parse settles (success OR failure — a failed parse
|
|
105
|
+
// must not poison the key).
|
|
106
|
+
parseInflight = new Map();
|
|
98
107
|
// Allowlist + settings caches. Both files live on the GDrive-mounted
|
|
99
108
|
// shared dir; their sync `readFileSync` calls in `loadAllowlist()` and
|
|
100
109
|
// `loadSettings()` can stall for seconds per call. getMessage runs on
|
|
@@ -262,13 +271,32 @@ export class Store {
|
|
|
262
271
|
// queue-contention parses. Synchronous parse makes the whole
|
|
263
272
|
// class structurally impossible: getMessage either returns the
|
|
264
273
|
// body or returns `cached:false` meaning exactly "fetch it."
|
|
265
|
-
|
|
266
|
-
|
|
267
|
-
|
|
268
|
-
const
|
|
269
|
-
|
|
270
|
-
|
|
271
|
-
|
|
274
|
+
// Coalesce: if another getMessage is already reading+parsing this
|
|
275
|
+
// exact body, await its parse instead of starting a duplicate.
|
|
276
|
+
let parsed;
|
|
277
|
+
const existingParse = this.parseInflight.get(cacheKey);
|
|
278
|
+
if (existingParse) {
|
|
279
|
+
parsed = await existingParse;
|
|
280
|
+
}
|
|
281
|
+
else {
|
|
282
|
+
const _parseT0 = Date.now();
|
|
283
|
+
const fresh = (async () => {
|
|
284
|
+
const raw = await this.bodyStore.readByPath(storedPath);
|
|
285
|
+
const adjusted = sniffAndFixCharset(raw);
|
|
286
|
+
const result = await parseSerial(adjusted);
|
|
287
|
+
const _parseMs = Date.now() - _parseT0;
|
|
288
|
+
if (_parseMs > 50) {
|
|
289
|
+
console.log(` [parse] simpleParser ${_parseMs}ms for ${(raw.length / 1024).toFixed(0)} KB (${storedPath})`);
|
|
290
|
+
}
|
|
291
|
+
return result;
|
|
292
|
+
})();
|
|
293
|
+
this.parseInflight.set(cacheKey, fresh);
|
|
294
|
+
try {
|
|
295
|
+
parsed = await fresh;
|
|
296
|
+
}
|
|
297
|
+
finally {
|
|
298
|
+
this.parseInflight.delete(cacheKey);
|
|
299
|
+
}
|
|
272
300
|
}
|
|
273
301
|
let bodyHtml = parsed.html || "";
|
|
274
302
|
const bodyText = parsed.text || "";
|