@bobfrankston/rmfmail 1.0.555 → 1.0.558
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/_dev-pop.ts +328 -0
- package/_dev-populate.ts +222 -0
- package/client/components/message-list.js +2 -2
- package/client/components/message-list.js.map +1 -1
- package/client/components/message-list.ts +2 -2
- package/package.json +3 -3
- package/packages/mailx-imap/index.d.ts +4 -1
- package/packages/mailx-imap/index.d.ts.map +1 -1
- package/packages/mailx-imap/index.js +49 -70
- package/packages/mailx-imap/index.js.map +1 -1
- package/packages/mailx-imap/index.ts +51 -56
- package/packages/mailx-imap/package-lock.json +2 -2
- package/packages/mailx-imap/package.json +1 -1
- package/packages/mailx-store/db.d.ts +17 -2
- package/packages/mailx-store/db.d.ts.map +1 -1
- package/packages/mailx-store/db.js +73 -9
- package/packages/mailx-store/db.js.map +1 -1
- package/packages/mailx-store/db.ts +94 -9
- package/packages/mailx-store/package.json +1 -1
package/_dev-pop.ts
ADDED
|
@@ -0,0 +1,328 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Standalone folder populator.
|
|
3
|
+
*
|
|
4
|
+
* Bypasses mailx's broken sync loop and directly fetches the latest N
|
|
5
|
+
* messages from every folder of every IMAP account, writing rows into the
|
|
6
|
+
* mailx.db. Designed to run while rmfmail.exe is killed (overnight job).
|
|
7
|
+
*
|
|
8
|
+
* Why: mailx's sync has been silently failing on bobma — bobma/Sent has
|
|
9
|
+
* never had real server-fetched rows in the local DB. Rather than fight
|
|
10
|
+
* the sync loop's connection-state corruption, this tool uses a single
|
|
11
|
+
* fresh IMAP connection (which we proved completes SELECT Sent in 26ms)
|
|
12
|
+
* and pumps messages into the DB directly via upsertMessage.
|
|
13
|
+
*
|
|
14
|
+
* Run:
|
|
15
|
+
* 1. Kill rmfmail.exe
|
|
16
|
+
* 2. cd Y:/dev/email/mailx/app
|
|
17
|
+
* 3. npx tsx Y:/dev/email/mailx/dev/populate-folders.ts [--n 500] [--account bobma] [--folder Sent]
|
|
18
|
+
*
|
|
19
|
+
* Defaults: N=500 latest messages per folder, all IMAP accounts, all folders.
|
|
20
|
+
*
|
|
21
|
+
* Notes:
|
|
22
|
+
* - Bodies are NOT fetched here — only envelope rows. Body fetch is
|
|
23
|
+
* mailx's prefetch loop's job; this tool just unblocks the metadata.
|
|
24
|
+
* - One connection, sequential folders. No parallelism that could
|
|
25
|
+
* trigger the same connection-state issues mailx hits.
|
|
26
|
+
* - Progress logged per folder. Failures logged + skipped, never bail.
|
|
27
|
+
* - Idempotent: re-running re-upserts. UNIQUE(account, folder, uid)
|
|
28
|
+
* constraint means existing rows are updated, not duplicated.
|
|
29
|
+
*/
|
|
30
|
+
|
|
31
|
+
import * as fs from "node:fs";
|
|
32
|
+
import * as path from "node:path";
|
|
33
|
+
import * as os from "node:os";
|
|
34
|
+
import { NativeImapClient } from "@bobfrankston/iflow-direct";
|
|
35
|
+
import { NodeTcpTransport } from "@bobfrankston/node-tcp-transport";
|
|
36
|
+
import { MailxDB } from "@bobfrankston/mailx-store";
|
|
37
|
+
|
|
38
|
+
// ── Args ──
|
|
39
|
+
const args = process.argv.slice(2);
|
|
40
|
+
const argVal = (name: string, def?: string): string | undefined => {
|
|
41
|
+
const i = args.indexOf(`--${name}`);
|
|
42
|
+
if (i < 0 || i === args.length - 1) return def;
|
|
43
|
+
return args[i + 1];
|
|
44
|
+
};
|
|
45
|
+
const FETCH_N = Number(argVal("n", "500"));
|
|
46
|
+
const ONLY_ACCOUNT = argVal("account");
|
|
47
|
+
const ONLY_FOLDER = argVal("folder");
|
|
48
|
+
const CONNECT_TIMEOUT_MS = Number(argVal("timeout", "60000"));
|
|
49
|
+
|
|
50
|
+
// ── Config ──
|
|
51
|
+
const home = process.env.USERPROFILE || os.homedir();
|
|
52
|
+
const dbDir = path.join(home, ".rmfmail");
|
|
53
|
+
const accountsPath = path.join(dbDir, "accounts.jsonc");
|
|
54
|
+
if (!fs.existsSync(accountsPath)) {
|
|
55
|
+
console.error(`accounts.jsonc not found at ${accountsPath}`);
|
|
56
|
+
process.exit(1);
|
|
57
|
+
}
|
|
58
|
+
|
|
59
|
+
// JSONC: strip // and /* */ before parsing
|
|
60
|
+
function parseJsonc(raw: string): any {
|
|
61
|
+
const stripped = raw
|
|
62
|
+
.replace(/\/\*[\s\S]*?\*\//g, "")
|
|
63
|
+
.replace(/^\s*\/\/.*$/gm, "");
|
|
64
|
+
return JSON.parse(stripped);
|
|
65
|
+
}
|
|
66
|
+
|
|
67
|
+
const accountsCfg = parseJsonc(fs.readFileSync(accountsPath, "utf-8"));
|
|
68
|
+
const accounts = accountsCfg.accounts || [];
|
|
69
|
+
|
|
70
|
+
// ── Helpers ──
|
|
71
|
+
function ts(): string { return new Date().toISOString().replace("T", " ").slice(0, 19); }
|
|
72
|
+
// log() writes synchronously to stdout AND to the log file via direct fs.write
|
|
73
|
+
// — console.log when redirected to a file via shell `>` gets line-buffered,
|
|
74
|
+
// which delays heartbeats and timeout messages until a chunk fills. Direct
|
|
75
|
+
// fs.writeSync flushes per call.
|
|
76
|
+
const logPath = process.env.POPULATE_LOG_FILE || path.join(home, ".rmfmail", "logs", "populate-overnight.log");
|
|
77
|
+
let logFd: number | null = null;
|
|
78
|
+
try { logFd = fs.openSync(logPath, "a"); } catch { /* fall through to console only */ }
|
|
79
|
+
function log(msg: string): void {
|
|
80
|
+
const line = `[${ts()}] ${msg}\n`;
|
|
81
|
+
if (logFd !== null) {
|
|
82
|
+
try { fs.writeSync(logFd, line); } catch { /* */ }
|
|
83
|
+
}
|
|
84
|
+
process.stdout.write(line);
|
|
85
|
+
}
|
|
86
|
+
|
|
87
|
+
// Skip Gmail / API-backed accounts — they don't speak IMAP through this
|
|
88
|
+
// tool's path, and Gmail's sync route is separate (REST). Same for any
|
|
89
|
+
// account without an `imap` block.
|
|
90
|
+
function shouldUseImap(acct: any): boolean {
|
|
91
|
+
if (!acct?.imap?.host) return false;
|
|
92
|
+
const host = String(acct.imap.host).toLowerCase();
|
|
93
|
+
if (host.includes("gmail.com") || host.includes("googlemail.com")) return false;
|
|
94
|
+
return true;
|
|
95
|
+
}
|
|
96
|
+
|
|
97
|
+
// Skip noselect / [Gmail] virtual folders / huge label folders (All Mail).
|
|
98
|
+
// Folders we DO want: every selectable real folder, including Sent / Drafts /
|
|
99
|
+
// custom user folders. The skip list is conservative.
|
|
100
|
+
//
|
|
101
|
+
// Optional: pass --skip <pattern>,<pattern2> to also skip folders whose
|
|
102
|
+
// names match any of the given prefixes/regexes. Useful for skipping slow
|
|
103
|
+
// archive folders like Added2.* on bobma where Dovecot serves envelopes at
|
|
104
|
+
// ~9s/msg — they'd block the run for hours.
|
|
105
|
+
const SKIP_PATTERNS = (argVal("skip", "") || "").split(",").map(s => s.trim()).filter(Boolean);
|
|
106
|
+
function matchesSkipPattern(name: string): string | null {
|
|
107
|
+
for (const pat of SKIP_PATTERNS) {
|
|
108
|
+
try {
|
|
109
|
+
if (new RegExp(pat).test(name)) return pat;
|
|
110
|
+
} catch { /* not a regex — try prefix */ }
|
|
111
|
+
if (name.startsWith(pat)) return pat;
|
|
112
|
+
}
|
|
113
|
+
return null;
|
|
114
|
+
}
|
|
115
|
+
function shouldSkipFolder(folder: { id: number; name: string; path?: string; specialUse: string | null; flags: string[] }): string | null {
|
|
116
|
+
const flags = folder.flags || [];
|
|
117
|
+
if (flags.some(f => /noselect/i.test(f))) return "noselect";
|
|
118
|
+
if (folder.name === "[Gmail]") return "Gmail virtual";
|
|
119
|
+
// Match against the FULL PATH (e.g. "Added2.Dewayne") not just the leaf
|
|
120
|
+
// name ("Dewayne"). Skip patterns like ^Added need to see the parent.
|
|
121
|
+
const target = folder.path || folder.name;
|
|
122
|
+
const m = matchesSkipPattern(target);
|
|
123
|
+
if (m) return `--skip ${m}`;
|
|
124
|
+
return null;
|
|
125
|
+
}
|
|
126
|
+
|
|
127
|
+
/** If --skip-populated is given, skip any folder that already has at least
|
|
128
|
+
* this many local rows. Lets a re-run focus on the un-populated ones. */
|
|
129
|
+
const SKIP_POPULATED_THRESHOLD = Number(argVal("skip-populated", "0"));
|
|
130
|
+
|
|
131
|
+
// Folder priority — process special-use first (Sent, Drafts, INBOX, Trash),
|
|
132
|
+
// then everything else alphabetically. Slow archive folders thus wait until
|
|
133
|
+
// the user-visible folders are done.
|
|
134
|
+
const SPECIAL_PRIORITY = ["sent", "drafts", "inbox", "trash", "junk"];
|
|
135
|
+
function folderPriority(f: { name: string; specialUse: string | null }): number {
|
|
136
|
+
const su = (f.specialUse || "").toLowerCase();
|
|
137
|
+
const i = SPECIAL_PRIORITY.indexOf(su);
|
|
138
|
+
if (i >= 0) return i;
|
|
139
|
+
if (f.name.toLowerCase() === "inbox") return SPECIAL_PRIORITY.indexOf("inbox");
|
|
140
|
+
return SPECIAL_PRIORITY.length + 100;
|
|
141
|
+
}
|
|
142
|
+
|
|
143
|
+
/** Per-folder timeout — the populate is best-effort. If any single folder
|
|
144
|
+
* hasn't finished SELECT+FETCH within this window, skip it and continue.
|
|
145
|
+
* Avoids one slow folder wedging the entire run. Set generously — we'd
|
|
146
|
+
* rather give a slow folder a chance than skip it; reconnect-on-error
|
|
147
|
+
* recovers the connection state for the next folder either way. */
|
|
148
|
+
const PER_FOLDER_TIMEOUT_MS = Number(argVal("foldertimeout", "180000"));
|
|
149
|
+
|
|
150
|
+
function withTimeout<T>(p: Promise<T>, ms: number, label: string): Promise<T> {
|
|
151
|
+
return Promise.race([
|
|
152
|
+
p,
|
|
153
|
+
new Promise<T>((_, reject) =>
|
|
154
|
+
setTimeout(() => reject(new Error(`${label} timed out after ${ms}ms`)), ms)
|
|
155
|
+
),
|
|
156
|
+
]);
|
|
157
|
+
}
|
|
158
|
+
|
|
159
|
+
// ── Main ──
|
|
160
|
+
async function populateAccount(db: MailxDB, account: any): Promise<{ folders: number; messages: number; failed: number }> {
|
|
161
|
+
const folders = db.getFolders(account.id);
|
|
162
|
+
if (folders.length === 0) {
|
|
163
|
+
log(` [${account.id}] no folders in DB — run mailx sync at least once first to populate the folder list`);
|
|
164
|
+
return { folders: 0, messages: 0, failed: 0 };
|
|
165
|
+
}
|
|
166
|
+
|
|
167
|
+
const port = account.imap.port || 993;
|
|
168
|
+
const useTls = account.imap.tls !== false;
|
|
169
|
+
log(` [${account.id}] connecting to ${account.imap.host}:${port} ${useTls ? "(TLS)" : "(plain)"}...`);
|
|
170
|
+
|
|
171
|
+
const newClient = (): NativeImapClient => new NativeImapClient(
|
|
172
|
+
{
|
|
173
|
+
server: account.imap.host,
|
|
174
|
+
port,
|
|
175
|
+
username: account.imap.user || account.imap.username,
|
|
176
|
+
password: account.imap.password,
|
|
177
|
+
verbose: false,
|
|
178
|
+
inactivityTimeout: CONNECT_TIMEOUT_MS,
|
|
179
|
+
greetingTimeout: 30_000,
|
|
180
|
+
},
|
|
181
|
+
() => new NodeTcpTransport(),
|
|
182
|
+
);
|
|
183
|
+
let client: NativeImapClient = newClient();
|
|
184
|
+
|
|
185
|
+
let counters = { folders: 0, messages: 0, failed: 0 };
|
|
186
|
+
try {
|
|
187
|
+
await client.connect();
|
|
188
|
+
log(` [${account.id}] connected.`);
|
|
189
|
+
|
|
190
|
+
// Order: special-use folders first (Sent, Drafts, INBOX, Trash),
|
|
191
|
+
// then everything else alphabetically. So if the run gets killed
|
|
192
|
+
// partway, the user-facing folders are already populated.
|
|
193
|
+
const targets = (ONLY_FOLDER
|
|
194
|
+
? folders.filter(f => f.name === ONLY_FOLDER || f.path === ONLY_FOLDER)
|
|
195
|
+
: [...folders].sort((a, b) => {
|
|
196
|
+
const pa = folderPriority(a as any);
|
|
197
|
+
const pb = folderPriority(b as any);
|
|
198
|
+
if (pa !== pb) return pa - pb;
|
|
199
|
+
return a.name.localeCompare(b.name);
|
|
200
|
+
}));
|
|
201
|
+
log(` [${account.id}] ${targets.length} folder(s) to process (special-use first)`);
|
|
202
|
+
|
|
203
|
+
for (const folder of targets) {
|
|
204
|
+
const skip = shouldSkipFolder(folder as any);
|
|
205
|
+
if (skip) {
|
|
206
|
+
log(` skip ${folder.path || folder.name} (${skip})`);
|
|
207
|
+
continue;
|
|
208
|
+
}
|
|
209
|
+
if (SKIP_POPULATED_THRESHOLD > 0) {
|
|
210
|
+
const localCount = db.getMessageCount(account.id, folder.id);
|
|
211
|
+
if (localCount >= SKIP_POPULATED_THRESHOLD) {
|
|
212
|
+
log(` skip ${folder.path || folder.name} (already has ${localCount} local rows)`);
|
|
213
|
+
continue;
|
|
214
|
+
}
|
|
215
|
+
}
|
|
216
|
+
const t0 = Date.now();
|
|
217
|
+
try {
|
|
218
|
+
const info = await withTimeout(
|
|
219
|
+
client.select(folder.path || folder.name),
|
|
220
|
+
PER_FOLDER_TIMEOUT_MS,
|
|
221
|
+
`SELECT ${folder.name}`,
|
|
222
|
+
);
|
|
223
|
+
const startUid = db.getHighestUid(account.id, folder.id);
|
|
224
|
+
if (info.exists === 0) {
|
|
225
|
+
log(` ${folder.name}: empty server-side`);
|
|
226
|
+
counters.folders++;
|
|
227
|
+
continue;
|
|
228
|
+
}
|
|
229
|
+
// Fetch the latest N by SEQUENCE number — sequence ranges
|
|
230
|
+
// are dense (1..exists, no gaps from deletions), so the
|
|
231
|
+
// server returns exactly N envelopes. fetchLatestN uses
|
|
232
|
+
// sequence FETCH internally, with BODY.PEEK[HEADER] for
|
|
233
|
+
// mailx's own field needs — that's fine for our purposes
|
|
234
|
+
// and we get progress via the chunk callback.
|
|
235
|
+
const before = Date.now();
|
|
236
|
+
let chunkN = 0;
|
|
237
|
+
const msgs = await withTimeout(
|
|
238
|
+
client.fetchLatestN(FETCH_N, { source: false }, (chunk) => {
|
|
239
|
+
chunkN += chunk.length;
|
|
240
|
+
if (chunkN > 0 && chunkN % 50 === 0) log(` ${folder.name}: streamed ${chunkN}, ${Date.now() - before}ms in`);
|
|
241
|
+
}),
|
|
242
|
+
PER_FOLDER_TIMEOUT_MS,
|
|
243
|
+
`FETCH ${folder.name}`,
|
|
244
|
+
);
|
|
245
|
+
let stored = 0;
|
|
246
|
+
for (const m of msgs) {
|
|
247
|
+
if (typeof m.uid !== "number" || m.uid <= 0) continue;
|
|
248
|
+
db.upsertMessage({
|
|
249
|
+
accountId: account.id,
|
|
250
|
+
folderId: folder.id,
|
|
251
|
+
uid: m.uid,
|
|
252
|
+
messageId: m.messageId || "",
|
|
253
|
+
inReplyTo: m.inReplyTo || "",
|
|
254
|
+
references: [],
|
|
255
|
+
date: m.date ? m.date.getTime() : 0,
|
|
256
|
+
subject: m.subject || "",
|
|
257
|
+
from: (m.from && m.from[0]) ? { name: m.from[0].name || "", address: m.from[0].address || "" } : { name: "", address: "" },
|
|
258
|
+
to: (m.to || []).map(a => ({ name: a.name || "", address: a.address || "" })),
|
|
259
|
+
cc: (m.cc || []).map(a => ({ name: a.name || "", address: a.address || "" })),
|
|
260
|
+
flags: Array.from(m.flags || []),
|
|
261
|
+
size: m.size || 0,
|
|
262
|
+
hasAttachments: false, // populated by body-prefetch later
|
|
263
|
+
preview: "", // populated by body-prefetch later
|
|
264
|
+
bodyPath: "", // body not fetched here
|
|
265
|
+
});
|
|
266
|
+
stored++;
|
|
267
|
+
}
|
|
268
|
+
db.recalcFolderCounts(folder.id);
|
|
269
|
+
counters.folders++;
|
|
270
|
+
counters.messages += stored;
|
|
271
|
+
const took = Date.now() - t0;
|
|
272
|
+
log(` ${folder.name}: server=${info.exists}, fetched=${msgs.length}, stored=${stored}, prevHighUid=${startUid} (${took}ms)`);
|
|
273
|
+
} catch (e: any) {
|
|
274
|
+
counters.failed++;
|
|
275
|
+
log(` ${folder.name}: ERROR ${e?.message || e}`);
|
|
276
|
+
// Connection state is suspect after a timeout/error. Tear
|
|
277
|
+
// it down and open a fresh one so the next folder doesn't
|
|
278
|
+
// inherit the wedge. Skip-on-failure path; never bail.
|
|
279
|
+
try { await client.logout(); } catch { /* */ }
|
|
280
|
+
try {
|
|
281
|
+
client = newClient();
|
|
282
|
+
await client.connect();
|
|
283
|
+
log(` [reconnect] ${account.id}: fresh connection after error`);
|
|
284
|
+
} catch (reErr: any) {
|
|
285
|
+
log(` [reconnect] FAILED: ${reErr?.message || reErr} — bailing on this account`);
|
|
286
|
+
break;
|
|
287
|
+
}
|
|
288
|
+
}
|
|
289
|
+
}
|
|
290
|
+
} finally {
|
|
291
|
+
try { await client.logout(); } catch { /* */ }
|
|
292
|
+
}
|
|
293
|
+
return counters;
|
|
294
|
+
}
|
|
295
|
+
|
|
296
|
+
async function main(): Promise<void> {
|
|
297
|
+
log(`mailx populate-folders starting`);
|
|
298
|
+
log(` config: dbDir=${dbDir} fetchN=${FETCH_N}${ONLY_ACCOUNT ? " account=" + ONLY_ACCOUNT : ""}${ONLY_FOLDER ? " folder=" + ONLY_FOLDER : ""}`);
|
|
299
|
+
|
|
300
|
+
const db = new MailxDB(dbDir);
|
|
301
|
+
let totalFolders = 0, totalMessages = 0, totalFailed = 0;
|
|
302
|
+
|
|
303
|
+
for (const acct of accounts) {
|
|
304
|
+
if (!acct.enabled && acct.enabled !== undefined) continue;
|
|
305
|
+
if (ONLY_ACCOUNT && acct.id !== ONLY_ACCOUNT) continue;
|
|
306
|
+
if (!shouldUseImap(acct)) {
|
|
307
|
+
log(`[${acct.id}] skipped (not IMAP / Gmail-API account)`);
|
|
308
|
+
continue;
|
|
309
|
+
}
|
|
310
|
+
log(`[${acct.id}] starting...`);
|
|
311
|
+
try {
|
|
312
|
+
const c = await populateAccount(db, acct);
|
|
313
|
+
log(`[${acct.id}] done: ${c.folders} folders, ${c.messages} messages, ${c.failed} failed`);
|
|
314
|
+
totalFolders += c.folders; totalMessages += c.messages; totalFailed += c.failed;
|
|
315
|
+
} catch (e: any) {
|
|
316
|
+
log(`[${acct.id}] FATAL ${e?.message || e}`);
|
|
317
|
+
}
|
|
318
|
+
}
|
|
319
|
+
|
|
320
|
+
log(`---`);
|
|
321
|
+
log(`Total: ${totalFolders} folders processed, ${totalMessages} messages stored, ${totalFailed} failures`);
|
|
322
|
+
log(`Done. Restart rmfmail to see populated folders.`);
|
|
323
|
+
}
|
|
324
|
+
|
|
325
|
+
main().catch(e => {
|
|
326
|
+
console.error("fatal:", e);
|
|
327
|
+
process.exit(1);
|
|
328
|
+
});
|
package/_dev-populate.ts
ADDED
|
@@ -0,0 +1,222 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Standalone folder populator.
|
|
3
|
+
*
|
|
4
|
+
* Bypasses mailx's broken sync loop and directly fetches the latest N
|
|
5
|
+
* messages from every folder of every IMAP account, writing rows into the
|
|
6
|
+
* mailx.db. Designed to run while rmfmail.exe is killed (overnight job).
|
|
7
|
+
*
|
|
8
|
+
* Why: mailx's sync has been silently failing on bobma — bobma/Sent has
|
|
9
|
+
* never had real server-fetched rows in the local DB. Rather than fight
|
|
10
|
+
* the sync loop's connection-state corruption, this tool uses a single
|
|
11
|
+
* fresh IMAP connection (which we proved completes SELECT Sent in 26ms)
|
|
12
|
+
* and pumps messages into the DB directly via upsertMessage.
|
|
13
|
+
*
|
|
14
|
+
* Run:
|
|
15
|
+
* 1. Kill rmfmail.exe
|
|
16
|
+
* 2. cd Y:/dev/email/mailx/app
|
|
17
|
+
* 3. npx tsx Y:/dev/email/mailx/dev/populate-folders.ts [--n 500] [--account bobma] [--folder Sent]
|
|
18
|
+
*
|
|
19
|
+
* Defaults: N=500 latest messages per folder, all IMAP accounts, all folders.
|
|
20
|
+
*
|
|
21
|
+
* Notes:
|
|
22
|
+
* - Bodies are NOT fetched here — only envelope rows. Body fetch is
|
|
23
|
+
* mailx's prefetch loop's job; this tool just unblocks the metadata.
|
|
24
|
+
* - One connection, sequential folders. No parallelism that could
|
|
25
|
+
* trigger the same connection-state issues mailx hits.
|
|
26
|
+
* - Progress logged per folder. Failures logged + skipped, never bail.
|
|
27
|
+
* - Idempotent: re-running re-upserts. UNIQUE(account, folder, uid)
|
|
28
|
+
* constraint means existing rows are updated, not duplicated.
|
|
29
|
+
*/
|
|
30
|
+
|
|
31
|
+
import * as fs from "node:fs";
|
|
32
|
+
import * as path from "node:path";
|
|
33
|
+
import * as os from "node:os";
|
|
34
|
+
import { NativeImapClient } from "@bobfrankston/iflow-direct";
|
|
35
|
+
import { NodeTcpTransport } from "@bobfrankston/node-tcp-transport";
|
|
36
|
+
import { MailxDB } from "@bobfrankston/mailx-store";
|
|
37
|
+
|
|
38
|
+
// ── Args ──
|
|
39
|
+
const args = process.argv.slice(2);
|
|
40
|
+
const argVal = (name: string, def?: string): string | undefined => {
|
|
41
|
+
const i = args.indexOf(`--${name}`);
|
|
42
|
+
if (i < 0 || i === args.length - 1) return def;
|
|
43
|
+
return args[i + 1];
|
|
44
|
+
};
|
|
45
|
+
const FETCH_N = Number(argVal("n", "500"));
|
|
46
|
+
const ONLY_ACCOUNT = argVal("account");
|
|
47
|
+
const ONLY_FOLDER = argVal("folder");
|
|
48
|
+
const CONNECT_TIMEOUT_MS = Number(argVal("timeout", "60000"));
|
|
49
|
+
|
|
50
|
+
// ── Config ──
|
|
51
|
+
const home = process.env.USERPROFILE || os.homedir();
|
|
52
|
+
const dbDir = path.join(home, ".rmfmail");
|
|
53
|
+
const accountsPath = path.join(dbDir, "accounts.jsonc");
|
|
54
|
+
if (!fs.existsSync(accountsPath)) {
|
|
55
|
+
console.error(`accounts.jsonc not found at ${accountsPath}`);
|
|
56
|
+
process.exit(1);
|
|
57
|
+
}
|
|
58
|
+
|
|
59
|
+
// JSONC: strip // and /* */ before parsing
|
|
60
|
+
function parseJsonc(raw: string): any {
|
|
61
|
+
const stripped = raw
|
|
62
|
+
.replace(/\/\*[\s\S]*?\*\//g, "")
|
|
63
|
+
.replace(/^\s*\/\/.*$/gm, "");
|
|
64
|
+
return JSON.parse(stripped);
|
|
65
|
+
}
|
|
66
|
+
|
|
67
|
+
const accountsCfg = parseJsonc(fs.readFileSync(accountsPath, "utf-8"));
|
|
68
|
+
const accounts = accountsCfg.accounts || [];
|
|
69
|
+
|
|
70
|
+
// ── Helpers ──
|
|
71
|
+
function ts(): string { return new Date().toISOString().replace("T", " ").slice(0, 19); }
|
|
72
|
+
function log(msg: string): void { console.log(`[${ts()}] ${msg}`); }
|
|
73
|
+
|
|
74
|
+
// Skip Gmail / API-backed accounts — they don't speak IMAP through this
|
|
75
|
+
// tool's path, and Gmail's sync route is separate (REST). Same for any
|
|
76
|
+
// account without an `imap` block.
|
|
77
|
+
function shouldUseImap(acct: any): boolean {
|
|
78
|
+
if (!acct?.imap?.host) return false;
|
|
79
|
+
const host = String(acct.imap.host).toLowerCase();
|
|
80
|
+
if (host.includes("gmail.com") || host.includes("googlemail.com")) return false;
|
|
81
|
+
return true;
|
|
82
|
+
}
|
|
83
|
+
|
|
84
|
+
// Skip noselect / [Gmail] virtual folders / huge label folders (All Mail).
|
|
85
|
+
// Folders we DO want: every selectable real folder, including Sent / Drafts /
|
|
86
|
+
// custom user folders. The skip list is conservative.
|
|
87
|
+
function shouldSkipFolder(folder: { id: number; name: string; specialUse: string | null; flags: string[] }): string | null {
|
|
88
|
+
const flags = folder.flags || [];
|
|
89
|
+
if (flags.some(f => /noselect/i.test(f))) return "noselect";
|
|
90
|
+
if (folder.name === "[Gmail]") return "Gmail virtual";
|
|
91
|
+
return null;
|
|
92
|
+
}
|
|
93
|
+
|
|
94
|
+
// ── Main ──
|
|
95
|
+
async function populateAccount(db: MailxDB, account: any): Promise<{ folders: number; messages: number; failed: number }> {
|
|
96
|
+
const folders = db.getFolders(account.id);
|
|
97
|
+
if (folders.length === 0) {
|
|
98
|
+
log(` [${account.id}] no folders in DB — run mailx sync at least once first to populate the folder list`);
|
|
99
|
+
return { folders: 0, messages: 0, failed: 0 };
|
|
100
|
+
}
|
|
101
|
+
|
|
102
|
+
const port = account.imap.port || 993;
|
|
103
|
+
const useTls = account.imap.tls !== false;
|
|
104
|
+
log(` [${account.id}] connecting to ${account.imap.host}:${port} ${useTls ? "(TLS)" : "(plain)"}...`);
|
|
105
|
+
|
|
106
|
+
const client = new NativeImapClient(
|
|
107
|
+
{
|
|
108
|
+
server: account.imap.host,
|
|
109
|
+
port,
|
|
110
|
+
username: account.imap.user || account.imap.username,
|
|
111
|
+
password: account.imap.password,
|
|
112
|
+
verbose: false,
|
|
113
|
+
inactivityTimeout: CONNECT_TIMEOUT_MS,
|
|
114
|
+
greetingTimeout: 30_000,
|
|
115
|
+
},
|
|
116
|
+
() => new NodeTcpTransport(),
|
|
117
|
+
);
|
|
118
|
+
|
|
119
|
+
let counters = { folders: 0, messages: 0, failed: 0 };
|
|
120
|
+
try {
|
|
121
|
+
await client.connect();
|
|
122
|
+
log(` [${account.id}] connected.`);
|
|
123
|
+
|
|
124
|
+
const targets = ONLY_FOLDER
|
|
125
|
+
? folders.filter(f => f.name === ONLY_FOLDER || f.path === ONLY_FOLDER)
|
|
126
|
+
: folders;
|
|
127
|
+
log(` [${account.id}] ${targets.length} folder(s) to process`);
|
|
128
|
+
|
|
129
|
+
for (const folder of targets) {
|
|
130
|
+
const skip = shouldSkipFolder(folder as any);
|
|
131
|
+
if (skip) {
|
|
132
|
+
log(` skip ${folder.name} (${skip})`);
|
|
133
|
+
continue;
|
|
134
|
+
}
|
|
135
|
+
const t0 = Date.now();
|
|
136
|
+
try {
|
|
137
|
+
const info = await client.select(folder.path || folder.name);
|
|
138
|
+
const startUid = db.getHighestUid(account.id, folder.id);
|
|
139
|
+
if (info.exists === 0) {
|
|
140
|
+
log(` ${folder.name}: empty server-side`);
|
|
141
|
+
counters.folders++;
|
|
142
|
+
continue;
|
|
143
|
+
}
|
|
144
|
+
// Fetch the latest N by sequence — fast, doesn't depend on
|
|
145
|
+
// having a known highestUid. Newer messages overwrite older
|
|
146
|
+
// upserts (UNIQUE constraint) so re-runs are idempotent.
|
|
147
|
+
const msgs = await client.fetchLatestN(FETCH_N, { source: false });
|
|
148
|
+
let stored = 0;
|
|
149
|
+
for (const m of msgs) {
|
|
150
|
+
if (typeof m.uid !== "number" || m.uid <= 0) continue;
|
|
151
|
+
db.upsertMessage({
|
|
152
|
+
accountId: account.id,
|
|
153
|
+
folderId: folder.id,
|
|
154
|
+
uid: m.uid,
|
|
155
|
+
messageId: m.messageId || "",
|
|
156
|
+
inReplyTo: m.inReplyTo || "",
|
|
157
|
+
references: [],
|
|
158
|
+
date: m.date ? m.date.getTime() : 0,
|
|
159
|
+
subject: m.subject || "",
|
|
160
|
+
from: (m.from && m.from[0]) ? { name: m.from[0].name || "", address: m.from[0].address || "" } : { name: "", address: "" },
|
|
161
|
+
to: (m.to || []).map(a => ({ name: a.name || "", address: a.address || "" })),
|
|
162
|
+
cc: (m.cc || []).map(a => ({ name: a.name || "", address: a.address || "" })),
|
|
163
|
+
flags: Array.from(m.flags || []),
|
|
164
|
+
size: m.size || 0,
|
|
165
|
+
hasAttachments: false, // populated by body-prefetch later
|
|
166
|
+
preview: "", // populated by body-prefetch later
|
|
167
|
+
bodyPath: "", // body not fetched here
|
|
168
|
+
});
|
|
169
|
+
stored++;
|
|
170
|
+
}
|
|
171
|
+
db.recalcFolderCounts(folder.id);
|
|
172
|
+
counters.folders++;
|
|
173
|
+
counters.messages += stored;
|
|
174
|
+
const took = Date.now() - t0;
|
|
175
|
+
log(` ${folder.name}: server=${info.exists}, fetched=${msgs.length}, stored=${stored}, prevHighUid=${startUid} (${took}ms)`);
|
|
176
|
+
} catch (e: any) {
|
|
177
|
+
counters.failed++;
|
|
178
|
+
log(` ${folder.name}: ERROR ${e?.message || e}`);
|
|
179
|
+
// Connection might be in a bad state — try to recover by
|
|
180
|
+
// closing the mailbox. Continue to next folder either way.
|
|
181
|
+
try { await client.closeMailbox(); } catch { /* */ }
|
|
182
|
+
}
|
|
183
|
+
}
|
|
184
|
+
} finally {
|
|
185
|
+
try { await client.logout(); } catch { /* */ }
|
|
186
|
+
}
|
|
187
|
+
return counters;
|
|
188
|
+
}
|
|
189
|
+
|
|
190
|
+
async function main(): Promise<void> {
|
|
191
|
+
log(`mailx populate-folders starting`);
|
|
192
|
+
log(` config: dbDir=${dbDir} fetchN=${FETCH_N}${ONLY_ACCOUNT ? " account=" + ONLY_ACCOUNT : ""}${ONLY_FOLDER ? " folder=" + ONLY_FOLDER : ""}`);
|
|
193
|
+
|
|
194
|
+
const db = new MailxDB(dbDir);
|
|
195
|
+
let totalFolders = 0, totalMessages = 0, totalFailed = 0;
|
|
196
|
+
|
|
197
|
+
for (const acct of accounts) {
|
|
198
|
+
if (!acct.enabled && acct.enabled !== undefined) continue;
|
|
199
|
+
if (ONLY_ACCOUNT && acct.id !== ONLY_ACCOUNT) continue;
|
|
200
|
+
if (!shouldUseImap(acct)) {
|
|
201
|
+
log(`[${acct.id}] skipped (not IMAP / Gmail-API account)`);
|
|
202
|
+
continue;
|
|
203
|
+
}
|
|
204
|
+
log(`[${acct.id}] starting...`);
|
|
205
|
+
try {
|
|
206
|
+
const c = await populateAccount(db, acct);
|
|
207
|
+
log(`[${acct.id}] done: ${c.folders} folders, ${c.messages} messages, ${c.failed} failed`);
|
|
208
|
+
totalFolders += c.folders; totalMessages += c.messages; totalFailed += c.failed;
|
|
209
|
+
} catch (e: any) {
|
|
210
|
+
log(`[${acct.id}] FATAL ${e?.message || e}`);
|
|
211
|
+
}
|
|
212
|
+
}
|
|
213
|
+
|
|
214
|
+
log(`---`);
|
|
215
|
+
log(`Total: ${totalFolders} folders processed, ${totalMessages} messages stored, ${totalFailed} failures`);
|
|
216
|
+
log(`Done. Restart rmfmail to see populated folders.`);
|
|
217
|
+
}
|
|
218
|
+
|
|
219
|
+
main().catch(e => {
|
|
220
|
+
console.error("fatal:", e);
|
|
221
|
+
process.exit(1);
|
|
222
|
+
});
|
|
@@ -420,7 +420,7 @@ export function removeMessagesAndReconcile(uids) {
|
|
|
420
420
|
el.remove();
|
|
421
421
|
}
|
|
422
422
|
if (state.getMessages().length === 0) {
|
|
423
|
-
body.innerHTML = `<div class="ml-empty">No messages
|
|
423
|
+
body.innerHTML = `<div class="ml-empty">No messages</div>`;
|
|
424
424
|
}
|
|
425
425
|
}
|
|
426
426
|
if (outcome.focusedWasRemoved) {
|
|
@@ -609,7 +609,7 @@ export async function loadMessages(accountId, folderId, page = 1, specialUse = "
|
|
|
609
609
|
updateSortIndicators();
|
|
610
610
|
if (result.items.length === 0) {
|
|
611
611
|
state.setMessages([]);
|
|
612
|
-
body.innerHTML = `<div class="ml-empty">${flaggedOnly ? "No flagged messages" : "No messages
|
|
612
|
+
body.innerHTML = `<div class="ml-empty">${flaggedOnly ? "No flagged messages" : "No messages"}</div>`;
|
|
613
613
|
return;
|
|
614
614
|
}
|
|
615
615
|
state.setMessages(result.items);
|