@nomadamas/mailcrawl 0.1.6 → 0.1.7

package/README.md

@@ -50,6 +50,21 @@ before multilingual search; the command re-analyzes existing messages and
 atomically records the new fingerprint. Embedding model changes are independent
 and require a new `mailcrawl index` generation.
 
+## Sync read concurrency
+
+`mailcrawl sync` reads a page of envelopes through a bounded pool of himalaya
+processes — 4 by default, `--concurrency <n>` to change it — instead of
+spawning one process per envelope. Gmail throttles accounts that open too many
+simultaneous IMAP connections, and an unbounded fan-out made one throttled
+read abort the entire sync. `--page-size` still controls only how many
+envelopes the IMAP window returns.
+
+A read that fails is retried with exponential backoff (three attempts by
+default). Messages that stay unreadable are reported in the sync JSON as
+`failures[]` with their `providerKey`, `attempts`, and the redacted himalaya
+error, while the messages that could be read are still synced. The command
+exits non-zero only when nothing could be read.
+
 ## Installation
 
 For the required Node setup, Kiwi model files, Go installation, Japanese and

package/dist/cli/index.js

@@ -18,6 +18,7 @@ program
     .option("--mailbox <name>", "mailbox name", "INBOX")
     .option("--backend <name>")
     .option("--page-size <n>", "envelopes per page", "1000")
+    .option("--concurrency <n>", "simultaneous message reads (default 4)")
     .option("--himalaya-config <path>")
     .option("--include-category <name>", "include a normally excluded category", collect, [])
     .option("--exclude-category <name>", "exclude a classification category", collect, [])
@@ -38,7 +39,11 @@ program
             : himalayaSource(options);
         const excludedCategories = (options.excludeCategory.length ? options.excludeCategory : ["spam", "promotions"])
             .filter((category) => !options.includeCategory.includes(category));
-        output(await archive.sync(await source.list(), { excludedCategories }), options.json);
+        const { messages, failures } = await source.collect();
+        if (messages.length === 0 && failures.length > 0) {
+            throw new Error(`source read failed for all ${failures.length} message(s); no messages could be read: ${failures.slice(0, 3).map((failure) => failure.providerKey).join(", ")}`);
+        }
+        output({ ...await archive.sync(messages, { excludedCategories }), failures }, options.json);
     }
     finally {
         archive.close();
@@ -325,7 +330,18 @@ function fixtureSource(options) {
 function himalayaSource(options) {
     if (!options.account)
         throw new Error("--account is required for selected source");
-    return new HimalayaSource(options.account, options.mailbox, options.backend, Number(options.pageSize), options.himalayaConfig);
+    return new HimalayaSource(options.account, options.mailbox, options.backend, Number(options.pageSize), options.himalayaConfig, {
+        concurrency: readConcurrency(options),
+    });
+}
+function readConcurrency(options) {
+    if (options.concurrency === undefined)
+        return undefined;
+    const concurrency = Number(options.concurrency);
+    if (!Number.isInteger(concurrency) || concurrency < 1) {
+        throw new Error(`--concurrency must be a positive integer: ${options.concurrency}`);
+    }
+    return concurrency;
 }
 function semanticStatus(archive, semantic) {
     return semantic.archiveRevision === archive.status().archiveRevision ? { ...semantic, status: "healthy" } : { ...semantic, status: "stale" };

package/dist/source.d.ts

@@ -1,10 +1,34 @@
-import type { MailMessage } from "./types.js";
+import type { MailMessage, SourceReadResult } from "./types.js";
+/** Default number of simultaneous `himalaya message read` processes. */
+export declare const DEFAULT_READ_CONCURRENCY = 4;
+/** Default number of attempts per message read, including the first one. */
+export declare const DEFAULT_RETRY_ATTEMPTS = 3;
+/** Default backoff before the second read attempt; doubles per round. */
+export declare const DEFAULT_RETRY_BASE_DELAY_MS = 250;
+/** Executes one himalaya invocation; replaced by tests. */
+export type HimalayaExec = (args: string[], maxBuffer: number) => Promise<{
+    stdout: string;
+    stderr?: string;
+}>;
+export interface HimalayaReadOptions {
+    /** Simultaneous message reads; defaults to DEFAULT_READ_CONCURRENCY. */
+    concurrency?: number;
+    /** Attempts per message read, including the first one; defaults to DEFAULT_RETRY_ATTEMPTS. */
+    retryAttempts?: number;
+    /** Backoff before the second attempt, doubling per round; defaults to DEFAULT_RETRY_BASE_DELAY_MS. */
+    retryBaseDelayMs?: number;
+    exec?: HimalayaExec;
+}
 export interface MailSource {
+    /** Messages that could be read, plus one record per message that could not. */
+    collect(): Promise<SourceReadResult>;
+    /** Strict variant of `collect()`: rejects when any message could not be read. */
     list(): Promise<MailMessage[]>;
 }
 export declare class FixtureSource implements MailSource {
     private readonly path;
     constructor(path: string);
+    collect(): Promise<SourceReadResult>;
     list(): Promise<MailMessage[]>;
 }
 export declare class HimalayaSource implements MailSource {
@@ -13,7 +37,17 @@ export declare class HimalayaSource implements MailSource {
     private readonly backend?;
     private readonly pageSize;
     private readonly config?;
-    constructor(account: string, mailbox?: string, backend?: string | undefined, pageSize?: number, config?: string | undefined);
+    private readonly readOptions;
+    constructor(account: string, mailbox?: string, backend?: string | undefined, pageSize?: number, config?: string | undefined, readOptions?: HimalayaReadOptions);
+    collect(): Promise<SourceReadResult>;
     list(): Promise<MailMessage[]>;
+    /** Reads a page through a bounded pool, retrying failed reads in later rounds. */
+    private readPage;
+    private get concurrency();
+    private envelopes;
+    private baseArgs;
     private read;
+    private run;
 }
+/** Runs `worker` over `items` with at most `limit` workers in flight, preserving input order. */
+export declare function mapWithConcurrency<T, R>(items: readonly T[], limit: number, worker: (item: T) => Promise<R>): Promise<R[]>;

package/dist/source.js

@@ -3,11 +3,20 @@ import { promisify } from "node:util";
 import { readFile } from "node:fs/promises";
 import { redactDiagnostic } from "./redact.js";
 const execFileAsync = promisify(execFile);
+/** Default number of simultaneous `himalaya message read` processes. */
+export const DEFAULT_READ_CONCURRENCY = 4;
+/** Default number of attempts per message read, including the first one. */
+export const DEFAULT_RETRY_ATTEMPTS = 3;
+/** Default backoff before the second read attempt; doubles per round. */
+export const DEFAULT_RETRY_BASE_DELAY_MS = 250;
 export class FixtureSource {
     path;
     constructor(path) {
         this.path = path;
     }
+    async collect() {
+        return { messages: await this.list(), failures: [] };
+    }
     async list() {
         const raw = await readFile(this.path, "utf8");
         return JSON.parse(raw);
@@ -19,62 +28,155 @@ export class HimalayaSource {
     backend;
     pageSize;
     config;
-    constructor(account, mailbox = "INBOX", backend, pageSize = 1000, config) {
+    readOptions;
+    constructor(account, mailbox = "INBOX", backend, pageSize = 1000, config, readOptions = {}) {
         this.account = account;
         this.mailbox = mailbox;
         this.backend = backend;
         this.pageSize = pageSize;
         this.config = config;
+        this.readOptions = readOptions;
+    }
+    async collect() {
+        return this.readPage(await this.envelopes());
     }
     async list() {
-        const args = this.config ? ["-c", this.config, "-a", this.account] : ["-a", this.account];
-        if (this.backend)
-            args.push("-b", this.backend);
+        const { messages, failures } = await this.collect();
+        if (failures.length > 0)
+            throw new Error(failures[0].error);
+        return messages;
+    }
+    /** Reads a page through a bounded pool, retrying failed reads in later rounds. */
+    async readPage(envelopes) {
+        const keys = envelopes.map(envelopeKey);
+        const rawMime = new Map();
+        const errors = new Map();
+        const attempts = Math.max(1, Math.floor(this.readOptions.retryAttempts ?? DEFAULT_RETRY_ATTEMPTS));
+        const baseDelayMs = Math.max(0, this.readOptions.retryBaseDelayMs ?? DEFAULT_RETRY_BASE_DELAY_MS);
+        let pending = envelopes.map((_, index) => index);
+        for (let attempt = 1; attempt <= attempts && pending.length > 0; attempt += 1) {
+            if (attempt > 1)
+                await delay(baseDelayMs * 2 ** (attempt - 2));
+            const failed = [];
+            await mapWithConcurrency(pending, this.concurrency, async (index) => {
+                try {
+                    rawMime.set(index, await this.read(keys[index]));
+                }
+                catch (error) {
+                    errors.set(index, error instanceof Error ? error : new Error(String(error)));
+                    failed.push(index);
+                }
+            });
+            pending = failed;
+        }
+        const messages = [];
+        const failures = [];
+        envelopes.forEach((envelope, index) => {
+            const raw = rawMime.get(index);
+            if (raw !== undefined) {
+                messages.push(envelopeMessage(this.account, this.mailbox, envelope, keys[index], raw));
+                return;
+            }
+            failures.push({ providerKey: keys[index], attempts, error: (errors.get(index) ?? new Error("message read failed")).message });
+        });
+        return { messages, failures };
+    }
+    get concurrency() {
+        return Math.max(1, Math.floor(this.readOptions.concurrency ?? DEFAULT_READ_CONCURRENCY));
+    }
+    async envelopes() {
+        const args = this.baseArgs();
         args.push("envelope", "list", "--mailbox", this.mailbox, "--page-size", String(this.pageSize), "--json");
-        const { stdout } = await runHimalaya(args, 16 * 1024 * 1024, "envelope list");
+        const { stdout } = await this.run(args, 16 * 1024 * 1024, "envelope list");
         const payload = JSON.parse(stdout);
-        const envelopes = payload.envelopes ?? (Array.isArray(payload) ? payload : []);
-        return Promise.all(envelopes.map(async (envelope) => {
-            const providerKey = String(envelope.id ?? envelope.uid ?? envelope["message-id"]);
-            const rawMime = await this.read(providerKey);
-            return {
-                accountId: this.account,
-                mailbox: this.mailbox,
-                providerKey,
-                messageId: envelope["message-id"],
-                inReplyTo: envelope["in-reply-to"]?.[0],
-                subject: envelope.subject ?? "",
-                from: address(envelope.from),
-                to: addresses(envelope.to),
-                cc: addresses(envelope.cc),
-                date: envelope.date ?? new Date(0).toISOString(),
-                text: envelope.body ?? envelope.snippet ?? "",
-                labels: strings(envelope.labels),
-                flags: strings(envelope.flags),
-                classifications: strings(envelope.classifications),
-                rawMime,
-            };
-        }));
+        return payload.envelopes ?? (Array.isArray(payload) ? payload : []);
     }
-    async read(id) {
+    baseArgs() {
         const args = this.config ? ["-c", this.config, "-a", this.account] : ["-a", this.account];
         if (this.backend)
             args.push("-b", this.backend);
+        return args;
+    }
+    async read(id) {
+        const args = this.baseArgs();
         args.push("--json", "message", "read", id, "--raw");
-        const { stdout } = await runHimalaya(args, 32 * 1024 * 1024, "message read");
+        const { stdout } = await this.run(args, 32 * 1024 * 1024, "message read");
         const payload = JSON.parse(stdout);
         return payload.message ?? stdout;
     }
+    run(args, maxBuffer, operation) {
+        return runHimalaya(args, maxBuffer, operation, this.readOptions.exec);
+    }
+}
+/** Runs `worker` over `items` with at most `limit` workers in flight, preserving input order. */
+export async function mapWithConcurrency(items, limit, worker) {
+    const results = new Array(items.length);
+    let cursor = 0;
+    let stopped = false;
+    const width = Math.max(1, Math.min(Math.floor(limit) || 1, items.length));
+    await Promise.all(Array.from({ length: width }, async () => {
+        while (!stopped) {
+            const index = cursor++;
+            if (index >= items.length)
+                return;
+            try {
+                results[index] = await worker(items[index]);
+            }
+            catch (error) {
+                // A failed page must not keep spawning provider processes for the
+                // items that were never read; only the in-flight workers finish.
+                stopped = true;
+                throw error;
+            }
+        }
+    }));
+    return results;
 }
-async function runHimalaya(args, maxBuffer, operation) {
+function envelopeKey(envelope) {
+    return String(envelope.id ?? envelope.uid ?? envelope["message-id"]);
+}
+function delay(ms) {
+    return new Promise((resolve) => setTimeout(resolve, ms));
+}
+function envelopeMessage(account, mailbox, envelope, providerKey, rawMime) {
+    return {
+        accountId: account,
+        mailbox,
+        providerKey,
+        messageId: envelope["message-id"],
+        inReplyTo: envelope["in-reply-to"]?.[0],
+        subject: envelope.subject ?? "",
+        from: address(envelope.from),
+        to: addresses(envelope.to),
+        cc: addresses(envelope.cc),
+        date: envelope.date ?? new Date(0).toISOString(),
+        text: envelope.body ?? envelope.snippet ?? "",
+        labels: strings(envelope.labels),
+        flags: strings(envelope.flags),
+        classifications: strings(envelope.classifications),
+        rawMime,
+    };
+}
+async function runHimalaya(args, maxBuffer, operation, exec) {
     try {
-        return await execFileAsync("himalaya", args, { maxBuffer });
+        return await (exec ?? defaultExec)(args, maxBuffer);
     }
     catch (error) {
         const detail = error instanceof Error ? error.message : String(error);
-        throw new Error(`himalaya ${operation} failed: ${redactDiagnostic(detail)}`);
+        throw new Error(`himalaya ${operation} failed: ${redactDiagnostic(detail)}${stderrDetail(error)}`);
     }
 }
+/** Keeps himalaya's own stderr in the surfaced error so throttling is distinguishable from a malformed message. */
+function stderrDetail(error) {
+    const stderr = error?.stderr;
+    if (typeof stderr !== "string")
+        return "";
+    const text = stderr.trim();
+    if (!text)
+        return "";
+    return `: ${String(redactDiagnostic(text.length > 2_000 ? `${text.slice(0, 2_000)} [truncated]` : text))}`;
+}
+const defaultExec = (args, maxBuffer) => execFileAsync("himalaya", args, { maxBuffer });
 function address(value) {
     if (typeof value === "string")
         return value;

package/dist/types.d.ts

@@ -96,6 +96,15 @@ export interface SyncReport {
     excluded: number;
     excludedByReason: Record<string, number>;
 }
+export interface SourceReadFailure {
+    providerKey: string;
+    attempts: number;
+    error: string;
+}
+export interface SourceReadResult {
+    messages: MailMessage[];
+    failures: SourceReadFailure[];
+}
 export interface ClassificationPolicy {
     excludedCategories?: string[];
 }

package/docs/architecture.md

@@ -158,13 +158,19 @@ All commands support `--json` where machine-readable output is useful.
 ```text
 mailcrawl doctor
 mailcrawl status
-mailcrawl sync
+mailcrawl sync [--page-size N] [--concurrency N]
 mailcrawl embed
 mailcrawl search --mode fts|bm25|keyword|semantic|hybrid [--limit N] [--mailbox NAME] QUERY
 mailcrawl message get MESSAGE_ID
 mailcrawl repair [--fts|--semantic|--all]
 ```
 
+`--page-size` is the IMAP envelope window and `--concurrency` is the number of
+simultaneous message reads (default 4). Reads run through that bounded pool,
+failures are retried with backoff, and the messages that could be read are
+synced even when some reads keep failing, so a throttled provider no longer
+aborts a whole page.
+
 Example sync response:
 
 ```json
@@ -175,10 +181,15 @@ Example sync response:
   "unchanged": 4821,
   "chunksAdded": 21,
   "chunksDeleted": 6,
-  "embeddingBacklog": 21
+  "embeddingBacklog": 21,
+  "failures": []
 }
 ```
 
+Each entry of `failures` carries the `providerKey`, the `attempts` spent on it,
+and the redacted himalaya error (including himalaya's own stderr). The command
+exits non-zero only when nothing could be read.
+
 Example search hit:
 
 ```json

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@nomadamas/mailcrawl",
-  "version": "0.1.6",
+  "version": "0.1.7",
   "description": "Local Himalaya-backed incremental email indexing and hybrid search CLI",
   "license": "MIT",
   "type": "module",

package/skills/mailcrawl/SKILL.md

@@ -63,6 +63,13 @@ command, such as a user-level cron or systemd timer. Run `sync` first and
 `index` afterward; inspect JSON exit status before handing results to an
 agent.
 
+Message reads use a bounded pool of himalaya processes (4 by default; change
+it with `--concurrency <n>`) because Gmail throttles accounts that open too
+many simultaneous IMAP connections. `--page-size` only sets the envelope
+window. A read that fails is retried with backoff, and messages that stay
+unreadable appear in the sync JSON `failures[]` while the readable messages
+are still synced; the command exits non-zero only when nothing could be read.
+
 Use a fixture for deterministic development:
 
 ```bash