@nomadamas/mailcrawl 0.1.5 → 0.1.7

package/README.md

@@ -20,6 +20,7 @@ mailcrawl embed --json
 mailcrawl search --mode hybrid --json "계약 갱신 조건"
 mailcrawl status --json
 mailcrawl doctor --json
+mailcrawl repair --all --json
 ```
 
 The CLI owns email synchronization and indexes. Consumers such as AutoRAG,
@@ -49,6 +50,21 @@ before multilingual search; the command re-analyzes existing messages and
 atomically records the new fingerprint. Embedding model changes are independent
 and require a new `mailcrawl index` generation.
 
+## Sync read concurrency
+
+`mailcrawl sync` reads a page of envelopes through a bounded pool of himalaya
+processes — 4 by default, `--concurrency <n>` to change it — instead of
+spawning one process per envelope. Gmail throttles accounts that open too many
+simultaneous IMAP connections, and an unbounded fan-out made one throttled
+read abort the entire sync. `--page-size` still controls only how many
+envelopes the IMAP window returns.
+
+A read that fails is retried with exponential backoff (three attempts by
+default). Messages that stay unreadable are reported in the sync JSON as
+`failures[]` with their `providerKey`, `attempts`, and the redacted himalaya
+error, while the messages that could be read are still synced. The command
+exits non-zero only when nothing could be read.
+
 ## Installation
 
 For the required Node setup, Kiwi model files, Go installation, Japanese and
@@ -56,6 +72,21 @@ Chinese helper builds, environment variables, smoke tests, and license
 requirements, follow [`docs/multilingual-installation.md`](docs/multilingual-installation.md)
 before using multilingual indexing or search.
 
+## Shared loopback embedding provider
+
+The local in-process model is the default. Opt into a local HTTP runtime with
+`--provider loopback-http`, `--embed-url`, `--embed-model`, and `--embed-dim`
+on `index`, `repair --semantic`, or semantic search. Only HTTP URLs for
+`127.0.0.1`, `localhost`, or `::1` are accepted. Query/passage prefixes and
+timeout are optional and are included in the provider identity. Any provider
+setting change rebuilds vectors, while a failed rebuild preserves the prior
+`CURRENT` generation.
+
+The equivalent environment contract is
+`MAILCRAWL_EMBEDDER_PROVIDER`, `MAILCRAWL_EMBED_URL`,
+`MAILCRAWL_EMBED_MODEL`, `MAILCRAWL_EMBED_DIM`, `MAILCRAWL_QUERY_PREFIX`,
+`MAILCRAWL_PASSAGE_PREFIX`, and `MAILCRAWL_EMBED_TIMEOUT`.
+
 ## Releasing
 
 GitHub Release `vX.Y.Z` (must match `package.json`) publishes `@nomadamas/mailcrawl` to npm with OIDC trusted publishing. No `NPM_TOKEN` is stored in GitHub.

package/dist/archive.d.ts

@@ -1,14 +1,25 @@
 import Database from "better-sqlite3";
-import type { ClassificationPolicy, Chunk, MailMessage, NormalizedMessage, SearchFilters, SearchHit, SyncReport } from "./types.js";
+import type { ClassificationPolicy, Chunk, LoopbackHttpConfig, MailMessage, NormalizedMessage, SearchFilters, SearchHit, SyncReport } from "./types.js";
 export declare class Archive {
     readonly db: Database.Database;
     private embedder?;
     private lexical?;
     private operationTail;
     private lexicalRebuildRequired;
-    constructor(path?: string);
+    private readonly embedderConfig?;
+    constructor(path?: string, embedderConfig?: LoopbackHttpConfig);
     close(): void;
     sync(messages: MailMessage[], policy?: ClassificationPolicy): Promise<SyncReport>;
+    status(): {
+        messageCount: number;
+        chunkCount: number;
+        embeddingBacklog: number;
+        archiveRevision: string;
+        fts: {
+            status: "healthy" | "stale";
+            rows: number;
+        };
+    };
     private syncUnlocked;
     searchBm25(query: string, filters?: SearchFilters, limit?: number): Promise<SearchHit[]>;
     indexSemantic(): Promise<{
@@ -24,6 +35,7 @@ export declare class Archive {
     }>;
     private indexSemanticGenerationUnlocked;
     private cleanupSemanticGenerations;
+    private generationTimestamp;
     semanticGeneration(root: string): {
         generation: string;
         archiveRevision: string;

package/dist/archive.js

@@ -1,5 +1,5 @@
 import Database from "better-sqlite3";
-import { createHash } from "node:crypto";
+import { createHash, randomBytes } from "node:crypto";
 import { mkdirSync, readFileSync, readdirSync, renameSync, rmSync, writeFileSync } from "node:fs";
 import { join } from "node:path";
 import { buildChunks } from "./chunk.js";
@@ -8,14 +8,16 @@ import { scopedId, snippet } from "./util.js";
 import { createEmbedder, embeddingModelName } from "./embedding.js";
 import { createLexicalAnalyzers, languagesForText, lexicalFields, LEXICAL_ANALYZER_VERSION } from "./lexical.js";
 const RETAINED_SEMANTIC_GENERATIONS = 2;
-const SEMANTIC_GENERATION_NAME = /^gen-[0-9a-f]{16}-[0-9]+$/;
+const SEMANTIC_GENERATION_NAME = /^gen-[0-9a-f]{16}-[0-9]+(?:-[0-9a-f]+)?$/;
 export class Archive {
     db;
     embedder;
     lexical;
     operationTail = Promise.resolve();
     lexicalRebuildRequired;
-    constructor(path = ":memory:") {
+    embedderConfig;
+    constructor(path = ":memory:", embedderConfig) {
+        this.embedderConfig = embedderConfig;
         this.db = new Database(path);
         this.db.pragma("journal_mode = WAL");
         this.db.pragma("foreign_keys = ON");
@@ -38,6 +40,19 @@ export class Archive {
     async sync(messages, policy = {}) {
         return this.runExclusive(() => this.syncUnlocked(messages, policy));
     }
+    status() {
+        const count = (table) => Number(this.db.prepare(`SELECT COUNT(*) AS count FROM ${table}`).get().count);
+        const ftsRows = Number(this.db.prepare("SELECT COUNT(*) AS count FROM chunks_fts").get().count);
+        const metadata = this.db.prepare("SELECT version FROM lexical_index_meta WHERE id = 1").get();
+        const chunkCount = count("chunks");
+        return {
+            messageCount: count("messages"),
+            chunkCount,
+            embeddingBacklog: Number(this.db.prepare("SELECT COUNT(*) AS count FROM embedding_queue WHERE state = 'pending'").get().count),
+            archiveRevision: this.revision(),
+            fts: { status: metadata?.version === LEXICAL_ANALYZER_VERSION && ftsRows === chunkCount ? "healthy" : "stale", rows: ftsRows },
+        };
+    }
     async syncUnlocked(messages, policy = {}) {
         for (const message of messages) {
             if (typeof message.providerKey !== "string" || !message.providerKey.trim())
@@ -160,7 +175,7 @@ export class Archive {
         const transaction = this.db.transaction(() => {
             for (const row of rows) {
                 const old = this.db.prepare("SELECT content_hash, model FROM semantic_vectors WHERE chunk_id = ?").get(row.chunk_id);
-                if (old?.content_hash === row.content_hash && old.model === embeddingModelName()) {
+                if (old?.content_hash === row.content_hash && old.model === embeddingModelName(this.embedderConfig)) {
                     completeQueueRow.run(row.chunk_id);
                     reused++;
                     continue;
@@ -175,7 +190,7 @@ export class Archive {
         const vectors = await embedder.embedDocuments(pending.map((row) => row.text));
         const write = this.db.transaction(() => {
             for (const [index, row] of pending.entries()) {
-                upsert.run(row.chunk_id, row.content_hash, embeddingModelName(), JSON.stringify(vectors[index]));
+                upsert.run(row.chunk_id, row.content_hash, embeddingModelName(this.embedderConfig), JSON.stringify(vectors[index]));
                 completeQueueRow.run(row.chunk_id);
                 embedded++;
             }
@@ -192,20 +207,22 @@ export class Archive {
         const previousVectors = this.db.prepare("SELECT chunk_id, content_hash, model, vector FROM semantic_vectors ORDER BY chunk_id").all();
         const previousQueue = this.db.prepare("SELECT chunk_id, content_hash, state, attempts FROM embedding_queue ORDER BY chunk_id").all();
         mkdirSync(generationRoot, { recursive: true });
-        const generation = `gen-${this.revision().slice(0, 16)}-${Date.now()}`;
+        const generation = `gen-${this.revision().slice(0, 16)}-${Date.now()}-${randomBytes(6).toString("hex")}`;
         const staging = join(generationRoot, `.${generation}.staging`);
         const publishedGeneration = join(generationRoot, generation);
+        const pointer = join(root, `.CURRENT.${process.pid}`);
         mkdirSync(staging);
+        let published = false;
         try {
             const report = await this.indexSemanticUnlocked();
             const vectors = this.db.prepare(`SELECT v.chunk_id, v.content_hash, v.vector
         FROM semantic_vectors v JOIN chunks c ON c.chunk_id = v.chunk_id
         ORDER BY v.chunk_id`).all();
             writeFileSync(join(staging, "manifest.json"), JSON.stringify({
-                archiveRevision: this.revision(), vectors, model: embeddingModelName(),
+                archiveRevision: this.revision(), vectors, model: embeddingModelName(this.embedderConfig),
             }));
             renameSync(staging, publishedGeneration);
-            const pointer = join(root, `.CURRENT.${process.pid}`);
+            published = true;
             writeFileSync(pointer, `${generation}\n`);
             renameSync(pointer, currentPath);
             this.cleanupSemanticGenerations(generationRoot, generation);
@@ -213,7 +230,9 @@ export class Archive {
         }
         catch (error) {
             rmSync(staging, { recursive: true, force: true });
-            rmSync(publishedGeneration, { recursive: true, force: true });
+            rmSync(pointer, { force: true });
+            if (!published)
+                rmSync(publishedGeneration, { recursive: true, force: true });
             const restore = this.db.transaction(() => {
                 this.db.exec("DELETE FROM semantic_vectors; DELETE FROM embedding_queue;");
                 const restoreVector = this.db.prepare("INSERT INTO semantic_vectors(chunk_id, content_hash, model, vector) VALUES (?, ?, ?, ?)");
@@ -232,8 +251,8 @@ export class Archive {
             .filter((entry) => entry.isDirectory() && SEMANTIC_GENERATION_NAME.test(entry.name))
             .map((entry) => entry.name)
             .sort((left, right) => {
-            const leftTimestamp = Number(left.slice(left.lastIndexOf("-") + 1));
-            const rightTimestamp = Number(right.slice(right.lastIndexOf("-") + 1));
+            const leftTimestamp = this.generationTimestamp(left);
+            const rightTimestamp = this.generationTimestamp(right);
             return rightTimestamp - leftTimestamp || right.localeCompare(left);
         });
         const retained = new Set(generations.slice(0, RETAINED_SEMANTIC_GENERATIONS));
@@ -243,8 +262,14 @@ export class Archive {
                 rmSync(join(generationRoot, generation), { recursive: true, force: true });
         }
     }
+    generationTimestamp(name) {
+        const match = /^gen-[0-9a-f]{16}-([0-9]+)(?:-[0-9a-f]+)?$/.exec(name);
+        return match ? Number(match[1]) : 0;
+    }
     semanticGeneration(root) {
         const generation = readFileSync(join(root, "CURRENT"), "utf8").trim();
+        if (!SEMANTIC_GENERATION_NAME.test(generation))
+            throw new Error("invalid semantic generation pointer");
         const manifest = JSON.parse(readFileSync(join(root, "generations", generation, "manifest.json"), "utf8"));
         return { generation, archiveRevision: manifest.archiveRevision, vectorCount: manifest.vectors.length };
     }
@@ -334,8 +359,6 @@ export class Archive {
         const rows = this.db.prepare("SELECT rowid, chunk_id, text, message_id FROM chunks").all();
         const rebuild = this.db.transaction(() => {
             this.db.exec("DELETE FROM chunks_fts");
-            for (const language of lexicalFields())
-                this.db.exec(`DELETE FROM chunks_fts_${language}`);
             for (const row of rows) {
                 const message = this.db.prepare("SELECT * FROM messages WHERE message_id = ?").get(row.message_id);
                 this.db.prepare(`INSERT INTO chunks_fts(rowid, subject, from_address, to_addresses, thread_subject,
@@ -344,8 +367,6 @@ export class Archive {
             }
         });
         rebuild();
-        this.db.prepare("DELETE FROM lexical_index_meta WHERE id = 1").run();
-        this.lexicalRebuildRequired = true;
         return { rows: rows.length, status: "repaired" };
     }
     upsertMessage(message) {
@@ -428,7 +449,7 @@ export class Archive {
         }
     }
     async getEmbedder() {
-        this.embedder ??= await createEmbedder();
+        this.embedder ??= await createEmbedder(this.embedderConfig);
         return this.embedder;
     }
     searchLexicalTable(table, query, filters, limit) {

package/dist/cli/index.js

@@ -2,9 +2,11 @@
 import { Command } from "commander";
 import { mkdir } from "node:fs/promises";
 import { existsSync } from "node:fs";
+import { join } from "node:path";
 import { Archive } from "../archive.js";
 import { FixtureSource, HimalayaSource } from "../source.js";
 import { redactDiagnostic } from "../redact.js";
+import { embeddingModelName, loopbackConfigFromEnvironment } from "../embedding.js";
 const program = new Command();
 program.name("mailcrawl").description("Local privacy-first email indexing CLI");
 program.option("--data-dir <path>", "archive directory", process.env.MAILCRAWL_DATA_DIR || ".mailcrawl");
@@ -16,21 +18,32 @@ program
     .option("--mailbox <name>", "mailbox name", "INBOX")
     .option("--backend <name>")
     .option("--page-size <n>", "envelopes per page", "1000")
+    .option("--concurrency <n>", "simultaneous message reads (default 4)")
     .option("--himalaya-config <path>")
     .option("--include-category <name>", "include a normally excluded category", collect, [])
     .option("--exclude-category <name>", "exclude a classification category", collect, [])
     .option("--json")
     .action(async (options, command) => {
     const dataDir = command.parent.opts().dataDir;
+    if (options.source !== "fixture" && options.source !== "himalaya")
+        throw new Error(`unsupported source: ${options.source}`);
+    if (options.source === "fixture" && !options.fixture)
+        throw new Error("--fixture is required for fixture source");
+    if (options.source === "himalaya" && !options.account)
+        throw new Error("--account is required for himalaya source");
     await mkdir(dataDir, { recursive: true });
-    const archive = new Archive(`${dataDir}/archive.sqlite`);
+    const archive = new Archive(join(dataDir, "archive.sqlite"));
     try {
         const source = options.source === "fixture"
-            ? new FixtureSource(options.fixture)
-            : new HimalayaSource(options.account, options.mailbox, options.backend, Number(options.pageSize), options.himalayaConfig);
+            ? fixtureSource(options)
+            : himalayaSource(options);
         const excludedCategories = (options.excludeCategory.length ? options.excludeCategory : ["spam", "promotions"])
             .filter((category) => !options.includeCategory.includes(category));
-        output(await archive.sync(await source.list(), { excludedCategories }), options.json);
+        const { messages, failures } = await source.collect();
+        if (messages.length === 0 && failures.length > 0) {
+            throw new Error(`source read failed for all ${failures.length} message(s); no messages could be read: ${failures.slice(0, 3).map((failure) => failure.providerKey).join(", ")}`);
+        }
+        output({ ...await archive.sync(messages, { excludedCategories }), failures }, options.json);
     }
     finally {
         archive.close();
@@ -38,12 +51,21 @@ program
 });
 program
     .command("index")
+    .alias("embed")
+    .option("--provider <provider>", "embedding provider", "local")
+    .option("--embed-url <url>")
+    .option("--embed-model <model>")
+    .option("--embed-dim <n>")
+    .option("--query-prefix <prefix>")
+    .option("--passage-prefix <prefix>")
+    .option("--embed-timeout <ms>")
     .option("--json")
     .action(async (options, command) => {
     const dataDir = command.parent.opts().dataDir;
-    const archive = new Archive(`${dataDir}/archive.sqlite`);
+    const config = embedderConfig(options);
+    const archive = new Archive(join(dataDir, "archive.sqlite"), config);
     try {
-        output({ ...await archive.indexSemanticGeneration(`${dataDir}/semantic`), embedder: "onnx-community/embeddinggemma-300m-ONNX" }, options.json);
+        output({ ...await archive.indexSemanticGeneration(join(dataDir, "semantic")), embedder: embeddingModelName(config) }, options.json);
     }
     finally {
         archive.close();
@@ -61,12 +83,19 @@ for (const mode of ["bm25", "keyword", "semantic", "hybrid"]) {
         .option("--after <date>")
         .option("--before <date>")
         .option("--limit <n>", "result limit", "10")
+        .option("--provider <provider>", "embedding provider", "local")
+        .option("--embed-url <url>")
+        .option("--embed-model <model>")
+        .option("--embed-dim <n>")
+        .option("--query-prefix <prefix>")
+        .option("--passage-prefix <prefix>")
+        .option("--embed-timeout <ms>")
         .option("--json")
         .action(async (query, options, command) => {
         const dataDir = command.parent.opts().dataDir;
-        const archive = new Archive(`${dataDir}/archive.sqlite`);
+        const archive = new Archive(join(dataDir, "archive.sqlite"), embedderConfig(options));
         try {
-            const result = mode === "bm25"
+            const result = mode === "bm25" || mode === "keyword"
                 ? await archive.searchBm25(query, filters(options), Number(options.limit))
                 : mode === "hybrid"
                     ? await archive.searchHybrid(query, filters(options), Number(options.limit))
@@ -81,7 +110,7 @@ for (const mode of ["bm25", "keyword", "semantic", "hybrid"]) {
 program
     .command("search")
     .argument("<query>")
-    .option("--mode <mode>", "keyword, bm25, semantic, hybrid", "bm25")
+    .option("--mode <mode>", "fts, keyword, bm25, semantic, hybrid", "bm25")
     .option("--account <id>")
     .option("--mailbox <name>")
     .option("--from <address>")
@@ -90,13 +119,20 @@ program
     .option("--after <date>")
     .option("--before <date>")
     .option("--limit <n>", "result limit", "10")
+    .option("--provider <provider>", "embedding provider", "local")
+    .option("--embed-url <url>")
+    .option("--embed-model <model>")
+    .option("--embed-dim <n>")
+    .option("--query-prefix <prefix>")
+    .option("--passage-prefix <prefix>")
+    .option("--embed-timeout <ms>")
     .option("--json")
     .action(async (query, options, command) => {
-    const archive = new Archive(`${command.parent.opts().dataDir}/archive.sqlite`);
+    const archive = new Archive(join(command.parent.opts().dataDir, "archive.sqlite"), embedderConfig(options));
     try {
         const filter = filters(options);
         const limit = Number(options.limit);
-        const result = options.mode === "bm25" ? await archive.searchBm25(query, filter, limit)
+        const result = options.mode === "bm25" || options.mode === "fts" || options.mode === "keyword" ? await archive.searchBm25(query, filter, limit)
             : options.mode === "semantic" ? await archive.searchSemantic(query, filter, limit)
                 : options.mode === "hybrid" ? await archive.searchHybrid(query, filter, limit)
                     : (() => { throw new Error(`unsupported search mode: ${options.mode}`); })();
@@ -111,7 +147,7 @@ program
     .command("get <messageId>")
     .option("--json")
     .action(async (messageId, options, command) => {
-    const archive = new Archive(`${command.parent.parent.opts().dataDir}/archive.sqlite`);
+    const archive = new Archive(join(command.parent.parent.opts().dataDir, "archive.sqlite"));
     try {
         output(archive.getMessage(messageId) ?? null, options.json);
     }
@@ -124,7 +160,7 @@ program
     .argument("<chunkId>")
     .option("--json")
     .action(async (chunkId, options, command) => {
-    const archive = new Archive(`${command.parent.opts().dataDir}/archive.sqlite`);
+    const archive = new Archive(join(command.parent.opts().dataDir, "archive.sqlite"));
     try {
         output(archive.getChunkContext(chunkId), options.json);
     }
@@ -141,7 +177,7 @@ program
     .option("--before <date>")
     .option("--json")
     .action(async (threadId, options, command) => {
-    const archive = new Archive(`${command.parent.parent.opts().dataDir}/archive.sqlite`);
+    const archive = new Archive(join(command.parent.parent.opts().dataDir, "archive.sqlite"));
     try {
         output(archive.getThread(threadId, filters(options)), options.json);
     }
@@ -155,7 +191,7 @@ program
     .option("--message <messageId>")
     .option("--json")
     .action(async (threadId, options, command) => {
-    const archive = new Archive(`${command.parent.opts().dataDir}/archive.sqlite`);
+    const archive = new Archive(join(command.parent.opts().dataDir, "archive.sqlite"));
     try {
         output(archive.getThreadContext(threadId, options.message), options.json);
     }
@@ -168,21 +204,21 @@ program
     .option("--json")
     .action(async (options, command) => {
     const dataDir = command.parent.opts().dataDir;
-    const archive = new Archive(`${dataDir}/archive.sqlite`);
+    const archive = new Archive(join(dataDir, "archive.sqlite"));
     try {
         let semantic = "missing";
         try {
-            semantic = archive.semanticGeneration(`${dataDir}/semantic`);
+            semantic = semanticStatus(archive, archive.semanticGeneration(join(dataDir, "semantic")));
         }
         catch (error) {
-            semantic = redactDiagnostic({ status: "stale", error: error instanceof Error ? error.message : String(error) });
+            semantic = redactDiagnostic({ status: semanticErrorStatus(error), error: error instanceof Error ? error.message : String(error) });
         }
         const semanticCommitted = typeof semantic === "object" && semantic !== null && "generation" in semantic;
         output({
             name: "mailcrawl",
-            archive: `${dataDir}/archive.sqlite`,
-            archivePresent: existsSync(`${dataDir}/archive.sqlite`),
-            fts: "available",
+            archive: join(dataDir, "archive.sqlite"),
+            archivePresent: existsSync(join(dataDir, "archive.sqlite")),
+            fts: archive.status().fts,
             semantic,
             recommendation: semanticCommitted ? "semantic index is committed" : "run sync, then index before semantic search",
         }, options.json);
@@ -191,16 +227,65 @@ program
         archive.close();
     }
 });
+program
+    .command("status")
+    .option("--json")
+    .action(async (options, command) => {
+    const dataDir = command.parent.opts().dataDir;
+    const archivePath = join(dataDir, "archive.sqlite");
+    if (!existsSync(archivePath)) {
+        output({ name: "mailcrawl", archive: archivePath, archivePresent: false, messageCount: 0, chunkCount: 0, embeddingBacklog: 0, fts: { status: "missing", rows: 0 }, semantic: { status: "missing" } }, options.json);
+        return;
+    }
+    const archive = new Archive(archivePath);
+    try {
+        let semantic = "missing";
+        try {
+            semantic = semanticStatus(archive, archive.semanticGeneration(join(dataDir, "semantic")));
+        }
+        catch (error) {
+            semantic = redactDiagnostic({ status: semanticErrorStatus(error), error: error instanceof Error ? error.message : String(error) });
+        }
+        output({ name: "mailcrawl", archive: archivePath, archivePresent: true, ...archive.status(), semantic }, options.json);
+    }
+    finally {
+        archive.close();
+    }
+});
 program
     .command("repair")
     .option("--fts")
+    .option("--semantic")
+    .option("--all")
+    .option("--provider <provider>", "embedding provider", "local")
+    .option("--embed-url <url>")
+    .option("--embed-model <model>")
+    .option("--embed-dim <n>")
+    .option("--query-prefix <prefix>")
+    .option("--passage-prefix <prefix>")
+    .option("--embed-timeout <ms>")
     .option("--json")
     .action(async (options, command) => {
-    const archive = new Archive(`${command.parent.opts().dataDir}/archive.sqlite`);
+    const dataDir = command.parent.opts().dataDir;
+    const archive = new Archive(join(dataDir, "archive.sqlite"));
     try {
-        if (!options.fts)
-            throw new Error("pass --fts");
-        output(archive.repairFts(), options.json);
+        if (!options.fts && !options.semantic && !options.all)
+            throw new Error("pass --fts, --semantic, or --all");
+        const result = {};
+        if (options.semantic || options.all) {
+            const config = embedderConfig(options);
+            const semanticArchive = config ? new Archive(join(dataDir, "archive.sqlite"), config) : archive;
+            try {
+                result.semantic = await semanticArchive.indexSemanticGeneration(join(dataDir, "semantic"));
+            }
+            finally {
+                if (semanticArchive !== archive)
+                    semanticArchive.close();
+            }
+        }
+        if (options.fts || options.all)
+            result.fts = archive.repairFts();
+        output(Object.keys(result).length === 1 ? Object.values(result)[0] : result, options.json);
     }
     finally {
         archive.close();
@@ -212,7 +297,7 @@ attachments
     .option("--message <messageId>")
     .option("--json")
     .action(async (options, command) => {
-    const archive = new Archive(`${command.parent.parent.opts().dataDir}/archive.sqlite`);
+    const archive = new Archive(join(command.parent.parent.opts().dataDir, "archive.sqlite"));
     try {
         output(archive.listAttachments(options.message), options.json);
     }
@@ -222,12 +307,48 @@ attachments
 });
 program.parseAsync().catch((error) => {
     const message = error instanceof Error ? error.message : String(error);
-    console.error(JSON.stringify({ error: message }));
+    console.error(JSON.stringify(redactDiagnostic({ error: message })));
     process.exitCode = 1;
 });
 function filters(options) {
     return { accountId: options.account, mailbox: options.mailbox, from: options.from, to: options.to, threadId: options.thread, after: options.after, before: options.before };
 }
+function embedderConfig(options) {
+    if (options.provider === "local")
+        return loopbackConfigFromEnvironment();
+    if (options.provider !== "loopback-http")
+        throw new Error(`unsupported embedding provider: ${options.provider}`);
+    if (!options.embedUrl || !options.embedModel || !options.embedDim)
+        throw new Error("--embed-url, --embed-model, and --embed-dim are required for loopback-http");
+    return { provider: "loopback-http", url: options.embedUrl, model: options.embedModel, dimension: Number(options.embedDim), queryPrefix: options.queryPrefix, passagePrefix: options.passagePrefix, timeoutMs: options.embedTimeout ? Number(options.embedTimeout) : undefined };
+}
+function fixtureSource(options) {
+    if (!options.fixture)
+        throw new Error("--fixture is required for selected source");
+    return new FixtureSource(options.fixture);
+}
+function himalayaSource(options) {
+    if (!options.account)
+        throw new Error("--account is required for selected source");
+    return new HimalayaSource(options.account, options.mailbox, options.backend, Number(options.pageSize), options.himalayaConfig, {
+        concurrency: readConcurrency(options),
+    });
+}
+function readConcurrency(options) {
+    if (options.concurrency === undefined)
+        return undefined;
+    const concurrency = Number(options.concurrency);
+    if (!Number.isInteger(concurrency) || concurrency < 1) {
+        throw new Error(`--concurrency must be a positive integer: ${options.concurrency}`);
+    }
+    return concurrency;
+}
+function semanticStatus(archive, semantic) {
+    return semantic.archiveRevision === archive.status().archiveRevision ? { ...semantic, status: "healthy" } : { ...semantic, status: "stale" };
+}
+function semanticErrorStatus(error) {
+    return error instanceof Error && "code" in error && error.code === "ENOENT" ? "missing" : "corrupt";
+}
 function output(value, json) {
     if (json)
         console.log(JSON.stringify(value));

package/dist/embedding.d.ts

@@ -1,6 +1,8 @@
+import type { LoopbackHttpConfig } from "./types.js";
 export interface Embedder {
     embedDocuments(texts: string[]): Promise<number[][]>;
     embedQuery(query: string): Promise<number[]>;
 }
-export declare function createEmbedder(): Promise<Embedder>;
-export declare function embeddingModelName(): string;
+export declare function createEmbedder(config?: LoopbackHttpConfig): Promise<Embedder>;
+export declare function embeddingModelName(config?: LoopbackHttpConfig): string;
+export declare function loopbackConfigFromEnvironment(): LoopbackHttpConfig | undefined;

package/dist/embedding.js

@@ -33,13 +33,76 @@ class TestEmbedder {
         return hashVector(query);
     }
 }
-export async function createEmbedder() {
+class LoopbackHttpEmbedder {
+    config;
+    constructor(config) {
+        this.config = config;
+        const url = new URL(config.url);
+        const hostname = url.hostname.startsWith("[") ? url.hostname.slice(1, -1) : url.hostname;
+        if (url.protocol !== "http:" || !["127.0.0.1", "localhost", "::1"].includes(hostname)) {
+            throw new Error("loopback HTTP embedding URL is required");
+        }
+        if (!Number.isInteger(config.dimension) || config.dimension <= 0)
+            throw new Error("embedding dimension must be positive");
+    }
+    async embedDocuments(texts) {
+        return this.request(texts.map((text) => (this.config.passagePrefix ?? "") + text));
+    }
+    async embedQuery(query) {
+        return (await this.request([(this.config.queryPrefix ?? "") + query.trim()]))[0];
+    }
+    async request(texts) {
+        const controller = new AbortController();
+        const timeout = setTimeout(() => controller.abort(), this.config.timeoutMs ?? 30_000);
+        try {
+            const response = await fetch(this.config.url, {
+                method: "POST",
+                headers: { "content-type": "application/json" },
+                body: JSON.stringify({ model: this.config.model, texts }),
+                signal: controller.signal,
+            });
+            if (!response.ok)
+                throw new Error(`embedding provider returned HTTP ${response.status}`);
+            const payload = await response.json();
+            if (!Array.isArray(payload.embeddings) || payload.embeddings.length !== texts.length)
+                throw new Error("embedding provider returned invalid embeddings");
+            const vectors = payload.embeddings;
+            if (vectors.some((vector) => !Array.isArray(vector) || vector.length !== this.config.dimension || vector.some((value) => typeof value !== "number"))) {
+                throw new Error("embedding provider returned invalid vector dimensions");
+            }
+            return vectors;
+        }
+        finally {
+            clearTimeout(timeout);
+        }
+    }
+}
+export async function createEmbedder(config) {
+    if (config?.provider === "loopback-http")
+        return new LoopbackHttpEmbedder(config);
     if (process.env.MAILCRAWL_EMBEDDER === "mock" || process.env.NODE_ENV === "test")
         return new TestEmbedder();
     return EmbeddingGemma.create();
 }
-export function embeddingModelName() {
-    return EMBEDDING_MODEL;
+export function embeddingModelName(config) {
+    return config
+        ? `loopback-http:${config.model}:${config.dimension}:${config.url}:${config.queryPrefix ?? ""}:${config.passagePrefix ?? ""}:${config.timeoutMs ?? 30_000}`
+        : EMBEDDING_MODEL;
+}
+export function loopbackConfigFromEnvironment() {
+    if (process.env.MAILCRAWL_EMBEDDER_PROVIDER !== "loopback-http")
+        return undefined;
+    const url = process.env.MAILCRAWL_EMBED_URL;
+    const model = process.env.MAILCRAWL_EMBED_MODEL;
+    const dimension = Number(process.env.MAILCRAWL_EMBED_DIM);
+    if (!url || !model || !Number.isInteger(dimension))
+        throw new Error("MAILCRAWL_EMBED_URL, MAILCRAWL_EMBED_MODEL, and MAILCRAWL_EMBED_DIM are required");
+    return {
+        provider: "loopback-http", url, model, dimension,
+        queryPrefix: process.env.MAILCRAWL_QUERY_PREFIX,
+        passagePrefix: process.env.MAILCRAWL_PASSAGE_PREFIX,
+        timeoutMs: process.env.MAILCRAWL_EMBED_TIMEOUT ? Number(process.env.MAILCRAWL_EMBED_TIMEOUT) : undefined,
+    };
 }
 function hashVector(text) {
     const vector = new Array(128).fill(0);

package/dist/redact.js

@@ -2,6 +2,7 @@ export function redactDiagnostic(value) {
     if (typeof value === "string") {
         return value
             .replace(/([?&](?:token|password|secret|key)=)[^&\s]+/giu, "$1[REDACTED]")
+            .replace(/(?<![?&])\b(?:token|password|secret|key|credential)\s*[=:]\s*[^,\s]+/giu, "[REDACTED]")
             .replace(/\b[A-Z0-9._%+-]+@[A-Z0-9.-]+\.[A-Z]{2,}\b/giu, "[EMAIL]");
     }
     if (Array.isArray(value))

package/dist/source.d.ts

@@ -1,10 +1,34 @@
-import type { MailMessage } from "./types.js";
+import type { MailMessage, SourceReadResult } from "./types.js";
+/** Default number of simultaneous `himalaya message read` processes. */
+export declare const DEFAULT_READ_CONCURRENCY = 4;
+/** Default number of attempts per message read, including the first one. */
+export declare const DEFAULT_RETRY_ATTEMPTS = 3;
+/** Default backoff before the second read attempt; doubles per round. */
+export declare const DEFAULT_RETRY_BASE_DELAY_MS = 250;
+/** Executes one himalaya invocation; replaced by tests. */
+export type HimalayaExec = (args: string[], maxBuffer: number) => Promise<{
+    stdout: string;
+    stderr?: string;
+}>;
+export interface HimalayaReadOptions {
+    /** Simultaneous message reads; defaults to DEFAULT_READ_CONCURRENCY. */
+    concurrency?: number;
+    /** Attempts per message read, including the first one; defaults to DEFAULT_RETRY_ATTEMPTS. */
+    retryAttempts?: number;
+    /** Backoff before the second attempt, doubling per round; defaults to DEFAULT_RETRY_BASE_DELAY_MS. */
+    retryBaseDelayMs?: number;
+    exec?: HimalayaExec;
+}
 export interface MailSource {
+    /** Messages that could be read, plus one record per message that could not. */
+    collect(): Promise<SourceReadResult>;
+    /** Strict variant of `collect()`: rejects when any message could not be read. */
     list(): Promise<MailMessage[]>;
 }
 export declare class FixtureSource implements MailSource {
     private readonly path;
     constructor(path: string);
+    collect(): Promise<SourceReadResult>;
     list(): Promise<MailMessage[]>;
 }
 export declare class HimalayaSource implements MailSource {
@@ -13,7 +37,17 @@ export declare class HimalayaSource implements MailSource {
     private readonly backend?;
     private readonly pageSize;
     private readonly config?;
-    constructor(account: string, mailbox?: string, backend?: string | undefined, pageSize?: number, config?: string | undefined);
+    private readonly readOptions;
+    constructor(account: string, mailbox?: string, backend?: string | undefined, pageSize?: number, config?: string | undefined, readOptions?: HimalayaReadOptions);
+    collect(): Promise<SourceReadResult>;
     list(): Promise<MailMessage[]>;
+    /** Reads a page through a bounded pool, retrying failed reads in later rounds. */
+    private readPage;
+    private get concurrency();
+    private envelopes;
+    private baseArgs;
     private read;
+    private run;
 }
+/** Runs `worker` over `items` with at most `limit` workers in flight, preserving input order. */
+export declare function mapWithConcurrency<T, R>(items: readonly T[], limit: number, worker: (item: T) => Promise<R>): Promise<R[]>;

package/dist/source.js

@@ -1,12 +1,22 @@
 import { execFile } from "node:child_process";
 import { promisify } from "node:util";
 import { readFile } from "node:fs/promises";
+import { redactDiagnostic } from "./redact.js";
 const execFileAsync = promisify(execFile);
+/** Default number of simultaneous `himalaya message read` processes. */
+export const DEFAULT_READ_CONCURRENCY = 4;
+/** Default number of attempts per message read, including the first one. */
+export const DEFAULT_RETRY_ATTEMPTS = 3;
+/** Default backoff before the second read attempt; doubles per round. */
+export const DEFAULT_RETRY_BASE_DELAY_MS = 250;
 export class FixtureSource {
     path;
     constructor(path) {
         this.path = path;
     }
+    async collect() {
+        return { messages: await this.list(), failures: [] };
+    }
     async list() {
         const raw = await readFile(this.path, "utf8");
         return JSON.parse(raw);
@@ -18,53 +28,155 @@ export class HimalayaSource {
     backend;
     pageSize;
     config;
-    constructor(account, mailbox = "INBOX", backend, pageSize = 1000, config) {
+    readOptions;
+    constructor(account, mailbox = "INBOX", backend, pageSize = 1000, config, readOptions = {}) {
         this.account = account;
         this.mailbox = mailbox;
         this.backend = backend;
         this.pageSize = pageSize;
         this.config = config;
+        this.readOptions = readOptions;
+    }
+    async collect() {
+        return this.readPage(await this.envelopes());
     }
     async list() {
-        const args = this.config ? ["-c", this.config, "-a", this.account] : ["-a", this.account];
-        if (this.backend)
-            args.push("-b", this.backend);
+        const { messages, failures } = await this.collect();
+        if (failures.length > 0)
+            throw new Error(failures[0].error);
+        return messages;
+    }
+    /** Reads a page through a bounded pool, retrying failed reads in later rounds. */
+    async readPage(envelopes) {
+        const keys = envelopes.map(envelopeKey);
+        const rawMime = new Map();
+        const errors = new Map();
+        const attempts = Math.max(1, Math.floor(this.readOptions.retryAttempts ?? DEFAULT_RETRY_ATTEMPTS));
+        const baseDelayMs = Math.max(0, this.readOptions.retryBaseDelayMs ?? DEFAULT_RETRY_BASE_DELAY_MS);
+        let pending = envelopes.map((_, index) => index);
+        for (let attempt = 1; attempt <= attempts && pending.length > 0; attempt += 1) {
+            if (attempt > 1)
+                await delay(baseDelayMs * 2 ** (attempt - 2));
+            const failed = [];
+            await mapWithConcurrency(pending, this.concurrency, async (index) => {
+                try {
+                    rawMime.set(index, await this.read(keys[index]));
+                }
+                catch (error) {
+                    errors.set(index, error instanceof Error ? error : new Error(String(error)));
+                    failed.push(index);
+                }
+            });
+            pending = failed;
+        }
+        const messages = [];
+        const failures = [];
+        envelopes.forEach((envelope, index) => {
+            const raw = rawMime.get(index);
+            if (raw !== undefined) {
+                messages.push(envelopeMessage(this.account, this.mailbox, envelope, keys[index], raw));
+                return;
+            }
+            failures.push({ providerKey: keys[index], attempts, error: (errors.get(index) ?? new Error("message read failed")).message });
+        });
+        return { messages, failures };
+    }
+    get concurrency() {
+        return Math.max(1, Math.floor(this.readOptions.concurrency ?? DEFAULT_READ_CONCURRENCY));
+    }
+    async envelopes() {
+        const args = this.baseArgs();
         args.push("envelope", "list", "--mailbox", this.mailbox, "--page-size", String(this.pageSize), "--json");
-        const { stdout } = await execFileAsync("himalaya", args, { maxBuffer: 16 * 1024 * 1024 });
+        const { stdout } = await this.run(args, 16 * 1024 * 1024, "envelope list");
         const payload = JSON.parse(stdout);
-        const envelopes = payload.envelopes ?? (Array.isArray(payload) ? payload : []);
-        return Promise.all(envelopes.map(async (envelope) => {
-            const providerKey = String(envelope.id ?? envelope.uid ?? envelope["message-id"]);
-            const rawMime = await this.read(providerKey);
-            return {
-                accountId: this.account,
-                mailbox: this.mailbox,
-                providerKey,
-                messageId: envelope["message-id"],
-                inReplyTo: envelope["in-reply-to"]?.[0],
-                subject: envelope.subject ?? "",
-                from: address(envelope.from),
-                to: addresses(envelope.to),
-                cc: addresses(envelope.cc),
-                date: envelope.date ?? new Date(0).toISOString(),
-                text: envelope.body ?? envelope.snippet ?? "",
-                labels: strings(envelope.labels),
-                flags: strings(envelope.flags),
-                classifications: strings(envelope.classifications),
-                rawMime,
-            };
-        }));
+        return payload.envelopes ?? (Array.isArray(payload) ? payload : []);
     }
-    async read(id) {
+    baseArgs() {
         const args = this.config ? ["-c", this.config, "-a", this.account] : ["-a", this.account];
         if (this.backend)
             args.push("-b", this.backend);
+        return args;
+    }
+    async read(id) {
+        const args = this.baseArgs();
         args.push("--json", "message", "read", id, "--raw");
-        const { stdout } = await execFileAsync("himalaya", args, { maxBuffer: 32 * 1024 * 1024 });
+        const { stdout } = await this.run(args, 32 * 1024 * 1024, "message read");
         const payload = JSON.parse(stdout);
         return payload.message ?? stdout;
     }
+    run(args, maxBuffer, operation) {
+        return runHimalaya(args, maxBuffer, operation, this.readOptions.exec);
+    }
+}
+/** Runs `worker` over `items` with at most `limit` workers in flight, preserving input order. */
+export async function mapWithConcurrency(items, limit, worker) {
+    const results = new Array(items.length);
+    let cursor = 0;
+    let stopped = false;
+    const width = Math.max(1, Math.min(Math.floor(limit) || 1, items.length));
+    await Promise.all(Array.from({ length: width }, async () => {
+        while (!stopped) {
+            const index = cursor++;
+            if (index >= items.length)
+                return;
+            try {
+                results[index] = await worker(items[index]);
+            }
+            catch (error) {
+                // A failed page must not keep spawning provider processes for the
+                // items that were never read; only the in-flight workers finish.
+                stopped = true;
+                throw error;
+            }
+        }
+    }));
+    return results;
+}
+function envelopeKey(envelope) {
+    return String(envelope.id ?? envelope.uid ?? envelope["message-id"]);
+}
+function delay(ms) {
+    return new Promise((resolve) => setTimeout(resolve, ms));
+}
+function envelopeMessage(account, mailbox, envelope, providerKey, rawMime) {
+    return {
+        accountId: account,
+        mailbox,
+        providerKey,
+        messageId: envelope["message-id"],
+        inReplyTo: envelope["in-reply-to"]?.[0],
+        subject: envelope.subject ?? "",
+        from: address(envelope.from),
+        to: addresses(envelope.to),
+        cc: addresses(envelope.cc),
+        date: envelope.date ?? new Date(0).toISOString(),
+        text: envelope.body ?? envelope.snippet ?? "",
+        labels: strings(envelope.labels),
+        flags: strings(envelope.flags),
+        classifications: strings(envelope.classifications),
+        rawMime,
+    };
+}
+async function runHimalaya(args, maxBuffer, operation, exec) {
+    try {
+        return await (exec ?? defaultExec)(args, maxBuffer);
+    }
+    catch (error) {
+        const detail = error instanceof Error ? error.message : String(error);
+        throw new Error(`himalaya ${operation} failed: ${redactDiagnostic(detail)}${stderrDetail(error)}`);
+    }
+}
+/** Keeps himalaya's own stderr in the surfaced error so throttling is distinguishable from a malformed message. */
+function stderrDetail(error) {
+    const stderr = error?.stderr;
+    if (typeof stderr !== "string")
+        return "";
+    const text = stderr.trim();
+    if (!text)
+        return "";
+    return `: ${String(redactDiagnostic(text.length > 2_000 ? `${text.slice(0, 2_000)} [truncated]` : text))}`;
 }
+const defaultExec = (args, maxBuffer) => execFileAsync("himalaya", args, { maxBuffer });
 function address(value) {
     if (typeof value === "string")
         return value;

package/dist/types.d.ts

@@ -1,4 +1,14 @@
 export type SearchMode = "keyword" | "bm25" | "semantic" | "hybrid";
+export type EmbedderProvider = "local" | "loopback-http";
+export interface LoopbackHttpConfig {
+    provider: "loopback-http";
+    url: string;
+    model: string;
+    dimension: number;
+    queryPrefix?: string;
+    passagePrefix?: string;
+    timeoutMs?: number;
+}
 export interface MailMessage {
     accountId: string;
     mailbox: string;
@@ -86,6 +96,15 @@ export interface SyncReport {
     excluded: number;
     excludedByReason: Record<string, number>;
 }
+export interface SourceReadFailure {
+    providerKey: string;
+    attempts: number;
+    error: string;
+}
+export interface SourceReadResult {
+    messages: MailMessage[];
+    failures: SourceReadFailure[];
+}
 export interface ClassificationPolicy {
     excludedCategories?: string[];
 }

package/docs/architecture.md

@@ -158,13 +158,19 @@ All commands support `--json` where machine-readable output is useful.
 ```text
 mailcrawl doctor
 mailcrawl status
-mailcrawl sync
-mailcrawl embed [--limit N]
-mailcrawl search --mode fts|semantic|hybrid [--limit N] [--mailbox NAME] QUERY
+mailcrawl sync [--page-size N] [--concurrency N]
+mailcrawl embed
+mailcrawl search --mode fts|bm25|keyword|semantic|hybrid [--limit N] [--mailbox NAME] QUERY
 mailcrawl message get MESSAGE_ID
-mailcrawl repair [--fts|--vectors|--all]
+mailcrawl repair [--fts|--semantic|--all]
 ```
 
+`--page-size` is the IMAP envelope window and `--concurrency` is the number of
+simultaneous message reads (default 4). Reads run through that bounded pool,
+failures are retried with backoff, and the messages that could be read are
+synced even when some reads keep failing, so a throttled provider no longer
+aborts a whole page.
+
 Example sync response:
 
 ```json
@@ -175,10 +181,15 @@ Example sync response:
   "unchanged": 4821,
   "chunksAdded": 21,
   "chunksDeleted": 6,
-  "embeddingBacklog": 21
+  "embeddingBacklog": 21,
+  "failures": []
 }
 ```
 
+Each entry of `failures` carries the `providerKey`, the `attempts` spent on it,
+and the redacted himalaya error (including himalaya's own stderr). The command
+exits non-zero only when nothing could be read.
+
 Example search hit:
 
 ```json

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@nomadamas/mailcrawl",
-  "version": "0.1.5",
+  "version": "0.1.7",
   "description": "Local Himalaya-backed incremental email indexing and hybrid search CLI",
   "license": "MIT",
   "type": "module",
@@ -48,6 +48,7 @@
     "@types/better-sqlite3": "^7.6.13",
     "@types/node": "^24.3.0",
     "tsx": "^4.20.5",
+    "typescript": "^5.9.3",
     "vitest": "^3.2.4"
   },
   "engines": {

package/skills/mailcrawl/SKILL.md

@@ -63,6 +63,13 @@ command, such as a user-level cron or systemd timer. Run `sync` first and
 `index` afterward; inspect JSON exit status before handing results to an
 agent.
 
+Message reads use a bounded pool of himalaya processes (4 by default; change
+it with `--concurrency <n>`) because Gmail throttles accounts that open too
+many simultaneous IMAP connections. `--page-size` only sets the envelope
+window. A read that fails is retried with backoff, and messages that stay
+unreadable appear in the sync JSON `failures[]` while the readable messages
+are still synced; the command exits non-zero only when nothing could be read.
+
 Use a fixture for deterministic development:
 
 ```bash
@@ -71,7 +78,8 @@ mailcrawl sync --source fixture --fixture ./messages.json --json
 
 ## Safe read and maintenance commands
 
-Search modes are `bm25`, `keyword`, `semantic`, and `hybrid`. Use metadata
+Search modes are `fts`, `bm25`, `keyword`, `semantic`, and `hybrid`; `fts` and
+`keyword` are aliases for BM25 lexical search. Use metadata
 filters such as `--mailbox`, `--from`, `--to`, `--thread`, `--after`, and
 `--before`. Empty queries and unsupported modes fail with a non-zero exit.
 
@@ -89,7 +97,11 @@ Check health and rebuild lexical data when needed:
 
 ```bash
 mailcrawl doctor --json
+mailcrawl status --json
+mailcrawl embed --json
 mailcrawl repair --fts --json
+mailcrawl repair --semantic --json
+mailcrawl repair --all --json
 ```
 
 `doctor` reports archive, FTS, and semantic-generation state. `repair` is a