@nomadamas/mailcrawl 0.1.4 → 0.1.6

package/README.md

@@ -20,6 +20,7 @@ mailcrawl embed --json
 mailcrawl search --mode hybrid --json "계약 갱신 조건"
 mailcrawl status --json
 mailcrawl doctor --json
+mailcrawl repair --all --json
 ```
 
 The CLI owns email synchronization and indexes. Consumers such as AutoRAG,
@@ -56,6 +57,21 @@ Chinese helper builds, environment variables, smoke tests, and license
 requirements, follow [`docs/multilingual-installation.md`](docs/multilingual-installation.md)
 before using multilingual indexing or search.
 
+## Shared loopback embedding provider
+
+The local in-process model is the default. Opt into a local HTTP runtime with
+`--provider loopback-http`, `--embed-url`, `--embed-model`, and `--embed-dim`
+on `index`, `repair --semantic`, or semantic search. Only HTTP URLs for
+`127.0.0.1`, `localhost`, or `::1` are accepted. Query/passage prefixes and
+timeout are optional and are included in the provider identity. Any provider
+setting change rebuilds vectors, while a failed rebuild preserves the prior
+`CURRENT` generation.
+
+The equivalent environment contract is
+`MAILCRAWL_EMBEDDER_PROVIDER`, `MAILCRAWL_EMBED_URL`,
+`MAILCRAWL_EMBED_MODEL`, `MAILCRAWL_EMBED_DIM`, `MAILCRAWL_QUERY_PREFIX`,
+`MAILCRAWL_PASSAGE_PREFIX`, and `MAILCRAWL_EMBED_TIMEOUT`.
+
 ## Releasing
 
 GitHub Release `vX.Y.Z` (must match `package.json`) publishes `@nomadamas/mailcrawl` to npm with OIDC trusted publishing. No `NPM_TOKEN` is stored in GitHub.

package/dist/archive.d.ts

@@ -1,24 +1,41 @@
 import Database from "better-sqlite3";
-import type { ClassificationPolicy, Chunk, MailMessage, NormalizedMessage, SearchFilters, SearchHit, SyncReport } from "./types.js";
+import type { ClassificationPolicy, Chunk, LoopbackHttpConfig, MailMessage, NormalizedMessage, SearchFilters, SearchHit, SyncReport } from "./types.js";
 export declare class Archive {
     readonly db: Database.Database;
     private embedder?;
     private lexical?;
+    private operationTail;
     private lexicalRebuildRequired;
-    constructor(path?: string);
+    private readonly embedderConfig?;
+    constructor(path?: string, embedderConfig?: LoopbackHttpConfig);
     close(): void;
     sync(messages: MailMessage[], policy?: ClassificationPolicy): Promise<SyncReport>;
+    status(): {
+        messageCount: number;
+        chunkCount: number;
+        embeddingBacklog: number;
+        archiveRevision: string;
+        fts: {
+            status: "healthy" | "stale";
+            rows: number;
+        };
+    };
+    private syncUnlocked;
     searchBm25(query: string, filters?: SearchFilters, limit?: number): Promise<SearchHit[]>;
     indexSemantic(): Promise<{
         embedded: number;
         reused: number;
         archiveRevision: string;
     }>;
+    private indexSemanticUnlocked;
     indexSemanticGeneration(root: string): Promise<{
         generation: string;
         embedded: number;
         reused: number;
     }>;
+    private indexSemanticGenerationUnlocked;
+    private cleanupSemanticGenerations;
+    private generationTimestamp;
     semanticGeneration(root: string): {
         generation: string;
         archiveRevision: string;
@@ -55,6 +72,7 @@ export declare class Archive {
     private removeMessage;
     private replaceMessageChunks;
     private revision;
+    private runExclusive;
     private getEmbedder;
     private searchLexicalTable;
 }

package/dist/archive.js

@@ -1,22 +1,28 @@
 import Database from "better-sqlite3";
-import { createHash } from "node:crypto";
-import { mkdirSync, readFileSync, renameSync, rmSync, writeFileSync } from "node:fs";
+import { createHash, randomBytes } from "node:crypto";
+import { mkdirSync, readFileSync, readdirSync, renameSync, rmSync, writeFileSync } from "node:fs";
 import { join } from "node:path";
 import { buildChunks } from "./chunk.js";
 import { normalizeMessage } from "./normalize.js";
-import { snippet } from "./util.js";
+import { scopedId, snippet } from "./util.js";
 import { createEmbedder, embeddingModelName } from "./embedding.js";
 import { createLexicalAnalyzers, languagesForText, lexicalFields, LEXICAL_ANALYZER_VERSION } from "./lexical.js";
+const RETAINED_SEMANTIC_GENERATIONS = 2;
+const SEMANTIC_GENERATION_NAME = /^gen-[0-9a-f]{16}-[0-9]+(?:-[0-9a-f]+)?$/;
 export class Archive {
     db;
     embedder;
     lexical;
+    operationTail = Promise.resolve();
     lexicalRebuildRequired;
-    constructor(path = ":memory:") {
+    embedderConfig;
+    constructor(path = ":memory:", embedderConfig) {
+        this.embedderConfig = embedderConfig;
         this.db = new Database(path);
         this.db.pragma("journal_mode = WAL");
         this.db.pragma("foreign_keys = ON");
         migrate(this.db);
+        cleanupOrphanedSemanticRows(this.db);
         this.lexicalRebuildRequired = this.db.prepare("SELECT version FROM lexical_index_meta WHERE id = 1").get() === undefined;
         if (!this.lexicalRebuildRequired) {
             const row = this.db.prepare("SELECT version FROM lexical_index_meta WHERE id = 1").get();
@@ -32,14 +38,36 @@ export class Archive {
         this.db.close();
     }
     async sync(messages, policy = {}) {
+        return this.runExclusive(() => this.syncUnlocked(messages, policy));
+    }
+    status() {
+        const count = (table) => Number(this.db.prepare(`SELECT COUNT(*) AS count FROM ${table}`).get().count);
+        const ftsRows = Number(this.db.prepare("SELECT COUNT(*) AS count FROM chunks_fts").get().count);
+        const metadata = this.db.prepare("SELECT version FROM lexical_index_meta WHERE id = 1").get();
+        const chunkCount = count("chunks");
+        return {
+            messageCount: count("messages"),
+            chunkCount,
+            embeddingBacklog: Number(this.db.prepare("SELECT COUNT(*) AS count FROM embedding_queue WHERE state = 'pending'").get().count),
+            archiveRevision: this.revision(),
+            fts: { status: metadata?.version === LEXICAL_ANALYZER_VERSION && ftsRows === chunkCount ? "healthy" : "stale", rows: ftsRows },
+        };
+    }
+    async syncUnlocked(messages, policy = {}) {
+        for (const message of messages) {
+            if (typeof message.providerKey !== "string" || !message.providerKey.trim())
+                throw new Error("message provider identity is required");
+        }
         const excludedCategories = new Set((policy.excludedCategories ?? ["spam", "promotions"]).map(normalizeCategory));
         const normalized = (await Promise.all(messages.map(normalizeMessage))).map((message) => ({
             ...message,
-            messageId: `${message.accountId}:${message.messageId}`,
-            threadId: `${message.accountId}:${message.threadId}`,
-            providerKey: `${message.accountId}:${message.providerKey}`,
-            inReplyTo: message.inReplyTo ? `${message.accountId}:${message.inReplyTo}` : undefined,
+            messageId: scopedId(message.accountId, message.mailbox, message.messageId),
+            threadId: scopedId(message.accountId, message.mailbox, message.threadId),
+            providerKey: scopedId(message.accountId, message.mailbox, message.providerKey),
+            inReplyTo: message.inReplyTo ? scopedId(message.accountId, message.mailbox, message.inReplyTo) : undefined,
         }));
+        validateIdentities(normalized);
+        validateStoredIdentities(this.db, normalized);
         const excluded = normalized.filter((message) => message.categories.some((category) => excludedCategories.has(category)));
         const included = normalized.filter((message) => !message.categories.some((category) => excludedCategories.has(category)));
         const existing = this.db.prepare("SELECT provider_key, normalized_hash FROM messages").all();
@@ -126,18 +154,29 @@ export class Archive {
             .slice(0, limit).map(({ hit, score }) => ({ ...hit, score }));
     }
     async indexSemantic() {
+        return this.runExclusive(() => this.indexSemanticUnlocked());
+    }
+    async indexSemanticUnlocked() {
+        const completeQueueRow = this.db.prepare("UPDATE embedding_queue SET state = 'complete' WHERE chunk_id = ?");
+        const reconcile = this.db.transaction(() => {
+            this.db.prepare("DELETE FROM embedding_queue WHERE chunk_id NOT IN (SELECT chunk_id FROM chunks)").run();
+            this.db.prepare(`INSERT INTO embedding_queue(chunk_id, content_hash, state, attempts)
+        SELECT chunk_id, content_hash, 'pending', 0 FROM chunks
+        WHERE chunk_id NOT IN (SELECT chunk_id FROM embedding_queue)`).run();
+        });
+        reconcile();
         const rows = this.db.prepare("SELECT chunk_id, text, content_hash FROM chunks ORDER BY chunk_id").all();
         let embedded = 0;
         let reused = 0;
         const upsert = this.db.prepare(`INSERT INTO semantic_vectors
       (chunk_id, content_hash, model, vector) VALUES (?, ?, ?, ?)
       ON CONFLICT(chunk_id) DO UPDATE SET content_hash=excluded.content_hash, model=excluded.model, vector=excluded.vector`);
-        const embedder = await this.getEmbedder();
         const pending = [];
         const transaction = this.db.transaction(() => {
             for (const row of rows) {
                 const old = this.db.prepare("SELECT content_hash, model FROM semantic_vectors WHERE chunk_id = ?").get(row.chunk_id);
-                if (old?.content_hash === row.content_hash && old.model === embeddingModelName()) {
+                if (old?.content_hash === row.content_hash && old.model === embeddingModelName(this.embedderConfig)) {
+                    completeQueueRow.run(row.chunk_id);
                     reused++;
                     continue;
                 }
@@ -147,10 +186,12 @@ export class Archive {
         transaction();
         if (!pending.length)
             return { embedded, reused, archiveRevision: this.revision() };
+        const embedder = await this.getEmbedder();
         const vectors = await embedder.embedDocuments(pending.map((row) => row.text));
         const write = this.db.transaction(() => {
             for (const [index, row] of pending.entries()) {
-                upsert.run(row.chunk_id, row.content_hash, embeddingModelName(), JSON.stringify(vectors[index]));
+                upsert.run(row.chunk_id, row.content_hash, embeddingModelName(this.embedderConfig), JSON.stringify(vectors[index]));
+                completeQueueRow.run(row.chunk_id);
                 embedded++;
             }
         });
@@ -158,31 +199,77 @@ export class Archive {
         return { embedded, reused, archiveRevision: this.revision() };
     }
     async indexSemanticGeneration(root) {
+        return this.runExclusive(() => this.indexSemanticGenerationUnlocked(root));
+    }
+    async indexSemanticGenerationUnlocked(root) {
         const currentPath = join(root, "CURRENT");
         const generationRoot = join(root, "generations");
+        const previousVectors = this.db.prepare("SELECT chunk_id, content_hash, model, vector FROM semantic_vectors ORDER BY chunk_id").all();
+        const previousQueue = this.db.prepare("SELECT chunk_id, content_hash, state, attempts FROM embedding_queue ORDER BY chunk_id").all();
         mkdirSync(generationRoot, { recursive: true });
-        const generation = `gen-${this.revision().slice(0, 16)}-${Date.now()}`;
+        const generation = `gen-${this.revision().slice(0, 16)}-${Date.now()}-${randomBytes(6).toString("hex")}`;
         const staging = join(generationRoot, `.${generation}.staging`);
+        const publishedGeneration = join(generationRoot, generation);
+        const pointer = join(root, `.CURRENT.${process.pid}`);
         mkdirSync(staging);
+        let published = false;
         try {
-            const report = await this.indexSemantic();
-            const vectors = this.db.prepare("SELECT chunk_id, content_hash, vector FROM semantic_vectors ORDER BY chunk_id").all();
+            const report = await this.indexSemanticUnlocked();
+            const vectors = this.db.prepare(`SELECT v.chunk_id, v.content_hash, v.vector
+        FROM semantic_vectors v JOIN chunks c ON c.chunk_id = v.chunk_id
+        ORDER BY v.chunk_id`).all();
             writeFileSync(join(staging, "manifest.json"), JSON.stringify({
-                archiveRevision: this.revision(), vectors, model: embeddingModelName(),
+                archiveRevision: this.revision(), vectors, model: embeddingModelName(this.embedderConfig),
             }));
-            renameSync(staging, join(generationRoot, generation));
-            const pointer = join(root, `.CURRENT.${process.pid}`);
+            renameSync(staging, publishedGeneration);
+            published = true;
             writeFileSync(pointer, `${generation}\n`);
             renameSync(pointer, currentPath);
+            this.cleanupSemanticGenerations(generationRoot, generation);
             return { generation, embedded: report.embedded, reused: report.reused };
         }
         catch (error) {
             rmSync(staging, { recursive: true, force: true });
+            rmSync(pointer, { force: true });
+            if (!published)
+                rmSync(publishedGeneration, { recursive: true, force: true });
+            const restore = this.db.transaction(() => {
+                this.db.exec("DELETE FROM semantic_vectors; DELETE FROM embedding_queue;");
+                const restoreVector = this.db.prepare("INSERT INTO semantic_vectors(chunk_id, content_hash, model, vector) VALUES (?, ?, ?, ?)");
+                for (const row of previousVectors)
+                    restoreVector.run(row.chunk_id, row.content_hash, row.model, row.vector);
+                const restoreQueue = this.db.prepare("INSERT INTO embedding_queue(chunk_id, content_hash, state, attempts) VALUES (?, ?, ?, ?)");
+                for (const row of previousQueue)
+                    restoreQueue.run(row.chunk_id, row.content_hash, row.state, row.attempts);
+            });
+            restore();
             throw error;
         }
     }
+    cleanupSemanticGenerations(generationRoot, activeGeneration) {
+        const generations = readdirSync(generationRoot, { withFileTypes: true })
+            .filter((entry) => entry.isDirectory() && SEMANTIC_GENERATION_NAME.test(entry.name))
+            .map((entry) => entry.name)
+            .sort((left, right) => {
+            const leftTimestamp = this.generationTimestamp(left);
+            const rightTimestamp = this.generationTimestamp(right);
+            return rightTimestamp - leftTimestamp || right.localeCompare(left);
+        });
+        const retained = new Set(generations.slice(0, RETAINED_SEMANTIC_GENERATIONS));
+        retained.add(activeGeneration);
+        for (const generation of generations) {
+            if (!retained.has(generation))
+                rmSync(join(generationRoot, generation), { recursive: true, force: true });
+        }
+    }
+    generationTimestamp(name) {
+        const match = /^gen-[0-9a-f]{16}-([0-9]+)(?:-[0-9a-f]+)?$/.exec(name);
+        return match ? Number(match[1]) : 0;
+    }
     semanticGeneration(root) {
         const generation = readFileSync(join(root, "CURRENT"), "utf8").trim();
+        if (!SEMANTIC_GENERATION_NAME.test(generation))
+            throw new Error("invalid semantic generation pointer");
         const manifest = JSON.parse(readFileSync(join(root, "generations", generation, "manifest.json"), "utf8"));
         return { generation, archiveRevision: manifest.archiveRevision, vectorCount: manifest.vectors.length };
     }
@@ -272,8 +359,6 @@ export class Archive {
         const rows = this.db.prepare("SELECT rowid, chunk_id, text, message_id FROM chunks").all();
         const rebuild = this.db.transaction(() => {
             this.db.exec("DELETE FROM chunks_fts");
-            for (const language of lexicalFields())
-                this.db.exec(`DELETE FROM chunks_fts_${language}`);
             for (const row of rows) {
                 const message = this.db.prepare("SELECT * FROM messages WHERE message_id = ?").get(row.message_id);
                 this.db.prepare(`INSERT INTO chunks_fts(rowid, subject, from_address, to_addresses, thread_subject,
@@ -282,8 +367,6 @@ export class Archive {
             }
         });
         rebuild();
-        this.db.prepare("DELETE FROM lexical_index_meta WHERE id = 1").run();
-        this.lexicalRebuildRequired = true;
         return { rows: rows.length, status: "repaired" };
     }
     upsertMessage(message) {
@@ -315,17 +398,14 @@ export class Archive {
         }
     }
     removeMessage(message) {
-        const stored = this.db.prepare("SELECT message_id FROM messages WHERE provider_key = ?").get(message.providerKey);
-        if (!stored)
-            return;
-        const rows = this.db.prepare("SELECT rowid FROM chunks WHERE message_id = ?").all(stored.message_id);
+        const rows = this.db.prepare("SELECT rowid FROM chunks WHERE message_id = (SELECT message_id FROM messages WHERE provider_key = ?)").all(message.providerKey);
         for (const row of rows)
             this.db.prepare("DELETE FROM chunks_fts WHERE rowid = ?").run(row.rowid);
         for (const language of lexicalFields())
-            this.db.prepare(`DELETE FROM chunks_fts_${language} WHERE chunk_id IN (SELECT chunk_id FROM chunks WHERE message_id = ?)`).run(stored.message_id);
-        this.db.prepare("DELETE FROM embedding_queue WHERE chunk_id IN (SELECT chunk_id FROM chunks WHERE message_id = ?)").run(stored.message_id);
-        this.db.prepare("DELETE FROM semantic_vectors WHERE chunk_id IN (SELECT chunk_id FROM chunks WHERE message_id = ?)").run(stored.message_id);
-        this.db.prepare("DELETE FROM messages WHERE message_id = ?").run(stored.message_id);
+            this.db.prepare(`DELETE FROM chunks_fts_${language} WHERE chunk_id IN (SELECT chunk_id FROM chunks WHERE message_id = (SELECT message_id FROM messages WHERE provider_key = ?))`).run(message.providerKey);
+        this.db.prepare("DELETE FROM embedding_queue WHERE chunk_id IN (SELECT chunk_id FROM chunks WHERE message_id = (SELECT message_id FROM messages WHERE provider_key = ?))").run(message.providerKey);
+        this.db.prepare("DELETE FROM semantic_vectors WHERE chunk_id IN (SELECT chunk_id FROM chunks WHERE message_id = (SELECT message_id FROM messages WHERE provider_key = ?))").run(message.providerKey);
+        this.db.prepare("DELETE FROM messages WHERE provider_key = ?").run(message.providerKey);
     }
     replaceMessageChunks(message, analyzedChunks) {
         const old = this.db.prepare("SELECT rowid, chunk_id FROM chunks WHERE message_id = ?").all(message.messageId);
@@ -333,6 +413,8 @@ export class Archive {
             this.db.prepare("DELETE FROM chunks_fts WHERE rowid = ?").run(row.rowid);
         for (const language of lexicalFields())
             this.db.prepare(`DELETE FROM chunks_fts_${language} WHERE chunk_id IN (SELECT chunk_id FROM chunks WHERE message_id = ?)`).run(message.messageId);
+        this.db.prepare("DELETE FROM embedding_queue WHERE chunk_id IN (SELECT chunk_id FROM chunks WHERE message_id = ?)").run(message.messageId);
+        this.db.prepare("DELETE FROM semantic_vectors WHERE chunk_id IN (SELECT chunk_id FROM chunks WHERE message_id = ?)").run(message.messageId);
         this.db.prepare("DELETE FROM chunks WHERE message_id = ?").run(message.messageId);
         for (const chunk of buildChunks(message)) {
             const result = this.db.prepare(`INSERT INTO chunks
@@ -353,8 +435,21 @@ export class Archive {
         const rows = this.db.prepare("SELECT chunk_id, content_hash FROM chunks ORDER BY chunk_id").all();
         return createHash("sha256").update(rows.map((row) => `${row.chunk_id}\0${row.content_hash}`).join("\0")).digest("hex");
     }
+    async runExclusive(operation) {
+        const previous = this.operationTail;
+        let release = () => { };
+        const turn = new Promise((resolve) => { release = resolve; });
+        this.operationTail = previous.then(() => turn);
+        await previous;
+        try {
+            return await operation();
+        }
+        finally {
+            release();
+        }
+    }
     async getEmbedder() {
-        this.embedder ??= await createEmbedder();
+        this.embedder ??= await createEmbedder(this.embedderConfig);
         return this.embedder;
     }
     searchLexicalTable(table, query, filters, limit) {
@@ -426,6 +521,12 @@ function migrate(db) {
             db.exec(`ALTER TABLE messages ADD COLUMN ${column} TEXT NOT NULL DEFAULT '[]'`);
     }
 }
+function cleanupOrphanedSemanticRows(db) {
+    db.prepare(`DELETE FROM embedding_queue
+    WHERE NOT EXISTS (SELECT 1 FROM chunks WHERE chunks.chunk_id = embedding_queue.chunk_id)`).run();
+    db.prepare(`DELETE FROM semantic_vectors
+    WHERE NOT EXISTS (SELECT 1 FROM chunks WHERE chunks.chunk_id = semantic_vectors.chunk_id)`).run();
+}
 function literalFtsQuery(query) {
     return query.trim().split(/\s+/).map((term) => `"${term.replaceAll('"', '""')}"`).join(" ");
 }
@@ -462,6 +563,28 @@ function addFilters(clauses, params, filters) {
 function normalizeCategory(value) {
     return value.trim().toLocaleLowerCase().replace(/^category[_-]/, "").replace(/^label[_-]/, "");
 }
+function validateIdentities(messages) {
+    const providerKeys = new Set();
+    const messageIds = new Set();
+    for (const message of messages) {
+        if (!message.providerKey)
+            throw new Error("message provider identity is required");
+        if (providerKeys.has(message.providerKey))
+            throw new Error("duplicate message provider identity");
+        providerKeys.add(message.providerKey);
+        if (messageIds.has(message.messageId))
+            throw new Error("duplicate message identity");
+        messageIds.add(message.messageId);
+    }
+}
+function validateStoredIdentities(db, messages) {
+    const byProviderKey = db.prepare("SELECT message_id FROM messages WHERE provider_key = ?");
+    for (const message of messages) {
+        const storedProvider = byProviderKey.get(message.providerKey);
+        if (storedProvider && storedProvider.message_id !== message.messageId)
+            throw new Error("provider identity already belongs to another message identity");
+    }
+}
 function countExcluded(messages, excluded) {
     const counts = {};
     for (const message of messages) {

package/dist/cli/index.js

@@ -2,9 +2,11 @@
 import { Command } from "commander";
 import { mkdir } from "node:fs/promises";
 import { existsSync } from "node:fs";
+import { join } from "node:path";
 import { Archive } from "../archive.js";
 import { FixtureSource, HimalayaSource } from "../source.js";
 import { redactDiagnostic } from "../redact.js";
+import { embeddingModelName, loopbackConfigFromEnvironment } from "../embedding.js";
 const program = new Command();
 program.name("mailcrawl").description("Local privacy-first email indexing CLI");
 program.option("--data-dir <path>", "archive directory", process.env.MAILCRAWL_DATA_DIR || ".mailcrawl");
@@ -22,12 +24,18 @@ program
     .option("--json")
     .action(async (options, command) => {
     const dataDir = command.parent.opts().dataDir;
+    if (options.source !== "fixture" && options.source !== "himalaya")
+        throw new Error(`unsupported source: ${options.source}`);
+    if (options.source === "fixture" && !options.fixture)
+        throw new Error("--fixture is required for fixture source");
+    if (options.source === "himalaya" && !options.account)
+        throw new Error("--account is required for himalaya source");
     await mkdir(dataDir, { recursive: true });
-    const archive = new Archive(`${dataDir}/archive.sqlite`);
+    const archive = new Archive(join(dataDir, "archive.sqlite"));
     try {
         const source = options.source === "fixture"
-            ? new FixtureSource(options.fixture)
-            : new HimalayaSource(options.account, options.mailbox, options.backend, Number(options.pageSize), options.himalayaConfig);
+            ? fixtureSource(options)
+            : himalayaSource(options);
         const excludedCategories = (options.excludeCategory.length ? options.excludeCategory : ["spam", "promotions"])
             .filter((category) => !options.includeCategory.includes(category));
         output(await archive.sync(await source.list(), { excludedCategories }), options.json);
@@ -38,12 +46,21 @@ program
 });
 program
     .command("index")
+    .alias("embed")
+    .option("--provider <provider>", "embedding provider", "local")
+    .option("--embed-url <url>")
+    .option("--embed-model <model>")
+    .option("--embed-dim <n>")
+    .option("--query-prefix <prefix>")
+    .option("--passage-prefix <prefix>")
+    .option("--embed-timeout <ms>")
     .option("--json")
     .action(async (options, command) => {
     const dataDir = command.parent.opts().dataDir;
-    const archive = new Archive(`${dataDir}/archive.sqlite`);
+    const config = embedderConfig(options);
+    const archive = new Archive(join(dataDir, "archive.sqlite"), config);
     try {
-        output({ ...await archive.indexSemanticGeneration(`${dataDir}/semantic`), embedder: "onnx-community/embeddinggemma-300m-ONNX" }, options.json);
+        output({ ...await archive.indexSemanticGeneration(join(dataDir, "semantic")), embedder: embeddingModelName(config) }, options.json);
     }
     finally {
         archive.close();
@@ -61,12 +78,19 @@ for (const mode of ["bm25", "keyword", "semantic", "hybrid"]) {
         .option("--after <date>")
         .option("--before <date>")
         .option("--limit <n>", "result limit", "10")
+        .option("--provider <provider>", "embedding provider", "local")
+        .option("--embed-url <url>")
+        .option("--embed-model <model>")
+        .option("--embed-dim <n>")
+        .option("--query-prefix <prefix>")
+        .option("--passage-prefix <prefix>")
+        .option("--embed-timeout <ms>")
         .option("--json")
         .action(async (query, options, command) => {
         const dataDir = command.parent.opts().dataDir;
-        const archive = new Archive(`${dataDir}/archive.sqlite`);
+        const archive = new Archive(join(dataDir, "archive.sqlite"), embedderConfig(options));
         try {
-            const result = mode === "bm25"
+            const result = mode === "bm25" || mode === "keyword"
                 ? await archive.searchBm25(query, filters(options), Number(options.limit))
                 : mode === "hybrid"
                     ? await archive.searchHybrid(query, filters(options), Number(options.limit))
@@ -81,7 +105,7 @@ for (const mode of ["bm25", "keyword", "semantic", "hybrid"]) {
 program
     .command("search")
     .argument("<query>")
-    .option("--mode <mode>", "keyword, bm25, semantic, hybrid", "bm25")
+    .option("--mode <mode>", "fts, keyword, bm25, semantic, hybrid", "bm25")
     .option("--account <id>")
     .option("--mailbox <name>")
     .option("--from <address>")
@@ -90,13 +114,20 @@ program
     .option("--after <date>")
     .option("--before <date>")
     .option("--limit <n>", "result limit", "10")
+    .option("--provider <provider>", "embedding provider", "local")
+    .option("--embed-url <url>")
+    .option("--embed-model <model>")
+    .option("--embed-dim <n>")
+    .option("--query-prefix <prefix>")
+    .option("--passage-prefix <prefix>")
+    .option("--embed-timeout <ms>")
     .option("--json")
     .action(async (query, options, command) => {
-    const archive = new Archive(`${command.parent.opts().dataDir}/archive.sqlite`);
+    const archive = new Archive(join(command.parent.opts().dataDir, "archive.sqlite"), embedderConfig(options));
     try {
         const filter = filters(options);
         const limit = Number(options.limit);
-        const result = options.mode === "bm25" ? await archive.searchBm25(query, filter, limit)
+        const result = options.mode === "bm25" || options.mode === "fts" || options.mode === "keyword" ? await archive.searchBm25(query, filter, limit)
             : options.mode === "semantic" ? await archive.searchSemantic(query, filter, limit)
                 : options.mode === "hybrid" ? await archive.searchHybrid(query, filter, limit)
                     : (() => { throw new Error(`unsupported search mode: ${options.mode}`); })();
@@ -111,7 +142,7 @@ program
     .command("get <messageId>")
     .option("--json")
     .action(async (messageId, options, command) => {
-    const archive = new Archive(`${command.parent.parent.opts().dataDir}/archive.sqlite`);
+    const archive = new Archive(join(command.parent.parent.opts().dataDir, "archive.sqlite"));
     try {
         output(archive.getMessage(messageId) ?? null, options.json);
     }
@@ -124,7 +155,7 @@ program
     .argument("<chunkId>")
     .option("--json")
     .action(async (chunkId, options, command) => {
-    const archive = new Archive(`${command.parent.opts().dataDir}/archive.sqlite`);
+    const archive = new Archive(join(command.parent.opts().dataDir, "archive.sqlite"));
     try {
         output(archive.getChunkContext(chunkId), options.json);
     }
@@ -141,7 +172,7 @@ program
     .option("--before <date>")
     .option("--json")
     .action(async (threadId, options, command) => {
-    const archive = new Archive(`${command.parent.parent.opts().dataDir}/archive.sqlite`);
+    const archive = new Archive(join(command.parent.parent.opts().dataDir, "archive.sqlite"));
     try {
         output(archive.getThread(threadId, filters(options)), options.json);
     }
@@ -155,7 +186,7 @@ program
     .option("--message <messageId>")
     .option("--json")
     .action(async (threadId, options, command) => {
-    const archive = new Archive(`${command.parent.opts().dataDir}/archive.sqlite`);
+    const archive = new Archive(join(command.parent.opts().dataDir, "archive.sqlite"));
     try {
         output(archive.getThreadContext(threadId, options.message), options.json);
     }
@@ -168,38 +199,88 @@ program
     .option("--json")
     .action(async (options, command) => {
     const dataDir = command.parent.opts().dataDir;
-    const archive = new Archive(`${dataDir}/archive.sqlite`);
+    const archive = new Archive(join(dataDir, "archive.sqlite"));
     try {
         let semantic = "missing";
         try {
-            semantic = archive.semanticGeneration(`${dataDir}/semantic`);
+            semantic = semanticStatus(archive, archive.semanticGeneration(join(dataDir, "semantic")));
         }
         catch (error) {
-            semantic = redactDiagnostic({ status: "stale", error: error instanceof Error ? error.message : String(error) });
+            semantic = redactDiagnostic({ status: semanticErrorStatus(error), error: error instanceof Error ? error.message : String(error) });
         }
+        const semanticCommitted = typeof semantic === "object" && semantic !== null && "generation" in semantic;
         output({
             name: "mailcrawl",
-            archive: `${dataDir}/archive.sqlite`,
-            archivePresent: existsSync(`${dataDir}/archive.sqlite`),
-            fts: "available",
+            archive: join(dataDir, "archive.sqlite"),
+            archivePresent: existsSync(join(dataDir, "archive.sqlite")),
+            fts: archive.status().fts,
             semantic,
-            recommendation: semantic === "missing" ? "run sync, then index before semantic search" : "semantic index is committed",
+            recommendation: semanticCommitted ? "semantic index is committed" : "run sync, then index before semantic search",
         }, options.json);
     }
     finally {
         archive.close();
     }
 });
+program
+    .command("status")
+    .option("--json")
+    .action(async (options, command) => {
+    const dataDir = command.parent.opts().dataDir;
+    const archivePath = join(dataDir, "archive.sqlite");
+    if (!existsSync(archivePath)) {
+        output({ name: "mailcrawl", archive: archivePath, archivePresent: false, messageCount: 0, chunkCount: 0, embeddingBacklog: 0, fts: { status: "missing", rows: 0 }, semantic: { status: "missing" } }, options.json);
+        return;
+    }
+    const archive = new Archive(archivePath);
+    try {
+        let semantic = "missing";
+        try {
+            semantic = semanticStatus(archive, archive.semanticGeneration(join(dataDir, "semantic")));
+        }
+        catch (error) {
+            semantic = redactDiagnostic({ status: semanticErrorStatus(error), error: error instanceof Error ? error.message : String(error) });
+        }
+        output({ name: "mailcrawl", archive: archivePath, archivePresent: true, ...archive.status(), semantic }, options.json);
+    }
+    finally {
+        archive.close();
+    }
+});
 program
     .command("repair")
     .option("--fts")
+    .option("--semantic")
+    .option("--all")
+    .option("--provider <provider>", "embedding provider", "local")
+    .option("--embed-url <url>")
+    .option("--embed-model <model>")
+    .option("--embed-dim <n>")
+    .option("--query-prefix <prefix>")
+    .option("--passage-prefix <prefix>")
+    .option("--embed-timeout <ms>")
     .option("--json")
     .action(async (options, command) => {
-    const archive = new Archive(`${command.parent.opts().dataDir}/archive.sqlite`);
+    const dataDir = command.parent.opts().dataDir;
+    const archive = new Archive(join(dataDir, "archive.sqlite"));
     try {
-        if (!options.fts)
-            throw new Error("pass --fts");
-        output(archive.repairFts(), options.json);
+        if (!options.fts && !options.semantic && !options.all)
+            throw new Error("pass --fts, --semantic, or --all");
+        const result = {};
+        if (options.semantic || options.all) {
+            const config = embedderConfig(options);
+            const semanticArchive = config ? new Archive(join(dataDir, "archive.sqlite"), config) : archive;
+            try {
+                result.semantic = await semanticArchive.indexSemanticGeneration(join(dataDir, "semantic"));
+            }
+            finally {
+                if (semanticArchive !== archive)
+                    semanticArchive.close();
+            }
+        }
+        if (options.fts || options.all)
+            result.fts = archive.repairFts();
+        output(Object.keys(result).length === 1 ? Object.values(result)[0] : result, options.json);
     }
     finally {
         archive.close();
@@ -211,7 +292,7 @@ attachments
     .option("--message <messageId>")
     .option("--json")
     .action(async (options, command) => {
-    const archive = new Archive(`${command.parent.parent.opts().dataDir}/archive.sqlite`);
+    const archive = new Archive(join(command.parent.parent.opts().dataDir, "archive.sqlite"));
     try {
         output(archive.listAttachments(options.message), options.json);
     }
@@ -221,12 +302,37 @@ attachments
 });
 program.parseAsync().catch((error) => {
     const message = error instanceof Error ? error.message : String(error);
-    console.error(JSON.stringify({ error: message }));
+    console.error(JSON.stringify(redactDiagnostic({ error: message })));
     process.exitCode = 1;
 });
 function filters(options) {
     return { accountId: options.account, mailbox: options.mailbox, from: options.from, to: options.to, threadId: options.thread, after: options.after, before: options.before };
 }
+function embedderConfig(options) {
+    if (options.provider === "local")
+        return loopbackConfigFromEnvironment();
+    if (options.provider !== "loopback-http")
+        throw new Error(`unsupported embedding provider: ${options.provider}`);
+    if (!options.embedUrl || !options.embedModel || !options.embedDim)
+        throw new Error("--embed-url, --embed-model, and --embed-dim are required for loopback-http");
+    return { provider: "loopback-http", url: options.embedUrl, model: options.embedModel, dimension: Number(options.embedDim), queryPrefix: options.queryPrefix, passagePrefix: options.passagePrefix, timeoutMs: options.embedTimeout ? Number(options.embedTimeout) : undefined };
+}
+function fixtureSource(options) {
+    if (!options.fixture)
+        throw new Error("--fixture is required for selected source");
+    return new FixtureSource(options.fixture);
+}
+function himalayaSource(options) {
+    if (!options.account)
+        throw new Error("--account is required for selected source");
+    return new HimalayaSource(options.account, options.mailbox, options.backend, Number(options.pageSize), options.himalayaConfig);
+}
+function semanticStatus(archive, semantic) {
+    return semantic.archiveRevision === archive.status().archiveRevision ? { ...semantic, status: "healthy" } : { ...semantic, status: "stale" };
+}
+function semanticErrorStatus(error) {
+    return error instanceof Error && "code" in error && error.code === "ENOENT" ? "missing" : "corrupt";
+}
 function output(value, json) {
     if (json)
         console.log(JSON.stringify(value));

package/dist/embedding.d.ts

@@ -1,6 +1,8 @@
+import type { LoopbackHttpConfig } from "./types.js";
 export interface Embedder {
     embedDocuments(texts: string[]): Promise<number[][]>;
     embedQuery(query: string): Promise<number[]>;
 }
-export declare function createEmbedder(): Promise<Embedder>;
-export declare function embeddingModelName(): string;
+export declare function createEmbedder(config?: LoopbackHttpConfig): Promise<Embedder>;
+export declare function embeddingModelName(config?: LoopbackHttpConfig): string;
+export declare function loopbackConfigFromEnvironment(): LoopbackHttpConfig | undefined;

package/dist/embedding.js

@@ -33,13 +33,76 @@ class TestEmbedder {
         return hashVector(query);
     }
 }
-export async function createEmbedder() {
+class LoopbackHttpEmbedder {
+    config;
+    constructor(config) {
+        this.config = config;
+        const url = new URL(config.url);
+        const hostname = url.hostname.startsWith("[") ? url.hostname.slice(1, -1) : url.hostname;
+        if (url.protocol !== "http:" || !["127.0.0.1", "localhost", "::1"].includes(hostname)) {
+            throw new Error("loopback HTTP embedding URL is required");
+        }
+        if (!Number.isInteger(config.dimension) || config.dimension <= 0)
+            throw new Error("embedding dimension must be positive");
+    }
+    async embedDocuments(texts) {
+        return this.request(texts.map((text) => (this.config.passagePrefix ?? "") + text));
+    }
+    async embedQuery(query) {
+        return (await this.request([(this.config.queryPrefix ?? "") + query.trim()]))[0];
+    }
+    async request(texts) {
+        const controller = new AbortController();
+        const timeout = setTimeout(() => controller.abort(), this.config.timeoutMs ?? 30_000);
+        try {
+            const response = await fetch(this.config.url, {
+                method: "POST",
+                headers: { "content-type": "application/json" },
+                body: JSON.stringify({ model: this.config.model, texts }),
+                signal: controller.signal,
+            });
+            if (!response.ok)
+                throw new Error(`embedding provider returned HTTP ${response.status}`);
+            const payload = await response.json();
+            if (!Array.isArray(payload.embeddings) || payload.embeddings.length !== texts.length)
+                throw new Error("embedding provider returned invalid embeddings");
+            const vectors = payload.embeddings;
+            if (vectors.some((vector) => !Array.isArray(vector) || vector.length !== this.config.dimension || vector.some((value) => typeof value !== "number"))) {
+                throw new Error("embedding provider returned invalid vector dimensions");
+            }
+            return vectors;
+        }
+        finally {
+            clearTimeout(timeout);
+        }
+    }
+}
+export async function createEmbedder(config) {
+    if (config?.provider === "loopback-http")
+        return new LoopbackHttpEmbedder(config);
     if (process.env.MAILCRAWL_EMBEDDER === "mock" || process.env.NODE_ENV === "test")
         return new TestEmbedder();
     return EmbeddingGemma.create();
 }
-export function embeddingModelName() {
-    return EMBEDDING_MODEL;
+export function embeddingModelName(config) {
+    return config
+        ? `loopback-http:${config.model}:${config.dimension}:${config.url}:${config.queryPrefix ?? ""}:${config.passagePrefix ?? ""}:${config.timeoutMs ?? 30_000}`
+        : EMBEDDING_MODEL;
+}
+export function loopbackConfigFromEnvironment() {
+    if (process.env.MAILCRAWL_EMBEDDER_PROVIDER !== "loopback-http")
+        return undefined;
+    const url = process.env.MAILCRAWL_EMBED_URL;
+    const model = process.env.MAILCRAWL_EMBED_MODEL;
+    const dimension = Number(process.env.MAILCRAWL_EMBED_DIM);
+    if (!url || !model || !Number.isInteger(dimension))
+        throw new Error("MAILCRAWL_EMBED_URL, MAILCRAWL_EMBED_MODEL, and MAILCRAWL_EMBED_DIM are required");
+    return {
+        provider: "loopback-http", url, model, dimension,
+        queryPrefix: process.env.MAILCRAWL_QUERY_PREFIX,
+        passagePrefix: process.env.MAILCRAWL_PASSAGE_PREFIX,
+        timeoutMs: process.env.MAILCRAWL_EMBED_TIMEOUT ? Number(process.env.MAILCRAWL_EMBED_TIMEOUT) : undefined,
+    };
 }
 function hashVector(text) {
     const vector = new Array(128).fill(0);

package/dist/normalize.js

@@ -26,7 +26,18 @@ export async function normalizeMessage(input) {
     const subject = input.subject.trim();
     const messageId = input.messageId || input.providerKey;
     const threadId = input.threadId || makeId(input.accountId, normalizeSubject(subject));
-    const normalizedHash = hash(JSON.stringify([subject, input.from, input.to, input.cc, latest, quoted, input.labels, input.flags, input.classifications]));
+    const attachmentInputs = (attachments || []).map((attachment) => [
+        attachment.name,
+        attachment.mimeType,
+        attachment.size ?? null,
+        attachment.text ?? null,
+        attachment.contentHash ?? null,
+    ]);
+    const normalizedHash = hash(JSON.stringify([
+        subject, input.from, input.to, input.cc, latest, quoted,
+        input.labels, input.flags, input.classifications,
+        ...(attachmentInputs.length ? [attachmentInputs] : []),
+    ]));
     const categories = [...new Set([
             ...(input.classifications || []),
             ...(input.labels || []),

package/dist/redact.js

@@ -2,6 +2,7 @@ export function redactDiagnostic(value) {
     if (typeof value === "string") {
         return value
             .replace(/([?&](?:token|password|secret|key)=)[^&\s]+/giu, "$1[REDACTED]")
+            .replace(/(?<![?&])\b(?:token|password|secret|key|credential)\s*[=:]\s*[^,\s]+/giu, "[REDACTED]")
             .replace(/\b[A-Z0-9._%+-]+@[A-Z0-9.-]+\.[A-Z]{2,}\b/giu, "[EMAIL]");
     }
     if (Array.isArray(value))

package/dist/source.js

@@ -1,6 +1,7 @@
 import { execFile } from "node:child_process";
 import { promisify } from "node:util";
 import { readFile } from "node:fs/promises";
+import { redactDiagnostic } from "./redact.js";
 const execFileAsync = promisify(execFile);
 export class FixtureSource {
     path;
@@ -30,7 +31,7 @@ export class HimalayaSource {
         if (this.backend)
             args.push("-b", this.backend);
         args.push("envelope", "list", "--mailbox", this.mailbox, "--page-size", String(this.pageSize), "--json");
-        const { stdout } = await execFileAsync("himalaya", args, { maxBuffer: 16 * 1024 * 1024 });
+        const { stdout } = await runHimalaya(args, 16 * 1024 * 1024, "envelope list");
         const payload = JSON.parse(stdout);
         const envelopes = payload.envelopes ?? (Array.isArray(payload) ? payload : []);
         return Promise.all(envelopes.map(async (envelope) => {
@@ -60,11 +61,20 @@ export class HimalayaSource {
         if (this.backend)
             args.push("-b", this.backend);
         args.push("--json", "message", "read", id, "--raw");
-        const { stdout } = await execFileAsync("himalaya", args, { maxBuffer: 32 * 1024 * 1024 });
+        const { stdout } = await runHimalaya(args, 32 * 1024 * 1024, "message read");
         const payload = JSON.parse(stdout);
         return payload.message ?? stdout;
     }
 }
+async function runHimalaya(args, maxBuffer, operation) {
+    try {
+        return await execFileAsync("himalaya", args, { maxBuffer });
+    }
+    catch (error) {
+        const detail = error instanceof Error ? error.message : String(error);
+        throw new Error(`himalaya ${operation} failed: ${redactDiagnostic(detail)}`);
+    }
+}
 function address(value) {
     if (typeof value === "string")
         return value;

package/dist/types.d.ts

@@ -1,4 +1,14 @@
 export type SearchMode = "keyword" | "bm25" | "semantic" | "hybrid";
+export type EmbedderProvider = "local" | "loopback-http";
+export interface LoopbackHttpConfig {
+    provider: "loopback-http";
+    url: string;
+    model: string;
+    dimension: number;
+    queryPrefix?: string;
+    passagePrefix?: string;
+    timeoutMs?: number;
+}
 export interface MailMessage {
     accountId: string;
     mailbox: string;

package/dist/util.d.ts

@@ -1,4 +1,5 @@
 export declare function hash(value: string): string;
 export declare function normalizeSubject(subject: string): string;
 export declare function makeId(...parts: string[]): string;
+export declare function scopedId(accountId: string, mailbox: string, value: string): string;
 export declare function snippet(text: string, query: string, maxChars?: number): string;

package/dist/util.js

@@ -8,6 +8,9 @@ export function normalizeSubject(subject) {
 export function makeId(...parts) {
     return hash(parts.join("\u0000")).slice(0, 24);
 }
+export function scopedId(accountId, mailbox, value) {
+    return makeId(accountId, mailbox, value);
+}
 export function snippet(text, query, maxChars = 240) {
     const clean = text.trim();
     if (clean.length <= maxChars)

package/docs/architecture.md

@@ -159,10 +159,10 @@ All commands support `--json` where machine-readable output is useful.
 mailcrawl doctor
 mailcrawl status
 mailcrawl sync
-mailcrawl embed [--limit N]
-mailcrawl search --mode fts|semantic|hybrid [--limit N] [--mailbox NAME] QUERY
+mailcrawl embed
+mailcrawl search --mode fts|bm25|keyword|semantic|hybrid [--limit N] [--mailbox NAME] QUERY
 mailcrawl message get MESSAGE_ID
-mailcrawl repair [--fts|--vectors|--all]
+mailcrawl repair [--fts|--semantic|--all]
 ```
 
 Example sync response:

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@nomadamas/mailcrawl",
-  "version": "0.1.4",
+  "version": "0.1.6",
   "description": "Local Himalaya-backed incremental email indexing and hybrid search CLI",
   "license": "MIT",
   "type": "module",
@@ -48,6 +48,7 @@
     "@types/better-sqlite3": "^7.6.13",
     "@types/node": "^24.3.0",
     "tsx": "^4.20.5",
+    "typescript": "^5.9.3",
     "vitest": "^3.2.4"
   },
   "engines": {

package/skills/mailcrawl/SKILL.md

@@ -71,7 +71,8 @@ mailcrawl sync --source fixture --fixture ./messages.json --json
 
 ## Safe read and maintenance commands
 
-Search modes are `bm25`, `keyword`, `semantic`, and `hybrid`. Use metadata
+Search modes are `fts`, `bm25`, `keyword`, `semantic`, and `hybrid`; `fts` and
+`keyword` are aliases for BM25 lexical search. Use metadata
 filters such as `--mailbox`, `--from`, `--to`, `--thread`, `--after`, and
 `--before`. Empty queries and unsupported modes fail with a non-zero exit.
 
@@ -89,7 +90,11 @@ Check health and rebuild lexical data when needed:
 
 ```bash
 mailcrawl doctor --json
+mailcrawl status --json
+mailcrawl embed --json
 mailcrawl repair --fts --json
+mailcrawl repair --semantic --json
+mailcrawl repair --all --json
 ```
 
 `doctor` reports archive, FTS, and semantic-generation state. `repair` is a