@nomadamas/mailcrawl 0.1.5 → 0.1.6

package/README.md

@@ -20,6 +20,7 @@ mailcrawl embed --json
 mailcrawl search --mode hybrid --json "계약 갱신 조건"
 mailcrawl status --json
 mailcrawl doctor --json
+mailcrawl repair --all --json
 ```
 
 The CLI owns email synchronization and indexes. Consumers such as AutoRAG,
@@ -56,6 +57,21 @@ Chinese helper builds, environment variables, smoke tests, and license
 requirements, follow [`docs/multilingual-installation.md`](docs/multilingual-installation.md)
 before using multilingual indexing or search.
 
+## Shared loopback embedding provider
+
+The local in-process model is the default. Opt into a local HTTP runtime with
+`--provider loopback-http`, `--embed-url`, `--embed-model`, and `--embed-dim`
+on `index`, `repair --semantic`, or semantic search. Only HTTP URLs for
+`127.0.0.1`, `localhost`, or `::1` are accepted. Query/passage prefixes and
+timeout are optional and are included in the provider identity. Any provider
+setting change rebuilds vectors, while a failed rebuild preserves the prior
+`CURRENT` generation.
+
+The equivalent environment contract is
+`MAILCRAWL_EMBEDDER_PROVIDER`, `MAILCRAWL_EMBED_URL`,
+`MAILCRAWL_EMBED_MODEL`, `MAILCRAWL_EMBED_DIM`, `MAILCRAWL_QUERY_PREFIX`,
+`MAILCRAWL_PASSAGE_PREFIX`, and `MAILCRAWL_EMBED_TIMEOUT`.
+
 ## Releasing
 
 GitHub Release `vX.Y.Z` (must match `package.json`) publishes `@nomadamas/mailcrawl` to npm with OIDC trusted publishing. No `NPM_TOKEN` is stored in GitHub.

package/dist/archive.d.ts

@@ -1,14 +1,25 @@
 import Database from "better-sqlite3";
-import type { ClassificationPolicy, Chunk, MailMessage, NormalizedMessage, SearchFilters, SearchHit, SyncReport } from "./types.js";
+import type { ClassificationPolicy, Chunk, LoopbackHttpConfig, MailMessage, NormalizedMessage, SearchFilters, SearchHit, SyncReport } from "./types.js";
 export declare class Archive {
     readonly db: Database.Database;
     private embedder?;
     private lexical?;
     private operationTail;
     private lexicalRebuildRequired;
-    constructor(path?: string);
+    private readonly embedderConfig?;
+    constructor(path?: string, embedderConfig?: LoopbackHttpConfig);
     close(): void;
     sync(messages: MailMessage[], policy?: ClassificationPolicy): Promise<SyncReport>;
+    status(): {
+        messageCount: number;
+        chunkCount: number;
+        embeddingBacklog: number;
+        archiveRevision: string;
+        fts: {
+            status: "healthy" | "stale";
+            rows: number;
+        };
+    };
     private syncUnlocked;
     searchBm25(query: string, filters?: SearchFilters, limit?: number): Promise<SearchHit[]>;
     indexSemantic(): Promise<{
@@ -24,6 +35,7 @@ export declare class Archive {
     }>;
     private indexSemanticGenerationUnlocked;
     private cleanupSemanticGenerations;
+    private generationTimestamp;
     semanticGeneration(root: string): {
         generation: string;
         archiveRevision: string;

package/dist/archive.js

@@ -1,5 +1,5 @@
 import Database from "better-sqlite3";
-import { createHash } from "node:crypto";
+import { createHash, randomBytes } from "node:crypto";
 import { mkdirSync, readFileSync, readdirSync, renameSync, rmSync, writeFileSync } from "node:fs";
 import { join } from "node:path";
 import { buildChunks } from "./chunk.js";
@@ -8,14 +8,16 @@ import { scopedId, snippet } from "./util.js";
 import { createEmbedder, embeddingModelName } from "./embedding.js";
 import { createLexicalAnalyzers, languagesForText, lexicalFields, LEXICAL_ANALYZER_VERSION } from "./lexical.js";
 const RETAINED_SEMANTIC_GENERATIONS = 2;
-const SEMANTIC_GENERATION_NAME = /^gen-[0-9a-f]{16}-[0-9]+$/;
+const SEMANTIC_GENERATION_NAME = /^gen-[0-9a-f]{16}-[0-9]+(?:-[0-9a-f]+)?$/;
 export class Archive {
     db;
     embedder;
     lexical;
     operationTail = Promise.resolve();
     lexicalRebuildRequired;
-    constructor(path = ":memory:") {
+    embedderConfig;
+    constructor(path = ":memory:", embedderConfig) {
+        this.embedderConfig = embedderConfig;
         this.db = new Database(path);
         this.db.pragma("journal_mode = WAL");
         this.db.pragma("foreign_keys = ON");
@@ -38,6 +40,19 @@ export class Archive {
     async sync(messages, policy = {}) {
         return this.runExclusive(() => this.syncUnlocked(messages, policy));
     }
+    status() {
+        const count = (table) => Number(this.db.prepare(`SELECT COUNT(*) AS count FROM ${table}`).get().count);
+        const ftsRows = Number(this.db.prepare("SELECT COUNT(*) AS count FROM chunks_fts").get().count);
+        const metadata = this.db.prepare("SELECT version FROM lexical_index_meta WHERE id = 1").get();
+        const chunkCount = count("chunks");
+        return {
+            messageCount: count("messages"),
+            chunkCount,
+            embeddingBacklog: Number(this.db.prepare("SELECT COUNT(*) AS count FROM embedding_queue WHERE state = 'pending'").get().count),
+            archiveRevision: this.revision(),
+            fts: { status: metadata?.version === LEXICAL_ANALYZER_VERSION && ftsRows === chunkCount ? "healthy" : "stale", rows: ftsRows },
+        };
+    }
     async syncUnlocked(messages, policy = {}) {
         for (const message of messages) {
             if (typeof message.providerKey !== "string" || !message.providerKey.trim())
@@ -160,7 +175,7 @@ export class Archive {
         const transaction = this.db.transaction(() => {
             for (const row of rows) {
                 const old = this.db.prepare("SELECT content_hash, model FROM semantic_vectors WHERE chunk_id = ?").get(row.chunk_id);
-                if (old?.content_hash === row.content_hash && old.model === embeddingModelName()) {
+                if (old?.content_hash === row.content_hash && old.model === embeddingModelName(this.embedderConfig)) {
                     completeQueueRow.run(row.chunk_id);
                     reused++;
                     continue;
@@ -175,7 +190,7 @@ export class Archive {
         const vectors = await embedder.embedDocuments(pending.map((row) => row.text));
         const write = this.db.transaction(() => {
             for (const [index, row] of pending.entries()) {
-                upsert.run(row.chunk_id, row.content_hash, embeddingModelName(), JSON.stringify(vectors[index]));
+                upsert.run(row.chunk_id, row.content_hash, embeddingModelName(this.embedderConfig), JSON.stringify(vectors[index]));
                 completeQueueRow.run(row.chunk_id);
                 embedded++;
             }
@@ -192,20 +207,22 @@ export class Archive {
         const previousVectors = this.db.prepare("SELECT chunk_id, content_hash, model, vector FROM semantic_vectors ORDER BY chunk_id").all();
         const previousQueue = this.db.prepare("SELECT chunk_id, content_hash, state, attempts FROM embedding_queue ORDER BY chunk_id").all();
         mkdirSync(generationRoot, { recursive: true });
-        const generation = `gen-${this.revision().slice(0, 16)}-${Date.now()}`;
+        const generation = `gen-${this.revision().slice(0, 16)}-${Date.now()}-${randomBytes(6).toString("hex")}`;
         const staging = join(generationRoot, `.${generation}.staging`);
         const publishedGeneration = join(generationRoot, generation);
+        const pointer = join(root, `.CURRENT.${process.pid}`);
         mkdirSync(staging);
+        let published = false;
         try {
             const report = await this.indexSemanticUnlocked();
             const vectors = this.db.prepare(`SELECT v.chunk_id, v.content_hash, v.vector
         FROM semantic_vectors v JOIN chunks c ON c.chunk_id = v.chunk_id
         ORDER BY v.chunk_id`).all();
             writeFileSync(join(staging, "manifest.json"), JSON.stringify({
-                archiveRevision: this.revision(), vectors, model: embeddingModelName(),
+                archiveRevision: this.revision(), vectors, model: embeddingModelName(this.embedderConfig),
             }));
             renameSync(staging, publishedGeneration);
-            const pointer = join(root, `.CURRENT.${process.pid}`);
+            published = true;
             writeFileSync(pointer, `${generation}\n`);
             renameSync(pointer, currentPath);
             this.cleanupSemanticGenerations(generationRoot, generation);
@@ -213,7 +230,9 @@ export class Archive {
         }
         catch (error) {
             rmSync(staging, { recursive: true, force: true });
-            rmSync(publishedGeneration, { recursive: true, force: true });
+            rmSync(pointer, { force: true });
+            if (!published)
+                rmSync(publishedGeneration, { recursive: true, force: true });
             const restore = this.db.transaction(() => {
                 this.db.exec("DELETE FROM semantic_vectors; DELETE FROM embedding_queue;");
                 const restoreVector = this.db.prepare("INSERT INTO semantic_vectors(chunk_id, content_hash, model, vector) VALUES (?, ?, ?, ?)");
@@ -232,8 +251,8 @@ export class Archive {
             .filter((entry) => entry.isDirectory() && SEMANTIC_GENERATION_NAME.test(entry.name))
             .map((entry) => entry.name)
             .sort((left, right) => {
-            const leftTimestamp = Number(left.slice(left.lastIndexOf("-") + 1));
-            const rightTimestamp = Number(right.slice(right.lastIndexOf("-") + 1));
+            const leftTimestamp = this.generationTimestamp(left);
+            const rightTimestamp = this.generationTimestamp(right);
             return rightTimestamp - leftTimestamp || right.localeCompare(left);
         });
         const retained = new Set(generations.slice(0, RETAINED_SEMANTIC_GENERATIONS));
@@ -243,8 +262,14 @@ export class Archive {
                 rmSync(join(generationRoot, generation), { recursive: true, force: true });
         }
     }
+    generationTimestamp(name) {
+        const match = /^gen-[0-9a-f]{16}-([0-9]+)(?:-[0-9a-f]+)?$/.exec(name);
+        return match ? Number(match[1]) : 0;
+    }
     semanticGeneration(root) {
         const generation = readFileSync(join(root, "CURRENT"), "utf8").trim();
+        if (!SEMANTIC_GENERATION_NAME.test(generation))
+            throw new Error("invalid semantic generation pointer");
         const manifest = JSON.parse(readFileSync(join(root, "generations", generation, "manifest.json"), "utf8"));
         return { generation, archiveRevision: manifest.archiveRevision, vectorCount: manifest.vectors.length };
     }
@@ -334,8 +359,6 @@ export class Archive {
         const rows = this.db.prepare("SELECT rowid, chunk_id, text, message_id FROM chunks").all();
         const rebuild = this.db.transaction(() => {
             this.db.exec("DELETE FROM chunks_fts");
-            for (const language of lexicalFields())
-                this.db.exec(`DELETE FROM chunks_fts_${language}`);
             for (const row of rows) {
                 const message = this.db.prepare("SELECT * FROM messages WHERE message_id = ?").get(row.message_id);
                 this.db.prepare(`INSERT INTO chunks_fts(rowid, subject, from_address, to_addresses, thread_subject,
@@ -344,8 +367,6 @@ export class Archive {
             }
         });
         rebuild();
-        this.db.prepare("DELETE FROM lexical_index_meta WHERE id = 1").run();
-        this.lexicalRebuildRequired = true;
         return { rows: rows.length, status: "repaired" };
     }
     upsertMessage(message) {
@@ -428,7 +449,7 @@ export class Archive {
         }
     }
     async getEmbedder() {
-        this.embedder ??= await createEmbedder();
+        this.embedder ??= await createEmbedder(this.embedderConfig);
         return this.embedder;
     }
     searchLexicalTable(table, query, filters, limit) {

package/dist/cli/index.js

@@ -2,9 +2,11 @@
 import { Command } from "commander";
 import { mkdir } from "node:fs/promises";
 import { existsSync } from "node:fs";
+import { join } from "node:path";
 import { Archive } from "../archive.js";
 import { FixtureSource, HimalayaSource } from "../source.js";
 import { redactDiagnostic } from "../redact.js";
+import { embeddingModelName, loopbackConfigFromEnvironment } from "../embedding.js";
 const program = new Command();
 program.name("mailcrawl").description("Local privacy-first email indexing CLI");
 program.option("--data-dir <path>", "archive directory", process.env.MAILCRAWL_DATA_DIR || ".mailcrawl");
@@ -22,12 +24,18 @@ program
     .option("--json")
     .action(async (options, command) => {
     const dataDir = command.parent.opts().dataDir;
+    if (options.source !== "fixture" && options.source !== "himalaya")
+        throw new Error(`unsupported source: ${options.source}`);
+    if (options.source === "fixture" && !options.fixture)
+        throw new Error("--fixture is required for fixture source");
+    if (options.source === "himalaya" && !options.account)
+        throw new Error("--account is required for himalaya source");
     await mkdir(dataDir, { recursive: true });
-    const archive = new Archive(`${dataDir}/archive.sqlite`);
+    const archive = new Archive(join(dataDir, "archive.sqlite"));
     try {
         const source = options.source === "fixture"
-            ? new FixtureSource(options.fixture)
-            : new HimalayaSource(options.account, options.mailbox, options.backend, Number(options.pageSize), options.himalayaConfig);
+            ? fixtureSource(options)
+            : himalayaSource(options);
         const excludedCategories = (options.excludeCategory.length ? options.excludeCategory : ["spam", "promotions"])
             .filter((category) => !options.includeCategory.includes(category));
         output(await archive.sync(await source.list(), { excludedCategories }), options.json);
@@ -38,12 +46,21 @@ program
 });
 program
     .command("index")
+    .alias("embed")
+    .option("--provider <provider>", "embedding provider", "local")
+    .option("--embed-url <url>")
+    .option("--embed-model <model>")
+    .option("--embed-dim <n>")
+    .option("--query-prefix <prefix>")
+    .option("--passage-prefix <prefix>")
+    .option("--embed-timeout <ms>")
     .option("--json")
     .action(async (options, command) => {
     const dataDir = command.parent.opts().dataDir;
-    const archive = new Archive(`${dataDir}/archive.sqlite`);
+    const config = embedderConfig(options);
+    const archive = new Archive(join(dataDir, "archive.sqlite"), config);
     try {
-        output({ ...await archive.indexSemanticGeneration(`${dataDir}/semantic`), embedder: "onnx-community/embeddinggemma-300m-ONNX" }, options.json);
+        output({ ...await archive.indexSemanticGeneration(join(dataDir, "semantic")), embedder: embeddingModelName(config) }, options.json);
     }
     finally {
         archive.close();
@@ -61,12 +78,19 @@ for (const mode of ["bm25", "keyword", "semantic", "hybrid"]) {
         .option("--after <date>")
         .option("--before <date>")
         .option("--limit <n>", "result limit", "10")
+        .option("--provider <provider>", "embedding provider", "local")
+        .option("--embed-url <url>")
+        .option("--embed-model <model>")
+        .option("--embed-dim <n>")
+        .option("--query-prefix <prefix>")
+        .option("--passage-prefix <prefix>")
+        .option("--embed-timeout <ms>")
         .option("--json")
         .action(async (query, options, command) => {
         const dataDir = command.parent.opts().dataDir;
-        const archive = new Archive(`${dataDir}/archive.sqlite`);
+        const archive = new Archive(join(dataDir, "archive.sqlite"), embedderConfig(options));
         try {
-            const result = mode === "bm25"
+            const result = mode === "bm25" || mode === "keyword"
                 ? await archive.searchBm25(query, filters(options), Number(options.limit))
                 : mode === "hybrid"
                     ? await archive.searchHybrid(query, filters(options), Number(options.limit))
@@ -81,7 +105,7 @@ for (const mode of ["bm25", "keyword", "semantic", "hybrid"]) {
 program
     .command("search")
     .argument("<query>")
-    .option("--mode <mode>", "keyword, bm25, semantic, hybrid", "bm25")
+    .option("--mode <mode>", "fts, keyword, bm25, semantic, hybrid", "bm25")
     .option("--account <id>")
     .option("--mailbox <name>")
     .option("--from <address>")
@@ -90,13 +114,20 @@ program
     .option("--after <date>")
     .option("--before <date>")
     .option("--limit <n>", "result limit", "10")
+    .option("--provider <provider>", "embedding provider", "local")
+    .option("--embed-url <url>")
+    .option("--embed-model <model>")
+    .option("--embed-dim <n>")
+    .option("--query-prefix <prefix>")
+    .option("--passage-prefix <prefix>")
+    .option("--embed-timeout <ms>")
     .option("--json")
     .action(async (query, options, command) => {
-    const archive = new Archive(`${command.parent.opts().dataDir}/archive.sqlite`);
+    const archive = new Archive(join(command.parent.opts().dataDir, "archive.sqlite"), embedderConfig(options));
     try {
         const filter = filters(options);
         const limit = Number(options.limit);
-        const result = options.mode === "bm25" ? await archive.searchBm25(query, filter, limit)
+        const result = options.mode === "bm25" || options.mode === "fts" || options.mode === "keyword" ? await archive.searchBm25(query, filter, limit)
             : options.mode === "semantic" ? await archive.searchSemantic(query, filter, limit)
                 : options.mode === "hybrid" ? await archive.searchHybrid(query, filter, limit)
                     : (() => { throw new Error(`unsupported search mode: ${options.mode}`); })();
@@ -111,7 +142,7 @@ program
     .command("get <messageId>")
     .option("--json")
     .action(async (messageId, options, command) => {
-    const archive = new Archive(`${command.parent.parent.opts().dataDir}/archive.sqlite`);
+    const archive = new Archive(join(command.parent.parent.opts().dataDir, "archive.sqlite"));
     try {
         output(archive.getMessage(messageId) ?? null, options.json);
     }
@@ -124,7 +155,7 @@ program
     .argument("<chunkId>")
     .option("--json")
     .action(async (chunkId, options, command) => {
-    const archive = new Archive(`${command.parent.opts().dataDir}/archive.sqlite`);
+    const archive = new Archive(join(command.parent.opts().dataDir, "archive.sqlite"));
     try {
         output(archive.getChunkContext(chunkId), options.json);
     }
@@ -141,7 +172,7 @@ program
     .option("--before <date>")
     .option("--json")
     .action(async (threadId, options, command) => {
-    const archive = new Archive(`${command.parent.parent.opts().dataDir}/archive.sqlite`);
+    const archive = new Archive(join(command.parent.parent.opts().dataDir, "archive.sqlite"));
     try {
         output(archive.getThread(threadId, filters(options)), options.json);
     }
@@ -155,7 +186,7 @@ program
     .option("--message <messageId>")
     .option("--json")
     .action(async (threadId, options, command) => {
-    const archive = new Archive(`${command.parent.opts().dataDir}/archive.sqlite`);
+    const archive = new Archive(join(command.parent.opts().dataDir, "archive.sqlite"));
     try {
         output(archive.getThreadContext(threadId, options.message), options.json);
     }
@@ -168,21 +199,21 @@ program
     .option("--json")
     .action(async (options, command) => {
     const dataDir = command.parent.opts().dataDir;
-    const archive = new Archive(`${dataDir}/archive.sqlite`);
+    const archive = new Archive(join(dataDir, "archive.sqlite"));
     try {
         let semantic = "missing";
         try {
-            semantic = archive.semanticGeneration(`${dataDir}/semantic`);
+            semantic = semanticStatus(archive, archive.semanticGeneration(join(dataDir, "semantic")));
         }
         catch (error) {
-            semantic = redactDiagnostic({ status: "stale", error: error instanceof Error ? error.message : String(error) });
+            semantic = redactDiagnostic({ status: semanticErrorStatus(error), error: error instanceof Error ? error.message : String(error) });
         }
         const semanticCommitted = typeof semantic === "object" && semantic !== null && "generation" in semantic;
         output({
             name: "mailcrawl",
-            archive: `${dataDir}/archive.sqlite`,
-            archivePresent: existsSync(`${dataDir}/archive.sqlite`),
-            fts: "available",
+            archive: join(dataDir, "archive.sqlite"),
+            archivePresent: existsSync(join(dataDir, "archive.sqlite")),
+            fts: archive.status().fts,
             semantic,
             recommendation: semanticCommitted ? "semantic index is committed" : "run sync, then index before semantic search",
         }, options.json);
@@ -191,16 +222,65 @@ program
         archive.close();
     }
 });
+program
+    .command("status")
+    .option("--json")
+    .action(async (options, command) => {
+    const dataDir = command.parent.opts().dataDir;
+    const archivePath = join(dataDir, "archive.sqlite");
+    if (!existsSync(archivePath)) {
+        output({ name: "mailcrawl", archive: archivePath, archivePresent: false, messageCount: 0, chunkCount: 0, embeddingBacklog: 0, fts: { status: "missing", rows: 0 }, semantic: { status: "missing" } }, options.json);
+        return;
+    }
+    const archive = new Archive(archivePath);
+    try {
+        let semantic = "missing";
+        try {
+            semantic = semanticStatus(archive, archive.semanticGeneration(join(dataDir, "semantic")));
+        }
+        catch (error) {
+            semantic = redactDiagnostic({ status: semanticErrorStatus(error), error: error instanceof Error ? error.message : String(error) });
+        }
+        output({ name: "mailcrawl", archive: archivePath, archivePresent: true, ...archive.status(), semantic }, options.json);
+    }
+    finally {
+        archive.close();
+    }
+});
 program
     .command("repair")
     .option("--fts")
+    .option("--semantic")
+    .option("--all")
+    .option("--provider <provider>", "embedding provider", "local")
+    .option("--embed-url <url>")
+    .option("--embed-model <model>")
+    .option("--embed-dim <n>")
+    .option("--query-prefix <prefix>")
+    .option("--passage-prefix <prefix>")
+    .option("--embed-timeout <ms>")
     .option("--json")
     .action(async (options, command) => {
-    const archive = new Archive(`${command.parent.opts().dataDir}/archive.sqlite`);
+    const dataDir = command.parent.opts().dataDir;
+    const archive = new Archive(join(dataDir, "archive.sqlite"));
     try {
-        if (!options.fts)
-            throw new Error("pass --fts");
-        output(archive.repairFts(), options.json);
+        if (!options.fts && !options.semantic && !options.all)
+            throw new Error("pass --fts, --semantic, or --all");
+        const result = {};
+        if (options.semantic || options.all) {
+            const config = embedderConfig(options);
+            const semanticArchive = config ? new Archive(join(dataDir, "archive.sqlite"), config) : archive;
+            try {
+                result.semantic = await semanticArchive.indexSemanticGeneration(join(dataDir, "semantic"));
+            }
+            finally {
+                if (semanticArchive !== archive)
+                    semanticArchive.close();
+            }
+        }
+        if (options.fts || options.all)
+            result.fts = archive.repairFts();
+        output(Object.keys(result).length === 1 ? Object.values(result)[0] : result, options.json);
     }
     finally {
         archive.close();
@@ -212,7 +292,7 @@ attachments
     .option("--message <messageId>")
     .option("--json")
     .action(async (options, command) => {
-    const archive = new Archive(`${command.parent.parent.opts().dataDir}/archive.sqlite`);
+    const archive = new Archive(join(command.parent.parent.opts().dataDir, "archive.sqlite"));
     try {
         output(archive.listAttachments(options.message), options.json);
     }
@@ -222,12 +302,37 @@ attachments
 });
 program.parseAsync().catch((error) => {
     const message = error instanceof Error ? error.message : String(error);
-    console.error(JSON.stringify({ error: message }));
+    console.error(JSON.stringify(redactDiagnostic({ error: message })));
     process.exitCode = 1;
 });
 function filters(options) {
     return { accountId: options.account, mailbox: options.mailbox, from: options.from, to: options.to, threadId: options.thread, after: options.after, before: options.before };
 }
+function embedderConfig(options) {
+    if (options.provider === "local")
+        return loopbackConfigFromEnvironment();
+    if (options.provider !== "loopback-http")
+        throw new Error(`unsupported embedding provider: ${options.provider}`);
+    if (!options.embedUrl || !options.embedModel || !options.embedDim)
+        throw new Error("--embed-url, --embed-model, and --embed-dim are required for loopback-http");
+    return { provider: "loopback-http", url: options.embedUrl, model: options.embedModel, dimension: Number(options.embedDim), queryPrefix: options.queryPrefix, passagePrefix: options.passagePrefix, timeoutMs: options.embedTimeout ? Number(options.embedTimeout) : undefined };
+}
+function fixtureSource(options) {
+    if (!options.fixture)
+        throw new Error("--fixture is required for selected source");
+    return new FixtureSource(options.fixture);
+}
+function himalayaSource(options) {
+    if (!options.account)
+        throw new Error("--account is required for selected source");
+    return new HimalayaSource(options.account, options.mailbox, options.backend, Number(options.pageSize), options.himalayaConfig);
+}
+function semanticStatus(archive, semantic) {
+    return semantic.archiveRevision === archive.status().archiveRevision ? { ...semantic, status: "healthy" } : { ...semantic, status: "stale" };
+}
+function semanticErrorStatus(error) {
+    return error instanceof Error && "code" in error && error.code === "ENOENT" ? "missing" : "corrupt";
+}
 function output(value, json) {
     if (json)
         console.log(JSON.stringify(value));

package/dist/embedding.d.ts

@@ -1,6 +1,8 @@
+import type { LoopbackHttpConfig } from "./types.js";
 export interface Embedder {
     embedDocuments(texts: string[]): Promise<number[][]>;
     embedQuery(query: string): Promise<number[]>;
 }
-export declare function createEmbedder(): Promise<Embedder>;
-export declare function embeddingModelName(): string;
+export declare function createEmbedder(config?: LoopbackHttpConfig): Promise<Embedder>;
+export declare function embeddingModelName(config?: LoopbackHttpConfig): string;
+export declare function loopbackConfigFromEnvironment(): LoopbackHttpConfig | undefined;

package/dist/embedding.js

@@ -33,13 +33,76 @@ class TestEmbedder {
         return hashVector(query);
     }
 }
-export async function createEmbedder() {
+class LoopbackHttpEmbedder {
+    config;
+    constructor(config) {
+        this.config = config;
+        const url = new URL(config.url);
+        const hostname = url.hostname.startsWith("[") ? url.hostname.slice(1, -1) : url.hostname;
+        if (url.protocol !== "http:" || !["127.0.0.1", "localhost", "::1"].includes(hostname)) {
+            throw new Error("loopback HTTP embedding URL is required");
+        }
+        if (!Number.isInteger(config.dimension) || config.dimension <= 0)
+            throw new Error("embedding dimension must be positive");
+    }
+    async embedDocuments(texts) {
+        return this.request(texts.map((text) => (this.config.passagePrefix ?? "") + text));
+    }
+    async embedQuery(query) {
+        return (await this.request([(this.config.queryPrefix ?? "") + query.trim()]))[0];
+    }
+    async request(texts) {
+        const controller = new AbortController();
+        const timeout = setTimeout(() => controller.abort(), this.config.timeoutMs ?? 30_000);
+        try {
+            const response = await fetch(this.config.url, {
+                method: "POST",
+                headers: { "content-type": "application/json" },
+                body: JSON.stringify({ model: this.config.model, texts }),
+                signal: controller.signal,
+            });
+            if (!response.ok)
+                throw new Error(`embedding provider returned HTTP ${response.status}`);
+            const payload = await response.json();
+            if (!Array.isArray(payload.embeddings) || payload.embeddings.length !== texts.length)
+                throw new Error("embedding provider returned invalid embeddings");
+            const vectors = payload.embeddings;
+            if (vectors.some((vector) => !Array.isArray(vector) || vector.length !== this.config.dimension || vector.some((value) => typeof value !== "number"))) {
+                throw new Error("embedding provider returned invalid vector dimensions");
+            }
+            return vectors;
+        }
+        finally {
+            clearTimeout(timeout);
+        }
+    }
+}
+export async function createEmbedder(config) {
+    if (config?.provider === "loopback-http")
+        return new LoopbackHttpEmbedder(config);
     if (process.env.MAILCRAWL_EMBEDDER === "mock" || process.env.NODE_ENV === "test")
         return new TestEmbedder();
     return EmbeddingGemma.create();
 }
-export function embeddingModelName() {
-    return EMBEDDING_MODEL;
+export function embeddingModelName(config) {
+    return config
+        ? `loopback-http:${config.model}:${config.dimension}:${config.url}:${config.queryPrefix ?? ""}:${config.passagePrefix ?? ""}:${config.timeoutMs ?? 30_000}`
+        : EMBEDDING_MODEL;
+}
+export function loopbackConfigFromEnvironment() {
+    if (process.env.MAILCRAWL_EMBEDDER_PROVIDER !== "loopback-http")
+        return undefined;
+    const url = process.env.MAILCRAWL_EMBED_URL;
+    const model = process.env.MAILCRAWL_EMBED_MODEL;
+    const dimension = Number(process.env.MAILCRAWL_EMBED_DIM);
+    if (!url || !model || !Number.isInteger(dimension))
+        throw new Error("MAILCRAWL_EMBED_URL, MAILCRAWL_EMBED_MODEL, and MAILCRAWL_EMBED_DIM are required");
+    return {
+        provider: "loopback-http", url, model, dimension,
+        queryPrefix: process.env.MAILCRAWL_QUERY_PREFIX,
+        passagePrefix: process.env.MAILCRAWL_PASSAGE_PREFIX,
+        timeoutMs: process.env.MAILCRAWL_EMBED_TIMEOUT ? Number(process.env.MAILCRAWL_EMBED_TIMEOUT) : undefined,
+    };
 }
 function hashVector(text) {
     const vector = new Array(128).fill(0);

package/dist/redact.js

@@ -2,6 +2,7 @@ export function redactDiagnostic(value) {
     if (typeof value === "string") {
         return value
             .replace(/([?&](?:token|password|secret|key)=)[^&\s]+/giu, "$1[REDACTED]")
+            .replace(/(?<![?&])\b(?:token|password|secret|key|credential)\s*[=:]\s*[^,\s]+/giu, "[REDACTED]")
             .replace(/\b[A-Z0-9._%+-]+@[A-Z0-9.-]+\.[A-Z]{2,}\b/giu, "[EMAIL]");
     }
     if (Array.isArray(value))

package/dist/source.js

@@ -1,6 +1,7 @@
 import { execFile } from "node:child_process";
 import { promisify } from "node:util";
 import { readFile } from "node:fs/promises";
+import { redactDiagnostic } from "./redact.js";
 const execFileAsync = promisify(execFile);
 export class FixtureSource {
     path;
@@ -30,7 +31,7 @@ export class HimalayaSource {
         if (this.backend)
             args.push("-b", this.backend);
         args.push("envelope", "list", "--mailbox", this.mailbox, "--page-size", String(this.pageSize), "--json");
-        const { stdout } = await execFileAsync("himalaya", args, { maxBuffer: 16 * 1024 * 1024 });
+        const { stdout } = await runHimalaya(args, 16 * 1024 * 1024, "envelope list");
         const payload = JSON.parse(stdout);
         const envelopes = payload.envelopes ?? (Array.isArray(payload) ? payload : []);
         return Promise.all(envelopes.map(async (envelope) => {
@@ -60,11 +61,20 @@ export class HimalayaSource {
         if (this.backend)
             args.push("-b", this.backend);
         args.push("--json", "message", "read", id, "--raw");
-        const { stdout } = await execFileAsync("himalaya", args, { maxBuffer: 32 * 1024 * 1024 });
+        const { stdout } = await runHimalaya(args, 32 * 1024 * 1024, "message read");
         const payload = JSON.parse(stdout);
         return payload.message ?? stdout;
     }
 }
+async function runHimalaya(args, maxBuffer, operation) {
+    try {
+        return await execFileAsync("himalaya", args, { maxBuffer });
+    }
+    catch (error) {
+        const detail = error instanceof Error ? error.message : String(error);
+        throw new Error(`himalaya ${operation} failed: ${redactDiagnostic(detail)}`);
+    }
+}
 function address(value) {
     if (typeof value === "string")
         return value;

package/dist/types.d.ts

@@ -1,4 +1,14 @@
 export type SearchMode = "keyword" | "bm25" | "semantic" | "hybrid";
+export type EmbedderProvider = "local" | "loopback-http";
+export interface LoopbackHttpConfig {
+    provider: "loopback-http";
+    url: string;
+    model: string;
+    dimension: number;
+    queryPrefix?: string;
+    passagePrefix?: string;
+    timeoutMs?: number;
+}
 export interface MailMessage {
     accountId: string;
     mailbox: string;

package/docs/architecture.md

@@ -159,10 +159,10 @@ All commands support `--json` where machine-readable output is useful.
 mailcrawl doctor
 mailcrawl status
 mailcrawl sync
-mailcrawl embed [--limit N]
-mailcrawl search --mode fts|semantic|hybrid [--limit N] [--mailbox NAME] QUERY
+mailcrawl embed
+mailcrawl search --mode fts|bm25|keyword|semantic|hybrid [--limit N] [--mailbox NAME] QUERY
 mailcrawl message get MESSAGE_ID
-mailcrawl repair [--fts|--vectors|--all]
+mailcrawl repair [--fts|--semantic|--all]
 ```
 
 Example sync response:

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@nomadamas/mailcrawl",
-  "version": "0.1.5",
+  "version": "0.1.6",
   "description": "Local Himalaya-backed incremental email indexing and hybrid search CLI",
   "license": "MIT",
   "type": "module",
@@ -48,6 +48,7 @@
     "@types/better-sqlite3": "^7.6.13",
     "@types/node": "^24.3.0",
     "tsx": "^4.20.5",
+    "typescript": "^5.9.3",
     "vitest": "^3.2.4"
   },
   "engines": {

package/skills/mailcrawl/SKILL.md

@@ -71,7 +71,8 @@ mailcrawl sync --source fixture --fixture ./messages.json --json
 
 ## Safe read and maintenance commands
 
-Search modes are `bm25`, `keyword`, `semantic`, and `hybrid`. Use metadata
+Search modes are `fts`, `bm25`, `keyword`, `semantic`, and `hybrid`; `fts` and
+`keyword` are aliases for BM25 lexical search. Use metadata
 filters such as `--mailbox`, `--from`, `--to`, `--thread`, `--after`, and
 `--before`. Empty queries and unsupported modes fail with a non-zero exit.
 
@@ -89,7 +90,11 @@ Check health and rebuild lexical data when needed:
 
 ```bash
 mailcrawl doctor --json
+mailcrawl status --json
+mailcrawl embed --json
 mailcrawl repair --fts --json
+mailcrawl repair --semantic --json
+mailcrawl repair --all --json
 ```
 
 `doctor` reports archive, FTS, and semantic-generation state. `repair` is a