@nomadamas/mailcrawl 0.1.0

package/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2026 NomaDamas
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

package/README.md

@@ -0,0 +1,61 @@
+# mailcrawl
+
+Local, privacy-first email indexing CLI for AI agents and humans.
+
+mailcrawl uses the configured [Himalaya](https://github.com/pimalaya/himalaya)
+account as its mail transport, then maintains a local normalized archive with
+incremental synchronization, email-aware chunking, full-text search (FTS5),
+semantic vector search, and hybrid retrieval.
+
+> Status: concept scaffold. The architecture is documented before the first
+> implementation increment.
+
+## Product goal
+
+Make one local command useful to any client:
+
+```bash
+mailcrawl sync --json
+mailcrawl embed --json
+mailcrawl search --mode hybrid --json "계약 갱신 조건"
+mailcrawl status --json
+mailcrawl doctor --json
+```
+
+The CLI owns email synchronization and indexes. Consumers such as AutoRAG,
+OpenClaw, MCP servers, Raycast, and custom scripts consume its stable JSON
+surface instead of opening the archive database.
+
+## Planned capabilities
+
+- Himalaya-backed IMAP/JMAP/Gmail/Microsoft Graph/Maildir access
+- Stable account/mailbox/message identity and cursor state
+- MIME normalization, HTML-to-text conversion, and quoted-reply handling
+- Email-aware, thread-aware chunking
+- Incremental archive, FTS5, and embedding updates
+- LanceDB vector storage with configurable local embedding providers
+- FTS, semantic, and hybrid search modes
+- JSON output, bounded diagnostics, `status`, `doctor`, and `repair`
+- No credential values in logs, diagnostics, or indexed metadata by default
+
+See [`docs/architecture.md`](docs/architecture.md) for the proposed design,
+data model, CLI contract, and implementation milestones.
+
+## Releasing
+
+GitHub Release `vX.Y.Z` (must match `package.json`) publishes `@nomadamas/mailcrawl` to npm with OIDC trusted publishing. No `NPM_TOKEN` is stored in GitHub.
+
+One-time setup:
+
+1. Create a GitHub Environment named `release` on this repository.
+2. On [npm trusted publishers](https://docs.npmjs.com/trusted-publishers) for `@nomadamas/mailcrawl`, add GitHub Actions:
+   - Organization: `NomaDamas`
+   - Repository: `mailcrawl`
+   - Workflow filename: `release.yml`
+   - Environment: `release`
+
+Then bump the version, push `main`, and publish a GitHub Release whose tag is `v<version>`. The workflow tests, builds, publishes with provenance, and attaches the tarball to the release.
+
+## License
+
+MIT. This project is not yet a production release.

package/dist/archive.d.ts

@@ -0,0 +1,55 @@
+import Database from "better-sqlite3";
+import type { ClassificationPolicy, Chunk, MailMessage, NormalizedMessage, SearchFilters, SearchHit, SyncReport } from "./types.js";
+export declare class Archive {
+    readonly db: Database.Database;
+    constructor(path?: string);
+    close(): void;
+    sync(messages: MailMessage[], policy?: ClassificationPolicy): Promise<SyncReport>;
+    searchBm25(query: string, filters?: SearchFilters, limit?: number): SearchHit[];
+    indexSemantic(): {
+        embedded: number;
+        reused: number;
+        archiveRevision: string;
+    };
+    indexSemanticGeneration(root: string): {
+        generation: string;
+        embedded: number;
+        reused: number;
+    };
+    semanticGeneration(root: string): {
+        generation: string;
+        archiveRevision: string;
+        vectorCount: number;
+    };
+    searchSemantic(query: string, filters?: SearchFilters, limit?: number): SearchHit[];
+    searchHybrid(query: string, filters?: SearchFilters, limit?: number): SearchHit[];
+    getMessage(messageId: string): NormalizedMessage | undefined;
+    listAttachments(messageId?: string): Array<{
+        attachmentId: string;
+        messageId: string;
+        name: string;
+        mimeType: string;
+        size: number | null;
+        contentHash: string | null;
+        extractedText: string | null;
+    }>;
+    getThread(threadId: string, filters?: SearchFilters): NormalizedMessage[];
+    getThreadContext(threadId: string, messageId?: string): {
+        previous: NormalizedMessage[];
+        current?: NormalizedMessage;
+        next: NormalizedMessage[];
+    };
+    getChunkContext(chunkId: string): {
+        previous?: Chunk;
+        current?: Chunk;
+        next?: Chunk;
+    };
+    repairFts(): {
+        rows: number;
+        status: "repaired";
+    };
+    private upsertMessage;
+    private removeMessage;
+    private replaceMessageChunks;
+    private revision;
+}

package/dist/archive.js

@@ -0,0 +1,388 @@
+import Database from "better-sqlite3";
+import { createHash } from "node:crypto";
+import { mkdirSync, readFileSync, renameSync, rmSync, writeFileSync } from "node:fs";
+import { join } from "node:path";
+import { buildChunks } from "./chunk.js";
+import { normalizeMessage } from "./normalize.js";
+import { snippet } from "./util.js";
+export class Archive {
+    db;
+    constructor(path = ":memory:") {
+        this.db = new Database(path);
+        this.db.pragma("journal_mode = WAL");
+        this.db.pragma("foreign_keys = ON");
+        migrate(this.db);
+    }
+    close() {
+        this.db.close();
+    }
+    async sync(messages, policy = {}) {
+        const excludedCategories = new Set((policy.excludedCategories ?? ["spam", "promotions"]).map(normalizeCategory));
+        const normalized = (await Promise.all(messages.map(normalizeMessage))).map((message) => ({
+            ...message,
+            messageId: `${message.accountId}:${message.messageId}`,
+            threadId: `${message.accountId}:${message.threadId}`,
+            providerKey: `${message.accountId}:${message.providerKey}`,
+            inReplyTo: message.inReplyTo ? `${message.accountId}:${message.inReplyTo}` : undefined,
+        }));
+        const excluded = normalized.filter((message) => message.categories.some((category) => excludedCategories.has(category)));
+        const included = normalized.filter((message) => !message.categories.some((category) => excludedCategories.has(category)));
+        const existing = this.db.prepare("SELECT provider_key, normalized_hash FROM messages").all();
+        const previous = new Map(existing.map((row) => [row.provider_key, row.normalized_hash]));
+        let added = 0;
+        let updated = 0;
+        let unchanged = 0;
+        const touched = new Set();
+        const transaction = this.db.transaction((items) => {
+            for (const message of items) {
+                const oldHash = previous.get(message.providerKey);
+                if (!oldHash)
+                    added++;
+                else if (oldHash !== message.normalizedHash)
+                    updated++;
+                else
+                    unchanged++;
+                if (oldHash !== message.normalizedHash)
+                    touched.add(message.threadId);
+                this.upsertMessage(message);
+                if (oldHash !== message.normalizedHash)
+                    this.replaceMessageChunks(message);
+            }
+        });
+        transaction(included);
+        for (const message of excluded)
+            this.removeMessage(message);
+        const chunks = Number(this.db.prepare("SELECT COUNT(*) AS count FROM chunks").get().count);
+        const backlog = Number(this.db.prepare("SELECT COUNT(*) AS count FROM embedding_queue WHERE state = 'pending'").get().count);
+        return {
+            added, updated, deleted: 0, unchanged, touchedThreads: touched.size,
+            rebuiltThreads: touched.size, chunksAdded: chunks, chunksDeleted: 0,
+            embeddingBacklog: backlog, archiveRevision: this.revision(),
+            excluded: excluded.length,
+            excludedByReason: countExcluded(excluded, excludedCategories),
+        };
+    }
+    searchBm25(query, filters = {}, limit = 10) {
+        if (!query.trim())
+            throw new Error("empty query");
+        const clauses = ["chunks_fts MATCH ?"];
+        const params = [literalFtsQuery(query)];
+        addFilters(clauses, params, filters);
+        const sql = `SELECT c.chunk_id, c.message_id, c.thread_id, c.account_id, c.mailbox,
+      m.subject, m.from_address, m.to_addresses, m.date, snippet(chunks_fts, 0, '[', ']', '…', 32) AS snippet,
+      bm25(chunks_fts, 8.0, 5.0, 2.0, 1.0, 1.0, 1.0, 0.5) AS score
+      FROM chunks_fts JOIN chunks c ON c.rowid = chunks_fts.rowid
+      JOIN messages m ON m.message_id = c.message_id
+      WHERE ${clauses.join(" AND ")} ORDER BY score LIMIT ?`;
+        params.push(limit);
+        return this.db.prepare(sql).all(...params).map((row) => hydrate(row, "bm25", query));
+    }
+    indexSemantic() {
+        const rows = this.db.prepare("SELECT chunk_id, text, content_hash FROM chunks ORDER BY chunk_id").all();
+        let embedded = 0;
+        let reused = 0;
+        const upsert = this.db.prepare(`INSERT INTO semantic_vectors
+      (chunk_id, content_hash, vector) VALUES (?, ?, ?)
+      ON CONFLICT(chunk_id) DO UPDATE SET content_hash=excluded.content_hash, vector=excluded.vector`);
+        const transaction = this.db.transaction(() => {
+            for (const row of rows) {
+                const old = this.db.prepare("SELECT content_hash FROM semantic_vectors WHERE chunk_id = ?").get(row.chunk_id);
+                if (old?.content_hash === row.content_hash) {
+                    reused++;
+                    continue;
+                }
+                upsert.run(row.chunk_id, row.content_hash, JSON.stringify(embed(row.text)));
+                embedded++;
+            }
+        });
+        transaction();
+        return { embedded, reused, archiveRevision: this.revision() };
+    }
+    indexSemanticGeneration(root) {
+        const currentPath = join(root, "CURRENT");
+        const generationRoot = join(root, "generations");
+        mkdirSync(generationRoot, { recursive: true });
+        const generation = `gen-${this.revision().slice(0, 16)}-${Date.now()}`;
+        const staging = join(generationRoot, `.${generation}.staging`);
+        mkdirSync(staging);
+        try {
+            const report = this.indexSemantic();
+            const vectors = this.db.prepare("SELECT chunk_id, content_hash, vector FROM semantic_vectors ORDER BY chunk_id").all();
+            writeFileSync(join(staging, "manifest.json"), JSON.stringify({
+                archiveRevision: this.revision(), vectors, model: "local-hash-v1",
+            }));
+            renameSync(staging, join(generationRoot, generation));
+            const pointer = join(root, `.CURRENT.${process.pid}`);
+            writeFileSync(pointer, `${generation}\n`);
+            renameSync(pointer, currentPath);
+            return { generation, embedded: report.embedded, reused: report.reused };
+        }
+        catch (error) {
+            rmSync(staging, { recursive: true, force: true });
+            throw error;
+        }
+    }
+    semanticGeneration(root) {
+        const generation = readFileSync(join(root, "CURRENT"), "utf8").trim();
+        const manifest = JSON.parse(readFileSync(join(root, "generations", generation, "manifest.json"), "utf8"));
+        return { generation, archiveRevision: manifest.archiveRevision, vectorCount: manifest.vectors.length };
+    }
+    searchSemantic(query, filters = {}, limit = 10) {
+        if (!query.trim())
+            throw new Error("empty query");
+        const queryVector = embed(query);
+        const clauses = ["1 = 1"];
+        const params = [];
+        addFilters(clauses, params, filters);
+        const rows = this.db.prepare(`SELECT v.vector, c.chunk_id, c.message_id, c.thread_id, c.account_id, c.mailbox,
+      m.subject, m.from_address, m.to_addresses, m.date, c.text
+      FROM semantic_vectors v JOIN chunks c ON c.chunk_id = v.chunk_id JOIN messages m ON m.message_id = c.message_id
+      WHERE ${clauses.join(" AND ")}`).all(...params);
+        return rows.map((row) => ({ row, score: dot(queryVector, JSON.parse(row.vector)) }))
+            .sort((a, b) => b.score - a.score || a.row.chunk_id.localeCompare(b.row.chunk_id))
+            .slice(0, limit)
+            .map(({ row, score }) => ({
+            chunkId: row.chunk_id, messageId: row.message_id, threadId: row.thread_id,
+            accountId: row.account_id, mailbox: row.mailbox, subject: row.subject,
+            from: row.from_address, to: JSON.parse(row.to_addresses), date: row.date,
+            snippet: snippet(row.text, query), score, mode: "semantic",
+        }));
+    }
+    searchHybrid(query, filters = {}, limit = 10) {
+        const lexical = this.searchBm25(query, filters, limit * 2);
+        const semantic = this.searchSemantic(query, filters, limit * 2);
+        const merged = new Map();
+        for (const [index, hit] of lexical.entries())
+            merged.set(hit.chunkId, { ...hit, score: 0.5 * (1 - index / Math.max(1, lexical.length)) });
+        for (const [index, hit] of semantic.entries()) {
+            const prior = merged.get(hit.chunkId);
+            const score = 0.5 * (1 - index / Math.max(1, semantic.length));
+            merged.set(hit.chunkId, prior ? { ...prior, score: prior.score + score, mode: "hybrid" } : { ...hit, score, mode: "hybrid" });
+        }
+        return [...merged.values()].sort((a, b) => b.score - a.score || a.chunkId.localeCompare(b.chunkId)).slice(0, limit);
+    }
+    getMessage(messageId) {
+        const row = this.db.prepare("SELECT * FROM messages WHERE message_id = ?").get(messageId);
+        return row && messageRow(row);
+    }
+    listAttachments(messageId) {
+        const rows = (messageId
+            ? this.db.prepare("SELECT * FROM attachments WHERE message_id = ? ORDER BY attachment_id").all(messageId)
+            : this.db.prepare("SELECT * FROM attachments ORDER BY message_id, attachment_id").all());
+        return rows.map((row) => ({
+            attachmentId: row.attachment_id,
+            messageId: row.message_id,
+            name: row.name,
+            mimeType: row.mime_type,
+            size: row.size,
+            contentHash: row.content_hash,
+            extractedText: row.extracted_text,
+        }));
+    }
+    getThread(threadId, filters = {}) {
+        const clauses = ["thread_id = ?"];
+        const params = [threadId];
+        addFilters(clauses, params, filters);
+        return this.db.prepare(`SELECT * FROM messages WHERE ${clauses.join(" AND ")} ORDER BY date, message_id`).all(...params).map(messageRow);
+    }
+    getThreadContext(threadId, messageId) {
+        const messages = this.getThread(threadId);
+        const index = messageId ? messages.findIndex((message) => message.messageId === messageId) : 0;
+        const current = index >= 0 ? messages[index] : undefined;
+        return {
+            previous: index > 0 ? messages.slice(0, index) : [],
+            current,
+            next: index >= 0 ? messages.slice(index + 1) : messages,
+        };
+    }
+    getChunkContext(chunkId) {
+        const current = this.db.prepare("SELECT * FROM chunks WHERE chunk_id = ?").get(chunkId);
+        if (!current)
+            return {};
+        const base = this.db.prepare("SELECT * FROM chunks WHERE thread_id = ? AND (started_at < ? OR (started_at = ? AND rowid < ?)) ORDER BY started_at DESC, rowid DESC LIMIT 1").get(current.thread_id, current.started_at, current.started_at, current.rowid);
+        const next = this.db.prepare("SELECT * FROM chunks WHERE thread_id = ? AND (started_at > ? OR (started_at = ? AND rowid > ?)) ORDER BY started_at, rowid LIMIT 1").get(current.thread_id, current.started_at, current.started_at, current.rowid);
+        return { previous: base && chunkRow(base), current: chunkRow(current), next: next && chunkRow(next) };
+    }
+    repairFts() {
+        const rows = this.db.prepare("SELECT rowid, chunk_id, text, message_id FROM chunks").all();
+        const rebuild = this.db.transaction(() => {
+            this.db.exec("DELETE FROM chunks_fts");
+            for (const row of rows) {
+                const message = this.db.prepare("SELECT * FROM messages WHERE message_id = ?").get(row.message_id);
+                this.db.prepare(`INSERT INTO chunks_fts(rowid, subject, from_address, to_addresses, thread_subject,
+          body_latest, body_quoted, forwarded_text, attachment_text)
+          VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)`).run(row.rowid, message.subject, message.from_address, message.to_addresses, message.subject, row.text, "", "", message.attachment_text || "");
+            }
+        });
+        rebuild();
+        return { rows: rows.length, status: "repaired" };
+    }
+    upsertMessage(message) {
+        this.db.prepare(`INSERT INTO messages
+      (message_id, account_id, mailbox, provider_key, thread_id, in_reply_to, subject, from_address,
+       to_addresses, cc_addresses, date, latest_text, quoted_text, attachment_text, normalized_hash,
+       labels, flags, classifications)
+      VALUES (@messageId, @accountId, @mailbox, @providerKey, @threadId, @inReplyTo, @subject, @from,
+       @to, @cc, @date, @latestText, @quotedText, @attachmentText, @normalizedHash,
+       @labels, @flags, @classifications)
+      ON CONFLICT(message_id) DO UPDATE SET account_id=excluded.account_id, mailbox=excluded.mailbox,
+       provider_key=excluded.provider_key, thread_id=excluded.thread_id, in_reply_to=excluded.in_reply_to,
+       subject=excluded.subject, from_address=excluded.from_address, to_addresses=excluded.to_addresses,
+       cc_addresses=excluded.cc_addresses, date=excluded.date, latest_text=excluded.latest_text,
+       quoted_text=excluded.quoted_text, attachment_text=excluded.attachment_text,
+       normalized_hash=excluded.normalized_hash, labels=excluded.labels, flags=excluded.flags,
+       classifications=excluded.classifications`).run({
+            ...message, inReplyTo: message.inReplyTo ?? null,
+            to: JSON.stringify(message.to), cc: JSON.stringify(message.cc),
+            labels: JSON.stringify(message.labels ?? []), flags: JSON.stringify(message.flags ?? []),
+            classifications: JSON.stringify(message.classifications ?? []),
+            attachmentText: attachmentText(message),
+        });
+        this.db.prepare("DELETE FROM attachments WHERE message_id = ?").run(message.messageId);
+        for (const [index, attachment] of (message.attachments || []).entries()) {
+            this.db.prepare(`INSERT INTO attachments
+        (attachment_id, message_id, name, mime_type, size, content_hash, extracted_text)
+        VALUES (?, ?, ?, ?, ?, ?, ?)`).run(`${message.messageId}:${index}`, message.messageId, attachment.name, attachment.mimeType, attachment.size ?? null, attachment.contentHash ?? null, attachment.text ?? null);
+        }
+    }
+    removeMessage(message) {
+        const stored = this.db.prepare("SELECT message_id FROM messages WHERE provider_key = ?").get(message.providerKey);
+        if (!stored)
+            return;
+        const rows = this.db.prepare("SELECT rowid FROM chunks WHERE message_id = ?").all(stored.message_id);
+        for (const row of rows)
+            this.db.prepare("DELETE FROM chunks_fts WHERE rowid = ?").run(row.rowid);
+        this.db.prepare("DELETE FROM embedding_queue WHERE chunk_id IN (SELECT chunk_id FROM chunks WHERE message_id = ?)").run(stored.message_id);
+        this.db.prepare("DELETE FROM semantic_vectors WHERE chunk_id IN (SELECT chunk_id FROM chunks WHERE message_id = ?)").run(stored.message_id);
+        this.db.prepare("DELETE FROM messages WHERE message_id = ?").run(stored.message_id);
+    }
+    replaceMessageChunks(message) {
+        const old = this.db.prepare("SELECT rowid, chunk_id FROM chunks WHERE message_id = ?").all(message.messageId);
+        for (const row of old)
+            this.db.prepare("DELETE FROM chunks_fts WHERE rowid = ?").run(row.rowid);
+        this.db.prepare("DELETE FROM chunks WHERE message_id = ?").run(message.messageId);
+        for (const chunk of buildChunks(message)) {
+            const result = this.db.prepare(`INSERT INTO chunks
+        (chunk_id, account_id, mailbox, message_id, thread_id, section, ordinal, text, started_at, ended_at, content_hash)
+        VALUES (@chunkId, @accountId, @mailbox, @messageId, @threadId, @section, @ordinal, @text, @startedAt, @endedAt, @contentHash)`).run(chunk);
+            this.db.prepare(`INSERT INTO chunks_fts(rowid, subject, from_address, to_addresses, thread_subject,
+        body_latest, body_quoted, forwarded_text, attachment_text)
+        VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)`).run(result.lastInsertRowid, message.subject, message.from, message.to.join(" "), message.normalizedSubject, chunk.section === "latest" ? chunk.text : "", chunk.section === "quoted" ? chunk.text : "", "", chunk.section === "attachment" ? chunk.text : "");
+            this.db.prepare(`INSERT INTO embedding_queue(chunk_id, content_hash, state, attempts)
+        VALUES (?, ?, 'pending', 0) ON CONFLICT(chunk_id) DO UPDATE SET content_hash=excluded.content_hash, state='pending'`).run(chunk.chunkId, chunk.contentHash);
+        }
+    }
+    revision() {
+        const rows = this.db.prepare("SELECT chunk_id, content_hash FROM chunks ORDER BY chunk_id").all();
+        return createHash("sha256").update(rows.map((row) => `${row.chunk_id}\0${row.content_hash}`).join("\0")).digest("hex");
+    }
+}
+function migrate(db) {
+    db.exec(`CREATE TABLE IF NOT EXISTS messages (
+    message_id TEXT PRIMARY KEY, account_id TEXT NOT NULL, mailbox TEXT NOT NULL,
+    provider_key TEXT NOT NULL UNIQUE, thread_id TEXT NOT NULL, in_reply_to TEXT,
+    subject TEXT NOT NULL, from_address TEXT NOT NULL, to_addresses TEXT NOT NULL,
+    cc_addresses TEXT NOT NULL, date TEXT NOT NULL, latest_text TEXT NOT NULL,
+    quoted_text TEXT NOT NULL, attachment_text TEXT NOT NULL DEFAULT '', normalized_hash TEXT NOT NULL,
+    labels TEXT NOT NULL DEFAULT '[]', flags TEXT NOT NULL DEFAULT '[]',
+    classifications TEXT NOT NULL DEFAULT '[]'
+  );
+  CREATE TABLE IF NOT EXISTS chunks (
+    chunk_id TEXT PRIMARY KEY, account_id TEXT NOT NULL, mailbox TEXT NOT NULL,
+    message_id TEXT NOT NULL REFERENCES messages(message_id) ON DELETE CASCADE,
+    thread_id TEXT NOT NULL, section TEXT NOT NULL, ordinal INTEGER NOT NULL,
+    text TEXT NOT NULL, started_at TEXT NOT NULL, ended_at TEXT NOT NULL, content_hash TEXT NOT NULL
+  );
+  CREATE INDEX IF NOT EXISTS idx_chunks_thread_time ON chunks(thread_id, started_at);
+  CREATE TABLE IF NOT EXISTS embedding_queue (
+    chunk_id TEXT PRIMARY KEY, content_hash TEXT NOT NULL, state TEXT NOT NULL, attempts INTEGER NOT NULL
+  );
+  CREATE TABLE IF NOT EXISTS semantic_vectors (
+    chunk_id TEXT PRIMARY KEY, content_hash TEXT NOT NULL, vector TEXT NOT NULL
+  );
+  CREATE TABLE IF NOT EXISTS attachments (
+    attachment_id TEXT PRIMARY KEY, message_id TEXT NOT NULL REFERENCES messages(message_id) ON DELETE CASCADE,
+    name TEXT NOT NULL, mime_type TEXT NOT NULL, size INTEGER, content_hash TEXT, extracted_text TEXT
+  );
+  CREATE VIRTUAL TABLE IF NOT EXISTS chunks_fts USING fts5(
+    subject, from_address, to_addresses, thread_subject, body_latest,
+    body_quoted, forwarded_text, attachment_text
+  );`);
+    const columns = db.prepare("PRAGMA table_info(messages)").all();
+    const existingColumns = new Set(columns.map((column) => column.name));
+    for (const column of ["labels", "flags", "classifications"]) {
+        if (!existingColumns.has(column))
+            db.exec(`ALTER TABLE messages ADD COLUMN ${column} TEXT NOT NULL DEFAULT '[]'`);
+    }
+}
+function literalFtsQuery(query) {
+    return query.trim().split(/\s+/).map((term) => `"${term.replaceAll('"', '""')}"`).join(" ");
+}
+function addFilters(clauses, params, filters) {
+    if (filters.accountId) {
+        clauses.push("c.account_id = ?");
+        params.push(filters.accountId);
+    }
+    if (filters.mailbox) {
+        clauses.push("c.mailbox = ?");
+        params.push(filters.mailbox);
+    }
+    if (filters.from) {
+        clauses.push("lower(m.from_address) = lower(?)");
+        params.push(filters.from);
+    }
+    if (filters.to) {
+        clauses.push("EXISTS (SELECT 1 FROM json_each(m.to_addresses) WHERE lower(value) = lower(?))");
+        params.push(filters.to);
+    }
+    if (filters.threadId) {
+        clauses.push("c.thread_id = ?");
+        params.push(filters.threadId);
+    }
+    if (filters.after) {
+        clauses.push("m.date >= ?");
+        params.push(filters.after);
+    }
+    if (filters.before) {
+        clauses.push("m.date <= ?");
+        params.push(filters.before);
+    }
+}
+function normalizeCategory(value) {
+    return value.trim().toLocaleLowerCase().replace(/^category[_-]/, "").replace(/^label[_-]/, "");
+}
+function countExcluded(messages, excluded) {
+    const counts = {};
+    for (const message of messages) {
+        for (const category of message.categories) {
+            if (excluded.has(category))
+                counts[category] = (counts[category] ?? 0) + 1;
+        }
+    }
+    return counts;
+}
+function hydrate(row, mode, query) {
+    return { chunkId: row.chunk_id, messageId: row.message_id, threadId: row.thread_id, accountId: row.account_id, mailbox: row.mailbox, subject: row.subject, from: row.from_address, to: JSON.parse(row.to_addresses), date: row.date, snippet: row.snippet || snippet(row.subject, query), score: row.score, mode };
+}
+function messageRow(row) {
+    return { accountId: row.account_id, mailbox: row.mailbox, providerKey: row.provider_key, messageId: row.message_id, threadId: row.thread_id, inReplyTo: row.in_reply_to || undefined, subject: row.subject, from: row.from_address, to: JSON.parse(row.to_addresses), cc: JSON.parse(row.cc_addresses), date: row.date, text: row.latest_text, latestText: row.latest_text, quotedText: row.quoted_text, normalizedSubject: row.subject.toLocaleLowerCase(), normalizedHash: row.normalized_hash, labels: JSON.parse(row.labels || "[]"), flags: JSON.parse(row.flags || "[]"), classifications: JSON.parse(row.classifications || "[]"), categories: [...new Set([...JSON.parse(row.labels || "[]"), ...JSON.parse(row.flags || "[]"), ...JSON.parse(row.classifications || "[]")].map(normalizeCategory))] };
+}
+function chunkRow(row) {
+    return { chunkId: row.chunk_id, accountId: row.account_id, mailbox: row.mailbox, messageId: row.message_id, threadId: row.thread_id, section: row.section, ordinal: row.ordinal, text: row.text, startedAt: row.started_at, endedAt: row.ended_at, contentHash: row.content_hash };
+}
+function attachmentText(message) {
+    return (message.attachments || []).map((attachment) => attachment.text || "").filter(Boolean).join("\n");
+}
+function embed(text) {
+    const vector = new Array(128).fill(0);
+    const normalized = text.toLocaleLowerCase().normalize("NFKC");
+    for (let index = 0; index < normalized.length; index++) {
+        const code = normalized.codePointAt(index) ?? 0;
+        vector[(code + index * 31) % vector.length] += 1;
+    }
+    const magnitude = Math.sqrt(vector.reduce((sum, value) => sum + value * value, 0)) || 1;
+    return vector.map((value) => value / magnitude);
+}
+function dot(left, right) {
+    return left.reduce((sum, value, index) => sum + value * (right[index] ?? 0), 0);
+}

package/dist/chunk.d.ts

@@ -0,0 +1,2 @@
+import type { Chunk, NormalizedMessage } from "./types.js";
+export declare function buildChunks(message: NormalizedMessage): Chunk[];

package/dist/chunk.js

@@ -0,0 +1,42 @@
+import { hash, makeId } from "./util.js";
+const MAX_CHARS = 2400;
+export function buildChunks(message) {
+    const sections = [
+        ["latest", message.latestText],
+        ["quoted", message.quotedText],
+        ["attachment", (message.attachments || []).map((a) => a.text || "").filter(Boolean).join("\n")],
+    ];
+    const chunks = [];
+    for (const [section, text] of sections) {
+        if (!text.trim())
+            continue;
+        const parts = splitText(text);
+        parts.forEach((part, ordinal) => {
+            const chunkId = makeId(message.accountId, message.mailbox, message.messageId, section, String(ordinal), part);
+            chunks.push({
+                chunkId,
+                accountId: message.accountId,
+                mailbox: message.mailbox,
+                messageId: message.messageId,
+                threadId: message.threadId,
+                section,
+                ordinal,
+                text: part,
+                startedAt: message.date,
+                endedAt: message.date,
+                contentHash: hash(part),
+            });
+        });
+    }
+    return chunks;
+}
+function splitText(text) {
+    const paragraphs = text.split(/\n\s*\n/).map((part) => part.trim()).filter(Boolean);
+    const output = [];
+    for (const paragraph of paragraphs.length ? paragraphs : [text.trim()]) {
+        for (let start = 0; start < paragraph.length; start += MAX_CHARS) {
+            output.push(paragraph.slice(start, start + MAX_CHARS).trim());
+        }
+    }
+    return output.filter(Boolean);
+}

package/dist/cli/index.d.ts

@@ -0,0 +1,2 @@
+#!/usr/bin/env node
+export {};

package/dist/cli/index.js

@@ -0,0 +1,238 @@
+#!/usr/bin/env node
+import { Command } from "commander";
+import { mkdir } from "node:fs/promises";
+import { existsSync } from "node:fs";
+import { Archive } from "../archive.js";
+import { FixtureSource, HimalayaSource } from "../source.js";
+import { redactDiagnostic } from "../redact.js";
+const program = new Command();
+program.name("mailcrawl").description("Local privacy-first email indexing CLI");
+program.option("--data-dir <path>", "archive directory", process.env.MAILCRAWL_DATA_DIR || ".mailcrawl");
+program
+    .command("sync")
+    .option("--source <source>", "fixture or himalaya", "himalaya")
+    .option("--fixture <path>")
+    .option("--account <name>")
+    .option("--mailbox <name>", "mailbox name", "INBOX")
+    .option("--backend <name>")
+    .option("--page-size <n>", "envelopes per page", "1000")
+    .option("--himalaya-config <path>")
+    .option("--include-category <name>", "include a normally excluded category", collect, [])
+    .option("--exclude-category <name>", "exclude a classification category", collect, [])
+    .option("--json")
+    .action(async (options, command) => {
+    const dataDir = command.parent.opts().dataDir;
+    await mkdir(dataDir, { recursive: true });
+    const archive = new Archive(`${dataDir}/archive.sqlite`);
+    try {
+        const source = options.source === "fixture"
+            ? new FixtureSource(options.fixture)
+            : new HimalayaSource(options.account, options.mailbox, options.backend, Number(options.pageSize), options.himalayaConfig);
+        const excludedCategories = (options.excludeCategory.length ? options.excludeCategory : ["spam", "promotions"])
+            .filter((category) => !options.includeCategory.includes(category));
+        output(await archive.sync(await source.list(), { excludedCategories }), options.json);
+    }
+    finally {
+        archive.close();
+    }
+});
+program
+    .command("index")
+    .option("--json")
+    .action(async (options, command) => {
+    const dataDir = command.parent.opts().dataDir;
+    const archive = new Archive(`${dataDir}/archive.sqlite`);
+    try {
+        output({ ...archive.indexSemanticGeneration(`${dataDir}/semantic`), embedder: "local-hash-v1" }, options.json);
+    }
+    finally {
+        archive.close();
+    }
+});
+for (const mode of ["bm25", "keyword", "semantic", "hybrid"]) {
+    program
+        .command(`search:${mode}`)
+        .argument("<query>")
+        .option("--account <id>")
+        .option("--mailbox <name>")
+        .option("--from <address>")
+        .option("--to <address>")
+        .option("--thread <id>")
+        .option("--after <date>")
+        .option("--before <date>")
+        .option("--limit <n>", "result limit", "10")
+        .option("--json")
+        .action(async (query, options, command) => {
+        const dataDir = command.parent.opts().dataDir;
+        const archive = new Archive(`${dataDir}/archive.sqlite`);
+        try {
+            const result = mode === "bm25"
+                ? archive.searchBm25(query, filters(options), Number(options.limit))
+                : mode === "hybrid"
+                    ? archive.searchHybrid(query, filters(options), Number(options.limit))
+                    : archive.searchSemantic(query, filters(options), Number(options.limit));
+            output(result, options.json);
+        }
+        finally {
+            archive.close();
+        }
+    });
+}
+program
+    .command("search")
+    .argument("<query>")
+    .option("--mode <mode>", "keyword, bm25, semantic, hybrid", "bm25")
+    .option("--account <id>")
+    .option("--mailbox <name>")
+    .option("--from <address>")
+    .option("--to <address>")
+    .option("--thread <id>")
+    .option("--after <date>")
+    .option("--before <date>")
+    .option("--limit <n>", "result limit", "10")
+    .option("--json")
+    .action(async (query, options, command) => {
+    const archive = new Archive(`${command.parent.opts().dataDir}/archive.sqlite`);
+    try {
+        const filter = filters(options);
+        const limit = Number(options.limit);
+        const result = options.mode === "bm25" ? archive.searchBm25(query, filter, limit)
+            : options.mode === "semantic" ? archive.searchSemantic(query, filter, limit)
+                : options.mode === "hybrid" ? archive.searchHybrid(query, filter, limit)
+                    : (() => { throw new Error(`unsupported search mode: ${options.mode}`); })();
+        output(result, options.json);
+    }
+    finally {
+        archive.close();
+    }
+});
+program
+    .command("message")
+    .command("get <messageId>")
+    .option("--json")
+    .action(async (messageId, options, command) => {
+    const archive = new Archive(`${command.parent.parent.opts().dataDir}/archive.sqlite`);
+    try {
+        output(archive.getMessage(messageId) ?? null, options.json);
+    }
+    finally {
+        archive.close();
+    }
+});
+program
+    .command("chunk-context")
+    .argument("<chunkId>")
+    .option("--json")
+    .action(async (chunkId, options, command) => {
+    const archive = new Archive(`${command.parent.opts().dataDir}/archive.sqlite`);
+    try {
+        output(archive.getChunkContext(chunkId), options.json);
+    }
+    finally {
+        archive.close();
+    }
+});
+program
+    .command("thread")
+    .command("get <threadId>")
+    .option("--from <address>")
+    .option("--to <address>")
+    .option("--after <date>")
+    .option("--before <date>")
+    .option("--json")
+    .action(async (threadId, options, command) => {
+    const archive = new Archive(`${command.parent.parent.opts().dataDir}/archive.sqlite`);
+    try {
+        output(archive.getThread(threadId, filters(options)), options.json);
+    }
+    finally {
+        archive.close();
+    }
+});
+program
+    .command("thread-context")
+    .argument("<threadId>")
+    .option("--message <messageId>")
+    .option("--json")
+    .action(async (threadId, options, command) => {
+    const archive = new Archive(`${command.parent.opts().dataDir}/archive.sqlite`);
+    try {
+        output(archive.getThreadContext(threadId, options.message), options.json);
+    }
+    finally {
+        archive.close();
+    }
+});
+program
+    .command("doctor")
+    .option("--json")
+    .action(async (options, command) => {
+    const dataDir = command.parent.opts().dataDir;
+    const archive = new Archive(`${dataDir}/archive.sqlite`);
+    try {
+        let semantic = "missing";
+        try {
+            semantic = archive.semanticGeneration(`${dataDir}/semantic`);
+        }
+        catch (error) {
+            semantic = redactDiagnostic({ status: "stale", error: error instanceof Error ? error.message : String(error) });
+        }
+        output({
+            name: "mailcrawl",
+            archive: `${dataDir}/archive.sqlite`,
+            archivePresent: existsSync(`${dataDir}/archive.sqlite`),
+            fts: "available",
+            semantic,
+            recommendation: semantic === "missing" ? "run sync, then index before semantic search" : "semantic index is committed",
+        }, options.json);
+    }
+    finally {
+        archive.close();
+    }
+});
+program
+    .command("repair")
+    .option("--fts")
+    .option("--json")
+    .action(async (options, command) => {
+    const archive = new Archive(`${command.parent.opts().dataDir}/archive.sqlite`);
+    try {
+        if (!options.fts)
+            throw new Error("pass --fts");
+        output(archive.repairFts(), options.json);
+    }
+    finally {
+        archive.close();
+    }
+});
+const attachments = program.command("attachments");
+attachments
+    .command("list")
+    .option("--message <messageId>")
+    .option("--json")
+    .action(async (options, command) => {
+    const archive = new Archive(`${command.parent.parent.opts().dataDir}/archive.sqlite`);
+    try {
+        output(archive.listAttachments(options.message), options.json);
+    }
+    finally {
+        archive.close();
+    }
+});
+program.parseAsync().catch((error) => {
+    const message = error instanceof Error ? error.message : String(error);
+    console.error(JSON.stringify({ error: message }));
+    process.exitCode = 1;
+});
+function filters(options) {
+    return { accountId: options.account, mailbox: options.mailbox, from: options.from, to: options.to, threadId: options.thread, after: options.after, before: options.before };
+}
+function output(value, json) {
+    if (json)
+        console.log(JSON.stringify(value));
+    else
+        console.log(JSON.stringify(value, null, 2));
+}
+function collect(value, previous) {
+    return previous.concat(value.toLocaleLowerCase());
+}

package/dist/normalize.d.ts

@@ -0,0 +1,2 @@
+import type { MailMessage, NormalizedMessage } from "./types.js";
+export declare function normalizeMessage(input: MailMessage): Promise<NormalizedMessage>;

package/dist/normalize.js

@@ -0,0 +1,56 @@
+import { simpleParser } from "mailparser";
+import { hash, makeId, normalizeSubject } from "./util.js";
+export async function normalizeMessage(input) {
+    let text = input.text || "";
+    let html = input.html;
+    let attachments = input.attachments;
+    if (input.rawMime) {
+        const parsed = await simpleParser(input.rawMime);
+        text = parsed.text || "";
+        html = typeof parsed.html === "string" ? parsed.html : undefined;
+        attachments = (parsed.attachments || []).map((attachment) => {
+            const mimeType = attachment.contentType || "application/octet-stream";
+            const content = attachment.content || Buffer.alloc(0);
+            const textLike = mimeType.startsWith("text/") && content.length <= 1_000_000;
+            return {
+                name: attachment.filename || "unnamed",
+                mimeType,
+                size: attachment.size ?? content.length,
+                contentHash: hash(content.toString("base64")),
+                text: textLike ? content.toString("utf8") : undefined,
+            };
+        });
+    }
+    const quoted = extractQuoted(text);
+    const latest = text.slice(0, text.length - quoted.length).trim();
+    const subject = input.subject.trim();
+    const messageId = input.messageId || input.providerKey;
+    const threadId = input.threadId || makeId(input.accountId, normalizeSubject(subject));
+    const normalizedHash = hash(JSON.stringify([subject, input.from, input.to, input.cc, latest, quoted, input.labels, input.flags, input.classifications]));
+    const categories = [...new Set([
+            ...(input.classifications || []),
+            ...(input.labels || []),
+            ...(input.flags || []),
+        ].map(normalizeCategory).filter(Boolean))];
+    return {
+        ...input,
+        messageId,
+        threadId,
+        html,
+        attachments,
+        normalizedSubject: normalizeSubject(subject),
+        latestText: latest,
+        quotedText: quoted,
+        text: latest,
+        normalizedHash,
+        categories,
+    };
+}
+function normalizeCategory(value) {
+    return value.trim().toLocaleLowerCase().replace(/^category[_-]/, "").replace(/^label[_-]/, "");
+}
+function extractQuoted(text) {
+    const lines = text.split(/\r?\n/);
+    const marker = lines.findIndex((line) => /^(?:On .+ wrote:|>{1,}|[- ]*Original Message[- ]*$)/iu.test(line.trim()));
+    return marker < 0 ? "" : lines.slice(marker).join("\n").trim();
+}

package/dist/redact.d.ts

@@ -0,0 +1 @@
+export declare function redactDiagnostic(value: unknown): unknown;

package/dist/redact.js

@@ -0,0 +1,15 @@
+export function redactDiagnostic(value) {
+    if (typeof value === "string") {
+        return value
+            .replace(/([?&](?:token|password|secret|key)=)[^&\s]+/giu, "$1[REDACTED]")
+            .replace(/\b[A-Z0-9._%+-]+@[A-Z0-9.-]+\.[A-Z]{2,}\b/giu, "[EMAIL]");
+    }
+    if (Array.isArray(value))
+        return value.map(redactDiagnostic);
+    if (value && typeof value === "object") {
+        return Object.fromEntries(Object.entries(value).map(([key, entry]) => [
+            key, /password|secret|token|credential/iu.test(key) ? "[REDACTED]" : redactDiagnostic(entry),
+        ]));
+    }
+    return value;
+}

package/dist/source.d.ts

@@ -0,0 +1,19 @@
+import type { MailMessage } from "./types.js";
+export interface MailSource {
+    list(): Promise<MailMessage[]>;
+}
+export declare class FixtureSource implements MailSource {
+    private readonly path;
+    constructor(path: string);
+    list(): Promise<MailMessage[]>;
+}
+export declare class HimalayaSource implements MailSource {
+    private readonly account;
+    private readonly mailbox;
+    private readonly backend?;
+    private readonly pageSize;
+    private readonly config?;
+    constructor(account: string, mailbox?: string, backend?: string | undefined, pageSize?: number, config?: string | undefined);
+    list(): Promise<MailMessage[]>;
+    private read;
+}

package/dist/source.js

@@ -0,0 +1,84 @@
+import { execFile } from "node:child_process";
+import { promisify } from "node:util";
+import { readFile } from "node:fs/promises";
+const execFileAsync = promisify(execFile);
+export class FixtureSource {
+    path;
+    constructor(path) {
+        this.path = path;
+    }
+    async list() {
+        const raw = await readFile(this.path, "utf8");
+        return JSON.parse(raw);
+    }
+}
+export class HimalayaSource {
+    account;
+    mailbox;
+    backend;
+    pageSize;
+    config;
+    constructor(account, mailbox = "INBOX", backend, pageSize = 1000, config) {
+        this.account = account;
+        this.mailbox = mailbox;
+        this.backend = backend;
+        this.pageSize = pageSize;
+        this.config = config;
+    }
+    async list() {
+        const args = this.config ? ["-c", this.config, "-a", this.account] : ["-a", this.account];
+        if (this.backend)
+            args.push("-b", this.backend);
+        args.push("envelope", "list", "--mailbox", this.mailbox, "--page-size", String(this.pageSize), "--json");
+        const { stdout } = await execFileAsync("himalaya", args, { maxBuffer: 16 * 1024 * 1024 });
+        const payload = JSON.parse(stdout);
+        const envelopes = payload.envelopes ?? (Array.isArray(payload) ? payload : []);
+        return Promise.all(envelopes.map(async (envelope) => {
+            const providerKey = String(envelope.id ?? envelope.uid ?? envelope["message-id"]);
+            const rawMime = await this.read(providerKey);
+            return {
+                accountId: this.account,
+                mailbox: this.mailbox,
+                providerKey,
+                messageId: envelope["message-id"],
+                inReplyTo: envelope["in-reply-to"]?.[0],
+                subject: envelope.subject ?? "",
+                from: address(envelope.from),
+                to: addresses(envelope.to),
+                cc: addresses(envelope.cc),
+                date: envelope.date ?? new Date(0).toISOString(),
+                text: envelope.body ?? envelope.snippet ?? "",
+                labels: strings(envelope.labels),
+                flags: strings(envelope.flags),
+                classifications: strings(envelope.classifications),
+                rawMime,
+            };
+        }));
+    }
+    async read(id) {
+        const args = this.config ? ["-c", this.config, "-a", this.account] : ["-a", this.account];
+        if (this.backend)
+            args.push("-b", this.backend);
+        args.push("--json", "message", "read", id, "--raw");
+        const { stdout } = await execFileAsync("himalaya", args, { maxBuffer: 32 * 1024 * 1024 });
+        const payload = JSON.parse(stdout);
+        return payload.message ?? stdout;
+    }
+}
+function address(value) {
+    if (typeof value === "string")
+        return value;
+    if (Array.isArray(value))
+        return value.length ? address(value[0]) : "";
+    if (value && typeof value === "object" && "email" in value)
+        return String(value.email);
+    return "";
+}
+function addresses(value) {
+    if (!Array.isArray(value))
+        return value ? [address(value)] : [];
+    return value.map(address).filter(Boolean);
+}
+function strings(value) {
+    return Array.isArray(value) ? value.map(String) : [];
+}

package/dist/types.d.ts

@@ -0,0 +1,90 @@
+export type SearchMode = "keyword" | "bm25" | "semantic" | "hybrid";
+export interface MailMessage {
+    accountId: string;
+    mailbox: string;
+    providerKey: string;
+    messageId?: string;
+    threadId?: string;
+    inReplyTo?: string;
+    subject: string;
+    from: string;
+    to: string[];
+    cc: string[];
+    date: string;
+    text: string;
+    html?: string;
+    rawMime?: string;
+    attachments?: AttachmentInput[];
+    labels?: string[];
+    flags?: string[];
+    classifications?: string[];
+}
+export interface AttachmentInput {
+    name: string;
+    mimeType: string;
+    size?: number;
+    text?: string;
+    contentHash?: string;
+}
+export interface NormalizedMessage extends MailMessage {
+    messageId: string;
+    threadId: string;
+    normalizedSubject: string;
+    latestText: string;
+    quotedText: string;
+    normalizedHash: string;
+    categories: string[];
+}
+export interface Chunk {
+    chunkId: string;
+    accountId: string;
+    mailbox: string;
+    messageId: string;
+    threadId: string;
+    section: string;
+    ordinal: number;
+    text: string;
+    startedAt: string;
+    endedAt: string;
+    contentHash: string;
+}
+export interface SearchFilters {
+    accountId?: string;
+    mailbox?: string;
+    from?: string;
+    to?: string;
+    threadId?: string;
+    after?: string;
+    before?: string;
+}
+export interface SearchHit {
+    chunkId: string;
+    messageId: string;
+    threadId: string;
+    accountId: string;
+    mailbox: string;
+    subject: string;
+    from: string;
+    to: string[];
+    date: string;
+    snippet: string;
+    score: number;
+    mode: SearchMode;
+}
+export interface SyncReport {
+    added: number;
+    updated: number;
+    deleted: number;
+    unchanged: number;
+    touchedThreads: number;
+    rebuiltThreads: number;
+    chunksAdded: number;
+    chunksDeleted: number;
+    embeddingBacklog: number;
+    archiveRevision: string;
+    excluded: number;
+    excludedByReason: Record<string, number>;
+}
+export interface ClassificationPolicy {
+    excludedCategories?: string[];
+}

package/dist/types.js

@@ -0,0 +1 @@
+export {};

package/dist/util.d.ts

@@ -0,0 +1,4 @@
+export declare function hash(value: string): string;
+export declare function normalizeSubject(subject: string): string;
+export declare function makeId(...parts: string[]): string;
+export declare function snippet(text: string, query: string, maxChars?: number): string;

package/dist/util.js

@@ -0,0 +1,18 @@
+import { createHash } from "node:crypto";
+export function hash(value) {
+    return createHash("sha256").update(value).digest("hex");
+}
+export function normalizeSubject(subject) {
+    return subject.replace(/^(?:(?:re|fw|fwd)\s*:\s*)+/giu, "").trim().toLocaleLowerCase();
+}
+export function makeId(...parts) {
+    return hash(parts.join("\u0000")).slice(0, 24);
+}
+export function snippet(text, query, maxChars = 240) {
+    const clean = text.trim();
+    if (clean.length <= maxChars)
+        return clean;
+    const index = clean.toLocaleLowerCase().indexOf(query.trim().toLocaleLowerCase());
+    const start = Math.max(0, index < 0 ? 0 : index - 60);
+    return clean.slice(start, start + maxChars);
+}

package/package.json

@@ -0,0 +1,52 @@
+{
+  "name": "@nomadamas/mailcrawl",
+  "version": "0.1.0",
+  "description": "Local Himalaya-backed incremental email indexing and hybrid search CLI",
+  "license": "MIT",
+  "type": "module",
+  "bin": {
+    "mailcrawl": "dist/cli/index.js"
+  },
+  "files": [
+    "dist",
+    "skills",
+    "README.md",
+    "LICENSE"
+  ],
+  "repository": {
+    "type": "git",
+    "url": "git+https://github.com/NomaDamas/mailcrawl.git"
+  },
+  "bugs": {
+    "url": "https://github.com/NomaDamas/mailcrawl/issues"
+  },
+  "homepage": "https://github.com/NomaDamas/mailcrawl#readme",
+  "publishConfig": {
+    "access": "public",
+    "registry": "https://registry.npmjs.org/"
+  },
+  "scripts": {
+    "build": "tsc -p tsconfig.json",
+    "typecheck": "tsc --noEmit",
+    "test": "vitest run",
+    "test:watch": "vitest",
+    "validate:skill": "node scripts/validate-skill.mjs",
+    "cli": "node dist/cli/index.js",
+    "commit:check": "git diff --check"
+  },
+  "dependencies": {
+    "better-sqlite3": "^12.4.1",
+    "commander": "^14.0.0",
+    "mailparser": "^3.9.0",
+    "mime-types": "^3.0.1"
+  },
+  "devDependencies": {
+    "@types/better-sqlite3": "^7.6.13",
+    "@types/node": "^24.3.0",
+    "tsx": "^4.20.5",
+    "vitest": "^3.2.4"
+  },
+  "engines": {
+    "node": ">=24"
+  }
+}

package/skills/mailcrawl/SKILL.md

@@ -0,0 +1,124 @@
+---
+name: mailcrawl
+description: Search and maintain a local, privacy-first email archive through the mailcrawl CLI.
+---
+
+# mailcrawl
+
+Use `mailcrawl` as a local, read-oriented email retrieval tool. It indexes
+messages obtained through the configured Himalaya account and returns bounded
+JSON suitable for an agent.
+
+## Installation and activation
+
+Install the package and build the CLI from the repository:
+
+```bash
+npm install
+npm run build
+```
+
+Expose `dist/cli/index.js` as `mailcrawl` on `PATH`, or invoke it with
+`node dist/cli/index.js`. Agent skill runners should discover this file through
+the `skills/mailcrawl/SKILL.md` directory and activate it only when the user
+requests local email synchronization or search.
+
+Required runtime dependencies are Node.js 24+, the package dependencies, and
+the Himalaya CLI for live synchronization. Fixture synchronization needs no
+mail account.
+
+## Data and credential boundaries
+
+The archive lives under `.mailcrawl` by default. Set `--data-dir` or
+`MAILCRAWL_DATA_DIR` to choose another local directory. The archive contains
+sensitive message metadata and content; keep its permissions private and do
+not commit it.
+
+Himalaya resolves credentials. Pass only account, backend, mailbox, and
+configuration path options to mailcrawl. Never place passwords, tokens, raw
+provider responses, or message bodies in logs, prompts, issue comments, or
+diagnostics.
+
+## Workflow
+
+Synchronization is explicit. mailcrawl does not run a background daemon:
+
+```bash
+mailcrawl sync --account personal --mailbox INBOX --json
+mailcrawl index --json
+mailcrawl search --mode hybrid --json "contract renewal"
+```
+
+For scheduled operation, configure an external scheduler with an explicit
+command, such as a user-level cron or systemd timer. Run `sync` first and
+`index` afterward; inspect JSON exit status before handing results to an
+agent.
+
+Use a fixture for deterministic development:
+
+```bash
+mailcrawl sync --source fixture --fixture ./messages.json --json
+```
+
+## Safe read and maintenance commands
+
+Search modes are `bm25`, `keyword`, `semantic`, and `hybrid`. Use metadata
+filters such as `--mailbox`, `--from`, `--to`, `--thread`, `--after`, and
+`--before`. Empty queries and unsupported modes fail with a non-zero exit.
+
+Navigate from a hit without dumping the entire archive:
+
+```bash
+mailcrawl message get MESSAGE_ID --json
+mailcrawl thread get THREAD_ID --json
+mailcrawl thread-context THREAD_ID --message MESSAGE_ID --json
+mailcrawl chunk-context CHUNK_ID --json
+mailcrawl attachments list --message MESSAGE_ID --json
+```
+
+Check health and rebuild lexical data when needed:
+
+```bash
+mailcrawl doctor --json
+mailcrawl repair --fts --json
+```
+
+`doctor` reports archive, FTS, and semantic-generation state. `repair` is a
+local maintenance operation and should be run only when diagnostics indicate
+that the corresponding index is inconsistent.
+
+## JSON contract
+
+Use `--json` for machine-readable output. Search results contain stable
+`chunkId`, `messageId`, `threadId`, `accountId`, `mailbox`, `subject`, `from`,
+`to`, `date`, `snippet`, `score`, and `mode` fields. Sync returns counts,
+`archiveRevision`, and classification exclusion diagnostics. Do not assume
+that an omitted result means a remote deletion.
+
+Treat snippets and identifiers as sensitive. Fetch message or thread content
+only when it is required to answer the user, and return the smallest useful
+excerpt.
+
+## Safety boundary
+
+This skill supports synchronization, indexing, search, inspection, and local
+repair only. It does not send, delete, move, or mutate remote email. Never
+invoke an external mail-sending command through this skill. If a user asks to
+send mail, obtain explicit approval and hand off to a separate, user-visible
+mail client workflow.
+
+## Verification
+
+From a checkout, verify installation and invocation with:
+
+```bash
+npm install
+npm test
+npm run typecheck
+npm run build
+node dist/cli/index.js --help
+node dist/cli/index.js doctor --json
+```
+
+The help output proves the CLI is callable. `doctor --json` proves that the
+agent can inspect a local data directory without opening a remote account.