@nomadamas/mailcrawl 0.1.3 → 0.1.5

package/dist/archive.d.ts

@@ -4,21 +4,26 @@ export declare class Archive {
     readonly db: Database.Database;
     private embedder?;
     private lexical?;
+    private operationTail;
     private lexicalRebuildRequired;
     constructor(path?: string);
     close(): void;
     sync(messages: MailMessage[], policy?: ClassificationPolicy): Promise<SyncReport>;
+    private syncUnlocked;
     searchBm25(query: string, filters?: SearchFilters, limit?: number): Promise<SearchHit[]>;
     indexSemantic(): Promise<{
         embedded: number;
         reused: number;
         archiveRevision: string;
     }>;
+    private indexSemanticUnlocked;
     indexSemanticGeneration(root: string): Promise<{
         generation: string;
         embedded: number;
         reused: number;
     }>;
+    private indexSemanticGenerationUnlocked;
+    private cleanupSemanticGenerations;
     semanticGeneration(root: string): {
         generation: string;
         archiveRevision: string;
@@ -55,6 +60,7 @@ export declare class Archive {
     private removeMessage;
     private replaceMessageChunks;
     private revision;
+    private runExclusive;
     private getEmbedder;
     private searchLexicalTable;
 }

package/dist/archive.js

@@ -1,22 +1,26 @@
 import Database from "better-sqlite3";
 import { createHash } from "node:crypto";
-import { mkdirSync, readFileSync, renameSync, rmSync, writeFileSync } from "node:fs";
+import { mkdirSync, readFileSync, readdirSync, renameSync, rmSync, writeFileSync } from "node:fs";
 import { join } from "node:path";
 import { buildChunks } from "./chunk.js";
 import { normalizeMessage } from "./normalize.js";
-import { snippet } from "./util.js";
+import { scopedId, snippet } from "./util.js";
 import { createEmbedder, embeddingModelName } from "./embedding.js";
 import { createLexicalAnalyzers, languagesForText, lexicalFields, LEXICAL_ANALYZER_VERSION } from "./lexical.js";
+const RETAINED_SEMANTIC_GENERATIONS = 2;
+const SEMANTIC_GENERATION_NAME = /^gen-[0-9a-f]{16}-[0-9]+$/;
 export class Archive {
     db;
     embedder;
     lexical;
+    operationTail = Promise.resolve();
     lexicalRebuildRequired;
     constructor(path = ":memory:") {
         this.db = new Database(path);
         this.db.pragma("journal_mode = WAL");
         this.db.pragma("foreign_keys = ON");
         migrate(this.db);
+        cleanupOrphanedSemanticRows(this.db);
         this.lexicalRebuildRequired = this.db.prepare("SELECT version FROM lexical_index_meta WHERE id = 1").get() === undefined;
         if (!this.lexicalRebuildRequired) {
             const row = this.db.prepare("SELECT version FROM lexical_index_meta WHERE id = 1").get();
@@ -32,14 +36,23 @@ export class Archive {
         this.db.close();
     }
     async sync(messages, policy = {}) {
+        return this.runExclusive(() => this.syncUnlocked(messages, policy));
+    }
+    async syncUnlocked(messages, policy = {}) {
+        for (const message of messages) {
+            if (typeof message.providerKey !== "string" || !message.providerKey.trim())
+                throw new Error("message provider identity is required");
+        }
         const excludedCategories = new Set((policy.excludedCategories ?? ["spam", "promotions"]).map(normalizeCategory));
         const normalized = (await Promise.all(messages.map(normalizeMessage))).map((message) => ({
             ...message,
-            messageId: `${message.accountId}:${message.messageId}`,
-            threadId: `${message.accountId}:${message.threadId}`,
-            providerKey: `${message.accountId}:${message.providerKey}`,
-            inReplyTo: message.inReplyTo ? `${message.accountId}:${message.inReplyTo}` : undefined,
+            messageId: scopedId(message.accountId, message.mailbox, message.messageId),
+            threadId: scopedId(message.accountId, message.mailbox, message.threadId),
+            providerKey: scopedId(message.accountId, message.mailbox, message.providerKey),
+            inReplyTo: message.inReplyTo ? scopedId(message.accountId, message.mailbox, message.inReplyTo) : undefined,
         }));
+        validateIdentities(normalized);
+        validateStoredIdentities(this.db, normalized);
         const excluded = normalized.filter((message) => message.categories.some((category) => excludedCategories.has(category)));
         const included = normalized.filter((message) => !message.categories.some((category) => excludedCategories.has(category)));
         const existing = this.db.prepare("SELECT provider_key, normalized_hash FROM messages").all();
@@ -126,18 +139,29 @@ export class Archive {
             .slice(0, limit).map(({ hit, score }) => ({ ...hit, score }));
     }
     async indexSemantic() {
+        return this.runExclusive(() => this.indexSemanticUnlocked());
+    }
+    async indexSemanticUnlocked() {
+        const completeQueueRow = this.db.prepare("UPDATE embedding_queue SET state = 'complete' WHERE chunk_id = ?");
+        const reconcile = this.db.transaction(() => {
+            this.db.prepare("DELETE FROM embedding_queue WHERE chunk_id NOT IN (SELECT chunk_id FROM chunks)").run();
+            this.db.prepare(`INSERT INTO embedding_queue(chunk_id, content_hash, state, attempts)
+        SELECT chunk_id, content_hash, 'pending', 0 FROM chunks
+        WHERE chunk_id NOT IN (SELECT chunk_id FROM embedding_queue)`).run();
+        });
+        reconcile();
         const rows = this.db.prepare("SELECT chunk_id, text, content_hash FROM chunks ORDER BY chunk_id").all();
         let embedded = 0;
         let reused = 0;
         const upsert = this.db.prepare(`INSERT INTO semantic_vectors
       (chunk_id, content_hash, model, vector) VALUES (?, ?, ?, ?)
       ON CONFLICT(chunk_id) DO UPDATE SET content_hash=excluded.content_hash, model=excluded.model, vector=excluded.vector`);
-        const embedder = await this.getEmbedder();
         const pending = [];
         const transaction = this.db.transaction(() => {
             for (const row of rows) {
                 const old = this.db.prepare("SELECT content_hash, model FROM semantic_vectors WHERE chunk_id = ?").get(row.chunk_id);
                 if (old?.content_hash === row.content_hash && old.model === embeddingModelName()) {
+                    completeQueueRow.run(row.chunk_id);
                     reused++;
                     continue;
                 }
@@ -145,10 +169,14 @@ export class Archive {
             }
         });
         transaction();
+        if (!pending.length)
+            return { embedded, reused, archiveRevision: this.revision() };
+        const embedder = await this.getEmbedder();
         const vectors = await embedder.embedDocuments(pending.map((row) => row.text));
         const write = this.db.transaction(() => {
             for (const [index, row] of pending.entries()) {
                 upsert.run(row.chunk_id, row.content_hash, embeddingModelName(), JSON.stringify(vectors[index]));
+                completeQueueRow.run(row.chunk_id);
                 embedded++;
             }
         });
@@ -156,29 +184,65 @@ export class Archive {
         return { embedded, reused, archiveRevision: this.revision() };
     }
     async indexSemanticGeneration(root) {
+        return this.runExclusive(() => this.indexSemanticGenerationUnlocked(root));
+    }
+    async indexSemanticGenerationUnlocked(root) {
         const currentPath = join(root, "CURRENT");
         const generationRoot = join(root, "generations");
+        const previousVectors = this.db.prepare("SELECT chunk_id, content_hash, model, vector FROM semantic_vectors ORDER BY chunk_id").all();
+        const previousQueue = this.db.prepare("SELECT chunk_id, content_hash, state, attempts FROM embedding_queue ORDER BY chunk_id").all();
         mkdirSync(generationRoot, { recursive: true });
         const generation = `gen-${this.revision().slice(0, 16)}-${Date.now()}`;
         const staging = join(generationRoot, `.${generation}.staging`);
+        const publishedGeneration = join(generationRoot, generation);
         mkdirSync(staging);
         try {
-            const report = await this.indexSemantic();
-            const vectors = this.db.prepare("SELECT chunk_id, content_hash, vector FROM semantic_vectors ORDER BY chunk_id").all();
+            const report = await this.indexSemanticUnlocked();
+            const vectors = this.db.prepare(`SELECT v.chunk_id, v.content_hash, v.vector
+        FROM semantic_vectors v JOIN chunks c ON c.chunk_id = v.chunk_id
+        ORDER BY v.chunk_id`).all();
             writeFileSync(join(staging, "manifest.json"), JSON.stringify({
                 archiveRevision: this.revision(), vectors, model: embeddingModelName(),
             }));
-            renameSync(staging, join(generationRoot, generation));
+            renameSync(staging, publishedGeneration);
             const pointer = join(root, `.CURRENT.${process.pid}`);
             writeFileSync(pointer, `${generation}\n`);
             renameSync(pointer, currentPath);
+            this.cleanupSemanticGenerations(generationRoot, generation);
             return { generation, embedded: report.embedded, reused: report.reused };
         }
         catch (error) {
             rmSync(staging, { recursive: true, force: true });
+            rmSync(publishedGeneration, { recursive: true, force: true });
+            const restore = this.db.transaction(() => {
+                this.db.exec("DELETE FROM semantic_vectors; DELETE FROM embedding_queue;");
+                const restoreVector = this.db.prepare("INSERT INTO semantic_vectors(chunk_id, content_hash, model, vector) VALUES (?, ?, ?, ?)");
+                for (const row of previousVectors)
+                    restoreVector.run(row.chunk_id, row.content_hash, row.model, row.vector);
+                const restoreQueue = this.db.prepare("INSERT INTO embedding_queue(chunk_id, content_hash, state, attempts) VALUES (?, ?, ?, ?)");
+                for (const row of previousQueue)
+                    restoreQueue.run(row.chunk_id, row.content_hash, row.state, row.attempts);
+            });
+            restore();
             throw error;
         }
     }
+    cleanupSemanticGenerations(generationRoot, activeGeneration) {
+        const generations = readdirSync(generationRoot, { withFileTypes: true })
+            .filter((entry) => entry.isDirectory() && SEMANTIC_GENERATION_NAME.test(entry.name))
+            .map((entry) => entry.name)
+            .sort((left, right) => {
+            const leftTimestamp = Number(left.slice(left.lastIndexOf("-") + 1));
+            const rightTimestamp = Number(right.slice(right.lastIndexOf("-") + 1));
+            return rightTimestamp - leftTimestamp || right.localeCompare(left);
+        });
+        const retained = new Set(generations.slice(0, RETAINED_SEMANTIC_GENERATIONS));
+        retained.add(activeGeneration);
+        for (const generation of generations) {
+            if (!retained.has(generation))
+                rmSync(join(generationRoot, generation), { recursive: true, force: true });
+        }
+    }
     semanticGeneration(root) {
         const generation = readFileSync(join(root, "CURRENT"), "utf8").trim();
         const manifest = JSON.parse(readFileSync(join(root, "generations", generation, "manifest.json"), "utf8"));
@@ -313,17 +377,14 @@ export class Archive {
         }
     }
     removeMessage(message) {
-        const stored = this.db.prepare("SELECT message_id FROM messages WHERE provider_key = ?").get(message.providerKey);
-        if (!stored)
-            return;
-        const rows = this.db.prepare("SELECT rowid FROM chunks WHERE message_id = ?").all(stored.message_id);
+        const rows = this.db.prepare("SELECT rowid FROM chunks WHERE message_id = (SELECT message_id FROM messages WHERE provider_key = ?)").all(message.providerKey);
         for (const row of rows)
             this.db.prepare("DELETE FROM chunks_fts WHERE rowid = ?").run(row.rowid);
         for (const language of lexicalFields())
-            this.db.prepare(`DELETE FROM chunks_fts_${language} WHERE chunk_id IN (SELECT chunk_id FROM chunks WHERE message_id = ?)`).run(stored.message_id);
-        this.db.prepare("DELETE FROM embedding_queue WHERE chunk_id IN (SELECT chunk_id FROM chunks WHERE message_id = ?)").run(stored.message_id);
-        this.db.prepare("DELETE FROM semantic_vectors WHERE chunk_id IN (SELECT chunk_id FROM chunks WHERE message_id = ?)").run(stored.message_id);
-        this.db.prepare("DELETE FROM messages WHERE message_id = ?").run(stored.message_id);
+            this.db.prepare(`DELETE FROM chunks_fts_${language} WHERE chunk_id IN (SELECT chunk_id FROM chunks WHERE message_id = (SELECT message_id FROM messages WHERE provider_key = ?))`).run(message.providerKey);
+        this.db.prepare("DELETE FROM embedding_queue WHERE chunk_id IN (SELECT chunk_id FROM chunks WHERE message_id = (SELECT message_id FROM messages WHERE provider_key = ?))").run(message.providerKey);
+        this.db.prepare("DELETE FROM semantic_vectors WHERE chunk_id IN (SELECT chunk_id FROM chunks WHERE message_id = (SELECT message_id FROM messages WHERE provider_key = ?))").run(message.providerKey);
+        this.db.prepare("DELETE FROM messages WHERE provider_key = ?").run(message.providerKey);
     }
     replaceMessageChunks(message, analyzedChunks) {
         const old = this.db.prepare("SELECT rowid, chunk_id FROM chunks WHERE message_id = ?").all(message.messageId);
@@ -331,6 +392,8 @@ export class Archive {
             this.db.prepare("DELETE FROM chunks_fts WHERE rowid = ?").run(row.rowid);
         for (const language of lexicalFields())
             this.db.prepare(`DELETE FROM chunks_fts_${language} WHERE chunk_id IN (SELECT chunk_id FROM chunks WHERE message_id = ?)`).run(message.messageId);
+        this.db.prepare("DELETE FROM embedding_queue WHERE chunk_id IN (SELECT chunk_id FROM chunks WHERE message_id = ?)").run(message.messageId);
+        this.db.prepare("DELETE FROM semantic_vectors WHERE chunk_id IN (SELECT chunk_id FROM chunks WHERE message_id = ?)").run(message.messageId);
         this.db.prepare("DELETE FROM chunks WHERE message_id = ?").run(message.messageId);
         for (const chunk of buildChunks(message)) {
             const result = this.db.prepare(`INSERT INTO chunks
@@ -351,6 +414,19 @@ export class Archive {
         const rows = this.db.prepare("SELECT chunk_id, content_hash FROM chunks ORDER BY chunk_id").all();
         return createHash("sha256").update(rows.map((row) => `${row.chunk_id}\0${row.content_hash}`).join("\0")).digest("hex");
     }
+    async runExclusive(operation) {
+        const previous = this.operationTail;
+        let release = () => { };
+        const turn = new Promise((resolve) => { release = resolve; });
+        this.operationTail = previous.then(() => turn);
+        await previous;
+        try {
+            return await operation();
+        }
+        finally {
+            release();
+        }
+    }
     async getEmbedder() {
         this.embedder ??= await createEmbedder();
         return this.embedder;
@@ -424,6 +500,12 @@ function migrate(db) {
             db.exec(`ALTER TABLE messages ADD COLUMN ${column} TEXT NOT NULL DEFAULT '[]'`);
     }
 }
+function cleanupOrphanedSemanticRows(db) {
+    db.prepare(`DELETE FROM embedding_queue
+    WHERE NOT EXISTS (SELECT 1 FROM chunks WHERE chunks.chunk_id = embedding_queue.chunk_id)`).run();
+    db.prepare(`DELETE FROM semantic_vectors
+    WHERE NOT EXISTS (SELECT 1 FROM chunks WHERE chunks.chunk_id = semantic_vectors.chunk_id)`).run();
+}
 function literalFtsQuery(query) {
     return query.trim().split(/\s+/).map((term) => `"${term.replaceAll('"', '""')}"`).join(" ");
 }
@@ -460,6 +542,28 @@ function addFilters(clauses, params, filters) {
 function normalizeCategory(value) {
     return value.trim().toLocaleLowerCase().replace(/^category[_-]/, "").replace(/^label[_-]/, "");
 }
+function validateIdentities(messages) {
+    const providerKeys = new Set();
+    const messageIds = new Set();
+    for (const message of messages) {
+        if (!message.providerKey)
+            throw new Error("message provider identity is required");
+        if (providerKeys.has(message.providerKey))
+            throw new Error("duplicate message provider identity");
+        providerKeys.add(message.providerKey);
+        if (messageIds.has(message.messageId))
+            throw new Error("duplicate message identity");
+        messageIds.add(message.messageId);
+    }
+}
+function validateStoredIdentities(db, messages) {
+    const byProviderKey = db.prepare("SELECT message_id FROM messages WHERE provider_key = ?");
+    for (const message of messages) {
+        const storedProvider = byProviderKey.get(message.providerKey);
+        if (storedProvider && storedProvider.message_id !== message.messageId)
+            throw new Error("provider identity already belongs to another message identity");
+    }
+}
 function countExcluded(messages, excluded) {
     const counts = {};
     for (const message of messages) {

package/dist/cli/index.js

@@ -177,13 +177,14 @@ program
         catch (error) {
             semantic = redactDiagnostic({ status: "stale", error: error instanceof Error ? error.message : String(error) });
         }
+        const semanticCommitted = typeof semantic === "object" && semantic !== null && "generation" in semantic;
         output({
             name: "mailcrawl",
             archive: `${dataDir}/archive.sqlite`,
             archivePresent: existsSync(`${dataDir}/archive.sqlite`),
             fts: "available",
             semantic,
-            recommendation: semantic === "missing" ? "run sync, then index before semantic search" : "semantic index is committed",
+            recommendation: semanticCommitted ? "semantic index is committed" : "run sync, then index before semantic search",
         }, options.json);
     }
     finally {

package/dist/normalize.js

@@ -26,7 +26,18 @@ export async function normalizeMessage(input) {
     const subject = input.subject.trim();
     const messageId = input.messageId || input.providerKey;
     const threadId = input.threadId || makeId(input.accountId, normalizeSubject(subject));
-    const normalizedHash = hash(JSON.stringify([subject, input.from, input.to, input.cc, latest, quoted, input.labels, input.flags, input.classifications]));
+    const attachmentInputs = (attachments || []).map((attachment) => [
+        attachment.name,
+        attachment.mimeType,
+        attachment.size ?? null,
+        attachment.text ?? null,
+        attachment.contentHash ?? null,
+    ]);
+    const normalizedHash = hash(JSON.stringify([
+        subject, input.from, input.to, input.cc, latest, quoted,
+        input.labels, input.flags, input.classifications,
+        ...(attachmentInputs.length ? [attachmentInputs] : []),
+    ]));
     const categories = [...new Set([
             ...(input.classifications || []),
             ...(input.labels || []),

package/dist/util.d.ts

@@ -1,4 +1,5 @@
 export declare function hash(value: string): string;
 export declare function normalizeSubject(subject: string): string;
 export declare function makeId(...parts: string[]): string;
+export declare function scopedId(accountId: string, mailbox: string, value: string): string;
 export declare function snippet(text: string, query: string, maxChars?: number): string;

package/dist/util.js

@@ -8,6 +8,9 @@ export function normalizeSubject(subject) {
 export function makeId(...parts) {
     return hash(parts.join("\u0000")).slice(0, 24);
 }
+export function scopedId(accountId, mailbox, value) {
+    return makeId(accountId, mailbox, value);
+}
 export function snippet(text, query, maxChars = 240) {
     const clean = text.trim();
     if (clean.length <= maxChars)

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@nomadamas/mailcrawl",
-  "version": "0.1.3",
+  "version": "0.1.5",
   "description": "Local Himalaya-backed incremental email indexing and hybrid search CLI",
   "license": "MIT",
   "type": "module",