@ouro.bot/cli 0.1.0-alpha.60 → 0.1.0-alpha.600

package/dist/mailroom/mbox-import.js

@@ -0,0 +1,393 @@
+"use strict";
+var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    var desc = Object.getOwnPropertyDescriptor(m, k);
+    if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
+      desc = { enumerable: true, get: function() { return m[k]; } };
+    }
+    Object.defineProperty(o, k2, desc);
+}) : (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    o[k2] = m[k];
+}));
+var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
+    Object.defineProperty(o, "default", { enumerable: true, value: v });
+}) : function(o, v) {
+    o["default"] = v;
+});
+var __importStar = (this && this.__importStar) || (function () {
+    var ownKeys = function(o) {
+        ownKeys = Object.getOwnPropertyNames || function (o) {
+            var ar = [];
+            for (var k in o) if (Object.prototype.hasOwnProperty.call(o, k)) ar[ar.length] = k;
+            return ar;
+        };
+        return ownKeys(o);
+    };
+    return function (mod) {
+        if (mod && mod.__esModule) return mod;
+        var result = {};
+        if (mod != null) for (var k = ownKeys(mod), i = 0; i < k.length; i++) if (k[i] !== "default") __createBinding(result, mod, k[i]);
+        __setModuleDefault(result, mod);
+        return result;
+    };
+})();
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.splitMboxMessages = splitMboxMessages;
+exports.importMboxToStore = importMboxToStore;
+exports.importMboxFileToStore = importMboxFileToStore;
+exports.cacheMatchingMailSearchDocumentsFromMboxFile = cacheMatchingMailSearchDocumentsFromMboxFile;
+const fs = __importStar(require("node:fs"));
+const runtime_1 = require("../nerves/runtime");
+const core_1 = require("./core");
+const search_cache_1 = require("./search-cache");
+const search_relevance_1 = require("./search-relevance");
+function splitMboxMessages(rawMbox) {
+    const text = rawMbox.toString("utf-8");
+    const separators = [...text.matchAll(/^From [^\r\n]*(?:\r?\n)/gm)];
+    if (separators.length === 0) {
+        const trimmed = text.trim();
+        return trimmed ? [Buffer.from(text, "utf-8")] : [];
+    }
+    const messages = separators
+        .map((match, index) => {
+        const start = match.index + match[0].length;
+        const end = index + 1 < separators.length ? separators[index + 1].index : text.length;
+        return text.slice(start, end).replace(/\r?\n$/, "");
+    })
+        .filter((message) => message.trim().length > 0)
+        .map((message) => Buffer.from(message, "utf-8"));
+    (0, runtime_1.emitNervesEvent)({
+        component: "senses",
+        event: "senses.mail_mbox_split",
+        message: "mbox payload split into messages",
+        meta: { messages: messages.length },
+    });
+    return messages;
+}
+function trimTrailingLineEnding(message) {
+    if (message.length >= 2 && message[message.length - 2] === 0x0d && message[message.length - 1] === 0x0a) {
+        return message.subarray(0, -2);
+    }
+    if (message.length >= 1 && message[message.length - 1] === 0x0a) {
+        return message.subarray(0, -1);
+    }
+    return message;
+}
+function isSeparatorLine(line) {
+    return line.length >= 5 &&
+        line[0] === 0x46 &&
+        line[1] === 0x72 &&
+        line[2] === 0x6f &&
+        line[3] === 0x6d &&
+        line[4] === 0x20;
+}
+async function* streamMboxMessagesFromFile(filePath) {
+    const stream = fs.createReadStream(filePath);
+    let pending = Buffer.alloc(0);
+    let currentParts = [];
+    const flushCurrent = () => {
+        if (currentParts.length === 0)
+            return null;
+        const message = trimTrailingLineEnding(Buffer.concat(currentParts));
+        currentParts = [];
+        return message;
+    };
+    for await (const chunk of stream) {
+        pending = pending.length > 0 ? Buffer.concat([pending, chunk]) : Buffer.from(chunk);
+        let newlineIndex = pending.indexOf(0x0a);
+        while (newlineIndex >= 0) {
+            const line = pending.subarray(0, newlineIndex + 1);
+            pending = pending.subarray(newlineIndex + 1);
+            if (isSeparatorLine(line)) {
+                const message = flushCurrent();
+                if (message && message.length > 0)
+                    yield message;
+            }
+            else {
+                currentParts.push(Buffer.from(line));
+            }
+            newlineIndex = pending.indexOf(0x0a);
+        }
+    }
+    if (pending.length > 0) {
+        if (isSeparatorLine(pending)) {
+            const message = flushCurrent();
+            if (message && message.length > 0)
+                yield message;
+        }
+        else {
+            currentParts.push(Buffer.from(pending));
+        }
+    }
+    const finalMessage = flushCurrent();
+    if (finalMessage && finalMessage.length > 0)
+        yield finalMessage;
+}
+function findSourceGrant(input) {
+    const ownerEmail = input.ownerEmail ? (0, core_1.normalizeMailAddress)(input.ownerEmail) : undefined;
+    const source = input.source?.trim().toLowerCase();
+    const grants = input.registry.sourceGrants.filter((grant) => {
+        if (!grant.enabled || grant.agentId !== input.agentId)
+            return false;
+        if (ownerEmail && (0, core_1.normalizeMailAddress)(grant.ownerEmail) !== ownerEmail)
+            return false;
+        if (source && grant.source.toLowerCase() !== source)
+            return false;
+        return true;
+    });
+    if (grants.length === 0) {
+        throw new Error(`No enabled Mailroom source grant found for ${input.agentId}${ownerEmail ? ` owner ${ownerEmail}` : ""}${source ? ` source ${source}` : ""}`);
+    }
+    if (grants.length > 1 && !ownerEmail && !source) {
+        throw new Error(`Multiple source grants found for ${input.agentId}; pass --owner-email or --source to choose one`);
+    }
+    return grants[0];
+}
+function parseMboxMessage(rawMessage, grant) {
+    const mailFrom = extractHeaderAddress(rawMessage, "from");
+    const messageDate = extractHeaderDate(rawMessage, "date");
+    return {
+        rawMessage,
+        envelope: {
+            mailFrom: mailFrom ? (0, core_1.normalizeMailAddress)(mailFrom) : "",
+            rcptTo: [(0, core_1.normalizeMailAddress)(grant.aliasAddress)],
+            remoteAddress: "mbox-import",
+        },
+        ...(messageDate ? { messageDate } : {}),
+    };
+}
+function latestMessageDate(messages) {
+    const timestamps = messages
+        .map((message) => message.messageDate?.getTime())
+        .filter((timestamp) => typeof timestamp === "number" && Number.isFinite(timestamp));
+    if (timestamps.length === 0)
+        return null;
+    return new Date(Math.max(...timestamps)).toISOString();
+}
+function readHeaderBlock(rawMessage) {
+    const crlfBoundary = rawMessage.indexOf("\r\n\r\n");
+    if (crlfBoundary >= 0) {
+        return rawMessage.subarray(0, crlfBoundary).toString("utf-8");
+    }
+    const lfBoundary = rawMessage.indexOf("\n\n");
+    if (lfBoundary >= 0) {
+        return rawMessage.subarray(0, lfBoundary).toString("utf-8");
+    }
+    return rawMessage.toString("utf-8");
+}
+function extractHeaderValue(rawMessage, headerName) {
+    const target = `${headerName.toLowerCase()}:`;
+    let currentHeader = "";
+    const unfoldedHeaders = [];
+    for (const line of readHeaderBlock(rawMessage).split(/\r?\n/)) {
+        if (/^[ \t]/.test(line) && currentHeader) {
+            currentHeader += ` ${line.trim()}`;
+            continue;
+        }
+        if (currentHeader)
+            unfoldedHeaders.push(currentHeader);
+        currentHeader = line;
+    }
+    if (currentHeader)
+        unfoldedHeaders.push(currentHeader);
+    const match = unfoldedHeaders.find((line) => line.toLowerCase().startsWith(target));
+    return match ? match.slice(target.length).trim() : "";
+}
+function extractHeaderAddress(rawMessage, headerName) {
+    const value = extractHeaderValue(rawMessage, headerName);
+    if (!value)
+        return "";
+    const bracketed = value.match(/<([^>]+)>/);
+    if (bracketed?.[1])
+        return bracketed[1];
+    const plain = value.match(/[A-Z0-9._%+-]+@[A-Z0-9.-]+\.[A-Z]{2,}/i);
+    return plain?.[0] ?? "";
+}
+function extractHeaderDate(rawMessage, headerName) {
+    const value = extractHeaderValue(rawMessage, headerName);
+    if (!value)
+        return undefined;
+    const parsed = new Date(value);
+    return Number.isFinite(parsed.getTime()) ? parsed : undefined;
+}
+function normalizeSearchTerms(queryTerms) {
+    return queryTerms
+        .map((term) => term.trim().toLowerCase())
+        .filter((term) => term.length > 0);
+}
+function historicalImportClassification(resolvedPlacement, sourceGrant) {
+    return {
+        placement: resolvedPlacement,
+        candidate: false,
+        trustReason: `delegated source grant ${sourceGrant.source} historical mbox import`,
+    };
+}
+async function importParsedMessagesToStore(input) {
+    let imported = 0;
+    let duplicates = 0;
+    const messages = [];
+    let scanned = 0;
+    const importedAt = (input.importedAt ?? new Date()).toISOString();
+    const sourceFreshThrough = input.sourceFreshThrough ?? null;
+    const maxConcurrency = Math.max(1, input.maxConcurrency ?? 1);
+    const inFlight = new Set();
+    const enqueue = (parsedMessage) => {
+        const task = (async () => {
+            const result = await input.store.putRawMessage({
+                resolved: input.resolved,
+                envelope: parsedMessage.envelope,
+                rawMime: parsedMessage.rawMessage,
+                receivedAt: parsedMessage.messageDate ?? input.importedAt,
+                ingest: {
+                    schemaVersion: 1,
+                    kind: "mbox-import",
+                    importedAt,
+                    sourceFreshThrough,
+                    attentionSuppressed: true,
+                },
+                classification: historicalImportClassification(input.resolved.defaultPlacement, input.sourceGrant),
+            });
+            if (input.collectMessages !== false)
+                messages.push(result.message);
+            if (result.created)
+                imported += 1;
+            else
+                duplicates += 1;
+            input.onProgress?.({ scanned, imported, duplicates });
+        })().finally(() => {
+            inFlight.delete(task);
+        });
+        inFlight.add(task);
+        return task;
+    };
+    for await (const parsedMessage of input.parsedMessages) {
+        scanned += 1;
+        input.onProgress?.({ scanned, imported, duplicates });
+        enqueue(parsedMessage);
+        if (inFlight.size >= maxConcurrency) {
+            await Promise.race(inFlight);
+        }
+    }
+    await Promise.all(inFlight);
+    (0, runtime_1.emitNervesEvent)({
+        component: "senses",
+        event: "senses.mail_mbox_imported",
+        message: "mbox mail imported",
+        meta: { agentId: input.sourceGrant.agentId.toLowerCase(), scanned, imported, duplicates, grantId: input.sourceGrant.grantId },
+    });
+    return {
+        agentId: input.sourceGrant.agentId.toLowerCase(),
+        sourceGrant: input.sourceGrant,
+        scanned,
+        imported,
+        duplicates,
+        sourceFreshThrough,
+        messages,
+    };
+}
+async function scanMboxFileFreshness(filePath, sourceGrant) {
+    let latestTimestamp = Number.NEGATIVE_INFINITY;
+    for await (const rawMessage of streamMboxMessagesFromFile(filePath)) {
+        const parsedMessage = parseMboxMessage(rawMessage, sourceGrant);
+        if (parsedMessage.messageDate) {
+            latestTimestamp = Math.max(latestTimestamp, parsedMessage.messageDate.getTime());
+        }
+    }
+    return Number.isFinite(latestTimestamp) ? new Date(latestTimestamp).toISOString() : null;
+}
+function resolveImportTarget(input) {
+    const agentId = input.agentId.toLowerCase();
+    const sourceGrant = findSourceGrant({
+        registry: input.registry,
+        agentId,
+        ownerEmail: input.ownerEmail,
+        source: input.source,
+    });
+    const resolved = (0, core_1.resolveMailAddress)(input.registry, sourceGrant.aliasAddress);
+    /* v8 ignore start -- findSourceGrant and resolveMailAddress share the same registry; this is a corruption guard. @preserve */
+    if (!resolved) {
+        throw new Error(`Source grant alias ${sourceGrant.aliasAddress} is not resolvable`);
+    }
+    /* v8 ignore stop */
+    return { agentId, sourceGrant, resolved };
+}
+async function importMboxToStore(input) {
+    const target = resolveImportTarget(input);
+    const rawMessages = splitMboxMessages(input.rawMbox);
+    const parsedMessages = rawMessages.map((rawMessage) => parseMboxMessage(rawMessage, target.sourceGrant));
+    const sourceFreshThrough = latestMessageDate(parsedMessages);
+    return importParsedMessagesToStore({
+        sourceGrant: target.sourceGrant,
+        resolved: target.resolved,
+        store: input.store,
+        parsedMessages,
+        importedAt: input.importedAt,
+        collectMessages: true,
+        sourceFreshThrough,
+    });
+}
+async function importMboxFileToStore(input) {
+    const target = resolveImportTarget(input);
+    const sourceFreshThrough = await scanMboxFileFreshness(input.filePath, target.sourceGrant);
+    async function* parsedMessages() {
+        for await (const rawMessage of streamMboxMessagesFromFile(input.filePath)) {
+            yield parseMboxMessage(rawMessage, target.sourceGrant);
+        }
+    }
+    return importParsedMessagesToStore({
+        sourceGrant: target.sourceGrant,
+        resolved: target.resolved,
+        store: input.store,
+        parsedMessages: parsedMessages(),
+        importedAt: input.importedAt,
+        collectMessages: false,
+        sourceFreshThrough,
+        maxConcurrency: 8,
+        onProgress: input.onProgress,
+    });
+}
+async function cacheMatchingMailSearchDocumentsFromMboxFile(input) {
+    const target = resolveImportTarget(input);
+    const queryTerms = normalizeSearchTerms(input.queryTerms);
+    if (queryTerms.length === 0 || input.limit <= 0)
+        return [];
+    const matches = [];
+    for await (const rawMessage of streamMboxMessagesFromFile(input.filePath)) {
+        const parsedMessage = parseMboxMessage(rawMessage, target.sourceGrant);
+        const { privateEnvelope, metadata } = await (0, core_1.buildStoredMailMessageMetadata)({
+            resolved: target.resolved,
+            envelope: parsedMessage.envelope,
+            rawMime: parsedMessage.rawMessage,
+            receivedAt: parsedMessage.messageDate,
+            ingest: {
+                schemaVersion: 1,
+                kind: "mbox-import",
+                attentionSuppressed: true,
+            },
+            classification: historicalImportClassification(target.resolved.defaultPlacement, target.sourceGrant),
+        });
+        // Projection-only: we need a `StoredMailMessage` shape for buildMailSearchCacheDocument
+        // but never persist this view; the live import (via store.putRawMessage) is
+        // what writes the durable record.
+        const { rawObjectStem, ...rest } = metadata;
+        const message = {
+            ...rest,
+            rawObject: `${rawObjectStem}.eml`,
+            bodyForm: "plaintext",
+            private: privateEnvelope,
+        };
+        const document = (0, search_cache_1.buildMailSearchCacheDocument)(message, privateEnvelope);
+        if (!queryTerms.some((term) => document.searchText.includes(term)))
+            continue;
+        (0, search_cache_1.upsertMailSearchCacheDocument)(message, privateEnvelope);
+        matches.push(document);
+        matches.sort((left, right) => (0, search_relevance_1.compareByRelevanceThenRecency)({ document: left, relevance: (0, search_relevance_1.scoreMailSearchDocument)(left, queryTerms) }, { document: right, relevance: (0, search_relevance_1.scoreMailSearchDocument)(right, queryTerms) }));
+        if (matches.length > input.limit)
+            matches.length = input.limit;
+    }
+    return matches
+        .map((document) => ({ document, relevance: (0, search_relevance_1.scoreMailSearchDocument)(document, queryTerms) }))
+        .sort(search_relevance_1.compareByRelevanceThenRecency)
+        .map((entry) => entry.document);
+}

package/dist/mailroom/migration.js

@@ -0,0 +1,164 @@
+"use strict";
+var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    var desc = Object.getOwnPropertyDescriptor(m, k);
+    if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
+      desc = { enumerable: true, get: function() { return m[k]; } };
+    }
+    Object.defineProperty(o, k2, desc);
+}) : (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    o[k2] = m[k];
+}));
+var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
+    Object.defineProperty(o, "default", { enumerable: true, value: v });
+}) : function(o, v) {
+    o["default"] = v;
+});
+var __importStar = (this && this.__importStar) || (function () {
+    var ownKeys = function(o) {
+        ownKeys = Object.getOwnPropertyNames || function (o) {
+            var ar = [];
+            for (var k in o) if (Object.prototype.hasOwnProperty.call(o, k)) ar[ar.length] = k;
+            return ar;
+        };
+        return ownKeys(o);
+    };
+    return function (mod) {
+        if (mod && mod.__esModule) return mod;
+        var result = {};
+        if (mod != null) for (var k = ownKeys(mod), i = 0; i < k.length; i++) if (k[i] !== "default") __createBinding(result, mod, k[i]);
+        __setModuleDefault(result, mod);
+        return result;
+    };
+})();
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.migrateLocalMailroomToPlaintext = migrateLocalMailroomToPlaintext;
+const fs = __importStar(require("node:fs"));
+const path = __importStar(require("node:path"));
+const runtime_1 = require("../nerves/runtime");
+function readJson(filePath) {
+    try {
+        return JSON.parse(fs.readFileSync(filePath, "utf-8"));
+    }
+    catch {
+        return null;
+    }
+}
+function listJsonEntries(dir) {
+    if (!fs.existsSync(dir))
+        return [];
+    return fs.readdirSync(dir).filter((name) => name.endsWith(".json"));
+}
+/**
+ * One-time, idempotent cleanup of pre-plaintext residue.
+ *
+ * Wipes:
+ *  - `messages/*.json` for this agent that are not already in the new plaintext
+ *    shape (no `bodyForm: "plaintext"` AND no `private` field). Unparseable JSON
+ *    also goes — we cannot safely keep ambiguous bytes.
+ *  - `raw/*.json` files that look like an `EncryptedPayload` JSON wrapper (the
+ *    pre-change raw-storage shape). Plaintext `.eml` files are preserved.
+ *    Non-EncryptedPayload JSON in `raw/` is left alone.
+ *  - Coverage records in `searchCacheRoot/coverage/` whose `storeKind` is no
+ *    longer reachable (`azure-blob` from a previous hosted attachment) or whose
+ *    JSON is unreadable. `file` coverage records are preserved.
+ *  - Orphan search-cache docs in `searchCacheRoot/*.json` that belong to this
+ *    agent but reference a `messageId` no longer present in `messages/`.
+ *
+ * Idempotent: a second run finds nothing to wipe and returns zeroed counts.
+ */
+function migrateLocalMailroomToPlaintext(input) {
+    const messagesDir = path.join(input.mailroomRoot, "messages");
+    const rawDir = path.join(input.mailroomRoot, "raw");
+    const coverageDir = path.join(input.searchCacheRoot, "coverage");
+    let wipedEnvelopes = 0;
+    let wipedRaw = 0;
+    let wipedCoverageRecords = 0;
+    let wipedOrphanSearchDocs = 0;
+    // 1. Messages: drop pre-plaintext or malformed shapes for this agent.
+    const surviving = new Set();
+    for (const entry of listJsonEntries(messagesDir)) {
+        const filePath = path.join(messagesDir, entry);
+        const value = readJson(filePath);
+        if (value === null) {
+            // Unparseable JSON — we cannot tell whose it is or what shape, but the
+            // pre-change codebase wrote message JSON here; treat as wipeable.
+            fs.unlinkSync(filePath);
+            wipedEnvelopes += 1;
+            continue;
+        }
+        const isCurrentShape = value.bodyForm === "plaintext" && value.private !== undefined && value.private !== null;
+        if (isCurrentShape) {
+            const stem = entry.slice(0, -".json".length);
+            surviving.add(stem);
+            continue;
+        }
+        // We don't restrict to a specific agentId here because the file store is
+        // single-agent: the bundle owns this directory and the migration is
+        // initiated from inside that bundle.
+        fs.unlinkSync(filePath);
+        wipedEnvelopes += 1;
+    }
+    // 2. Raw artifacts: drop pre-change encrypted JSON wrappers.
+    if (fs.existsSync(rawDir)) {
+        for (const entry of fs.readdirSync(rawDir)) {
+            if (!entry.endsWith(".json"))
+                continue;
+            const filePath = path.join(rawDir, entry);
+            const value = readJson(filePath);
+            const isEncryptedPayload = value !== null
+                && typeof value.algorithm === "string"
+                && value.algorithm.startsWith("RSA-OAEP")
+                && typeof value.ciphertext === "string";
+            if (!isEncryptedPayload)
+                continue;
+            fs.unlinkSync(filePath);
+            wipedRaw += 1;
+        }
+    }
+    // 3. Coverage records: drop azure-blob (no longer reachable) and malformed.
+    for (const entry of listJsonEntries(coverageDir)) {
+        const filePath = path.join(coverageDir, entry);
+        const value = readJson(filePath);
+        if (value === null) {
+            fs.unlinkSync(filePath);
+            wipedCoverageRecords += 1;
+            continue;
+        }
+        if (value.agentId !== input.agentId)
+            continue;
+        if (value.storeKind === "file")
+            continue;
+        fs.unlinkSync(filePath);
+        wipedCoverageRecords += 1;
+    }
+    // 4. Orphan search-cache docs.
+    for (const entry of listJsonEntries(input.searchCacheRoot)) {
+        const filePath = path.join(input.searchCacheRoot, entry);
+        const value = readJson(filePath);
+        if (value === null)
+            continue;
+        if (value.agentId !== input.agentId)
+            continue;
+        if (typeof value.messageId !== "string")
+            continue;
+        if (surviving.has(value.messageId))
+            continue;
+        fs.unlinkSync(filePath);
+        wipedOrphanSearchDocs += 1;
+    }
+    (0, runtime_1.emitNervesEvent)({
+        component: "senses",
+        event: "senses.mail_local_migration_executed",
+        message: "local mailroom plaintext migration executed",
+        meta: {
+            agentId: input.agentId,
+            wipedEnvelopes,
+            wipedRaw,
+            wipedCoverageRecords,
+            wipedOrphanSearchDocs,
+        },
+    });
+    return { wipedEnvelopes, wipedRaw, wipedCoverageRecords, wipedOrphanSearchDocs };
+}