@ouro.bot/cli 0.1.0-alpha.73 → 0.1.0-alpha.731

package/dist/mailroom/search-cache.js

@@ -0,0 +1,404 @@
+"use strict";
+var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    var desc = Object.getOwnPropertyDescriptor(m, k);
+    if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
+      desc = { enumerable: true, get: function() { return m[k]; } };
+    }
+    Object.defineProperty(o, k2, desc);
+}) : (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    o[k2] = m[k];
+}));
+var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
+    Object.defineProperty(o, "default", { enumerable: true, value: v });
+}) : function(o, v) {
+    o["default"] = v;
+});
+var __importStar = (this && this.__importStar) || (function () {
+    var ownKeys = function(o) {
+        ownKeys = Object.getOwnPropertyNames || function (o) {
+            var ar = [];
+            for (var k in o) if (Object.prototype.hasOwnProperty.call(o, k)) ar[ar.length] = k;
+            return ar;
+        };
+        return ownKeys(o);
+    };
+    return function (mod) {
+        if (mod && mod.__esModule) return mod;
+        var result = {};
+        if (mod != null) for (var k = ownKeys(mod), i = 0; i < k.length; i++) if (k[i] !== "default") __createBinding(result, mod, k[i]);
+        __setModuleDefault(result, mod);
+        return result;
+    };
+})();
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.MAIL_SEARCH_TEXT_PROJECTION_VERSION = void 0;
+exports.buildMailSearchCacheDocument = buildMailSearchCacheDocument;
+exports.upsertMailSearchCacheDocument = upsertMailSearchCacheDocument;
+exports.inspectMailSearchCacheFiles = inspectMailSearchCacheFiles;
+exports.reloadMailSearchCache = reloadMailSearchCache;
+exports.removeMailSearchCacheDocument = removeMailSearchCacheDocument;
+exports.removeMailSearchCacheFile = removeMailSearchCacheFile;
+exports.syncMailSearchCacheMetadata = syncMailSearchCacheMetadata;
+exports.searchMailSearchCache = searchMailSearchCache;
+exports.snapshotMailSearchCacheForFilters = snapshotMailSearchCacheForFilters;
+exports.snapshotMailSearchCache = snapshotMailSearchCache;
+exports.readMailSearchCoverageRecord = readMailSearchCoverageRecord;
+exports.writeMailSearchCoverageRecord = writeMailSearchCoverageRecord;
+exports.resetMailSearchCacheForTests = resetMailSearchCacheForTests;
+const fs = __importStar(require("node:fs"));
+const path = __importStar(require("node:path"));
+const identity_1 = require("../heart/identity");
+const runtime_1 = require("../nerves/runtime");
+const core_1 = require("./core");
+const search_relevance_1 = require("./search-relevance");
+const SEARCH_TEXT_EXCERPT_LIMIT = 16_384;
+exports.MAIL_SEARCH_TEXT_PROJECTION_VERSION = 2;
+const cacheStates = new Map();
+function defaultCacheDir(agentId) {
+    return path.join((0, identity_1.getAgentRoot)(agentId), "state", "mail-search");
+}
+function cacheDir(agentId, options = {}) {
+    const resolve = options.cacheDirForAgent ?? defaultCacheDir;
+    return resolve(agentId);
+}
+function cachePath(agentId, messageId, options) {
+    return path.join(cacheDir(agentId, options), `${messageId}.json`);
+}
+function coverageDir(agentId, options) {
+    return path.join(cacheDir(agentId, options), "coverage");
+}
+function normalizedCoverageKey(key) {
+    return {
+        agentId: key.agentId,
+        storeKind: key.storeKind,
+        ...(key.placement ? { placement: key.placement } : {}),
+        ...(key.compartmentKind ? { compartmentKind: key.compartmentKind } : {}),
+        ...(key.source ? { source: key.source.toLowerCase() } : {}),
+    };
+}
+function coveragePath(key, options) {
+    const normalized = normalizedCoverageKey(key);
+    const encoded = Buffer.from(JSON.stringify(normalized)).toString("base64url");
+    return path.join(coverageDir(normalized.agentId, options), `${encoded}.json`);
+}
+function normalizeSearchText(privateEnvelope) {
+    const readableText = (0, core_1.privateMailEnvelopeReadableText)(privateEnvelope);
+    return [
+        privateEnvelope.subject,
+        privateEnvelope.snippet,
+        readableText.slice(0, SEARCH_TEXT_EXCERPT_LIMIT),
+        privateEnvelope.from.join(" "),
+    ].join("\n").toLowerCase();
+}
+function readJsonDocument(filePath) {
+    try {
+        return JSON.parse(fs.readFileSync(filePath, "utf-8"));
+    }
+    catch {
+        return null;
+    }
+}
+function writeJsonAtomically(filePath, value) {
+    const tempPath = `${filePath}.${process.pid}.${Date.now()}.tmp`;
+    try {
+        fs.writeFileSync(tempPath, `${JSON.stringify(value)}\n`, "utf-8");
+        fs.renameSync(tempPath, filePath);
+    }
+    finally {
+        if (fs.existsSync(tempPath))
+            fs.unlinkSync(tempPath);
+    }
+}
+function cacheState(agentId, options) {
+    const key = `${agentId}:${cacheDir(agentId, options)}`;
+    let state = cacheStates.get(key);
+    if (state)
+        return state;
+    state = { loaded: false, docs: new Map() };
+    cacheStates.set(key, state);
+    return state;
+}
+function loadCache(agentId, options) {
+    const state = cacheState(agentId, options);
+    if (state.loaded)
+        return state.docs;
+    state.loaded = true;
+    const dir = cacheDir(agentId, options);
+    if (!fs.existsSync(dir))
+        return state.docs;
+    for (const entry of fs.readdirSync(dir)) {
+        if (!entry.endsWith(".json"))
+            continue;
+        const document = readJsonDocument(path.join(dir, entry));
+        if (!document || document.agentId !== agentId)
+            continue;
+        state.docs.set(document.messageId, document);
+    }
+    return state.docs;
+}
+function buildMailSearchCacheDocument(message, privateEnvelope, provenance = {}) {
+    const readableText = (0, core_1.privateMailEnvelopeReadableText)(privateEnvelope);
+    const decryptionKeyId = provenance.decryptionKeyId
+        ?? (message.bodyForm === "encrypted" ? message.privateEnvelope?.keyId : undefined);
+    return {
+        schemaVersion: 1,
+        messageId: message.id,
+        agentId: message.agentId,
+        receivedAt: message.receivedAt,
+        placement: message.placement,
+        compartmentKind: message.compartmentKind,
+        ...(message.ownerEmail ? { ownerEmail: message.ownerEmail } : {}),
+        ...(message.source ? { source: message.source } : {}),
+        from: [...privateEnvelope.from],
+        subject: privateEnvelope.subject,
+        snippet: privateEnvelope.snippet,
+        textExcerpt: readableText.slice(0, SEARCH_TEXT_EXCERPT_LIMIT),
+        untrustedContentWarning: privateEnvelope.untrustedContentWarning,
+        searchText: normalizeSearchText(privateEnvelope),
+        textProjectionVersion: exports.MAIL_SEARCH_TEXT_PROJECTION_VERSION,
+        attachmentCount: privateEnvelope.attachments.length,
+        ...(decryptionKeyId ? { decryptionKeyId } : {}),
+    };
+}
+function upsertMailSearchCacheDocument(message, privateEnvelope, options, provenance) {
+    const document = buildMailSearchCacheDocument(message, privateEnvelope, provenance);
+    const dir = cacheDir(message.agentId, options);
+    fs.mkdirSync(dir, { recursive: true });
+    writeJsonAtomically(cachePath(message.agentId, message.id, options), document);
+    const state = cacheState(message.agentId, options);
+    if (state.loaded)
+        state.docs.set(document.messageId, document);
+    (0, runtime_1.emitNervesEvent)({
+        component: "senses",
+        event: "senses.mail_search_cache_upserted",
+        message: "mail search cache entry written",
+        meta: {
+            agentId: message.agentId,
+            messageId: document.messageId,
+            placement: document.placement,
+            compartmentKind: document.compartmentKind,
+        },
+    });
+    return document;
+}
+function inspectMailSearchCacheFiles(agentId, options) {
+    const dir = cacheDir(agentId, options);
+    if (!fs.existsSync(dir))
+        return [];
+    return fs.readdirSync(dir, { withFileTypes: true })
+        .filter((entry) => entry.isFile() && entry.name.endsWith(".json"))
+        .map((entry) => {
+        const filePath = path.join(dir, entry.name);
+        return { fileName: entry.name, filePath, document: readJsonDocument(filePath) };
+    })
+        .sort((left, right) => left.fileName.localeCompare(right.fileName));
+}
+function reloadMailSearchCache(agentId, options) {
+    const state = cacheState(agentId, options);
+    state.loaded = false;
+    state.docs.clear();
+}
+function removeMailSearchCacheDocument(agentId, messageId, options) {
+    const filePath = cachePath(agentId, messageId, options);
+    let removed = false;
+    try {
+        if (fs.lstatSync(filePath).isFile()) {
+            fs.unlinkSync(filePath);
+            removed = true;
+        }
+    }
+    catch (error) {
+        if (!(typeof error === "object" && error !== null && "code" in error && error.code === "ENOENT"))
+            throw error;
+    }
+    const state = cacheState(agentId, options);
+    if (state.loaded)
+        state.docs.delete(messageId);
+    return removed;
+}
+function removeMailSearchCacheFile(agentId, fileName, options) {
+    if (path.basename(fileName) !== fileName || !fileName.endsWith(".json"))
+        return false;
+    const filePath = path.join(cacheDir(agentId, options), fileName);
+    try {
+        if (!fs.lstatSync(filePath).isFile())
+            return false;
+        fs.unlinkSync(filePath);
+        reloadMailSearchCache(agentId, options);
+        return true;
+    }
+    catch (error) {
+        if (!(typeof error === "object" && error !== null && "code" in error && error.code === "ENOENT"))
+            throw error;
+        return false;
+    }
+}
+function syncMailSearchCacheMetadata(message, options) {
+    const existing = readJsonDocument(cachePath(message.agentId, message.id, options));
+    if (!existing)
+        return;
+    const updated = {
+        ...existing,
+        receivedAt: message.receivedAt,
+        placement: message.placement,
+        compartmentKind: message.compartmentKind,
+        ...(message.ownerEmail ? { ownerEmail: message.ownerEmail } : {}),
+        ...(message.source ? { source: message.source } : {}),
+    };
+    writeJsonAtomically(cachePath(message.agentId, message.id, options), updated);
+    const state = cacheState(message.agentId, options);
+    if (state.loaded)
+        state.docs.set(updated.messageId, updated);
+}
+function sourceMatches(source, filter) {
+    if (!filter)
+        return true;
+    if (!source)
+        return false;
+    return source.toLowerCase() === filter.toLowerCase();
+}
+function documentMatchesFilters(document, filters) {
+    const queryTerms = filters.queryTerms ?? [];
+    return (filters.placement ? document.placement === filters.placement : true)
+        && (filters.compartmentKind ? document.compartmentKind === filters.compartmentKind : true)
+        && sourceMatches(document.source, filters.source)
+        && (queryTerms.length
+            ? queryTerms.some((term) => document.searchText.includes(term))
+            : true);
+}
+function* readCacheDocuments(agentId, options) {
+    const dir = cacheDir(agentId, options);
+    if (!fs.existsSync(dir))
+        return;
+    for (const entry of fs.readdirSync(dir, { withFileTypes: true })) {
+        if (!entry.isFile() || !entry.name.endsWith(".json"))
+            continue;
+        const document = readJsonDocument(path.join(dir, entry.name));
+        if (!document || document.agentId !== agentId)
+            continue;
+        yield document;
+    }
+}
+function insertLimitedMatch(matches, candidate, filters) {
+    matches.push(candidate);
+    const queryTerms = filters.queryTerms ?? [];
+    if (queryTerms.length > 0) {
+        matches.sort((left, right) => (0, search_relevance_1.compareByRelevanceThenRecency)({ document: left, relevance: (0, search_relevance_1.scoreMailSearchDocument)(left, queryTerms) }, { document: right, relevance: (0, search_relevance_1.scoreMailSearchDocument)(right, queryTerms) }));
+    }
+    else {
+        matches.sort((left, right) => right.receivedAt.localeCompare(left.receivedAt));
+    }
+    if (matches.length > filters.limit)
+        matches.length = filters.limit;
+}
+function streamSearchMailSearchCache(filters, options) {
+    const matches = [];
+    for (const document of readCacheDocuments(filters.agentId, options)) {
+        if (!documentMatchesFilters(document, filters))
+            continue;
+        insertLimitedMatch(matches, document, filters);
+    }
+    return matches;
+}
+function searchMailSearchCache(filters, options) {
+    if (typeof filters.limit === "number" && !cacheState(filters.agentId, options).loaded) {
+        return streamSearchMailSearchCache({ ...filters, limit: filters.limit }, options);
+    }
+    const queryTerms = filters.queryTerms ?? [];
+    const docs = [...loadCache(filters.agentId, options).values()]
+        .filter((document) => documentMatchesFilters(document, filters));
+    let ordered;
+    if (queryTerms.length > 0) {
+        ordered = docs
+            .map((document) => ({ document, relevance: (0, search_relevance_1.scoreMailSearchDocument)(document, queryTerms) }))
+            .sort(search_relevance_1.compareByRelevanceThenRecency)
+            .map((entry) => entry.document);
+    }
+    else {
+        ordered = docs.sort((left, right) => right.receivedAt.localeCompare(left.receivedAt));
+    }
+    return typeof filters.limit === "number" ? ordered.slice(0, filters.limit) : ordered;
+}
+function snapshotMailSearchCacheForFilters(filters, options) {
+    let totalDocuments = 0;
+    let currentProjectionDocuments = 0;
+    let oldestReceivedAt;
+    let newestReceivedAt;
+    for (const document of readCacheDocuments(filters.agentId, options)) {
+        if (!documentMatchesFilters(document, filters))
+            continue;
+        totalDocuments += 1;
+        if (document.textProjectionVersion === exports.MAIL_SEARCH_TEXT_PROJECTION_VERSION) {
+            currentProjectionDocuments += 1;
+        }
+        if (!oldestReceivedAt || document.receivedAt < oldestReceivedAt)
+            oldestReceivedAt = document.receivedAt;
+        if (!newestReceivedAt || document.receivedAt > newestReceivedAt)
+            newestReceivedAt = document.receivedAt;
+    }
+    return {
+        totalDocuments,
+        currentProjectionDocuments,
+        ...(oldestReceivedAt ? { oldestReceivedAt } : {}),
+        ...(newestReceivedAt ? { newestReceivedAt } : {}),
+    };
+}
+function snapshotMailSearchCache(agentId, options) {
+    const dir = cacheDir(agentId, options);
+    if (!fs.existsSync(dir))
+        return { totalDocuments: 0 };
+    let total = 0;
+    for (const entry of fs.readdirSync(dir, { withFileTypes: true })) {
+        if (entry.isFile() && entry.name.endsWith(".json"))
+            total += 1;
+    }
+    return { totalDocuments: total };
+}
+function readMailSearchCoverageRecord(key, options) {
+    const document = readJsonDocument(coveragePath(key, options));
+    if (!document || document.schemaVersion !== 1 || document.agentId !== key.agentId)
+        return null;
+    const normalized = normalizedCoverageKey(key);
+    const stored = normalizedCoverageKey(document);
+    if (JSON.stringify(stored) !== JSON.stringify(normalized))
+        return null;
+    return document;
+}
+function writeMailSearchCoverageRecord(record, options) {
+    const normalized = normalizedCoverageKey(record);
+    const document = {
+        schemaVersion: 1,
+        ...normalized,
+        indexedAt: record.indexedAt,
+        visibleMessageCount: record.visibleMessageCount,
+        cachedMessageCount: record.cachedMessageCount,
+        decryptableMessageCount: record.decryptableMessageCount,
+        skippedMessageCount: record.skippedMessageCount,
+        ...(record.messageIndexFingerprint ? { messageIndexFingerprint: record.messageIndexFingerprint } : {}),
+        textProjectionVersion: record.textProjectionVersion,
+        ...(record.oldestReceivedAt ? { oldestReceivedAt: record.oldestReceivedAt } : {}),
+        ...(record.newestReceivedAt ? { newestReceivedAt: record.newestReceivedAt } : {}),
+    };
+    fs.mkdirSync(coverageDir(document.agentId, options), { recursive: true });
+    writeJsonAtomically(coveragePath(document, options), document);
+    (0, runtime_1.emitNervesEvent)({
+        component: "senses",
+        event: "senses.mail_search_coverage_written",
+        message: "mail search coverage record written",
+        meta: {
+            agentId: document.agentId,
+            placement: document.placement ?? null,
+            compartmentKind: document.compartmentKind ?? null,
+            source: document.source ?? null,
+            storeKind: document.storeKind,
+            visibleMessageCount: document.visibleMessageCount,
+            decryptableMessageCount: document.decryptableMessageCount,
+        },
+    });
+    return document;
+}
+function resetMailSearchCacheForTests() {
+    cacheStates.clear();
+}