@ouro.bot/cli 0.1.0-alpha.730 → 0.1.0-alpha.731

package/dist/repertoire/tools-mail.js

@@ -8,7 +8,6 @@ exports.renderCachedMessageSummary = renderCachedMessageSummary;
 exports.mergeCachedMailSearchDocuments = mergeCachedMailSearchDocuments;
 exports.searchSuccessfulImportArchives = searchSuccessfulImportArchives;
 const node_fs_1 = __importDefault(require("node:fs"));
-const node_crypto_1 = require("node:crypto");
 const friends_1 = require("@ouro.bot/friends");
 const file_store_1 = require("../mailroom/file-store");
 const reader_1 = require("../mailroom/reader");
@@ -25,8 +24,8 @@ const credential_access_1 = require("./credential-access");
 const background_operations_1 = require("../heart/background-operations");
 const mail_import_discovery_1 = require("../heart/mail-import-discovery");
 const identity_1 = require("../heart/identity");
-const MAIL_SEARCH_INDEX_FETCH_CONCURRENCY = 80;
-const MAIL_SEARCH_INDEX_BATCH_SIZE = 500;
+const blob_store_1 = require("../mailroom/blob-store");
+const hosted_cache_sync_1 = require("../mailroom/hosted-cache-sync");
 function trustAllowsMailRead(ctx) {
     const trustLevel = ctx?.context?.friend?.trustLevel;
     const allowed = trustLevel === undefined || (0, friends_1.isTrustedLevel)(trustLevel);
@@ -300,24 +299,6 @@ function appendDelegatedSearchCoverage(body, input) {
         ].filter(Boolean).join(" "),
     ].join("\n\n");
 }
-async function mapWithConcurrency(items, concurrency, worker) {
-    /* v8 ignore next -- refresh callers pass only non-empty slices into the worker pool. @preserve */
-    if (items.length === 0)
-        return [];
-    const results = new Array(items.length);
-    let nextIndex = 0;
-    const workerLoop = async () => {
-        while (true) {
-            const current = nextIndex;
-            nextIndex += 1;
-            if (current >= items.length)
-                return;
-            results[current] = await worker(items[current], current);
-        }
-    };
-    await Promise.all(Array.from({ length: Math.min(concurrency, items.length) }, async () => workerLoop()));
-    return results;
-}
 function mailListFilters(input) {
     return {
         agentId: input.agentId,
@@ -338,28 +319,6 @@ function mailSearchCoverageKey(input) {
         ...(input.source ? { source: input.source } : {}),
     };
 }
-function mailSearchCacheDocumentNeedsProjectionRefresh(document) {
-    if (document.textProjectionVersion === search_cache_1.MAIL_SEARCH_TEXT_PROJECTION_VERSION)
-        return false;
-    return document.textExcerpt.trim().length === 0;
-}
-function mailSearchCoverageSnapshot(records) {
-    const normalizedRecords = records
-        .map((record) => ({
-        id: record.id,
-        receivedAt: record.receivedAt,
-        placement: record.placement,
-        compartmentKind: record.compartmentKind,
-        source: record.source?.toLowerCase() ?? null,
-    }))
-        .sort((left, right) => left.id.localeCompare(right.id));
-    return {
-        visibleMessageCount: records.length,
-        messageIndexFingerprint: (0, node_crypto_1.createHash)("sha256").update(JSON.stringify(normalizedRecords)).digest("hex"),
-        ...(oldestReceivedAt(records) ? { oldestReceivedAt: oldestReceivedAt(records) } : {}),
-        ...(newestReceivedAt(records) ? { newestReceivedAt: newestReceivedAt(records) } : {}),
-    };
-}
 function mailSearchCoverageStalenessReason(record, currentSnapshot) {
     if (!record)
         return "hosted search index incomplete";
@@ -389,113 +348,6 @@ function mailSearchCoverageUnsearchableReason(record) {
     const noun = unsearchableCount === 1 ? "message was" : "messages were";
     return `${unsearchableCount} visible ${noun} not searchable because the index skipped missing/decryption-key mail`;
 }
-function newestReceivedAt(records) {
-    return records.reduce((newest, record) => {
-        if (!newest || Date.parse(record.receivedAt) > Date.parse(newest))
-            return record.receivedAt;
-        return newest;
-    }, undefined);
-}
-function oldestReceivedAt(records) {
-    return records.reduce((oldest, record) => {
-        if (!oldest || Date.parse(record.receivedAt) < Date.parse(oldest))
-            return record.receivedAt;
-        return oldest;
-    }, undefined);
-}
-async function refreshMailSearchIndex(input) {
-    const filters = mailListFilters(input);
-    const indexedRecords = await input.store.listMessageIndexRecords?.(filters);
-    const visibleRecords = indexedRecords ?? (await input.store.listMessages(filters)).map((message) => ({
-        schemaVersion: 1,
-        id: message.id,
-        agentId: message.agentId,
-        compartmentKind: message.compartmentKind,
-        placement: message.placement,
-        ...(message.source ? { source: message.source } : {}),
-        receivedAt: message.receivedAt,
-    }));
-    const visibleIds = new Set(visibleRecords.map((record) => record.id));
-    const coverageSnapshot = mailSearchCoverageSnapshot(visibleRecords);
-    const cachedForScope = (0, search_cache_1.searchMailSearchCache)({
-        agentId: input.agentId,
-        placement: input.placement,
-        compartmentKind: input.scope,
-        source: input.source,
-    }, input.cacheOptions);
-    const cachedVisibleIds = new Set(cachedForScope
-        .filter((document) => visibleIds.has(document.messageId))
-        .filter((document) => !mailSearchCacheDocumentNeedsProjectionRefresh(document))
-        .map((document) => document.messageId));
-    const idsToFetch = visibleRecords
-        .filter((record) => !cachedVisibleIds.has(record.id))
-        .map((record) => record.id);
-    const failures = [];
-    let fetchedCount = 0;
-    let skippedCount = 0;
-    for (let start = 0; start < idsToFetch.length; start += MAIL_SEARCH_INDEX_BATCH_SIZE) {
-        const batchIds = idsToFetch.slice(start, start + MAIL_SEARCH_INDEX_BATCH_SIZE);
-        const fetchedMessages = await mapWithConcurrency(batchIds, MAIL_SEARCH_INDEX_FETCH_CONCURRENCY, async (id) => {
-            try {
-                const message = input.store.getIndexedMessageById
-                    ? await input.store.getIndexedMessageById(id)
-                    : await input.store.getMessage(id);
-                return { id, message, error: message ? null : "indexed message was not retrievable" };
-            }
-            catch (error) {
-                return { id, message: null, error: error instanceof Error ? error.message : String(error) };
-            }
-        });
-        failures.push(...fetchedMessages.filter((entry) => entry.error !== null));
-        const fetchedStored = fetchedMessages
-            .map((entry) => entry.message)
-            .filter((message) => message !== null);
-        const result = decryptVisibleMessages(fetchedStored, input.privateKeys);
-        cacheDecryptedMessages(result.decrypted, input.cacheOptions);
-        fetchedCount += fetchedStored.length;
-        skippedCount += result.skipped.length;
-        (0, runtime_1.emitNervesEvent)({
-            component: "repertoire",
-            event: "repertoire.mail_search_index_refresh_progress",
-            message: "mail search index refresh cached a batch",
-            meta: {
-                agentId: input.agentId,
-                fetched: fetchedCount,
-                totalToFetch: idsToFetch.length,
-                alreadyCached: cachedVisibleIds.size,
-                failures: failures.length,
-            },
-        });
-    }
-    const refreshedCachedForScope = (0, search_cache_1.searchMailSearchCache)({
-        agentId: input.agentId,
-        placement: input.placement,
-        compartmentKind: input.scope,
-        source: input.source,
-    }, input.cacheOptions).filter((document) => visibleIds.has(document.messageId));
-    if (failures.length > 0) {
-        const sample = failures.slice(0, 3).map((entry) => `${entry.id}: ${entry.error}`).join("; ");
-        throw new Error(`mail search index refresh incomplete after fetching ${fetchedCount}/${idsToFetch.length} missing message(s); ${failures.length} fetch failed. first failure(s): ${sample}`);
-    }
-    const coverage = (0, search_cache_1.writeMailSearchCoverageRecord)({
-        schemaVersion: 1,
-        ...mailSearchCoverageKey(input),
-        indexedAt: new Date().toISOString(),
-        visibleMessageCount: visibleRecords.length,
-        cachedMessageCount: refreshedCachedForScope.length,
-        decryptableMessageCount: refreshedCachedForScope.length,
-        skippedMessageCount: skippedCount,
-        messageIndexFingerprint: coverageSnapshot.messageIndexFingerprint,
-        textProjectionVersion: search_cache_1.MAIL_SEARCH_TEXT_PROJECTION_VERSION,
-        ...(coverageSnapshot.oldestReceivedAt ? { oldestReceivedAt: coverageSnapshot.oldestReceivedAt } : {}),
-        ...(coverageSnapshot.newestReceivedAt ? { newestReceivedAt: coverageSnapshot.newestReceivedAt } : {}),
-    }, input.cacheOptions);
-    return {
-        coverage,
-        fetched: fetchedCount,
-        alreadyCached: cachedVisibleIds.size,
-    };
-}
 function accessProvenance(message) {
     const provenance = (0, core_1.describeMailProvenance)(message);
     return {
@@ -1390,7 +1242,7 @@ exports.mailToolDefinitions = [
             }
             if (hostedDelegatedSearch && storedCoverageRecord) {
                 if (currentIndexRecordsLoaded) {
-                    currentCoverageSnapshot = mailSearchCoverageSnapshot(currentIndexRecords);
+                    currentCoverageSnapshot = (0, blob_store_1.canonicalMailIndexRecordSnapshot)(currentIndexRecords);
                 }
                 else if (!coverageValidationError) {
                     coverageValidationError = "hosted store does not expose message index records";
@@ -1575,8 +1427,9 @@ exports.mailToolDefinitions = [
                 : requestedScope ?? (args.source ? "delegated" : (familyOrAgentSelf(ctx) ? undefined : "native"));
             try {
                 const cacheOptions = mailSearchCacheOptionsForStore(resolved.store);
-                const refreshed = await refreshMailSearchIndex({
+                const refreshed = await (0, hosted_cache_sync_1.syncHostedMailSearchCache)({
                     agentId: resolved.agentName,
+                    mode: "scoped-upsert",
                     store: resolved.store,
                     privateKeys: resolved.config.privateKeys,
                     storeKind: resolved.storeKind,

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@ouro.bot/cli",
-  "version": "0.1.0-alpha.730",
+  "version": "0.1.0-alpha.731",
   "engines": {
     "node": ">=22"
   },