@ouro.bot/cli 0.1.0-alpha.48 → 0.1.0-alpha.480

package/dist/mailroom/blob-store.js

@@ -0,0 +1,558 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.AzureBlobMailroomStore = void 0;
+exports.decryptBlobMessages = decryptBlobMessages;
+const runtime_1 = require("../nerves/runtime");
+const core_1 = require("./core");
+const MESSAGE_INDEX_PREFIX = "message-index";
+const MESSAGE_INDEX_SORT_MAX_MS = 9_999_999_999_999;
+const MESSAGE_INDEX_SORT_WIDTH = 13;
+const MESSAGE_INDEX_NO_SOURCE = "~";
+const DEFAULT_BLOB_OPERATION_TIMEOUT_MS = 20_000;
+const DEFAULT_MESSAGE_FETCH_CONCURRENCY = 20;
+const DEFAULT_MESSAGE_INDEX_BACKFILL_CONCURRENCY = 8;
+const MESSAGE_LIST_SCAN_CONCURRENCY = 32;
+function compareNewestFirst(left, right) {
+    return Date.parse(right.receivedAt) - Date.parse(left.receivedAt);
+}
+function compareCandidatesNewestFirst(left, right) {
+    return Date.parse(right.lastSeenAt) - Date.parse(left.lastSeenAt);
+}
+function blobText(value) {
+    return Buffer.from(`${JSON.stringify(value, null, 2)}\n`, "utf-8");
+}
+function positiveInteger(value, fallback) {
+    if (typeof value !== "number" || !Number.isFinite(value))
+        return fallback;
+    const normalized = Math.floor(value);
+    return normalized > 0 ? normalized : fallback;
+}
+function blobClientName(blob) {
+    return typeof blob.name === "string" && blob.name.trim().length > 0 ? blob.name : "<unknown-blob>";
+}
+function timeoutSignal(timeoutMs) {
+    if (typeof AbortSignal.timeout === "function") {
+        return {
+            signal: AbortSignal.timeout(timeoutMs),
+            dispose() {
+                return undefined;
+            },
+        };
+    }
+    const controller = new AbortController();
+    const timer = setTimeout(() => controller.abort(new Error(`The operation timed out after ${timeoutMs}ms`)), timeoutMs);
+    return {
+        signal: controller.signal,
+        dispose() {
+            clearTimeout(timer);
+        },
+    };
+}
+async function withBlobOperationTimeout(timeoutMs, operation) {
+    const timeout = timeoutSignal(timeoutMs);
+    try {
+        return await operation(timeout.signal);
+    }
+    finally {
+        timeout.dispose();
+    }
+}
+function normalizeBlobOperationError(action, blob, timeoutMs, error) {
+    const message = error instanceof Error ? error.message : String(error);
+    if ((error instanceof Error && error.name === "AbortError") || message.toLowerCase().includes("aborted")) {
+        return new Error(`${action} ${blobClientName(blob)} timed out after ${timeoutMs}ms`);
+    }
+    return new Error(`${action} ${blobClientName(blob)} failed: ${message}`);
+}
+async function downloadJson(blob, timeoutMs) {
+    if (!await blob.exists())
+        return null;
+    try {
+        const buffer = await withBlobOperationTimeout(timeoutMs, (abortSignal) => {
+            return blob.downloadToBuffer(undefined, undefined, { abortSignal });
+        });
+        return JSON.parse(buffer.toString("utf-8"));
+    }
+    catch (error) {
+        throw normalizeBlobOperationError("download", blob, timeoutMs, error);
+    }
+}
+async function uploadJson(blob, value, timeoutMs) {
+    try {
+        await withBlobOperationTimeout(timeoutMs, (abortSignal) => {
+            return blob.uploadData(blobText(value), { abortSignal });
+        });
+    }
+    catch (error) {
+        throw normalizeBlobOperationError("upload", blob, timeoutMs, error);
+    }
+}
+async function mapWithConcurrency(items, concurrency, worker) {
+    if (items.length === 0)
+        return [];
+    const results = new Array(items.length);
+    let nextIndex = 0;
+    const workerLoop = async () => {
+        while (true) {
+            const current = nextIndex;
+            nextIndex += 1;
+            if (current >= items.length)
+                return;
+            results[current] = await worker(items[current], current);
+        }
+    };
+    await Promise.all(Array.from({ length: Math.min(concurrency, items.length) }, async () => workerLoop()));
+    return results;
+}
+function encodeSourceToken(source) {
+    return source ? encodeURIComponent(source.toLowerCase()) : MESSAGE_INDEX_NO_SOURCE;
+}
+function decodeSourceToken(token) {
+    return token === MESSAGE_INDEX_NO_SOURCE ? undefined : decodeURIComponent(token);
+}
+function parseSortMs(receivedAt) {
+    const parsed = Date.parse(receivedAt);
+    if (!Number.isFinite(parsed))
+        return 0;
+    return Math.max(0, Math.min(MESSAGE_INDEX_SORT_MAX_MS, parsed));
+}
+function messageIndexPrefix(agentId) {
+    return `${MESSAGE_INDEX_PREFIX}/${agentId}/`;
+}
+function messageIndexBlobName(message) {
+    const sortKey = String(MESSAGE_INDEX_SORT_MAX_MS - parseSortMs(message.receivedAt)).padStart(MESSAGE_INDEX_SORT_WIDTH, "0");
+    return `${messageIndexPrefix(message.agentId)}${sortKey}__${message.compartmentKind}__${message.placement}__${encodeSourceToken(message.source)}__${message.id}.json`;
+}
+function messageIndexRecord(message) {
+    return {
+        schemaVersion: 1,
+        id: message.id,
+        agentId: message.agentId,
+        compartmentKind: message.compartmentKind,
+        placement: message.placement,
+        ...(message.source ? { source: message.source } : {}),
+        receivedAt: message.receivedAt,
+    };
+}
+function parseMessageIndexBlobName(name) {
+    if (!name.startsWith(`${MESSAGE_INDEX_PREFIX}/`) || !name.endsWith(".json"))
+        return null;
+    const parts = name.split("/");
+    if (parts.length !== 3)
+        return null;
+    const agentId = parts[1];
+    const stem = parts[2].slice(0, -5);
+    const [sortKey, compartmentKind, placement, sourceToken, ...idParts] = stem.split("__");
+    if (!sortKey || !compartmentKind || !placement || !sourceToken || idParts.length === 0)
+        return null;
+    if (compartmentKind !== "native" && compartmentKind !== "delegated")
+        return null;
+    const receivedAtMs = MESSAGE_INDEX_SORT_MAX_MS - Number.parseInt(sortKey, 10);
+    return {
+        schemaVersion: 1,
+        id: idParts.join("__"),
+        agentId,
+        compartmentKind,
+        placement: placement,
+        ...(decodeSourceToken(sourceToken) ? { source: decodeSourceToken(sourceToken) } : {}),
+        receivedAt: Number.isFinite(receivedAtMs) ? new Date(receivedAtMs).toISOString() : new Date(0).toISOString(),
+    };
+}
+function sourceMatchesFilter(source, filter) {
+    if (!filter)
+        return true;
+    if (!source)
+        return false;
+    return source.toLowerCase() === filter.toLowerCase();
+}
+function messageMatchesFilters(message, filters) {
+    return message.agentId === filters.agentId &&
+        (filters.placement ? message.placement === filters.placement : true) &&
+        (filters.compartmentKind ? message.compartmentKind === filters.compartmentKind : true) &&
+        sourceMatchesFilter(message.source, filters.source);
+}
+class AzureBlobMailroomStore {
+    serviceClient;
+    containerName;
+    blobOperationTimeoutMs;
+    messageFetchConcurrency;
+    backfillConcurrency;
+    containerReady = null;
+    constructor(options) {
+        this.serviceClient = options.serviceClient;
+        this.containerName = options.containerName;
+        this.blobOperationTimeoutMs = positiveInteger(options.blobOperationTimeoutMs, DEFAULT_BLOB_OPERATION_TIMEOUT_MS);
+        this.messageFetchConcurrency = positiveInteger(options.messageFetchConcurrency, DEFAULT_MESSAGE_FETCH_CONCURRENCY);
+        this.backfillConcurrency = positiveInteger(options.backfillConcurrency, DEFAULT_MESSAGE_INDEX_BACKFILL_CONCURRENCY);
+        (0, runtime_1.emitNervesEvent)({
+            component: "senses",
+            event: "senses.mail_blob_store_init",
+            message: "azure blob mailroom store initialized",
+            meta: { containerName: this.containerName },
+        });
+    }
+    get container() {
+        return this.serviceClient.getContainerClient(this.containerName);
+    }
+    async ensureContainer() {
+        if (!this.containerReady) {
+            this.containerReady = this.container.createIfNotExists().then(() => undefined);
+        }
+        await this.containerReady;
+    }
+    messageBlob(id) {
+        return this.container.getBlockBlobClient(`messages/${id}.json`);
+    }
+    messageIndexBlob(name) {
+        return this.container.getBlockBlobClient(name);
+    }
+    candidateBlob(id) {
+        return this.container.getBlockBlobClient(`candidates/${id}.json`);
+    }
+    rawBlob(objectName) {
+        return this.container.getBlockBlobClient(objectName);
+    }
+    decisionsBlob(agentId) {
+        return this.container.getBlockBlobClient(`decisions/${agentId}.json`);
+    }
+    accessLogBlob(agentId) {
+        return this.container.getBlockBlobClient(`access-log/${agentId}.jsonl`);
+    }
+    outboundBlob(id) {
+        return this.container.getBlockBlobClient(`outbound/${id}.json`);
+    }
+    async putMessageIndex(message) {
+        await uploadJson(this.messageIndexBlob(messageIndexBlobName(message)), messageIndexRecord(message), this.blobOperationTimeoutMs);
+    }
+    async removeMessageIndex(message) {
+        await this.messageIndexBlob(messageIndexBlobName(message)).deleteIfExists();
+    }
+    async listMessagesLegacy(filters) {
+        const messageBlobNames = [];
+        for await (const item of this.container.listBlobsFlat({ prefix: "messages/" })) {
+            messageBlobNames.push(item.name);
+        }
+        const matches = [];
+        const limit = filters.limit ?? 20;
+        let nextIndex = 0;
+        const worker = async () => {
+            while (nextIndex < messageBlobNames.length) {
+                const current = messageBlobNames[nextIndex];
+                nextIndex += 1;
+                const message = await downloadJson(this.container.getBlockBlobClient(current), this.blobOperationTimeoutMs);
+                if (!message || !messageMatchesFilters(message, filters))
+                    continue;
+                matches.push(message);
+                matches.sort(compareNewestFirst);
+                if (matches.length > limit)
+                    matches.length = limit;
+            }
+        };
+        await Promise.all(Array.from({ length: Math.min(MESSAGE_LIST_SCAN_CONCURRENCY, Math.max(messageBlobNames.length, 1)) }, async () => worker()));
+        return matches.sort(compareNewestFirst).slice(0, limit);
+    }
+    async listMessagesFromIndexes(filters) {
+        const messageIds = [];
+        let sawIndex = false;
+        for await (const item of this.container.listBlobsFlat({ prefix: messageIndexPrefix(filters.agentId) })) {
+            sawIndex = true;
+            const parsed = parseMessageIndexBlobName(item.name);
+            if (!parsed || !messageMatchesFilters(parsed, filters))
+                continue;
+            messageIds.push(parsed.id);
+            if (messageIds.length >= (filters.limit ?? 20))
+                break;
+        }
+        if (!sawIndex)
+            return null;
+        return (await mapWithConcurrency(messageIds, this.messageFetchConcurrency, async (id) => {
+            return downloadJson(this.messageBlob(id), this.blobOperationTimeoutMs);
+        }))
+            .filter((message) => message !== null)
+            .filter((message) => messageMatchesFilters(message, filters))
+            .sort(compareNewestFirst)
+            .slice(0, filters.limit ?? 20);
+    }
+    async backfillMessageIndexes(agentId, onProgress) {
+        await this.ensureContainer();
+        const messageBlobNames = [];
+        for await (const item of this.container.listBlobsFlat({ prefix: "messages/" })) {
+            messageBlobNames.push(item.name);
+        }
+        let indexed = 0;
+        const failures = [];
+        let scanned = 0;
+        let nextIndex = 0;
+        const worker = async () => {
+            while (nextIndex < messageBlobNames.length) {
+                const current = messageBlobNames[nextIndex];
+                nextIndex += 1;
+                try {
+                    const message = await downloadJson(this.container.getBlockBlobClient(current), this.blobOperationTimeoutMs);
+                    if (!message)
+                        continue;
+                    if (agentId && message.agentId !== agentId)
+                        continue;
+                    await uploadJson(this.messageIndexBlob(messageIndexBlobName(message)), messageIndexRecord(message), this.blobOperationTimeoutMs);
+                    indexed += 1;
+                }
+                catch (error) {
+                    failures.push(error instanceof Error ? error.message : String(error));
+                }
+                finally {
+                    scanned += 1;
+                    onProgress?.({ scanned, indexed, failures: failures.length, total: messageBlobNames.length });
+                }
+            }
+        };
+        await Promise.all(Array.from({ length: Math.min(this.backfillConcurrency, Math.max(messageBlobNames.length, 1)) }, async () => worker()));
+        if (failures.length > 0) {
+            const sample = failures.slice(0, 3).join("; ");
+            throw new Error(`hosted message index backfill incomplete after indexing ${indexed} message(s); ${failures.length} blob operation(s) failed. first failure(s): ${sample}. rerun the command to retry remaining messages.`);
+        }
+        (0, runtime_1.emitNervesEvent)({
+            component: "senses",
+            event: "senses.mail_blob_index_backfilled",
+            message: "azure blob mailroom message indexes backfilled",
+            meta: { agentId: agentId ?? null, indexed },
+        });
+        return indexed;
+    }
+    async putRawMessage(input) {
+        await this.ensureContainer();
+        const { message, rawPayload, candidate } = await (0, core_1.buildStoredMailMessage)(input);
+        const existing = await downloadJson(this.messageBlob(message.id), this.blobOperationTimeoutMs);
+        if (existing) {
+            await this.putMessageIndex(existing);
+            (0, runtime_1.emitNervesEvent)({
+                component: "senses",
+                event: "senses.mail_blob_store_dedupe",
+                message: "azure blob mailroom store deduped existing message",
+                meta: { id: message.id, agentId: message.agentId },
+            });
+            return { created: false, message: existing };
+        }
+        await this.rawBlob(message.rawObject).uploadData(blobText(rawPayload));
+        await this.messageBlob(message.id).uploadData(blobText(message));
+        await this.putMessageIndex(message);
+        if (candidate) {
+            await this.candidateBlob(candidate.id).uploadData(blobText(candidate));
+        }
+        (0, runtime_1.emitNervesEvent)({
+            component: "senses",
+            event: "senses.mail_blob_store_message_written",
+            message: "azure blob mailroom store wrote message",
+            meta: { id: message.id, agentId: message.agentId, candidate: candidate !== undefined },
+        });
+        return { created: true, message };
+    }
+    async getMessage(id) {
+        await this.ensureContainer();
+        const message = await downloadJson(this.messageBlob(id), this.blobOperationTimeoutMs);
+        (0, runtime_1.emitNervesEvent)({
+            component: "senses",
+            event: "senses.mail_blob_store_message_read",
+            message: "azure blob mailroom store read message",
+            meta: { id, found: message !== null },
+        });
+        return message;
+    }
+    async listMessages(filters) {
+        await this.ensureContainer();
+        let filtered = await this.listMessagesFromIndexes(filters);
+        let source = "index";
+        if (filtered === null) {
+            filtered = await this.listMessagesLegacy(filters);
+            source = "legacy";
+        }
+        (0, runtime_1.emitNervesEvent)({
+            component: "senses",
+            event: "senses.mail_blob_store_messages_listed",
+            message: "azure blob mailroom store listed messages",
+            meta: { agentId: filters.agentId, count: filtered.length, source },
+        });
+        return filtered;
+    }
+    async updateMessagePlacement(id, placement) {
+        await this.ensureContainer();
+        const blob = this.messageBlob(id);
+        const message = await downloadJson(blob, this.blobOperationTimeoutMs);
+        if (!message) {
+            (0, runtime_1.emitNervesEvent)({
+                component: "senses",
+                event: "senses.mail_blob_store_message_placement_updated",
+                message: "azure blob mailroom store message placement update missed",
+                meta: { id, placement, found: false },
+            });
+            return null;
+        }
+        const updated = { ...message, placement };
+        await blob.uploadData(blobText(updated));
+        await this.removeMessageIndex(message);
+        await this.putMessageIndex(updated);
+        (0, runtime_1.emitNervesEvent)({
+            component: "senses",
+            event: "senses.mail_blob_store_message_placement_updated",
+            message: "azure blob mailroom store updated message placement",
+            meta: { id, placement, found: true },
+        });
+        return updated;
+    }
+    async readRawPayload(objectName) {
+        await this.ensureContainer();
+        const payload = await downloadJson(this.rawBlob(objectName), this.blobOperationTimeoutMs);
+        (0, runtime_1.emitNervesEvent)({
+            component: "senses",
+            event: "senses.mail_blob_store_raw_read",
+            message: "azure blob mailroom store read raw payload",
+            meta: { objectName, found: payload !== null },
+        });
+        return payload;
+    }
+    async putScreenerCandidate(candidate) {
+        await this.ensureContainer();
+        await this.candidateBlob(candidate.id).uploadData(blobText(candidate));
+        (0, runtime_1.emitNervesEvent)({
+            component: "senses",
+            event: "senses.mail_blob_screener_candidate_written",
+            message: "azure blob mail screener candidate written",
+            meta: { id: candidate.id, agentId: candidate.agentId, status: candidate.status },
+        });
+        return candidate;
+    }
+    async updateScreenerCandidate(candidate) {
+        return this.putScreenerCandidate(candidate);
+    }
+    async listScreenerCandidates(filters) {
+        await this.ensureContainer();
+        const candidates = [];
+        for await (const item of this.container.listBlobsFlat({ prefix: "candidates/" })) {
+            const candidate = await downloadJson(this.container.getBlockBlobClient(item.name), this.blobOperationTimeoutMs);
+            if (candidate)
+                candidates.push(candidate);
+        }
+        const filtered = candidates
+            .filter((candidate) => candidate.agentId === filters.agentId)
+            .filter((candidate) => filters.status ? candidate.status === filters.status : true)
+            .filter((candidate) => filters.placement ? candidate.placement === filters.placement : true)
+            .sort(compareCandidatesNewestFirst)
+            .slice(0, filters.limit ?? 50);
+        (0, runtime_1.emitNervesEvent)({
+            component: "senses",
+            event: "senses.mail_blob_screener_candidates_listed",
+            message: "azure blob mail screener candidates listed",
+            meta: { agentId: filters.agentId, count: filtered.length },
+        });
+        return filtered;
+    }
+    async recordMailDecision(entry) {
+        await this.ensureContainer();
+        const complete = {
+            schemaVersion: 1,
+            ...entry,
+            id: entry.id ?? `decision_${Date.now()}_${Math.random().toString(36).slice(2, 8)}`,
+            createdAt: entry.createdAt ?? new Date().toISOString(),
+        };
+        const blob = this.decisionsBlob(entry.agentId);
+        const existing = await downloadJson(blob, this.blobOperationTimeoutMs).catch(() => null);
+        const entries = Array.isArray(existing) ? existing : [];
+        entries.push(complete);
+        await blob.uploadData(blobText(entries));
+        (0, runtime_1.emitNervesEvent)({
+            component: "senses",
+            event: "senses.mail_blob_decision_recorded",
+            message: "azure blob mail decision recorded",
+            meta: { agentId: entry.agentId, messageId: entry.messageId, action: entry.action },
+        });
+        return complete;
+    }
+    async listMailDecisions(agentId) {
+        await this.ensureContainer();
+        const entries = await downloadJson(this.decisionsBlob(agentId), this.blobOperationTimeoutMs);
+        const safeEntries = Array.isArray(entries) ? entries : [];
+        (0, runtime_1.emitNervesEvent)({
+            component: "senses",
+            event: "senses.mail_blob_decisions_listed",
+            message: "azure blob mail decisions listed",
+            meta: { agentId, count: safeEntries.length },
+        });
+        return safeEntries;
+    }
+    async upsertMailOutbound(record) {
+        await this.ensureContainer();
+        await this.outboundBlob(record.id).uploadData(blobText(record));
+        (0, runtime_1.emitNervesEvent)({
+            component: "senses",
+            event: "senses.mail_blob_outbound_record_written",
+            message: "azure blob mail outbound record written",
+            meta: { agentId: record.agentId, id: record.id, status: record.status },
+        });
+        return record;
+    }
+    async getMailOutbound(id) {
+        await this.ensureContainer();
+        const record = await downloadJson(this.outboundBlob(id), this.blobOperationTimeoutMs);
+        (0, runtime_1.emitNervesEvent)({
+            component: "senses",
+            event: "senses.mail_blob_outbound_record_read",
+            message: "azure blob mail outbound record read",
+            meta: { id, found: record !== null },
+        });
+        return record;
+    }
+    async listMailOutbound(agentId) {
+        await this.ensureContainer();
+        const records = [];
+        for await (const item of this.container.listBlobsFlat({ prefix: "outbound/" })) {
+            const record = await downloadJson(this.container.getBlockBlobClient(item.name), this.blobOperationTimeoutMs);
+            if (record)
+                records.push(record);
+        }
+        const filtered = records
+            .filter((record) => record.agentId === agentId)
+            .sort((left, right) => right.updatedAt.localeCompare(left.updatedAt));
+        (0, runtime_1.emitNervesEvent)({
+            component: "senses",
+            event: "senses.mail_blob_outbound_records_listed",
+            message: "azure blob mail outbound records listed",
+            meta: { agentId, count: filtered.length },
+        });
+        return filtered;
+    }
+    async recordAccess(entry) {
+        await this.ensureContainer();
+        const complete = {
+            ...entry,
+            id: `access_${Date.now()}_${Math.random().toString(36).slice(2, 8)}`,
+            accessedAt: new Date().toISOString(),
+        };
+        const blob = this.accessLogBlob(entry.agentId);
+        const existing = await downloadJson(blob, this.blobOperationTimeoutMs).catch(() => null);
+        const entries = Array.isArray(existing) ? existing : [];
+        entries.push(complete);
+        await blob.uploadData(blobText(entries));
+        (0, runtime_1.emitNervesEvent)({
+            component: "senses",
+            event: "senses.mail_blob_access_recorded",
+            message: "azure blob mail access recorded",
+            meta: { agentId: entry.agentId, messageId: entry.messageId ?? null, tool: entry.tool },
+        });
+        return complete;
+    }
+    async listAccessLog(agentId) {
+        await this.ensureContainer();
+        const entries = await downloadJson(this.accessLogBlob(agentId), this.blobOperationTimeoutMs);
+        const safeEntries = Array.isArray(entries) ? entries : [];
+        (0, runtime_1.emitNervesEvent)({
+            component: "senses",
+            event: "senses.mail_blob_access_log_listed",
+            message: "azure blob mail access log listed",
+            meta: { agentId, count: safeEntries.length },
+        });
+        return safeEntries;
+    }
+}
+exports.AzureBlobMailroomStore = AzureBlobMailroomStore;
+function decryptBlobMessages(messages, privateKeys) {
+    return messages.map((message) => (0, core_1.decryptStoredMailMessage)(message, privateKeys));
+}