@ouro.bot/cli 0.1.0-alpha.92 → 0.1.0-alpha.921

package/dist/senses/bluebubbles/latest-turn.js

@@ -0,0 +1,311 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.reserveObservation = reserveObservation;
+exports.beginObservationBatch = beginObservationBatch;
+exports.reserveObservationFromBatch = reserveObservationFromBatch;
+exports.clearPending = clearPending;
+exports.reactivateObservation = reactivateObservation;
+exports.mergeObservationReservations = mergeObservationReservations;
+exports.promote = promote;
+exports.isCurrent = isCurrent;
+exports.observationSchedulingKeys = observationSchedulingKeys;
+exports.awaitDeliveryAdmission = awaitDeliveryAdmission;
+exports.isDeliveryAdmittedNow = isDeliveryAdmittedNow;
+exports.cancel = cancel;
+exports.finish = finish;
+exports.__resetBlueBubblesLatestTurnsForTests = __resetBlueBubblesLatestTurnsForTests;
+const node_crypto_1 = require("node:crypto");
+const runtime_1 = require("../../nerves/runtime");
+const AMBIGUOUS_IDENTIFIER = Symbol("ambiguous-bluebubbles-identifier");
+const UNRESOLVED_SCHEDULING_KEY = "unresolved:*";
+const pending = new Map();
+const lanes = new Map();
+const identifierBindings = new Map();
+const reservationHints = new WeakMap();
+const batchReservations = new WeakMap();
+const observationEpoch = `${new Date().toISOString()}/${(0, node_crypto_1.randomUUID)()}`;
+let nextOrdinal = 0;
+function clean(value) {
+    const trimmed = value?.trim();
+    return trimmed && trimmed.toLowerCase() !== "unknown" ? trimmed : null;
+}
+function canonicalIdentifier(value) {
+    const trimmed = clean(value);
+    if (!trimmed)
+        return null;
+    if (trimmed.includes("@"))
+        return trimmed.toLowerCase();
+    if (/^[+\d\s().-]+$/.test(trimmed)) {
+        const compact = trimmed.replace(/[^\d+]/g, "");
+        if (compact)
+            return compact;
+    }
+    return trimmed;
+}
+function guidAlias(guid) {
+    return `guid:${guid}`;
+}
+function identifierAlias(identifier) {
+    return `identifier:${identifier}`;
+}
+function observationHints(input) {
+    const guid = clean(input.chatGuid);
+    const identifier = canonicalIdentifier(input.chatIdentifier);
+    return [
+        ...(guid ? [guidAlias(guid)] : []),
+        ...(identifier ? [identifierAlias(identifier)] : []),
+    ];
+}
+function settlePending(ordinal) {
+    const entry = pending.get(ordinal);
+    if (!entry)
+        return;
+    pending.delete(ordinal);
+    entry.settle();
+}
+function createReservation(ordinal, input) {
+    const knownHints = new Set(observationHints(input));
+    const reservation = Object.freeze({
+        observationEpoch,
+        ordinal,
+        get hints() {
+            return Object.freeze([...knownHints]);
+        },
+    });
+    reservationHints.set(reservation, knownHints);
+    let settle;
+    const settled = new Promise((resolve) => {
+        settle = resolve;
+    });
+    pending.set(ordinal, {
+        reservation,
+        hints: new Set(knownHints),
+        settled,
+        settle,
+    });
+    return reservation;
+}
+function reserveObservation(input) {
+    return createReservation(++nextOrdinal, input);
+}
+function beginObservationBatch(size) {
+    if (!Number.isSafeInteger(size) || size <= 0)
+        throw new Error("bluebubbles_observation_batch_size_invalid");
+    // BlueBubbles' `before` filter is inclusive. Subtract one millisecond so a
+    // row created in the same clock tick after this reservation cannot be
+    // backdated into the already-reserved ordinal range.
+    const batch = Object.freeze({
+        highOrdinal: nextOrdinal + size,
+        size,
+        beforeTimestamp: Date.now() - 1,
+    });
+    nextOrdinal = batch.highOrdinal;
+    batchReservations.set(batch, new Set());
+    return batch;
+}
+function reserveObservationFromBatch(batch, newestOffset, input) {
+    const used = batchReservations.get(batch);
+    if (!used || !Number.isSafeInteger(newestOffset) || newestOffset < 0 || newestOffset >= batch.size) {
+        throw new Error("bluebubbles_observation_batch_offset_invalid");
+    }
+    const ordinal = batch.highOrdinal - newestOffset;
+    if (used.has(ordinal))
+        throw new Error("bluebubbles_observation_batch_offset_reused");
+    used.add(ordinal);
+    return createReservation(ordinal, input);
+}
+function clearPending(reservation) {
+    settlePending(reservation.ordinal);
+}
+function reactivateObservation(reservation) {
+    if (pending.has(reservation.ordinal))
+        return;
+    const knownHints = reservationHints.get(reservation);
+    if (!knownHints)
+        throw new Error("bluebubbles_observation_reservation_unknown");
+    let settle;
+    const settled = new Promise((resolve) => {
+        settle = resolve;
+    });
+    pending.set(reservation.ordinal, {
+        reservation,
+        hints: new Set(knownHints),
+        settled,
+        settle,
+    });
+}
+function mergeObservationReservations(primary, duplicate) {
+    if (primary === duplicate)
+        return primary;
+    const primaryHints = reservationHints.get(primary);
+    const duplicateHints = reservationHints.get(duplicate);
+    if (!primaryHints || !duplicateHints)
+        throw new Error("bluebubbles_observation_reservation_unknown");
+    for (const hint of duplicateHints)
+        primaryHints.add(hint);
+    const primaryPending = pending.get(primary.ordinal);
+    if (primaryPending) {
+        for (const hint of duplicateHints)
+            primaryPending.hints.add(hint);
+    }
+    settlePending(duplicate.ordinal);
+    return primary;
+}
+function bindIdentifier(identifier, guid) {
+    const existing = identifierBindings.get(identifier);
+    if (existing === undefined) {
+        identifierBindings.set(identifier, guid);
+        return true;
+    }
+    if (existing === guid)
+        return true;
+    identifierBindings.set(identifier, AMBIGUOUS_IDENTIFIER);
+    return false;
+}
+function resolveCanonicalChat(input) {
+    const inputGuid = clean(input.chatGuid);
+    const observedIdentifier = clean(input.chatIdentifier);
+    const identifier = canonicalIdentifier(input.chatIdentifier);
+    let guid = inputGuid;
+    if (guid && identifier)
+        bindIdentifier(identifier, guid);
+    if (!guid && identifier) {
+        const binding = identifierBindings.get(identifier);
+        if (typeof binding === "string")
+            guid = binding;
+    }
+    if (!guid)
+        return null;
+    return {
+        chatGuid: guid,
+        ...(observedIdentifier && identifier && identifierBindings.get(identifier) === guid
+            ? { chatIdentifier: observedIdentifier }
+            : {}),
+        sessionKey: `chat:${guid}`,
+    };
+}
+function promote(reservation, input, options = {}) {
+    const canonicalChat = resolveCanonicalChat(input);
+    settlePending(reservation.ordinal);
+    if (!canonicalChat)
+        return { status: "unresolved" };
+    const chatKey = guidAlias(canonicalChat.chatGuid);
+    const lane = lanes.get(chatKey) ?? { highWater: 0, current: null };
+    const sameGenerationRetry = reservation.ordinal === lane.highWater
+        && options.allowSameGenerationRetry === true
+        && lane.current === null;
+    if (reservation.ordinal < lane.highWater || (reservation.ordinal === lane.highWater && !sameGenerationRetry)) {
+        return { status: "stale" };
+    }
+    lane.current?.controller.abort(new Error("superseded"));
+    const controller = new AbortController();
+    const capability = Object.freeze({
+        ordinal: reservation.ordinal,
+        chatKey,
+        canonicalChat: Object.freeze(canonicalChat),
+        signal: controller.signal,
+        controller,
+    });
+    lane.highWater = reservation.ordinal;
+    lane.current = capability;
+    lanes.set(chatKey, lane);
+    (0, runtime_1.emitNervesEvent)({
+        component: "senses",
+        event: "senses.bluebubbles_latest_turn_promoted",
+        message: "promoted the latest process-local bluebubbles turn",
+        meta: {
+            chatKey,
+            ordinal: reservation.ordinal,
+            aliasCount: 1 + [...identifierBindings.values()].filter((binding) => binding === canonicalChat.chatGuid).length,
+        },
+    });
+    return { status: "promoted", capability };
+}
+function internal(capability) {
+    return capability;
+}
+function isCurrent(capability) {
+    const lane = lanes.get(capability.chatKey);
+    return lane?.current === capability && !capability.signal.aborted;
+}
+function pendingIntersects(capability, entry) {
+    if (entry.reservation.ordinal <= capability.ordinal)
+        return false;
+    const hasGuidHint = [...entry.hints].some((hint) => hint.startsWith("guid:"));
+    if (entry.hints.size === 0)
+        return true;
+    for (const hint of entry.hints) {
+        if (hint === capability.chatKey)
+            return true;
+        if (hint.startsWith("identifier:")) {
+            const identifier = hint.slice("identifier:".length);
+            const binding = identifierBindings.get(identifier);
+            if (binding === capability.canonicalChat.chatGuid)
+                return true;
+            // An identifier-only observation can repair into any GUID lane. Until
+            // that relation is proved, it is a temporary global delivery fence—not
+            // an active shared chat lane. A direct GUID hint keeps paired events
+            // independently schedulable through that exact coordinate.
+            if (!hasGuidHint && typeof binding !== "string")
+                return true;
+        }
+    }
+    return false;
+}
+function observationSchedulingKeys(reservation) {
+    const hints = reservationHints.get(reservation);
+    if (!hints)
+        throw new Error("bluebubbles_observation_reservation_unknown");
+    const keys = new Set();
+    const hasGuidHint = [...hints].some((hint) => hint.startsWith("guid:"));
+    if (hints.size === 0)
+        keys.add(UNRESOLVED_SCHEDULING_KEY);
+    for (const hint of hints) {
+        if (hint.startsWith("identifier:")) {
+            const identifier = hint.slice("identifier:".length);
+            const binding = identifierBindings.get(identifier);
+            keys.add(typeof binding === "string"
+                ? guidAlias(binding)
+                : hasGuidHint ? hint : UNRESOLVED_SCHEDULING_KEY);
+        }
+        else {
+            keys.add(hint);
+        }
+    }
+    return Object.freeze([...keys]);
+}
+async function awaitDeliveryAdmission(capability) {
+    const value = internal(capability);
+    while (isCurrent(value)) {
+        const blockers = [...pending.values()].filter((entry) => pendingIntersects(value, entry));
+        if (blockers.length === 0)
+            return true;
+        await Promise.all(blockers.map((entry) => entry.settled));
+    }
+    return false;
+}
+function isDeliveryAdmittedNow(capability) {
+    const value = internal(capability);
+    return isCurrent(value)
+        && ![...pending.values()].some((entry) => pendingIntersects(value, entry));
+}
+function cancel(capability, reason = "cancelled") {
+    const lane = lanes.get(capability.chatKey);
+    if (lane?.current !== capability)
+        return;
+    internal(capability).controller.abort(new Error(reason));
+    lane.current = null;
+}
+function finish(capability) {
+    const lane = lanes.get(capability.chatKey);
+    if (lane?.current === capability)
+        lane.current = null;
+}
+function __resetBlueBubblesLatestTurnsForTests() {
+    for (const entry of pending.values())
+        entry.settle();
+    pending.clear();
+    lanes.clear();
+    identifierBindings.clear();
+    nextOrdinal = 0;
+}

package/dist/senses/{bluebubbles-media.js → bluebubbles/media.js}

@@ -33,16 +33,31 @@ var __importStar = (this && this.__importStar) || (function () {
     };
 })();
 Object.defineProperty(exports, "__esModule", { value: true });
+exports.downloadBlueBubblesAttachment = exports.VLM_TEXT_WRAPPERS = void 0;
 exports.hydrateBlueBubblesAttachments = hydrateBlueBubblesAttachments;
 const node_child_process_1 = require("node:child_process");
 const fs = __importStar(require("node:fs/promises"));
 const os = __importStar(require("node:os"));
 const path = __importStar(require("node:path"));
-const runtime_1 = require("../nerves/runtime");
-const identity_1 = require("../heart/identity");
-const MAX_ATTACHMENT_BYTES = 8 * 1024 * 1024;
+const runtime_1 = require("../../nerves/runtime");
+const identity_1 = require("../../heart/identity");
+const model_capabilities_1 = require("../../heart/model-capabilities");
+const image_normalize_1 = require("../../heart/attachments/image-normalize");
+const store_1 = require("../../heart/attachments/store");
+const bluebubbles_1 = require("../../heart/attachments/sources/bluebubbles");
+const attachment_download_1 = require("./attachment-download");
+// Pin the exact wrapper strings so tests and code read from one source.
+exports.VLM_TEXT_WRAPPERS = {
+    description: (desc) => `[image description: ${desc}]`,
+    failure: (reason) => `[image description failed: ${reason}]`,
+};
+// D3 prompt template. Included here (not templated out) because the contract
+// is load-bearing — see planning doc section D3 and AX-4.
+function buildVlmPrompt(userText) {
+    const body = userText ?? "";
+    return `User message: "${body}"\n\nDescribe this image in detail, focusing on anything relevant to what the user said above.\nInclude any text visible in the image verbatim.`;
+}
 const AUDIO_EXTENSIONS = new Set([".mp3", ".wav", ".m4a", ".caf", ".ogg"]);
-const IMAGE_EXTENSIONS = new Set([".jpg", ".jpeg", ".png", ".gif", ".webp", ".heic", ".heif"]);
 const AUDIO_EXTENSION_BY_CONTENT_TYPE = {
     "audio/wav": ".wav",
     "audio/x-wav": ".wav",
@@ -66,31 +81,19 @@ const AUDIO_INPUT_FORMAT_BY_EXTENSION = {
 const WHISPER_CPP_FORMULA = "whisper-cpp";
 const WHISPER_CPP_MODEL_NAME = "ggml-base.en.bin";
 const WHISPER_CPP_MODEL_URL = `https://huggingface.co/ggerganov/whisper.cpp/resolve/main/${WHISPER_CPP_MODEL_NAME}`;
-const WHISPER_CPP_TOOLS_DIR = path.join((0, identity_1.getAgentToolsRoot)(), "whisper-cpp");
-const WHISPER_CPP_MODELS_DIR = path.join(WHISPER_CPP_TOOLS_DIR, "models");
-const WHISPER_CPP_MODEL_PATH = path.join(WHISPER_CPP_MODELS_DIR, WHISPER_CPP_MODEL_NAME);
-function buildBlueBubblesApiUrl(baseUrl, endpoint, password) {
-    const root = baseUrl.endsWith("/") ? baseUrl : `${baseUrl}/`;
-    const url = new URL(endpoint.replace(/^\//, ""), root);
-    url.searchParams.set("password", password);
-    return url.toString();
+// Lazy — getAgentToolsRoot() requires identity to be resolved, which isn't
+// true at module-load time in most unit test contexts. Tests that only touch
+// the image/VLM path shouldn't break because the audio path asked for a
+// tools root they don't need.
+function whisperCppPaths() {
+    const toolsDir = path.join((0, identity_1.getAgentToolsRoot)(), "whisper-cpp");
+    const modelsDir = path.join(toolsDir, "models");
+    const modelPath = path.join(modelsDir, WHISPER_CPP_MODEL_NAME);
+    return { toolsDir, modelsDir, modelPath };
 }
 function describeAttachment(attachment) {
     return attachment.transferName?.trim() || attachment.guid?.trim() || "attachment";
 }
-function inferContentType(attachment, responseType) {
-    const normalizedResponseType = responseType?.split(";")[0]?.trim().toLowerCase();
-    if (normalizedResponseType) {
-        return normalizedResponseType;
-    }
-    return attachment.mimeType?.trim().toLowerCase() || undefined;
-}
-function isImageAttachment(attachment, contentType) {
-    if (contentType?.startsWith("image/"))
-        return true;
-    const extension = path.extname(attachment.transferName ?? "").toLowerCase();
-    return IMAGE_EXTENSIONS.has(extension);
-}
 function isAudioAttachment(attachment, contentType) {
     if (contentType?.startsWith("audio/"))
         return true;
@@ -170,12 +173,13 @@ async function resolveWhisperCppBinary(timeoutMs) {
     return candidate;
 }
 async function ensureWhisperCppModel(timeoutMs, fetchImpl) {
+    const { modelsDir, modelPath } = whisperCppPaths();
     try {
-        await fs.access(WHISPER_CPP_MODEL_PATH);
-        return WHISPER_CPP_MODEL_PATH;
+        await fs.access(modelPath);
+        return modelPath;
     }
     catch {
-        await fs.mkdir(WHISPER_CPP_MODELS_DIR, { recursive: true });
+        await fs.mkdir(modelsDir, { recursive: true });
         const response = await fetchImpl(WHISPER_CPP_MODEL_URL, {
             method: "GET",
             signal: AbortSignal.timeout(Math.max(timeoutMs, 300_000)),
@@ -183,8 +187,8 @@ async function ensureWhisperCppModel(timeoutMs, fetchImpl) {
         if (!response.ok) {
             throw new Error(`failed to download whisper.cpp model: HTTP ${response.status}`);
         }
-        await fs.writeFile(WHISPER_CPP_MODEL_PATH, Buffer.from(await response.arrayBuffer()));
-        return WHISPER_CPP_MODEL_PATH;
+        await fs.writeFile(modelPath, Buffer.from(await response.arrayBuffer()));
+        return modelPath;
     }
 }
 async function convertAudioForWhisperCpp(sourcePath, outputPath, timeoutMs) {
@@ -236,59 +240,105 @@ async function transcribeAudioWithWhisperCpp(params, modelFetchImpl = fetch) {
         await fs.rm(workDir, { recursive: true, force: true }).catch(() => undefined);
     }
 }
-async function downloadAttachment(attachment, config, channelConfig, fetchImpl) {
-    const guid = attachment.guid?.trim();
-    if (!guid) {
-        throw new Error("attachment guid missing");
-    }
-    if (typeof attachment.totalBytes === "number" && attachment.totalBytes > MAX_ATTACHMENT_BYTES) {
-        throw new Error(`attachment exceeds ${MAX_ATTACHMENT_BYTES} byte limit`);
-    }
-    const url = buildBlueBubblesApiUrl(config.serverUrl, `/api/v1/attachment/${encodeURIComponent(guid)}/download`, config.password);
-    const response = await fetchImpl(url, {
-        method: "GET",
-        signal: AbortSignal.timeout(channelConfig.requestTimeoutMs),
-    });
-    if (!response.ok) {
-        throw new Error(`HTTP ${response.status}`);
-    }
-    const buffer = Buffer.from(await response.arrayBuffer());
-    if (buffer.length > MAX_ATTACHMENT_BYTES) {
-        throw new Error(`attachment exceeds ${MAX_ATTACHMENT_BYTES} byte limit`);
-    }
-    return {
-        buffer,
-        contentType: inferContentType(attachment, response.headers.get("content-type")),
-    };
-}
+var attachment_download_2 = require("./attachment-download");
+Object.defineProperty(exports, "downloadBlueBubblesAttachment", { enumerable: true, get: function () { return attachment_download_2.downloadBlueBubblesAttachment; } });
 async function hydrateBlueBubblesAttachments(attachments, config, channelConfig, deps = {}) {
-    (0, runtime_1.emitNervesEvent)({
-        component: "senses",
-        event: "senses.bluebubbles_media_hydrate",
-        message: "hydrating bluebubbles attachments",
-        meta: {
-            attachmentCount: attachments.length,
-            preferAudioInput: deps.preferAudioInput ?? false,
-        },
-    });
     const fetchImpl = deps.fetchImpl ?? fetch;
     const modelFetchImpl = deps.modelFetchImpl ?? fetch;
     const transcribeAudio = deps.transcribeAudio ?? ((params) => transcribeAudioWithWhisperCpp(params, modelFetchImpl));
     const preferAudioInput = deps.preferAudioInput ?? false;
+    const chatModel = deps.chatModel;
+    const visionCapable = chatModel ? (0, model_capabilities_1.getModelCapabilities)(chatModel).vision === true : true;
+    const agentName = (0, identity_1.getAgentName)();
+    const agentRoot = (0, identity_1.getAgentRoot)(agentName);
     const inputParts = [];
     const transcriptAdditions = [];
     const notices = [];
     for (const attachment of attachments) {
         const name = describeAttachment(attachment);
+        const initialRecord = attachment.guid?.trim()
+            ? (0, store_1.cacheRecentAttachment)(agentName, (0, bluebubbles_1.buildBlueBubblesAttachmentRecord)(attachment), agentRoot)
+            : null;
         try {
-            const downloaded = await downloadAttachment(attachment, config, channelConfig, fetchImpl);
+            const downloaded = await (0, attachment_download_1.downloadBlueBubblesAttachment)(attachment, config, channelConfig, fetchImpl);
             const base64 = downloaded.buffer.toString("base64");
-            if (isImageAttachment(attachment, downloaded.contentType)) {
-                inputParts.push({
-                    type: "image_url",
-                    image_url: {
-                        url: `data:${downloaded.contentType ?? "application/octet-stream"};base64,${base64}`,
-                        detail: "auto",
+            const byteCount = downloaded.buffer.length;
+            const persistedRecord = initialRecord && initialRecord.source === "bluebubbles"
+                ? await (0, bluebubbles_1.persistBlueBubblesAttachmentSource)(agentName, initialRecord, {
+                    buffer: downloaded.buffer,
+                    mimeType: downloaded.contentType,
+                    byteCount,
+                }, agentRoot)
+                : null;
+            if ((0, attachment_download_1.isBlueBubblesImageAttachment)(attachment, downloaded.contentType)) {
+                const mimeType = downloaded.contentType ?? "application/octet-stream";
+                if (visionCapable) {
+                    inputParts.push({
+                        type: "image_url",
+                        image_url: {
+                            url: `data:${mimeType};base64,${base64}`,
+                            detail: "auto",
+                        },
+                    });
+                    (0, runtime_1.emitNervesEvent)({
+                        level: "warn",
+                        component: "senses",
+                        event: "senses.bluebubbles_media_hydrate",
+                        message: "bluebubbles media hydrate",
+                        meta: {
+                            attachmentGuid: attachment.guid,
+                            mimeType,
+                            byteCount,
+                            hydrationPath: "native-passthrough",
+                        },
+                    });
+                    continue;
+                }
+                try {
+                    if (!deps.vlmDescribe) {
+                        throw new Error("no VLM describer configured — wire a vlmDescribe dep or configure a vision-capable chat model");
+                    }
+                    if (!persistedRecord?.sourceData.localPath) {
+                        throw new Error("image attachment could not be persisted for normalization");
+                    }
+                    const normalized = await (0, image_normalize_1.normalizeImageForVision)({
+                        attachment: persistedRecord,
+                        sourcePath: persistedRecord.sourceData.localPath,
+                        agentName,
+                        agentRoot,
+                    });
+                    const normalizedBuffer = await fs.readFile(normalized.path);
+                    const normalizedMime = normalized.mimeType ?? persistedRecord.mimeType ?? "image/jpeg";
+                    const dataUrl = `data:${normalizedMime};base64,${normalizedBuffer.toString("base64")}`;
+                    const description = await deps.vlmDescribe({
+                        prompt: buildVlmPrompt(deps.userText),
+                        imageDataUrl: dataUrl,
+                        attachmentGuid: attachment.guid,
+                        mimeType: normalizedMime,
+                        chatModel,
+                    });
+                    inputParts.push({
+                        type: "text",
+                        text: exports.VLM_TEXT_WRAPPERS.description(description),
+                    });
+                }
+                catch (vlmError) {
+                    const reason = vlmError instanceof Error ? vlmError.message : String(vlmError);
+                    inputParts.push({
+                        type: "text",
+                        text: exports.VLM_TEXT_WRAPPERS.failure(reason),
+                    });
+                }
+                (0, runtime_1.emitNervesEvent)({
+                    level: "warn",
+                    component: "senses",
+                    event: "senses.bluebubbles_media_hydrate",
+                    message: "bluebubbles media hydrate",
+                    meta: {
+                        attachmentGuid: attachment.guid,
+                        mimeType: persistedRecord?.mimeType ?? mimeType,
+                        byteCount,
+                        hydrationPath: "vlm-describe",
                     },
                 });
                 continue;