@kybernesis/buzz 0.6.3 → 0.7.0

package/dist/bridge.js

@@ -1,4 +1,5 @@
 import { channelIdentity } from "@kybernesis/enterprise";
+import { fetchMedia, isImage, parseMedia } from "./media.js";
 import { speakerCredentials } from "./credentials.js";
 import { Client } from "eve/client";
 import { BuzzRelay } from "./relay.js";
@@ -103,7 +104,49 @@ export function buzzBridge(options) {
      * nothing. The bridge knew the answer the whole time: it is the connection
      * the message arrived on.
      */
-    async function ask(channel, text, pubkey, community) {
+    /**
+     * One inbound message, as the parts a turn can carry.
+     *
+     * @remarks
+     * Returns a plain string when there is nothing but text, so the common case
+     * stays exactly as it was — a change in the shape of every ordinary turn is
+     * not worth paying for the rare one.
+     *
+     * A failed fetch does NOT throw. The person is owed an answer either way, and
+     * an agent told "there was an attachment I could not retrieve" can say so;
+     * an agent told nothing answers the caption and looks like it ignored the
+     * picture, which is the bug being fixed.
+     */
+    async function composeMessage(text, attachments) {
+        if (attachments.length === 0)
+            return text;
+        const parts = [];
+        if (text)
+            parts.push({ type: "text", text });
+        for (const ref of attachments) {
+            try {
+                const media = await fetchMedia(key, ref);
+                if (isImage(media.mediaType)) {
+                    parts.push({ type: "image", image: media.bytes, mediaType: media.mediaType });
+                }
+                else {
+                    parts.push({
+                        type: "text",
+                        text: `[The sender attached a ${media.mediaType} file, which cannot be read as an image. Tell them what you received and ask for it in a readable form if you need it.]`,
+                    });
+                }
+            }
+            catch (error) {
+                log(`could not fetch an attachment: ${error.message}`);
+                parts.push({
+                    type: "text",
+                    text: `[The sender attached a file, and it could not be retrieved (${error.message}). Say so rather than answering as if there were no attachment.]`,
+                });
+            }
+        }
+        return parts;
+    }
+    async function ask(channel, message, pubkey, community) {
         // Built per turn, for the person whose turn it is — and kept current for as
         // long as that turn runs.
         const client = clientFor(pubkey);
@@ -138,7 +181,7 @@ export function buzzBridge(options) {
                 if (unread > 0) {
                     log(`caught up on ${unread} unread event(s) in ${channel.slice(0, 8)} before answering`);
                 }
-                const response = await session.send(text, { clientContext: { buzzCommunity: community, buzzChannel: channel } });
+                const response = await session.send(message, { clientContext: { buzzCommunity: community, buzzChannel: channel } });
                 const result = await response.result();
                 // Where this turn left the conversation, so the next one starts after it.
                 sessions.set(channel, { id: existing.id, streamIndex: session.state.streamIndex });
@@ -152,15 +195,15 @@ export function buzzBridge(options) {
             }
         }
         const created = await client.sessions.create({
-            message: text,
+            message,
             clientContext: { buzzCommunity: community, buzzChannel: channel },
         });
-        const message = (await created.response.result()).message ?? "";
+        const reply = (await created.response.result()).message ?? "";
         sessions.set(channel, {
             id: created.session.state.sessionId,
             streamIndex: created.session.state.streamIndex,
         });
-        return message;
+        return reply;
     }
     /**
      * Send someone the link that makes them known, privately.
@@ -223,8 +266,21 @@ export function buzzBridge(options) {
         if (!channel || !addressed)
             return;
         const text = String(event.content ?? "").trim();
-        if (!text)
+        const attachments = parseMedia(event);
+        /**
+         * Drop only a message with NOTHING in it — and say so when we do.
+         *
+         * This guard used to test the text alone, and sat above the log line, so an
+         * image sent without a caption was discarded before anything was written
+         * down. From the outside the agent had simply ignored someone; from the
+         * inside there was no record that a message had ever arrived. The reply
+         * path was hardened against exactly this failure — an empty answer now
+         * says so out loud — and the receiving path was left as it was.
+         */
+        if (!text && attachments.length === 0) {
+            log(`nothing to answer in ${channel.slice(0, 8)} from ${event.pubkey.slice(0, 8)} — no text, no attachments`);
             return;
+        }
         let speaker;
         try {
             speaker = await identity.resolve(PROVIDER, event.pubkey, npubEncode(event.pubkey));
@@ -242,11 +298,23 @@ export function buzzBridge(options) {
                 await refuse(relay, event.pubkey, speaker.reason);
             return;
         }
-        log(`asked in ${channel.slice(0, 8)} by ${speaker.user.email}: ${text.slice(0, 60)}`);
+        log(`asked in ${channel.slice(0, 8)} by ${speaker.user.email}: ${text.slice(0, 60)}` +
+            (attachments.length ? ` [+${attachments.length} attachment(s)]` : ""));
+        /**
+         * Attachments become parts of the turn, or become words about themselves.
+         *
+         * An image the model can see goes in as bytes. Anything else — a zip, a
+         * PDF the model cannot read, a fetch that failed — goes in as a sentence
+         * saying so, because the alternative is an agent that answers the caption
+         * while the person waits for a reply about the file they sent. Being told
+         * "I got a file I cannot read" is a worse answer than the truth only if
+         * you never wanted the truth.
+         */
+        const message = await composeMessage(text, attachments);
         void relay.react(event.id, SEEN);
         const stopTyping = relay.typingIn(channel);
         try {
-            const reply = await serialize(channel, () => ask(channel, text, event.pubkey, from));
+            const reply = await serialize(channel, () => ask(channel, message, event.pubkey, from));
             if (!reply) {
                 /**
                  * Silence is the worst answer available.

package/dist/media.d.ts

@@ -0,0 +1,48 @@
+import type { AgentKey } from "./keys.js";
+import type { NostrEvent } from "./relay.js";
+/** One attachment on an inbound message. */
+export interface MediaRef {
+    url: string;
+    /** From the `m` field when the client sends one; the response may correct it. */
+    mediaType?: string;
+    /** From the `x` field: the hash the blob is addressed by. */
+    sha256?: string;
+}
+/** Fetched bytes, ready to become a message part. */
+export interface FetchedMedia {
+    url: string;
+    mediaType: string;
+    bytes: Uint8Array;
+}
+/**
+ * The attachments on a message, from its `imeta` tags (NIP-92).
+ *
+ * @remarks
+ * A Buzz client does NOT inline an attachment into the message text. The
+ * content stays prose — often prose *about* the picture — and the media lives
+ * in a tag. A bridge that reads only `event.content` therefore sees a caption
+ * with nothing attached, and, when there is no caption, sees nothing at all.
+ *
+ * Both were reported from a live deployment: an image with a caption was
+ * answered as though the caption were the whole message, and an image without
+ * one produced no response and no log line. The agent could not tell either had
+ * happened, so it speculated about its own vision support — which was fine.
+ */
+export declare function parseMedia(event: NostrEvent): MediaRef[];
+/**
+ * Fetch one attachment as the agent.
+ *
+ * @remarks
+ * Buzz media is authenticated: an anonymous GET answers 401, not 404. So the
+ * URL cannot simply be handed to the model — a provider fetching it would be
+ * refused, and the failure would surface as a broken image rather than as an
+ * auth problem. The bytes are fetched here, with the same Blossom
+ * authorisation this package already signs to upload an agent's avatar, and
+ * passed inline.
+ *
+ * `t get` rather than `t upload`, and the blob's own hash when the sender told
+ * us one, so the authorisation is scoped to reading that specific blob.
+ */
+export declare function fetchMedia(key: AgentKey, ref: MediaRef, fetchImpl?: typeof fetch): Promise<FetchedMedia>;
+/** Whether the model can be shown this directly, or only told about it. */
+export declare function isImage(mediaType: string): boolean;

package/dist/media.js

@@ -0,0 +1,85 @@
+import { createHash } from "node:crypto";
+import { finalizeEvent } from "nostr-tools/pure";
+/**
+ * The attachments on a message, from its `imeta` tags (NIP-92).
+ *
+ * @remarks
+ * A Buzz client does NOT inline an attachment into the message text. The
+ * content stays prose — often prose *about* the picture — and the media lives
+ * in a tag. A bridge that reads only `event.content` therefore sees a caption
+ * with nothing attached, and, when there is no caption, sees nothing at all.
+ *
+ * Both were reported from a live deployment: an image with a caption was
+ * answered as though the caption were the whole message, and an image without
+ * one produced no response and no log line. The agent could not tell either had
+ * happened, so it speculated about its own vision support — which was fine.
+ */
+export function parseMedia(event) {
+    const media = [];
+    for (const tag of event.tags) {
+        if (tag[0] !== "imeta")
+            continue;
+        // An imeta tag is space-separated "key value" pairs, one per element.
+        const fields = new Map();
+        for (const entry of tag.slice(1)) {
+            const space = entry.indexOf(" ");
+            if (space > 0)
+                fields.set(entry.slice(0, space), entry.slice(space + 1));
+        }
+        const url = fields.get("url");
+        if (url)
+            media.push({ url, mediaType: fields.get("m"), sha256: fields.get("x") });
+    }
+    return media;
+}
+/**
+ * Fetch one attachment as the agent.
+ *
+ * @remarks
+ * Buzz media is authenticated: an anonymous GET answers 401, not 404. So the
+ * URL cannot simply be handed to the model — a provider fetching it would be
+ * refused, and the failure would surface as a broken image rather than as an
+ * auth problem. The bytes are fetched here, with the same Blossom
+ * authorisation this package already signs to upload an agent's avatar, and
+ * passed inline.
+ *
+ * `t get` rather than `t upload`, and the blob's own hash when the sender told
+ * us one, so the authorisation is scoped to reading that specific blob.
+ */
+export async function fetchMedia(key, ref, fetchImpl = fetch) {
+    const expires = Math.floor(Date.now() / 1000) + 300;
+    const auth = finalizeEvent({
+        kind: 24242,
+        created_at: Math.floor(Date.now() / 1000),
+        tags: [
+            ["t", "get"],
+            ...(ref.sha256 ? [["x", ref.sha256]] : []),
+            ["expiration", String(expires)],
+        ],
+        content: "Fetch buzz-media",
+    }, key.secretKey);
+    const response = await fetchImpl(ref.url, {
+        headers: { authorization: `Nostr ${Buffer.from(JSON.stringify(auth)).toString("base64url")}` },
+    });
+    if (!response.ok) {
+        throw new Error(`${response.status} fetching ${ref.url}`);
+    }
+    const bytes = new Uint8Array(await response.arrayBuffer());
+    // The server's answer wins over the sender's claim: the sender's `m` field is
+    // whatever their client asserted, and a wrong media type reaches the model as
+    // an image it cannot decode.
+    const mediaType = response.headers.get("content-type")?.split(";")[0]?.trim() || ref.mediaType || "application/octet-stream";
+    if (ref.sha256) {
+        // Blossom addresses a blob BY its hash, so a mismatch means the bytes are
+        // not the ones that were sent. Cheap to check and not worth trusting past.
+        const actual = createHash("sha256").update(bytes).digest("hex");
+        if (actual !== ref.sha256) {
+            throw new Error(`content hash mismatch for ${ref.url}`);
+        }
+    }
+    return { url: ref.url, mediaType, bytes };
+}
+/** Whether the model can be shown this directly, or only told about it. */
+export function isImage(mediaType) {
+    return mediaType.startsWith("image/");
+}

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@kybernesis/buzz",
-  "version": "0.6.3",
+  "version": "0.7.0",
   "description": "Put an eve agent in a Buzz workspace as a member: per-speaker verified identity resolved through the control plane, self-service identity linking, presence, typing and seen signals.",
   "license": "Apache-2.0",
   "type": "module",