@kybernesis/buzz 0.15.0 → 0.16.0

package/README.md

@@ -95,6 +95,13 @@ share link as the person who asked (`POST /eve/v1/kyb/artifacts/:id/share`, `@ky
 app and opens it beside the conversation; everywhere else the link is in the text. The person
 can withdraw the link from the agent. `BUZZ_CARDS=off` stops card messages.
 
+A screenshot the agent shows (`show_screen` in `@kybernesis/computer`) goes out as an ordinary Buzz
+picture, not a card: the bridge fetches it from the agent as the person who asked
+(`GET /eve/v1/kyb/screens/:id`, served only to them) and stores it in the community's own media
+store, which serves members only. Every Buzz app draws it, with or without the plugin. Like an
+artifact, a screenshot asked for in a channel is shared with that channel. If it cannot be
+fetched or stored, the card goes out as before.
+
 ## Questions and approvals
 
 When the agent needs a yes or an answer, the bridge posts the question as a reply to the message

package/dist/bridge.js

@@ -1,6 +1,7 @@
 import { cardMessage, cardText, readHiddenAnswers } from "@kybernesis/cards";
 import { channelIdentity } from "@kybernesis/enterprise";
 import { fetchMedia, parseMedia } from "./media.js";
+import { isAgentScreenshot, pictureMessage } from "./pictures.js";
 import { speakerCredentials } from "./credentials.js";
 import { surfaceWriter } from "./surface.js";
 import { SessionStore } from "./sessions.js";
@@ -78,6 +79,27 @@ export function buzzBridge(options) {
             return card;
         }
     };
+    /**
+     * A screenshot card as a picture everyone in the channel sees: fetched from
+     * the agent as the person who asked (it is theirs), stored in this
+     * community's media store. See pictures.ts. If that fails, the card goes out
+     * as it was.
+     */
+    const screenshotIn = async (relay, card, speaker) => {
+        try {
+            const credentials = credentialsFor(speaker);
+            return await pictureMessage(card, {
+                agentUrl: options.agentUrl,
+                headers: async () => ({ ...(await credentials.headers()), authorization: `Bearer ${await credentials.bearer()}` }),
+                relay: relay.url,
+                key,
+            });
+        }
+        catch (error) {
+            log(`could not post a screenshot as a picture (${error.message})`);
+            return null;
+        }
+    };
     /**
      * A turn's cards, one message each, threaded where the reply went.
      *
@@ -90,6 +112,12 @@ export function buzzBridge(options) {
             return;
         let posted = 0;
         for (const card of await Promise.all(cards.map((c) => sharedInChannel(c, speaker)))) {
+            const picture = isAgentScreenshot(card) ? await screenshotIn(relay, card, speaker) : null;
+            if (picture) {
+                if (relay.reply(channel, picture.content, replyTo, { mention: false, tags: picture.tags }))
+                    posted += 1;
+                continue;
+            }
             const content = cardMessage(card);
             if (content && relay.reply(channel, content, replyTo, { mention: false }))
                 posted += 1;

package/dist/pictures.d.ts

@@ -0,0 +1,26 @@
+import type { Card } from "@kybernesis/cards";
+import type { AgentKey } from "./keys.js";
+import { type UploadedMedia } from "./profile.js";
+export interface PictureMessage {
+    content: string;
+    tags: string[][];
+}
+export interface PictureOptions {
+    /** The agent's base URL, where the screenshot is served. */
+    agentUrl: string;
+    /** The asker's headers for the agent: the screenshot is theirs to read. */
+    headers: () => Promise<Record<string, string>>;
+    /** The community the message goes to: its media store holds the picture. */
+    relay: string;
+    key: AgentKey;
+    fetchImpl?: typeof fetch;
+    upload?: (relay: string, key: AgentKey, bytes: Uint8Array, mediaType: string) => Promise<UploadedMedia>;
+}
+/** Whether this card is a screenshot the bridge can carry as a picture. */
+export declare function isAgentScreenshot(card: Card): boolean;
+/**
+ * The screenshot card as a Buzz picture message, or null when it is not one
+ * the bridge carries. Throws when the picture cannot be fetched or stored; the
+ * caller then posts the card as it was.
+ */
+export declare function pictureMessage(card: Card, options: PictureOptions): Promise<PictureMessage | null>;

package/dist/pictures.js

@@ -0,0 +1,90 @@
+import { uploadMedia } from "./profile.js";
+/**
+ * A screenshot the agent showed someone, as a picture in the conversation.
+ *
+ * @remarks
+ * `show_screen` keeps the image on the agent's machine and names it by a path
+ * on the agent (`/eve/v1/kyb/screens/<id>.png`) that only the person it was
+ * taken for can read. KYBER Studio fetches it with that person's token. Buzz
+ * cannot: the cards plugin has no token, so it drew "This picture is on the
+ * agent and can't be shown here", and Buzz without the plugin showed only the
+ * caption.
+ *
+ * So the bridge fetches it, as the person who asked, and puts it in the
+ * community's own media store, which serves only members of that community.
+ * The message is an ordinary Buzz picture message (the words, a Markdown image
+ * and its `imeta` tag, as Buzz's own composer writes them), so every Buzz app
+ * draws it, with or without the plugin. Asking for a screenshot in a channel
+ * shares it with that channel, the same as an artifact posted there.
+ */
+/** The only address a screenshot card may name for the bridge to fetch: one on the agent, by id. */
+const SCREEN_PATH = /^\/eve\/v1\/kyb\/screens\/[0-9a-f]{32}(?:\.png)?$/;
+/** Larger than any screen; the community's own limit for images is higher still. */
+const MAX_BYTES = 20 * 1024 * 1024;
+/** Whether this card is a screenshot the bridge can carry as a picture. */
+export function isAgentScreenshot(card) {
+    if (card.card !== "screenshot" || !card.data || typeof card.data !== "object")
+        return false;
+    const image = card.data.image;
+    return typeof image === "string" && SCREEN_PATH.test(image);
+}
+/**
+ * The screenshot card as a Buzz picture message, or null when it is not one
+ * the bridge carries. Throws when the picture cannot be fetched or stored; the
+ * caller then posts the card as it was.
+ */
+export async function pictureMessage(card, options) {
+    if (!isAgentScreenshot(card))
+        return null;
+    const data = card.data;
+    const fetchImpl = options.fetchImpl ?? fetch;
+    const upload = options.upload ?? uploadMedia;
+    const response = await fetchImpl(`${options.agentUrl.replace(/\/$/, "")}${data.image}`, {
+        headers: await options.headers(),
+        signal: AbortSignal.timeout(15_000),
+    });
+    if (!response.ok)
+        throw new Error(`${response.status} fetching the screenshot from the agent`);
+    if (Number(response.headers.get("content-length") ?? 0) > MAX_BYTES)
+        throw new Error("the screenshot is too large");
+    const bytes = new Uint8Array(await response.arrayBuffer());
+    const size = pngSize(bytes);
+    if (!size || bytes.byteLength > MAX_BYTES)
+        throw new Error("the agent did not answer with a PNG");
+    const stored = await upload(options.relay, options.key, bytes, "image/png");
+    const caption = typeof data.caption === "string" ? oneLine(data.caption) : "";
+    const site = typeof data.site === "string" ? oneLine(data.site) : "";
+    const words = caption && site ? `${caption} (${site})` : caption || site || "A screenshot from the agent's computer.";
+    return {
+        content: `${words}\n\n![${markdownLabel(caption || "Screenshot")}](<${stored.url}>)`,
+        tags: [
+            [
+                "imeta",
+                `url ${stored.url}`,
+                "m image/png",
+                `x ${stored.sha256}`,
+                `size ${stored.size}`,
+                `dim ${size.width}x${size.height}`,
+                "filename screenshot.png",
+            ],
+        ],
+    };
+}
+/** Width and height from a PNG's header, when the bytes are one. */
+function pngSize(bytes) {
+    if (bytes.length < 24 || bytes[0] !== 0x89 || bytes[1] !== 0x50 || bytes[2] !== 0x4e || bytes[3] !== 0x47)
+        return null;
+    const view = new DataView(bytes.buffer, bytes.byteOffset, bytes.byteLength);
+    const width = view.getUint32(16);
+    const height = view.getUint32(20);
+    return width > 0 && height > 0 ? { width, height } : null;
+}
+function oneLine(text) {
+    return text.replace(/\s+/g, " ").trim().slice(0, 200);
+}
+/** A Markdown image label, escaped the way Buzz escapes an attachment's name. */
+function markdownLabel(text) {
+    return Array.from(text, (char) => (char.charCodeAt(0) < 32 ? " " : char))
+        .join("")
+        .replace(/[\\`*_{}[\]()!<>&]/g, "\\$&");
+}

package/dist/profile.d.ts

@@ -46,6 +46,14 @@ export declare function copy(from: string, to: readonly string[], key: AgentKey)
  * up, and nobody should need a blob token to give their agent a face.
  */
 export declare function uploadImage(relay: string, key: AgentKey, bytes: Uint8Array, mediaType: string): Promise<string>;
+/** What a community's media store holds after an upload: what a message's `imeta` tag names. */
+export type UploadedMedia = {
+    url: string;
+    sha256: string;
+    size: number;
+};
+/** {@link uploadImage}, answering with the hash and size of what was stored too. */
+export declare function uploadMedia(relay: string, key: AgentKey, bytes: Uint8Array, mediaType: string): Promise<UploadedMedia>;
 /**
  * Remove everything from an image that is not the image.
  *

package/dist/profile.js

@@ -147,6 +147,10 @@ export async function copy(from, to, key) {
  * up, and nobody should need a blob token to give their agent a face.
  */
 export async function uploadImage(relay, key, bytes, mediaType) {
+    return (await uploadMedia(relay, key, bytes, mediaType)).url;
+}
+/** {@link uploadImage}, answering with the hash and size of what was stored too. */
+export async function uploadMedia(relay, key, bytes, mediaType) {
     const { createHash } = await import("node:crypto");
     const base = relay.replace(/^wss:/, "https:").replace(/^ws:/, "http:").replace(/\/$/, "");
     // Stripped BEFORE hashing: the hash has to be of what is actually sent, and a
@@ -183,10 +187,10 @@ export async function uploadImage(relay, key, bytes, mediaType) {
             body: body,
         });
         if (response.ok) {
-            const body = (await response.json().catch(() => ({})));
-            if (typeof body.url === "string")
-                return body.url;
-            return `${base}/${sha256}`;
+            const answer = (await response.json().catch(() => ({})));
+            // Absolute either way: a message names its picture by full address, and a server may answer with a path.
+            const url = typeof answer.url === "string" ? new URL(answer.url, `${base}/`).toString() : `${base}/${sha256}`;
+            return { url, sha256, size: body.byteLength };
         }
         last = `${response.status} ${(await response.text().catch(() => "")).slice(0, 120)}`;
     }

package/dist/relay.d.ts

@@ -105,7 +105,8 @@ export declare class BuzzRelay {
     private discovery;
     /** Since when a newly found DM is read back, so a message sent before it was found is not lost. */
     private discoveredAt;
-    private readonly url;
+    /** The community's address, as configured. */
+    readonly url: string;
     private readonly rest;
     private readonly key;
     private readonly pollMs;
@@ -164,6 +165,7 @@ export declare class BuzzRelay {
     /** `mention: false` threads under `replyTo` without notifying its author again, as a follow-up to a reply does. */
     reply(channel: string, text: string, replyTo?: NostrEvent, options?: {
         mention?: boolean;
+        tags?: readonly string[][];
     }): string | null;
     /**
      * Replace the words of a message this agent posted. Every client shows the

package/dist/relay.js

@@ -104,6 +104,7 @@ export class BuzzRelay {
     discovery = null;
     /** Since when a newly found DM is read back, so a message sent before it was found is not lost. */
     discoveredAt = null;
+    /** The community's address, as configured. */
     url;
     rest;
     key;
@@ -475,6 +476,9 @@ export class BuzzRelay {
             if (options.mention !== false)
                 tags.push(["p", replyTo.pubkey]);
         }
+        // Attachments (`imeta`), which the words name.
+        for (const tag of options.tags ?? [])
+            tags.push([...tag]);
         return this.publish({ kind: KIND_MESSAGE, tags, content: text });
     }
     /**

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@kybernesis/buzz",
-  "version": "0.15.0",
+  "version": "0.16.0",
   "description": "Put an eve agent in a Buzz workspace as a member: per-speaker verified identity resolved through the control plane, self-service identity linking, presence, typing and seen signals.",
   "license": "Apache-2.0",
   "type": "module",