@friendlyrobot/discord-pi-agent 0.12.0 → 0.13.0

package/dist/image-description.d.ts

@@ -1,10 +1,11 @@
 import type { Model } from "@earendil-works/pi-ai";
 import type { AgentService } from "./agent-service";
 /**
- * Use a vision-capable model to describe an image, returning a text
- * description that can be inlined into a prompt for a non-vision model.
+ * Use a vision-capable model to describe a media attachment (image or PDF),
+ * returning a text description that can be inlined into a prompt for a
+ * non-vision model.
  *
- * Creates a temporary in-memory session, sends the image, extracts the
+ * Creates a temporary in-memory session, sends the media, extracts the
  * assistant's text reply, then disposes the session.
  */
 export declare function describeImage(agentService: AgentService, imageData: string, mimeType: string, userText: string, visionModel: Model<any>): Promise<string>;

package/dist/index.js

@@ -870,26 +870,32 @@ var logger5 = createModuleLogger("image-description");
 async function describeImage(agentService, imageData, mimeType, userText, visionModel) {
   const session = await agentService.createTemporarySession();
   await session.setModel(visionModel);
+  const isPdf = mimeType === "application/pdf";
   const imageContent = {
     type: "image",
     data: imageData,
     mimeType
   };
-  const promptText = userText.trim().length > 0 ? `The user sent this image with the following message: "${userText}". Please describe the image in detail and address any questions from the user's message.` : "Please describe this image in detail. What do you see?";
+  let promptText;
+  if (isPdf) {
+    promptText = userText.trim().length > 0 ? `The user sent a PDF document with the following message: "${userText}". Please extract and summarize the text content of this PDF. Be thorough — include all important details, sections, and data from the document.` : "Please extract and summarize the text content of this PDF document. Be thorough — include all important details, sections, data, and key points.";
+  } else {
+    promptText = userText.trim().length > 0 ? `The user sent this image with the following message: "${userText}". Please describe the image in detail and address any questions from the user's message.` : "Please describe this image in detail. What do you see?";
+  }
   let text = "";
   try {
     await session.prompt(promptText, { images: [imageContent] });
     text = extractLastAssistantText(session);
   } catch (error) {
-    logger5.error({ error }, "vision model prompt failed");
-    text = "(Vision model failed to process the image.)";
+    logger5.error({ error, mimeType }, "vision model prompt failed");
+    text = "(Vision model failed to process the file.)";
   } finally {
     session.dispose();
   }
   if (!text) {
     return "(Vision model returned no description.)";
   }
-  logger5.debug({ textLength: text.length }, "image described");
+  logger5.debug({ textLength: text.length, mimeType }, "media described");
   return text;
 }
 function extractLastAssistantText(session) {
@@ -1211,28 +1217,42 @@ async function readTextAttachments(message) {
   }
   return results;
 }
-var IMAGE_ATTACHMENT_EXTENSIONS = [".png", ".jpg", ".jpeg", ".gif", ".webp"];
-var MAX_IMAGE_ATTACHMENT_SIZE = 10 * 1024 * 1024;
-async function readImageAttachments(message) {
+var MEDIA_ATTACHMENT_EXTENSIONS = [
+  ".png",
+  ".jpg",
+  ".jpeg",
+  ".gif",
+  ".webp",
+  ".pdf"
+];
+var MAX_MEDIA_ATTACHMENT_SIZE = 25 * 1024 * 1024;
+function isMediaAttachment(attachment) {
+  const ext = attachment.name?.slice(attachment.name.lastIndexOf(".")).toLowerCase();
+  if (!ext || !MEDIA_ATTACHMENT_EXTENSIONS.includes(ext)) {
+    return false;
+  }
+  const ct = attachment.contentType;
+  if (!ct) {
+    return false;
+  }
+  return ct.startsWith("image/") || ct === "application/pdf";
+}
+async function readMediaAttachments(message) {
   const attachments = message.attachments;
   if (attachments.size === 0) {
     return [];
   }
   const results = [];
   for (const [, attachment] of attachments) {
-    const ext = attachment.name?.slice(attachment.name.lastIndexOf(".")).toLowerCase();
-    if (!ext || !IMAGE_ATTACHMENT_EXTENSIONS.includes(ext)) {
-      continue;
-    }
-    if (!attachment.contentType?.startsWith("image/")) {
+    if (!isMediaAttachment(attachment)) {
       continue;
     }
-    if (attachment.size > MAX_IMAGE_ATTACHMENT_SIZE) {
+    if (attachment.size > MAX_MEDIA_ATTACHMENT_SIZE) {
       logger6.warn({
         messageId: message.id,
         filename: attachment.name,
         size: attachment.size
-      }, "image attachment too large, skipping");
+      }, "media attachment too large, skipping");
       continue;
     }
     try {
@@ -1240,14 +1260,14 @@ async function readImageAttachments(message) {
         messageId: message.id,
         filename: attachment.name,
         size: attachment.size
-      }, "fetching image attachment");
+      }, "fetching media attachment");
       const response = await fetch(attachment.url);
       if (!response.ok) {
         logger6.warn({
           messageId: message.id,
           filename: attachment.name,
           status: response.status
-        }, "failed to fetch image attachment");
+        }, "failed to fetch media attachment");
         continue;
       }
       const buffer = await response.arrayBuffer();
@@ -1255,10 +1275,10 @@ async function readImageAttachments(message) {
       results.push({
         filename: attachment.name,
         data: base64,
-        mimeType: attachment.contentType ?? "image/png"
+        mimeType: attachment.contentType ?? "application/octet-stream"
       });
     } catch (error) {
-      logger6.error({ messageId: message.id, filename: attachment.name, error }, "error fetching image attachment");
+      logger6.error({ messageId: message.id, filename: attachment.name, error }, "error fetching media attachment");
     }
   }
   return results;
@@ -1277,27 +1297,29 @@ function parseVisionModelId(visionModelId) {
     modelId: trimmed.substring(slashIndex + 1)
   };
 }
-async function resolveImageAttachments(imageAttachments, content, currentModel, config, agentService) {
+async function resolveMediaAttachments(media, content, currentModel, config, agentService) {
   const modelSupportsVision = currentModel?.input.includes("image") ?? false;
   if (modelSupportsVision) {
+    const names = media.map((m) => m.filename).join(", ");
     logger6.info({
-      imageCount: imageAttachments.length,
+      count: media.length,
+      filenames: names,
       model: currentModel ? `${currentModel.provider}/${currentModel.id}` : "none"
-    }, "passing images natively to vision-capable model");
-    const images = imageAttachments.map((img) => ({
+    }, "passing media natively to vision-capable model");
+    const images = media.map((m) => ({
       type: "image",
-      data: img.data,
-      mimeType: img.mimeType
+      data: m.data,
+      mimeType: m.mimeType
     }));
     return { content, images };
   }
   if (!config.visionModelId) {
-    const imageNames = imageAttachments.map((i) => i.filename).join(", ");
-    logger6.info({ imageNames }, "image attachments received but vision model not configured");
+    const names = media.map((m) => m.filename).join(", ");
+    logger6.info({ filenames: names }, "media attachments received but vision model not configured");
     const note = `
 
-[User sent image attachment(s): ${imageNames}]
-` + "(Image vision not configured. Set visionModelId to enable image understanding.)";
+[User sent media attachment(s): ${names}]
+` + "(Media vision not configured. Set visionModelId to enable image/PDF understanding.)";
     return { content: content ? content + note : note, images: [] };
   }
   const parsed = parseVisionModelId(config.visionModelId);
@@ -1307,21 +1329,23 @@ async function resolveImageAttachments(imageAttachments, content, currentModel,
   const visionModel = agentService.findModel(parsed.provider, parsed.modelId);
   if (!visionModel) {
     logger6.warn({ visionModelId: config.visionModelId }, "vision model not found in registry");
-    const imageNames = imageAttachments.map((i) => i.filename).join(", ");
+    const names = media.map((m) => m.filename).join(", ");
     const note = `
 
-[User sent image attachment(s): ${imageNames}]
+[User sent media attachment(s): ${names}]
 (Vision model not found: ${config.visionModelId})`;
     return { content: content ? content + note : note, images: [] };
   }
   logger6.info({
-    imageCount: imageAttachments.length,
+    count: media.length,
     visionModel: `${visionModel.provider}/${visionModel.id}`
-  }, "describing images with vision model");
+  }, "describing media with vision model");
   const descriptions = [];
-  for (const img of imageAttachments) {
-    const description = await describeImage(agentService, img.data, img.mimeType, content, visionModel);
-    descriptions.push(`[Image: ${img.filename}]
+  for (const m of media) {
+    const isPdf = m.mimeType === "application/pdf";
+    const description = await describeImage(agentService, m.data, m.mimeType, content, visionModel);
+    const label = isPdf ? `[PDF: ${m.filename}]` : `[Image: ${m.filename}]`;
+    descriptions.push(`${label}
 ${description}`);
   }
   if (descriptions.length > 0) {
@@ -1414,8 +1438,8 @@ async function onMessage(message, config, agentService, sessionRegistry, authCon
 ${a.content}`).join("");
     content = content ? content + suffix : attachmentContents[0].content;
   }
-  const imageAttachments = await readImageAttachments(message);
-  if (!content && imageAttachments.length === 0) {
+  const mediaAttachments = await readMediaAttachments(message);
+  if (!content && mediaAttachments.length === 0) {
     logger6.debug({ messageId: message.id }, "ignored empty message (no text or images)");
     return;
   }
@@ -1488,8 +1512,8 @@ ${a.content}`).join("");
     response = await promptQueue.enqueue(async () => {
       let promptContent = content;
       let promptImages;
-      if (imageAttachments.length > 0) {
-        const resolved = await resolveImageAttachments(imageAttachments, promptContent, session.model, config, agentService);
+      if (mediaAttachments.length > 0) {
+        const resolved = await resolveMediaAttachments(mediaAttachments, promptContent, session.model, config, agentService);
         promptContent = resolved.content;
         if (resolved.images.length > 0) {
           promptImages = resolved.images;

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@friendlyrobot/discord-pi-agent",
-  "version": "0.12.0",
+  "version": "0.13.0",
   "description": "Reusable Discord gateway bridge for persistent pi agent sessions",
   "license": "MIT",
   "type": "module",