@elizaos/plugin-bootstrap 1.0.0-beta.74 → 1.0.0-beta.76

package/dist/index.d.ts

@@ -1,4 +1,4 @@
-import { Action, Evaluator, Provider, Plugin, Media } from '@elizaos/core';
+import { Action, Evaluator, Provider, Plugin, Media, IAgentRuntime } from '@elizaos/core';
 
 /**
  * Represents an action that allows selecting an option for a pending task that has multiple options.
@@ -425,6 +425,15 @@ type MediaData = {
  * @returns {Promise<MediaData[]>} - A Promise that resolves with an array of MediaData objects.
  */
 declare function fetchMediaData(attachments: Media[]): Promise<MediaData[]>;
+/**
+ * Processes attachments by generating descriptions for supported media types.
+ * Currently supports image description generation.
+ *
+ * @param {Media[]} attachments - Array of attachments to process
+ * @param {IAgentRuntime} runtime - The agent runtime for accessing AI models
+ * @returns {Promise<Media[]>} - Returns a new array of processed attachments with added description, title, and text properties
+ */
+declare function processAttachments(attachments: Media[], runtime: IAgentRuntime): Promise<Media[]>;
 declare const bootstrapPlugin: Plugin;
 
-export { actionsProvider, anxietyProvider, attachmentsProvider, bootstrapPlugin, capabilitiesProvider, characterProvider, choiceAction, choiceProvider, bootstrapPlugin as default, entitiesProvider, evaluatorsProvider, factsProvider, fetchMediaData, followRoomAction, ignoreAction, muteRoomAction, noneAction, providersProvider, recentMessagesProvider, reflectionEvaluator, relationshipsProvider, replyAction, roleProvider, sendMessageAction, settingsProvider, timeProvider, unfollowRoomAction, unmuteRoomAction, updateEntityAction, updateRoleAction, updateSettingsAction, worldProvider };
+export { actionsProvider, anxietyProvider, attachmentsProvider, bootstrapPlugin, capabilitiesProvider, characterProvider, choiceAction, choiceProvider, bootstrapPlugin as default, entitiesProvider, evaluatorsProvider, factsProvider, fetchMediaData, followRoomAction, ignoreAction, muteRoomAction, noneAction, processAttachments, providersProvider, recentMessagesProvider, reflectionEvaluator, relationshipsProvider, replyAction, roleProvider, sendMessageAction, settingsProvider, timeProvider, unfollowRoomAction, unmuteRoomAction, updateEntityAction, updateRoleAction, updateSettingsAction, worldProvider };

package/dist/index.js

@@ -82,7 +82,9 @@ import {
   postCreationTemplate,
   shouldRespondTemplate,
   truncateToCompleteSentence,
-  parseKeyValueXml
+  parseKeyValueXml,
+  imageDescriptionTemplate,
+  ContentType
 } from "@elizaos/core";
 
 // ../../node_modules/uuid/dist/esm/stringify.js
@@ -3820,7 +3822,8 @@ var attachmentsProvider = {
   description: "List of attachments sent during the current conversation, including names, descriptions, and summaries",
   dynamic: true,
   get: async (runtime, message) => {
-    let allAttachments = message.content.attachments || [];
+    const currentMessageAttachments = message.content.attachments || [];
+    let allAttachments = [...currentMessageAttachments];
     const { roomId } = message;
     const conversationLength = runtime.getConversationLength();
     const recentMessagesData = await runtime.getMemories({
@@ -3836,17 +3839,24 @@ var attachmentsProvider = {
       if (lastMessageWithAttachment) {
         const lastMessageTime = lastMessageWithAttachment?.createdAt ?? Date.now();
         const oneHourBeforeLastMessage = lastMessageTime - 60 * 60 * 1e3;
-        allAttachments = recentMessagesData.reverse().flatMap((msg) => {
+        const currentAttachmentsMap = new Map(
+          currentMessageAttachments.map((att) => [att.id, att])
+        );
+        const recentAttachments = recentMessagesData.reverse().flatMap((msg) => {
           const msgTime = msg.createdAt ?? Date.now();
           const isWithinTime = msgTime >= oneHourBeforeLastMessage;
           const attachments = msg.content.attachments || [];
-          if (!isWithinTime) {
-            for (const attachment of attachments) {
-              attachment.text = "[Hidden]";
+          return attachments.map((attachment) => {
+            if (currentAttachmentsMap.has(attachment.id)) {
+              return null;
             }
-          }
-          return attachments;
+            if (!isWithinTime) {
+              return { ...attachment, text: "[Hidden]" };
+            }
+            return attachment;
+          }).filter((att) => att !== null);
         });
+        allAttachments = [...currentMessageAttachments, ...recentAttachments];
       }
     }
     const formattedAttachments = allAttachments.map(
@@ -5641,6 +5651,56 @@ async function fetchMediaData(attachments) {
     })
   );
 }
+async function processAttachments(attachments, runtime) {
+  if (!attachments || attachments.length === 0) {
+    return [];
+  }
+  logger19.debug(`[Bootstrap] Processing ${attachments.length} attachment(s)`);
+  const processedAttachments = [];
+  for (const attachment of attachments) {
+    try {
+      const processedAttachment = { ...attachment };
+      if (attachment.contentType === ContentType.IMAGE && !attachment.description) {
+        logger19.debug(`[Bootstrap] Generating description for image: ${attachment.url}`);
+        try {
+          const response = await runtime.useModel(ModelType13.IMAGE_DESCRIPTION, {
+            prompt: imageDescriptionTemplate,
+            imageUrl: attachment.url
+          });
+          if (typeof response === "string") {
+            const parsedXml = parseKeyValueXml(response);
+            if (parsedXml?.description && parsedXml?.text) {
+              processedAttachment.description = parsedXml.description;
+              processedAttachment.title = parsedXml.title || "Image";
+              processedAttachment.text = parsedXml.text;
+              logger19.debug(
+                `[Bootstrap] Generated description: ${processedAttachment.description?.substring(0, 100)}...`
+              );
+            } else {
+              logger19.warn(`[Bootstrap] Failed to parse XML response for image description`);
+            }
+          } else if (response && typeof response === "object" && "description" in response) {
+            processedAttachment.description = response.description;
+            processedAttachment.title = response.title || "Image";
+            processedAttachment.text = response.description;
+            logger19.debug(
+              `[Bootstrap] Generated description: ${processedAttachment.description?.substring(0, 100)}...`
+            );
+          } else {
+            logger19.warn(`[Bootstrap] Unexpected response format for image description`);
+          }
+        } catch (error) {
+          logger19.error(`[Bootstrap] Error generating image description:`, error);
+        }
+      }
+      processedAttachments.push(processedAttachment);
+    } catch (error) {
+      logger19.error(`[Bootstrap] Failed to process attachment ${attachment.url}:`, error);
+      processedAttachments.push(attachment);
+    }
+  }
+  return processedAttachments;
+}
 var messageReceivedHandler = async ({
   runtime,
   message,
@@ -5728,6 +5788,12 @@ var messageReceivedHandler = async ({
         logger19.debug(
           `[Bootstrap] Skipping shouldRespond check for ${runtime.character.name} because ${room?.type} ${room?.source}`
         );
+        if (message.content.attachments && message.content.attachments.length > 0) {
+          message.content.attachments = await processAttachments(
+            message.content.attachments,
+            runtime
+          );
+        }
         let shouldRespond = true;
         if (!shouldSkipShouldRespond) {
           const shouldRespondPrompt = composePromptFromState9({
@@ -6404,6 +6470,7 @@ export {
   ignoreAction,
   muteRoomAction,
   noneAction,
+  processAttachments,
   providersProvider,
   recentMessagesProvider,
   reflectionEvaluator,