mem0ai 2.1.0 → 2.1.2

package/dist/oss/index.mjs

@@ -73,14 +73,17 @@ import OpenAI2 from "openai";
 var OpenAILLM = class {
   constructor(config) {
     this.openai = new OpenAI2({ apiKey: config.apiKey });
-    this.model = config.model || "gpt-4-turbo-preview";
+    this.model = config.model || "gpt-4o-mini";
   }
   async generateResponse(messages, responseFormat, tools) {
     const completion = await this.openai.chat.completions.create({
-      messages: messages.map((msg) => ({
-        role: msg.role,
-        content: msg.content
-      })),
+      messages: messages.map((msg) => {
+        const role = msg.role;
+        return {
+          role,
+          content: typeof msg.content === "string" ? msg.content : JSON.stringify(msg.content)
+        };
+      }),
       model: this.model,
       response_format: responseFormat,
       ...tools && { tools, tool_choice: "auto" }
@@ -100,10 +103,13 @@ var OpenAILLM = class {
   }
   async generateChat(messages) {
     const completion = await this.openai.chat.completions.create({
-      messages: messages.map((msg) => ({
-        role: msg.role,
-        content: msg.content
-      })),
+      messages: messages.map((msg) => {
+        const role = msg.role;
+        return {
+          role,
+          content: typeof msg.content === "string" ? msg.content : JSON.stringify(msg.content)
+        };
+      }),
       model: this.model
     });
     const response = completion.choices[0].message;
@@ -125,7 +131,7 @@ var OpenAIStructuredLLM = class {
     const completion = await this.openai.chat.completions.create({
       messages: messages.map((msg) => ({
         role: msg.role,
-        content: msg.content
+        content: typeof msg.content === "string" ? msg.content : JSON.stringify(msg.content)
       })),
       model: this.model,
       ...tools ? {
@@ -161,7 +167,7 @@ var OpenAIStructuredLLM = class {
     const completion = await this.openai.chat.completions.create({
       messages: messages.map((msg) => ({
         role: msg.role,
-        content: msg.content
+        content: typeof msg.content === "string" ? msg.content : JSON.stringify(msg.content)
       })),
       model: this.model
     });
@@ -191,9 +197,9 @@ var AnthropicLLM = class {
       model: this.model,
       messages: otherMessages.map((msg) => ({
         role: msg.role,
-        content: msg.content
+        content: typeof msg.content === "string" ? msg.content : msg.content.image_url.url
       })),
-      system: systemMessage == null ? void 0 : systemMessage.content,
+      system: typeof (systemMessage == null ? void 0 : systemMessage.content) === "string" ? systemMessage.content : void 0,
       max_tokens: 4096
     });
     return response.content[0].text;
@@ -223,7 +229,7 @@ var GroqLLM = class {
       model: this.model,
       messages: messages.map((msg) => ({
         role: msg.role,
-        content: msg.content
+        content: typeof msg.content === "string" ? msg.content : JSON.stringify(msg.content)
       })),
       response_format: responseFormat
     });
@@ -234,7 +240,7 @@ var GroqLLM = class {
       model: this.model,
       messages: messages.map((msg) => ({
         role: msg.role,
-        content: msg.content
+        content: typeof msg.content === "string" ? msg.content : JSON.stringify(msg.content)
       }))
     });
     const message = response.choices[0].message;
@@ -2147,6 +2153,43 @@ Text: ${data}`
   }
 };
 
+// src/oss/src/utils/memory.ts
+var get_image_description = async (image_url) => {
+  const llm = new OpenAILLM({
+    apiKey: process.env.OPENAI_API_KEY
+  });
+  const response = await llm.generateResponse([
+    {
+      role: "user",
+      content: "Provide a description of the image and do not include any additional text."
+    },
+    {
+      role: "user",
+      content: { type: "image_url", image_url: { url: image_url } }
+    }
+  ]);
+  return response;
+};
+var parse_vision_messages = async (messages) => {
+  const parsed_messages = [];
+  for (const message of messages) {
+    let new_message = {
+      role: message.role,
+      content: ""
+    };
+    if (message.role !== "system") {
+      if (typeof message.content === "object" && message.content.type === "image_url") {
+        const description = await get_image_description(
+          message.content.image_url.url
+        );
+        new_message.content = typeof description === "string" ? description : JSON.stringify(description);
+        parsed_messages.push(new_message);
+      } else parsed_messages.push(message);
+    }
+  }
+  return parsed_messages;
+};
+
 // src/oss/src/memory/index.ts
 var Memory = class _Memory {
   constructor(config = {}) {
@@ -2199,8 +2242,9 @@ var Memory = class _Memory {
       );
     }
     const parsedMessages = Array.isArray(messages) ? messages : [{ role: "user", content: messages }];
+    const final_parsedMessages = await parse_vision_messages(parsedMessages);
     const vectorStoreResult = await this.addToVectorStore(
-      parsedMessages,
+      final_parsedMessages,
       metadata,
       filters
     );
@@ -2208,7 +2252,7 @@ var Memory = class _Memory {
     if (this.graphMemory) {
       try {
         graphResult = await this.graphMemory.add(
-          parsedMessages.map((m) => m.content).join("\n"),
+          final_parsedMessages.map((m) => m.content).join("\n"),
           filters
         );
       } catch (error) {