@convex-dev/agent 0.1.18 → 0.2.0

package/dist/client/files.js

@@ -1,16 +1,17 @@
+import { assert } from "convex-helpers";
 export const MAX_FILE_SIZE = 1024 * 64;
 /**
  * Store a file in the file storage and return the URL and fileId.
  * @param ctx A ctx object from an action.
  * @param component The agent component.
  * @param blob The blob to store.
- * @param filename The filename to store.
- * @param sha256 The sha256 hash of the file. If not provided, it will be
+ * @param args.filename The filename to store.
+ * @param args.sha256 The sha256 hash of the file. If not provided, it will be
  *   computed. However, to ensure no corruption during transfer, you can
  *   calculate this on the client to enforce integrity.
  * @returns The URL, fileId, and storageId of the stored file.
  */
-export async function storeFile(ctx, component, blob, filename, sha256) {
+export async function storeFile(ctx, component, blob, { filename, sha256 } = {}) {
     if (!("runAction" in ctx) || !("storage" in ctx)) {
         throw new Error("You're trying to save a file that's too large in a mutation / workflow. " +
             "You can store the file in file storage from an action first, then pass a URL instead. " +
@@ -105,20 +106,74 @@ export async function getFile(ctx, component, fileId) {
         },
     };
 }
-function getParts(url, mimeType, filename) {
+function getParts(url, mediaType, filename) {
     const filePart = {
         type: "file",
         data: new URL(url),
-        mimeType,
+        mediaType,
         filename,
     };
-    const imagePart = mimeType.startsWith("image/")
-        ? {
-            type: "image",
-            image: new URL(url),
-            mimeType,
-        }
+    const imagePart = mediaType.startsWith("image/")
+        ? { type: "image", image: new URL(url), mediaType }
         : undefined;
     return { filePart, imagePart };
 }
+/**
+ * Check if a URL points to localhost
+ */
+function isLocalhostUrl(url) {
+    return (url.hostname === "localhost" ||
+        url.hostname === "127.0.0.1" ||
+        url.hostname === "::1" ||
+        url.hostname === "0.0.0.0");
+}
+/**
+ * Download a file from a URL
+ */
+async function downloadFile(url) {
+    // Fetch the file
+    const response = await fetch(url);
+    if (!response.ok) {
+        throw new Error(`Failed to fetch ${url}: ${response.statusText}`);
+    }
+    return await response.arrayBuffer();
+}
+/**
+ * Process messages to inline file and image URLs that point to localhost
+ * by converting them to base64. This solves the problem of LLMs not being
+ * able to access localhost URLs.
+ */
+export async function inlineMessagesFiles(messages) {
+    // Process each message to convert localhost URLs to base64
+    return Promise.all(messages.map(async (message) => {
+        if ((message.role !== "user" && message.role !== "assistant") ||
+            typeof message.content === "string" ||
+            !Array.isArray(message.content)) {
+            return message;
+        }
+        const processedContent = await Promise.all(message.content.map(async (part) => {
+            if (part.type === "image" && part.image instanceof URL) {
+                assert(message.role === "user", "Images can only be in user messages");
+                if (isLocalhostUrl(part.image)) {
+                    const imageData = await downloadFile(part.image);
+                    return { ...part, image: imageData };
+                }
+            }
+            // Handle file parts
+            if (part.type === "file" && part.data instanceof URL) {
+                if (isLocalhostUrl(part.data)) {
+                    const fileData = await downloadFile(part.data);
+                    return { ...part, data: fileData };
+                }
+            }
+            return part;
+        }));
+        if (message.role === "user") {
+            return { ...message, content: processedContent };
+        }
+        else {
+            return { ...message, content: processedContent };
+        }
+    }));
+}
 //# sourceMappingURL=files.js.map

package/dist/client/files.js.map

@@ -1 +1 @@
-{"version":3,"file":"files.js","sourceRoot":"","sources":["../../src/client/files.ts"],"names":[],"mappings":"AASA,MAAM,CAAC,MAAM,aAAa,GAAG,IAAI,GAAG,EAAE,CAAC;AAUvC;;;;;;;;;;GAUG;AACH,MAAM,CAAC,KAAK,UAAU,SAAS,CAC7B,GAA+B,EAC/B,SAAyB,EACzB,IAAU,EACV,QAAiB,EACjB,MAAe;IAMf,IAAI,CAAC,CAAC,WAAW,IAAI,GAAG,CAAC,IAAI,CAAC,CAAC,SAAS,IAAI,GAAG,CAAC,EAAE,CAAC;QACjD,MAAM,IAAI,KAAK,CACb,0EAA0E;YACxE,wFAAwF;YACxF,sIAAsI;YACtI,wBAAwB,CAC3B,CAAC;IACJ,CAAC;IACD,MAAM,IAAI,GACR,MAAM;QACN,KAAK,CAAC,IAAI,CACR,IAAI,UAAU,CACZ,MAAM,MAAM,CAAC,MAAM,CAAC,MAAM,CAAC,SAAS,EAAE,MAAM,IAAI,CAAC,KAAK,EAAE,CAAC,WAAW,EAAE,CAAC,CACxE,CACF;aACE,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,QAAQ,CAAC,EAAE,CAAC,CAAC,QAAQ,CAAC,CAAC,EAAE,GAAG,CAAC,CAAC;aAC3C,IAAI,CAAC,EAAE,CAAC,CAAC;IACd,MAAM,MAAM,GAAG,MAAM,GAAG,CAAC,WAAW,CAAC,SAAS,CAAC,KAAK,CAAC,eAAe,EAAE;QACpE,IAAI;QACJ,QAAQ;KACT,CAAC,CAAC;IACH,IAAI,MAAM,EAAE,CAAC;QACX,MAAM,GAAG,GAAG,CAAC,MAAM,GAAG,CAAC,OAAO,CAAC,MAAM,CAAC,MAAM,CAAC,SAAS,CAAC,CAAE,CAAC;QAC1D,OAAO;YACL,GAAG,QAAQ,CAAC,GAAG,EAAE,IAAI,CAAC,IAAI,EAAE,QAAQ,CAAC;YACrC,IAAI,EAAE;gBACJ,GAAG;gBACH,MAAM,EAAE,MAAM,CAAC,MAAM;gBACrB,SAAS,EAAE,MAAM,CAAC,SAA2B;gBAC7C,IAAI;gBACJ,QAAQ;aACT;SACF,CAAC;IACJ,CAAC;IACD,MAAM,YAAY,GAAG,MAAM,GAAG,CAAC,OAAO,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;IACnD,IAAI,MAAM,EAAE,CAAC;QACX,MAAM,QAAQ,GAAG,MAAM,GAAG,CAAC,OAAO,CAAC,WAAW,CAAC,YAAY,CAAC,CAAC;QAC7D,IAAI,QAAQ,EAAE,MAAM,KAAK,MAAM,EAAE,CAAC;YAChC,MAAM,IAAI,KAAK,CAAC,iBAAiB,GAAG,QAAQ,EAAE,MAAM,GAAG,MAAM,GAAG,MAAM,CAAC,CAAC;QAC1E,CAAC;IACH,CAAC;IACD,MAAM,EAAE,MAAM,EAAE,SAAS,EAAE,GAAG,MAAM,GAAG,CAAC,WAAW,CAAC,SAAS,CAAC,KAAK,CAAC,OAAO,EAAE;QAC3E,SAAS,EAAE,YAAY;QACvB,IAAI;QACJ,QAAQ;QACR,QAAQ,EAAE,IAAI,CAAC,IAAI;KACpB,CAAC,CAAC;IACH,MAAM,GAAG,GAAG,CAAC,MAAM,GAAG,CAAC,OAAO,CAAC,MAAM,CAAC,SAA2B,CAAC,CAAE,CAAC;IACrE,IAAI,SAAS,KAAK,YAAY,EAAE,CAAC;QAC/B,0CAA0C;QAC1C,wEAAwE;QACxE,+DAA+D;QAC/D,MAAM,GAAG,CAAC,OAAO,CAAC,MAAM,CAAC,YAAY,CAAC,CAAC;IACzC,CAAC;IACD,OAAO;QACL,GAAG,QAAQ,CAAC,GAAG,EAAE,IAAI,CAAC,IAAI,EAAE,QAAQ,CAAC;QACrC,IAAI,EAAE;YACJ,GAAG;YACH,MAAM;YACN,SAAS,EAAE,SAA2B;YACtC,IAAI;YACJ,QAAQ;SACT;KACF,CAAC;AACJ,CAAC;AAED;;;;;;;;;;;;;;;GAeG;AACH,MAAM,CAAC,KAAK,UAAU,OAAO,CAC3B,GAAyB,EACzB,SAAyB,EACzB,MAAc;IAEd,MAAM,IAAI,GAAG,MAAM,GAAG,CAAC,QAAQ,CAAC,SAAS,CAAC,KAAK,CAAC,GAAG,EAAE,EAAE,MAAM,EAAE,CAAC,CAAC;IACjE,IAAI,CAAC,IAAI,EAAE,CAAC;QACV,MAAM,IAAI,KAAK,CAAC,gCAAgC,MAAM,EAAE,CAAC,CAAC;IAC5D,CAAC;IACD,MAAM,GAAG,GAAG,MAAM,GAAG,CAAC,OAAO,CAAC,MAAM,CAAC,IAAI,CAAC,SAA2B,CAAC,CAAC;IACvE,IAAI,CAAC,GAAG,EAAE,CAAC;QACT,MAAM,IAAI,KAAK,CAAC,8BAA8B,IAAI,CAAC,SAAS,EAAE,CAAC,CAAC;IAClE,CAAC;IACD,OAAO;QACL,GAAG,QAAQ,CAAC,GAAG,EAAE,IAAI,CAAC,QAAQ,EAAE,IAAI,CAAC,QAAQ,CAAC;QAC9C,IAAI,EAAE;YACJ,MAAM;YACN,GAAG;YACH,SAAS,EAAE,IAAI,CAAC,SAA2B;YAC3C,IAAI,EAAE,IAAI,CAAC,IAAI;YACf,QAAQ,EAAE,IAAI,CAAC,QAAQ;SACxB;KACF,CAAC;AACJ,CAAC;AAED,SAAS,QAAQ,CACf,GAAW,EACX,QAAgB,EAChB,QAA4B;IAK5B,MAAM,QAAQ,GAAa;QACzB,IAAI,EAAE,MAAM;QACZ,IAAI,EAAE,IAAI,GAAG,CAAC,GAAG,CAAC;QAClB,QAAQ;QACR,QAAQ;KACT,CAAC;IACF,MAAM,SAAS,GAA0B,QAAQ,CAAC,UAAU,CAAC,QAAQ,CAAC;QACpE,CAAC,CAAC;YACE,IAAI,EAAE,OAAO;YACb,KAAK,EAAE,IAAI,GAAG,CAAC,GAAG,CAAC;YACnB,QAAQ;SACT;QACH,CAAC,CAAC,SAAS,CAAC;IACd,OAAO,EAAE,QAAQ,EAAE,SAAS,EAAE,CAAC;AACjC,CAAC"}
+{"version":3,"file":"files.js","sourceRoot":"","sources":["../../src/client/files.ts"],"names":[],"mappings":"AAeA,OAAO,EAAE,MAAM,EAAE,MAAM,gBAAgB,CAAC;AAExC,MAAM,CAAC,MAAM,aAAa,GAAG,IAAI,GAAG,EAAE,CAAC;AAUvC;;;;;;;;;;GAUG;AACH,MAAM,CAAC,KAAK,UAAU,SAAS,CAC7B,GAA+B,EAC/B,SAAyB,EACzB,IAAU,EACV,EAAE,QAAQ,EAAE,MAAM,KAA6C,EAAE;IAMjE,IAAI,CAAC,CAAC,WAAW,IAAI,GAAG,CAAC,IAAI,CAAC,CAAC,SAAS,IAAI,GAAG,CAAC,EAAE,CAAC;QACjD,MAAM,IAAI,KAAK,CACb,0EAA0E;YACxE,wFAAwF;YACxF,sIAAsI;YACtI,wBAAwB,CAC3B,CAAC;IACJ,CAAC;IACD,MAAM,IAAI,GACR,MAAM;QACN,KAAK,CAAC,IAAI,CACR,IAAI,UAAU,CACZ,MAAM,MAAM,CAAC,MAAM,CAAC,MAAM,CAAC,SAAS,EAAE,MAAM,IAAI,CAAC,KAAK,EAAE,CAAC,WAAW,EAAE,CAAC,CACxE,CACF;aACE,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,QAAQ,CAAC,EAAE,CAAC,CAAC,QAAQ,CAAC,CAAC,EAAE,GAAG,CAAC,CAAC;aAC3C,IAAI,CAAC,EAAE,CAAC,CAAC;IACd,MAAM,MAAM,GAAG,MAAM,GAAG,CAAC,WAAW,CAAC,SAAS,CAAC,KAAK,CAAC,eAAe,EAAE;QACpE,IAAI;QACJ,QAAQ;KACT,CAAC,CAAC;IACH,IAAI,MAAM,EAAE,CAAC;QACX,MAAM,GAAG,GAAG,CAAC,MAAM,GAAG,CAAC,OAAO,CAAC,MAAM,CAAC,MAAM,CAAC,SAAS,CAAC,CAAE,CAAC;QAC1D,OAAO;YACL,GAAG,QAAQ,CAAC,GAAG,EAAE,IAAI,CAAC,IAAI,EAAE,QAAQ,CAAC;YACrC,IAAI,EAAE;gBACJ,GAAG;gBACH,MAAM,EAAE,MAAM,CAAC,MAAM;gBACrB,SAAS,EAAE,MAAM,CAAC,SAA2B;gBAC7C,IAAI;gBACJ,QAAQ;aACT;SACF,CAAC;IACJ,CAAC;IACD,MAAM,YAAY,GAAG,MAAM,GAAG,CAAC,OAAO,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;IACnD,IAAI,MAAM,EAAE,CAAC;QACX,MAAM,QAAQ,GAAG,MAAM,GAAG,CAAC,OAAO,CAAC,WAAW,CAAC,YAAY,CAAC,CAAC;QAC7D,IAAI,QAAQ,EAAE,MAAM,KAAK,MAAM,EAAE,CAAC;YAChC,MAAM,IAAI,KAAK,CAAC,iBAAiB,GAAG,QAAQ,EAAE,MAAM,GAAG,MAAM,GAAG,MAAM,CAAC,CAAC;QAC1E,CAAC;IACH,CAAC;IACD,MAAM,EAAE,MAAM,EAAE,SAAS,EAAE,GAAG,MAAM,GAAG,CAAC,WAAW,CAAC,SAAS,CAAC,KAAK,CAAC,OAAO,EAAE;QAC3E,SAAS,EAAE,YAAY;QACvB,IAAI;QACJ,QAAQ;QACR,QAAQ,EAAE,IAAI,CAAC,IAAI;KACpB,CAAC,CAAC;IACH,MAAM,GAAG,GAAG,CAAC,MAAM,GAAG,CAAC,OAAO,CAAC,MAAM,CAAC,SAA2B,CAAC,CAAE,CAAC;IACrE,IAAI,SAAS,KAAK,YAAY,EAAE,CAAC;QAC/B,0CAA0C;QAC1C,wEAAwE;QACxE,+DAA+D;QAC/D,MAAM,GAAG,CAAC,OAAO,CAAC,MAAM,CAAC,YAAY,CAAC,CAAC;IACzC,CAAC;IACD,OAAO;QACL,GAAG,QAAQ,CAAC,GAAG,EAAE,IAAI,CAAC,IAAI,EAAE,QAAQ,CAAC;QACrC,IAAI,EAAE;YACJ,GAAG;YACH,MAAM;YACN,SAAS,EAAE,SAA2B;YACtC,IAAI;YACJ,QAAQ;SACT;KACF,CAAC;AACJ,CAAC;AAED;;;;;;;;;;;;;;;GAeG;AACH,MAAM,CAAC,KAAK,UAAU,OAAO,CAC3B,GAAyB,EACzB,SAAyB,EACzB,MAAc;IAEd,MAAM,IAAI,GAAG,MAAM,GAAG,CAAC,QAAQ,CAAC,SAAS,CAAC,KAAK,CAAC,GAAG,EAAE,EAAE,MAAM,EAAE,CAAC,CAAC;IACjE,IAAI,CAAC,IAAI,EAAE,CAAC;QACV,MAAM,IAAI,KAAK,CAAC,gCAAgC,MAAM,EAAE,CAAC,CAAC;IAC5D,CAAC;IACD,MAAM,GAAG,GAAG,MAAM,GAAG,CAAC,OAAO,CAAC,MAAM,CAAC,IAAI,CAAC,SAA2B,CAAC,CAAC;IACvE,IAAI,CAAC,GAAG,EAAE,CAAC;QACT,MAAM,IAAI,KAAK,CAAC,8BAA8B,IAAI,CAAC,SAAS,EAAE,CAAC,CAAC;IAClE,CAAC;IACD,OAAO;QACL,GAAG,QAAQ,CAAC,GAAG,EAAE,IAAI,CAAC,QAAQ,EAAE,IAAI,CAAC,QAAQ,CAAC;QAC9C,IAAI,EAAE;YACJ,MAAM;YACN,GAAG;YACH,SAAS,EAAE,IAAI,CAAC,SAA2B;YAC3C,IAAI,EAAE,IAAI,CAAC,IAAI;YACf,QAAQ,EAAE,IAAI,CAAC,QAAQ;SACxB;KACF,CAAC;AACJ,CAAC;AAED,SAAS,QAAQ,CACf,GAAW,EACX,SAAiB,EACjB,QAA4B;IAE5B,MAAM,QAAQ,GAAa;QACzB,IAAI,EAAE,MAAM;QACZ,IAAI,EAAE,IAAI,GAAG,CAAC,GAAG,CAAC;QAClB,SAAS;QACT,QAAQ;KACT,CAAC;IACF,MAAM,SAAS,GAA0B,SAAS,CAAC,UAAU,CAAC,QAAQ,CAAC;QACrE,CAAC,CAAC,EAAE,IAAI,EAAE,OAAO,EAAE,KAAK,EAAE,IAAI,GAAG,CAAC,GAAG,CAAC,EAAE,SAAS,EAAE;QACnD,CAAC,CAAC,SAAS,CAAC;IACd,OAAO,EAAE,QAAQ,EAAE,SAAS,EAAE,CAAC;AACjC,CAAC;AAED;;GAEG;AACH,SAAS,cAAc,CAAC,GAAQ;IAC9B,OAAO,CACL,GAAG,CAAC,QAAQ,KAAK,WAAW;QAC5B,GAAG,CAAC,QAAQ,KAAK,WAAW;QAC5B,GAAG,CAAC,QAAQ,KAAK,KAAK;QACtB,GAAG,CAAC,QAAQ,KAAK,SAAS,CAC3B,CAAC;AACJ,CAAC;AAED;;GAEG;AACH,KAAK,UAAU,YAAY,CAAC,GAAQ;IAClC,iBAAiB;IACjB,MAAM,QAAQ,GAAG,MAAM,KAAK,CAAC,GAAG,CAAC,CAAC;IAClC,IAAI,CAAC,QAAQ,CAAC,EAAE,EAAE,CAAC;QACjB,MAAM,IAAI,KAAK,CAAC,mBAAmB,GAAG,KAAK,QAAQ,CAAC,UAAU,EAAE,CAAC,CAAC;IACpE,CAAC;IAED,OAAO,MAAM,QAAQ,CAAC,WAAW,EAAE,CAAC;AACtC,CAAC;AAED;;;;GAIG;AACH,MAAM,CAAC,KAAK,UAAU,mBAAmB,CACvC,QAAoC;IAEpC,2DAA2D;IAC3D,OAAO,OAAO,CAAC,GAAG,CAChB,QAAQ,CAAC,GAAG,CAAC,KAAK,EAAE,OAAO,EAAmC,EAAE;QAC9D,IACE,CAAC,OAAO,CAAC,IAAI,KAAK,MAAM,IAAI,OAAO,CAAC,IAAI,KAAK,WAAW,CAAC;YACzD,OAAO,OAAO,CAAC,OAAO,KAAK,QAAQ;YACnC,CAAC,KAAK,CAAC,OAAO,CAAC,OAAO,CAAC,OAAO,CAAC,EAC/B,CAAC;YACD,OAAO,OAAO,CAAC;QACjB,CAAC;QAED,MAAM,gBAAgB,GAAG,MAAM,OAAO,CAAC,GAAG,CACxC,OAAO,CAAC,OAAO,CAAC,GAAG,CAAC,KAAK,EAAE,IAAI,EAAE,EAAE;YACjC,IAAI,IAAI,CAAC,IAAI,KAAK,OAAO,IAAI,IAAI,CAAC,KAAK,YAAY,GAAG,EAAE,CAAC;gBACvD,MAAM,CACJ,OAAO,CAAC,IAAI,KAAK,MAAM,EACvB,qCAAqC,CACtC,CAAC;gBACF,IAAI,cAAc,CAAC,IAAI,CAAC,KAAK,CAAC,EAAE,CAAC;oBAC/B,MAAM,SAAS,GAAG,MAAM,YAAY,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;oBACjD,OAAO,EAAE,GAAG,IAAI,EAAE,KAAK,EAAE,SAAS,EAAe,CAAC;gBACpD,CAAC;YACH,CAAC;YAED,oBAAoB;YACpB,IAAI,IAAI,CAAC,IAAI,KAAK,MAAM,IAAI,IAAI,CAAC,IAAI,YAAY,GAAG,EAAE,CAAC;gBACrD,IAAI,cAAc,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC;oBAC9B,MAAM,QAAQ,GAAG,MAAM,YAAY,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;oBAC/C,OAAO,EAAE,GAAG,IAAI,EAAE,IAAI,EAAE,QAAQ,EAAc,CAAC;gBACjD,CAAC;YACH,CAAC;YAED,OAAO,IAAI,CAAC;QACd,CAAC,CAAC,CACH,CAAC;QACF,IAAI,OAAO,CAAC,IAAI,KAAK,MAAM,EAAE,CAAC;YAC5B,OAAO,EAAE,GAAG,OAAO,EAAE,OAAO,EAAE,gBAA+B,EAAE,CAAC;QAClE,CAAC;aAAM,CAAC;YACN,OAAO,EAAE,GAAG,OAAO,EAAE,OAAO,EAAE,gBAAoC,EAAE,CAAC;QACvE,CAAC;IACH,CAAC,CAAC,CACH,CAAC;AACJ,CAAC"}