@convex-dev/agent 0.2.0-alpha.2 → 0.2.0

package/dist/client/createTool.d.ts

@@ -1,5 +1,5 @@
-import type { Schema, Tool, ToolCallOptions, ToolSet } from "ai";
-import { z } from "zod/v3";
+import type { FlexibleSchema } from "@ai-sdk/provider-utils";
+import type { Tool, ToolCallOptions, ToolSet } from "ai";
 import type { Agent } from "./index.js";
 import type { GenericActionCtx, GenericDataModel } from "convex/server";
 import type { ProviderOptions } from "../validators.js";
@@ -28,7 +28,7 @@ export declare function createTool<INPUT, OUTPUT, Ctx extends ToolCtx = ToolCtx>
     It is also used to validate the output of the language model.
     Use descriptions to make the input understandable for the language model.
        */
-    args: ToolParameters<INPUT>;
+    args: FlexibleSchema<INPUT>;
     /**
     An async function that is called with the arguments from the tool call and produces a result.
     If not provided, the tool will not be executed automatically.
@@ -63,6 +63,4 @@ export declare function createTool<INPUT, OUTPUT, Ctx extends ToolCtx = ToolCtx>
     providerOptions?: ProviderOptions;
 }): Tool<INPUT, OUTPUT>;
 export declare function wrapTools(ctx: ToolCtx, ...toolSets: (ToolSet | undefined)[]): ToolSet;
-type ToolParameters<T> = z.Schema<T> | Schema<T>;
-export {};
 //# sourceMappingURL=createTool.d.ts.map

package/dist/client/createTool.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"createTool.d.ts","sourceRoot":"","sources":["../../src/client/createTool.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,MAAM,EAAE,IAAI,EAAE,eAAe,EAAE,OAAO,EAAE,MAAM,IAAI,CAAC;AAEjE,OAAO,EAAE,CAAC,EAAE,MAAM,QAAQ,CAAC;AAC3B,OAAO,KAAK,EAAE,KAAK,EAAE,MAAM,YAAY,CAAC;AACxC,OAAO,KAAK,EAAE,gBAAgB,EAAE,gBAAgB,EAAE,MAAM,eAAe,CAAC;AACxE,OAAO,KAAK,EAAE,eAAe,EAAE,MAAM,kBAAkB,CAAC;AAExD,MAAM,MAAM,OAAO,CAAC,SAAS,SAAS,gBAAgB,GAAG,gBAAgB,IACvE,gBAAgB,CAAC,SAAS,CAAC,GAAG;IAC5B,KAAK,EAAE,KAAK,CAAC;IACb,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,SAAS,CAAC,EAAE,MAAM,CAAC;CACpB,CAAC;AAEJ;;;;;;GAMG;AACH,wBAAgB,UAAU,CAAC,KAAK,EAAE,MAAM,EAAE,GAAG,SAAS,OAAO,GAAG,OAAO,EAAE,GAAG,EAAE;IAC5E;;;;SAIK;IACL,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB;;;;SAIK;IACL,IAAI,EAAE,cAAc,CAAC,KAAK,CAAC,CAAC;IAC5B;;;;;;SAMK;IACL,OAAO,EAAE,CACP,GAAG,EAAE,GAAG,EACR,IAAI,EAAE,KAAK,EACX,OAAO,EAAE,eAAe,KACrB,WAAW,CAAC,MAAM,CAAC,CAAC;IACzB;;OAEG;IACH,GAAG,CAAC,EAAE,GAAG,CAAC;IACV;;;OAGG;IACH,YAAY,CAAC,EAAE,CACb,GAAG,EAAE,GAAG,EACR,OAAO,EAAE,eAAe,KACrB,IAAI,GAAG,WAAW,CAAC,IAAI,CAAC,CAAC;IAC9B;;;OAGG;IACH,YAAY,CAAC,EAAE,CACb,GAAG,EAAE,GAAG,EACR,OAAO,EAAE;QACP,cAAc,EAAE,MAAM,CAAC;KACxB,GAAG,eAAe,KAChB,IAAI,GAAG,WAAW,CAAC,IAAI,CAAC,CAAC;IAC9B;;;OAGG;IACH,gBAAgB,CAAC,EAAE,CACjB,GAAG,EAAE,GAAG,EACR,OAAO,EAAE;QACP,KAAK,EAAE,CAAC,KAAK,CAAC,SAAS,CAAC,KAAK,CAAC,GAAG,SAAS,GAAG,KAAK,CAAC;KACpD,GAAG,eAAe,KAChB,IAAI,GAAG,WAAW,CAAC,IAAI,CAAC,CAAC;IAG9B,eAAe,CAAC,EAAE,eAAe,CAAC;CACnC,GAAG,IAAI,CAAC,KAAK,EAAE,MAAM,CAAC,CA6BtB;AAOD,wBAAgB,SAAS,CACvB,GAAG,EAAE,OAAO,EACZ,GAAG,QAAQ,EAAE,CAAC,OAAO,GAAG,SAAS,CAAC,EAAE,GACnC,OAAO,CAiBT;AAGD,KAAK,cAAc,CAAC,CAAC,IAAI,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,GAAG,MAAM,CAAC,CAAC,CAAC,CAAC"}
+{"version":3,"file":"createTool.d.ts","sourceRoot":"","sources":["../../src/client/createTool.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,cAAc,EAAE,MAAM,wBAAwB,CAAC;AAC7D,OAAO,KAAK,EAAE,IAAI,EAAE,eAAe,EAAE,OAAO,EAAE,MAAM,IAAI,CAAC;AAEzD,OAAO,KAAK,EAAE,KAAK,EAAE,MAAM,YAAY,CAAC;AACxC,OAAO,KAAK,EAAE,gBAAgB,EAAE,gBAAgB,EAAE,MAAM,eAAe,CAAC;AACxE,OAAO,KAAK,EAAE,eAAe,EAAE,MAAM,kBAAkB,CAAC;AAExD,MAAM,MAAM,OAAO,CAAC,SAAS,SAAS,gBAAgB,GAAG,gBAAgB,IACvE,gBAAgB,CAAC,SAAS,CAAC,GAAG;IAC5B,KAAK,EAAE,KAAK,CAAC;IACb,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,SAAS,CAAC,EAAE,MAAM,CAAC;CACpB,CAAC;AAEJ;;;;;;GAMG;AACH,wBAAgB,UAAU,CAAC,KAAK,EAAE,MAAM,EAAE,GAAG,SAAS,OAAO,GAAG,OAAO,EAAE,GAAG,EAAE;IAC5E;;;;SAIK;IACL,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB;;;;SAIK;IACL,IAAI,EAAE,cAAc,CAAC,KAAK,CAAC,CAAC;IAC5B;;;;;;SAMK;IACL,OAAO,EAAE,CACP,GAAG,EAAE,GAAG,EACR,IAAI,EAAE,KAAK,EACX,OAAO,EAAE,eAAe,KACrB,WAAW,CAAC,MAAM,CAAC,CAAC;IACzB;;OAEG;IACH,GAAG,CAAC,EAAE,GAAG,CAAC;IACV;;;OAGG;IACH,YAAY,CAAC,EAAE,CACb,GAAG,EAAE,GAAG,EACR,OAAO,EAAE,eAAe,KACrB,IAAI,GAAG,WAAW,CAAC,IAAI,CAAC,CAAC;IAC9B;;;OAGG;IACH,YAAY,CAAC,EAAE,CACb,GAAG,EAAE,GAAG,EACR,OAAO,EAAE;QAAE,cAAc,EAAE,MAAM,CAAA;KAAE,GAAG,eAAe,KAClD,IAAI,GAAG,WAAW,CAAC,IAAI,CAAC,CAAC;IAC9B;;;OAGG;IACH,gBAAgB,CAAC,EAAE,CACjB,GAAG,EAAE,GAAG,EACR,OAAO,EAAE;QACP,KAAK,EAAE,CAAC,KAAK,CAAC,SAAS,CAAC,KAAK,CAAC,GAAG,SAAS,GAAG,KAAK,CAAC;KACpD,GAAG,eAAe,KAChB,IAAI,GAAG,WAAW,CAAC,IAAI,CAAC,CAAC;IAG9B,eAAe,CAAC,EAAE,eAAe,CAAC;CACnC,GAAG,IAAI,CAAC,KAAK,EAAE,MAAM,CAAC,CA6BtB;AAOD,wBAAgB,SAAS,CACvB,GAAG,EAAE,OAAO,EACZ,GAAG,QAAQ,EAAE,CAAC,OAAO,GAAG,SAAS,CAAC,EAAE,GACnC,OAAO,CAiBT"}

package/dist/client/createTool.js

@@ -1,5 +1,4 @@
 import { tool } from "ai";
-import { z } from "zod/v3";
 /**
  * This is a wrapper around the ai.tool function that adds extra context to the
  * tool call, including the action context, userId, threadId, and messageId.

package/dist/client/createTool.js.map

@@ -1 +1 @@
-{"version":3,"file":"createTool.js","sourceRoot":"","sources":["../../src/client/createTool.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,IAAI,EAAE,MAAM,IAAI,CAAC;AAC1B,OAAO,EAAE,CAAC,EAAE,MAAM,QAAQ,CAAC;AAa3B;;;;;;GAMG;AACH,MAAM,UAAU,UAAU,CAA+C,GA4DxE;IACC,MAAM,CAAC,GAAG,IAAI,CAAC;QACb,IAAI,EAAE,UAAU;QAChB,YAAY,EAAE,IAAI;QAClB,GAAG,EAAE,GAAG,CAAC,GAAG;QACZ,WAAW,EAAE,GAAG,CAAC,WAAW;QAC5B,WAAW,EAAE,GAAG,CAAC,IAAI;QACrB,KAAK,CAAC,OAAO,CAAC,IAAW,EAAE,OAAwB;YACjD,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,EAAE,CAAC;gBAClB,MAAM,IAAI,KAAK,CACb,uDAAuD;oBACrD,qEAAqE;oBACrE,uDAAuD,CAC1D,CAAC;YACJ,CAAC;YACD,OAAO,GAAG,CAAC,OAAO,CAAC,MAAM,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,OAAO,CAAC,CAAC;QAClD,CAAC;QACD,eAAe,EAAE,GAAG,CAAC,eAAe;KACrC,CAAC,CAAC;IACH,IAAI,GAAG,CAAC,YAAY,EAAE,CAAC;QACrB,CAAC,CAAC,YAAY,GAAG,GAAG,CAAC,YAAY,CAAC,IAAI,CAAC,CAAC,EAAE,MAAM,CAAC,CAAC,CAAC,CAAC,CAAC;IACvD,CAAC;IACD,IAAI,GAAG,CAAC,YAAY,EAAE,CAAC;QACrB,CAAC,CAAC,YAAY,GAAG,GAAG,CAAC,YAAY,CAAC,IAAI,CAAC,CAAC,EAAE,MAAM,CAAC,CAAC,CAAC,CAAC,CAAC;IACvD,CAAC;IACD,IAAI,GAAG,CAAC,gBAAgB,EAAE,CAAC;QACzB,CAAC,CAAC,gBAAgB,GAAG,GAAG,CAAC,gBAAgB,CAAC,IAAI,CAAC,CAAC,EAAE,MAAM,CAAC,CAAC,CAAC,CAAC,CAAC;IAC/D,CAAC;IACD,OAAO,CAAC,CAAC;AACX,CAAC;AAED,8DAA8D;AAC9D,SAAS,MAAM,CAAsB,IAAS;IAC5C,OAAQ,IAAqB,CAAC,GAAG,CAAC;AACpC,CAAC;AAED,MAAM,UAAU,SAAS,CACvB,GAAY,EACZ,GAAG,QAAiC;IAEpC,MAAM,MAAM,GAAG,EAAa,CAAC;IAC7B,KAAK,MAAM,OAAO,IAAI,QAAQ,EAAE,CAAC;QAC/B,IAAI,CAAC,OAAO,EAAE,CAAC;YACb,SAAS;QACX,CAAC;QACD,KAAK,MAAM,CAAC,IAAI,EAAE,IAAI,CAAC,IAAI,MAAM,CAAC,OAAO,CAAC,OAAO,CAAC,EAAE,CAAC;YACnD,8DAA8D;YAC9D,IAAI,CAAE,IAAY,CAAC,YAAY,EAAE,CAAC;gBAChC,MAAM,CAAC,IAAI,CAAC,GAAG,IAAI,CAAC;YACtB,CAAC;iBAAM,CAAC;gBACN,MAAM,GAAG,GAAG,EAAE,GAAG,IAAI,EAAE,GAAG,EAAE,CAAC;gBAC7B,MAAM,CAAC,IAAI,CAAC,GAAG,GAAG,CAAC;YACrB,CAAC;QACH,CAAC;IACH,CAAC;IACD,OAAO,MAAM,CAAC;AAChB,CAAC"}
+{"version":3,"file":"createTool.js","sourceRoot":"","sources":["../../src/client/createTool.ts"],"names":[],"mappings":"AAEA,OAAO,EAAE,IAAI,EAAE,MAAM,IAAI,CAAC;AAa1B;;;;;;GAMG;AACH,MAAM,UAAU,UAAU,CAA+C,GA0DxE;IACC,MAAM,CAAC,GAAG,IAAI,CAAC;QACb,IAAI,EAAE,UAAU;QAChB,YAAY,EAAE,IAAI;QAClB,GAAG,EAAE,GAAG,CAAC,GAAG;QACZ,WAAW,EAAE,GAAG,CAAC,WAAW;QAC5B,WAAW,EAAE,GAAG,CAAC,IAAI;QACrB,KAAK,CAAC,OAAO,CAAC,IAAW,EAAE,OAAwB;YACjD,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,EAAE,CAAC;gBAClB,MAAM,IAAI,KAAK,CACb,uDAAuD;oBACrD,qEAAqE;oBACrE,uDAAuD,CAC1D,CAAC;YACJ,CAAC;YACD,OAAO,GAAG,CAAC,OAAO,CAAC,MAAM,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,OAAO,CAAC,CAAC;QAClD,CAAC;QACD,eAAe,EAAE,GAAG,CAAC,eAAe;KACrC,CAAC,CAAC;IACH,IAAI,GAAG,CAAC,YAAY,EAAE,CAAC;QACrB,CAAC,CAAC,YAAY,GAAG,GAAG,CAAC,YAAY,CAAC,IAAI,CAAC,CAAC,EAAE,MAAM,CAAC,CAAC,CAAC,CAAC,CAAC;IACvD,CAAC;IACD,IAAI,GAAG,CAAC,YAAY,EAAE,CAAC;QACrB,CAAC,CAAC,YAAY,GAAG,GAAG,CAAC,YAAY,CAAC,IAAI,CAAC,CAAC,EAAE,MAAM,CAAC,CAAC,CAAC,CAAC,CAAC;IACvD,CAAC;IACD,IAAI,GAAG,CAAC,gBAAgB,EAAE,CAAC;QACzB,CAAC,CAAC,gBAAgB,GAAG,GAAG,CAAC,gBAAgB,CAAC,IAAI,CAAC,CAAC,EAAE,MAAM,CAAC,CAAC,CAAC,CAAC,CAAC;IAC/D,CAAC;IACD,OAAO,CAAC,CAAC;AACX,CAAC;AAED,8DAA8D;AAC9D,SAAS,MAAM,CAAsB,IAAS;IAC5C,OAAQ,IAAqB,CAAC,GAAG,CAAC;AACpC,CAAC;AAED,MAAM,UAAU,SAAS,CACvB,GAAY,EACZ,GAAG,QAAiC;IAEpC,MAAM,MAAM,GAAG,EAAa,CAAC;IAC7B,KAAK,MAAM,OAAO,IAAI,QAAQ,EAAE,CAAC;QAC/B,IAAI,CAAC,OAAO,EAAE,CAAC;YACb,SAAS;QACX,CAAC;QACD,KAAK,MAAM,CAAC,IAAI,EAAE,IAAI,CAAC,IAAI,MAAM,CAAC,OAAO,CAAC,OAAO,CAAC,EAAE,CAAC;YACnD,8DAA8D;YAC9D,IAAI,CAAE,IAAY,CAAC,YAAY,EAAE,CAAC;gBAChC,MAAM,CAAC,IAAI,CAAC,GAAG,IAAI,CAAC;YACtB,CAAC;iBAAM,CAAC;gBACN,MAAM,GAAG,GAAG,EAAE,GAAG,IAAI,EAAE,GAAG,EAAE,CAAC;gBAC7B,MAAM,CAAC,IAAI,CAAC,GAAG,GAAG,CAAC;YACrB,CAAC;QACH,CAAC;IACH,CAAC;IACD,OAAO,MAAM,CAAC;AAChB,CAAC"}

package/dist/client/definePlaygroundAPI.d.ts

@@ -173,9 +173,9 @@ export declare function definePlaygroundAPI<DataModel extends GenericDataModel>(
             title?: string | undefined;
             type?: "source" | undefined;
             providerOptions?: Record<string, Record<string, any>> | undefined;
-            url?: string | undefined;
             id: string;
             sourceType: "url";
+            url: string;
         } | {
             filename?: string | undefined;
             providerMetadata?: Record<string, Record<string, any>> | undefined;
@@ -185,6 +185,19 @@ export declare function definePlaygroundAPI<DataModel extends GenericDataModel>(
             sourceType: "document";
             mediaType: string;
         })[] | undefined;
+        warnings?: ({
+            details?: string | undefined;
+            type: "unsupported-setting";
+            setting: string;
+        } | {
+            details?: string | undefined;
+            type: "unsupported-tool";
+            tool: any;
+        } | {
+            type: "other";
+            message: string;
+        })[] | undefined;
+        finishReason?: "length" | "error" | "other" | "stop" | "content-filter" | "tool-calls" | "unknown" | undefined;
         reasoningDetails?: ({
             providerOptions?: Record<string, Record<string, any>> | undefined;
             signature?: string | undefined;
@@ -199,19 +212,6 @@ export declare function definePlaygroundAPI<DataModel extends GenericDataModel>(
             type: "redacted";
             data: string;
         })[] | undefined;
-        warnings?: ({
-            details?: string | undefined;
-            type: "unsupported-setting";
-            setting: string;
-        } | {
-            details?: string | undefined;
-            type: "unsupported-tool";
-            tool: any;
-        } | {
-            type: "other";
-            message: string;
-        })[] | undefined;
-        finishReason?: "length" | "error" | "other" | "stop" | "content-filter" | "tool-calls" | "unknown" | undefined;
         _id: string;
         _creationTime: number;
         status: "pending" | "success" | "failed";
@@ -543,9 +543,9 @@ export declare function definePlaygroundAPI<DataModel extends GenericDataModel>(
             title?: string | undefined;
             type?: "source" | undefined;
             providerOptions?: Record<string, Record<string, any>> | undefined;
-            url?: string | undefined;
             id: string;
             sourceType: "url";
+            url: string;
         } | {
             filename?: string | undefined;
             providerMetadata?: Record<string, Record<string, any>> | undefined;
@@ -555,6 +555,19 @@ export declare function definePlaygroundAPI<DataModel extends GenericDataModel>(
             sourceType: "document";
             mediaType: string;
         })[] | undefined;
+        warnings?: ({
+            details?: string | undefined;
+            type: "unsupported-setting";
+            setting: string;
+        } | {
+            details?: string | undefined;
+            type: "unsupported-tool";
+            tool: any;
+        } | {
+            type: "other";
+            message: string;
+        })[] | undefined;
+        finishReason?: "length" | "error" | "other" | "stop" | "content-filter" | "tool-calls" | "unknown" | undefined;
         reasoningDetails?: ({
             providerOptions?: Record<string, Record<string, any>> | undefined;
             signature?: string | undefined;
@@ -569,19 +582,6 @@ export declare function definePlaygroundAPI<DataModel extends GenericDataModel>(
             type: "redacted";
             data: string;
         })[] | undefined;
-        warnings?: ({
-            details?: string | undefined;
-            type: "unsupported-setting";
-            setting: string;
-        } | {
-            details?: string | undefined;
-            type: "unsupported-tool";
-            tool: any;
-        } | {
-            type: "other";
-            message: string;
-        })[] | undefined;
-        finishReason?: "length" | "error" | "other" | "stop" | "content-filter" | "tool-calls" | "unknown" | undefined;
         _id: string;
         _creationTime: number;
         status: "pending" | "success" | "failed";

package/dist/client/files.d.ts

@@ -1,6 +1,7 @@
-import type { FilePart, ImagePart } from "ai";
+import type { FilePart, ImagePart, ModelMessage } from "ai";
 import type { Id } from "../component/_generated/dataModel.js";
 import type { ActionCtx, AgentComponent, QueryCtx, RunMutationCtx } from "./types.js";
+import type { Message } from "../validators.js";
 export declare const MAX_FILE_SIZE: number;
 type File = {
     url: string;
@@ -55,5 +56,11 @@ export declare function getFile(ctx: ActionCtx | QueryCtx, component: AgentCompo
     filePart: FilePart;
     imagePart: ImagePart | undefined;
 }>;
+/**
+ * Process messages to inline file and image URLs that point to localhost
+ * by converting them to base64. This solves the problem of LLMs not being
+ * able to access localhost URLs.
+ */
+export declare function inlineMessagesFiles(messages: (ModelMessage | Message)[]): Promise<(ModelMessage | Message)[]>;
 export {};
 //# sourceMappingURL=files.d.ts.map

package/dist/client/files.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"files.d.ts","sourceRoot":"","sources":["../../src/client/files.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,QAAQ,EAAE,SAAS,EAAE,MAAM,IAAI,CAAC;AAC9C,OAAO,KAAK,EAAE,EAAE,EAAE,MAAM,sCAAsC,CAAC;AAC/D,OAAO,KAAK,EACV,SAAS,EACT,cAAc,EACd,QAAQ,EACR,cAAc,EACf,MAAM,YAAY,CAAC;AAEpB,eAAO,MAAM,aAAa,QAAY,CAAC;AAEvC,KAAK,IAAI,GAAG;IACV,GAAG,EAAE,MAAM,CAAC;IACZ,MAAM,EAAE,MAAM,CAAC;IACf,SAAS,EAAE,EAAE,CAAC,UAAU,CAAC,CAAC;IAC1B,IAAI,EAAE,MAAM,CAAC;IACb,QAAQ,EAAE,MAAM,GAAG,SAAS,CAAC;CAC9B,CAAC;AAEF;;;;;;;;;;GAUG;AACH,wBAAsB,SAAS,CAC7B,GAAG,EAAE,SAAS,GAAG,cAAc,EAC/B,SAAS,EAAE,cAAc,EACzB,IAAI,EAAE,IAAI,EACV,EAAE,QAAQ,EAAE,MAAM,EAAE,GAAE;IAAE,QAAQ,CAAC,EAAE,MAAM,CAAC;IAAC,MAAM,CAAC,EAAE,MAAM,CAAA;CAAO,GAChE,OAAO,CAAC;IACT,IAAI,EAAE,IAAI,CAAC;IACX,QAAQ,EAAE,QAAQ,CAAC;IACnB,SAAS,EAAE,SAAS,GAAG,SAAS,CAAC;CAClC,CAAC,CAiED;AAED;;;;;;;;;;;;;;;GAeG;AACH,wBAAsB,OAAO,CAC3B,GAAG,EAAE,SAAS,GAAG,QAAQ,EACzB,SAAS,EAAE,cAAc,EACzB,MAAM,EAAE,MAAM;;;;mBAemB,EAAE,CAAC,UAAU,CAAC;;;;cAYrC,QAAQ;eACP,SAAS,GAAG,SAAS;GARjC"}
+{"version":3,"file":"files.d.ts","sourceRoot":"","sources":["../../src/client/files.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAEV,QAAQ,EACR,SAAS,EACT,YAAY,EAEb,MAAM,IAAI,CAAC;AACZ,OAAO,KAAK,EAAE,EAAE,EAAE,MAAM,sCAAsC,CAAC;AAC/D,OAAO,KAAK,EACV,SAAS,EACT,cAAc,EACd,QAAQ,EACR,cAAc,EACf,MAAM,YAAY,CAAC;AACpB,OAAO,KAAK,EAAE,OAAO,EAAE,MAAM,kBAAkB,CAAC;AAGhD,eAAO,MAAM,aAAa,QAAY,CAAC;AAEvC,KAAK,IAAI,GAAG;IACV,GAAG,EAAE,MAAM,CAAC;IACZ,MAAM,EAAE,MAAM,CAAC;IACf,SAAS,EAAE,EAAE,CAAC,UAAU,CAAC,CAAC;IAC1B,IAAI,EAAE,MAAM,CAAC;IACb,QAAQ,EAAE,MAAM,GAAG,SAAS,CAAC;CAC9B,CAAC;AAEF;;;;;;;;;;GAUG;AACH,wBAAsB,SAAS,CAC7B,GAAG,EAAE,SAAS,GAAG,cAAc,EAC/B,SAAS,EAAE,cAAc,EACzB,IAAI,EAAE,IAAI,EACV,EAAE,QAAQ,EAAE,MAAM,EAAE,GAAE;IAAE,QAAQ,CAAC,EAAE,MAAM,CAAC;IAAC,MAAM,CAAC,EAAE,MAAM,CAAA;CAAO,GAChE,OAAO,CAAC;IACT,IAAI,EAAE,IAAI,CAAC;IACX,QAAQ,EAAE,QAAQ,CAAC;IACnB,SAAS,EAAE,SAAS,GAAG,SAAS,CAAC;CAClC,CAAC,CAiED;AAED;;;;;;;;;;;;;;;GAeG;AACH,wBAAsB,OAAO,CAC3B,GAAG,EAAE,SAAS,GAAG,QAAQ,EACzB,SAAS,EAAE,cAAc,EACzB,MAAM,EAAE,MAAM;;;;mBAemB,EAAE,CAAC,UAAU,CAAC;;;;cAWlC,QAAQ;eAAa,SAAS,GAAG,SAAS;GANxD;AA4CD;;;;GAIG;AACH,wBAAsB,mBAAmB,CACvC,QAAQ,EAAE,CAAC,YAAY,GAAG,OAAO,CAAC,EAAE,GACnC,OAAO,CAAC,CAAC,YAAY,GAAG,OAAO,CAAC,EAAE,CAAC,CA2CrC"}

package/dist/client/files.js

@@ -1,3 +1,4 @@
+import { assert } from "convex-helpers";
 export const MAX_FILE_SIZE = 1024 * 64;
 /**
  * Store a file in the file storage and return the URL and fileId.
@@ -113,12 +114,66 @@ function getParts(url, mediaType, filename) {
         filename,
     };
     const imagePart = mediaType.startsWith("image/")
-        ? {
-            type: "image",
-            image: new URL(url),
-            mediaType,
-        }
+        ? { type: "image", image: new URL(url), mediaType }
         : undefined;
     return { filePart, imagePart };
 }
+/**
+ * Check if a URL points to localhost
+ */
+function isLocalhostUrl(url) {
+    return (url.hostname === "localhost" ||
+        url.hostname === "127.0.0.1" ||
+        url.hostname === "::1" ||
+        url.hostname === "0.0.0.0");
+}
+/**
+ * Download a file from a URL
+ */
+async function downloadFile(url) {
+    // Fetch the file
+    const response = await fetch(url);
+    if (!response.ok) {
+        throw new Error(`Failed to fetch ${url}: ${response.statusText}`);
+    }
+    return await response.arrayBuffer();
+}
+/**
+ * Process messages to inline file and image URLs that point to localhost
+ * by converting them to base64. This solves the problem of LLMs not being
+ * able to access localhost URLs.
+ */
+export async function inlineMessagesFiles(messages) {
+    // Process each message to convert localhost URLs to base64
+    return Promise.all(messages.map(async (message) => {
+        if ((message.role !== "user" && message.role !== "assistant") ||
+            typeof message.content === "string" ||
+            !Array.isArray(message.content)) {
+            return message;
+        }
+        const processedContent = await Promise.all(message.content.map(async (part) => {
+            if (part.type === "image" && part.image instanceof URL) {
+                assert(message.role === "user", "Images can only be in user messages");
+                if (isLocalhostUrl(part.image)) {
+                    const imageData = await downloadFile(part.image);
+                    return { ...part, image: imageData };
+                }
+            }
+            // Handle file parts
+            if (part.type === "file" && part.data instanceof URL) {
+                if (isLocalhostUrl(part.data)) {
+                    const fileData = await downloadFile(part.data);
+                    return { ...part, data: fileData };
+                }
+            }
+            return part;
+        }));
+        if (message.role === "user") {
+            return { ...message, content: processedContent };
+        }
+        else {
+            return { ...message, content: processedContent };
+        }
+    }));
+}
 //# sourceMappingURL=files.js.map

package/dist/client/files.js.map

@@ -1 +1 @@
-{"version":3,"file":"files.js","sourceRoot":"","sources":["../../src/client/files.ts"],"names":[],"mappings":"AASA,MAAM,CAAC,MAAM,aAAa,GAAG,IAAI,GAAG,EAAE,CAAC;AAUvC;;;;;;;;;;GAUG;AACH,MAAM,CAAC,KAAK,UAAU,SAAS,CAC7B,GAA+B,EAC/B,SAAyB,EACzB,IAAU,EACV,EAAE,QAAQ,EAAE,MAAM,KAA6C,EAAE;IAMjE,IAAI,CAAC,CAAC,WAAW,IAAI,GAAG,CAAC,IAAI,CAAC,CAAC,SAAS,IAAI,GAAG,CAAC,EAAE,CAAC;QACjD,MAAM,IAAI,KAAK,CACb,0EAA0E;YACxE,wFAAwF;YACxF,sIAAsI;YACtI,wBAAwB,CAC3B,CAAC;IACJ,CAAC;IACD,MAAM,IAAI,GACR,MAAM;QACN,KAAK,CAAC,IAAI,CACR,IAAI,UAAU,CACZ,MAAM,MAAM,CAAC,MAAM,CAAC,MAAM,CAAC,SAAS,EAAE,MAAM,IAAI,CAAC,KAAK,EAAE,CAAC,WAAW,EAAE,CAAC,CACxE,CACF;aACE,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,QAAQ,CAAC,EAAE,CAAC,CAAC,QAAQ,CAAC,CAAC,EAAE,GAAG,CAAC,CAAC;aAC3C,IAAI,CAAC,EAAE,CAAC,CAAC;IACd,MAAM,MAAM,GAAG,MAAM,GAAG,CAAC,WAAW,CAAC,SAAS,CAAC,KAAK,CAAC,eAAe,EAAE;QACpE,IAAI;QACJ,QAAQ;KACT,CAAC,CAAC;IACH,IAAI,MAAM,EAAE,CAAC;QACX,MAAM,GAAG,GAAG,CAAC,MAAM,GAAG,CAAC,OAAO,CAAC,MAAM,CAAC,MAAM,CAAC,SAAS,CAAC,CAAE,CAAC;QAC1D,OAAO;YACL,GAAG,QAAQ,CAAC,GAAG,EAAE,IAAI,CAAC,IAAI,EAAE,QAAQ,CAAC;YACrC,IAAI,EAAE;gBACJ,GAAG;gBACH,MAAM,EAAE,MAAM,CAAC,MAAM;gBACrB,SAAS,EAAE,MAAM,CAAC,SAA2B;gBAC7C,IAAI;gBACJ,QAAQ;aACT;SACF,CAAC;IACJ,CAAC;IACD,MAAM,YAAY,GAAG,MAAM,GAAG,CAAC,OAAO,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;IACnD,IAAI,MAAM,EAAE,CAAC;QACX,MAAM,QAAQ,GAAG,MAAM,GAAG,CAAC,OAAO,CAAC,WAAW,CAAC,YAAY,CAAC,CAAC;QAC7D,IAAI,QAAQ,EAAE,MAAM,KAAK,MAAM,EAAE,CAAC;YAChC,MAAM,IAAI,KAAK,CAAC,iBAAiB,GAAG,QAAQ,EAAE,MAAM,GAAG,MAAM,GAAG,MAAM,CAAC,CAAC;QAC1E,CAAC;IACH,CAAC;IACD,MAAM,EAAE,MAAM,EAAE,SAAS,EAAE,GAAG,MAAM,GAAG,CAAC,WAAW,CAAC,SAAS,CAAC,KAAK,CAAC,OAAO,EAAE;QAC3E,SAAS,EAAE,YAAY;QACvB,IAAI;QACJ,QAAQ;QACR,QAAQ,EAAE,IAAI,CAAC,IAAI;KACpB,CAAC,CAAC;IACH,MAAM,GAAG,GAAG,CAAC,MAAM,GAAG,CAAC,OAAO,CAAC,MAAM,CAAC,SAA2B,CAAC,CAAE,CAAC;IACrE,IAAI,SAAS,KAAK,YAAY,EAAE,CAAC;QAC/B,0CAA0C;QAC1C,wEAAwE;QACxE,+DAA+D;QAC/D,MAAM,GAAG,CAAC,OAAO,CAAC,MAAM,CAAC,YAAY,CAAC,CAAC;IACzC,CAAC;IACD,OAAO;QACL,GAAG,QAAQ,CAAC,GAAG,EAAE,IAAI,CAAC,IAAI,EAAE,QAAQ,CAAC;QACrC,IAAI,EAAE;YACJ,GAAG;YACH,MAAM;YACN,SAAS,EAAE,SAA2B;YACtC,IAAI;YACJ,QAAQ;SACT;KACF,CAAC;AACJ,CAAC;AAED;;;;;;;;;;;;;;;GAeG;AACH,MAAM,CAAC,KAAK,UAAU,OAAO,CAC3B,GAAyB,EACzB,SAAyB,EACzB,MAAc;IAEd,MAAM,IAAI,GAAG,MAAM,GAAG,CAAC,QAAQ,CAAC,SAAS,CAAC,KAAK,CAAC,GAAG,EAAE,EAAE,MAAM,EAAE,CAAC,CAAC;IACjE,IAAI,CAAC,IAAI,EAAE,CAAC;QACV,MAAM,IAAI,KAAK,CAAC,gCAAgC,MAAM,EAAE,CAAC,CAAC;IAC5D,CAAC;IACD,MAAM,GAAG,GAAG,MAAM,GAAG,CAAC,OAAO,CAAC,MAAM,CAAC,IAAI,CAAC,SAA2B,CAAC,CAAC;IACvE,IAAI,CAAC,GAAG,EAAE,CAAC;QACT,MAAM,IAAI,KAAK,CAAC,8BAA8B,IAAI,CAAC,SAAS,EAAE,CAAC,CAAC;IAClE,CAAC;IACD,OAAO;QACL,GAAG,QAAQ,CAAC,GAAG,EAAE,IAAI,CAAC,QAAQ,EAAE,IAAI,CAAC,QAAQ,CAAC;QAC9C,IAAI,EAAE;YACJ,MAAM;YACN,GAAG;YACH,SAAS,EAAE,IAAI,CAAC,SAA2B;YAC3C,IAAI,EAAE,IAAI,CAAC,IAAI;YACf,QAAQ,EAAE,IAAI,CAAC,QAAQ;SACxB;KACF,CAAC;AACJ,CAAC;AAED,SAAS,QAAQ,CACf,GAAW,EACX,SAAiB,EACjB,QAA4B;IAK5B,MAAM,QAAQ,GAAa;QACzB,IAAI,EAAE,MAAM;QACZ,IAAI,EAAE,IAAI,GAAG,CAAC,GAAG,CAAC;QAClB,SAAS;QACT,QAAQ;KACT,CAAC;IACF,MAAM,SAAS,GAA0B,SAAS,CAAC,UAAU,CAAC,QAAQ,CAAC;QACrE,CAAC,CAAC;YACE,IAAI,EAAE,OAAO;YACb,KAAK,EAAE,IAAI,GAAG,CAAC,GAAG,CAAC;YACnB,SAAS;SACV;QACH,CAAC,CAAC,SAAS,CAAC;IACd,OAAO,EAAE,QAAQ,EAAE,SAAS,EAAE,CAAC;AACjC,CAAC"}
+{"version":3,"file":"files.js","sourceRoot":"","sources":["../../src/client/files.ts"],"names":[],"mappings":"AAeA,OAAO,EAAE,MAAM,EAAE,MAAM,gBAAgB,CAAC;AAExC,MAAM,CAAC,MAAM,aAAa,GAAG,IAAI,GAAG,EAAE,CAAC;AAUvC;;;;;;;;;;GAUG;AACH,MAAM,CAAC,KAAK,UAAU,SAAS,CAC7B,GAA+B,EAC/B,SAAyB,EACzB,IAAU,EACV,EAAE,QAAQ,EAAE,MAAM,KAA6C,EAAE;IAMjE,IAAI,CAAC,CAAC,WAAW,IAAI,GAAG,CAAC,IAAI,CAAC,CAAC,SAAS,IAAI,GAAG,CAAC,EAAE,CAAC;QACjD,MAAM,IAAI,KAAK,CACb,0EAA0E;YACxE,wFAAwF;YACxF,sIAAsI;YACtI,wBAAwB,CAC3B,CAAC;IACJ,CAAC;IACD,MAAM,IAAI,GACR,MAAM;QACN,KAAK,CAAC,IAAI,CACR,IAAI,UAAU,CACZ,MAAM,MAAM,CAAC,MAAM,CAAC,MAAM,CAAC,SAAS,EAAE,MAAM,IAAI,CAAC,KAAK,EAAE,CAAC,WAAW,EAAE,CAAC,CACxE,CACF;aACE,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,QAAQ,CAAC,EAAE,CAAC,CAAC,QAAQ,CAAC,CAAC,EAAE,GAAG,CAAC,CAAC;aAC3C,IAAI,CAAC,EAAE,CAAC,CAAC;IACd,MAAM,MAAM,GAAG,MAAM,GAAG,CAAC,WAAW,CAAC,SAAS,CAAC,KAAK,CAAC,eAAe,EAAE;QACpE,IAAI;QACJ,QAAQ;KACT,CAAC,CAAC;IACH,IAAI,MAAM,EAAE,CAAC;QACX,MAAM,GAAG,GAAG,CAAC,MAAM,GAAG,CAAC,OAAO,CAAC,MAAM,CAAC,MAAM,CAAC,SAAS,CAAC,CAAE,CAAC;QAC1D,OAAO;YACL,GAAG,QAAQ,CAAC,GAAG,EAAE,IAAI,CAAC,IAAI,EAAE,QAAQ,CAAC;YACrC,IAAI,EAAE;gBACJ,GAAG;gBACH,MAAM,EAAE,MAAM,CAAC,MAAM;gBACrB,SAAS,EAAE,MAAM,CAAC,SAA2B;gBAC7C,IAAI;gBACJ,QAAQ;aACT;SACF,CAAC;IACJ,CAAC;IACD,MAAM,YAAY,GAAG,MAAM,GAAG,CAAC,OAAO,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;IACnD,IAAI,MAAM,EAAE,CAAC;QACX,MAAM,QAAQ,GAAG,MAAM,GAAG,CAAC,OAAO,CAAC,WAAW,CAAC,YAAY,CAAC,CAAC;QAC7D,IAAI,QAAQ,EAAE,MAAM,KAAK,MAAM,EAAE,CAAC;YAChC,MAAM,IAAI,KAAK,CAAC,iBAAiB,GAAG,QAAQ,EAAE,MAAM,GAAG,MAAM,GAAG,MAAM,CAAC,CAAC;QAC1E,CAAC;IACH,CAAC;IACD,MAAM,EAAE,MAAM,EAAE,SAAS,EAAE,GAAG,MAAM,GAAG,CAAC,WAAW,CAAC,SAAS,CAAC,KAAK,CAAC,OAAO,EAAE;QAC3E,SAAS,EAAE,YAAY;QACvB,IAAI;QACJ,QAAQ;QACR,QAAQ,EAAE,IAAI,CAAC,IAAI;KACpB,CAAC,CAAC;IACH,MAAM,GAAG,GAAG,CAAC,MAAM,GAAG,CAAC,OAAO,CAAC,MAAM,CAAC,SAA2B,CAAC,CAAE,CAAC;IACrE,IAAI,SAAS,KAAK,YAAY,EAAE,CAAC;QAC/B,0CAA0C;QAC1C,wEAAwE;QACxE,+DAA+D;QAC/D,MAAM,GAAG,CAAC,OAAO,CAAC,MAAM,CAAC,YAAY,CAAC,CAAC;IACzC,CAAC;IACD,OAAO;QACL,GAAG,QAAQ,CAAC,GAAG,EAAE,IAAI,CAAC,IAAI,EAAE,QAAQ,CAAC;QACrC,IAAI,EAAE;YACJ,GAAG;YACH,MAAM;YACN,SAAS,EAAE,SAA2B;YACtC,IAAI;YACJ,QAAQ;SACT;KACF,CAAC;AACJ,CAAC;AAED;;;;;;;;;;;;;;;GAeG;AACH,MAAM,CAAC,KAAK,UAAU,OAAO,CAC3B,GAAyB,EACzB,SAAyB,EACzB,MAAc;IAEd,MAAM,IAAI,GAAG,MAAM,GAAG,CAAC,QAAQ,CAAC,SAAS,CAAC,KAAK,CAAC,GAAG,EAAE,EAAE,MAAM,EAAE,CAAC,CAAC;IACjE,IAAI,CAAC,IAAI,EAAE,CAAC;QACV,MAAM,IAAI,KAAK,CAAC,gCAAgC,MAAM,EAAE,CAAC,CAAC;IAC5D,CAAC;IACD,MAAM,GAAG,GAAG,MAAM,GAAG,CAAC,OAAO,CAAC,MAAM,CAAC,IAAI,CAAC,SAA2B,CAAC,CAAC;IACvE,IAAI,CAAC,GAAG,EAAE,CAAC;QACT,MAAM,IAAI,KAAK,CAAC,8BAA8B,IAAI,CAAC,SAAS,EAAE,CAAC,CAAC;IAClE,CAAC;IACD,OAAO;QACL,GAAG,QAAQ,CAAC,GAAG,EAAE,IAAI,CAAC,QAAQ,EAAE,IAAI,CAAC,QAAQ,CAAC;QAC9C,IAAI,EAAE;YACJ,MAAM;YACN,GAAG;YACH,SAAS,EAAE,IAAI,CAAC,SAA2B;YAC3C,IAAI,EAAE,IAAI,CAAC,IAAI;YACf,QAAQ,EAAE,IAAI,CAAC,QAAQ;SACxB;KACF,CAAC;AACJ,CAAC;AAED,SAAS,QAAQ,CACf,GAAW,EACX,SAAiB,EACjB,QAA4B;IAE5B,MAAM,QAAQ,GAAa;QACzB,IAAI,EAAE,MAAM;QACZ,IAAI,EAAE,IAAI,GAAG,CAAC,GAAG,CAAC;QAClB,SAAS;QACT,QAAQ;KACT,CAAC;IACF,MAAM,SAAS,GAA0B,SAAS,CAAC,UAAU,CAAC,QAAQ,CAAC;QACrE,CAAC,CAAC,EAAE,IAAI,EAAE,OAAO,EAAE,KAAK,EAAE,IAAI,GAAG,CAAC,GAAG,CAAC,EAAE,SAAS,EAAE;QACnD,CAAC,CAAC,SAAS,CAAC;IACd,OAAO,EAAE,QAAQ,EAAE,SAAS,EAAE,CAAC;AACjC,CAAC;AAED;;GAEG;AACH,SAAS,cAAc,CAAC,GAAQ;IAC9B,OAAO,CACL,GAAG,CAAC,QAAQ,KAAK,WAAW;QAC5B,GAAG,CAAC,QAAQ,KAAK,WAAW;QAC5B,GAAG,CAAC,QAAQ,KAAK,KAAK;QACtB,GAAG,CAAC,QAAQ,KAAK,SAAS,CAC3B,CAAC;AACJ,CAAC;AAED;;GAEG;AACH,KAAK,UAAU,YAAY,CAAC,GAAQ;IAClC,iBAAiB;IACjB,MAAM,QAAQ,GAAG,MAAM,KAAK,CAAC,GAAG,CAAC,CAAC;IAClC,IAAI,CAAC,QAAQ,CAAC,EAAE,EAAE,CAAC;QACjB,MAAM,IAAI,KAAK,CAAC,mBAAmB,GAAG,KAAK,QAAQ,CAAC,UAAU,EAAE,CAAC,CAAC;IACpE,CAAC;IAED,OAAO,MAAM,QAAQ,CAAC,WAAW,EAAE,CAAC;AACtC,CAAC;AAED;;;;GAIG;AACH,MAAM,CAAC,KAAK,UAAU,mBAAmB,CACvC,QAAoC;IAEpC,2DAA2D;IAC3D,OAAO,OAAO,CAAC,GAAG,CAChB,QAAQ,CAAC,GAAG,CAAC,KAAK,EAAE,OAAO,EAAmC,EAAE;QAC9D,IACE,CAAC,OAAO,CAAC,IAAI,KAAK,MAAM,IAAI,OAAO,CAAC,IAAI,KAAK,WAAW,CAAC;YACzD,OAAO,OAAO,CAAC,OAAO,KAAK,QAAQ;YACnC,CAAC,KAAK,CAAC,OAAO,CAAC,OAAO,CAAC,OAAO,CAAC,EAC/B,CAAC;YACD,OAAO,OAAO,CAAC;QACjB,CAAC;QAED,MAAM,gBAAgB,GAAG,MAAM,OAAO,CAAC,GAAG,CACxC,OAAO,CAAC,OAAO,CAAC,GAAG,CAAC,KAAK,EAAE,IAAI,EAAE,EAAE;YACjC,IAAI,IAAI,CAAC,IAAI,KAAK,OAAO,IAAI,IAAI,CAAC,KAAK,YAAY,GAAG,EAAE,CAAC;gBACvD,MAAM,CACJ,OAAO,CAAC,IAAI,KAAK,MAAM,EACvB,qCAAqC,CACtC,CAAC;gBACF,IAAI,cAAc,CAAC,IAAI,CAAC,KAAK,CAAC,EAAE,CAAC;oBAC/B,MAAM,SAAS,GAAG,MAAM,YAAY,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;oBACjD,OAAO,EAAE,GAAG,IAAI,EAAE,KAAK,EAAE,SAAS,EAAe,CAAC;gBACpD,CAAC;YACH,CAAC;YAED,oBAAoB;YACpB,IAAI,IAAI,CAAC,IAAI,KAAK,MAAM,IAAI,IAAI,CAAC,IAAI,YAAY,GAAG,EAAE,CAAC;gBACrD,IAAI,cAAc,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC;oBAC9B,MAAM,QAAQ,GAAG,MAAM,YAAY,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;oBAC/C,OAAO,EAAE,GAAG,IAAI,EAAE,IAAI,EAAE,QAAQ,EAAc,CAAC;gBACjD,CAAC;YACH,CAAC;YAED,OAAO,IAAI,CAAC;QACd,CAAC,CAAC,CACH,CAAC;QACF,IAAI,OAAO,CAAC,IAAI,KAAK,MAAM,EAAE,CAAC;YAC5B,OAAO,EAAE,GAAG,OAAO,EAAE,OAAO,EAAE,gBAA+B,EAAE,CAAC;QAClE,CAAC;aAAM,CAAC;YACN,OAAO,EAAE,GAAG,OAAO,EAAE,OAAO,EAAE,gBAAoC,EAAE,CAAC;QACvE,CAAC;IACH,CAAC,CAAC,CACH,CAAC;AACJ,CAAC"}