@assistant-ui/mcp-docs-server 0.1.32 → 0.1.34

package/.docs/organized/code-examples/with-ffmpeg.md

@@ -267,30 +267,26 @@ export default dynamic(() => Promise.resolve(NoSSRWrapper), {
 
 import {
   useAssistantInstructions,
-  useAssistantTool,
   useAui,
   useAuiState,
   AuiProvider,
   Suggestions,
+  Tools,
 } from "@assistant-ui/react";
-import { z } from "zod";
 import { FFmpeg } from "@ffmpeg/ffmpeg";
 import { toBlobURL } from "@ffmpeg/util";
-import { type FC, useCallback, useEffect, useRef, useState } from "react";
-import {
-  CircleCheckIcon,
-  DownloadIcon,
-  FileIcon,
-  RefreshCcwIcon,
-  TriangleAlertIcon,
-} from "lucide-react";
+import { type FC, type ReactNode, useEffect, useRef, useState } from "react";
 import { Thread } from "@/components/assistant-ui/thread";
+import { useFfmpegToolkit } from "./toolkit";
 
 // MVP: upload file, enter command
 // MVP: convert command to tool call
 // MVP: tool call: ffmpeg
 
-const FfmpegTool: FC<{ file: File }> = ({ file }) => {
+const FfmpegToolsProvider: FC<{ file: File; children: ReactNode }> = ({
+  file,
+  children,
+}) => {
   const loadingRef = useRef(false);
   const ffmpegRef = useRef(new FFmpeg());
 
@@ -321,272 +317,13 @@ const FfmpegTool: FC<{ file: File }> = ({ file }) => {
     `The user has attached a file: ${file.name}. To add text overlays, use the render_overlay tool to render HTML to a PNG image, then use run_ffmpeg with the "overlay" filter to composite it onto the video. Do NOT use the drawtext filter.`,
   );
 
-  useAssistantTool({
-    toolName: "run_ffmpeg",
-    parameters: z.object({
-      command: z
-        .string()
-        .array()
-        .describe("The ffmpeg command line arguments to provide"),
-    }),
-    execute: async ({ command }) => {
-      const transcode = async () => {
-        const ffmpeg = ffmpegRef.current;
-
-        const logs: string[] = [];
-        const logger = ({ message }: { message: string }) => {
-          logs.push(message);
-        };
-        ffmpeg.on("log", logger);
-
-        await ffmpeg.writeFile(
-          file.name,
-          new Uint8Array(await file.arrayBuffer()),
-        );
-
-        const code = await ffmpeg.exec(command);
-        ffmpeg.off("log", logger);
-
-        return { code, logs };
-      };
-      const { code, logs } = await transcode();
-
-      return {
-        success: code === 0,
-        hint:
-          code === 0
-            ? "Success. Now call display_file to show the output to the user."
-            : `some error happened, logs: ${logs.join("\n")}`,
-      };
-    },
-    render: function RenderFfmpeg({
-      args: { command },
-      result: { success } = {},
-    }) {
-      return (
-        <div className="mb-2 flex flex-col gap-2 rounded-lg border px-5 py-4">
-          <div className="flex items-center gap-2">
-            {success == null && (
-              <RefreshCcwIcon className="size-4 animate-spin text-blue-600" />
-            )}
-            {success === false && (
-              <TriangleAlertIcon className="size-4 text-red-600" />
-            )}
-            {success === true && (
-              <CircleCheckIcon className="size-4 text-green-600" />
-            )}
-            <p>Running ffmpeg</p>
-          </div>
-          <pre className="overflow-y-scroll font-sm">
-            ffmpeg {command?.join(" ")}
-          </pre>
-          {success === false && (
-            <div className="mt-2 border-t border-dashed pt-3">
-              Encountered an error.
-            </div>
-          )}
-        </div>
-      );
-    },
-  });
-
-  useAssistantTool({
-    toolName: "render_overlay",
-    parameters: z.object({
-      html: z
-        .string()
-        .describe(
-          "HTML content to render. Use inline styles for all styling. The background is transparent by default.",
-        ),
-      width: z.number().describe("Width of the output image in pixels"),
-      height: z.number().describe("Height of the output image in pixels"),
-      fileName: z
-        .string()
-        .describe("Output filename in ffmpeg filesystem, e.g. overlay.png"),
-    }),
-    execute: async ({ html, width, height, fileName }) => {
-      const svg = `<svg xmlns="http://www.w3.org/2000/svg" width="${width}" height="${height}">
-        <foreignObject width="100%" height="100%">
-          <div xmlns="http://www.w3.org/1999/xhtml">${html}</div>
-        </foreignObject>
-      </svg>`;
-
-      const img = new Image();
-      img.src = `data:image/svg+xml;charset=utf-8,${encodeURIComponent(svg)}`;
-
-      await new Promise<void>((resolve, reject) => {
-        img.onload = () => resolve();
-        img.onerror = () => reject(new Error("Failed to render HTML"));
-      });
-
-      const canvas = document.createElement("canvas");
-      canvas.width = width;
-      canvas.height = height;
-      const ctx = canvas.getContext("2d")!;
-      ctx.drawImage(img, 0, 0);
-
-      const blob = await new Promise<Blob>((resolve) =>
-        canvas.toBlob((b) => resolve(b!), "image/png"),
-      );
-      const data = new Uint8Array(await blob.arrayBuffer());
+  const toolkit = useFfmpegToolkit(file, ffmpegRef);
 
-      const ffmpeg = ffmpegRef.current;
-      await ffmpeg.writeFile(fileName, data);
-
-      return {
-        success: true,
-        hint: `Overlay image "${fileName}" (${width}x${height}) written to ffmpeg filesystem. Use it with the overlay filter in run_ffmpeg, e.g. -i ${fileName} -filter_complex "overlay=x=0:y=0"`,
-      };
-    },
-    render: function RenderOverlay({ args, result }) {
-      return (
-        <div className="mb-2 flex items-center gap-2 rounded-lg border px-5 py-4">
-          {!result && (
-            <RefreshCcwIcon className="size-4 animate-spin text-blue-600" />
-          )}
-          {result?.success && (
-            <CircleCheckIcon className="size-4 text-green-600" />
-          )}
-          <p>
-            {result?.success
-              ? `Rendered overlay → ${args.fileName}`
-              : "Rendering overlay..."}
-          </p>
-        </div>
-      );
-    },
-  });
-
-  useAssistantTool({
-    toolName: "display_file",
-    parameters: z.object({
-      fileName: z
-        .string()
-        .describe("The name of the file to display from the ffmpeg filesystem"),
-      mimeType: z
-        .string()
-        .describe("The mime type of the file, e.g. image/png, video/mp4"),
-    }),
-    execute: async ({ fileName }) => {
-      const ffmpeg = ffmpegRef.current;
-      try {
-        const data = (await ffmpeg.readFile(
-          fileName,
-        )) as Uint8Array<ArrayBuffer>;
-        return {
-          success: true,
-          size: data.byteLength,
-          hint: "A file preview and download button is now visible to the user. Do not describe the file or repeat its contents.",
-        };
-      } catch {
-        return {
-          success: false,
-          error: `File "${fileName}" not found in ffmpeg filesystem`,
-        };
-      }
-    },
-    render: function RenderDisplayFile({
-      args: { fileName, mimeType },
-      result,
-    }) {
-      // biome-ignore lint/correctness/useHookAtTopLevel: intentional conditional/nested hook usage
-      const [blobUrl, setBlobUrl] = useState<string | null>(null);
-
-      // biome-ignore lint/correctness/useHookAtTopLevel: intentional conditional/nested hook usage
-      const readFile = useCallback(async () => {
-        const ffmpeg = ffmpegRef.current;
-        const data = (await ffmpeg.readFile(
-          fileName,
-        )) as Uint8Array<ArrayBuffer>;
-        return URL.createObjectURL(new Blob([data.buffer], { type: mimeType }));
-      }, [fileName, mimeType]);
-
-      // biome-ignore lint/correctness/useHookAtTopLevel: intentional conditional/nested hook usage
-      useEffect(() => {
-        if (!result?.success) return;
-        let revoked = false;
-        readFile().then((url) => {
-          if (revoked) {
-            URL.revokeObjectURL(url);
-            return;
-          }
-          setBlobUrl(url);
-        });
-        return () => {
-          revoked = true;
-          setBlobUrl((prev) => {
-            if (prev) URL.revokeObjectURL(prev);
-            return null;
-          });
-        };
-      }, [result?.success, readFile]);
-
-      const handleDownload = () => {
-        if (!blobUrl) return;
-        const a = document.createElement("a");
-        a.href = blobUrl;
-        a.download = fileName;
-        a.click();
-      };
-
-      if (!result) {
-        return (
-          <div className="flex items-center gap-2 rounded-lg border px-5 py-4">
-            <RefreshCcwIcon className="size-4 animate-spin text-blue-600" />
-            <p>Loading file...</p>
-          </div>
-        );
-      }
-
-      if (!result.success) {
-        return (
-          <div className="flex items-center gap-2 rounded-lg border px-5 py-4">
-            <TriangleAlertIcon className="size-4 text-red-600" />
-            <p>File not found</p>
-          </div>
-        );
-      }
-
-      const isImage = mimeType?.startsWith("image/");
-      const isVideo = mimeType?.startsWith("video/");
-      const isAudio = mimeType?.startsWith("audio/");
-
-      return (
-        <div className="mb-2 flex flex-col gap-3 rounded-lg border px-5 py-4">
-          {blobUrl && isImage && (
-            // biome-ignore lint/performance/noImgElement: blob URL display
-            <img
-              src={blobUrl}
-              alt={fileName}
-              className="max-h-64 w-fit rounded"
-            />
-          )}
-          {blobUrl && isVideo && (
-            // biome-ignore lint/a11y/useMediaCaption: generated output
-            <video src={blobUrl} controls className="max-h-64 w-fit rounded" />
-          )}
-          {/* biome-ignore lint/a11y/useMediaCaption: generated output */}
-          {blobUrl && isAudio && <audio src={blobUrl} controls />}
-          {blobUrl && !isImage && !isVideo && !isAudio && (
-            <div className="flex items-center gap-2 text-muted-foreground">
-              <FileIcon className="size-5" />
-              <span>{fileName}</span>
-            </div>
-          )}
-          <button
-            type="button"
-            onClick={handleDownload}
-            className="flex w-fit items-center gap-2 rounded-md border bg-background px-4 py-2 text-sm hover:bg-accent"
-          >
-            <DownloadIcon className="size-4" />
-            Download {fileName}
-          </button>
-        </div>
-      );
-    },
+  const aui = useAui({
+    tools: Tools({ toolkit }),
   });
 
-  return null;
+  return <AuiProvider value={aui}>{children}</AuiProvider>;
 };
 
 export default function Home() {
@@ -616,15 +353,313 @@ export default function Home() {
   return (
     <div className="flex h-full flex-col">
       <AuiProvider value={aui}>
-        <Thread />
+        {lastFile ? (
+          <FfmpegToolsProvider file={lastFile}>
+            <Thread />
+          </FfmpegToolsProvider>
+        ) : (
+          <Thread />
+        )}
       </AuiProvider>
-      {lastFile && <FfmpegTool file={lastFile} />}
     </div>
   );
 }
 
 ```
 
+## app/toolkit.tsx
+
+```tsx
+"use client";
+
+import type { MutableRefObject } from "react";
+import { useCallback, useEffect, useMemo, useState } from "react";
+import { defineToolkit } from "@assistant-ui/react";
+import type { FFmpeg } from "@ffmpeg/ffmpeg";
+import {
+  CircleCheckIcon,
+  DownloadIcon,
+  FileIcon,
+  RefreshCcwIcon,
+  TriangleAlertIcon,
+} from "lucide-react";
+import { z } from "zod";
+
+export function useFfmpegToolkit(
+  file: File,
+  ffmpegRef: MutableRefObject<FFmpeg>,
+) {
+  return useMemo(
+    () =>
+      defineToolkit({
+        run_ffmpeg: {
+          type: "frontend",
+          parameters: z.object({
+            command: z
+              .string()
+              .array()
+              .describe("The ffmpeg command line arguments to provide"),
+          }),
+          execute: async ({ command }) => {
+            const ffmpeg = ffmpegRef.current;
+
+            const logs: string[] = [];
+            const logger = ({ message }: { message: string }) => {
+              logs.push(message);
+            };
+            ffmpeg.on("log", logger);
+
+            await ffmpeg.writeFile(
+              file.name,
+              new Uint8Array(await file.arrayBuffer()),
+            );
+
+            const code = await ffmpeg.exec(command);
+            ffmpeg.off("log", logger);
+
+            return {
+              success: code === 0,
+              hint:
+                code === 0
+                  ? "Success. Now call display_file to show the output to the user."
+                  : `some error happened, logs: ${logs.join("\n")}`,
+            };
+          },
+          render: function RenderFfmpeg({
+            args: { command },
+            result: { success } = {},
+          }) {
+            return (
+              <div className="mb-2 flex flex-col gap-2 rounded-lg border px-5 py-4">
+                <div className="flex items-center gap-2">
+                  {success == null && (
+                    <RefreshCcwIcon className="size-4 animate-spin text-blue-600" />
+                  )}
+                  {success === false && (
+                    <TriangleAlertIcon className="size-4 text-red-600" />
+                  )}
+                  {success === true && (
+                    <CircleCheckIcon className="size-4 text-green-600" />
+                  )}
+                  <p>Running ffmpeg</p>
+                </div>
+                <pre className="font-sm overflow-y-scroll">
+                  ffmpeg {command?.join(" ")}
+                </pre>
+                {success === false && (
+                  <div className="mt-2 border-t border-dashed pt-3">
+                    Encountered an error.
+                  </div>
+                )}
+              </div>
+            );
+          },
+        },
+        render_overlay: {
+          type: "frontend",
+          parameters: z.object({
+            html: z
+              .string()
+              .describe(
+                "HTML content to render. Use inline styles for all styling. The background is transparent by default.",
+              ),
+            width: z.number().describe("Width of the output image in pixels"),
+            height: z.number().describe("Height of the output image in pixels"),
+            fileName: z
+              .string()
+              .describe(
+                "Output filename in ffmpeg filesystem, e.g. overlay.png",
+              ),
+          }),
+          execute: async ({ html, width, height, fileName }) => {
+            const svg = `<svg xmlns="http://www.w3.org/2000/svg" width="${width}" height="${height}">
+        <foreignObject width="100%" height="100%">
+          <div xmlns="http://www.w3.org/1999/xhtml">${html}</div>
+        </foreignObject>
+      </svg>`;
+
+            const img = new Image();
+            img.src = `data:image/svg+xml;charset=utf-8,${encodeURIComponent(svg)}`;
+
+            await new Promise<void>((resolve, reject) => {
+              img.onload = () => resolve();
+              img.onerror = () => reject(new Error("Failed to render HTML"));
+            });
+
+            const canvas = document.createElement("canvas");
+            canvas.width = width;
+            canvas.height = height;
+            const ctx = canvas.getContext("2d")!;
+            ctx.drawImage(img, 0, 0);
+
+            const blob = await new Promise<Blob>((resolve) =>
+              canvas.toBlob((b) => resolve(b!), "image/png"),
+            );
+            const data = new Uint8Array(await blob.arrayBuffer());
+
+            const ffmpeg = ffmpegRef.current;
+            await ffmpeg.writeFile(fileName, data);
+
+            return {
+              success: true,
+              hint: `Overlay image "${fileName}" (${width}x${height}) written to ffmpeg filesystem. Use it with the overlay filter in run_ffmpeg, e.g. -i ${fileName} -filter_complex "overlay=x=0:y=0"`,
+            };
+          },
+          render: function RenderOverlay({ args, result }) {
+            return (
+              <div className="mb-2 flex items-center gap-2 rounded-lg border px-5 py-4">
+                {!result && (
+                  <RefreshCcwIcon className="size-4 animate-spin text-blue-600" />
+                )}
+                {result?.success && (
+                  <CircleCheckIcon className="size-4 text-green-600" />
+                )}
+                <p>
+                  {result?.success
+                    ? `Rendered overlay -> ${args.fileName}`
+                    : "Rendering overlay..."}
+                </p>
+              </div>
+            );
+          },
+        },
+        display_file: {
+          type: "frontend",
+          parameters: z.object({
+            fileName: z
+              .string()
+              .describe(
+                "The name of the file to display from the ffmpeg filesystem",
+              ),
+            mimeType: z
+              .string()
+              .describe("The mime type of the file, e.g. image/png, video/mp4"),
+          }),
+          execute: async ({ fileName }) => {
+            const ffmpeg = ffmpegRef.current;
+            try {
+              const data = (await ffmpeg.readFile(
+                fileName,
+              )) as Uint8Array<ArrayBuffer>;
+              return {
+                success: true,
+                size: data.byteLength,
+                hint: "A file preview and download button is now visible to the user. Do not describe the file or repeat its contents.",
+              };
+            } catch {
+              return {
+                success: false,
+                error: `File "${fileName}" not found in ffmpeg filesystem`,
+              };
+            }
+          },
+          render: function RenderDisplayFile({
+            args: { fileName, mimeType },
+            result,
+          }) {
+            const [blobUrl, setBlobUrl] = useState<string | null>(null);
+
+            const readFile = useCallback(async () => {
+              const ffmpeg = ffmpegRef.current;
+              const data = (await ffmpeg.readFile(
+                fileName,
+              )) as Uint8Array<ArrayBuffer>;
+              return URL.createObjectURL(
+                new Blob([data.buffer], { type: mimeType }),
+              );
+            }, [fileName, mimeType]);
+
+            useEffect(() => {
+              if (!result?.success) return;
+              let revoked = false;
+              readFile().then((url) => {
+                if (revoked) {
+                  URL.revokeObjectURL(url);
+                  return;
+                }
+                setBlobUrl(url);
+              });
+              return () => {
+                revoked = true;
+                setBlobUrl((prev) => {
+                  if (prev) URL.revokeObjectURL(prev);
+                  return null;
+                });
+              };
+            }, [result?.success, readFile]);
+
+            const handleDownload = () => {
+              if (!blobUrl) return;
+              const a = document.createElement("a");
+              a.href = blobUrl;
+              a.download = fileName;
+              a.click();
+            };
+
+            if (!result) {
+              return (
+                <div className="flex items-center gap-2 rounded-lg border px-5 py-4">
+                  <RefreshCcwIcon className="size-4 animate-spin text-blue-600" />
+                  <p>Loading file...</p>
+                </div>
+              );
+            }
+
+            if (!result.success) {
+              return (
+                <div className="flex items-center gap-2 rounded-lg border px-5 py-4">
+                  <TriangleAlertIcon className="size-4 text-red-600" />
+                  <p>File not found</p>
+                </div>
+              );
+            }
+
+            const isImage = mimeType?.startsWith("image/");
+            const isVideo = mimeType?.startsWith("video/");
+            const isAudio = mimeType?.startsWith("audio/");
+
+            return (
+              <div className="mb-2 flex flex-col gap-3 rounded-lg border px-5 py-4">
+                {blobUrl && isImage && (
+                  <img
+                    src={blobUrl}
+                    alt={fileName}
+                    className="max-h-64 w-fit rounded"
+                  />
+                )}
+                {blobUrl && isVideo && (
+                  <video
+                    src={blobUrl}
+                    controls
+                    className="max-h-64 w-fit rounded"
+                  />
+                )}
+                {blobUrl && isAudio && <audio src={blobUrl} controls />}
+                {blobUrl && !isImage && !isVideo && !isAudio && (
+                  <div className="text-muted-foreground flex items-center gap-2">
+                    <FileIcon className="size-5" />
+                    <span>{fileName}</span>
+                  </div>
+                )}
+                <button
+                  type="button"
+                  onClick={handleDownload}
+                  className="bg-background hover:bg-accent flex w-fit items-center gap-2 rounded-md border px-4 py-2 text-sm"
+                >
+                  <DownloadIcon className="size-4" />
+                  Download {fileName}
+                </button>
+              </div>
+            );
+          },
+        },
+      }),
+    [ffmpegRef, file],
+  );
+}
+
+```
+
 ## components.json
 
 ```json
@@ -658,13 +693,14 @@ export default function Home() {
 ## next.config.ts
 
 ```typescript
+import { withAui } from "@assistant-ui/next";
 import type { NextConfig } from "next";
 
 const nextConfig: NextConfig = {
   /* config options here */
 };
 
-export default nextConfig;
+export default withAui(nextConfig);
 
 ```
 
@@ -682,28 +718,29 @@ export default nextConfig;
     "start": "next start"
   },
   "dependencies": {
-    "@ai-sdk/openai": "^3.0.64",
+    "@ai-sdk/openai": "^3.0.68",
     "@assistant-ui/react": "workspace:*",
     "@assistant-ui/react-ai-sdk": "workspace:*",
     "@assistant-ui/react-markdown": "workspace:*",
     "@assistant-ui/ui": "workspace:*",
     "@ffmpeg/ffmpeg": "^0.12.15",
     "@ffmpeg/util": "^0.12.2",
-    "ai": "^6.0.188",
+    "ai": "^6.0.197",
     "class-variance-authority": "^0.7.1",
     "clsx": "^2.1.1",
-    "lucide-react": "^1.16.0",
-    "next": "^16.2.6",
-    "react": "^19.2.6",
-    "react-dom": "^19.2.6",
+    "lucide-react": "^1.17.0",
+    "next": "^16.2.7",
+    "react": "^19.2.7",
+    "react-dom": "^19.2.7",
     "tailwind-merge": "^3.6.0",
     "zod": "^4.4.3"
   },
   "devDependencies": {
+    "@assistant-ui/next": "workspace:*",
     "@assistant-ui/x-buildutils": "workspace:*",
     "@tailwindcss/postcss": "^4.3.0",
-    "@types/node": "^25.9.1",
-    "@types/react": "^19.2.15",
+    "@types/node": "^25.9.2",
+    "@types/react": "^19.2.17",
     "@types/react-dom": "^19.2.3",
     "postcss": "^8.5.15",
     "tailwindcss": "^4.3.0",