@assistant-ui/core 0.3.18 → 0.3.20

package/src/react/adapters/LocalStorageThreadListAdapter.test.ts

@@ -95,6 +95,94 @@ describe("parseStoredMessageRepository", () => {
     expect(repo.headId).toBe("message-2");
   });
 
+  it("preserves modality on user and assistant messages", () => {
+    const repo = parseStoredMessageRepository(
+      JSON.stringify({
+        messages: [
+          {
+            message: {
+              ...storedMessage("voice-user"),
+              metadata: { modality: "voice", custom: {} },
+            },
+            parentId: null,
+          },
+          {
+            message: {
+              ...storedMessage("voice-assistant", "assistant"),
+              metadata: { modality: "voice", custom: {} },
+            },
+            parentId: "voice-user",
+          },
+        ],
+      }),
+    );
+
+    expect(
+      repo.messages.map(({ message }) => message.metadata.modality),
+    ).toEqual(["voice", "voice"]);
+  });
+
+  it("restores submitted feedback with and without a comment", () => {
+    const assistant = (id: string, submittedFeedback: unknown) => ({
+      message: {
+        ...storedMessage(id, "assistant"),
+        metadata: { submittedFeedback, custom: {} },
+      },
+      parentId: null,
+    });
+    const repo = parseStoredMessageRepository(
+      JSON.stringify({
+        messages: [
+          assistant("commented", {
+            type: "negative",
+            comment: "Quoted the wrong date",
+          }),
+          assistant("legacy", { type: "positive" }),
+          assistant("blank", { type: "positive", comment: "" }),
+          assistant("invalid", { type: "neutral", comment: "ignored" }),
+        ],
+      }),
+    );
+
+    expect(
+      repo.messages.map(({ message }) => message.metadata.submittedFeedback),
+    ).toStrictEqual([
+      { type: "negative", comment: "Quoted the wrong date" },
+      { type: "positive" },
+      { type: "positive" },
+      undefined,
+    ]);
+  });
+
+  it("omits modality when it is missing, unsupported, or on a system message", () => {
+    const repo = parseStoredMessageRepository(
+      JSON.stringify({
+        messages: [
+          {
+            message: {
+              ...storedMessage("system", "system"),
+              content: [{ type: "text", text: "Be brief" }],
+              metadata: { modality: "voice", custom: {} },
+            },
+            parentId: null,
+          },
+          { message: storedMessage("typed-user"), parentId: "system" },
+          {
+            message: {
+              ...storedMessage("video-assistant", "assistant"),
+              metadata: { modality: "video", custom: {} },
+            },
+            parentId: "typed-user",
+          },
+        ],
+      }),
+    );
+
+    expect(
+      repo.messages.map(({ message }) => "modality" in message.metadata),
+    ).toEqual([false, false, false]);
+  });
+
   it("drops a head id that points at a skipped message", () => {
     const repo = parseStoredMessageRepository(
       JSON.stringify({
@@ -139,6 +227,279 @@ describe("parseStoredMessageRepository", () => {
     expect(repo.messages.map((item) => item.message.id)).toEqual(["valid"]);
   });
 
+  it("drops unreadable parts and attachments while keeping their messages", () => {
+    const attachment = {
+      id: "attachment-1",
+      type: "document",
+      name: "notes.txt",
+      status: { type: "complete" },
+      content: [null, { type: "text", text: "notes" }],
+    };
+    const repo = parseStoredMessageRepository(
+      JSON.stringify({
+        headId: "answer",
+        messages: [
+          {
+            message: {
+              ...storedMessage("question"),
+              content: [
+                null,
+                "text",
+                { type: 1 },
+                { type: "text", text: "hi" },
+              ],
+              attachments: [
+                null,
+                attachment,
+                { ...attachment, id: 2 },
+                { ...attachment, id: "uploading", status: { type: "running" } },
+                { ...attachment, id: "no-content", content: undefined },
+                { ...attachment, id: "no-name", name: undefined },
+              ],
+            },
+            parentId: null,
+          },
+          {
+            message: {
+              ...storedMessage("answer", "assistant"),
+              content: [null, { type: "future-part", value: 1 }],
+            },
+            parentId: "question",
+          },
+        ],
+      }),
+    );
+
+    const [question, answer] = repo.messages.map((item) => item.message);
+    expect(repo.headId).toBe("answer");
+    expect(question?.content).toEqual([{ type: "text", text: "hi" }]);
+    expect(question?.attachments).toEqual([
+      { ...attachment, content: [{ type: "text", text: "notes" }] },
+    ]);
+    expect(answer?.content).toEqual([{ type: "future-part", value: 1 }]);
+  });
+
+  it("keeps an attachment part type it does not know", () => {
+    const repo = parseStoredMessageRepository(
+      JSON.stringify({
+        headId: "question",
+        messages: [
+          {
+            message: {
+              ...storedMessage("question"),
+              attachments: [
+                {
+                  id: "attachment-1",
+                  type: "document",
+                  name: "notes.txt",
+                  status: { type: "complete" },
+                  content: [{ type: "future-part", value: 1 }],
+                },
+              ],
+            },
+            parentId: null,
+          },
+        ],
+      }),
+    );
+
+    expect(repo.messages[0]?.message.attachments?.[0]?.content).toEqual([
+      { type: "future-part", value: 1 },
+    ]);
+  });
+
+  it("drops known parts that are missing a required field", () => {
+    const parts = {
+      text: { type: "text", text: "hi" },
+      reasoning: { type: "reasoning", text: "because" },
+      summary: { type: "reasoning", unstable_summary: "Searching the docs" },
+      image: { type: "image", image: "https://example.com/a.png" },
+      file: { type: "file", data: "SGk=", mimeType: "text/plain" },
+      audio: { type: "audio", audio: { data: "SGk=", format: "mp3" } },
+      data: { type: "data", name: "weather", data: { sunny: true } },
+      url: {
+        type: "source",
+        sourceType: "url",
+        id: "source-1",
+        url: "https://example.com",
+      },
+      document: {
+        type: "source",
+        sourceType: "document",
+        id: "source-2",
+        title: "Notes",
+        mediaType: "text/plain",
+      },
+      generativeUI: { type: "generative-ui", spec: { root: "hi" } },
+      toolCall: {
+        type: "tool-call",
+        toolCallId: "call-1",
+        toolName: "search",
+        args: {},
+        argsText: "{}",
+      },
+    };
+    const repo = parseStoredMessageRepository(
+      JSON.stringify({
+        messages: [
+          {
+            message: {
+              ...storedMessage("assistant", "assistant"),
+              content: [
+                { type: "text" },
+                { ...parts.reasoning, text: 1 },
+                { type: "image" },
+                { ...parts.file, mimeType: undefined },
+                { type: "audio", audio: null },
+                { type: "data", data: {} },
+                { ...parts.url, url: undefined },
+                { ...parts.document, sourceType: "unknown" },
+                { type: "generative-ui" },
+                { ...parts.toolCall, argsText: undefined },
+                ...Object.values(parts),
+              ],
+            },
+            parentId: null,
+          },
+        ],
+      }),
+    );
+
+    expect(repo.messages[0]?.message.content).toEqual(Object.values(parts));
+  });
+
+  it("applies the same rules to nested tool call messages", () => {
+    const toolCall = {
+      type: "tool-call",
+      toolCallId: "call-1",
+      toolName: "delegate",
+      args: {},
+      argsText: "{}",
+    };
+    const repo = parseStoredMessageRepository(
+      JSON.stringify({
+        messages: [
+          {
+            message: {
+              ...storedMessage("parent", "assistant"),
+              content: [
+                {
+                  ...toolCall,
+                  messages: [
+                    null,
+                    { id: "missing-shell" },
+                    {
+                      ...storedMessage("nested", "assistant"),
+                      content: [null, { type: "text", text: "nested" }],
+                    },
+                  ],
+                },
+                { ...toolCall, toolCallId: "call-2", messages: "invalid" },
+              ],
+            },
+            parentId: null,
+          },
+        ],
+      }),
+    );
+
+    expect(repo.messages[0]?.message.content).toEqual([
+      {
+        ...toolCall,
+        messages: [
+          {
+            ...storedMessage("nested", "assistant"),
+            content: [{ type: "text", text: "nested" }],
+            createdAt: new Date("2026-01-01T00:00:00.000Z"),
+            metadata: {
+              unstable_state: null,
+              unstable_annotations: [],
+              unstable_data: [],
+              steps: [],
+              custom: {},
+            },
+          },
+        ],
+      },
+      { ...toolCall, toolCallId: "call-2" },
+    ]);
+  });
+
+  it("keeps a system message when exactly one readable part remains", () => {
+    const repo = parseStoredMessageRepository(
+      JSON.stringify({
+        messages: [
+          {
+            message: {
+              ...storedMessage("unreadable", "system"),
+              content: [null],
+            },
+            parentId: null,
+          },
+          {
+            message: {
+              ...storedMessage("non-text", "system"),
+              content: [
+                { type: "text" },
+                { type: "image", image: "https://example.com/a.png" },
+              ],
+            },
+            parentId: null,
+          },
+          {
+            message: {
+              ...storedMessage("recovered", "system"),
+              content: [null, { type: "text", text: "Be brief." }],
+            },
+            parentId: null,
+          },
+        ],
+      }),
+    );
+
+    expect(
+      repo.messages.map(({ message }) => [message.id, message.content]),
+    ).toEqual([
+      ["non-text", [{ type: "image", image: "https://example.com/a.png" }]],
+      ["recovered", [{ type: "text", text: "Be brief." }]],
+    ]);
+  });
+
+  it("stops parsing nested tool call messages past the depth limit", () => {
+    let stored: unknown = storedMessage("leaf", "assistant");
+    for (let level = 0; level < 150; level += 1) {
+      stored = {
+        ...storedMessage(`level-${level}`, "assistant"),
+        content: [
+          {
+            type: "tool-call",
+            toolCallId: `call-${level}`,
+            toolName: "delegate",
+            args: {},
+            argsText: "{}",
+            messages: [stored],
+          },
+        ],
+      };
+    }
+
+    const repo = parseStoredMessageRepository(
+      JSON.stringify({ messages: [{ message: stored, parentId: null }] }),
+    );
+
+    let message = repo.messages[0]?.message;
+    let depth = 0;
+    while (message?.role === "assistant") {
+      const part = message.content[0];
+      const nested =
+        part?.type === "tool-call" ? part.messages?.[0] : undefined;
+      if (!nested) break;
+      message = nested;
+      depth += 1;
+    }
+    expect(depth).toBe(100);
+  });
+
   it("skips messages whose parent is missing, skipped, or appears later", () => {
     const repo = parseStoredMessageRepository(
       JSON.stringify({

package/src/react/adapters/LocalStorageThreadListAdapter.tsx

@@ -9,6 +9,7 @@ import {
 } from "react";
 import { useAui } from "@assistant-ui/store";
 import type {
+  MessageModality,
   RemoteThreadInitializeResponse,
   RemoteThreadListAdapter,
   RemoteThreadListResponse,
@@ -22,6 +23,13 @@ import type {
   ExportedMessageRepositoryItem,
 } from "../../internal";
 import { isRecord } from "../../utils/json/is-json";
+import {
+  MAX_STORED_MESSAGE_DEPTH,
+  isStoredMessagePart,
+  isStoredMessageRole,
+  parseStoredAttachment,
+  parseStoredDate,
+} from "../../runtime/utils/stored-message-parts";
 import {
   RuntimeAdapterProvider,
   type RuntimeAdapters,
@@ -109,28 +117,53 @@ const parseStoredThread = (value: unknown): StoredThreadMetadata | null => {
   };
 };
 
-const parseDate = (value: unknown): Date | null => {
-  if (value instanceof Date && !Number.isNaN(value.getTime())) return value;
-  if (typeof value !== "string" && typeof value !== "number") return null;
-
-  const date = new Date(value);
-  return Number.isNaN(date.getTime()) ? null : date;
-};
-
-const isMessageRole = (value: unknown): value is ThreadMessage["role"] =>
-  value === "system" || value === "user" || value === "assistant";
+const messageModalities = {
+  voice: true,
+} satisfies Record<MessageModality, true>;
+
+const isMessageModality = (value: unknown): value is MessageModality =>
+  typeof value === "string" && Object.hasOwn(messageModalities, value);
+
+const parseStoredMessageParts = (
+  content: unknown[],
+  depth: number,
+): unknown[] =>
+  content.flatMap((part) => {
+    if (!isStoredMessagePart(part)) return [];
+    if (part.type !== "tool-call" || part.messages === undefined) return [part];
+
+    const { messages, ...toolCall } = part;
+    if (!Array.isArray(messages)) return [toolCall];
+    return [
+      {
+        ...toolCall,
+        messages: messages.flatMap((item) => {
+          const message = parseStoredThreadMessage(item, depth + 1);
+          return message ? [message] : [];
+        }),
+      },
+    ];
+  });
 
-const parseStoredThreadMessage = (value: unknown): ThreadMessage | null => {
+const parseStoredThreadMessage = (
+  value: unknown,
+  depth: number,
+): ThreadMessage | null => {
+  if (depth > MAX_STORED_MESSAGE_DEPTH) return null;
   if (!isRecord(value) || typeof value.id !== "string") return null;
-  if (!isMessageRole(value.role)) return null;
+  if (!isStoredMessageRole(value.role)) return null;
   if (!Array.isArray(value.content)) return null;
 
-  const createdAt = parseDate(value.createdAt);
+  const createdAt = parseStoredDate(value.createdAt);
   if (!createdAt) return null;
 
   const metadata = value.metadata;
   if (!isRecord(metadata) || !isRecord(metadata.custom)) return null;
 
+  const modality = isMessageModality(metadata.modality)
+    ? metadata.modality
+    : undefined;
+
   if (value.role === "assistant") {
     const status = value.status;
     if (!isRecord(status) || typeof status.type !== "string") return null;
@@ -139,11 +172,15 @@ const parseStoredThreadMessage = (value: unknown): ThreadMessage | null => {
       ? metadata.submittedFeedback
       : undefined;
     const submittedFeedbackType = submittedFeedback?.type;
+    const submittedFeedbackComment = submittedFeedback?.comment;
 
     return {
       id: value.id,
       role: "assistant",
-      content: value.content as StoredAssistantMessage["content"],
+      content: parseStoredMessageParts(
+        value.content,
+        depth,
+      ) as StoredAssistantMessage["content"],
       status: status as StoredAssistantMessage["status"],
       createdAt,
       metadata: {
@@ -163,6 +200,10 @@ const parseStoredThreadMessage = (value: unknown): ThreadMessage | null => {
           ? {
               submittedFeedback: {
                 type: submittedFeedbackType,
+                ...(typeof submittedFeedbackComment === "string" &&
+                submittedFeedbackComment !== ""
+                  ? { comment: submittedFeedbackComment }
+                  : undefined),
               },
             }
           : undefined),
@@ -176,6 +217,7 @@ const parseStoredThreadMessage = (value: unknown): ThreadMessage | null => {
         ...(metadata.isOptimistic === true
           ? { isOptimistic: true }
           : undefined),
+        ...(modality !== undefined ? { modality } : undefined),
         custom: metadata.custom,
       },
     };
@@ -185,23 +227,31 @@ const parseStoredThreadMessage = (value: unknown): ThreadMessage | null => {
     return {
       id: value.id,
       role: "user",
-      content: value.content as StoredUserMessage["content"],
+      content: parseStoredMessageParts(
+        value.content,
+        depth,
+      ) as StoredUserMessage["content"],
       attachments: Array.isArray(value.attachments)
-        ? (value.attachments as StoredUserMessage["attachments"])
+        ? value.attachments.flatMap((item) => {
+            const attachment = parseStoredAttachment(item, isStoredMessagePart);
+            return attachment ? [attachment] : [];
+          })
         : [],
       createdAt,
       metadata: {
+        ...(modality !== undefined ? { modality } : undefined),
         custom: metadata.custom,
       },
     };
   }
 
-  if (value.content.length !== 1) return null;
+  const content = parseStoredMessageParts(value.content, depth);
+  if (content.length !== 1) return null;
 
   return {
     id: value.id,
     role: "system",
-    content: [value.content[0] as StoredSystemMessage["content"][0]],
+    content: [content[0] as StoredSystemMessage["content"][0]],
     createdAt,
     metadata: {
       custom: metadata.custom,
@@ -226,7 +276,7 @@ const parseStoredMessageRepositoryItem = (
 ): ExportedMessageRepositoryItem | null => {
   if (!isRecord(value)) return null;
 
-  const message = parseStoredThreadMessage(value.message);
+  const message = parseStoredThreadMessage(value.message, 0);
   if (!message) return null;
 
   const parentId = value.parentId;

package/src/react/client/DataRenderers.test.tsx

@@ -0,0 +1,45 @@
+// @vitest-environment jsdom
+
+import { act, cleanup, render } from "@testing-library/react";
+import { useAui } from "@assistant-ui/store";
+import { afterEach, describe, expect, it, vi } from "vitest";
+import { DataRenderers } from "./DataRenderers";
+
+type AnyClient = Record<string, any>;
+
+afterEach(() => {
+  cleanup();
+});
+
+describe("DataRenderers", () => {
+  it.each(["__proto__", "constructor"])(
+    "registers and removes a renderer named %s",
+    async (name) => {
+      let aui!: AnyClient;
+      const Harness = () => {
+        aui = useAui({ dataRenderers: DataRenderers() } as never);
+        return null;
+      };
+      render(<Harness />);
+
+      let remove!: () => void;
+      await act(async () => {
+        remove = aui.dataRenderers().setDataUI(name, () => null);
+        await vi.waitFor(() => {
+          const renderers = aui.dataRenderers().getState().renderers;
+          expect(Object.hasOwn(renderers, name)).toBe(true);
+          expect(renderers[name]).toHaveLength(1);
+        });
+      });
+
+      await act(async () => {
+        remove();
+        await vi.waitFor(() =>
+          expect(
+            Object.hasOwn(aui.dataRenderers().getState().renderers, name),
+          ).toBe(false),
+        );
+      });
+    },
+  );
+});

package/src/react/client/DataRenderers.ts

@@ -3,6 +3,7 @@ import { resource } from "@assistant-ui/tap";
 import type { ClientOutput } from "@assistant-ui/store";
 import type { DataRenderersState } from "../types/scopes/dataRenderers";
 import type { DataMessagePartComponent } from "../types/MessagePartComponentTypes";
+import { nullProtoRecord } from "../../utils/record";
 
 /**
  * Registers renderers for `data` message parts.
@@ -13,30 +14,33 @@ import type { DataMessagePartComponent } from "../types/MessagePartComponentType
  */
 const useDataRenderers = (): ClientOutput<"dataRenderers"> => {
   const [state, setState] = useState<DataRenderersState>(() => ({
-    renderers: {},
+    renderers: nullProtoRecord(),
     fallbacks: [],
   }));
 
   const setDataUI = useCallback(
     (name: string, render: DataMessagePartComponent) => {
       setState((prev) => {
+        const renderers = nullProtoRecord(prev.renderers);
+        renderers[name] = [...(renderers[name] ?? []), render];
         return {
           ...prev,
-          renderers: {
-            ...prev.renderers,
-            [name]: [...(prev.renderers[name] ?? []), render],
-          },
+          renderers,
         };
       });
 
       return () => {
         setState((prev) => {
+          const renderers = nullProtoRecord(prev.renderers);
+          const remaining = renderers[name]?.filter((r) => r !== render) ?? [];
+          if (remaining.length > 0) {
+            renderers[name] = remaining;
+          } else {
+            delete renderers[name];
+          }
           return {
             ...prev,
-            renderers: {
-              ...prev.renderers,
-              [name]: prev.renderers[name]?.filter((r) => r !== render) ?? [],
-            },
+            renderers,
           };
         });
       };