@assistant-ui/react-langchain 0.0.35 → 0.0.36

package/src/converter.ts

@@ -10,6 +10,7 @@ import {
   resolveFilePartSource,
 } from "@assistant-ui/core/internal";
 import type { StreamingTimingAccessors } from "@assistant-ui/core/react";
+import type { ReadonlyJSONObject } from "assistant-stream/utils";
 
 /** Known content block types from @langchain/core messages. */
 export type LangChainContentBlock =
@@ -51,12 +52,143 @@ export type LangChainContentBlock =
     }
   | { type: "tool_use" | "input_json_delta" };
 
+/**
+ * One block covers both multimodal vocabularies: the standard content blocks
+ * key their payload off a camelCase JavaScript field or snake_case Python
+ * field, while the legacy data content blocks use a `source_type` discriminator.
+ */
+type LangChainMediaBlock = {
+  type: "image" | "video" | "audio" | "file" | "text-plain";
+  mimeType?: string;
+  mime_type?: string;
+  data?: string | Uint8Array;
+  base64?: string | Uint8Array;
+  url?: string;
+  fileId?: string;
+  file_id?: string;
+  id?: string;
+  text?: string;
+  source_type?: "base64" | "url" | "id" | "text";
+  metadata?: { filename?: string };
+};
+
 type ConvertedContentPart =
   | ThreadUserMessage["content"][number]
   | ThreadAssistantMessage["content"][number];
 
+type MediaSource = { data: string; sourceType?: "url" | "id" };
+
+/**
+ * `id` names the block, not the payload, in the standard content blocks, so it
+ * only identifies a file under the legacy `source_type: "id"` discriminator.
+ */
+const resolveMediaSource = (
+  part: LangChainMediaBlock,
+): MediaSource | undefined => {
+  switch (part.source_type) {
+    case "url":
+      return typeof part.url === "string"
+        ? { data: part.url, sourceType: "url" }
+        : undefined;
+    case "id":
+      return typeof part.id === "string"
+        ? { data: part.id, sourceType: "id" }
+        : undefined;
+    case "base64": {
+      const data = part.data ?? part.base64;
+      return typeof data === "string" ? { data } : undefined;
+    }
+  }
+  if (typeof part.url === "string")
+    return { data: part.url, sourceType: "url" };
+  const fileId = part.fileId ?? part.file_id;
+  if (typeof fileId === "string") return { data: fileId, sourceType: "id" };
+  const data = part.data ?? part.base64;
+  if (typeof data === "string") return { data };
+  if (part.type === "text-plain" && typeof part.text === "string")
+    return { data: textToBase64(part.text) };
+  return undefined;
+};
+
+const textToBase64 = (text: string): string => {
+  const bytes = new TextEncoder().encode(text);
+  let binary = "";
+  for (let i = 0; i < bytes.length; i += 0x8000) {
+    binary += String.fromCharCode(...bytes.subarray(i, i + 0x8000));
+  }
+  return btoa(binary);
+};
+
+const resolveMediaMimeType = (part: LangChainMediaBlock): string | undefined =>
+  typeof part.mimeType === "string"
+    ? part.mimeType
+    : typeof part.mime_type === "string"
+      ? part.mime_type
+      : undefined;
+
+const defaultMediaFilename = (type: string, mimeType: string): string => {
+  if (type !== "audio" && type !== "video") return "file";
+  const subtype = mimeType.startsWith(`${type}/`)
+    ? mimeType.slice(type.length + 1)
+    : undefined;
+  return subtype ? `${type}.${subtype}` : type;
+};
+
+const resolveImage = (
+  source: MediaSource,
+  mimeType: string | undefined,
+): string | undefined => {
+  if (source.sourceType === "id") return undefined;
+  // `parseDataUrl` only matches the base64 form, so a percent-encoded data URL
+  // would otherwise be wrapped in a second base64 envelope.
+  if (source.sourceType === "url" || /^data:/i.test(source.data))
+    return source.data;
+  return mimeType ? `data:${mimeType};base64,${source.data}` : undefined;
+};
+
+const convertMediaBlock = (
+  part: LangChainMediaBlock,
+): ConvertedContentPart | undefined => {
+  const source = resolveMediaSource(part);
+  if (!source) return undefined;
+
+  const mimeType = resolveMediaMimeType(part);
+  if (part.type === "image") {
+    const image = resolveImage(source, mimeType);
+    if (image) return { type: "image" as const, image };
+  }
+
+  const resolvedMimeType =
+    mimeType ??
+    (part.type === "text-plain" ? "text/plain" : "application/octet-stream");
+  return {
+    type: "file" as const,
+    filename:
+      part.metadata?.filename ??
+      defaultMediaFilename(part.type, resolvedMimeType),
+    data: source.data,
+    mimeType: resolvedMimeType,
+    ...(source.sourceType && { sourceType: source.sourceType }),
+  };
+};
+
+export const normalizeToolCallArgs = (args: unknown): ReadonlyJSONObject => {
+  if (typeof args !== "object" || args === null || Array.isArray(args)) {
+    return {};
+  }
+
+  try {
+    const prototype = Object.getPrototypeOf(args);
+    return prototype === Object.prototype || prototype === null
+      ? (args as ReadonlyJSONObject)
+      : {};
+  } catch {
+    return {};
+  }
+};
+
 export const convertLangChainContentBlock = (
-  part: LangChainContentBlock,
+  part: LangChainContentBlock | LangChainMediaBlock,
 ): ConvertedContentPart | null | undefined => {
   const type = part.type;
   switch (type) {
@@ -74,33 +206,12 @@ export const convertLangChainContentBlock = (
       if (!image) return null;
       return { type: "image" as const, image };
     }
+    case "image":
+    case "video":
+    case "audio":
     case "file":
-      return {
-        type: "file" as const,
-        filename: part.metadata?.filename ?? "file",
-        data:
-          part.source_type === "url"
-            ? part.url
-            : part.source_type === "id"
-              ? part.id
-              : part.data,
-        mimeType: part.mime_type ?? "application/octet-stream",
-        ...((part.source_type === "url" || part.source_type === "id") && {
-          sourceType: part.source_type,
-        }),
-      };
-    case "audio": {
-      const mimeType = part.mime_type ?? "application/octet-stream";
-      const subtype = mimeType.startsWith("audio/")
-        ? mimeType.slice("audio/".length)
-        : undefined;
-      return {
-        type: "file" as const,
-        filename: subtype ? `audio.${subtype}` : "audio",
-        data: part.data,
-        mimeType,
-      };
-    }
+    case "text-plain":
+      return convertMediaBlock(part);
     case "thinking":
       return hasVisibleText(part.thinking)
         ? { type: "reasoning" as const, text: part.thinking }

package/src/getPendingToolCalls.test.ts

@@ -0,0 +1,35 @@
+import { describe, expect, it } from "vitest";
+import type { LangChainBaseMessage } from "./types";
+import { getPendingToolCalls } from "./useStreamRuntime";
+
+const aiWithNullToolCall = {
+  id: "ai-1",
+  _getType: () => "ai",
+  content: "",
+  tool_calls: [null, { id: "call-1", name: "lookup", args: {} }],
+} as unknown as LangChainBaseMessage;
+
+describe("getPendingToolCalls", () => {
+  it("skips a null tool_calls entry and returns the remaining pending call", () => {
+    expect(
+      getPendingToolCalls([
+        { id: "human-1", _getType: () => "human", content: "look it up" },
+        aiWithNullToolCall,
+      ]),
+    ).toEqual([{ id: "call-1", name: "lookup", args: {} }]);
+  });
+
+  it("treats a call answered by a tool message as settled next to a null entry", () => {
+    expect(
+      getPendingToolCalls([
+        aiWithNullToolCall,
+        {
+          id: "tool-1",
+          _getType: () => "tool",
+          content: "done",
+          tool_call_id: "call-1",
+        } as LangChainBaseMessage,
+      ]),
+    ).toEqual([]);
+  });
+});

package/src/langChainThreadState.test.ts

@@ -0,0 +1,203 @@
+import { describe, expect, it } from "vitest";
+import {
+  createLangChainThreadState,
+  reduceLangChainThreadState,
+  type StagedEntry,
+  type StagedMessage,
+} from "./langChainThreadState";
+
+const message = (
+  id: string,
+  type = "human",
+  content: unknown = id,
+): StagedMessage => ({ id, _getType: () => type, content });
+
+const entry = (
+  stagedMessage: StagedMessage,
+  overrides: Partial<StagedEntry> = {},
+): StagedEntry => ({
+  message: stagedMessage,
+  runConfig: undefined,
+  reconcileOnEcho: false,
+  baseMessageCount: 0,
+  ...overrides,
+});
+
+describe("reduceLangChainThreadState", () => {
+  it("stages sends and transcripts without changing the previous snapshot", () => {
+    const base = message("base");
+    const send = message("send");
+    const transcript = message("transcript", "ai");
+    const initial = createLangChainThreadState();
+    const staged = reduceLangChainThreadState(initial, {
+      type: "stage",
+      entry: entry(send, { reconcileOnEcho: true, baseMessageCount: 1 }),
+      visibleMessages: [base],
+    });
+    const withTranscript = reduceLangChainThreadState(staged, {
+      type: "stage",
+      entry: entry(transcript, { transcriptStatus: "unsent" }),
+      visibleMessages: staged.visibleStagedMessages!,
+    });
+
+    expect(initial.stagedEntries.size).toBe(0);
+    expect(staged.visibleStagedMessages).toEqual([base, send]);
+    expect(staged.stagedEntries.has("transcript")).toBe(false);
+    expect(withTranscript.visibleStagedMessages).toEqual([
+      base,
+      send,
+      transcript,
+    ]);
+    expect(
+      withTranscript.stagedEntries.get("transcript")?.transcriptStatus,
+    ).toBe("unsent");
+  });
+
+  it("keeps an edit's truncated base while stream messages change", () => {
+    const base = message("base");
+    const edit = message("edit");
+    const staged = reduceLangChainThreadState(createLangChainThreadState(), {
+      type: "stageEdit",
+      entry: entry(edit),
+      baseMessages: [base],
+    });
+    const reconciled = reduceLangChainThreadState(staged, {
+      type: "reconcile",
+      messages: [base, message("later")],
+      visibleMessages: staged.visibleStagedMessages!,
+    });
+
+    expect(reconciled.stagedBaseMessages).toEqual([base]);
+    expect(reconciled.visibleStagedMessages).toEqual([base, edit]);
+  });
+
+  it("matches echoes by id or eligible human content after the base count", () => {
+    const base = message("base", "human", "same");
+    const first = message("first", "human", [{ type: "text", text: "same" }]);
+    const second = message("second", "human", "same");
+    const stagedOnce = reduceLangChainThreadState(
+      createLangChainThreadState(),
+      {
+        type: "stage",
+        entry: entry(first, { reconcileOnEcho: true, baseMessageCount: 1 }),
+        visibleMessages: [base],
+      },
+    );
+    const staged = reduceLangChainThreadState(stagedOnce, {
+      type: "stage",
+      entry: entry(second, { reconcileOnEcho: true, baseMessageCount: 1 }),
+      visibleMessages: stagedOnce.visibleStagedMessages!,
+    });
+    const rejected = reduceLangChainThreadState(staged, {
+      type: "reconcile",
+      messages: [base, message("assistant", "ai", "same")],
+      visibleMessages: staged.visibleStagedMessages!,
+    });
+    const matched = reduceLangChainThreadState(rejected, {
+      type: "reconcile",
+      messages: [base, message("echo", "human", "same"), second],
+      visibleMessages: rejected.visibleStagedMessages!,
+    });
+
+    expect(rejected.stagedEntries.size).toBe(2);
+    expect(matched.stagedEntries.size).toBe(0);
+    expect(matched.visibleStagedMessages).toBeNull();
+    expect(staged.stagedEntries.size).toBe(2);
+  });
+
+  it("returns the same state when a reconcile changes nothing", () => {
+    const staged = reduceLangChainThreadState(createLangChainThreadState(), {
+      type: "stage",
+      entry: entry(message("hidden")),
+      visibleMessages: [],
+    });
+    const settled = { ...staged, visibleStagedMessages: null };
+
+    expect(
+      reduceLangChainThreadState(settled, {
+        type: "reconcile",
+        messages: [message("base")],
+        visibleMessages: [message("base")],
+      }),
+    ).toBe(settled);
+  });
+
+  it("does not match content for entries that require an id echo", () => {
+    const stagedMessage = message("voice", "human", "spoken");
+    const staged = reduceLangChainThreadState(createLangChainThreadState(), {
+      type: "stage",
+      entry: entry(stagedMessage, { transcriptStatus: "unsent" }),
+      visibleMessages: [],
+    });
+    const noMatch = reduceLangChainThreadState(staged, {
+      type: "reconcile",
+      messages: [message("other", "human", "spoken")],
+      visibleMessages: staged.visibleStagedMessages!,
+    });
+    const matched = reduceLangChainThreadState(noMatch, {
+      type: "reconcile",
+      messages: [message("voice", "human", "different")],
+      visibleMessages: noMatch.visibleStagedMessages!,
+    });
+
+    expect(noMatch.stagedEntries.has("voice")).toBe(true);
+    expect(matched.stagedEntries.has("voice")).toBe(false);
+  });
+
+  it("reserves transcripts, restores them, and promotes only nontranscripts", () => {
+    const voice = message("voice");
+    const send = message("send");
+    const first = reduceLangChainThreadState(createLangChainThreadState(), {
+      type: "stage",
+      entry: entry(voice, { transcriptStatus: "unsent" }),
+      visibleMessages: [],
+    });
+    const staged = reduceLangChainThreadState(first, {
+      type: "stage",
+      entry: entry(send),
+      visibleMessages: first.visibleStagedMessages!,
+    });
+    const sent = reduceLangChainThreadState(staged, {
+      type: "markTranscript",
+      messages: [voice, send],
+      status: "sent",
+    });
+    const restored = reduceLangChainThreadState(sent, {
+      type: "markTranscript",
+      messages: [voice],
+      status: "unsent",
+    });
+    const promoted = reduceLangChainThreadState(restored, {
+      type: "promote",
+      messages: [voice, send],
+      visibleMessages: restored.visibleStagedMessages!,
+    });
+
+    expect(staged.stagedEntries.get("voice")?.transcriptStatus).toBe("unsent");
+    expect(sent.stagedEntries.get("voice")?.transcriptStatus).toBe("sent");
+    expect(sent.stagedEntries.get("send")?.transcriptStatus).toBeUndefined();
+    expect(promoted.stagedEntries.has("send")).toBe(false);
+    expect(promoted.stagedEntries.get("voice")?.transcriptStatus).toBe(
+      "unsent",
+    );
+    expect(promoted.visibleStagedMessages).toEqual([voice]);
+  });
+
+  it("removes a failed stage and clears the edit base", () => {
+    const stagedMessage = message("failed");
+    const staged = reduceLangChainThreadState(createLangChainThreadState(), {
+      type: "stageEdit",
+      entry: entry(stagedMessage),
+      baseMessages: [message("base")],
+    });
+    const removed = reduceLangChainThreadState(staged, {
+      type: "remove",
+      id: "failed",
+      visibleMessages: staged.visibleStagedMessages!,
+    });
+
+    expect(removed.stagedEntries.size).toBe(0);
+    expect(removed.stagedBaseMessages).toBeNull();
+    expect(removed.visibleStagedMessages).toBeNull();
+  });
+});

package/src/langChainThreadState.ts

@@ -0,0 +1,239 @@
+import type { AppendMessage } from "@assistant-ui/core";
+import type { LangChainBaseMessage } from "./types";
+import { getMessageType } from "./convertMessages";
+
+export type StagedMessage = LangChainBaseMessage & { id: string };
+
+export type StagedEntry = {
+  message: StagedMessage;
+  runConfig: AppendMessage["runConfig"];
+  reconcileOnEcho: boolean;
+  baseMessageCount: number;
+  transcriptStatus?: "unsent" | "sent";
+};
+
+export type LangChainThreadState = {
+  forkGeneration: number;
+  forkPending: boolean;
+  stagedEntries: ReadonlyMap<string, StagedEntry>;
+  stagedBaseMessages: LangChainBaseMessage[] | null;
+  visibleStagedMessages: LangChainBaseMessage[] | null;
+};
+
+export type LangChainThreadAction =
+  | { type: "supersedeFork" }
+  | { type: "finishFork"; generation: number }
+  | { type: "cancelFork" }
+  | {
+      type: "stage";
+      entry: StagedEntry;
+      visibleMessages: readonly LangChainBaseMessage[];
+    }
+  | {
+      type: "stageEdit";
+      entry: StagedEntry;
+      baseMessages: LangChainBaseMessage[];
+    }
+  | {
+      type: "reconcile";
+      messages: LangChainBaseMessage[];
+      visibleMessages: readonly LangChainBaseMessage[];
+    }
+  | {
+      type: "markTranscript";
+      messages: readonly LangChainBaseMessage[];
+      status: "unsent" | "sent";
+    }
+  | {
+      type: "remove";
+      id: string;
+      visibleMessages: readonly LangChainBaseMessage[];
+    }
+  | {
+      type: "promote";
+      messages: readonly LangChainBaseMessage[];
+      visibleMessages: readonly LangChainBaseMessage[];
+    };
+
+export const createLangChainThreadState = (): LangChainThreadState => ({
+  forkGeneration: 0,
+  forkPending: false,
+  stagedEntries: new Map(),
+  stagedBaseMessages: null,
+  visibleStagedMessages: null,
+});
+
+const humanContentText = (content: LangChainBaseMessage["content"]) => {
+  if (typeof content === "string") return content;
+  if (!Array.isArray(content)) return "";
+  return content
+    .filter(
+      (part): part is { type: "text"; text: string } =>
+        typeof part === "object" &&
+        part !== null &&
+        part.type === "text" &&
+        typeof part.text === "string",
+    )
+    .map((part) => part.text)
+    .join("");
+};
+
+const hasSameMessageContent = (
+  a: LangChainBaseMessage,
+  b: LangChainBaseMessage,
+) => humanContentText(a.content) === humanContentText(b.content);
+
+export const reduceLangChainThreadState = (
+  state: LangChainThreadState,
+  action: LangChainThreadAction,
+): LangChainThreadState => {
+  switch (action.type) {
+    case "supersedeFork":
+      return {
+        ...state,
+        forkGeneration: state.forkGeneration + 1,
+        forkPending: true,
+      };
+    case "finishFork":
+      return state.forkGeneration === action.generation
+        ? { ...state, forkPending: false }
+        : state;
+    case "cancelFork":
+      return {
+        ...state,
+        forkGeneration: state.forkGeneration + 1,
+        forkPending: false,
+      };
+    case "stage": {
+      const stagedEntries = new Map(state.stagedEntries);
+      stagedEntries.set(action.entry.message.id, action.entry);
+      return {
+        ...state,
+        stagedEntries,
+        visibleStagedMessages: [
+          ...action.visibleMessages,
+          action.entry.message,
+        ],
+      };
+    }
+    case "stageEdit": {
+      const stagedEntries = new Map(state.stagedEntries);
+      stagedEntries.set(action.entry.message.id, action.entry);
+      return {
+        ...state,
+        stagedEntries,
+        stagedBaseMessages: action.baseMessages,
+        visibleStagedMessages: [...action.baseMessages, action.entry.message],
+      };
+    }
+    case "reconcile": {
+      if (state.stagedEntries.size === 0) return state;
+      // Staged edits must keep their truncated base while stream updates arrive before promotion.
+      const baseMessages = state.stagedBaseMessages ?? action.messages;
+      const stagedEntries = new Map(state.stagedEntries);
+      const remainingStagedMessages: LangChainBaseMessage[] = [];
+      const matchedBaseMessageIndexes = new Set<number>();
+      const visibleStagedIds = new Set(
+        action.visibleMessages.flatMap((message) =>
+          message.id ? [message.id] : [],
+        ),
+      );
+      for (const [id, staged] of state.stagedEntries) {
+        if (!visibleStagedIds.has(id)) continue;
+        const echoed = baseMessages.some((message, index) => {
+          if (matchedBaseMessageIndexes.has(index)) return false;
+          if (message.id === id) {
+            matchedBaseMessageIndexes.add(index);
+            return true;
+          }
+          if (
+            !staged.reconcileOnEcho ||
+            index < staged.baseMessageCount ||
+            getMessageType(message) !== "human" ||
+            !hasSameMessageContent(message, staged.message)
+          ) {
+            return false;
+          }
+          matchedBaseMessageIndexes.add(index);
+          return true;
+        });
+        if (echoed) stagedEntries.delete(id);
+        else remainingStagedMessages.push(staged.message);
+      }
+      if (remainingStagedMessages.length === 0) {
+        if (
+          stagedEntries.size === state.stagedEntries.size &&
+          state.stagedBaseMessages === null &&
+          state.visibleStagedMessages === null
+        ) {
+          return state;
+        }
+        return {
+          ...state,
+          stagedEntries,
+          stagedBaseMessages: null,
+          visibleStagedMessages: null,
+        };
+      }
+      return {
+        ...state,
+        stagedEntries,
+        visibleStagedMessages: [...baseMessages, ...remainingStagedMessages],
+      };
+    }
+    case "markTranscript": {
+      let stagedEntries: Map<string, StagedEntry> | null = null;
+      for (const message of action.messages) {
+        const entry = message.id
+          ? (stagedEntries ?? state.stagedEntries).get(message.id)
+          : undefined;
+        if (entry?.transcriptStatus) {
+          stagedEntries ??= new Map(state.stagedEntries);
+          stagedEntries.set(message.id!, {
+            ...entry,
+            transcriptStatus: action.status,
+          });
+        }
+      }
+      return stagedEntries ? { ...state, stagedEntries } : state;
+    }
+    case "remove": {
+      if (!state.stagedEntries.has(action.id)) return state;
+      const stagedEntries = new Map(state.stagedEntries);
+      stagedEntries.delete(action.id);
+      return {
+        ...state,
+        stagedEntries,
+        stagedBaseMessages:
+          stagedEntries.size === 0 ? null : state.stagedBaseMessages,
+        visibleStagedMessages:
+          stagedEntries.size === 0
+            ? null
+            : action.visibleMessages.filter(
+                (message) => message.id !== action.id,
+              ),
+      };
+    }
+    case "promote": {
+      const stagedEntries = new Map(state.stagedEntries);
+      const promotedIds = new Set<string>();
+      for (const message of action.messages) {
+        if (!message.id || stagedEntries.get(message.id)?.transcriptStatus)
+          continue;
+        promotedIds.add(message.id);
+        stagedEntries.delete(message.id);
+      }
+      return {
+        ...state,
+        stagedEntries,
+        stagedBaseMessages: null,
+        visibleStagedMessages:
+          stagedEntries.size > 0
+            ? action.visibleMessages.filter(
+                (message) => !message.id || !promotedIds.has(message.id),
+              )
+            : null,
+      };
+    }
+  }
+};

package/src/types.ts

@@ -118,6 +118,11 @@ export type LangChainRuntimeExtraOptions = ExternalStoreSharedOptions & {
    */
   onThreadIdChange?: ((threadId: string | undefined) => void) | undefined;
   cloud?: AssistantCloud | undefined;
+  /**
+   * Stable identity for the account or workspace owning Cloud runtime state.
+   * Provide it from the first render and change it when that scope changes.
+   */
+  scopeId?: string | undefined;
   adapters?:
     | {
         attachments?: AttachmentAdapter | undefined;
@@ -137,11 +142,15 @@ export type LangChainRuntimeExtraOptions = ExternalStoreSharedOptions & {
   /**
    * Routes the Cancel button's click to `useStream().stop()`. On by
    * default. Pass `false` to disable the Cancel button.
+   *
+   * @deprecated Experimental since 2025-01-03. Not scheduled for removal; the API may change in any release.
    */
   unstable_allowCancellation?: boolean | undefined;
   /**
    * Custom `RemoteThreadListAdapter`. When provided, replaces the
    * cloud-backed thread list adapter.
+   *
+   * @deprecated Experimental since 2026-04-16. Not scheduled for removal; the API may change in any release.
    */
   unstable_threadListAdapter?: RemoteThreadListAdapter | undefined;
   /** Custom thread-creation hook, forwarded to the cloud adapter. */