@assistant-ui/core 0.1.0

package/src/adapters/index.ts

@@ -0,0 +1,30 @@
+// Attachment adapters
+export type { AttachmentAdapter } from "./attachment";
+export {
+  SimpleImageAttachmentAdapter,
+  SimpleTextAttachmentAdapter,
+  CompositeAttachmentAdapter,
+} from "./attachment";
+
+// Speech adapters
+export type { SpeechSynthesisAdapter, DictationAdapter } from "./speech";
+export {
+  WebSpeechSynthesisAdapter,
+  WebSpeechDictationAdapter,
+} from "./speech";
+
+// Feedback adapter
+export type { FeedbackAdapter } from "./feedback";
+
+// Suggestion adapter
+export type { SuggestionAdapter } from "./suggestion";
+
+// Thread history adapters
+export type {
+  ThreadHistoryAdapter,
+  GenericThreadHistoryAdapter,
+  MessageFormatAdapter,
+  MessageFormatItem,
+  MessageFormatRepository,
+  MessageStorageEntry,
+} from "./thread-history";

package/src/adapters/speech.ts

@@ -0,0 +1,338 @@
+import type { Unsubscribe } from "../types";
+
+// =============================================================================
+// Speech Synthesis Adapter
+// =============================================================================
+
+export namespace SpeechSynthesisAdapter {
+  export type Status =
+    | {
+        type: "starting" | "running";
+      }
+    | {
+        type: "ended";
+        reason: "finished" | "cancelled" | "error";
+        error?: unknown;
+      };
+
+  export type Utterance = {
+    status: Status;
+    cancel: () => void;
+    subscribe: (callback: () => void) => Unsubscribe;
+  };
+}
+
+export type SpeechSynthesisAdapter = {
+  speak: (text: string) => SpeechSynthesisAdapter.Utterance;
+};
+
+// =============================================================================
+// Dictation Adapter
+// =============================================================================
+
+export namespace DictationAdapter {
+  export type Status =
+    | {
+        type: "starting" | "running";
+      }
+    | {
+        type: "ended";
+        reason: "stopped" | "cancelled" | "error";
+      };
+
+  export type Result = {
+    transcript: string;
+    isFinal?: boolean;
+  };
+
+  export type Session = {
+    status: Status;
+    stop: () => Promise<void>;
+    cancel: () => void;
+    onSpeechStart: (callback: () => void) => Unsubscribe;
+    onSpeechEnd: (callback: (result: Result) => void) => Unsubscribe;
+    onSpeech: (callback: (result: Result) => void) => Unsubscribe;
+  };
+}
+
+export type DictationAdapter = {
+  listen: () => DictationAdapter.Session;
+  disableInputDuringDictation?: boolean;
+};
+
+// =============================================================================
+// Web Speech Synthesis Adapter
+// =============================================================================
+
+export class WebSpeechSynthesisAdapter implements SpeechSynthesisAdapter {
+  speak(text: string): SpeechSynthesisAdapter.Utterance {
+    const utterance = new SpeechSynthesisUtterance(text);
+
+    const subscribers = new Set<() => void>();
+    const handleEnd = (
+      reason: "finished" | "error" | "cancelled",
+      error?: unknown,
+    ) => {
+      if (res.status.type === "ended") return;
+
+      res.status = { type: "ended", reason, error };
+      subscribers.forEach((handler) => handler());
+    };
+
+    utterance.addEventListener("end", () => handleEnd("finished"));
+    utterance.addEventListener("error", (e) => handleEnd("error", e.error));
+
+    window.speechSynthesis.speak(utterance);
+
+    const res: SpeechSynthesisAdapter.Utterance = {
+      status: { type: "running" },
+      cancel: () => {
+        window.speechSynthesis.cancel();
+        handleEnd("cancelled");
+      },
+      subscribe: (callback) => {
+        if (res.status.type === "ended") {
+          let cancelled = false;
+          queueMicrotask(() => {
+            if (!cancelled) callback();
+          });
+          return () => {
+            cancelled = true;
+          };
+        } else {
+          subscribers.add(callback);
+          return () => {
+            subscribers.delete(callback);
+          };
+        }
+      },
+    };
+    return res;
+  }
+}
+
+// =============================================================================
+// Web Speech Dictation Adapter
+// =============================================================================
+
+interface SpeechRecognitionResultItem {
+  transcript: string;
+  confidence: number;
+}
+
+interface SpeechRecognitionResult {
+  readonly length: number;
+  item(index: number): SpeechRecognitionResultItem;
+  readonly isFinal: boolean;
+  [index: number]: SpeechRecognitionResultItem;
+}
+
+interface SpeechRecognitionResultList {
+  readonly length: number;
+  item(index: number): SpeechRecognitionResult;
+  [index: number]: SpeechRecognitionResult;
+}
+
+interface SpeechRecognitionEvent extends Event {
+  readonly resultIndex: number;
+  readonly results: SpeechRecognitionResultList;
+}
+
+interface SpeechRecognitionErrorEvent extends Event {
+  readonly error: string;
+  readonly message: string;
+}
+
+interface SpeechRecognitionInstance extends EventTarget {
+  lang: string;
+  continuous: boolean;
+  interimResults: boolean;
+  start(): void;
+  stop(): void;
+  abort(): void;
+}
+
+interface SpeechRecognitionConstructor {
+  new (): SpeechRecognitionInstance;
+}
+
+declare global {
+  interface Window {
+    SpeechRecognition?: SpeechRecognitionConstructor;
+    webkitSpeechRecognition?: SpeechRecognitionConstructor;
+  }
+}
+
+const getSpeechRecognitionAPI = ():
+  | SpeechRecognitionConstructor
+  | undefined => {
+  if (typeof window === "undefined") return undefined;
+  return window.SpeechRecognition ?? window.webkitSpeechRecognition;
+};
+
+export class WebSpeechDictationAdapter implements DictationAdapter {
+  private _language: string;
+  private _continuous: boolean;
+  private _interimResults: boolean;
+
+  constructor(
+    options: {
+      language?: string;
+      continuous?: boolean;
+      interimResults?: boolean;
+    } = {},
+  ) {
+    const defaultLanguage =
+      typeof navigator !== "undefined" && navigator.language
+        ? navigator.language
+        : "en-US";
+    this._language = options.language ?? defaultLanguage;
+    this._continuous = options.continuous ?? true;
+    this._interimResults = options.interimResults ?? true;
+  }
+
+  static isSupported(): boolean {
+    return getSpeechRecognitionAPI() !== undefined;
+  }
+
+  listen(): DictationAdapter.Session {
+    const SpeechRecognitionAPI = getSpeechRecognitionAPI();
+    if (!SpeechRecognitionAPI) {
+      throw new Error(
+        "SpeechRecognition is not supported in this browser. " +
+          "Try using Chrome, Edge, or Safari.",
+      );
+    }
+
+    const recognition = new SpeechRecognitionAPI();
+    recognition.lang = this._language;
+    recognition.continuous = this._continuous;
+    recognition.interimResults = this._interimResults;
+
+    const speechStartCallbacks = new Set<() => void>();
+    const speechEndCallbacks = new Set<
+      (result: DictationAdapter.Result) => void
+    >();
+    const speechCallbacks = new Set<
+      (result: DictationAdapter.Result) => void
+    >();
+
+    let finalTranscript = "";
+
+    const session: DictationAdapter.Session = {
+      status: { type: "starting" },
+
+      stop: async () => {
+        recognition.stop();
+        return new Promise<void>((resolve) => {
+          const checkEnded = () => {
+            if (session.status.type === "ended") {
+              resolve();
+            } else {
+              setTimeout(checkEnded, 50);
+            }
+          };
+          checkEnded();
+        });
+      },
+
+      cancel: () => {
+        recognition.abort();
+      },
+
+      onSpeechStart: (callback: () => void): Unsubscribe => {
+        speechStartCallbacks.add(callback);
+        return () => {
+          speechStartCallbacks.delete(callback);
+        };
+      },
+
+      onSpeechEnd: (
+        callback: (result: DictationAdapter.Result) => void,
+      ): Unsubscribe => {
+        speechEndCallbacks.add(callback);
+        return () => {
+          speechEndCallbacks.delete(callback);
+        };
+      },
+
+      onSpeech: (
+        callback: (result: DictationAdapter.Result) => void,
+      ): Unsubscribe => {
+        speechCallbacks.add(callback);
+        return () => {
+          speechCallbacks.delete(callback);
+        };
+      },
+    };
+
+    const updateStatus = (newStatus: DictationAdapter.Status) => {
+      session.status = newStatus;
+    };
+
+    recognition.addEventListener("speechstart", () => {
+      for (const cb of speechStartCallbacks) cb();
+    });
+
+    recognition.addEventListener("start", () => {
+      updateStatus({ type: "running" });
+    });
+
+    recognition.addEventListener("result", (event) => {
+      const speechEvent = event as unknown as SpeechRecognitionEvent;
+
+      for (
+        let i = speechEvent.resultIndex;
+        i < speechEvent.results.length;
+        i++
+      ) {
+        const result = speechEvent.results[i];
+        if (!result) continue;
+
+        const transcript = result[0]?.transcript ?? "";
+
+        if (result.isFinal) {
+          finalTranscript += transcript;
+          for (const cb of speechCallbacks) cb({ transcript, isFinal: true });
+        } else {
+          for (const cb of speechCallbacks) cb({ transcript, isFinal: false });
+        }
+      }
+    });
+
+    recognition.addEventListener("speechend", () => {
+      // speechend fires when user stops speaking, but 'end' handles final cleanup
+    });
+
+    recognition.addEventListener("end", () => {
+      const currentStatus = session.status;
+      if (currentStatus.type !== "ended") {
+        updateStatus({ type: "ended", reason: "stopped" });
+      }
+      if (finalTranscript) {
+        for (const cb of speechEndCallbacks)
+          cb({ transcript: finalTranscript });
+        finalTranscript = "";
+      }
+    });
+
+    recognition.addEventListener("error", (event) => {
+      const errorEvent = event as unknown as SpeechRecognitionErrorEvent;
+      if (errorEvent.error === "aborted") {
+        updateStatus({ type: "ended", reason: "cancelled" });
+      } else {
+        updateStatus({ type: "ended", reason: "error" });
+        console.error("Dictation error:", errorEvent.error, errorEvent.message);
+      }
+    });
+
+    try {
+      recognition.start();
+    } catch (error) {
+      updateStatus({ type: "ended", reason: "error" });
+      throw error;
+    }
+
+    return session;
+  }
+}

package/src/adapters/suggestion.ts

@@ -0,0 +1,14 @@
+import type { ThreadMessage } from "../types";
+import type { ThreadSuggestion } from "../runtime/interfaces/thread-runtime-core";
+
+type SuggestionAdapterGenerateOptions = {
+  messages: readonly ThreadMessage[];
+};
+
+export type SuggestionAdapter = {
+  generate: (
+    options: SuggestionAdapterGenerateOptions,
+  ) =>
+    | Promise<readonly ThreadSuggestion[]>
+    | AsyncGenerator<readonly ThreadSuggestion[], void>;
+};

package/src/adapters/thread-history.ts

@@ -0,0 +1,72 @@
+import type {
+  ChatModelRunOptions,
+  ChatModelRunResult,
+} from "../runtime/utils/chat-model-adapter";
+import type {
+  ExportedMessageRepository,
+  ExportedMessageRepositoryItem,
+} from "../runtime/utils/message-repository";
+
+// =============================================================================
+// Message Format Adapter
+// =============================================================================
+
+export interface MessageStorageEntry<TPayload> {
+  id: string;
+  parent_id: string | null;
+  format: string;
+  content: TPayload;
+}
+
+export interface MessageFormatItem<TMessage> {
+  parentId: string | null;
+  message: TMessage;
+}
+
+export interface MessageFormatRepository<TMessage> {
+  headId?: string | null;
+  messages: MessageFormatItem<TMessage>[];
+}
+
+export interface MessageFormatAdapter<
+  TMessage,
+  TStorageFormat extends Record<string, unknown>,
+> {
+  format: string;
+  encode(item: MessageFormatItem<TMessage>): TStorageFormat;
+  decode(
+    stored: MessageStorageEntry<TStorageFormat>,
+  ): MessageFormatItem<TMessage>;
+  getId(message: TMessage): string;
+}
+
+// =============================================================================
+// Thread History Adapter
+// =============================================================================
+
+export type GenericThreadHistoryAdapter<TMessage> = {
+  load(): Promise<MessageFormatRepository<TMessage>>;
+  append(item: MessageFormatItem<TMessage>): Promise<void>;
+  update?(
+    item: MessageFormatItem<TMessage>,
+    localMessageId: string,
+  ): Promise<void>;
+  reportTelemetry?(
+    items: MessageFormatItem<TMessage>[],
+    options?: {
+      durationMs?: number;
+      stepTimestamps?: { start_ms: number; end_ms: number }[];
+    },
+  ): void;
+};
+
+export type ThreadHistoryAdapter = {
+  load(): Promise<ExportedMessageRepository & { unstable_resume?: boolean }>;
+  resume?(
+    options: ChatModelRunOptions,
+  ): AsyncGenerator<ChatModelRunResult, void, unknown>;
+  append(item: ExportedMessageRepositoryItem): Promise<void>;
+  withFormat?<TMessage, TStorageFormat extends Record<string, unknown>>(
+    formatAdapter: MessageFormatAdapter<TMessage, TStorageFormat>,
+  ): GenericThreadHistoryAdapter<TMessage>;
+};

package/src/index.ts

@@ -0,0 +1,7 @@
+// @assistant-ui/core - Framework-agnostic core runtime (public API)
+
+export * from "./types";
+export * from "./model-context";
+export * from "./adapters";
+export * from "./runtime";
+export * from "./runtimes";

package/src/internal.ts

@@ -0,0 +1,7 @@
+// @assistant-ui/core/internal - Internal implementation details
+// Not part of the public API. Used by @assistant-ui/react and other framework bindings.
+
+export * from "./subscribable";
+export * from "./utils";
+export * from "./runtime/internal";
+export * from "./runtimes/internal";

package/src/model-context/frame/host.ts

@@ -0,0 +1,195 @@
+import { ModelContextProvider, ModelContext } from "../types";
+import type { Unsubscribe } from "../../types";
+import { Tool } from "assistant-stream";
+import {
+  FrameMessage,
+  FRAME_MESSAGE_CHANNEL,
+  SerializedModelContext,
+  SerializedTool,
+} from "./types";
+
+/**
+ * Deserializes tools from JSON Schema format back to Tool objects
+ */
+const deserializeTool = (serializedTool: SerializedTool): Tool<any, any> =>
+  ({
+    parameters: serializedTool.parameters,
+    ...(serializedTool.description && {
+      description: serializedTool.description,
+    }),
+    ...(serializedTool.disabled !== undefined && {
+      disabled: serializedTool.disabled,
+    }),
+    ...(serializedTool.type && { type: serializedTool.type }),
+  }) as Tool<any, any>;
+
+/**
+ * Deserializes a ModelContext from transmission format
+ */
+const deserializeModelContext = (
+  serialized: SerializedModelContext,
+): ModelContext => ({
+  ...(serialized.system !== undefined && { system: serialized.system }),
+  ...(serialized.tools && {
+    tools: Object.fromEntries(
+      Object.entries(serialized.tools).map(([name, tool]) => [
+        name,
+        deserializeTool(tool),
+      ]),
+    ),
+  }),
+});
+
+export class AssistantFrameHost implements ModelContextProvider {
+  private _context: ModelContext = {};
+  private _subscribers = new Set<() => void>();
+  private _pendingRequests = new Map<
+    string,
+    {
+      resolve: (value: any) => void;
+      reject: (error: any) => void;
+    }
+  >();
+  private _requestCounter = 0;
+  private _iframeWindow: Window;
+  private _targetOrigin: string;
+
+  constructor(iframeWindow: Window, targetOrigin: string = "*") {
+    this._iframeWindow = iframeWindow;
+    this._targetOrigin = targetOrigin;
+
+    this.handleMessage = this.handleMessage.bind(this);
+    window.addEventListener("message", this.handleMessage);
+
+    this.requestContext();
+  }
+
+  private handleMessage(event: MessageEvent) {
+    if (this._targetOrigin !== "*" && event.origin !== this._targetOrigin)
+      return;
+    if (event.source !== this._iframeWindow) return;
+    if (event.data?.channel !== FRAME_MESSAGE_CHANNEL) return;
+
+    const message = event.data.message as FrameMessage;
+
+    switch (message.type) {
+      case "model-context-update": {
+        this.updateContext(message.context);
+        break;
+      }
+
+      case "tool-result": {
+        const pending = this._pendingRequests.get(message.id);
+        if (pending) {
+          if (message.error) {
+            pending.reject(new Error(message.error));
+          } else {
+            pending.resolve(message.result);
+          }
+          this._pendingRequests.delete(message.id);
+        }
+        break;
+      }
+    }
+  }
+
+  private updateContext(serializedContext: SerializedModelContext) {
+    const context = deserializeModelContext(serializedContext);
+    this._context = {
+      ...context,
+      tools:
+        context.tools &&
+        Object.fromEntries(
+          Object.entries(context.tools).map(([name, tool]) => [
+            name,
+            {
+              ...tool,
+              execute: (args: any) => this.callTool(name, args),
+            } as Tool<any, any>,
+          ]),
+        ),
+    };
+    this.notifySubscribers();
+  }
+
+  private callTool(toolName: string, args: any): Promise<any> {
+    return this.sendRequest(
+      {
+        type: "tool-call",
+        id: `tool-${this._requestCounter++}`,
+        toolName,
+        args,
+      },
+      30000,
+      `Tool call "${toolName}" timed out`,
+    );
+  }
+
+  private sendRequest<T extends FrameMessage & { id: string }>(
+    message: T,
+    timeout = 30000,
+    timeoutMessage = "Request timed out",
+  ): Promise<any> {
+    return new Promise((resolve, reject) => {
+      this._pendingRequests.set(message.id, { resolve, reject });
+
+      this._iframeWindow.postMessage(
+        { channel: FRAME_MESSAGE_CHANNEL, message },
+        this._targetOrigin,
+      );
+
+      const timeoutId = setTimeout(() => {
+        const pending = this._pendingRequests.get(message.id);
+        if (pending) {
+          pending.reject(new Error(timeoutMessage));
+          this._pendingRequests.delete(message.id);
+        }
+      }, timeout);
+
+      const originalResolve = this._pendingRequests.get(message.id)!.resolve;
+      const originalReject = this._pendingRequests.get(message.id)!.reject;
+
+      this._pendingRequests.set(message.id, {
+        resolve: (value: any) => {
+          clearTimeout(timeoutId);
+          originalResolve(value);
+        },
+        reject: (error: any) => {
+          clearTimeout(timeoutId);
+          originalReject(error);
+        },
+      });
+    });
+  }
+
+  private requestContext() {
+    this._iframeWindow.postMessage(
+      {
+        channel: FRAME_MESSAGE_CHANNEL,
+        message: {
+          type: "model-context-request",
+        } as FrameMessage,
+      },
+      this._targetOrigin,
+    );
+  }
+
+  private notifySubscribers() {
+    this._subscribers.forEach((callback) => callback());
+  }
+
+  getModelContext(): ModelContext {
+    return this._context;
+  }
+
+  subscribe(callback: () => void): Unsubscribe {
+    this._subscribers.add(callback);
+    return () => this._subscribers.delete(callback);
+  }
+
+  dispose() {
+    window.removeEventListener("message", this.handleMessage);
+    this._subscribers.clear();
+    this._pendingRequests.clear();
+  }
+}

package/src/model-context/frame/index.ts

@@ -0,0 +1,10 @@
+export { AssistantFrameHost } from "./host";
+export { AssistantFrameProvider } from "./provider";
+
+export type {
+  SerializedTool,
+  SerializedModelContext,
+  FrameMessageType,
+  FrameMessage,
+} from "./types";
+export { FRAME_MESSAGE_CHANNEL } from "./types";