@ulearning/ai-generative-frame 0.0.0-stage → 1.0.0-next.0

package/src/spec/assistant-ui/thread.ts

@@ -0,0 +1,381 @@
+import {
+  hasLaterToolCallInMessage,
+  toolCalls,
+  type MessageLike,
+} from "../../assistant-ui/tool-calls";
+import type { ActionHandler } from "../actions";
+import type { Catalog } from "../catalog";
+import { formulaValueReads } from "../calc";
+import { conditionGuards, templateStatePaths } from "../expressions";
+import { escapePointerToken, getAtPointer } from "../pointer";
+import type { SpecPromptOptions } from "../prompt";
+import {
+  createSpecTools,
+  SPEC_TOOL_RESULTS_RULE,
+  specGuidanceModule,
+  type RenderSpecInput,
+} from "../render-spec";
+import type { SpecStateWrite } from "../state";
+import { parseSpecStream } from "../stream";
+import { childIdsOf, safePointer } from "../tree";
+import type { Spec, SpecElement } from "../types";
+import { formatSpecIssues, type SpecIssue } from "../validate";
+
+/** The arguments of a `render_spec` call, which stream in. */
+export type SpecArgs = Partial<RenderSpecInput>;
+
+/** The interactions a host records on a tool call, as its `unstable_interactions` prop holds them. */
+export type InteractionLog = {
+  readonly entries: readonly (
+    | {
+        readonly type: "action";
+        readonly payload: Readonly<Record<string, unknown>>;
+      }
+    | { readonly type: "human-response"; readonly payload: unknown }
+  )[];
+};
+
+/**
+ * The state writes saved on a `render_spec` call: the newest `setState` interaction recorded for that call. An entry whose `writes` is not a list marks a save that failed, so the call starts from the model's state rather than from an older snapshot.
+ */
+export function savedWrites(
+  log: InteractionLog | undefined,
+  toolCallId: string,
+): SpecStateWrite[] {
+  const entries = log?.entries ?? [];
+  for (let index = entries.length - 1; index >= 0; index--) {
+    const entry = entries[index]!;
+    if (
+      entry.type !== "action" ||
+      entry.payload["type"] !== "setState" ||
+      entry.payload["toolCallId"] !== toolCallId
+    )
+      continue;
+    const writes = entry.payload["writes"];
+    if (!Array.isArray(writes)) return [];
+    return writes.filter(
+      (write): write is SpecStateWrite =>
+        Array.isArray(write) &&
+        typeof write[0] === "string" &&
+        (write.length === 2 ||
+          (write.length === 3 && write[2] === "replace") ||
+          (write.length === 4 &&
+            (write[2] === "add" || write[2] === "replace") &&
+            Array.isArray(write[3]) &&
+            write[3].every((path: unknown) => typeof path === "string"))),
+    );
+  }
+  return [];
+}
+
+const WIDGET_TOOLS = new Set([
+  "read_me",
+  "show_widget",
+  "edit_widget",
+  "preview_widget",
+  "render_spec",
+]);
+
+/** Results of the tool calls before this one, by tool name and newest first, leaving out the widget toolkit's own tools. Tool names are keys every model can see, unlike call ids, which some formats never show, and counting back from this call keeps an index stable when a host prunes older history. */
+export function resolveToolResults(
+  messages: readonly MessageLike[],
+  toolCallId: string,
+): Record<string, unknown[]> {
+  const results: Record<string, unknown[]> = Object.create(null);
+  for (const call of toolCalls(messages)) {
+    if (call.id === toolCallId) break;
+    if (WIDGET_TOOLS.has(call.name)) continue;
+    (results[call.name] ??= []).unshift(decodeResult(call.result));
+  }
+  return results;
+}
+
+/** A string result that holds a JSON object or array is decoded, since the model read that string as the structure. */
+function decodeResult(result: unknown): unknown {
+  if (typeof result !== "string") return result;
+  try {
+    const value: unknown = JSON.parse(result);
+    return typeof value === "object" && value !== null ? value : result;
+  } catch {
+    return result;
+  }
+}
+
+/** A `spec` argument as the renderers read it, with `elements` an object even while the argument is still streaming in. */
+export const specFromArgs = (spec: object): Spec => {
+  const value = { state: {}, ...spec } as Spec;
+  return typeof value.elements === "object" && value.elements !== null
+    ? value
+    : { ...value, elements: {} };
+};
+
+/**
+ * The spec a `render_spec` call's patches apply onto: the result of the
+ * earlier `render_spec` calls with the same title in the thread.
+ */
+export function resolveSpecBase(
+  messages: readonly MessageLike[],
+  toolCallId: string,
+  title: string,
+): Spec | undefined {
+  let spec: Spec | undefined;
+  for (const call of toolCalls(messages)) {
+    if (call.id === toolCallId) return spec;
+    if (call.name !== "render_spec" || call.args["title"] !== title) continue;
+    if (call.args["spec"] && typeof call.args["spec"] === "object") {
+      spec = specFromArgs(call.args["spec"]);
+    } else if (typeof call.args["patches"] === "string") {
+      spec = parseSpecStream(
+        call.args["patches"],
+        spec ? { initial: spec } : {},
+      ).spec;
+    }
+  }
+  return spec;
+}
+
+/** Whether a later `render_spec` call in the same message has this call's title. That call shows the spec this one's patches lead to, so this one shows nothing rather than a second copy. */
+export function supersededInMessage(
+  messages: readonly MessageLike[],
+  toolCallId: string,
+  title: string,
+): boolean {
+  return hasLaterToolCallInMessage(
+    messages,
+    toolCallId,
+    title,
+    (call) => call.name === "render_spec",
+  );
+}
+
+function priorSpecCalls(
+  messages: readonly MessageLike[],
+  toolCallId: string,
+  title: string,
+): number {
+  let count = 0;
+  for (const call of toolCalls(messages)) {
+    if (call.id === toolCallId) break;
+    if (call.name === "render_spec" && call.args["title"] === title) count++;
+  }
+  return count;
+}
+
+type SpecHistory = {
+  base(title: string): { spec: Spec | undefined; version: number };
+  toolResults: Record<string, unknown[]> | undefined;
+};
+
+/** What a mounted call registers for `execute`: the spec each title held before it in its own thread, how many calls built it, and the earlier results it mounts under `/tools`. */
+export const callHistory = (
+  messages: readonly MessageLike[],
+  toolCallId: string,
+  toolResults: Record<string, unknown[]> | undefined,
+): SpecHistory => ({
+  base: (title) => ({
+    spec: resolveSpecBase(messages, toolCallId, title),
+    version: priorSpecCalls(messages, toolCallId, title),
+  }),
+  toolResults,
+});
+
+const readsTools = (pointer: string) =>
+  pointer === "/tools" || pointer.startsWith("/tools/");
+
+/** Collects the state paths a value shows, leaving out reads in a branch that only renders while that path has a value: the `$then` of a `$cond` that tests it, and the `?` branch of a `$calc` condition on it. */
+function collectShown(
+  value: unknown,
+  guarded: ReadonlySet<string>,
+  shown: string[],
+): void {
+  if (Array.isArray(value)) {
+    for (const item of value) collectShown(item, guarded, shown);
+    return;
+  }
+  if (typeof value !== "object" || value === null) return;
+  const record = value as Record<string, unknown>;
+  if ("$cond" in record) {
+    const tested = conditionGuards(record["$cond"]).map(safePointer);
+    collectShown(record["$then"], new Set([...guarded, ...tested]), shown);
+    collectShown(record["$else"], guarded, shown);
+    return;
+  }
+  for (const [key, child] of Object.entries(record)) {
+    if (typeof child !== "string") {
+      collectShown(child, guarded, shown);
+      continue;
+    }
+    const paths =
+      key === "$state" || key === "$bindState"
+        ? [child]
+        : key === "$template"
+          ? templateStatePaths(child)
+          : key === "$calc"
+            ? formulaValueReads(child)
+            : [];
+    for (const path of paths) {
+      const pointer = safePointer(path);
+      if (!guarded.has(pointer)) shown.push(pointer);
+    }
+  }
+}
+
+/** Each place a spec can render an element, with the paths a `visible` condition on it or an ancestor guards there: reachable from the root through children and slots, leaving out one hidden with `visible: false` along with what it holds, and the repeated children of a `/tools` path the results hold no array at. An element under parents that guard different paths appears once per set of guards. */
+function renderedElements(
+  spec: Spec,
+  state: unknown,
+): [string, SpecElement, ReadonlySet<string>][] {
+  const elements = spec.elements ?? {};
+  const rendered: [string, SpecElement, ReadonlySet<string>][] = [];
+  const seen = new Set<string>();
+  const pending: [string, ReadonlySet<string>][] =
+    typeof spec.root === "string" ? [[spec.root, new Set()]] : [];
+  while (pending.length > 0) {
+    const [id, inherited] = pending.pop()!;
+    if (!Object.hasOwn(elements, id)) continue;
+    const element = elements[id];
+    if (typeof element !== "object" || element === null) continue;
+    if (element.visible === false) continue;
+    const guarded = new Set([
+      ...inherited,
+      ...conditionGuards(element.visible).map(safePointer),
+    ]);
+    const visit = JSON.stringify([id, ...[...guarded].sort()]);
+    if (seen.has(visit)) continue;
+    seen.add(visit);
+    rendered.push([id, element, guarded]);
+    const repeat =
+      typeof element.repeat?.path === "string"
+        ? safePointer(element.repeat.path)
+        : undefined;
+    const repeatsNothing =
+      repeat !== undefined &&
+      readsTools(repeat) &&
+      !Array.isArray(getAtPointer(state, repeat));
+    for (const slot of ["default", ...Object.keys(element.slots ?? {})]) {
+      if (slot === "default" && repeatsNothing) continue;
+      for (const child of childIdsOf(element, slot))
+        if (typeof child === "string") pending.push([child, guarded]);
+    }
+  }
+  return rendered;
+}
+
+/** Each place a spec shows state under `/tools` that the mounted results do not have, as a warning, since the model may have read the value from output the thread does not store or may mean a field that is absent. A read that only renders while its path has a value, under a `visible` condition or `$cond` test on it, is left out. */
+function missingToolResults(
+  spec: Spec,
+  toolResults: Record<string, unknown[]>,
+): SpecIssue[] {
+  const issues: SpecIssue[] = [];
+  const state = { tools: toolResults };
+  const reported = new Set<string>();
+  for (const [id, element, guarded] of renderedElements(spec, state)) {
+    const shown: string[] = [];
+    collectShown([element.props, element.on, element.watch], guarded, shown);
+    if (typeof element.repeat?.path === "string") {
+      const pointer = safePointer(element.repeat.path);
+      if (!guarded.has(pointer)) shown.push(pointer);
+    }
+    for (const pointer of new Set(shown)) {
+      if (!readsTools(pointer)) continue;
+      if (getAtPointer(state, pointer) !== undefined) continue;
+      const report = JSON.stringify([id, pointer]);
+      if (reported.has(report)) continue;
+      reported.add(report);
+      issues.push({
+        code: "missing-tool-result",
+        severity: "warning",
+        elementId: id,
+        path: `/elements/${escapePointerToken(id)}`,
+        message: `"${id}" shows \`${pointer}\`, which the earlier tool results do not have, so it shows nothing there. If you read that value from the tool's output, write it into the spec instead.`,
+      });
+    }
+  }
+  return issues;
+}
+
+/** The app's rules and where earlier tool results are, which `render_spec`'s description and the `spec` module both carry. */
+const toolkitRules = (
+  specPrompt: Omit<SpecPromptOptions, "mode"> | undefined,
+) => [...(specPrompt?.customRules ?? []), SPEC_TOOL_RESULTS_RULE];
+
+/**
+ * The `render_spec` tool a toolkit shares across its calls. Each mounted call
+ * registers its {@link callHistory} in `histories` under its tool call id, so
+ * `execute` patches the base that call's own thread held, even after a reload
+ * or in another thread.
+ */
+export function createSpecCalls(
+  catalog: Catalog,
+  specPrompt: Omit<SpecPromptOptions, "mode"> | undefined,
+) {
+  const specs = new Map<string, { spec: Spec; version: number }>();
+  const tools = createSpecTools(catalog, {
+    specs,
+    customRules: toolkitRules(specPrompt),
+  });
+  const histories = new Map<string, SpecHistory>();
+  const execute = async (
+    input: RenderSpecInput,
+    call: { toolCallId: string },
+  ) => {
+    const history = histories.get(call.toolCallId);
+    if (history) {
+      const { spec, version } = history.base(input.title);
+      if (spec) specs.set(input.title, { spec, version });
+      else specs.delete(input.title);
+    }
+    const running = tools.render_spec.execute(input);
+    // `execute` stores this call's spec before it returns, so the spec is read before an overlapping call with the same title stores its own.
+    const spec = specs.get(input.title)?.spec;
+    const result = await running;
+    // The results are mounted only while the spec writes no `tools` of its own.
+    if (!history?.toolResults || !spec || spec.state?.["tools"] !== undefined)
+      return result;
+    const missing = missingToolResults(spec, history.toolResults);
+    if (missing.length === 0) return result;
+    const issues = [...result.issues, ...missing];
+    return { ...result, issues, feedback: formatSpecIssues(issues) };
+  };
+  return { tools, histories, execute };
+}
+
+/** The `spec` module of `read_me`: the catalog guidance with the built-in actions every assistant-ui toolkit handles, the app's rules, and where earlier tool results are. */
+export const specModules = (
+  catalog: Catalog,
+  specPrompt: Omit<SpecPromptOptions, "mode"> | undefined,
+  host: { clipboard: boolean },
+) => [
+  specGuidanceModule(catalog, {
+    ...specPrompt,
+    hostApi: {
+      prompt: true,
+      modelContext: true,
+      clipboard: host.clipboard,
+      ...specPrompt?.hostApi,
+    },
+    customRules: toolkitRules(specPrompt),
+  }),
+];
+
+/** Writes to the browser's clipboard, rejecting where the Clipboard API is missing. */
+export const writeClipboard = (text: string): Promise<void> =>
+  typeof navigator !== "undefined" && navigator.clipboard
+    ? navigator.clipboard.writeText(text)
+    : Promise.reject(new Error("Clipboard API is unavailable"));
+
+/** The built-in `copyText` handler. A copy that does not happen fails, so the rest of the list, such as a confirmation, does not run. */
+export const copyTextHandler =
+  (write: (text: string) => Promise<void>): ActionHandler =>
+  (params, source) => {
+    if (source.kind === "watch") {
+      throw new Error(
+        "copyText runs only from an event a component emits, never from `watch`",
+      );
+    }
+    const text = params["text"];
+    if (typeof text !== "string") {
+      throw new Error("copyText needs a string `text`");
+    }
+    return write(text);
+  };

package/src/spec/assistant-ui/toolkit.tsx

@@ -0,0 +1,295 @@
+import {
+  computed,
+  defineComponent,
+  h,
+  onActivated,
+  onDeactivated,
+  onMounted,
+  onScopeDispose,
+  toValue,
+  watch,
+  type Component,
+  type MaybeRefOrGetter,
+} from "vue";
+import { toolkitEntry } from "../../assistant-ui/entry";
+import type { WidgetToolkitExtension } from "../../assistant-ui/extension";
+import {
+  savedContext,
+  toSharedContext,
+  type ModelContextSource,
+  type SharedContext,
+} from "../../assistant-ui/modelContext";
+import { createSnapshotSaver } from "../../assistant-ui/snapshotSaver";
+import type { MessageLike } from "../../assistant-ui/tool-calls";
+import { useSharedContext } from "../../assistant-ui/useSharedContext";
+import type { ActionHandler } from "../actions";
+import type { Catalog } from "../catalog";
+import type { SpecPromptOptions } from "../prompt";
+import type { SpecComponents } from "../vue/SpecRenderer";
+import { useSpecStream } from "../vue/useSpecStream";
+import {
+  createStateStore,
+  type SpecStateStore,
+  type SpecStateWrite,
+} from "../state";
+import {
+  callHistory,
+  copyTextHandler,
+  createSpecCalls,
+  resolveSpecBase,
+  resolveToolResults,
+  savedWrites,
+  specFromArgs,
+  specModules,
+  supersededInMessage,
+  type InteractionLog,
+  type SpecArgs,
+} from "./thread";
+export type SpecToolkitOptions = {
+  components: SpecComponents;
+  handlers?: Record<string, ActionHandler>;
+  onAction?: (
+    name: string,
+    params: Record<string, unknown>,
+    context: { elementId?: string; trigger?: string; state: SpecStateStore },
+  ) => unknown;
+  placeholder?: Component;
+  specPrompt?: Omit<SpecPromptOptions, "mode">;
+  toolResults?: boolean;
+};
+export type SpecToolkitPlatform = {
+  useMessages(): MaybeRefOrGetter<readonly MessageLike[]>;
+  useArgsComplete(): MaybeRefOrGetter<boolean>;
+  SpecRenderer: Component;
+  /** The frame renders its content through its default Vue slot. */
+  Frame: Component;
+  useSendMessage(): (text: string) => void;
+  useModelContext(source: ModelContextSource): void;
+  copyText?: (text: string) => Promise<void>;
+};
+type RenderSpecProps = {
+  args: SpecArgs;
+  toolCallId: string;
+  /** @deprecated Experimental since 2026-09-23. Not scheduled for removal; the API may change in any release. */
+  unstable_interactions?: InteractionLog | undefined;
+  /** @deprecated Experimental since 2026-09-23. Not scheduled for removal; the API may change in any release. */
+  unstable_recordInteraction?:
+    | ((input: { type: "action"; payload: unknown }) => Promise<void>)
+    | undefined;
+};
+export const specToolkitFor =
+  ({
+    useMessages,
+    useArgsComplete,
+    SpecRenderer,
+    Frame,
+    useSendMessage,
+    useModelContext,
+    copyText,
+  }: SpecToolkitPlatform) =>
+  (catalog: Catalog, options: SpecToolkitOptions): WidgetToolkitExtension => {
+    const { tools, histories, execute } = createSpecCalls(
+      catalog,
+      options.specPrompt,
+    );
+    const renderSpecUI = (mountsToolResults: boolean) =>
+      defineComponent(
+        (props: RenderSpecProps) => {
+          const complete = useArgsComplete();
+          const messages = useMessages();
+          const streaming = computed(() => !toValue(complete));
+          const title = computed(() =>
+            typeof props.args.title === "string" ? props.args.title : "",
+          );
+          const earlier = toValue(messages);
+          const superseded = computed(() =>
+            supersededInMessage(
+              toValue(messages),
+              props.toolCallId,
+              title.value,
+            ),
+          );
+          const base = computed(() =>
+            resolveSpecBase(earlier, props.toolCallId, title.value),
+          );
+          const toolResults = mountsToolResults
+            ? resolveToolResults(earlier, props.toolCallId)
+            : undefined;
+          const history = callHistory(earlier, props.toolCallId, toolResults);
+          const streamed = useSpecStream(() => ({
+            source:
+              typeof props.args.patches === "string" ? props.args.patches : "",
+            complete: !streaming.value,
+            ...(base.value ? { initial: base.value } : {}),
+          }));
+          watch(base, () => streamed.reset(), { flush: "sync" });
+          const specInput = computed(() => props.args.spec);
+          const spec = computed(() => {
+            const input = specInput.value;
+            return input &&
+              typeof input === "object" &&
+              typeof input.root === "string"
+              ? specFromArgs(input)
+              : streamed.spec.value;
+          });
+          const emptyState: Record<string, unknown> = {};
+          const modelState = computed(() => spec.value.state ?? emptyState);
+          const stateWithTools = computed(() =>
+            toolResults && modelState.value["tools"] === undefined
+              ? { ...modelState.value, tools: toolResults }
+              : modelState.value,
+          );
+          const renderSpec = computed(() =>
+            stateWithTools.value === spec.value.state
+              ? spec.value
+              : { ...spec.value, state: stateWithTools.value },
+          );
+          const store = createStateStore(stateWithTools.value, {
+            writes: savedWrites(props.unstable_interactions, props.toolCallId),
+          });
+          watch(stateWithTools, (value) => store.seed(value), {
+            flush: "sync",
+          });
+          const initialContext = savedContext(
+            props.unstable_interactions,
+            props.toolCallId,
+            "setState",
+          );
+          let context = initialContext;
+          const snapshot = () => ({
+            writes: store.writes(),
+            ...(context ? { context } : {}),
+          });
+          const saver = createSnapshotSaver<{
+            writes: readonly SpecStateWrite[];
+            context?: SharedContext;
+          }>(snapshot(), (saved) =>
+            props.unstable_recordInteraction?.({
+              type: "action",
+              payload: {
+                type: "setState",
+                toolCallId: props.toolCallId,
+                writes: saved?.writes ?? null,
+                ...(saved?.context ? { context: saved.context } : {}),
+              },
+            }),
+          );
+          const sendMessage = useSendMessage();
+          const share = useSharedContext(() => ({
+            title: title.value,
+            toolCallId: props.toolCallId,
+            initial: options.handlers?.["updateModelContext"]
+              ? undefined
+              : initialContext,
+            useModelContext,
+            onShare(next) {
+              context = next;
+              saver.schedule(snapshot);
+            },
+            muted: superseded.value,
+          }));
+          const handlers = {
+            sendMessage(
+              params: Record<string, unknown>,
+              source: { kind?: string },
+            ) {
+              const text = params["text"];
+              if (source.kind === "watch")
+                throw new Error(
+                  "sendMessage runs only from an event a component emits, never from `watch`",
+                );
+              if (typeof text !== "string" || !text.trim())
+                throw new Error("sendMessage needs a non-empty string `text`");
+              sendMessage(text);
+            },
+            updateModelContext: (params: Record<string, unknown>) =>
+              share(toSharedContext(params["text"], params["data"])),
+            ...(copyText ? { copyText: copyTextHandler(copyText) } : {}),
+            ...options.handlers,
+          };
+          let enabled = true;
+          let unsubscribe: (() => void) | undefined;
+          let historyId = props.toolCallId;
+          const stop = () => {
+            unsubscribe?.();
+            unsubscribe = undefined;
+            if (histories.get(historyId) === history)
+              histories.delete(historyId);
+            saver.flush();
+          };
+          const start = () => {
+            if (unsubscribe || !enabled) return;
+            historyId = props.toolCallId;
+            histories.set(historyId, history);
+            unsubscribe = store.subscribe(() => saver.schedule(snapshot));
+          };
+          onMounted(start);
+          watch(
+            () => props.toolCallId,
+            () => {
+              stop();
+              start();
+            },
+            { flush: "sync" },
+          );
+          onDeactivated(() => {
+            enabled = false;
+            stop();
+          });
+          onActivated(() => {
+            enabled = true;
+            start();
+          });
+          onScopeDispose(stop);
+          return () =>
+            superseded.value
+              ? null
+              : h(
+                  Frame,
+                  { title: title.value, streaming: streaming.value },
+                  {
+                    default: () =>
+                      h(SpecRenderer, {
+                        spec: renderSpec.value,
+                        state: store,
+                        catalog,
+                        components: options.components,
+                        streaming: streaming.value,
+                        handlers,
+                        ...(options.onAction
+                          ? { onAction: options.onAction }
+                          : {}),
+                        ...(options.placeholder
+                          ? { placeholder: options.placeholder }
+                          : {}),
+                      }),
+                  },
+                );
+        },
+        {
+          name: "RenderSpecUI",
+          props: [
+            "args",
+            "toolCallId",
+            "unstable_interactions",
+            "unstable_recordInteraction",
+          ],
+        },
+      );
+    return {
+      tools,
+      modules: specModules(catalog, options.specPrompt, {
+        clipboard: copyText !== undefined,
+      }),
+      entries: (context) => ({
+        render_spec: toolkitEntry(
+          tools.render_spec,
+          renderSpecUI(
+            context.execution === "frontend" || options.toolResults === true,
+          ),
+          context,
+          execute,
+        ),
+      }),
+    };
+  };