@hyperframes/studio 0.8.16 → 0.8.18

package/src/webmcp/tools/lookTools.ts

@@ -0,0 +1,201 @@
+/**
+ * `studio_look`: the one call that orients an agent.
+ *
+ * Deliberately fat. Every field here is one the agent would otherwise have to
+ * spend a round trip discovering.
+ *
+ * The building is a pure function over a snapshot so it can be tested with
+ * values. Gathering the snapshot is the React layer's job.
+ */
+
+import type { DomEditSelection } from "../../components/editor/domEditingTypes";
+import type { TimelineElement } from "../../player/store/timelineElement";
+import { mintElementHandle, patchTargetAddress, timelineElementAddress } from "../handles";
+import { toolOk, type ToolResult } from "../toolResult";
+
+export interface StudioLookSnapshot {
+  projectId: string | null;
+  compositionPath: string | null;
+  currentTime: number;
+  duration: number;
+  isPlaying: boolean;
+  elements: readonly TimelineElement[];
+  selection: DomEditSelection | null;
+  /** Live animations for the current selection arrive outside DomEditSelection. */
+  selectionAnimationCount: number;
+  /**
+   * The undo stack as Studio's shell actually exposes it.
+   *
+   * This is a weaker signal than a revision counter, and deliberately not
+   * dressed up as one: the depth lives in component-local state and is not
+   * reachable here without plumbing it through the shell context. What an agent
+   * CAN do is checkpoint `undoLabel` before a batch and notice it change to
+   * something it did not do, which means a human pressed undo and its earlier
+   * edits are gone.
+   */
+  history: {
+    canUndo: boolean;
+    canRedo: boolean;
+    undoLabel: string | null;
+    redoLabel: string | null;
+  };
+}
+
+interface LookElement {
+  /** Pass back to any tool that takes a handle. Null means unaddressable. */
+  handle: string | null;
+  label: string | null;
+  tag: string;
+  kind: string | null;
+  start: number;
+  duration: number;
+  track: number;
+  zIndex: number | null;
+}
+
+interface LookSelection {
+  handle: string | null;
+  label: string;
+  tagName: string;
+  sourceFile: string;
+  box: { x: number; y: number; width: number; height: number };
+  text: string | null;
+  /** What this element will and will not accept, straight from Studio. */
+  can: {
+    editStyles: boolean;
+    move: boolean;
+    resize: boolean;
+    editText: boolean;
+    reasonIfDisabled: string | null;
+  };
+  animationCount: number;
+}
+
+/**
+ * Session-scoped response shape. There is intentionally no schema version:
+ * WebMCP consumers discover the current tool and schema when they connect
+ * rather than pinning a cached REST response contract.
+ */
+export interface StudioLook {
+  projectId: string | null;
+  compositionPath: string | null;
+  playhead: number;
+  duration: number;
+  isPlaying: boolean;
+  history: StudioLookSnapshot["history"];
+  selection: LookSelection | null;
+  elementCount: number;
+  elements: LookElement[];
+}
+
+export interface StudioLookInput {
+  /** Case-insensitive substring match against label, tag, and handle. */
+  filter?: string;
+  /** Cap the returned list. The full count is always reported separately. */
+  limit?: number;
+}
+
+const DEFAULT_LIMIT = 200;
+const MAX_FILTER_LENGTH = 128;
+
+function describeElement(element: TimelineElement): LookElement {
+  return {
+    handle: mintElementHandle(timelineElementAddress(element)),
+    label: element.label ?? null,
+    tag: element.tag,
+    kind: element.kind ?? null,
+    start: element.start,
+    duration: element.duration,
+    track: element.track,
+    zIndex: element.zIndex ?? null,
+  };
+}
+
+function describeSelection(selection: DomEditSelection, animationCount: number): LookSelection {
+  const { capabilities } = selection;
+  return {
+    handle: mintElementHandle(patchTargetAddress(selection)),
+    label: selection.label,
+    tagName: selection.tagName,
+    sourceFile: selection.sourceFile,
+    box: selection.boundingBox,
+    text: selection.textContent,
+    can: {
+      editStyles: capabilities.canEditStyles,
+      move: capabilities.canMove || capabilities.canApplyManualOffset,
+      resize: capabilities.canResize || capabilities.canApplyManualSize,
+      editText: selection.textFields.length > 0,
+      reasonIfDisabled: capabilities.reasonIfDisabled ?? null,
+    },
+    animationCount,
+  };
+}
+
+function matchesFilter(element: LookElement, needle: string): boolean {
+  return (
+    (element.label?.toLowerCase().includes(needle) ?? false) ||
+    element.tag.toLowerCase().includes(needle) ||
+    (element.handle?.toLowerCase().includes(needle) ?? false)
+  );
+}
+
+export function buildStudioLook(
+  snapshot: StudioLookSnapshot,
+  input: StudioLookInput = {},
+): ToolResult<StudioLook> {
+  const described = snapshot.elements.map(describeElement);
+  const needle = input.filter?.slice(0, MAX_FILTER_LENGTH).trim().toLowerCase();
+  const matched = needle
+    ? described.filter((element) => matchesFilter(element, needle))
+    : described;
+
+  // Clamp rather than reject: a bad limit should not cost the agent a round trip
+  // when the answer it wants is right here.
+  const requested =
+    Number.isInteger(input.limit) && input.limit! > 0 ? input.limit! : DEFAULT_LIMIT;
+  const limit = Math.min(requested, DEFAULT_LIMIT);
+
+  return toolOk<StudioLook>({
+    projectId: snapshot.projectId,
+    compositionPath: snapshot.compositionPath,
+    playhead: snapshot.currentTime,
+    duration: snapshot.duration,
+    isPlaying: snapshot.isPlaying,
+    history: snapshot.history,
+    selection: snapshot.selection
+      ? describeSelection(snapshot.selection, snapshot.selectionAnimationCount)
+      : null,
+    // The count is of everything that MATCHED, so a truncated list is visible
+    // as a truncated list rather than reading as "that is all there is".
+    elementCount: matched.length,
+    elements: matched.slice(0, limit),
+  });
+}
+
+export const STUDIO_LOOK_INPUT_SCHEMA = {
+  type: "object",
+  properties: {
+    filter: {
+      type: "string",
+      maxLength: MAX_FILTER_LENGTH,
+      description: "Case-insensitive substring matched against element label, tag, and handle.",
+    },
+    limit: {
+      type: "integer",
+      minimum: 1,
+      maximum: DEFAULT_LIMIT,
+      description: `Cap the returned elements (default and max ${DEFAULT_LIMIT}). elementCount always reports the full match count.`,
+    },
+  },
+  additionalProperties: false,
+} as const;
+
+export const STUDIO_LOOK_DESCRIPTION = [
+  "Read HyperFrames Studio's live state in one call: the open project and composition,",
+  "the playhead and duration, what the human currently has selected (including what that",
+  "element will and will not accept), and the timeline's elements with a handle for each.",
+  "Pass a handle back to any tool that edits an element.",
+  "Returns an object with `ok: true`, or `ok: false` with `kind`, `reason` and often a `hint`.",
+  "`history.undoLabel` is worth checkpointing before a batch: if it later names something",
+  "you did not do, a human pressed undo and your earlier edits are gone.",
+].join(" ");

package/src/webmcp/types.ts

@@ -0,0 +1,90 @@
+/**
+ * The slice of the WebMCP browser API that Studio uses.
+ *
+ * Mirrors the WebIDL in the W3C spec (`webmachinelearning/webmcp`, `index.bs`)
+ * as of 2026-08-26. Two things worth knowing before editing this file:
+ *
+ * - The API hangs off `document`, NOT `navigator`. `navigator.modelContext` is
+ *   a polyfill compatibility shim, not a spec member, so feature-detecting it
+ *   is wrong even where an article's sample "works".
+ * - The spec is pre-stable (Origin Trial). This file and `registrar.ts` are the
+ *   only places that touch the API, so a spec change is a two-file edit. Re-read
+ *   `index.bs` rather than trusting this transcription.
+ *
+ * Only the surface Studio registers against is declared. `getTools` and
+ * `executeTool` are the consumer side; Studio registers, it does not call.
+ *
+ * These stay hand-written rather than imported from `@mcp-b/webmcp-types`,
+ * which the polyfill pulls in. That package's `registerTool` is overloaded to
+ * infer argument types from a literal `inputSchema`, which is useful when you
+ * register one tool inline and actively hostile when you register a uniform
+ * list of them, as `registerStudioTools` does. Narrower is the safe operation
+ * here. It does mean this file can drift from the spec, hence the note above.
+ */
+
+export interface ModelContextToolAnnotations {
+  /** The tool does not change state. Lets an agent decide when calling is free. */
+  readOnlyHint?: boolean;
+  /** The tool's output contains data the page's author does not vouch for. */
+  untrustedContentHint?: boolean;
+}
+
+export interface ToolExecuteCallbackOptions {
+  /**
+   * Aborted when the caller cancels. Studio's commit path is not cancellable
+   * once dispatched, so tools check this BEFORE dispatching and document that a
+   * late abort does not unwind a write.
+   */
+  signal: AbortSignal;
+}
+
+export interface ModelContextTool {
+  /**
+   * Max 128 characters, ASCII alphanumeric plus `_`, `-`, `.`. Registering a
+   * name that already exists REJECTS with InvalidStateError; it does not
+   * replace.
+   */
+  name: string;
+  title?: string;
+  /** Required and non-empty; an empty string rejects with InvalidStateError. */
+  description: string;
+  /** JSON Schema. Nothing in the platform validates input against it. */
+  inputSchema?: object;
+  /**
+   * The user agent JSON-serializes whatever this resolves with, so it must
+   * return an object. Returning `undefined` fails the serialization.
+   *
+   * A rejection is NOT a usable error channel: the spec discards the reason and
+   * rejects the caller with a bare UnknownError. Resolve with a tagged failure
+   * instead. See `toolResult.ts`.
+   */
+  execute: (input: object, options: ToolExecuteCallbackOptions) => Promise<unknown>;
+  annotations?: ModelContextToolAnnotations;
+}
+
+export interface ModelContextRegisterToolOptions {
+  exposedTo?: string[];
+  /** Aborting unregisters the tool. It does not cancel a running `execute`. */
+  signal?: AbortSignal;
+}
+
+export interface ModelContext {
+  registerTool(tool: ModelContextTool, options?: ModelContextRegisterToolOptions): Promise<void>;
+}
+
+function isModelContext(value: unknown): value is ModelContext {
+  if (typeof value !== "object" || value === null) return false;
+  return typeof Reflect.get(value, "registerTool") === "function";
+}
+
+/**
+ * The live WebMCP entry point, or null when this browser has not shipped it.
+ *
+ * Reads through a guard rather than augmenting the `Document` interface. The
+ * polyfill's typings already declare `Document.modelContext` globally, and a
+ * second, narrower declaration of the same property is a type error.
+ */
+export function getModelContext(doc: Document = document): ModelContext | null {
+  const candidate = Reflect.get(doc, "modelContext");
+  return isModelContext(candidate) ? candidate : null;
+}

package/src/webmcp/useStudioAgentTools.test.tsx

@@ -0,0 +1,221 @@
+// @vitest-environment jsdom
+import { act } from "react";
+import { afterEach, beforeEach, describe, expect, it, vi } from "vitest";
+import { mountReactHarness } from "../hooks/domSelectionTestHarness";
+import { writeStudioUiPreferences } from "../utils/studioUiPreferences";
+import { useStudioAgentTools, type StudioAgentToolsDeps } from "./useStudioAgentTools";
+import type { ModelContext, ModelContextRegisterToolOptions, ModelContextTool } from "./types";
+import type { StudioLookSnapshot } from "./tools/lookTools";
+
+const trackEvent = vi.hoisted(() => vi.fn());
+vi.mock("../telemetry/client", () => ({ trackEvent }));
+
+Reflect.set(globalThis, "IS_REACT_ACT_ENVIRONMENT", true);
+
+let cleanup: (() => void) | null = null;
+
+function snapshot(overrides: Partial<StudioLookSnapshot> = {}): StudioLookSnapshot {
+  return {
+    projectId: "demo",
+    compositionPath: "index.html",
+    currentTime: 0,
+    duration: 10,
+    isPlaying: false,
+    elements: [],
+    selection: null,
+    selectionAnimationCount: 0,
+    history: { canUndo: false, canRedo: false, undoLabel: null, redoLabel: null },
+    ...overrides,
+  };
+}
+
+/** Install a fake `document.modelContext` and report what got registered. */
+function installModelContext() {
+  const registered: ModelContextTool[] = [];
+  const registerTool = vi.fn(
+    async (tool: ModelContextTool, _options?: ModelContextRegisterToolOptions) => {
+      registered.push(tool);
+    },
+  );
+  const modelContext: ModelContext = { registerTool };
+  Object.defineProperty(document, "modelContext", {
+    value: modelContext,
+    configurable: true,
+    writable: true,
+  });
+  return { registered, registerTool };
+}
+
+function removeModelContext() {
+  Reflect.deleteProperty(document, "modelContext");
+}
+
+function mountTools(deps: StudioAgentToolsDeps) {
+  function Probe({ current }: { current: StudioAgentToolsDeps }) {
+    useStudioAgentTools(current);
+    return null;
+  }
+  const root = mountReactHarness(<Probe current={deps} />);
+  cleanup = () => act(() => root.unmount());
+  return {
+    rerenderWith(next: StudioAgentToolsDeps) {
+      act(() => root.render(<Probe current={next} />));
+    },
+  };
+}
+
+beforeEach(() => {
+  window.localStorage.clear();
+  trackEvent.mockReset();
+});
+
+afterEach(() => {
+  cleanup?.();
+  cleanup = null;
+  removeModelContext();
+  window.localStorage.clear();
+  vi.restoreAllMocks();
+});
+
+describe("useStudioAgentTools", () => {
+  it("registers the tool set once on mount", async () => {
+    const { registered } = installModelContext();
+
+    await act(async () => {
+      mountTools({ getSnapshot: () => snapshot() });
+    });
+
+    expect(registered.map((tool) => tool.name)).toEqual(["studio_look"]);
+    expect(trackEvent).toHaveBeenCalledWith("webmcp.native_present");
+  });
+
+  it("does not re-register when the deps object changes identity", async () => {
+    // The regression test for the whole design. The DomEdit actions object
+    // changes identity on nearly every interaction; if registration depended on
+    // it, the signal would abort and unregister the tools each time.
+    const { registerTool } = installModelContext();
+
+    let harness: ReturnType<typeof mountTools> | null = null;
+    await act(async () => {
+      harness = mountTools({ getSnapshot: () => snapshot() });
+    });
+    expect(registerTool).toHaveBeenCalledTimes(1);
+
+    await act(async () => {
+      harness?.rerenderWith({ getSnapshot: () => snapshot({ currentTime: 5 }) });
+      harness?.rerenderWith({ getSnapshot: () => snapshot({ currentTime: 6 }) });
+    });
+
+    expect(registerTool).toHaveBeenCalledTimes(1);
+  });
+
+  it("executes against the LATEST deps, not the ones present at registration", async () => {
+    // The other half of the ref: registering once must not freeze the state the
+    // tools read, or every answer after the first render would be stale.
+    const { registered } = installModelContext();
+
+    let harness: ReturnType<typeof mountTools> | null = null;
+    await act(async () => {
+      harness = mountTools({ getSnapshot: () => snapshot({ currentTime: 1 }) });
+    });
+
+    await act(async () => {
+      harness?.rerenderWith({ getSnapshot: () => snapshot({ currentTime: 42 }) });
+    });
+
+    const look = registered[0];
+    if (!look) throw new Error("expected studio_look to be registered");
+    const result = (await look.execute({}, { signal: new AbortController().signal })) as {
+      ok: boolean;
+      playhead: number;
+    };
+
+    expect(result.ok).toBe(true);
+    expect(result.playhead).toBe(42);
+  });
+
+  it("unregisters on unmount by aborting the registration signal", async () => {
+    const { registerTool } = installModelContext();
+
+    await act(async () => {
+      mountTools({ getSnapshot: () => snapshot() });
+    });
+    const signal = registerTool.mock.calls[0]?.[1]?.signal;
+    expect(signal?.aborted).toBe(false);
+
+    cleanup?.();
+    cleanup = null;
+
+    expect(signal?.aborted).toBe(true);
+  });
+
+  it("boots cleanly when the browser has no native WebMCP", async () => {
+    removeModelContext();
+
+    await act(async () => {
+      mountTools({ getSnapshot: () => snapshot() });
+    });
+
+    // The assertion is that mounting did not throw; a browser without the
+    // native API must still boot Studio. The polyfill may install
+    // document.modelContext as a fallback — that is expected.
+  });
+
+  it("registers nothing when the preference is turned off", async () => {
+    writeStudioUiPreferences({ agentToolsEnabled: false });
+    const { registerTool } = installModelContext();
+
+    await act(async () => {
+      mountTools({ getSnapshot: () => snapshot() });
+    });
+
+    expect(registerTool).not.toHaveBeenCalled();
+  });
+
+  it("registers when the preference is absent, because on is the default", async () => {
+    const { registerTool } = installModelContext();
+
+    await act(async () => {
+      mountTools({ getSnapshot: () => snapshot() });
+    });
+
+    expect(registerTool).toHaveBeenCalledTimes(1);
+  });
+
+  it("reports a non-abort registration failure through production telemetry", async () => {
+    const { registerTool } = installModelContext();
+    registerTool.mockRejectedValue(new DOMException("blocked", "NotAllowedError"));
+
+    await act(async () => {
+      mountTools({ getSnapshot: () => snapshot() });
+    });
+
+    expect(trackEvent).toHaveBeenCalledWith("webmcp_registration_failed", {
+      error_name: "NotAllowedError",
+      tool_name: "studio_look",
+    });
+  });
+
+  it("reports a tool that throws as an internal failure instead of rejecting", async () => {
+    const { registered } = installModelContext();
+
+    await act(async () => {
+      mountTools({
+        getSnapshot: () => {
+          throw new TypeError("handler signature moved");
+        },
+      });
+    });
+    vi.spyOn(console, "error").mockImplementation(() => {});
+
+    const look = registered[0];
+    if (!look) throw new Error("expected studio_look to be registered");
+    const result = (await look.execute({}, { signal: new AbortController().signal })) as {
+      ok: boolean;
+      kind: string;
+    };
+
+    expect(result.ok).toBe(false);
+    expect(result.kind).toBe("internal");
+  });
+});

package/src/webmcp/useStudioAgentTools.ts

@@ -0,0 +1,119 @@
+import { useEffect, useRef } from "react";
+import { trackEvent } from "../telemetry/client";
+import { readStudioUiPreferences } from "../utils/studioUiPreferences";
+import { makeStudioDebugLogger } from "../utils/studioDebug";
+import { loadModelContextPolyfill } from "./polyfill";
+import { registerStudioTools, type ToolRegistrationReport } from "./registrar";
+import { runToolBody, type ToolResult } from "./toolResult";
+import { getModelContext, type ModelContext, type ModelContextTool } from "./types";
+import {
+  buildStudioLook,
+  STUDIO_LOOK_DESCRIPTION,
+  STUDIO_LOOK_INPUT_SCHEMA,
+  type StudioLook,
+  type StudioLookInput,
+  type StudioLookSnapshot,
+} from "./tools/lookTools";
+
+const log = makeStudioDebugLogger("webmcp");
+
+function reportRegistration(report: ToolRegistrationReport, native: boolean): void {
+  log("registered", { native, ...report });
+  for (const failure of report.failed) {
+    trackEvent("webmcp_registration_failed", {
+      error_name: failure.name,
+      tool_name: failure.tool,
+    });
+  }
+}
+
+export interface StudioAgentToolsDeps {
+  /** Read Studio's current state. Called per tool invocation, never cached. */
+  getSnapshot: () => StudioLookSnapshot;
+}
+
+/**
+ * Build the tool list once.
+ *
+ * Every `execute` reads `depsRef.current` at CALL time rather than closing over
+ * a snapshot, which is what lets the list be built once and still see live
+ * state. That is the whole point of the ref: see the registration note below.
+ */
+function buildStudioTools(depsRef: { readonly current: StudioAgentToolsDeps }): ModelContextTool[] {
+  return [
+    {
+      name: "studio_look",
+      title: "Look at the composition",
+      description: STUDIO_LOOK_DESCRIPTION,
+      inputSchema: STUDIO_LOOK_INPUT_SCHEMA,
+      annotations: {
+        readOnlyHint: true,
+        // The labels, text and ids come from the user's composition, which can
+        // contain anything. This is a hint to the agent, not a sanitiser.
+        untrustedContentHint: true,
+      },
+      execute: (input): Promise<ToolResult<StudioLook>> =>
+        runToolBody("studio_look", async () =>
+          buildStudioLook(depsRef.current.getSnapshot(), input as StudioLookInput),
+        ),
+    },
+  ];
+}
+
+/**
+ * Register Studio's tools with the browser, exactly once per mount.
+ *
+ * The effect has an EMPTY dependency array on purpose, and the deps live in a
+ * ref that every render refreshes. The obvious alternative — depend on the
+ * handlers — re-runs on nearly every interaction, because the DomEdit actions
+ * object changes identity whenever the selection or the element list does.
+ * Re-running means the registration signal aborts and unregisters everything,
+ * `toolchange` fires constantly so a connected agent watches the tool list
+ * churn, and the spec warns that a quick unregister-then-reregister can apply
+ * an old call's arguments against the new schema.
+ *
+ * `useStudioTestHooks` carries a comment about the same class of bug already hit
+ * in this codebase, where effect teardown revoked a lease moments after it was
+ * taken because writing state changed the effect's dependency identities.
+ *
+ * Any fallback must be awaited inside this effect before registration and then
+ * re-read here. Installing one from a sibling effect would race this mount-only
+ * lookup. Hot-module replacement can still create a brief unregister/register
+ * window in development; production has one document-scoped registration.
+ */
+export function useStudioAgentTools(deps: StudioAgentToolsDeps): void {
+  const depsRef = useRef(deps);
+  depsRef.current = deps;
+
+  // eslint-disable-next-line no-restricted-syntax
+  useEffect(() => {
+    if (readStudioUiPreferences().agentToolsEnabled === false) {
+      log("skipped", { why: "disabled by preference" });
+      return;
+    }
+
+    const controller = new AbortController();
+
+    void (async () => {
+      const native: ModelContext | null = getModelContext();
+      if (native) trackEvent("webmcp.native_present");
+      // Native browsers never download the polyfill.
+      const modelContext = native ?? (await loadModelContextPolyfill());
+      if (!modelContext) {
+        log("skipped", { why: "no model context, native or polyfilled" });
+        return;
+      }
+      // The import is async, so the component may already be gone.
+      if (controller.signal.aborted) return;
+
+      const report = await registerStudioTools(
+        modelContext,
+        buildStudioTools(depsRef),
+        controller.signal,
+      );
+      reportRegistration(report, native !== null);
+    })();
+
+    return () => controller.abort();
+  }, []);
+}